phoenix/litellm
1
0
Fork 0
forked from phoenix/litellm-mirror
Code Pull requests Activity
litellm/cookbook/Claude_(Anthropic)_with_Streaming_liteLLM_Examples.ipynb
ishaan-jaff f29a353796 (docs) update how to use litellm streaming
2023-11-21 11:19:32 -08:00

406 lines
13 KiB
Text
Vendored
Raw Blame History

{
"cells": [
{
"cell_type": "code",
"execution_count": 1,
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "ZwuaylskLxFu",
"outputId": "d684d6a3-32fe-4beb-c378-c39134bcf8cc"
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Collecting litellm==0.1.363\n",
" Downloading litellm-0.1.363-py3-none-any.whl (34 kB)\n",
"Requirement already satisfied: openai<0.28.0,>=0.27.8 in /usr/local/lib/python3.10/dist-packages (from litellm==0.1.363) (0.27.8)\n",
"Requirement already satisfied: python-dotenv<2.0.0,>=1.0.0 in /usr/local/lib/python3.10/dist-packages (from litellm==0.1.363) (1.0.0)\n",
"Requirement already satisfied: tiktoken<0.5.0,>=0.4.0 in /usr/local/lib/python3.10/dist-packages (from litellm==0.1.363) (0.4.0)\n",
"Requirement already satisfied: requests>=2.20 in /usr/local/lib/python3.10/dist-packages (from openai<0.28.0,>=0.27.8->litellm==0.1.363) (2.31.0)\n",
"Requirement already satisfied: tqdm in /usr/local/lib/python3.10/dist-packages (from openai<0.28.0,>=0.27.8->litellm==0.1.363) (4.65.0)\n",
"Requirement already satisfied: aiohttp in /usr/local/lib/python3.10/dist-packages (from openai<0.28.0,>=0.27.8->litellm==0.1.363) (3.8.5)\n",
"Requirement already satisfied: regex>=2022.1.18 in /usr/local/lib/python3.10/dist-packages (from tiktoken<0.5.0,>=0.4.0->litellm==0.1.363) (2022.10.31)\n",
"Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests>=2.20->openai<0.28.0,>=0.27.8->litellm==0.1.363) (3.2.0)\n",
"Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests>=2.20->openai<0.28.0,>=0.27.8->litellm==0.1.363) (3.4)\n",
"Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests>=2.20->openai<0.28.0,>=0.27.8->litellm==0.1.363) (1.26.16)\n",
"Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests>=2.20->openai<0.28.0,>=0.27.8->litellm==0.1.363) (2023.7.22)\n",
"Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->openai<0.28.0,>=0.27.8->litellm==0.1.363) (23.1.0)\n",
"Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.10/dist-packages (from aiohttp->openai<0.28.0,>=0.27.8->litellm==0.1.363) (6.0.4)\n",
"Requirement already satisfied: async-timeout<5.0,>=4.0.0a3 in /usr/local/lib/python3.10/dist-packages (from aiohttp->openai<0.28.0,>=0.27.8->litellm==0.1.363) (4.0.2)\n",
"Requirement already satisfied: yarl<2.0,>=1.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->openai<0.28.0,>=0.27.8->litellm==0.1.363) (1.9.2)\n",
"Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/lib/python3.10/dist-packages (from aiohttp->openai<0.28.0,>=0.27.8->litellm==0.1.363) (1.4.0)\n",
"Requirement already satisfied: aiosignal>=1.1.2 in /usr/local/lib/python3.10/dist-packages (from aiohttp->openai<0.28.0,>=0.27.8->litellm==0.1.363) (1.3.1)\n",
"Installing collected packages: litellm\n",
" Attempting uninstall: litellm\n",
" Found existing installation: litellm 0.1.362\n",
" Uninstalling litellm-0.1.362:\n",
" Successfully uninstalled litellm-0.1.362\n",
"Successfully installed litellm-0.1.363\n"
]
}
],
"source": [
"!pip install litellm==\"0.1.363\""
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {
"id": "W216G__XL19Q"
},
"outputs": [],
"source": [
"# @title Import litellm & Set env variables\n",
"import litellm\n",
"import os\n",
"\n",
"os.environ[\"ANTHROPIC_API_KEY\"] = \" \" #@param"
]
},
{
"cell_type": "code",
"execution_count": 9,
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "ff1lKwUMMLJj",
"outputId": "bfddf6f8-36d4-45e5-92dc-349083fa41b8"
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"\n",
"\n",
" Result from claude-instant-1 {'choices': [{'finish_reason': 'stop', 'index': 0, 'message': {'role': 'assistant', 'content': \" The Los Angeles Dodgers won the 2020 World Series, defeating the Tampa Bay Rays 4-2. It was the Dodgers' first World Series title since 1988.\"}}], 'created': 1691536677.2676156, 'model': 'claude-instant-1', 'usage': {'prompt_tokens': 30, 'completion_tokens': 32, 'total_tokens': 62}}\n",
"\n",
"\n",
" Result from claude-2 {'choices': [{'finish_reason': 'stop', 'index': 0, 'message': {'role': 'assistant', 'content': ' The Los Angeles Dodgers won'}}], 'created': 1691536677.944753, 'model': 'claude-2', 'usage': {'prompt_tokens': 30, 'completion_tokens': 5, 'total_tokens': 35}}\n"
]
}
],
"source": [
"# @title Request Claude Instant-1 and Claude-2\n",
"messages = [\n",
" {\"role\": \"system\", \"content\": \"You are a helpful assistant.\"},\n",
" {\"role\": \"user\", \"content\": \"Who won the world series in 2020?\"}\n",
" ]\n",
"\n",
"result = litellm.completion('claude-instant-1', messages)\n",
"print(\"\\n\\n Result from claude-instant-1\", result)\n",
"result = litellm.completion('claude-2', messages, max_tokens=5, temperature=0.2)\n",
"print(\"\\n\\n Result from claude-2\", result)"
]
},
{
"cell_type": "code",
"execution_count": 7,
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "06hWKnNQMrV-",
"outputId": "7fdec0eb-d4a9-4882-f9c4-987ff9a31114"
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
" Here\n",
"'s\n",
" a\n",
" quick\n",
" overview\n",
" of\n",
" how\n",
" a\n",
" court\n",
" case\n",
" can\n",
" reach\n",
" the\n",
" U\n",
".\n",
"S\n",
".\n",
" Supreme\n",
" Court\n",
":\n",
"\n",
"\n",
"-\n",
" The\n",
" case\n",
" must\n",
" first\n",
" be\n",
" heard\n",
" in\n",
" a\n",
" lower\n",
" trial\n",
" court\n",
" (\n",
"either\n",
" a\n",
" state\n",
" court\n",
" or\n",
" federal\n",
" district\n",
" court\n",
").\n",
" The\n",
" trial\n",
" court\n",
" makes\n",
" initial\n",
" r\n",
"ulings\n",
" and\n",
" produces\n",
" a\n",
" record\n",
" of\n",
" the\n",
" case\n",
".\n",
"\n",
"\n",
"-\n",
" The\n",
" losing\n",
" party\n",
" can\n",
" appeal\n",
" the\n",
" decision\n",
" to\n",
" an\n",
" appeals\n",
" court\n",
" (\n",
"a\n",
" state\n",
" appeals\n",
" court\n",
" for\n",
" state\n",
" cases\n",
",\n",
" or\n",
" a\n",
" federal\n",
" circuit\n",
" court\n",
" for\n",
" federal\n",
" cases\n",
").\n",
" The\n",
" appeals\n",
" court\n",
" reviews\n",
" the\n",
" trial\n",
" court\n",
"'s\n",
" r\n",
"ulings\n",
" and\n",
" can\n",
" affirm\n",
",\n",
" reverse\n",
",\n",
" or\n",
" modify\n",
" the\n",
" decision\n",
".\n",
"\n",
"\n",
"-\n",
" If\n",
" a\n",
" party\n",
" is\n",
" still\n",
" unsat\n",
"isf\n",
"ied\n",
" after\n",
" the\n",
" appeals\n",
" court\n",
" rules\n",
",\n",
" they\n",
" can\n",
" petition\n",
" the\n",
" Supreme\n",
" Court\n",
" to\n",
" hear\n",
" the\n",
" case\n",
" through\n",
" a\n",
" writ\n",
" of\n",
" cert\n",
"ior\n",
"ari\n",
".\n",
" \n",
"\n",
"\n",
"-\n",
" The\n",
" Supreme\n",
" Court\n",
" gets\n",
" thousands\n",
" of\n",
" cert\n",
" petitions\n",
" every\n",
" year\n",
" but\n",
" usually\n",
" only\n",
" agrees\n",
" to\n",
" hear\n",
" about\n",
" 100\n",
"-\n",
"150\n",
" of\n",
" cases\n",
" that\n",
" have\n",
" significant\n",
" national\n",
" importance\n",
" or\n",
" where\n",
" lower\n",
" courts\n",
" disagree\n",
" on\n",
" federal\n",
" law\n",
".\n",
" \n",
"\n",
"\n",
"-\n",
" If\n",
" 4\n",
" out\n",
" of\n",
" the\n",
" 9\n",
" Just\n",
"ices\n",
" vote\n",
" to\n",
" grant\n",
" cert\n",
" (\n",
"agree\n",
" to\n",
" hear\n",
" the\n",
" case\n",
"),\n",
" it\n",
" goes\n",
" on\n",
" the\n",
" Supreme\n",
" Court\n",
"'s\n",
" do\n",
"cket\n",
" for\n",
" arguments\n",
".\n",
"\n",
"\n",
"-\n",
" The\n",
" Supreme\n",
" Court\n",
" then\n",
" hears\n",
" oral\n",
" arguments\n",
",\n",
" considers\n",
" written\n",
" brief\n",
"s\n",
",\n",
" examines\n",
" the\n",
" lower\n",
" court\n",
" records\n",
",\n",
" and\n",
" issues\n",
" a\n",
" final\n",
" ruling\n",
" on\n",
" the\n",
" case\n",
",\n",
" which\n",
" serves\n",
" as\n",
" binding\n",
" precedent\n"
]
}
],
"source": [
"# @title Streaming Example: Request Claude-2\n",
"messages = [\n",
" {\"role\": \"system\", \"content\": \"You are a helpful assistant.\"},\n",
" {\"role\": \"user\", \"content\": \"how does a court case get to the Supreme Court?\"}\n",
" ]\n",
"\n",
"result = litellm.completion('claude-2', messages, stream=True)\n",
"for part in result:\n",
" print(part.choices[0].delta.content or \"\")\n",
"\n"
]
}
],
"metadata": {
"colab": {
"provenance": []
},
"kernelspec": {
"display_name": "Python 3",
"name": "python3"
},
"language_info": {
"name": "python"
}
},
"nbformat": 4,
"nbformat_minor": 0
}
View git blame Copy permalink