forked from phoenix/litellm-mirror
example with Claude+streaming
This commit is contained in:
ishaan-jaff
parent
1b81f4ad79
commit
5161573bce
1 changed files with 406 additions and 0 deletions
|
|
@ -0,0 +1,406 @@
|
|||
{
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 0,
|
||||
"metadata": {
|
||||
"colab": {
|
||||
"provenance": []
|
||||
},
|
||||
"kernelspec": {
|
||||
"name": "python3",
|
||||
"display_name": "Python 3"
|
||||
},
|
||||
"language_info": {
|
||||
"name": "python"
|
||||
}
|
||||
},
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 1,
|
||||
"metadata": {
|
||||
"colab": {
|
||||
"base_uri": "https://localhost:8080/"
|
||||
},
|
||||
"id": "ZwuaylskLxFu",
|
||||
"outputId": "d684d6a3-32fe-4beb-c378-c39134bcf8cc"
|
||||
},
|
||||
"outputs": [
|
||||
{
|
||||
"output_type": "stream",
|
||||
"name": "stdout",
|
||||
"text": [
|
||||
"Collecting litellm==0.1.363\n",
|
||||
" Downloading litellm-0.1.363-py3-none-any.whl (34 kB)\n",
|
||||
"Requirement already satisfied: openai<0.28.0,>=0.27.8 in /usr/local/lib/python3.10/dist-packages (from litellm==0.1.363) (0.27.8)\n",
|
||||
"Requirement already satisfied: python-dotenv<2.0.0,>=1.0.0 in /usr/local/lib/python3.10/dist-packages (from litellm==0.1.363) (1.0.0)\n",
|
||||
"Requirement already satisfied: tiktoken<0.5.0,>=0.4.0 in /usr/local/lib/python3.10/dist-packages (from litellm==0.1.363) (0.4.0)\n",
|
||||
"Requirement already satisfied: requests>=2.20 in /usr/local/lib/python3.10/dist-packages (from openai<0.28.0,>=0.27.8->litellm==0.1.363) (2.31.0)\n",
|
||||
"Requirement already satisfied: tqdm in /usr/local/lib/python3.10/dist-packages (from openai<0.28.0,>=0.27.8->litellm==0.1.363) (4.65.0)\n",
|
||||
"Requirement already satisfied: aiohttp in /usr/local/lib/python3.10/dist-packages (from openai<0.28.0,>=0.27.8->litellm==0.1.363) (3.8.5)\n",
|
||||
"Requirement already satisfied: regex>=2022.1.18 in /usr/local/lib/python3.10/dist-packages (from tiktoken<0.5.0,>=0.4.0->litellm==0.1.363) (2022.10.31)\n",
|
||||
"Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests>=2.20->openai<0.28.0,>=0.27.8->litellm==0.1.363) (3.2.0)\n",
|
||||
"Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests>=2.20->openai<0.28.0,>=0.27.8->litellm==0.1.363) (3.4)\n",
|
||||
"Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests>=2.20->openai<0.28.0,>=0.27.8->litellm==0.1.363) (1.26.16)\n",
|
||||
"Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests>=2.20->openai<0.28.0,>=0.27.8->litellm==0.1.363) (2023.7.22)\n",
|
||||
"Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->openai<0.28.0,>=0.27.8->litellm==0.1.363) (23.1.0)\n",
|
||||
"Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.10/dist-packages (from aiohttp->openai<0.28.0,>=0.27.8->litellm==0.1.363) (6.0.4)\n",
|
||||
"Requirement already satisfied: async-timeout<5.0,>=4.0.0a3 in /usr/local/lib/python3.10/dist-packages (from aiohttp->openai<0.28.0,>=0.27.8->litellm==0.1.363) (4.0.2)\n",
|
||||
"Requirement already satisfied: yarl<2.0,>=1.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->openai<0.28.0,>=0.27.8->litellm==0.1.363) (1.9.2)\n",
|
||||
"Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/lib/python3.10/dist-packages (from aiohttp->openai<0.28.0,>=0.27.8->litellm==0.1.363) (1.4.0)\n",
|
||||
"Requirement already satisfied: aiosignal>=1.1.2 in /usr/local/lib/python3.10/dist-packages (from aiohttp->openai<0.28.0,>=0.27.8->litellm==0.1.363) (1.3.1)\n",
|
||||
"Installing collected packages: litellm\n",
|
||||
" Attempting uninstall: litellm\n",
|
||||
" Found existing installation: litellm 0.1.362\n",
|
||||
" Uninstalling litellm-0.1.362:\n",
|
||||
" Successfully uninstalled litellm-0.1.362\n",
|
||||
"Successfully installed litellm-0.1.363\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"!pip install litellm==\"0.1.363\""
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"source": [
|
||||
"# @title Import litellm & Set env variables\n",
|
||||
"import litellm\n",
|
||||
"import os\n",
|
||||
"\n",
|
||||
"os.environ[\"ANTHROPIC_API_KEY\"] = \" \" #@param"
|
||||
],
|
||||
"metadata": {
|
||||
"id": "W216G__XL19Q"
|
||||
},
|
||||
"execution_count": 2,
|
||||
"outputs": []
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"source": [
|
||||
"# @title Request Claude Instant-1 and Claude-2\n",
|
||||
"messages = [\n",
|
||||
" {\"role\": \"system\", \"content\": \"You are a helpful assistant.\"},\n",
|
||||
" {\"role\": \"user\", \"content\": \"Who won the world series in 2020?\"}\n",
|
||||
" ]\n",
|
||||
"\n",
|
||||
"result = litellm.completion('claude-instant-1', messages)\n",
|
||||
"print(\"\\n\\n Result from claude-instant-1\", result)\n",
|
||||
"result = litellm.completion('claude-2', messages, max_tokens=5, temperature=0.2)\n",
|
||||
"print(\"\\n\\n Result from claude-2\", result)"
|
||||
],
|
||||
"metadata": {
|
||||
"colab": {
|
||||
"base_uri": "https://localhost:8080/"
|
||||
},
|
||||
"id": "ff1lKwUMMLJj",
|
||||
"outputId": "bfddf6f8-36d4-45e5-92dc-349083fa41b8"
|
||||
},
|
||||
"execution_count": 9,
|
||||
"outputs": [
|
||||
{
|
||||
"output_type": "stream",
|
||||
"name": "stdout",
|
||||
"text": [
|
||||
"\n",
|
||||
"\n",
|
||||
" Result from claude-instant-1 {'choices': [{'finish_reason': 'stop', 'index': 0, 'message': {'role': 'assistant', 'content': \" The Los Angeles Dodgers won the 2020 World Series, defeating the Tampa Bay Rays 4-2. It was the Dodgers' first World Series title since 1988.\"}}], 'created': 1691536677.2676156, 'model': 'claude-instant-1', 'usage': {'prompt_tokens': 30, 'completion_tokens': 32, 'total_tokens': 62}}\n",
|
||||
"\n",
|
||||
"\n",
|
||||
" Result from claude-2 {'choices': [{'finish_reason': 'stop', 'index': 0, 'message': {'role': 'assistant', 'content': ' The Los Angeles Dodgers won'}}], 'created': 1691536677.944753, 'model': 'claude-2', 'usage': {'prompt_tokens': 30, 'completion_tokens': 5, 'total_tokens': 35}}\n"
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"source": [
|
||||
"# @title Streaming Example: Request Claude-2\n",
|
||||
"messages = [\n",
|
||||
" {\"role\": \"system\", \"content\": \"You are a helpful assistant.\"},\n",
|
||||
" {\"role\": \"user\", \"content\": \"how does a court case get to the Supreme Court?\"}\n",
|
||||
" ]\n",
|
||||
"\n",
|
||||
"result = litellm.completion('claude-2', messages, stream=True)\n",
|
||||
"for chunk in result:\n",
|
||||
" print(chunk['choices'][0]['delta'])\n",
|
||||
"\n"
|
||||
],
|
||||
"metadata": {
|
||||
"colab": {
|
||||
"base_uri": "https://localhost:8080/"
|
||||
},
|
||||
"id": "06hWKnNQMrV-",
|
||||
"outputId": "7fdec0eb-d4a9-4882-f9c4-987ff9a31114"
|
||||
},
|
||||
"execution_count": 7,
|
||||
"outputs": [
|
||||
{
|
||||
"output_type": "stream",
|
||||
"name": "stdout",
|
||||
"text": [
|
||||
" Here\n",
|
||||
"'s\n",
|
||||
" a\n",
|
||||
" quick\n",
|
||||
" overview\n",
|
||||
" of\n",
|
||||
" how\n",
|
||||
" a\n",
|
||||
" court\n",
|
||||
" case\n",
|
||||
" can\n",
|
||||
" reach\n",
|
||||
" the\n",
|
||||
" U\n",
|
||||
".\n",
|
||||
"S\n",
|
||||
".\n",
|
||||
" Supreme\n",
|
||||
" Court\n",
|
||||
":\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"-\n",
|
||||
" The\n",
|
||||
" case\n",
|
||||
" must\n",
|
||||
" first\n",
|
||||
" be\n",
|
||||
" heard\n",
|
||||
" in\n",
|
||||
" a\n",
|
||||
" lower\n",
|
||||
" trial\n",
|
||||
" court\n",
|
||||
" (\n",
|
||||
"either\n",
|
||||
" a\n",
|
||||
" state\n",
|
||||
" court\n",
|
||||
" or\n",
|
||||
" federal\n",
|
||||
" district\n",
|
||||
" court\n",
|
||||
").\n",
|
||||
" The\n",
|
||||
" trial\n",
|
||||
" court\n",
|
||||
" makes\n",
|
||||
" initial\n",
|
||||
" r\n",
|
||||
"ulings\n",
|
||||
" and\n",
|
||||
" produces\n",
|
||||
" a\n",
|
||||
" record\n",
|
||||
" of\n",
|
||||
" the\n",
|
||||
" case\n",
|
||||
".\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"-\n",
|
||||
" The\n",
|
||||
" losing\n",
|
||||
" party\n",
|
||||
" can\n",
|
||||
" appeal\n",
|
||||
" the\n",
|
||||
" decision\n",
|
||||
" to\n",
|
||||
" an\n",
|
||||
" appeals\n",
|
||||
" court\n",
|
||||
" (\n",
|
||||
"a\n",
|
||||
" state\n",
|
||||
" appeals\n",
|
||||
" court\n",
|
||||
" for\n",
|
||||
" state\n",
|
||||
" cases\n",
|
||||
",\n",
|
||||
" or\n",
|
||||
" a\n",
|
||||
" federal\n",
|
||||
" circuit\n",
|
||||
" court\n",
|
||||
" for\n",
|
||||
" federal\n",
|
||||
" cases\n",
|
||||
").\n",
|
||||
" The\n",
|
||||
" appeals\n",
|
||||
" court\n",
|
||||
" reviews\n",
|
||||
" the\n",
|
||||
" trial\n",
|
||||
" court\n",
|
||||
"'s\n",
|
||||
" r\n",
|
||||
"ulings\n",
|
||||
" and\n",
|
||||
" can\n",
|
||||
" affirm\n",
|
||||
",\n",
|
||||
" reverse\n",
|
||||
",\n",
|
||||
" or\n",
|
||||
" modify\n",
|
||||
" the\n",
|
||||
" decision\n",
|
||||
".\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"-\n",
|
||||
" If\n",
|
||||
" a\n",
|
||||
" party\n",
|
||||
" is\n",
|
||||
" still\n",
|
||||
" unsat\n",
|
||||
"isf\n",
|
||||
"ied\n",
|
||||
" after\n",
|
||||
" the\n",
|
||||
" appeals\n",
|
||||
" court\n",
|
||||
" rules\n",
|
||||
",\n",
|
||||
" they\n",
|
||||
" can\n",
|
||||
" petition\n",
|
||||
" the\n",
|
||||
" Supreme\n",
|
||||
" Court\n",
|
||||
" to\n",
|
||||
" hear\n",
|
||||
" the\n",
|
||||
" case\n",
|
||||
" through\n",
|
||||
" a\n",
|
||||
" writ\n",
|
||||
" of\n",
|
||||
" cert\n",
|
||||
"ior\n",
|
||||
"ari\n",
|
||||
".\n",
|
||||
" \n",
|
||||
"\n",
|
||||
"\n",
|
||||
"-\n",
|
||||
" The\n",
|
||||
" Supreme\n",
|
||||
" Court\n",
|
||||
" gets\n",
|
||||
" thousands\n",
|
||||
" of\n",
|
||||
" cert\n",
|
||||
" petitions\n",
|
||||
" every\n",
|
||||
" year\n",
|
||||
" but\n",
|
||||
" usually\n",
|
||||
" only\n",
|
||||
" agrees\n",
|
||||
" to\n",
|
||||
" hear\n",
|
||||
" about\n",
|
||||
" 100\n",
|
||||
"-\n",
|
||||
"150\n",
|
||||
" of\n",
|
||||
" cases\n",
|
||||
" that\n",
|
||||
" have\n",
|
||||
" significant\n",
|
||||
" national\n",
|
||||
" importance\n",
|
||||
" or\n",
|
||||
" where\n",
|
||||
" lower\n",
|
||||
" courts\n",
|
||||
" disagree\n",
|
||||
" on\n",
|
||||
" federal\n",
|
||||
" law\n",
|
||||
".\n",
|
||||
" \n",
|
||||
"\n",
|
||||
"\n",
|
||||
"-\n",
|
||||
" If\n",
|
||||
" 4\n",
|
||||
" out\n",
|
||||
" of\n",
|
||||
" the\n",
|
||||
" 9\n",
|
||||
" Just\n",
|
||||
"ices\n",
|
||||
" vote\n",
|
||||
" to\n",
|
||||
" grant\n",
|
||||
" cert\n",
|
||||
" (\n",
|
||||
"agree\n",
|
||||
" to\n",
|
||||
" hear\n",
|
||||
" the\n",
|
||||
" case\n",
|
||||
"),\n",
|
||||
" it\n",
|
||||
" goes\n",
|
||||
" on\n",
|
||||
" the\n",
|
||||
" Supreme\n",
|
||||
" Court\n",
|
||||
"'s\n",
|
||||
" do\n",
|
||||
"cket\n",
|
||||
" for\n",
|
||||
" arguments\n",
|
||||
".\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"-\n",
|
||||
" The\n",
|
||||
" Supreme\n",
|
||||
" Court\n",
|
||||
" then\n",
|
||||
" hears\n",
|
||||
" oral\n",
|
||||
" arguments\n",
|
||||
",\n",
|
||||
" considers\n",
|
||||
" written\n",
|
||||
" brief\n",
|
||||
"s\n",
|
||||
",\n",
|
||||
" examines\n",
|
||||
" the\n",
|
||||
" lower\n",
|
||||
" court\n",
|
||||
" records\n",
|
||||
",\n",
|
||||
" and\n",
|
||||
" issues\n",
|
||||
" a\n",
|
||||
" final\n",
|
||||
" ruling\n",
|
||||
" on\n",
|
||||
" the\n",
|
||||
" case\n",
|
||||
",\n",
|
||||
" which\n",
|
||||
" serves\n",
|
||||
" as\n",
|
||||
" binding\n",
|
||||
" precedent\n"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue