From 5161573bce96cb53098056f794c5349e47a2dccc Mon Sep 17 00:00:00 2001 From: ishaan-jaff Date: Tue, 8 Aug 2023 16:20:49 -0700 Subject: [PATCH] example with Claude+streaming --- ...pic)_with_Streaming_liteLLM_Examples.ipynb | 406 ++++++++++++++++++ 1 file changed, 406 insertions(+) create mode 100644 cookbook/Claude_(Anthropic)_with_Streaming_liteLLM_Examples.ipynb diff --git a/cookbook/Claude_(Anthropic)_with_Streaming_liteLLM_Examples.ipynb b/cookbook/Claude_(Anthropic)_with_Streaming_liteLLM_Examples.ipynb new file mode 100644 index 000000000..f3875ae60 --- /dev/null +++ b/cookbook/Claude_(Anthropic)_with_Streaming_liteLLM_Examples.ipynb @@ -0,0 +1,406 @@ +{ + "nbformat": 4, + "nbformat_minor": 0, + "metadata": { + "colab": { + "provenance": [] + }, + "kernelspec": { + "name": "python3", + "display_name": "Python 3" + }, + "language_info": { + "name": "python" + } + }, + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "ZwuaylskLxFu", + "outputId": "d684d6a3-32fe-4beb-c378-c39134bcf8cc" + }, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Collecting litellm==0.1.363\n", + " Downloading litellm-0.1.363-py3-none-any.whl (34 kB)\n", + "Requirement already satisfied: openai<0.28.0,>=0.27.8 in /usr/local/lib/python3.10/dist-packages (from litellm==0.1.363) (0.27.8)\n", + "Requirement already satisfied: python-dotenv<2.0.0,>=1.0.0 in /usr/local/lib/python3.10/dist-packages (from litellm==0.1.363) (1.0.0)\n", + "Requirement already satisfied: tiktoken<0.5.0,>=0.4.0 in /usr/local/lib/python3.10/dist-packages (from litellm==0.1.363) (0.4.0)\n", + "Requirement already satisfied: requests>=2.20 in /usr/local/lib/python3.10/dist-packages (from openai<0.28.0,>=0.27.8->litellm==0.1.363) (2.31.0)\n", + "Requirement already satisfied: tqdm in /usr/local/lib/python3.10/dist-packages (from openai<0.28.0,>=0.27.8->litellm==0.1.363) (4.65.0)\n", + "Requirement already satisfied: aiohttp in /usr/local/lib/python3.10/dist-packages (from openai<0.28.0,>=0.27.8->litellm==0.1.363) (3.8.5)\n", + "Requirement already satisfied: regex>=2022.1.18 in /usr/local/lib/python3.10/dist-packages (from tiktoken<0.5.0,>=0.4.0->litellm==0.1.363) (2022.10.31)\n", + "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests>=2.20->openai<0.28.0,>=0.27.8->litellm==0.1.363) (3.2.0)\n", + "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests>=2.20->openai<0.28.0,>=0.27.8->litellm==0.1.363) (3.4)\n", + "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests>=2.20->openai<0.28.0,>=0.27.8->litellm==0.1.363) (1.26.16)\n", + "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests>=2.20->openai<0.28.0,>=0.27.8->litellm==0.1.363) (2023.7.22)\n", + "Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->openai<0.28.0,>=0.27.8->litellm==0.1.363) (23.1.0)\n", + "Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.10/dist-packages (from aiohttp->openai<0.28.0,>=0.27.8->litellm==0.1.363) (6.0.4)\n", + "Requirement already satisfied: async-timeout<5.0,>=4.0.0a3 in /usr/local/lib/python3.10/dist-packages (from aiohttp->openai<0.28.0,>=0.27.8->litellm==0.1.363) (4.0.2)\n", + "Requirement already satisfied: yarl<2.0,>=1.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->openai<0.28.0,>=0.27.8->litellm==0.1.363) (1.9.2)\n", + "Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/lib/python3.10/dist-packages (from aiohttp->openai<0.28.0,>=0.27.8->litellm==0.1.363) (1.4.0)\n", + "Requirement already satisfied: aiosignal>=1.1.2 in /usr/local/lib/python3.10/dist-packages (from aiohttp->openai<0.28.0,>=0.27.8->litellm==0.1.363) (1.3.1)\n", + "Installing collected packages: litellm\n", + " Attempting uninstall: litellm\n", + " Found existing installation: litellm 0.1.362\n", + " Uninstalling litellm-0.1.362:\n", + " Successfully uninstalled litellm-0.1.362\n", + "Successfully installed litellm-0.1.363\n" + ] + } + ], + "source": [ + "!pip install litellm==\"0.1.363\"" + ] + }, + { + "cell_type": "code", + "source": [ + "# @title Import litellm & Set env variables\n", + "import litellm\n", + "import os\n", + "\n", + "os.environ[\"ANTHROPIC_API_KEY\"] = \" \" #@param" + ], + "metadata": { + "id": "W216G__XL19Q" + }, + "execution_count": 2, + "outputs": [] + }, + { + "cell_type": "code", + "source": [ + "# @title Request Claude Instant-1 and Claude-2\n", + "messages = [\n", + " {\"role\": \"system\", \"content\": \"You are a helpful assistant.\"},\n", + " {\"role\": \"user\", \"content\": \"Who won the world series in 2020?\"}\n", + " ]\n", + "\n", + "result = litellm.completion('claude-instant-1', messages)\n", + "print(\"\\n\\n Result from claude-instant-1\", result)\n", + "result = litellm.completion('claude-2', messages, max_tokens=5, temperature=0.2)\n", + "print(\"\\n\\n Result from claude-2\", result)" + ], + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "ff1lKwUMMLJj", + "outputId": "bfddf6f8-36d4-45e5-92dc-349083fa41b8" + }, + "execution_count": 9, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "\n", + "\n", + " Result from claude-instant-1 {'choices': [{'finish_reason': 'stop', 'index': 0, 'message': {'role': 'assistant', 'content': \" The Los Angeles Dodgers won the 2020 World Series, defeating the Tampa Bay Rays 4-2. It was the Dodgers' first World Series title since 1988.\"}}], 'created': 1691536677.2676156, 'model': 'claude-instant-1', 'usage': {'prompt_tokens': 30, 'completion_tokens': 32, 'total_tokens': 62}}\n", + "\n", + "\n", + " Result from claude-2 {'choices': [{'finish_reason': 'stop', 'index': 0, 'message': {'role': 'assistant', 'content': ' The Los Angeles Dodgers won'}}], 'created': 1691536677.944753, 'model': 'claude-2', 'usage': {'prompt_tokens': 30, 'completion_tokens': 5, 'total_tokens': 35}}\n" + ] + } + ] + }, + { + "cell_type": "code", + "source": [ + "# @title Streaming Example: Request Claude-2\n", + "messages = [\n", + " {\"role\": \"system\", \"content\": \"You are a helpful assistant.\"},\n", + " {\"role\": \"user\", \"content\": \"how does a court case get to the Supreme Court?\"}\n", + " ]\n", + "\n", + "result = litellm.completion('claude-2', messages, stream=True)\n", + "for chunk in result:\n", + " print(chunk['choices'][0]['delta'])\n", + "\n" + ], + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "06hWKnNQMrV-", + "outputId": "7fdec0eb-d4a9-4882-f9c4-987ff9a31114" + }, + "execution_count": 7, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + " Here\n", + "'s\n", + " a\n", + " quick\n", + " overview\n", + " of\n", + " how\n", + " a\n", + " court\n", + " case\n", + " can\n", + " reach\n", + " the\n", + " U\n", + ".\n", + "S\n", + ".\n", + " Supreme\n", + " Court\n", + ":\n", + "\n", + "\n", + "-\n", + " The\n", + " case\n", + " must\n", + " first\n", + " be\n", + " heard\n", + " in\n", + " a\n", + " lower\n", + " trial\n", + " court\n", + " (\n", + "either\n", + " a\n", + " state\n", + " court\n", + " or\n", + " federal\n", + " district\n", + " court\n", + ").\n", + " The\n", + " trial\n", + " court\n", + " makes\n", + " initial\n", + " r\n", + "ulings\n", + " and\n", + " produces\n", + " a\n", + " record\n", + " of\n", + " the\n", + " case\n", + ".\n", + "\n", + "\n", + "-\n", + " The\n", + " losing\n", + " party\n", + " can\n", + " appeal\n", + " the\n", + " decision\n", + " to\n", + " an\n", + " appeals\n", + " court\n", + " (\n", + "a\n", + " state\n", + " appeals\n", + " court\n", + " for\n", + " state\n", + " cases\n", + ",\n", + " or\n", + " a\n", + " federal\n", + " circuit\n", + " court\n", + " for\n", + " federal\n", + " cases\n", + ").\n", + " The\n", + " appeals\n", + " court\n", + " reviews\n", + " the\n", + " trial\n", + " court\n", + "'s\n", + " r\n", + "ulings\n", + " and\n", + " can\n", + " affirm\n", + ",\n", + " reverse\n", + ",\n", + " or\n", + " modify\n", + " the\n", + " decision\n", + ".\n", + "\n", + "\n", + "-\n", + " If\n", + " a\n", + " party\n", + " is\n", + " still\n", + " unsat\n", + "isf\n", + "ied\n", + " after\n", + " the\n", + " appeals\n", + " court\n", + " rules\n", + ",\n", + " they\n", + " can\n", + " petition\n", + " the\n", + " Supreme\n", + " Court\n", + " to\n", + " hear\n", + " the\n", + " case\n", + " through\n", + " a\n", + " writ\n", + " of\n", + " cert\n", + "ior\n", + "ari\n", + ".\n", + " \n", + "\n", + "\n", + "-\n", + " The\n", + " Supreme\n", + " Court\n", + " gets\n", + " thousands\n", + " of\n", + " cert\n", + " petitions\n", + " every\n", + " year\n", + " but\n", + " usually\n", + " only\n", + " agrees\n", + " to\n", + " hear\n", + " about\n", + " 100\n", + "-\n", + "150\n", + " of\n", + " cases\n", + " that\n", + " have\n", + " significant\n", + " national\n", + " importance\n", + " or\n", + " where\n", + " lower\n", + " courts\n", + " disagree\n", + " on\n", + " federal\n", + " law\n", + ".\n", + " \n", + "\n", + "\n", + "-\n", + " If\n", + " 4\n", + " out\n", + " of\n", + " the\n", + " 9\n", + " Just\n", + "ices\n", + " vote\n", + " to\n", + " grant\n", + " cert\n", + " (\n", + "agree\n", + " to\n", + " hear\n", + " the\n", + " case\n", + "),\n", + " it\n", + " goes\n", + " on\n", + " the\n", + " Supreme\n", + " Court\n", + "'s\n", + " do\n", + "cket\n", + " for\n", + " arguments\n", + ".\n", + "\n", + "\n", + "-\n", + " The\n", + " Supreme\n", + " Court\n", + " then\n", + " hears\n", + " oral\n", + " arguments\n", + ",\n", + " considers\n", + " written\n", + " brief\n", + "s\n", + ",\n", + " examines\n", + " the\n", + " lower\n", + " court\n", + " records\n", + ",\n", + " and\n", + " issues\n", + " a\n", + " final\n", + " ruling\n", + " on\n", + " the\n", + " case\n", + ",\n", + " which\n", + " serves\n", + " as\n", + " binding\n", + " precedent\n" + ] + } + ] + } + ] +} \ No newline at end of file