From f29a353796b57a0d1dd27976375fdfc37d02032b Mon Sep 17 00:00:00 2001 From: ishaan-jaff Date: Tue, 21 Nov 2023 11:19:32 -0800 Subject: [PATCH] (docs) update how to use litellm streaming --- README.md | 10 +- ...pic)_with_Streaming_liteLLM_Examples.ipynb | 118 +++++++++--------- 2 files changed, 64 insertions(+), 64 deletions(-) diff --git a/README.md b/README.md index 3e453257b..e3ef0a5c8 100644 --- a/README.md +++ b/README.md @@ -68,13 +68,13 @@ Streaming is supported for all models (Bedrock, Huggingface, TogetherAI, Azure, ```python from litellm import completion response = completion(model="gpt-3.5-turbo", messages=messages, stream=True) -for chunk in response: - print(chunk['choices'][0]['delta']) +for part in response: + print(part.choices[0].delta.content or "") # claude 2 -result = completion('claude-2', messages, stream=True) -for chunk in result: - print(chunk['choices'][0]['delta']) +response = completion('claude-2', messages, stream=True) +for part in response: + print(part.choices[0].delta.content or "") ``` # Router - load balancing([Docs](https://docs.litellm.ai/docs/routing)) diff --git a/cookbook/Claude_(Anthropic)_with_Streaming_liteLLM_Examples.ipynb b/cookbook/Claude_(Anthropic)_with_Streaming_liteLLM_Examples.ipynb index f3875ae60..338785ea5 100644 --- a/cookbook/Claude_(Anthropic)_with_Streaming_liteLLM_Examples.ipynb +++ b/cookbook/Claude_(Anthropic)_with_Streaming_liteLLM_Examples.ipynb @@ -1,18 +1,4 @@ { - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "colab": { - "provenance": [] - }, - "kernelspec": { - "name": "python3", - "display_name": "Python 3" - }, - "language_info": { - "name": "python" - } - }, "cells": [ { "cell_type": "code", @@ -26,8 +12,8 @@ }, "outputs": [ { - "output_type": "stream", "name": "stdout", + "output_type": "stream", "text": [ "Collecting litellm==0.1.363\n", " Downloading litellm-0.1.363-py3-none-any.whl (34 kB)\n", @@ -63,21 +49,43 @@ }, { "cell_type": "code", + "execution_count": 2, + "metadata": { + "id": "W216G__XL19Q" + }, + "outputs": [], "source": [ "# @title Import litellm & Set env variables\n", "import litellm\n", "import os\n", "\n", "os.environ[\"ANTHROPIC_API_KEY\"] = \" \" #@param" - ], - "metadata": { - "id": "W216G__XL19Q" - }, - "execution_count": 2, - "outputs": [] + ] }, { "cell_type": "code", + "execution_count": 9, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "ff1lKwUMMLJj", + "outputId": "bfddf6f8-36d4-45e5-92dc-349083fa41b8" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "\n", + " Result from claude-instant-1 {'choices': [{'finish_reason': 'stop', 'index': 0, 'message': {'role': 'assistant', 'content': \" The Los Angeles Dodgers won the 2020 World Series, defeating the Tampa Bay Rays 4-2. It was the Dodgers' first World Series title since 1988.\"}}], 'created': 1691536677.2676156, 'model': 'claude-instant-1', 'usage': {'prompt_tokens': 30, 'completion_tokens': 32, 'total_tokens': 62}}\n", + "\n", + "\n", + " Result from claude-2 {'choices': [{'finish_reason': 'stop', 'index': 0, 'message': {'role': 'assistant', 'content': ' The Los Angeles Dodgers won'}}], 'created': 1691536677.944753, 'model': 'claude-2', 'usage': {'prompt_tokens': 30, 'completion_tokens': 5, 'total_tokens': 35}}\n" + ] + } + ], "source": [ "# @title Request Claude Instant-1 and Claude-2\n", "messages = [\n", @@ -89,44 +97,11 @@ "print(\"\\n\\n Result from claude-instant-1\", result)\n", "result = litellm.completion('claude-2', messages, max_tokens=5, temperature=0.2)\n", "print(\"\\n\\n Result from claude-2\", result)" - ], - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "ff1lKwUMMLJj", - "outputId": "bfddf6f8-36d4-45e5-92dc-349083fa41b8" - }, - "execution_count": 9, - "outputs": [ - { - "output_type": "stream", - "name": "stdout", - "text": [ - "\n", - "\n", - " Result from claude-instant-1 {'choices': [{'finish_reason': 'stop', 'index': 0, 'message': {'role': 'assistant', 'content': \" The Los Angeles Dodgers won the 2020 World Series, defeating the Tampa Bay Rays 4-2. It was the Dodgers' first World Series title since 1988.\"}}], 'created': 1691536677.2676156, 'model': 'claude-instant-1', 'usage': {'prompt_tokens': 30, 'completion_tokens': 32, 'total_tokens': 62}}\n", - "\n", - "\n", - " Result from claude-2 {'choices': [{'finish_reason': 'stop', 'index': 0, 'message': {'role': 'assistant', 'content': ' The Los Angeles Dodgers won'}}], 'created': 1691536677.944753, 'model': 'claude-2', 'usage': {'prompt_tokens': 30, 'completion_tokens': 5, 'total_tokens': 35}}\n" - ] - } ] }, { "cell_type": "code", - "source": [ - "# @title Streaming Example: Request Claude-2\n", - "messages = [\n", - " {\"role\": \"system\", \"content\": \"You are a helpful assistant.\"},\n", - " {\"role\": \"user\", \"content\": \"how does a court case get to the Supreme Court?\"}\n", - " ]\n", - "\n", - "result = litellm.completion('claude-2', messages, stream=True)\n", - "for chunk in result:\n", - " print(chunk['choices'][0]['delta'])\n", - "\n" - ], + "execution_count": 7, "metadata": { "colab": { "base_uri": "https://localhost:8080/" @@ -134,11 +109,10 @@ "id": "06hWKnNQMrV-", "outputId": "7fdec0eb-d4a9-4882-f9c4-987ff9a31114" }, - "execution_count": 7, "outputs": [ { - "output_type": "stream", "name": "stdout", + "output_type": "stream", "text": [ " Here\n", "'s\n", @@ -400,7 +374,33 @@ " precedent\n" ] } + ], + "source": [ + "# @title Streaming Example: Request Claude-2\n", + "messages = [\n", + " {\"role\": \"system\", \"content\": \"You are a helpful assistant.\"},\n", + " {\"role\": \"user\", \"content\": \"how does a court case get to the Supreme Court?\"}\n", + " ]\n", + "\n", + "result = litellm.completion('claude-2', messages, stream=True)\n", + "for part in result:\n", + " print(part.choices[0].delta.content or \"\")\n", + "\n" ] } - ] -} \ No newline at end of file + ], + "metadata": { + "colab": { + "provenance": [] + }, + "kernelspec": { + "display_name": "Python 3", + "name": "python3" + }, + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +}