phoenix/litellm-mirror
0
0
Fork 1
mirror of https://github.com/BerriAI/litellm.git synced 2025-04-24 10:14:26 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

(docs) update how to use litellm streaming

Browse source
This commit is contained in:
ishaan-jaff 2023-11-21 11:19:32 -08:00
parent 37187e8e88
commit f29a353796
2 changed files with 64 additions and 64 deletions
Download patch file Download diff file Expand all files Collapse all files

10
README.md
View file

@ -68,13 +68,13 @@ Streaming is supported for all models (Bedrock, Huggingface, TogetherAI, Azure,
```python
from litellm import completion
response = completion(model="gpt-3.5-turbo", messages=messages, stream=True)
for chunk in response:
print(chunk['choices'][0]['delta'])
for part in response:
print(part.choices[0].delta.content or "")
# claude 2
result = completion('claude-2', messages, stream=True)
for chunk in result:
print(chunk['choices'][0]['delta'])
response = completion('claude-2', messages, stream=True)
for part in response:
print(part.choices[0].delta.content or "")
```
# Router - load balancing([Docs](https://docs.litellm.ai/docs/routing))

118
cookbook/Claude_(Anthropic)_with_Streaming_liteLLM_Examples.ipynb vendored
View file

@ -1,18 +1,4 @@
{
"nbformat": 4,
"nbformat_minor": 0,
"metadata": {
"colab": {
"provenance": []
},
"kernelspec": {
"name": "python3",
"display_name": "Python 3"
},
"language_info": {
"name": "python"
}
},
"cells": [
{
"cell_type": "code",
@ -26,8 +12,8 @@
},
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"output_type": "stream",
"text": [
"Collecting litellm==0.1.363\n",
" Downloading litellm-0.1.363-py3-none-any.whl (34 kB)\n",
@ -63,21 +49,43 @@
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {
"id": "W216G__XL19Q"
},
"outputs": [],
"source": [
"# @title Import litellm & Set env variables\n",
"import litellm\n",
"import os\n",
"\n",
"os.environ[\"ANTHROPIC_API_KEY\"] = \" \" #@param"
],
"metadata": {
"id": "W216G__XL19Q"
},
"execution_count": 2,
"outputs": []
]
},
{
"cell_type": "code",
"execution_count": 9,
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "ff1lKwUMMLJj",
"outputId": "bfddf6f8-36d4-45e5-92dc-349083fa41b8"
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"\n",
"\n",
" Result from claude-instant-1 {'choices': [{'finish_reason': 'stop', 'index': 0, 'message': {'role': 'assistant', 'content': \" The Los Angeles Dodgers won the 2020 World Series, defeating the Tampa Bay Rays 4-2. It was the Dodgers' first World Series title since 1988.\"}}], 'created': 1691536677.2676156, 'model': 'claude-instant-1', 'usage': {'prompt_tokens': 30, 'completion_tokens': 32, 'total_tokens': 62}}\n",
"\n",
"\n",
" Result from claude-2 {'choices': [{'finish_reason': 'stop', 'index': 0, 'message': {'role': 'assistant', 'content': ' The Los Angeles Dodgers won'}}], 'created': 1691536677.944753, 'model': 'claude-2', 'usage': {'prompt_tokens': 30, 'completion_tokens': 5, 'total_tokens': 35}}\n"
]
}
],
"source": [
"# @title Request Claude Instant-1 and Claude-2\n",
"messages = [\n",
@ -89,44 +97,11 @@
"print(\"\\n\\n Result from claude-instant-1\", result)\n",
"result = litellm.completion('claude-2', messages, max_tokens=5, temperature=0.2)\n",
"print(\"\\n\\n Result from claude-2\", result)"
],
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "ff1lKwUMMLJj",
"outputId": "bfddf6f8-36d4-45e5-92dc-349083fa41b8"
},
"execution_count": 9,
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"\n",
"\n",
" Result from claude-instant-1 {'choices': [{'finish_reason': 'stop', 'index': 0, 'message': {'role': 'assistant', 'content': \" The Los Angeles Dodgers won the 2020 World Series, defeating the Tampa Bay Rays 4-2. It was the Dodgers' first World Series title since 1988.\"}}], 'created': 1691536677.2676156, 'model': 'claude-instant-1', 'usage': {'prompt_tokens': 30, 'completion_tokens': 32, 'total_tokens': 62}}\n",
"\n",
"\n",
" Result from claude-2 {'choices': [{'finish_reason': 'stop', 'index': 0, 'message': {'role': 'assistant', 'content': ' The Los Angeles Dodgers won'}}], 'created': 1691536677.944753, 'model': 'claude-2', 'usage': {'prompt_tokens': 30, 'completion_tokens': 5, 'total_tokens': 35}}\n"
]
}
]
},
{
"cell_type": "code",
"source": [
"# @title Streaming Example: Request Claude-2\n",
"messages = [\n",
" {\"role\": \"system\", \"content\": \"You are a helpful assistant.\"},\n",
" {\"role\": \"user\", \"content\": \"how does a court case get to the Supreme Court?\"}\n",
" ]\n",
"\n",
"result = litellm.completion('claude-2', messages, stream=True)\n",
"for chunk in result:\n",
" print(chunk['choices'][0]['delta'])\n",
"\n"
],
"execution_count": 7,
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
@ -134,11 +109,10 @@
"id": "06hWKnNQMrV-",
"outputId": "7fdec0eb-d4a9-4882-f9c4-987ff9a31114"
},
"execution_count": 7,
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"output_type": "stream",
"text": [
" Here\n",
"'s\n",
@ -400,7 +374,33 @@
" precedent\n"
]
}
],
"source": [
"# @title Streaming Example: Request Claude-2\n",
"messages = [\n",
" {\"role\": \"system\", \"content\": \"You are a helpful assistant.\"},\n",
" {\"role\": \"user\", \"content\": \"how does a court case get to the Supreme Court?\"}\n",
" ]\n",
"\n",
"result = litellm.completion('claude-2', messages, stream=True)\n",
"for part in result:\n",
" print(part.choices[0].delta.content or \"\")\n",
"\n"
]
}
]
}
],
"metadata": {
"colab": {
"provenance": []
},
"kernelspec": {
"display_name": "Python 3",
"name": "python3"
},
"language_info": {
"name": "python"
}
},
"nbformat": 4,
"nbformat_minor": 0
}