phoenix/litellm-mirror
0
0
Fork 1
mirror of https://github.com/BerriAI/litellm.git synced 2025-04-25 10:44:24 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

tg ai demo

Browse source
This commit is contained in:
ishaan-jaff 2023-08-09 18:24:00 -07:00
parent 654e413500
commit 19008e1b4d
1 changed files with 142 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

142
cookbook/TogetherAI_liteLLM.ipynb Normal file
View file

@ -0,0 +1,142 @@
{
"nbformat": 4,
"nbformat_minor": 0,
"metadata": {
"colab": {
"provenance": []
},
"kernelspec": {
"name": "python3",
"display_name": "Python 3"
},
"language_info": {
"name": "python"
}
},
"cells": [
{
"cell_type": "markdown",
"source": [
"## liteLLM Together AI Tutorial\n",
"https://together.ai/\n"
],
"metadata": {
"id": "WemkFEdDAnJL"
}
},
{
"cell_type": "code",
"source": [
"!pip install litellm==0.1.371"
],
"metadata": {
"id": "pc6IO4V99O25"
},
"execution_count": null,
"outputs": []
},
{
"cell_type": "code",
"source": [
"import os\n",
"from litellm import completion\n",
"os.environ[\"TOGETHER_AI_TOKEN\"] = \"\" #@param\n",
"user_message = \"Hello, whats the weather in San Francisco??\"\n",
"messages = [{ \"content\": user_message,\"role\": \"user\"}]"
],
"metadata": {
"id": "TMI3739_9q97"
},
"execution_count": 5,
"outputs": []
},
{
"cell_type": "markdown",
"source": [
"## Calling togethercomputer/llama-2-70b-chat\n",
"https://api.together.xyz/playground/chat?model=togethercomputer%2Fllama-2-70b-chat"
],
"metadata": {
"id": "bEqJ2HHjBJqq"
}
},
{
"cell_type": "code",
"source": [
"model_name = \"togethercomputer/llama-2-70b-chat\"\n",
"response = completion(model=model_name, messages=messages, together_ai=True)\n",
"print(response)"
],
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "Jrrt8puj523f",
"outputId": "5a5b5beb-cda3-413e-8e83-4423d392cb44"
},
"execution_count": 6,
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"{'choices': [{'finish_reason': 'stop', 'index': 0, 'message': {'role': 'assistant', 'content': \"\\n\\nI'm not able to provide real-time weather information. However, I can suggest\"}}], 'created': 1691629657.9288375, 'model': 'togethercomputer/llama-2-70b-chat', 'usage': {'prompt_tokens': 9, 'completion_tokens': 17, 'total_tokens': 26}}\n"
]
}
]
},
{
"cell_type": "markdown",
"source": [
"## With Streaming"
],
"metadata": {
"id": "sfWtgf-mBQcM"
}
},
{
"cell_type": "code",
"source": [
"response = completion(model=model_name, messages=messages, together_ai=True, stream=True)\n",
"print(response)\n",
"for chunk in response:\n",
" print(chunk['choices'][0]['delta']) # same as openai format"
],
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "wuBhlZtC6MH5",
"outputId": "fcb82177-6494-4963-8e37-8716d3b9e616"
},
"execution_count": 8,
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"<litellm.utils.CustomStreamWrapper object at 0x7ad005e93ee0>\n",
"{'role': 'assistant', 'content': '\\\\n'}\n",
"{'role': 'assistant', 'content': '\\\\n'}\n",
"{'role': 'assistant', 'content': 'I'}\n",
"{'role': 'assistant', 'content': 'm'}\n",
"{'role': 'assistant', 'content': ' not'}\n",
"{'role': 'assistant', 'content': ' able'}\n",
"{'role': 'assistant', 'content': ' to'}\n",
"{'role': 'assistant', 'content': ' provide'}\n",
"{'role': 'assistant', 'content': ' real'}\n",
"{'role': 'assistant', 'content': '-'}\n",
"{'role': 'assistant', 'content': 'time'}\n",
"{'role': 'assistant', 'content': ' weather'}\n",
"{'role': 'assistant', 'content': ' information'}\n",
"{'role': 'assistant', 'content': '.'}\n",
"{'role': 'assistant', 'content': ' However'}\n",
"{'role': 'assistant', 'content': ','}\n",
"{'role': 'assistant', 'content': ' I'}\n",
"{'role': 'assistant', 'content': ' can'}\n"
]
}
]
}
]
}