From a775e9b27820afa22b712a91963c9cc95c959346 Mon Sep 17 00:00:00 2001 From: ishaan-jaff Date: Sat, 23 Sep 2023 10:38:59 -0700 Subject: [PATCH] getting started cookbook litellm --- ...AI.ipynb => liteLLM_Getting_Started.ipynb} | 186 ++++++++++++------ 1 file changed, 123 insertions(+), 63 deletions(-) rename cookbook/{liteLLM_OpenAI.ipynb => liteLLM_Getting_Started.ipynb} (51%) diff --git a/cookbook/liteLLM_OpenAI.ipynb b/cookbook/liteLLM_Getting_Started.ipynb similarity index 51% rename from cookbook/liteLLM_OpenAI.ipynb rename to cookbook/liteLLM_Getting_Started.ipynb index 79ef6a7d3..b43c51dce 100644 --- a/cookbook/liteLLM_OpenAI.ipynb +++ b/cookbook/liteLLM_Getting_Started.ipynb @@ -7,15 +7,11 @@ "id": "MZ01up0p7wOJ" }, "source": [ - "## 🚅 liteLLM Demo\n", + "## 🚅 liteLLM Quick Start Demo\n", "### TLDR: Call 50+ LLM APIs using chatGPT Input/Output format\n", "https://github.com/BerriAI/litellm\n", "\n", "liteLLM is package to simplify calling **OpenAI, Azure, Llama2, Cohere, Anthropic, Huggingface API Endpoints**. LiteLLM manages\n", - "\n", - "* Translating inputs to the provider's `completion()` and `embedding()` endpoints\n", - "* Guarantees consistent output, text responses will always be available at `['choices'][0]['message']['content']`\n", - "* Exception mapping - common exceptions across providers are mapped to the OpenAI exception types\n", "\n" ] }, @@ -42,7 +38,7 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": 2, "metadata": { "id": "ArrWyG5b7QAG" }, @@ -67,7 +63,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 18, "metadata": { "id": "-h8Ga5cR7SvV" }, @@ -76,23 +72,11 @@ "# Only set keys for the LLMs you want to use\n", "os.environ['OPENAI_API_KEY'] = \"\" #@param\n", "os.environ[\"ANTHROPIC_API_KEY\"] = \"\" #@param\n", + "os.environ[\"REPLICATE_API_KEY\"] = \"\" #@param\n", + "os.environ[\"COHERE_API_KEY\"] = \"\" #@param\n", "os.environ[\"AZURE_API_BASE\"] = \"\" #@param\n", "os.environ[\"AZURE_API_VERSION\"] = \"\" #@param\n", - "os.environ[\"AZURE_API_KEY\"] = \"\" #@param\n", - "os.environ[\"REPLICATE_API_TOKEN\"] = \"\" #@param\n", - "os.environ[\"COHERE_API_KEY\"] = \"\" #@param\n", - "os.environ[\"HF_TOKEN\"] = \"\" #@param" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": { - "id": "MBujGiby8YBu" - }, - "outputs": [], - "source": [ - "messages = [{ \"content\": \"what's the weather in SF\",\"role\": \"user\"}]" + "os.environ[\"AZURE_API_KEY\"] = \"\" #@param" ] }, { @@ -107,7 +91,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 13, "metadata": { "colab": { "base_uri": "https://localhost:8080/" @@ -119,36 +103,37 @@ { "data": { "text/plain": [ - " JSON: {\n", - " \"id\": \"chatcmpl-7vYWJYYUeFuhjCiOjI9JXK6gNmWk3\",\n", + " JSON: {\n", + " \"id\": \"chatcmpl-820kPkRwSLml4X6165fWbZlEDOedr\",\n", " \"object\": \"chat.completion\",\n", - " \"created\": 1693951747,\n", + " \"created\": 1695490221,\n", " \"model\": \"gpt-3.5-turbo-0613\",\n", " \"choices\": [\n", " {\n", " \"index\": 0,\n", " \"message\": {\n", " \"role\": \"assistant\",\n", - " \"content\": \"I apologize, but as an AI language model, I do not have real-time data. However, you can easily find the current weather conditions in San Francisco, California by checking a trusted weather website or using a weather app on your smartphone.\"\n", + " \"content\": \"I'm sorry, but as an AI text-based model, I don't have real-time information. However, you can check the current weather in San Francisco by searching for \\\"weather in SF\\\" on any search engine or checking a weather website or app.\"\n", " },\n", " \"finish_reason\": \"stop\"\n", " }\n", " ],\n", " \"usage\": {\n", " \"prompt_tokens\": 13,\n", - " \"completion_tokens\": 48,\n", - " \"total_tokens\": 61\n", - " }\n", + " \"completion_tokens\": 51,\n", + " \"total_tokens\": 64\n", + " },\n", + " \"response_ms\": 2385.592\n", "}" ] }, - "execution_count": 3, + "execution_count": 13, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "completion(model=\"gpt-3.5-turbo\", messages=messages)" + "completion(model=\"gpt-3.5-turbo\", messages=[{ \"content\": \"what's the weather in SF\",\"role\": \"user\"}])" ] }, { @@ -163,7 +148,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 19, "metadata": { "colab": { "base_uri": "https://localhost:8080/" @@ -175,22 +160,38 @@ { "data": { "text/plain": [ - "{'choices': [{'finish_reason': 'stop',\n", - " 'index': 0,\n", - " 'message': {'role': 'assistant',\n", - " 'content': \" Unfortunately I do not have enough context to provide the current weather in San Francisco. To get the most accurate weather report, it's helpful if I know details like:\\n\\n- Exact location (city name, zip code, etc)\\n- Time frame (current conditions, forecast for a certain day/week, etc)\\n\\nIf you can provide some more specifics about what weather information you need for San Francisco, I'd be happy to look that up for you!\"}}],\n", - " 'created': 1691880836.974166,\n", - " 'model': 'claude-2',\n", - " 'usage': {'prompt_tokens': 18, 'completion_tokens': 95, 'total_tokens': 113}}" + " JSON: {\n", + " \"object\": \"chat.completion\",\n", + " \"choices\": [\n", + " {\n", + " \"finish_reason\": \"stop_sequence\",\n", + " \"index\": 0,\n", + " \"message\": {\n", + " \"content\": \" Unfortunately I don't have enough context to know the exact location you are asking about when you say \\\"SF\\\". SF could refer to San Francisco, California, or potentially other cities that go by SF as an abbreviation. To get an accurate weather report, it would be helpful if you could provide the full city name and state/country. If you are looking for the weather in San Francisco, California, I would be happy to provide that forecast. Please let me know the specific location you want the weather for.\",\n", + " \"role\": \"assistant\",\n", + " \"logprobs\": null\n", + " }\n", + " }\n", + " ],\n", + " \"id\": \"chatcmpl-6d1a40c0-19c0-4bd7-9ca2-a91d8b8c2295\",\n", + " \"created\": 1695490260.983768,\n", + " \"response_ms\": 6351.544,\n", + " \"model\": \"claude-2\",\n", + " \"usage\": {\n", + " \"prompt_tokens\": 14,\n", + " \"completion_tokens\": 102,\n", + " \"total_tokens\": 116\n", + " }\n", + "}" ] }, - "execution_count": 11, + "execution_count": 19, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "completion(model=\"claude-2\", messages=messages)" + "completion(model=\"claude-2\", messages=[{ \"content\": \"what's the weather in SF\",\"role\": \"user\"}])" ] }, { @@ -205,7 +206,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 17, "metadata": { "colab": { "base_uri": "https://localhost:8080/" @@ -217,23 +218,40 @@ { "data": { "text/plain": [ - "{'choices': [{'finish_reason': 'stop',\n", - " 'index': 0,\n", - " 'message': {'role': 'assistant',\n", - " 'content': ' I\\'m happy to help! However, I must point out that the question \"what\\'s the weather in SF\" doesn\\'t make sense as \"SF\" could refer to multiple locations (San Francisco, South Florida, San Fernando, etc.). Could you please provide more context or specify which location you\\'re referring to? That way, I can give you an accurate answer.'}}],\n", - " 'created': 1691880930.9003325,\n", - " 'model': 'replicate/llama-2-70b-chat:2c1608e18606fad2812020dc541930f2d0495ce32eee50074220b87300bc16e1',\n", - " 'usage': {'prompt_tokens': 6, 'completion_tokens': 74, 'total_tokens': 80}}" + " JSON: {\n", + " \"object\": \"chat.completion\",\n", + " \"choices\": [\n", + " {\n", + " \"finish_reason\": \"stop\",\n", + " \"index\": 0,\n", + " \"message\": {\n", + " \"content\": \" I'm happy to help! However, I must point out that the question \\\"what's the weather in SF\\\" doesn't make sense as \\\"SF\\\" could refer to multiple locations. Could you please clarify which location you are referring to? San Francisco, California or Sioux Falls, South Dakota? Once I have more context, I would be happy to provide you with accurate and reliable information.\",\n", + " \"role\": \"assistant\",\n", + " \"logprobs\": null\n", + " }\n", + " }\n", + " ],\n", + " \"id\": \"chatcmpl-3151c2eb-b26f-4c96-89b5-ed1746b219e0\",\n", + " \"created\": 1695490237.714101,\n", + " \"response_ms\": 12109.565,\n", + " \"model\": \"replicate/llama-2-70b-chat:2c1608e18606fad2812020dc541930f2d0495ce32eee50074220b87300bc16e1\",\n", + " \"usage\": {\n", + " \"prompt_tokens\": 6,\n", + " \"completion_tokens\": 78,\n", + " \"total_tokens\": 84\n", + " },\n", + " \"ended\": 1695490249.821266\n", + "}" ] }, - "execution_count": 13, + "execution_count": 17, "metadata": {}, "output_type": "execute_result" } ], "source": [ "model = \"replicate/llama-2-70b-chat:2c1608e18606fad2812020dc541930f2d0495ce32eee50074220b87300bc16e1\"\n", - "completion(model=model, messages=messages)" + "completion(model=model, messages=[{ \"content\": \"what's the weather in SF\",\"role\": \"user\"}])" ] }, { @@ -248,7 +266,7 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 16, "metadata": { "colab": { "base_uri": "https://localhost:8080/" @@ -260,35 +278,38 @@ { "data": { "text/plain": [ - " JSON: {\n", + " JSON: {\n", + " \"object\": \"chat.completion\",\n", " \"choices\": [\n", " {\n", " \"finish_reason\": \"stop\",\n", " \"index\": 0,\n", " \"message\": {\n", - " \"content\": \" The weather in San Francisco can be quite unpredictable and varies throughout the year. In general, the city\",\n", + " \"content\": \" As an AI model I don't have access to real-time data, so I can't tell\",\n", " \"role\": \"assistant\",\n", " \"logprobs\": null\n", " }\n", " }\n", " ],\n", - " \"created\": 1693951797.3149078,\n", + " \"id\": \"chatcmpl-dc0d8ead-071d-486c-a111-78975b38794b\",\n", + " \"created\": 1695490235.936903,\n", + " \"response_ms\": 1022.6759999999999,\n", " \"model\": \"command-nightly\",\n", " \"usage\": {\n", " \"prompt_tokens\": 6,\n", - " \"completion_tokens\": 20,\n", - " \"total_tokens\": 26\n", + " \"completion_tokens\": 19,\n", + " \"total_tokens\": 25\n", " }\n", "}" ] }, - "execution_count": 7, + "execution_count": 16, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "completion(model=\"command-nightly\", messages=messages)" + "completion(model=\"command-nightly\", messages=[{ \"content\": \"what's the weather in SF\",\"role\": \"user\"}])" ] }, { @@ -311,7 +332,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 15, "metadata": { "colab": { "base_uri": "https://localhost:8080/" @@ -319,10 +340,49 @@ "id": "AvLjR-PF-lt0", "outputId": "deff2db3-b003-48cd-ea62-c03a68a4464a" }, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + " JSON: {\n", + " \"id\": \"chatcmpl-820kZyCwbNvZATiLkNmXmpxxzvTKO\",\n", + " \"object\": \"chat.completion\",\n", + " \"created\": 1695490231,\n", + " \"model\": \"gpt-35-turbo\",\n", + " \"choices\": [\n", + " {\n", + " \"index\": 0,\n", + " \"finish_reason\": \"stop\",\n", + " \"message\": {\n", + " \"role\": \"assistant\",\n", + " \"content\": \"Sorry, as an AI language model, I don't have real-time information. Please check your preferred weather website or app for the latest weather updates of San Francisco.\"\n", + " }\n", + " }\n", + " ],\n", + " \"usage\": {\n", + " \"completion_tokens\": 33,\n", + " \"prompt_tokens\": 14,\n", + " \"total_tokens\": 47\n", + " },\n", + " \"response_ms\": 1499.529\n", + "}" + ] + }, + "execution_count": 15, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ - "completion(model=\"azure/chatgpt-test\", messages=messages)" + "completion(model=\"azure/chatgpt-v-2\", messages=[{ \"content\": \"what's the weather in SF\",\"role\": \"user\"}])" ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] } ], "metadata": {