diff --git a/cookbook/LiteLLM_AB_TestLLMs.ipynb b/cookbook/LiteLLM_AB_TestLLMs.ipynb new file mode 100644 index 000000000..c0d710162 --- /dev/null +++ b/cookbook/LiteLLM_AB_TestLLMs.ipynb @@ -0,0 +1,198 @@ +{ + "nbformat": 4, + "nbformat_minor": 0, + "metadata": { + "colab": { + "provenance": [] + }, + "kernelspec": { + "name": "python3", + "display_name": "Python 3" + }, + "language_info": { + "name": "python" + } + }, + "cells": [ + { + "cell_type": "markdown", + "source": [ + "# LiteLLM A/B Testing LLMs in production\n", + "\n", + "* LiteLLM allows you to use 100+ LLMs as a drop in replacement for `gpt-3.5-turbo`\n", + "\n", + "This tutorial walks through how to use LiteLLM to easily A/B Test LLMs in production" + ], + "metadata": { + "id": "ODpmJQ5u4rXI" + } + }, + { + "cell_type": "markdown", + "source": [ + "# Example 1: A/B Test GPT-4 & GPT-3.5\n", + "\n", + "# Step 1\n", + "👉 Get your `id` from here: https://admin.litellm.ai/" + ], + "metadata": { + "id": "YamUetcC5Ke7" + } + }, + { + "cell_type": "code", + "source": [ + "from litellm import completion_with_split_tests\n", + "import os\n", + "\n", + "## set ENV variables\n", + "os.environ[\"OPENAI_API_KEY\"] = \"\"\n", + "\n", + "\n", + "# define a dict of model id and % of requests for model\n", + "# see models here: https://docs.litellm.ai/docs/completion/supported\n", + "split_per_model = {\n", + "\t\"gpt-4\": 0.3,\n", + "\t\"gpt-3.5-turbo\": 0.7\n", + "}\n", + "\n", + "messages = [{ \"content\": \"Hello, how are you?\",\"role\": \"user\"}]\n", + "\n", + "completion_with_split_tests(messages=messages, use_client=True,\n", + " id=\"91fad14a-8c0f-4e99-8eaa-68245435aa80\") # enter your id" + ], + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "7XGfv0958k70", + "outputId": "91a069a5-c7d4-4fb0-e345-5ebf383edbbc" + }, + "execution_count": 4, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "last_fetched_at: 1693624804.2941535\n" + ] + }, + { + "output_type": "execute_result", + "data": { + "text/plain": [ + " JSON: {\n", + " \"id\": \"chatcmpl-7uBT4QHc8BAoZKkU7JoH4ahmXvu0M\",\n", + " \"object\": \"chat.completion\",\n", + " \"created\": 1693624806,\n", + " \"model\": \"gpt-3.5-turbo-0613\",\n", + " \"choices\": [\n", + " {\n", + " \"index\": 0,\n", + " \"message\": {\n", + " \"role\": \"assistant\",\n", + " \"content\": \"Hello! I'm an AI, so I don't have emotions, but I'm here to assist you. How can I help you today?\"\n", + " },\n", + " \"finish_reason\": \"stop\"\n", + " }\n", + " ],\n", + " \"usage\": {\n", + " \"prompt_tokens\": 13,\n", + " \"completion_tokens\": 29,\n", + " \"total_tokens\": 42\n", + " }\n", + "}" + ] + }, + "metadata": {}, + "execution_count": 4 + } + ] + }, + { + "cell_type": "markdown", + "source": [ + "## A/B Test GPT-4 and Claude-2" + ], + "metadata": { + "id": "Y12cxhZt58v8" + } + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "0k6rshtR8i9q", + "outputId": "31ac9d73-9e35-4697-d1ff-5d51048566f8" + }, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "last_fetched_at: 1693624809.3467667\n" + ] + }, + { + "output_type": "execute_result", + "data": { + "text/plain": [ + " JSON: {\n", + " \"id\": \"chatcmpl-7uBTA6gotsTksvCU7GffJ64ybfHUw\",\n", + " \"object\": \"chat.completion\",\n", + " \"created\": 1693624812,\n", + " \"model\": \"gpt-4-0613\",\n", + " \"choices\": [\n", + " {\n", + " \"index\": 0,\n", + " \"message\": {\n", + " \"role\": \"assistant\",\n", + " \"content\": \"As an AI, I don't have feelings, but I'm here and ready to assist you. How can I help you today?\"\n", + " },\n", + " \"finish_reason\": \"stop\"\n", + " }\n", + " ],\n", + " \"usage\": {\n", + " \"prompt_tokens\": 13,\n", + " \"completion_tokens\": 27,\n", + " \"total_tokens\": 40\n", + " }\n", + "}" + ] + }, + "metadata": {}, + "execution_count": 5 + } + ], + "source": [ + "from litellm import completion_with_split_tests\n", + "import os\n", + "\n", + "## set ENV variables\n", + "os.environ[\"ANTHROPIC_API_KEY\"] = \"\"\n", + "\n", + "# define a dict of model id and % of requests for model\n", + "split_per_model = {\n", + "\t\"gpt-4\": 0.3,\n", + "\t\"claude-2\": 0.7\n", + "}\n", + "\n", + "messages = [{ \"content\": \"Hello, how are you?\",\"role\": \"user\"}]\n", + "\n", + "\n", + "completion_with_split_tests(messages=messages, use_client=True,\n", + " id=\"91fad14a-8c0f-4e99-8eaa-68245435aa80\") # enter your id" + ] + }, + { + "cell_type": "markdown", + "source": [], + "metadata": { + "id": "hzzbsAIp4pnr" + } + } + ] +} \ No newline at end of file diff --git a/cookbook/litellm_ab_testllms.py b/cookbook/litellm_ab_testllms.py deleted file mode 100644 index ee77442c5..000000000 --- a/cookbook/litellm_ab_testllms.py +++ /dev/null @@ -1,58 +0,0 @@ -# -*- coding: utf-8 -*- -"""LiteLLM_AB_TestLLMs.ipynb - -Automatically generated by Colaboratory. - -Original file is located at - https://colab.research.google.com/drive/1Y7zcpuJT5rqJYEELeg9UfP1Zf8gFVR_o - -# LiteLLM A/B Testing LLMs in production - -* LiteLLM allows you to use 100+ LLMs as a drop in replacement for `gpt-3.5-turbo` - -This tutorial walks through how to use LiteLLM to easily A/B Test LLMs in production - -# Example 1: A/B Test GPT-4 & GPT-3.5 - -# Step 1 -👉 Get your `id` from here: https://admin.litellm.ai/ -""" - -from litellm import completion_with_split_tests -import os - -## set ENV variables -os.environ["OPENAI_API_KEY"] = "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" - - -# define a dict of model id and % of requests for model -# see models here: https://docs.litellm.ai/docs/completion/supported -split_per_model = { - "gpt-4": 0.3, - "gpt-3.5-turbo": 0.7 -} - -messages = [{ "content": "Hello, how are you?","role": "user"}] - -completion_with_split_tests(messages=messages, use_client=True, - id="91fad14a-8c0f-4e99-8eaa-68245435aa80") # enter your id - -"""## A/B Test GPT-4 and Claude-2""" - -from litellm import completion_with_split_tests -import os - -## set ENV variables -os.environ["ANTHROPIC_API_KEY"] = "" - -# define a dict of model id and % of requests for model -split_per_model = { - "gpt-4": 0.3, - "claude-2": 0.7 -} - -messages = [{ "content": "Hello, how are you?","role": "user"}] - - -completion_with_split_tests(messages=messages, use_client=True, - id="91fad14a-8c0f-4e99-8eaa-68245435aa80") # enter your id \ No newline at end of file