forked from phoenix/litellm-mirror
update cookbook
This commit is contained in:
ishaan-jaff
parent
8a2be96c67
commit
5914dbb666
2 changed files with 198 additions and 58 deletions
198
cookbook/LiteLLM_AB_TestLLMs.ipynb
vendored
Normal file
198
cookbook/LiteLLM_AB_TestLLMs.ipynb
vendored
Normal file
|
|
@ -0,0 +1,198 @@
|
||||||
|
{
|
||||||
|
"nbformat": 4,
|
||||||
|
"nbformat_minor": 0,
|
||||||
|
"metadata": {
|
||||||
|
"colab": {
|
||||||
|
"provenance": []
|
||||||
|
},
|
||||||
|
"kernelspec": {
|
||||||
|
"name": "python3",
|
||||||
|
"display_name": "Python 3"
|
||||||
|
},
|
||||||
|
"language_info": {
|
||||||
|
"name": "python"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"cells": [
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"source": [
|
||||||
|
"# LiteLLM A/B Testing LLMs in production\n",
|
||||||
|
"\n",
|
||||||
|
"* LiteLLM allows you to use 100+ LLMs as a drop in replacement for `gpt-3.5-turbo`\n",
|
||||||
|
"\n",
|
||||||
|
"This tutorial walks through how to use LiteLLM to easily A/B Test LLMs in production"
|
||||||
|
],
|
||||||
|
"metadata": {
|
||||||
|
"id": "ODpmJQ5u4rXI"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"source": [
|
||||||
|
"# Example 1: A/B Test GPT-4 & GPT-3.5\n",
|
||||||
|
"\n",
|
||||||
|
"# Step 1\n",
|
||||||
|
"👉 Get your `id` from here: https://admin.litellm.ai/"
|
||||||
|
],
|
||||||
|
"metadata": {
|
||||||
|
"id": "YamUetcC5Ke7"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"source": [
|
||||||
|
"from litellm import completion_with_split_tests\n",
|
||||||
|
"import os\n",
|
||||||
|
"\n",
|
||||||
|
"## set ENV variables\n",
|
||||||
|
"os.environ[\"OPENAI_API_KEY\"] = \"\"\n",
|
||||||
|
"\n",
|
||||||
|
"\n",
|
||||||
|
"# define a dict of model id and % of requests for model\n",
|
||||||
|
"# see models here: https://docs.litellm.ai/docs/completion/supported\n",
|
||||||
|
"split_per_model = {\n",
|
||||||
|
"\t\"gpt-4\": 0.3,\n",
|
||||||
|
"\t\"gpt-3.5-turbo\": 0.7\n",
|
||||||
|
"}\n",
|
||||||
|
"\n",
|
||||||
|
"messages = [{ \"content\": \"Hello, how are you?\",\"role\": \"user\"}]\n",
|
||||||
|
"\n",
|
||||||
|
"completion_with_split_tests(messages=messages, use_client=True,\n",
|
||||||
|
" id=\"91fad14a-8c0f-4e99-8eaa-68245435aa80\") # enter your id"
|
||||||
|
],
|
||||||
|
"metadata": {
|
||||||
|
"colab": {
|
||||||
|
"base_uri": "https://localhost:8080/"
|
||||||
|
},
|
||||||
|
"id": "7XGfv0958k70",
|
||||||
|
"outputId": "91a069a5-c7d4-4fb0-e345-5ebf383edbbc"
|
||||||
|
},
|
||||||
|
"execution_count": 4,
|
||||||
|
"outputs": [
|
||||||
|
{
|
||||||
|
"output_type": "stream",
|
||||||
|
"name": "stdout",
|
||||||
|
"text": [
|
||||||
|
"last_fetched_at: 1693624804.2941535\n"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"output_type": "execute_result",
|
||||||
|
"data": {
|
||||||
|
"text/plain": [
|
||||||
|
"<OpenAIObject chat.completion id=chatcmpl-7uBT4QHc8BAoZKkU7JoH4ahmXvu0M at 0x7c2895c9e890> JSON: {\n",
|
||||||
|
" \"id\": \"chatcmpl-7uBT4QHc8BAoZKkU7JoH4ahmXvu0M\",\n",
|
||||||
|
" \"object\": \"chat.completion\",\n",
|
||||||
|
" \"created\": 1693624806,\n",
|
||||||
|
" \"model\": \"gpt-3.5-turbo-0613\",\n",
|
||||||
|
" \"choices\": [\n",
|
||||||
|
" {\n",
|
||||||
|
" \"index\": 0,\n",
|
||||||
|
" \"message\": {\n",
|
||||||
|
" \"role\": \"assistant\",\n",
|
||||||
|
" \"content\": \"Hello! I'm an AI, so I don't have emotions, but I'm here to assist you. How can I help you today?\"\n",
|
||||||
|
" },\n",
|
||||||
|
" \"finish_reason\": \"stop\"\n",
|
||||||
|
" }\n",
|
||||||
|
" ],\n",
|
||||||
|
" \"usage\": {\n",
|
||||||
|
" \"prompt_tokens\": 13,\n",
|
||||||
|
" \"completion_tokens\": 29,\n",
|
||||||
|
" \"total_tokens\": 42\n",
|
||||||
|
" }\n",
|
||||||
|
"}"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"metadata": {},
|
||||||
|
"execution_count": 4
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"source": [
|
||||||
|
"## A/B Test GPT-4 and Claude-2"
|
||||||
|
],
|
||||||
|
"metadata": {
|
||||||
|
"id": "Y12cxhZt58v8"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 5,
|
||||||
|
"metadata": {
|
||||||
|
"colab": {
|
||||||
|
"base_uri": "https://localhost:8080/"
|
||||||
|
},
|
||||||
|
"id": "0k6rshtR8i9q",
|
||||||
|
"outputId": "31ac9d73-9e35-4697-d1ff-5d51048566f8"
|
||||||
|
},
|
||||||
|
"outputs": [
|
||||||
|
{
|
||||||
|
"output_type": "stream",
|
||||||
|
"name": "stdout",
|
||||||
|
"text": [
|
||||||
|
"last_fetched_at: 1693624809.3467667\n"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"output_type": "execute_result",
|
||||||
|
"data": {
|
||||||
|
"text/plain": [
|
||||||
|
"<OpenAIObject chat.completion id=chatcmpl-7uBTA6gotsTksvCU7GffJ64ybfHUw at 0x7c28aa288630> JSON: {\n",
|
||||||
|
" \"id\": \"chatcmpl-7uBTA6gotsTksvCU7GffJ64ybfHUw\",\n",
|
||||||
|
" \"object\": \"chat.completion\",\n",
|
||||||
|
" \"created\": 1693624812,\n",
|
||||||
|
" \"model\": \"gpt-4-0613\",\n",
|
||||||
|
" \"choices\": [\n",
|
||||||
|
" {\n",
|
||||||
|
" \"index\": 0,\n",
|
||||||
|
" \"message\": {\n",
|
||||||
|
" \"role\": \"assistant\",\n",
|
||||||
|
" \"content\": \"As an AI, I don't have feelings, but I'm here and ready to assist you. How can I help you today?\"\n",
|
||||||
|
" },\n",
|
||||||
|
" \"finish_reason\": \"stop\"\n",
|
||||||
|
" }\n",
|
||||||
|
" ],\n",
|
||||||
|
" \"usage\": {\n",
|
||||||
|
" \"prompt_tokens\": 13,\n",
|
||||||
|
" \"completion_tokens\": 27,\n",
|
||||||
|
" \"total_tokens\": 40\n",
|
||||||
|
" }\n",
|
||||||
|
"}"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"metadata": {},
|
||||||
|
"execution_count": 5
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"source": [
|
||||||
|
"from litellm import completion_with_split_tests\n",
|
||||||
|
"import os\n",
|
||||||
|
"\n",
|
||||||
|
"## set ENV variables\n",
|
||||||
|
"os.environ[\"ANTHROPIC_API_KEY\"] = \"\"\n",
|
||||||
|
"\n",
|
||||||
|
"# define a dict of model id and % of requests for model\n",
|
||||||
|
"split_per_model = {\n",
|
||||||
|
"\t\"gpt-4\": 0.3,\n",
|
||||||
|
"\t\"claude-2\": 0.7\n",
|
||||||
|
"}\n",
|
||||||
|
"\n",
|
||||||
|
"messages = [{ \"content\": \"Hello, how are you?\",\"role\": \"user\"}]\n",
|
||||||
|
"\n",
|
||||||
|
"\n",
|
||||||
|
"completion_with_split_tests(messages=messages, use_client=True,\n",
|
||||||
|
" id=\"91fad14a-8c0f-4e99-8eaa-68245435aa80\") # enter your id"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"source": [],
|
||||||
|
"metadata": {
|
||||||
|
"id": "hzzbsAIp4pnr"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
|
@ -1,58 +0,0 @@
|
||||||
# -*- coding: utf-8 -*-
|
|
||||||
"""LiteLLM_AB_TestLLMs.ipynb
|
|
||||||
|
|
||||||
Automatically generated by Colaboratory.
|
|
||||||
|
|
||||||
Original file is located at
|
|
||||||
https://colab.research.google.com/drive/1Y7zcpuJT5rqJYEELeg9UfP1Zf8gFVR_o
|
|
||||||
|
|
||||||
# LiteLLM A/B Testing LLMs in production
|
|
||||||
|
|
||||||
* LiteLLM allows you to use 100+ LLMs as a drop in replacement for `gpt-3.5-turbo`
|
|
||||||
|
|
||||||
This tutorial walks through how to use LiteLLM to easily A/B Test LLMs in production
|
|
||||||
|
|
||||||
# Example 1: A/B Test GPT-4 & GPT-3.5
|
|
||||||
|
|
||||||
# Step 1
|
|
||||||
👉 Get your `id` from here: https://admin.litellm.ai/
|
|
||||||
"""
|
|
||||||
|
|
||||||
from litellm import completion_with_split_tests
|
|
||||||
import os
|
|
||||||
|
|
||||||
## set ENV variables
|
|
||||||
os.environ["OPENAI_API_KEY"] = "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"
|
|
||||||
|
|
||||||
|
|
||||||
# define a dict of model id and % of requests for model
|
|
||||||
# see models here: https://docs.litellm.ai/docs/completion/supported
|
|
||||||
split_per_model = {
|
|
||||||
"gpt-4": 0.3,
|
|
||||||
"gpt-3.5-turbo": 0.7
|
|
||||||
}
|
|
||||||
|
|
||||||
messages = [{ "content": "Hello, how are you?","role": "user"}]
|
|
||||||
|
|
||||||
completion_with_split_tests(messages=messages, use_client=True,
|
|
||||||
id="91fad14a-8c0f-4e99-8eaa-68245435aa80") # enter your id
|
|
||||||
|
|
||||||
"""## A/B Test GPT-4 and Claude-2"""
|
|
||||||
|
|
||||||
from litellm import completion_with_split_tests
|
|
||||||
import os
|
|
||||||
|
|
||||||
## set ENV variables
|
|
||||||
os.environ["ANTHROPIC_API_KEY"] = ""
|
|
||||||
|
|
||||||
# define a dict of model id and % of requests for model
|
|
||||||
split_per_model = {
|
|
||||||
"gpt-4": 0.3,
|
|
||||||
"claude-2": 0.7
|
|
||||||
}
|
|
||||||
|
|
||||||
messages = [{ "content": "Hello, how are you?","role": "user"}]
|
|
||||||
|
|
||||||
|
|
||||||
completion_with_split_tests(messages=messages, use_client=True,
|
|
||||||
id="91fad14a-8c0f-4e99-8eaa-68245435aa80") # enter your id
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue