forked from phoenix/litellm-mirror
(fix) remove bloat completion_with_split_tests
This commit is contained in:
ishaan-jaff
parent
224a0b9367
commit
d5e6777973
3 changed files with 0 additions and 311 deletions
198
cookbook/LiteLLM_AB_TestLLMs.ipynb
vendored
198
cookbook/LiteLLM_AB_TestLLMs.ipynb
vendored
|
|
@ -1,198 +0,0 @@
|
||||||
{
|
|
||||||
"nbformat": 4,
|
|
||||||
"nbformat_minor": 0,
|
|
||||||
"metadata": {
|
|
||||||
"colab": {
|
|
||||||
"provenance": []
|
|
||||||
},
|
|
||||||
"kernelspec": {
|
|
||||||
"name": "python3",
|
|
||||||
"display_name": "Python 3"
|
|
||||||
},
|
|
||||||
"language_info": {
|
|
||||||
"name": "python"
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"cells": [
|
|
||||||
{
|
|
||||||
"cell_type": "markdown",
|
|
||||||
"source": [
|
|
||||||
"# LiteLLM A/B Testing LLMs in production\n",
|
|
||||||
"\n",
|
|
||||||
"* LiteLLM allows you to use 100+ LLMs as a drop in replacement for `gpt-3.5-turbo`\n",
|
|
||||||
"\n",
|
|
||||||
"This tutorial walks through how to use LiteLLM to easily A/B Test LLMs in production"
|
|
||||||
],
|
|
||||||
"metadata": {
|
|
||||||
"id": "ODpmJQ5u4rXI"
|
|
||||||
}
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "markdown",
|
|
||||||
"source": [
|
|
||||||
"# Example 1: A/B Test GPT-4 & GPT-3.5\n",
|
|
||||||
"\n",
|
|
||||||
"# Step 1\n",
|
|
||||||
"👉 Get your `id` from here: https://admin.litellm.ai/"
|
|
||||||
],
|
|
||||||
"metadata": {
|
|
||||||
"id": "YamUetcC5Ke7"
|
|
||||||
}
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"source": [
|
|
||||||
"from litellm import completion_with_split_tests\n",
|
|
||||||
"import os\n",
|
|
||||||
"\n",
|
|
||||||
"## set ENV variables\n",
|
|
||||||
"os.environ[\"OPENAI_API_KEY\"] = \"\"\n",
|
|
||||||
"\n",
|
|
||||||
"\n",
|
|
||||||
"# define a dict of model id and % of requests for model\n",
|
|
||||||
"# see models here: https://docs.litellm.ai/docs/providers\n",
|
|
||||||
"split_per_model = {\n",
|
|
||||||
"\t\"gpt-4\": 0.3,\n",
|
|
||||||
"\t\"gpt-3.5-turbo\": 0.7\n",
|
|
||||||
"}\n",
|
|
||||||
"\n",
|
|
||||||
"messages = [{ \"content\": \"Hello, how are you?\",\"role\": \"user\"}]\n",
|
|
||||||
"\n",
|
|
||||||
"completion_with_split_tests(messages=messages, use_client=True,\n",
|
|
||||||
" id=\"91fad14a-8c0f-4e99-8eaa-68245435aa80\") # [Optional Set your own ID]"
|
|
||||||
],
|
|
||||||
"metadata": {
|
|
||||||
"colab": {
|
|
||||||
"base_uri": "https://localhost:8080/"
|
|
||||||
},
|
|
||||||
"id": "7XGfv0958k70",
|
|
||||||
"outputId": "91a069a5-c7d4-4fb0-e345-5ebf383edbbc"
|
|
||||||
},
|
|
||||||
"execution_count": 4,
|
|
||||||
"outputs": [
|
|
||||||
{
|
|
||||||
"output_type": "stream",
|
|
||||||
"name": "stdout",
|
|
||||||
"text": [
|
|
||||||
"last_fetched_at: 1693624804.2941535\n"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"output_type": "execute_result",
|
|
||||||
"data": {
|
|
||||||
"text/plain": [
|
|
||||||
"<OpenAIObject chat.completion id=chatcmpl-7uBT4QHc8BAoZKkU7JoH4ahmXvu0M at 0x7c2895c9e890> JSON: {\n",
|
|
||||||
" \"id\": \"chatcmpl-7uBT4QHc8BAoZKkU7JoH4ahmXvu0M\",\n",
|
|
||||||
" \"object\": \"chat.completion\",\n",
|
|
||||||
" \"created\": 1693624806,\n",
|
|
||||||
" \"model\": \"gpt-3.5-turbo-0613\",\n",
|
|
||||||
" \"choices\": [\n",
|
|
||||||
" {\n",
|
|
||||||
" \"index\": 0,\n",
|
|
||||||
" \"message\": {\n",
|
|
||||||
" \"role\": \"assistant\",\n",
|
|
||||||
" \"content\": \"Hello! I'm an AI, so I don't have emotions, but I'm here to assist you. How can I help you today?\"\n",
|
|
||||||
" },\n",
|
|
||||||
" \"finish_reason\": \"stop\"\n",
|
|
||||||
" }\n",
|
|
||||||
" ],\n",
|
|
||||||
" \"usage\": {\n",
|
|
||||||
" \"prompt_tokens\": 13,\n",
|
|
||||||
" \"completion_tokens\": 29,\n",
|
|
||||||
" \"total_tokens\": 42\n",
|
|
||||||
" }\n",
|
|
||||||
"}"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
"metadata": {},
|
|
||||||
"execution_count": 4
|
|
||||||
}
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "markdown",
|
|
||||||
"source": [
|
|
||||||
"## A/B Test GPT-4 and Claude-2"
|
|
||||||
],
|
|
||||||
"metadata": {
|
|
||||||
"id": "Y12cxhZt58v8"
|
|
||||||
}
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": 5,
|
|
||||||
"metadata": {
|
|
||||||
"colab": {
|
|
||||||
"base_uri": "https://localhost:8080/"
|
|
||||||
},
|
|
||||||
"id": "0k6rshtR8i9q",
|
|
||||||
"outputId": "31ac9d73-9e35-4697-d1ff-5d51048566f8"
|
|
||||||
},
|
|
||||||
"outputs": [
|
|
||||||
{
|
|
||||||
"output_type": "stream",
|
|
||||||
"name": "stdout",
|
|
||||||
"text": [
|
|
||||||
"last_fetched_at: 1693624809.3467667\n"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"output_type": "execute_result",
|
|
||||||
"data": {
|
|
||||||
"text/plain": [
|
|
||||||
"<OpenAIObject chat.completion id=chatcmpl-7uBTA6gotsTksvCU7GffJ64ybfHUw at 0x7c28aa288630> JSON: {\n",
|
|
||||||
" \"id\": \"chatcmpl-7uBTA6gotsTksvCU7GffJ64ybfHUw\",\n",
|
|
||||||
" \"object\": \"chat.completion\",\n",
|
|
||||||
" \"created\": 1693624812,\n",
|
|
||||||
" \"model\": \"gpt-4-0613\",\n",
|
|
||||||
" \"choices\": [\n",
|
|
||||||
" {\n",
|
|
||||||
" \"index\": 0,\n",
|
|
||||||
" \"message\": {\n",
|
|
||||||
" \"role\": \"assistant\",\n",
|
|
||||||
" \"content\": \"As an AI, I don't have feelings, but I'm here and ready to assist you. How can I help you today?\"\n",
|
|
||||||
" },\n",
|
|
||||||
" \"finish_reason\": \"stop\"\n",
|
|
||||||
" }\n",
|
|
||||||
" ],\n",
|
|
||||||
" \"usage\": {\n",
|
|
||||||
" \"prompt_tokens\": 13,\n",
|
|
||||||
" \"completion_tokens\": 27,\n",
|
|
||||||
" \"total_tokens\": 40\n",
|
|
||||||
" }\n",
|
|
||||||
"}"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
"metadata": {},
|
|
||||||
"execution_count": 5
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"source": [
|
|
||||||
"from litellm import completion_with_split_tests\n",
|
|
||||||
"import os\n",
|
|
||||||
"\n",
|
|
||||||
"## set ENV variables\n",
|
|
||||||
"os.environ[\"ANTHROPIC_API_KEY\"] = \"\"\n",
|
|
||||||
"\n",
|
|
||||||
"# define a dict of model id and % of requests for model\n",
|
|
||||||
"split_per_model = {\n",
|
|
||||||
"\t\"gpt-4\": 0.3,\n",
|
|
||||||
"\t\"claude-2\": 0.7\n",
|
|
||||||
"}\n",
|
|
||||||
"\n",
|
|
||||||
"messages = [{ \"content\": \"Hello, how are you?\",\"role\": \"user\"}]\n",
|
|
||||||
"\n",
|
|
||||||
"\n",
|
|
||||||
"completion_with_split_tests(messages=messages, use_client=True,\n",
|
|
||||||
" id=\"91fad14a-8c0f-4e99-8eaa-68245435aa80\") # [Optional Set your own ID]"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "markdown",
|
|
||||||
"source": [],
|
|
||||||
"metadata": {
|
|
||||||
"id": "hzzbsAIp4pnr"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
]
|
|
||||||
}
|
|
||||||
|
|
@ -324,7 +324,6 @@ from .utils import (
|
||||||
Logging,
|
Logging,
|
||||||
acreate,
|
acreate,
|
||||||
get_model_list,
|
get_model_list,
|
||||||
completion_with_split_tests,
|
|
||||||
get_max_tokens,
|
get_max_tokens,
|
||||||
register_prompt_template,
|
register_prompt_template,
|
||||||
validate_environment,
|
validate_environment,
|
||||||
|
|
|
||||||
|
|
@ -1,112 +0,0 @@
|
||||||
#### What this tests ####
|
|
||||||
# This tests if logging to the litedebugger integration actually works
|
|
||||||
|
|
||||||
# Test Scenarios (test across normal completion, streaming)
|
|
||||||
## 1: Pre-API-Call
|
|
||||||
## 2: Post-API-Call
|
|
||||||
## 3: On LiteLLM Call success
|
|
||||||
## 4: On LiteLLM Call failure
|
|
||||||
|
|
||||||
|
|
||||||
import sys, os, io
|
|
||||||
import traceback, logging
|
|
||||||
import pytest
|
|
||||||
import dotenv
|
|
||||||
dotenv.load_dotenv()
|
|
||||||
|
|
||||||
# Create logger
|
|
||||||
logger = logging.getLogger(__name__)
|
|
||||||
logger.setLevel(logging.DEBUG)
|
|
||||||
|
|
||||||
# Create a stream handler
|
|
||||||
stream_handler = logging.StreamHandler(sys.stdout)
|
|
||||||
logger.addHandler(stream_handler)
|
|
||||||
|
|
||||||
# Create a function to log information
|
|
||||||
def logger_fn(message):
|
|
||||||
logger.info(message)
|
|
||||||
|
|
||||||
sys.path.insert(
|
|
||||||
0, os.path.abspath("../..")
|
|
||||||
) # Adds the parent directory to the system path
|
|
||||||
import litellm
|
|
||||||
from litellm import completion
|
|
||||||
from openai.error import AuthenticationError
|
|
||||||
litellm.set_verbose = True
|
|
||||||
|
|
||||||
score = 0
|
|
||||||
split_per_model = {
|
|
||||||
"gpt-4": 0,
|
|
||||||
"claude-instant-1.2": 1
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
user_message = "Hello, how are you?"
|
|
||||||
messages = [{"content": user_message, "role": "user"}]
|
|
||||||
|
|
||||||
# # #Test 1: On completion call - without setting client to true -> ensure litedebugger is not initialized
|
|
||||||
# try:
|
|
||||||
# # Redirect stdout
|
|
||||||
# old_stdout = sys.stdout
|
|
||||||
# sys.stdout = new_stdout = io.StringIO()
|
|
||||||
|
|
||||||
# response = completion(model="gpt-3.5-turbo", messages=messages)
|
|
||||||
|
|
||||||
# # Restore stdout
|
|
||||||
# sys.stdout = old_stdout
|
|
||||||
# output = new_stdout.getvalue().strip()
|
|
||||||
|
|
||||||
# if "LiteLLMDebugger" in output:
|
|
||||||
# raise Exception("LiteLLM Debugger should not be called!")
|
|
||||||
# score += 1
|
|
||||||
# except Exception as e:
|
|
||||||
# pytest.fail(f"Error occurred: {e}")
|
|
||||||
|
|
||||||
|
|
||||||
# # Test 2: On normal completion call - setting client to true
|
|
||||||
# litellm.use_client=True
|
|
||||||
# def test_completion_with_client():
|
|
||||||
# try:
|
|
||||||
# # Redirect stdout
|
|
||||||
# old_stdout = sys.stdout
|
|
||||||
# sys.stdout = new_stdout = io.StringIO()
|
|
||||||
# litellm.token = "1e6795ea-a75e-4231-8110-dcc721dcffc3" # generate one here - https://www.uuidgenerator.net/version4
|
|
||||||
|
|
||||||
# completion(model="gpt-3.5-turbo", messages=messages)
|
|
||||||
# completion(model="claude-instant-1", messages=messages)
|
|
||||||
|
|
||||||
# # Restore stdout
|
|
||||||
# sys.stdout = old_stdout
|
|
||||||
# output = new_stdout.getvalue().strip()
|
|
||||||
# print(output)
|
|
||||||
# if "LiteDebugger: Pre-API Call Logging" not in output:
|
|
||||||
# raise Exception(f"LiteLLMDebugger: pre-api call not logged!")
|
|
||||||
# if "LiteDebugger: Post-API Call Logging" not in output:
|
|
||||||
# raise Exception("LiteLLMDebugger: post-api call not logged!")
|
|
||||||
# if "LiteDebugger: Success/Failure Call Logging" not in output:
|
|
||||||
# raise Exception("LiteLLMDebugger: success/failure call not logged!")
|
|
||||||
# except Exception as e:
|
|
||||||
# pytest.fail(f"Error occurred: {e}")
|
|
||||||
# test_completion_with_client()
|
|
||||||
# # Test 3: On streaming completion call - setting client to true
|
|
||||||
# try:
|
|
||||||
# # Redirect stdout
|
|
||||||
# old_stdout = sys.stdout
|
|
||||||
# sys.stdout = new_stdout = io.StringIO()
|
|
||||||
|
|
||||||
# response = completion_with_split_tests(models=split_per_model, messages=messages, stream=True, use_client=True, override_client=True, id="6d383c99-488d-481d-aa1b-1f94935cec44")
|
|
||||||
# for data in response:
|
|
||||||
# continue
|
|
||||||
# # Restore stdout
|
|
||||||
# sys.stdout = old_stdout
|
|
||||||
# output = new_stdout.getvalue().strip()
|
|
||||||
|
|
||||||
# if "LiteDebugger: Pre-API Call Logging" not in output:
|
|
||||||
# raise Exception("LiteLLMDebugger: pre-api call not logged!")
|
|
||||||
# if "LiteDebugger: Post-API Call Logging" not in output:
|
|
||||||
# raise Exception("LiteLLMDebugger: post-api call not logged!")
|
|
||||||
# if "LiteDebugger: Success/Failure Call Logging" not in output:
|
|
||||||
# raise Exception("LiteLLMDebugger: success/failure call not logged!")
|
|
||||||
# except Exception as e:
|
|
||||||
# pytest.fail(f"Error occurred: {e}")
|
|
||||||
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue