From 39b5b03ac3f63c362b5783fa57fabad7924f0c0a Mon Sep 17 00:00:00 2001 From: ishaan-jaff Date: Mon, 13 Nov 2023 14:41:21 -0800 Subject: [PATCH] (docs) remove bloat cookbook --- cookbook/llm-ab-test-server/main.py | 45 ------- cookbook/llm-ab-test-server/readme.md | 174 -------------------------- 2 files changed, 219 deletions(-) delete mode 100644 cookbook/llm-ab-test-server/main.py delete mode 100644 cookbook/llm-ab-test-server/readme.md diff --git a/cookbook/llm-ab-test-server/main.py b/cookbook/llm-ab-test-server/main.py deleted file mode 100644 index c0c235ca05..0000000000 --- a/cookbook/llm-ab-test-server/main.py +++ /dev/null @@ -1,45 +0,0 @@ -from flask import Flask, request, jsonify, abort, Response -from flask_cors import CORS -from litellm import completion -import os, dotenv -import random -dotenv.load_dotenv() - -# TODO: set your keys in .env or here: -# os.environ["OPENAI_API_KEY"] = "" # set your openai key here or in your .env -# see supported models, keys here: - - -app = Flask(__name__) -CORS(app) - -@app.route('/') -def index(): - return 'received!', 200 - -# Dictionary of LLM functions with their A/B test ratios, should sum to 1 :) -llm_dict = { - "gpt-4": 0.2, - "together_ai/togethercomputer/llama-2-70b-chat": 0.4, - "claude-2": 0.2, - "claude-1.2": 0.2 -} - - -@app.route('/chat/completions', methods=["POST"]) -def api_completion(): - data = request.json - try: - # pass in data to completion function, unpack data - selected_llm = random.choices(list(llm_dict.keys()), weights=list(llm_dict.values()))[0] - response = completion(**data, model=selected_llm) - except Exception as e: - print(f"got error{e}") - return response, 200 - - -if __name__ == "__main__": - from waitress import serve - print("starting server") - serve(app, host="0.0.0.0", port=5000, threads=500) - diff --git a/cookbook/llm-ab-test-server/readme.md b/cookbook/llm-ab-test-server/readme.md deleted file mode 100644 index 763912a199..0000000000 --- a/cookbook/llm-ab-test-server/readme.md +++ /dev/null @@ -1,174 +0,0 @@ -

- 🚅 LiteLLM - A/B Testing LLMs in Production -

-

-

Call all LLM APIs using the OpenAI format [Anthropic, Huggingface, Cohere, Azure OpenAI etc.]

-

- -

- - PyPI Version - - - Stable Version - - - CircleCI - - Downloads - - - -

- -

- 100+ Supported Models | - Docs | - Demo Website -

- -

- -![pika-1693451518986-1x](https://github.com/BerriAI/litellm/assets/29436595/1cbf29a3-5313-4f61-ad6e-481dd8737309) -

- -LiteLLM allows you to call 100+ LLMs using completion - -## Usage - A/B Test LLMs in Production - -### Set your A/B Test Ratios -```python -llm_dict = { - "gpt-4": 0.2, - "together_ai/togethercomputer/llama-2-70b-chat": 0.4, - "claude-2": 0.2, - "claude-1.2": 0.2 -} -``` - -### Select LLM + Make Completion call -Use weighted selection, and call the model using litellm.completion -```python -from litellm import completion -selected_llm = random.choices(list(llm_dict.keys()), weights=list(llm_dict.values()))[0] - -response = completion(model=selected_model, messages=[{ "content": "Hello, how are you?","role": "user"}]) - -``` - -### Viewing Logs, Feedback -In order to view logs set `litellm.token=` -```python -import litellm -litellm.token='ishaan_discord@berri.ai' -``` - -Your logs will be available at: -https://lite-llm-abtest-nckmhi7ue-clerkieai.vercel.app/ - -### Live Demo UI -👉https://lite-llm-abtest-nckmhi7ue-clerkieai.vercel.app/ishaan_discord@berri.ai - -## Viewing Responses + Custom Scores -LiteLLM UI allows you to view responses and set custom scores for each response - -Screenshot 2023-08-30 at 8 08 59 PM - -# Using LiteLLM A/B Testing Server -## Setup - -### Install LiteLLM -``` -pip install litellm -``` - - -### Clone LiteLLM Git Repo -``` -git clone https://github.com/BerriAI/litellm/ -``` - -### Navigate to LiteLLM-A/B Test Server -``` -cd litellm/cookbook/llm-ab-test-server -``` - -### Run the Server -``` -python3 main.py -``` - -### Set your LLM Configs -Set your LLMs and LLM weights you want to run A/B testing with -In main.py set your selected LLMs you want to AB test in `llm_dict` -You can A/B test more than 100+ LLMs using LiteLLM https://docs.litellm.ai/docs/providers -```python -llm_dict = { - "gpt-4": 0.2, - "together_ai/togethercomputer/llama-2-70b-chat": 0.4, - "claude-2": 0.2, - "claude-1.2": 0.2 -} -``` - -#### Setting your API Keys -Set your LLM API keys in a .env file in the directory or set them as `os.environ` variables. - -See https://docs.litellm.ai/docs/providers for the format of API keys - -LiteLLM generalizes api keys to follow the following format -`PROVIDER_API_KEY` - -## Making Requests to the LiteLLM Server Locally -The server follows the Input/Output format set by the OpenAI Chat Completions API -Here is an example request made the LiteLLM Server - -### Python -```python -import requests -import json - -url = "http://localhost:5000/chat/completions" - -payload = json.dumps({ - "messages": [ - { - "content": "who is CTO of litellm", - "role": "user" - } - ] -}) -headers = { - 'Content-Type': 'application/json' -} - -response = requests.request("POST", url, headers=headers, data=payload) - -print(response.text) - -``` - -### Curl Command -``` -curl --location 'http://localhost:5000/chat/completions' \ ---header 'Content-Type: application/json' \ ---data '{ - "messages": [ - { - "content": "who is CTO of litellm", - "role": "user" - } - ] - -} -' -``` - -# support / talk with founders -- [Schedule Demo 👋](https://calendly.com/d/4mp-gd3-k5k/berriai-1-1-onboarding-litellm-hosted-version) -- [Community Discord 💭](https://discord.gg/wuPM9dRgDw) -- Our numbers 📞 +1 (770) 8783-106 / ‭+1 (412) 618-6238‬ -- Our emails ✉️ ishaan@berri.ai / krrish@berri.ai - -# why did we build this -- **Need for simplicity**: Our code started to get extremely complicated managing & translating calls between Azure, OpenAI, Cohere