diff --git a/docs/my-website/docs/observability/lago.md b/docs/my-website/docs/observability/lago.md index aed02e9916..337a2b553e 100644 --- a/docs/my-website/docs/observability/lago.md +++ b/docs/my-website/docs/observability/lago.md @@ -14,9 +14,6 @@ Use just 1 lines of code, to instantly log your responses **across all providers Get your Lago [API Key](https://docs.getlago.com/guide/self-hosted/docker#find-your-api-key) ```python - - - litellm.callbacks = ["lago"] # logs cost + usage of successful calls to lago ``` @@ -44,7 +41,8 @@ response = litellm.completion( model="gpt-3.5-turbo", messages=[ {"role": "user", "content": "Hi 👋 - i'm openai"} - ] + ], + user="your_customer_id" # 👈 SET YOUR CUSTOMER ID HERE ) ``` @@ -72,6 +70,9 @@ litellm --config /path/to/config.yaml 3. Test it! + + + ```bash curl --location 'http://0.0.0.0:4000/chat/completions' \ --header 'Content-Type: application/json' \ @@ -83,10 +84,68 @@ curl --location 'http://0.0.0.0:4000/chat/completions' \ "content": "what llm are you" } ], + "user": "your-customer-id" # 👈 SET YOUR CUSTOMER ID } ' ``` + + +```python +import openai +client = openai.OpenAI( + api_key="anything", + base_url="http://0.0.0.0:4000" +) + +# request sent to model set on litellm proxy, `litellm --model` +response = client.chat.completions.create(model="gpt-3.5-turbo", messages = [ + { + "role": "user", + "content": "this is a test request, write a short poem" + } +], user="my_customer_id") # 👈 whatever your customer id is + +print(response) +``` + + + +```python +from langchain.chat_models import ChatOpenAI +from langchain.prompts.chat import ( + ChatPromptTemplate, + HumanMessagePromptTemplate, + SystemMessagePromptTemplate, +) +from langchain.schema import HumanMessage, SystemMessage +import os + +os.environ["OPENAI_API_KEY"] = "anything" + +chat = ChatOpenAI( + openai_api_base="http://0.0.0.0:4000", + model = "gpt-3.5-turbo", + temperature=0.1, + extra_body={ + "user": "my_customer_id" # 👈 whatever your customer id is + } +) + +messages = [ + SystemMessage( + content="You are a helpful assistant that im using to make a test request to." + ), + HumanMessage( + content="test from litellm. tell me why it's amazing in 1 sentence" + ), +] +response = chat(messages) + +print(response) +``` + + diff --git a/docs/my-website/docs/proxy/billing.md b/docs/my-website/docs/proxy/billing.md new file mode 100644 index 0000000000..97e944f258 --- /dev/null +++ b/docs/my-website/docs/proxy/billing.md @@ -0,0 +1,229 @@ +import Image from '@theme/IdealImage'; +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +# 💰 Billing + +Bill users for their usage. + +Requirements: +- Setup a billing plan on Lago, for usage-based billing. We recommend following their Stripe tutorial - https://docs.getlago.com/templates/per-transaction/stripe#step-1-create-billable-metrics-for-transaction + +Steps: +- Connect the proxy to Lago +- Set the id you want to bill for (customers, internal users, teams) +- Start! + +## 1. Connect proxy to Lago + +Add your Lago keys to the environment + +```bash +export LAGO_API_BASE="http://localhost:3000" # self-host - https://docs.getlago.com/guide/self-hosted/docker#run-the-app +export LAGO_API_KEY="3e29d607-de54-49aa-a019-ecf585729070" # Get key - https://docs.getlago.com/guide/self-hosted/docker#find-your-api-key +export LAGO_API_EVENT_CODE="openai_tokens" # name of lago billing code +``` + +Set 'lago' as a callback on your proxy config.yaml + +```yaml +... +litellm_settings: + callbacks: ["lago"] +``` + +## 2. Set the id you want to bill for + +For: +- Customers (id passed via 'user' param in /chat/completion call) = 'end_user_id' +- Internal Users (id set when [creating keys](https://docs.litellm.ai/docs/proxy/virtual_keys#advanced---spend-tracking)) = 'user_id' +- Teams (id set when [creating keys](https://docs.litellm.ai/docs/proxy/virtual_keys#advanced---spend-tracking)) = 'team_id' + +```yaml +export LAGO_API_CHARGE_BY="end_user_id" # 👈 Charge 'Customers'. Default is 'end_user_id'. +``` + +## 3. Start billing! + + + + + +### **Curl** +```shell +curl --location 'http://0.0.0.0:4000/chat/completions' \ +--header 'Content-Type: application/json' \ +--data ' { + "model": "gpt-3.5-turbo", + "messages": [ + { + "role": "user", + "content": "what llm are you" + } + ], + "user": "my_customer_id" # 👈 whatever your customer id is + } +' +``` + +### **OpenAI Python SDK** + +```python +import openai +client = openai.OpenAI( + api_key="anything", + base_url="http://0.0.0.0:4000" +) + +# request sent to model set on litellm proxy, `litellm --model` +response = client.chat.completions.create(model="gpt-3.5-turbo", messages = [ + { + "role": "user", + "content": "this is a test request, write a short poem" + } +], user="my_customer_id") # 👈 whatever your customer id is + +print(response) +``` + +### **Langchain** + +```python +from langchain.chat_models import ChatOpenAI +from langchain.prompts.chat import ( + ChatPromptTemplate, + HumanMessagePromptTemplate, + SystemMessagePromptTemplate, +) +from langchain.schema import HumanMessage, SystemMessage +import os + +os.environ["OPENAI_API_KEY"] = "anything" + +chat = ChatOpenAI( + openai_api_base="http://0.0.0.0:4000", + model = "gpt-3.5-turbo", + temperature=0.1, + extra_body={ + "user": "my_customer_id" # 👈 whatever your customer id is + } +) + +messages = [ + SystemMessage( + content="You are a helpful assistant that im using to make a test request to." + ), + HumanMessage( + content="test from litellm. tell me why it's amazing in 1 sentence" + ), +] +response = chat(messages) + +print(response) +``` + + + +1. Create a key for that user + +```bash +curl 'http://0.0.0.0:4000/key/generate' \ +--header 'Authorization: Bearer ' \ +--header 'Content-Type: application/json' \ +--data-raw '{"user_id": "my-unique-id"}' +``` + +Response Object: + +```bash +{ + "key": "sk-tXL0wt5-lOOVK9sfY2UacA", +} +``` + +2. Make API Calls with that Key + +```python +import openai +client = openai.OpenAI( + api_key="sk-tXL0wt5-lOOVK9sfY2UacA", # 👈 Generated key + base_url="http://0.0.0.0:4000" +) + +# request sent to model set on litellm proxy, `litellm --model` +response = client.chat.completions.create(model="gpt-3.5-turbo", messages = [ + { + "role": "user", + "content": "this is a test request, write a short poem" + } +]) + +print(response) +``` + + + + +1. Create a key for that team + +```bash +curl 'http://0.0.0.0:4000/key/generate' \ +--header 'Authorization: Bearer ' \ +--header 'Content-Type: application/json' \ +--data-raw '{"team_id": "my-unique-id"}' +``` + +Response Object: + +```bash +{ + "key": "sk-tXL0wt5-lOOVK9sfY2UacA", +} +``` + +2. Make API Calls with that Key + +```python +import openai +client = openai.OpenAI( + api_key="sk-tXL0wt5-lOOVK9sfY2UacA", # 👈 Generated key + base_url="http://0.0.0.0:4000" +) + +# request sent to model set on litellm proxy, `litellm --model` +response = client.chat.completions.create(model="gpt-3.5-turbo", messages = [ + { + "role": "user", + "content": "this is a test request, write a short poem" + } +]) + +print(response) +``` + + + +**See Results on Lago** + + + + +## Advanced - Lago Logging object + +This is what LiteLLM will log to Lagos + +``` +{ + "event": { + "transaction_id": "", + "external_customer_id": , # either 'end_user_id', 'user_id', or 'team_id'. Default 'end_user_id'. + "code": os.getenv("LAGO_API_EVENT_CODE"), + "properties": { + "input_tokens": , + "output_tokens": , + "model": , + "response_cost": , # 👈 LITELLM CALCULATED RESPONSE COST - https://github.com/BerriAI/litellm/blob/d43f75150a65f91f60dc2c0c9462ce3ffc713c1f/litellm/utils.py#L1473 + } + } +} +``` diff --git a/docs/my-website/sidebars.js b/docs/my-website/sidebars.js index 1cd3286659..f840ed7897 100644 --- a/docs/my-website/sidebars.js +++ b/docs/my-website/sidebars.js @@ -41,6 +41,7 @@ const sidebars = { "proxy/reliability", "proxy/cost_tracking", "proxy/users", + "proxy/billing", "proxy/user_keys", "proxy/enterprise", "proxy/virtual_keys", diff --git a/litellm/integrations/lago.py b/litellm/integrations/lago.py index cf2f604754..e6d38f530c 100644 --- a/litellm/integrations/lago.py +++ b/litellm/integrations/lago.py @@ -7,7 +7,7 @@ import traceback, httpx from litellm.integrations.custom_logger import CustomLogger from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler import uuid -from typing import Optional +from typing import Optional, Literal def get_utc_datetime(): @@ -32,6 +32,10 @@ class LagoLogger(CustomLogger): Expects LAGO_API_BASE, LAGO_API_KEY, + LAGO_API_EVENT_CODE, + + Optional: + LAGO_API_CHARGE_BY in the environment """ @@ -72,15 +76,37 @@ class LagoLogger(CustomLogger): team_id = litellm_params["metadata"].get("user_api_key_team_id", None) org_id = litellm_params["metadata"].get("user_api_key_org_id", None) - if end_user_id is None: - raise Exception("LAGO: user is required") + charge_by: Literal["end_user_id", "team_id", "user_id"] = "end_user_id" + external_customer_id: Optional[str] = None + + if os.getenv("LAGO_API_CHARGE_BY", None) is not None and isinstance( + os.environ["LAGO_API_CHARGE_BY"], str + ): + if os.environ["LAGO_API_CHARGE_BY"] in [ + "end_user_id", + "user_id", + "team_id", + ]: + charge_by = os.environ["LAGO_API_CHARGE_BY"] # type: ignore + else: + raise Exception("invalid LAGO_API_CHARGE_BY set") + + if charge_by == "end_user_id": + external_customer_id = end_user_id + elif charge_by == "team_id": + external_customer_id = team_id + elif charge_by == "user_id": + external_customer_id = user_id + + if external_customer_id is None: + raise Exception("External Customer ID is not set") return { "event": { "transaction_id": str(uuid.uuid4()), - "external_customer_id": end_user_id, + "external_customer_id": external_customer_id, "code": os.getenv("LAGO_API_EVENT_CODE"), - "properties": {"model": model, "response_cost": 10000, **usage}, + "properties": {"model": model, "response_cost": cost, **usage}, } }