diff --git a/cookbook/logging_observability/LiteLLM_Arize.ipynb b/cookbook/logging_observability/LiteLLM_Arize.ipynb index 72a082f874..d40b4c4bc4 100644 --- a/cookbook/logging_observability/LiteLLM_Arize.ipynb +++ b/cookbook/logging_observability/LiteLLM_Arize.ipynb @@ -105,11 +105,16 @@ "import os\n", "from getpass import getpass\n", "\n", - "os.environ[\"ARIZE_SPACE_KEY\"] = getpass(\"Enter your Arize space key: \")\n", + "os.environ[\"ARIZE_SPACE_ID\"] = getpass(\"Enter your Arize space id: \")\n", "os.environ[\"ARIZE_API_KEY\"] = getpass(\"Enter your Arize API key: \")\n", "os.environ['OPENAI_API_KEY']= getpass(\"Enter your OpenAI API key: \")" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [] + }, { "cell_type": "markdown", "metadata": {}, diff --git a/docs/my-website/docs/observability/arize_integration.md b/docs/my-website/docs/observability/arize_integration.md index 1cd36a1111..ebdf043466 100644 --- a/docs/my-website/docs/observability/arize_integration.md +++ b/docs/my-website/docs/observability/arize_integration.md @@ -11,12 +11,12 @@ https://github.com/BerriAI/litellm ::: - - ## Pre-Requisites + Make an account on [Arize AI](https://app.arize.com/auth/login) ## Quick Start + Use just 2 lines of code, to instantly log your responses **across all providers** with arize You can also use the instrumentor option instead of the callback, which you can find [here](https://docs.arize.com/arize/llm-tracing/tracing-integrations-auto/litellm). @@ -24,11 +24,12 @@ You can also use the instrumentor option instead of the callback, which you can ```python litellm.callbacks = ["arize"] ``` + ```python import litellm import os -os.environ["ARIZE_SPACE_KEY"] = "" +os.environ["ARIZE_SPACE_ID"] = "" os.environ["ARIZE_API_KEY"] = "" # LLM API Keys @@ -36,7 +37,7 @@ os.environ['OPENAI_API_KEY']="" # set arize as a callback, litellm will send the data to arize litellm.callbacks = ["arize"] - + # openai call response = litellm.completion( model="gpt-3.5-turbo", @@ -48,7 +49,6 @@ response = litellm.completion( ### Using with LiteLLM Proxy - ```yaml model_list: - model_name: gpt-4 @@ -61,10 +61,10 @@ litellm_settings: callbacks: ["arize"] environment_variables: - ARIZE_SPACE_KEY: "d0*****" - ARIZE_API_KEY: "141a****" - ARIZE_ENDPOINT: "https://otlp.arize.com/v1" # OPTIONAL - your custom arize GRPC api endpoint - ARIZE_HTTP_ENDPOINT: "https://otlp.arize.com/v1" # OPTIONAL - your custom arize HTTP api endpoint. Set either this or ARIZE_ENDPOINT + ARIZE_SPACE_ID: "d0*****" + ARIZE_API_KEY: "141a****" + ARIZE_ENDPOINT: "https://otlp.arize.com/v1" # OPTIONAL - your custom arize GRPC api endpoint + ARIZE_HTTP_ENDPOINT: "https://otlp.arize.com/v1" # OPTIONAL - your custom arize HTTP api endpoint. Set either this or ARIZE_ENDPOINT ``` ## Support & Talk to Founders diff --git a/docs/my-website/docs/proxy/config_settings.md b/docs/my-website/docs/proxy/config_settings.md index 9e24437449..21c95e5f40 100644 --- a/docs/my-website/docs/proxy/config_settings.md +++ b/docs/my-website/docs/proxy/config_settings.md @@ -302,7 +302,7 @@ router_settings: | AISPEND_API_KEY | API Key for AI Spend | ALLOWED_EMAIL_DOMAINS | List of email domains allowed for access | ARIZE_API_KEY | API key for Arize platform integration -| ARIZE_SPACE_KEY | Space key for Arize platform +| ARIZE_SPACE_ID | Space key for Arize platform | ARGILLA_BATCH_SIZE | Batch size for Argilla logging | ARGILLA_API_KEY | API key for Argilla platform | ARGILLA_SAMPLING_RATE | Sampling rate for Argilla logging diff --git a/docs/my-website/docs/proxy/logging.md b/docs/my-website/docs/proxy/logging.md index e13a403634..3493d37391 100644 --- a/docs/my-website/docs/proxy/logging.md +++ b/docs/my-website/docs/proxy/logging.md @@ -17,8 +17,6 @@ Log Proxy input, output, and exceptions using: - DynamoDB - etc. - - ## Getting the LiteLLM Call ID LiteLLM generates a unique `call_id` for each request. This `call_id` can be @@ -52,31 +50,29 @@ A number of these headers could be useful for troubleshooting, but the `x-litellm-call-id` is the one that is most useful for tracking a request across components in your system, including in logging tools. - ## Logging Features ### Conditional Logging by Virtual Keys, Teams Use this to: + 1. Conditionally enable logging for some virtual keys/teams 2. Set different logging providers for different virtual keys/teams [👉 **Get Started** - Team/Key Based Logging](team_logging) +### Redacting UserAPIKeyInfo -### Redacting UserAPIKeyInfo - -Redact information about the user api key (hashed token, user_id, team id, etc.), from logs. +Redact information about the user api key (hashed token, user_id, team id, etc.), from logs. Currently supported for Langfuse, OpenTelemetry, Logfire, ArizeAI logging. ```yaml -litellm_settings: +litellm_settings: callbacks: ["langfuse"] redact_user_api_key_info: true ``` - ### Redact Messages, Response Content Set `litellm.turn_off_message_logging=True` This will prevent the messages and responses from being logged to your logging provider, but request metadata - e.g. spend, will still be tracked. @@ -86,6 +82,7 @@ Set `litellm.turn_off_message_logging=True` This will prevent the messages and r **1. Setup config.yaml ** + ```yaml model_list: - model_name: gpt-3.5-turbo @@ -97,6 +94,7 @@ litellm_settings: ``` **2. Send request** + ```shell curl --location 'http://0.0.0.0:4000/chat/completions' \ --header 'Content-Type: application/json' \ @@ -111,14 +109,12 @@ curl --location 'http://0.0.0.0:4000/chat/completions' \ }' ``` - - :::info -Dynamic request message redaction is in BETA. +Dynamic request message redaction is in BETA. ::: @@ -170,13 +166,11 @@ curl -L -X POST 'http://0.0.0.0:4000/v1/chat/completions' \ - ### Disable Message Redaction If you have `litellm.turn_on_message_logging` turned on, you can override it for specific requests by setting a request header `LiteLLM-Disable-Message-Redaction: true`. - ```shell curl --location 'http://0.0.0.0:4000/chat/completions' \ --header 'Content-Type: application/json' \ @@ -192,7 +186,6 @@ curl --location 'http://0.0.0.0:4000/chat/completions' \ }' ``` - ### Turn off all tracking/logging For some use cases, you may want to turn off all tracking/logging. You can do this by passing `no-log=True` in the request body. @@ -205,6 +198,7 @@ Disable this by setting `global_disable_no_log_param:true` in your config.yaml f litellm_settings: global_disable_no_log_param: True ``` + ::: @@ -262,13 +256,12 @@ print(response) -**Expected Console Log** +**Expected Console Log** ``` LiteLLM.Info: "no-log request, skipping logging" ``` - ## What gets logged? Found under `kwargs["standard_logging_object"]`. This is a standard payload, logged for every response. @@ -431,10 +424,8 @@ print(response) Set `tags` as part of your request body - - ```python @@ -462,6 +453,7 @@ response = client.chat.completions.create( print(response) ``` + @@ -486,6 +478,7 @@ curl --location 'http://0.0.0.0:4000/chat/completions' \ } }' ``` + @@ -528,28 +521,26 @@ print(response) - - ### LiteLLM Tags - `cache_hit`, `cache_key` Use this if you want to control which LiteLLM-specific fields are logged as tags by the LiteLLM proxy. By default LiteLLM Proxy logs no LiteLLM-specific fields -| LiteLLM specific field | Description | Example Value | -|------------------------|-------------------------------------------------------|------------------------------------------------| -| `cache_hit` | Indicates whether a cache hit occured (True) or not (False) | `true`, `false` | -| `cache_key` | The Cache key used for this request | `d2b758c****`| -| `proxy_base_url` | The base URL for the proxy server, the value of env var `PROXY_BASE_URL` on your server | `https://proxy.example.com`| -| `user_api_key_alias` | An alias for the LiteLLM Virtual Key.| `prod-app1` | -| `user_api_key_user_id` | The unique ID associated with a user's API key. | `user_123`, `user_456` | -| `user_api_key_user_email` | The email associated with a user's API key. | `user@example.com`, `admin@example.com` | -| `user_api_key_team_alias` | An alias for a team associated with an API key. | `team_alpha`, `dev_team` | - +| LiteLLM specific field | Description | Example Value | +| ------------------------- | --------------------------------------------------------------------------------------- | --------------------------------------- | +| `cache_hit` | Indicates whether a cache hit occured (True) or not (False) | `true`, `false` | +| `cache_key` | The Cache key used for this request | `d2b758c****` | +| `proxy_base_url` | The base URL for the proxy server, the value of env var `PROXY_BASE_URL` on your server | `https://proxy.example.com` | +| `user_api_key_alias` | An alias for the LiteLLM Virtual Key. | `prod-app1` | +| `user_api_key_user_id` | The unique ID associated with a user's API key. | `user_123`, `user_456` | +| `user_api_key_user_email` | The email associated with a user's API key. | `user@example.com`, `admin@example.com` | +| `user_api_key_team_alias` | An alias for a team associated with an API key. | `team_alpha`, `dev_team` | **Usage** Specify `langfuse_default_tags` to control what litellm fields get logged on Langfuse -Example config.yaml +Example config.yaml + ```yaml model_list: - model_name: gpt-4 @@ -562,12 +553,23 @@ litellm_settings: success_callback: ["langfuse"] # 👇 Key Change - langfuse_default_tags: ["cache_hit", "cache_key", "proxy_base_url", "user_api_key_alias", "user_api_key_user_id", "user_api_key_user_email", "user_api_key_team_alias", "semantic-similarity", "proxy_base_url"] + langfuse_default_tags: + [ + "cache_hit", + "cache_key", + "proxy_base_url", + "user_api_key_alias", + "user_api_key_user_id", + "user_api_key_user_email", + "user_api_key_team_alias", + "semantic-similarity", + "proxy_base_url", + ] ``` ### View POST sent from LiteLLM to provider -Use this when you want to view the RAW curl request sent from LiteLLM to the LLM API +Use this when you want to view the RAW curl request sent from LiteLLM to the LLM API @@ -670,7 +672,7 @@ You will see `raw_request` in your Langfuse Metadata. This is the RAW CURL comma ## OpenTelemetry -:::info +:::info [Optional] Customize OTEL Service Name and OTEL TRACER NAME by setting the following variables in your environment @@ -730,30 +732,30 @@ This is the Span from OTEL Logging ```json { - "name": "litellm-acompletion", - "context": { - "trace_id": "0x8d354e2346060032703637a0843b20a3", - "span_id": "0xd8d3476a2eb12724", - "trace_state": "[]" - }, - "kind": "SpanKind.INTERNAL", - "parent_id": null, - "start_time": "2024-06-04T19:46:56.415888Z", - "end_time": "2024-06-04T19:46:56.790278Z", - "status": { - "status_code": "OK" - }, + "name": "litellm-acompletion", + "context": { + "trace_id": "0x8d354e2346060032703637a0843b20a3", + "span_id": "0xd8d3476a2eb12724", + "trace_state": "[]" + }, + "kind": "SpanKind.INTERNAL", + "parent_id": null, + "start_time": "2024-06-04T19:46:56.415888Z", + "end_time": "2024-06-04T19:46:56.790278Z", + "status": { + "status_code": "OK" + }, + "attributes": { + "model": "llama3-8b-8192" + }, + "events": [], + "links": [], + "resource": { "attributes": { - "model": "llama3-8b-8192" + "service.name": "litellm" }, - "events": [], - "links": [], - "resource": { - "attributes": { - "service.name": "litellm" - }, - "schema_url": "" - } + "schema_url": "" + } } ``` @@ -965,6 +967,7 @@ callback_settings: ``` ### Traceparent Header + ##### Context propagation across Services `Traceparent HTTP Header` ❓ Use this when you want to **pass information about the incoming request in a distributed tracing system** @@ -1042,25 +1045,23 @@ Log LLM Logs to [Google Cloud Storage Buckets](https://cloud.google.com/storage? ::: - -| Property | Details | -|----------|---------| -| Description | Log LLM Input/Output to cloud storage buckets | -| Load Test Benchmarks | [Benchmarks](https://docs.litellm.ai/docs/benchmarks) | +| Property | Details | +| ---------------------------- | -------------------------------------------------------------- | +| Description | Log LLM Input/Output to cloud storage buckets | +| Load Test Benchmarks | [Benchmarks](https://docs.litellm.ai/docs/benchmarks) | | Google Docs on Cloud Storage | [Google Cloud Storage](https://cloud.google.com/storage?hl=en) | - - #### Usage 1. Add `gcs_bucket` to LiteLLM Config.yaml + ```yaml model_list: -- litellm_params: - api_base: https://exampleopenaiendpoint-production.up.railway.app/ - api_key: my-fake-key - model: openai/my-fake-model - model_name: fake-openai-endpoint + - litellm_params: + api_base: https://exampleopenaiendpoint-production.up.railway.app/ + api_key: my-fake-key + model: openai/my-fake-model + model_name: fake-openai-endpoint litellm_settings: callbacks: ["gcs_bucket"] # 👈 KEY CHANGE # 👈 KEY CHANGE @@ -1079,7 +1080,7 @@ GCS_PATH_SERVICE_ACCOUNT="/Users/ishaanjaffer/Downloads/adroit-crow-413218-a956e litellm --config /path/to/config.yaml ``` -4. Test it! +4. Test it! ```bash curl --location 'http://0.0.0.0:4000/chat/completions' \ @@ -1096,7 +1097,6 @@ curl --location 'http://0.0.0.0:4000/chat/completions' \ ' ``` - #### Expected Logs on GCS Buckets @@ -1105,7 +1105,6 @@ curl --location 'http://0.0.0.0:4000/chat/completions' \ [**The standard logging object is logged on GCS Bucket**](../proxy/logging_spec) - #### Getting `service_account.json` from Google Cloud Console 1. Go to [Google Cloud Console](https://console.cloud.google.com/) @@ -1115,8 +1114,6 @@ curl --location 'http://0.0.0.0:4000/chat/completions' \ 5. Click on 'Keys' -> Add Key -> Create New Key -> JSON 6. Save the JSON file and add the path to `GCS_PATH_SERVICE_ACCOUNT` - - ## Google Cloud Storage - PubSub Topic Log LLM Logs/SpendLogs to [Google Cloud Storage PubSub Topic](https://cloud.google.com/pubsub/docs/reference/rest) @@ -1127,26 +1124,25 @@ Log LLM Logs/SpendLogs to [Google Cloud Storage PubSub Topic](https://cloud.goog ::: - -| Property | Details | -|----------|---------| +| Property | Details | +| ----------- | ------------------------------------------------------------------ | | Description | Log LiteLLM `SpendLogs Table` to Google Cloud Storage PubSub Topic | When to use `gcs_pubsub`? - If your LiteLLM Database has crossed 1M+ spend logs and you want to send `SpendLogs` to a PubSub Topic that can be consumed by GCS BigQuery - #### Usage 1. Add `gcs_pubsub` to LiteLLM Config.yaml + ```yaml model_list: -- litellm_params: - api_base: https://exampleopenaiendpoint-production.up.railway.app/ - api_key: my-fake-key - model: openai/my-fake-model - model_name: fake-openai-endpoint + - litellm_params: + api_base: https://exampleopenaiendpoint-production.up.railway.app/ + api_key: my-fake-key + model: openai/my-fake-model + model_name: fake-openai-endpoint litellm_settings: callbacks: ["gcs_pubsub"] # 👈 KEY CHANGE # 👈 KEY CHANGE @@ -1165,7 +1161,7 @@ GCS_PUBSUB_PROJECT_ID="reliableKeys" litellm --config /path/to/config.yaml ``` -4. Test it! +4. Test it! ```bash curl --location 'http://0.0.0.0:4000/chat/completions' \ @@ -1182,13 +1178,11 @@ curl --location 'http://0.0.0.0:4000/chat/completions' \ ' ``` - - ## s3 Buckets -We will use the `--config` to set +We will use the `--config` to set -- `litellm.success_callback = ["s3"]` +- `litellm.success_callback = ["s3"]` This will log all successfull LLM calls to s3 Bucket @@ -1276,17 +1270,15 @@ Log LLM Logs to [Azure Data Lake Storage](https://learn.microsoft.com/en-us/azur ::: - -| Property | Details | -|----------|---------| -| Description | Log LLM Input/Output to Azure Blob Storag (Bucket) | +| Property | Details | +| ------------------------------- | --------------------------------------------------------------------------------------------------------------- | +| Description | Log LLM Input/Output to Azure Blob Storag (Bucket) | | Azure Docs on Data Lake Storage | [Azure Data Lake Storage](https://learn.microsoft.com/en-us/azure/storage/blobs/data-lake-storage-introduction) | - - #### Usage 1. Add `azure_storage` to LiteLLM Config.yaml + ```yaml model_list: - model_name: fake-openai-endpoint @@ -1322,7 +1314,7 @@ AZURE_STORAGE_CLIENT_SECRET="uMS8Qxxxxxxxxxx" # The Application Client Secret to litellm --config /path/to/config.yaml ``` -4. Test it! +4. Test it! ```bash curl --location 'http://0.0.0.0:4000/chat/completions' \ @@ -1339,7 +1331,6 @@ curl --location 'http://0.0.0.0:4000/chat/completions' \ ' ``` - #### Expected Logs on Azure Data Lake Storage @@ -1348,11 +1339,10 @@ curl --location 'http://0.0.0.0:4000/chat/completions' \ [**The standard logging object is logged on Azure Data Lake Storage**](../proxy/logging_spec) - - ## DataDog LiteLLM Supports logging to the following Datdog Integrations: + - `datadog` [Datadog Logs](https://docs.datadoghq.com/logs/) - `datadog_llm_observability` [Datadog LLM Observability](https://www.datadoghq.com/product/llm-observability/) - `ddtrace-run` [Datadog Tracing](#datadog-tracing) @@ -1447,26 +1437,29 @@ docker run \ LiteLLM supports customizing the following Datadog environment variables -| Environment Variable | Description | Default Value | Required | -|---------------------|-------------|---------------|----------| -| `DD_API_KEY` | Your Datadog API key for authentication | None | ✅ Yes | -| `DD_SITE` | Your Datadog site (e.g., "us5.datadoghq.com") | None | ✅ Yes | -| `DD_ENV` | Environment tag for your logs (e.g., "production", "staging") | "unknown" | ❌ No | -| `DD_SERVICE` | Service name for your logs | "litellm-server" | ❌ No | -| `DD_SOURCE` | Source name for your logs | "litellm" | ❌ No | -| `DD_VERSION` | Version tag for your logs | "unknown" | ❌ No | -| `HOSTNAME` | Hostname tag for your logs | "" | ❌ No | -| `POD_NAME` | Pod name tag (useful for Kubernetes deployments) | "unknown" | ❌ No | - +| Environment Variable | Description | Default Value | Required | +| -------------------- | ------------------------------------------------------------- | ---------------- | -------- | +| `DD_API_KEY` | Your Datadog API key for authentication | None | ✅ Yes | +| `DD_SITE` | Your Datadog site (e.g., "us5.datadoghq.com") | None | ✅ Yes | +| `DD_ENV` | Environment tag for your logs (e.g., "production", "staging") | "unknown" | ❌ No | +| `DD_SERVICE` | Service name for your logs | "litellm-server" | ❌ No | +| `DD_SOURCE` | Source name for your logs | "litellm" | ❌ No | +| `DD_VERSION` | Version tag for your logs | "unknown" | ❌ No | +| `HOSTNAME` | Hostname tag for your logs | "" | ❌ No | +| `POD_NAME` | Pod name tag (useful for Kubernetes deployments) | "unknown" | ❌ No | ## Lunary -#### Step1: Install dependencies and set your environment variables + +#### Step1: Install dependencies and set your environment variables + Install the dependencies + ```shell pip install litellm lunary ``` -Get you Lunary public key from from https://app.lunary.ai/settings +Get you Lunary public key from from https://app.lunary.ai/settings + ```shell export LUNARY_PUBLIC_KEY="" ``` @@ -1484,6 +1477,7 @@ litellm_settings: ``` #### Step 3: Start the LiteLLM proxy + ```shell litellm --config config.yaml ``` @@ -1510,8 +1504,8 @@ curl -X POST 'http://0.0.0.0:4000/chat/completions' \ ## MLflow - #### Step1: Install dependencies + Install the dependencies. ```shell @@ -1531,6 +1525,7 @@ litellm_settings: ``` #### Step 3: Start the LiteLLM proxy + ```shell litellm --config config.yaml ``` @@ -1559,8 +1554,6 @@ Run the following command to start MLflow UI and review recorded traces. mlflow ui ``` - - ## Custom Callback Class [Async] Use this when you want to run custom callbacks in `python` @@ -1571,7 +1564,7 @@ We use `litellm.integrations.custom_logger` for this, **more details about litel Define your custom callback class in a python file. -Here's an example custom logger for tracking `key, user, model, prompt, response, tokens, cost`. We create a file called `custom_callbacks.py` and initialize `proxy_handler_instance` +Here's an example custom logger for tracking `key, user, model, prompt, response, tokens, cost`. We create a file called `custom_callbacks.py` and initialize `proxy_handler_instance` ```python from litellm.integrations.custom_logger import CustomLogger @@ -1580,16 +1573,16 @@ import litellm # This file includes the custom callbacks for LiteLLM Proxy # Once defined, these can be passed in proxy_config.yaml class MyCustomHandler(CustomLogger): - def log_pre_api_call(self, model, messages, kwargs): + def log_pre_api_call(self, model, messages, kwargs): print(f"Pre-API Call") - - def log_post_api_call(self, kwargs, response_obj, start_time, end_time): + + def log_post_api_call(self, kwargs, response_obj, start_time, end_time): print(f"Post-API Call") - - def log_success_event(self, kwargs, response_obj, start_time, end_time): + + def log_success_event(self, kwargs, response_obj, start_time, end_time): print("On Success") - def log_failure_event(self, kwargs, response_obj, start_time, end_time): + def log_failure_event(self, kwargs, response_obj, start_time, end_time): print(f"On Failure") async def async_log_success_event(self, kwargs, response_obj, start_time, end_time): @@ -1607,7 +1600,7 @@ class MyCustomHandler(CustomLogger): # Calculate cost using litellm.completion_cost() cost = litellm.completion_cost(completion_response=response_obj) response = response_obj - # tokens used in response + # tokens used in response usage = response_obj["usage"] print( @@ -1623,7 +1616,7 @@ class MyCustomHandler(CustomLogger): ) return - async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time): + async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time): try: print(f"On Async Failure !") print("\nkwargs", kwargs) @@ -1643,7 +1636,7 @@ class MyCustomHandler(CustomLogger): # Calculate cost using litellm.completion_cost() cost = litellm.completion_cost(completion_response=response_obj) print("now checking response obj") - + print( f""" Model: {model}, @@ -1667,7 +1660,7 @@ proxy_handler_instance = MyCustomHandler() #### Step 2 - Pass your custom callback class in `config.yaml` -We pass the custom callback class defined in **Step1** to the config.yaml. +We pass the custom callback class defined in **Step1** to the config.yaml. Set `callbacks` to `python_filename.logger_instance_name` In the config below, we pass @@ -1685,7 +1678,6 @@ model_list: litellm_settings: callbacks: custom_callbacks.proxy_handler_instance # sets litellm.callbacks = [proxy_handler_instance] - ``` #### Step 3 - Start proxy + test request @@ -1766,7 +1758,7 @@ class MyCustomHandler(CustomLogger): } ``` -#### Logging `model_info` set in config.yaml +#### Logging `model_info` set in config.yaml Here is how to log the `model_info` set in your proxy `config.yaml`. Information on setting `model_info` on [config.yaml](https://docs.litellm.ai/docs/proxy/configs) @@ -1783,7 +1775,7 @@ class MyCustomHandler(CustomLogger): **Expected Output** ```json -{'mode': 'embedding', 'input_cost_per_token': 0.002} +{ "mode": "embedding", "input_cost_per_token": 0.002 } ``` ##### Logging responses from proxy @@ -1863,7 +1855,7 @@ Use this if you: #### Step 1. Create your generic logging API endpoint -Set up a generic API endpoint that can receive data in JSON format. The data will be included within a "data" field. +Set up a generic API endpoint that can receive data in JSON format. The data will be included within a "data" field. Your server should support the following Request format: @@ -1946,7 +1938,7 @@ litellm_settings: success_callback: ["generic"] ``` -Start the LiteLLM Proxy and make a test request to verify the logs reached your callback API +Start the LiteLLM Proxy and make a test request to verify the logs reached your callback API ## Langsmith @@ -1966,14 +1958,13 @@ environment_variables: LANGSMITH_BASE_URL: "https://api.smith.langchain.com" # (Optional - only needed if you have a custom Langsmith instance) ``` - 2. Start Proxy ``` litellm --config /path/to/config.yaml ``` -3. Test it! +3. Test it! ```bash curl --location 'http://0.0.0.0:4000/chat/completions' \ @@ -1989,10 +1980,10 @@ curl --location 'http://0.0.0.0:4000/chat/completions' \ } ' ``` + Expect to see your log on Langfuse - ## Arize AI 1. Set `success_callback: ["arize"]` on litellm config.yaml @@ -2009,10 +2000,10 @@ litellm_settings: callbacks: ["arize"] environment_variables: - ARIZE_SPACE_KEY: "d0*****" - ARIZE_API_KEY: "141a****" - ARIZE_ENDPOINT: "https://otlp.arize.com/v1" # OPTIONAL - your custom arize GRPC api endpoint - ARIZE_HTTP_ENDPOINT: "https://otlp.arize.com/v1" # OPTIONAL - your custom arize HTTP api endpoint. Set either this or ARIZE_ENDPOINT + ARIZE_SPACE_ID: "d0*****" + ARIZE_API_KEY: "141a****" + ARIZE_ENDPOINT: "https://otlp.arize.com/v1" # OPTIONAL - your custom arize GRPC api endpoint + ARIZE_HTTP_ENDPOINT: "https://otlp.arize.com/v1" # OPTIONAL - your custom arize HTTP api endpoint. Set either this or ARIZE_ENDPOINT ``` 2. Start Proxy @@ -2021,7 +2012,7 @@ environment_variables: litellm --config /path/to/config.yaml ``` -3. Test it! +3. Test it! ```bash curl --location 'http://0.0.0.0:4000/chat/completions' \ @@ -2037,10 +2028,10 @@ curl --location 'http://0.0.0.0:4000/chat/completions' \ } ' ``` + Expect to see your log on Langfuse - ## Langtrace 1. Set `success_callback: ["langtrace"]` on litellm config.yaml @@ -2057,7 +2048,7 @@ litellm_settings: callbacks: ["langtrace"] environment_variables: - LANGTRACE_API_KEY: "141a****" + LANGTRACE_API_KEY: "141a****" ``` 2. Start Proxy @@ -2066,7 +2057,7 @@ environment_variables: litellm --config /path/to/config.yaml ``` -3. Test it! +3. Test it! ```bash curl --location 'http://0.0.0.0:4000/chat/completions' \ @@ -2104,17 +2095,17 @@ export GALILEO_USERNAME="" export GALILEO_PASSWORD="" ``` -#### Quick Start +#### Quick Start 1. Add to Config.yaml ```yaml model_list: -- litellm_params: - api_base: https://exampleopenaiendpoint-production.up.railway.app/ - api_key: my-fake-key - model: openai/my-fake-model - model_name: fake-openai-endpoint + - litellm_params: + api_base: https://exampleopenaiendpoint-production.up.railway.app/ + api_key: my-fake-key + model: openai/my-fake-model + model_name: fake-openai-endpoint litellm_settings: success_callback: ["galileo"] # 👈 KEY CHANGE @@ -2126,7 +2117,7 @@ litellm_settings: litellm --config /path/to/config.yaml ``` -3. Test it! +3. Test it! ```bash curl --location 'http://0.0.0.0:4000/chat/completions' \ @@ -2157,17 +2148,17 @@ export OPENMETER_API_ENDPOINT="" # defaults to https://openmeter.cloud export OPENMETER_API_KEY="" ``` -##### Quick Start +##### Quick Start 1. Add to Config.yaml ```yaml model_list: -- litellm_params: - api_base: https://openai-function-calling-workers.tasslexyz.workers.dev/ - api_key: my-fake-key - model: openai/my-fake-model - model_name: fake-openai-endpoint + - litellm_params: + api_base: https://openai-function-calling-workers.tasslexyz.workers.dev/ + api_key: my-fake-key + model: openai/my-fake-model + model_name: fake-openai-endpoint litellm_settings: success_callback: ["openmeter"] # 👈 KEY CHANGE @@ -2179,7 +2170,7 @@ litellm_settings: litellm --config /path/to/config.yaml ``` -3. Test it! +3. Test it! ```bash curl --location 'http://0.0.0.0:4000/chat/completions' \ @@ -2200,9 +2191,9 @@ curl --location 'http://0.0.0.0:4000/chat/completions' \ ## DynamoDB -We will use the `--config` to set +We will use the `--config` to set -- `litellm.success_callback = ["dynamodb"]` +- `litellm.success_callback = ["dynamodb"]` - `litellm.dynamodb_table_name = "your-table-name"` This will log all successfull LLM calls to DynamoDB @@ -2326,7 +2317,7 @@ Your logs should be available on DynamoDB ## Sentry -If api calls fail (llm/database) you can log those to Sentry: +If api calls fail (llm/database) you can log those to Sentry: **Step 1** Install Sentry @@ -2340,7 +2331,7 @@ pip install --upgrade sentry-sdk export SENTRY_DSN="your-sentry-dsn" ``` -```yaml +```yaml model_list: - model_name: gpt-3.5-turbo litellm_params: @@ -2348,7 +2339,7 @@ model_list: litellm_settings: # other settings failure_callback: ["sentry"] -general_settings: +general_settings: database_url: "my-bad-url" # set a fake url to trigger a sentry exception ``` @@ -2413,7 +2404,6 @@ curl --location 'http://0.0.0.0:4000/chat/completions' \ }' ``` - \ No newline at end of file +::: --> diff --git a/litellm/integrations/arize/_utils.py b/litellm/integrations/arize/_utils.py index 9921d47aff..aded5f7469 100644 --- a/litellm/integrations/arize/_utils.py +++ b/litellm/integrations/arize/_utils.py @@ -12,7 +12,7 @@ else: def set_attributes(span: Span, kwargs, response_obj): - from openinference.semconv.trace import ( + from litellm.integrations._types.open_inference import ( MessageAttributes, OpenInferenceSpanKindValues, SpanAttributes, diff --git a/litellm/integrations/arize/arize.py b/litellm/integrations/arize/arize.py index 652957e1ee..b7e2bc4545 100644 --- a/litellm/integrations/arize/arize.py +++ b/litellm/integrations/arize/arize.py @@ -40,11 +40,11 @@ class ArizeLogger: Raises: ValueError: If required environment variables are not set. """ - space_key = os.environ.get("ARIZE_SPACE_KEY") + space_id = os.environ.get("ARIZE_SPACE_ID") api_key = os.environ.get("ARIZE_API_KEY") - if not space_key: - raise ValueError("ARIZE_SPACE_KEY not found in environment variables") + if not space_id: + raise ValueError("ARIZE_SPACE_ID not found in environment variables") if not api_key: raise ValueError("ARIZE_API_KEY not found in environment variables") @@ -65,7 +65,7 @@ class ArizeLogger: endpoint = "https://otlp.arize.com/v1" return ArizeConfig( - space_key=space_key, + space_id=space_id, api_key=api_key, protocol=protocol, endpoint=endpoint, diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 0945c45491..1fc9c95458 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -2654,7 +2654,7 @@ def _init_custom_logger_compatible_class( # noqa: PLR0915 ) os.environ["OTEL_EXPORTER_OTLP_TRACES_HEADERS"] = ( - f"space_key={arize_config.space_key},api_key={arize_config.api_key}" + f"space_id={arize_config.space_id},api_key={arize_config.api_key}" ) for callback in _in_memory_loggers: if ( @@ -2899,8 +2899,8 @@ def get_custom_logger_compatible_class( # noqa: PLR0915 elif logging_integration == "arize": from litellm.integrations.opentelemetry import OpenTelemetry - if "ARIZE_SPACE_KEY" not in os.environ: - raise ValueError("ARIZE_SPACE_KEY not found in environment variables") + if "ARIZE_SPACE_ID" not in os.environ: + raise ValueError("ARIZE_SPACE_ID not found in environment variables") if "ARIZE_API_KEY" not in os.environ: raise ValueError("ARIZE_API_KEY not found in environment variables") for callback in _in_memory_loggers: diff --git a/litellm/types/integrations/arize.py b/litellm/types/integrations/arize.py index 24298fc363..d8c02f438c 100644 --- a/litellm/types/integrations/arize.py +++ b/litellm/types/integrations/arize.py @@ -1,4 +1,4 @@ -from typing import TYPE_CHECKING, Literal, Any +from typing import TYPE_CHECKING, Literal, Any, Optional from pydantic import BaseModel @@ -8,7 +8,7 @@ else: Protocol = Any class ArizeConfig(BaseModel): - space_key: str - api_key: str + space_id: str + api_key: str protocol: Protocol endpoint: str diff --git a/tests/local_testing/test_arize_ai.py b/tests/local_testing/test_arize_ai.py index 9d2f27ca30..37ad4685b0 100644 --- a/tests/local_testing/test_arize_ai.py +++ b/tests/local_testing/test_arize_ai.py @@ -1,11 +1,13 @@ import asyncio import logging +from litellm import Choices import pytest from dotenv import load_dotenv import litellm from litellm._logging import verbose_logger, verbose_proxy_logger +from litellm.integrations._types.open_inference import SpanAttributes from litellm.integrations.arize.arize import ArizeConfig, ArizeLogger load_dotenv() @@ -32,7 +34,7 @@ async def test_async_otel_callback(): @pytest.fixture def mock_env_vars(monkeypatch): - monkeypatch.setenv("ARIZE_SPACE_KEY", "test_space_key") + monkeypatch.setenv("ARIZE_SPACE_ID", "test_space_id") monkeypatch.setenv("ARIZE_API_KEY", "test_api_key") @@ -42,7 +44,7 @@ def test_get_arize_config(mock_env_vars): """ config = ArizeLogger.get_arize_config() assert isinstance(config, ArizeConfig) - assert config.space_key == "test_space_key" + assert config.space_id == "test_space_id" assert config.api_key == "test_api_key" assert config.endpoint == "https://otlp.arize.com/v1" assert config.protocol == "otlp_grpc" @@ -58,3 +60,41 @@ def test_get_arize_config_with_endpoints(mock_env_vars, monkeypatch): config = ArizeLogger.get_arize_config() assert config.endpoint == "grpc://test.endpoint" assert config.protocol == "otlp_grpc" + + +def test_arize_set_attributes(): + """ + Test setting attributes for Arize + """ + from unittest.mock import MagicMock + from litellm.types.utils import ModelResponse + + span = MagicMock() + kwargs = { + "role": "user", + "content": "simple arize test", + "model": "gpt-4o", + "messages": [{"role": "user", "content": "basic arize test"}], + "litellm_params": {"metadata": {"key": "value"}}, + "standard_logging_object": {"model_parameters": {"user": "test_user"}} + } + response_obj = ModelResponse(usage={"total_tokens": 100, "completion_tokens": 60, "prompt_tokens": 40}, + choices=[Choices(message={"role": "assistant", "content": "response content"})]) + + ArizeLogger.set_arize_attributes(span, kwargs, response_obj) + + assert span.set_attribute.call_count == 14 + span.set_attribute.assert_any_call(SpanAttributes.METADATA, str({"key": "value"})) + span.set_attribute.assert_any_call(SpanAttributes.LLM_MODEL_NAME, "gpt-4o") + span.set_attribute.assert_any_call(SpanAttributes.OPENINFERENCE_SPAN_KIND, "LLM") + span.set_attribute.assert_any_call(SpanAttributes.INPUT_VALUE, "basic arize test") + span.set_attribute.assert_any_call("llm.input_messages.0.message.role", "user") + span.set_attribute.assert_any_call("llm.input_messages.0.message.content", "basic arize test") + span.set_attribute.assert_any_call(SpanAttributes.LLM_INVOCATION_PARAMETERS, '{"user": "test_user"}') + span.set_attribute.assert_any_call(SpanAttributes.USER_ID, "test_user") + span.set_attribute.assert_any_call(SpanAttributes.OUTPUT_VALUE, "response content") + span.set_attribute.assert_any_call("llm.output_messages.0.message.role", "assistant") + span.set_attribute.assert_any_call("llm.output_messages.0.message.content", "response content") + span.set_attribute.assert_any_call(SpanAttributes.LLM_TOKEN_COUNT_TOTAL, 100) + span.set_attribute.assert_any_call(SpanAttributes.LLM_TOKEN_COUNT_COMPLETION, 60) + span.set_attribute.assert_any_call(SpanAttributes.LLM_TOKEN_COUNT_PROMPT, 40) \ No newline at end of file diff --git a/tests/logging_callback_tests/test_arize_logging.py b/tests/logging_callback_tests/test_arize_logging.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/logging_callback_tests/test_unit_tests_init_callbacks.py b/tests/logging_callback_tests/test_unit_tests_init_callbacks.py index fcba3ebbc3..654a4b0615 100644 --- a/tests/logging_callback_tests/test_unit_tests_init_callbacks.py +++ b/tests/logging_callback_tests/test_unit_tests_init_callbacks.py @@ -89,7 +89,7 @@ expected_env_vars = { "OPIK_API_KEY": "opik_api_key", "LANGTRACE_API_KEY": "langtrace_api_key", "LOGFIRE_TOKEN": "logfire_token", - "ARIZE_SPACE_KEY": "arize_space_key", + "ARIZE_SPACE_ID": "arize_space_id", "ARIZE_API_KEY": "arize_api_key", "PHOENIX_API_KEY": "phoenix_api_key", "ARGILLA_API_KEY": "argilla_api_key",