bump: version 1.53.1 → 1.53.2

* fix(factory.py): ensure tool call converts image url

Fixes https://github.com/BerriAI/litellm/issues/6953

* fix(transformation.py): support mp4 + pdf url's for vertex ai

Fixes https://github.com/BerriAI/litellm/issues/6936

* fix(http_handler.py): mask gemini api key in error logs

Fixes https://github.com/BerriAI/litellm/issues/6963

* docs(prometheus.md): update prometheus FAQs

* feat(auth_checks.py): ensure specific model access > wildcard model access

if wildcard model is in access group, but specific model is not - deny access

* fix(auth_checks.py): handle auth checks for team based model access groups

handles scenario where model access group used for wildcard models

* fix(internal_user_endpoints.py): support adding guardrails on `/user/update`

Fixes https://github.com/BerriAI/litellm/issues/6942

* fix(key_management_endpoints.py): fix prepare_metadata_fields helper

* fix: fix tests

* build(requirements.txt): bump openai dep version

fixes proxies argument

* test: fix tests

* fix(http_handler.py): fix error message masking

* fix(bedrock_guardrails.py): pass in prepped data

* test: fix test

* test: fix nvidia nim test

* fix(http_handler.py): return original response headers

* fix: revert maskedhttpstatuserror

* test: update tests

* test: cleanup test

* fix(key_management_endpoints.py): fix metadata field update logic

* fix(key_management_endpoints.py): maintain initial order of guardrails in key update

* fix(key_management_endpoints.py): handle prepare metadata

* fix: fix linting errors

* fix: fix linting errors

* fix: fix linting errors

* fix: fix key management errors

* fix(key_management_endpoints.py): update metadata

* test: update test

* refactor: add more debug statements

* test: skip flaky test

* test: fix test

* fix: fix test

* fix: fix update metadata logic

* fix: fix test

* ci(config.yml): change db url for e2e ui testing

.circleci/config.yml

@@ -625,6 +625,48 @@ jobs:
           paths:
             - llm_translation_coverage.xml
             - llm_translation_coverage
+  pass_through_unit_testing:
+    docker:
+      - image: cimg/python:3.11
+        auth:
+          username: ${DOCKERHUB_USERNAME}
+          password: ${DOCKERHUB_PASSWORD}
+    working_directory: ~/project
+
+    steps:
+      - checkout
+      - run:
+          name: Install Dependencies
+          command: |
+            python -m pip install --upgrade pip
+            python -m pip install -r requirements.txt
+            pip install "pytest==7.3.1"
+            pip install "pytest-retry==1.6.3"
+            pip install "pytest-cov==5.0.0"
+            pip install "pytest-asyncio==0.21.1"
+            pip install "respx==0.21.1"
+      # Run pytest and generate JUnit XML report
+      - run:
+          name: Run tests
+          command: |
+            pwd
+            ls
+            python -m pytest -vv tests/pass_through_unit_tests --cov=litellm --cov-report=xml -x -s -v --junitxml=test-results/junit.xml --durations=5
+          no_output_timeout: 120m
+      - run:
+          name: Rename the coverage files
+          command: |
+            mv coverage.xml pass_through_unit_tests_coverage.xml
+            mv .coverage pass_through_unit_tests_coverage
+        
+      # Store test results
+      - store_test_results:
+          path: test-results
+      - persist_to_workspace:
+          root: .
+          paths:
+            - pass_through_unit_tests_coverage.xml
+            - pass_through_unit_tests_coverage
   image_gen_testing:
     docker:
       - image: cimg/python:3.11
@@ -690,6 +732,7 @@ jobs:
             pip install "respx==0.21.1"
             pip install "google-generativeai==0.3.2"
             pip install "google-cloud-aiplatform==1.43.0"
+            pip install "mlflow==2.17.2"
       # Run pytest and generate JUnit XML report
       - run:
           name: Run tests
@@ -764,11 +807,14 @@ jobs:
             curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash
       - run: python -c "from litellm import *" || (echo '🚨 import failed, this means you introduced unprotected imports! 🚨'; exit 1)
       - run: ruff check ./litellm
-      - run: python ./tests/documentation_tests/test_general_setting_keys.py
+      # - run: python ./tests/documentation_tests/test_general_setting_keys.py
       - run: python ./tests/code_coverage_tests/router_code_coverage.py
       - run: python ./tests/code_coverage_tests/test_router_strategy_async.py
       - run: python ./tests/code_coverage_tests/litellm_logging_code_coverage.py
       - run: python ./tests/documentation_tests/test_env_keys.py
+      - run: python ./tests/documentation_tests/test_router_settings.py
+      - run: python ./tests/documentation_tests/test_api_docs.py
+      - run: python ./tests/code_coverage_tests/ensure_async_clients_test.py
       - run: helm lint ./deploy/charts/litellm-helm
 
   db_migration_disable_update_check:
@@ -920,7 +966,7 @@ jobs:
           command: |
             pwd
             ls
-            python -m pytest -s -vv tests/*.py -x --junitxml=test-results/junit.xml --durations=5 --ignore=tests/otel_tests --ignore=tests/pass_through_tests --ignore=tests/proxy_admin_ui_tests --ignore=tests/load_tests --ignore=tests/llm_translation --ignore=tests/image_gen_tests
+            python -m pytest -s -vv tests/*.py -x --junitxml=test-results/junit.xml --durations=5 --ignore=tests/otel_tests --ignore=tests/pass_through_tests --ignore=tests/proxy_admin_ui_tests --ignore=tests/load_tests --ignore=tests/llm_translation --ignore=tests/image_gen_tests --ignore=tests/pass_through_unit_tests
           no_output_timeout: 120m
 
       # Store test results
@@ -1037,6 +1083,48 @@ jobs:
             ls
             python -m pytest -vv tests/otel_tests -x --junitxml=test-results/junit.xml --durations=5
           no_output_timeout: 120m
+           # Clean up first container
+      - run:
+          name: Stop and remove first container
+          command: |
+            docker stop my-app
+            docker rm my-app
+
+      # Second Docker Container Run with Different Config
+      # NOTE: We intentionally pass a "bad" license here. We need to ensure proxy starts and serves request even with bad license
+      - run:
+          name: Run Second Docker container
+          command: |
+            docker run -d \
+              -p 4000:4000 \
+              -e DATABASE_URL=$PROXY_DATABASE_URL \
+              -e REDIS_HOST=$REDIS_HOST \
+              -e REDIS_PASSWORD=$REDIS_PASSWORD \
+              -e REDIS_PORT=$REDIS_PORT \
+              -e LITELLM_MASTER_KEY="sk-1234" \
+              -e OPENAI_API_KEY=$OPENAI_API_KEY \
+              -e LITELLM_LICENSE="bad-license" \
+              --name my-app-3 \
+              -v $(pwd)/litellm/proxy/example_config_yaml/enterprise_config.yaml:/app/config.yaml \
+              my-app:latest \
+              --config /app/config.yaml \
+              --port 4000 \
+              --detailed_debug
+
+      - run:
+          name: Start outputting logs for second container
+          command: docker logs -f my-app-2
+          background: true
+
+      - run:
+          name: Wait for second app to be ready
+          command: dockerize -wait http://localhost:4000 -timeout 5m
+
+      - run:
+          name: Run second round of tests
+          command: |
+            python -m pytest -vv tests/basic_proxy_startup_tests -x --junitxml=test-results/junit-2.xml --durations=5
+          no_output_timeout: 120m
 
       # Store test results
       - store_test_results:
@@ -1091,6 +1179,8 @@ jobs:
             pip install "asyncio==3.4.3"
             pip install "PyGithub==1.59.1"
             pip install "google-cloud-aiplatform==1.59.0"
+            pip install anthropic
+      # Run pytest and generate JUnit XML report
       - run:
           name: Build Docker image
           command: docker build -t my-app:latest -f ./docker/Dockerfile.database .
@@ -1102,6 +1192,8 @@ jobs:
               -e DATABASE_URL=$PROXY_DATABASE_URL \
               -e LITELLM_MASTER_KEY="sk-1234" \
               -e OPENAI_API_KEY=$OPENAI_API_KEY \
+              -e GEMINI_API_KEY=$GEMINI_API_KEY \
+              -e ANTHROPIC_API_KEY=$ANTHROPIC_API_KEY \
               -e LITELLM_LICENSE=$LITELLM_LICENSE \
               --name my-app \
               -v $(pwd)/litellm/proxy/example_config_yaml/pass_through_config.yaml:/app/config.yaml \
@@ -1125,6 +1217,27 @@ jobs:
       - run:
           name: Wait for app to be ready
           command: dockerize -wait http://localhost:4000 -timeout 5m
+      # New steps to run Node.js test
+      - run:
+          name: Install Node.js
+          command: |
+            curl -fsSL https://deb.nodesource.com/setup_18.x | sudo -E bash -
+            sudo apt-get install -y nodejs
+            node --version
+            npm --version
+
+      - run:
+          name: Install Node.js dependencies
+          command: |
+            npm install @google-cloud/vertexai
+            npm install @google/generative-ai
+            npm install --save-dev jest
+
+      - run:
+          name: Run Vertex AI, Google AI Studio Node.js tests
+          command: |
+            npx jest tests/pass_through_tests --verbose
+          no_output_timeout: 30m
       - run:
           name: Run tests
           command: |
@@ -1132,7 +1245,6 @@ jobs:
             ls
             python -m pytest -vv tests/pass_through_tests/ -x --junitxml=test-results/junit.xml --durations=5
           no_output_timeout: 120m
-
       # Store test results
       - store_test_results:
           path: test-results
@@ -1158,7 +1270,7 @@ jobs:
             python -m venv venv
             . venv/bin/activate
             pip install coverage
-            coverage combine llm_translation_coverage logging_coverage litellm_router_coverage local_testing_coverage litellm_assistants_api_coverage auth_ui_unit_tests_coverage langfuse_coverage caching_coverage litellm_proxy_unit_tests_coverage image_gen_coverage
+            coverage combine llm_translation_coverage logging_coverage litellm_router_coverage local_testing_coverage litellm_assistants_api_coverage auth_ui_unit_tests_coverage langfuse_coverage caching_coverage litellm_proxy_unit_tests_coverage image_gen_coverage pass_through_unit_tests_coverage
             coverage xml
       - codecov/upload:
           file: ./coverage.xml
@@ -1264,6 +1376,7 @@ jobs:
           name: Install Dependencies
           command: |
             npm install -D @playwright/test
+            npm install @google-cloud/vertexai
             pip install "pytest==7.3.1"
             pip install "pytest-retry==1.6.3"
             pip install "pytest-asyncio==0.21.1"
@@ -1295,7 +1408,7 @@ jobs:
           command: |
             docker run -d \
               -p 4000:4000 \
-              -e DATABASE_URL=$PROXY_DATABASE_URL \
+              -e DATABASE_URL=$PROXY_DATABASE_URL_2 \
               -e LITELLM_MASTER_KEY="sk-1234" \
               -e OPENAI_API_KEY=$OPENAI_API_KEY \
               -e UI_USERNAME="admin" \
@@ -1325,7 +1438,7 @@ jobs:
       - run:
           name: Run Playwright Tests
           command: |
-            npx playwright test --reporter=html --output=test-results
+            npx playwright test e2e_ui_tests/ --reporter=html --output=test-results
           no_output_timeout: 120m
       - store_test_results:
           path: test-results
@@ -1447,6 +1560,12 @@ workflows:
               only:
                 - main
                 - /litellm_.*/
+      - pass_through_unit_testing:
+          filters:
+            branches:
+              only:
+                - main
+                - /litellm_.*/
       - image_gen_testing:
           filters:
             branches:
@@ -1462,6 +1581,7 @@ workflows:
       - upload-coverage:
           requires:
             - llm_translation_testing
+            - pass_through_unit_testing
             - image_gen_testing
             - logging_testing
             - litellm_router_testing
@@ -1502,6 +1622,7 @@ workflows:
             - load_testing
             - test_bad_database_url
             - llm_translation_testing
+            - pass_through_unit_testing
             - image_gen_testing
             - logging_testing
             - litellm_router_testing

README.md

@@ -113,7 +113,7 @@ for part in response:
 
 ## Logging Observability ([Docs](https://docs.litellm.ai/docs/observability/callbacks))
 
-LiteLLM exposes pre defined callbacks to send data to Lunary, Langfuse, DynamoDB, s3 Buckets, Helicone, Promptlayer, Traceloop, Athina, Slack
+LiteLLM exposes pre defined callbacks to send data to Lunary, Langfuse, DynamoDB, s3 Buckets, Helicone, Promptlayer, Traceloop, Athina, Slack, MLflow
 
 ```python
 from litellm import completion
@@ -305,6 +305,36 @@ Step 4: Submit a PR with your changes! 🚀
 - push your fork to your GitHub repo
 - submit a PR from there
 
+### Building LiteLLM Docker Image 
+
+Follow these instructions if you want to build / run the LiteLLM Docker Image yourself.
+
+Step 1: Clone the repo
+
+```
+git clone https://github.com/BerriAI/litellm.git
+```
+
+Step 2: Build the Docker Image
+
+Build using Dockerfile.non_root
+```
+docker build -f docker/Dockerfile.non_root -t litellm_test_image .
+```
+
+Step 3: Run the Docker Image
+
+Make sure config.yaml is present in the root directory. This is your litellm proxy config file.
+```
+docker run \
+    -v $(pwd)/proxy_config.yaml:/app/config.yaml \
+    -e DATABASE_URL="postgresql://xxxxxxxx" \
+    -e LITELLM_MASTER_KEY="sk-1234" \
+    -p 4000:4000 \
+    litellm_test_image \
+    --config /app/config.yaml --detailed_debug
+```
+
 # Enterprise
 For companies that need better security, user management and professional support
 

deploy/charts/litellm-helm/templates/migrations-job.yaml

@@ -6,24 +6,25 @@ metadata:
   name: {{ include "litellm.fullname" . }}-migrations
   annotations:
     argocd.argoproj.io/hook: PreSync
-    argocd.argoproj.io/hook-delete-policy: HookSucceeded
+    argocd.argoproj.io/hook-delete-policy: Never # keep this resource so we can debug status on ArgoCD
+    checksum/config: {{ toYaml .Values | sha256sum }}
 spec:
   template:
     spec:
       containers:
         - name: prisma-migrations
-          image: "ghcr.io/berriai/litellm:main-stable"
+          image: ghcr.io/berriai/litellm-database:main-latest
           command: ["python", "litellm/proxy/prisma_migration.py"]
           workingDir: "/app"
           env:
-            {{- if .Values.db.deployStandalone }}
-            - name: DATABASE_URL
-              value: postgresql://{{ .Values.postgresql.auth.username }}:{{ .Values.postgresql.auth.password }}@{{ .Release.Name }}-postgresql/{{ .Values.postgresql.auth.database }}
-            {{- else if .Values.db.useExisting }}
+            {{- if .Values.db.useExisting }}
             - name: DATABASE_URL
               value: {{ .Values.db.url | quote }}
+            {{- else }}
+            - name: DATABASE_URL
+              value: postgresql://{{ .Values.postgresql.auth.username }}:{{ .Values.postgresql.auth.password }}@{{ .Release.Name }}-postgresql/{{ .Values.postgresql.auth.database }}
             {{- end }}
             - name: DISABLE_SCHEMA_UPDATE
-              value: "{{ .Values.migrationJob.disableSchemaUpdate }}"
+              value: "false" # always run the migration from the Helm PreSync hook, override the value set
       restartPolicy: OnFailure
   backoffLimit: {{ .Values.migrationJob.backoffLimit }}

docs/my-website/docs/anthropic_completion.md

@@ -1,54 +0,0 @@
-# [BETA] Anthropic `/v1/messages`
-
-Call 100+ LLMs in the Anthropic format. 
-
-
-1. Setup config.yaml 
-
-```yaml
-model_list:
-  - model_name: my-test-model
-    litellm_params:
-      model: gpt-3.5-turbo
-```
-
-2. Start proxy 
-
-```bash
-litellm --config /path/to/config.yaml
-```
-
-3. Test it! 
-
-```bash
-curl -X POST 'http://0.0.0.0:4000/v1/messages' \
--H 'x-api-key: sk-1234' \
--H 'content-type: application/json' \
--D '{
-    "model": "my-test-model",
-    "max_tokens": 1024,
-    "messages": [
-        {"role": "user", "content": "Hello, world"}
-    ]
-}'
-```
-
-## Test with Anthropic SDK 
-
-```python
-import os
-from anthropic import Anthropic
-
-client = Anthropic(api_key="sk-1234", base_url="http://0.0.0.0:4000") # 👈 CONNECT TO PROXY
-
-message = client.messages.create(
-    messages=[
-        {
-            "role": "user",
-            "content": "Hello, Claude",
-        }
-    ],
-    model="my-test-model", # 👈 set 'model_name'
-)
-print(message.content)
-```

docs/my-website/docs/completion/input.md

@@ -41,7 +41,7 @@ Use `litellm.get_supported_openai_params()` for an updated list of params for ea
 
 | Provider | temperature | max_completion_tokens | max_tokens | top_p | stream | stream_options | stop | n | presence_penalty | frequency_penalty | functions | function_call | logit_bias | user | response_format | seed | tools | tool_choice | logprobs | top_logprobs | extra_headers |
 |---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
-|Anthropic| ✅ | ✅ | ✅ |✅ | ✅ | ✅ | ✅ |  |  |   |  |  |  |✅ | ✅ | ✅ | ✅ | ✅ |  |  | ✅ |
+|Anthropic| ✅ | ✅ | ✅ |✅ | ✅ | ✅ | ✅ |  |  |   |  |  |  |✅ | ✅ | | ✅ | ✅ |  |  | ✅ |
 |OpenAI| ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |✅ | ✅ | ✅ | ✅ |✅ | ✅ | ✅ | ✅ | ✅ |
 |Azure OpenAI| ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |✅ | ✅ | ✅ | ✅ |✅ | ✅ |  |  | ✅ |
 |Replicate | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | |  |   |  |   |

docs/my-website/docs/completion/json_mode.md

@@ -75,6 +75,9 @@ Works for:
 - Google AI Studio - Gemini models
 - Vertex AI models (Gemini + Anthropic)
 - Bedrock Models
+- Anthropic API Models
+- Groq Models
+- Ollama Models
 
 <Tabs>
 <TabItem value="sdk" label="SDK">

docs/my-website/docs/completion/prefix.md

@@ -93,7 +93,7 @@ curl http://0.0.0.0:4000/v1/chat/completions \
 
 ## Check Model Support 
 
-Call `litellm.get_model_info` to check if a model/provider supports `response_format`. 
+Call `litellm.get_model_info` to check if a model/provider supports `prefix`. 
 
 <Tabs>
 <TabItem value="sdk" label="SDK">
@@ -116,4 +116,4 @@ curl -X GET 'http://0.0.0.0:4000/v1/model/info' \
 -H 'Authorization: Bearer $LITELLM_KEY' \
 ```
 </TabItem>
-</Tabs>
+</Tabs>

docs/my-website/docs/embedding/supported_embedding.md

@@ -1,7 +1,7 @@
 import Tabs from '@theme/Tabs';
 import TabItem from '@theme/TabItem';
 
-# Embedding Models
+# Embeddings
 
 ## Quick Start
 ```python

docs/my-website/docs/guides/finetuned_models.md

@@ -0,0 +1,74 @@
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+
+
+# Calling Finetuned Models
+
+## OpenAI
+
+
+| Model Name                | Function Call                                                          |
+|---------------------------|-----------------------------------------------------------------|
+| fine tuned `gpt-4-0613`    | `response = completion(model="ft:gpt-4-0613", messages=messages)`     |
+| fine tuned `gpt-4o-2024-05-13` | `response = completion(model="ft:gpt-4o-2024-05-13", messages=messages)` |
+| fine tuned `gpt-3.5-turbo-0125` | `response = completion(model="ft:gpt-3.5-turbo-0125", messages=messages)` |
+| fine tuned `gpt-3.5-turbo-1106` | `response = completion(model="ft:gpt-3.5-turbo-1106", messages=messages)` |
+| fine tuned `gpt-3.5-turbo-0613` | `response = completion(model="ft:gpt-3.5-turbo-0613", messages=messages)` |
+
+
+## Vertex AI
+
+Fine tuned models on vertex have a numerical model/endpoint id. 
+
+<Tabs>
+<TabItem value="sdk" label="SDK">
+
+```python
+from litellm import completion
+import os
+
+## set ENV variables
+os.environ["VERTEXAI_PROJECT"] = "hardy-device-38811"
+os.environ["VERTEXAI_LOCATION"] = "us-central1"
+
+response = completion(
+  model="vertex_ai/<your-finetuned-model>",  # e.g. vertex_ai/4965075652664360960
+  messages=[{ "content": "Hello, how are you?","role": "user"}],
+  base_model="vertex_ai/gemini-1.5-pro" # the base model - used for routing
+)
+```
+
+</TabItem>
+<TabItem value="proxy" label="PROXY">
+
+1. Add Vertex Credentials to your env 
+
+```bash
+!gcloud auth application-default login
+```
+
+2. Setup config.yaml 
+
+```yaml
+- model_name: finetuned-gemini
+  litellm_params:
+    model: vertex_ai/<ENDPOINT_ID>
+    vertex_project: <PROJECT_ID>
+    vertex_location: <LOCATION>
+  model_info:
+    base_model: vertex_ai/gemini-1.5-pro # IMPORTANT
+```
+
+3. Test it! 
+
+```bash
+curl --location 'https://0.0.0.0:4000/v1/chat/completions' \
+--header 'Content-Type: application/json' \
+--header 'Authorization: <LITELLM_KEY>' \
+--data '{"model": "finetuned-gemini" ,"messages":[{"role": "user", "content":[{"type": "text", "text": "hi"}]}]}'
+```
+
+</TabItem>
+</Tabs>
+
+

docs/my-website/docs/image_generation.md

@@ -1,4 +1,4 @@
-# Image Generation
+# Images
 
 ## Quick Start
 

docs/my-website/docs/moderation.md

@@ -0,0 +1,135 @@
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+
+# Moderation
+
+
+### Usage
+<Tabs>
+<TabItem value="python" label="LiteLLM Python SDK">
+
+```python
+from litellm import moderation
+
+response = moderation(
+    input="hello from litellm",
+    model="text-moderation-stable"
+)
+```
+
+</TabItem>
+<TabItem value="proxy" label="LiteLLM Proxy Server">
+
+For `/moderations` endpoint, there is **no need to specify `model` in the request or on the litellm config.yaml**
+
+Start litellm proxy server 
+
+```
+litellm
+```
+
+
+<Tabs>
+<TabItem value="python" label="OpenAI Python SDK">
+
+```python
+from openai import OpenAI
+
+# set base_url to your proxy server
+# set api_key to send to proxy server
+client = OpenAI(api_key="<proxy-api-key>", base_url="http://0.0.0.0:4000")
+
+response = client.moderations.create(
+    input="hello from litellm",
+    model="text-moderation-stable" # optional, defaults to `omni-moderation-latest`
+)
+
+print(response)
+```
+</TabItem>
+
+<TabItem value="curl" label="Curl Request">
+
+```shell
+curl --location 'http://0.0.0.0:4000/moderations' \
+    --header 'Content-Type: application/json' \
+    --header 'Authorization: Bearer sk-1234' \
+    --data '{"input": "Sample text goes here", "model": "text-moderation-stable"}'
+```
+</TabItem>
+</Tabs>
+
+</TabItem>
+</Tabs>
+
+## Input Params
+LiteLLM accepts and translates the [OpenAI Moderation params](https://platform.openai.com/docs/api-reference/moderations) across all supported providers.
+
+### Required Fields
+
+- `input`: *string or array* - Input (or inputs) to classify. Can be a single string, an array of strings, or an array of multi-modal input objects similar to other models.
+  - If string: A string of text to classify for moderation
+  - If array of strings: An array of strings to classify for moderation
+  - If array of objects: An array of multi-modal inputs to the moderation model, where each object can be:
+    - An object describing an image to classify with:
+      - `type`: *string, required* - Always `image_url`
+      - `image_url`: *object, required* - Contains either an image URL or a data URL for a base64 encoded image
+    - An object describing text to classify with:
+      - `type`: *string, required* - Always `text`
+      - `text`: *string, required* - A string of text to classify
+
+### Optional Fields
+
+- `model`: *string (optional)* - The moderation model to use. Defaults to `omni-moderation-latest`.
+
+## Output Format
+Here's the exact json output and type you can expect from all moderation calls:
+
+[**LiteLLM follows OpenAI's output format**](https://platform.openai.com/docs/api-reference/moderations/object)
+
+
+```python
+{
+  "id": "modr-AB8CjOTu2jiq12hp1AQPfeqFWaORR",
+  "model": "text-moderation-007",
+  "results": [
+    {
+      "flagged": true,
+      "categories": {
+        "sexual": false,
+        "hate": false,
+        "harassment": true,
+        "self-harm": false,
+        "sexual/minors": false,
+        "hate/threatening": false,
+        "violence/graphic": false,
+        "self-harm/intent": false,
+        "self-harm/instructions": false,
+        "harassment/threatening": true,
+        "violence": true
+      },
+      "category_scores": {
+        "sexual": 0.000011726012417057063,
+        "hate": 0.22706663608551025,
+        "harassment": 0.5215635299682617,
+        "self-harm": 2.227119921371923e-6,
+        "sexual/minors": 7.107352217872176e-8,
+        "hate/threatening": 0.023547329008579254,
+        "violence/graphic": 0.00003391829886822961,
+        "self-harm/intent": 1.646940972932498e-6,
+        "self-harm/instructions": 1.1198755256458526e-9,
+        "harassment/threatening": 0.5694745779037476,
+        "violence": 0.9971134662628174
+      }
+    }
+  ]
+}
+
+```
+
+
+## **Supported Providers**
+
+| Provider    |
+|-------------|
+| OpenAI      |  

docs/my-website/docs/observability/argilla.md

@@ -4,24 +4,63 @@ import TabItem from '@theme/TabItem';
 
 # Argilla 
 
-Argilla is a tool for annotating datasets. 
+Argilla is a collaborative annotation tool for AI engineers and domain experts who need to build high-quality datasets for their projects.
 
 
+## Getting Started
 
-## Usage 
+To log the data to Argilla, first you need to deploy the Argilla server. If you have not deployed the Argilla server, please follow the instructions [here](https://docs.argilla.io/latest/getting_started/quickstart/).
+
+Next, you will need to configure and create the Argilla dataset.
+
+```python
+import argilla as rg
+
+client = rg.Argilla(api_url="<api_url>", api_key="<api_key>")
+
+settings = rg.Settings(
+    guidelines="These are some guidelines.",
+    fields=[
+        rg.ChatField(
+            name="user_input",
+        ),
+        rg.TextField(
+            name="llm_output",
+        ),
+    ],
+    questions=[
+        rg.RatingQuestion(
+            name="rating",
+            values=[1, 2, 3, 4, 5, 6, 7],
+        ),
+    ],
+)
+
+dataset = rg.Dataset(
+    name="my_first_dataset",
+    settings=settings,
+)
+
+dataset.create()
+```
+
+For further configuration, please refer to the [Argilla documentation](https://docs.argilla.io/latest/how_to_guides/dataset/).
+
+
+## Usage
 
 <Tabs>
 <Tab value="sdk" label="SDK">
 
 ```python
-from litellm import completion
+import os
 import litellm
-import os 
+from litellm import completion
 
 # add env vars
 os.environ["ARGILLA_API_KEY"]="argilla.apikey"
 os.environ["ARGILLA_BASE_URL"]="http://localhost:6900"
-os.environ["ARGILLA_DATASET_NAME"]="my_second_dataset"   
+os.environ["ARGILLA_DATASET_NAME"]="my_first_dataset"   
 os.environ["OPENAI_API_KEY"]="sk-proj-..."
 
 litellm.callbacks = ["argilla"]

docs/my-website/docs/observability/mlflow.md

@@ -0,0 +1,108 @@
+# MLflow
+
+## What is MLflow?
+
+**MLflow** is an end-to-end open source MLOps platform for [experiment tracking](https://www.mlflow.org/docs/latest/tracking.html), [model management](https://www.mlflow.org/docs/latest/models.html), [evaluation](https://www.mlflow.org/docs/latest/llms/llm-evaluate/index.html), [observability (tracing)](https://www.mlflow.org/docs/latest/llms/tracing/index.html), and [deployment](https://www.mlflow.org/docs/latest/deployment/index.html). MLflow empowers teams to collaboratively develop and refine LLM applications efficiently.
+
+MLflow’s integration with LiteLLM supports advanced observability compatible with OpenTelemetry.
+
+
+<Image img={require('../../img/mlflow_tracing.png')} />
+
+
+## Getting Started
+
+Install MLflow:
+
+```shell
+pip install mlflow
+```
+
+To enable LiteLLM tracing:
+
+```python
+import mlflow
+
+mlflow.litellm.autolog()
+
+# Alternative, you can set the callback manually in LiteLLM
+# litellm.callbacks = ["mlflow"]
+```
+
+Since MLflow is open-source, no sign-up or API key is needed to log traces!
+
+```
+import litellm
+import os
+
+# Set your LLM provider's API key
+os.environ["OPENAI_API_KEY"] = ""
+
+# Call LiteLLM as usual
+response = litellm.completion(
+    model="gpt-4o-mini",
+    messages=[
+      {"role": "user", "content": "Hi 👋 - i'm openai"}
+    ]
+)
+```
+
+Open the MLflow UI and go to the `Traces` tab to view logged traces:
+
+```bash
+mlflow ui
+```
+
+## Exporting Traces to OpenTelemetry collectors
+
+MLflow traces are compatible with OpenTelemetry. You can export traces to any OpenTelemetry collector (e.g., Jaeger, Zipkin, Datadog, New Relic) by setting the endpoint URL in the environment variables.
+
+```
+# Set the endpoint of the OpenTelemetry Collector
+os.environ["OTEL_EXPORTER_OTLP_TRACES_ENDPOINT"] = "http://localhost:4317/v1/traces"
+# Optionally, set the service name to group traces
+os.environ["OTEL_SERVICE_NAME"] = "<your-service-name>"
+```
+
+See [MLflow documentation](https://mlflow.org/docs/latest/llms/tracing/index.html#using-opentelemetry-collector-for-exporting-traces) for more details.
+
+## Combine LiteLLM Trace with Your Application Trace
+
+LiteLLM is often part of larger LLM applications, such as agentic models. MLflow Tracing allows you to instrument custom Python code, which can then be combined with LiteLLM traces.
+
+```python
+import litellm
+import mlflow
+from mlflow.entities import SpanType
+
+# Enable LiteLLM tracing
+mlflow.litellm.autolog()
+
+
+class CustomAgent:
+    # Use @mlflow.trace to instrument Python functions.
+    @mlflow.trace(span_type=SpanType.AGENT)
+    def run(self, query: str):
+        # do something
+
+        while i < self.max_turns:
+            response = litellm.completion(
+                model="gpt-4o-mini",
+                messages=messages,
+            )
+
+            action = self.get_action(response)
+            ...
+
+    @mlflow.trace
+    def get_action(llm_response):
+        ...
+```
+
+This approach generates a unified trace, combining your custom Python code with LiteLLM calls.
+
+
+## Support
+
+* For advanced usage and integrations of tracing, visit the [MLflow Tracing documentation](https://mlflow.org/docs/latest/llms/tracing/index.html).
+* For any question or issue with this integration, please [submit an issue](https://github.com/mlflow/mlflow/issues/new/choose) on our [Github](https://github.com/mlflow/mlflow) repository!

docs/my-website/docs/observability/opentelemetry_integration.md

@@ -86,3 +86,20 @@ Be aware that if you are continuing an existing trace, and you set `update_trace
 ## Support
 
 For any question or issue with the integration you can reach out to the OpenLLMetry maintainers on [Slack](https://traceloop.com/slack) or via [email](mailto:dev@traceloop.com).
+
+## Troubleshooting
+
+### Trace LiteLLM Proxy user/key/org/team information on failed requests
+
+LiteLLM emits the user_api_key_metadata
+- key hash
+- key_alias
+- org_id
+- user_id
+- team_id
+
+for successful + failed requests 
+
+click under `litellm_request` in the trace
+
+<Image img={require('../../img/otel_debug_trace.png')} />

docs/my-website/docs/pass_through/anthropic_completion.md

@@ -0,0 +1,371 @@
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+
+# Anthropic SDK
+
+Pass-through endpoints for Anthropic - call provider-specific endpoint, in native format (no translation).
+
+Just replace `https://api.anthropic.com` with `LITELLM_PROXY_BASE_URL/anthropic`
+
+#### **Example Usage**
+
+
+<Tabs>
+<TabItem value="curl" label="curl">
+
+```bash
+curl --request POST \
+  --url http://0.0.0.0:4000/anthropic/v1/messages \
+  --header 'accept: application/json' \
+  --header 'content-type: application/json' \
+  --header "Authorization: bearer sk-anything" \
+  --data '{
+        "model": "claude-3-5-sonnet-20241022",
+        "max_tokens": 1024,
+        "messages": [
+            {"role": "user", "content": "Hello, world"}
+        ]
+    }'
+```
+
+</TabItem>
+<TabItem value="python" label="Anthropic Python SDK">
+
+```python
+from anthropic import Anthropic
+
+# Initialize client with proxy base URL
+client = Anthropic(
+    base_url="http://0.0.0.0:4000/anthropic", # <proxy-base-url>/anthropic
+    api_key="sk-anything" # proxy virtual key
+)
+
+# Make a completion request
+response = client.messages.create(
+    model="claude-3-5-sonnet-20241022",
+    max_tokens=1024,
+    messages=[
+        {"role": "user", "content": "Hello, world"}
+    ]
+)
+
+print(response)
+```
+
+</TabItem>
+</Tabs>
+
+Supports **ALL** Anthropic Endpoints (including streaming).
+
+[**See All Anthropic Endpoints**](https://docs.anthropic.com/en/api/messages)
+
+## Quick Start
+
+Let's call the Anthropic [`/messages` endpoint](https://docs.anthropic.com/en/api/messages)
+
+1. Add Anthropic API Key to your environment 
+
+```bash
+export ANTHROPIC_API_KEY=""
+```
+
+2. Start LiteLLM Proxy 
+
+```bash
+litellm
+
+# RUNNING on http://0.0.0.0:4000
+```
+
+3. Test it! 
+
+Let's call the Anthropic /messages endpoint
+
+```bash
+curl http://0.0.0.0:4000/anthropic/v1/messages \
+     --header "x-api-key: $LITELLM_API_KEY" \
+     --header "anthropic-version: 2023-06-01" \
+     --header "content-type: application/json" \
+     --data \
+    '{
+        "model": "claude-3-5-sonnet-20241022",
+        "max_tokens": 1024,
+        "messages": [
+            {"role": "user", "content": "Hello, world"}
+        ]
+    }'
+```
+
+
+## Examples
+
+Anything after `http://0.0.0.0:4000/anthropic` is treated as a provider-specific route, and handled accordingly.
+
+Key Changes: 
+
+| **Original Endpoint**                                | **Replace With**                  |
+|------------------------------------------------------|-----------------------------------|
+| `https://api.anthropic.com`          | `http://0.0.0.0:4000/anthropic` (LITELLM_PROXY_BASE_URL="http://0.0.0.0:4000")      |
+| `bearer $ANTHROPIC_API_KEY`                                 | `bearer anything` (use `bearer LITELLM_VIRTUAL_KEY` if Virtual Keys are setup on proxy)                    |
+    
+
+### **Example 1: Messages endpoint**
+
+#### LiteLLM Proxy Call 
+
+```bash
+curl --request POST \
+  --url http://0.0.0.0:4000/anthropic/v1/messages \
+  --header "x-api-key: $LITELLM_API_KEY" \
+    --header "anthropic-version: 2023-06-01" \
+    --header "content-type: application/json" \
+  --data '{
+    "model": "claude-3-5-sonnet-20241022",
+    "max_tokens": 1024,
+    "messages": [
+        {"role": "user", "content": "Hello, world"}
+    ]
+  }'
+```
+
+#### Direct Anthropic API Call 
+
+```bash
+curl https://api.anthropic.com/v1/messages \
+     --header "x-api-key: $ANTHROPIC_API_KEY" \
+     --header "anthropic-version: 2023-06-01" \
+     --header "content-type: application/json" \
+     --data \
+    '{
+        "model": "claude-3-5-sonnet-20241022",
+        "max_tokens": 1024,
+        "messages": [
+            {"role": "user", "content": "Hello, world"}
+        ]
+    }'
+```
+
+### **Example 2: Token Counting API**
+
+#### LiteLLM Proxy Call 
+
+```bash
+curl --request POST \
+    --url http://0.0.0.0:4000/anthropic/v1/messages/count_tokens \
+    --header "x-api-key: $LITELLM_API_KEY" \
+    --header "anthropic-version: 2023-06-01" \
+    --header "anthropic-beta: token-counting-2024-11-01" \
+    --header "content-type: application/json" \
+    --data \
+    '{
+        "model": "claude-3-5-sonnet-20241022",
+        "messages": [
+            {"role": "user", "content": "Hello, world"}
+        ]
+    }'
+```
+
+#### Direct Anthropic API Call 
+
+```bash
+curl https://api.anthropic.com/v1/messages/count_tokens \
+     --header "x-api-key: $ANTHROPIC_API_KEY" \
+     --header "anthropic-version: 2023-06-01" \
+     --header "anthropic-beta: token-counting-2024-11-01" \
+     --header "content-type: application/json" \
+     --data \
+'{
+    "model": "claude-3-5-sonnet-20241022",
+    "messages": [
+        {"role": "user", "content": "Hello, world"}
+    ]
+}'
+```
+
+### **Example 3: Batch Messages**
+
+
+#### LiteLLM Proxy Call 
+
+```bash
+curl --request POST \
+    --url http://0.0.0.0:4000/anthropic/v1/messages/batches \
+    --header "x-api-key: $LITELLM_API_KEY" \
+    --header "anthropic-version: 2023-06-01" \
+    --header "anthropic-beta: message-batches-2024-09-24" \
+    --header "content-type: application/json" \
+    --data \
+'{
+    "requests": [
+        {
+            "custom_id": "my-first-request",
+            "params": {
+                "model": "claude-3-5-sonnet-20241022",
+                "max_tokens": 1024,
+                "messages": [
+                    {"role": "user", "content": "Hello, world"}
+                ]
+            }
+        },
+        {
+            "custom_id": "my-second-request",
+            "params": {
+                "model": "claude-3-5-sonnet-20241022",
+                "max_tokens": 1024,
+                "messages": [
+                    {"role": "user", "content": "Hi again, friend"}
+                ]
+            }
+        }
+    ]
+}'
+```
+
+#### Direct Anthropic API Call 
+
+```bash
+curl https://api.anthropic.com/v1/messages/batches \
+     --header "x-api-key: $ANTHROPIC_API_KEY" \
+     --header "anthropic-version: 2023-06-01" \
+     --header "anthropic-beta: message-batches-2024-09-24" \
+     --header "content-type: application/json" \
+     --data \
+'{
+    "requests": [
+        {
+            "custom_id": "my-first-request",
+            "params": {
+                "model": "claude-3-5-sonnet-20241022",
+                "max_tokens": 1024,
+                "messages": [
+                    {"role": "user", "content": "Hello, world"}
+                ]
+            }
+        },
+        {
+            "custom_id": "my-second-request",
+            "params": {
+                "model": "claude-3-5-sonnet-20241022",
+                "max_tokens": 1024,
+                "messages": [
+                    {"role": "user", "content": "Hi again, friend"}
+                ]
+            }
+        }
+    ]
+}'
+```
+
+
+## Advanced
+
+Pre-requisites
+- [Setup proxy with DB](../proxy/virtual_keys.md#setup)
+
+Use this, to avoid giving developers the raw Anthropic API key, but still letting them use Anthropic endpoints.
+
+### Use with Virtual Keys 
+
+1. Setup environment
+
+```bash
+export DATABASE_URL=""
+export LITELLM_MASTER_KEY=""
+export COHERE_API_KEY=""
+```
+
+```bash
+litellm
+
+# RUNNING on http://0.0.0.0:4000
+```
+
+2. Generate virtual key 
+
+```bash
+curl -X POST 'http://0.0.0.0:4000/key/generate' \
+-H 'Authorization: Bearer sk-1234' \
+-H 'Content-Type: application/json' \
+-d '{}'
+```
+
+Expected Response 
+
+```bash
+{
+    ...
+    "key": "sk-1234ewknldferwedojwojw"
+}
+```
+
+3. Test it! 
+
+
+```bash
+curl --request POST \
+  --url http://0.0.0.0:4000/anthropic/v1/messages \
+  --header 'accept: application/json' \
+  --header 'content-type: application/json' \
+  --header "Authorization: bearer sk-1234ewknldferwedojwojw" \
+  --data '{
+    "model": "claude-3-5-sonnet-20241022",
+    "max_tokens": 1024,
+    "messages": [
+        {"role": "user", "content": "Hello, world"}
+    ]
+  }'
+```
+
+
+### Send `litellm_metadata` (tags)
+
+<Tabs>
+<TabItem value="curl" label="curl">
+
+```bash
+curl --request POST \
+  --url http://0.0.0.0:4000/anthropic/v1/messages \
+  --header 'accept: application/json' \
+  --header 'content-type: application/json' \
+  --header "Authorization: bearer sk-anything" \
+  --data '{
+    "model": "claude-3-5-sonnet-20241022",
+    "max_tokens": 1024,
+    "messages": [
+        {"role": "user", "content": "Hello, world"}
+    ],
+    "litellm_metadata": {
+        "tags": ["test-tag-1", "test-tag-2"]
+    }
+  }'
+```
+
+</TabItem>
+<TabItem value="python" label="Anthropic Python SDK">
+
+```python
+from anthropic import Anthropic
+
+client = Anthropic(
+    base_url="http://0.0.0.0:4000/anthropic",
+    api_key="sk-anything"
+)
+
+response = client.messages.create(
+    model="claude-3-5-sonnet-20241022",
+    max_tokens=1024,
+    messages=[
+        {"role": "user", "content": "Hello, world"}
+    ],
+    extra_body={
+        "litellm_metadata": {
+            "tags": ["test-tag-1", "test-tag-2"]
+        }
+    }
+)
+
+print(response)
+```
+
+</TabItem>
+</Tabs>

docs/my-website/docs/pass_through/cohere.md

@@ -1,4 +1,4 @@
-# Cohere API
+# Cohere SDK
 
 Pass-through endpoints for Cohere - call provider-specific endpoint, in native format (no translation).
 

docs/my-website/docs/pass_through/google_ai_studio.md

@@ -1,12 +1,21 @@
-# Google AI Studio
+import Image from '@theme/IdealImage';
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+
+
+# Google AI Studio SDK
 
 Pass-through endpoints for Google AI Studio - call provider-specific endpoint, in native format (no translation).
 
-Just replace `https://generativelanguage.googleapis.com` with `LITELLM_PROXY_BASE_URL/gemini` 🚀
+Just replace `https://generativelanguage.googleapis.com` with `LITELLM_PROXY_BASE_URL/gemini`
 
 #### **Example Usage**
+
+<Tabs>
+<TabItem value="curl" label="curl">
+
 ```bash
-http://0.0.0.0:4000/gemini/v1beta/models/gemini-1.5-flash:countTokens?key=sk-anything' \
+curl 'http://0.0.0.0:4000/gemini/v1beta/models/gemini-1.5-flash:countTokens?key=sk-anything' \
 -H 'Content-Type: application/json' \
 -d '{
     "contents": [{
@@ -17,6 +26,53 @@ http://0.0.0.0:4000/gemini/v1beta/models/gemini-1.5-flash:countTokens?key=sk-any
 }'
 ```
 
+</TabItem>
+<TabItem value="js" label="Google AI Node.js SDK">
+
+```javascript
+const { GoogleGenerativeAI } = require("@google/generative-ai");
+
+const modelParams = {
+    model: 'gemini-pro',
+};
+  
+const requestOptions = {
+    baseUrl: 'http://localhost:4000/gemini', // http://<proxy-base-url>/gemini
+};
+  
+const genAI = new GoogleGenerativeAI("sk-1234"); // litellm proxy API key
+const model = genAI.getGenerativeModel(modelParams, requestOptions);
+
+async function main() {
+    try {
+        const result = await model.generateContent("Explain how AI works");
+        console.log(result.response.text());
+    } catch (error) {
+        console.error('Error:', error);
+    }
+}
+
+// For streaming responses
+async function main_streaming() {
+    try {
+        const streamingResult = await model.generateContentStream("Explain how AI works");
+        for await (const chunk of streamingResult.stream) {
+            console.log('Stream chunk:', JSON.stringify(chunk));
+        }
+        const aggregatedResponse = await streamingResult.response;
+        console.log('Aggregated response:', JSON.stringify(aggregatedResponse));
+    } catch (error) {
+        console.error('Error:', error);
+    }
+}
+
+main();
+// main_streaming();
+```
+
+</TabItem>
+</Tabs>
+
 Supports **ALL** Google AI Studio Endpoints (including streaming).
 
 [**See All Google AI Studio Endpoints**](https://ai.google.dev/api)
@@ -166,14 +222,14 @@ curl -X POST "https://generativelanguage.googleapis.com/v1beta/models/gemini-1.5
 ```
 
 
-## Advanced - Use with Virtual Keys 
+## Advanced 
 
 Pre-requisites
 - [Setup proxy with DB](../proxy/virtual_keys.md#setup)
 
 Use this, to avoid giving developers the raw Google AI Studio key, but still letting them use Google AI Studio endpoints.
 
-### Usage
+### Use with Virtual Keys
 
 1. Setup environment
 
@@ -220,4 +276,66 @@ http://0.0.0.0:4000/gemini/v1beta/models/gemini-1.5-flash:countTokens?key=sk-123
           }]
         }]
 }'
-```
+```
+
+
+### Send `tags` in request headers
+
+Use this if you want `tags` to be tracked in the LiteLLM DB and on logging callbacks.
+
+Pass tags in request headers as a comma separated list. In the example below the following tags will be tracked
+
+```
+tags: ["gemini-js-sdk", "pass-through-endpoint"]
+```
+
+<Tabs>
+<TabItem value="curl" label="curl">
+
+```bash
+curl 'http://0.0.0.0:4000/gemini/v1beta/models/gemini-1.5-flash:generateContent?key=sk-anything' \
+-H 'Content-Type: application/json' \
+-H 'tags: gemini-js-sdk,pass-through-endpoint' \
+-d '{
+    "contents": [{
+        "parts":[{
+          "text": "The quick brown fox jumps over the lazy dog."
+          }]
+        }]
+}'
+```
+
+</TabItem>
+<TabItem value="js" label="Google AI Node.js SDK">
+
+```javascript
+const { GoogleGenerativeAI } = require("@google/generative-ai");
+
+const modelParams = {
+    model: 'gemini-pro',
+};
+  
+const requestOptions = {
+    baseUrl: 'http://localhost:4000/gemini', // http://<proxy-base-url>/gemini
+    customHeaders: {
+        "tags": "gemini-js-sdk,pass-through-endpoint"
+    }
+};
+  
+const genAI = new GoogleGenerativeAI("sk-1234");
+const model = genAI.getGenerativeModel(modelParams, requestOptions);
+
+async function main() {
+    try {
+        const result = await model.generateContent("Explain how AI works");
+        console.log(result.response.text());
+    } catch (error) {
+        console.error('Error:', error);
+    }
+}
+
+main();
+```
+
+</TabItem>
+</Tabs>

docs/my-website/docs/pass_through/langfuse.md

@@ -1,4 +1,4 @@
-# Langfuse Endpoints 
+# Langfuse SDK
 
 Pass-through endpoints for Langfuse - call langfuse endpoints with LiteLLM Virtual Key.
 

docs/my-website/docs/providers/anthropic.md

@@ -10,6 +10,35 @@ LiteLLM supports all anthropic models.
 - `claude-2.1`
 - `claude-instant-1.2`
 
+
+| Property | Details |
+|-------|-------|
+| Description | Claude is a highly performant, trustworthy, and intelligent AI platform built by Anthropic. Claude excels at tasks involving language, reasoning, analysis, coding, and more. |
+| Provider Route on LiteLLM | `anthropic/` (add this prefix to the model name, to route any requests to Anthropic - e.g. `anthropic/claude-3-5-sonnet-20240620`) |
+| Provider Doc | [Anthropic ↗](https://docs.anthropic.com/en/docs/build-with-claude/overview) |
+| API Endpoint for Provider | https://api.anthropic.com |
+| Supported Endpoints | `/chat/completions` |
+
+
+## Supported OpenAI Parameters
+
+Check this in code, [here](../completion/input.md#translated-openai-params)
+
+```
+"stream",
+"stop",
+"temperature",
+"top_p",
+"max_tokens",
+"max_completion_tokens",
+"tools",
+"tool_choice",
+"extra_headers",
+"parallel_tool_calls",
+"response_format",
+"user"
+```
+
 :::info
 
 Anthropic API fails requests when `max_tokens` are not passed. Due to this litellm passes `max_tokens=4096` when no `max_tokens` are passed.
@@ -957,3 +986,52 @@ curl http://0.0.0.0:4000/v1/chat/completions \
 ```
 </TabItem>
 </Tabs>
+
+## Usage - passing 'user_id' to Anthropic
+
+LiteLLM translates the OpenAI `user` param to Anthropic's `metadata[user_id]` param.
+
+<Tabs>
+<TabItem value="sdk" label="SDK">
+
+```python
+response = completion(
+    model="claude-3-5-sonnet-20240620",
+    messages=messages,
+    user="user_123",
+)
+```
+</TabItem>
+<TabItem value="proxy" label="PROXY">
+
+1. Setup config.yaml
+
+```yaml
+model_list:
+    - model_name: claude-3-5-sonnet-20240620
+      litellm_params:
+        model: anthropic/claude-3-5-sonnet-20240620
+        api_key: os.environ/ANTHROPIC_API_KEY
+```
+
+2. Start Proxy
+
+```
+litellm --config /path/to/config.yaml
+```
+
+3. Test it! 
+
+```bash
+curl http://0.0.0.0:4000/v1/chat/completions \
+  -H "Content-Type: application/json" \
+  -H "Authorization: Bearer <YOUR-LITELLM-KEY>" \
+  -d '{
+    "model": "claude-3-5-sonnet-20240620",
+    "messages": [{"role": "user", "content": "What is Anthropic?"}],
+    "user": "user_123"
+  }'
+```
+
+</TabItem>
+</Tabs>

docs/my-website/docs/providers/gemini.md

@@ -10,6 +10,7 @@ import TabItem from '@theme/TabItem';
 | Provider Route on LiteLLM | `gemini/` |
 | Provider Doc | [Google AI Studio ↗](https://ai.google.dev/aistudio) |
 | API Endpoint for Provider | https://generativelanguage.googleapis.com |
+| Supported Endpoints | `/chat/completions`, `/embeddings` |
 
 <br />
 

docs/my-website/docs/providers/huggingface.md

@@ -37,7 +37,7 @@ os.environ["HUGGINGFACE_API_KEY"] = "huggingface_api_key"
 messages = [{ "content": "There's a llama in my garden 😱 What should I do?","role": "user"}]
 
 # e.g. Call 'https://huggingface.co/meta-llama/Meta-Llama-3.1-8B-Instruct' from Serverless Inference API
-response = litellm.completion(
+response = completion(
     model="huggingface/meta-llama/Meta-Llama-3.1-8B-Instruct",
     messages=[{ "content": "Hello, how are you?","role": "user"}],
     stream=True
@@ -165,14 +165,14 @@ Steps to use
 
 ```python
 import os
-import litellm
+from litellm import completion
 
 os.environ["HUGGINGFACE_API_KEY"] = ""
 
 # TGI model: Call https://huggingface.co/glaiveai/glaive-coder-7b
 # add the 'huggingface/' prefix to the model to set huggingface as the provider
 # set api base to your deployed api endpoint from hugging face
-response = litellm.completion(
+response = completion(
     model="huggingface/glaiveai/glaive-coder-7b",
     messages=[{ "content": "Hello, how are you?","role": "user"}],
     api_base="https://wjiegasee9bmqke2.us-east-1.aws.endpoints.huggingface.cloud"
@@ -383,6 +383,8 @@ def default_pt(messages):
 #### Custom prompt templates
 
 ```python
+import litellm
+
 # Create your own custom prompt template works
 litellm.register_prompt_template(
 	    model="togethercomputer/LLaMA-2-7B-32K",

docs/my-website/docs/providers/jina_ai.md

@@ -1,6 +1,13 @@
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+
 # Jina AI
 https://jina.ai/embeddings/
 
+Supported endpoints: 
+- /embeddings
+- /rerank
+
 ## API Key
 ```python
 # env variable
@@ -8,6 +15,10 @@ os.environ['JINA_AI_API_KEY']
 ```
 
 ## Sample Usage - Embedding
+
+<Tabs>
+<TabItem value="sdk" label="SDK">
+
 ```python
 from litellm import embedding
 import os
@@ -19,6 +30,142 @@ response = embedding(
 )
 print(response)
 ```
+</TabItem>
+<TabItem value="proxy" label="PROXY">
+
+1. Add to config.yaml
+```yaml
+model_list:
+  - model_name: embedding-model
+    litellm_params:
+      model: jina_ai/jina-embeddings-v3
+      api_key: os.environ/JINA_AI_API_KEY
+```
+
+2. Start proxy 
+
+```bash
+litellm --config /path/to/config.yaml
+
+# RUNNING on http://0.0.0.0:4000/
+```
+
+3. Test it! 
+
+```bash 
+curl -L -X POST 'http://0.0.0.0:4000/embeddings' \
+-H 'Authorization: Bearer sk-1234' \
+-H 'Content-Type: application/json' \
+-d '{"input": ["hello world"], "model": "embedding-model"}'
+```
+
+</TabItem>
+</Tabs>
+
+## Sample Usage - Rerank
+
+<Tabs>
+<TabItem value="sdk" label="SDK">
+
+```python
+from litellm import rerank
+import os
+
+os.environ["JINA_AI_API_KEY"] = "sk-..."
+
+query = "What is the capital of the United States?"
+documents = [
+    "Carson City is the capital city of the American state of Nevada.",
+    "The Commonwealth of the Northern Mariana Islands is a group of islands in the Pacific Ocean. Its capital is Saipan.",
+    "Washington, D.C. is the capital of the United States.",
+    "Capital punishment has existed in the United States since before it was a country.",
+]
+
+response = rerank(
+    model="jina_ai/jina-reranker-v2-base-multilingual",
+    query=query,
+    documents=documents,
+    top_n=3,
+)
+print(response)
+```
+</TabItem>
+<TabItem value="proxy" label="PROXY">
+
+1. Add to config.yaml
+```yaml
+model_list:
+  - model_name: rerank-model
+    litellm_params:
+      model: jina_ai/jina-reranker-v2-base-multilingual
+      api_key: os.environ/JINA_AI_API_KEY
+```
+
+2. Start proxy 
+
+```bash
+litellm --config /path/to/config.yaml
+```
+
+3. Test it! 
+
+```bash 
+curl -L -X POST 'http://0.0.0.0:4000/rerank' \
+-H 'Authorization: Bearer sk-1234' \
+-H 'Content-Type: application/json' \
+-d '{
+    "model": "rerank-model",
+    "query": "What is the capital of the United States?",
+    "documents": [
+        "Carson City is the capital city of the American state of Nevada.",
+        "The Commonwealth of the Northern Mariana Islands is a group of islands in the Pacific Ocean. Its capital is Saipan.",
+        "Washington, D.C. is the capital of the United States.",
+        "Capital punishment has existed in the United States since before it was a country."
+    ],
+    "top_n": 3
+}'
+```
+
+</TabItem>
+</Tabs>
 
 ## Supported Models
 All models listed here https://jina.ai/embeddings/ are supported
+
+## Supported Optional Rerank Parameters
+
+All cohere rerank parameters are supported. 
+
+## Supported Optional Embeddings Parameters
+
+```
+dimensions
+```
+
+## Provider-specific parameters
+
+Pass any jina ai specific parameters as a keyword argument to the `embedding` or `rerank` function, e.g. 
+
+<Tabs>
+<TabItem value="sdk" label="SDK">
+
+```python
+response = embedding(
+    model="jina_ai/jina-embeddings-v3",
+    input=["good morning from litellm"],
+    dimensions=1536,
+    my_custom_param="my_custom_value", # any other jina ai specific parameters
+)
+```
+</TabItem>
+<TabItem value="proxy" label="PROXY">
+
+```bash
+curl -L -X POST 'http://0.0.0.0:4000/embeddings' \
+-H 'Authorization: Bearer sk-1234' \
+-H 'Content-Type: application/json' \
+-d '{"input": ["good morning from litellm"], "model": "jina_ai/jina-embeddings-v3", "dimensions": 1536, "my_custom_param": "my_custom_value"}'
+```
+
+</TabItem>
+</Tabs>

docs/my-website/docs/providers/vertex.md

@@ -572,6 +572,96 @@ Here's how to use Vertex AI with the LiteLLM Proxy Server
 
   </Tabs>
 
+
+## Authentication - vertex_project, vertex_location, etc. 
+
+Set your vertex credentials via:
+- dynamic params
+OR
+- env vars 
+
+
+### **Dynamic Params**
+
+You can set:
+- `vertex_credentials` (str) - can be a json string or filepath to your vertex ai service account.json
+- `vertex_location` (str) - place where vertex model is deployed (us-central1, asia-southeast1, etc.)
+- `vertex_project` Optional[str] - use if vertex project different from the one in vertex_credentials
+
+as dynamic params for a `litellm.completion` call. 
+
+<Tabs>
+<TabItem value="sdk" label="SDK">
+
+```python
+from litellm import completion
+import json 
+
+## GET CREDENTIALS 
+file_path = 'path/to/vertex_ai_service_account.json'
+
+# Load the JSON file
+with open(file_path, 'r') as file:
+    vertex_credentials = json.load(file)
+
+# Convert to JSON string
+vertex_credentials_json = json.dumps(vertex_credentials)
+
+
+response = completion(
+  model="vertex_ai/gemini-pro",
+  messages=[{"content": "You are a good bot.","role": "system"}, {"content": "Hello, how are you?","role": "user"}], 
+  vertex_credentials=vertex_credentials_json,
+  vertex_project="my-special-project", 
+  vertex_location="my-special-location"
+)
+```
+
+</TabItem>
+<TabItem value="proxy" label="PROXY">
+
+```yaml
+model_list:
+    - model_name: gemini-1.5-pro
+      litellm_params:
+        model: gemini-1.5-pro
+        vertex_credentials: os.environ/VERTEX_FILE_PATH_ENV_VAR # os.environ["VERTEX_FILE_PATH_ENV_VAR"] = "/path/to/service_account.json" 
+        vertex_project: "my-special-project"
+        vertex_location: "my-special-location:
+```
+
+</TabItem>
+</Tabs>
+
+
+
+
+### **Environment Variables**
+
+You can set:
+- `GOOGLE_APPLICATION_CREDENTIALS` - store the filepath for your service_account.json in here (used by vertex sdk directly).
+- VERTEXAI_LOCATION - place where vertex model is deployed (us-central1, asia-southeast1, etc.)
+- VERTEXAI_PROJECT - Optional[str] - use if vertex project different from the one in vertex_credentials
+
+1. GOOGLE_APPLICATION_CREDENTIALS
+
+```bash
+export GOOGLE_APPLICATION_CREDENTIALS="/path/to/service_account.json"
+```
+
+2. VERTEXAI_LOCATION
+
+```bash
+export VERTEXAI_LOCATION="us-central1" # can be any vertex location
+```
+
+3. VERTEXAI_PROJECT
+
+```bash
+export VERTEXAI_PROJECT="my-test-project" # ONLY use if model project is different from service account project
+```
+
+
 ## Specifying Safety Settings 
 In certain use-cases you may need to make calls to the models and pass [safety settigns](https://ai.google.dev/docs/safety_setting_gemini) different from the defaults. To do so, simple pass the `safety_settings` argument to `completion` or `acompletion`. For example:
 
@@ -1161,12 +1251,96 @@ curl --location 'http://0.0.0.0:4000/chat/completions' \
 
 
 ## Model Garden
-| Model Name       | Function Call                        |
-|------------------|--------------------------------------|
-| llama2   | `completion('vertex_ai/<endpoint_id>', messages)` |
+
+:::tip
+
+All OpenAI compatible models from Vertex Model Garden are supported. 
+
+:::
 
 #### Using Model Garden
 
+**Almost all Vertex Model Garden models are OpenAI compatible.**
+
+<Tabs>
+
+<TabItem value="openai" label="OpenAI Compatible Models">
+
+| Property | Details |
+|----------|---------|
+| Provider Route | `vertex_ai/openai/{MODEL_ID}` |
+| Vertex Documentation | [Vertex Model Garden - OpenAI Chat Completions](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/model_garden/model_garden_gradio_streaming_chat_completions.ipynb), [Vertex Model Garden](https://cloud.google.com/model-garden?hl=en) |
+| Supported Operations | `/chat/completions`, `/embeddings` |
+
+<Tabs>
+<TabItem value="sdk" label="SDK">
+
+```python
+from litellm import completion
+import os
+
+## set ENV variables
+os.environ["VERTEXAI_PROJECT"] = "hardy-device-38811"
+os.environ["VERTEXAI_LOCATION"] = "us-central1"
+
+response = completion(
+  model="vertex_ai/openai/<your-endpoint-id>", 
+  messages=[{ "content": "Hello, how are you?","role": "user"}]
+)
+```
+
+</TabItem>
+
+<TabItem value="proxy" label="Proxy">
+
+
+**1. Add to config**
+
+```yaml
+model_list:
+    - model_name: llama3-1-8b-instruct
+      litellm_params:
+        model: vertex_ai/openai/5464397967697903616
+        vertex_ai_project: "my-test-project"
+        vertex_ai_location: "us-east-1"
+```
+
+**2. Start proxy**
+
+```bash
+litellm --config /path/to/config.yaml
+
+# RUNNING at http://0.0.0.0:4000
+```
+
+**3. Test it!**
+
+```bash
+curl --location 'http://0.0.0.0:4000/chat/completions' \
+      --header 'Authorization: Bearer sk-1234' \
+      --header 'Content-Type: application/json' \
+      --data '{
+            "model": "llama3-1-8b-instruct", # 👈 the 'model_name' in config
+            "messages": [
+                {
+                "role": "user",
+                "content": "what llm are you"
+                }
+            ],
+        }'
+```
+
+
+
+
+</TabItem>
+
+</Tabs>
+
+</TabItem>
+
+<TabItem value="non-openai" label="Non-OpenAI Compatible Models">
+
 ```python
 from litellm import completion
 import os
@@ -1181,6 +1355,11 @@ response = completion(
 )
 ```
 
+</TabItem>
+
+</Tabs>
+
+
 ## Gemini Pro
 | Model Name       | Function Call                        |
 |------------------|--------------------------------------|
@@ -1562,6 +1741,10 @@ curl http://0.0.0.0:4000/v1/chat/completions \
 ## **Embedding Models**
 
 #### Usage - Embedding
+
+<Tabs>
+<TabItem value="sdk" label="SDK">
+
 ```python
 import litellm
 from litellm import embedding
@@ -1574,6 +1757,49 @@ response = embedding(
 )
 print(response)
 ```
+</TabItem>
+
+<TabItem value="proxy" label="LiteLLM PROXY">
+
+
+1. Add model to config.yaml
+```yaml
+model_list:
+  - model_name: snowflake-arctic-embed-m-long-1731622468876
+    litellm_params:
+      model: vertex_ai/<your-model-id>
+      vertex_project: "adroit-crow-413218"
+      vertex_location: "us-central1"
+      vertex_credentials: adroit-crow-413218-a956eef1a2a8.json 
+
+litellm_settings:
+  drop_params: True
+```
+
+2. Start Proxy 
+
+```
+$ litellm --config /path/to/config.yaml
+```
+
+3. Make Request using OpenAI Python SDK, Langchain Python SDK
+
+```python
+import openai
+
+client = openai.OpenAI(api_key="sk-1234", base_url="http://0.0.0.0:4000")
+
+response = client.embeddings.create(
+    model="snowflake-arctic-embed-m-long-1731622468876", 
+    input = ["good morning from litellm", "this is another item"],
+)
+
+print(response)
+```
+
+
+</TabItem>
+</Tabs>
 
 #### Supported Embedding Models
 All models listed [here](https://github.com/BerriAI/litellm/blob/57f37f743886a0249f630a6792d49dffc2c5d9b7/model_prices_and_context_window.json#L835) are supported
@@ -1589,6 +1815,7 @@ All models listed [here](https://github.com/BerriAI/litellm/blob/57f37f743886a02
 | textembedding-gecko@003 | `embedding(model="vertex_ai/textembedding-gecko@003", input)` | 
 | text-embedding-preview-0409 | `embedding(model="vertex_ai/text-embedding-preview-0409", input)` |
 | text-multilingual-embedding-preview-0409 | `embedding(model="vertex_ai/text-multilingual-embedding-preview-0409", input)` | 
+| Fine-tuned OR Custom Embedding models | `embedding(model="vertex_ai/<your-model-id>", input)` | 
 
 ### Supported OpenAI (Unified) Params
 
@@ -2166,97 +2393,6 @@ print("response from proxy", response)
 </TabItem>
 </Tabs>
 
-
-
-## Authentication - vertex_project, vertex_location, etc. 
-
-Set your vertex credentials via:
-- dynamic params
-OR
-- env vars 
-
-
-### **Dynamic Params**
-
-You can set:
-- `vertex_credentials` (str) - can be a json string or filepath to your vertex ai service account.json
-- `vertex_location` (str) - place where vertex model is deployed (us-central1, asia-southeast1, etc.)
-- `vertex_project` Optional[str] - use if vertex project different from the one in vertex_credentials
-
-as dynamic params for a `litellm.completion` call. 
-
-<Tabs>
-<TabItem value="sdk" label="SDK">
-
-```python
-from litellm import completion
-import json 
-
-## GET CREDENTIALS 
-file_path = 'path/to/vertex_ai_service_account.json'
-
-# Load the JSON file
-with open(file_path, 'r') as file:
-    vertex_credentials = json.load(file)
-
-# Convert to JSON string
-vertex_credentials_json = json.dumps(vertex_credentials)
-
-
-response = completion(
-  model="vertex_ai/gemini-pro",
-  messages=[{"content": "You are a good bot.","role": "system"}, {"content": "Hello, how are you?","role": "user"}], 
-  vertex_credentials=vertex_credentials_json,
-  vertex_project="my-special-project", 
-  vertex_location="my-special-location"
-)
-```
-
-</TabItem>
-<TabItem value="proxy" label="PROXY">
-
-```yaml
-model_list:
-    - model_name: gemini-1.5-pro
-      litellm_params:
-        model: gemini-1.5-pro
-        vertex_credentials: os.environ/VERTEX_FILE_PATH_ENV_VAR # os.environ["VERTEX_FILE_PATH_ENV_VAR"] = "/path/to/service_account.json" 
-        vertex_project: "my-special-project"
-        vertex_location: "my-special-location:
-```
-
-</TabItem>
-</Tabs>
-
-
-
-
-### **Environment Variables**
-
-You can set:
-- `GOOGLE_APPLICATION_CREDENTIALS` - store the filepath for your service_account.json in here (used by vertex sdk directly).
-- VERTEXAI_LOCATION - place where vertex model is deployed (us-central1, asia-southeast1, etc.)
-- VERTEXAI_PROJECT - Optional[str] - use if vertex project different from the one in vertex_credentials
-
-1. GOOGLE_APPLICATION_CREDENTIALS
-
-```bash
-export GOOGLE_APPLICATION_CREDENTIALS="/path/to/service_account.json"
-```
-
-2. VERTEXAI_LOCATION
-
-```bash
-export VERTEXAI_LOCATION="us-central1" # can be any vertex location
-```
-
-3. VERTEXAI_PROJECT
-
-```bash
-export VERTEXAI_PROJECT="my-test-project" # ONLY use if model project is different from service account project
-```
-
-
 ## Extra
 
 ### Using `GOOGLE_APPLICATION_CREDENTIALS`

docs/my-website/docs/proxy/alerting.md

@@ -2,7 +2,7 @@ import Image from '@theme/IdealImage';
 import Tabs from '@theme/Tabs';
 import TabItem from '@theme/TabItem';
 
-# 🚨 Alerting / Webhooks
+# Alerting / Webhooks
 
 Get alerts for:
 

docs/my-website/docs/proxy/caching.md

@@ -136,6 +136,7 @@ litellm_settings:
     type: "redis"
     service_name: "mymaster"
     sentinel_nodes: [["localhost", 26379]]
+    sentinel_password: "password" # [OPTIONAL]
 ```
 
 </TabItem>
@@ -149,6 +150,7 @@ You can configure redis sentinel in your .env by setting `REDIS_SENTINEL_NODES`
 ```env
 REDIS_SENTINEL_NODES='[["localhost", 26379]]'
 REDIS_SERVICE_NAME = "mymaster"
+REDIS_SENTINEL_PASSWORD = "password"
 ```
 
 :::note

docs/my-website/docs/proxy/config_management.md

@@ -0,0 +1,59 @@
+# File Management
+
+## `include` external YAML files in a config.yaml 
+
+You can use `include` to include external YAML files in a config.yaml. 
+
+**Quick Start Usage:**
+
+To include a config file, use `include` with either a single file or a list of files. 
+
+Contents of `parent_config.yaml`:
+```yaml
+include:
+  - model_config.yaml # 👈 Key change, will include the contents of model_config.yaml
+
+litellm_settings:
+  callbacks: ["prometheus"] 
+```
+
+
+Contents of `model_config.yaml`:
+```yaml
+model_list:
+  - model_name: gpt-4o
+    litellm_params:
+      model: openai/gpt-4o
+      api_base: https://exampleopenaiendpoint-production.up.railway.app/
+  - model_name: fake-anthropic-endpoint
+    litellm_params:
+      model: anthropic/fake
+      api_base: https://exampleanthropicendpoint-production.up.railway.app/
+
+```
+
+Start proxy server 
+
+This will start the proxy server with config `parent_config.yaml`. Since the `include` directive is used, the server will also include the contents of `model_config.yaml`.
+```
+litellm --config parent_config.yaml --detailed_debug
+```
+
+
+
+
+
+## Examples using `include`
+
+Include a single file:
+```yaml
+include:
+  - model_config.yaml
+```
+
+Include multiple files:
+```yaml
+include:
+  - model_config.yaml
+  - another_config.yaml
+```

docs/my-website/docs/proxy/config_settings.md

@@ -0,0 +1,507 @@
+# All settings
+
+
+```yaml
+environment_variables: {}
+
+model_list:
+  - model_name: string
+    litellm_params: {}
+    model_info:
+      id: string
+      mode: embedding
+      input_cost_per_token: 0
+      output_cost_per_token: 0
+      max_tokens: 2048
+      base_model: gpt-4-1106-preview
+      additionalProp1: {}
+
+litellm_settings:
+  # Logging/Callback settings
+  success_callback: ["langfuse"]  # list of success callbacks
+  failure_callback: ["sentry"]  # list of failure callbacks
+  callbacks: ["otel"]  # list of callbacks - runs on success and failure
+  service_callbacks: ["datadog", "prometheus"]  # logs redis, postgres failures on datadog, prometheus
+  turn_off_message_logging: boolean  # prevent the messages and responses from being logged to on your callbacks, but request metadata will still be logged.
+  redact_user_api_key_info: boolean  # Redact information about the user api key (hashed token, user_id, team id, etc.), from logs. Currently supported for Langfuse, OpenTelemetry, Logfire, ArizeAI logging.
+  langfuse_default_tags: ["cache_hit", "cache_key", "proxy_base_url", "user_api_key_alias", "user_api_key_user_id", "user_api_key_user_email", "user_api_key_team_alias", "semantic-similarity", "proxy_base_url"] # default tags for Langfuse Logging
+  
+  # Networking settings
+  request_timeout: 10 # (int) llm requesttimeout in seconds. Raise Timeout error if call takes longer than 10s. Sets litellm.request_timeout 
+  force_ipv4: boolean # If true, litellm will force ipv4 for all LLM requests. Some users have seen httpx ConnectionError when using ipv6 + Anthropic API
+  
+  set_verbose: boolean # sets litellm.set_verbose=True to view verbose debug logs. DO NOT LEAVE THIS ON IN PRODUCTION
+  json_logs: boolean # if true, logs will be in json format
+
+  # Fallbacks, reliability
+  default_fallbacks: ["claude-opus"] # set default_fallbacks, in case a specific model group is misconfigured / bad.
+  content_policy_fallbacks: [{"gpt-3.5-turbo-small": ["claude-opus"]}] # fallbacks for ContentPolicyErrors
+  context_window_fallbacks: [{"gpt-3.5-turbo-small": ["gpt-3.5-turbo-large", "claude-opus"]}] # fallbacks for ContextWindowExceededErrors
+
+
+
+  # Caching settings
+  cache: true 
+  cache_params:        # set cache params for redis
+    type: redis        # type of cache to initialize
+
+    # Optional - Redis Settings
+    host: "localhost"  # The host address for the Redis cache. Required if type is "redis".
+    port: 6379  # The port number for the Redis cache. Required if type is "redis".
+    password: "your_password"  # The password for the Redis cache. Required if type is "redis".
+    namespace: "litellm.caching.caching" # namespace for redis cache
+  
+    # Optional - Redis Cluster Settings
+    redis_startup_nodes: [{"host": "127.0.0.1", "port": "7001"}] 
+
+    # Optional - Redis Sentinel Settings
+    service_name: "mymaster"
+    sentinel_nodes: [["localhost", 26379]]
+
+    # Optional - Qdrant Semantic Cache Settings
+    qdrant_semantic_cache_embedding_model: openai-embedding # the model should be defined on the model_list
+    qdrant_collection_name: test_collection
+    qdrant_quantization_config: binary
+    similarity_threshold: 0.8   # similarity threshold for semantic cache
+
+    # Optional - S3 Cache Settings
+    s3_bucket_name: cache-bucket-litellm   # AWS Bucket Name for S3
+    s3_region_name: us-west-2              # AWS Region Name for S3
+    s3_aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID  # us os.environ/<variable name> to pass environment variables. This is AWS Access Key ID for S3
+    s3_aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY  # AWS Secret Access Key for S3
+    s3_endpoint_url: https://s3.amazonaws.com  # [OPTIONAL] S3 endpoint URL, if you want to use Backblaze/cloudflare s3 bucket
+
+    # Common Cache settings
+    # Optional - Supported call types for caching
+    supported_call_types: ["acompletion", "atext_completion", "aembedding", "atranscription"]
+                          # /chat/completions, /completions, /embeddings, /audio/transcriptions
+    mode: default_off # if default_off, you need to opt in to caching on a per call basis
+    ttl: 600 # ttl for caching
+
+
+callback_settings:
+  otel:
+    message_logging: boolean  # OTEL logging callback specific settings
+
+general_settings:
+  completion_model: string
+  disable_spend_logs: boolean  # turn off writing each transaction to the db
+  disable_master_key_return: boolean  # turn off returning master key on UI (checked on '/user/info' endpoint)
+  disable_retry_on_max_parallel_request_limit_error: boolean  # turn off retries when max parallel request limit is reached
+  disable_reset_budget: boolean  # turn off reset budget scheduled task
+  disable_adding_master_key_hash_to_db: boolean  # turn off storing master key hash in db, for spend tracking
+  enable_jwt_auth: boolean  # allow proxy admin to auth in via jwt tokens with 'litellm_proxy_admin' in claims
+  enforce_user_param: boolean  # requires all openai endpoint requests to have a 'user' param
+  allowed_routes: ["route1", "route2"]  # list of allowed proxy API routes - a user can access. (currently JWT-Auth only)
+  key_management_system: google_kms  # either google_kms or azure_kms
+  master_key: string
+
+  # Database Settings
+  database_url: string
+  database_connection_pool_limit: 0  # default 100
+  database_connection_timeout: 0  # default 60s
+  allow_requests_on_db_unavailable: boolean  # if true, will allow requests that can not connect to the DB to verify Virtual Key to still work 
+
+  custom_auth: string
+  max_parallel_requests: 0  # the max parallel requests allowed per deployment 
+  global_max_parallel_requests: 0  # the max parallel requests allowed on the proxy all up 
+  infer_model_from_keys: true
+  background_health_checks: true
+  health_check_interval: 300
+  alerting: ["slack", "email"]
+  alerting_threshold: 0
+  use_client_credentials_pass_through_routes: boolean  # use client credentials for all pass through routes like "/vertex-ai", /bedrock/. When this is True Virtual Key auth will not be applied on these endpoints
+```
+
+### litellm_settings - Reference
+
+| Name | Type | Description |
+|------|------|-------------|
+| success_callback | array of strings | List of success callbacks. [Doc Proxy logging callbacks](logging), [Doc Metrics](prometheus) |
+| failure_callback | array of strings | List of failure callbacks [Doc Proxy logging callbacks](logging), [Doc Metrics](prometheus) |
+| callbacks | array of strings | List of callbacks - runs on success and failure [Doc Proxy logging callbacks](logging), [Doc Metrics](prometheus) |
+| service_callbacks | array of strings | System health monitoring - Logs redis, postgres failures on specified services (e.g. datadog, prometheus) [Doc Metrics](prometheus) |
+| turn_off_message_logging | boolean | If true, prevents messages and responses from being logged to callbacks, but request metadata will still be logged [Proxy Logging](logging) |
+| modify_params | boolean | If true, allows modifying the parameters of the request before it is sent to the LLM provider |
+| enable_preview_features | boolean | If true, enables preview features - e.g. Azure O1 Models with streaming support.|
+| redact_user_api_key_info | boolean | If true, redacts information about the user api key from logs [Proxy Logging](logging#redacting-userapikeyinfo) |
+| langfuse_default_tags | array of strings | Default tags for Langfuse Logging. Use this if you want to control which LiteLLM-specific fields are logged as tags by the LiteLLM proxy. By default LiteLLM Proxy logs no LiteLLM-specific fields as tags. [Further docs](./logging#litellm-specific-tags-on-langfuse---cache_hit-cache_key) |
+| set_verbose | boolean | If true, sets litellm.set_verbose=True to view verbose debug logs. DO NOT LEAVE THIS ON IN PRODUCTION |
+| json_logs | boolean | If true, logs will be in json format. If you need to store the logs as JSON, just set the `litellm.json_logs = True`. We currently just log the raw POST request from litellm as a JSON [Further docs](./debugging) |
+| default_fallbacks | array of strings | List of fallback models to use if a specific model group is misconfigured / bad. [Further docs](./reliability#default-fallbacks) |
+| request_timeout | integer | The timeout for requests in seconds. If not set, the default value is `6000 seconds`. [For reference OpenAI Python SDK defaults to `600 seconds`.](https://github.com/openai/openai-python/blob/main/src/openai/_constants.py) |
+| force_ipv4 | boolean | If true, litellm will force ipv4 for all LLM requests. Some users have seen httpx ConnectionError when using ipv6 + Anthropic API |
+| content_policy_fallbacks | array of objects | Fallbacks to use when a ContentPolicyViolationError is encountered. [Further docs](./reliability#content-policy-fallbacks) |
+| context_window_fallbacks | array of objects | Fallbacks to use when a ContextWindowExceededError is encountered. [Further docs](./reliability#context-window-fallbacks) |
+| cache | boolean | If true, enables caching. [Further docs](./caching) |
+| cache_params | object | Parameters for the cache. [Further docs](./caching) |
+| cache_params.type | string | The type of cache to initialize. Can be one of ["local", "redis", "redis-semantic", "s3", "disk", "qdrant-semantic"]. Defaults to "redis". [Furher docs](./caching) |
+| cache_params.host | string | The host address for the Redis cache. Required if type is "redis". |
+| cache_params.port | integer | The port number for the Redis cache. Required if type is "redis". |
+| cache_params.password | string | The password for the Redis cache. Required if type is "redis". |
+| cache_params.namespace | string | The namespace for the Redis cache. |
+| cache_params.redis_startup_nodes | array of objects | Redis Cluster Settings. [Further docs](./caching) |
+| cache_params.service_name | string | Redis Sentinel Settings. [Further docs](./caching) |
+| cache_params.sentinel_nodes | array of arrays | Redis Sentinel Settings. [Further docs](./caching) |
+| cache_params.ttl | integer | The time (in seconds) to store entries in cache. |
+| cache_params.qdrant_semantic_cache_embedding_model | string | The embedding model to use for qdrant semantic cache. |
+| cache_params.qdrant_collection_name | string | The name of the collection to use for qdrant semantic cache. |
+| cache_params.qdrant_quantization_config | string | The quantization configuration for the qdrant semantic cache. |
+| cache_params.similarity_threshold | float | The similarity threshold for the semantic cache. |
+| cache_params.s3_bucket_name | string | The name of the S3 bucket to use for the semantic cache. |
+| cache_params.s3_region_name | string | The region name for the S3 bucket. |
+| cache_params.s3_aws_access_key_id | string | The AWS access key ID for the S3 bucket. |
+| cache_params.s3_aws_secret_access_key | string | The AWS secret access key for the S3 bucket. |
+| cache_params.s3_endpoint_url | string | Optional - The endpoint URL for the S3 bucket. |
+| cache_params.supported_call_types | array of strings | The types of calls to cache. [Further docs](./caching) |
+| cache_params.mode | string | The mode of the cache. [Further docs](./caching) |
+| disable_end_user_cost_tracking | boolean | If true, turns off end user cost tracking on prometheus metrics + litellm spend logs table on proxy. |
+| key_generation_settings | object | Restricts who can generate keys. [Further docs](./virtual_keys.md#restricting-key-generation) |
+
+### general_settings - Reference
+
+| Name | Type | Description |
+|------|------|-------------|
+| completion_model | string | The default model to use for completions when `model` is not specified in the request |
+| disable_spend_logs | boolean | If true, turns off writing each transaction to the database |
+| disable_master_key_return | boolean | If true, turns off returning master key on UI. (checked on '/user/info' endpoint) |
+| disable_retry_on_max_parallel_request_limit_error | boolean | If true, turns off retries when max parallel request limit is reached |
+| disable_reset_budget | boolean | If true, turns off reset budget scheduled task |
+| disable_adding_master_key_hash_to_db | boolean | If true, turns off storing master key hash in db |
+| enable_jwt_auth | boolean | allow proxy admin to auth in via jwt tokens with 'litellm_proxy_admin' in claims. [Doc on JWT Tokens](token_auth) |
+| enforce_user_param | boolean | If true, requires all OpenAI endpoint requests to have a 'user' param. [Doc on call hooks](call_hooks)|
+| allowed_routes | array of strings | List of allowed proxy API routes a user can access [Doc on controlling allowed routes](enterprise#control-available-public-private-routes)|
+| key_management_system | string | Specifies the key management system. [Doc Secret Managers](../secret) |
+| master_key | string | The master key for the proxy [Set up Virtual Keys](virtual_keys) |
+| database_url | string | The URL for the database connection [Set up Virtual Keys](virtual_keys) |
+| database_connection_pool_limit | integer | The limit for database connection pool [Setting DB Connection Pool limit](#configure-db-pool-limits--connection-timeouts) |
+| database_connection_timeout | integer | The timeout for database connections in seconds [Setting DB Connection Pool limit, timeout](#configure-db-pool-limits--connection-timeouts) |
+| allow_requests_on_db_unavailable | boolean | If true, allows requests to succeed even if DB is unreachable. **Only use this if running LiteLLM in your VPC** This will allow requests to work even when LiteLLM cannot connect to the DB to verify a Virtual Key |
+| custom_auth | string | Write your own custom authentication logic [Doc Custom Auth](virtual_keys#custom-auth) |
+| max_parallel_requests | integer | The max parallel requests allowed per deployment |
+| global_max_parallel_requests | integer | The max parallel requests allowed on the proxy overall |
+| infer_model_from_keys | boolean | If true, infers the model from the provided keys |
+| background_health_checks | boolean | If true, enables background health checks. [Doc on health checks](health) |
+| health_check_interval | integer | The interval for health checks in seconds [Doc on health checks](health) |
+| alerting | array of strings | List of alerting methods [Doc on Slack Alerting](alerting) |
+| alerting_threshold | integer | The threshold for triggering alerts [Doc on Slack Alerting](alerting) |
+| use_client_credentials_pass_through_routes | boolean | If true, uses client credentials for all pass-through routes. [Doc on pass through routes](pass_through) |
+| health_check_details | boolean | If false, hides health check details (e.g. remaining rate limit). [Doc on health checks](health) |
+| public_routes | List[str] | (Enterprise Feature) Control list of public routes |
+| alert_types | List[str] | Control list of alert types to send to slack (Doc on alert types)[./alerting.md] |
+| enforced_params | List[str] | (Enterprise Feature) List of params that must be included in all requests to the proxy |
+| enable_oauth2_auth | boolean | (Enterprise Feature) If true, enables oauth2.0 authentication |
+| use_x_forwarded_for | str | If true, uses the X-Forwarded-For header to get the client IP address |
+| service_account_settings | List[Dict[str, Any]] | Set `service_account_settings` if you want to create settings that only apply to service account keys (Doc on service accounts)[./service_accounts.md] | 
+| image_generation_model | str | The default model to use for image generation - ignores model set in request |
+| store_model_in_db | boolean | If true, allows `/model/new` endpoint to store model information in db. Endpoint disabled by default. [Doc on `/model/new` endpoint](./model_management.md#create-a-new-model) |
+| max_request_size_mb | int | The maximum size for requests in MB. Requests above this size will be rejected. |
+| max_response_size_mb | int | The maximum size for responses in MB. LLM Responses above this size will not be sent. |
+| proxy_budget_rescheduler_min_time | int | The minimum time (in seconds) to wait before checking db for budget resets. **Default is 597 seconds** |
+| proxy_budget_rescheduler_max_time | int | The maximum time (in seconds) to wait before checking db for budget resets. **Default is 605 seconds** |
+| proxy_batch_write_at | int | Time (in seconds) to wait before batch writing spend logs to the db. **Default is 10 seconds** |
+| alerting_args | dict | Args for Slack Alerting [Doc on Slack Alerting](./alerting.md) |
+| custom_key_generate | str | Custom function for key generation [Doc on custom key generation](./virtual_keys.md#custom--key-generate) |
+| allowed_ips | List[str] | List of IPs allowed to access the proxy. If not set, all IPs are allowed. |
+| embedding_model | str | The default model to use for embeddings - ignores model set in request |
+| default_team_disabled | boolean | If true, users cannot create 'personal' keys (keys with no team_id). |
+| alert_to_webhook_url | Dict[str] | [Specify a webhook url for each alert type.](./alerting.md#set-specific-slack-channels-per-alert-type) |
+| key_management_settings | List[Dict[str, Any]] | Settings for key management system (e.g. AWS KMS, Azure Key Vault) [Doc on key management](../secret.md) |
+| allow_user_auth | boolean | (Deprecated) old approach for user authentication. |
+| user_api_key_cache_ttl | int | The time (in seconds) to cache user api keys in memory. |
+| disable_prisma_schema_update | boolean | If true, turns off automatic schema updates to DB |
+| litellm_key_header_name | str | If set, allows passing LiteLLM keys as a custom header. [Doc on custom headers](./virtual_keys.md#custom-headers) |
+| moderation_model | str | The default model to use for moderation. |
+| custom_sso | str | Path to a python file that implements custom SSO logic. [Doc on custom SSO](./custom_sso.md) |
+| allow_client_side_credentials | boolean | If true, allows passing client side credentials to the proxy. (Useful when testing finetuning models) [Doc on client side credentials](./virtual_keys.md#client-side-credentials) |
+| admin_only_routes | List[str] | (Enterprise Feature) List of routes that are only accessible to admin users. [Doc on admin only routes](./enterprise#control-available-public-private-routes) |
+| use_azure_key_vault | boolean | If true, load keys from azure key vault | 
+| use_google_kms | boolean | If true, load keys from google kms |
+| spend_report_frequency | str | Specify how often you want a Spend Report to be sent (e.g. "1d", "2d", "30d") [More on this](./alerting.md#spend-report-frequency) |
+| ui_access_mode | Literal["admin_only"] | If set, restricts access to the UI to admin users only. [Docs](./ui.md#restrict-ui-access) |
+| litellm_jwtauth | Dict[str, Any] | Settings for JWT authentication. [Docs](./token_auth.md) |
+| litellm_license | str | The license key for the proxy. [Docs](../enterprise.md#how-does-deployment-with-enterprise-license-work) |
+| oauth2_config_mappings | Dict[str, str] | Define the OAuth2 config mappings | 
+| pass_through_endpoints | List[Dict[str, Any]] | Define the pass through endpoints. [Docs](./pass_through) |
+| enable_oauth2_proxy_auth | boolean | (Enterprise Feature) If true, enables oauth2.0 authentication |
+| forward_openai_org_id | boolean | If true, forwards the OpenAI Organization ID to the backend LLM call (if it's OpenAI). |
+| forward_client_headers_to_llm_api | boolean | If true, forwards the client headers (any `x-` headers) to the backend LLM call |
+
+### router_settings - Reference
+
+:::info
+
+Most values can also be set via `litellm_settings`. If you see overlapping values, settings on `router_settings` will override those on `litellm_settings`.
+:::
+
+```yaml
+router_settings:
+  routing_strategy: usage-based-routing-v2 # Literal["simple-shuffle", "least-busy", "usage-based-routing","latency-based-routing"], default="simple-shuffle"
+  redis_host: <your-redis-host>           # string
+  redis_password: <your-redis-password>   # string
+  redis_port: <your-redis-port>           # string
+  enable_pre_call_check: true             # bool - Before call is made check if a call is within model context window 
+  allowed_fails: 3 # cooldown model if it fails > 1 call in a minute. 
+  cooldown_time: 30 # (in seconds) how long to cooldown model if fails/min > allowed_fails
+  disable_cooldowns: True                  # bool - Disable cooldowns for all models 
+  enable_tag_filtering: True                # bool - Use tag based routing for requests
+  retry_policy: {                          # Dict[str, int]: retry policy for different types of exceptions
+    "AuthenticationErrorRetries": 3,
+    "TimeoutErrorRetries": 3,
+    "RateLimitErrorRetries": 3,
+    "ContentPolicyViolationErrorRetries": 4,
+    "InternalServerErrorRetries": 4
+  }
+  allowed_fails_policy: {
+    "BadRequestErrorAllowedFails": 1000, # Allow 1000 BadRequestErrors before cooling down a deployment
+    "AuthenticationErrorAllowedFails": 10, # int 
+    "TimeoutErrorAllowedFails": 12, # int 
+    "RateLimitErrorAllowedFails": 10000, # int 
+    "ContentPolicyViolationErrorAllowedFails": 15, # int 
+    "InternalServerErrorAllowedFails": 20, # int 
+  }
+  content_policy_fallbacks=[{"claude-2": ["my-fallback-model"]}] # List[Dict[str, List[str]]]: Fallback model for content policy violations
+  fallbacks=[{"claude-2": ["my-fallback-model"]}] # List[Dict[str, List[str]]]: Fallback model for all errors
+```
+
+| Name | Type | Description |
+|------|------|-------------|
+| routing_strategy | string | The strategy used for routing requests. Options: "simple-shuffle", "least-busy", "usage-based-routing", "latency-based-routing". Default is "simple-shuffle". [More information here](../routing) |
+| redis_host | string | The host address for the Redis server. **Only set this if you have multiple instances of LiteLLM Proxy and want current tpm/rpm tracking to be shared across them** |
+| redis_password | string | The password for the Redis server. **Only set this if you have multiple instances of LiteLLM Proxy and want current tpm/rpm tracking to be shared across them** |
+| redis_port | string | The port number for the Redis server. **Only set this if you have multiple instances of LiteLLM Proxy and want current tpm/rpm tracking to be shared across them**|
+| enable_pre_call_check | boolean | If true, checks if a call is within the model's context window before making the call. [More information here](reliability) |
+| content_policy_fallbacks | array of objects | Specifies fallback models for content policy violations. [More information here](reliability) |
+| fallbacks | array of objects | Specifies fallback models for all types of errors. [More information here](reliability) |
+| enable_tag_filtering | boolean | If true, uses tag based routing for requests [Tag Based Routing](tag_routing) |
+| cooldown_time | integer | The duration (in seconds) to cooldown a model if it exceeds the allowed failures. |
+| disable_cooldowns | boolean | If true, disables cooldowns for all models. [More information here](reliability) |
+| retry_policy | object | Specifies the number of retries for different types of exceptions. [More information here](reliability) |
+| allowed_fails | integer | The number of failures allowed before cooling down a model. [More information here](reliability) |
+| allowed_fails_policy | object | Specifies the number of allowed failures for different error types before cooling down a deployment. [More information here](reliability) |
+| default_max_parallel_requests | Optional[int] | The default maximum number of parallel requests for a deployment. |
+| default_priority | (Optional[int]) | The default priority for a request. Only for '.scheduler_acompletion()'. Default is None. | 
+| polling_interval | (Optional[float]) | frequency of polling queue. Only for '.scheduler_acompletion()'. Default is 3ms. |
+| max_fallbacks | Optional[int] | The maximum number of fallbacks to try before exiting the call. Defaults to 5. |
+| default_litellm_params | Optional[dict] | The default litellm parameters to add to all requests (e.g. `temperature`, `max_tokens`). |
+| timeout | Optional[float] | The default timeout for a request. |
+| debug_level | Literal["DEBUG", "INFO"] | The debug level for the logging library in the router. Defaults to "INFO". |
+| client_ttl | int | Time-to-live for cached clients in seconds. Defaults to 3600. |
+| cache_kwargs | dict | Additional keyword arguments for the cache initialization. |
+| routing_strategy_args | dict | Additional keyword arguments for the routing strategy - e.g. lowest latency routing default ttl |
+| model_group_alias | dict | Model group alias mapping. E.g. `{"claude-3-haiku": "claude-3-haiku-20240229"}` |
+| num_retries | int | Number of retries for a request. Defaults to 3. |
+| default_fallbacks | Optional[List[str]] | Fallbacks to try if no model group-specific fallbacks are defined. |
+| caching_groups | Optional[List[tuple]] | List of model groups for caching across model groups. Defaults to None. - e.g. caching_groups=[("openai-gpt-3.5-turbo", "azure-gpt-3.5-turbo")]|
+| alerting_config | AlertingConfig | [SDK-only arg] Slack alerting configuration. Defaults to None. [Further Docs](../routing.md#alerting-) |
+| assistants_config | AssistantsConfig | Set on proxy via `assistant_settings`. [Further docs](../assistants.md) |
+| set_verbose | boolean | [DEPRECATED PARAM - see debug docs](./debugging.md) If true, sets the logging level to verbose. |
+| retry_after | int | Time to wait before retrying a request in seconds. Defaults to 0. If `x-retry-after` is received from LLM API, this value is overridden. |
+| provider_budget_config | ProviderBudgetConfig | Provider budget configuration. Use this to set llm_provider budget limits. example $100/day to OpenAI, $100/day to Azure, etc. Defaults to None. [Further Docs](./provider_budget_routing.md) |
+| enable_pre_call_checks | boolean | If true, checks if a call is within the model's context window before making the call. [More information here](reliability) |
+| model_group_retry_policy | Dict[str, RetryPolicy] | [SDK-only arg] Set retry policy for model groups. |
+| context_window_fallbacks | List[Dict[str, List[str]]] | Fallback models for context window violations. |
+| redis_url | str | URL for Redis server. **Known performance issue with Redis URL.** |
+| cache_responses | boolean | Flag to enable caching LLM Responses, if cache set under `router_settings`. If true, caches responses. Defaults to False. |
+| router_general_settings | RouterGeneralSettings | [SDK-Only] Router general settings - contains optimizations like 'async_only_mode'. [Docs](../routing.md#router-general-settings) |
+
+### environment variables - Reference
+
+| Name | Description |
+|------|-------------|
+| ACTIONS_ID_TOKEN_REQUEST_TOKEN | Token for requesting ID in GitHub Actions
+| ACTIONS_ID_TOKEN_REQUEST_URL | URL for requesting ID token in GitHub Actions
+| AISPEND_ACCOUNT_ID | Account ID for AI Spend
+| AISPEND_API_KEY | API Key for AI Spend
+| ALLOWED_EMAIL_DOMAINS | List of email domains allowed for access
+| ARIZE_API_KEY | API key for Arize platform integration
+| ARIZE_SPACE_KEY | Space key for Arize platform
+| ARGILLA_BATCH_SIZE | Batch size for Argilla logging
+| ARGILLA_API_KEY | API key for Argilla platform
+| ARGILLA_SAMPLING_RATE | Sampling rate for Argilla logging
+| ARGILLA_DATASET_NAME | Dataset name for Argilla logging
+| ARGILLA_BASE_URL | Base URL for Argilla service
+| ATHINA_API_KEY | API key for Athina service
+| AUTH_STRATEGY | Strategy used for authentication (e.g., OAuth, API key)
+| AWS_ACCESS_KEY_ID | Access Key ID for AWS services
+| AWS_PROFILE_NAME | AWS CLI profile name to be used
+| AWS_REGION_NAME | Default AWS region for service interactions
+| AWS_ROLE_NAME | Role name for AWS IAM usage
+| AWS_SECRET_ACCESS_KEY | Secret Access Key for AWS services
+| AWS_SESSION_NAME | Name for AWS session
+| AWS_WEB_IDENTITY_TOKEN | Web identity token for AWS
+| AZURE_API_VERSION | Version of the Azure API being used
+| AZURE_AUTHORITY_HOST | Azure authority host URL
+| AZURE_CLIENT_ID | Client ID for Azure services
+| AZURE_CLIENT_SECRET | Client secret for Azure services
+| AZURE_FEDERATED_TOKEN_FILE | File path to Azure federated token
+| AZURE_KEY_VAULT_URI | URI for Azure Key Vault
+| AZURE_TENANT_ID | Tenant ID for Azure Active Directory
+| BERRISPEND_ACCOUNT_ID | Account ID for BerriSpend service
+| BRAINTRUST_API_KEY | API key for Braintrust integration
+| CIRCLE_OIDC_TOKEN | OpenID Connect token for CircleCI
+| CIRCLE_OIDC_TOKEN_V2 | Version 2 of the OpenID Connect token for CircleCI
+| CONFIG_FILE_PATH | File path for configuration file
+| CUSTOM_TIKTOKEN_CACHE_DIR | Custom directory for Tiktoken cache
+| DATABASE_HOST | Hostname for the database server
+| DATABASE_NAME | Name of the database
+| DATABASE_PASSWORD | Password for the database user
+| DATABASE_PORT | Port number for database connection
+| DATABASE_SCHEMA | Schema name used in the database
+| DATABASE_URL | Connection URL for the database
+| DATABASE_USER | Username for database connection
+| DATABASE_USERNAME | Alias for database user
+| DATABRICKS_API_BASE | Base URL for Databricks API
+| DD_BASE_URL | Base URL for Datadog integration
+| DATADOG_BASE_URL | (Alternative to DD_BASE_URL) Base URL for Datadog integration
+| _DATADOG_BASE_URL | (Alternative to DD_BASE_URL) Base URL for Datadog integration
+| DD_API_KEY | API key for Datadog integration
+| DD_SITE | Site URL for Datadog (e.g., datadoghq.com)
+| DD_SOURCE | Source identifier for Datadog logs
+| DD_ENV | Environment identifier for Datadog logs. Only supported for `datadog_llm_observability` callback
+| DD_SERVICE | Service identifier for Datadog logs. Defaults to "litellm-server"
+| DD_VERSION | Version identifier for Datadog logs. Defaults to "unknown"
+| DEBUG_OTEL | Enable debug mode for OpenTelemetry
+| DIRECT_URL | Direct URL for service endpoint
+| DISABLE_ADMIN_UI | Toggle to disable the admin UI
+| DISABLE_SCHEMA_UPDATE | Toggle to disable schema updates
+| DOCS_DESCRIPTION | Description text for documentation pages
+| DOCS_FILTERED | Flag indicating filtered documentation
+| DOCS_TITLE | Title of the documentation pages
+| DOCS_URL | The path to the Swagger API documentation. **By default this is "/"**
+| EMAIL_SUPPORT_CONTACT | Support contact email address
+| GCS_BUCKET_NAME | Name of the Google Cloud Storage bucket
+| GCS_PATH_SERVICE_ACCOUNT | Path to the Google Cloud service account JSON file
+| GCS_FLUSH_INTERVAL | Flush interval for GCS logging (in seconds). Specify how often you want a log to be sent to GCS. **Default is 20 seconds**
+| GCS_BATCH_SIZE | Batch size for GCS logging. Specify after how many logs you want to flush to GCS. If `BATCH_SIZE` is set to 10, logs are flushed every 10 logs. **Default is 2048**
+| GENERIC_AUTHORIZATION_ENDPOINT | Authorization endpoint for generic OAuth providers
+| GENERIC_CLIENT_ID | Client ID for generic OAuth providers
+| GENERIC_CLIENT_SECRET | Client secret for generic OAuth providers
+| GENERIC_CLIENT_STATE | State parameter for generic client authentication
+| GENERIC_INCLUDE_CLIENT_ID | Include client ID in requests for OAuth
+| GENERIC_SCOPE | Scope settings for generic OAuth providers
+| GENERIC_TOKEN_ENDPOINT | Token endpoint for generic OAuth providers
+| GENERIC_USER_DISPLAY_NAME_ATTRIBUTE | Attribute for user's display name in generic auth
+| GENERIC_USER_EMAIL_ATTRIBUTE | Attribute for user's email in generic auth
+| GENERIC_USER_FIRST_NAME_ATTRIBUTE | Attribute for user's first name in generic auth
+| GENERIC_USER_ID_ATTRIBUTE | Attribute for user ID in generic auth
+| GENERIC_USER_LAST_NAME_ATTRIBUTE | Attribute for user's last name in generic auth
+| GENERIC_USER_PROVIDER_ATTRIBUTE | Attribute specifying the user's provider
+| GENERIC_USER_ROLE_ATTRIBUTE | Attribute specifying the user's role
+| GENERIC_USERINFO_ENDPOINT | Endpoint to fetch user information in generic OAuth
+| GALILEO_BASE_URL | Base URL for Galileo platform
+| GALILEO_PASSWORD | Password for Galileo authentication
+| GALILEO_PROJECT_ID | Project ID for Galileo usage
+| GALILEO_USERNAME | Username for Galileo authentication
+| GREENSCALE_API_KEY | API key for Greenscale service
+| GREENSCALE_ENDPOINT | Endpoint URL for Greenscale service
+| GOOGLE_APPLICATION_CREDENTIALS | Path to Google Cloud credentials JSON file
+| GOOGLE_CLIENT_ID | Client ID for Google OAuth
+| GOOGLE_CLIENT_SECRET | Client secret for Google OAuth
+| GOOGLE_KMS_RESOURCE_NAME | Name of the resource in Google KMS
+| HF_API_BASE | Base URL for Hugging Face API
+| HELICONE_API_KEY | API key for Helicone service
+| HUGGINGFACE_API_BASE | Base URL for Hugging Face API
+| IAM_TOKEN_DB_AUTH | IAM token for database authentication
+| JSON_LOGS | Enable JSON formatted logging
+| JWT_AUDIENCE | Expected audience for JWT tokens
+| JWT_PUBLIC_KEY_URL | URL to fetch public key for JWT verification
+| LAGO_API_BASE | Base URL for Lago API
+| LAGO_API_CHARGE_BY | Parameter to determine charge basis in Lago
+| LAGO_API_EVENT_CODE | Event code for Lago API events
+| LAGO_API_KEY | API key for accessing Lago services
+| LANGFUSE_DEBUG | Toggle debug mode for Langfuse
+| LANGFUSE_FLUSH_INTERVAL | Interval for flushing Langfuse logs
+| LANGFUSE_HOST | Host URL for Langfuse service
+| LANGFUSE_PUBLIC_KEY | Public key for Langfuse authentication
+| LANGFUSE_RELEASE | Release version of Langfuse integration
+| LANGFUSE_SECRET_KEY | Secret key for Langfuse authentication
+| LANGSMITH_API_KEY | API key for Langsmith platform
+| LANGSMITH_BASE_URL | Base URL for Langsmith service
+| LANGSMITH_BATCH_SIZE | Batch size for operations in Langsmith
+| LANGSMITH_DEFAULT_RUN_NAME | Default name for Langsmith run
+| LANGSMITH_PROJECT | Project name for Langsmith integration
+| LANGSMITH_SAMPLING_RATE | Sampling rate for Langsmith logging
+| LANGTRACE_API_KEY | API key for Langtrace service
+| LITERAL_API_KEY | API key for Literal integration
+| LITERAL_API_URL | API URL for Literal service
+| LITERAL_BATCH_SIZE | Batch size for Literal operations
+| LITELLM_DONT_SHOW_FEEDBACK_BOX | Flag to hide feedback box in LiteLLM UI
+| LITELLM_DROP_PARAMS | Parameters to drop in LiteLLM requests
+| LITELLM_EMAIL | Email associated with LiteLLM account
+| LITELLM_GLOBAL_MAX_PARALLEL_REQUEST_RETRIES | Maximum retries for parallel requests in LiteLLM
+| LITELLM_GLOBAL_MAX_PARALLEL_REQUEST_RETRY_TIMEOUT | Timeout for retries of parallel requests in LiteLLM
+| LITELLM_HOSTED_UI | URL of the hosted UI for LiteLLM
+| LITELLM_LICENSE | License key for LiteLLM usage
+| LITELLM_LOCAL_MODEL_COST_MAP | Local configuration for model cost mapping in LiteLLM
+| LITELLM_LOG | Enable detailed logging for LiteLLM
+| LITELLM_MODE | Operating mode for LiteLLM (e.g., production, development)
+| LITELLM_SALT_KEY | Salt key for encryption in LiteLLM
+| LITELLM_SECRET_AWS_KMS_LITELLM_LICENSE | AWS KMS encrypted license for LiteLLM
+| LITELLM_TOKEN | Access token for LiteLLM integration
+| LOGFIRE_TOKEN | Token for Logfire logging service
+| MICROSOFT_CLIENT_ID | Client ID for Microsoft services
+| MICROSOFT_CLIENT_SECRET | Client secret for Microsoft services
+| MICROSOFT_TENANT | Tenant ID for Microsoft Azure
+| NO_DOCS | Flag to disable documentation generation
+| NO_PROXY | List of addresses to bypass proxy
+| OAUTH_TOKEN_INFO_ENDPOINT | Endpoint for OAuth token info retrieval
+| OPENAI_API_BASE | Base URL for OpenAI API
+| OPENAI_API_KEY | API key for OpenAI services
+| OPENAI_ORGANIZATION | Organization identifier for OpenAI
+| OPENID_BASE_URL | Base URL for OpenID Connect services
+| OPENID_CLIENT_ID | Client ID for OpenID Connect authentication
+| OPENID_CLIENT_SECRET | Client secret for OpenID Connect authentication
+| OPENMETER_API_ENDPOINT | API endpoint for OpenMeter integration
+| OPENMETER_API_KEY | API key for OpenMeter services
+| OPENMETER_EVENT_TYPE | Type of events sent to OpenMeter
+| OTEL_ENDPOINT | OpenTelemetry endpoint for traces
+| OTEL_ENVIRONMENT_NAME | Environment name for OpenTelemetry
+| OTEL_EXPORTER | Exporter type for OpenTelemetry
+| OTEL_HEADERS | Headers for OpenTelemetry requests
+| OTEL_SERVICE_NAME | Service name identifier for OpenTelemetry
+| OTEL_TRACER_NAME | Tracer name for OpenTelemetry tracing
+| PREDIBASE_API_BASE | Base URL for Predibase API
+| PRESIDIO_ANALYZER_API_BASE | Base URL for Presidio Analyzer service
+| PRESIDIO_ANONYMIZER_API_BASE | Base URL for Presidio Anonymizer service
+| PROMETHEUS_URL | URL for Prometheus service
+| PROMPTLAYER_API_KEY | API key for PromptLayer integration
+| PROXY_ADMIN_ID | Admin identifier for proxy server
+| PROXY_BASE_URL | Base URL for proxy service
+| PROXY_LOGOUT_URL | URL for logging out of the proxy service
+| PROXY_MASTER_KEY | Master key for proxy authentication
+| QDRANT_API_BASE | Base URL for Qdrant API
+| QDRANT_API_KEY | API key for Qdrant service
+| QDRANT_URL | Connection URL for Qdrant database
+| REDIS_HOST | Hostname for Redis server
+| REDIS_PASSWORD | Password for Redis service
+| REDIS_PORT | Port number for Redis server
+| REDOC_URL | The path to the Redoc Fast API documentation. **By default this is "/redoc"**
+| SERVER_ROOT_PATH | Root path for the server application
+| SET_VERBOSE | Flag to enable verbose logging
+| SLACK_DAILY_REPORT_FREQUENCY | Frequency of daily Slack reports (e.g., daily, weekly)
+| SLACK_WEBHOOK_URL | Webhook URL for Slack integration
+| SMTP_HOST | Hostname for the SMTP server
+| SMTP_PASSWORD | Password for SMTP authentication
+| SMTP_PORT | Port number for SMTP server
+| SMTP_SENDER_EMAIL | Email address used as the sender in SMTP transactions
+| SMTP_SENDER_LOGO | Logo used in emails sent via SMTP
+| SMTP_TLS | Flag to enable or disable TLS for SMTP connections
+| SMTP_USERNAME | Username for SMTP authentication
+| SPEND_LOGS_URL | URL for retrieving spend logs
+| SSL_CERTIFICATE | Path to the SSL certificate file
+| SSL_VERIFY | Flag to enable or disable SSL certificate verification
+| SUPABASE_KEY | API key for Supabase service
+| SUPABASE_URL | Base URL for Supabase instance
+| TEST_EMAIL_ADDRESS | Email address used for testing purposes
+| UI_LOGO_PATH | Path to the logo image used in the UI
+| UI_PASSWORD | Password for accessing the UI
+| UI_USERNAME | Username for accessing the UI
+| UPSTREAM_LANGFUSE_DEBUG | Flag to enable debugging for upstream Langfuse
+| UPSTREAM_LANGFUSE_HOST | Host URL for upstream Langfuse service
+| UPSTREAM_LANGFUSE_PUBLIC_KEY | Public key for upstream Langfuse authentication
+| UPSTREAM_LANGFUSE_RELEASE | Release version identifier for upstream Langfuse
+| UPSTREAM_LANGFUSE_SECRET_KEY | Secret key for upstream Langfuse authentication
+| USE_AWS_KMS | Flag to enable AWS Key Management Service for encryption
+| WEBHOOK_URL | URL for receiving webhooks from external services
+

docs/my-website/docs/proxy/configs.md

@@ -2,7 +2,7 @@ import Image from '@theme/IdealImage';
 import Tabs from '@theme/Tabs';
 import TabItem from '@theme/TabItem';
 
-# Proxy Config.yaml
+# Overview
 Set model list, `api_base`, `api_key`, `temperature` & proxy server settings (`master-key`) on the config.yaml. 
 
 | Param Name           | Description                                                   |
@@ -357,77 +357,6 @@ curl --location 'http://0.0.0.0:4000/v1/model/info' \
 --data ''
 ```
 
- 
-### Provider specific wildcard routing 
-**Proxy all models from a provider**
-
-Use this if you want to **proxy all models from a specific provider without defining them on the config.yaml**
-
-**Step 1** - define provider specific routing on config.yaml
-```yaml
-model_list:
-  # provider specific wildcard routing
-  - model_name: "anthropic/*"
-    litellm_params:
-      model: "anthropic/*"
-      api_key: os.environ/ANTHROPIC_API_KEY
-  - model_name: "groq/*"
-    litellm_params:
-      model: "groq/*"
-      api_key: os.environ/GROQ_API_KEY
-  - model_name: "fo::*:static::*" # all requests matching this pattern will be routed to this deployment, example: model="fo::hi::static::hi" will be routed to deployment: "openai/fo::*:static::*"
-    litellm_params:
-      model: "openai/fo::*:static::*"
-      api_key: os.environ/OPENAI_API_KEY
-```
-
-Step 2 - Run litellm proxy 
-
-```shell
-$ litellm --config /path/to/config.yaml
-```
-
-Step 3 Test it 
-
-Test with `anthropic/` - all models with `anthropic/` prefix will get routed to `anthropic/*`
-```shell
-curl http://localhost:4000/v1/chat/completions \
-  -H "Content-Type: application/json" \
-  -H "Authorization: Bearer sk-1234" \
-  -d '{
-    "model": "anthropic/claude-3-sonnet-20240229",
-    "messages": [
-      {"role": "user", "content": "Hello, Claude!"}
-    ]
-  }'
-```
-
-Test with `groq/` - all models with `groq/` prefix will get routed to `groq/*`
-```shell
-curl http://localhost:4000/v1/chat/completions \
-  -H "Content-Type: application/json" \
-  -H "Authorization: Bearer sk-1234" \
-  -d '{
-    "model": "groq/llama3-8b-8192",
-    "messages": [
-      {"role": "user", "content": "Hello, Claude!"}
-    ]
-  }'
-```
-
-Test with `fo::*::static::*` - all requests matching this pattern will be routed to `openai/fo::*:static::*`
-```shell
-curl http://localhost:4000/v1/chat/completions \
-  -H "Content-Type: application/json" \
-  -H "Authorization: Bearer sk-1234" \
-  -d '{
-    "model": "fo::hi::static::hi",
-    "messages": [
-      {"role": "user", "content": "Hello, Claude!"}
-    ]
-  }'
-```
-
 ### Load Balancing 
 
 :::info
@@ -597,474 +526,6 @@ general_settings:
   database_connection_timeout: 60 # sets a 60s timeout for any connection call to the db 
 ```
 
-## **All settings**
-
-
-```yaml
-environment_variables: {}
-
-model_list:
-  - model_name: string
-    litellm_params: {}
-    model_info:
-      id: string
-      mode: embedding
-      input_cost_per_token: 0
-      output_cost_per_token: 0
-      max_tokens: 2048
-      base_model: gpt-4-1106-preview
-      additionalProp1: {}
-
-litellm_settings:
-  # Logging/Callback settings
-  success_callback: ["langfuse"]  # list of success callbacks
-  failure_callback: ["sentry"]  # list of failure callbacks
-  callbacks: ["otel"]  # list of callbacks - runs on success and failure
-  service_callbacks: ["datadog", "prometheus"]  # logs redis, postgres failures on datadog, prometheus
-  turn_off_message_logging: boolean  # prevent the messages and responses from being logged to on your callbacks, but request metadata will still be logged.
-  redact_user_api_key_info: boolean  # Redact information about the user api key (hashed token, user_id, team id, etc.), from logs. Currently supported for Langfuse, OpenTelemetry, Logfire, ArizeAI logging.
-  langfuse_default_tags: ["cache_hit", "cache_key", "proxy_base_url", "user_api_key_alias", "user_api_key_user_id", "user_api_key_user_email", "user_api_key_team_alias", "semantic-similarity", "proxy_base_url"] # default tags for Langfuse Logging
-  
-  request_timeout: 10 # (int) llm requesttimeout in seconds. Raise Timeout error if call takes longer than 10s. Sets litellm.request_timeout 
-  
-  set_verbose: boolean # sets litellm.set_verbose=True to view verbose debug logs. DO NOT LEAVE THIS ON IN PRODUCTION
-  json_logs: boolean # if true, logs will be in json format
-
-  # Fallbacks, reliability
-  default_fallbacks: ["claude-opus"] # set default_fallbacks, in case a specific model group is misconfigured / bad.
-  content_policy_fallbacks: [{"gpt-3.5-turbo-small": ["claude-opus"]}] # fallbacks for ContentPolicyErrors
-  context_window_fallbacks: [{"gpt-3.5-turbo-small": ["gpt-3.5-turbo-large", "claude-opus"]}] # fallbacks for ContextWindowExceededErrors
-
-
-
-  # Caching settings
-  cache: true 
-  cache_params:        # set cache params for redis
-    type: redis        # type of cache to initialize
-
-    # Optional - Redis Settings
-    host: "localhost"  # The host address for the Redis cache. Required if type is "redis".
-    port: 6379  # The port number for the Redis cache. Required if type is "redis".
-    password: "your_password"  # The password for the Redis cache. Required if type is "redis".
-    namespace: "litellm.caching.caching" # namespace for redis cache
-  
-    # Optional - Redis Cluster Settings
-    redis_startup_nodes: [{"host": "127.0.0.1", "port": "7001"}] 
-
-    # Optional - Redis Sentinel Settings
-    service_name: "mymaster"
-    sentinel_nodes: [["localhost", 26379]]
-
-    # Optional - Qdrant Semantic Cache Settings
-    qdrant_semantic_cache_embedding_model: openai-embedding # the model should be defined on the model_list
-    qdrant_collection_name: test_collection
-    qdrant_quantization_config: binary
-    similarity_threshold: 0.8   # similarity threshold for semantic cache
-
-    # Optional - S3 Cache Settings
-    s3_bucket_name: cache-bucket-litellm   # AWS Bucket Name for S3
-    s3_region_name: us-west-2              # AWS Region Name for S3
-    s3_aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID  # us os.environ/<variable name> to pass environment variables. This is AWS Access Key ID for S3
-    s3_aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY  # AWS Secret Access Key for S3
-    s3_endpoint_url: https://s3.amazonaws.com  # [OPTIONAL] S3 endpoint URL, if you want to use Backblaze/cloudflare s3 bucket
-
-    # Common Cache settings
-    # Optional - Supported call types for caching
-    supported_call_types: ["acompletion", "atext_completion", "aembedding", "atranscription"]
-                          # /chat/completions, /completions, /embeddings, /audio/transcriptions
-    mode: default_off # if default_off, you need to opt in to caching on a per call basis
-    ttl: 600 # ttl for caching
-
-
-callback_settings:
-  otel:
-    message_logging: boolean  # OTEL logging callback specific settings
-
-general_settings:
-  completion_model: string
-  disable_spend_logs: boolean  # turn off writing each transaction to the db
-  disable_master_key_return: boolean  # turn off returning master key on UI (checked on '/user/info' endpoint)
-  disable_retry_on_max_parallel_request_limit_error: boolean  # turn off retries when max parallel request limit is reached
-  disable_reset_budget: boolean  # turn off reset budget scheduled task
-  disable_adding_master_key_hash_to_db: boolean  # turn off storing master key hash in db, for spend tracking
-  enable_jwt_auth: boolean  # allow proxy admin to auth in via jwt tokens with 'litellm_proxy_admin' in claims
-  enforce_user_param: boolean  # requires all openai endpoint requests to have a 'user' param
-  allowed_routes: ["route1", "route2"]  # list of allowed proxy API routes - a user can access. (currently JWT-Auth only)
-  key_management_system: google_kms  # either google_kms or azure_kms
-  master_key: string
-
-  # Database Settings
-  database_url: string
-  database_connection_pool_limit: 0  # default 100
-  database_connection_timeout: 0  # default 60s
-  allow_requests_on_db_unavailable: boolean  # if true, will allow requests that can not connect to the DB to verify Virtual Key to still work 
-
-  custom_auth: string
-  max_parallel_requests: 0  # the max parallel requests allowed per deployment 
-  global_max_parallel_requests: 0  # the max parallel requests allowed on the proxy all up 
-  infer_model_from_keys: true
-  background_health_checks: true
-  health_check_interval: 300
-  alerting: ["slack", "email"]
-  alerting_threshold: 0
-  use_client_credentials_pass_through_routes: boolean  # use client credentials for all pass through routes like "/vertex-ai", /bedrock/. When this is True Virtual Key auth will not be applied on these endpoints
-```
-
-### litellm_settings - Reference
-
-| Name | Type | Description |
-|------|------|-------------|
-| success_callback | array of strings | List of success callbacks. [Doc Proxy logging callbacks](logging), [Doc Metrics](prometheus) |
-| failure_callback | array of strings | List of failure callbacks [Doc Proxy logging callbacks](logging), [Doc Metrics](prometheus) |
-| callbacks | array of strings | List of callbacks - runs on success and failure [Doc Proxy logging callbacks](logging), [Doc Metrics](prometheus) |
-| service_callbacks | array of strings | System health monitoring - Logs redis, postgres failures on specified services (e.g. datadog, prometheus) [Doc Metrics](prometheus) |
-| turn_off_message_logging | boolean | If true, prevents messages and responses from being logged to callbacks, but request metadata will still be logged [Proxy Logging](logging) |
-| modify_params | boolean | If true, allows modifying the parameters of the request before it is sent to the LLM provider |
-| enable_preview_features | boolean | If true, enables preview features - e.g. Azure O1 Models with streaming support.|
-| redact_user_api_key_info | boolean | If true, redacts information about the user api key from logs [Proxy Logging](logging#redacting-userapikeyinfo) |
-| langfuse_default_tags | array of strings | Default tags for Langfuse Logging. Use this if you want to control which LiteLLM-specific fields are logged as tags by the LiteLLM proxy. By default LiteLLM Proxy logs no LiteLLM-specific fields as tags. [Further docs](./logging#litellm-specific-tags-on-langfuse---cache_hit-cache_key) |
-| set_verbose | boolean | If true, sets litellm.set_verbose=True to view verbose debug logs. DO NOT LEAVE THIS ON IN PRODUCTION |
-| json_logs | boolean | If true, logs will be in json format. If you need to store the logs as JSON, just set the `litellm.json_logs = True`. We currently just log the raw POST request from litellm as a JSON [Further docs](./debugging) |
-| default_fallbacks | array of strings | List of fallback models to use if a specific model group is misconfigured / bad. [Further docs](./reliability#default-fallbacks) |
-| request_timeout | integer | The timeout for requests in seconds. If not set, the default value is `6000 seconds`. [For reference OpenAI Python SDK defaults to `600 seconds`.](https://github.com/openai/openai-python/blob/main/src/openai/_constants.py) |
-| content_policy_fallbacks | array of objects | Fallbacks to use when a ContentPolicyViolationError is encountered. [Further docs](./reliability#content-policy-fallbacks) |
-| context_window_fallbacks | array of objects | Fallbacks to use when a ContextWindowExceededError is encountered. [Further docs](./reliability#context-window-fallbacks) |
-| cache | boolean | If true, enables caching. [Further docs](./caching) |
-| cache_params | object | Parameters for the cache. [Further docs](./caching) |
-| cache_params.type | string | The type of cache to initialize. Can be one of ["local", "redis", "redis-semantic", "s3", "disk", "qdrant-semantic"]. Defaults to "redis". [Furher docs](./caching) |
-| cache_params.host | string | The host address for the Redis cache. Required if type is "redis". |
-| cache_params.port | integer | The port number for the Redis cache. Required if type is "redis". |
-| cache_params.password | string | The password for the Redis cache. Required if type is "redis". |
-| cache_params.namespace | string | The namespace for the Redis cache. |
-| cache_params.redis_startup_nodes | array of objects | Redis Cluster Settings. [Further docs](./caching) |
-| cache_params.service_name | string | Redis Sentinel Settings. [Further docs](./caching) |
-| cache_params.sentinel_nodes | array of arrays | Redis Sentinel Settings. [Further docs](./caching) |
-| cache_params.ttl | integer | The time (in seconds) to store entries in cache. |
-| cache_params.qdrant_semantic_cache_embedding_model | string | The embedding model to use for qdrant semantic cache. |
-| cache_params.qdrant_collection_name | string | The name of the collection to use for qdrant semantic cache. |
-| cache_params.qdrant_quantization_config | string | The quantization configuration for the qdrant semantic cache. |
-| cache_params.similarity_threshold | float | The similarity threshold for the semantic cache. |
-| cache_params.s3_bucket_name | string | The name of the S3 bucket to use for the semantic cache. |
-| cache_params.s3_region_name | string | The region name for the S3 bucket. |
-| cache_params.s3_aws_access_key_id | string | The AWS access key ID for the S3 bucket. |
-| cache_params.s3_aws_secret_access_key | string | The AWS secret access key for the S3 bucket. |
-| cache_params.s3_endpoint_url | string | Optional - The endpoint URL for the S3 bucket. |
-| cache_params.supported_call_types | array of strings | The types of calls to cache. [Further docs](./caching) |
-| cache_params.mode | string | The mode of the cache. [Further docs](./caching) |
-
-### general_settings - Reference
-
-| Name | Type | Description |
-|------|------|-------------|
-| completion_model | string | The default model to use for completions when `model` is not specified in the request |
-| disable_spend_logs | boolean | If true, turns off writing each transaction to the database |
-| disable_master_key_return | boolean | If true, turns off returning master key on UI. (checked on '/user/info' endpoint) |
-| disable_retry_on_max_parallel_request_limit_error | boolean | If true, turns off retries when max parallel request limit is reached |
-| disable_reset_budget | boolean | If true, turns off reset budget scheduled task |
-| disable_adding_master_key_hash_to_db | boolean | If true, turns off storing master key hash in db |
-| enable_jwt_auth | boolean | allow proxy admin to auth in via jwt tokens with 'litellm_proxy_admin' in claims. [Doc on JWT Tokens](token_auth) |
-| enforce_user_param | boolean | If true, requires all OpenAI endpoint requests to have a 'user' param. [Doc on call hooks](call_hooks)|
-| allowed_routes | array of strings | List of allowed proxy API routes a user can access [Doc on controlling allowed routes](enterprise#control-available-public-private-routes)|
-| key_management_system | string | Specifies the key management system. [Doc Secret Managers](../secret) |
-| master_key | string | The master key for the proxy [Set up Virtual Keys](virtual_keys) |
-| database_url | string | The URL for the database connection [Set up Virtual Keys](virtual_keys) |
-| database_connection_pool_limit | integer | The limit for database connection pool [Setting DB Connection Pool limit](#configure-db-pool-limits--connection-timeouts) |
-| database_connection_timeout | integer | The timeout for database connections in seconds [Setting DB Connection Pool limit, timeout](#configure-db-pool-limits--connection-timeouts) |
-| allow_requests_on_db_unavailable | boolean | If true, allows requests to succeed even if DB is unreachable. **Only use this if running LiteLLM in your VPC** This will allow requests to work even when LiteLLM cannot connect to the DB to verify a Virtual Key |
-| custom_auth | string | Write your own custom authentication logic [Doc Custom Auth](virtual_keys#custom-auth) |
-| max_parallel_requests | integer | The max parallel requests allowed per deployment |
-| global_max_parallel_requests | integer | The max parallel requests allowed on the proxy overall |
-| infer_model_from_keys | boolean | If true, infers the model from the provided keys |
-| background_health_checks | boolean | If true, enables background health checks. [Doc on health checks](health) |
-| health_check_interval | integer | The interval for health checks in seconds [Doc on health checks](health) |
-| alerting | array of strings | List of alerting methods [Doc on Slack Alerting](alerting) |
-| alerting_threshold | integer | The threshold for triggering alerts [Doc on Slack Alerting](alerting) |
-| use_client_credentials_pass_through_routes | boolean | If true, uses client credentials for all pass-through routes. [Doc on pass through routes](pass_through) |
-| health_check_details | boolean | If false, hides health check details (e.g. remaining rate limit). [Doc on health checks](health) |
-| public_routes | List[str] | (Enterprise Feature) Control list of public routes |
-| alert_types | List[str] | Control list of alert types to send to slack (Doc on alert types)[./alerting.md] |
-| enforced_params | List[str] | (Enterprise Feature) List of params that must be included in all requests to the proxy |
-| enable_oauth2_auth | boolean | (Enterprise Feature) If true, enables oauth2.0 authentication |
-| use_x_forwarded_for | str | If true, uses the X-Forwarded-For header to get the client IP address |
-| service_account_settings | List[Dict[str, Any]] | Set `service_account_settings` if you want to create settings that only apply to service account keys (Doc on service accounts)[./service_accounts.md] | 
-| image_generation_model | str | The default model to use for image generation - ignores model set in request |
-| store_model_in_db | boolean | If true, allows `/model/new` endpoint to store model information in db. Endpoint disabled by default. [Doc on `/model/new` endpoint](./model_management.md#create-a-new-model) |
-| max_request_size_mb | int | The maximum size for requests in MB. Requests above this size will be rejected. |
-| max_response_size_mb | int | The maximum size for responses in MB. LLM Responses above this size will not be sent. |
-| proxy_budget_rescheduler_min_time | int | The minimum time (in seconds) to wait before checking db for budget resets. |
-| proxy_budget_rescheduler_max_time | int | The maximum time (in seconds) to wait before checking db for budget resets. |
-| proxy_batch_write_at | int | Time (in seconds) to wait before batch writing spend logs to the db. |
-| alerting_args | dict | Args for Slack Alerting [Doc on Slack Alerting](./alerting.md) |
-| custom_key_generate | str | Custom function for key generation [Doc on custom key generation](./virtual_keys.md#custom--key-generate) |
-| allowed_ips | List[str] | List of IPs allowed to access the proxy. If not set, all IPs are allowed. |
-| embedding_model | str | The default model to use for embeddings - ignores model set in request |
-| default_team_disabled | boolean | If true, users cannot create 'personal' keys (keys with no team_id). |
-| alert_to_webhook_url | Dict[str] | [Specify a webhook url for each alert type.](./alerting.md#set-specific-slack-channels-per-alert-type) |
-| key_management_settings | List[Dict[str, Any]] | Settings for key management system (e.g. AWS KMS, Azure Key Vault) [Doc on key management](../secret.md) |
-| allow_user_auth | boolean | (Deprecated) old approach for user authentication. |
-| user_api_key_cache_ttl | int | The time (in seconds) to cache user api keys in memory. |
-| disable_prisma_schema_update | boolean | If true, turns off automatic schema updates to DB |
-| litellm_key_header_name | str | If set, allows passing LiteLLM keys as a custom header. [Doc on custom headers](./virtual_keys.md#custom-headers) |
-| moderation_model | str | The default model to use for moderation. |
-| custom_sso | str | Path to a python file that implements custom SSO logic. [Doc on custom SSO](./custom_sso.md) |
-| allow_client_side_credentials | boolean | If true, allows passing client side credentials to the proxy. (Useful when testing finetuning models) [Doc on client side credentials](./virtual_keys.md#client-side-credentials) |
-| admin_only_routes | List[str] | (Enterprise Feature) List of routes that are only accessible to admin users. [Doc on admin only routes](./enterprise#control-available-public-private-routes) |
-| use_azure_key_vault | boolean | If true, load keys from azure key vault | 
-| use_google_kms | boolean | If true, load keys from google kms |
-| spend_report_frequency | str | Specify how often you want a Spend Report to be sent (e.g. "1d", "2d", "30d") [More on this](./alerting.md#spend-report-frequency) |
-| ui_access_mode | Literal["admin_only"] | If set, restricts access to the UI to admin users only. [Docs](./ui.md#restrict-ui-access) |
-| litellm_jwtauth | Dict[str, Any] | Settings for JWT authentication. [Docs](./token_auth.md) |
-| litellm_license | str | The license key for the proxy. [Docs](../enterprise.md#how-does-deployment-with-enterprise-license-work) |
-| oauth2_config_mappings | Dict[str, str] | Define the OAuth2 config mappings | 
-| pass_through_endpoints | List[Dict[str, Any]] | Define the pass through endpoints. [Docs](./pass_through) |
-| enable_oauth2_proxy_auth | boolean | (Enterprise Feature) If true, enables oauth2.0 authentication |
-| forward_openai_org_id | boolean | If true, forwards the OpenAI Organization ID to the backend LLM call (if it's OpenAI). |
-| forward_client_headers_to_llm_api | boolean | If true, forwards the client headers (any `x-` headers) to the backend LLM call |
-
-### router_settings - Reference
-
-```yaml
-router_settings:
-  routing_strategy: usage-based-routing-v2 # Literal["simple-shuffle", "least-busy", "usage-based-routing","latency-based-routing"], default="simple-shuffle"
-  redis_host: <your-redis-host>           # string
-  redis_password: <your-redis-password>   # string
-  redis_port: <your-redis-port>           # string
-  enable_pre_call_check: true             # bool - Before call is made check if a call is within model context window 
-  allowed_fails: 3 # cooldown model if it fails > 1 call in a minute. 
-  cooldown_time: 30 # (in seconds) how long to cooldown model if fails/min > allowed_fails
-  disable_cooldowns: True                  # bool - Disable cooldowns for all models 
-  enable_tag_filtering: True                # bool - Use tag based routing for requests
-  retry_policy: {                          # Dict[str, int]: retry policy for different types of exceptions
-    "AuthenticationErrorRetries": 3,
-    "TimeoutErrorRetries": 3,
-    "RateLimitErrorRetries": 3,
-    "ContentPolicyViolationErrorRetries": 4,
-    "InternalServerErrorRetries": 4
-  }
-  allowed_fails_policy: {
-    "BadRequestErrorAllowedFails": 1000, # Allow 1000 BadRequestErrors before cooling down a deployment
-    "AuthenticationErrorAllowedFails": 10, # int 
-    "TimeoutErrorAllowedFails": 12, # int 
-    "RateLimitErrorAllowedFails": 10000, # int 
-    "ContentPolicyViolationErrorAllowedFails": 15, # int 
-    "InternalServerErrorAllowedFails": 20, # int 
-  }
-  content_policy_fallbacks=[{"claude-2": ["my-fallback-model"]}] # List[Dict[str, List[str]]]: Fallback model for content policy violations
-  fallbacks=[{"claude-2": ["my-fallback-model"]}] # List[Dict[str, List[str]]]: Fallback model for all errors
-```
-
-| Name | Type | Description |
-|------|------|-------------|
-| routing_strategy | string | The strategy used for routing requests. Options: "simple-shuffle", "least-busy", "usage-based-routing", "latency-based-routing". Default is "simple-shuffle". [More information here](../routing) |
-| redis_host | string | The host address for the Redis server. **Only set this if you have multiple instances of LiteLLM Proxy and want current tpm/rpm tracking to be shared across them** |
-| redis_password | string | The password for the Redis server. **Only set this if you have multiple instances of LiteLLM Proxy and want current tpm/rpm tracking to be shared across them** |
-| redis_port | string | The port number for the Redis server. **Only set this if you have multiple instances of LiteLLM Proxy and want current tpm/rpm tracking to be shared across them**|
-| enable_pre_call_check | boolean | If true, checks if a call is within the model's context window before making the call. [More information here](reliability) |
-| content_policy_fallbacks | array of objects | Specifies fallback models for content policy violations. [More information here](reliability) |
-| fallbacks | array of objects | Specifies fallback models for all types of errors. [More information here](reliability) |
-| enable_tag_filtering | boolean | If true, uses tag based routing for requests [Tag Based Routing](tag_routing) |
-| cooldown_time | integer | The duration (in seconds) to cooldown a model if it exceeds the allowed failures. |
-| disable_cooldowns | boolean | If true, disables cooldowns for all models. [More information here](reliability) |
-| retry_policy | object | Specifies the number of retries for different types of exceptions. [More information here](reliability) |
-| allowed_fails | integer | The number of failures allowed before cooling down a model. [More information here](reliability) |
-| allowed_fails_policy | object | Specifies the number of allowed failures for different error types before cooling down a deployment. [More information here](reliability) |
-
-
-### environment variables - Reference
-
-| Name | Description |
-|------|-------------|
-| ACTIONS_ID_TOKEN_REQUEST_TOKEN | Token for requesting ID in GitHub Actions
-| ACTIONS_ID_TOKEN_REQUEST_URL | URL for requesting ID token in GitHub Actions
-| AISPEND_ACCOUNT_ID | Account ID for AI Spend
-| AISPEND_API_KEY | API Key for AI Spend
-| ALLOWED_EMAIL_DOMAINS | List of email domains allowed for access
-| ARIZE_API_KEY | API key for Arize platform integration
-| ARIZE_SPACE_KEY | Space key for Arize platform
-| ARGILLA_BATCH_SIZE | Batch size for Argilla logging
-| ARGILLA_API_KEY | API key for Argilla platform
-| ARGILLA_SAMPLING_RATE | Sampling rate for Argilla logging
-| ARGILLA_DATASET_NAME | Dataset name for Argilla logging
-| ARGILLA_BASE_URL | Base URL for Argilla service
-| ATHINA_API_KEY | API key for Athina service
-| AUTH_STRATEGY | Strategy used for authentication (e.g., OAuth, API key)
-| AWS_ACCESS_KEY_ID | Access Key ID for AWS services
-| AWS_PROFILE_NAME | AWS CLI profile name to be used
-| AWS_REGION_NAME | Default AWS region for service interactions
-| AWS_ROLE_NAME | Role name for AWS IAM usage
-| AWS_SECRET_ACCESS_KEY | Secret Access Key for AWS services
-| AWS_SESSION_NAME | Name for AWS session
-| AWS_WEB_IDENTITY_TOKEN | Web identity token for AWS
-| AZURE_API_VERSION | Version of the Azure API being used
-| AZURE_AUTHORITY_HOST | Azure authority host URL
-| AZURE_CLIENT_ID | Client ID for Azure services
-| AZURE_CLIENT_SECRET | Client secret for Azure services
-| AZURE_FEDERATED_TOKEN_FILE | File path to Azure federated token
-| AZURE_KEY_VAULT_URI | URI for Azure Key Vault
-| AZURE_TENANT_ID | Tenant ID for Azure Active Directory
-| BERRISPEND_ACCOUNT_ID | Account ID for BerriSpend service
-| BRAINTRUST_API_KEY | API key for Braintrust integration
-| CIRCLE_OIDC_TOKEN | OpenID Connect token for CircleCI
-| CIRCLE_OIDC_TOKEN_V2 | Version 2 of the OpenID Connect token for CircleCI
-| CONFIG_FILE_PATH | File path for configuration file
-| CUSTOM_TIKTOKEN_CACHE_DIR | Custom directory for Tiktoken cache
-| DATABASE_HOST | Hostname for the database server
-| DATABASE_NAME | Name of the database
-| DATABASE_PASSWORD | Password for the database user
-| DATABASE_PORT | Port number for database connection
-| DATABASE_SCHEMA | Schema name used in the database
-| DATABASE_URL | Connection URL for the database
-| DATABASE_USER | Username for database connection
-| DATABASE_USERNAME | Alias for database user
-| DATABRICKS_API_BASE | Base URL for Databricks API
-| DD_BASE_URL | Base URL for Datadog integration
-| DATADOG_BASE_URL | (Alternative to DD_BASE_URL) Base URL for Datadog integration
-| _DATADOG_BASE_URL | (Alternative to DD_BASE_URL) Base URL for Datadog integration
-| DD_API_KEY | API key for Datadog integration
-| DD_SITE | Site URL for Datadog (e.g., datadoghq.com)
-| DD_SOURCE | Source identifier for Datadog logs
-| DD_ENV | Environment identifier for Datadog logs. Only supported for `datadog_llm_observability` callback
-| DEBUG_OTEL | Enable debug mode for OpenTelemetry
-| DIRECT_URL | Direct URL for service endpoint
-| DISABLE_ADMIN_UI | Toggle to disable the admin UI
-| DISABLE_SCHEMA_UPDATE | Toggle to disable schema updates
-| DOCS_DESCRIPTION | Description text for documentation pages
-| DOCS_FILTERED | Flag indicating filtered documentation
-| DOCS_TITLE | Title of the documentation pages
-| EMAIL_SUPPORT_CONTACT | Support contact email address
-| GCS_BUCKET_NAME | Name of the Google Cloud Storage bucket
-| GCS_PATH_SERVICE_ACCOUNT | Path to the Google Cloud service account JSON file
-| GCS_FLUSH_INTERVAL | Flush interval for GCS logging (in seconds). Specify how often you want a log to be sent to GCS. **Default is 20 seconds**
-| GCS_BATCH_SIZE | Batch size for GCS logging. Specify after how many logs you want to flush to GCS. If `BATCH_SIZE` is set to 10, logs are flushed every 10 logs. **Default is 2048**
-| GENERIC_AUTHORIZATION_ENDPOINT | Authorization endpoint for generic OAuth providers
-| GENERIC_CLIENT_ID | Client ID for generic OAuth providers
-| GENERIC_CLIENT_SECRET | Client secret for generic OAuth providers
-| GENERIC_CLIENT_STATE | State parameter for generic client authentication
-| GENERIC_INCLUDE_CLIENT_ID | Include client ID in requests for OAuth
-| GENERIC_SCOPE | Scope settings for generic OAuth providers
-| GENERIC_TOKEN_ENDPOINT | Token endpoint for generic OAuth providers
-| GENERIC_USER_DISPLAY_NAME_ATTRIBUTE | Attribute for user's display name in generic auth
-| GENERIC_USER_EMAIL_ATTRIBUTE | Attribute for user's email in generic auth
-| GENERIC_USER_FIRST_NAME_ATTRIBUTE | Attribute for user's first name in generic auth
-| GENERIC_USER_ID_ATTRIBUTE | Attribute for user ID in generic auth
-| GENERIC_USER_LAST_NAME_ATTRIBUTE | Attribute for user's last name in generic auth
-| GENERIC_USER_PROVIDER_ATTRIBUTE | Attribute specifying the user's provider
-| GENERIC_USER_ROLE_ATTRIBUTE | Attribute specifying the user's role
-| GENERIC_USERINFO_ENDPOINT | Endpoint to fetch user information in generic OAuth
-| GALILEO_BASE_URL | Base URL for Galileo platform
-| GALILEO_PASSWORD | Password for Galileo authentication
-| GALILEO_PROJECT_ID | Project ID for Galileo usage
-| GALILEO_USERNAME | Username for Galileo authentication
-| GREENSCALE_API_KEY | API key for Greenscale service
-| GREENSCALE_ENDPOINT | Endpoint URL for Greenscale service
-| GOOGLE_APPLICATION_CREDENTIALS | Path to Google Cloud credentials JSON file
-| GOOGLE_CLIENT_ID | Client ID for Google OAuth
-| GOOGLE_CLIENT_SECRET | Client secret for Google OAuth
-| GOOGLE_KMS_RESOURCE_NAME | Name of the resource in Google KMS
-| HF_API_BASE | Base URL for Hugging Face API
-| HELICONE_API_KEY | API key for Helicone service
-| HUGGINGFACE_API_BASE | Base URL for Hugging Face API
-| IAM_TOKEN_DB_AUTH | IAM token for database authentication
-| JSON_LOGS | Enable JSON formatted logging
-| JWT_AUDIENCE | Expected audience for JWT tokens
-| JWT_PUBLIC_KEY_URL | URL to fetch public key for JWT verification
-| LAGO_API_BASE | Base URL for Lago API
-| LAGO_API_CHARGE_BY | Parameter to determine charge basis in Lago
-| LAGO_API_EVENT_CODE | Event code for Lago API events
-| LAGO_API_KEY | API key for accessing Lago services
-| LANGFUSE_DEBUG | Toggle debug mode for Langfuse
-| LANGFUSE_FLUSH_INTERVAL | Interval for flushing Langfuse logs
-| LANGFUSE_HOST | Host URL for Langfuse service
-| LANGFUSE_PUBLIC_KEY | Public key for Langfuse authentication
-| LANGFUSE_RELEASE | Release version of Langfuse integration
-| LANGFUSE_SECRET_KEY | Secret key for Langfuse authentication
-| LANGSMITH_API_KEY | API key for Langsmith platform
-| LANGSMITH_BASE_URL | Base URL for Langsmith service
-| LANGSMITH_BATCH_SIZE | Batch size for operations in Langsmith
-| LANGSMITH_DEFAULT_RUN_NAME | Default name for Langsmith run
-| LANGSMITH_PROJECT | Project name for Langsmith integration
-| LANGSMITH_SAMPLING_RATE | Sampling rate for Langsmith logging
-| LANGTRACE_API_KEY | API key for Langtrace service
-| LITERAL_API_KEY | API key for Literal integration
-| LITERAL_API_URL | API URL for Literal service
-| LITERAL_BATCH_SIZE | Batch size for Literal operations
-| LITELLM_DONT_SHOW_FEEDBACK_BOX | Flag to hide feedback box in LiteLLM UI
-| LITELLM_DROP_PARAMS | Parameters to drop in LiteLLM requests
-| LITELLM_EMAIL | Email associated with LiteLLM account
-| LITELLM_GLOBAL_MAX_PARALLEL_REQUEST_RETRIES | Maximum retries for parallel requests in LiteLLM
-| LITELLM_GLOBAL_MAX_PARALLEL_REQUEST_RETRY_TIMEOUT | Timeout for retries of parallel requests in LiteLLM
-| LITELLM_HOSTED_UI | URL of the hosted UI for LiteLLM
-| LITELLM_LICENSE | License key for LiteLLM usage
-| LITELLM_LOCAL_MODEL_COST_MAP | Local configuration for model cost mapping in LiteLLM
-| LITELLM_LOG | Enable detailed logging for LiteLLM
-| LITELLM_MODE | Operating mode for LiteLLM (e.g., production, development)
-| LITELLM_SALT_KEY | Salt key for encryption in LiteLLM
-| LITELLM_SECRET_AWS_KMS_LITELLM_LICENSE | AWS KMS encrypted license for LiteLLM
-| LITELLM_TOKEN | Access token for LiteLLM integration
-| LOGFIRE_TOKEN | Token for Logfire logging service
-| MICROSOFT_CLIENT_ID | Client ID for Microsoft services
-| MICROSOFT_CLIENT_SECRET | Client secret for Microsoft services
-| MICROSOFT_TENANT | Tenant ID for Microsoft Azure
-| NO_DOCS | Flag to disable documentation generation
-| NO_PROXY | List of addresses to bypass proxy
-| OAUTH_TOKEN_INFO_ENDPOINT | Endpoint for OAuth token info retrieval
-| OPENAI_API_BASE | Base URL for OpenAI API
-| OPENAI_API_KEY | API key for OpenAI services
-| OPENAI_ORGANIZATION | Organization identifier for OpenAI
-| OPENID_BASE_URL | Base URL for OpenID Connect services
-| OPENID_CLIENT_ID | Client ID for OpenID Connect authentication
-| OPENID_CLIENT_SECRET | Client secret for OpenID Connect authentication
-| OPENMETER_API_ENDPOINT | API endpoint for OpenMeter integration
-| OPENMETER_API_KEY | API key for OpenMeter services
-| OPENMETER_EVENT_TYPE | Type of events sent to OpenMeter
-| OTEL_ENDPOINT | OpenTelemetry endpoint for traces
-| OTEL_ENVIRONMENT_NAME | Environment name for OpenTelemetry
-| OTEL_EXPORTER | Exporter type for OpenTelemetry
-| OTEL_HEADERS | Headers for OpenTelemetry requests
-| OTEL_SERVICE_NAME | Service name identifier for OpenTelemetry
-| OTEL_TRACER_NAME | Tracer name for OpenTelemetry tracing
-| PREDIBASE_API_BASE | Base URL for Predibase API
-| PRESIDIO_ANALYZER_API_BASE | Base URL for Presidio Analyzer service
-| PRESIDIO_ANONYMIZER_API_BASE | Base URL for Presidio Anonymizer service
-| PROMETHEUS_URL | URL for Prometheus service
-| PROMPTLAYER_API_KEY | API key for PromptLayer integration
-| PROXY_ADMIN_ID | Admin identifier for proxy server
-| PROXY_BASE_URL | Base URL for proxy service
-| PROXY_LOGOUT_URL | URL for logging out of the proxy service
-| PROXY_MASTER_KEY | Master key for proxy authentication
-| QDRANT_API_BASE | Base URL for Qdrant API
-| QDRANT_API_KEY | API key for Qdrant service
-| QDRANT_URL | Connection URL for Qdrant database
-| REDIS_HOST | Hostname for Redis server
-| REDIS_PASSWORD | Password for Redis service
-| REDIS_PORT | Port number for Redis server
-| SERVER_ROOT_PATH | Root path for the server application
-| SET_VERBOSE | Flag to enable verbose logging
-| SLACK_DAILY_REPORT_FREQUENCY | Frequency of daily Slack reports (e.g., daily, weekly)
-| SLACK_WEBHOOK_URL | Webhook URL for Slack integration
-| SMTP_HOST | Hostname for the SMTP server
-| SMTP_PASSWORD | Password for SMTP authentication
-| SMTP_PORT | Port number for SMTP server
-| SMTP_SENDER_EMAIL | Email address used as the sender in SMTP transactions
-| SMTP_SENDER_LOGO | Logo used in emails sent via SMTP
-| SMTP_TLS | Flag to enable or disable TLS for SMTP connections
-| SMTP_USERNAME | Username for SMTP authentication
-| SPEND_LOGS_URL | URL for retrieving spend logs
-| SSL_CERTIFICATE | Path to the SSL certificate file
-| SSL_VERIFY | Flag to enable or disable SSL certificate verification
-| SUPABASE_KEY | API key for Supabase service
-| SUPABASE_URL | Base URL for Supabase instance
-| TEST_EMAIL_ADDRESS | Email address used for testing purposes
-| UI_LOGO_PATH | Path to the logo image used in the UI
-| UI_PASSWORD | Password for accessing the UI
-| UI_USERNAME | Username for accessing the UI
-| UPSTREAM_LANGFUSE_DEBUG | Flag to enable debugging for upstream Langfuse
-| UPSTREAM_LANGFUSE_HOST | Host URL for upstream Langfuse service
-| UPSTREAM_LANGFUSE_PUBLIC_KEY | Public key for upstream Langfuse authentication
-| UPSTREAM_LANGFUSE_RELEASE | Release version identifier for upstream Langfuse
-| UPSTREAM_LANGFUSE_SECRET_KEY | Secret key for upstream Langfuse authentication
-| USE_AWS_KMS | Flag to enable AWS Key Management Service for encryption
-| WEBHOOK_URL | URL for receiving webhooks from external services
 ## Extras
 
 

docs/my-website/docs/proxy/db_info.md

@@ -0,0 +1,71 @@
+# What is stored in the DB
+
+The LiteLLM Proxy uses a PostgreSQL database to store various information. Here's are the main features the DB is used for:
+- Virtual Keys, Organizations, Teams, Users, Budgets, and more.
+- Per request Usage Tracking
+
+## Link to DB Schema
+
+You can see the full DB Schema [here](https://github.com/BerriAI/litellm/blob/main/schema.prisma)
+
+## DB Tables
+
+### Organizations, Teams, Users, End Users
+
+| Table Name | Description | Row Insert Frequency |
+|------------|-------------|---------------------|
+| LiteLLM_OrganizationTable | Manages organization-level configurations. Tracks organization spend, model access, and metadata. Links to budget configurations and teams. | Low |
+| LiteLLM_TeamTable | Handles team-level settings within organizations. Manages team members, admins, and their roles. Controls team-specific budgets, rate limits, and model access. | Low |
+| LiteLLM_UserTable | Stores user information and their settings. Tracks individual user spend, model access, and rate limits. Manages user roles and team memberships. | Low |
+| LiteLLM_EndUserTable | Manages end-user configurations. Controls model access and regional requirements. Tracks end-user spend. | Low |
+| LiteLLM_TeamMembership | Tracks user participation in teams. Manages team-specific user budgets and spend. | Low |
+| LiteLLM_OrganizationMembership | Manages user roles within organizations. Tracks organization-specific user permissions and spend. | Low |
+| LiteLLM_InvitationLink | Handles user invitations. Manages invitation status and expiration. Tracks who created and accepted invitations. | Low |
+| LiteLLM_UserNotifications | Handles model access requests. Tracks user requests for model access. Manages approval status. | Low |
+
+### Authentication
+
+| Table Name | Description | Row Insert Frequency |
+|------------|-------------|---------------------|
+| LiteLLM_VerificationToken | Manages Virtual Keys and their permissions. Controls token-specific budgets, rate limits, and model access. Tracks key-specific spend and metadata. | **Medium** - stores all Virtual Keys |
+
+### Model (LLM) Management
+
+| Table Name | Description | Row Insert Frequency |
+|------------|-------------|---------------------|
+| LiteLLM_ProxyModelTable | Stores model configurations. Defines available models and their parameters. Contains model-specific information and settings. | Low - Configuration only |
+
+### Budget Management
+
+| Table Name | Description | Row Insert Frequency |
+|------------|-------------|---------------------|
+| LiteLLM_BudgetTable | Stores budget and rate limit configurations for organizations, keys, and end users. Tracks max budgets, soft budgets, TPM/RPM limits, and model-specific budgets. Handles budget duration and reset timing. | Low - Configuration only |
+
+
+### Tracking & Logging
+
+| Table Name | Description | Row Insert Frequency |
+|------------|-------------|---------------------|
+| LiteLLM_SpendLogs | Detailed logs of all API requests. Records token usage, spend, and timing information. Tracks which models and keys were used. | **High - every LLM API request** |
+| LiteLLM_ErrorLogs | Captures failed requests and errors. Stores exception details and request information. Helps with debugging and monitoring. | **Medium - on errors only** |
+| LiteLLM_AuditLog | Tracks changes to system configuration. Records who made changes and what was modified. Maintains history of updates to teams, users, and models. | **Off by default**, **High - when enabled** |
+
+## Disable `LiteLLM_SpendLogs` & `LiteLLM_ErrorLogs`
+
+You can disable spend_logs and error_logs by setting `disable_spend_logs` and `disable_error_logs` to `True` on the `general_settings` section of your proxy_config.yaml file.
+
+```yaml
+general_settings:
+  disable_spend_logs: True   # Disable writing spend logs to DB
+  disable_error_logs: True   # Disable writing error logs to DB
+```
+
+### What is the impact of disabling these logs?
+
+When disabling spend logs (`disable_spend_logs: True`):
+- You **will not** be able to view Usage on the LiteLLM UI
+- You **will** continue seeing cost metrics on s3, Prometheus, Langfuse (any other Logging integration you are using)
+
+When disabling error logs (`disable_error_logs: True`):
+- You **will not** be able to view Errors on the LiteLLM UI
+- You **will** continue seeing error logs in your application logs and any other logging integrations you are using

docs/my-website/docs/proxy/deploy.md

@@ -2,7 +2,7 @@ import Tabs from '@theme/Tabs';
 import TabItem from '@theme/TabItem';
 import Image from '@theme/IdealImage';
 
-# 🐳 Docker, Deployment
+# Docker, Deployment
 
 You can find the Dockerfile to build litellm proxy [here](https://github.com/BerriAI/litellm/blob/main/Dockerfile)
 
@@ -688,8 +688,35 @@ Provide an ssl certificate when starting litellm proxy server
 
 Use this if you want to run the proxy with hypercorn to support http/2
 
-**Usage**
-Pass the `--run_hypercorn` flag when starting the proxy
+Step 1. Build your custom docker image with hypercorn
+
+```shell
+# Use the provided base image
+FROM ghcr.io/berriai/litellm:main-latest
+
+# Set the working directory to /app
+WORKDIR /app
+
+# Copy the configuration file into the container at /app
+COPY config.yaml .
+
+# Make sure your docker/entrypoint.sh is executable
+RUN chmod +x ./docker/entrypoint.sh
+
+# Expose the necessary port
+EXPOSE 4000/tcp
+
+# 👉 Key Change: Install hypercorn
+RUN pip install hypercorn
+
+# Override the CMD instruction with your desired command and arguments
+# WARNING: FOR PROD DO NOT USE `--detailed_debug` it slows down response times, instead use the following CMD
+# CMD ["--port", "4000", "--config", "config.yaml"]
+
+CMD ["--port", "4000", "--config", "config.yaml", "--detailed_debug"]
+```
+
+Step 2. Pass the `--run_hypercorn` flag when starting the proxy
 
 ```shell
 docker run \
@@ -699,7 +726,7 @@ docker run \
     -e SERVER_ROOT_PATH="/api/v1"\
     -e DATABASE_URL=postgresql://<user>:<password>@<host>:<port>/<dbname> \
     -e LITELLM_MASTER_KEY="sk-1234"\
-    ghcr.io/berriai/litellm:main-latest \
+    your_custom_docker_image \
     --config /app/config.yaml
     --run_hypercorn
 ```

docs/my-website/docs/proxy/docker_quick_start.md

@@ -1,3 +1,7 @@
+
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+
 # Getting Started - E2E Tutorial
 
 End-to-End tutorial for LiteLLM Proxy to:
@@ -9,7 +13,11 @@ End-to-End tutorial for LiteLLM Proxy to:
 
 ## Pre-Requisites 
 
-- Install LiteLLM Docker Image 
+- Install LiteLLM Docker Image ** OR ** LiteLLM CLI (pip package)
+
+<Tabs>
+
+<TabItem value="docker" label="Docker">
 
 ```
 docker pull ghcr.io/berriai/litellm:main-latest
@@ -17,6 +25,18 @@ docker pull ghcr.io/berriai/litellm:main-latest
 
 [**See all docker images**](https://github.com/orgs/BerriAI/packages)
 
+</TabItem>
+
+<TabItem value="pip" label="LiteLLM CLI (pip package)">
+
+```shell
+$ pip install 'litellm[proxy]'
+```
+
+</TabItem>
+
+</Tabs>
+
 ## 1. Add a model 
 
 Control LiteLLM Proxy with a config.yaml file.
@@ -58,6 +78,11 @@ LiteLLM Proxy is 100% OpenAI-compatible. Test your azure model via the `/chat/co
 
 Save your config.yaml from step 1. as `litellm_config.yaml`.
 
+<Tabs>
+
+
+<TabItem value="docker" label="Docker">
+
 ```bash
 docker run \
     -v $(pwd)/litellm_config.yaml:/app/config.yaml \
@@ -70,6 +95,20 @@ docker run \
 # RUNNING on http://0.0.0.0:4000
 ```
 
+</TabItem>
+
+<TabItem value="pip" label="LiteLLM CLI (pip package)">
+
+```shell
+$ litellm --config /app/config.yaml --detailed_debug
+```
+
+</TabItem>
+
+
+</Tabs>
+
+
 Confirm your config.yaml got mounted correctly
 
 ```bash

docs/my-website/docs/proxy/ip_address.md

@@ -1,5 +1,5 @@
 
-# ✨ IP Address Filtering
+# IP Address Filtering
 
 :::info
 

docs/my-website/docs/proxy/load_balancing.md

@@ -1,4 +1,4 @@
-# Multiple Instances
+# Proxy - Load Balancing
 Load balance multiple instances of the same model
 
 The proxy will handle routing requests (using LiteLLM's Router). **Set `rpm` in the config if you want maximize throughput**

docs/my-website/docs/proxy/logging.md

@@ -4,6 +4,7 @@ Log Proxy input, output, and exceptions using:
 
 - Langfuse
 - OpenTelemetry
+- GCS and s3 Buckets
 - Custom Callbacks
 - Langsmith
 - DataDog
@@ -47,7 +48,19 @@ A number of these headers could be useful for troubleshooting, but the
 `x-litellm-call-id` is the one that is most useful for tracking a request across
 components in your system, including in logging tools.
 
-## Redacting UserAPIKeyInfo 
+
+## Logging Features
+
+### Conditional Logging by Virtual Keys, Teams
+
+Use this to:
+1. Conditionally enable logging for some virtual keys/teams
+2. Set different logging providers for different virtual keys/teams
+
+[👉 **Get Started** - Team/Key Based Logging](team_logging)
+
+
+### Redacting UserAPIKeyInfo 
 
 Redact information about the user api key (hashed token, user_id, team id, etc.), from logs. 
 
@@ -59,17 +72,58 @@ litellm_settings:
   redact_user_api_key_info: true
 ```
 
+
+### Redact Messages, Response Content
+
+Set `litellm.turn_off_message_logging=True` This will prevent the messages and responses from being logged to your logging provider, but request metadata will still be logged.
+
+
+Example config.yaml
+```yaml
+model_list:
+ - model_name: gpt-3.5-turbo
+    litellm_params:
+      model: gpt-3.5-turbo
+litellm_settings:
+  success_callback: ["langfuse"]
+  turn_off_message_logging: True # 👈 Key Change
+```
+
+If you have this feature turned on, you can override it for specific requests by
+setting a request header `LiteLLM-Disable-Message-Redaction: true`.
+
+```shell
+curl --location 'http://0.0.0.0:4000/chat/completions' \
+    --header 'Content-Type: application/json' \
+    --header 'LiteLLM-Disable-Message-Redaction: true' \
+    --data '{
+    "model": "gpt-3.5-turbo",
+    "messages": [
+        {
+        "role": "user",
+        "content": "what llm are you"
+        }
+    ]
+}'
+```
+
 Removes any field with `user_api_key_*` from metadata.
 
-## What gets logged? StandardLoggingPayload
+## What gets logged?
 
 Found under `kwargs["standard_logging_object"]`. This is a standard payload, logged for every response.
 
 ```python
+
 class StandardLoggingPayload(TypedDict):
     id: str
+    trace_id: str  # Trace multiple LLM calls belonging to same overall request (e.g. fallbacks/retries)
     call_type: str
     response_cost: float
+    response_cost_failure_debug_info: Optional[
+        StandardLoggingModelCostFailureDebugInformation
+    ]
+    status: StandardLoggingPayloadStatus
     total_tokens: int
     prompt_tokens: int
     completion_tokens: int
@@ -84,13 +138,13 @@ class StandardLoggingPayload(TypedDict):
     metadata: StandardLoggingMetadata
     cache_hit: Optional[bool]
     cache_key: Optional[str]
-    saved_cache_cost: Optional[float]
-    request_tags: list                         
+    saved_cache_cost: float
+    request_tags: list
     end_user: Optional[str]
-    requester_ip_address: Optional[str]         # IP address of requester
-    requester_metadata: Optional[dict]          # metadata passed in request in the "metadata" field
+    requester_ip_address: Optional[str]
     messages: Optional[Union[str, list, dict]]
     response: Optional[Union[str, list, dict]]
+    error_str: Optional[str]
     model_parameters: dict
     hidden_params: StandardLoggingHiddenParams
 
@@ -99,14 +153,50 @@ class StandardLoggingHiddenParams(TypedDict):
     cache_key: Optional[str]
     api_base: Optional[str]
     response_cost: Optional[str]
-    additional_headers: Optional[dict]
+    additional_headers: Optional[StandardLoggingAdditionalHeaders]
 
+class StandardLoggingAdditionalHeaders(TypedDict, total=False):
+    x_ratelimit_limit_requests: int
+    x_ratelimit_limit_tokens: int
+    x_ratelimit_remaining_requests: int
+    x_ratelimit_remaining_tokens: int
+
+class StandardLoggingMetadata(StandardLoggingUserAPIKeyMetadata):
+    """
+    Specific metadata k,v pairs logged to integration for easier cost tracking
+    """
+
+    spend_logs_metadata: Optional[
+        dict
+    ]  # special param to log k,v pairs to spendlogs for a call
+    requester_ip_address: Optional[str]
+    requester_metadata: Optional[dict]
 
 class StandardLoggingModelInformation(TypedDict):
     model_map_key: str
     model_map_value: Optional[ModelInfo]
+  
+
+StandardLoggingPayloadStatus = Literal["success", "failure"]
+
+class StandardLoggingModelCostFailureDebugInformation(TypedDict, total=False):
+    """
+    Debug information, if cost tracking fails.
+
+    Avoid logging sensitive information like response or optional params
+    """
+
+    error_str: Required[str]
+    traceback_str: Required[str]
+    model: str
+    cache_hit: Optional[bool]
+    custom_llm_provider: Optional[str]
+    base_model: Optional[str]
+    call_type: str
+    custom_pricing: Optional[bool]
 ```
 
+
 ## Langfuse
 
 We will use the `--config` to set `litellm.success_callback = ["langfuse"]` this will log all successfull LLM calls to langfuse. Make sure to set `LANGFUSE_PUBLIC_KEY` and `LANGFUSE_SECRET_KEY` in your environment
@@ -259,73 +349,8 @@ print(response)
 </TabItem>
 </Tabs>
 
-### Team based Logging to Langfuse
 
-[👉 Tutorial - Allow each team to use their own Langfuse Project / custom callbacks](team_logging)
-<!-- 
-
-**Example:**
-
-This config would send langfuse logs to 2 different langfuse projects, based on the team id 
-
-```yaml
-litellm_settings:
-  default_team_settings: 
-    - team_id: my-secret-project
-      success_callback: ["langfuse"]
-      langfuse_public_key: os.environ/LANGFUSE_PUB_KEY_1 # Project 1
-      langfuse_secret: os.environ/LANGFUSE_PRIVATE_KEY_1 # Project 1
-    - team_id: ishaans-secret-project
-      success_callback: ["langfuse"]
-      langfuse_public_key: os.environ/LANGFUSE_PUB_KEY_2 # Project 2
-      langfuse_secret: os.environ/LANGFUSE_SECRET_2 # Project 2
-```
-
-Now, when you [generate keys](./virtual_keys.md) for this team-id 
-
-```bash
-curl -X POST 'http://0.0.0.0:4000/key/generate' \
--H 'Authorization: Bearer sk-1234' \
--H 'Content-Type: application/json' \
--d '{"team_id": "ishaans-secret-project"}'
-```
-
-All requests made with these keys will log data to their team-specific logging. -->
-
-### Redacting Messages, Response Content from Langfuse Logging 
-
-Set `litellm.turn_off_message_logging=True` This will prevent the messages and responses from being logged to langfuse, but request metadata will still be logged.
-
-```yaml
-model_list:
- - model_name: gpt-3.5-turbo
-    litellm_params:
-      model: gpt-3.5-turbo
-litellm_settings:
-  success_callback: ["langfuse"]
-  turn_off_message_logging: True
-```
-
-If you have this feature turned on, you can override it for specific requests by
-setting a request header `LiteLLM-Disable-Message-Redaction: true`.
-
-```shell
-curl --location 'http://0.0.0.0:4000/chat/completions' \
-    --header 'Content-Type: application/json' \
-    --header 'LiteLLM-Disable-Message-Redaction: true' \
-    --data '{
-    "model": "gpt-3.5-turbo",
-    "messages": [
-        {
-        "role": "user",
-        "content": "what llm are you"
-        }
-    ]
-}'
-```
-
-
-### LiteLLM-specific Tags on Langfuse - `cache_hit`, `cache_key`
+### LiteLLM Tags - `cache_hit`, `cache_key`
 
 Use this if you want to control which LiteLLM-specific fields are logged as tags by the LiteLLM proxy. By default LiteLLM Proxy logs no LiteLLM-specific fields
 
@@ -360,7 +385,7 @@ litellm_settings:
   langfuse_default_tags: ["cache_hit", "cache_key", "proxy_base_url", "user_api_key_alias", "user_api_key_user_id", "user_api_key_user_email", "user_api_key_team_alias", "semantic-similarity", "proxy_base_url"]
 ```
 
-### 🔧 Debugging - Viewing RAW CURL sent from LiteLLM to provider
+### View POST sent from LiteLLM to provider
 
 Use this when you want to view the RAW curl request sent from LiteLLM to the LLM API 
 
@@ -463,7 +488,7 @@ You will see `raw_request` in your Langfuse Metadata. This is the RAW CURL comma
 
 <Image img={require('../../img/debug_langfuse.png')} />
 
-## OpenTelemetry format
+## OpenTelemetry
 
 :::info 
 
@@ -745,7 +770,7 @@ curl --location 'http://0.0.0.0:4000/chat/completions' \
 
 ** 🎉 Expect to see this trace logged in your OTEL collector**
 
-### Redacting Messages, Response Content from OTEL Logging
+### Redacting Messages, Response Content
 
 Set `message_logging=False` for `otel`, no messages / response will be logged
 
@@ -759,7 +784,8 @@ callback_settings:
     message_logging: False
 ```
 
-### Context propagation across Services `Traceparent HTTP Header`
+### Traceparent Header
+##### Context propagation across Services `Traceparent HTTP Header`
 
 ❓ Use this when you want to **pass information about the incoming request in a distributed tracing system**
 
@@ -809,7 +835,7 @@ Search for Trace=`80e1afed08e019fc1110464cfa66635c` on your OTEL Collector
 
 <Image img={require('../../img/otel_parent.png')} />
 
-### Forwarding `Traceparent HTTP Header` to LLM APIs
+##### Forwarding `Traceparent HTTP Header` to LLM APIs
 
 Use this if you want to forward the traceparent headers to your self hosted LLMs like vLLM
 
@@ -826,6 +852,151 @@ litellm_settings:
   forward_traceparent_to_llm_provider: True
 ```
 
+## Google Cloud Storage Buckets
+
+Log LLM Logs to [Google Cloud Storage Buckets](https://cloud.google.com/storage?hl=en)
+
+:::info
+
+✨ This is an Enterprise only feature [Get Started with Enterprise here](https://calendly.com/d/4mp-gd3-k5k/litellm-1-1-onboarding-chat)
+
+:::
+
+
+| Property | Details |
+|----------|---------|
+| Description | Log LLM Input/Output to cloud storage buckets |
+| Load Test Benchmarks | [Benchmarks](https://docs.litellm.ai/docs/benchmarks) |
+| Google Docs on Cloud Storage | [Google Cloud Storage](https://cloud.google.com/storage?hl=en) |
+
+
+
+#### Usage
+
+1. Add `gcs_bucket` to LiteLLM Config.yaml
+```yaml
+model_list:
+- litellm_params:
+    api_base: https://openai-function-calling-workers.tasslexyz.workers.dev/
+    api_key: my-fake-key
+    model: openai/my-fake-model
+  model_name: fake-openai-endpoint
+
+litellm_settings:
+  callbacks: ["gcs_bucket"] # 👈 KEY CHANGE # 👈 KEY CHANGE
+```
+
+2. Set required env variables
+
+```shell
+GCS_BUCKET_NAME="<your-gcs-bucket-name>"
+GCS_PATH_SERVICE_ACCOUNT="/Users/ishaanjaffer/Downloads/adroit-crow-413218-a956eef1a2a8.json" # Add path to service account.json
+```
+
+3. Start Proxy
+
+```
+litellm --config /path/to/config.yaml
+```
+
+4. Test it! 
+
+```bash
+curl --location 'http://0.0.0.0:4000/chat/completions' \
+--header 'Content-Type: application/json' \
+--data ' {
+      "model": "fake-openai-endpoint",
+      "messages": [
+        {
+          "role": "user",
+          "content": "what llm are you"
+        }
+      ],
+    }
+'
+```
+
+
+#### Expected Logs on GCS Buckets
+
+<Image img={require('../../img/gcs_bucket.png')} />
+
+#### Fields Logged on GCS Buckets
+
+[**The standard logging object is logged on GCS Bucket**](../proxy/logging)
+
+
+#### Getting `service_account.json` from Google Cloud Console
+
+1. Go to [Google Cloud Console](https://console.cloud.google.com/)
+2. Search for IAM & Admin
+3. Click on Service Accounts
+4. Select a Service Account
+5. Click on 'Keys' -> Add Key -> Create New Key -> JSON
+6. Save the JSON file and add the path to `GCS_PATH_SERVICE_ACCOUNT`
+
+
+## s3 Buckets
+
+We will use the `--config` to set 
+
+- `litellm.success_callback = ["s3"]` 
+
+This will log all successfull LLM calls to s3 Bucket
+
+**Step 1** Set AWS Credentials in .env
+
+```shell
+AWS_ACCESS_KEY_ID = ""
+AWS_SECRET_ACCESS_KEY = ""
+AWS_REGION_NAME = ""
+```
+
+**Step 2**: Create a `config.yaml` file and set `litellm_settings`: `success_callback`
+
+```yaml
+model_list:
+ - model_name: gpt-3.5-turbo
+    litellm_params:
+      model: gpt-3.5-turbo
+litellm_settings:
+  success_callback: ["s3"]
+  s3_callback_params:
+    s3_bucket_name: logs-bucket-litellm   # AWS Bucket Name for S3
+    s3_region_name: us-west-2              # AWS Region Name for S3
+    s3_aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID  # us os.environ/<variable name> to pass environment variables. This is AWS Access Key ID for S3
+    s3_aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY  # AWS Secret Access Key for S3
+    s3_path: my-test-path # [OPTIONAL] set path in bucket you want to write logs to
+    s3_endpoint_url: https://s3.amazonaws.com  # [OPTIONAL] S3 endpoint URL, if you want to use Backblaze/cloudflare s3 buckets
+```
+
+**Step 3**: Start the proxy, make a test request
+
+Start proxy
+
+```shell
+litellm --config config.yaml --debug
+```
+
+Test Request
+
+```shell
+curl --location 'http://0.0.0.0:4000/chat/completions' \
+    --header 'Content-Type: application/json' \
+    --data ' {
+    "model": "Azure OpenAI GPT-4 East",
+    "messages": [
+        {
+        "role": "user",
+        "content": "what llm are you"
+        }
+    ]
+    }'
+```
+
+Your logs should be available on the specified s3 Bucket
+
+
 ## Custom Callback Class [Async]
 
 Use this when you want to run custom callbacks in `python`
@@ -1054,7 +1225,7 @@ class MyCustomHandler(CustomLogger):
 {'mode': 'embedding', 'input_cost_per_token': 0.002}
 ```
 
-### Logging responses from proxy
+##### Logging responses from proxy
 
 Both `/chat/completions` and `/embeddings` responses are available as `response_obj`
 
@@ -1372,7 +1543,7 @@ export GALILEO_USERNAME=""
 export GALILEO_PASSWORD=""
 ```
 
-### Quick Start 
+#### Quick Start 
 
 1. Add to Config.yaml
 
@@ -1413,7 +1584,7 @@ curl --location 'http://0.0.0.0:4000/chat/completions' \
 
 🎉 That's it - Expect to see your Logs on your Galileo Dashboard
 
-## Logging Proxy Cost + Usage - OpenMeter
+## OpenMeter
 
 Bill customers according to their LLM API usage with [OpenMeter](../observability/openmeter.md)
 
@@ -1425,7 +1596,7 @@ export OPENMETER_API_ENDPOINT="" # defaults to https://openmeter.cloud
 export OPENMETER_API_KEY=""
 ```
 
-### Quick Start 
+##### Quick Start 
 
 1. Add to Config.yaml
 
@@ -1758,7 +1929,10 @@ curl --location 'http://0.0.0.0:4000/chat/completions' \
     }'
 ```
 
-## (BETA) Moderation with Azure Content Safety
+
+<!-- ## (BETA) Moderation with Azure Content Safety
+
+Note: This page is for logging callbacks and this is a moderation service. Commenting until we found a better location for this.
 
 [Azure Content-Safety](https://azure.microsoft.com/en-us/products/ai-services/ai-content-safety) is a Microsoft Azure service that provides content moderation APIs to detect potential offensive, harmful, or risky content in text.
 
@@ -1843,4 +2017,4 @@ litellm_settings:
 :::info
 `thresholds` are not required by default, but you can tune the values to your needs.
 Default values is `4` for all categories
-:::
+::: -->

docs/my-website/docs/proxy/pass_through.md

@@ -1,6 +1,6 @@
 import Image from '@theme/IdealImage';
 
-# ➡️ Create Pass Through Endpoints 
+# Create Pass Through Endpoints 
 
 Add pass through routes to LiteLLM Proxy
 

docs/my-website/docs/proxy/prod.md

@@ -1,5 +1,6 @@
 import Tabs from '@theme/Tabs';
 import TabItem from '@theme/TabItem';
+import Image from '@theme/IdealImage';
 
 # ⚡ Best Practices for Production
 
@@ -22,6 +23,7 @@ general_settings:
 
   # OPTIONAL Best Practices
   disable_spend_logs: True # turn off writing each transaction to the db. We recommend doing this is you don't need to see Usage on the LiteLLM UI and are tracking metrics via Prometheus
+  disable_error_logs: True # turn off writing LLM Exceptions to DB
   allow_requests_on_db_unavailable: True # Only USE when running LiteLLM on your VPC. Allow requests to still be processed even if the DB is unavailable. We recommend doing this if you're running LiteLLM on VPC that cannot be accessed from the public internet.
 
 litellm_settings:
@@ -101,18 +103,51 @@ general_settings:
   allow_requests_on_db_unavailable: True
 ```
 
-## 6. Disable spend_logs if you're not using the LiteLLM UI
+## 6. Disable spend_logs & error_logs if not using the LiteLLM UI
 
-By default LiteLLM will write every request to the `LiteLLM_SpendLogs` table. This is used for viewing Usage on the LiteLLM UI. 
+By default, LiteLLM writes several types of logs to the database:
+- Every LLM API request to the `LiteLLM_SpendLogs` table
+- LLM Exceptions to the `LiteLLM_LogsErrors` table
 
-If you're not viewing Usage on the LiteLLM UI (most users use Prometheus when this is disabled), you can disable spend_logs by setting `disable_spend_logs` to `True`.
+If you're not viewing these logs on the LiteLLM UI (most users use Prometheus for monitoring), you can disable them by setting the following flags to `True`:
 
 ```yaml
 general_settings:
-  disable_spend_logs: True
+  disable_spend_logs: True    # Disable writing spend logs to DB
+  disable_error_logs: True    # Disable writing error logs to DB
 ```
 
-## 7. Set LiteLLM Salt Key 
+[More information about what the Database is used for here](db_info)
+
+## 7. Use Helm PreSync Hook for Database Migrations [BETA]
+
+To ensure only one service manages database migrations, use our [Helm PreSync hook for Database Migrations](https://github.com/BerriAI/litellm/blob/main/deploy/charts/litellm-helm/templates/migrations-job.yaml). This ensures migrations are handled during `helm upgrade` or `helm install`, while LiteLLM pods explicitly disable migrations.
+
+
+1. **Helm PreSync Hook**:
+   - The Helm PreSync hook is configured in the chart to run database migrations during deployments.
+   - The hook always sets `DISABLE_SCHEMA_UPDATE=false`, ensuring migrations are executed reliably.
+  
+  Reference Settings to set on ArgoCD for `values.yaml`
+
+  ```yaml
+  db:
+    useExisting: true # use existing Postgres DB
+    url: postgresql://ishaanjaffer0324:3rnwpOBau6hT@ep-withered-mud-a5dkdpke.us-east-2.aws.neon.tech/test-argo-cd?sslmode=require # url of existing Postgres DB
+  ```
+
+2. **LiteLLM Pods**:
+   - Set `DISABLE_SCHEMA_UPDATE=true` in LiteLLM pod configurations to prevent them from running migrations.
+   
+   Example configuration for LiteLLM pod:
+   ```yaml
+   env:
+     - name: DISABLE_SCHEMA_UPDATE
+       value: "true"
+   ```
+
+
+## 8. Set LiteLLM Salt Key 
 
 If you plan on using the DB, set a salt key for encrypting/decrypting variables in the DB. 
 

docs/my-website/docs/proxy/prometheus.md

@@ -192,3 +192,13 @@ Here is a screenshot of the metrics you can monitor with the LiteLLM Grafana Das
 |----------------------|--------------------------------------|
 | `litellm_llm_api_failed_requests_metric`             | **deprecated** use `litellm_proxy_failed_requests_metric` |
 | `litellm_requests_metric`             | **deprecated** use `litellm_proxy_total_requests_metric` |
+
+
+## FAQ 
+
+### What are `_created` vs. `_total` metrics?
+
+- `_created` metrics are metrics that are created when the proxy starts
+- `_total` metrics are metrics that are incremented for each request
+
+You should consume the `_total` metrics for your counting purposes

docs/my-website/docs/proxy/provider_budget_routing.md

@@ -0,0 +1,191 @@
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+
+# Provider Budget Routing
+Use this to set budgets for LLM Providers - example $100/day for OpenAI, $100/day for Azure.
+
+## Quick Start
+
+Set provider budgets in your `proxy_config.yaml` file
+### Proxy Config setup
+```yaml
+model_list:
+    - model_name: gpt-3.5-turbo
+      litellm_params:
+        model: openai/gpt-3.5-turbo
+        api_key: os.environ/OPENAI_API_KEY
+
+router_settings:
+  provider_budget_config: 
+    openai: 
+      budget_limit: 0.000000000001 # float of $ value budget for time period
+      time_period: 1d # can be 1d, 2d, 30d, 1mo, 2mo
+    azure:
+      budget_limit: 100
+      time_period: 1d
+    anthropic:
+      budget_limit: 100
+      time_period: 10d
+    vertex_ai:
+      budget_limit: 100
+      time_period: 12d
+    gemini:
+      budget_limit: 100
+      time_period: 12d
+  
+  # OPTIONAL: Set Redis Host, Port, and Password if using multiple instance of LiteLLM
+  redis_host: os.environ/REDIS_HOST
+  redis_port: os.environ/REDIS_PORT
+  redis_password: os.environ/REDIS_PASSWORD
+
+general_settings:
+  master_key: sk-1234
+```
+
+### Make a test request
+
+We expect the first request to succeed, and the second request to fail since we cross the budget for `openai`
+
+
+**[Langchain, OpenAI SDK Usage Examples](../proxy/user_keys#request-format)**
+
+<Tabs>
+<TabItem label="Successful Call " value = "allowed">
+
+```shell
+curl -i http://localhost:4000/v1/chat/completions \
+  -H "Content-Type: application/json" \
+  -H "Authorization: Bearer sk-1234" \
+  -d '{
+    "model": "gpt-4o",
+    "messages": [
+      {"role": "user", "content": "hi my name is test request"}
+    ]
+  }'
+```
+
+</TabItem>
+<TabItem label="Unsuccessful call" value = "not-allowed">
+
+Expect this to fail since since `ishaan@berri.ai` in the request is PII
+
+```shell
+curl -i http://localhost:4000/v1/chat/completions \
+  -H "Content-Type: application/json" \
+  -H "Authorization: Bearer sk-1234" \
+  -d '{
+    "model": "gpt-4o",
+    "messages": [
+      {"role": "user", "content": "hi my name is test request"}
+    ]
+  }'
+```
+
+Expected response on failure
+
+```json
+{
+  "error": {
+    "message": "No deployments available - crossed budget for provider: Exceeded budget for provider openai: 0.0007350000000000001 >= 1e-12",
+    "type": "None",
+    "param": "None",
+    "code": "429"
+  }
+}
+```
+
+</TabItem>
+
+
+</Tabs>
+
+
+
+## How provider budget routing works
+
+1. **Budget Tracking**: 
+   - Uses Redis to track spend for each provider
+   - Tracks spend over specified time periods (e.g., "1d", "30d")
+   - Automatically resets spend after time period expires
+
+2. **Routing Logic**:
+   - Routes requests to providers under their budget limits
+   - Skips providers that have exceeded their budget
+   - If all providers exceed budget, raises an error
+
+3. **Supported Time Periods**:
+   - Seconds: "Xs" (e.g., "30s")
+   - Minutes: "Xm" (e.g., "10m")
+   - Hours: "Xh" (e.g., "24h")
+   - Days: "Xd" (e.g., "1d", "30d")
+   - Months: "Xmo" (e.g., "1mo", "2mo")
+
+4. **Requirements**:
+   - Redis required for tracking spend across instances
+   - Provider names must be litellm provider names. See [Supported Providers](https://docs.litellm.ai/docs/providers)
+
+## Monitoring Provider Remaining Budget
+
+LiteLLM will emit the following metric on Prometheus to track the remaining budget for each provider
+
+This metric indicates the remaining budget for a provider in dollars (USD)
+
+```
+litellm_provider_remaining_budget_metric{api_provider="openai"} 10
+```
+
+## Multi-instance setup
+
+If you are using a multi-instance setup, you will need to set the Redis host, port, and password in the `proxy_config.yaml` file. Redis is used to sync the spend across LiteLLM instances.
+
+```yaml
+model_list:
+    - model_name: gpt-3.5-turbo
+      litellm_params:
+        model: openai/gpt-3.5-turbo
+        api_key: os.environ/OPENAI_API_KEY
+
+router_settings:
+  provider_budget_config: 
+    openai: 
+      budget_limit: 0.000000000001 # float of $ value budget for time period
+      time_period: 1d # can be 1d, 2d, 30d, 1mo, 2mo
+  
+  # 👇 Add this: Set Redis Host, Port, and Password if using multiple instance of LiteLLM
+  redis_host: os.environ/REDIS_HOST
+  redis_port: os.environ/REDIS_PORT
+  redis_password: os.environ/REDIS_PASSWORD
+
+general_settings:
+  master_key: sk-1234
+```
+
+## Spec for provider_budget_config
+
+The `provider_budget_config` is a dictionary where:
+- **Key**: Provider name (string) - Must be a valid [LiteLLM provider name](https://docs.litellm.ai/docs/providers)
+- **Value**: Budget configuration object with the following parameters:
+  - `budget_limit`: Float value representing the budget in USD
+  - `time_period`: Duration string in one of the following formats:
+    - Seconds: `"Xs"` (e.g., "30s")
+    - Minutes: `"Xm"` (e.g., "10m")
+    - Hours: `"Xh"` (e.g., "24h")
+    - Days: `"Xd"` (e.g., "1d", "30d")
+    - Months: `"Xmo"` (e.g., "1mo", "2mo")
+
+Example structure:
+```yaml
+provider_budget_config:
+  openai:
+    budget_limit: 100.0    # $100 USD
+    time_period: "1d"      # 1 day period
+  azure:
+    budget_limit: 500.0    # $500 USD
+    time_period: "30d"     # 30 day period
+  anthropic:
+    budget_limit: 200.0    # $200 USD
+    time_period: "1mo"     # 1 month period
+  gemini:
+    budget_limit: 50.0     # $50 USD
+    time_period: "24h"     # 24 hour period
+```

docs/my-website/docs/proxy/reliability.md

@@ -2,7 +2,7 @@ import Image from '@theme/IdealImage';
 import Tabs from '@theme/Tabs';
 import TabItem from '@theme/TabItem';
 
-# Fallbacks, Load Balancing, Retries
+# Proxy - Fallbacks, Retries
 
 - Quick Start [load balancing](#test---load-balancing)
 - Quick Start [client side fallbacks](#test---client-side-fallbacks)
@@ -748,4 +748,19 @@ curl -L -X POST 'http://0.0.0.0:4000/v1/chat/completions' \
     "max_tokens": 300,
     "mock_testing_fallbacks": true
 }'
+```
+
+### Disable Fallbacks per key
+
+You can disable fallbacks per key by setting `disable_fallbacks: true` in your key metadata.
+
+```bash
+curl -L -X POST 'http://0.0.0.0:4000/key/generate' \
+-H 'Authorization: Bearer sk-1234' \
+-H 'Content-Type: application/json' \
+-d '{
+    "metadata": {
+        "disable_fallbacks": true
+    }
+}'
 ```

docs/my-website/docs/proxy/self_serve.md

@@ -217,4 +217,10 @@ litellm_settings:
     max_parallel_requests: 1000 # (Optional[int], optional): Max number of requests that can be made in parallel. Defaults to None.
     tpm_limit: 1000 #(Optional[int], optional): Tpm limit. Defaults to None.
     rpm_limit: 1000 #(Optional[int], optional): Rpm limit. Defaults to None.
-```
+
+  key_generation_settings: # Restricts who can generate keys. [Further docs](./virtual_keys.md#restricting-key-generation)
+    team_key_generation:
+      allowed_team_member_roles: ["admin"]
+    personal_key_generation: # maps to 'Default Team' on UI 
+      allowed_user_roles: ["proxy_admin"]
+```

docs/my-website/docs/proxy/team_based_routing.md

@@ -1,4 +1,4 @@
-# 👥 Team-based Routing
+# Team-based Routing
 
 ## Routing
 Route calls to different model groups based on the team-id

docs/my-website/docs/proxy/team_logging.md

@@ -2,7 +2,7 @@ import Image from '@theme/IdealImage';
 import Tabs from '@theme/Tabs';
 import TabItem from '@theme/TabItem';
 
-# 👥📊 Team/Key Based Logging
+# Team/Key Based Logging
 
 Allow each key/team to use their own Langfuse Project / custom callbacks
 
@@ -11,15 +11,13 @@ Allow each key/team to use their own Langfuse Project / custom callbacks
 Team 1 -> Logs to Langfuse Project 1 
 Team 2 -> Logs to Langfuse Project 2
 Team 3 -> Disabled Logging (for GDPR compliance)
-
 ```
 
 ## Team Based Logging
 
-[👉 Tutorial - Allow each team to use their own Langfuse Project / custom callbacks](team_logging.md)
 
 
-## Logging / Caching
+### Setting Team Logging via `config.yaml`
 
 Turn on/off logging and caching for a specific team id. 
 

docs/my-website/docs/proxy/ui.md

@@ -64,7 +64,7 @@ Allow others to create/delete their own keys.
 Features here are behind a commercial license in our `/enterprise` folder. [**See Code**](https://github.com/BerriAI/litellm/tree/main/enterprise)
 
 
-### Setup SSO/Auth for UI
+### SSO for UI
 
 #### Step 1: Set upperbounds for keys
 Control the upperbound that users can use for `max_budget`, `budget_duration` or any `key/generate` param per key. 
@@ -88,12 +88,6 @@ litellm_settings:
 
 #### Step 2: Setup Oauth Client
 
-:::tip
-
-Looking for how to use Oauth 2.0 for /chat, /completions API requests to the proxy? [Follow this doc](oauth2)
-
-:::
-
 <Tabs>
 <TabItem value="okta" label="Okta SSO">
 
@@ -196,6 +190,13 @@ GENERIC_SCOPE = "openid profile email" # default scope openid is sometimes not e
 
 </Tabs>
 
+### Default Login, Logout URLs
+
+Some SSO providers require a specific redirect url for login and logout. You can input the following values.
+
+- Login: `<your-proxy-base-url>/sso/key/generate`
+- Logout: `<your-proxy-base-url>`
+
 #### Step 3. Set `PROXY_BASE_URL` in your .env
 
 Set this in your .env (so the proxy can set the correct redirect url)
@@ -216,9 +217,9 @@ export ALLOWED_EMAIL_DOMAINS="berri.ai"
 
 This will check if the user email we receive from SSO contains this domain, before allowing access.
 
-### Set Admin view w/ SSO 
+### Set Proxy Admin
 
-You just need to set Proxy Admin ID
+Set a Proxy Admin when SSO is enabled. Once SSO is enabled, the `user_id` for users is retrieved from the SSO provider. In order to set a Proxy Admin, you need to copy the `user_id` from the UI and set it in your `.env` as `PROXY_ADMIN_ID`.
 
 #### Step 1: Copy your ID from the UI 
 
@@ -256,7 +257,7 @@ general_settings:
   default_team_disabled: true # OR you can set env var PROXY_DEFAULT_TEAM_DISABLED="true"
 ```
 
-### Sign in with Username, Password when SSO is on
+### Use Username, Password when SSO is on
 
 If you need to access the UI via username/password when SSO is on navigate to `/fallback/login`. This route will allow you to sign in with your username/password credentials.
 

docs/my-website/docs/proxy/virtual_keys.md

@@ -1,7 +1,7 @@
 import Tabs from '@theme/Tabs';
 import TabItem from '@theme/TabItem';
 
-# 🔑 Virtual Keys
+# Virtual Keys
 Track Spend, and control model access via virtual keys for the proxy
 
 :::info
@@ -811,6 +811,78 @@ litellm_settings:
     team_id: "core-infra"
 ```
 
+### Restricting Key Generation
+
+Use this to control who can generate keys. Useful when letting others create keys on the UI. 
+
+```yaml
+litellm_settings:
+  key_generation_settings:
+    team_key_generation:
+      allowed_team_member_roles: ["admin"]
+      required_params: ["tags"] # require team admins to set tags for cost-tracking when generating a team key
+    personal_key_generation: # maps to 'Default Team' on UI 
+      allowed_user_roles: ["proxy_admin"]
+```
+
+#### Spec 
+
+```python
+class TeamUIKeyGenerationConfig(TypedDict):
+    allowed_team_member_roles: List[str]
+    required_params: List[str] # require params on `/key/generate` to be set if a team key (team_id in request) is being generated
+
+
+class PersonalUIKeyGenerationConfig(TypedDict):
+    allowed_user_roles: List[LitellmUserRoles] 
+    required_params: List[str] # require params on `/key/generate` to be set if a personal key (no team_id in request) is being generated
+
+
+class StandardKeyGenerationConfig(TypedDict, total=False):
+    team_key_generation: TeamUIKeyGenerationConfig
+    personal_key_generation: PersonalUIKeyGenerationConfig
+
+
+class LitellmUserRoles(str, enum.Enum):
+    """
+    Admin Roles:
+    PROXY_ADMIN: admin over the platform
+    PROXY_ADMIN_VIEW_ONLY: can login, view all own keys, view all spend
+    ORG_ADMIN: admin over a specific organization, can create teams, users only within their organization
+
+    Internal User Roles:
+    INTERNAL_USER: can login, view/create/delete their own keys, view their spend
+    INTERNAL_USER_VIEW_ONLY: can login, view their own keys, view their own spend
+
+
+    Team Roles:
+    TEAM: used for JWT auth
+
+
+    Customer Roles:
+    CUSTOMER: External users -> these are customers
+
+    """
+
+    # Admin Roles
+    PROXY_ADMIN = "proxy_admin"
+    PROXY_ADMIN_VIEW_ONLY = "proxy_admin_viewer"
+
+    # Organization admins
+    ORG_ADMIN = "org_admin"
+
+    # Internal User Roles
+    INTERNAL_USER = "internal_user"
+    INTERNAL_USER_VIEW_ONLY = "internal_user_viewer"
+
+    # Team Roles
+    TEAM = "team"
+
+    # Customer Roles - External users of proxy
+    CUSTOMER = "customer"
+```
+
+
 ## **Next Steps - Set Budgets, Rate Limits per Virtual Key**
 
 [Follow this doc to set budgets, rate limiters per virtual key with LiteLLM](users)

docs/my-website/docs/rerank.md

@@ -113,4 +113,5 @@ curl http://0.0.0.0:4000/rerank \
 |-------------|--------------------|
 | Cohere      |   [Usage](#quick-start)                 |
 | Together AI|   [Usage](../docs/providers/togetherai)                 |  
-| Azure AI|   [Usage](../docs/providers/azure_ai)                 |  
+| Azure AI|   [Usage](../docs/providers/azure_ai)                 |  
+| Jina AI|   [Usage](../docs/providers/jina_ai)                 |  

docs/my-website/docs/router_architecture.md

@@ -0,0 +1,24 @@
+import Image from '@theme/IdealImage';
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+
+# Router Architecture (Fallbacks / Retries)
+
+## High Level architecture
+
+<Image img={require('../img/router_architecture.png')} style={{ width: '100%', maxWidth: '4000px' }} />
+
+### Request Flow 
+
+1. **User Sends Request**: The process begins when a user sends a request to the LiteLLM Router endpoint. All unified endpoints (`.completion`, `.embeddings`, etc) are supported by LiteLLM Router.
+
+2. **function_with_fallbacks**: The initial request is sent to the `function_with_fallbacks` function. This function wraps the initial request in a try-except block, to handle any exceptions - doing fallbacks if needed. This request is then sent to the `function_with_retries` function.
+
+
+3. **function_with_retries**: The `function_with_retries` function wraps the request in a try-except block and passes the initial request to a base litellm unified function (`litellm.completion`, `litellm.embeddings`, etc) to handle LLM API calling. `function_with_retries` handles any exceptions - doing retries on the model group if needed (i.e. if the request fails, it will retry on an available model within the model group). 
+
+4. **litellm.completion**: The `litellm.completion` function is a base function that handles the LLM API calling. It is used by `function_with_retries` to make the actual request to the LLM API.
+
+## Legend 
+
+**model_group**: A group of LLM API deployments that share the same `model_name`, are part of the same `model_group`, and can be load balanced across.

docs/my-website/docs/routing.md

@@ -281,7 +281,7 @@ Picks the deployment with the lowest response time.
 
 It caches, and updates the response times for deployments based on when a request was sent and received from a deployment.
 
-[**How to test**](https://github.com/BerriAI/litellm/blob/main/litellm/tests/test_lowest_latency_routing.py)
+[**How to test**](https://github.com/BerriAI/litellm/blob/main/tests/local_testing/test_lowest_latency_routing.py)
 
 ```python
 from litellm import Router 
@@ -567,7 +567,7 @@ print(response)
 
 Picks a deployment with the least number of ongoing calls, it's handling.
 
-[**How to test**](https://github.com/BerriAI/litellm/blob/main/litellm/tests/test_least_busy_routing.py)
+[**How to test**](https://github.com/BerriAI/litellm/blob/main/tests/local_testing/test_least_busy_routing.py)
 
 ```python
 from litellm import Router 
@@ -1035,7 +1035,7 @@ print(f"response: {response}")
 
 ### [Advanced]: Custom Retries, Cooldowns based on Error Type
 
-- Use `RetryPolicy` if you want to set a `num_retries` based on the Exception receieved
+- Use `RetryPolicy` if you want to set a `num_retries` based on the Exception received
 - Use `AllowedFailsPolicy` to set a custom number of `allowed_fails`/minute before cooling down a deployment
 
 [**See All Exception Types**](https://github.com/BerriAI/litellm/blob/ccda616f2f881375d4e8586c76fe4662909a7d22/litellm/types/router.py#L436)
@@ -1891,3 +1891,22 @@ router = Router(
     debug_level="DEBUG"  # defaults to INFO
 )
 ```
+
+## Router General Settings
+
+### Usage 
+
+```python
+router = Router(model_list=..., router_general_settings=RouterGeneralSettings(async_only_mode=True))
+```
+
+### Spec 
+```python
+class RouterGeneralSettings(BaseModel):
+    async_only_mode: bool = Field(
+        default=False
+    )  # this will only initialize async clients. Good for memory utils
+    pass_through_all_models: bool = Field(
+        default=False
+    )  # if passed a model not llm_router model list, pass through the request to litellm.acompletion/embedding
+```

docs/my-website/docs/secret.md

@@ -1,3 +1,6 @@
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+
 # Secret Manager
 LiteLLM supports reading secrets from Azure Key Vault, Google Secret Manager
 
@@ -59,14 +62,36 @@ os.environ["AWS_REGION_NAME"] = "" # us-east-1, us-east-2, us-west-1, us-west-2
 ```
 
 2. Enable AWS Secret Manager in config. 
+
+<Tabs>
+<TabItem value="read_only" label="Read Keys from AWS Secret Manager">
+
 ```yaml
 general_settings:
   master_key: os.environ/litellm_master_key 
   key_management_system: "aws_secret_manager" # 👈 KEY CHANGE
   key_management_settings: 
     hosted_keys: ["litellm_master_key"] # 👈 Specify which env keys you stored on AWS 
+
 ```
 
+</TabItem>
+
+<TabItem value="write_only" label="Write Virtual Keys to AWS Secret Manager">
+
+This will only store virtual keys in AWS Secret Manager. No keys will be read from AWS Secret Manager.
+
+```yaml
+general_settings:
+  key_management_system: "aws_secret_manager" # 👈 KEY CHANGE
+  key_management_settings: 
+    store_virtual_keys: true # OPTIONAL. Defaults to False, when True will store virtual keys in secret manager
+    prefix_for_stored_virtual_keys: "litellm/" # OPTIONAL. If set, this prefix will be used for stored virtual keys in the secret manager
+    access_mode: "write_only" # Literal["read_only", "write_only", "read_and_write"]
+```
+</TabItem>
+</Tabs>
+
 3. Run proxy
 
 ```bash
@@ -181,16 +206,14 @@ litellm --config /path/to/config.yaml
 
 Use encrypted keys from Google KMS on the proxy
 
-### Usage with LiteLLM Proxy Server
-
-## Step 1. Add keys to env 
+Step 1. Add keys to env 
 ```
 export GOOGLE_APPLICATION_CREDENTIALS="/path/to/credentials.json"
 export GOOGLE_KMS_RESOURCE_NAME="projects/*/locations/*/keyRings/*/cryptoKeys/*"
 export PROXY_DATABASE_URL_ENCRYPTED=b'\n$\x00D\xac\xb4/\x8e\xc...'
 ```
 
-## Step 2: Update Config
+Step 2: Update Config
 
 ```yaml
 general_settings:
@@ -199,7 +222,7 @@ general_settings:
   master_key: sk-1234
 ```
 
-## Step 3: Start + test proxy
+Step 3: Start + test proxy
 
 ```
 $ litellm --config /path/to/config.yaml
@@ -215,3 +238,24 @@ $ litellm --test
 <!-- 
 ## .env Files
 If no secret manager client is specified, Litellm automatically uses the `.env` file to manage sensitive data. -->
+
+
+## All Secret Manager Settings
+
+All settings related to secret management
+
+```yaml
+general_settings:
+  key_management_system: "aws_secret_manager" # REQUIRED
+  key_management_settings:  
+
+    # Storing Virtual Keys Settings
+    store_virtual_keys: true # OPTIONAL. Defaults to False, when True will store virtual keys in secret manager
+    prefix_for_stored_virtual_keys: "litellm/" # OPTIONAL.I f set, this prefix will be used for stored virtual keys in the secret manager
+    
+    # Access Mode Settings
+    access_mode: "write_only" # OPTIONAL. Literal["read_only", "write_only", "read_and_write"]. Defaults to "read_only"
+    
+    # Hosted Keys Settings
+    hosted_keys: ["litellm_master_key"] # OPTIONAL. Specify which env keys you stored on AWS
+```

docs/my-website/docs/text_completion.md

@@ -0,0 +1,174 @@
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+
+# Text Completion
+
+### Usage
+<Tabs>
+<TabItem value="python" label="LiteLLM Python SDK">
+
+```python
+from litellm import text_completion
+
+response = text_completion(
+    model="gpt-3.5-turbo-instruct",
+    prompt="Say this is a test",
+    max_tokens=7
+)
+```
+
+</TabItem>
+<TabItem value="proxy" label="LiteLLM Proxy Server">
+
+1. Define models on config.yaml
+
+```yaml
+model_list:
+  - model_name: gpt-3.5-turbo-instruct
+    litellm_params:
+      model: text-completion-openai/gpt-3.5-turbo-instruct # The `text-completion-openai/` prefix will call openai.completions.create
+      api_key: os.environ/OPENAI_API_KEY
+  - model_name: text-davinci-003
+    litellm_params:
+      model: text-completion-openai/text-davinci-003
+      api_key: os.environ/OPENAI_API_KEY
+```
+
+2. Start litellm proxy server 
+
+```
+litellm --config config.yaml
+```
+
+<Tabs>
+<TabItem value="python" label="OpenAI Python SDK">
+
+```python
+from openai import OpenAI
+
+# set base_url to your proxy server
+# set api_key to send to proxy server
+client = OpenAI(api_key="<proxy-api-key>", base_url="http://0.0.0.0:4000")
+
+response = client.completions.create(
+    model="gpt-3.5-turbo-instruct",
+    prompt="Say this is a test",
+    max_tokens=7
+)
+
+print(response)
+```
+</TabItem>
+
+<TabItem value="curl" label="Curl Request">
+
+```shell
+curl --location 'http://0.0.0.0:4000/completions' \
+    --header 'Content-Type: application/json' \
+    --header 'Authorization: Bearer sk-1234' \
+    --data '{
+        "model": "gpt-3.5-turbo-instruct",
+        "prompt": "Say this is a test",
+        "max_tokens": 7
+    }'
+```
+</TabItem>
+</Tabs>
+
+</TabItem>
+</Tabs>
+
+## Input Params
+
+LiteLLM accepts and translates the [OpenAI Text Completion params](https://platform.openai.com/docs/api-reference/completions) across all supported providers.
+
+### Required Fields
+
+- `model`: *string* - ID of the model to use
+- `prompt`: *string or array* - The prompt(s) to generate completions for
+
+### Optional Fields
+
+- `best_of`: *integer* - Generates best_of completions server-side and returns the "best" one
+- `echo`: *boolean* - Echo back the prompt in addition to the completion.
+- `frequency_penalty`: *number* - Number between -2.0 and 2.0. Positive values penalize new tokens based on their existing frequency.
+- `logit_bias`: *map* - Modify the likelihood of specified tokens appearing in the completion
+- `logprobs`: *integer* - Include the log probabilities on the logprobs most likely tokens. Max value of 5
+- `max_tokens`: *integer* - The maximum number of tokens to generate.
+- `n`: *integer* - How many completions to generate for each prompt.
+- `presence_penalty`: *number* - Number between -2.0 and 2.0. Positive values penalize new tokens based on whether they appear in the text so far.
+- `seed`: *integer* - If specified, system will attempt to make deterministic samples
+- `stop`: *string or array* - Up to 4 sequences where the API will stop generating tokens
+- `stream`: *boolean* - Whether to stream back partial progress. Defaults to false
+- `suffix`: *string* - The suffix that comes after a completion of inserted text
+- `temperature`: *number* - What sampling temperature to use, between 0 and 2. 
+- `top_p`: *number* - An alternative to sampling with temperature, called nucleus sampling. 
+- `user`: *string* - A unique identifier representing your end-user
+
+## Output Format
+Here's the exact JSON output format you can expect from completion calls:
+
+
+[**Follows OpenAI's output format**](https://platform.openai.com/docs/api-reference/completions/object)
+
+<Tabs>
+
+<TabItem value="non-streaming" label="Non-Streaming Response">
+
+```python
+{
+  "id": "cmpl-uqkvlQyYK7bGYrRHQ0eXlWi7",
+  "object": "text_completion",
+  "created": 1589478378,
+  "model": "gpt-3.5-turbo-instruct",
+  "system_fingerprint": "fp_44709d6fcb",
+  "choices": [
+    {
+      "text": "\n\nThis is indeed a test",
+      "index": 0,
+      "logprobs": null,
+      "finish_reason": "length"
+    }
+  ],
+  "usage": {
+    "prompt_tokens": 5,
+    "completion_tokens": 7,
+    "total_tokens": 12
+  }
+}
+
+```
+</TabItem>
+<TabItem value="streaming" label="Streaming Response">
+
+```python
+{
+  "id": "cmpl-7iA7iJjj8V2zOkCGvWF2hAkDWBQZe",
+  "object": "text_completion",
+  "created": 1690759702,
+  "choices": [
+    {
+      "text": "This",
+      "index": 0,
+      "logprobs": null,
+      "finish_reason": null
+    }
+  ],
+  "model": "gpt-3.5-turbo-instruct"
+  "system_fingerprint": "fp_44709d6fcb",
+}
+
+```
+
+</TabItem>
+</Tabs>
+
+
+## **Supported Providers**
+
+| Provider    | Link to Usage      |
+|-------------|--------------------|
+| OpenAI      |   [Usage](../docs/providers/text_completion_openai)                 | 
+| Azure OpenAI|   [Usage](../docs/providers/azure)                 |  
+
+

docs/my-website/docs/wildcard_routing.md

@@ -0,0 +1,140 @@
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+
+# Provider specific Wildcard routing 
+
+**Proxy all models from a provider**
+
+Use this if you want to **proxy all models from a specific provider without defining them on the config.yaml**
+
+## Step 1. Define provider specific routing 
+
+<Tabs>
+<TabItem value="sdk" label="SDK">
+
+```python
+from litellm import Router
+
+router = Router(
+    model_list=[
+        {
+            "model_name": "anthropic/*",
+            "litellm_params": {
+                "model": "anthropic/*",
+                "api_key": os.environ["ANTHROPIC_API_KEY"]
+            }
+        }, 
+        {
+            "model_name": "groq/*",
+            "litellm_params": {
+                "model": "groq/*",
+                "api_key": os.environ["GROQ_API_KEY"]
+            }
+        }, 
+        {
+            "model_name": "fo::*:static::*", # all requests matching this pattern will be routed to this deployment, example: model="fo::hi::static::hi" will be routed to deployment: "openai/fo::*:static::*"
+            "litellm_params": {
+                "model": "openai/fo::*:static::*",
+                "api_key": os.environ["OPENAI_API_KEY"]
+            }
+        }
+    ]
+)
+```
+
+</TabItem>
+<TabItem value="proxy" label="PROXY">
+
+**Step 1** - define provider specific routing on config.yaml
+```yaml
+model_list:
+  # provider specific wildcard routing
+  - model_name: "anthropic/*"
+    litellm_params:
+      model: "anthropic/*"
+      api_key: os.environ/ANTHROPIC_API_KEY
+  - model_name: "groq/*"
+    litellm_params:
+      model: "groq/*"
+      api_key: os.environ/GROQ_API_KEY
+  - model_name: "fo::*:static::*" # all requests matching this pattern will be routed to this deployment, example: model="fo::hi::static::hi" will be routed to deployment: "openai/fo::*:static::*"
+    litellm_params:
+      model: "openai/fo::*:static::*"
+      api_key: os.environ/OPENAI_API_KEY
+```
+</TabItem>
+</Tabs>
+
+## [PROXY-Only] Step 2 - Run litellm proxy 
+
+```shell
+$ litellm --config /path/to/config.yaml
+```
+
+## Step 3 - Test it 
+
+<Tabs>  
+<TabItem value="sdk" label="SDK">
+
+```python
+from litellm import Router
+
+router = Router(model_list=...)
+
+# Test with `anthropic/` - all models with `anthropic/` prefix will get routed to `anthropic/*`
+resp = completion(model="anthropic/claude-3-sonnet-20240229", messages=[{"role": "user", "content": "Hello, Claude!"}])
+print(resp)
+
+# Test with `groq/` - all models with `groq/` prefix will get routed to `groq/*`
+resp = completion(model="groq/llama3-8b-8192", messages=[{"role": "user", "content": "Hello, Groq!"}])
+print(resp)
+
+# Test with `fo::*::static::*` - all requests matching this pattern will be routed to `openai/fo::*:static::*`
+resp = completion(model="fo::hi::static::hi", messages=[{"role": "user", "content": "Hello, Claude!"}])
+print(resp)
+```
+
+</TabItem>
+<TabItem value="proxy" label="PROXY">
+
+Test with `anthropic/` - all models with `anthropic/` prefix will get routed to `anthropic/*`
+```bash
+curl http://localhost:4000/v1/chat/completions \
+  -H "Content-Type: application/json" \
+  -H "Authorization: Bearer sk-1234" \
+  -d '{
+    "model": "anthropic/claude-3-sonnet-20240229",
+    "messages": [
+      {"role": "user", "content": "Hello, Claude!"}
+    ]
+  }'
+```
+
+Test with `groq/` - all models with `groq/` prefix will get routed to `groq/*`
+```shell
+curl http://localhost:4000/v1/chat/completions \
+  -H "Content-Type: application/json" \
+  -H "Authorization: Bearer sk-1234" \
+  -d '{
+    "model": "groq/llama3-8b-8192",
+    "messages": [
+      {"role": "user", "content": "Hello, Claude!"}
+    ]
+  }'
+```
+
+Test with `fo::*::static::*` - all requests matching this pattern will be routed to `openai/fo::*:static::*`
+```shell
+curl http://localhost:4000/v1/chat/completions \
+  -H "Content-Type: application/json" \
+  -H "Authorization: Bearer sk-1234" \
+  -d '{
+    "model": "fo::hi::static::hi",
+    "messages": [
+      {"role": "user", "content": "Hello, Claude!"}
+    ]
+  }'
+```
+
+</TabItem>
+</Tabs>

docs/my-website/docusaurus.config.js

@@ -113,7 +113,7 @@ const config = {
           {
             sidebarId: 'tutorialSidebar',
             position: 'left',
-            label: '🚀 Hosted',
+            label: 'Hosted',
             to: "docs/hosted"
           },
           {

docs/my-website/sidebars.js

@@ -20,61 +20,72 @@ const sidebars = {
     { type: "doc", id: "index" }, // NEW
     {
       type: "category",
-      label: "💥 LiteLLM Proxy Server",
+      label: "LiteLLM Proxy Server",
       link: {
         type: "generated-index",
-        title: "💥 LiteLLM Proxy Server (LLM Gateway)",
+        title: "LiteLLM Proxy Server (LLM Gateway)",
         description: `OpenAI Proxy Server (LLM Gateway) to call 100+ LLMs in a unified interface & track spend, set budgets per virtual key/user`,
         slug: "/simple_proxy",
       },
       items: [
-        "proxy/quick_start",
-        "proxy/docker_quick_start",
-        "proxy/deploy", 
+        "proxy/docker_quick_start", 
+        {
+          "type": "category", 
+          "label": "Config.yaml",
+          "items": ["proxy/configs", "proxy/config_management", "proxy/config_settings"]
+        },
+        {
+          type: "category",
+          label: "Setup & Deployment",
+          items: [
+            "proxy/deploy", 
+            "proxy/prod", 
+            "proxy/cli",
+            "proxy/model_management",
+            "proxy/health",
+            "proxy/debugging",
+            "proxy/pass_through",
+        ],
+        },
         "proxy/demo",
-        "proxy/prod",
         {
           type: "category",
           label: "Architecture",
-          items: ["proxy/architecture"],
+          items: ["proxy/architecture", "proxy/db_info", "router_architecture"],
         }, 
         {
           type: "link",
-          label: "📖 All Endpoints (Swagger)",
+          label: "All Endpoints (Swagger)",
           href: "https://litellm-api.up.railway.app/",
         },
         "proxy/enterprise",
-        "proxy/user_keys",
-        "proxy/configs",
-        "proxy/response_headers", 
-        "proxy/reliability",
         {
           type: "category",
-          label: "🔑 Authentication",
-          items: ["proxy/virtual_keys", "proxy/token_auth", "proxy/service_accounts", "proxy/access_control","proxy/ip_address"],
-        },
-        {
-          type: "category",
-          label: "💸 Spend Tracking + Budgets",
-          items: ["proxy/cost_tracking", "proxy/users", "proxy/custom_pricing", "proxy/team_budgets", "proxy/billing", "proxy/customers"],
-        },
-        {
-          type: "category",
-          label: "Routing",
-          items: ["proxy/load_balancing", "proxy/tag_routing", "proxy/team_based_routing", "proxy/customer_routing",],
-        },
-        {
-          type: "category",
-          label: "Use with Provider SDKs",
+          label: "Making LLM Requests",
           items: [
+            "proxy/user_keys",
+            "proxy/response_headers", 
             "pass_through/vertex_ai",
             "pass_through/google_ai_studio",
             "pass_through/cohere",
-            "anthropic_completion",
+            "pass_through/anthropic_completion",
             "pass_through/bedrock",
             "pass_through/langfuse"
           ],
         },
+        {
+          type: "category",
+          label: "Authentication",
+          items: [
+            "proxy/virtual_keys", 
+            "proxy/token_auth", 
+            "proxy/service_accounts", 
+            "proxy/access_control",
+            "proxy/ip_address",
+            "proxy/email",
+            "proxy/multiple_admins",
+          ],
+        },
         {
           type: "category",
           label: "Admin UI",
@@ -86,12 +97,22 @@ const sidebars = {
         },
         {
           type: "category",
-          label: "🪢 Logging, Alerting, Metrics",
-          items: ["proxy/logging", "proxy/bucket", "proxy/team_logging","proxy/streaming_logging", "proxy/alerting", "proxy/prometheus",],
+          label: "Spend Tracking + Budgets",
+          items: ["proxy/cost_tracking", "proxy/users", "proxy/custom_pricing", "proxy/team_budgets", "proxy/billing", "proxy/customers"],
+        },
+        {
+          type: "link",
+          label: "Load Balancing, Routing, Fallbacks",
+          href: "https://docs.litellm.ai/docs/routing-load-balancing",
         },
         {
           type: "category",
-          label: "🛡️ [Beta] Guardrails",
+          label: "Logging, Alerting, Metrics",
+          items: ["proxy/logging", "proxy/team_logging","proxy/alerting", "proxy/prometheus",],
+        },
+        {
+          type: "category",
+          label: "[Beta] Guardrails",
           items: [
             "proxy/guardrails/quick_start", 
             "proxy/guardrails/aporia_api", 
@@ -106,27 +127,20 @@ const sidebars = {
         },
         {
           type: "category", 
-          label: "Secret Manager - storing LLM API Keys", 
+          label: "Secret Managers", 
           items: [
             "secret", 
             "oidc"
           ]
         },
         "proxy/caching",
-        "proxy/pass_through",
-        "proxy/email",
-        "proxy/multiple_admins",
-        "proxy/model_management",
-        "proxy/health",
-        "proxy/debugging",
         "proxy/call_hooks",
-        "proxy/rules",
-        "proxy/cli", 
+        "proxy/rules", 
       ]
     },
     {
       type: "category",
-      label: "💯 Supported Models & Providers",
+      label: "Supported Models & Providers",
       link: {
         type: "generated-index",
         title: "Providers",
@@ -183,7 +197,6 @@ const sidebars = {
         "providers/openrouter", 
         "providers/palm", 
         "providers/sambanova", 
-        // "providers/custom_openai_proxy",
         "providers/custom_llm_server",
         "providers/petals",
         
@@ -191,27 +204,19 @@ const sidebars = {
     },
     {
       type: "category",
-      label: "Chat Completions (litellm.completion + PROXY)",
-      link: {
-        type: "generated-index",
-        title: "Chat Completions",
-        description: "Details on the completion() function",
-        slug: "/completion",
-      },
+      label: "Guides",
       items: [
-        "completion/input",
+        "exception_mapping",
         "completion/provider_specific_params",
-        "completion/json_mode",
-        "completion/prompt_caching",
+        "guides/finetuned_models",
         "completion/audio",
         "completion/vision",
+        "completion/json_mode",
+        "completion/prompt_caching",
         "completion/predict_outputs",
         "completion/prefix",
         "completion/drop_params",
         "completion/prompt_formatting",
-        "completion/output",
-        "completion/usage",
-        "exception_mapping",
         "completion/stream",
         "completion/message_trimming",
         "completion/function_call",
@@ -219,21 +224,45 @@ const sidebars = {
         "completion/batching",
         "completion/mock_requests",
         "completion/reliable_completions",
-      ],
+        
+      ]
     },
     {
       type: "category",
-      label: "Supported Endpoints - /images, /audio/speech, /assistants etc",
+      label: "Supported Endpoints",
       items: [
+        {
+          type: "category",
+          label: "Chat",
+          link: {
+            type: "generated-index",
+            title: "Chat Completions",
+            description: "Details on the completion() function",
+            slug: "/completion",
+          },
+          items: [
+            "completion/input",
+            "completion/output",
+            "completion/usage",
+          ],
+        },
+        "text_completion",
         "embedding/supported_embedding",
         "image_generation",
-        "audio_transcription",
-        "text_to_speech",
+        {
+          type: "category",
+          label: "Audio",
+          "items": [
+            "audio_transcription",
+            "text_to_speech",
+          ]
+        },
         "rerank",
         "assistants",
         "batches",
         "realtime",
         "fine_tuning",
+        "moderation",
         {
           type: "link",
           label: "Use LiteLLM Proxy with Vertex, Bedrock SDK",
@@ -241,11 +270,20 @@ const sidebars = {
         },
       ],
     },
-    "routing",
-    "scheduler",
     {
       type: "category",
-      label: "🚅 LiteLLM Python SDK",
+      label: "Routing, Loadbalancing & Fallbacks",
+      link: {
+        type: "generated-index",
+        title: "Routing, Loadbalancing & Fallbacks",
+        description: "Learn how to load balance, route, and set fallbacks for your LLM requests",
+        slug: "/routing-load-balancing",
+      },
+      items: ["routing", "scheduler", "proxy/load_balancing", "proxy/reliability", "proxy/tag_routing", "proxy/provider_budget_routing", "proxy/team_based_routing", "proxy/customer_routing", "wildcard_routing"],
+    },
+    {
+      type: "category",
+      label: "LiteLLM Python SDK",
       items: [
         "set_keys",
         "completion/token_usage",

enterprise/utils.py

@@ -2,7 +2,9 @@
 from typing import Optional, List
 from litellm._logging import verbose_logger
 from litellm.proxy.proxy_server import PrismaClient, HTTPException
+from litellm.llms.custom_httpx.http_handler import HTTPHandler
 import collections
+import httpx
 from datetime import datetime
 
 
@@ -114,7 +116,6 @@ async def ui_get_spend_by_tags(
 
 
 def _forecast_daily_cost(data: list):
-    import requests  # type: ignore
     from datetime import datetime, timedelta
 
     if len(data) == 0:
@@ -136,17 +137,17 @@ def _forecast_daily_cost(data: list):
 
     print("last entry date", last_entry_date)
 
-    # Assuming today_date is a datetime object
-    today_date = datetime.now()
-
     # Calculate the last day of the month
     last_day_of_todays_month = datetime(
         today_date.year, today_date.month % 12 + 1, 1
     ) - timedelta(days=1)
 
+    print("last day of todays month", last_day_of_todays_month)
     # Calculate the remaining days in the month
     remaining_days = (last_day_of_todays_month - last_entry_date).days
 
+    print("remaining days", remaining_days)
+
     current_spend_this_month = 0
     series = {}
     for entry in data:
@@ -176,13 +177,19 @@ def _forecast_daily_cost(data: list):
         "Content-Type": "application/json",
     }
 
-    response = requests.post(
-        url="https://trend-api-production.up.railway.app/forecast",
-        json=payload,
-        headers=headers,
-    )
-    # check the status code
-    response.raise_for_status()
+    client = HTTPHandler()
+
+    try:
+        response = client.post(
+            url="https://trend-api-production.up.railway.app/forecast",
+            json=payload,
+            headers=headers,
+        )
+    except httpx.HTTPStatusError as e:
+        raise HTTPException(
+            status_code=500,
+            detail={"error": f"Error getting forecast: {e.response.text}"},
+        )
 
     json_response = response.json()
     forecast_data = json_response["forecast"]
@@ -206,13 +213,3 @@ def _forecast_daily_cost(data: list):
         f"Predicted Spend for { today_month } 2024, ${total_predicted_spend}"
     )
     return {"response": response_data, "predicted_spend": predicted_spend}
-
-    # print(f"Date: {entry['date']}, Spend: {entry['spend']}, Response: {response.text}")
-
-
-# _forecast_daily_cost(
-#     [
-#         {"date": "2022-01-01", "spend": 100},
-
-#     ]
-# )

litellm/__init__.py

@@ -24,6 +24,7 @@ from litellm.proxy._types import (
     KeyManagementSettings,
     LiteLLM_UpperboundKeyGenerateParams,
 )
+from litellm.types.utils import StandardKeyGenerationConfig
 import httpx
 import dotenv
 from enum import Enum
@@ -57,6 +58,7 @@ _custom_logger_compatible_callbacks_literal = Literal[
     "gcs_bucket",
     "opik",
     "argilla",
+    "mlflow",
 ]
 logged_real_time_event_types: Optional[Union[List[str], Literal["*"]]] = None
 _known_custom_logger_compatible_callbacks: List = list(
@@ -66,6 +68,7 @@ callbacks: List[Union[Callable, _custom_logger_compatible_callbacks_literal]] =
 langfuse_default_tags: Optional[List[str]] = None
 langsmith_batch_size: Optional[int] = None
 argilla_batch_size: Optional[int] = None
+datadog_use_v1: Optional[bool] = False  # if you want to use v1 datadog logged payload
 argilla_transformation_object: Optional[Dict[str, Any]] = None
 _async_input_callback: List[Callable] = (
     []
@@ -132,7 +135,7 @@ use_client: bool = False
 ssl_verify: Union[str, bool] = True
 ssl_certificate: Optional[str] = None
 disable_streaming_logging: bool = False
-in_memory_llm_clients_cache: dict = {}
+in_memory_llm_clients_cache: InMemoryCache = InMemoryCache()
 safe_memory_mode: bool = False
 enable_azure_ad_token_refresh: Optional[bool] = False
 ### DEFAULT AZURE API VERSION ###
@@ -272,6 +275,7 @@ s3_callback_params: Optional[Dict] = None
 generic_logger_headers: Optional[Dict] = None
 default_key_generate_params: Optional[Dict] = None
 upperbound_key_generate_params: Optional[LiteLLM_UpperboundKeyGenerateParams] = None
+key_generation_settings: Optional[StandardKeyGenerationConfig] = None
 default_internal_user_params: Optional[Dict] = None
 default_team_settings: Optional[List] = None
 max_user_budget: Optional[float] = None
@@ -279,15 +283,23 @@ default_max_internal_user_budget: Optional[float] = None
 max_internal_user_budget: Optional[float] = None
 internal_user_budget_duration: Optional[str] = None
 max_end_user_budget: Optional[float] = None
+disable_end_user_cost_tracking: Optional[bool] = None
 #### REQUEST PRIORITIZATION ####
 priority_reservation: Optional[Dict[str, float]] = None
 #### RELIABILITY ####
 REPEATED_STREAMING_CHUNK_LIMIT = 100  # catch if model starts looping the same chunk while streaming. Uses high default to prevent false positives.
+
+#### Networking settings ####
 request_timeout: float = 6000  # time in seconds
+force_ipv4: bool = (
+    False  # when True, litellm will force ipv4 for all LLM requests. Some users have seen httpx ConnectionError when using ipv6.
+)
 module_level_aclient = AsyncHTTPHandler(
     timeout=request_timeout, client_alias="module level aclient"
 )
 module_level_client = HTTPHandler(timeout=request_timeout)
+
+#### RETRIES ####
 num_retries: Optional[int] = None  # per model endpoint
 max_fallbacks: Optional[int] = None
 default_fallbacks: Optional[List] = None
@@ -304,7 +316,7 @@ secret_manager_client: Optional[Any] = (
 )
 _google_kms_resource_name: Optional[str] = None
 _key_management_system: Optional[KeyManagementSystem] = None
-_key_management_settings: Optional[KeyManagementSettings] = None
+_key_management_settings: KeyManagementSettings = KeyManagementSettings()
 #### PII MASKING ####
 output_parse_pii: bool = False
 #############################################
@@ -961,6 +973,8 @@ from .utils import (
     supports_response_schema,
     supports_parallel_function_calling,
     supports_vision,
+    supports_audio_input,
+    supports_audio_output,
     supports_system_messages,
     get_litellm_params,
     acreate,
@@ -1122,6 +1136,7 @@ from .llms.AzureOpenAI.chat.gpt_transformation import AzureOpenAIConfig
 from .llms.hosted_vllm.chat.transformation import HostedVLLMChatConfig
 from .llms.deepseek.chat.transformation import DeepSeekChatConfig
 from .llms.lm_studio.chat.transformation import LMStudioChatConfig
+from .llms.lm_studio.embed.transformation import LmStudioEmbeddingConfig
 from .llms.perplexity.chat.transformation import PerplexityChatConfig
 from .llms.AzureOpenAI.chat.o1_transformation import AzureOpenAIO1Config
 from .llms.watsonx.completion.handler import IBMWatsonXAIConfig

litellm/_redis.py

@@ -12,13 +12,13 @@ import json
 
 # s/o [@Frank Colson](https://www.linkedin.com/in/frank-colson-422b9b183/) for this redis implementation
 import os
-from typing import List, Optional, Union
+from typing import Dict, List, Optional, Union
 
 import redis  # type: ignore
 import redis.asyncio as async_redis  # type: ignore
 
 import litellm
-from litellm import get_secret
+from litellm import get_secret, get_secret_str
 
 from ._logging import verbose_logger
 
@@ -141,6 +141,13 @@ def _get_redis_client_logic(**env_overrides):
     if _sentinel_nodes is not None and isinstance(_sentinel_nodes, str):
         redis_kwargs["sentinel_nodes"] = json.loads(_sentinel_nodes)
 
+    _sentinel_password: Optional[str] = redis_kwargs.get(
+        "sentinel_password", None
+    ) or get_secret_str("REDIS_SENTINEL_PASSWORD")
+
+    if _sentinel_password is not None:
+        redis_kwargs["sentinel_password"] = _sentinel_password
+
     _service_name: Optional[str] = redis_kwargs.get("service_name", None) or get_secret(  # type: ignore
         "REDIS_SERVICE_NAME"
     )
@@ -217,6 +224,7 @@ def _init_redis_sentinel(redis_kwargs) -> redis.Redis:
 
 def _init_async_redis_sentinel(redis_kwargs) -> async_redis.Redis:
     sentinel_nodes = redis_kwargs.get("sentinel_nodes")
+    sentinel_password = redis_kwargs.get("sentinel_password")
     service_name = redis_kwargs.get("service_name")
 
     if not sentinel_nodes or not service_name:
@@ -227,7 +235,11 @@ def _init_async_redis_sentinel(redis_kwargs) -> async_redis.Redis:
     verbose_logger.debug("init_redis_sentinel: sentinel nodes are being initialized.")
 
     # Set up the Sentinel client
-    sentinel = async_redis.Sentinel(sentinel_nodes, socket_timeout=0.1)
+    sentinel = async_redis.Sentinel(
+        sentinel_nodes,
+        socket_timeout=0.1,
+        password=sentinel_password,
+    )
 
     # Return the master instance for the given service
 
@@ -301,12 +313,13 @@ def get_redis_async_client(**env_overrides) -> async_redis.Redis:
 
 def get_redis_connection_pool(**env_overrides):
     redis_kwargs = _get_redis_client_logic(**env_overrides)
+    verbose_logger.debug("get_redis_connection_pool: redis_kwargs", redis_kwargs)
     if "url" in redis_kwargs and redis_kwargs["url"] is not None:
         return async_redis.BlockingConnectionPool.from_url(
             timeout=5, url=redis_kwargs["url"]
         )
     connection_class = async_redis.Connection
-    if "ssl" in redis_kwargs and redis_kwargs["ssl"] is not None:
+    if "ssl" in redis_kwargs:
         connection_class = async_redis.SSLConnection
         redis_kwargs.pop("ssl", None)
         redis_kwargs["connection_class"] = connection_class

litellm/caching/redis_cache.py

@@ -20,6 +20,7 @@ from typing import TYPE_CHECKING, Any, List, Optional, Tuple
 import litellm
 from litellm._logging import print_verbose, verbose_logger
 from litellm.litellm_core_utils.core_helpers import _get_parent_otel_span_from_kwargs
+from litellm.types.caching import RedisPipelineIncrementOperation
 from litellm.types.services import ServiceLoggerPayload, ServiceTypes
 from litellm.types.utils import all_litellm_params
 
@@ -890,3 +891,92 @@ class RedisCache(BaseCache):
 
     def delete_cache(self, key):
         self.redis_client.delete(key)
+
+    async def _pipeline_increment_helper(
+        self,
+        pipe: pipeline,
+        increment_list: List[RedisPipelineIncrementOperation],
+    ) -> Optional[List[float]]:
+        """Helper function for pipeline increment operations"""
+        # Iterate through each increment operation and add commands to pipeline
+        for increment_op in increment_list:
+            cache_key = self.check_and_fix_namespace(key=increment_op["key"])
+            print_verbose(
+                f"Increment ASYNC Redis Cache PIPELINE: key: {cache_key}\nValue {increment_op['increment_value']}\nttl={increment_op['ttl']}"
+            )
+            pipe.incrbyfloat(cache_key, increment_op["increment_value"])
+            if increment_op["ttl"] is not None:
+                _td = timedelta(seconds=increment_op["ttl"])
+                pipe.expire(cache_key, _td)
+        # Execute the pipeline and return results
+        results = await pipe.execute()
+        print_verbose(f"Increment ASYNC Redis Cache PIPELINE: results: {results}")
+        return results
+
+    async def async_increment_pipeline(
+        self, increment_list: List[RedisPipelineIncrementOperation], **kwargs
+    ) -> Optional[List[float]]:
+        """
+        Use Redis Pipelines for bulk increment operations
+        Args:
+            increment_list: List of RedisPipelineIncrementOperation dicts containing:
+                - key: str
+                - increment_value: float
+                - ttl_seconds: int
+        """
+        # don't waste a network request if there's nothing to increment
+        if len(increment_list) == 0:
+            return None
+
+        from redis.asyncio import Redis
+
+        _redis_client: Redis = self.init_async_client()  # type: ignore
+        start_time = time.time()
+
+        print_verbose(
+            f"Increment Async Redis Cache Pipeline: increment list: {increment_list}"
+        )
+
+        try:
+            async with _redis_client as redis_client:
+                async with redis_client.pipeline(transaction=True) as pipe:
+                    results = await self._pipeline_increment_helper(
+                        pipe, increment_list
+                    )
+
+            print_verbose(f"pipeline increment results: {results}")
+
+            ## LOGGING ##
+            end_time = time.time()
+            _duration = end_time - start_time
+            asyncio.create_task(
+                self.service_logger_obj.async_service_success_hook(
+                    service=ServiceTypes.REDIS,
+                    duration=_duration,
+                    call_type="async_increment_pipeline",
+                    start_time=start_time,
+                    end_time=end_time,
+                    parent_otel_span=_get_parent_otel_span_from_kwargs(kwargs),
+                )
+            )
+            return results
+        except Exception as e:
+            ## LOGGING ##
+            end_time = time.time()
+            _duration = end_time - start_time
+            asyncio.create_task(
+                self.service_logger_obj.async_service_failure_hook(
+                    service=ServiceTypes.REDIS,
+                    duration=_duration,
+                    error=e,
+                    call_type="async_increment_pipeline",
+                    start_time=start_time,
+                    end_time=end_time,
+                    parent_otel_span=_get_parent_otel_span_from_kwargs(kwargs),
+                )
+            )
+            verbose_logger.error(
+                "LiteLLM Redis Caching: async increment_pipeline() - Got exception from REDIS %s",
+                str(e),
+            )
+            raise e

litellm/cost_calculator.py

@@ -46,6 +46,9 @@ from litellm.llms.OpenAI.cost_calculation import (
 from litellm.llms.OpenAI.cost_calculation import cost_per_token as openai_cost_per_token
 from litellm.llms.OpenAI.cost_calculation import cost_router as openai_cost_router
 from litellm.llms.together_ai.cost_calculator import get_model_params_and_category
+from litellm.llms.vertex_ai_and_google_ai_studio.image_generation.cost_calculator import (
+    cost_calculator as vertex_ai_image_cost_calculator,
+)
 from litellm.types.llms.openai import HttpxBinaryResponseContent
 from litellm.types.rerank import RerankResponse
 from litellm.types.router import SPECIAL_MODEL_INFO_PARAMS
@@ -171,7 +174,6 @@ def cost_per_token(  # noqa: PLR0915
                 model_with_provider = model_with_provider_and_region
     else:
         _, custom_llm_provider, _, _ = litellm.get_llm_provider(model=model)
-
     model_without_prefix = model
     model_parts = model.split("/", 1)
     if len(model_parts) > 1:
@@ -454,7 +456,6 @@ def _select_model_name_for_cost_calc(
 
     if base_model is not None:
         return base_model
-
     return_model = model
     if isinstance(completion_response, str):
         return return_model
@@ -620,7 +621,8 @@ def completion_cost(  # noqa: PLR0915
                 f"completion_response response ms: {getattr(completion_response, '_response_ms', None)} "
             )
             model = _select_model_name_for_cost_calc(
-                model=model, completion_response=completion_response
+                model=model,
+                completion_response=completion_response,
             )
             hidden_params = getattr(completion_response, "_hidden_params", None)
             if hidden_params is not None:
@@ -668,9 +670,11 @@ def completion_cost(  # noqa: PLR0915
         ):
             ### IMAGE GENERATION COST CALCULATION ###
             if custom_llm_provider == "vertex_ai":
-                # https://cloud.google.com/vertex-ai/generative-ai/pricing
-                # Vertex Charges Flat $0.20 per image
-                return 0.020
+                if isinstance(completion_response, ImageResponse):
+                    return vertex_ai_image_cost_calculator(
+                        model=model,
+                        image_response=completion_response,
+                    )
             elif custom_llm_provider == "bedrock":
                 if isinstance(completion_response, ImageResponse):
                     return bedrock_image_cost_calculator(
@@ -853,6 +857,8 @@ def response_cost_calculator(
             if isinstance(response_object, BaseModel):
                 response_object._hidden_params["optional_params"] = optional_params
             if isinstance(response_object, ImageResponse):
+                if base_model is not None:
+                    model = base_model
                 response_cost = completion_cost(
                     completion_response=response_object,
                     model=model,

litellm/integrations/datadog/datadog.py

@@ -32,9 +32,11 @@ from litellm.llms.custom_httpx.http_handler import (
     get_async_httpx_client,
     httpxSpecialProvider,
 )
+from litellm.proxy._types import UserAPIKeyAuth
+from litellm.types.integrations.datadog import *
 from litellm.types.services import ServiceLoggerPayload
+from litellm.types.utils import StandardLoggingPayload
 
-from .types import DD_ERRORS, DatadogPayload, DataDogStatus
 from .utils import make_json_serializable
 
 DD_MAX_BATCH_SIZE = 1000  # max number of logs DD API can accept
@@ -106,20 +108,20 @@ class DataDogLogger(CustomBatchLogger):
             verbose_logger.debug(
                 "Datadog: Logging - Enters logging function for model %s", kwargs
             )
-            dd_payload = self.create_datadog_logging_payload(
-                kwargs=kwargs,
-                response_obj=response_obj,
-                start_time=start_time,
-                end_time=end_time,
-            )
+            await self._log_async_event(kwargs, response_obj, start_time, end_time)
 
-            self.log_queue.append(dd_payload)
+        except Exception as e:
+            verbose_logger.exception(
+                f"Datadog Layer Error - {str(e)}\n{traceback.format_exc()}"
+            )
+            pass
+
+    async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time):
+        try:
             verbose_logger.debug(
-                f"Datadog, event added to queue. Will flush in {self.flush_interval} seconds..."
+                "Datadog: Logging - Enters logging function for model %s", kwargs
             )
-
-            if len(self.log_queue) >= self.batch_size:
-                await self.async_send_batch()
+            await self._log_async_event(kwargs, response_obj, start_time, end_time)
 
         except Exception as e:
             verbose_logger.exception(
@@ -181,12 +183,20 @@ class DataDogLogger(CustomBatchLogger):
             verbose_logger.debug(
                 "Datadog: Logging - Enters logging function for model %s", kwargs
             )
-            dd_payload = self.create_datadog_logging_payload(
-                kwargs=kwargs,
-                response_obj=response_obj,
-                start_time=start_time,
-                end_time=end_time,
-            )
+            if litellm.datadog_use_v1 is True:
+                dd_payload = self._create_v0_logging_payload(
+                    kwargs=kwargs,
+                    response_obj=response_obj,
+                    start_time=start_time,
+                    end_time=end_time,
+                )
+            else:
+                dd_payload = self.create_datadog_logging_payload(
+                    kwargs=kwargs,
+                    response_obj=response_obj,
+                    start_time=start_time,
+                    end_time=end_time,
+                )
 
             response = self.sync_client.post(
                 url=self.intake_url,
@@ -215,6 +225,22 @@ class DataDogLogger(CustomBatchLogger):
             pass
         pass
 
+    async def _log_async_event(self, kwargs, response_obj, start_time, end_time):
+        dd_payload = self.create_datadog_logging_payload(
+            kwargs=kwargs,
+            response_obj=response_obj,
+            start_time=start_time,
+            end_time=end_time,
+        )
+
+        self.log_queue.append(dd_payload)
+        verbose_logger.debug(
+            f"Datadog, event added to queue. Will flush in {self.flush_interval} seconds..."
+        )
+
+        if len(self.log_queue) >= self.batch_size:
+            await self.async_send_batch()
+
     def create_datadog_logging_payload(
         self,
         kwargs: Union[dict, Any],
@@ -236,73 +262,29 @@ class DataDogLogger(CustomBatchLogger):
         """
         import json
 
-        litellm_params = kwargs.get("litellm_params", {})
-        metadata = (
-            litellm_params.get("metadata", {}) or {}
-        )  # if litellm_params['metadata'] == None
-        messages = kwargs.get("messages")
-        optional_params = kwargs.get("optional_params", {})
-        call_type = kwargs.get("call_type", "litellm.completion")
-        cache_hit = kwargs.get("cache_hit", False)
-        usage = response_obj["usage"]
-        id = response_obj.get("id", str(uuid.uuid4()))
-        usage = dict(usage)
-        try:
-            response_time = (end_time - start_time).total_seconds() * 1000
-        except Exception:
-            response_time = None
+        standard_logging_object: Optional[StandardLoggingPayload] = kwargs.get(
+            "standard_logging_object", None
+        )
+        if standard_logging_object is None:
+            raise ValueError("standard_logging_object not found in kwargs")
 
-        try:
-            response_obj = dict(response_obj)
-        except Exception:
-            response_obj = response_obj
-
-        # Clean Metadata before logging - never log raw metadata
-        # the raw metadata can contain circular references which leads to infinite recursion
-        # we clean out all extra litellm metadata params before logging
-        clean_metadata = {}
-        if isinstance(metadata, dict):
-            for key, value in metadata.items():
-                # clean litellm metadata before logging
-                if key in [
-                    "endpoint",
-                    "caching_groups",
-                    "previous_models",
-                ]:
-                    continue
-                else:
-                    clean_metadata[key] = value
+        status = DataDogStatus.INFO
+        if standard_logging_object.get("status") == "failure":
+            status = DataDogStatus.ERROR
 
         # Build the initial payload
-        payload = {
-            "id": id,
-            "call_type": call_type,
-            "cache_hit": cache_hit,
-            "start_time": start_time,
-            "end_time": end_time,
-            "response_time": response_time,
-            "model": kwargs.get("model", ""),
-            "user": kwargs.get("user", ""),
-            "model_parameters": optional_params,
-            "spend": kwargs.get("response_cost", 0),
-            "messages": messages,
-            "response": response_obj,
-            "usage": usage,
-            "metadata": clean_metadata,
-        }
-
-        make_json_serializable(payload)
-        json_payload = json.dumps(payload)
+        make_json_serializable(standard_logging_object)
+        json_payload = json.dumps(standard_logging_object)
 
         verbose_logger.debug("Datadog: Logger - Logging payload = %s", json_payload)
 
         dd_payload = DatadogPayload(
-            ddsource=os.getenv("DD_SOURCE", "litellm"),
-            ddtags="",
-            hostname="",
+            ddsource=self._get_datadog_source(),
+            ddtags=self._get_datadog_tags(),
+            hostname=self._get_datadog_hostname(),
             message=json_payload,
-            service="litellm-server",
-            status=DataDogStatus.INFO,
+            service=self._get_datadog_service(),
+            status=status,
         )
         return dd_payload
 
@@ -382,3 +364,140 @@ class DataDogLogger(CustomBatchLogger):
         No user has asked for this so far, this might be spammy on datatdog. If need arises we can implement this
         """
         return
+
+    async def async_post_call_failure_hook(
+        self,
+        request_data: dict,
+        original_exception: Exception,
+        user_api_key_dict: UserAPIKeyAuth,
+    ):
+        """
+        Handles Proxy Errors (not-related to LLM API), ex: Authentication Errors
+        """
+        import json
+
+        _exception_payload = DatadogProxyFailureHookJsonMessage(
+            exception=str(original_exception),
+            error_class=str(original_exception.__class__.__name__),
+            status_code=getattr(original_exception, "status_code", None),
+            traceback=traceback.format_exc(),
+            user_api_key_dict=user_api_key_dict.model_dump(),
+        )
+
+        json_payload = json.dumps(_exception_payload)
+        verbose_logger.debug("Datadog: Logger - Logging payload = %s", json_payload)
+        dd_payload = DatadogPayload(
+            ddsource=self._get_datadog_source(),
+            ddtags=self._get_datadog_tags(),
+            hostname=self._get_datadog_hostname(),
+            message=json_payload,
+            service=self._get_datadog_service(),
+            status=DataDogStatus.ERROR,
+        )
+
+        self.log_queue.append(dd_payload)
+
+    def _create_v0_logging_payload(
+        self,
+        kwargs: Union[dict, Any],
+        response_obj: Any,
+        start_time: datetime.datetime,
+        end_time: datetime.datetime,
+    ) -> DatadogPayload:
+        """
+        Note: This is our V1 Version of DataDog Logging Payload
+
+
+        (Not Recommended) If you want this to get logged set `litellm.datadog_use_v1 = True`
+        """
+        import json
+
+        litellm_params = kwargs.get("litellm_params", {})
+        metadata = (
+            litellm_params.get("metadata", {}) or {}
+        )  # if litellm_params['metadata'] == None
+        messages = kwargs.get("messages")
+        optional_params = kwargs.get("optional_params", {})
+        call_type = kwargs.get("call_type", "litellm.completion")
+        cache_hit = kwargs.get("cache_hit", False)
+        usage = response_obj["usage"]
+        id = response_obj.get("id", str(uuid.uuid4()))
+        usage = dict(usage)
+        try:
+            response_time = (end_time - start_time).total_seconds() * 1000
+        except Exception:
+            response_time = None
+
+        try:
+            response_obj = dict(response_obj)
+        except Exception:
+            response_obj = response_obj
+
+        # Clean Metadata before logging - never log raw metadata
+        # the raw metadata can contain circular references which leads to infinite recursion
+        # we clean out all extra litellm metadata params before logging
+        clean_metadata = {}
+        if isinstance(metadata, dict):
+            for key, value in metadata.items():
+                # clean litellm metadata before logging
+                if key in [
+                    "endpoint",
+                    "caching_groups",
+                    "previous_models",
+                ]:
+                    continue
+                else:
+                    clean_metadata[key] = value
+
+        # Build the initial payload
+        payload = {
+            "id": id,
+            "call_type": call_type,
+            "cache_hit": cache_hit,
+            "start_time": start_time,
+            "end_time": end_time,
+            "response_time": response_time,
+            "model": kwargs.get("model", ""),
+            "user": kwargs.get("user", ""),
+            "model_parameters": optional_params,
+            "spend": kwargs.get("response_cost", 0),
+            "messages": messages,
+            "response": response_obj,
+            "usage": usage,
+            "metadata": clean_metadata,
+        }
+
+        make_json_serializable(payload)
+        json_payload = json.dumps(payload)
+
+        verbose_logger.debug("Datadog: Logger - Logging payload = %s", json_payload)
+
+        dd_payload = DatadogPayload(
+            ddsource=self._get_datadog_source(),
+            ddtags=self._get_datadog_tags(),
+            hostname=self._get_datadog_hostname(),
+            message=json_payload,
+            service=self._get_datadog_service(),
+            status=DataDogStatus.INFO,
+        )
+        return dd_payload
+
+    @staticmethod
+    def _get_datadog_tags():
+        return f"env:{os.getenv('DD_ENV', 'unknown')},service:{os.getenv('DD_SERVICE', 'litellm')},version:{os.getenv('DD_VERSION', 'unknown')}"
+
+    @staticmethod
+    def _get_datadog_source():
+        return os.getenv("DD_SOURCE", "litellm")
+
+    @staticmethod
+    def _get_datadog_service():
+        return os.getenv("DD_SERVICE", "litellm-server")
+
+    @staticmethod
+    def _get_datadog_hostname():
+        return ""
+
+    @staticmethod
+    def _get_datadog_env():
+        return os.getenv("DD_ENV", "unknown")

litellm/integrations/mlflow.py

@@ -0,0 +1,247 @@
+import json
+import threading
+from typing import Optional
+
+from litellm._logging import verbose_logger
+from litellm.integrations.custom_logger import CustomLogger
+
+
+class MlflowLogger(CustomLogger):
+    def __init__(self):
+        from mlflow.tracking import MlflowClient
+
+        self._client = MlflowClient()
+
+        self._stream_id_to_span = {}
+        self._lock = threading.Lock()  # lock for _stream_id_to_span
+
+    def log_success_event(self, kwargs, response_obj, start_time, end_time):
+        self._handle_success(kwargs, response_obj, start_time, end_time)
+
+    async def async_log_success_event(self, kwargs, response_obj, start_time, end_time):
+        self._handle_success(kwargs, response_obj, start_time, end_time)
+
+    def _handle_success(self, kwargs, response_obj, start_time, end_time):
+        """
+        Log the success event as an MLflow span.
+        Note that this method is called asynchronously in the background thread.
+        """
+        from mlflow.entities import SpanStatusCode
+
+        try:
+            verbose_logger.debug("MLflow logging start for success event")
+
+            if kwargs.get("stream"):
+                self._handle_stream_event(kwargs, response_obj, start_time, end_time)
+            else:
+                span = self._start_span_or_trace(kwargs, start_time)
+                end_time_ns = int(end_time.timestamp() * 1e9)
+                self._end_span_or_trace(
+                    span=span,
+                    outputs=response_obj,
+                    status=SpanStatusCode.OK,
+                    end_time_ns=end_time_ns,
+                )
+        except Exception:
+            verbose_logger.debug("MLflow Logging Error", stack_info=True)
+
+    def log_failure_event(self, kwargs, response_obj, start_time, end_time):
+        self._handle_failure(kwargs, response_obj, start_time, end_time)
+
+    async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time):
+        self._handle_failure(kwargs, response_obj, start_time, end_time)
+
+    def _handle_failure(self, kwargs, response_obj, start_time, end_time):
+        """
+        Log the failure event as an MLflow span.
+        Note that this method is called *synchronously* unlike the success handler.
+        """
+        from mlflow.entities import SpanEvent, SpanStatusCode
+
+        try:
+            span = self._start_span_or_trace(kwargs, start_time)
+
+            end_time_ns = int(end_time.timestamp() * 1e9)
+
+            # Record exception info as event
+            if exception := kwargs.get("exception"):
+                span.add_event(SpanEvent.from_exception(exception))
+
+            self._end_span_or_trace(
+                span=span,
+                outputs=response_obj,
+                status=SpanStatusCode.ERROR,
+                end_time_ns=end_time_ns,
+            )
+
+        except Exception as e:
+            verbose_logger.debug(f"MLflow Logging Error - {e}", stack_info=True)
+
+    def _handle_stream_event(self, kwargs, response_obj, start_time, end_time):
+        """
+        Handle the success event for a streaming response. For streaming calls,
+        log_success_event handle is triggered for every chunk of the stream.
+        We create a single span for the entire stream request as follows:
+
+        1. For the first chunk, start a new span and store it in the map.
+        2. For subsequent chunks, add the chunk as an event to the span.
+        3. For the final chunk, end the span and remove the span from the map.
+        """
+        from mlflow.entities import SpanStatusCode
+
+        litellm_call_id = kwargs.get("litellm_call_id")
+
+        if litellm_call_id not in self._stream_id_to_span:
+            with self._lock:
+                # Check again after acquiring lock
+                if litellm_call_id not in self._stream_id_to_span:
+                    # Start a new span for the first chunk of the stream
+                    span = self._start_span_or_trace(kwargs, start_time)
+                    self._stream_id_to_span[litellm_call_id] = span
+
+        # Add chunk as event to the span
+        span = self._stream_id_to_span[litellm_call_id]
+        self._add_chunk_events(span, response_obj)
+
+        # If this is the final chunk, end the span. The final chunk
+        # has complete_streaming_response that gathers the full response.
+        if final_response := kwargs.get("complete_streaming_response"):
+            end_time_ns = int(end_time.timestamp() * 1e9)
+            self._end_span_or_trace(
+                span=span,
+                outputs=final_response,
+                status=SpanStatusCode.OK,
+                end_time_ns=end_time_ns,
+            )
+
+            # Remove the stream_id from the map
+            with self._lock:
+                self._stream_id_to_span.pop(litellm_call_id)
+
+    def _add_chunk_events(self, span, response_obj):
+        from mlflow.entities import SpanEvent
+
+        try:
+            for choice in response_obj.choices:
+                span.add_event(
+                    SpanEvent(
+                        name="streaming_chunk",
+                        attributes={"delta": json.dumps(choice.delta.model_dump())},
+                    )
+                )
+        except Exception:
+            verbose_logger.debug("Error adding chunk events to span", stack_info=True)
+
+    def _construct_input(self, kwargs):
+        """Construct span inputs with optional parameters"""
+        inputs = {"messages": kwargs.get("messages")}
+        for key in ["functions", "tools", "stream", "tool_choice", "user"]:
+            if value := kwargs.get("optional_params", {}).pop(key, None):
+                inputs[key] = value
+        return inputs
+
+    def _extract_attributes(self, kwargs):
+        """
+        Extract span attributes from kwargs.
+
+        With the latest version of litellm, the standard_logging_object contains
+        canonical information for logging. If it is not present, we extract
+        subset of attributes from other kwargs.
+        """
+        attributes = {
+            "litellm_call_id": kwargs.get("litellm_call_id"),
+            "call_type": kwargs.get("call_type"),
+            "model": kwargs.get("model"),
+        }
+        standard_obj = kwargs.get("standard_logging_object")
+        if standard_obj:
+            attributes.update(
+                {
+                    "api_base": standard_obj.get("api_base"),
+                    "cache_hit": standard_obj.get("cache_hit"),
+                    "usage": {
+                        "completion_tokens": standard_obj.get("completion_tokens"),
+                        "prompt_tokens": standard_obj.get("prompt_tokens"),
+                        "total_tokens": standard_obj.get("total_tokens"),
+                    },
+                    "raw_llm_response": standard_obj.get("response"),
+                    "response_cost": standard_obj.get("response_cost"),
+                    "saved_cache_cost": standard_obj.get("saved_cache_cost"),
+                }
+            )
+        else:
+            litellm_params = kwargs.get("litellm_params", {})
+            attributes.update(
+                {
+                    "model": kwargs.get("model"),
+                    "cache_hit": kwargs.get("cache_hit"),
+                    "custom_llm_provider": kwargs.get("custom_llm_provider"),
+                    "api_base": litellm_params.get("api_base"),
+                    "response_cost": kwargs.get("response_cost"),
+                }
+            )
+        return attributes
+
+    def _get_span_type(self, call_type: Optional[str]) -> str:
+        from mlflow.entities import SpanType
+
+        if call_type in ["completion", "acompletion"]:
+            return SpanType.LLM
+        elif call_type == "embeddings":
+            return SpanType.EMBEDDING
+        else:
+            return SpanType.LLM
+
+    def _start_span_or_trace(self, kwargs, start_time):
+        """
+        Start an MLflow span or a trace.
+
+        If there is an active span, we start a new span as a child of
+        that span. Otherwise, we start a new trace.
+        """
+        import mlflow
+
+        call_type = kwargs.get("call_type", "completion")
+        span_name = f"litellm-{call_type}"
+        span_type = self._get_span_type(call_type)
+        start_time_ns = int(start_time.timestamp() * 1e9)
+
+        inputs = self._construct_input(kwargs)
+        attributes = self._extract_attributes(kwargs)
+
+        if active_span := mlflow.get_current_active_span():  # type: ignore
+            return self._client.start_span(
+                name=span_name,
+                request_id=active_span.request_id,
+                parent_id=active_span.span_id,
+                span_type=span_type,
+                inputs=inputs,
+                attributes=attributes,
+                start_time_ns=start_time_ns,
+            )
+        else:
+            return self._client.start_trace(
+                name=span_name,
+                span_type=span_type,
+                inputs=inputs,
+                attributes=attributes,
+                start_time_ns=start_time_ns,
+            )
+
+    def _end_span_or_trace(self, span, outputs, end_time_ns, status):
+        """End an MLflow span or a trace."""
+        if span.parent_id is None:
+            self._client.end_trace(
+                request_id=span.request_id,
+                outputs=outputs,
+                status=status,
+                end_time_ns=end_time_ns,
+            )
+        else:
+            self._client.end_span(
+                request_id=span.request_id,
+                span_id=span.span_id,
+                outputs=outputs,
+                status=status,
+                end_time_ns=end_time_ns,
+            )

litellm/integrations/prometheus.py

@@ -18,6 +18,7 @@ from litellm.integrations.custom_logger import CustomLogger
 from litellm.proxy._types import UserAPIKeyAuth
 from litellm.types.integrations.prometheus import *
 from litellm.types.utils import StandardLoggingPayload
+from litellm.utils import get_end_user_id_for_cost_tracking
 
 
 class PrometheusLogger(CustomLogger):
@@ -228,6 +229,13 @@ class PrometheusLogger(CustomLogger):
                     "api_key_alias",
                 ],
             )
+            # llm api provider budget metrics
+            self.litellm_provider_remaining_budget_metric = Gauge(
+                "litellm_provider_remaining_budget_metric",
+                "Remaining budget for provider - used when you set provider budget limits",
+                labelnames=["api_provider"],
+            )
+
             # Get all keys
             _logged_llm_labels = [
                 "litellm_model_name",
@@ -357,8 +365,7 @@ class PrometheusLogger(CustomLogger):
         model = kwargs.get("model", "")
         litellm_params = kwargs.get("litellm_params", {}) or {}
         _metadata = litellm_params.get("metadata", {})
-        proxy_server_request = litellm_params.get("proxy_server_request") or {}
-        end_user_id = proxy_server_request.get("body", {}).get("user", None)
+        end_user_id = get_end_user_id_for_cost_tracking(litellm_params)
         user_id = standard_logging_payload["metadata"]["user_api_key_user_id"]
         user_api_key = standard_logging_payload["metadata"]["user_api_key_hash"]
         user_api_key_alias = standard_logging_payload["metadata"]["user_api_key_alias"]
@@ -657,13 +664,11 @@ class PrometheusLogger(CustomLogger):
 
         # unpack kwargs
         model = kwargs.get("model", "")
-        litellm_params = kwargs.get("litellm_params", {}) or {}
         standard_logging_payload: StandardLoggingPayload = kwargs.get(
             "standard_logging_object", {}
         )
-        proxy_server_request = litellm_params.get("proxy_server_request") or {}
-
-        end_user_id = proxy_server_request.get("body", {}).get("user", None)
+        litellm_params = kwargs.get("litellm_params", {}) or {}
+        end_user_id = get_end_user_id_for_cost_tracking(litellm_params)
         user_id = standard_logging_payload["metadata"]["user_api_key_user_id"]
         user_api_key = standard_logging_payload["metadata"]["user_api_key_hash"]
         user_api_key_alias = standard_logging_payload["metadata"]["user_api_key_alias"]
@@ -1130,6 +1135,19 @@ class PrometheusLogger(CustomLogger):
             litellm_model_name, model_id, api_base, api_provider, exception_status
         ).inc()
 
+    def track_provider_remaining_budget(
+        self, provider: str, spend: float, budget_limit: float
+    ):
+        """
+        Track provider remaining budget in Prometheus
+        """
+        self.litellm_provider_remaining_budget_metric.labels(provider).set(
+            self._safe_get_remaining_budget(
+                max_budget=budget_limit,
+                spend=spend,
+            )
+        )
+
     def _safe_get_remaining_budget(
         self, max_budget: Optional[float], spend: Optional[float]
     ) -> float:

litellm/litellm_core_utils/README.md

@@ -8,4 +8,5 @@ Core files:
 - `exception_mapping_utils.py`: utils for mapping exceptions to openai-compatible error types. 
 - `default_encoding.py`: code for loading the default encoding (tiktoken)
 - `get_llm_provider_logic.py`: code for inferring the LLM provider from a given model name. 
+- `duration_parser.py`: code for parsing durations - e.g. "1d", "1mo", "10s"
 

litellm/litellm_core_utils/duration_parser.py

@@ -0,0 +1,92 @@
+"""
+Helper utilities for parsing durations - 1s, 1d, 10d, 30d, 1mo, 2mo
+
+duration_in_seconds is used in diff parts of the code base, example 
+- Router - Provider budget routing
+- Proxy - Key, Team Generation
+"""
+
+import re
+import time
+from datetime import datetime, timedelta
+from typing import Tuple
+
+
+def _extract_from_regex(duration: str) -> Tuple[int, str]:
+    match = re.match(r"(\d+)(mo|[smhd]?)", duration)
+
+    if not match:
+        raise ValueError("Invalid duration format")
+
+    value, unit = match.groups()
+    value = int(value)
+
+    return value, unit
+
+
+def get_last_day_of_month(year, month):
+    # Handle December case
+    if month == 12:
+        return 31
+    # Next month is January, so subtract a day from March 1st
+    next_month = datetime(year=year, month=month + 1, day=1)
+    last_day_of_month = (next_month - timedelta(days=1)).day
+    return last_day_of_month
+
+
+def duration_in_seconds(duration: str) -> int:
+    """
+    Parameters:
+    - duration:
+        - "<number>s" - seconds
+        - "<number>m" - minutes
+        - "<number>h" - hours
+        - "<number>d" - days
+        - "<number>mo" - months
+
+    Returns time in seconds till when budget needs to be reset
+    """
+    value, unit = _extract_from_regex(duration=duration)
+
+    if unit == "s":
+        return value
+    elif unit == "m":
+        return value * 60
+    elif unit == "h":
+        return value * 3600
+    elif unit == "d":
+        return value * 86400
+    elif unit == "mo":
+        now = time.time()
+        current_time = datetime.fromtimestamp(now)
+
+        if current_time.month == 12:
+            target_year = current_time.year + 1
+            target_month = 1
+        else:
+            target_year = current_time.year
+            target_month = current_time.month + value
+
+        # Determine the day to set for next month
+        target_day = current_time.day
+        last_day_of_target_month = get_last_day_of_month(target_year, target_month)
+
+        if target_day > last_day_of_target_month:
+            target_day = last_day_of_target_month
+
+        next_month = datetime(
+            year=target_year,
+            month=target_month,
+            day=target_day,
+            hour=current_time.hour,
+            minute=current_time.minute,
+            second=current_time.second,
+            microsecond=current_time.microsecond,
+        )
+
+        # Calculate the duration until the first day of the next month
+        duration_until_next_month = next_month - current_time
+        return int(duration_until_next_month.total_seconds())
+
+    else:
+        raise ValueError(f"Unsupported duration unit, passed duration: {duration}")

litellm/litellm_core_utils/exception_mapping_utils.py

@@ -239,7 +239,7 @@ def exception_type(  # type: ignore  # noqa: PLR0915
                         message=f"ContextWindowExceededError: {exception_provider} - {message}",
                         llm_provider=custom_llm_provider,
                         model=model,
-                        response=original_exception.response,
+                        response=getattr(original_exception, "response", None),
                         litellm_debug_info=extra_information,
                     )
                 elif (
@@ -251,7 +251,7 @@ def exception_type(  # type: ignore  # noqa: PLR0915
                         message=f"{exception_provider} - {message}",
                         llm_provider=custom_llm_provider,
                         model=model,
-                        response=original_exception.response,
+                        response=getattr(original_exception, "response", None),
                         litellm_debug_info=extra_information,
                     )
                 elif "A timeout occurred" in error_str:
@@ -271,7 +271,7 @@ def exception_type(  # type: ignore  # noqa: PLR0915
                         message=f"ContentPolicyViolationError: {exception_provider} - {message}",
                         llm_provider=custom_llm_provider,
                         model=model,
-                        response=original_exception.response,
+                        response=getattr(original_exception, "response", None),
                         litellm_debug_info=extra_information,
                     )
                 elif (
@@ -283,7 +283,7 @@ def exception_type(  # type: ignore  # noqa: PLR0915
                         message=f"{exception_provider} - {message}",
                         llm_provider=custom_llm_provider,
                         model=model,
-                        response=original_exception.response,
+                        response=getattr(original_exception, "response", None),
                         litellm_debug_info=extra_information,
                     )
                 elif "Web server is returning an unknown error" in error_str:
@@ -299,7 +299,7 @@ def exception_type(  # type: ignore  # noqa: PLR0915
                         message=f"RateLimitError: {exception_provider} - {message}",
                         model=model,
                         llm_provider=custom_llm_provider,
-                        response=original_exception.response,
+                        response=getattr(original_exception, "response", None),
                         litellm_debug_info=extra_information,
                     )
                 elif (
@@ -311,7 +311,7 @@ def exception_type(  # type: ignore  # noqa: PLR0915
                         message=f"AuthenticationError: {exception_provider} - {message}",
                         llm_provider=custom_llm_provider,
                         model=model,
-                        response=original_exception.response,
+                        response=getattr(original_exception, "response", None),
                         litellm_debug_info=extra_information,
                     )
                 elif "Mistral API raised a streaming error" in error_str:
@@ -335,7 +335,7 @@ def exception_type(  # type: ignore  # noqa: PLR0915
                             message=f"{exception_provider} - {message}",
                             llm_provider=custom_llm_provider,
                             model=model,
-                            response=original_exception.response,
+                            response=getattr(original_exception, "response", None),
                             litellm_debug_info=extra_information,
                         )
                     elif original_exception.status_code == 401:
@@ -344,7 +344,7 @@ def exception_type(  # type: ignore  # noqa: PLR0915
                             message=f"AuthenticationError: {exception_provider} - {message}",
                             llm_provider=custom_llm_provider,
                             model=model,
-                            response=original_exception.response,
+                            response=getattr(original_exception, "response", None),
                             litellm_debug_info=extra_information,
                         )
                     elif original_exception.status_code == 404:
@@ -353,7 +353,7 @@ def exception_type(  # type: ignore  # noqa: PLR0915
                             message=f"NotFoundError: {exception_provider} - {message}",
                             model=model,
                             llm_provider=custom_llm_provider,
-                            response=original_exception.response,
+                            response=getattr(original_exception, "response", None),
                             litellm_debug_info=extra_information,
                         )
                     elif original_exception.status_code == 408:
@@ -516,7 +516,7 @@ def exception_type(  # type: ignore  # noqa: PLR0915
                         message=f"ReplicateException - {error_str}",
                         llm_provider="replicate",
                         model=model,
-                        response=original_exception.response,
+                        response=getattr(original_exception, "response", None),
                     )
                 elif "input is too long" in error_str:
                     exception_mapping_worked = True
@@ -524,7 +524,7 @@ def exception_type(  # type: ignore  # noqa: PLR0915
                         message=f"ReplicateException - {error_str}",
                         model=model,
                         llm_provider="replicate",
-                        response=original_exception.response,
+                        response=getattr(original_exception, "response", None),
                     )
                 elif exception_type == "ModelError":
                     exception_mapping_worked = True
@@ -532,7 +532,7 @@ def exception_type(  # type: ignore  # noqa: PLR0915
                         message=f"ReplicateException - {error_str}",
                         model=model,
                         llm_provider="replicate",
-                        response=original_exception.response,
+                        response=getattr(original_exception, "response", None),
                     )
                 elif "Request was throttled" in error_str:
                     exception_mapping_worked = True
@@ -540,7 +540,7 @@ def exception_type(  # type: ignore  # noqa: PLR0915
                         message=f"ReplicateException - {error_str}",
                         llm_provider="replicate",
                         model=model,
-                        response=original_exception.response,
+                        response=getattr(original_exception, "response", None),
                     )
                 elif hasattr(original_exception, "status_code"):
                     if original_exception.status_code == 401:
@@ -549,7 +549,7 @@ def exception_type(  # type: ignore  # noqa: PLR0915
                             message=f"ReplicateException - {original_exception.message}",
                             llm_provider="replicate",
                             model=model,
-                            response=original_exception.response,
+                            response=getattr(original_exception, "response", None),
                         )
                     elif (
                         original_exception.status_code == 400
@@ -560,7 +560,7 @@ def exception_type(  # type: ignore  # noqa: PLR0915
                             message=f"ReplicateException - {original_exception.message}",
                             model=model,
                             llm_provider="replicate",
-                            response=original_exception.response,
+                            response=getattr(original_exception, "response", None),
                         )
                     elif original_exception.status_code == 422:
                         exception_mapping_worked = True
@@ -568,7 +568,7 @@ def exception_type(  # type: ignore  # noqa: PLR0915
                             message=f"ReplicateException - {original_exception.message}",
                             model=model,
                             llm_provider="replicate",
-                            response=original_exception.response,
+                            response=getattr(original_exception, "response", None),
                         )
                     elif original_exception.status_code == 408:
                         exception_mapping_worked = True
@@ -583,7 +583,7 @@ def exception_type(  # type: ignore  # noqa: PLR0915
                             message=f"ReplicateException - {original_exception.message}",
                             llm_provider="replicate",
                             model=model,
-                            response=original_exception.response,
+                            response=getattr(original_exception, "response", None),
                         )
                     elif original_exception.status_code == 429:
                         exception_mapping_worked = True
@@ -591,7 +591,7 @@ def exception_type(  # type: ignore  # noqa: PLR0915
                             message=f"ReplicateException - {original_exception.message}",
                             llm_provider="replicate",
                             model=model,
-                            response=original_exception.response,
+                            response=getattr(original_exception, "response", None),
                         )
                     elif original_exception.status_code == 500:
                         exception_mapping_worked = True
@@ -599,7 +599,7 @@ def exception_type(  # type: ignore  # noqa: PLR0915
                             message=f"ReplicateException - {original_exception.message}",
                             llm_provider="replicate",
                             model=model,
-                            response=original_exception.response,
+                            response=getattr(original_exception, "response", None),
                         )
                 exception_mapping_worked = True
                 raise APIError(
@@ -631,7 +631,7 @@ def exception_type(  # type: ignore  # noqa: PLR0915
                         message=f"{custom_llm_provider}Exception: Authentication Error - {error_str}",
                         llm_provider=custom_llm_provider,
                         model=model,
-                        response=original_exception.response,
+                        response=getattr(original_exception, "response", None),
                         litellm_debug_info=extra_information,
                     )
                 elif "token_quota_reached" in error_str:
@@ -640,7 +640,7 @@ def exception_type(  # type: ignore  # noqa: PLR0915
                         message=f"{custom_llm_provider}Exception: Rate Limit Errror - {error_str}",
                         llm_provider=custom_llm_provider,
                         model=model,
-                        response=original_exception.response,
+                        response=getattr(original_exception, "response", None),
                     )
                 elif (
                     "The server received an invalid response from an upstream server."
@@ -750,7 +750,7 @@ def exception_type(  # type: ignore  # noqa: PLR0915
                         message=f"BedrockException - {error_str}\n. Enable 'litellm.modify_params=True' (for PROXY do: `litellm_settings::modify_params: True`) to insert a dummy assistant message and fix this error.",
                         model=model,
                         llm_provider="bedrock",
-                        response=original_exception.response,
+                        response=getattr(original_exception, "response", None),
                     )
                 elif "Malformed input request" in error_str:
                     exception_mapping_worked = True
@@ -758,7 +758,7 @@ def exception_type(  # type: ignore  # noqa: PLR0915
                         message=f"BedrockException - {error_str}",
                         model=model,
                         llm_provider="bedrock",
-                        response=original_exception.response,
+                        response=getattr(original_exception, "response", None),
                     )
                 elif "A conversation must start with a user message." in error_str:
                     exception_mapping_worked = True
@@ -766,7 +766,7 @@ def exception_type(  # type: ignore  # noqa: PLR0915
                         message=f"BedrockException - {error_str}\n. Pass in default user message via `completion(..,user_continue_message=)` or enable `litellm.modify_params=True`.\nFor Proxy: do via `litellm_settings::modify_params: True` or user_continue_message under `litellm_params`",
                         model=model,
                         llm_provider="bedrock",
-                        response=original_exception.response,
+                        response=getattr(original_exception, "response", None),
                     )
                 elif (
                     "Unable to locate credentials" in error_str
@@ -778,7 +778,7 @@ def exception_type(  # type: ignore  # noqa: PLR0915
                         message=f"BedrockException Invalid Authentication - {error_str}",
                         model=model,
                         llm_provider="bedrock",
-                        response=original_exception.response,
+                        response=getattr(original_exception, "response", None),
                     )
                 elif "AccessDeniedException" in error_str:
                     exception_mapping_worked = True
@@ -786,7 +786,7 @@ def exception_type(  # type: ignore  # noqa: PLR0915
                         message=f"BedrockException PermissionDeniedError - {error_str}",
                         model=model,
                         llm_provider="bedrock",
-                        response=original_exception.response,
+                        response=getattr(original_exception, "response", None),
                     )
                 elif (
                     "throttlingException" in error_str
@@ -797,7 +797,7 @@ def exception_type(  # type: ignore  # noqa: PLR0915
                         message=f"BedrockException: Rate Limit Error - {error_str}",
                         model=model,
                         llm_provider="bedrock",
-                        response=original_exception.response,
+                        response=getattr(original_exception, "response", None),
                     )
                 elif (
                     "Connect timeout on endpoint URL" in error_str
@@ -836,7 +836,7 @@ def exception_type(  # type: ignore  # noqa: PLR0915
                             message=f"BedrockException - {original_exception.message}",
                             llm_provider="bedrock",
                             model=model,
-                            response=original_exception.response,
+                            response=getattr(original_exception, "response", None),
                         )
                     elif original_exception.status_code == 400:
                         exception_mapping_worked = True
@@ -844,7 +844,7 @@ def exception_type(  # type: ignore  # noqa: PLR0915
                             message=f"BedrockException - {original_exception.message}",
                             llm_provider="bedrock",
                             model=model,
-                            response=original_exception.response,
+                            response=getattr(original_exception, "response", None),
                         )
                     elif original_exception.status_code == 404:
                         exception_mapping_worked = True
@@ -852,7 +852,7 @@ def exception_type(  # type: ignore  # noqa: PLR0915
                             message=f"BedrockException - {original_exception.message}",
                             llm_provider="bedrock",
                             model=model,
-                            response=original_exception.response,
+                            response=getattr(original_exception, "response", None),
                         )
                     elif original_exception.status_code == 408:
                         exception_mapping_worked = True
@@ -868,7 +868,7 @@ def exception_type(  # type: ignore  # noqa: PLR0915
                             message=f"BedrockException - {original_exception.message}",
                             model=model,
                             llm_provider=custom_llm_provider,
-                            response=original_exception.response,
+                            response=getattr(original_exception, "response", None),
                             litellm_debug_info=extra_information,
                         )
                     elif original_exception.status_code == 429:
@@ -877,7 +877,7 @@ def exception_type(  # type: ignore  # noqa: PLR0915
                             message=f"BedrockException - {original_exception.message}",
                             model=model,
                             llm_provider=custom_llm_provider,
-                            response=original_exception.response,
+                            response=getattr(original_exception, "response", None),
                             litellm_debug_info=extra_information,
                         )
                     elif original_exception.status_code == 503:
@@ -886,7 +886,7 @@ def exception_type(  # type: ignore  # noqa: PLR0915
                             message=f"BedrockException - {original_exception.message}",
                             model=model,
                             llm_provider=custom_llm_provider,
-                            response=original_exception.response,
+                            response=getattr(original_exception, "response", None),
                             litellm_debug_info=extra_information,
                         )
                     elif original_exception.status_code == 504:  # gateway timeout error
@@ -907,7 +907,7 @@ def exception_type(  # type: ignore  # noqa: PLR0915
                         message=f"litellm.BadRequestError: SagemakerException - {error_str}",
                         model=model,
                         llm_provider="sagemaker",
-                        response=original_exception.response,
+                        response=getattr(original_exception, "response", None),
                     )
                 elif (
                     "Input validation error: `best_of` must be > 0 and <= 2"
@@ -918,7 +918,7 @@ def exception_type(  # type: ignore  # noqa: PLR0915
                         message="SagemakerException - the value of 'n' must be > 0 and <= 2 for sagemaker endpoints",
                         model=model,
                         llm_provider="sagemaker",
-                        response=original_exception.response,
+                        response=getattr(original_exception, "response", None),
                     )
                 elif (
                     "`inputs` tokens + `max_new_tokens` must be <=" in error_str
@@ -929,7 +929,7 @@ def exception_type(  # type: ignore  # noqa: PLR0915
                         message=f"SagemakerException - {error_str}",
                         model=model,
                         llm_provider="sagemaker",
-                        response=original_exception.response,
+                        response=getattr(original_exception, "response", None),
                     )
                 elif hasattr(original_exception, "status_code"):
                     if original_exception.status_code == 500:
@@ -951,7 +951,7 @@ def exception_type(  # type: ignore  # noqa: PLR0915
                             message=f"SagemakerException - {original_exception.message}",
                             llm_provider=custom_llm_provider,
                             model=model,
-                            response=original_exception.response,
+                            response=getattr(original_exception, "response", None),
                         )
                     elif original_exception.status_code == 400:
                         exception_mapping_worked = True
@@ -959,7 +959,7 @@ def exception_type(  # type: ignore  # noqa: PLR0915
                             message=f"SagemakerException - {original_exception.message}",
                             llm_provider=custom_llm_provider,
                             model=model,
-                            response=original_exception.response,
+                            response=getattr(original_exception, "response", None),
                         )
                     elif original_exception.status_code == 404:
                         exception_mapping_worked = True
@@ -967,7 +967,7 @@ def exception_type(  # type: ignore  # noqa: PLR0915
                             message=f"SagemakerException - {original_exception.message}",
                             llm_provider=custom_llm_provider,
                             model=model,
-                            response=original_exception.response,
+                            response=getattr(original_exception, "response", None),
                         )
                     elif original_exception.status_code == 408:
                         exception_mapping_worked = True
@@ -986,7 +986,7 @@ def exception_type(  # type: ignore  # noqa: PLR0915
                             message=f"SagemakerException - {original_exception.message}",
                             model=model,
                             llm_provider=custom_llm_provider,
-                            response=original_exception.response,
+                            response=getattr(original_exception, "response", None),
                             litellm_debug_info=extra_information,
                         )
                     elif original_exception.status_code == 429:
@@ -995,7 +995,7 @@ def exception_type(  # type: ignore  # noqa: PLR0915
                             message=f"SagemakerException - {original_exception.message}",
                             model=model,
                             llm_provider=custom_llm_provider,
-                            response=original_exception.response,
+                            response=getattr(original_exception, "response", None),
                             litellm_debug_info=extra_information,
                         )
                     elif original_exception.status_code == 503:
@@ -1004,7 +1004,7 @@ def exception_type(  # type: ignore  # noqa: PLR0915
                             message=f"SagemakerException - {original_exception.message}",
                             model=model,
                             llm_provider=custom_llm_provider,
-                            response=original_exception.response,
+                            response=getattr(original_exception, "response", None),
                             litellm_debug_info=extra_information,
                         )
                     elif original_exception.status_code == 504:  # gateway timeout error
@@ -1124,10 +1124,13 @@ def exception_type(  # type: ignore  # noqa: PLR0915
                             ),
                         ),
                     )
-                elif "500 Internal Server Error" in error_str:
+                elif (
+                    "500 Internal Server Error" in error_str
+                    or "The model is overloaded." in error_str
+                ):
                     exception_mapping_worked = True
-                    raise ServiceUnavailableError(
-                        message=f"litellm.ServiceUnavailableError: VertexAIException - {error_str}",
+                    raise litellm.InternalServerError(
+                        message=f"litellm.InternalServerError: VertexAIException - {error_str}",
                         model=model,
                         llm_provider="vertex_ai",
                         litellm_debug_info=extra_information,
@@ -1214,7 +1217,7 @@ def exception_type(  # type: ignore  # noqa: PLR0915
                         message="GeminiException - Invalid api key",
                         model=model,
                         llm_provider="palm",
-                        response=original_exception.response,
+                        response=getattr(original_exception, "response", None),
                     )
                 if (
                     "504 Deadline expired before operation could complete." in error_str
@@ -1232,7 +1235,7 @@ def exception_type(  # type: ignore  # noqa: PLR0915
                         message=f"GeminiException - {error_str}",
                         model=model,
                         llm_provider="palm",
-                        response=original_exception.response,
+                        response=getattr(original_exception, "response", None),
                     )
                 if (
                     "500 An internal error has occurred." in error_str
@@ -1259,7 +1262,7 @@ def exception_type(  # type: ignore  # noqa: PLR0915
                             message=f"GeminiException - {error_str}",
                             model=model,
                             llm_provider="palm",
-                            response=original_exception.response,
+                            response=getattr(original_exception, "response", None),
                         )
                 # Dailed: Error occurred: 400 Request payload size exceeds the limit: 20000 bytes
             elif custom_llm_provider == "cloudflare":
@@ -1269,7 +1272,7 @@ def exception_type(  # type: ignore  # noqa: PLR0915
                         message=f"Cloudflare Exception - {original_exception.message}",
                         llm_provider="cloudflare",
                         model=model,
-                        response=original_exception.response,
+                        response=getattr(original_exception, "response", None),
                     )
                 if "must have required property" in error_str:
                     exception_mapping_worked = True
@@ -1277,7 +1280,7 @@ def exception_type(  # type: ignore  # noqa: PLR0915
                         message=f"Cloudflare Exception - {original_exception.message}",
                         llm_provider="cloudflare",
                         model=model,
-                        response=original_exception.response,
+                        response=getattr(original_exception, "response", None),
                     )
             elif (
                 custom_llm_provider == "cohere" or custom_llm_provider == "cohere_chat"
@@ -1291,7 +1294,7 @@ def exception_type(  # type: ignore  # noqa: PLR0915
                         message=f"CohereException - {original_exception.message}",
                         llm_provider="cohere",
                         model=model,
-                        response=original_exception.response,
+                        response=getattr(original_exception, "response", None),
                     )
                 elif "too many tokens" in error_str:
                     exception_mapping_worked = True
@@ -1299,7 +1302,7 @@ def exception_type(  # type: ignore  # noqa: PLR0915
                         message=f"CohereException - {original_exception.message}",
                         model=model,
                         llm_provider="cohere",
-                        response=original_exception.response,
+                        response=getattr(original_exception, "response", None),
                     )
                 elif hasattr(original_exception, "status_code"):
                     if (
@@ -1311,7 +1314,7 @@ def exception_type(  # type: ignore  # noqa: PLR0915
                             message=f"CohereException - {original_exception.message}",
                             llm_provider="cohere",
                             model=model,
-                            response=original_exception.response,
+                            response=getattr(original_exception, "response", None),
                         )
                     elif original_exception.status_code == 408:
                         exception_mapping_worked = True
@@ -1326,7 +1329,7 @@ def exception_type(  # type: ignore  # noqa: PLR0915
                             message=f"CohereException - {original_exception.message}",
                             llm_provider="cohere",
                             model=model,
-                            response=original_exception.response,
+                            response=getattr(original_exception, "response", None),
                         )
                 elif (
                     "CohereConnectionError" in exception_type
@@ -1336,7 +1339,7 @@ def exception_type(  # type: ignore  # noqa: PLR0915
                         message=f"CohereException - {original_exception.message}",
                         llm_provider="cohere",
                         model=model,
-                        response=original_exception.response,
+                        response=getattr(original_exception, "response", None),
                     )
                 elif "invalid type:" in error_str:
                     exception_mapping_worked = True
@@ -1344,7 +1347,7 @@ def exception_type(  # type: ignore  # noqa: PLR0915
                         message=f"CohereException - {original_exception.message}",
                         llm_provider="cohere",
                         model=model,
-                        response=original_exception.response,
+                        response=getattr(original_exception, "response", None),
                     )
                 elif "Unexpected server error" in error_str:
                     exception_mapping_worked = True
@@ -1352,7 +1355,7 @@ def exception_type(  # type: ignore  # noqa: PLR0915
                         message=f"CohereException - {original_exception.message}",
                         llm_provider="cohere",
                         model=model,
-                        response=original_exception.response,
+                        response=getattr(original_exception, "response", None),
                     )
                 else:
                     if hasattr(original_exception, "status_code"):
@@ -1372,7 +1375,7 @@ def exception_type(  # type: ignore  # noqa: PLR0915
                         message=error_str,
                         model=model,
                         llm_provider="huggingface",
-                        response=original_exception.response,
+                        response=getattr(original_exception, "response", None),
                     )
                 elif "A valid user token is required" in error_str:
                     exception_mapping_worked = True
@@ -1380,7 +1383,7 @@ def exception_type(  # type: ignore  # noqa: PLR0915
                         message=error_str,
                         llm_provider="huggingface",
                         model=model,
-                        response=original_exception.response,
+                        response=getattr(original_exception, "response", None),
                     )
                 elif "Rate limit reached" in error_str:
                     exception_mapping_worked = True
@@ -1388,7 +1391,7 @@ def exception_type(  # type: ignore  # noqa: PLR0915
                         message=error_str,
                         llm_provider="huggingface",
                         model=model,
-                        response=original_exception.response,
+                        response=getattr(original_exception, "response", None),
                     )
                 if hasattr(original_exception, "status_code"):
                     if original_exception.status_code == 401:
@@ -1397,7 +1400,7 @@ def exception_type(  # type: ignore  # noqa: PLR0915
                             message=f"HuggingfaceException - {original_exception.message}",
                             llm_provider="huggingface",
                             model=model,
-                            response=original_exception.response,
+                            response=getattr(original_exception, "response", None),
                         )
                     elif original_exception.status_code == 400:
                         exception_mapping_worked = True
@@ -1405,7 +1408,7 @@ def exception_type(  # type: ignore  # noqa: PLR0915
                             message=f"HuggingfaceException - {original_exception.message}",
                             model=model,
                             llm_provider="huggingface",
-                            response=original_exception.response,
+                            response=getattr(original_exception, "response", None),
                         )
                     elif original_exception.status_code == 408:
                         exception_mapping_worked = True
@@ -1420,7 +1423,7 @@ def exception_type(  # type: ignore  # noqa: PLR0915
                             message=f"HuggingfaceException - {original_exception.message}",
                             llm_provider="huggingface",
                             model=model,
-                            response=original_exception.response,
+                            response=getattr(original_exception, "response", None),
                         )
                     elif original_exception.status_code == 503:
                         exception_mapping_worked = True
@@ -1428,7 +1431,7 @@ def exception_type(  # type: ignore  # noqa: PLR0915
                             message=f"HuggingfaceException - {original_exception.message}",
                             llm_provider="huggingface",
                             model=model,
-                            response=original_exception.response,
+                            response=getattr(original_exception, "response", None),
                         )
                     else:
                         exception_mapping_worked = True
@@ -1447,7 +1450,7 @@ def exception_type(  # type: ignore  # noqa: PLR0915
                             message=f"AI21Exception - {original_exception.message}",
                             model=model,
                             llm_provider="ai21",
-                            response=original_exception.response,
+                            response=getattr(original_exception, "response", None),
                         )
                     if "Bad or missing API token." in original_exception.message:
                         exception_mapping_worked = True
@@ -1455,7 +1458,7 @@ def exception_type(  # type: ignore  # noqa: PLR0915
                             message=f"AI21Exception - {original_exception.message}",
                             model=model,
                             llm_provider="ai21",
-                            response=original_exception.response,
+                            response=getattr(original_exception, "response", None),
                         )
                 if hasattr(original_exception, "status_code"):
                     if original_exception.status_code == 401:
@@ -1464,7 +1467,7 @@ def exception_type(  # type: ignore  # noqa: PLR0915
                             message=f"AI21Exception - {original_exception.message}",
                             llm_provider="ai21",
                             model=model,
-                            response=original_exception.response,
+                            response=getattr(original_exception, "response", None),
                         )
                     elif original_exception.status_code == 408:
                         exception_mapping_worked = True
@@ -1479,7 +1482,7 @@ def exception_type(  # type: ignore  # noqa: PLR0915
                             message=f"AI21Exception - {original_exception.message}",
                             model=model,
                             llm_provider="ai21",
-                            response=original_exception.response,
+                            response=getattr(original_exception, "response", None),
                         )
                     elif original_exception.status_code == 429:
                         exception_mapping_worked = True
@@ -1487,7 +1490,7 @@ def exception_type(  # type: ignore  # noqa: PLR0915
                             message=f"AI21Exception - {original_exception.message}",
                             llm_provider="ai21",
                             model=model,
-                            response=original_exception.response,
+                            response=getattr(original_exception, "response", None),
                         )
                     else:
                         exception_mapping_worked = True
@@ -1506,7 +1509,7 @@ def exception_type(  # type: ignore  # noqa: PLR0915
                             message=f"NLPCloudException - {error_str}",
                             model=model,
                             llm_provider="nlp_cloud",
-                            response=original_exception.response,
+                            response=getattr(original_exception, "response", None),
                         )
                     elif "value is not a valid" in error_str:
                         exception_mapping_worked = True
@@ -1514,7 +1517,7 @@ def exception_type(  # type: ignore  # noqa: PLR0915
                             message=f"NLPCloudException - {error_str}",
                             model=model,
                             llm_provider="nlp_cloud",
-                            response=original_exception.response,
+                            response=getattr(original_exception, "response", None),
                         )
                     else:
                         exception_mapping_worked = True
@@ -1539,7 +1542,7 @@ def exception_type(  # type: ignore  # noqa: PLR0915
                             message=f"NLPCloudException - {original_exception.message}",
                             llm_provider="nlp_cloud",
                             model=model,
-                            response=original_exception.response,
+                            response=getattr(original_exception, "response", None),
                         )
                     elif (
                         original_exception.status_code == 401
@@ -1550,7 +1553,7 @@ def exception_type(  # type: ignore  # noqa: PLR0915
                             message=f"NLPCloudException - {original_exception.message}",
                             llm_provider="nlp_cloud",
                             model=model,
-                            response=original_exception.response,
+                            response=getattr(original_exception, "response", None),
                         )
                     elif (
                         original_exception.status_code == 522
@@ -1571,7 +1574,7 @@ def exception_type(  # type: ignore  # noqa: PLR0915
                             message=f"NLPCloudException - {original_exception.message}",
                             llm_provider="nlp_cloud",
                             model=model,
-                            response=original_exception.response,
+                            response=getattr(original_exception, "response", None),
                         )
                     elif (
                         original_exception.status_code == 500
@@ -1594,7 +1597,7 @@ def exception_type(  # type: ignore  # noqa: PLR0915
                             message=f"NLPCloudException - {original_exception.message}",
                             model=model,
                             llm_provider="nlp_cloud",
-                            response=original_exception.response,
+                            response=getattr(original_exception, "response", None),
                         )
                     else:
                         exception_mapping_worked = True
@@ -1620,7 +1623,7 @@ def exception_type(  # type: ignore  # noqa: PLR0915
                         message=f"TogetherAIException - {error_response['error']}",
                         model=model,
                         llm_provider="together_ai",
-                        response=original_exception.response,
+                        response=getattr(original_exception, "response", None),
                     )
                 elif (
                     "error" in error_response
@@ -1631,7 +1634,7 @@ def exception_type(  # type: ignore  # noqa: PLR0915
                         message=f"TogetherAIException - {error_response['error']}",
                         llm_provider="together_ai",
                         model=model,
-                        response=original_exception.response,
+                        response=getattr(original_exception, "response", None),
                     )
                 elif (
                     "error" in error_response
@@ -1642,7 +1645,7 @@ def exception_type(  # type: ignore  # noqa: PLR0915
                         message=f"TogetherAIException - {error_response['error']}",
                         model=model,
                         llm_provider="together_ai",
-                        response=original_exception.response,
+                        response=getattr(original_exception, "response", None),
                     )
                 elif "A timeout occurred" in error_str:
                     exception_mapping_worked = True
@@ -1661,7 +1664,7 @@ def exception_type(  # type: ignore  # noqa: PLR0915
                         message=f"TogetherAIException - {error_response['error']}",
                         model=model,
                         llm_provider="together_ai",
-                        response=original_exception.response,
+                        response=getattr(original_exception, "response", None),
                     )
                 elif (
                     "error_type" in error_response
@@ -1672,7 +1675,7 @@ def exception_type(  # type: ignore  # noqa: PLR0915
                         message=f"TogetherAIException - {error_response['error']}",
                         model=model,
                         llm_provider="together_ai",
-                        response=original_exception.response,
+                        response=getattr(original_exception, "response", None),
                     )
                 if hasattr(original_exception, "status_code"):
                     if original_exception.status_code == 408:
@@ -1688,7 +1691,7 @@ def exception_type(  # type: ignore  # noqa: PLR0915
                             message=f"TogetherAIException - {error_response['error']}",
                             model=model,
                             llm_provider="together_ai",
-                            response=original_exception.response,
+                            response=getattr(original_exception, "response", None),
                         )
                     elif original_exception.status_code == 429:
                         exception_mapping_worked = True
@@ -1696,7 +1699,7 @@ def exception_type(  # type: ignore  # noqa: PLR0915
                             message=f"TogetherAIException - {original_exception.message}",
                             llm_provider="together_ai",
                             model=model,
-                            response=original_exception.response,
+                            response=getattr(original_exception, "response", None),
                         )
                     elif original_exception.status_code == 524:
                         exception_mapping_worked = True
@@ -1724,7 +1727,7 @@ def exception_type(  # type: ignore  # noqa: PLR0915
                         message=f"AlephAlphaException - {original_exception.message}",
                         llm_provider="aleph_alpha",
                         model=model,
-                        response=original_exception.response,
+                        response=getattr(original_exception, "response", None),
                     )
                 elif "InvalidToken" in error_str or "No token provided" in error_str:
                     exception_mapping_worked = True
@@ -1732,7 +1735,7 @@ def exception_type(  # type: ignore  # noqa: PLR0915
                         message=f"AlephAlphaException - {original_exception.message}",
                         llm_provider="aleph_alpha",
                         model=model,
-                        response=original_exception.response,
+                        response=getattr(original_exception, "response", None),
                     )
                 elif hasattr(original_exception, "status_code"):
                     verbose_logger.debug(
@@ -1751,7 +1754,7 @@ def exception_type(  # type: ignore  # noqa: PLR0915
                             message=f"AlephAlphaException - {original_exception.message}",
                             llm_provider="aleph_alpha",
                             model=model,
-                            response=original_exception.response,
+                            response=getattr(original_exception, "response", None),
                         )
                     elif original_exception.status_code == 429:
                         exception_mapping_worked = True
@@ -1759,7 +1762,7 @@ def exception_type(  # type: ignore  # noqa: PLR0915
                             message=f"AlephAlphaException - {original_exception.message}",
                             llm_provider="aleph_alpha",
                             model=model,
-                            response=original_exception.response,
+                            response=getattr(original_exception, "response", None),
                         )
                     elif original_exception.status_code == 500:
                         exception_mapping_worked = True
@@ -1767,7 +1770,7 @@ def exception_type(  # type: ignore  # noqa: PLR0915
                             message=f"AlephAlphaException - {original_exception.message}",
                             llm_provider="aleph_alpha",
                             model=model,
-                            response=original_exception.response,
+                            response=getattr(original_exception, "response", None),
                         )
                     raise original_exception
                 raise original_exception
@@ -1784,7 +1787,7 @@ def exception_type(  # type: ignore  # noqa: PLR0915
                         message=f"OllamaException: Invalid Model/Model not loaded - {original_exception}",
                         model=model,
                         llm_provider="ollama",
-                        response=original_exception.response,
+                        response=getattr(original_exception, "response", None),
                     )
                 elif "Failed to establish a new connection" in error_str:
                     exception_mapping_worked = True
@@ -1792,7 +1795,7 @@ def exception_type(  # type: ignore  # noqa: PLR0915
                         message=f"OllamaException: {original_exception}",
                         llm_provider="ollama",
                         model=model,
-                        response=original_exception.response,
+                        response=getattr(original_exception, "response", None),
                     )
                 elif "Invalid response object from API" in error_str:
                     exception_mapping_worked = True
@@ -1800,7 +1803,7 @@ def exception_type(  # type: ignore  # noqa: PLR0915
                         message=f"OllamaException: {original_exception}",
                         llm_provider="ollama",
                         model=model,
-                        response=original_exception.response,
+                        response=getattr(original_exception, "response", None),
                     )
                 elif "Read timed out" in error_str:
                     exception_mapping_worked = True
@@ -1834,6 +1837,7 @@ def exception_type(  # type: ignore  # noqa: PLR0915
                         llm_provider="azure",
                         model=model,
                         litellm_debug_info=extra_information,
+                        response=getattr(original_exception, "response", None),
                     )
                 elif "This model's maximum context length is" in error_str:
                     exception_mapping_worked = True
@@ -1842,6 +1846,7 @@ def exception_type(  # type: ignore  # noqa: PLR0915
                         llm_provider="azure",
                         model=model,
                         litellm_debug_info=extra_information,
+                        response=getattr(original_exception, "response", None),
                     )
                 elif "DeploymentNotFound" in error_str:
                     exception_mapping_worked = True
@@ -1850,6 +1855,7 @@ def exception_type(  # type: ignore  # noqa: PLR0915
                         llm_provider="azure",
                         model=model,
                         litellm_debug_info=extra_information,
+                        response=getattr(original_exception, "response", None),
                     )
                 elif (
                     (
@@ -1870,6 +1876,7 @@ def exception_type(  # type: ignore  # noqa: PLR0915
                         llm_provider="azure",
                         model=model,
                         litellm_debug_info=extra_information,
+                        response=getattr(original_exception, "response", None),
                     )
                 elif "invalid_request_error" in error_str:
                     exception_mapping_worked = True
@@ -1878,6 +1885,7 @@ def exception_type(  # type: ignore  # noqa: PLR0915
                         llm_provider="azure",
                         model=model,
                         litellm_debug_info=extra_information,
+                        response=getattr(original_exception, "response", None),
                     )
                 elif (
                     "The api_key client option must be set either by passing api_key to the client or by setting"
@@ -1889,6 +1897,7 @@ def exception_type(  # type: ignore  # noqa: PLR0915
                         llm_provider=custom_llm_provider,
                         model=model,
                         litellm_debug_info=extra_information,
+                        response=getattr(original_exception, "response", None),
                     )
                 elif "Connection error" in error_str:
                     exception_mapping_worked = True
@@ -1907,6 +1916,7 @@ def exception_type(  # type: ignore  # noqa: PLR0915
                             llm_provider="azure",
                             model=model,
                             litellm_debug_info=extra_information,
+                            response=getattr(original_exception, "response", None),
                         )
                     elif original_exception.status_code == 401:
                         exception_mapping_worked = True
@@ -1915,6 +1925,7 @@ def exception_type(  # type: ignore  # noqa: PLR0915
                             llm_provider="azure",
                             model=model,
                             litellm_debug_info=extra_information,
+                            response=getattr(original_exception, "response", None),
                         )
                     elif original_exception.status_code == 408:
                         exception_mapping_worked = True
@@ -1931,6 +1942,7 @@ def exception_type(  # type: ignore  # noqa: PLR0915
                             model=model,
                             llm_provider="azure",
                             litellm_debug_info=extra_information,
+                            response=getattr(original_exception, "response", None),
                         )
                     elif original_exception.status_code == 429:
                         exception_mapping_worked = True
@@ -1939,6 +1951,7 @@ def exception_type(  # type: ignore  # noqa: PLR0915
                             model=model,
                             llm_provider="azure",
                             litellm_debug_info=extra_information,
+                            response=getattr(original_exception, "response", None),
                         )
                     elif original_exception.status_code == 503:
                         exception_mapping_worked = True
@@ -1947,6 +1960,7 @@ def exception_type(  # type: ignore  # noqa: PLR0915
                             model=model,
                             llm_provider="azure",
                             litellm_debug_info=extra_information,
+                            response=getattr(original_exception, "response", None),
                         )
                     elif original_exception.status_code == 504:  # gateway timeout error
                         exception_mapping_worked = True
@@ -1986,7 +2000,7 @@ def exception_type(  # type: ignore  # noqa: PLR0915
                             message=f"{exception_provider} - {error_str}",
                             llm_provider=custom_llm_provider,
                             model=model,
-                            response=original_exception.response,
+                            response=getattr(original_exception, "response", None),
                             litellm_debug_info=extra_information,
                         )
                     elif original_exception.status_code == 401:
@@ -1995,7 +2009,7 @@ def exception_type(  # type: ignore  # noqa: PLR0915
                             message=f"AuthenticationError: {exception_provider} - {error_str}",
                             llm_provider=custom_llm_provider,
                             model=model,
-                            response=original_exception.response,
+                            response=getattr(original_exception, "response", None),
                             litellm_debug_info=extra_information,
                         )
                     elif original_exception.status_code == 404:
@@ -2004,7 +2018,7 @@ def exception_type(  # type: ignore  # noqa: PLR0915
                             message=f"NotFoundError: {exception_provider} - {error_str}",
                             model=model,
                             llm_provider=custom_llm_provider,
-                            response=original_exception.response,
+                            response=getattr(original_exception, "response", None),
                             litellm_debug_info=extra_information,
                         )
                     elif original_exception.status_code == 408:
@@ -2021,7 +2035,7 @@ def exception_type(  # type: ignore  # noqa: PLR0915
                             message=f"BadRequestError: {exception_provider} - {error_str}",
                             model=model,
                             llm_provider=custom_llm_provider,
-                            response=original_exception.response,
+                            response=getattr(original_exception, "response", None),
                             litellm_debug_info=extra_information,
                         )
                     elif original_exception.status_code == 429:
@@ -2030,7 +2044,7 @@ def exception_type(  # type: ignore  # noqa: PLR0915
                             message=f"RateLimitError: {exception_provider} - {error_str}",
                             model=model,
                             llm_provider=custom_llm_provider,
-                            response=original_exception.response,
+                            response=getattr(original_exception, "response", None),
                             litellm_debug_info=extra_information,
                         )
                     elif original_exception.status_code == 503:
@@ -2039,7 +2053,7 @@ def exception_type(  # type: ignore  # noqa: PLR0915
                             message=f"ServiceUnavailableError: {exception_provider} - {error_str}",
                             model=model,
                             llm_provider=custom_llm_provider,
-                            response=original_exception.response,
+                            response=getattr(original_exception, "response", None),
                             litellm_debug_info=extra_information,
                         )
                     elif original_exception.status_code == 504:  # gateway timeout error

litellm/litellm_core_utils/get_supported_openai_params.py

@@ -161,17 +161,7 @@ def get_supported_openai_params(  # noqa: PLR0915
     elif custom_llm_provider == "huggingface":
         return litellm.HuggingfaceConfig().get_supported_openai_params()
     elif custom_llm_provider == "together_ai":
-        return [
-            "stream",
-            "temperature",
-            "max_tokens",
-            "top_p",
-            "stop",
-            "frequency_penalty",
-            "tools",
-            "tool_choice",
-            "response_format",
-        ]
+        return litellm.TogetherAIConfig().get_supported_openai_params(model=model)
     elif custom_llm_provider == "ai21":
         return [
             "stream",

litellm/litellm_core_utils/litellm_logging.py

@@ -28,6 +28,7 @@ from litellm.caching.caching_handler import LLMCachingHandler
 from litellm.cost_calculator import _select_model_name_for_cost_calc
 from litellm.integrations.custom_guardrail import CustomGuardrail
 from litellm.integrations.custom_logger import CustomLogger
+from litellm.integrations.mlflow import MlflowLogger
 from litellm.litellm_core_utils.redact_messages import (
     redact_message_input_output_from_custom_logger,
     redact_message_input_output_from_logging,
@@ -201,6 +202,7 @@ class Logging:
         start_time,
         litellm_call_id: str,
         function_id: str,
+        litellm_trace_id: Optional[str] = None,
         dynamic_input_callbacks: Optional[
             List[Union[str, Callable, CustomLogger]]
         ] = None,
@@ -238,6 +240,7 @@ class Logging:
         self.start_time = start_time  # log the call start time
         self.call_type = call_type
         self.litellm_call_id = litellm_call_id
+        self.litellm_trace_id = litellm_trace_id
         self.function_id = function_id
         self.streaming_chunks: List[Any] = []  # for generating complete stream response
         self.sync_streaming_chunks: List[Any] = (
@@ -274,6 +277,11 @@ class Logging:
         self.completion_start_time: Optional[datetime.datetime] = None
         self._llm_caching_handler: Optional[LLMCachingHandler] = None
 
+        self.model_call_details = {
+            "litellm_trace_id": litellm_trace_id,
+            "litellm_call_id": litellm_call_id,
+        }
+
     def process_dynamic_callbacks(self):
         """
         Initializes CustomLogger compatible callbacks in self.dynamic_* callbacks
@@ -381,21 +389,23 @@ class Logging:
         self.logger_fn = litellm_params.get("logger_fn", None)
         verbose_logger.debug(f"self.optional_params: {self.optional_params}")
 
-        self.model_call_details = {
-            "model": self.model,
-            "messages": self.messages,
-            "optional_params": self.optional_params,
-            "litellm_params": self.litellm_params,
-            "start_time": self.start_time,
-            "stream": self.stream,
-            "user": user,
-            "call_type": str(self.call_type),
-            "litellm_call_id": self.litellm_call_id,
-            "completion_start_time": self.completion_start_time,
-            "standard_callback_dynamic_params": self.standard_callback_dynamic_params,
-            **self.optional_params,
-            **additional_params,
-        }
+        self.model_call_details.update(
+            {
+                "model": self.model,
+                "messages": self.messages,
+                "optional_params": self.optional_params,
+                "litellm_params": self.litellm_params,
+                "start_time": self.start_time,
+                "stream": self.stream,
+                "user": user,
+                "call_type": str(self.call_type),
+                "litellm_call_id": self.litellm_call_id,
+                "completion_start_time": self.completion_start_time,
+                "standard_callback_dynamic_params": self.standard_callback_dynamic_params,
+                **self.optional_params,
+                **additional_params,
+            }
+        )
 
         ## check if stream options is set ##  - used by CustomStreamWrapper for easy instrumentation
         if "stream_options" in additional_params:
@@ -554,6 +564,7 @@ class Logging:
                             message=f"Model Call Details pre-call: {details_to_log}",
                             level="info",
                         )
+
                     elif isinstance(callback, CustomLogger):  # custom logger class
                         callback.log_pre_api_call(
                             model=self.model,
@@ -923,19 +934,10 @@ class Logging:
                         status="success",
                     )
                 )
-            if self.dynamic_success_callbacks is not None and isinstance(
-                self.dynamic_success_callbacks, list
-            ):
-                callbacks = self.dynamic_success_callbacks
-                ## keep the internal functions ##
-                for callback in litellm.success_callback:
-                    if (
-                        isinstance(callback, CustomLogger)
-                        and "_PROXY_" in callback.__class__.__name__
-                    ):
-                        callbacks.append(callback)
-            else:
-                callbacks = litellm.success_callback
+            callbacks = get_combined_callback_list(
+                dynamic_success_callbacks=self.dynamic_success_callbacks,
+                global_callbacks=litellm.success_callback,
+            )
 
             ## REDACT MESSAGES ##
             result = redact_message_input_output_from_logging(
@@ -1249,6 +1251,7 @@ class Logging:
                                 end_time=end_time,
                                 print_verbose=print_verbose,
                             )
+
                     if (
                         callback == "openmeter"
                         and self.model_call_details.get("litellm_params", {}).get(
@@ -1356,8 +1359,11 @@ class Logging:
                         and customLogger is not None
                     ):  # custom logger functions
                         print_verbose(
-                            "success callbacks: Running Custom Callback Function"
+                            "success callbacks: Running Custom Callback Function - {}".format(
+                                callback
+                            )
                         )
+
                         customLogger.log_event(
                             kwargs=self.model_call_details,
                             response_obj=result,
@@ -1454,21 +1460,10 @@ class Logging:
                     status="success",
                 )
             )
-        if self.dynamic_async_success_callbacks is not None and isinstance(
-            self.dynamic_async_success_callbacks, list
-        ):
-            callbacks = self.dynamic_async_success_callbacks
-            ## keep the internal functions ##
-            for callback in litellm._async_success_callback:
-                callback_name = ""
-                if isinstance(callback, CustomLogger):
-                    callback_name = callback.__class__.__name__
-                if callable(callback):
-                    callback_name = callback.__name__
-                if "_PROXY_" in callback_name:
-                    callbacks.append(callback)
-        else:
-            callbacks = litellm._async_success_callback
+        callbacks = get_combined_callback_list(
+            dynamic_success_callbacks=self.dynamic_async_success_callbacks,
+            global_callbacks=litellm._async_success_callback,
+        )
 
         result = redact_message_input_output_from_logging(
             model_call_details=(
@@ -1735,21 +1730,10 @@ class Logging:
                 start_time=start_time,
                 end_time=end_time,
             )
-            callbacks = []  # init this to empty incase it's not created
-
-            if self.dynamic_failure_callbacks is not None and isinstance(
-                self.dynamic_failure_callbacks, list
-            ):
-                callbacks = self.dynamic_failure_callbacks
-                ## keep the internal functions ##
-                for callback in litellm.failure_callback:
-                    if (
-                        isinstance(callback, CustomLogger)
-                        and "_PROXY_" in callback.__class__.__name__
-                    ):
-                        callbacks.append(callback)
-            else:
-                callbacks = litellm.failure_callback
+            callbacks = get_combined_callback_list(
+                dynamic_success_callbacks=self.dynamic_failure_callbacks,
+                global_callbacks=litellm.failure_callback,
+            )
 
             result = None  # result sent to all loggers, init this to None incase it's not created
 
@@ -1932,21 +1916,10 @@ class Logging:
             end_time=end_time,
         )
 
-        callbacks = []  # init this to empty incase it's not created
-
-        if self.dynamic_async_failure_callbacks is not None and isinstance(
-            self.dynamic_async_failure_callbacks, list
-        ):
-            callbacks = self.dynamic_async_failure_callbacks
-            ## keep the internal functions ##
-            for callback in litellm._async_failure_callback:
-                if (
-                    isinstance(callback, CustomLogger)
-                    and "_PROXY_" in callback.__class__.__name__
-                ):
-                    callbacks.append(callback)
-        else:
-            callbacks = litellm._async_failure_callback
+        callbacks = get_combined_callback_list(
+            dynamic_success_callbacks=self.dynamic_async_failure_callbacks,
+            global_callbacks=litellm._async_failure_callback,
+        )
 
         result = None  # result sent to all loggers, init this to None incase it's not created
         for callback in callbacks:
@@ -2338,6 +2311,15 @@ def _init_custom_logger_compatible_class(  # noqa: PLR0915
         _in_memory_loggers.append(_otel_logger)
         return _otel_logger  # type: ignore
 
+    elif logging_integration == "mlflow":
+        for callback in _in_memory_loggers:
+            if isinstance(callback, MlflowLogger):
+                return callback  # type: ignore
+
+        _mlflow_logger = MlflowLogger()
+        _in_memory_loggers.append(_mlflow_logger)
+        return _mlflow_logger  # type: ignore
+
 
 def get_custom_logger_compatible_class(
     logging_integration: litellm._custom_logger_compatible_callbacks_literal,
@@ -2439,6 +2421,12 @@ def get_custom_logger_compatible_class(
                 and callback.callback_name == "langtrace"
             ):
                 return callback
+
+    elif logging_integration == "mlflow":
+        for callback in _in_memory_loggers:
+            if isinstance(callback, MlflowLogger):
+                return callback
+
     return None
 
 
@@ -2806,6 +2794,7 @@ def get_standard_logging_object_payload(
 
         payload: StandardLoggingPayload = StandardLoggingPayload(
             id=str(id),
+            trace_id=kwargs.get("litellm_trace_id"),  # type: ignore
             call_type=call_type or "",
             cache_hit=cache_hit,
             status=status,
@@ -2922,3 +2911,11 @@ def modify_integration(integration_name, integration_params):
     if integration_name == "supabase":
         if "table_name" in integration_params:
             Supabase.supabase_table_name = integration_params["table_name"]
+
+
+def get_combined_callback_list(
+    dynamic_success_callbacks: Optional[List], global_callbacks: List
+) -> List:
+    if dynamic_success_callbacks is None:
+        return global_callbacks
+    return list(set(dynamic_success_callbacks + global_callbacks))

litellm/litellm_core_utils/streaming_handler.py

@@ -1793,7 +1793,7 @@ class CustomStreamWrapper:
                 or self.custom_llm_provider == "bedrock"
                 or self.custom_llm_provider == "triton"
                 or self.custom_llm_provider == "watsonx"
-                or self.custom_llm_provider in litellm.openai_compatible_endpoints
+                or self.custom_llm_provider in litellm.openai_compatible_providers
                 or self.custom_llm_provider in litellm._custom_providers
             ):
                 async for chunk in self.completion_stream:

litellm/llms/AzureOpenAI/azure.py

@@ -12,7 +12,11 @@ from typing_extensions import overload
 import litellm
 from litellm.caching.caching import DualCache
 from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
-from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
+from litellm.llms.custom_httpx.http_handler import (
+    AsyncHTTPHandler,
+    HTTPHandler,
+    get_async_httpx_client,
+)
 from litellm.types.utils import EmbeddingResponse
 from litellm.utils import (
     CustomStreamWrapper,
@@ -977,7 +981,10 @@ class AzureChatCompletion(BaseLLM):
             else:
                 _params["timeout"] = httpx.Timeout(timeout=600.0, connect=5.0)
 
-            async_handler = AsyncHTTPHandler(**_params)  # type: ignore
+            async_handler = get_async_httpx_client(
+                llm_provider=litellm.LlmProviders.AZURE,
+                params=_params,
+            )
         else:
             async_handler = client  # type: ignore
 
@@ -1521,7 +1528,8 @@ class AzureChatCompletion(BaseLLM):
         prompt: Optional[str] = None,
     ) -> dict:
         client_session = (
-            litellm.aclient_session or httpx.AsyncClient()
+            litellm.aclient_session
+            or get_async_httpx_client(llm_provider=litellm.LlmProviders.AZURE).client
         )  # handle dall-e-2 calls
 
         if "gateway.ai.cloudflare.com" in api_base:

litellm/llms/OpenAI/chat/o1_handler.py

@@ -17,22 +17,6 @@ from litellm.utils import CustomStreamWrapper
 
 class OpenAIO1ChatCompletion(OpenAIChatCompletion):
 
-    async def mock_async_streaming(
-        self,
-        response: Any,
-        model: Optional[str],
-        logging_obj: Any,
-    ):
-        model_response = await response
-        completion_stream = MockResponseIterator(model_response=model_response)
-        streaming_response = CustomStreamWrapper(
-            completion_stream=completion_stream,
-            model=model,
-            custom_llm_provider="openai",
-            logging_obj=logging_obj,
-        )
-        return streaming_response
-
     def completion(
         self,
         model_response: ModelResponse,
@@ -54,7 +38,7 @@ class OpenAIO1ChatCompletion(OpenAIChatCompletion):
         custom_llm_provider: Optional[str] = None,
         drop_params: Optional[bool] = None,
     ):
-        stream: Optional[bool] = optional_params.pop("stream", False)
+        # stream: Optional[bool] = optional_params.pop("stream", False)
         response = super().completion(
             model_response,
             timeout,
@@ -76,20 +60,4 @@ class OpenAIO1ChatCompletion(OpenAIChatCompletion):
             drop_params,
         )
 
-        if stream is True:
-            if asyncio.iscoroutine(response):
-                return self.mock_async_streaming(
-                    response=response, model=model, logging_obj=logging_obj  # type: ignore
-                )
-
-            completion_stream = MockResponseIterator(model_response=response)
-            streaming_response = CustomStreamWrapper(
-                completion_stream=completion_stream,
-                model=model,
-                custom_llm_provider="openai",
-                logging_obj=logging_obj,
-            )
-
-            return streaming_response
-        else:
-            return response
+        return response

litellm/llms/OpenAI/openai.py

@@ -18,6 +18,7 @@ import litellm
 from litellm import LlmProviders
 from litellm._logging import verbose_logger
 from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
+from litellm.llms.custom_httpx.http_handler import _DEFAULT_TTL_FOR_HTTPX_CLIENTS
 from litellm.secret_managers.main import get_secret_str
 from litellm.types.utils import ProviderField
 from litellm.utils import (
@@ -562,8 +563,9 @@ class OpenAIChatCompletion(BaseLLM):
 
             _cache_key = f"hashed_api_key={hashed_api_key},api_base={api_base},timeout={timeout},max_retries={max_retries},organization={organization},is_async={is_async}"
 
-            if _cache_key in litellm.in_memory_llm_clients_cache:
-                return litellm.in_memory_llm_clients_cache[_cache_key]
+            _cached_client = litellm.in_memory_llm_clients_cache.get_cache(_cache_key)
+            if _cached_client:
+                return _cached_client
             if is_async:
                 _new_client: Union[OpenAI, AsyncOpenAI] = AsyncOpenAI(
                     api_key=api_key,
@@ -584,7 +586,11 @@ class OpenAIChatCompletion(BaseLLM):
                 )
 
             ## SAVE CACHE KEY
-            litellm.in_memory_llm_clients_cache[_cache_key] = _new_client
+            litellm.in_memory_llm_clients_cache.set_cache(
+                key=_cache_key,
+                value=_new_client,
+                ttl=_DEFAULT_TTL_FOR_HTTPX_CLIENTS,
+            )
             return _new_client
 
         else:

litellm/llms/anthropic/chat/handler.py

@@ -44,8 +44,8 @@ from litellm.types.llms.openai import (
     ChatCompletionToolCallFunctionChunk,
     ChatCompletionUsageBlock,
 )
-from litellm.types.utils import GenericStreamingChunk, PromptTokensDetailsWrapper
-from litellm.utils import CustomStreamWrapper, ModelResponse, Usage
+from litellm.types.utils import GenericStreamingChunk
+from litellm.utils import CustomStreamWrapper, ModelResponse
 
 from ...base import BaseLLM
 from ..common_utils import AnthropicError, process_anthropic_headers
@@ -58,6 +58,7 @@ def validate_environment(
     user_headers,
     model,
     messages: List[AllMessageValues],
+    is_vertex_request: bool,
     tools: Optional[List[AllAnthropicToolsValues]],
     anthropic_version: Optional[str] = None,
 ):
@@ -78,6 +79,7 @@ def validate_environment(
         prompt_caching_set=prompt_caching_set,
         pdf_used=pdf_used,
         api_key=api_key,
+        is_vertex_request=is_vertex_request,
     )
 
     if user_headers is not None and isinstance(user_headers, dict):
@@ -94,6 +96,7 @@ async def make_call(
     messages: list,
     logging_obj,
     timeout: Optional[Union[float, httpx.Timeout]],
+    json_mode: bool,
 ) -> Tuple[Any, httpx.Headers]:
     if client is None:
         client = litellm.module_level_aclient
@@ -119,7 +122,9 @@ async def make_call(
         raise AnthropicError(status_code=500, message=str(e))
 
     completion_stream = ModelResponseIterator(
-        streaming_response=response.aiter_lines(), sync_stream=False
+        streaming_response=response.aiter_lines(),
+        sync_stream=False,
+        json_mode=json_mode,
     )
 
     # LOGGING
@@ -142,6 +147,7 @@ def make_sync_call(
     messages: list,
     logging_obj,
     timeout: Optional[Union[float, httpx.Timeout]],
+    json_mode: bool,
 ) -> Tuple[Any, httpx.Headers]:
     if client is None:
         client = litellm.module_level_client  # re-use a module level client
@@ -175,7 +181,7 @@ def make_sync_call(
         )
 
     completion_stream = ModelResponseIterator(
-        streaming_response=response.iter_lines(), sync_stream=True
+        streaming_response=response.iter_lines(), sync_stream=True, json_mode=json_mode
     )
 
     # LOGGING
@@ -193,131 +199,6 @@ class AnthropicChatCompletion(BaseLLM):
     def __init__(self) -> None:
         super().__init__()
 
-    def _process_response(
-        self,
-        model: str,
-        response: Union[requests.Response, httpx.Response],
-        model_response: ModelResponse,
-        stream: bool,
-        logging_obj: litellm.litellm_core_utils.litellm_logging.Logging,  # type: ignore
-        optional_params: dict,
-        api_key: str,
-        data: Union[dict, str],
-        messages: List,
-        print_verbose,
-        encoding,
-        json_mode: bool,
-    ) -> ModelResponse:
-        _hidden_params: Dict = {}
-        _hidden_params["additional_headers"] = process_anthropic_headers(
-            dict(response.headers)
-        )
-        ## LOGGING
-        logging_obj.post_call(
-            input=messages,
-            api_key=api_key,
-            original_response=response.text,
-            additional_args={"complete_input_dict": data},
-        )
-        print_verbose(f"raw model_response: {response.text}")
-        ## RESPONSE OBJECT
-        try:
-            completion_response = response.json()
-        except Exception as e:
-            response_headers = getattr(response, "headers", None)
-            raise AnthropicError(
-                message="Unable to get json response - {}, Original Response: {}".format(
-                    str(e), response.text
-                ),
-                status_code=response.status_code,
-                headers=response_headers,
-            )
-        if "error" in completion_response:
-            response_headers = getattr(response, "headers", None)
-            raise AnthropicError(
-                message=str(completion_response["error"]),
-                status_code=response.status_code,
-                headers=response_headers,
-            )
-        else:
-            text_content = ""
-            tool_calls: List[ChatCompletionToolCallChunk] = []
-            for idx, content in enumerate(completion_response["content"]):
-                if content["type"] == "text":
-                    text_content += content["text"]
-                ## TOOL CALLING
-                elif content["type"] == "tool_use":
-                    tool_calls.append(
-                        ChatCompletionToolCallChunk(
-                            id=content["id"],
-                            type="function",
-                            function=ChatCompletionToolCallFunctionChunk(
-                                name=content["name"],
-                                arguments=json.dumps(content["input"]),
-                            ),
-                            index=idx,
-                        )
-                    )
-
-            _message = litellm.Message(
-                tool_calls=tool_calls,
-                content=text_content or None,
-            )
-
-            ## HANDLE JSON MODE - anthropic returns single function call
-            if json_mode and len(tool_calls) == 1:
-                json_mode_content_str: Optional[str] = tool_calls[0]["function"].get(
-                    "arguments"
-                )
-                if json_mode_content_str is not None:
-                    args = json.loads(json_mode_content_str)
-                    values: Optional[dict] = args.get("values")
-                    if values is not None:
-                        _message = litellm.Message(content=json.dumps(values))
-                        completion_response["stop_reason"] = "stop"
-            model_response.choices[0].message = _message  # type: ignore
-            model_response._hidden_params["original_response"] = completion_response[
-                "content"
-            ]  # allow user to access raw anthropic tool calling response
-
-            model_response.choices[0].finish_reason = map_finish_reason(
-                completion_response["stop_reason"]
-            )
-
-        ## CALCULATING USAGE
-        prompt_tokens = completion_response["usage"]["input_tokens"]
-        completion_tokens = completion_response["usage"]["output_tokens"]
-        _usage = completion_response["usage"]
-        cache_creation_input_tokens: int = 0
-        cache_read_input_tokens: int = 0
-
-        model_response.created = int(time.time())
-        model_response.model = model
-        if "cache_creation_input_tokens" in _usage:
-            cache_creation_input_tokens = _usage["cache_creation_input_tokens"]
-            prompt_tokens += cache_creation_input_tokens
-        if "cache_read_input_tokens" in _usage:
-            cache_read_input_tokens = _usage["cache_read_input_tokens"]
-            prompt_tokens += cache_read_input_tokens
-
-        prompt_tokens_details = PromptTokensDetailsWrapper(
-            cached_tokens=cache_read_input_tokens
-        )
-        total_tokens = prompt_tokens + completion_tokens
-        usage = Usage(
-            prompt_tokens=prompt_tokens,
-            completion_tokens=completion_tokens,
-            total_tokens=total_tokens,
-            prompt_tokens_details=prompt_tokens_details,
-            cache_creation_input_tokens=cache_creation_input_tokens,
-            cache_read_input_tokens=cache_read_input_tokens,
-        )
-
-        setattr(model_response, "usage", usage)  # type: ignore
-
-        model_response._hidden_params = _hidden_params
-        return model_response
-
     async def acompletion_stream_function(
         self,
         model: str,
@@ -334,6 +215,7 @@ class AnthropicChatCompletion(BaseLLM):
         stream,
         _is_function_call,
         data: dict,
+        json_mode: bool,
         optional_params=None,
         litellm_params=None,
         logger_fn=None,
@@ -350,6 +232,7 @@ class AnthropicChatCompletion(BaseLLM):
             messages=messages,
             logging_obj=logging_obj,
             timeout=timeout,
+            json_mode=json_mode,
         )
         streamwrapper = CustomStreamWrapper(
             completion_stream=completion_stream,
@@ -412,7 +295,7 @@ class AnthropicChatCompletion(BaseLLM):
                 headers=error_headers,
             )
 
-        return self._process_response(
+        return AnthropicConfig._process_response(
             model=model,
             response=response,
             model_response=model_response,
@@ -440,30 +323,32 @@ class AnthropicChatCompletion(BaseLLM):
         logging_obj,
         optional_params: dict,
         timeout: Union[float, httpx.Timeout],
+        litellm_params: dict,
         acompletion=None,
-        litellm_params=None,
         logger_fn=None,
         headers={},
         client=None,
     ):
+        optional_params = copy.deepcopy(optional_params)
+        stream = optional_params.pop("stream", None)
+        json_mode: bool = optional_params.pop("json_mode", False)
+        is_vertex_request: bool = optional_params.pop("is_vertex_request", False)
+        _is_function_call = False
+        messages = copy.deepcopy(messages)
         headers = validate_environment(
             api_key,
             headers,
             model,
             messages=messages,
             tools=optional_params.get("tools"),
+            is_vertex_request=is_vertex_request,
         )
-        _is_function_call = False
-        messages = copy.deepcopy(messages)
-        optional_params = copy.deepcopy(optional_params)
-        stream = optional_params.pop("stream", None)
-        json_mode: bool = optional_params.pop("json_mode", False)
-        is_vertex_request: bool = optional_params.pop("is_vertex_request", False)
 
         data = AnthropicConfig()._transform_request(
             model=model,
             messages=messages,
             optional_params=optional_params,
+            litellm_params=litellm_params,
             headers=headers,
             _is_function_call=_is_function_call,
             is_vertex_request=is_vertex_request,
@@ -500,6 +385,7 @@ class AnthropicChatCompletion(BaseLLM):
                     optional_params=optional_params,
                     stream=stream,
                     _is_function_call=_is_function_call,
+                    json_mode=json_mode,
                     litellm_params=litellm_params,
                     logger_fn=logger_fn,
                     headers=headers,
@@ -547,6 +433,7 @@ class AnthropicChatCompletion(BaseLLM):
                     messages=messages,
                     logging_obj=logging_obj,
                     timeout=timeout,
+                    json_mode=json_mode,
                 )
                 return CustomStreamWrapper(
                     completion_stream=completion_stream,
@@ -584,7 +471,7 @@ class AnthropicChatCompletion(BaseLLM):
                         headers=error_headers,
                     )
 
-        return self._process_response(
+        return AnthropicConfig._process_response(
             model=model,
             response=response,
             model_response=model_response,
@@ -605,11 +492,14 @@ class AnthropicChatCompletion(BaseLLM):
 
 
 class ModelResponseIterator:
-    def __init__(self, streaming_response, sync_stream: bool):
+    def __init__(
+        self, streaming_response, sync_stream: bool, json_mode: Optional[bool] = False
+    ):
         self.streaming_response = streaming_response
         self.response_iterator = self.streaming_response
         self.content_blocks: List[ContentBlockDelta] = []
         self.tool_index = -1
+        self.json_mode = json_mode
 
     def check_empty_tool_call_args(self) -> bool:
         """
@@ -771,6 +661,8 @@ class ModelResponseIterator:
                     status_code=500,  # it looks like Anthropic API does not return a status code in the chunk error - default to 500
                 )
 
+            text, tool_use = self._handle_json_mode_chunk(text=text, tool_use=tool_use)
+
             returned_chunk = GenericStreamingChunk(
                 text=text,
                 tool_use=tool_use,
@@ -785,6 +677,34 @@ class ModelResponseIterator:
         except json.JSONDecodeError:
             raise ValueError(f"Failed to decode JSON from chunk: {chunk}")
 
+    def _handle_json_mode_chunk(
+        self, text: str, tool_use: Optional[ChatCompletionToolCallChunk]
+    ) -> Tuple[str, Optional[ChatCompletionToolCallChunk]]:
+        """
+        If JSON mode is enabled, convert the tool call to a message.
+
+        Anthropic returns the JSON schema as part of the tool call
+        OpenAI returns the JSON schema as part of the content, this handles placing it in the content
+
+        Args:
+            text: str
+            tool_use: Optional[ChatCompletionToolCallChunk]
+        Returns:
+            Tuple[str, Optional[ChatCompletionToolCallChunk]]
+
+            text: The text to use in the content
+            tool_use: The ChatCompletionToolCallChunk to use in the chunk response
+        """
+        if self.json_mode is True and tool_use is not None:
+            message = AnthropicConfig._convert_tool_response_to_message(
+                tool_calls=[tool_use]
+            )
+            if message is not None:
+                text = message.content or ""
+                tool_use = None
+
+        return text, tool_use
+
     # Sync iterator
     def __iter__(self):
         return self
@@ -859,3 +779,32 @@ class ModelResponseIterator:
             raise StopAsyncIteration
         except ValueError as e:
             raise RuntimeError(f"Error parsing chunk: {e},\nReceived chunk: {chunk}")
+
+    def convert_str_chunk_to_generic_chunk(self, chunk: str) -> GenericStreamingChunk:
+        """
+        Convert a string chunk to a GenericStreamingChunk
+
+        Note: This is used for Anthropic pass through streaming logging
+
+        We can move __anext__, and __next__ to use this function since it's common logic.
+        Did not migrate them to minmize changes made in 1 PR.
+        """
+        str_line = chunk
+        if isinstance(chunk, bytes):  # Handle binary data
+            str_line = chunk.decode("utf-8")  # Convert bytes to string
+            index = str_line.find("data:")
+            if index != -1:
+                str_line = str_line[index:]
+
+        if str_line.startswith("data:"):
+            data_json = json.loads(str_line[5:])
+            return self.chunk_parser(chunk=data_json)
+        else:
+            return GenericStreamingChunk(
+                text="",
+                is_finished=False,
+                finish_reason="",
+                usage=None,
+                index=0,
+                tool_use=None,
+            )

litellm/llms/anthropic/chat/transformation.py

@@ -1,12 +1,20 @@
+import json
+import time
 import types
-from typing import List, Literal, Optional, Tuple, Union
+from re import A
+from typing import Dict, List, Literal, Optional, Tuple, Union
+
+import httpx
+import requests
 
 import litellm
+from litellm.litellm_core_utils.core_helpers import map_finish_reason
 from litellm.llms.prompt_templates.factory import anthropic_messages_pt
 from litellm.types.llms.anthropic import (
     AllAnthropicToolsValues,
     AnthropicComputerTool,
     AnthropicHostedTools,
+    AnthropicInputSchema,
     AnthropicMessageRequestBase,
     AnthropicMessagesRequest,
     AnthropicMessagesTool,
@@ -17,12 +25,23 @@ from litellm.types.llms.openai import (
     AllMessageValues,
     ChatCompletionCachedContent,
     ChatCompletionSystemMessage,
+    ChatCompletionToolCallChunk,
+    ChatCompletionToolCallFunctionChunk,
     ChatCompletionToolParam,
     ChatCompletionToolParamFunctionChunk,
+    ChatCompletionUsageBlock,
+)
+from litellm.types.utils import Message as LitellmMessage
+from litellm.types.utils import PromptTokensDetailsWrapper
+from litellm.utils import (
+    CustomStreamWrapper,
+    ModelResponse,
+    Usage,
+    add_dummy_tool,
+    has_tool_call_blocks,
 )
-from litellm.utils import add_dummy_tool, has_tool_call_blocks
 
-from ..common_utils import AnthropicError
+from ..common_utils import AnthropicError, process_anthropic_headers
 
 
 class AnthropicConfig:
@@ -90,6 +109,7 @@ class AnthropicConfig:
             "extra_headers",
             "parallel_tool_calls",
             "response_format",
+            "user",
         ]
 
     def get_cache_control_headers(self) -> dict:
@@ -105,6 +125,7 @@ class AnthropicConfig:
         computer_tool_used: bool = False,
         prompt_caching_set: bool = False,
         pdf_used: bool = False,
+        is_vertex_request: bool = False,
     ) -> dict:
         import json
 
@@ -121,8 +142,13 @@ class AnthropicConfig:
             "accept": "application/json",
             "content-type": "application/json",
         }
-        if len(betas) > 0:
+
+        # Don't send any beta headers to Vertex, Vertex has failed requests when they are sent
+        if is_vertex_request is True:
+            pass
+        elif len(betas) > 0:
             headers["anthropic-beta"] = ",".join(betas)
+
         return headers
 
     def _map_tool_choice(
@@ -159,15 +185,17 @@ class AnthropicConfig:
         returned_tool: Optional[AllAnthropicToolsValues] = None
 
         if tool["type"] == "function" or tool["type"] == "custom":
+            _input_schema: dict = tool["function"].get(
+                "parameters",
+                {
+                    "type": "object",
+                    "properties": {},
+                },
+            )
+            input_schema: AnthropicInputSchema = AnthropicInputSchema(**_input_schema)
             _tool = AnthropicMessagesTool(
                 name=tool["function"]["name"],
-                input_schema=tool["function"].get(
-                    "parameters",
-                    {
-                        "type": "object",
-                        "properties": {},
-                    },
-                ),
+                input_schema=input_schema,
             )
 
             _description = tool["function"].get("description")
@@ -243,6 +271,28 @@ class AnthropicConfig:
                 anthropic_tools.append(new_tool)
         return anthropic_tools
 
+    def _map_stop_sequences(
+        self, stop: Optional[Union[str, List[str]]]
+    ) -> Optional[List[str]]:
+        new_stop: Optional[List[str]] = None
+        if isinstance(stop, str):
+            if (
+                stop == "\n"
+            ) and litellm.drop_params is True:  # anthropic doesn't allow whitespace characters as stop-sequences
+                return new_stop
+            new_stop = [stop]
+        elif isinstance(stop, list):
+            new_v = []
+            for v in stop:
+                if (
+                    v == "\n"
+                ) and litellm.drop_params is True:  # anthropic doesn't allow whitespace characters as stop-sequences
+                    continue
+                new_v.append(v)
+            if len(new_v) > 0:
+                new_stop = new_v
+        return new_stop
+
     def map_openai_params(
         self,
         non_default_params: dict,
@@ -268,26 +318,10 @@ class AnthropicConfig:
                     optional_params["tool_choice"] = _tool_choice
             if param == "stream" and value is True:
                 optional_params["stream"] = value
-            if param == "stop":
-                if isinstance(value, str):
-                    if (
-                        value == "\n"
-                    ) and litellm.drop_params is True:  # anthropic doesn't allow whitespace characters as stop-sequences
-                        continue
-                    value = [value]
-                elif isinstance(value, list):
-                    new_v = []
-                    for v in value:
-                        if (
-                            v == "\n"
-                        ) and litellm.drop_params is True:  # anthropic doesn't allow whitespace characters as stop-sequences
-                            continue
-                        new_v.append(v)
-                    if len(new_v) > 0:
-                        value = new_v
-                    else:
-                        continue
-                optional_params["stop_sequences"] = value
+            if param == "stop" and (isinstance(value, str) or isinstance(value, list)):
+                _value = self._map_stop_sequences(value)
+                if _value is not None:
+                    optional_params["stop_sequences"] = _value
             if param == "temperature":
                 optional_params["temperature"] = value
             if param == "top_p":
@@ -304,21 +338,15 @@ class AnthropicConfig:
                 - You should set tool_choice (see Forcing tool use) to instruct the model to explicitly use that tool
                 - Remember that the model will pass the input to the tool, so the name of the tool and description should be from the model’s perspective.
                 """
-                _tool_choice = None
                 _tool_choice = {"name": "json_tool_call", "type": "tool"}
-
-                _tool = AnthropicMessagesTool(
-                    name="json_tool_call",
-                    input_schema={
-                        "type": "object",
-                        "properties": {"values": json_schema},  # type: ignore
-                    },
+                _tool = self._create_json_tool_call_for_response_format(
+                    json_schema=json_schema,
                 )
-
                 optional_params["tools"] = [_tool]
                 optional_params["tool_choice"] = _tool_choice
                 optional_params["json_mode"] = True
-
+            if param == "user":
+                optional_params["metadata"] = {"user_id": value}
         ## VALIDATE REQUEST
         """
         Anthropic doesn't support tool calling without `tools=` param specified.
@@ -341,6 +369,34 @@ class AnthropicConfig:
 
         return optional_params
 
+    def _create_json_tool_call_for_response_format(
+        self,
+        json_schema: Optional[dict] = None,
+    ) -> AnthropicMessagesTool:
+        """
+        Handles creating a tool call for getting responses in JSON format.
+
+        Args:
+            json_schema (Optional[dict]): The JSON schema the response should be in
+
+        Returns:
+            AnthropicMessagesTool: The tool call to send to Anthropic API to get responses in JSON format
+        """
+        _input_schema: AnthropicInputSchema = AnthropicInputSchema(
+            type="object",
+        )
+
+        if json_schema is None:
+            # Anthropic raises a 400 BadRequest error if properties is passed as None
+            # see usage with additionalProperties (Example 5) https://github.com/anthropics/anthropic-cookbook/blob/main/tool_use/extracting_structured_json.ipynb
+            _input_schema["additionalProperties"] = True
+            _input_schema["properties"] = {}
+        else:
+            _input_schema["properties"] = {"values": json_schema}
+
+        _tool = AnthropicMessagesTool(name="json_tool_call", input_schema=_input_schema)
+        return _tool
+
     def is_cache_control_set(self, messages: List[AllMessageValues]) -> bool:
         """
         Return if {"cache_control": ..} in message content block
@@ -371,6 +427,7 @@ class AnthropicConfig:
     def is_pdf_used(self, messages: List[AllMessageValues]) -> bool:
         """
         Set to true if media passed into messages.
+
         """
         for message in messages:
             if (
@@ -441,6 +498,7 @@ class AnthropicConfig:
         model: str,
         messages: List[AllMessageValues],
         optional_params: dict,
+        litellm_params: dict,
         headers: dict,
         _is_function_call: bool,
         is_vertex_request: bool,
@@ -478,6 +536,15 @@ class AnthropicConfig:
         if "tools" in optional_params:
             _is_function_call = True
 
+        ## Handle user_id in metadata
+        _litellm_metadata = litellm_params.get("metadata", None)
+        if (
+            _litellm_metadata
+            and isinstance(_litellm_metadata, dict)
+            and "user_id" in _litellm_metadata
+        ):
+            optional_params["metadata"] = {"user_id": _litellm_metadata["user_id"]}
+
         data = {
             "messages": anthropic_messages,
             **optional_params,
@@ -485,3 +552,162 @@ class AnthropicConfig:
         if not is_vertex_request:
             data["model"] = model
         return data
+
+    @staticmethod
+    def _process_response(
+        model: str,
+        response: Union[requests.Response, httpx.Response],
+        model_response: ModelResponse,
+        stream: bool,
+        logging_obj: litellm.litellm_core_utils.litellm_logging.Logging,  # type: ignore
+        optional_params: dict,
+        api_key: str,
+        data: Union[dict, str],
+        messages: List,
+        print_verbose,
+        encoding,
+        json_mode: bool,
+    ) -> ModelResponse:
+        _hidden_params: Dict = {}
+        _hidden_params["additional_headers"] = process_anthropic_headers(
+            dict(response.headers)
+        )
+        ## LOGGING
+        logging_obj.post_call(
+            input=messages,
+            api_key=api_key,
+            original_response=response.text,
+            additional_args={"complete_input_dict": data},
+        )
+        print_verbose(f"raw model_response: {response.text}")
+        ## RESPONSE OBJECT
+        try:
+            completion_response = response.json()
+        except Exception as e:
+            response_headers = getattr(response, "headers", None)
+            raise AnthropicError(
+                message="Unable to get json response - {}, Original Response: {}".format(
+                    str(e), response.text
+                ),
+                status_code=response.status_code,
+                headers=response_headers,
+            )
+        if "error" in completion_response:
+            response_headers = getattr(response, "headers", None)
+            raise AnthropicError(
+                message=str(completion_response["error"]),
+                status_code=response.status_code,
+                headers=response_headers,
+            )
+        else:
+            text_content = ""
+            tool_calls: List[ChatCompletionToolCallChunk] = []
+            for idx, content in enumerate(completion_response["content"]):
+                if content["type"] == "text":
+                    text_content += content["text"]
+                ## TOOL CALLING
+                elif content["type"] == "tool_use":
+                    tool_calls.append(
+                        ChatCompletionToolCallChunk(
+                            id=content["id"],
+                            type="function",
+                            function=ChatCompletionToolCallFunctionChunk(
+                                name=content["name"],
+                                arguments=json.dumps(content["input"]),
+                            ),
+                            index=idx,
+                        )
+                    )
+
+            _message = litellm.Message(
+                tool_calls=tool_calls,
+                content=text_content or None,
+            )
+
+            ## HANDLE JSON MODE - anthropic returns single function call
+            if json_mode and len(tool_calls) == 1:
+                json_mode_content_str: Optional[str] = tool_calls[0]["function"].get(
+                    "arguments"
+                )
+                if json_mode_content_str is not None:
+                    _converted_message = (
+                        AnthropicConfig._convert_tool_response_to_message(
+                            tool_calls=tool_calls,
+                        )
+                    )
+                    if _converted_message is not None:
+                        completion_response["stop_reason"] = "stop"
+                        _message = _converted_message
+            model_response.choices[0].message = _message  # type: ignore
+            model_response._hidden_params["original_response"] = completion_response[
+                "content"
+            ]  # allow user to access raw anthropic tool calling response
+
+            model_response.choices[0].finish_reason = map_finish_reason(
+                completion_response["stop_reason"]
+            )
+
+        ## CALCULATING USAGE
+        prompt_tokens = completion_response["usage"]["input_tokens"]
+        completion_tokens = completion_response["usage"]["output_tokens"]
+        _usage = completion_response["usage"]
+        cache_creation_input_tokens: int = 0
+        cache_read_input_tokens: int = 0
+
+        model_response.created = int(time.time())
+        model_response.model = model
+        if "cache_creation_input_tokens" in _usage:
+            cache_creation_input_tokens = _usage["cache_creation_input_tokens"]
+            prompt_tokens += cache_creation_input_tokens
+        if "cache_read_input_tokens" in _usage:
+            cache_read_input_tokens = _usage["cache_read_input_tokens"]
+            prompt_tokens += cache_read_input_tokens
+
+        prompt_tokens_details = PromptTokensDetailsWrapper(
+            cached_tokens=cache_read_input_tokens
+        )
+        total_tokens = prompt_tokens + completion_tokens
+        usage = Usage(
+            prompt_tokens=prompt_tokens,
+            completion_tokens=completion_tokens,
+            total_tokens=total_tokens,
+            prompt_tokens_details=prompt_tokens_details,
+            cache_creation_input_tokens=cache_creation_input_tokens,
+            cache_read_input_tokens=cache_read_input_tokens,
+        )
+
+        setattr(model_response, "usage", usage)  # type: ignore
+
+        model_response._hidden_params = _hidden_params
+        return model_response
+
+    @staticmethod
+    def _convert_tool_response_to_message(
+        tool_calls: List[ChatCompletionToolCallChunk],
+    ) -> Optional[LitellmMessage]:
+        """
+        In JSON mode, Anthropic API returns JSON schema as a tool call, we need to convert it to a message to follow the OpenAI format
+
+        """
+        ## HANDLE JSON MODE - anthropic returns single function call
+        json_mode_content_str: Optional[str] = tool_calls[0]["function"].get(
+            "arguments"
+        )
+        try:
+            if json_mode_content_str is not None:
+                args = json.loads(json_mode_content_str)
+                if (
+                    isinstance(args, dict)
+                    and (values := args.get("values")) is not None
+                ):
+                    _message = litellm.Message(content=json.dumps(values))
+                    return _message
+                else:
+                    # a lot of the times the `values` key is not present in the tool response
+                    # relevant issue: https://github.com/BerriAI/litellm/issues/6741
+                    _message = litellm.Message(content=json.dumps(args))
+                    return _message
+        except json.JSONDecodeError:
+            # json decode error does occur, return the original tool response str
+            return litellm.Message(content=json_mode_content_str)
+        return None

litellm/llms/anthropic/completion.py

@@ -13,7 +13,11 @@ import httpx
 import requests
 
 import litellm
-from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
+from litellm.llms.custom_httpx.http_handler import (
+    AsyncHTTPHandler,
+    HTTPHandler,
+    get_async_httpx_client,
+)
 from litellm.utils import CustomStreamWrapper, ModelResponse, Usage
 
 from ..base import BaseLLM
@@ -162,7 +166,10 @@ class AnthropicTextCompletion(BaseLLM):
         client=None,
     ):
         if client is None:
-            client = AsyncHTTPHandler(timeout=httpx.Timeout(timeout=600.0, connect=5.0))
+            client = get_async_httpx_client(
+                llm_provider=litellm.LlmProviders.ANTHROPIC,
+                params={"timeout": httpx.Timeout(timeout=600.0, connect=5.0)},
+            )
 
         response = await client.post(api_base, headers=headers, data=json.dumps(data))
 
@@ -198,7 +205,10 @@ class AnthropicTextCompletion(BaseLLM):
         client=None,
     ):
         if client is None:
-            client = AsyncHTTPHandler(timeout=httpx.Timeout(timeout=600.0, connect=5.0))
+            client = get_async_httpx_client(
+                llm_provider=litellm.LlmProviders.ANTHROPIC,
+                params={"timeout": httpx.Timeout(timeout=600.0, connect=5.0)},
+            )
 
         response = await client.post(api_base, headers=headers, data=json.dumps(data))
 

litellm/llms/azure_ai/chat/transformation.py

@@ -3,7 +3,10 @@ from typing import List, Optional, Tuple
 import litellm
 from litellm._logging import verbose_logger
 from litellm.llms.OpenAI.openai import OpenAIConfig
-from litellm.llms.prompt_templates.common_utils import convert_content_list_to_str
+from litellm.llms.prompt_templates.common_utils import (
+    _audio_or_image_in_message_content,
+    convert_content_list_to_str,
+)
 from litellm.secret_managers.main import get_secret_str
 from litellm.types.llms.openai import AllMessageValues
 from litellm.types.utils import ProviderField
@@ -27,8 +30,21 @@ class AzureAIStudioConfig(OpenAIConfig):
             ),
         ]
 
-    def _transform_messages(self, messages: List[AllMessageValues]) -> List:
+    def _transform_messages(
+        self,
+        messages: List[AllMessageValues],
+    ) -> List:
+        """
+        - Azure AI Studio doesn't support content as a list. This handles:
+            1. Transforms list content to a string.
+            2. If message contains an image or audio, send as is (user-intended)
+        """
         for message in messages:
+
+            # Do nothing if the message contains an image or audio
+            if _audio_or_image_in_message_content(message):
+                continue
+
             texts = convert_content_list_to_str(message=message)
             if texts:
                 message["content"] = texts

litellm/llms/azure_ai/embed/handler.py

@@ -74,7 +74,10 @@ class AzureAIEmbedding(OpenAIChatCompletion):
         client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None,
     ) -> EmbeddingResponse:
         if client is None or not isinstance(client, AsyncHTTPHandler):
-            client = AsyncHTTPHandler(timeout=timeout, concurrent_limit=1)
+            client = get_async_httpx_client(
+                llm_provider=litellm.LlmProviders.AZURE_AI,
+                params={"timeout": timeout},
+            )
 
         url = "{}/images/embeddings".format(api_base)
 

litellm/llms/azure_ai/rerank/handler.py

@@ -4,6 +4,7 @@ import httpx
 
 from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
 from litellm.llms.cohere.rerank import CohereRerank
+from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
 from litellm.types.rerank import RerankResponse
 
 
@@ -73,6 +74,7 @@ class AzureAIRerank(CohereRerank):
         return_documents: Optional[bool] = True,
         max_chunks_per_doc: Optional[int] = None,
         _is_async: Optional[bool] = False,
+        client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None,
     ) -> RerankResponse:
 
         if headers is None:

litellm/llms/bedrock/chat/converse_transformation.py

@@ -458,7 +458,7 @@ class AmazonConverseConfig:
         """
         Abbreviations of regions AWS Bedrock supports for cross region inference
         """
-        return ["us", "eu"]
+        return ["us", "eu", "apac"]
 
     def _get_base_model(self, model: str) -> str:
         """

litellm/llms/bedrock/image/amazon_stability3_transformation.py

@@ -53,9 +53,15 @@ class AmazonStability3Config:
             sd3-medium
             sd3.5-large
             sd3.5-large-turbo
+
+        Stability ultra models
+            stable-image-ultra-v1
         """
-        if model and ("sd3" in model or "sd3.5" in model):
-            return True
+        if model:
+            if "sd3" in model or "sd3.5" in model:
+                return True
+            if "stable-image-ultra-v1" in model:
+                return True
         return False
 
     @classmethod

litellm/llms/clarifai.py

@@ -9,7 +9,10 @@ import httpx
 import requests
 
 import litellm
-from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
+from litellm.llms.custom_httpx.http_handler import (
+    AsyncHTTPHandler,
+    get_async_httpx_client,
+)
 from litellm.utils import Choices, CustomStreamWrapper, Message, ModelResponse, Usage
 
 from .prompt_templates.factory import custom_prompt, prompt_factory
@@ -185,7 +188,10 @@ async def async_completion(
     headers={},
 ):
 
-    async_handler = AsyncHTTPHandler(timeout=httpx.Timeout(timeout=600.0, connect=5.0))
+    async_handler = get_async_httpx_client(
+        llm_provider=litellm.LlmProviders.CLARIFAI,
+        params={"timeout": 600.0},
+    )
     response = await async_handler.post(
         url=model, headers=headers, data=json.dumps(data)
     )

litellm/llms/cohere/embed/handler.py

@@ -11,7 +11,11 @@ import requests  # type: ignore
 
 import litellm
 from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
-from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
+from litellm.llms.custom_httpx.http_handler import (
+    AsyncHTTPHandler,
+    HTTPHandler,
+    get_async_httpx_client,
+)
 from litellm.types.llms.bedrock import CohereEmbeddingRequest
 from litellm.utils import Choices, Message, ModelResponse, Usage
 
@@ -70,8 +74,12 @@ async def async_embedding(
         },
     )
     ## COMPLETION CALL
+
     if client is None:
-        client = AsyncHTTPHandler(concurrent_limit=1, timeout=timeout)
+        client = get_async_httpx_client(
+            llm_provider=litellm.LlmProviders.COHERE,
+            params={"timeout": timeout},
+        )
 
     try:
         response = await client.post(api_base, headers=headers, data=json.dumps(data))
@@ -144,6 +152,11 @@ def embedding(
             api_key=api_key,
             headers=headers,
             encoding=encoding,
+            client=(
+                client
+                if client is not None and isinstance(client, AsyncHTTPHandler)
+                else None
+            ),
         )
 
     ## LOGGING

litellm/llms/cohere/rerank.py

@@ -6,10 +6,14 @@ LiteLLM supports the re rank API format, no paramter transformation occurs
 
 from typing import Any, Dict, List, Optional, Union
 
+import httpx
+
 import litellm
 from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
 from litellm.llms.base import BaseLLM
 from litellm.llms.custom_httpx.http_handler import (
+    AsyncHTTPHandler,
+    HTTPHandler,
     _get_httpx_client,
     get_async_httpx_client,
 )
@@ -34,6 +38,23 @@ class CohereRerank(BaseLLM):
         # Merge other headers, overriding any default ones except Authorization
         return {**default_headers, **headers}
 
+    def ensure_rerank_endpoint(self, api_base: str) -> str:
+        """
+        Ensures the `/v1/rerank` endpoint is appended to the given `api_base`.
+        If `/v1/rerank` is already present, the original URL is returned.
+
+        :param api_base: The base API URL.
+        :return: A URL with `/v1/rerank` appended if missing.
+        """
+        # Parse the base URL to ensure proper structure
+        url = httpx.URL(api_base)
+
+        # Check if the URL already ends with `/v1/rerank`
+        if not url.path.endswith("/v1/rerank"):
+            url = url.copy_with(path=f"{url.path.rstrip('/')}/v1/rerank")
+
+        return str(url)
+
     def rerank(
         self,
         model: str,
@@ -48,9 +69,10 @@ class CohereRerank(BaseLLM):
         return_documents: Optional[bool] = True,
         max_chunks_per_doc: Optional[int] = None,
         _is_async: Optional[bool] = False,  # New parameter
+        client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None,
     ) -> RerankResponse:
         headers = self.validate_environment(api_key=api_key, headers=headers)
-
+        api_base = self.ensure_rerank_endpoint(api_base)
         request_data = RerankRequest(
             model=model,
             query=query,
@@ -76,9 +98,13 @@ class CohereRerank(BaseLLM):
         if _is_async:
             return self.async_rerank(request_data=request_data, api_key=api_key, api_base=api_base, headers=headers)  # type: ignore # Call async method
 
-        client = _get_httpx_client()
+        if client is not None and isinstance(client, HTTPHandler):
+            client = client
+        else:
+            client = _get_httpx_client()
+
         response = client.post(
-            api_base,
+            url=api_base,
             headers=headers,
             json=request_data_dict,
         )
@@ -100,10 +126,13 @@ class CohereRerank(BaseLLM):
         api_key: str,
         api_base: str,
         headers: dict,
+        client: Optional[AsyncHTTPHandler] = None,
     ) -> RerankResponse:
         request_data_dict = request_data.dict(exclude_none=True)
 
-        client = get_async_httpx_client(llm_provider=litellm.LlmProviders.COHERE)
+        client = client or get_async_httpx_client(
+            llm_provider=litellm.LlmProviders.COHERE
+        )
 
         response = await client.post(
             api_base,

litellm/llms/custom_httpx/http_handler.py

@@ -4,11 +4,11 @@ import traceback
 from typing import TYPE_CHECKING, Any, Callable, List, Mapping, Optional, Union
 
 import httpx
-from httpx import USE_CLIENT_DEFAULT
+from httpx import USE_CLIENT_DEFAULT, AsyncHTTPTransport, HTTPTransport
 
 import litellm
-
-from .types import httpxSpecialProvider
+from litellm.caching import InMemoryCache
+from litellm.types.llms.custom_http import *
 
 if TYPE_CHECKING:
     from litellm import LlmProviders
@@ -26,6 +26,63 @@ headers = {
 
 # https://www.python-httpx.org/advanced/timeouts
 _DEFAULT_TIMEOUT = httpx.Timeout(timeout=5.0, connect=5.0)
+_DEFAULT_TTL_FOR_HTTPX_CLIENTS = 3600  # 1 hour, re-use the same httpx client for 1 hour
+
+import re
+
+
+def mask_sensitive_info(error_message):
+    # Find the start of the key parameter
+    if isinstance(error_message, str):
+        key_index = error_message.find("key=")
+    else:
+        return error_message
+
+    # If key is found
+    if key_index != -1:
+        # Find the end of the key parameter (next & or end of string)
+        next_param = error_message.find("&", key_index)
+
+        if next_param == -1:
+            # If no more parameters, mask until the end of the string
+            masked_message = error_message[: key_index + 4] + "[REDACTED_API_KEY]"
+        else:
+            # Replace the key with redacted value, keeping other parameters
+            masked_message = (
+                error_message[: key_index + 4]
+                + "[REDACTED_API_KEY]"
+                + error_message[next_param:]
+            )
+
+        return masked_message
+
+    return error_message
+
+
+class MaskedHTTPStatusError(httpx.HTTPStatusError):
+    def __init__(
+        self, original_error, message: Optional[str] = None, text: Optional[str] = None
+    ):
+        # Create a new error with the masked URL
+        masked_url = mask_sensitive_info(str(original_error.request.url))
+        # Create a new error that looks like the original, but with a masked URL
+
+        super().__init__(
+            message=original_error.message,
+            request=httpx.Request(
+                method=original_error.request.method,
+                url=masked_url,
+                headers=original_error.request.headers,
+                content=original_error.request.content,
+            ),
+            response=httpx.Response(
+                status_code=original_error.response.status_code,
+                content=original_error.response.content,
+                headers=original_error.response.headers,
+            ),
+        )
+        self.message = message
+        self.text = text
 
 
 class AsyncHTTPHandler:
@@ -60,8 +117,10 @@ class AsyncHTTPHandler:
         if timeout is None:
             timeout = _DEFAULT_TIMEOUT
         # Create a client with a connection pool
+        transport = self._create_async_transport()
 
         return httpx.AsyncClient(
+            transport=transport,
             event_hooks=event_hooks,
             timeout=timeout,
             limits=httpx.Limits(
@@ -152,13 +211,16 @@ class AsyncHTTPHandler:
                 headers=headers,
             )
         except httpx.HTTPStatusError as e:
-            setattr(e, "status_code", e.response.status_code)
+
             if stream is True:
                 setattr(e, "message", await e.response.aread())
                 setattr(e, "text", await e.response.aread())
             else:
-                setattr(e, "message", e.response.text)
-                setattr(e, "text", e.response.text)
+                setattr(e, "message", mask_sensitive_info(e.response.text))
+                setattr(e, "text", mask_sensitive_info(e.response.text))
+
+            setattr(e, "status_code", e.response.status_code)
+
             raise e
         except Exception as e:
             raise e
@@ -297,6 +359,18 @@ class AsyncHTTPHandler:
         except Exception:
             pass
 
+    def _create_async_transport(self) -> Optional[AsyncHTTPTransport]:
+        """
+        Create an async transport with IPv4 only if litellm.force_ipv4 is True.
+        Otherwise, return None.
+
+        Some users have seen httpx ConnectionError when using ipv6 - forcing ipv4 resolves the issue for them
+        """
+        if litellm.force_ipv4:
+            return AsyncHTTPTransport(local_address="0.0.0.0")
+        else:
+            return None
+
 
 class HTTPHandler:
     def __init__(
@@ -316,8 +390,11 @@ class HTTPHandler:
         cert = os.getenv("SSL_CERTIFICATE", litellm.ssl_certificate)
 
         if client is None:
+            transport = self._create_sync_transport()
+
             # Create a client with a connection pool
             self.client = httpx.Client(
+                transport=transport,
                 timeout=timeout,
                 limits=httpx.Limits(
                     max_connections=concurrent_limit,
@@ -381,11 +458,17 @@ class HTTPHandler:
                 llm_provider="litellm-httpx-handler",
             )
         except httpx.HTTPStatusError as e:
-            setattr(e, "status_code", e.response.status_code)
+
             if stream is True:
-                setattr(e, "message", e.response.read())
+                setattr(e, "message", mask_sensitive_info(e.response.read()))
+                setattr(e, "text", mask_sensitive_info(e.response.read()))
             else:
-                setattr(e, "message", e.response.text)
+                error_text = mask_sensitive_info(e.response.text)
+                setattr(e, "message", error_text)
+                setattr(e, "text", error_text)
+
+            setattr(e, "status_code", e.response.status_code)
+
             raise e
         except Exception as e:
             raise e
@@ -427,6 +510,18 @@ class HTTPHandler:
         except Exception:
             pass
 
+    def _create_sync_transport(self) -> Optional[HTTPTransport]:
+        """
+        Create an HTTP transport with IPv4 only if litellm.force_ipv4 is True.
+        Otherwise, return None.
+
+        Some users have seen httpx ConnectionError when using ipv6 - forcing ipv4 resolves the issue for them
+        """
+        if litellm.force_ipv4:
+            return HTTPTransport(local_address="0.0.0.0")
+        else:
+            return None
+
 
 def get_async_httpx_client(
     llm_provider: Union[LlmProviders, httpxSpecialProvider],
@@ -447,8 +542,9 @@ def get_async_httpx_client(
                 pass
 
     _cache_key_name = "async_httpx_client" + _params_key_name + llm_provider
-    if _cache_key_name in litellm.in_memory_llm_clients_cache:
-        return litellm.in_memory_llm_clients_cache[_cache_key_name]
+    _cached_client = litellm.in_memory_llm_clients_cache.get_cache(_cache_key_name)
+    if _cached_client:
+        return _cached_client
 
     if params is not None:
         _new_client = AsyncHTTPHandler(**params)
@@ -456,7 +552,11 @@ def get_async_httpx_client(
         _new_client = AsyncHTTPHandler(
             timeout=httpx.Timeout(timeout=600.0, connect=5.0)
         )
-    litellm.in_memory_llm_clients_cache[_cache_key_name] = _new_client
+    litellm.in_memory_llm_clients_cache.set_cache(
+        key=_cache_key_name,
+        value=_new_client,
+        ttl=_DEFAULT_TTL_FOR_HTTPX_CLIENTS,
+    )
     return _new_client
 
 
@@ -476,13 +576,18 @@ def _get_httpx_client(params: Optional[dict] = None) -> HTTPHandler:
                 pass
 
     _cache_key_name = "httpx_client" + _params_key_name
-    if _cache_key_name in litellm.in_memory_llm_clients_cache:
-        return litellm.in_memory_llm_clients_cache[_cache_key_name]
+    _cached_client = litellm.in_memory_llm_clients_cache.get_cache(_cache_key_name)
+    if _cached_client:
+        return _cached_client
 
     if params is not None:
         _new_client = HTTPHandler(**params)
     else:
         _new_client = HTTPHandler(timeout=httpx.Timeout(timeout=600.0, connect=5.0))
 
-    litellm.in_memory_llm_clients_cache[_cache_key_name] = _new_client
+    litellm.in_memory_llm_clients_cache.set_cache(
+        key=_cache_key_name,
+        value=_new_client,
+        ttl=_DEFAULT_TTL_FOR_HTTPX_CLIENTS,
+    )
     return _new_client

litellm/llms/custom_httpx/types.py

@@ -1,10 +0,0 @@
-from enum import Enum
-
-import litellm
-
-
-class httpxSpecialProvider(str, Enum):
-    LoggingCallback = "logging_callback"
-    GuardrailCallback = "guardrail_callback"
-    Caching = "caching"
-    Oauth2Check = "oauth2_check"

litellm/llms/databricks/chat.py

@@ -393,7 +393,10 @@ class DatabricksChatCompletion(BaseLLM):
         if timeout is None:
             timeout = httpx.Timeout(timeout=600.0, connect=5.0)
 
-        self.async_handler = AsyncHTTPHandler(timeout=timeout)
+        self.async_handler = get_async_httpx_client(
+            llm_provider=litellm.LlmProviders.DATABRICKS,
+            params={"timeout": timeout},
+        )
 
         try:
             response = await self.async_handler.post(
@@ -470,6 +473,9 @@ class DatabricksChatCompletion(BaseLLM):
                 optional_params[k] = v
 
         stream: bool = optional_params.get("stream", None) or False
+        optional_params.pop(
+            "max_retries", None
+        )  # [TODO] add max retry support at llm api call level
         optional_params["stream"] = stream
 
         data = {
@@ -607,7 +613,10 @@ class DatabricksChatCompletion(BaseLLM):
         response = None
         try:
             if client is None or isinstance(client, AsyncHTTPHandler):
-                self.async_client = AsyncHTTPHandler(timeout=timeout)  # type: ignore
+                self.async_client = get_async_httpx_client(
+                    llm_provider=litellm.LlmProviders.DATABRICKS,
+                    params={"timeout": timeout},
+                )
             else:
                 self.async_client = client
 

litellm/llms/fine_tuning_apis/vertex_ai.py

@@ -5,9 +5,14 @@ from typing import Any, Coroutine, Literal, Optional, Union
 import httpx
 from openai.types.fine_tuning.fine_tuning_job import FineTuningJob, Hyperparameters
 
+import litellm
 from litellm._logging import verbose_logger
 from litellm.llms.base import BaseLLM
-from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
+from litellm.llms.custom_httpx.http_handler import (
+    AsyncHTTPHandler,
+    HTTPHandler,
+    get_async_httpx_client,
+)
 from litellm.llms.vertex_ai_and_google_ai_studio.gemini.vertex_and_google_ai_studio_gemini import (
     VertexLLM,
 )
@@ -26,8 +31,9 @@ class VertexFineTuningAPI(VertexLLM):
 
     def __init__(self) -> None:
         super().__init__()
-        self.async_handler = AsyncHTTPHandler(
-            timeout=httpx.Timeout(timeout=600.0, connect=5.0)
+        self.async_handler = get_async_httpx_client(
+            llm_provider=litellm.LlmProviders.VERTEX_AI,
+            params={"timeout": 600.0},
         )
 
     def convert_response_created_at(self, response: ResponseTuningJob):

litellm/llms/groq/chat/handler.py

@@ -6,55 +6,68 @@ from typing import Any, Callable, Optional, Union
 
 from httpx._config import Timeout
 
+from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
+from litellm.types.utils import CustomStreamingDecoder
 from litellm.utils import ModelResponse
 
 from ...groq.chat.transformation import GroqChatConfig
-from ...OpenAI.openai import OpenAIChatCompletion
+from ...openai_like.chat.handler import OpenAILikeChatHandler
 
 
-class GroqChatCompletion(OpenAIChatCompletion):
+class GroqChatCompletion(OpenAILikeChatHandler):
     def __init__(self, **kwargs):
         super().__init__(**kwargs)
 
     def completion(
         self,
+        *,
+        model: str,
+        messages: list,
+        api_base: str,
+        custom_llm_provider: str,
+        custom_prompt_dict: dict,
         model_response: ModelResponse,
-        timeout: Union[float, Timeout],
+        print_verbose: Callable,
+        encoding,
+        api_key: Optional[str],
+        logging_obj,
         optional_params: dict,
-        logging_obj: Any,
-        model: Optional[str] = None,
-        messages: Optional[list] = None,
-        print_verbose: Optional[Callable[..., Any]] = None,
-        api_key: Optional[str] = None,
-        api_base: Optional[str] = None,
-        acompletion: bool = False,
+        acompletion=None,
         litellm_params=None,
         logger_fn=None,
         headers: Optional[dict] = None,
-        custom_prompt_dict: dict = {},
-        client=None,
-        organization: Optional[str] = None,
-        custom_llm_provider: Optional[str] = None,
-        drop_params: Optional[bool] = None,
+        timeout: Optional[Union[float, Timeout]] = None,
+        client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None,
+        custom_endpoint: Optional[bool] = None,
+        streaming_decoder: Optional[CustomStreamingDecoder] = None,
+        fake_stream: bool = False
     ):
         messages = GroqChatConfig()._transform_messages(messages)  # type: ignore
+
+        if optional_params.get("stream") is True:
+            fake_stream = GroqChatConfig()._should_fake_stream(optional_params)
+        else:
+            fake_stream = False
+
         return super().completion(
-            model_response,
-            timeout,
-            optional_params,
-            logging_obj,
-            model,
-            messages,
-            print_verbose,
-            api_key,
-            api_base,
-            acompletion,
-            litellm_params,
-            logger_fn,
-            headers,
-            custom_prompt_dict,
-            client,
-            organization,
-            custom_llm_provider,
-            drop_params,
+            model=model,
+            messages=messages,
+            api_base=api_base,
+            custom_llm_provider=custom_llm_provider,
+            custom_prompt_dict=custom_prompt_dict,
+            model_response=model_response,
+            print_verbose=print_verbose,
+            encoding=encoding,
+            api_key=api_key,
+            logging_obj=logging_obj,
+            optional_params=optional_params,
+            acompletion=acompletion,
+            litellm_params=litellm_params,
+            logger_fn=logger_fn,
+            headers=headers,
+            timeout=timeout,
+            client=client,
+            custom_endpoint=custom_endpoint,
+            streaming_decoder=streaming_decoder,
+            fake_stream=fake_stream,
         )

litellm/llms/groq/chat/transformation.py

@@ -2,6 +2,7 @@
 Translate from OpenAI's `/v1/chat/completions` to Groq's `/v1/chat/completions`
 """
 
+import json
 import types
 from typing import List, Optional, Tuple, Union
 
@@ -9,7 +10,12 @@ from pydantic import BaseModel
 
 import litellm
 from litellm.secret_managers.main import get_secret_str
-from litellm.types.llms.openai import AllMessageValues, ChatCompletionAssistantMessage
+from litellm.types.llms.openai import (
+    AllMessageValues,
+    ChatCompletionAssistantMessage,
+    ChatCompletionToolParam,
+    ChatCompletionToolParamFunctionChunk,
+)
 
 from ...OpenAI.chat.gpt_transformation import OpenAIGPTConfig
 
@@ -99,3 +105,69 @@ class GroqChatConfig(OpenAIGPTConfig):
         )  # type: ignore
         dynamic_api_key = api_key or get_secret_str("GROQ_API_KEY")
         return api_base, dynamic_api_key
+
+    def _should_fake_stream(self, optional_params: dict) -> bool:
+        """
+        Groq doesn't support 'response_format' while streaming
+        """
+        if optional_params.get("response_format") is not None:
+            return True
+
+        return False
+
+    def _create_json_tool_call_for_response_format(
+        self,
+        json_schema: dict,
+    ):
+        """
+        Handles creating a tool call for getting responses in JSON format.
+
+        Args:
+            json_schema (Optional[dict]): The JSON schema the response should be in
+
+        Returns:
+            AnthropicMessagesTool: The tool call to send to Anthropic API to get responses in JSON format
+        """
+        return ChatCompletionToolParam(
+            type="function",
+            function=ChatCompletionToolParamFunctionChunk(
+                name="json_tool_call",
+                parameters=json_schema,
+            ),
+        )
+
+    def map_openai_params(
+        self,
+        non_default_params: dict,
+        optional_params: dict,
+        model: str,
+        drop_params: bool = False,
+    ) -> dict:
+        _response_format = non_default_params.get("response_format")
+        if _response_format is not None and isinstance(_response_format, dict):
+            json_schema: Optional[dict] = None
+            if "response_schema" in _response_format:
+                json_schema = _response_format["response_schema"]
+            elif "json_schema" in _response_format:
+                json_schema = _response_format["json_schema"]["schema"]
+            """
+            When using tools in this way: - https://docs.anthropic.com/en/docs/build-with-claude/tool-use#json-mode
+            - You usually want to provide a single tool
+            - You should set tool_choice (see Forcing tool use) to instruct the model to explicitly use that tool
+            - Remember that the model will pass the input to the tool, so the name of the tool and description should be from the model’s perspective.
+            """
+            if json_schema is not None:
+                _tool_choice = {
+                    "type": "function",
+                    "function": {"name": "json_tool_call"},
+                }
+                _tool = self._create_json_tool_call_for_response_format(
+                    json_schema=json_schema,
+                )
+                optional_params["tools"] = [_tool]
+                optional_params["tool_choice"] = _tool_choice
+                optional_params["json_mode"] = True
+            non_default_params.pop("response_format", None)
+        return super().map_openai_params(
+            non_default_params, optional_params, model, drop_params
+        )

litellm/llms/huggingface_restapi.py

@@ -263,7 +263,11 @@ def get_hf_task_for_model(model: str) -> Tuple[hf_tasks, str]:
         return "text-generation-inference", model  # default to tgi
 
 
-from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
+from litellm.llms.custom_httpx.http_handler import (
+    AsyncHTTPHandler,
+    HTTPHandler,
+    get_async_httpx_client,
+)
 
 
 def get_hf_task_embedding_for_model(
@@ -301,7 +305,9 @@ async def async_get_hf_task_embedding_for_model(
                     task_type, hf_tasks_embeddings
                 )
             )
-    http_client = AsyncHTTPHandler(concurrent_limit=1)
+    http_client = get_async_httpx_client(
+        llm_provider=litellm.LlmProviders.HUGGINGFACE,
+    )
 
     model_info = await http_client.get(url=api_base)
 
@@ -1067,7 +1073,9 @@ class Huggingface(BaseLLM):
         )
         ## COMPLETION CALL
         if client is None:
-            client = AsyncHTTPHandler(concurrent_limit=1)
+            client = get_async_httpx_client(
+                llm_provider=litellm.LlmProviders.HUGGINGFACE,
+            )
 
         response = await client.post(api_base, headers=headers, data=json.dumps(data))
 

litellm/llms/jina_ai/embedding/transformation.py

@@ -76,4 +76,4 @@ class JinaAIEmbeddingConfig:
             or get_secret_str("JINA_AI_API_KEY")
             or get_secret_str("JINA_AI_TOKEN")
         )
-        return LlmProviders.OPENAI_LIKE.value, api_base, dynamic_api_key
+        return LlmProviders.JINA_AI.value, api_base, dynamic_api_key

litellm/llms/jina_ai/rerank/handler.py

@@ -0,0 +1,96 @@
+"""
+Re rank api
+
+LiteLLM supports the re rank API format, no paramter transformation occurs
+"""
+
+import uuid
+from typing import Any, Dict, List, Optional, Union
+
+import httpx
+from pydantic import BaseModel
+
+import litellm
+from litellm.llms.base import BaseLLM
+from litellm.llms.custom_httpx.http_handler import (
+    _get_httpx_client,
+    get_async_httpx_client,
+)
+from litellm.llms.jina_ai.rerank.transformation import JinaAIRerankConfig
+from litellm.types.rerank import RerankRequest, RerankResponse
+
+
+class JinaAIRerank(BaseLLM):
+    def rerank(
+        self,
+        model: str,
+        api_key: str,
+        query: str,
+        documents: List[Union[str, Dict[str, Any]]],
+        top_n: Optional[int] = None,
+        rank_fields: Optional[List[str]] = None,
+        return_documents: Optional[bool] = True,
+        max_chunks_per_doc: Optional[int] = None,
+        _is_async: Optional[bool] = False,
+    ) -> RerankResponse:
+        client = _get_httpx_client()
+
+        request_data = RerankRequest(
+            model=model,
+            query=query,
+            top_n=top_n,
+            documents=documents,
+            rank_fields=rank_fields,
+            return_documents=return_documents,
+        )
+
+        # exclude None values from request_data
+        request_data_dict = request_data.dict(exclude_none=True)
+
+        if _is_async:
+            return self.async_rerank(request_data_dict, api_key)  # type: ignore # Call async method
+
+        response = client.post(
+            "https://api.jina.ai/v1/rerank",
+            headers={
+                "accept": "application/json",
+                "content-type": "application/json",
+                "authorization": f"Bearer {api_key}",
+            },
+            json=request_data_dict,
+        )
+
+        if response.status_code != 200:
+            raise Exception(response.text)
+
+        _json_response = response.json()
+
+        return JinaAIRerankConfig()._transform_response(_json_response)
+
+    async def async_rerank(  # New async method
+        self,
+        request_data_dict: Dict[str, Any],
+        api_key: str,
+    ) -> RerankResponse:
+        client = get_async_httpx_client(
+            llm_provider=litellm.LlmProviders.JINA_AI
+        )  # Use async client
+
+        response = await client.post(
+            "https://api.jina.ai/v1/rerank",
+            headers={
+                "accept": "application/json",
+                "content-type": "application/json",
+                "authorization": f"Bearer {api_key}",
+            },
+            json=request_data_dict,
+        )
+
+        if response.status_code != 200:
+            raise Exception(response.text)
+
+        _json_response = response.json()
+
+        return JinaAIRerankConfig()._transform_response(_json_response)
+
+    pass

litellm/llms/jina_ai/rerank/transformation.py

@@ -0,0 +1,36 @@
+"""
+Transformation logic from Cohere's /v1/rerank format to Jina AI's  `/v1/rerank` format. 
+
+Why separate file? Make it easy to see how transformation works
+
+Docs - https://jina.ai/reranker
+"""
+
+import uuid
+from typing import List, Optional
+
+from litellm.types.rerank import (
+    RerankBilledUnits,
+    RerankResponse,
+    RerankResponseMeta,
+    RerankTokens,
+)
+
+
+class JinaAIRerankConfig:
+    def _transform_response(self, response: dict) -> RerankResponse:
+
+        _billed_units = RerankBilledUnits(**response.get("usage", {}))
+        _tokens = RerankTokens(**response.get("usage", {}))
+        rerank_meta = RerankResponseMeta(billed_units=_billed_units, tokens=_tokens)
+
+        _results: Optional[List[dict]] = response.get("results")
+
+        if _results is None:
+            raise ValueError(f"No results found in the response={response}")
+
+        return RerankResponse(
+            id=response.get("id") or str(uuid.uuid4()),
+            results=_results,
+            meta=rerank_meta,
+        )  # Return response

litellm/llms/lm_studio/embed/transformation.py

@@ -0,0 +1,54 @@
+"""
+Transformation logic from OpenAI /v1/embeddings format to LM Studio's  `/v1/embeddings` format. 
+
+Why separate file? Make it easy to see how transformation works
+
+Docs - https://lmstudio.ai/docs/basics/server
+"""
+
+import types
+from typing import List, Optional, Tuple
+
+from litellm import LlmProviders
+from litellm.secret_managers.main import get_secret_str
+from litellm.types.utils import Embedding, EmbeddingResponse, Usage
+
+
+class LmStudioEmbeddingConfig:
+    """
+    Reference: https://lmstudio.ai/docs/basics/server
+    """
+
+    def __init__(
+        self,
+    ) -> None:
+        locals_ = locals()
+        for key, value in locals_.items():
+            if key != "self" and value is not None:
+                setattr(self.__class__, key, value)
+
+    @classmethod
+    def get_config(cls):
+        return {
+            k: v
+            for k, v in cls.__dict__.items()
+            if not k.startswith("__")
+            and not isinstance(
+                v,
+                (
+                    types.FunctionType,
+                    types.BuiltinFunctionType,
+                    classmethod,
+                    staticmethod,
+                ),
+            )
+            and v is not None
+        }
+
+    def get_supported_openai_params(self) -> List[str]:
+        return []
+
+    def map_openai_params(
+        self, non_default_params: dict, optional_params: dict
+    ) -> dict:
+        return optional_params

litellm/llms/ollama.py

@@ -14,6 +14,7 @@ import requests  # type: ignore
 
 import litellm
 from litellm import verbose_logger
+from litellm.llms.custom_httpx.http_handler import get_async_httpx_client
 from litellm.secret_managers.main import get_secret_str
 from litellm.types.utils import ModelInfo, ProviderField, StreamingChoices
 
@@ -164,6 +165,30 @@ class OllamaConfig:
             "response_format",
         ]
 
+    def map_openai_params(
+        self, optional_params: dict, non_default_params: dict
+    ) -> dict:
+        for param, value in non_default_params.items():
+            if param == "max_tokens":
+                optional_params["num_predict"] = value
+            if param == "stream":
+                optional_params["stream"] = value
+            if param == "temperature":
+                optional_params["temperature"] = value
+            if param == "seed":
+                optional_params["seed"] = value
+            if param == "top_p":
+                optional_params["top_p"] = value
+            if param == "frequency_penalty":
+                optional_params["repeat_penalty"] = value
+            if param == "stop":
+                optional_params["stop"] = value
+            if param == "response_format" and isinstance(value, dict):
+                if value["type"] == "json_object":
+                    optional_params["format"] = "json"
+
+        return optional_params
+
     def _supports_function_calling(self, ollama_model_info: dict) -> bool:
         """
         Check if the 'template' field in the ollama_model_info contains a 'tools' or 'function' key.
@@ -185,6 +210,8 @@ class OllamaConfig:
           "name": "mistral"
         }'
         """
+        if model.startswith("ollama/") or model.startswith("ollama_chat/"):
+            model = model.split("/", 1)[1]
         api_base = get_secret_str("OLLAMA_API_BASE") or "http://localhost:11434"
 
         try:
@@ -430,7 +457,10 @@ def ollama_completion_stream(url, data, logging_obj):
 
 async def ollama_async_streaming(url, data, model_response, encoding, logging_obj):
     try:
-        client = httpx.AsyncClient()
+        _async_http_client = get_async_httpx_client(
+            llm_provider=litellm.LlmProviders.OLLAMA
+        )
+        client = _async_http_client.client
         async with client.stream(
             url=f"{url}", json=data, method="POST", timeout=litellm.request_timeout
         ) as response:

litellm/llms/ollama_chat.py

@@ -13,6 +13,7 @@ from pydantic import BaseModel
 
 import litellm
 from litellm import verbose_logger
+from litellm.llms.custom_httpx.http_handler import get_async_httpx_client
 from litellm.types.llms.ollama import OllamaToolCall, OllamaToolCallFunction
 from litellm.types.llms.openai import ChatCompletionAssistantToolCall
 from litellm.types.utils import StreamingChoices
@@ -445,7 +446,10 @@ async def ollama_async_streaming(
     url, api_key, data, model_response, encoding, logging_obj
 ):
     try:
-        client = httpx.AsyncClient()
+        _async_http_client = get_async_httpx_client(
+            llm_provider=litellm.LlmProviders.OLLAMA
+        )
+        client = _async_http_client.client
         _request = {
             "url": f"{url}",
             "json": data,

litellm/llms/openai_like/chat/handler.py

@@ -17,7 +17,9 @@ import httpx  # type: ignore
 import requests  # type: ignore
 
 import litellm
+from litellm import LlmProviders
 from litellm.litellm_core_utils.core_helpers import map_finish_reason
+from litellm.llms.bedrock.chat.invoke_handler import MockResponseIterator
 from litellm.llms.custom_httpx.http_handler import (
     AsyncHTTPHandler,
     HTTPHandler,
@@ -25,9 +27,19 @@ from litellm.llms.custom_httpx.http_handler import (
 )
 from litellm.llms.databricks.streaming_utils import ModelResponseIterator
 from litellm.types.utils import CustomStreamingDecoder, ModelResponse
-from litellm.utils import CustomStreamWrapper, EmbeddingResponse
+from litellm.utils import (
+    Choices,
+    CustomStreamWrapper,
+    EmbeddingResponse,
+    Message,
+    ProviderConfigManager,
+    TextCompletionResponse,
+    Usage,
+    convert_to_model_response_object,
+)
 
 from ..common_utils import OpenAILikeBase, OpenAILikeError
+from .transformation import OpenAILikeChatConfig
 
 
 async def make_call(
@@ -39,16 +51,22 @@ async def make_call(
     messages: list,
     logging_obj,
     streaming_decoder: Optional[CustomStreamingDecoder] = None,
+    fake_stream: bool = False,
 ):
     if client is None:
         client = litellm.module_level_aclient
 
-    response = await client.post(api_base, headers=headers, data=data, stream=True)
+    response = await client.post(
+        api_base, headers=headers, data=data, stream=not fake_stream
+    )
 
     if streaming_decoder is not None:
         completion_stream: Any = streaming_decoder.aiter_bytes(
             response.aiter_bytes(chunk_size=1024)
         )
+    elif fake_stream:
+        model_response = ModelResponse(**response.json())
+        completion_stream = MockResponseIterator(model_response=model_response)
     else:
         completion_stream = ModelResponseIterator(
             streaming_response=response.aiter_lines(), sync_stream=False
@@ -73,11 +91,12 @@ def make_sync_call(
     messages: list,
     logging_obj,
     streaming_decoder: Optional[CustomStreamingDecoder] = None,
+    fake_stream: bool = False,
 ):
     if client is None:
         client = litellm.module_level_client  # Create a new client if none provided
 
-    response = client.post(api_base, headers=headers, data=data, stream=True)
+    response = client.post(api_base, headers=headers, data=data, stream=not fake_stream)
 
     if response.status_code != 200:
         raise OpenAILikeError(status_code=response.status_code, message=response.read())
@@ -86,6 +105,9 @@ def make_sync_call(
         completion_stream = streaming_decoder.iter_bytes(
             response.iter_bytes(chunk_size=1024)
         )
+    elif fake_stream:
+        model_response = ModelResponse(**response.json())
+        completion_stream = MockResponseIterator(model_response=model_response)
     else:
         completion_stream = ModelResponseIterator(
             streaming_response=response.iter_lines(), sync_stream=True
@@ -126,8 +148,8 @@ class OpenAILikeChatHandler(OpenAILikeBase):
         headers={},
         client: Optional[AsyncHTTPHandler] = None,
         streaming_decoder: Optional[CustomStreamingDecoder] = None,
+        fake_stream: bool = False,
     ) -> CustomStreamWrapper:
-
         data["stream"] = True
         completion_stream = await make_call(
             client=client,
@@ -169,6 +191,7 @@ class OpenAILikeChatHandler(OpenAILikeBase):
         logger_fn=None,
         headers={},
         timeout: Optional[Union[float, httpx.Timeout]] = None,
+        json_mode: bool = False,
     ) -> ModelResponse:
         if timeout is None:
             timeout = httpx.Timeout(timeout=600.0, connect=5.0)
@@ -181,8 +204,6 @@ class OpenAILikeChatHandler(OpenAILikeBase):
                 api_base, headers=headers, data=json.dumps(data), timeout=timeout
             )
             response.raise_for_status()
-
-            response_json = response.json()
         except httpx.HTTPStatusError as e:
             raise OpenAILikeError(
                 status_code=e.response.status_code,
@@ -193,22 +214,26 @@ class OpenAILikeChatHandler(OpenAILikeBase):
         except Exception as e:
             raise OpenAILikeError(status_code=500, message=str(e))
 
-        logging_obj.post_call(
-            input=messages,
-            api_key="",
-            original_response=response_json,
-            additional_args={"complete_input_dict": data},
+        return OpenAILikeChatConfig._transform_response(
+            model=model,
+            response=response,
+            model_response=model_response,
+            stream=stream,
+            logging_obj=logging_obj,
+            optional_params=optional_params,
+            api_key=api_key,
+            data=data,
+            messages=messages,
+            print_verbose=print_verbose,
+            encoding=encoding,
+            json_mode=json_mode,
+            custom_llm_provider=custom_llm_provider,
+            base_model=base_model,
         )
-        response = ModelResponse(**response_json)
-
-        response.model = custom_llm_provider + "/" + (response.model or "")
-
-        if base_model is not None:
-            response._hidden_params["model"] = base_model
-        return response
 
     def completion(
         self,
+        *,
         model: str,
         messages: list,
         api_base: str,
@@ -230,6 +255,7 @@ class OpenAILikeChatHandler(OpenAILikeBase):
         streaming_decoder: Optional[
             CustomStreamingDecoder
         ] = None,  # if openai-compatible api needs custom stream decoder - e.g. sagemaker
+        fake_stream: bool = False,
     ):
         custom_endpoint = custom_endpoint or optional_params.pop(
             "custom_endpoint", None
@@ -243,13 +269,24 @@ class OpenAILikeChatHandler(OpenAILikeBase):
             headers=headers,
         )
 
-        stream: bool = optional_params.get("stream", None) or False
-        optional_params["stream"] = stream
+        stream: bool = optional_params.pop("stream", None) or False
+        extra_body = optional_params.pop("extra_body", {})
+        json_mode = optional_params.pop("json_mode", None)
+        optional_params.pop("max_retries", None)
+        if not fake_stream:
+            optional_params["stream"] = stream
+
+        if messages is not None and custom_llm_provider is not None:
+            provider_config = ProviderConfigManager.get_provider_config(
+                model=model, provider=LlmProviders(custom_llm_provider)
+            )
+            messages = provider_config._transform_messages(messages)
 
         data = {
             "model": model,
             "messages": messages,
             **optional_params,
+            **extra_body,
         }
 
         ## LOGGING
@@ -288,6 +325,7 @@ class OpenAILikeChatHandler(OpenAILikeBase):
                     client=client,
                     custom_llm_provider=custom_llm_provider,
                     streaming_decoder=streaming_decoder,
+                    fake_stream=fake_stream,
                 )
             else:
                 return self.acompletion_function(
@@ -327,6 +365,7 @@ class OpenAILikeChatHandler(OpenAILikeBase):
                     messages=messages,
                     logging_obj=logging_obj,
                     streaming_decoder=streaming_decoder,
+                    fake_stream=fake_stream,
                 )
                 # completion_stream.__iter__()
                 return CustomStreamWrapper(
@@ -344,7 +383,6 @@ class OpenAILikeChatHandler(OpenAILikeBase):
                     )
                     response.raise_for_status()
 
-                    response_json = response.json()
                 except httpx.HTTPStatusError as e:
                     raise OpenAILikeError(
                         status_code=e.response.status_code,
@@ -356,17 +394,19 @@ class OpenAILikeChatHandler(OpenAILikeBase):
                     )
                 except Exception as e:
                     raise OpenAILikeError(status_code=500, message=str(e))
-        logging_obj.post_call(
-            input=messages,
-            api_key="",
-            original_response=response_json,
-            additional_args={"complete_input_dict": data},
+        return OpenAILikeChatConfig._transform_response(
+            model=model,
+            response=response,
+            model_response=model_response,
+            stream=stream,
+            logging_obj=logging_obj,
+            optional_params=optional_params,
+            api_key=api_key,
+            data=data,
+            messages=messages,
+            print_verbose=print_verbose,
+            encoding=encoding,
+            json_mode=json_mode,
+            custom_llm_provider=custom_llm_provider,
+            base_model=base_model,
         )
-        response = ModelResponse(**response_json)
-
-        response.model = custom_llm_provider + "/" + (response.model or "")
-
-        if base_model is not None:
-            response._hidden_params["model"] = base_model
-
-        return response

litellm/llms/openai_like/chat/transformation.py

@@ -0,0 +1,98 @@
+"""
+OpenAI-like chat completion transformation
+"""
+
+import types
+from typing import List, Optional, Tuple, Union
+
+import httpx
+from pydantic import BaseModel
+
+import litellm
+from litellm.secret_managers.main import get_secret_str
+from litellm.types.llms.openai import AllMessageValues, ChatCompletionAssistantMessage
+from litellm.types.utils import ModelResponse
+
+from ....utils import _remove_additional_properties, _remove_strict_from_schema
+from ...OpenAI.chat.gpt_transformation import OpenAIGPTConfig
+
+
+class OpenAILikeChatConfig(OpenAIGPTConfig):
+    def _get_openai_compatible_provider_info(
+        self, api_base: Optional[str], api_key: Optional[str]
+    ) -> Tuple[Optional[str], Optional[str]]:
+        api_base = api_base or get_secret_str("OPENAI_LIKE_API_BASE")  # type: ignore
+        dynamic_api_key = (
+            api_key or get_secret_str("OPENAI_LIKE_API_KEY") or ""
+        )  # vllm does not require an api key
+        return api_base, dynamic_api_key
+
+    @staticmethod
+    def _convert_tool_response_to_message(
+        message: ChatCompletionAssistantMessage, json_mode: bool
+    ) -> ChatCompletionAssistantMessage:
+        """
+        if json_mode is true, convert the returned tool call response to a content with json str
+
+        e.g. input:
+
+        {"role": "assistant", "tool_calls": [{"id": "call_5ms4", "type": "function", "function": {"name": "json_tool_call", "arguments": "{\"key\": \"question\", \"value\": \"What is the capital of France?\"}"}}]}
+
+        output:
+
+        {"role": "assistant", "content": "{\"key\": \"question\", \"value\": \"What is the capital of France?\"}"}
+        """
+        if not json_mode:
+            return message
+
+        _tool_calls = message.get("tool_calls")
+
+        if _tool_calls is None or len(_tool_calls) != 1:
+            return message
+
+        message["content"] = _tool_calls[0]["function"].get("arguments") or ""
+        message["tool_calls"] = None
+
+        return message
+
+    @staticmethod
+    def _transform_response(
+        model: str,
+        response: httpx.Response,
+        model_response: ModelResponse,
+        stream: bool,
+        logging_obj: litellm.litellm_core_utils.litellm_logging.Logging,  # type: ignore
+        optional_params: dict,
+        api_key: Optional[str],
+        data: Union[dict, str],
+        messages: List,
+        print_verbose,
+        encoding,
+        json_mode: bool,
+        custom_llm_provider: str,
+        base_model: Optional[str],
+    ) -> ModelResponse:
+        response_json = response.json()
+        logging_obj.post_call(
+            input=messages,
+            api_key="",
+            original_response=response_json,
+            additional_args={"complete_input_dict": data},
+        )
+
+        if json_mode:
+            for choice in response_json["choices"]:
+                message = OpenAILikeChatConfig._convert_tool_response_to_message(
+                    choice.get("message"), json_mode
+                )
+                choice["message"] = message
+
+        returned_response = ModelResponse(**response_json)
+
+        returned_response.model = (
+            custom_llm_provider + "/" + (returned_response.model or "")
+        )
+
+        if base_model is not None:
+            returned_response._hidden_params["model"] = base_model
+        return returned_response

litellm/llms/openai_like/embedding/handler.py

@@ -45,7 +45,10 @@ class OpenAILikeEmbeddingHandler(OpenAILikeBase):
         response = None
         try:
             if client is None or isinstance(client, AsyncHTTPHandler):
-                self.async_client = AsyncHTTPHandler(timeout=timeout)  # type: ignore
+                self.async_client = get_async_httpx_client(
+                    llm_provider=litellm.LlmProviders.OPENAI,
+                    params={"timeout": timeout},
+                )
             else:
                 self.async_client = client
 
@@ -62,7 +65,7 @@ class OpenAILikeEmbeddingHandler(OpenAILikeBase):
             except httpx.HTTPStatusError as e:
                 raise OpenAILikeError(
                     status_code=e.response.status_code,
-                    message=response.text if response else str(e),
+                    message=e.response.text if e.response else str(e),
                 )
             except httpx.TimeoutException:
                 raise OpenAILikeError(