phoenix/litellm-mirror
0
0
Fork 1
mirror of https://github.com/BerriAI/litellm.git synced 2025-04-27 03:34:10 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

resolve merge conflicts

Browse source
This commit is contained in:
Sophia Loris 2024-07-19 09:45:53 -05:00
commit adae0777d6
597 changed files with 276126 additions and 25044 deletions
Download patch file Download diff file Expand all files Collapse all files

120
.circleci/config.yml
View file

@ -2,7 +2,7 @@ version: 4.3.4
jobs:
local_testing:
docker:
- image: circleci/python:3.9
- image: cimg/python:3.11
working_directory: ~/project
steps:
@ -43,9 +43,13 @@ jobs:
pip install "langfuse==2.27.1"
pip install "logfire==0.29.0"
pip install numpydoc
pip install traceloop-sdk==0.18.2
pip install traceloop-sdk==0.21.1
pip install opentelemetry-api==1.25.0
pip install opentelemetry-sdk==1.25.0
pip install opentelemetry-exporter-otlp==1.25.0
pip install openai
pip install prisma
pip install "detect_secrets==1.5.0"
pip install "httpx==0.24.1"
pip install fastapi
pip install "gunicorn==21.2.0"
@ -61,6 +65,8 @@ jobs:
pip install prometheus-client==0.20.0
pip install "pydantic==2.7.1"
pip install "diskcache==5.6.1"
pip install "Pillow==10.3.0"
pip install "jsonschema==4.22.0"
- save_cache:
paths:
- ./venv
@ -96,7 +102,7 @@ jobs:
command: |
pwd
ls
python -m pytest -vv litellm/tests/ -x --junitxml=test-results/junit.xml --durations=5
python -m pytest -vv litellm/tests/ -x --junitxml=test-results/junit.xml --durations=5 -k "not test_python_38.py"
no_output_timeout: 120m
# Store test results
@ -122,6 +128,7 @@ jobs:
pip install jinja2
pip install tokenizers
pip install openai
pip install jsonschema
- run:
name: Run tests
command: |
@ -176,6 +183,7 @@ jobs:
pip install numpydoc
pip install prisma
pip install fastapi
pip install jsonschema
pip install "httpx==0.24.1"
pip install "gunicorn==21.2.0"
pip install "anyio==3.7.1"
@ -198,11 +206,13 @@ jobs:
-e REDIS_PORT=$REDIS_PORT \
-e AZURE_FRANCE_API_KEY=$AZURE_FRANCE_API_KEY \
-e AZURE_EUROPE_API_KEY=$AZURE_EUROPE_API_KEY \
-e MISTRAL_API_KEY=$MISTRAL_API_KEY \
-e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID \
-e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY \
-e AWS_REGION_NAME=$AWS_REGION_NAME \
-e AUTO_INFER_REGION=True \
-e OPENAI_API_KEY=$OPENAI_API_KEY \
-e LITELLM_LICENSE=$LITELLM_LICENSE \
-e LANGFUSE_PROJECT1_PUBLIC=$LANGFUSE_PROJECT1_PUBLIC \
-e LANGFUSE_PROJECT2_PUBLIC=$LANGFUSE_PROJECT2_PUBLIC \
-e LANGFUSE_PROJECT1_SECRET=$LANGFUSE_PROJECT1_SECRET \
@ -233,7 +243,102 @@ jobs:
command: |
pwd
ls
python -m pytest -vv tests/ -x --junitxml=test-results/junit.xml --durations=5
python -m pytest -vv tests/ -x --junitxml=test-results/junit.xml --durations=5 --ignore=tests/otel_tests
no_output_timeout: 120m
# Store test results
- store_test_results:
path: test-results
proxy_log_to_otel_tests:
machine:
image: ubuntu-2204:2023.10.1
resource_class: xlarge
working_directory: ~/project
steps:
- checkout
- run:
name: Install Docker CLI (In case it's not already installed)
command: |
sudo apt-get update
sudo apt-get install -y docker-ce docker-ce-cli containerd.io
- run:
name: Install Python 3.9
command: |
curl https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh --output miniconda.sh
bash miniconda.sh -b -p $HOME/miniconda
export PATH="$HOME/miniconda/bin:$PATH"
conda init bash
source ~/.bashrc
conda create -n myenv python=3.9 -y
conda activate myenv
python --version
- run:
name: Install Dependencies
command: |
pip install "pytest==7.3.1"
pip install "pytest-asyncio==0.21.1"
pip install aiohttp
pip install openai
python -m pip install --upgrade pip
python -m pip install -r .circleci/requirements.txt
pip install "pytest==7.3.1"
pip install "pytest-mock==3.12.0"
pip install "pytest-asyncio==0.21.1"
pip install mypy
pip install pyarrow
pip install numpydoc
pip install prisma
pip install fastapi
pip install jsonschema
pip install "httpx==0.24.1"
pip install "anyio==3.7.1"
pip install "asyncio==3.4.3"
pip install "PyGithub==1.59.1"
- run:
name: Build Docker image
command: docker build -t my-app:latest -f Dockerfile.database .
- run:
name: Run Docker container
# intentionally give bad redis credentials here
# the OTEL test - should get this as a trace
command: |
docker run -d \
-p 4000:4000 \
-e DATABASE_URL=$PROXY_DATABASE_URL \
-e REDIS_HOST=$REDIS_HOST \
-e REDIS_PASSWORD=$REDIS_PASSWORD \
-e REDIS_PORT=$REDIS_PORT \
-e LITELLM_MASTER_KEY="sk-1234" \
-e OPENAI_API_KEY=$OPENAI_API_KEY \
-e LITELLM_LICENSE=$LITELLM_LICENSE \
-e OTEL_EXPORTER="in_memory" \
--name my-app \
-v $(pwd)/litellm/proxy/example_config_yaml/otel_test_config.yaml:/app/config.yaml \
my-app:latest \
--config /app/config.yaml \
--port 4000 \
--detailed_debug \
- run:
name: Install curl and dockerize
command: |
sudo apt-get update
sudo apt-get install -y curl
sudo wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz
sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz
sudo rm dockerize-linux-amd64-v0.6.1.tar.gz
- run:
name: Start outputting logs
command: docker logs -f my-app
background: true
- run:
name: Wait for app to be ready
command: dockerize -wait http://localhost:4000 -timeout 5m
- run:
name: Run tests
command: |
pwd
ls
python -m pytest -vv tests/otel_tests/test_otel.py -x --junitxml=test-results/junit.xml --durations=5
no_output_timeout: 120m
# Store test results
@ -327,6 +432,12 @@ workflows:
only:
- main
- /litellm_.*/
- proxy_log_to_otel_tests:
filters:
branches:
only:
- main
- /litellm_.*/
- installing_litellm_on_python:
filters:
branches:
@ -337,6 +448,7 @@ workflows:
requires:
- local_testing
- build_and_test
- proxy_log_to_otel_tests
filters:
branches:
only:

3
.circleci/requirements.txt
View file

@ -7,6 +7,5 @@ cohere
redis
anthropic
orjson
pydantic==1.10.14
pydantic==2.7.1
google-cloud-aiplatform==1.43.0
redisvl==0.0.7 # semantic caching

10
.github/dependabot.yaml vendored Normal file
View file

@ -0,0 +1,10 @@
version: 2
updates:
- package-ecosystem: "github-actions"
directory: "/"
schedule:
interval: "daily"
groups:
github-actions:
patterns:
- "*"

32
.github/workflows/ghcr_deploy.yml vendored
View file

@ -25,6 +25,11 @@ jobs:
if: github.repository == 'BerriAI/litellm'
runs-on: ubuntu-latest
steps:
-
name: Checkout
uses: actions/checkout@v4
with:
ref: ${{ github.event.inputs.commit_hash }}
-
name: Set up QEMU
uses: docker/setup-qemu-action@v3
@ -41,12 +46,14 @@ jobs:
name: Build and push
uses: docker/build-push-action@v5
with:
context: .
push: true
tags: litellm/litellm:${{ github.event.inputs.tag || 'latest' }}
-
name: Build and push litellm-database image
uses: docker/build-push-action@v5
with:
context: .
push: true
file: Dockerfile.database
tags: litellm/litellm-database:${{ github.event.inputs.tag || 'latest' }}
@ -54,6 +61,7 @@ jobs:
name: Build and push litellm-spend-logs image
uses: docker/build-push-action@v5
with:
context: .
push: true
file: ./litellm-js/spend-logs/Dockerfile
tags: litellm/litellm-spend_logs:${{ github.event.inputs.tag || 'latest' }}
@ -68,6 +76,8 @@ jobs:
steps:
- name: Checkout repository
uses: actions/checkout@v4
with:
ref: ${{ github.event.inputs.commit_hash }}
# Uses the `docker/login-action` action to log in to the Container registry registry using the account and password that will publish the packages. Once published, the packages are scoped to the account defined here.
- name: Log in to the Container registry
uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1
@ -92,7 +102,7 @@ jobs:
- name: Build and push Docker image
uses: docker/build-push-action@4976231911ebf5f32aad765192d35f942aa48cb8
with:
context: https://github.com/BerriAI/litellm.git#${{ github.event.inputs.commit_hash}}
context: .
push: true
tags: ${{ steps.meta.outputs.tags }}-${{ github.event.inputs.tag || 'latest' }}, ${{ steps.meta.outputs.tags }}-${{ github.event.inputs.release_type }} # if a tag is provided, use that, otherwise use the release tag, and if neither is available, use 'latest'
labels: ${{ steps.meta.outputs.labels }}
@ -106,6 +116,8 @@ jobs:
steps:
- name: Checkout repository
uses: actions/checkout@v4
with:
ref: ${{ github.event.inputs.commit_hash }}
- name: Log in to the Container registry
uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1
@ -128,7 +140,7 @@ jobs:
- name: Build and push Database Docker image
uses: docker/build-push-action@f2a1d5e99d037542a71f64918e516c093c6f3fc4
with:
context: https://github.com/BerriAI/litellm.git#${{ github.event.inputs.commit_hash}}
context: .
file: Dockerfile.database
push: true
tags: ${{ steps.meta-database.outputs.tags }}-${{ github.event.inputs.tag || 'latest' }}, ${{ steps.meta-database.outputs.tags }}-${{ github.event.inputs.release_type }}
@ -143,6 +155,8 @@ jobs:
steps:
- name: Checkout repository
uses: actions/checkout@v4
with:
ref: ${{ github.event.inputs.commit_hash }}
- name: Log in to the Container registry
uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1
@ -165,7 +179,7 @@ jobs:
- name: Build and push Database Docker image
uses: docker/build-push-action@f2a1d5e99d037542a71f64918e516c093c6f3fc4
with:
context: https://github.com/BerriAI/litellm.git#${{ github.event.inputs.commit_hash}}
context: .
file: ./litellm-js/spend-logs/Dockerfile
push: true
tags: ${{ steps.meta-spend-logs.outputs.tags }}-${{ github.event.inputs.tag || 'latest' }}, ${{ steps.meta-spend-logs.outputs.tags }}-${{ github.event.inputs.release_type }}
@ -176,6 +190,8 @@ jobs:
steps:
- name: Checkout repository
uses: actions/checkout@v4
with:
ref: ${{ github.event.inputs.commit_hash }}
- name: Log in to the Container registry
uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1
@ -273,7 +289,8 @@ jobs:
repo: context.repo.repo,
release_id: process.env.RELEASE_ID,
});
return response.data.body;
const formattedBody = JSON.stringify(response.data.body).slice(1, -1);
return formattedBody;
} catch (error) {
core.setFailed(error.message);
}
@ -286,14 +303,15 @@ jobs:
RELEASE_NOTES: ${{ steps.release-notes.outputs.result }}
run: |
curl -H "Content-Type: application/json" -X POST -d '{
"content": "New LiteLLM release ${{ env.RELEASE_TAG }}",
"content": "New LiteLLM release '"${RELEASE_TAG}"'",
"username": "Release Changelog",
"avatar_url": "https://cdn.discordapp.com/avatars/487431320314576937/bd64361e4ba6313d561d54e78c9e7171.png",
"embeds": [
{
"title": "Changelog for LiteLLM ${{ env.RELEASE_TAG }}",
"description": "${{ env.RELEASE_NOTES }}",
"title": "Changelog for LiteLLM '"${RELEASE_TAG}"'",
"description": "'"${RELEASE_NOTES}"'",
"color": 2105893
}
]
}' $WEBHOOK_URL

34
.github/workflows/main.yml vendored Normal file
View file

@ -0,0 +1,34 @@
name: Publish Dev Release to PyPI
on:
workflow_dispatch:
jobs:
publish-dev-release:
runs-on: ubuntu-latest
steps:
- name: Checkout code
uses: actions/checkout@v2
- name: Set up Python
uses: actions/setup-python@v2
with:
python-version: 3.8 # Adjust the Python version as needed
- name: Install dependencies
run: pip install toml twine
- name: Read version from pyproject.toml
id: read-version
run: |
version=$(python -c 'import toml; print(toml.load("pyproject.toml")["tool"]["commitizen"]["version"])')
printf "LITELLM_VERSION=%s" "$version" >> $GITHUB_ENV
- name: Check if version exists on PyPI
id: check-version
run: |
set -e
if twine check --repository-url https://pypi.org/simple/ "litellm==$LITELLM_VERSION" >/dev/null 2>&1; then
echo "Version $LITELLM_VERSION already exists on PyPI. Skipping publish."

8
.gitignore vendored
View file

@ -1,5 +1,7 @@
.venv
.env
.newenv
newenv/*
litellm/proxy/myenv/*
litellm_uuid.txt
__pycache__/
@ -56,3 +58,9 @@ litellm/proxy/_super_secret_config.yaml
litellm/proxy/myenv/bin/activate
litellm/proxy/myenv/bin/Activate.ps1
myenv/*
litellm/proxy/_experimental/out/404/index.html
litellm/proxy/_experimental/out/model_hub/index.html
litellm/proxy/_experimental/out/onboarding/index.html
litellm/tests/log.txt
litellm/tests/langfuse.log
litellm/tests/langfuse.log

52
.pre-commit-config.yaml
View file

@ -1,21 +1,4 @@
repos:
- repo: https://github.com/psf/black
rev: 24.2.0
hooks:
- id: black
- repo: https://github.com/pycqa/flake8
rev: 7.0.0 # The version of flake8 to use
hooks:
- id: flake8
exclude: ^litellm/tests/|^litellm/proxy/proxy_cli.py|^litellm/proxy/tests/
additional_dependencies: [flake8-print]
files: litellm/.*\.py
- repo: local
hooks:
- id: check-files-match
name: Check if files match
entry: python3 ci_cd/check_files_match.py
language: system
- repo: local
hooks:
- id: mypy
@ -24,3 +7,38 @@ repos:
language: system
types: [python]
files: ^litellm/
- id: isort
name: isort
entry: isort
language: system
types: [python]
files: litellm/.*\.py
exclude: ^litellm/__init__.py$
- repo: https://github.com/psf/black
rev: 24.2.0
hooks:
- id: black
- repo: https://github.com/pycqa/flake8
rev: 7.0.0 # The version of flake8 to use
hooks:
- id: flake8
exclude: ^litellm/tests/|^litellm/proxy/tests/
additional_dependencies: [flake8-print]
files: litellm/.*\.py
- repo: https://github.com/python-poetry/poetry
rev: 1.8.0
hooks:
- id: poetry-check
- repo: local
hooks:
- id: check-files-match
name: Check if files match
entry: python3 ci_cd/check_files_match.py
language: system
# - id: check-file-length
# name: Check file length
# entry: python check_file_length.py
# args: ["10000"] # set your desired maximum number of lines
# language: python
# files: litellm/.*\.py
# exclude: ^litellm/tests/

90
README.md
View file

@ -48,6 +48,7 @@ Support for more providers. Missing a provider or LLM Platform, raise a [feature
> [!IMPORTANT]
> LiteLLM v1.0.0 now requires `openai>=1.0.0`. Migration guide [here](https://docs.litellm.ai/docs/migration)
> LiteLLM v1.40.14+ now requires `pydantic>=2.0.0`. No changes required.
<a target="_blank" href="https://colab.research.google.com/github/BerriAI/litellm/blob/main/cookbook/liteLLM_Getting_Started.ipynb">
<img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"/>
@ -119,6 +120,7 @@ from litellm import completion
## set env variables for logging tools
os.environ["LUNARY_PUBLIC_KEY"] = "your-lunary-public-key"
os.environ["HELICONE_API_KEY"] = "your-helicone-auth-key"
os.environ["LANGFUSE_PUBLIC_KEY"] = ""
os.environ["LANGFUSE_SECRET_KEY"] = ""
os.environ["ATHINA_API_KEY"] = "your-athina-api-key"
@ -126,7 +128,7 @@ os.environ["ATHINA_API_KEY"] = "your-athina-api-key"
os.environ["OPENAI_API_KEY"]
# set callbacks
litellm.success_callback = ["lunary", "langfuse", "athina"] # log input/output to lunary, langfuse, supabase, athina etc
litellm.success_callback = ["lunary", "langfuse", "athina", "helicone"] # log input/output to lunary, langfuse, supabase, athina, helicone etc
#openai call
response = completion(model="gpt-3.5-turbo", messages=[{"role": "user", "content": "Hi 👋 - i'm openai"}])
@ -147,6 +149,7 @@ The proxy provides:
## 📖 Proxy Endpoints - [Swagger Docs](https://litellm-api.up.railway.app/)
## Quick Start Proxy - CLI
```shell
@ -179,6 +182,31 @@ print(response)
## Proxy Key Management ([Docs](https://docs.litellm.ai/docs/proxy/virtual_keys))
Connect the proxy with a Postgres DB to create proxy keys
```bash
# Get the code
git clone https://github.com/BerriAI/litellm
# Go to folder
cd litellm
# Add the master key - you can change this after setup
echo 'LITELLM_MASTER_KEY="sk-1234"' > .env
# Add the litellm salt key - you cannot change this after adding a model
# It is used to encrypt / decrypt your LLM API Key credentials
# We recommned - https://1password.com/password-generator/
# password generator to get a random hash for litellm salt key
echo 'LITELLM_SALT_KEY="sk-1234"' > .env
source .env
# Start
docker-compose up
```
UI on `/ui` on your proxy server
![ui_3](https://github.com/BerriAI/litellm/assets/29436595/47c97d5e-b9be-4839-b28c-43d7f4f10033)
@ -206,37 +234,39 @@ curl 'http://0.0.0.0:4000/key/generate' \
## Supported Providers ([Docs](https://docs.litellm.ai/docs/providers))
| Provider | [Completion](https://docs.litellm.ai/docs/#basic-usage) | [Streaming](https://docs.litellm.ai/docs/completion/stream#streaming-responses) | [Async Completion](https://docs.litellm.ai/docs/completion/stream#async-completion) | [Async Streaming](https://docs.litellm.ai/docs/completion/stream#async-streaming) | [Async Embedding](https://docs.litellm.ai/docs/embedding/supported_embedding) | [Async Image Generation](https://docs.litellm.ai/docs/image_generation) |
| ----------------------------------------------------------------------------------- | ------------------------------------------------------- | ------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------- | --------------------------------------------------------------------------------- | ----------------------------------------------------------------------------- | ----------------------------------------------------------------------- |
|-------------------------------------------------------------------------------------|---------------------------------------------------------|---------------------------------------------------------------------------------|-------------------------------------------------------------------------------------|-----------------------------------------------------------------------------------|-------------------------------------------------------------------------------|-------------------------------------------------------------------------|
| [openai](https://docs.litellm.ai/docs/providers/openai) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |
| [azure](https://docs.litellm.ai/docs/providers/azure) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |
| [aws - sagemaker](https://docs.litellm.ai/docs/providers/aws_sagemaker) | ✅ | ✅ | ✅ | ✅ | ✅ |
| [aws - bedrock](https://docs.litellm.ai/docs/providers/bedrock) | ✅ | ✅ | ✅ | ✅ | ✅ |
| [google - vertex_ai [Gemini]](https://docs.litellm.ai/docs/providers/vertex) | ✅ | ✅ | ✅ | ✅ |
| [google - palm](https://docs.litellm.ai/docs/providers/palm) | ✅ | ✅ | ✅ | ✅ |
| [google AI Studio - gemini](https://docs.litellm.ai/docs/providers/gemini) | ✅ | ✅ | ✅ | ✅ | |
| [mistral ai api](https://docs.litellm.ai/docs/providers/mistral) | ✅ | ✅ | ✅ | ✅ | ✅ |
| [cloudflare AI Workers](https://docs.litellm.ai/docs/providers/cloudflare_workers) | ✅ | ✅ | ✅ | ✅ |
| [cohere](https://docs.litellm.ai/docs/providers/cohere) | ✅ | ✅ | ✅ | ✅ | ✅ |
| [anthropic](https://docs.litellm.ai/docs/providers/anthropic) | ✅ | ✅ | ✅ | ✅ |
| [huggingface](https://docs.litellm.ai/docs/providers/huggingface) | ✅ | ✅ | ✅ | ✅ | ✅ |
| [replicate](https://docs.litellm.ai/docs/providers/replicate) | ✅ | ✅ | ✅ | ✅ |
| [together_ai](https://docs.litellm.ai/docs/providers/togetherai) | ✅ | ✅ | ✅ | ✅ |
| [openrouter](https://docs.litellm.ai/docs/providers/openrouter) | ✅ | ✅ | ✅ | ✅ |
| [ai21](https://docs.litellm.ai/docs/providers/ai21) | ✅ | ✅ | ✅ | ✅ |
| [baseten](https://docs.litellm.ai/docs/providers/baseten) | ✅ | ✅ | ✅ | ✅ |
| [vllm](https://docs.litellm.ai/docs/providers/vllm) | ✅ | ✅ | ✅ | ✅ |
| [nlp_cloud](https://docs.litellm.ai/docs/providers/nlp_cloud) | ✅ | ✅ | ✅ | ✅ |
| [aleph alpha](https://docs.litellm.ai/docs/providers/aleph_alpha) | ✅ | ✅ | ✅ | ✅ |
| [petals](https://docs.litellm.ai/docs/providers/petals) | ✅ | ✅ | ✅ | ✅ |
| [ollama](https://docs.litellm.ai/docs/providers/ollama) | ✅ | ✅ | ✅ | ✅ | ✅ |
| [deepinfra](https://docs.litellm.ai/docs/providers/deepinfra) | ✅ | ✅ | ✅ | ✅ |
| [perplexity-ai](https://docs.litellm.ai/docs/providers/perplexity) | ✅ | ✅ | ✅ | ✅ |
| [Groq AI](https://docs.litellm.ai/docs/providers/groq) | ✅ | ✅ | ✅ | ✅ |
| [Deepseek](https://docs.litellm.ai/docs/providers/deepseek) | ✅ | ✅ | ✅ | ✅ |
| [anyscale](https://docs.litellm.ai/docs/providers/anyscale) | ✅ | ✅ | ✅ | ✅ |
| [IBM - watsonx.ai](https://docs.litellm.ai/docs/providers/watsonx) | ✅ | ✅ | ✅ | ✅ | ✅
| [voyage ai](https://docs.litellm.ai/docs/providers/voyage) | | | | | ✅ |
| [xinference [Xorbits Inference]](https://docs.litellm.ai/docs/providers/xinference) | | | | | ✅ |
| [aws - sagemaker](https://docs.litellm.ai/docs/providers/aws_sagemaker) | ✅ | ✅ | ✅ | ✅ | ✅ | |
| [aws - bedrock](https://docs.litellm.ai/docs/providers/bedrock) | ✅ | ✅ | ✅ | ✅ | ✅ | |
| [google - vertex_ai](https://docs.litellm.ai/docs/providers/vertex) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |
| [google - palm](https://docs.litellm.ai/docs/providers/palm) | ✅ | ✅ | ✅ | ✅ | | |
| [google AI Studio - gemini](https://docs.litellm.ai/docs/providers/gemini) | ✅ | ✅ | ✅ | ✅ | | |
| [mistral ai api](https://docs.litellm.ai/docs/providers/mistral) | ✅ | ✅ | ✅ | ✅ | ✅ | |
| [cloudflare AI Workers](https://docs.litellm.ai/docs/providers/cloudflare_workers) | ✅ | ✅ | ✅ | ✅ | | |
| [cohere](https://docs.litellm.ai/docs/providers/cohere) | ✅ | ✅ | ✅ | ✅ | ✅ | |
| [anthropic](https://docs.litellm.ai/docs/providers/anthropic) | ✅ | ✅ | ✅ | ✅ | | |
| [empower](https://docs.litellm.ai/docs/providers/empower) | ✅ | ✅ | ✅ | ✅ |
| [huggingface](https://docs.litellm.ai/docs/providers/huggingface) | ✅ | ✅ | ✅ | ✅ | ✅ | |
| [replicate](https://docs.litellm.ai/docs/providers/replicate) | ✅ | ✅ | ✅ | ✅ | | |
| [together_ai](https://docs.litellm.ai/docs/providers/togetherai) | ✅ | ✅ | ✅ | ✅ | | |
| [openrouter](https://docs.litellm.ai/docs/providers/openrouter) | ✅ | ✅ | ✅ | ✅ | | |
| [ai21](https://docs.litellm.ai/docs/providers/ai21) | ✅ | ✅ | ✅ | ✅ | | |
| [baseten](https://docs.litellm.ai/docs/providers/baseten) | ✅ | ✅ | ✅ | ✅ | | |
| [vllm](https://docs.litellm.ai/docs/providers/vllm) | ✅ | ✅ | ✅ | ✅ | | |
| [nlp_cloud](https://docs.litellm.ai/docs/providers/nlp_cloud) | ✅ | ✅ | ✅ | ✅ | | |
| [aleph alpha](https://docs.litellm.ai/docs/providers/aleph_alpha) | ✅ | ✅ | ✅ | ✅ | | |
| [petals](https://docs.litellm.ai/docs/providers/petals) | ✅ | ✅ | ✅ | ✅ | | |
| [ollama](https://docs.litellm.ai/docs/providers/ollama) | ✅ | ✅ | ✅ | ✅ | ✅ | |
| [deepinfra](https://docs.litellm.ai/docs/providers/deepinfra) | ✅ | ✅ | ✅ | ✅ | | |
| [perplexity-ai](https://docs.litellm.ai/docs/providers/perplexity) | ✅ | ✅ | ✅ | ✅ | | |
| [Groq AI](https://docs.litellm.ai/docs/providers/groq) | ✅ | ✅ | ✅ | ✅ | | |
| [Deepseek](https://docs.litellm.ai/docs/providers/deepseek) | ✅ | ✅ | ✅ | ✅ | | |
| [anyscale](https://docs.litellm.ai/docs/providers/anyscale) | ✅ | ✅ | ✅ | ✅ | | |
| [IBM - watsonx.ai](https://docs.litellm.ai/docs/providers/watsonx) | ✅ | ✅ | ✅ | ✅ | ✅ | |
| [voyage ai](https://docs.litellm.ai/docs/providers/voyage) | | | | | ✅ | |
| [xinference [Xorbits Inference]](https://docs.litellm.ai/docs/providers/xinference) | | | | | ✅ | |
| [FriendliAI](https://docs.litellm.ai/docs/providers/friendliai) | ✅ | ✅ | ✅ | ✅ | | |
[**Read the Docs**](https://docs.litellm.ai/docs/)

28
check_file_length.py Normal file
View file

@ -0,0 +1,28 @@
import sys
def check_file_length(max_lines, filenames):
bad_files = []
for filename in filenames:
with open(filename, "r") as file:
lines = file.readlines()
if len(lines) > max_lines:
bad_files.append((filename, len(lines)))
return bad_files
if __name__ == "__main__":
max_lines = int(sys.argv[1])
filenames = sys.argv[2:]
bad_files = check_file_length(max_lines, filenames)
if bad_files:
bad_files.sort(
key=lambda x: x[1], reverse=True
) # Sort files by length in descending order
for filename, length in bad_files:
print(f"{filename}: {length} lines")
sys.exit(1)
else:
sys.exit(0)

594
cookbook/litellm_proxy_server/grafana_dashboard/dashboard_1/grafana_dashboard.json Normal file
View file

@ -0,0 +1,594 @@
{
"annotations": {
"list": [
{
"builtIn": 1,
"datasource": {
"type": "grafana",
"uid": "-- Grafana --"
},
"enable": true,
"hide": true,
"iconColor": "rgba(0, 211, 255, 1)",
"name": "Annotations & Alerts",
"target": {
"limit": 100,
"matchAny": false,
"tags": [],
"type": "dashboard"
},
"type": "dashboard"
}
]
},
"description": "",
"editable": true,
"fiscalYearStartMonth": 0,
"graphTooltip": 0,
"id": 2039,
"links": [],
"liveNow": false,
"panels": [
{
"datasource": {
"type": "prometheus",
"uid": "rMzWaBvIk"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
},
"unit": "s"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 0
},
"id": 10,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "rMzWaBvIk"
},
"editorMode": "code",
"expr": "histogram_quantile(0.99, sum(rate(litellm_self_latency_bucket{self=\"self\"}[1m])) by (le))",
"legendFormat": "Time to first token",
"range": true,
"refId": "A"
}
],
"title": "Time to first token (latency)",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "rMzWaBvIk"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
},
"unit": "currencyUSD"
},
"overrides": [
{
"matcher": {
"id": "byName",
"options": "7e4b0627fd32efdd2313c846325575808aadcf2839f0fde90723aab9ab73c78f"
},
"properties": [
{
"id": "displayName",
"value": "Translata"
}
]
}
]
},
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 8
},
"id": 11,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "rMzWaBvIk"
},
"editorMode": "code",
"expr": "sum(increase(litellm_spend_metric_total[30d])) by (hashed_api_key)",
"legendFormat": "{{team}}",
"range": true,
"refId": "A"
}
],
"title": "Spend by team",
"transformations": [],
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "rMzWaBvIk"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 9,
"w": 12,
"x": 0,
"y": 16
},
"id": 2,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "rMzWaBvIk"
},
"editorMode": "code",
"expr": "sum by (model) (increase(litellm_requests_metric_total[5m]))",
"legendFormat": "{{model}}",
"range": true,
"refId": "A"
}
],
"title": "Requests by model",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "rMzWaBvIk"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"mappings": [],
"noValue": "0",
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 7,
"w": 3,
"x": 0,
"y": 25
},
"id": 8,
"options": {
"colorMode": "value",
"graphMode": "area",
"justifyMode": "auto",
"orientation": "auto",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"textMode": "auto"
},
"pluginVersion": "9.4.17",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "rMzWaBvIk"
},
"editorMode": "code",
"expr": "sum(increase(litellm_llm_api_failed_requests_metric_total[1h]))",
"legendFormat": "__auto",
"range": true,
"refId": "A"
}
],
"title": "Faild Requests",
"type": "stat"
},
{
"datasource": {
"type": "prometheus",
"uid": "rMzWaBvIk"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
},
"unit": "currencyUSD"
},
"overrides": []
},
"gridPos": {
"h": 7,
"w": 3,
"x": 3,
"y": 25
},
"id": 6,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "rMzWaBvIk"
},
"editorMode": "code",
"expr": "sum(increase(litellm_spend_metric_total[30d])) by (model)",
"legendFormat": "{{model}}",
"range": true,
"refId": "A"
}
],
"title": "Spend",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "rMzWaBvIk"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 7,
"w": 6,
"x": 6,
"y": 25
},
"id": 4,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "rMzWaBvIk"
},
"editorMode": "code",
"expr": "sum(increase(litellm_total_tokens_total[5m])) by (model)",
"legendFormat": "__auto",
"range": true,
"refId": "A"
}
],
"title": "Tokens",
"type": "timeseries"
}
],
"refresh": "1m",
"revision": 1,
"schemaVersion": 38,
"style": "dark",
"tags": [],
"templating": {
"list": []
},
"time": {
"from": "now-1h",
"to": "now"
},
"timepicker": {},
"timezone": "",
"title": "LLM Proxy",
"uid": "rgRrHxESz",
"version": 15,
"weekStart": ""
}

6
cookbook/litellm_proxy_server/grafana_dashboard/dashboard_1/readme.md Normal file
View file

@ -0,0 +1,6 @@
## This folder contains the `json` for creating the following Grafana Dashboard
### Pre-Requisites
- Setup LiteLLM Proxy Prometheus Metrics https://docs.litellm.ai/docs/proxy/prometheus
![1716623265684](https://github.com/BerriAI/litellm/assets/29436595/0e12c57e-4a2d-4850-bd4f-e4294f87a814)

6
cookbook/litellm_proxy_server/grafana_dashboard/readme.md Normal file
View file

@ -0,0 +1,6 @@
## Contains example Grafana Dashboard made for LiteLLM Proxy Server
This folder contains the `json` for creating Grafana Dashboards
### Pre-Requisites
- Setup LiteLLM Proxy Prometheus Metrics https://docs.litellm.ai/docs/proxy/prometheus

0
cookbook/proxy-server/readme.md → cookbook/litellm_proxy_server/readme.md
View file

72
cookbook/misc/add_new_models.py Normal file
View file

@ -0,0 +1,72 @@
import requests
import json
def get_initial_config():
proxy_base_url = input("Enter your proxy base URL (e.g., http://localhost:4000): ")
master_key = input("Enter your LITELLM_MASTER_KEY ")
return proxy_base_url, master_key
def get_user_input():
model_name = input(
"Enter model_name (this is the 'model' passed in /chat/completions requests):"
)
model = input("litellm_params: Enter model eg. 'azure/<your-deployment-name>': ")
tpm = int(input("litellm_params: Enter tpm (tokens per minute): "))
rpm = int(input("litellm_params: Enter rpm (requests per minute): "))
api_key = input("litellm_params: Enter api_key: ")
api_base = input("litellm_params: Enter api_base: ")
api_version = input("litellm_params: Enter api_version: ")
timeout = int(input("litellm_params: Enter timeout (0 for default): "))
stream_timeout = int(
input("litellm_params: Enter stream_timeout (0 for default): ")
)
max_retries = int(input("litellm_params: Enter max_retries (0 for default): "))
return {
"model_name": model_name,
"litellm_params": {
"model": model,
"tpm": tpm,
"rpm": rpm,
"api_key": api_key,
"api_base": api_base,
"api_version": api_version,
"timeout": timeout,
"stream_timeout": stream_timeout,
"max_retries": max_retries,
},
}
def make_request(proxy_base_url, master_key, data):
url = f"{proxy_base_url}/model/new"
headers = {
"Content-Type": "application/json",
"Authorization": f"Bearer {master_key}",
}
response = requests.post(url, headers=headers, json=data)
print(f"Status Code: {response.status_code}")
print(f"Response from adding model: {response.text}")
def main():
proxy_base_url, master_key = get_initial_config()
while True:
print("Adding new Model to your proxy server...")
data = get_user_input()
make_request(proxy_base_url, master_key, data)
add_another = input("Do you want to add another model? (yes/no): ").lower()
if add_another != "yes":
break
print("Script finished.")
if __name__ == "__main__":
main()

4
deploy/charts/litellm-helm/Chart.yaml
View file

@ -18,13 +18,13 @@ type: application
# This is the chart version. This version number should be incremented each time you make changes
# to the chart and its templates, including the app version.
# Versions are expected to follow Semantic Versioning (https://semver.org/)
version: 0.2.0
version: 0.2.1
# This is the version number of the application being deployed. This version number should be
# incremented each time you make changes to the application. Versions are not expected to
# follow Semantic Versioning. They should reflect the version the application is using.
# It is recommended to use it with quotes.
appVersion: v1.35.38
appVersion: v1.41.8
dependencies:
- name: "postgresql"

31
docker-compose.yml
View file

@ -1,16 +1,35 @@
version: "3.9"
version: "3.11"
services:
litellm:
build:
context: .
args:
target: runtime
image: ghcr.io/berriai/litellm:main-latest
image: ghcr.io/berriai/litellm:main-stable
#########################################
## Uncomment these lines to start proxy with a config.yaml file ##
# volumes:
# - ./proxy_server_config.yaml:/app/config.yaml
# command: [ "--config", "./config.yaml", "--port", "4000"]
###############################################
ports:
- "4000:4000" # Map the container port to the host, change the host port if necessary
volumes:
- ./litellm-config.yaml:/app/config.yaml # Mount the local configuration file
# You can change the port or number of workers as per your requirements or pass any new supported CLI augument. Make sure the port passed here matches with the container port defined above in `ports` value
command: [ "--config", "/app/config.yaml", "--port", "4000", "--num_workers", "8" ]
environment:
DATABASE_URL: "postgresql://postgres:example@db:5432/postgres"
STORE_MODEL_IN_DB: "True" # allows adding models to proxy via UI
env_file:
- .env # Load local .env file
db:
image: postgres
restart: always
environment:
POSTGRES_PASSWORD: example
healthcheck:
test: ["CMD-SHELL", "pg_isready"]
interval: 1s
timeout: 5s
retries: 10
# ...rest of your docker-compose config if any

54
docs/my-website/docs/anthropic_completion.md Normal file
View file

@ -0,0 +1,54 @@
# [BETA] Anthropic `/v1/messages`
Call 100+ LLMs in the Anthropic format.
1. Setup config.yaml
```yaml
model_list:
- model_name: my-test-model
litellm_params:
model: gpt-3.5-turbo
```
2. Start proxy
```bash
litellm --config /path/to/config.yaml
```
3. Test it!
```bash
curl -X POST 'http://0.0.0.0:4000/v1/messages' \
-H 'x-api-key: sk-1234' \
-H 'content-type: application/json' \
-D '{
"model": "my-test-model",
"max_tokens": 1024,
"messages": [
{"role": "user", "content": "Hello, world"}
]
}'
```
## Test with Anthropic SDK
```python
import os
from anthropic import Anthropic
client = Anthropic(api_key="sk-1234", base_url="http://0.0.0.0:4000") # 👈 CONNECT TO PROXY
message = client.messages.create(
messages=[
{
"role": "user",
"content": "Hello, Claude",
}
],
model="my-test-model", # 👈 set 'model_name'
)
print(message.content)
```

312
docs/my-website/docs/assistants.md Normal file
View file

@ -0,0 +1,312 @@
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
# Assistants API
Covers Threads, Messages, Assistants.
LiteLLM currently covers:
- Create Assistants
- Get Assistants
- Create Thread
- Get Thread
- Add Messages
- Get Messages
- Run Thread
## Quick Start
Call an existing Assistant.
- Get the Assistant
- Create a Thread when a user starts a conversation.
- Add Messages to the Thread as the user asks questions.
- Run the Assistant on the Thread to generate a response by calling the model and the tools.
### SDK + PROXY
<Tabs>
<TabItem value="sdk" label="SDK">
**Create an Assistant**
```python
import litellm
import os
# setup env
os.environ["OPENAI_API_KEY"] = "sk-.."
assistant = litellm.create_assistants(
custom_llm_provider="openai",
model="gpt-4-turbo",
instructions="You are a personal math tutor. When asked a question, write and run Python code to answer the question.",
name="Math Tutor",
tools=[{"type": "code_interpreter"}],
)
### ASYNC USAGE ###
# assistant = await litellm.acreate_assistants(
# custom_llm_provider="openai",
# model="gpt-4-turbo",
# instructions="You are a personal math tutor. When asked a question, write and run Python code to answer the question.",
# name="Math Tutor",
# tools=[{"type": "code_interpreter"}],
# )
```
**Get the Assistant**
```python
from litellm import get_assistants, aget_assistants
import os
# setup env
os.environ["OPENAI_API_KEY"] = "sk-.."
assistants = get_assistants(custom_llm_provider="openai")
### ASYNC USAGE ###
# assistants = await aget_assistants(custom_llm_provider="openai")
```
**Create a Thread**
```python
from litellm import create_thread, acreate_thread
import os
os.environ["OPENAI_API_KEY"] = "sk-.."
new_thread = create_thread(
custom_llm_provider="openai",
messages=[{"role": "user", "content": "Hey, how's it going?"}], # type: ignore
)
### ASYNC USAGE ###
# new_thread = await acreate_thread(custom_llm_provider="openai",messages=[{"role": "user", "content": "Hey, how's it going?"}])
```
**Add Messages to the Thread**
```python
from litellm import create_thread, get_thread, aget_thread, add_message, a_add_message
import os
os.environ["OPENAI_API_KEY"] = "sk-.."
## CREATE A THREAD
_new_thread = create_thread(
custom_llm_provider="openai",
messages=[{"role": "user", "content": "Hey, how's it going?"}], # type: ignore
)
## OR retrieve existing thread
received_thread = get_thread(
custom_llm_provider="openai",
thread_id=_new_thread.id,
)
### ASYNC USAGE ###
# received_thread = await aget_thread(custom_llm_provider="openai", thread_id=_new_thread.id,)
## ADD MESSAGE TO THREAD
message = {"role": "user", "content": "Hey, how's it going?"}
added_message = add_message(
thread_id=_new_thread.id, custom_llm_provider="openai", **message
)
### ASYNC USAGE ###
# added_message = await a_add_message(thread_id=_new_thread.id, custom_llm_provider="openai", **message)
```
**Run the Assistant on the Thread**
```python
from litellm import get_assistants, create_thread, add_message, run_thread, arun_thread
import os
os.environ["OPENAI_API_KEY"] = "sk-.."
assistants = get_assistants(custom_llm_provider="openai")
## get the first assistant ###
assistant_id = assistants.data[0].id
## GET A THREAD
_new_thread = create_thread(
custom_llm_provider="openai",
messages=[{"role": "user", "content": "Hey, how's it going?"}], # type: ignore
)
## ADD MESSAGE
message = {"role": "user", "content": "Hey, how's it going?"}
added_message = add_message(
thread_id=_new_thread.id, custom_llm_provider="openai", **message
)
## 🚨 RUN THREAD
response = run_thread(
custom_llm_provider="openai", thread_id=thread_id, assistant_id=assistant_id
)
### ASYNC USAGE ###
# response = await arun_thread(custom_llm_provider="openai", thread_id=thread_id, assistant_id=assistant_id)
print(f"run_thread: {run_thread}")
```
</TabItem>
<TabItem value="proxy" label="PROXY">
```yaml
assistant_settings:
custom_llm_provider: azure
litellm_params:
api_key: os.environ/AZURE_API_KEY
api_base: os.environ/AZURE_API_BASE
api_version: os.environ/AZURE_API_VERSION
```
```bash
$ litellm --config /path/to/config.yaml
# RUNNING on http://0.0.0.0:4000
```
**Create the Assistant**
```bash
curl "http://localhost:4000/v1/assistants" \
-H "Content-Type: application/json" \
-H "Authorization: Bearer sk-1234" \
-d '{
"instructions": "You are a personal math tutor. When asked a question, write and run Python code to answer the question.",
"name": "Math Tutor",
"tools": [{"type": "code_interpreter"}],
"model": "gpt-4-turbo"
}'
```
**Get the Assistant**
```bash
curl "http://0.0.0.0:4000/v1/assistants?order=desc&limit=20" \
-H "Content-Type: application/json" \
-H "Authorization: Bearer sk-1234"
```
**Create a Thread**
```bash
curl http://0.0.0.0:4000/v1/threads \
-H "Content-Type: application/json" \
-H "Authorization: Bearer sk-1234" \
-d ''
```
**Get a Thread**
```bash
curl http://0.0.0.0:4000/v1/threads/{thread_id} \
-H "Content-Type: application/json" \
-H "Authorization: Bearer sk-1234"
```
**Add Messages to the Thread**
```bash
curl http://0.0.0.0:4000/v1/threads/{thread_id}/messages \
-H "Content-Type: application/json" \
-H "Authorization: Bearer sk-1234" \
-d '{
"role": "user",
"content": "How does AI work? Explain it in simple terms."
}'
```
**Run the Assistant on the Thread**
```bash
curl http://0.0.0.0:4000/v1/threads/thread_abc123/runs \
-H "Authorization: Bearer sk-1234" \
-H "Content-Type: application/json" \
-d '{
"assistant_id": "asst_abc123"
}'
```
</TabItem>
</Tabs>
## Streaming
<Tabs>
<TabItem value="sdk" label="SDK">
```python
from litellm import run_thread_stream
import os
os.environ["OPENAI_API_KEY"] = "sk-.."
message = {"role": "user", "content": "Hey, how's it going?"}
data = {"custom_llm_provider": "openai", "thread_id": _new_thread.id, "assistant_id": assistant_id, **message}
run = run_thread_stream(**data)
with run as run:
assert isinstance(run, AssistantEventHandler)
for chunk in run:
print(f"chunk: {chunk}")
run.until_done()
```
</TabItem>
<TabItem value="proxy" label="PROXY">
```bash
curl -X POST 'http://0.0.0.0:4000/threads/{thread_id}/runs' \
-H 'Authorization: Bearer sk-1234' \
-H 'Content-Type: application/json' \
-D '{
"assistant_id": "asst_6xVZQFFy1Kw87NbnYeNebxTf",
"stream": true
}'
```
</TabItem>
</Tabs>
## [👉 Proxy API Reference](https://litellm-api.up.railway.app/#/assistants)
## OpenAI-Compatible APIs
To call openai-compatible Assistants API's (eg. Astra Assistants API), just add `openai/` to the model name:
**config**
```yaml
assistant_settings:
custom_llm_provider: openai
litellm_params:
api_key: os.environ/ASTRA_API_KEY
api_base: os.environ/ASTRA_API_BASE
```
**curl**
```bash
curl -X POST "http://localhost:4000/v1/assistants" \
-H "Content-Type: application/json" \
-H "Authorization: Bearer sk-1234" \
-d '{
"instructions": "You are a personal math tutor. When asked a question, write and run Python code to answer the question.",
"name": "Math Tutor",
"tools": [{"type": "code_interpreter"}],
"model": "openai/<my-astra-model-name>"
}'
```

2
docs/my-website/docs/audio_transcription.md
View file

@ -1,7 +1,7 @@
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
# Audio Transcription
# Speech to Text
Use this to loadbalance across Azure + OpenAI.

124
docs/my-website/docs/batches.md Normal file
View file

@ -0,0 +1,124 @@
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
# Batches API
Covers Batches, Files
## Quick Start
Call an existing Assistant.
- Create File for Batch Completion
- Create Batch Request
- Retrieve the Specific Batch and File Content
<Tabs>
<TabItem value="sdk" label="SDK">
**Create File for Batch Completion**
```python
from litellm
import os
os.environ["OPENAI_API_KEY"] = "sk-.."
file_name = "openai_batch_completions.jsonl"
_current_dir = os.path.dirname(os.path.abspath(__file__))
file_path = os.path.join(_current_dir, file_name)
file_obj = await litellm.acreate_file(
file=open(file_path, "rb"),
purpose="batch",
custom_llm_provider="openai",
)
print("Response from creating file=", file_obj)
```
**Create Batch Request**
```python
from litellm
import os
create_batch_response = await litellm.acreate_batch(
completion_window="24h",
endpoint="/v1/chat/completions",
input_file_id=batch_input_file_id,
custom_llm_provider="openai",
metadata={"key1": "value1", "key2": "value2"},
)
print("response from litellm.create_batch=", create_batch_response)
```
**Retrieve the Specific Batch and File Content**
```python
retrieved_batch = await litellm.aretrieve_batch(
batch_id=create_batch_response.id, custom_llm_provider="openai"
)
print("retrieved batch=", retrieved_batch)
# just assert that we retrieved a non None batch
assert retrieved_batch.id == create_batch_response.id
# try to get file content for our original file
file_content = await litellm.afile_content(
file_id=batch_input_file_id, custom_llm_provider="openai"
)
print("file content = ", file_content)
```
</TabItem>
<TabItem value="proxy" label="PROXY">
```bash
$ export OPENAI_API_KEY="sk-..."
$ litellm
# RUNNING on http://0.0.0.0:4000
```
**Create File for Batch Completion**
```shell
curl https://api.openai.com/v1/files \
-H "Authorization: Bearer sk-1234" \
-F purpose="batch" \
-F file="@mydata.jsonl"
```
**Create Batch Request**
```bash
curl http://localhost:4000/v1/batches \
-H "Authorization: Bearer sk-1234" \
-H "Content-Type: application/json" \
-d '{
"input_file_id": "file-abc123",
"endpoint": "/v1/chat/completions",
"completion_window": "24h"
}'
```
**Retrieve the Specific Batch**
```bash
curl http://localhost:4000/v1/batches/batch_abc123 \
-H "Authorization: Bearer sk-1234" \
-H "Content-Type: application/json" \
```
</TabItem>
</Tabs>
## [👉 Proxy API Reference](https://litellm-api.up.railway.app/#/batch)

88
docs/my-website/docs/caching/all_caches.md
View file

@ -212,6 +212,94 @@ If you run the code two times, response1 will use the cache from the first run t
</TabItem>
</Tabs>
## Switch Cache On / Off Per LiteLLM Call
LiteLLM supports 4 cache-controls:
- `no-cache`: *Optional(bool)* When `True`, Will not return a cached response, but instead call the actual endpoint.
- `no-store`: *Optional(bool)* When `True`, Will not cache the response.
- `ttl`: *Optional(int)* - Will cache the response for the user-defined amount of time (in seconds).
- `s-maxage`: *Optional(int)* Will only accept cached responses that are within user-defined range (in seconds).
[Let us know if you need more](https://github.com/BerriAI/litellm/issues/1218)
<Tabs>
<TabItem value="no-cache" label="No-Cache">
Example usage `no-cache` - When `True`, Will not return a cached response
```python
response = litellm.completion(
model="gpt-3.5-turbo",
messages=[
{
"role": "user",
"content": "hello who are you"
}
],
cache={"no-cache": True},
)
```
</TabItem>
<TabItem value="no-store" label="No-Store">
Example usage `no-store` - When `True`, Will not cache the response.
```python
response = litellm.completion(
model="gpt-3.5-turbo",
messages=[
{
"role": "user",
"content": "hello who are you"
}
],
cache={"no-store": True},
)
```
</TabItem>
<TabItem value="ttl" label="ttl">
Example usage `ttl` - cache the response for 10 seconds
```python
response = litellm.completion(
model="gpt-3.5-turbo",
messages=[
{
"role": "user",
"content": "hello who are you"
}
],
cache={"ttl": 10},
)
```
</TabItem>
<TabItem value="s-maxage" label="s-maxage">
Example usage `s-maxage` - Will only accept cached responses for 60 seconds
```python
response = litellm.completion(
model="gpt-3.5-turbo",
messages=[
{
"role": "user",
"content": "hello who are you"
}
],
cache={"s-maxage": 60},
)
```
</TabItem>
</Tabs>
## Cache Context Manager - Enable, Disable, Update Cache

94
docs/my-website/docs/completion/batching.md
View file

@ -1,3 +1,6 @@
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
# Batching Completion()
LiteLLM allows you to:
* Send many completion calls to 1 model
@ -51,6 +54,9 @@ This makes parallel calls to the specified `models` and returns the first respon
Use this to reduce latency
<Tabs>
<TabItem value="sdk" label="SDK">
### Example Code
```python
import litellm
@ -68,8 +74,93 @@ response = batch_completion_models(
print(result)
```
</TabItem>
<TabItem value="proxy" label="PROXY">
[how to setup proxy config](#example-setup)
Just pass a comma-separated string of model names and the flag `fastest_response=True`.
<Tabs>
<TabItem value="curl" label="curl">
```bash
curl -X POST 'http://localhost:4000/chat/completions' \
-H 'Content-Type: application/json' \
-H 'Authorization: Bearer sk-1234' \
-D '{
"model": "gpt-4o, groq-llama", # 👈 Comma-separated models
"messages": [
{
"role": "user",
"content": "What's the weather like in Boston today?"
}
],
"stream": true,
"fastest_response": true # 👈 FLAG
}
'
```
</TabItem>
<TabItem value="openai" label="OpenAI SDK">
```python
import openai
client = openai.OpenAI(
api_key="anything",
base_url="http://0.0.0.0:4000"
)
# request sent to model set on litellm proxy, `litellm --model`
response = client.chat.completions.create(
model="gpt-4o, groq-llama", # 👈 Comma-separated models
messages = [
{
"role": "user",
"content": "this is a test request, write a short poem"
}
],
extra_body={"fastest_response": true} # 👈 FLAG
)
print(response)
```
</TabItem>
</Tabs>
---
### Example Setup:
```yaml
model_list:
- model_name: groq-llama
litellm_params:
model: groq/llama3-8b-8192
api_key: os.environ/GROQ_API_KEY
- model_name: gpt-4o
litellm_params:
model: gpt-4o
api_key: os.environ/OPENAI_API_KEY
```
```bash
litellm --config /path/to/config.yaml
# RUNNING on http://0.0.0.0:4000
```
</TabItem>
</Tabs>
### Output
Returns the first response
Returns the first response in OpenAI format. Cancels other LLM API calls.
```json
{
"object": "chat.completion",
@ -95,6 +186,7 @@ Returns the first response
}
```
## Send 1 completion call to many models: Return All Responses
This makes parallel calls to the specified models and returns all responses

110
docs/my-website/docs/completion/drop_params.md Normal file
View file

@ -0,0 +1,110 @@
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
# Drop Unsupported Params
Drop unsupported OpenAI params by your LLM Provider.
## Quick Start
```python
import litellm
import os
# set keys
os.environ["COHERE_API_KEY"] = "co-.."
litellm.drop_params = True # 👈 KEY CHANGE
response = litellm.completion(
model="command-r",
messages=[{"role": "user", "content": "Hey, how's it going?"}],
response_format={"key": "value"},
)
```
LiteLLM maps all supported openai params by provider + model (e.g. function calling is supported by anthropic on bedrock but not titan).
See `litellm.get_supported_openai_params("command-r")` [**Code**](https://github.com/BerriAI/litellm/blob/main/litellm/utils.py#L3584)
If a provider/model doesn't support a particular param, you can drop it.
## OpenAI Proxy Usage
```yaml
litellm_settings:
drop_params: true
```
## Pass drop_params in `completion(..)`
Just drop_params when calling specific models
<Tabs>
<TabItem value="sdk" label="SDK">
```python
import litellm
import os
# set keys
os.environ["COHERE_API_KEY"] = "co-.."
response = litellm.completion(
model="command-r",
messages=[{"role": "user", "content": "Hey, how's it going?"}],
response_format={"key": "value"},
drop_params=True
)
```
</TabItem>
<TabItem value="proxy" label="PROXY">
```yaml
- litellm_params:
api_base: my-base
model: openai/my-model
drop_params: true # 👈 KEY CHANGE
model_name: my-model
```
</TabItem>
</Tabs>
## Specify params to drop
To drop specific params when calling a provider (E.g. 'logit_bias' for vllm)
Use `additional_drop_params`
<Tabs>
<TabItem value="sdk" label="SDK">
```python
import litellm
import os
# set keys
os.environ["COHERE_API_KEY"] = "co-.."
response = litellm.completion(
model="command-r",
messages=[{"role": "user", "content": "Hey, how's it going?"}],
response_format={"key": "value"},
additional_drop_params=["response_format"]
)
```
</TabItem>
<TabItem value="proxy" label="PROXY">
```yaml
- litellm_params:
api_base: my-base
model: openai/my-model
additional_drop_params: ["response_format"] # 👈 KEY CHANGE
model_name: my-model
```
</TabItem>
</Tabs>
**additional_drop_params**: List or null - Is a list of openai params you want to drop when making a call to the model.

4
docs/my-website/docs/completion/function_call.md
View file

@ -502,10 +502,10 @@ response = completion(model="gpt-3.5-turbo-0613", messages=messages, functions=f
print(response)
```
## Function calling for Non-OpenAI LLMs
## Function calling for Models w/out function-calling support
### Adding Function to prompt
For Non OpenAI LLMs LiteLLM allows you to add the function to the prompt set: `litellm.add_function_to_prompt = True`
For Models/providers without function calling support, LiteLLM allows you to add the function to the prompt set: `litellm.add_function_to_prompt = True`
#### Usage
```python

435
docs/my-website/docs/completion/input.md
View file

@ -39,38 +39,38 @@ This is a list of openai params we translate across providers.
Use `litellm.get_supported_openai_params()` for an updated list of params for each model + provider
| Provider | temperature | max_tokens | top_p | stream | stop | n | presence_penalty | frequency_penalty | functions | function_call | logit_bias | user | response_format | seed | tools | tool_choice | logprobs | top_logprobs | extra_headers |
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|--|
|Anthropic| ✅ | ✅ | ✅ | ✅ | ✅ | | | | | | |✅ | ✅ | ✅ | ✅ | ✅ | | | ✅
|OpenAI| ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |✅ | ✅ | ✅ | ✅ |✅ | ✅ | ✅ | ✅ | ✅ |
|Azure OpenAI| ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |✅ | ✅ | ✅ | ✅ |✅ | ✅ | | | ✅ |
| Provider | temperature | max_tokens | top_p | stream | stream_options | stop | n | presence_penalty | frequency_penalty | functions | function_call | logit_bias | user | response_format | seed | tools | tool_choice | logprobs | top_logprobs | extra_headers |
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
|Anthropic| ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | | | | | | |✅ | ✅ | ✅ | ✅ | ✅ | | | ✅ |
|OpenAI| ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |✅ | ✅ | ✅ | ✅ |✅ | ✅ | ✅ | ✅ | ✅ |
|Azure OpenAI| ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |✅ | ✅ | ✅ | ✅ |✅ | ✅ | | | ✅ |
|Replicate | ✅ | ✅ | ✅ | ✅ | ✅ | | | | | |
|Anyscale | ✅ | ✅ | ✅ | ✅ |
|Anyscale | ✅ | ✅ | ✅ | ✅ | ✅ |
|Cohere| ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | | |
|Huggingface| ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | | | | |
|Huggingface| ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | | | |
|Openrouter| ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | | | | | ✅ | | | | |
|AI21| ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | | |
|VertexAI| ✅ | ✅ | | ✅ | | | | | | | | | | | ✅ | | |
|VertexAI| ✅ | ✅ | | ✅ | ✅ | | | | | | | | | ✅ | ✅ | | |
|Bedrock| ✅ | ✅ | ✅ | ✅ | ✅ | | | | | | | | | | ✅ (for anthropic) | |
|Sagemaker| ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | | | | |
|Sagemaker| ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | | | |
|TogetherAI| ✅ | ✅ | ✅ | ✅ | ✅ | | | | | | ✅ |
|AlephAlpha| ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | | | | |
|Palm| ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | | | | |
|AlephAlpha| ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | | | |
|Palm| ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | | | |
|NLP Cloud| ✅ | ✅ | ✅ | ✅ | ✅ | | | | | |
|Petals| ✅ | ✅ | | ✅ | | | | | | |
|Petals| ✅ | ✅ | | ✅ | ✅ | | | | | |
|Ollama| ✅ | ✅ | ✅ | ✅ | ✅ | | | ✅ | | | | | ✅ | | |
|Databricks| ✅ | ✅ | ✅ | ✅ | ✅ | | | | | | | | | | |
|ClarifAI| ✅ | ✅ | | | | | | | | | | | | | |
|ClarifAI| ✅ | ✅ | |✅ | ✅ | | | | | | | | | | |
:::note
By default, LiteLLM raises an exception if the openai param being passed in isn't supported.
To drop the param instead, set `litellm.drop_params = True`.
To drop the param instead, set `litellm.drop_params = True` or `completion(..drop_params=True)`.
**For function calling:**
This **ONLY DROPS UNSUPPORTED OPENAI PARAMS**.
LiteLLM assumes any non-openai param is provider specific and passes it in as a kwarg in the request body
Add to prompt for non-openai models, set: `litellm.add_function_to_prompt = True`.
:::
## Input Params
@ -97,6 +97,7 @@ def completion(
seed: Optional[int] = None,
tools: Optional[List] = None,
tool_choice: Optional[str] = None,
parallel_tool_calls: Optional[bool] = None,
logprobs: Optional[bool] = None,
top_logprobs: Optional[int] = None,
deployment_id=None,
@ -166,10 +167,12 @@ def completion(
- `function`: *object* - Required.
- `tool_choice`: *string or object (optional)* - Controls which (if any) function is called by the model. none means the model will not call a function and instead generates a message. auto means the model can pick between generating a message or calling a function. Specifying a particular function via {"type: "function", "function": {"name": "my_function"}} forces the model to call that function.
- `tool_choice`: *string or object (optional)* - Controls which (if any) function is called by the model. none means the model will not call a function and instead generates a message. auto means the model can pick between generating a message or calling a function. Specifying a particular function via `{"type: "function", "function": {"name": "my_function"}}` forces the model to call that function.
- `none` is the default when no functions are present. `auto` is the default if functions are present.
- `parallel_tool_calls`: *boolean (optional)* - Whether to enable parallel function calling during tool use.. OpenAI default is true.
- `frequency_penalty`: *number or null (optional)* - It is used to penalize new tokens based on their frequency in the text so far.
- `logit_bias`: *map (optional)* - Used to modify the probability of specific tokens appearing in the completion.
@ -226,399 +229,3 @@ def completion(
- `hf_model_name`: *string (optional)* - [Sagemaker Only] The corresponding huggingface name of the model, used to pull the right chat template for the model.
## Provider-specific Params
Providers might offer params not supported by OpenAI (e.g. top_k). You can pass those in 2 ways:
- via completion(): We'll pass the non-openai param, straight to the provider as part of the request body.
- e.g. `completion(model="claude-instant-1", top_k=3)`
- via provider-specific config variable (e.g. `litellm.OpenAIConfig()`).
<Tabs>
<TabItem value="openai" label="OpenAI">
```python
import litellm, os
# set env variables
os.environ["OPENAI_API_KEY"] = "your-openai-key"
## SET MAX TOKENS - via completion()
response_1 = litellm.completion(
model="gpt-3.5-turbo",
messages=[{ "content": "Hello, how are you?","role": "user"}],
max_tokens=10
)
response_1_text = response_1.choices[0].message.content
## SET MAX TOKENS - via config
litellm.OpenAIConfig(max_tokens=10)
response_2 = litellm.completion(
model="gpt-3.5-turbo",
messages=[{ "content": "Hello, how are you?","role": "user"}],
)
response_2_text = response_2.choices[0].message.content
## TEST OUTPUT
assert len(response_2_text) > len(response_1_text)
```
</TabItem>
<TabItem value="openai-text" label="OpenAI Text Completion">
```python
import litellm, os
# set env variables
os.environ["OPENAI_API_KEY"] = "your-openai-key"
## SET MAX TOKENS - via completion()
response_1 = litellm.completion(
model="text-davinci-003",
messages=[{ "content": "Hello, how are you?","role": "user"}],
max_tokens=10
)
response_1_text = response_1.choices[0].message.content
## SET MAX TOKENS - via config
litellm.OpenAITextCompletionConfig(max_tokens=10)
response_2 = litellm.completion(
model="text-davinci-003",
messages=[{ "content": "Hello, how are you?","role": "user"}],
)
response_2_text = response_2.choices[0].message.content
## TEST OUTPUT
assert len(response_2_text) > len(response_1_text)
```
</TabItem>
<TabItem value="azure-openai" label="Azure OpenAI">
```python
import litellm, os
# set env variables
os.environ["AZURE_API_BASE"] = "your-azure-api-base"
os.environ["AZURE_API_TYPE"] = "azure" # [OPTIONAL]
os.environ["AZURE_API_VERSION"] = "2023-07-01-preview" # [OPTIONAL]
## SET MAX TOKENS - via completion()
response_1 = litellm.completion(
model="azure/chatgpt-v-2",
messages=[{ "content": "Hello, how are you?","role": "user"}],
max_tokens=10
)
response_1_text = response_1.choices[0].message.content
## SET MAX TOKENS - via config
litellm.AzureOpenAIConfig(max_tokens=10)
response_2 = litellm.completion(
model="azure/chatgpt-v-2",
messages=[{ "content": "Hello, how are you?","role": "user"}],
)
response_2_text = response_2.choices[0].message.content
## TEST OUTPUT
assert len(response_2_text) > len(response_1_text)
```
</TabItem>
<TabItem value="anthropic" label="Anthropic">
```python
import litellm, os
# set env variables
os.environ["ANTHROPIC_API_KEY"] = "your-anthropic-key"
## SET MAX TOKENS - via completion()
response_1 = litellm.completion(
model="claude-instant-1",
messages=[{ "content": "Hello, how are you?","role": "user"}],
max_tokens=10
)
response_1_text = response_1.choices[0].message.content
## SET MAX TOKENS - via config
litellm.AnthropicConfig(max_tokens_to_sample=200)
response_2 = litellm.completion(
model="claude-instant-1",
messages=[{ "content": "Hello, how are you?","role": "user"}],
)
response_2_text = response_2.choices[0].message.content
## TEST OUTPUT
assert len(response_2_text) > len(response_1_text)
```
</TabItem>
<TabItem value="huggingface" label="Huggingface">
```python
import litellm, os
# set env variables
os.environ["HUGGINGFACE_API_KEY"] = "your-huggingface-key" #[OPTIONAL]
## SET MAX TOKENS - via completion()
response_1 = litellm.completion(
model="huggingface/mistralai/Mistral-7B-Instruct-v0.1",
messages=[{ "content": "Hello, how are you?","role": "user"}],
api_base="https://your-huggingface-api-endpoint",
max_tokens=10
)
response_1_text = response_1.choices[0].message.content
## SET MAX TOKENS - via config
litellm.HuggingfaceConfig(max_new_tokens=200)
response_2 = litellm.completion(
model="huggingface/mistralai/Mistral-7B-Instruct-v0.1",
messages=[{ "content": "Hello, how are you?","role": "user"}],
api_base="https://your-huggingface-api-endpoint"
)
response_2_text = response_2.choices[0].message.content
## TEST OUTPUT
assert len(response_2_text) > len(response_1_text)
```
</TabItem>
<TabItem value="together_ai" label="TogetherAI">
```python
import litellm, os
# set env variables
os.environ["TOGETHERAI_API_KEY"] = "your-togetherai-key"
## SET MAX TOKENS - via completion()
response_1 = litellm.completion(
model="together_ai/togethercomputer/llama-2-70b-chat",
messages=[{ "content": "Hello, how are you?","role": "user"}],
max_tokens=10
)
response_1_text = response_1.choices[0].message.content
## SET MAX TOKENS - via config
litellm.TogetherAIConfig(max_tokens_to_sample=200)
response_2 = litellm.completion(
model="together_ai/togethercomputer/llama-2-70b-chat",
messages=[{ "content": "Hello, how are you?","role": "user"}],
)
response_2_text = response_2.choices[0].message.content
## TEST OUTPUT
assert len(response_2_text) > len(response_1_text)
```
</TabItem>
<TabItem value="ollama" label="Ollama">
```python
import litellm, os
## SET MAX TOKENS - via completion()
response_1 = litellm.completion(
model="ollama/llama2",
messages=[{ "content": "Hello, how are you?","role": "user"}],
max_tokens=10
)
response_1_text = response_1.choices[0].message.content
## SET MAX TOKENS - via config
litellm.OllamConfig(num_predict=200)
response_2 = litellm.completion(
model="ollama/llama2",
messages=[{ "content": "Hello, how are you?","role": "user"}],
)
response_2_text = response_2.choices[0].message.content
## TEST OUTPUT
assert len(response_2_text) > len(response_1_text)
```
</TabItem>
<TabItem value="replicate" label="Replicate">
```python
import litellm, os
# set env variables
os.environ["REPLICATE_API_KEY"] = "your-replicate-key"
## SET MAX TOKENS - via completion()
response_1 = litellm.completion(
model="replicate/meta/llama-2-70b-chat:02e509c789964a7ea8736978a43525956ef40397be9033abf9fd2badfe68c9e3",
messages=[{ "content": "Hello, how are you?","role": "user"}],
max_tokens=10
)
response_1_text = response_1.choices[0].message.content
## SET MAX TOKENS - via config
litellm.ReplicateConfig(max_new_tokens=200)
response_2 = litellm.completion(
model="replicate/meta/llama-2-70b-chat:02e509c789964a7ea8736978a43525956ef40397be9033abf9fd2badfe68c9e3",
messages=[{ "content": "Hello, how are you?","role": "user"}],
)
response_2_text = response_2.choices[0].message.content
## TEST OUTPUT
assert len(response_2_text) > len(response_1_text)
```
</TabItem>
<TabItem value="petals" label="Petals">
```python
import litellm
## SET MAX TOKENS - via completion()
response_1 = litellm.completion(
model="petals/petals-team/StableBeluga2",
messages=[{ "content": "Hello, how are you?","role": "user"}],
api_base="https://chat.petals.dev/api/v1/generate",
max_tokens=10
)
response_1_text = response_1.choices[0].message.content
## SET MAX TOKENS - via config
litellm.PetalsConfig(max_new_tokens=10)
response_2 = litellm.completion(
model="petals/petals-team/StableBeluga2",
messages=[{ "content": "Hello, how are you?","role": "user"}],
api_base="https://chat.petals.dev/api/v1/generate",
)
response_2_text = response_2.choices[0].message.content
## TEST OUTPUT
assert len(response_2_text) > len(response_1_text)
```
</TabItem>
<TabItem value="palm" label="Palm">
```python
import litellm, os
# set env variables
os.environ["PALM_API_KEY"] = "your-palm-key"
## SET MAX TOKENS - via completion()
response_1 = litellm.completion(
model="palm/chat-bison",
messages=[{ "content": "Hello, how are you?","role": "user"}],
max_tokens=10
)
response_1_text = response_1.choices[0].message.content
## SET MAX TOKENS - via config
litellm.PalmConfig(maxOutputTokens=10)
response_2 = litellm.completion(
model="palm/chat-bison",
messages=[{ "content": "Hello, how are you?","role": "user"}],
)
response_2_text = response_2.choices[0].message.content
## TEST OUTPUT
assert len(response_2_text) > len(response_1_text)
```
</TabItem>
<TabItem value="ai21" label="AI21">
```python
import litellm, os
# set env variables
os.environ["AI21_API_KEY"] = "your-ai21-key"
## SET MAX TOKENS - via completion()
response_1 = litellm.completion(
model="j2-mid",
messages=[{ "content": "Hello, how are you?","role": "user"}],
max_tokens=10
)
response_1_text = response_1.choices[0].message.content
## SET MAX TOKENS - via config
litellm.AI21Config(maxOutputTokens=10)
response_2 = litellm.completion(
model="j2-mid",
messages=[{ "content": "Hello, how are you?","role": "user"}],
)
response_2_text = response_2.choices[0].message.content
## TEST OUTPUT
assert len(response_2_text) > len(response_1_text)
```
</TabItem>
<TabItem value="cohere" label="Cohere">
```python
import litellm, os
# set env variables
os.environ["COHERE_API_KEY"] = "your-cohere-key"
## SET MAX TOKENS - via completion()
response_1 = litellm.completion(
model="command-nightly",
messages=[{ "content": "Hello, how are you?","role": "user"}],
max_tokens=10
)
response_1_text = response_1.choices[0].message.content
## SET MAX TOKENS - via config
litellm.CohereConfig(max_tokens=200)
response_2 = litellm.completion(
model="command-nightly",
messages=[{ "content": "Hello, how are you?","role": "user"}],
)
response_2_text = response_2.choices[0].message.content
## TEST OUTPUT
assert len(response_2_text) > len(response_1_text)
```
</TabItem>
</Tabs>
[**Check out the tutorial!**](../tutorials/provider_specific_params.md)

137
docs/my-website/docs/completion/json_mode.md Normal file
View file

@ -0,0 +1,137 @@
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
# JSON Mode
## Quick Start
<Tabs>
<TabItem value="sdk" label="SDK">
```python
from litellm import completion
import os
os.environ["OPENAI_API_KEY"] = ""
response = completion(
model="gpt-4o-mini",
response_format={ "type": "json_object" },
messages=[
{"role": "system", "content": "You are a helpful assistant designed to output JSON."},
{"role": "user", "content": "Who won the world series in 2020?"}
]
)
print(response.choices[0].message.content)
```
</TabItem>
<TabItem value="proxy" label="PROXY">
```bash
curl http://0.0.0.0:4000/v1/chat/completions \
-H "Content-Type: application/json" \
-H "Authorization: Bearer $LITELLM_KEY" \
-d '{
"model": "gpt-4o-mini",
"response_format": { "type": "json_object" },
"messages": [
{
"role": "system",
"content": "You are a helpful assistant designed to output JSON."
},
{
"role": "user",
"content": "Who won the world series in 2020?"
}
]
}'
```
</TabItem>
</Tabs>
## Check Model Support
Call `litellm.get_supported_openai_params` to check if a model/provider supports `response_format`.
```python
from litellm import get_supported_openai_params
params = get_supported_openai_params(model="anthropic.claude-3", custom_llm_provider="bedrock")
assert "response_format" in params
```
## Validate JSON Schema
For VertexAI models, LiteLLM supports passing the `response_schema` and validating the JSON output.
This works across Gemini (`vertex_ai_beta/`) + Anthropic (`vertex_ai/`) models.
<Tabs>
<TabItem value="sdk" label="SDK">
```python
# !gcloud auth application-default login - run this to add vertex credentials to your env
from litellm import completion
messages = [{"role": "user", "content": "List 5 cookie recipes"}]
response_schema = {
"type": "array",
"items": {
"type": "object",
"properties": {
"recipe_name": {
"type": "string",
},
},
"required": ["recipe_name"],
},
}
resp = completion(
model="vertex_ai_beta/gemini-1.5-pro",
messages=messages,
response_format={
"type": "json_object",
"response_schema": response_schema,
"enforce_validation": True, # client-side json schema validation
},
vertex_location="us-east5",
)
print("Received={}".format(resp))
```
</TabItem>
<TabItem value="proxy" label="PROXY">
```bash
curl http://0.0.0.0:4000/v1/chat/completions \
-H "Content-Type: application/json" \
-H "Authorization: Bearer $LITELLM_API_KEY" \
-d '{
"model": "vertex_ai_beta/gemini-1.5-pro",
"messages": [{"role": "user", "content": "List 5 cookie recipes"}]
"response_format": {
"type": "json_object",
"enforce_validation: true,
"response_schema": {
"type": "array",
"items": {
"type": "object",
"properties": {
"recipe_name": {
"type": "string",
},
},
"required": ["recipe_name"],
},
}
},
}'
```
</TabItem>
</Tabs>

436
docs/my-website/docs/completion/provider_specific_params.md Normal file
View file

@ -0,0 +1,436 @@
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
# Provider-specific Params
Providers might offer params not supported by OpenAI (e.g. top_k). LiteLLM treats any non-openai param, as a provider-specific param, and passes it to the provider in the request body, as a kwarg. [**See Reserved Params**](https://github.com/BerriAI/litellm/blob/aa2fd29e48245f360e771a8810a69376464b195e/litellm/main.py#L700)
You can pass those in 2 ways:
- via completion(): We'll pass the non-openai param, straight to the provider as part of the request body.
- e.g. `completion(model="claude-instant-1", top_k=3)`
- via provider-specific config variable (e.g. `litellm.OpenAIConfig()`).
## SDK Usage
<Tabs>
<TabItem value="openai" label="OpenAI">
```python
import litellm, os
# set env variables
os.environ["OPENAI_API_KEY"] = "your-openai-key"
## SET MAX TOKENS - via completion()
response_1 = litellm.completion(
model="gpt-3.5-turbo",
messages=[{ "content": "Hello, how are you?","role": "user"}],
max_tokens=10
)
response_1_text = response_1.choices[0].message.content
## SET MAX TOKENS - via config
litellm.OpenAIConfig(max_tokens=10)
response_2 = litellm.completion(
model="gpt-3.5-turbo",
messages=[{ "content": "Hello, how are you?","role": "user"}],
)
response_2_text = response_2.choices[0].message.content
## TEST OUTPUT
assert len(response_2_text) > len(response_1_text)
```
</TabItem>
<TabItem value="openai-text" label="OpenAI Text Completion">
```python
import litellm, os
# set env variables
os.environ["OPENAI_API_KEY"] = "your-openai-key"
## SET MAX TOKENS - via completion()
response_1 = litellm.completion(
model="text-davinci-003",
messages=[{ "content": "Hello, how are you?","role": "user"}],
max_tokens=10
)
response_1_text = response_1.choices[0].message.content
## SET MAX TOKENS - via config
litellm.OpenAITextCompletionConfig(max_tokens=10)
response_2 = litellm.completion(
model="text-davinci-003",
messages=[{ "content": "Hello, how are you?","role": "user"}],
)
response_2_text = response_2.choices[0].message.content
## TEST OUTPUT
assert len(response_2_text) > len(response_1_text)
```
</TabItem>
<TabItem value="azure-openai" label="Azure OpenAI">
```python
import litellm, os
# set env variables
os.environ["AZURE_API_BASE"] = "your-azure-api-base"
os.environ["AZURE_API_TYPE"] = "azure" # [OPTIONAL]
os.environ["AZURE_API_VERSION"] = "2023-07-01-preview" # [OPTIONAL]
## SET MAX TOKENS - via completion()
response_1 = litellm.completion(
model="azure/chatgpt-v-2",
messages=[{ "content": "Hello, how are you?","role": "user"}],
max_tokens=10
)
response_1_text = response_1.choices[0].message.content
## SET MAX TOKENS - via config
litellm.AzureOpenAIConfig(max_tokens=10)
response_2 = litellm.completion(
model="azure/chatgpt-v-2",
messages=[{ "content": "Hello, how are you?","role": "user"}],
)
response_2_text = response_2.choices[0].message.content
## TEST OUTPUT
assert len(response_2_text) > len(response_1_text)
```
</TabItem>
<TabItem value="anthropic" label="Anthropic">
```python
import litellm, os
# set env variables
os.environ["ANTHROPIC_API_KEY"] = "your-anthropic-key"
## SET MAX TOKENS - via completion()
response_1 = litellm.completion(
model="claude-instant-1",
messages=[{ "content": "Hello, how are you?","role": "user"}],
max_tokens=10
)
response_1_text = response_1.choices[0].message.content
## SET MAX TOKENS - via config
litellm.AnthropicConfig(max_tokens_to_sample=200)
response_2 = litellm.completion(
model="claude-instant-1",
messages=[{ "content": "Hello, how are you?","role": "user"}],
)
response_2_text = response_2.choices[0].message.content
## TEST OUTPUT
assert len(response_2_text) > len(response_1_text)
```
</TabItem>
<TabItem value="huggingface" label="Huggingface">
```python
import litellm, os
# set env variables
os.environ["HUGGINGFACE_API_KEY"] = "your-huggingface-key" #[OPTIONAL]
## SET MAX TOKENS - via completion()
response_1 = litellm.completion(
model="huggingface/mistralai/Mistral-7B-Instruct-v0.1",
messages=[{ "content": "Hello, how are you?","role": "user"}],
api_base="https://your-huggingface-api-endpoint",
max_tokens=10
)
response_1_text = response_1.choices[0].message.content
## SET MAX TOKENS - via config
litellm.HuggingfaceConfig(max_new_tokens=200)
response_2 = litellm.completion(
model="huggingface/mistralai/Mistral-7B-Instruct-v0.1",
messages=[{ "content": "Hello, how are you?","role": "user"}],
api_base="https://your-huggingface-api-endpoint"
)
response_2_text = response_2.choices[0].message.content
## TEST OUTPUT
assert len(response_2_text) > len(response_1_text)
```
</TabItem>
<TabItem value="together_ai" label="TogetherAI">
```python
import litellm, os
# set env variables
os.environ["TOGETHERAI_API_KEY"] = "your-togetherai-key"
## SET MAX TOKENS - via completion()
response_1 = litellm.completion(
model="together_ai/togethercomputer/llama-2-70b-chat",
messages=[{ "content": "Hello, how are you?","role": "user"}],
max_tokens=10
)
response_1_text = response_1.choices[0].message.content
## SET MAX TOKENS - via config
litellm.TogetherAIConfig(max_tokens_to_sample=200)
response_2 = litellm.completion(
model="together_ai/togethercomputer/llama-2-70b-chat",
messages=[{ "content": "Hello, how are you?","role": "user"}],
)
response_2_text = response_2.choices[0].message.content
## TEST OUTPUT
assert len(response_2_text) > len(response_1_text)
```
</TabItem>
<TabItem value="ollama" label="Ollama">
```python
import litellm, os
## SET MAX TOKENS - via completion()
response_1 = litellm.completion(
model="ollama/llama2",
messages=[{ "content": "Hello, how are you?","role": "user"}],
max_tokens=10
)
response_1_text = response_1.choices[0].message.content
## SET MAX TOKENS - via config
litellm.OllamConfig(num_predict=200)
response_2 = litellm.completion(
model="ollama/llama2",
messages=[{ "content": "Hello, how are you?","role": "user"}],
)
response_2_text = response_2.choices[0].message.content
## TEST OUTPUT
assert len(response_2_text) > len(response_1_text)
```
</TabItem>
<TabItem value="replicate" label="Replicate">
```python
import litellm, os
# set env variables
os.environ["REPLICATE_API_KEY"] = "your-replicate-key"
## SET MAX TOKENS - via completion()
response_1 = litellm.completion(
model="replicate/meta/llama-2-70b-chat:02e509c789964a7ea8736978a43525956ef40397be9033abf9fd2badfe68c9e3",
messages=[{ "content": "Hello, how are you?","role": "user"}],
max_tokens=10
)
response_1_text = response_1.choices[0].message.content
## SET MAX TOKENS - via config
litellm.ReplicateConfig(max_new_tokens=200)
response_2 = litellm.completion(
model="replicate/meta/llama-2-70b-chat:02e509c789964a7ea8736978a43525956ef40397be9033abf9fd2badfe68c9e3",
messages=[{ "content": "Hello, how are you?","role": "user"}],
)
response_2_text = response_2.choices[0].message.content
## TEST OUTPUT
assert len(response_2_text) > len(response_1_text)
```
</TabItem>
<TabItem value="petals" label="Petals">
```python
import litellm
## SET MAX TOKENS - via completion()
response_1 = litellm.completion(
model="petals/petals-team/StableBeluga2",
messages=[{ "content": "Hello, how are you?","role": "user"}],
api_base="https://chat.petals.dev/api/v1/generate",
max_tokens=10
)
response_1_text = response_1.choices[0].message.content
## SET MAX TOKENS - via config
litellm.PetalsConfig(max_new_tokens=10)
response_2 = litellm.completion(
model="petals/petals-team/StableBeluga2",
messages=[{ "content": "Hello, how are you?","role": "user"}],
api_base="https://chat.petals.dev/api/v1/generate",
)
response_2_text = response_2.choices[0].message.content
## TEST OUTPUT
assert len(response_2_text) > len(response_1_text)
```
</TabItem>
<TabItem value="palm" label="Palm">
```python
import litellm, os
# set env variables
os.environ["PALM_API_KEY"] = "your-palm-key"
## SET MAX TOKENS - via completion()
response_1 = litellm.completion(
model="palm/chat-bison",
messages=[{ "content": "Hello, how are you?","role": "user"}],
max_tokens=10
)
response_1_text = response_1.choices[0].message.content
## SET MAX TOKENS - via config
litellm.PalmConfig(maxOutputTokens=10)
response_2 = litellm.completion(
model="palm/chat-bison",
messages=[{ "content": "Hello, how are you?","role": "user"}],
)
response_2_text = response_2.choices[0].message.content
## TEST OUTPUT
assert len(response_2_text) > len(response_1_text)
```
</TabItem>
<TabItem value="ai21" label="AI21">
```python
import litellm, os
# set env variables
os.environ["AI21_API_KEY"] = "your-ai21-key"
## SET MAX TOKENS - via completion()
response_1 = litellm.completion(
model="j2-mid",
messages=[{ "content": "Hello, how are you?","role": "user"}],
max_tokens=10
)
response_1_text = response_1.choices[0].message.content
## SET MAX TOKENS - via config
litellm.AI21Config(maxOutputTokens=10)
response_2 = litellm.completion(
model="j2-mid",
messages=[{ "content": "Hello, how are you?","role": "user"}],
)
response_2_text = response_2.choices[0].message.content
## TEST OUTPUT
assert len(response_2_text) > len(response_1_text)
```
</TabItem>
<TabItem value="cohere" label="Cohere">
```python
import litellm, os
# set env variables
os.environ["COHERE_API_KEY"] = "your-cohere-key"
## SET MAX TOKENS - via completion()
response_1 = litellm.completion(
model="command-nightly",
messages=[{ "content": "Hello, how are you?","role": "user"}],
max_tokens=10
)
response_1_text = response_1.choices[0].message.content
## SET MAX TOKENS - via config
litellm.CohereConfig(max_tokens=200)
response_2 = litellm.completion(
model="command-nightly",
messages=[{ "content": "Hello, how are you?","role": "user"}],
)
response_2_text = response_2.choices[0].message.content
## TEST OUTPUT
assert len(response_2_text) > len(response_1_text)
```
</TabItem>
</Tabs>
[**Check out the tutorial!**](../tutorials/provider_specific_params.md)
## Proxy Usage
**via Config**
```yaml
model_list:
- model_name: llama-3-8b-instruct
litellm_params:
model: predibase/llama-3-8b-instruct
api_key: os.environ/PREDIBASE_API_KEY
tenant_id: os.environ/PREDIBASE_TENANT_ID
max_tokens: 256
adapter_base: <my-special_base> # 👈 PROVIDER-SPECIFIC PARAM
```
**via Request**
```bash
curl -X POST 'http://0.0.0.0:4000/chat/completions' \
-H 'Content-Type: application/json' \
-H 'Authorization: Bearer sk-1234' \
-D '{
"model": "llama-3-8b-instruct",
"messages": [
{
"role": "user",
"content": "What'\''s the weather like in Boston today?"
}
],
"adapater_id": "my-special-adapter-id" # 👈 PROVIDER-SPECIFIC PARAM
}'
```

14
docs/my-website/docs/completion/reliable_completions.md
View file

@ -31,9 +31,15 @@ response = completion(
)
```
## Fallbacks
## Fallbacks (SDK)
### Context Window Fallbacks
:::info
[See how to do on PROXY](../proxy/reliability.md)
:::
### Context Window Fallbacks (SDK)
```python
from litellm import completion
@ -43,7 +49,7 @@ messages = [{"content": "how does a court case get to the Supreme Court?" * 500,
completion(model="gpt-3.5-turbo", messages=messages, context_window_fallback_dict=ctx_window_fallback_dict)
```
### Fallbacks - Switch Models/API Keys/API Bases
### Fallbacks - Switch Models/API Keys/API Bases (SDK)
LLM APIs can be unstable, completion() with fallbacks ensures you'll always get a response from your calls
@ -69,7 +75,7 @@ response = completion(model="azure/gpt-4", messages=messages, api_key=api_key,
[Check out this section for implementation details](#fallbacks-1)
## Implementation Details
## Implementation Details (SDK)
### Fallbacks
#### Output from calls

18
docs/my-website/docs/completion/token_usage.md
View file

@ -1,7 +1,21 @@
# Completion Token Usage & Cost
By default LiteLLM returns token usage in all completion requests ([See here](https://litellm.readthedocs.io/en/latest/output/))
However, we also expose some helper functions + **[NEW]** an API to calculate token usage across providers:
LiteLLM returns `response_cost` in all calls.
```python
from litellm import completion
response = litellm.completion(
model="gpt-3.5-turbo",
messages=[{"role": "user", "content": "Hey, how's it going?"}],
mock_response="Hello world",
)
print(response._hidden_params["response_cost"])
```
LiteLLM also exposes some helper functions:
- `encode`: This encodes the text passed in, using the model-specific tokenizer. [**Jump to code**](#1-encode)
@ -23,7 +37,7 @@ However, we also expose some helper functions + **[NEW]** an API to calculate to
- `api.litellm.ai`: Live token + price count across [all supported models](https://github.com/BerriAI/litellm/blob/main/model_prices_and_context_window.json). [**Jump to code**](#10-apilitellmai)
📣 This is a community maintained list. Contributions are welcome! ❤️
📣 [This is a community maintained list](https://github.com/BerriAI/litellm/blob/main/model_prices_and_context_window.json). Contributions are welcome! ❤️
## Example Usage

2
docs/my-website/docs/completion/vision.md
View file

@ -39,7 +39,7 @@ Use `litellm.supports_vision(model="")` -> returns `True` if model supports `vis
```python
assert litellm.supports_vision(model="gpt-4-vision-preview") == True
assert litellm.supports_vision(model="gemini-1.0-pro-visionn") == True
assert litellm.supports_vision(model="gemini-1.0-pro-vision") == True
assert litellm.supports_vision(model="gpt-3.5-turbo") == False
```

42
docs/my-website/docs/data_security.md Normal file
View file

@ -0,0 +1,42 @@
# Data Privacy and Security
## Security Measures
### LiteLLM Cloud
- We encrypt all data stored using your `LITELLM_MASTER_KEY` and in transit using TLS.
- Our database and application run on GCP, AWS infrastructure, partly managed by NeonDB.
- US data region: Northern California (AWS/GCP `us-west-1`) & Virginia (AWS `us-east-1`)
- EU data region Germany/Frankfurt (AWS/GCP `eu-central-1`)
- All users have access to SSO (Single Sign-On) through OAuth 2.0 with Google, Okta, Microsoft, KeyCloak.
- Audit Logs with retention policy
- Control Allowed IP Addresses that can access your Cloud LiteLLM Instance
For security inquiries, please contact us at support@berri.ai
## Self-hosted Instances LiteLLM
- ** No data or telemetry is stored on LiteLLM Servers when you self host **
- For installation and configuration, see: [Self-hosting guided](../docs/proxy/deploy.md)
- **Telemetry** We run no telemetry when you self host LiteLLM
For security inquiries, please contact us at support@berri.ai
### Supported data regions for LiteLLM Cloud
LiteLLM supports the following data regions:
- US, Northern California (AWS/GCP `us-west-1`)
- Europe, Frankfurt, Germany (AWS/GCP `eu-central-1`)
All data, user accounts, and infrastructure are completely separated between these two regions
### Security Vulnerability Reporting Guidelines
We value the security community's role in protecting our systems and users. To report a security vulnerability:
- Email support@berri.ai with details
- Include steps to reproduce the issue
- Provide any relevant additional information
We'll review all reports promptly. Note that we don't currently offer a bug bounty program.

90
docs/my-website/docs/debugging/hosted_debugging.md
View file

@ -1,90 +0,0 @@
import Image from '@theme/IdealImage';
import QueryParamReader from '../../src/components/queryParamReader.js'
# [Beta] Monitor Logs in Production
:::note
This is in beta. Expect frequent updates, as we improve based on your feedback.
:::
LiteLLM provides an integration to let you monitor logs in production.
👉 Jump to our sample LiteLLM Dashboard: https://admin.litellm.ai/
<Image img={require('../../img/alt_dashboard.png')} alt="Dashboard" />
## Debug your first logs
<a target="_blank" href="https://colab.research.google.com/github/BerriAI/litellm/blob/main/cookbook/liteLLM_OpenAI.ipynb">
<img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"/>
</a>
### 1. Get your LiteLLM Token
Go to [admin.litellm.ai](https://admin.litellm.ai/) and copy the code snippet with your unique token
<Image img={require('../../img/hosted_debugger_usage_page.png')} alt="Usage" />
### 2. Set up your environment
**Add it to your .env**
```python
import os
os.env["LITELLM_TOKEN"] = "e24c4c06-d027-4c30-9e78-18bc3a50aebb" # replace with your unique token
```
**Turn on LiteLLM Client**
```python
import litellm
litellm.client = True
```
### 3. Make a normal `completion()` call
```python
import litellm
from litellm import completion
import os
# set env variables
os.environ["LITELLM_TOKEN"] = "e24c4c06-d027-4c30-9e78-18bc3a50aebb" # replace with your unique token
os.environ["OPENAI_API_KEY"] = "openai key"
litellm.use_client = True # enable logging dashboard
messages = [{ "content": "Hello, how are you?","role": "user"}]
# openai call
response = completion(model="gpt-3.5-turbo", messages=messages)
```
Your `completion()` call print with a link to your session dashboard (https://admin.litellm.ai/<your_unique_token>)
In the above case it would be: [`admin.litellm.ai/e24c4c06-d027-4c30-9e78-18bc3a50aebb`](https://admin.litellm.ai/e24c4c06-d027-4c30-9e78-18bc3a50aebb)
Click on your personal dashboard link. Here's how you can find it 👇
<Image img={require('../../img/dash_output.png')} alt="Dashboard" />
[👋 Tell us if you need better privacy controls](https://calendly.com/d/4mp-gd3-k5k/berriai-1-1-onboarding-litellm-hosted-version?month=2023-08)
### 3. Review request log
Oh! Looks like our request was made successfully. Let's click on it and see exactly what got sent to the LLM provider.
Ah! So we can see that this request was made to a **Baseten** (see litellm_params > custom_llm_provider) for a model with ID - **7qQNLDB** (see model). The message sent was - `"Hey, how's it going?"` and the response received was - `"As an AI language model, I don't have feelings or emotions, but I can assist you with your queries. How can I assist you today?"`
<Image img={require('../../img/dashboard_log.png')} alt="Dashboard Log Row" />
:::info
🎉 Congratulations! You've successfully debugger your first log!
:::

74
docs/my-website/docs/embedding/supported_embedding.md
View file

@ -85,6 +85,17 @@ print(query_result[:5])
</Tabs>
## Input Params for `litellm.embedding()`
:::info
Any non-openai params, will be treated as provider-specific params, and sent in the request body as kwargs to the provider.
[**See Reserved Params**](https://github.com/BerriAI/litellm/blob/2f5f85cb52f36448d1f8bbfbd3b8af8167d0c4c8/litellm/main.py#L3130)
[**See Example**](#example)
:::
### Required Fields
- `model`: *string* - ID of the model to use. `model='text-embedding-ada-002'`
@ -363,3 +374,66 @@ All models listed here https://docs.voyageai.com/embeddings/#models-and-specific
| voyage-01 | `embedding(model="voyage/voyage-01", input)` |
| voyage-lite-01 | `embedding(model="voyage/voyage-lite-01", input)` |
| voyage-lite-01-instruct | `embedding(model="voyage/voyage-lite-01-instruct", input)` |
## Provider-specific Params
:::info
Any non-openai params, will be treated as provider-specific params, and sent in the request body as kwargs to the provider.
[**See Reserved Params**](https://github.com/BerriAI/litellm/blob/2f5f85cb52f36448d1f8bbfbd3b8af8167d0c4c8/litellm/main.py#L3130)
:::
### **Example**
Cohere v3 Models have a required parameter: `input_type`, it can be one of the following four values:
- `input_type="search_document"`: (default) Use this for texts (documents) you want to store in your vector database
- `input_type="search_query"`: Use this for search queries to find the most relevant documents in your vector database
- `input_type="classification"`: Use this if you use the embeddings as an input for a classification system
- `input_type="clustering"`: Use this if you use the embeddings for text clustering
https://txt.cohere.com/introducing-embed-v3/
<Tabs>
<TabItem value="sdk" label="SDK">
```python
from litellm import embedding
os.environ["COHERE_API_KEY"] = "cohere key"
# cohere call
response = embedding(
model="embed-english-v3.0",
input=["good morning from litellm", "this is another item"],
input_type="search_document" # 👈 PROVIDER-SPECIFIC PARAM
)
```
</TabItem>
<TabItem value="proxy" label="PROXY">
**via config**
```yaml
model_list:
- model_name: "cohere-embed"
litellm_params:
model: embed-english-v3.0
input_type: search_document # 👈 PROVIDER-SPECIFIC PARAM
```
**via request**
```bash
curl -X POST 'http://0.0.0.0:4000/v1/embeddings' \
-H 'Authorization: Bearer sk-54d77cd67b9febbb' \
-H 'Content-Type: application/json' \
-d '{
"model": "cohere-embed",
"input": ["Are you authorized to work in United States of America?"],
"input_type": "search_document" # 👈 PROVIDER-SPECIFIC PARAM
}'
```
</TabItem>
</Tabs>

58
docs/my-website/docs/enterprise.md
View file

@ -2,38 +2,64 @@
For companies that need SSO, user management and professional support for LiteLLM Proxy
:::info
Interested in Enterprise? Schedule a meeting with us here 👉
[Talk to founders](https://calendly.com/d/4mp-gd3-k5k/litellm-1-1-onboarding-chat)
:::
This covers:
- ✅ **Features under the [LiteLLM Commercial License (Content Mod, Custom Tags, etc.)](https://docs.litellm.ai/docs/proxy/enterprise)**
- ✅ [**Secure UI access with Single Sign-On**](../docs/proxy/ui.md#setup-ssoauth-for-ui)
- ✅ [**JWT-Auth**](../docs/proxy/token_auth.md)
- ✅ [**Prompt Injection Detection**](#prompt-injection-detection-lakeraai)
- ✅ [**Invite Team Members to access `/spend` Routes**](../docs/proxy/cost_tracking#allowing-non-proxy-admins-to-access-spend-endpoints)
- ✅ **Feature Prioritization**
- ✅ **Custom Integrations**
- ✅ **Professional Support - Dedicated discord + slack**
- ✅ **Custom SLAs**
## [COMING SOON] AWS Marketplace Support
Deploy managed LiteLLM Proxy within your VPC.
Includes all enterprise features.
[**View AWS Marketplace Listing**](https://aws.amazon.com/marketplace/pp/prodview-gdm3gswgjhgjo?sr=0-1&ref_=beagle&applicationId=AWSMPContessa)
[**Get early access**](https://calendly.com/d/4mp-gd3-k5k/litellm-1-1-onboarding-chat)
This covers:
- **Enterprise Features**
- **Security**
- ✅ [SSO for Admin UI](./proxy/ui#✨-enterprise-features)
- ✅ [Audit Logs with retention policy](./proxy/enterprise#audit-logs)
- ✅ [JWT-Auth](../docs/proxy/token_auth.md)
- ✅ [Control available public, private routes](./proxy/enterprise#control-available-public-private-routes)
- ✅ [[BETA] AWS Key Manager v2 - Key Decryption](./proxy/enterprise#beta-aws-key-manager---key-decryption)
- ✅ IP address‑based access control lists
- ✅ Track Request IP Address
- ✅ [Use LiteLLM keys/authentication on Pass Through Endpoints](./proxy/pass_through#✨-enterprise---use-litellm-keysauthentication-on-pass-through-endpoints)
- ✅ Set Max Request / File Size on Requests
- ✅ [Enforce Required Params for LLM Requests (ex. Reject requests missing ["metadata"]["generation_name"])](./proxy/enterprise#enforce-required-params-for-llm-requests)
- **Spend Tracking**
- ✅ [Tracking Spend for Custom Tags](./proxy/enterprise#tracking-spend-for-custom-tags)
- ✅ [API Endpoints to get Spend Reports per Team, API Key, Customer](./proxy/cost_tracking.md#✨-enterprise-api-endpoints-to-get-spend)
- **Advanced Metrics**
- ✅ [`x-ratelimit-remaining-requests`, `x-ratelimit-remaining-tokens` for LLM APIs on Prometheus](./proxy/prometheus#✨-enterprise-llm-remaining-requests-and-remaining-tokens)
- **Guardrails, PII Masking, Content Moderation**
- ✅ [Content Moderation with LLM Guard, LlamaGuard, Secret Detection, Google Text Moderations](./proxy/enterprise#content-moderation)
- ✅ [Prompt Injection Detection (with LakeraAI API)](./proxy/enterprise#prompt-injection-detection---lakeraai)
- ✅ Reject calls from Blocked User list
- ✅ Reject calls (incoming / outgoing) with Banned Keywords (e.g. competitors)
- **Custom Branding**
- ✅ [Custom Branding + Routes on Swagger Docs](./proxy/enterprise#swagger-docs---custom-routes--branding)
- ✅ [Public Model Hub](../docs/proxy/enterprise.md#public-model-hub)
- ✅ [Custom Email Branding](../docs/proxy/email.md#customizing-email-branding)
- ✅ **Feature Prioritization**
- ✅ **Custom Integrations**
- ✅ **Professional Support - Dedicated discord + slack**
## Frequently Asked Questions
### What topics does Professional support cover and what SLAs do you offer?
Professional Support can assist with LLM/Provider integrations, deployment, upgrade management, and LLM Provider troubleshooting. We can’t solve your own infrastructure-related issues but we will guide you to fix them.
We offer custom SLAs based on your needs and the severity of the issue. The standard SLA is 6 hours for Sev0-Sev1 severity and 24h for Sev2-Sev3 between 7am – 7pm PT (Monday through Saturday).
- 1 hour for Sev0 issues
- 6 hours for Sev1
- 24h for Sev2-Sev3 between 7am – 7pm PT (Monday through Saturday)
**We can offer custom SLAs** based on your needs and the severity of the issue
### What’s the cost of the Self-Managed Enterprise edition?

3
docs/my-website/docs/getting_started.md
View file

@ -87,13 +87,14 @@ from litellm import completion
## set env variables for logging tools
os.environ["LUNARY_PUBLIC_KEY"] = "your-lunary-public-key"
os.environ["HELICONE_API_KEY"] = "your-helicone-key"
os.environ["LANGFUSE_PUBLIC_KEY"] = ""
os.environ["LANGFUSE_SECRET_KEY"] = ""
os.environ["OPENAI_API_KEY"]
# set callbacks
litellm.success_callback = ["lunary", "langfuse"] # log input/output to langfuse, lunary, supabase
litellm.success_callback = ["lunary", "langfuse", "helicone"] # log input/output to langfuse, lunary, supabase, helicone
#openai call
response = completion(model="gpt-3.5-turbo", messages=[{"role": "user", "content": "Hi 👋 - i'm openai"}])

8
docs/my-website/docs/hosted.md
View file

@ -21,6 +21,14 @@ See our status page for [**live reliability**](https://status.litellm.ai/)
- **Reliable**: Our hosted proxy is tested on 1k requests per second, making it reliable for high load.
- **Secure**: LiteLLM is currently undergoing SOC-2 compliance, to make sure your data is as secure as possible.
## Data Privacy & Security
You can find our [data privacy & security policy for cloud litellm here](../docs/data_security#litellm-cloud)
## Supported data regions for LiteLLM Cloud
You can find [supported data regions litellm here](../docs/data_security#supported-data-regions-for-litellm-cloud)
### Pricing
Pricing is based on usage. We can figure out a price that works for your team, on the call.

73
docs/my-website/docs/image_generation.md
View file

@ -14,7 +14,76 @@ response = image_generation(prompt="A cute baby sea otter", model="dall-e-3")
print(f"response: {response}")
```
### Input Params for `litellm.image_generation()`
## Proxy Usage
### Setup config.yaml
```yaml
model_list:
- model_name: dall-e-2 ### RECEIVED MODEL NAME ###
litellm_params: # all params accepted by litellm.image_generation()
model: azure/dall-e-2 ### MODEL NAME sent to `litellm.image_generation()` ###
api_base: https://my-endpoint-europe-berri-992.openai.azure.com/
api_key: "os.environ/AZURE_API_KEY_EU" # does os.getenv("AZURE_API_KEY_EU")
rpm: 6 # [OPTIONAL] Rate limit for this deployment: in requests per minute (rpm)
```
### Start proxy
```bash
litellm --config /path/to/config.yaml
# RUNNING on http://0.0.0.0:4000
```
### Test
<Tabs>
<TabItem value="curl" label="Curl">
```bash
curl -X POST 'http://0.0.0.0:4000/v1/images/generations' \
-H 'Content-Type: application/json' \
-H 'Authorization: Bearer sk-1234' \
-D '{
"model": "dall-e-2",
"prompt": "A cute baby sea otter",
"n": 1,
"size": "1024x1024"
}'
```
</TabItem>
<TabItem value="openai" label="OpenAI">
```python
from openai import OpenAI
client = openai.OpenAI(
api_key="sk-1234",
base_url="http://0.0.0.0:4000"
)
image = client.images.generate(
prompt="A cute baby sea otter",
model="dall-e-3",
)
print(image)
```
</TabItem>
</Tabs>
## Input Params for `litellm.image_generation()`
:::info
Any non-openai params, will be treated as provider-specific params, and sent in the request body as kwargs to the provider.
[**See Reserved Params**](https://github.com/BerriAI/litellm/blob/2f5f85cb52f36448d1f8bbfbd3b8af8167d0c4c8/litellm/main.py#L4082)
:::
### Required Fields
- `prompt`: *string* - A text description of the desired image(s).
@ -51,7 +120,7 @@ print(f"response: {response}")
- `api_base`: *string (optional)* - The api endpoint you want to call the model with
- `api_version`: *string (optional)* - (Azure-specific) the api version for the call
- `api_version`: *string (optional)* - (Azure-specific) the api version for the call; required for dall-e-3 on Azure
- `api_key`: *string (optional)* - The API key to authenticate and authorize requests. If not provided, the default API key is used.

3
docs/my-website/docs/index.md
View file

@ -310,6 +310,7 @@ LiteLLM exposes pre defined callbacks to send data to Lunary, Langfuse, Helicone
from litellm import completion
## set env variables for logging tools
os.environ["HELICONE_API_KEY"] = "your-helicone-key"
os.environ["LANGFUSE_PUBLIC_KEY"] = ""
os.environ["LANGFUSE_SECRET_KEY"] = ""
os.environ["LUNARY_PUBLIC_KEY"] = "your-lunary-public-key"
@ -317,7 +318,7 @@ os.environ["LUNARY_PUBLIC_KEY"] = "your-lunary-public-key"
os.environ["OPENAI_API_KEY"]
# set callbacks
litellm.success_callback = ["lunary", "langfuse"] # log input/output to lunary, langfuse, supabase
litellm.success_callback = ["lunary", "langfuse", "helicone"] # log input/output to lunary, langfuse, supabase, helicone
#openai call
response = completion(model="gpt-3.5-turbo", messages=[{"role": "user", "content": "Hi 👋 - i'm openai"}])

9
docs/my-website/docs/observability/athina_integration.md
View file

@ -2,6 +2,15 @@ import Image from '@theme/IdealImage';
# Athina
:::tip
This is community maintained, Please make an issue if you run into a bug
https://github.com/BerriAI/litellm
:::
[Athina](https://athina.ai/) is an evaluation framework and production monitoring platform for your LLM-powered app. Athina is designed to enhance the performance and reliability of AI applications through real-time monitoring, granular analytics, and plug-and-play evaluations.
<Image img={require('../../img/athina_dashboard.png')} />

4
docs/my-website/docs/observability/callbacks.md
View file

@ -7,15 +7,17 @@ liteLLM provides `input_callbacks`, `success_callbacks` and `failure_callbacks`,
liteLLM supports:
- [Custom Callback Functions](https://docs.litellm.ai/docs/observability/custom_callback)
- [Lunary](https://lunary.ai/docs)
- [Langfuse](https://langfuse.com/docs)
- [Helicone](https://docs.helicone.ai/introduction)
- [Traceloop](https://traceloop.com/docs)
- [Lunary](https://lunary.ai/docs)
- [Athina](https://docs.athina.ai/)
- [Sentry](https://docs.sentry.io/platforms/python/)
- [PostHog](https://posthog.com/docs/libraries/python)
- [Slack](https://slack.dev/bolt-python/concepts)
This is **not** an extensive list. Please check the dropdown for all logging integrations.
### Quick Start
```python

2
docs/my-website/docs/observability/custom_callback.md
View file

@ -38,7 +38,7 @@ class MyCustomHandler(CustomLogger):
print(f"On Async Success")
async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time):
print(f"On Async Success")
print(f"On Async Failure")
customHandler = MyCustomHandler()

9
docs/my-website/docs/observability/greenscale_integration.md
View file

@ -1,5 +1,14 @@
# Greenscale - Track LLM Spend and Responsible Usage
:::tip
This is community maintained, Please make an issue if you run into a bug
https://github.com/BerriAI/litellm
:::
[Greenscale](https://greenscale.ai/) is a production monitoring platform for your LLM-powered app that provides you granular key insights into your GenAI spending and responsible usage. Greenscale only captures metadata to minimize the exposure risk of personally identifiable information (PII).
## Getting Started

191
docs/my-website/docs/observability/helicone_integration.md
View file

@ -1,55 +1,170 @@
# Helicone Tutorial
[Helicone](https://helicone.ai/) is an open source observability platform that proxies your OpenAI traffic and provides you key insights into your spend, latency and usage.
# 🧠 Helicone - OSS LLM Observability Platform
## Use Helicone to log requests across all LLM Providers (OpenAI, Azure, Anthropic, Cohere, Replicate, PaLM)
liteLLM provides `success_callbacks` and `failure_callbacks`, making it easy for you to send data to a particular provider depending on the status of your responses.
:::tip
In this case, we want to log requests to Helicone when a request succeeds.
This is community maintained. Please make an issue if you run into a bug:
https://github.com/BerriAI/litellm
:::
[Helicone](https://helicone.ai/) is an open source observability platform that proxies your LLM requests and provides key insights into your usage, spend, latency and more.
## Using Helicone with LiteLLM
LiteLLM provides `success_callbacks` and `failure_callbacks`, allowing you to easily log data to Helicone based on the status of your responses.
### Supported LLM Providers
Helicone can log requests across [various LLM providers](https://docs.helicone.ai/getting-started/quick-start), including:
- OpenAI
- Azure
- Anthropic
- Gemini
- Groq
- Cohere
- Replicate
- And more
### Integration Methods
There are two main approaches to integrate Helicone with LiteLLM:
1. Using callbacks
2. Using Helicone as a proxy
Let's explore each method in detail.
### Approach 1: Use Callbacks
Use just 1 line of code, to instantly log your responses **across all providers** with helicone:
Use just 1 line of code to instantly log your responses **across all providers** with Helicone:
```python
litellm.success_callback = ["helicone"]
```
Complete code
```python
from litellm import completion
## set env variables
os.environ["HELICONE_API_KEY"] = "your-helicone-key"
os.environ["OPENAI_API_KEY"], os.environ["COHERE_API_KEY"] = "", ""
# set callbacks
litellm.success_callback=["helicone"]
#openai call
response = completion(model="gpt-3.5-turbo", messages=[{"role": "user", "content": "Hi 👋 - i'm openai"}])
#cohere call
response = completion(model="command-nightly", messages=[{"role": "user", "content": "Hi 👋 - i'm cohere"}])
```
### Approach 2: [OpenAI + Azure only] Use Helicone as a proxy
Helicone provides advanced functionality like caching, etc. Helicone currently supports this for Azure and OpenAI.
If you want to use Helicone to proxy your OpenAI/Azure requests, then you can -
- Set helicone as your base url via: `litellm.api_url`
- Pass in helicone request headers via: `litellm.headers`
Complete Code
```python
import litellm
import os
from litellm import completion
litellm.api_base = "https://oai.hconeai.com/v1"
litellm.headers = {"Helicone-Auth": f"Bearer {os.getenv('HELICONE_API_KEY')}"}
## Set env variables
os.environ["HELICONE_API_KEY"] = "your-helicone-key"
os.environ["OPENAI_API_KEY"] = "your-openai-key"
response = litellm.completion(
model="gpt-3.5-turbo",
messages=[{"role": "user", "content": "how does a court case get to the Supreme Court?"}]
# Set callbacks
litellm.success_callback = ["helicone"]
# OpenAI call
response = completion(
model="gpt-4o",
messages=[{"role": "user", "content": "Hi 👋 - I'm OpenAI"}],
)
print(response)
```
### Approach 2: Use Helicone as a proxy
Helicone's proxy provides [advanced functionality](https://docs.helicone.ai/getting-started/proxy-vs-async) like caching, rate limiting, LLM security through [PromptArmor](https://promptarmor.com/) and more.
To use Helicone as a proxy for your LLM requests:
1. Set Helicone as your base URL via: litellm.api_base
2. Pass in Helicone request headers via: litellm.metadata
Complete Code:
```python
import os
import litellm
from litellm import completion
litellm.api_base = "https://oai.hconeai.com/v1"
litellm.headers = {
"Helicone-Auth": f"Bearer {os.getenv('HELICONE_API_KEY')}", # Authenticate to send requests to Helicone API
}
response = litellm.completion(
model="gpt-3.5-turbo",
messages=[{"role": "user", "content": "How does a court case get to the Supreme Court?"}]
)
print(response)
```
### Advanced Usage
You can add custom metadata and properties to your requests using Helicone headers. Here are some examples:
```python
litellm.metadata = {
"Helicone-Auth": f"Bearer {os.getenv('HELICONE_API_KEY')}", # Authenticate to send requests to Helicone API
"Helicone-User-Id": "user-abc", # Specify the user making the request
"Helicone-Property-App": "web", # Custom property to add additional information
"Helicone-Property-Custom": "any-value", # Add any custom property
"Helicone-Prompt-Id": "prompt-supreme-court", # Assign an ID to associate this prompt with future versions
"Helicone-Cache-Enabled": "true", # Enable caching of responses
"Cache-Control": "max-age=3600", # Set cache limit to 1 hour
"Helicone-RateLimit-Policy": "10;w=60;s=user", # Set rate limit policy
"Helicone-Retry-Enabled": "true", # Enable retry mechanism
"helicone-retry-num": "3", # Set number of retries
"helicone-retry-factor": "2", # Set exponential backoff factor
"Helicone-Model-Override": "gpt-3.5-turbo-0613", # Override the model used for cost calculation
"Helicone-Session-Id": "session-abc-123", # Set session ID for tracking
"Helicone-Session-Path": "parent-trace/child-trace", # Set session path for hierarchical tracking
"Helicone-Omit-Response": "false", # Include response in logging (default behavior)
"Helicone-Omit-Request": "false", # Include request in logging (default behavior)
"Helicone-LLM-Security-Enabled": "true", # Enable LLM security features
"Helicone-Moderations-Enabled": "true", # Enable content moderation
"Helicone-Fallbacks": '["gpt-3.5-turbo", "gpt-4"]', # Set fallback models
}
```
### Caching and Rate Limiting
Enable caching and set up rate limiting policies:
```python
litellm.metadata = {
"Helicone-Auth": f"Bearer {os.getenv('HELICONE_API_KEY')}", # Authenticate to send requests to Helicone API
"Helicone-Cache-Enabled": "true", # Enable caching of responses
"Cache-Control": "max-age=3600", # Set cache limit to 1 hour
"Helicone-RateLimit-Policy": "100;w=3600;s=user", # Set rate limit policy
}
```
### Session Tracking and Tracing
Track multi-step and agentic LLM interactions using session IDs and paths:
```python
litellm.metadata = {
"Helicone-Auth": f"Bearer {os.getenv('HELICONE_API_KEY')}", # Authenticate to send requests to Helicone API
"Helicone-Session-Id": "session-abc-123", # The session ID you want to track
"Helicone-Session-Path": "parent-trace/child-trace", # The path of the session
}
```
- `Helicone-Session-Id`: Use this to specify the unique identifier for the session you want to track. This allows you to group related requests together.
- `Helicone-Session-Path`: This header defines the path of the session, allowing you to represent parent and child traces. For example, "parent/child" represents a child trace of a parent trace.
By using these two headers, you can effectively group and visualize multi-step LLM interactions, gaining insights into complex AI workflows.
### Retry and Fallback Mechanisms
Set up retry mechanisms and fallback options:
```python
litellm.metadata = {
"Helicone-Auth": f"Bearer {os.getenv('HELICONE_API_KEY')}", # Authenticate to send requests to Helicone API
"Helicone-Retry-Enabled": "true", # Enable retry mechanism
"helicone-retry-num": "3", # Set number of retries
"helicone-retry-factor": "2", # Set exponential backoff factor
"Helicone-Fallbacks": '["gpt-3.5-turbo", "gpt-4"]', # Set fallback models
}
```
> **Supported Headers** - For a full list of supported Helicone headers and their descriptions, please refer to the [Helicone documentation](https://docs.helicone.ai/getting-started/quick-start).
> By utilizing these headers and metadata options, you can gain deeper insights into your LLM usage, optimize performance, and better manage your AI workflows with Helicone and LiteLLM.

28
docs/my-website/docs/observability/langfuse_integration.md
View file

@ -1,6 +1,6 @@
import Image from '@theme/IdealImage';
# Langfuse - Logging LLM Input/Output
# 🔥 Langfuse - Logging LLM Input/Output
LangFuse is open Source Observability & Analytics for LLM Apps
Detailed production traces and a granular view on quality, cost and latency
@ -122,10 +122,12 @@ response = completion(
metadata={
"generation_name": "ishaan-test-generation", # set langfuse Generation Name
"generation_id": "gen-id22", # set langfuse Generation ID
"parent_observation_id": "obs-id9" # set langfuse Parent Observation ID
"version": "test-generation-version" # set langfuse Generation Version
"trace_user_id": "user-id2", # set langfuse Trace User ID
"session_id": "session-1", # set langfuse Session ID
"tags": ["tag1", "tag2"], # set langfuse Tags
"trace_name": "new-trace-name" # set langfuse Trace Name
"trace_id": "trace-id22", # set langfuse Trace ID
"trace_metadata": {"key": "value"}, # set langfuse Trace Metadata
"trace_version": "test-trace-version", # set langfuse Trace Version (if not set, defaults to Generation Version)
@ -144,6 +146,27 @@ print(response)
```
You can also pass `metadata` as part of the request header with a `langfuse_*` prefix:
```shell
curl --location --request POST 'http://0.0.0.0:4000/chat/completions' \
--header 'Content-Type: application/json' \
--header 'Authorization: Bearer sk-1234' \
--header 'langfuse_trace_id: trace-id2' \
--header 'langfuse_trace_user_id: user-id2' \
--header 'langfuse_trace_metadata: {"key":"value"}' \
--data '{
"model": "gpt-3.5-turbo",
"messages": [
{
"role": "user",
"content": "what llm are you"
}
]
}'
```
### Trace & Generation Parameters
#### Trace Specific Parameters
@ -172,7 +195,8 @@ The following parameters can be updated on a continuation of a trace by passing
* `generation_id` - Identifier for the generation, auto-generated by default
* `generation_name` - Identifier for the generation, auto-generated by default
* `prompt` - Langfuse prompt object used for the generation, defaults to None
* `parent_observation_id` - Identifier for the parent observation, defaults to `None`
* `prompt` - Langfuse prompt object used for the generation, defaults to `None`
Any other key value pairs passed into the metadata not listed in the above spec for a `litellm` completion will be added as a metadata key value pair for the generation.

12
docs/my-website/docs/observability/langsmith_integration.md
View file

@ -1,10 +1,20 @@
import Image from '@theme/IdealImage';
# Langsmith - Logging LLM Input/Output
:::tip
This is community maintained, Please make an issue if you run into a bug
https://github.com/BerriAI/litellm
:::
An all-in-one developer platform for every step of the application lifecycle
https://smith.langchain.com/
<Image img={require('../../img/langsmith.png')} />
<Image img={require('../../img/langsmith_new.png')} />
:::info
We want to learn how we can make the callbacks better! Meet the LiteLLM [founders](https://calendly.com/d/4mp-gd3-k5k/berriai-1-1-onboarding-litellm-hosted-version) or

13
docs/my-website/docs/observability/logfire_integration.md
View file

@ -1,6 +1,6 @@
import Image from '@theme/IdealImage';
# Logfire - Logging LLM Input/Output
# 🔥 Logfire - Logging LLM Input/Output
Logfire is open Source Observability & Analytics for LLM Apps
Detailed production traces and a granular view on quality, cost and latency
@ -14,10 +14,14 @@ join our [discord](https://discord.gg/wuPM9dRgDw)
## Pre-Requisites
Ensure you have run `pip install logfire` for this integration
Ensure you have installed the following packages to use this integration
```shell
pip install logfire litellm
pip install litellm
pip install opentelemetry-api==1.25.0
pip install opentelemetry-sdk==1.25.0
pip install opentelemetry-exporter-otlp==1.25.0
```
## Quick Start
@ -25,8 +29,7 @@ pip install logfire litellm
Get your Logfire token from [Logfire](https://logfire.pydantic.dev/)
```python
litellm.success_callback = ["logfire"]
litellm.failure_callback = ["logfire"] # logs errors to logfire
litellm.callbacks = ["logfire"]
```
```python

8
docs/my-website/docs/observability/lunary_integration.md
View file

@ -1,5 +1,13 @@
# Lunary - Logging and tracing LLM input/output
:::tip
This is community maintained, Please make an issue if you run into a bug
https://github.com/BerriAI/litellm
:::
[Lunary](https://lunary.ai/) is an open-source AI developer platform providing observability, prompt management, and evaluation tools for AI developers.
<video controls width='900' >

11
docs/my-website/docs/observability/promptlayer_integration.md
View file

@ -1,5 +1,16 @@
import Image from '@theme/IdealImage';
# Promptlayer Tutorial
:::tip
This is community maintained, Please make an issue if you run into a bug
https://github.com/BerriAI/litellm
:::
Promptlayer is a platform for prompt engineers. Log OpenAI requests. Search usage history. Track performance. Visually manage prompt templates.
<Image img={require('../../img/promptlayer.png')} />

46
docs/my-website/docs/observability/raw_request_response.md Normal file
View file

@ -0,0 +1,46 @@
import Image from '@theme/IdealImage';
# Raw Request/Response Logging
See the raw request/response sent by LiteLLM in your logging provider (OTEL/Langfuse/etc.).
**on SDK**
```python
# pip install langfuse
import litellm
import os
# log raw request/response
litellm.log_raw_request_response = True
# from https://cloud.langfuse.com/
os.environ["LANGFUSE_PUBLIC_KEY"] = ""
os.environ["LANGFUSE_SECRET_KEY"] = ""
# Optional, defaults to https://cloud.langfuse.com
os.environ["LANGFUSE_HOST"] # optional
# LLM API Keys
os.environ['OPENAI_API_KEY']=""
# set langfuse as a callback, litellm will send the data to langfuse
litellm.success_callback = ["langfuse"]
# openai call
response = litellm.completion(
model="gpt-3.5-turbo",
messages=[
{"role": "user", "content": "Hi 👋 - i'm openai"}
]
)
```
**on Proxy**
```yaml
litellm_settings:
log_raw_request_response: True
```
**Expected Log**
<Image img={require('../../img/raw_request_log.png')}/>

97
docs/my-website/docs/observability/scrub_data.md Normal file
View file

@ -0,0 +1,97 @@
# Scrub Logged Data
Redact messages / mask PII before sending data to logging integrations (langfuse/etc.).
See our [**Presidio PII Masking**](https://github.com/BerriAI/litellm/blob/a176feeacc5fdf504747978d82056eb84679c4be/litellm/proxy/hooks/presidio_pii_masking.py#L286) for reference.
1. Setup a custom callback
```python
from litellm.integrations.custom_logger import CustomLogger
class MyCustomHandler(CustomLogger):
async def async_logging_hook(
self, kwargs: dict, result: Any, call_type: str
) -> Tuple[dict, Any]:
"""
For masking logged request/response. Return a modified version of the request/result.
Called before `async_log_success_event`.
"""
if (
call_type == "completion" or call_type == "acompletion"
): # /chat/completions requests
messages: Optional[List] = kwargs.get("messages", None)
kwargs["messages"] = [{"role": "user", "content": "MASK_THIS_ASYNC_VALUE"}]
return kwargs, responses
def logging_hook(
self, kwargs: dict, result: Any, call_type: str
) -> Tuple[dict, Any]:
"""
For masking logged request/response. Return a modified version of the request/result.
Called before `log_success_event`.
"""
if (
call_type == "completion" or call_type == "acompletion"
): # /chat/completions requests
messages: Optional[List] = kwargs.get("messages", None)
kwargs["messages"] = [{"role": "user", "content": "MASK_THIS_SYNC_VALUE"}]
return kwargs, responses
customHandler = MyCustomHandler()
```
2. Connect custom handler to LiteLLM
```python
import litellm
litellm.callbacks = [customHandler]
```
3. Test it!
```python
# pip install langfuse
import os
import litellm
from litellm import completion
os.environ["LANGFUSE_PUBLIC_KEY"] = ""
os.environ["LANGFUSE_SECRET_KEY"] = ""
# Optional, defaults to https://cloud.langfuse.com
os.environ["LANGFUSE_HOST"] # optional
# LLM API Keys
os.environ['OPENAI_API_KEY']=""
litellm.callbacks = [customHandler]
litellm.success_callback = ["langfuse"]
## sync
response = completion(model="gpt-3.5-turbo", messages=[{ "role": "user", "content": "Hi 👋 - i'm openai"}],
stream=True)
for chunk in response:
continue
## async
import asyncio
def async completion():
response = await acompletion(model="gpt-3.5-turbo", messages=[{ "role": "user", "content": "Hi 👋 - i'm openai"}],
stream=True)
async for chunk in response:
continue
asyncio.run(completion())
```

9
docs/my-website/docs/observability/sentry.md
View file

@ -1,5 +1,14 @@
import Image from '@theme/IdealImage';
:::tip
This is community maintained, Please make an issue if you run into a bug
https://github.com/BerriAI/litellm
:::
# Sentry - Log LLM Exceptions
[Sentry](https://sentry.io/) provides error monitoring for production. LiteLLM can add breadcrumbs and send exceptions to Sentry with this integration

8
docs/my-website/docs/observability/supabase_integration.md
View file

@ -1,4 +1,12 @@
# Supabase Tutorial
:::tip
This is community maintained, Please make an issue if you run into a bug
https://github.com/BerriAI/litellm
:::
[Supabase](https://supabase.com/) is an open source Firebase alternative.
Start your project with a Postgres database, Authentication, instant APIs, Edge Functions, Realtime subscriptions, Storage, and Vector embeddings.

9
docs/my-website/docs/observability/telemetry.md
View file

@ -1,13 +1,8 @@
# Telemetry
LiteLLM contains a telemetry feature that tells us what models are used, and what errors are hit.
There is no Telemetry on LiteLLM - no data is stored by us
## What is logged?
Only the model name and exception raised is logged.
NOTHING - no data is sent to LiteLLM Servers
## Why?
We use this information to help us understand how LiteLLM is used, and improve stability.
## Opting out
If you prefer to opt out of telemetry, you can do this by setting `litellm.telemetry = False`.

10
docs/my-website/docs/observability/wandb_integration.md
View file

@ -1,6 +1,16 @@
import Image from '@theme/IdealImage';
# Weights & Biases - Logging LLM Input/Output
:::tip
This is community maintained, Please make an issue if you run into a bug
https://github.com/BerriAI/litellm
:::
Weights & Biases helps AI developers build better models faster https://wandb.ai
<Image img={require('../../img/wandb.png')} />

5
docs/my-website/docs/projects/llm_cord.md Normal file
View file

@ -0,0 +1,5 @@
# llmcord.py
llmcord.py lets you and your friends chat with LLMs directly in your Discord server. It works with practically any LLM, remote or locally hosted.
Github: https://github.com/jakobdylanc/discord-llm-chatbot

34
docs/my-website/docs/providers/anthropic.md
View file

@ -2,8 +2,9 @@ import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
# Anthropic
LiteLLM supports
LiteLLM supports all anthropic models.
- `claude-3.5` (`claude-3-5-sonnet-20240620`)
- `claude-3` (`claude-3-haiku-20240307`, `claude-3-opus-20240229`, `claude-3-sonnet-20240229`)
- `claude-2`
- `claude-2.1`
@ -11,7 +12,7 @@ LiteLLM supports
:::info
Anthropic API fails requests when `max_tokens` are not passed. Due to this litellm passes `max_tokens=4096` when no `max_tokens` are passed
Anthropic API fails requests when `max_tokens` are not passed. Due to this litellm passes `max_tokens=4096` when no `max_tokens` are passed.
:::
@ -167,16 +168,34 @@ print(response)
## Supported Models
`Model Name` 👉 Human-friendly name.
`Function Call` 👉 How to call the model in LiteLLM.
| Model Name | Function Call |
|------------------|--------------------------------------------|
| claude-3-5-sonnet | `completion('claude-3-5-sonnet-20240620', messages)` | `os.environ['ANTHROPIC_API_KEY']` |
| claude-3-haiku | `completion('claude-3-haiku-20240307', messages)` | `os.environ['ANTHROPIC_API_KEY']` |
| claude-3-opus | `completion('claude-3-opus-20240229', messages)` | `os.environ['ANTHROPIC_API_KEY']` |
| claude-3-5-sonnet-20240620 | `completion('claude-3-5-sonnet-20240620', messages)` | `os.environ['ANTHROPIC_API_KEY']` |
| claude-3-sonnet | `completion('claude-3-sonnet-20240229', messages)` | `os.environ['ANTHROPIC_API_KEY']` |
| claude-2.1 | `completion('claude-2.1', messages)` | `os.environ['ANTHROPIC_API_KEY']` |
| claude-2 | `completion('claude-2', messages)` | `os.environ['ANTHROPIC_API_KEY']` |
| claude-instant-1.2 | `completion('claude-instant-1.2', messages)` | `os.environ['ANTHROPIC_API_KEY']` |
| claude-instant-1 | `completion('claude-instant-1', messages)` | `os.environ['ANTHROPIC_API_KEY']` |
## Passing Extra Headers to Anthropic API
Pass `extra_headers: dict` to `litellm.completion`
```python
from litellm import completion
messages = [{"role": "user", "content": "What is Anthropic?"}]
response = completion(
model="claude-3-5-sonnet-20240620",
messages=messages,
extra_headers={"anthropic-beta": "max-tokens-3-5-sonnet-2024-07-15"}
)
```
## Advanced
## Usage - Function Calling
@ -229,17 +248,6 @@ assert isinstance(
```
### Setting `anthropic-beta` Header in Requests
Pass the the `extra_headers` param to litellm, All headers will be forwarded to Anthropic API
```python
response = completion(
model="anthropic/claude-3-opus-20240229",
messages=messages,
tools=tools,
)
```
### Forcing Anthropic Tool Use

4
docs/my-website/docs/providers/azure.md
View file

@ -68,6 +68,7 @@ response = litellm.completion(
| Model Name | Function Call |
|------------------|----------------------------------------|
| gpt-4o | `completion('azure/<your deployment name>', messages)` |
| gpt-4 | `completion('azure/<your deployment name>', messages)` |
| gpt-4-0314 | `completion('azure/<your deployment name>', messages)` |
| gpt-4-0613 | `completion('azure/<your deployment name>', messages)` |
@ -85,7 +86,8 @@ response = litellm.completion(
## Azure OpenAI Vision Models
| Model Name | Function Call |
|-----------------------|-----------------------------------------------------------------|
| gpt-4-vision | `response = completion(model="azure/<your deployment name>", messages=messages)` |
| gpt-4-vision | `completion(model="azure/<your deployment name>", messages=messages)` |
| gpt-4o | `completion('azure/<your deployment name>', messages)` |
#### Usage
```python

163
docs/my-website/docs/providers/azure_ai.md
View file

@ -3,49 +3,151 @@ import TabItem from '@theme/TabItem';
# Azure AI Studio
**Ensure the following:**
1. The API Base passed ends in the `/v1/` prefix
example:
```python
api_base = "https://Mistral-large-dfgfj-serverless.eastus2.inference.ai.azure.com/v1/"
```
LiteLLM supports all models on Azure AI Studio
2. The `model` passed is listed in [supported models](#supported-models). You **DO NOT** Need to pass your deployment name to litellm. Example `model=azure/Mistral-large-nmefg`
## Usage
<Tabs>
<TabItem value="sdk" label="SDK">
### ENV VAR
```python
import litellm
response = litellm.completion(
model="azure/command-r-plus",
api_base="<your-deployment-base>/v1/"
api_key="eskk******"
messages=[{"role": "user", "content": "What is the meaning of life?"}],
import os
os.environ["AZURE_AI_API_KEY"] = ""
os.environ["AZURE_AI_API_BASE"] = ""
```
### Example Call
```python
from litellm import completion
import os
## set ENV variables
os.environ["AZURE_AI_API_KEY"] = "azure ai key"
os.environ["AZURE_AI_API_BASE"] = "azure ai base url" # e.g.: https://Mistral-large-dfgfj-serverless.eastus2.inference.ai.azure.com/
# predibase llama-3 call
response = completion(
model="azure_ai/command-r-plus",
messages = [{ "content": "Hello, how are you?","role": "user"}]
)
```
</TabItem>
<TabItem value="proxy" label="PROXY">
## Sample Usage - LiteLLM Proxy
1. Add models to your config.yaml
```yaml
model_list:
- model_name: mistral
litellm_params:
model: azure/mistral-large-latest
api_base: https://Mistral-large-dfgfj-serverless.eastus2.inference.ai.azure.com/v1/
api_key: JGbKodRcTp****
- model_name: command-r-plus
litellm_params:
model: azure/command-r-plus
api_key: os.environ/AZURE_COHERE_API_KEY
api_base: os.environ/AZURE_COHERE_API_BASE
model: azure_ai/command-r-plus
api_key: os.environ/AZURE_AI_API_KEY
api_base: os.environ/AZURE_AI_API_BASE
```
2. Start the proxy
```bash
$ litellm --config /path/to/config.yaml --debug
```
3. Send Request to LiteLLM Proxy Server
<Tabs>
<TabItem value="openai" label="OpenAI Python v1.0.0+">
```python
import openai
client = openai.OpenAI(
api_key="sk-1234", # pass litellm proxy key, if you're using virtual keys
base_url="http://0.0.0.0:4000" # litellm-proxy-base url
)
response = client.chat.completions.create(
model="command-r-plus",
messages = [
{
"role": "system",
"content": "Be a good human!"
},
{
"role": "user",
"content": "What do you know about earth?"
}
]
)
print(response)
```
</TabItem>
<TabItem value="curl" label="curl">
```shell
curl --location 'http://0.0.0.0:4000/chat/completions' \
--header 'Authorization: Bearer sk-1234' \
--header 'Content-Type: application/json' \
--data '{
"model": "command-r-plus",
"messages": [
{
"role": "system",
"content": "Be a good human!"
},
{
"role": "user",
"content": "What do you know about earth?"
}
],
}'
```
</TabItem>
</Tabs>
</TabItem>
</Tabs>
## Passing additional params - max_tokens, temperature
See all litellm.completion supported params [here](../completion/input.md#translated-openai-params)
```python
# !pip install litellm
from litellm import completion
import os
## set ENV variables
os.environ["AZURE_AI_API_KEY"] = "azure ai api key"
os.environ["AZURE_AI_API_BASE"] = "azure ai api base"
# command r plus call
response = completion(
model="azure_ai/command-r-plus",
messages = [{ "content": "Hello, how are you?","role": "user"}],
max_tokens=20,
temperature=0.5
)
```
**proxy**
```yaml
model_list:
- model_name: command-r-plus
litellm_params:
model: azure_ai/command-r-plus
api_key: os.environ/AZURE_AI_API_KEY
api_base: os.environ/AZURE_AI_API_BASE
max_tokens: 20
temperature: 0.5
```
@ -103,9 +205,6 @@ response = litellm.completion(
</Tabs>
</TabItem>
</Tabs>
## Function Calling
<Tabs>
@ -115,8 +214,8 @@ response = litellm.completion(
from litellm import completion
# set env
os.environ["AZURE_MISTRAL_API_KEY"] = "your-api-key"
os.environ["AZURE_MISTRAL_API_BASE"] = "your-api-base"
os.environ["AZURE_AI_API_KEY"] = "your-api-key"
os.environ["AZURE_AI_API_BASE"] = "your-api-base"
tools = [
{
@ -141,9 +240,7 @@ tools = [
messages = [{"role": "user", "content": "What's the weather like in Boston today?"}]
response = completion(
model="azure/mistral-large-latest",
api_base=os.getenv("AZURE_MISTRAL_API_BASE")
api_key=os.getenv("AZURE_MISTRAL_API_KEY")
model="azure_ai/mistral-large-latest",
messages=messages,
tools=tools,
tool_choice="auto",
@ -206,10 +303,12 @@ curl http://0.0.0.0:4000/v1/chat/completions \
## Supported Models
LiteLLM supports **ALL** azure ai models. Here's a few examples:
| Model Name | Function Call |
|--------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------|
| Cohere command-r-plus | `completion(model="azure/command-r-plus", messages)` |
| Cohere ommand-r | `completion(model="azure/command-r", messages)` |
| Cohere command-r | `completion(model="azure/command-r", messages)` |
| mistral-large-latest | `completion(model="azure/mistral-large-latest", messages)` |

275
docs/my-website/docs/providers/bedrock.md
View file

@ -40,36 +40,36 @@ response = completion(
Here's how to call Anthropic with the LiteLLM Proxy Server
### 1. Save key in your environment
```bash
export AWS_ACCESS_KEY_ID=""
export AWS_SECRET_ACCESS_KEY=""
export AWS_REGION_NAME=""
```
### 2. Start the proxy
<Tabs>
<TabItem value="cli" label="CLI">
```bash
$ litellm --model anthropic.claude-3-sonnet-20240229-v1:0
# Server running on http://0.0.0.0:4000
```
</TabItem>
<TabItem value="config" label="config.yaml">
### 1. Setup config.yaml
```yaml
model_list:
- model_name: bedrock-claude-v1
litellm_params:
model: bedrock/anthropic.claude-instant-v1
aws_access_key_id: os.environ/CUSTOM_AWS_ACCESS_KEY_ID
aws_secret_access_key: os.environ/CUSTOM_AWS_SECRET_ACCESS_KEY
aws_region_name: os.environ/CUSTOM_AWS_REGION_NAME
```
</TabItem>
</Tabs>
All possible auth params:
```
aws_access_key_id: Optional[str],
aws_secret_access_key: Optional[str],
aws_session_token: Optional[str],
aws_region_name: Optional[str],
aws_session_name: Optional[str],
aws_profile_name: Optional[str],
aws_role_name: Optional[str],
aws_web_identity_token: Optional[str],
```
### 2. Start the proxy
```bash
litellm --config /path/to/config.yaml
```
### 3. Test it
@ -144,16 +144,135 @@ print(response)
</TabItem>
</Tabs>
## Set temperature, top p, etc.
<Tabs>
<TabItem value="sdk" label="SDK">
```python
import os
from litellm import completion
os.environ["AWS_ACCESS_KEY_ID"] = ""
os.environ["AWS_SECRET_ACCESS_KEY"] = ""
os.environ["AWS_REGION_NAME"] = ""
response = completion(
model="bedrock/anthropic.claude-3-sonnet-20240229-v1:0",
messages=[{ "content": "Hello, how are you?","role": "user"}],
temperature=0.7,
top_p=1
)
```
</TabItem>
<TabItem value="proxy" label="PROXY">
**Set on yaml**
```yaml
model_list:
- model_name: bedrock-claude-v1
litellm_params:
model: bedrock/anthropic.claude-instant-v1
temperature: <your-temp>
top_p: <your-top-p>
```
**Set on request**
```python
import openai
client = openai.OpenAI(
api_key="anything",
base_url="http://0.0.0.0:4000"
)
# request sent to model set on litellm proxy, `litellm --model`
response = client.chat.completions.create(model="bedrock-claude-v1", messages = [
{
"role": "user",
"content": "this is a test request, write a short poem"
}
],
temperature=0.7,
top_p=1
)
print(response)
```
</TabItem>
</Tabs>
## Pass provider-specific params
If you pass a non-openai param to litellm, we'll assume it's provider-specific and send it as a kwarg in the request body. [See more](../completion/input.md#provider-specific-params)
<Tabs>
<TabItem value="sdk" label="SDK">
```python
import os
from litellm import completion
os.environ["AWS_ACCESS_KEY_ID"] = ""
os.environ["AWS_SECRET_ACCESS_KEY"] = ""
os.environ["AWS_REGION_NAME"] = ""
response = completion(
model="bedrock/anthropic.claude-3-sonnet-20240229-v1:0",
messages=[{ "content": "Hello, how are you?","role": "user"}],
top_k=1 # 👈 PROVIDER-SPECIFIC PARAM
)
```
</TabItem>
<TabItem value="proxy" label="PROXY">
**Set on yaml**
```yaml
model_list:
- model_name: bedrock-claude-v1
litellm_params:
model: bedrock/anthropic.claude-instant-v1
top_k: 1 # 👈 PROVIDER-SPECIFIC PARAM
```
**Set on request**
```python
import openai
client = openai.OpenAI(
api_key="anything",
base_url="http://0.0.0.0:4000"
)
# request sent to model set on litellm proxy, `litellm --model`
response = client.chat.completions.create(model="bedrock-claude-v1", messages = [
{
"role": "user",
"content": "this is a test request, write a short poem"
}
],
temperature=0.7,
extra_body={
top_k=1 # 👈 PROVIDER-SPECIFIC PARAM
}
)
print(response)
```
</TabItem>
</Tabs>
## Usage - Function Calling
:::info
Claude returns it's output as an XML Tree. [Here is how we translate it](https://github.com/BerriAI/litellm/blob/49642a5b00a53b1babc1a753426a8afcac85dbbe/litellm/llms/prompt_templates/factory.py#L734).
You can see the raw response via `response._hidden_params["original_response"]`.
Claude hallucinates, e.g. returning the list param `value` as `<value>\n<item>apple</item>\n<item>banana</item>\n</value>` or `<value>\n<list>\n<item>apple</item>\n<item>banana</item>\n</list>\n</value>`.
:::
LiteLLM uses Bedrock's Converse API for making tool calls
```python
from litellm import completion
@ -361,47 +480,6 @@ response = completion(
)
```
### Passing an external BedrockRuntime.Client as a parameter - Completion()
Pass an external BedrockRuntime.Client object as a parameter to litellm.completion. Useful when using an AWS credentials profile, SSO session, assumed role session, or if environment variables are not available for auth.
Create a client from session credentials:
```python
import boto3
from litellm import completion
bedrock = boto3.client(
service_name="bedrock-runtime",
region_name="us-east-1",
aws_access_key_id="",
aws_secret_access_key="",
aws_session_token="",
)
response = completion(
model="bedrock/anthropic.claude-instant-v1",
messages=[{ "content": "Hello, how are you?","role": "user"}],
aws_bedrock_client=bedrock,
)
```
Create a client from AWS profile in `~/.aws/config`:
```python
import boto3
from litellm import completion
dev_session = boto3.Session(profile_name="dev-profile")
bedrock = dev_session.client(
service_name="bedrock-runtime",
region_name="us-east-1",
)
response = completion(
model="bedrock/anthropic.claude-instant-v1",
messages=[{ "content": "Hello, how are you?","role": "user"}],
aws_bedrock_client=bedrock,
)
```
### SSO Login (AWS Profile)
- Set `AWS_PROFILE` environment variable
- Make bedrock completion call
@ -464,6 +542,60 @@ response = completion(
)
```
### Passing an external BedrockRuntime.Client as a parameter - Completion()
:::warning
This is a deprecated flow. Boto3 is not async. And boto3.client does not let us make the http call through httpx. Pass in your aws params through the method above 👆. [See Auth Code](https://github.com/BerriAI/litellm/blob/55a20c7cce99a93d36a82bf3ae90ba3baf9a7f89/litellm/llms/bedrock_httpx.py#L284) [Add new auth flow](https://github.com/BerriAI/litellm/issues)
Experimental - 2024-Jun-23:
`aws_access_key_id`, `aws_secret_access_key`, and `aws_session_token` will be extracted from boto3.client and be passed into the httpx client
:::
Pass an external BedrockRuntime.Client object as a parameter to litellm.completion. Useful when using an AWS credentials profile, SSO session, assumed role session, or if environment variables are not available for auth.
Create a client from session credentials:
```python
import boto3
from litellm import completion
bedrock = boto3.client(
service_name="bedrock-runtime",
region_name="us-east-1",
aws_access_key_id="",
aws_secret_access_key="",
aws_session_token="",
)
response = completion(
model="bedrock/anthropic.claude-instant-v1",
messages=[{ "content": "Hello, how are you?","role": "user"}],
aws_bedrock_client=bedrock,
)
```
Create a client from AWS profile in `~/.aws/config`:
```python
import boto3
from litellm import completion
dev_session = boto3.Session(profile_name="dev-profile")
bedrock = dev_session.client(
service_name="bedrock-runtime",
region_name="us-east-1",
)
response = completion(
model="bedrock/anthropic.claude-instant-v1",
messages=[{ "content": "Hello, how are you?","role": "user"}],
aws_bedrock_client=bedrock,
)
```
## Provisioned throughput models
To use provisioned throughput Bedrock models pass
- `model=bedrock/<base-model>`, example `model=bedrock/anthropic.claude-v2`. Set `model` to any of the [Supported AWS models](#supported-aws-bedrock-models)
@ -495,6 +627,7 @@ Here's an example of using a bedrock model with LiteLLM
| Model Name | Command |
|----------------------------|------------------------------------------------------------------|
| Anthropic Claude-V3.5 Sonnet | `completion(model='bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0', messages=messages)` | `os.environ['AWS_ACCESS_KEY_ID']`, `os.environ['AWS_SECRET_ACCESS_KEY']` |
| Anthropic Claude-V3 sonnet | `completion(model='bedrock/anthropic.claude-3-sonnet-20240229-v1:0', messages=messages)` | `os.environ['AWS_ACCESS_KEY_ID']`, `os.environ['AWS_SECRET_ACCESS_KEY']` |
| Anthropic Claude-V3 Haiku | `completion(model='bedrock/anthropic.claude-3-haiku-20240307-v1:0', messages=messages)` | `os.environ['AWS_ACCESS_KEY_ID']`, `os.environ['AWS_SECRET_ACCESS_KEY']` |
| Anthropic Claude-V3 Opus | `completion(model='bedrock/anthropic.claude-3-opus-20240229-v1:0', messages=messages)` | `os.environ['AWS_ACCESS_KEY_ID']`, `os.environ['AWS_SECRET_ACCESS_KEY']` |

14
docs/my-website/docs/providers/clarifai.md
View file

@ -1,10 +1,13 @@
# 🆕 Clarifai
# Clarifai
Anthropic, OpenAI, Mistral, Llama and Gemini LLMs are Supported on Clarifai.
:::warning
Streaming is not yet supported on using clarifai and litellm. Tracking support here: https://github.com/BerriAI/litellm/issues/4162
:::
## Pre-Requisites
`pip install clarifai`
`pip install litellm`
## Required Environment Variables
@ -12,6 +15,7 @@ To obtain your Clarifai Personal access token follow this [link](https://docs.cl
```python
os.environ["CLARIFAI_API_KEY"] = "YOUR_CLARIFAI_PAT" # CLARIFAI_PAT
```
## Usage
@ -68,7 +72,7 @@ Example Usage - Note: liteLLM supports all models deployed on Clarifai
| clarifai/meta.Llama-2.codeLlama-70b-Python | `completion('clarifai/meta.Llama-2.codeLlama-70b-Python', messages)`|
| clarifai/meta.Llama-2.codeLlama-70b-Instruct | `completion('clarifai/meta.Llama-2.codeLlama-70b-Instruct', messages)` |
## Mistal LLMs
## Mistral LLMs
| Model Name | Function Call |
|---------------------------------------------|------------------------------------------------------------------------|
| clarifai/mistralai.completion.mixtral-8x22B | `completion('clarifai/mistralai.completion.mixtral-8x22B', messages)` |

255
docs/my-website/docs/providers/codestral.md Normal file
View file

@ -0,0 +1,255 @@
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
# Codestral API [Mistral AI]
Codestral is available in select code-completion plugins but can also be queried directly. See the documentation for more details.
## API Key
```python
# env variable
os.environ['CODESTRAL_API_KEY']
```
## FIM / Completions
:::info
Official Mistral API Docs: https://docs.mistral.ai/api/#operation/createFIMCompletion
:::
<Tabs>
<TabItem value="no-streaming" label="No Streaming">
#### Sample Usage
```python
import os
import litellm
os.environ['CODESTRAL_API_KEY']
response = await litellm.atext_completion(
model="text-completion-codestral/codestral-2405",
prompt="def is_odd(n): \n return n % 2 == 1 \ndef test_is_odd():",
suffix="return True", # optional
temperature=0, # optional
top_p=1, # optional
max_tokens=10, # optional
min_tokens=10, # optional
seed=10, # optional
stop=["return"], # optional
)
```
#### Expected Response
```json
{
"id": "b41e0df599f94bc1a46ea9fcdbc2aabe",
"object": "text_completion",
"created": 1589478378,
"model": "codestral-latest",
"choices": [
{
"text": "\n assert is_odd(1)\n assert",
"index": 0,
"logprobs": null,
"finish_reason": "length"
}
],
"usage": {
"prompt_tokens": 5,
"completion_tokens": 7,
"total_tokens": 12
}
}
```
</TabItem>
<TabItem value="stream" label="Streaming">
#### Sample Usage - Streaming
```python
import os
import litellm
os.environ['CODESTRAL_API_KEY']
response = await litellm.atext_completion(
model="text-completion-codestral/codestral-2405",
prompt="def is_odd(n): \n return n % 2 == 1 \ndef test_is_odd():",
suffix="return True", # optional
temperature=0, # optional
top_p=1, # optional
stream=True,
seed=10, # optional
stop=["return"], # optional
)
async for chunk in response:
print(chunk)
```
#### Expected Response
```json
{
"id": "726025d3e2d645d09d475bb0d29e3640",
"object": "text_completion",
"created": 1718659669,
"choices": [
{
"text": "This",
"index": 0,
"logprobs": null,
"finish_reason": null
}
],
"model": "codestral-2405",
}
```
</TabItem>
</Tabs>
### Supported Models
All models listed here https://docs.mistral.ai/platform/endpoints are supported. We actively maintain the list of models, pricing, token window, etc. [here](https://github.com/BerriAI/litellm/blob/main/model_prices_and_context_window.json).
| Model Name | Function Call |
|----------------|--------------------------------------------------------------|
| Codestral Latest | `completion(model="text-completion-codestral/codestral-latest", messages)` |
| Codestral 2405 | `completion(model="text-completion-codestral/codestral-2405", messages)`|
## Chat Completions
:::info
Official Mistral API Docs: https://docs.mistral.ai/api/#operation/createChatCompletion
:::
<Tabs>
<TabItem value="no-streaming" label="No Streaming">
#### Sample Usage
```python
import os
import litellm
os.environ['CODESTRAL_API_KEY']
response = await litellm.acompletion(
model="codestral/codestral-latest",
messages=[
{
"role": "user",
"content": "Hey, how's it going?",
}
],
temperature=0.0, # optional
top_p=1, # optional
max_tokens=10, # optional
safe_prompt=False, # optional
seed=12, # optional
)
```
#### Expected Response
```json
{
"id": "chatcmpl-123",
"object": "chat.completion",
"created": 1677652288,
"model": "codestral/codestral-latest",
"system_fingerprint": None,
"choices": [{
"index": 0,
"message": {
"role": "assistant",
"content": "\n\nHello there, how may I assist you today?",
},
"logprobs": null,
"finish_reason": "stop"
}],
"usage": {
"prompt_tokens": 9,
"completion_tokens": 12,
"total_tokens": 21
}
}
```
</TabItem>
<TabItem value="stream" label="Streaming">
#### Sample Usage - Streaming
```python
import os
import litellm
os.environ['CODESTRAL_API_KEY']
response = await litellm.acompletion(
model="codestral/codestral-latest",
messages=[
{
"role": "user",
"content": "Hey, how's it going?",
}
],
stream=True, # optional
temperature=0.0, # optional
top_p=1, # optional
max_tokens=10, # optional
safe_prompt=False, # optional
seed=12, # optional
)
async for chunk in response:
print(chunk)
```
#### Expected Response
```json
{
"id":"chatcmpl-123",
"object":"chat.completion.chunk",
"created":1694268190,
"model": "codestral/codestral-latest",
"system_fingerprint": None,
"choices":[
{
"index":0,
"delta":{"role":"assistant","content":"gm"},
"logprobs":null,
" finish_reason":null
}
]
}
```
</TabItem>
</Tabs>
### Supported Models
All models listed here https://docs.mistral.ai/platform/endpoints are supported. We actively maintain the list of models, pricing, token window, etc. [here](https://github.com/BerriAI/litellm/blob/main/model_prices_and_context_window.json).
| Model Name | Function Call |
|----------------|--------------------------------------------------------------|
| Codestral Latest | `completion(model="codestral/codestral-latest", messages)` |
| Codestral 2405 | `completion(model="codestral/codestral-2405", messages)`|

4
docs/my-website/docs/providers/cohere.md
View file

@ -68,7 +68,7 @@ response = embedding(
```
### Setting - Input Type for v3 models
v3 Models have a required parameter: `input_type`, it can be one of the following four values:
v3 Models have a required parameter: `input_type`. LiteLLM defaults to `search_document`. It can be one of the following four values:
- `input_type="search_document"`: (default) Use this for texts (documents) you want to store in your vector database
- `input_type="search_query"`: Use this for search queries to find the most relevant documents in your vector database
@ -76,6 +76,8 @@ v3 Models have a required parameter: `input_type`, it can be one of the followin
- `input_type="clustering"`: Use this if you use the embeddings for text clustering
https://txt.cohere.com/introducing-embed-v3/
```python
from litellm import embedding
os.environ["COHERE_API_KEY"] = "cohere key"

22
docs/my-website/docs/providers/databricks.md
View file

@ -27,7 +27,7 @@ import os
os.environ["DATABRICKS_API_KEY"] = "databricks key"
os.environ["DATABRICKS_API_BASE"] = "databricks base url" # e.g.: https://adb-3064715882934586.6.azuredatabricks.net/serving-endpoints
# predibase llama-3 call
# Databricks dbrx-instruct call
response = completion(
model="databricks/databricks-dbrx-instruct",
messages = [{ "content": "Hello, how are you?","role": "user"}]
@ -125,11 +125,12 @@ See all litellm.completion supported params [here](../completion/input.md#transl
from litellm import completion
import os
## set ENV variables
os.environ["PREDIBASE_API_KEY"] = "predibase key"
os.environ["DATABRICKS_API_KEY"] = "databricks key"
os.environ["DATABRICKS_API_BASE"] = "databricks api base"
# predibae llama-3 call
# databricks dbrx call
response = completion(
model="predibase/llama3-8b-instruct",
model="databricks/databricks-dbrx-instruct",
messages = [{ "content": "Hello, how are you?","role": "user"}],
max_tokens=20,
temperature=0.5
@ -142,13 +143,13 @@ response = completion(
model_list:
- model_name: llama-3
litellm_params:
model: predibase/llama-3-8b-instruct
api_key: os.environ/PREDIBASE_API_KEY
model: databricks/databricks-meta-llama-3-70b-instruct
api_key: os.environ/DATABRICKS_API_KEY
max_tokens: 20
temperature: 0.5
```
## Passings Database specific params - 'instruction'
## Passings Databricks specific params - 'instruction'
For embedding models, databricks lets you pass in an additional param 'instruction'. [Full Spec](https://github.com/BerriAI/litellm/blob/43353c28b341df0d9992b45c6ce464222ebd7984/litellm/llms/databricks.py#L164)
@ -161,7 +162,7 @@ import os
os.environ["DATABRICKS_API_KEY"] = "databricks key"
os.environ["DATABRICKS_API_BASE"] = "databricks url"
# predibase llama3 call
# Databricks bge-large-en call
response = litellm.embedding(
model="databricks/databricks-bge-large-en",
input=["good morning from litellm"],
@ -183,7 +184,6 @@ response = litellm.embedding(
## Supported Databricks Chat Completion Models
Here's an example of using a Databricks models with LiteLLM
| Model Name | Command |
|----------------------------|------------------------------------------------------------------|
@ -195,8 +195,8 @@ Here's an example of using a Databricks models with LiteLLM
| databricks-mpt-7b-instruct | `completion(model='databricks/databricks-mpt-7b-instruct', messages=messages)` |
## Supported Databricks Embedding Models
Here's an example of using a databricks models with LiteLLM
| Model Name | Command |
|----------------------------|------------------------------------------------------------------|
| databricks-bge-large-en | `completion(model='databricks/databricks-bge-large-en', messages=messages)` |
| databricks-bge-large-en | `embedding(model='databricks/databricks-bge-large-en', messages=messages)` |
| databricks-gte-large-en | `embedding(model='databricks/databricks-gte-large-en', messages=messages)` |

13
docs/my-website/docs/providers/deepinfra.md
View file

@ -1,6 +1,13 @@
# DeepInfra
https://deepinfra.com/
:::tip
**We support ALL DeepInfra models, just set `model=deepinfra/<any-model-on-deepinfra>` as a prefix when sending litellm requests**
:::
## API Key
```python
# env variable
@ -38,13 +45,11 @@ for chunk in response:
## Chat Models
| Model Name | Function Call |
|------------------|--------------------------------------|
| meta-llama/Meta-Llama-3-8B-Instruct | `completion(model="deepinfra/meta-llama/Meta-Llama-3-8B-Instruct", messages)` |
| meta-llama/Meta-Llama-3-70B-Instruct | `completion(model="deepinfra/meta-llama/Meta-Llama-3-70B-Instruct", messages)` |
| meta-llama/Llama-2-70b-chat-hf | `completion(model="deepinfra/meta-llama/Llama-2-70b-chat-hf", messages)` |
| meta-llama/Llama-2-7b-chat-hf | `completion(model="deepinfra/meta-llama/Llama-2-7b-chat-hf", messages)` |
| meta-llama/Llama-2-13b-chat-hf | `completion(model="deepinfra/meta-llama/Llama-2-13b-chat-hf", messages)` |
| codellama/CodeLlama-34b-Instruct-hf | `completion(model="deepinfra/codellama/CodeLlama-34b-Instruct-hf", messages)` |
| mistralai/Mistral-7B-Instruct-v0.1 | `completion(model="deepinfra/mistralai/Mistral-7B-Instruct-v0.1", messages)` |
| jondurbin/airoboros-l2-70b-gpt4-1.4.1 | `completion(model="deepinfra/jondurbin/airoboros-l2-70b-gpt4-1.4.1", messages)` |

2
docs/my-website/docs/providers/deepseek.md
View file

@ -49,6 +49,6 @@ We support ALL Deepseek models, just set `deepseek/` as a prefix when sending co
| Model Name | Function Call |
|--------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------|
| deepseek-chat | `completion(model="deepseek/deepseek-chat", messages)` |
| deepseek-coder | `completion(model="deepseek/deepseek-chat", messages)` |
| deepseek-coder | `completion(model="deepseek/deepseek-coder", messages)` |

89
docs/my-website/docs/providers/empower.md Normal file
View file

@ -0,0 +1,89 @@
# Empower
LiteLLM supports all models on Empower.
## API Keys
```python
import os
os.environ["EMPOWER_API_KEY"] = "your-api-key"
```
## Example Usage
```python
from litellm import completion
import os
os.environ["EMPOWER_API_KEY"] = "your-api-key"
messages = [{"role": "user", "content": "Write me a poem about the blue sky"}]
response = completion(model="empower/empower-functions", messages=messages)
print(response)
```
## Example Usage - Streaming
```python
from litellm import completion
import os
os.environ["EMPOWER_API_KEY"] = "your-api-key"
messages = [{"role": "user", "content": "Write me a poem about the blue sky"}]
response = completion(model="empower/empower-functions", messages=messages, streaming=True)
for chunk in response:
print(chunk['choices'][0]['delta'])
```
## Example Usage - Automatic Tool Calling
```python
from litellm import completion
import os
os.environ["EMPOWER_API_KEY"] = "your-api-key"
messages = [{"role": "user", "content": "What's the weather like in San Francisco, Tokyo, and Paris?"}]
tools = [
{
"type": "function",
"function": {
"name": "get_current_weather",
"description": "Get the current weather in a given location",
"parameters": {
"type": "object",
"properties": {
"location": {
"type": "string",
"description": "The city and state, e.g. San Francisco, CA",
},
"unit": {"type": "string", "enum": ["celsius", "fahrenheit"]},
},
"required": ["location"],
},
},
}
]
response = completion(
model="empower/empower-functions-small",
messages=messages,
tools=tools,
tool_choice="auto", # auto is default, but we'll be explicit
)
print("\nLLM Response:\n", response)
```
## Empower Models
liteLLM supports `non-streaming` and `streaming` requests to all models on https://empower.dev/
Example Empower Usage - Note: liteLLM supports all models deployed on Empower
### Empower LLMs - Automatic Tool Using models
| Model Name | Function Call | Required OS Variables |
|-----------------------------------|------------------------------------------------------------------------|---------------------------------|
| empower/empower-functions | `completion('empower/empower-functions', messages)` | `os.environ['TOGETHERAI_API_KEY']` |
| empower/empower-functions-small | `completion('empower/empower-functions-small', messages)` | `os.environ['TOGETHERAI_API_KEY']` |

46
docs/my-website/docs/providers/gemini.md
View file

@ -45,6 +45,52 @@ response = completion(
)
```
## Tool Calling
```python
from litellm import completion
import os
# set env
os.environ["GEMINI_API_KEY"] = ".."
tools = [
{
"type": "function",
"function": {
"name": "get_current_weather",
"description": "Get the current weather in a given location",
"parameters": {
"type": "object",
"properties": {
"location": {
"type": "string",
"description": "The city and state, e.g. San Francisco, CA",
},
"unit": {"type": "string", "enum": ["celsius", "fahrenheit"]},
},
"required": ["location"],
},
},
}
]
messages = [{"role": "user", "content": "What's the weather like in Boston today?"}]
response = completion(
model="gemini/gemini-1.5-flash",
messages=messages,
tools=tools,
)
# Add any assertions, here to check response args
print(response)
assert isinstance(response.choices[0].message.tool_calls[0].function.name, str)
assert isinstance(
response.choices[0].message.tool_calls[0].function.arguments, str
)
```
# Gemini-Pro-Vision
LiteLLM Supports the following image types passed in `url`
- Images with direct links - https://storage.googleapis.com/github-repo/img/gemini/intro/landmark3.jpg

25
docs/my-website/docs/providers/groq.md
View file

@ -1,7 +1,11 @@
# Groq
https://groq.com/
**We support ALL Groq models, just set `groq/` as a prefix when sending completion requests**
:::tip
**We support ALL Groq models, just set `model=groq/<any-model-on-groq>` as a prefix when sending litellm requests**
:::
## API Key
```python
@ -47,7 +51,7 @@ for chunk in response:
We support ALL Groq models, just set `groq/` as a prefix when sending completion requests
| Model Name | Function Call |
|--------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------|
|--------------------|---------------------------------------------------------|
| llama3-8b-8192 | `completion(model="groq/llama3-8b-8192", messages)` |
| llama3-70b-8192 | `completion(model="groq/llama3-70b-8192", messages)` |
| llama2-70b-4096 | `completion(model="groq/llama2-70b-4096", messages)` |
@ -154,3 +158,20 @@ if tool_calls:
) # get a new response from the model where it can see the function response
print("second response\n", second_response)
```
## Speech to Text - Whisper
```python
os.environ["GROQ_API_KEY"] = ""
audio_file = open("/path/to/audio.mp3", "rb")
transcript = litellm.transcription(
model="groq/whisper-large-v3",
file=audio_file,
prompt="Specify context or spelling",
temperature=0,
response_format="json"
)
print("response=", transcript)
```

3
docs/my-website/docs/providers/mistral.md
View file

@ -42,7 +42,7 @@ for chunk in response:
## Supported Models
All models listed here https://docs.mistral.ai/platform/endpoints are supported. We actively maintain the list of models, pricing, token window, etc. [here](https://github.com/BerriAI/litellm/blob/c1b25538277206b9f00de5254d80d6a83bb19a29/model_prices_and_context_window.json).
All models listed here https://docs.mistral.ai/platform/endpoints are supported. We actively maintain the list of models, pricing, token window, etc. [here](https://github.com/BerriAI/litellm/blob/main/model_prices_and_context_window.json).
| Model Name | Function Call |
|----------------|--------------------------------------------------------------|
@ -52,6 +52,7 @@ All models listed here https://docs.mistral.ai/platform/endpoints are supported.
| Mistral 7B | `completion(model="mistral/open-mistral-7b", messages)` |
| Mixtral 8x7B | `completion(model="mistral/open-mixtral-8x7b", messages)` |
| Mixtral 8x22B | `completion(model="mistral/open-mixtral-8x22b", messages)` |
| Codestral | `completion(model="mistral/codestral-latest", messages)` |
## Function Calling

103
docs/my-website/docs/providers/nvidia_nim.md Normal file
View file

@ -0,0 +1,103 @@
# Nvidia NIM
https://docs.api.nvidia.com/nim/reference/
:::tip
**We support ALL Nvidia NIM models, just set `model=nvidia_nim/<any-model-on-nvidia_nim>` as a prefix when sending litellm requests**
:::
## API Key
```python
# env variable
os.environ['NVIDIA_NIM_API_KEY']
```
## Sample Usage
```python
from litellm import completion
import os
os.environ['NVIDIA_NIM_API_KEY'] = ""
response = completion(
model="nvidia_nim/meta/llama3-70b-instruct",
messages=[
{
"role": "user",
"content": "What's the weather like in Boston today in Fahrenheit?",
}
],
temperature=0.2, # optional
top_p=0.9, # optional
frequency_penalty=0.1, # optional
presence_penalty=0.1, # optional
max_tokens=10, # optional
stop=["\n\n"], # optional
)
print(response)
```
## Sample Usage - Streaming
```python
from litellm import completion
import os
os.environ['NVIDIA_NIM_API_KEY'] = ""
response = completion(
model="nvidia_nim/meta/llama3-70b-instruct",
messages=[
{
"role": "user",
"content": "What's the weather like in Boston today in Fahrenheit?",
}
],
stream=True,
temperature=0.2, # optional
top_p=0.9, # optional
frequency_penalty=0.1, # optional
presence_penalty=0.1, # optional
max_tokens=10, # optional
stop=["\n\n"], # optional
)
for chunk in response:
print(chunk)
```
## Supported Models - 💥 ALL Nvidia NIM Models Supported!
We support ALL `nvidia_nim` models, just set `nvidia_nim/` as a prefix when sending completion requests
| Model Name | Function Call |
|------------|---------------|
| nvidia/nemotron-4-340b-reward | `completion(model="nvidia_nim/nvidia/nemotron-4-340b-reward", messages)` |
| 01-ai/yi-large | `completion(model="nvidia_nim/01-ai/yi-large", messages)` |
| aisingapore/sea-lion-7b-instruct | `completion(model="nvidia_nim/aisingapore/sea-lion-7b-instruct", messages)` |
| databricks/dbrx-instruct | `completion(model="nvidia_nim/databricks/dbrx-instruct", messages)` |
| google/gemma-7b | `completion(model="nvidia_nim/google/gemma-7b", messages)` |
| google/gemma-2b | `completion(model="nvidia_nim/google/gemma-2b", messages)` |
| google/codegemma-1.1-7b | `completion(model="nvidia_nim/google/codegemma-1.1-7b", messages)` |
| google/codegemma-7b | `completion(model="nvidia_nim/google/codegemma-7b", messages)` |
| google/recurrentgemma-2b | `completion(model="nvidia_nim/google/recurrentgemma-2b", messages)` |
| ibm/granite-34b-code-instruct | `completion(model="nvidia_nim/ibm/granite-34b-code-instruct", messages)` |
| ibm/granite-8b-code-instruct | `completion(model="nvidia_nim/ibm/granite-8b-code-instruct", messages)` |
| mediatek/breeze-7b-instruct | `completion(model="nvidia_nim/mediatek/breeze-7b-instruct", messages)` |
| meta/codellama-70b | `completion(model="nvidia_nim/meta/codellama-70b", messages)` |
| meta/llama2-70b | `completion(model="nvidia_nim/meta/llama2-70b", messages)` |
| meta/llama3-8b | `completion(model="nvidia_nim/meta/llama3-8b", messages)` |
| meta/llama3-70b | `completion(model="nvidia_nim/meta/llama3-70b", messages)` |
| microsoft/phi-3-medium-4k-instruct | `completion(model="nvidia_nim/microsoft/phi-3-medium-4k-instruct", messages)` |
| microsoft/phi-3-mini-128k-instruct | `completion(model="nvidia_nim/microsoft/phi-3-mini-128k-instruct", messages)` |
| microsoft/phi-3-mini-4k-instruct | `completion(model="nvidia_nim/microsoft/phi-3-mini-4k-instruct", messages)` |
| microsoft/phi-3-small-128k-instruct | `completion(model="nvidia_nim/microsoft/phi-3-small-128k-instruct", messages)` |
| microsoft/phi-3-small-8k-instruct | `completion(model="nvidia_nim/microsoft/phi-3-small-8k-instruct", messages)` |
| mistralai/codestral-22b-instruct-v0.1 | `completion(model="nvidia_nim/mistralai/codestral-22b-instruct-v0.1", messages)` |
| mistralai/mistral-7b-instruct | `completion(model="nvidia_nim/mistralai/mistral-7b-instruct", messages)` |
| mistralai/mistral-7b-instruct-v0.3 | `completion(model="nvidia_nim/mistralai/mistral-7b-instruct-v0.3", messages)` |
| mistralai/mixtral-8x7b-instruct | `completion(model="nvidia_nim/mistralai/mixtral-8x7b-instruct", messages)` |
| mistralai/mixtral-8x22b-instruct | `completion(model="nvidia_nim/mistralai/mixtral-8x22b-instruct", messages)` |
| mistralai/mistral-large | `completion(model="nvidia_nim/mistralai/mistral-large", messages)` |
| nvidia/nemotron-4-340b-instruct | `completion(model="nvidia_nim/nvidia/nemotron-4-340b-instruct", messages)` |
| seallms/seallm-7b-v2.5 | `completion(model="nvidia_nim/seallms/seallm-7b-v2.5", messages)` |
| snowflake/arctic | `completion(model="nvidia_nim/snowflake/arctic", messages)` |
| upstage/solar-10.7b-instruct | `completion(model="nvidia_nim/upstage/solar-10.7b-instruct", messages)` |

13
docs/my-website/docs/providers/openai.md
View file

@ -163,6 +163,8 @@ os.environ["OPENAI_API_BASE"] = "openaiai-api-base" # OPTIONAL
| Model Name | Function Call |
|-----------------------|-----------------------------------------------------------------|
| gpt-4o-mini | `response = completion(model="gpt-4o-mini", messages=messages)` |
| gpt-4o-mini-2024-07-18 | `response = completion(model="gpt-4o-mini-2024-07-18", messages=messages)` |
| gpt-4o | `response = completion(model="gpt-4o", messages=messages)` |
| gpt-4o-2024-05-13 | `response = completion(model="gpt-4o-2024-05-13", messages=messages)` |
| gpt-4-turbo | `response = completion(model="gpt-4-turbo", messages=messages)` |
@ -223,6 +225,17 @@ response = completion(
```
## OpenAI Fine Tuned Models
| Model Name | Function Call |
|---------------------------|-----------------------------------------------------------------|
| fine tuned `gpt-4-0613` | `response = completion(model="ft:gpt-4-0613", messages=messages)` |
| fine tuned `gpt-4o-2024-05-13` | `response = completion(model="ft:gpt-4o-2024-05-13", messages=messages)` |
| fine tuned `gpt-3.5-turbo-0125` | `response = completion(model="ft:gpt-3.5-turbo-0125", messages=messages)` |
| fine tuned `gpt-3.5-turbo-1106` | `response = completion(model="ft:gpt-3.5-turbo-1106", messages=messages)` |
| fine tuned `gpt-3.5-turbo-0613` | `response = completion(model="ft:gpt-3.5-turbo-0613", messages=messages)` |
## Advanced
### Parallel Function calling

25
docs/my-website/docs/providers/openai_compatible.md
View file

@ -18,7 +18,7 @@ import litellm
import os
response = litellm.completion(
model="openai/mistral, # add `openai/` prefix to model so litellm knows to route to OpenAI
model="openai/mistral", # add `openai/` prefix to model so litellm knows to route to OpenAI
api_key="sk-1234", # api key to your openai compatible endpoint
api_base="http://0.0.0.0:4000", # set API Base of your Custom OpenAI Endpoint
messages=[
@ -63,6 +63,14 @@ Here's how to call an OpenAI-Compatible Endpoint with the LiteLLM Proxy Server
api_key: api-key # api key to send your model
```
:::info
If you see `Not Found Error` when testing make sure your `api_base` has the `/v1` postfix
Example: `http://vllm-endpoint.xyz/v1`
:::
2. Start the proxy
```bash
@ -115,3 +123,18 @@ Here's how to call an OpenAI-Compatible Endpoint with the LiteLLM Proxy Server
</TabItem>
</Tabs>
### Advanced - Disable System Messages
Some VLLM models (e.g. gemma) don't support system messages. To map those requests to 'user' messages, use the `supports_system_message` flag.
```yaml
model_list:
- model_name: my-custom-model
litellm_params:
model: openai/google/gemma
api_base: http://my-custom-base
api_key: ""
supports_system_message: False # 👈 KEY CHANGE
```

3
docs/my-website/docs/providers/text_completion_openai.md
View file

@ -1,3 +1,6 @@
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
# OpenAI (Text Completion)
LiteLLM supports OpenAI text completion models

16
docs/my-website/docs/providers/togetherai.md
View file

@ -27,12 +27,12 @@ Example TogetherAI Usage - Note: liteLLM supports all models deployed on Togethe
### Llama LLMs - Chat
| Model Name | Function Call | Required OS Variables |
|-----------------------------------|------------------------------------------------------------------------|---------------------------------|
|-----------------------------------|-------------------------------------------------------------------------|------------------------------------|
| togethercomputer/llama-2-70b-chat | `completion('together_ai/togethercomputer/llama-2-70b-chat', messages)` | `os.environ['TOGETHERAI_API_KEY']` |
### Llama LLMs - Language / Instruct
| Model Name | Function Call | Required OS Variables |
|-----------------------------------|------------------------------------------------------------------------|---------------------------------|
|------------------------------------------|--------------------------------------------------------------------------------|------------------------------------|
| togethercomputer/llama-2-70b | `completion('together_ai/togethercomputer/llama-2-70b', messages)` | `os.environ['TOGETHERAI_API_KEY']` |
| togethercomputer/LLaMA-2-7B-32K | `completion('together_ai/togethercomputer/LLaMA-2-7B-32K', messages)` | `os.environ['TOGETHERAI_API_KEY']` |
| togethercomputer/Llama-2-7B-32K-Instruct | `completion('together_ai/togethercomputer/Llama-2-7B-32K-Instruct', messages)` | `os.environ['TOGETHERAI_API_KEY']` |
@ -40,23 +40,23 @@ Example TogetherAI Usage - Note: liteLLM supports all models deployed on Togethe
### Falcon LLMs
| Model Name | Function Call | Required OS Variables |
|-----------------------------------|------------------------------------------------------------------------|---------------------------------|
|--------------------------------------|----------------------------------------------------------------------------|------------------------------------|
| togethercomputer/falcon-40b-instruct | `completion('together_ai/togethercomputer/falcon-40b-instruct', messages)` | `os.environ['TOGETHERAI_API_KEY']` |
| togethercomputer/falcon-7b-instruct | `completion('together_ai/togethercomputer/falcon-7b-instruct', messages)` | `os.environ['TOGETHERAI_API_KEY']` |
### Alpaca LLMs
| Model Name | Function Call | Required OS Variables |
|-----------------------------------|------------------------------------------------------------------------|---------------------------------|
|----------------------------|------------------------------------------------------------------|------------------------------------|
| togethercomputer/alpaca-7b | `completion('together_ai/togethercomputer/alpaca-7b', messages)` | `os.environ['TOGETHERAI_API_KEY']` |
### Other Chat LLMs
| Model Name | Function Call | Required OS Variables |
|-----------------------------------|------------------------------------------------------------------------|---------------------------------|
|------------------------------|--------------------------------------------------------------------|------------------------------------|
| HuggingFaceH4/starchat-alpha | `completion('together_ai/HuggingFaceH4/starchat-alpha', messages)` | `os.environ['TOGETHERAI_API_KEY']` |
### Code LLMs
| Model Name | Function Call | Required OS Variables |
|-----------------------------------|------------------------------------------------------------------------|---------------------------------|
|-----------------------------------------|-------------------------------------------------------------------------------|------------------------------------|
| togethercomputer/CodeLlama-34b | `completion('together_ai/togethercomputer/CodeLlama-34b', messages)` | `os.environ['TOGETHERAI_API_KEY']` |
| togethercomputer/CodeLlama-34b-Instruct | `completion('together_ai/togethercomputer/CodeLlama-34b-Instruct', messages)` | `os.environ['TOGETHERAI_API_KEY']` |
| togethercomputer/CodeLlama-34b-Python | `completion('together_ai/togethercomputer/CodeLlama-34b-Python', messages)` | `os.environ['TOGETHERAI_API_KEY']` |
@ -67,7 +67,7 @@ Example TogetherAI Usage - Note: liteLLM supports all models deployed on Togethe
### Language LLMs
| Model Name | Function Call | Required OS Variables |
|-----------------------------------|------------------------------------------------------------------------|---------------------------------|
|-------------------------------------|---------------------------------------------------------------------------|------------------------------------|
| NousResearch/Nous-Hermes-Llama2-13b | `completion('together_ai/NousResearch/Nous-Hermes-Llama2-13b', messages)` | `os.environ['TOGETHERAI_API_KEY']` |
| Austism/chronos-hermes-13b | `completion('together_ai/Austism/chronos-hermes-13b', messages)` | `os.environ['TOGETHERAI_API_KEY']` |
| upstage/SOLAR-0-70b-16bit | `completion('together_ai/upstage/SOLAR-0-70b-16bit', messages)` | `os.environ['TOGETHERAI_API_KEY']` |
@ -208,7 +208,7 @@ print(response)
Instead of using the `custom_llm_provider` arg to specify which provider you're using (e.g. together ai), you can just pass the provider name as part of the model name, and LiteLLM will parse it out.
Expected format: <custom_llm_provider>/<model_name>
Expected format: `<custom_llm_provider>/<model_name>`
e.g. completion(model="together_ai/togethercomputer/Llama-2-7B-32K-Instruct", ...)

646
docs/my-website/docs/providers/vertex.md
View file

@ -8,6 +8,425 @@ import TabItem from '@theme/TabItem';
<img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"/>
</a>
## 🆕 `vertex_ai_beta/` route
New `vertex_ai_beta/` route. Adds support for system messages, tool_choice params, etc. by moving to httpx client (instead of vertex sdk). This implementation uses [VertexAI's REST API](https://cloud.google.com/vertex-ai/generative-ai/docs/model-reference/inference#syntax).
```python
from litellm import completion
import json
## GET CREDENTIALS
file_path = 'path/to/vertex_ai_service_account.json'
# Load the JSON file
with open(file_path, 'r') as file:
vertex_credentials = json.load(file)
# Convert to JSON string
vertex_credentials_json = json.dumps(vertex_credentials)
## COMPLETION CALL
response = completion(
model="vertex_ai_beta/gemini-pro",
messages=[{ "content": "Hello, how are you?","role": "user"}],
vertex_credentials=vertex_credentials_json
)
```
### **System Message**
```python
from litellm import completion
import json
## GET CREDENTIALS
file_path = 'path/to/vertex_ai_service_account.json'
# Load the JSON file
with open(file_path, 'r') as file:
vertex_credentials = json.load(file)
# Convert to JSON string
vertex_credentials_json = json.dumps(vertex_credentials)
response = completion(
model="vertex_ai_beta/gemini-pro",
messages=[{"content": "You are a good bot.","role": "system"}, {"content": "Hello, how are you?","role": "user"}],
vertex_credentials=vertex_credentials_json
)
```
### **Function Calling**
Force Gemini to make tool calls with `tool_choice="required"`.
```python
from litellm import completion
import json
## GET CREDENTIALS
file_path = 'path/to/vertex_ai_service_account.json'
# Load the JSON file
with open(file_path, 'r') as file:
vertex_credentials = json.load(file)
# Convert to JSON string
vertex_credentials_json = json.dumps(vertex_credentials)
messages = [
{
"role": "system",
"content": "Your name is Litellm Bot, you are a helpful assistant",
},
# User asks for their name and weather in San Francisco
{
"role": "user",
"content": "Hello, what is your name and can you tell me the weather?",
},
]
tools = [
{
"type": "function",
"function": {
"name": "get_weather",
"description": "Get the current weather in a given location",
"parameters": {
"type": "object",
"properties": {
"location": {
"type": "string",
"description": "The city and state, e.g. San Francisco, CA",
}
},
"required": ["location"],
},
},
}
]
data = {
"model": "vertex_ai_beta/gemini-1.5-pro-preview-0514"),
"messages": messages,
"tools": tools,
"tool_choice": "required",
"vertex_credentials": vertex_credentials_json
}
## COMPLETION CALL
print(completion(**data))
```
### **JSON Schema**
From v`1.40.1+` LiteLLM supports sending `response_schema` as a param for Gemini-1.5-Pro on Vertex AI. For other models (e.g. `gemini-1.5-flash` or `claude-3-5-sonnet`), LiteLLM adds the schema to the message list with a user-controlled prompt.
**Response Schema**
<Tabs>
<TabItem value="sdk" label="SDK">
```python
from litellm import completion
import json
## SETUP ENVIRONMENT
# !gcloud auth application-default login - run this to add vertex credentials to your env
messages = [
{
"role": "user",
"content": "List 5 popular cookie recipes."
}
]
response_schema = {
"type": "array",
"items": {
"type": "object",
"properties": {
"recipe_name": {
"type": "string",
},
},
"required": ["recipe_name"],
},
}
completion(
model="vertex_ai_beta/gemini-1.5-pro",
messages=messages,
response_format={"type": "json_object", "response_schema": response_schema} # 👈 KEY CHANGE
)
print(json.loads(completion.choices[0].message.content))
```
</TabItem>
<TabItem value="proxy" label="PROXY">
1. Add model to config.yaml
```yaml
model_list:
- model_name: gemini-pro
litellm_params:
model: vertex_ai_beta/gemini-1.5-pro
vertex_project: "project-id"
vertex_location: "us-central1"
vertex_credentials: "/path/to/service_account.json" # [OPTIONAL] Do this OR `!gcloud auth application-default login` - run this to add vertex credentials to your env
```
2. Start Proxy
```
$ litellm --config /path/to/config.yaml
```
3. Make Request!
```bash
curl -X POST 'http://0.0.0.0:4000/chat/completions' \
-H 'Content-Type: application/json' \
-H 'Authorization: Bearer sk-1234' \
-D '{
"model": "gemini-pro",
"messages": [
{"role": "user", "content": "List 5 popular cookie recipes."}
],
"response_format": {"type": "json_object", "response_schema": {
"type": "array",
"items": {
"type": "object",
"properties": {
"recipe_name": {
"type": "string",
},
},
"required": ["recipe_name"],
},
}}
}
'
```
</TabItem>
</Tabs>
**Validate Schema**
To validate the response_schema, set `enforce_validation: true`.
<Tabs>
<TabItem value="sdk" label="SDK">
```python
from litellm import completion, JSONSchemaValidationError
try:
completion(
model="vertex_ai_beta/gemini-1.5-pro",
messages=messages,
response_format={
"type": "json_object",
"response_schema": response_schema,
"enforce_validation": true # 👈 KEY CHANGE
}
)
except JSONSchemaValidationError as e:
print("Raw Response: {}".format(e.raw_response))
raise e
```
</TabItem>
<TabItem value="proxy" label="PROXY">
1. Add model to config.yaml
```yaml
model_list:
- model_name: gemini-pro
litellm_params:
model: vertex_ai_beta/gemini-1.5-pro
vertex_project: "project-id"
vertex_location: "us-central1"
vertex_credentials: "/path/to/service_account.json" # [OPTIONAL] Do this OR `!gcloud auth application-default login` - run this to add vertex credentials to your env
```
2. Start Proxy
```
$ litellm --config /path/to/config.yaml
```
3. Make Request!
```bash
curl -X POST 'http://0.0.0.0:4000/chat/completions' \
-H 'Content-Type: application/json' \
-H 'Authorization: Bearer sk-1234' \
-D '{
"model": "gemini-pro",
"messages": [
{"role": "user", "content": "List 5 popular cookie recipes."}
],
"response_format": {"type": "json_object", "response_schema": {
"type": "array",
"items": {
"type": "object",
"properties": {
"recipe_name": {
"type": "string",
},
},
"required": ["recipe_name"],
},
},
"enforce_validation": true
}
}
'
```
</TabItem>
</Tabs>
LiteLLM will validate the response against the schema, and raise a `JSONSchemaValidationError` if the response does not match the schema.
JSONSchemaValidationError inherits from `openai.APIError`
Access the raw response with `e.raw_response`
**Add to prompt yourself**
```python
from litellm import completion
## GET CREDENTIALS
file_path = 'path/to/vertex_ai_service_account.json'
# Load the JSON file
with open(file_path, 'r') as file:
vertex_credentials = json.load(file)
# Convert to JSON string
vertex_credentials_json = json.dumps(vertex_credentials)
messages = [
{
"role": "user",
"content": """
List 5 popular cookie recipes.
Using this JSON schema:
Recipe = {"recipe_name": str}
Return a `list[Recipe]`
"""
}
]
completion(model="vertex_ai_beta/gemini-1.5-flash-preview-0514", messages=messages, response_format={ "type": "json_object" })
```
### **Grounding**
Add Google Search Result grounding to vertex ai calls.
[**Relevant VertexAI Docs**](https://cloud.google.com/vertex-ai/generative-ai/docs/model-reference/grounding#examples)
See the grounding metadata with `response_obj._hidden_params["vertex_ai_grounding_metadata"]`
<Tabs>
<TabItem value="sdk" label="SDK">
```python
from litellm import completion
## SETUP ENVIRONMENT
# !gcloud auth application-default login - run this to add vertex credentials to your env
tools = [{"googleSearchRetrieval": {}}] # 👈 ADD GOOGLE SEARCH
resp = litellm.completion(
model="vertex_ai_beta/gemini-1.0-pro-001",
messages=[{"role": "user", "content": "Who won the world cup?"}],
tools=tools,
)
print(resp)
```
</TabItem>
<TabItem value="proxy" label="PROXY">
```bash
curl http://0.0.0.0:4000/v1/chat/completions \
-H "Content-Type: application/json" \
-H "Authorization: Bearer $OPENAI_API_KEY" \
-d '{
"model": "gpt-4o",
"messages": [{"role": "user", "content": "Who won the world cup?"}],
"tools": [
{
"googleSearchResults": {}
}
]
}'
```
</TabItem>
</Tabs>
#### **Moving from Vertex AI SDK to LiteLLM (GROUNDING)**
If this was your initial VertexAI Grounding code,
```python
import vertexai
vertexai.init(project=project_id, location="us-central1")
model = GenerativeModel("gemini-1.5-flash-001")
# Use Google Search for grounding
tool = Tool.from_google_search_retrieval(grounding.GoogleSearchRetrieval(disable_attributon=False))
prompt = "When is the next total solar eclipse in US?"
response = model.generate_content(
prompt,
tools=[tool],
generation_config=GenerationConfig(
temperature=0.0,
),
)
print(response)
```
then, this is what it looks like now
```python
from litellm import completion
# !gcloud auth application-default login - run this to add vertex credentials to your env
tools = [{"googleSearchRetrieval": {"disable_attributon": False}}] # 👈 ADD GOOGLE SEARCH
resp = litellm.completion(
model="vertex_ai_beta/gemini-1.0-pro-001",
messages=[{"role": "user", "content": "Who won the world cup?"}],
tools=tools,
vertex_project="project-id"
)
print(resp)
```
## Pre-requisites
* `pip install google-cloud-aiplatform` (pre-installed on proxy docker image)
* Authentication:
@ -140,7 +559,7 @@ In certain use-cases you may need to make calls to the models and pass [safety s
```python
response = completion(
model="gemini/gemini-pro",
model="vertex_ai/gemini-pro",
messages=[{"role": "user", "content": "write code for saying hi from LiteLLM"}]
safety_settings=[
{
@ -254,6 +673,7 @@ litellm.vertex_location = "us-central1 # Your Location
| Model Name | Function Call |
|------------------|--------------------------------------|
| claude-3-opus@20240229 | `completion('vertex_ai/claude-3-opus@20240229', messages)` |
| claude-3-5-sonnet@20240620 | `completion('vertex_ai/claude-3-5-sonnet@20240620', messages)` |
| claude-3-sonnet@20240229 | `completion('vertex_ai/claude-3-sonnet@20240229', messages)` |
| claude-3-haiku@20240307 | `completion('vertex_ai/claude-3-haiku@20240307', messages)` |
@ -363,8 +783,8 @@ response = completion(
## Gemini 1.5 Pro (and Vision)
| Model Name | Function Call |
|------------------|--------------------------------------|
| gemini-1.5-pro | `completion('gemini-1.5-pro', messages)`, `completion('vertex_ai/gemini-pro', messages)` |
| gemini-1.5-flash-preview-0514 | `completion('gemini-1.5-flash-preview-0514', messages)`, `completion('vertex_ai/gemini-pro', messages)` |
| gemini-1.5-pro | `completion('gemini-1.5-pro', messages)`, `completion('vertex_ai/gemini-1.5-pro', messages)` |
| gemini-1.5-flash-preview-0514 | `completion('gemini-1.5-flash-preview-0514', messages)`, `completion('vertex_ai/gemini-1.5-flash-preview-0514', messages)` |
| gemini-1.5-pro-preview-0514 | `completion('gemini-1.5-pro-preview-0514', messages)`, `completion('vertex_ai/gemini-1.5-pro-preview-0514', messages)` |
@ -449,6 +869,198 @@ print(response)
</TabItem>
</Tabs>
## Usage - Function Calling
LiteLLM supports Function Calling for Vertex AI gemini models.
```python
from litellm import completion
import os
# set env
os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = ".."
os.environ["VERTEX_AI_PROJECT"] = ".."
os.environ["VERTEX_AI_LOCATION"] = ".."
tools = [
{
"type": "function",
"function": {
"name": "get_current_weather",
"description": "Get the current weather in a given location",
"parameters": {
"type": "object",
"properties": {
"location": {
"type": "string",
"description": "The city and state, e.g. San Francisco, CA",
},
"unit": {"type": "string", "enum": ["celsius", "fahrenheit"]},
},
"required": ["location"],
},
},
}
]
messages = [{"role": "user", "content": "What's the weather like in Boston today?"}]
response = completion(
model="vertex_ai/gemini-pro-vision",
messages=messages,
tools=tools,
)
# Add any assertions, here to check response args
print(response)
assert isinstance(response.choices[0].message.tool_calls[0].function.name, str)
assert isinstance(
response.choices[0].message.tool_calls[0].function.arguments, str
)
```
## Usage - PDF / Videos / etc. Files
Pass any file supported by Vertex AI, through LiteLLM.
<Tabs>
<TabItem value="sdk" label="SDK">
### **Using `gs://`**
```python
from litellm import completion
response = completion(
model="vertex_ai/gemini-1.5-flash",
messages=[
{
"role": "user",
"content": [
{"type": "text", "text": "You are a very professional document summarization specialist. Please summarize the given document."},
{
"type": "image_url",
"image_url": "gs://cloud-samples-data/generative-ai/pdf/2403.05530.pdf", # 👈 PDF
},
],
}
],
max_tokens=300,
)
print(response.choices[0])
```
### **using base64**
```python
from litellm import completion
import base64
import requests
# URL of the file
url = "https://storage.googleapis.com/cloud-samples-data/generative-ai/pdf/2403.05530.pdf"
# Download the file
response = requests.get(url)
file_data = response.content
encoded_file = base64.b64encode(file_data).decode("utf-8")
response = completion(
model="vertex_ai/gemini-1.5-flash",
messages=[
{
"role": "user",
"content": [
{"type": "text", "text": "You are a very professional document summarization specialist. Please summarize the given document."},
{
"type": "image_url",
"image_url": f"data:application/pdf;base64,{encoded_file}", # 👈 PDF
},
],
}
],
max_tokens=300,
)
print(response.choices[0])
```
</TabItem>
<TabItem value="proxy" lable="PROXY">
1. Add model to config
```yaml
- model_name: gemini-1.5-flash
litellm_params:
model: vertex_ai/gemini-1.5-flash
vertex_credentials: "/path/to/service_account.json"
```
2. Start Proxy
```
litellm --config /path/to/config.yaml
```
3. Test it!
**Using `gs://`**
```bash
curl http://0.0.0.0:4000/v1/chat/completions \
-H "Content-Type: application/json" \
-H "Authorization: Bearer <YOUR-LITELLM-KEY>" \
-d '{
"model": "gemini-1.5-flash",
"messages": [
{
"role": "user",
"content": [
{
"type": "text",
"text": "You are a very professional document summarization specialist. Please summarize the given document"
},
{
"type": "image_url",
"image_url": "gs://cloud-samples-data/generative-ai/pdf/2403.05530.pdf" # 👈 PDF
}
}
]
}
],
"max_tokens": 300
}'
```
```bash
curl http://0.0.0.0:4000/v1/chat/completions \
-H "Content-Type: application/json" \
-H "Authorization: Bearer <YOUR-LITELLM-KEY>" \
-d '{
"model": "gemini-1.5-flash",
"messages": [
{
"role": "user",
"content": [
{
"type": "text",
"text": "You are a very professional document summarization specialist. Please summarize the given document"
},
{
"type": "image_url",
"image_url": "data:application/pdf;base64,{encoded_file}" # 👈 PDF
}
}
]
}
],
"max_tokens": 300
}'
```
</TabItem>
</Tabs>
## Chat Models
| Model Name | Function Call |
@ -500,6 +1112,8 @@ All models listed [here](https://github.com/BerriAI/litellm/blob/57f37f743886a02
| Model Name | Function Call |
|--------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------|
| text-embedding-004 | `embedding(model="vertex_ai/text-embedding-004", input)` |
| text-multilingual-embedding-002 | `embedding(model="vertex_ai/text-multilingual-embedding-002", input)` |
| textembedding-gecko | `embedding(model="vertex_ai/textembedding-gecko", input)` |
| textembedding-gecko-multilingual | `embedding(model="vertex_ai/textembedding-gecko-multilingual", input)` |
| textembedding-gecko-multilingual@001 | `embedding(model="vertex_ai/textembedding-gecko-multilingual@001", input)` |
@ -508,6 +1122,29 @@ All models listed [here](https://github.com/BerriAI/litellm/blob/57f37f743886a02
| text-embedding-preview-0409 | `embedding(model="vertex_ai/text-embedding-preview-0409", input)` |
| text-multilingual-embedding-preview-0409 | `embedding(model="vertex_ai/text-multilingual-embedding-preview-0409", input)` |
### Advanced Use `task_type` and `title` (Vertex Specific Params)
👉 `task_type` and `title` are vertex specific params
LiteLLM Supported Vertex Specific Params
```python
auto_truncate: Optional[bool] = None
task_type: Optional[Literal["RETRIEVAL_QUERY","RETRIEVAL_DOCUMENT", "SEMANTIC_SIMILARITY", "CLASSIFICATION", "CLUSTERING", "QUESTION_ANSWERING", "FACT_VERIFICATION"]] = None
title: Optional[str] = None # The title of the document to be embedded. (only valid with task_type=RETRIEVAL_DOCUMENT).
```
**Example Usage with LiteLLM**
```python
response = litellm.embedding(
model="vertex_ai/text-embedding-004",
input=["good morning from litellm", "gm"]
task_type = "RETRIEVAL_DOCUMENT",
dimensions=1,
auto_truncate=True,
)
```
## Image Generation Models
Usage
@ -607,6 +1244,3 @@ s/o @[Darien Kindlund](https://www.linkedin.com/in/kindlund/) for this tutorial

2
docs/my-website/docs/providers/vllm.md
View file

@ -156,7 +156,7 @@ def default_pt(messages):
#### Models we already have Prompt Templates for
| Model Name | Works for Models | Function Call |
| -------- | -------- | -------- |
|--------------------------------------|-----------------------------------|------------------------------------------------------------------------------------------------------------------|
| meta-llama/Llama-2-7b-chat | All meta-llama llama2 chat models | `completion(model='vllm/meta-llama/Llama-2-7b', messages=messages, api_base="your_api_endpoint")` |
| tiiuae/falcon-7b-instruct | All falcon instruct models | `completion(model='vllm/tiiuae/falcon-7b-instruct', messages=messages, api_base="your_api_endpoint")` |
| mosaicml/mpt-7b-chat | All mpt chat models | `completion(model='vllm/mosaicml/mpt-7b-chat', messages=messages, api_base="your_api_endpoint")` |

98
docs/my-website/docs/providers/volcano.md Normal file
View file

@ -0,0 +1,98 @@
# Volcano Engine (Volcengine)
https://www.volcengine.com/docs/82379/1263482
:::tip
**We support ALL Volcengine NIM models, just set `model=volcengine/<any-model-on-volcengine>` as a prefix when sending litellm requests**
:::
## API Key
```python
# env variable
os.environ['VOLCENGINE_API_KEY']
```
## Sample Usage
```python
from litellm import completion
import os
os.environ['VOLCENGINE_API_KEY'] = ""
response = completion(
model="volcengine/<OUR_ENDPOINT_ID>",
messages=[
{
"role": "user",
"content": "What's the weather like in Boston today in Fahrenheit?",
}
],
temperature=0.2, # optional
top_p=0.9, # optional
frequency_penalty=0.1, # optional
presence_penalty=0.1, # optional
max_tokens=10, # optional
stop=["\n\n"], # optional
)
print(response)
```
## Sample Usage - Streaming
```python
from litellm import completion
import os
os.environ['VOLCENGINE_API_KEY'] = ""
response = completion(
model="volcengine/<OUR_ENDPOINT_ID>",
messages=[
{
"role": "user",
"content": "What's the weather like in Boston today in Fahrenheit?",
}
],
stream=True,
temperature=0.2, # optional
top_p=0.9, # optional
frequency_penalty=0.1, # optional
presence_penalty=0.1, # optional
max_tokens=10, # optional
stop=["\n\n"], # optional
)
for chunk in response:
print(chunk)
```
## Supported Models - 💥 ALL Volcengine NIM Models Supported!
We support ALL `volcengine` models, just set `volcengine/<OUR_ENDPOINT_ID>` as a prefix when sending completion requests
## Sample Usage - LiteLLM Proxy
### Config.yaml setting
```yaml
model_list:
- model_name: volcengine-model
litellm_params:
model: volcengine/<OUR_ENDPOINT_ID>
api_key: os.environ/VOLCENGINE_API_KEY
```
### Send Request
```shell
curl --location 'http://localhost:4000/chat/completions' \
--header 'Authorization: Bearer sk-1234' \
--header 'Content-Type: application/json' \
--data '{
"model": "volcengine-model",
"messages": [
{
"role": "user",
"content": "here is my api key. openai_api_key=sk-1234"
}
]
}'
```

4
docs/my-website/docs/providers/watsonx.md
View file

@ -252,7 +252,7 @@ response = completion(
Here are some examples of models available in IBM watsonx.ai that you can use with LiteLLM:
| Mode Name | Command |
| ---------- | --------- |
|------------------------------------|------------------------------------------------------------------------------------------|
| Flan T5 XXL | `completion(model=watsonx/google/flan-t5-xxl, messages=messages)` |
| Flan Ul2 | `completion(model=watsonx/google/flan-ul2, messages=messages)` |
| Mt0 XXL | `completion(model=watsonx/bigscience/mt0-xxl, messages=messages)` |
@ -276,7 +276,7 @@ For a list of all available models in watsonx.ai, see [here](https://dataplatfor
## Supported IBM watsonx.ai Embedding Models
| Model Name | Function Call |
|----------------------|---------------------------------------------|
|------------|------------------------------------------------------------------------|
| Slate 30m | `embedding(model="watsonx/ibm/slate-30m-english-rtrvr", input=input)` |
| Slate 125m | `embedding(model="watsonx/ibm/slate-125m-english-rtrvr", input=input)` |

2
docs/my-website/docs/providers/xinference.md
View file

@ -38,7 +38,7 @@ print(response)
All models listed here https://inference.readthedocs.io/en/latest/models/builtin/embedding/index.html are supported
| Model Name | Function Call |
|------------------------------|--------------------------------------------------------|
|-----------------------------|--------------------------------------------------------------------|
| bge-base-en | `embedding(model="xinference/bge-base-en", input)` |
| bge-base-en-v1.5 | `embedding(model="xinference/bge-base-en-v1.5", input)` |
| bge-base-zh | `embedding(model="xinference/bge-base-zh", input)` |

141
docs/my-website/docs/proxy/alerting.md
View file

@ -1,3 +1,5 @@
import Image from '@theme/IdealImage';
# 🚨 Alerting / Webhooks
Get alerts for:
@ -15,6 +17,11 @@ Get alerts for:
- **Spend** Weekly & Monthly spend per Team, Tag
Works across:
- [Slack](#quick-start)
- [Discord](#advanced---using-discord-webhooks)
- [Microsoft Teams](#advanced---using-ms-teams-webhooks)
## Quick Start
Set up a slack alert channel to receive alerts from proxy.
@ -25,43 +32,79 @@ Get a slack webhook url from https://api.slack.com/messaging/webhooks
You can also use Discord Webhooks, see [here](#using-discord-webhooks)
### Step 2: Update config.yaml
- Set `SLACK_WEBHOOK_URL` in your proxy env to enable Slack alerts.
- Just for testing purposes, let's save a bad key to our proxy.
Set `SLACK_WEBHOOK_URL` in your proxy env to enable Slack alerts.
```bash
export SLACK_WEBHOOK_URL="https://hooks.slack.com/services/<>/<>/<>"
```
### Step 2: Setup Proxy
```yaml
model_list:
model_name: "azure-model"
litellm_params:
model: "azure/gpt-35-turbo"
api_key: "my-bad-key" # 👈 bad key
general_settings:
alerting: ["slack"]
alerting_threshold: 300 # sends alerts if requests hang for 5min+ and responses take 5min+
environment_variables:
SLACK_WEBHOOK_URL: "https://hooks.slack.com/services/<>/<>/<>"
SLACK_DAILY_REPORT_FREQUENCY: "86400" # 24 hours; Optional: defaults to 12 hours
```
### Step 3: Start proxy
Start proxy
```bash
$ litellm --config /path/to/config.yaml
```
## Testing Alerting is Setup Correctly
Make a GET request to `/health/services`, expect to see a test slack alert in your provided webhook slack channel
### Step 3: Test it!
```shell
curl -X GET 'http://localhost:4000/health/services?service=slack' \
```bash
curl -X GET 'http://0.0.0.0:4000/health/services?service=slack' \
-H 'Authorization: Bearer sk-1234'
```
## Advanced - Redacting Messages from Alerts
By default alerts show the `messages/input` passed to the LLM. If you want to redact this from slack alerting set the following setting on your config
```shell
general_settings:
alerting: ["slack"]
alert_types: ["spend_reports"]
litellm_settings:
redact_messages_in_exceptions: True
```
## Advanced - Add Metadata to alerts
Add alerting metadata to proxy calls for debugging.
```python
import openai
client = openai.OpenAI(
api_key="anything",
base_url="http://0.0.0.0:4000"
)
# request sent to model set on litellm proxy, `litellm --model`
response = client.chat.completions.create(
model="gpt-3.5-turbo",
messages = [],
extra_body={
"metadata": {
"alerting_metadata": {
"hello": "world"
}
}
}
)
```
**Expected Response**
<Image img={require('../../img/alerting_metadata.png')}/>
## Advanced - Opting into specific alert types
Set `alert_types` if you want to Opt into only specific alert types
@ -91,6 +134,48 @@ AlertType = Literal[
```
## Advanced - Using MS Teams Webhooks
MS Teams provides a slack compatible webhook url that you can use for alerting
##### Quick Start
1. [Get a webhook url](https://learn.microsoft.com/en-us/microsoftteams/platform/webhooks-and-connectors/how-to/add-incoming-webhook?tabs=newteams%2Cdotnet#create-an-incoming-webhook) for your Microsoft Teams channel
2. Add it to your .env
```bash
SLACK_WEBHOOK_URL="https://berriai.webhook.office.com/webhookb2/...6901/IncomingWebhook/b55fa0c2a48647be8e6effedcd540266/e04b1092-4a3e-44a2-ab6b-29a0a4854d1d"
```
3. Add it to your litellm config
```yaml
model_list:
model_name: "azure-model"
litellm_params:
model: "azure/gpt-35-turbo"
api_key: "my-bad-key" # 👈 bad key
general_settings:
alerting: ["slack"]
alerting_threshold: 300 # sends alerts if requests hang for 5min+ and responses take 5min+
```
4. Run health check!
Call the proxy `/health/services` endpoint to test if your alerting connection is correctly setup.
```bash
curl --location 'http://0.0.0.0:4000/health/services?service=slack' \
--header 'Authorization: Bearer sk-1234'
```
**Expected Response**
<Image img={require('../../img/ms_teams_alerting.png')}/>
## Advanced - Using Discord Webhooks
Discord provides a slack compatible webhook url that you can use for alerting
@ -122,7 +207,6 @@ environment_variables:
SLACK_WEBHOOK_URL: "https://discord.com/api/webhooks/1240030362193760286/cTLWt5ATn1gKmcy_982rl5xmYHsrM1IWJdmCL1AyOmU9JdQXazrp8L1_PYgUtgxj8x4f/slack"
```
That's it ! You're ready to go !
## Advanced - [BETA] Webhooks for Budget Alerts
@ -178,23 +262,26 @@ curl -X GET --location 'http://0.0.0.0:4000/health/services?service=webhook' \
}
```
**API Spec for Webhook Event**
## **API Spec for Webhook Event**
- `spend` *float*: The current spend amount for the 'event_group'.
- `max_budget` *float*: The maximum allowed budget for the 'event_group'.
- `max_budget` *float or null*: The maximum allowed budget for the 'event_group'. null if not set.
- `token` *str*: A hashed value of the key, used for authentication or identification purposes.
- `user_id` *str or null*: The ID of the user associated with the event (optional).
- `customer_id` *str or null*: The ID of the customer associated with the event (optional).
- `internal_user_id` *str or null*: The ID of the internal user associated with the event (optional).
- `team_id` *str or null*: The ID of the team associated with the event (optional).
- `user_email` *str or null*: The email of the user associated with the event (optional).
- `user_email` *str or null*: The email of the internal user associated with the event (optional).
- `key_alias` *str or null*: An alias for the key associated with the event (optional).
- `projected_exceeded_date` *str or null*: The date when the budget is projected to be exceeded, returned when 'soft_budget' is set for key (optional).
- `projected_spend` *float or null*: The projected spend amount, returned when 'soft_budget' is set for key (optional).
- `event` *Literal["budget_crossed", "threshold_crossed", "projected_limit_exceeded"]*: The type of event that triggered the webhook. Possible values are:
* "spend_tracked": Emitted whenver spend is tracked for a customer id.
* "budget_crossed": Indicates that the spend has exceeded the max budget.
* "threshold_crossed": Indicates that spend has crossed a threshold (currently sent when 85% and 95% of budget is reached).
* "projected_limit_exceeded": For "key" only - Indicates that the projected spend is expected to exceed the soft budget threshold.
- `event_group` *Literal["user", "key", "team", "proxy"]*: The group associated with the event. Possible values are:
* "user": The event is related to a specific user.
- `event_group` *Literal["customer", "internal_user", "key", "team", "proxy"]*: The group associated with the event. Possible values are:
* "customer": The event is related to a specific customer
* "internal_user": The event is related to a specific internal user.
* "key": The event is related to a specific key.
* "team": The event is related to a team.
* "proxy": The event is related to a proxy.

29
docs/my-website/docs/proxy/caching.md
View file

@ -283,7 +283,7 @@ litellm_settings:
### Turn on / off caching per request.
The proxy support 3 cache-controls:
The proxy support 4 cache-controls:
- `ttl`: *Optional(int)* - Will cache the response for the user-defined amount of time (in seconds).
- `s-maxage`: *Optional(int)* Will only accept cached responses that are within user-defined range (in seconds).
@ -374,6 +374,33 @@ chat_completion = client.chat.completions.create(
)
```
### Turn on / off caching per Key.
1. Add cache params when creating a key [full list](#turn-on--off-caching-per-key)
```bash
curl -X POST 'http://0.0.0.0:4000/key/generate' \
-H 'Authorization: Bearer sk-1234' \
-H 'Content-Type: application/json' \
-D '{
"user_id": "222",
"metadata": {
"cache": {
"no-cache": true
}
}
}'
```
2. Test it!
```bash
curl -X POST 'http://localhost:4000/chat/completions' \
-H 'Content-Type: application/json' \
-H 'Authorization: Bearer <YOUR_NEW_KEY>' \
-D '{"model": "gpt-3.5-turbo", "messages": [{"role": "user", "content": "bom dia"}]}'
```
### Deleting Cache Keys - `/cache/delete`
In order to delete a cache key, send a request to `/cache/delete` with the `keys` you want to delete

114
docs/my-website/docs/proxy/configs.md
View file

@ -80,6 +80,13 @@ For more provider-specific info, [go here](../providers/)
$ litellm --config /path/to/config.yaml
```
:::tip
Run with `--detailed_debug` if you need detailed debug logs
```shell
$ litellm --config /path/to/config.yaml --detailed_debug
:::
### Using Proxy - Curl Request, OpenAI Package, Langchain, Langchain JS
Calling a model group
@ -245,13 +252,86 @@ $ litellm --config /path/to/config.yaml
```
## Multiple OpenAI Organizations
Add all openai models across all OpenAI organizations with just 1 model definition
```yaml
- model_name: *
litellm_params:
model: openai/*
api_key: os.environ/OPENAI_API_KEY
organization:
- org-1
- org-2
- org-3
```
LiteLLM will automatically create separate deployments for each org.
Confirm this via
```bash
curl --location 'http://0.0.0.0:4000/v1/model/info' \
--header 'Authorization: Bearer ${LITELLM_KEY}' \
--data ''
```
## Wildcard Model Name (Add ALL MODELS from env)
Dynamically call any model from any given provider without the need to predefine it in the config YAML file. As long as the relevant keys are in the environment (see [providers list](../providers/)), LiteLLM will make the call correctly.
1. Setup config.yaml
```
model_list:
- model_name: "*" # all requests where model not in your config go to this deployment
litellm_params:
model: "openai/*" # passes our validation check that a real provider is given
```
2. Start LiteLLM proxy
```
litellm --config /path/to/config.yaml
```
3. Try claude 3-5 sonnet from anthropic
```bash
curl -X POST 'http://0.0.0.0:4000/chat/completions' \
-H 'Content-Type: application/json' \
-H 'Authorization: Bearer sk-1234' \
-D '{
"model": "claude-3-5-sonnet-20240620",
"messages": [
{"role": "user", "content": "Hey, how'\''s it going?"},
{
"role": "assistant",
"content": "I'\''m doing well. Would like to hear the rest of the story?"
},
{"role": "user", "content": "Na"},
{
"role": "assistant",
"content": "No problem, is there anything else i can help you with today?"
},
{
"role": "user",
"content": "I think you'\''re getting cut off sometimes"
}
]
}
'
```
## Load Balancing
:::info
For more on this, go to [this page](./load_balancing.md)
For more on this, go to [this page](https://docs.litellm.ai/docs/proxy/load_balancing)
:::
Use this to call multiple instances of the same model and configure things like [routing strategy](../routing.md#advanced).
Use this to call multiple instances of the same model and configure things like [routing strategy](https://docs.litellm.ai/docs/routing#advanced).
For optimal performance:
- Set `tpm/rpm` per model deployment. Weighted picks are then based on the established tpm/rpm.
@ -559,6 +639,36 @@ curl --location 'http://0.0.0.0:4000/chat/completions' \
}'
```
## ✨ IP Address Filtering
:::info
You need a LiteLLM License to unlock this feature. [Grab time](https://calendly.com/d/4mp-gd3-k5k/litellm-1-1-onboarding-chat), to get one today!
:::
Restrict which IP's can call the proxy endpoints.
```yaml
general_settings:
allowed_ips: ["192.168.1.1"]
```
**Expected Response** (if IP not listed)
```bash
{
"error": {
"message": "Access forbidden: IP address not allowed.",
"type": "auth_error",
"param": "None",
"code": 403
}
}
```
## Disable Swagger UI
To disable the Swagger docs from the base url, set

398
docs/my-website/docs/proxy/cost_tracking.md
View file

@ -1,22 +1,174 @@
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
import Image from '@theme/IdealImage';
# 💸 Spend Tracking
Track spend for keys, users, and teams across 100+ LLMs.
## Getting Spend Reports - To Charge Other Teams, API Keys
### How to Track Spend with LiteLLM
Use the `/global/spend/report` endpoint to get daily spend per team, with a breakdown of spend per API Key, Model
**Step 1**
### Example Request
👉 [Setup LiteLLM with a Database](https://docs.litellm.ai/docs/proxy/deploy)
**Step2** Send `/chat/completions` request
<Tabs>
<TabItem value="openai" label="OpenAI Python v1.0.0+">
```python
import openai
client = openai.OpenAI(
api_key="sk-1234",
base_url="http://0.0.0.0:4000"
)
response = client.chat.completions.create(
model="llama3",
messages = [
{
"role": "user",
"content": "this is a test request, write a short poem"
}
],
user="palantir",
extra_body={
"metadata": {
"tags": ["jobID:214590dsff09fds", "taskName:run_page_classification"]
}
}
)
print(response)
```
</TabItem>
<TabItem value="Curl" label="Curl Request">
Pass `metadata` as part of the request body
```shell
curl -X GET 'http://localhost:4000/global/spend/report?start_date=2024-04-01&end_date=2024-06-30' \
curl --location 'http://0.0.0.0:4000/chat/completions' \
--header 'Content-Type: application/json' \
--header 'Authorization: Bearer sk-1234' \
--data '{
"model": "llama3",
"messages": [
{
"role": "user",
"content": "what llm are you"
}
],
"user": "palantir",
"metadata": {
"tags": ["jobID:214590dsff09fds", "taskName:run_page_classification"]
}
}'
```
</TabItem>
<TabItem value="langchain" label="Langchain">
```python
from langchain.chat_models import ChatOpenAI
from langchain.prompts.chat import (
ChatPromptTemplate,
HumanMessagePromptTemplate,
SystemMessagePromptTemplate,
)
from langchain.schema import HumanMessage, SystemMessage
import os
os.environ["OPENAI_API_KEY"] = "sk-1234"
chat = ChatOpenAI(
openai_api_base="http://0.0.0.0:4000",
model = "llama3",
user="palantir",
extra_body={
"metadata": {
"tags": ["jobID:214590dsff09fds", "taskName:run_page_classification"]
}
}
)
messages = [
SystemMessage(
content="You are a helpful assistant that im using to make a test request to."
),
HumanMessage(
content="test from litellm. tell me why it's amazing in 1 sentence"
),
]
response = chat(messages)
print(response)
```
</TabItem>
</Tabs>
**Step3 - Verify Spend Tracked**
That's IT. Now Verify your spend was tracked
<Tabs>
<TabItem value="curl" label="Response Headers">
Expect to see `x-litellm-response-cost` in the response headers with calculated cost
<Image img={require('../../img/response_cost_img.png')} />
</TabItem>
<TabItem value="db" label="DB + UI">
The following spend gets tracked in Table `LiteLLM_SpendLogs`
```json
{
"api_key": "fe6b0cab4ff5a5a8df823196cc8a450*****", # Hash of API Key used
"user": "default_user", # Internal User (LiteLLM_UserTable) that owns `api_key=sk-1234`.
"team_id": "e8d1460f-846c-45d7-9b43-55f3cc52ac32", # Team (LiteLLM_TeamTable) that owns `api_key=sk-1234`
"request_tags": ["jobID:214590dsff09fds", "taskName:run_page_classification"],# Tags sent in request
"end_user": "palantir", # Customer - the `user` sent in the request
"model_group": "llama3", # "model" passed to LiteLLM
"api_base": "https://api.groq.com/openai/v1/", # "api_base" of model used by LiteLLM
"spend": 0.000002, # Spend in $
"total_tokens": 100,
"completion_tokens": 80,
"prompt_tokens": 20,
}
```
Navigate to the Usage Tab on the LiteLLM UI (found on https://your-proxy-endpoint/ui) and verify you see spend tracked under `Usage`
<Image img={require('../../img/admin_ui_spend.png')} />
</TabItem>
</Tabs>
## ✨ (Enterprise) API Endpoints to get Spend
#### Getting Spend Reports - To Charge Other Teams, Customers, Users
Use the `/global/spend/report` endpoint to get spend reports
<Tabs>
<TabItem value="per team" label="Spend Per Team">
##### Example Request
👉 Key Change: Specify `group_by=team`
```shell
curl -X GET 'http://localhost:4000/global/spend/report?start_date=2024-04-01&end_date=2024-06-30&group_by=team' \
-H 'Authorization: Bearer sk-1234'
```
### Example Response
##### Example Response
<Tabs>
<TabItem value="response" label="Expected Response">
@ -125,7 +277,202 @@ Output from script
</Tabs>
## Allowing Non-Proxy Admins to access `/spend` endpoints
</TabItem>
<TabItem value="per customer" label="Spend Per Customer">
:::info
Customer This is the value of `user_id` passed when calling [`/key/generate`](https://litellm-api.up.railway.app/#/key%20management/generate_key_fn_key_generate_post)
[this is `user` passed to `/chat/completions` request](#how-to-track-spend-with-litellm)
- [LiteLLM API key](virtual_keys.md)
:::
##### Example Request
👉 Key Change: Specify `group_by=customer`
```shell
curl -X GET 'http://localhost:4000/global/spend/report?start_date=2024-04-01&end_date=2024-06-30&group_by=customer' \
-H 'Authorization: Bearer sk-1234'
```
##### Example Response
```shell
[
{
"group_by_day": "2024-04-30T00:00:00+00:00",
"customers": [
{
"customer": "palantir",
"total_spend": 0.0015265,
"metadata": [ # see the spend by unique(key + model)
{
"model": "gpt-4",
"spend": 0.00123,
"total_tokens": 28,
"api_key": "88dc28.." # the hashed api key
},
{
"model": "gpt-4",
"spend": 0.00123,
"total_tokens": 28,
"api_key": "a73dc2.." # the hashed api key
},
{
"model": "chatgpt-v-2",
"spend": 0.000214,
"total_tokens": 122,
"api_key": "898c28.." # the hashed api key
},
{
"model": "gpt-3.5-turbo",
"spend": 0.0000825,
"total_tokens": 85,
"api_key": "84dc28.." # the hashed api key
}
]
}
]
}
]
```
</TabItem>
<TabItem value="per key" label="Spend for Specific API Key">
👉 Key Change: Specify `api_key=sk-1234`
```shell
curl -X GET 'http://localhost:4000/global/spend/report?start_date=2024-04-01&end_date=2024-06-30&api_key=sk-1234' \
-H 'Authorization: Bearer sk-1234'
```
##### Example Response
```shell
[
{
"api_key": "88dc28d0f030c55ed4ab77ed8faf098196cb1c05df778539800c9f1243fe6b4b",
"total_cost": 0.3201286305151999,
"total_input_tokens": 36.0,
"total_output_tokens": 1593.0,
"model_details": [
{
"model": "dall-e-3",
"total_cost": 0.31999939051519993,
"total_input_tokens": 0,
"total_output_tokens": 0
},
{
"model": "llama3-8b-8192",
"total_cost": 0.00012924,
"total_input_tokens": 36,
"total_output_tokens": 1593
}
]
}
]
```
</TabItem>
<TabItem value="per user" label="Spend for Internal User (Key Owner)">
:::info
Internal User (Key Owner): This is the value of `user_id` passed when calling [`/key/generate`](https://litellm-api.up.railway.app/#/key%20management/generate_key_fn_key_generate_post)
:::
👉 Key Change: Specify `internal_user_id=ishaan`
```shell
curl -X GET 'http://localhost:4000/global/spend/report?start_date=2024-04-01&end_date=2024-12-30&internal_user_id=ishaan' \
-H 'Authorization: Bearer sk-1234'
```
##### Example Response
```shell
[
{
"api_key": "88dc28d0f030c55ed4ab77ed8faf098196cb1c05df778539800c9f1243fe6b4b",
"total_cost": 0.00013132,
"total_input_tokens": 105.0,
"total_output_tokens": 872.0,
"model_details": [
{
"model": "gpt-3.5-turbo-instruct",
"total_cost": 5.85e-05,
"total_input_tokens": 15,
"total_output_tokens": 18
},
{
"model": "llama3-8b-8192",
"total_cost": 7.282000000000001e-05,
"total_input_tokens": 90,
"total_output_tokens": 854
}
]
},
{
"api_key": "151e85e46ab8c9c7fad090793e3fe87940213f6ae665b543ca633b0b85ba6dc6",
"total_cost": 5.2699999999999993e-05,
"total_input_tokens": 26.0,
"total_output_tokens": 27.0,
"model_details": [
{
"model": "gpt-3.5-turbo",
"total_cost": 5.2499999999999995e-05,
"total_input_tokens": 24,
"total_output_tokens": 27
},
{
"model": "text-embedding-ada-002",
"total_cost": 2e-07,
"total_input_tokens": 2,
"total_output_tokens": 0
}
]
},
{
"api_key": "60cb83a2dcbf13531bd27a25f83546ecdb25a1a6deebe62d007999dc00e1e32a",
"total_cost": 9.42e-06,
"total_input_tokens": 30.0,
"total_output_tokens": 99.0,
"model_details": [
{
"model": "llama3-8b-8192",
"total_cost": 9.42e-06,
"total_input_tokens": 30,
"total_output_tokens": 99
}
]
}
]
```
</TabItem>
</Tabs>
#### Allowing Non-Proxy Admins to access `/spend` endpoints
Use this when you want non-proxy admins to access `/spend` endpoints
@ -135,7 +482,7 @@ Schedule a [meeting with us to get your Enterprise License](https://calendly.com
:::
### Create Key
##### Create Key
Create Key with with `permissions={"get_spend_routes": true}`
```shell
curl --location 'http://0.0.0.0:4000/key/generate' \
@ -146,7 +493,7 @@ curl --location 'http://0.0.0.0:4000/key/generate' \
}'
```
### Use generated key on `/spend` endpoints
##### Use generated key on `/spend` endpoints
Access spend Routes with newly generate keys
```shell
@ -156,14 +503,14 @@ curl -X GET 'http://localhost:4000/global/spend/report?start_date=2024-04-01&end
## Reset Team, API Key Spend - MASTER KEY ONLY
#### Reset Team, API Key Spend - MASTER KEY ONLY
Use `/global/spend/reset` if you want to:
- Reset the Spend for all API Keys, Teams. The `spend` for ALL Teams and Keys in `LiteLLM_TeamTable` and `LiteLLM_VerificationToken` will be set to `spend=0`
- LiteLLM will maintain all the logs in `LiteLLMSpendLogs` for Auditing Purposes
### Request
##### Request
Only the `LITELLM_MASTER_KEY` you set can access this route
```shell
curl -X POST \
@ -172,7 +519,7 @@ curl -X POST \
-H 'Content-Type: application/json'
```
### Expected Responses
##### Expected Responses
```shell
{"message":"Spend for all API Keys and Teams reset successfully","status":"success"}
@ -181,11 +528,11 @@ curl -X POST \
## Spend Tracking for Azure
## Spend Tracking for Azure OpenAI Models
Set base model for cost tracking azure image-gen call
### Image Generation
#### Image Generation
```yaml
model_list:
@ -200,7 +547,7 @@ model_list:
mode: image_generation
```
### Chat Completions / Embeddings
#### Chat Completions / Embeddings
**Problem**: Azure returns `gpt-4` in the response when `azure/gpt-4-1106-preview` is used. This leads to inaccurate cost tracking
@ -220,3 +567,26 @@ model_list:
model_info:
base_model: azure/gpt-4-1106-preview
```
## Custom Input/Output Pricing
👉 Head to [Custom Input/Output Pricing](https://docs.litellm.ai/docs/proxy/custom_pricing) to setup custom pricing or your models
## ✨ Custom k,v pairs
Log specific key,value pairs as part of the metadata for a spend log
:::info
Logging specific key,value pairs in spend logs metadata is an enterprise feature. [See here](./enterprise.md#tracking-spend-with-custom-metadata)
:::
## ✨ Custom Tags
:::info
Tracking spend with Custom tags is an enterprise feature. [See here](./enterprise.md#tracking-spend-for-custom-tags)
:::

251
docs/my-website/docs/proxy/customers.md Normal file
View file

@ -0,0 +1,251 @@
import Image from '@theme/IdealImage';
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
# 🙋‍♂️ Customers
Track spend, set budgets for your customers.
## Tracking Customer Credit
### 1. Make LLM API call w/ Customer ID
Make a /chat/completions call, pass 'user' - First call Works
```bash
curl -X POST 'http://0.0.0.0:4000/chat/completions' \
--header 'Content-Type: application/json' \
--header 'Authorization: Bearer sk-1234' \ # 👈 YOUR PROXY KEY
--data ' {
"model": "azure-gpt-3.5",
"user": "ishaan3", # 👈 CUSTOMER ID
"messages": [
{
"role": "user",
"content": "what time is it"
}
]
}'
```
The customer_id will be upserted into the DB with the new spend.
If the customer_id already exists, spend will be incremented.
### 2. Get Customer Spend
<Tabs>
<TabItem value="all-up" label="All-up spend">
Call `/customer/info` to get a customer's all up spend
```bash
curl -X GET 'http://0.0.0.0:4000/customer/info?end_user_id=ishaan3' \ # 👈 CUSTOMER ID
-H 'Authorization: Bearer sk-1234' \ # 👈 YOUR PROXY KEY
```
Expected Response:
```
{
"user_id": "ishaan3",
"blocked": false,
"alias": null,
"spend": 0.001413,
"allowed_model_region": null,
"default_model": null,
"litellm_budget_table": null
}
```
</TabItem>
<TabItem value="event-webhook" label="Event Webhook">
To update spend in your client-side DB, point the proxy to your webhook.
E.g. if your server is `https://webhook.site` and your listening on `6ab090e8-c55f-4a23-b075-3209f5c57906`
1. Add webhook url to your proxy environment:
```bash
export WEBHOOK_URL="https://webhook.site/6ab090e8-c55f-4a23-b075-3209f5c57906"
```
2. Add 'webhook' to config.yaml
```yaml
general_settings:
alerting: ["webhook"] # 👈 KEY CHANGE
```
3. Test it!
```bash
curl -X POST 'http://localhost:4000/chat/completions' \
-H 'Content-Type: application/json' \
-H 'Authorization: Bearer sk-1234' \
-D '{
"model": "mistral",
"messages": [
{
"role": "user",
"content": "What's the weather like in Boston today?"
}
],
"user": "krrish12"
}
'
```
Expected Response
```json
{
"spend": 0.0011120000000000001, # 👈 SPEND
"max_budget": null,
"token": "88dc28d0f030c55ed4ab77ed8faf098196cb1c05df778539800c9f1243fe6b4b",
"customer_id": "krrish12", # 👈 CUSTOMER ID
"user_id": null,
"team_id": null,
"user_email": null,
"key_alias": null,
"projected_exceeded_date": null,
"projected_spend": null,
"event": "spend_tracked",
"event_group": "customer",
"event_message": "Customer spend tracked. Customer=krrish12, spend=0.0011120000000000001"
}
```
[See Webhook Spec](./alerting.md#api-spec-for-webhook-event)
</TabItem>
</Tabs>
## Setting Customer Budgets
Set customer budgets (e.g. monthly budgets, tpm/rpm limits) on LiteLLM Proxy
### Quick Start
Create / Update a customer with budget
**Create New Customer w/ budget**
```bash
curl -X POST 'http://0.0.0.0:4000/customer/new'
-H 'Authorization: Bearer sk-1234'
-H 'Content-Type: application/json'
-D '{
"user_id" : "my-customer-id",
"max_budget": "0", # 👈 CAN BE FLOAT
}'
```
**Test it!**
```bash
curl -X POST 'http://localhost:4000/chat/completions' \
-H 'Content-Type: application/json' \
-H 'Authorization: Bearer sk-1234' \
-D '{
"model": "mistral",
"messages": [
{
"role": "user",
"content": "What'\''s the weather like in Boston today?"
}
],
"user": "ishaan-jaff-48"
}
```
### Assign Pricing Tiers
Create and assign customers to pricing tiers.
#### 1. Create a budget
<Tabs>
<TabItem value="ui" label="UI">
- Go to the 'Budgets' tab on the UI.
- Click on '+ Create Budget'.
- Create your pricing tier (e.g. 'my-free-tier' with budget $4). This means each user on this pricing tier will have a max budget of $4.
<Image img={require('../../img/create_budget_modal.png')} />
</TabItem>
<TabItem value="api" label="API">
Use the `/budget/new` endpoint for creating a new budget. [API Reference](https://litellm-api.up.railway.app/#/budget%20management/new_budget_budget_new_post)
```bash
curl -X POST 'http://localhost:4000/budget/new' \
-H 'Content-Type: application/json' \
-H 'Authorization: Bearer sk-1234' \
-D '{
"budget_id": "my-free-tier",
"max_budget": 4
}
```
</TabItem>
</Tabs>
#### 2. Assign Budget to Customer
In your application code, assign budget when creating a new customer.
Just use the `budget_id` used when creating the budget. In our example, this is `my-free-tier`.
```bash
curl -X POST 'http://localhost:4000/customer/new' \
-H 'Content-Type: application/json' \
-H 'Authorization: Bearer sk-1234' \
-D '{
"user_id": "my-customer-id",
"budget_id": "my-free-tier" # 👈 KEY CHANGE
}
```
#### 3. Test it!
<Tabs>
<TabItem value="curl" label="curl">
```bash
curl -X POST 'http://localhost:4000/customer/new' \
-H 'Content-Type: application/json' \
-H 'Authorization: Bearer sk-1234' \
-D '{
"user_id": "my-customer-id",
"budget_id": "my-free-tier" # 👈 KEY CHANGE
}
```
</TabItem>
<TabItem value="openai" label="OpenAI">
```python
from openai import OpenAI
client = OpenAI(
base_url="<your_proxy_base_url>",
api_key="<your_proxy_key>"
)
completion = client.chat.completions.create(
model="gpt-3.5-turbo",
messages=[
{"role": "system", "content": "You are a helpful assistant."},
{"role": "user", "content": "Hello!"}
],
user="my-customer-id"
)
print(completion.choices[0].message)
```
</TabItem>
</Tabs>

66
docs/my-website/docs/proxy/debugging.md
View file

@ -42,6 +42,14 @@ Set `JSON_LOGS="True"` in your env:
```bash
export JSON_LOGS="True"
```
**OR**
Set `json_logs: true` in your yaml:
```yaml
litellm_settings:
json_logs: true
```
Start proxy
@ -50,3 +58,61 @@ $ litellm
```
The proxy will now all logs in json format.
## Control Log Output
Turn off fastapi's default 'INFO' logs
1. Turn on 'json logs'
```yaml
litellm_settings:
json_logs: true
```
2. Set `LITELLM_LOG` to 'ERROR'
Only get logs if an error occurs.
```bash
LITELLM_LOG="ERROR"
```
3. Start proxy
```bash
$ litellm
```
Expected Output:
```bash
# no info statements
```
## Common Errors
1. "No available deployments..."
```
No deployments available for selected model, Try again in 60 seconds. Passed model=claude-3-5-sonnet. pre-call-checks=False, allowed_model_region=n/a.
```
This can be caused due to all your models hitting rate limit errors, causing the cooldown to kick in.
How to control this?
- Adjust the cooldown time
```yaml
router_settings:
cooldown_time: 0 # 👈 KEY CHANGE
```
- Disable Cooldowns [NOT RECOMMENDED]
```yaml
router_settings:
disable_cooldowns: True
```
This is not recommended, as it will lead to requests being routed to deployments over their tpm/rpm limit.

65
docs/my-website/docs/proxy/deploy.md
View file

@ -1,5 +1,6 @@
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
import Image from '@theme/IdealImage';
# 🐳 Docker, Deploying LiteLLM Proxy
@ -7,9 +8,33 @@ You can find the Dockerfile to build litellm proxy [here](https://github.com/Ber
## Quick Start
To start using Litellm, run the following commands in a shell:
```bash
# Get the code
git clone https://github.com/BerriAI/litellm
# Go to folder
cd litellm
# Add the master key - you can change this after setup
echo 'LITELLM_MASTER_KEY="sk-1234"' > .env
# Add the litellm salt key - you cannot change this after adding a model
# It is used to encrypt / decrypt your LLM API Key credentials
# We recommned - https://1password.com/password-generator/
# password generator to get a random hash for litellm salt key
echo 'LITELLM_SALT_KEY="sk-1234"' > .env
source .env
# Start
docker-compose up
```
<Tabs>
<TabItem value="basic" label="Basic">
<TabItem value="basic" label="Basic (No DB)">
### Step 1. CREATE config.yaml
@ -80,7 +105,13 @@ docker run ghcr.io/berriai/litellm:main-latest --port 8002 --num_workers 8
```
</TabItem>
<TabItem value="terraform" label="Terraform">
s/o [Nicholas Cecere](https://www.linkedin.com/in/nicholas-cecere-24243549/) for his LiteLLM User Management Terraform
👉 [Go here for Terraform](https://github.com/ncecere/terraform-litellm-user-mgmt)
</TabItem>
<TabItem value="base-image" label="use litellm as a base image">
```shell
@ -243,7 +274,7 @@ Requirements:
<TabItem value="docker-deploy" label="Dockerfile">
We maintain a [seperate Dockerfile](https://github.com/BerriAI/litellm/pkgs/container/litellm-database) for reducing build time when running LiteLLM proxy with a connected Postgres Database
We maintain a [separate Dockerfile](https://github.com/BerriAI/litellm/pkgs/container/litellm-database) for reducing build time when running LiteLLM proxy with a connected Postgres Database
```shell
docker pull ghcr.io/berriai/litellm-database:main-latest
@ -362,6 +393,7 @@ kubectl port-forward service/litellm-service 4000:4000
Your OpenAI proxy server is now running on `http://0.0.0.0:4000`.
</TabItem>
<TabItem value="helm-deploy" label="Helm">
@ -407,7 +439,6 @@ If you need to set your litellm proxy config.yaml, you can find this in [values.
</TabItem>
<TabItem value="helm-oci" label="Helm OCI Registry (GHCR)">
:::info
@ -520,7 +551,9 @@ ghcr.io/berriai/litellm-database:main-latest --config your_config.yaml
## Advanced Deployment Settings
### Customization of the server root path
### 1. Customization of the server root path (custom Proxy base url)
💥 Use this when you want to serve LiteLLM on a custom base url path like `https://localhost:4000/api/v1`
:::info
@ -531,9 +564,29 @@ In a Kubernetes deployment, it's possible to utilize a shared DNS to host multip
Customize the root path to eliminate the need for employing multiple DNS configurations during deployment.
👉 Set `SERVER_ROOT_PATH` in your .env and this will be set as your server root path
```
export SERVER_ROOT_PATH="/api/v1"
```
**Step 1. Run Proxy with `SERVER_ROOT_PATH` set in your env **
### Setting SSL Certification
```shell
docker run --name litellm-proxy \
-e DATABASE_URL=postgresql://<user>:<password>@<host>:<port>/<dbname> \
-e SERVER_ROOT_PATH="/api/v1" \
-p 4000:4000 \
ghcr.io/berriai/litellm-database:main-latest --config your_config.yaml
```
After running the proxy you can access it on `http://0.0.0.0:4000/api/v1/` (since we set `SERVER_ROOT_PATH="/api/v1"`)
**Step 2. Verify Running on correct path**
<Image img={require('../../img/custom_root_path.png')} />
**That's it**, that's all you need to run the proxy on a custom root path
### 2. Setting SSL Certification
Use this, If you need to set ssl certificates for your on prem litellm proxy
@ -629,7 +682,7 @@ Once the stack is created, get the DatabaseURL of the Database resource, copy th
#### 3. Connect to the EC2 Instance and deploy litellm on the EC2 container
From the EC2 console, connect to the instance created by the stack (e.g., using SSH).
Run the following command, replacing <database_url> with the value you copied in step 2
Run the following command, replacing `<database_url>` with the value you copied in step 2
```shell
docker run --name litellm-proxy \

1
docs/my-website/docs/proxy/email.md
View file

@ -5,6 +5,7 @@ import Image from '@theme/IdealImage';
Send an Email to your users when:
- A Proxy API Key is created for them
- Their API Key crosses it's Budget
- All Team members of a LiteLLM Team -> when the team crosses it's budget
<Image img={require('../../img/email_notifs.png')} style={{ width: '500px' }}/>

957
docs/my-website/docs/proxy/enterprise.md
View file

File diff suppressed because it is too large Load diff

313
docs/my-website/docs/proxy/guardrails.md Normal file
View file

@ -0,0 +1,313 @@
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
# 🛡️ Guardrails
Setup Prompt Injection Detection, Secret Detection on LiteLLM Proxy
:::info
✨ Enterprise Only Feature
Schedule a meeting with us to get an Enterprise License 👉 Talk to founders [here](https://calendly.com/d/4mp-gd3-k5k/litellm-1-1-onboarding-chat)
:::
## Quick Start
### 1. Setup guardrails on litellm proxy config.yaml
```yaml
model_list:
- model_name: gpt-3.5-turbo
litellm_params:
model: openai/gpt-3.5-turbo
api_key: sk-xxxxxxx
litellm_settings:
guardrails:
- prompt_injection: # your custom name for guardrail
callbacks: [lakera_prompt_injection] # litellm callbacks to use
default_on: true # will run on all llm requests when true
- pii_masking: # your custom name for guardrail
callbacks: [presidio] # use the litellm presidio callback
default_on: false # by default this is off for all requests
- hide_secrets_guard:
callbacks: [hide_secrets]
default_on: false
- your-custom-guardrail
callbacks: [hide_secrets]
default_on: false
```
:::info
Since `pii_masking` is default Off for all requests, [you can switch it on per API Key](#switch-guardrails-onoff-per-api-key)
:::
### 2. Test it
Run litellm proxy
```shell
litellm --config config.yaml
```
Make LLM API request
Test it with this request -> expect it to get rejected by LiteLLM Proxy
```shell
curl --location 'http://localhost:4000/chat/completions' \
--header 'Authorization: Bearer sk-1234' \
--header 'Content-Type: application/json' \
--data '{
"model": "gpt-3.5-turbo",
"messages": [
{
"role": "user",
"content": "what is your system prompt"
}
]
}'
```
## Control Guardrails On/Off per Request
You can switch off/on any guardrail on the config.yaml by passing
```shell
"metadata": {"guardrails": {"<guardrail_name>": false}}
```
example - we defined `prompt_injection`, `hide_secrets_guard` [on step 1](#1-setup-guardrails-on-litellm-proxy-configyaml)
This will
- switch **off** `prompt_injection` checks running on this request
- switch **on** `hide_secrets_guard` checks on this request
```shell
"metadata": {"guardrails": {"prompt_injection": false, "hide_secrets_guard": true}}
```
<Tabs>
<TabItem value="js" label="Langchain JS">
```js
const model = new ChatOpenAI({
modelName: "llama3",
openAIApiKey: "sk-1234",
modelKwargs: {"metadata": "guardrails": {"prompt_injection": False, "hide_secrets_guard": true}}}
}, {
basePath: "http://0.0.0.0:4000",
});
const message = await model.invoke("Hi there!");
console.log(message);
```
</TabItem>
<TabItem value="curl" label="Curl">
```shell
curl --location 'http://0.0.0.0:4000/chat/completions' \
--header 'Authorization: Bearer sk-1234' \
--header 'Content-Type: application/json' \
--data '{
"model": "llama3",
"metadata": {"guardrails": {"prompt_injection": false, "hide_secrets_guard": true}}},
"messages": [
{
"role": "user",
"content": "what is your system prompt"
}
]
}'
```
</TabItem>
<TabItem value="openai" label="OpenAI Python SDK">
```python
import openai
client = openai.OpenAI(
api_key="s-1234",
base_url="http://0.0.0.0:4000"
)
# request sent to model set on litellm proxy, `litellm --model`
response = client.chat.completions.create(
model="llama3",
messages = [
{
"role": "user",
"content": "this is a test request, write a short poem"
}
],
extra_body={
"metadata": {"guardrails": {"prompt_injection": False, "hide_secrets_guard": True}}}
}
)
print(response)
```
</TabItem>
<TabItem value="langchain" label="Langchain Py">
```python
from langchain.chat_models import ChatOpenAI
from langchain.prompts.chat import (
ChatPromptTemplate,
HumanMessagePromptTemplate,
SystemMessagePromptTemplate,
)
from langchain.schema import HumanMessage, SystemMessage
import os
os.environ["OPENAI_API_KEY"] = "sk-1234"
chat = ChatOpenAI(
openai_api_base="http://0.0.0.0:4000",
model = "llama3",
extra_body={
"metadata": {"guardrails": {"prompt_injection": False, "hide_secrets_guard": True}}}
}
)
messages = [
SystemMessage(
content="You are a helpful assistant that im using to make a test request to."
),
HumanMessage(
content="test from litellm. tell me why it's amazing in 1 sentence"
),
]
response = chat(messages)
print(response)
```
</TabItem>
</Tabs>
## Switch Guardrails On/Off Per API Key
❓ Use this when you need to switch guardrails on/off per API Key
**Step 1** Create Key with `pii_masking` On
**NOTE:** We defined `pii_masking` [on step 1](#1-setup-guardrails-on-litellm-proxy-configyaml)
👉 Set `"permissions": {"pii_masking": true}` with either `/key/generate` or `/key/update`
This means the `pii_masking` guardrail is on for all requests from this API Key
:::info
If you need to switch `pii_masking` off for an API Key set `"permissions": {"pii_masking": false}` with either `/key/generate` or `/key/update`
:::
<Tabs>
<TabItem value="/key/generate" label="/key/generate">
```shell
curl -X POST 'http://0.0.0.0:4000/key/generate' \
-H 'Authorization: Bearer sk-1234' \
-H 'Content-Type: application/json' \
-D '{
"permissions": {"pii_masking": true}
}'
```
```shell
# {"permissions":{"pii_masking":true},"key":"sk-jNm1Zar7XfNdZXp49Z1kSQ"}
```
</TabItem>
<TabItem value="/key/update" label="/key/update">
```shell
curl --location 'http://0.0.0.0:4000/key/update' \
--header 'Authorization: Bearer sk-1234' \
--header 'Content-Type: application/json' \
--data '{
"key": "sk-jNm1Zar7XfNdZXp49Z1kSQ",
"permissions": {"pii_masking": true}
}'
```
```shell
# {"permissions":{"pii_masking":true},"key":"sk-jNm1Zar7XfNdZXp49Z1kSQ"}
```
</TabItem>
</Tabs>
**Step 2** Test it with new key
```shell
curl --location 'http://0.0.0.0:4000/chat/completions' \
--header 'Authorization: Bearer sk-jNm1Zar7XfNdZXp49Z1kSQ' \
--header 'Content-Type: application/json' \
--data '{
"model": "llama3",
"messages": [
{
"role": "user",
"content": "does my phone number look correct - +1 412-612-9992"
}
]
}'
```
Expect to NOT see `+1 412-612-9992` in your server logs on your callback.
:::info
The `pii_masking` guardrail ran on this request because api key=sk-jNm1Zar7XfNdZXp49Z1kSQ has `"permissions": {"pii_masking": true}`
:::
## Spec for `guardrails` on litellm config
```yaml
litellm_settings:
guardrails:
- string: GuardrailItemSpec
```
- `string` - Your custom guardrail name
- `GuardrailItemSpec`:
- `callbacks`: List[str], list of supported guardrail callbacks.
- Full List: presidio, lakera_prompt_injection, hide_secrets, llmguard_moderations, llamaguard_moderations, google_text_moderation
- `default_on`: bool, will run on all llm requests when true
- `logging_only`: Optional[bool], if true, run guardrail only on logged output, not on the actual LLM API call. Currently only supported for presidio pii masking. Requires `default_on` to be True as well.
Example:
```yaml
litellm_settings:
guardrails:
- prompt_injection: # your custom name for guardrail
callbacks: [lakera_prompt_injection, hide_secrets, llmguard_moderations, llamaguard_moderations, google_text_moderation] # litellm callbacks to use
default_on: true # will run on all llm requests when true
- hide_secrets:
callbacks: [hide_secrets]
default_on: true
- pii_masking:
callback: ["presidio"]
default_on: true
logging_only: true
- your-custom-guardrail
callbacks: [hide_secrets]
default_on: false
```

97
docs/my-website/docs/proxy/health.md
View file

@ -112,6 +112,31 @@ model_list:
mode: completion # 👈 ADD THIS
```
### Speech to Text Models
```yaml
model_list:
- model_name: whisper
litellm_params:
model: whisper-1
api_key: os.environ/OPENAI_API_KEY
model_info:
mode: audio_transcription
```
### Hide details
The health check response contains details like endpoint URLs, error messages,
and other LiteLLM params. While this is useful for debugging, it can be
problematic when exposing the proxy server to a broad audience.
You can hide these details by setting the `health_check_details` setting to `False`.
```yaml
general_settings:
health_check_details: False
```
## `/health/readiness`
Unprotected endpoint for checking if proxy is ready to accept requests
@ -119,30 +144,32 @@ Unprotected endpoint for checking if proxy is ready to accept requests
Example Request:
```bash
curl --location 'http://0.0.0.0:4000/health/readiness'
curl http://0.0.0.0:4000/health/readiness
```
Example Response:
*If proxy connected to a database*
```json
{
"status": "healthy",
"status": "connected",
"db": "connected",
"litellm_version":"1.19.2",
"cache": null,
"litellm_version": "1.40.21",
"success_callbacks": [
"langfuse",
"_PROXY_track_cost_callback",
"response_taking_too_long_callback",
"_PROXY_MaxParallelRequestsHandler",
"_PROXY_MaxBudgetLimiter",
"_PROXY_CacheControlCheck",
"ServiceLogging"
],
"last_updated": "2024-07-10T18:59:10.616968"
}
```
*If proxy not connected to a database*
```json
{
"status": "healthy",
"db": "Not connected",
"litellm_version":"1.19.2",
}
```
If the proxy is not connected to a database, then the `"db"` field will be `"Not
connected"` instead of `"connected"` and the `"last_updated"` field will not be present.
## `/health/liveliness`
@ -162,3 +189,45 @@ Example Response:
```json
"I'm alive!"
```
## Advanced - Call specific models
To check health of specific models, here's how to call them:
### 1. Get model id via `/model/info`
```bash
curl -X GET 'http://0.0.0.0:4000/v1/model/info' \
--header 'Authorization: Bearer sk-1234' \
```
**Expected Response**
```bash
{
"model_name": "bedrock-anthropic-claude-3",
"litellm_params": {
"model": "anthropic.claude-3-sonnet-20240229-v1:0"
},
"model_info": {
"id": "634b87c444..", # 👈 UNIQUE MODEL ID
}
```
### 2. Call specific model via `/chat/completions`
```bash
curl -X POST 'http://localhost:4000/chat/completions' \
-H 'Content-Type: application/json' \
-H 'Authorization: Bearer sk-1234' \
-D '{
"model": "634b87c444.." # 👈 UNIQUE MODEL ID
"messages": [
{
"role": "user",
"content": "ping"
}
],
}
'
```

1083
docs/my-website/docs/proxy/logging.md
View file

File diff suppressed because it is too large Load diff

7
docs/my-website/docs/proxy/model_management.md
View file

@ -1,3 +1,6 @@
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
# Model Management
Add new models + Get model info without restarting proxy.
@ -12,9 +15,9 @@ model_list:
metadata: "here's additional metadata on the model" # returned via GET /model/info
```
## Get Model Information
## Get Model Information - `/model/info`
Retrieve detailed information about each model listed in the `/models` endpoint, including descriptions from the `config.yaml` file, and additional model info (e.g. max tokens, cost per input token, etc.) pulled the model_info you set and the litellm model cost map. Sensitive details like API keys are excluded for security purposes.
Retrieve detailed information about each model listed in the `/model/info` endpoint, including descriptions from the `config.yaml` file, and additional model info (e.g. max tokens, cost per input token, etc.) pulled the model_info you set and the litellm model cost map. Sensitive details like API keys are excluded for security purposes.
<Tabs
defaultValue="curl"

99
docs/my-website/docs/proxy/multiple_admins.md Normal file
View file

@ -0,0 +1,99 @@
# ✨ Attribute Management changes to Users
Call management endpoints on behalf of a user. (Useful when connecting proxy to your development platform).
:::tip
Requires Enterprise License, Get in touch with us [here](https://calendly.com/d/4mp-gd3-k5k/litellm-1-1-onboarding-chat)
:::
## 1. Switch on audit Logs
Add `store_audit_logs` to your litellm config.yaml and then start the proxy
```shell
litellm_settings:
store_audit_logs: true
```
## 2. Set `LiteLLM-Changed-By` in request headers
Set the 'user_id' in request headers, when calling a management endpoint. [View Full List](https://litellm-api.up.railway.app/#/team%20management).
- Update Team budget with master key.
- Attribute change to 'krrish@berri.ai'.
**👉 Key change:** Passing `-H 'LiteLLM-Changed-By: krrish@berri.ai'`
```shell
curl -X POST 'http://0.0.0.0:4000/team/update' \
-H 'Authorization: Bearer sk-1234' \
-H 'LiteLLM-Changed-By: krrish@berri.ai' \
-H 'Content-Type: application/json' \
-d '{
"team_id" : "8bf18b11-7f52-4717-8e1f-7c65f9d01e52",
"max_budget": 2000
}'
```
## 3. Emitted Audit Log
```bash
{
"id": "bd136c28-edd0-4cb6-b963-f35464cf6f5a",
"updated_at": "2024-06-08 23:41:14.793",
"changed_by": "krrish@berri.ai", # 👈 CHANGED BY
"changed_by_api_key": "88dc28d0f030c55ed4ab77ed8faf098196cb1c05df778539800c9f1243fe6b4b",
"action": "updated",
"table_name": "LiteLLM_TeamTable",
"object_id": "8bf18b11-7f52-4717-8e1f-7c65f9d01e52",
"before_value": {
"spend": 0,
"max_budget": 0,
},
"updated_values": {
"team_id": "8bf18b11-7f52-4717-8e1f-7c65f9d01e52",
"max_budget": 2000 # 👈 CHANGED TO
},
}
```
## API SPEC of Audit Log
### `id`
- **Type:** `String`
- **Description:** This is the unique identifier for each audit log entry. It is automatically generated as a UUID (Universally Unique Identifier) by default.
### `updated_at`
- **Type:** `DateTime`
- **Description:** This field stores the timestamp of when the audit log entry was created or updated. It is automatically set to the current date and time by default.
### `changed_by`
- **Type:** `String`
- **Description:** The `user_id` that performed the audited action. If `LiteLLM-Changed-By` Header is passed then `changed_by=<value passed for LiteLLM-Changed-By header>`
### `changed_by_api_key`
- **Type:** `String`
- **Description:** This field stores the hashed API key that was used to perform the audited action. If left blank, it defaults to an empty string.
### `action`
- **Type:** `String`
- **Description:** The type of action that was performed. One of "create", "update", or "delete".
### `table_name`
- **Type:** `String`
- **Description:** This field stores the name of the table that was affected by the audited action. It can be one of the following values: `LiteLLM_TeamTable`, `LiteLLM_UserTable`, `LiteLLM_VerificationToken`
### `object_id`
- **Type:** `String`
- **Description:** This field stores the ID of the object that was affected by the audited action. It can be the key ID, team ID, user ID
### `before_value`
- **Type:** `Json?`
- **Description:** This field stores the value of the row before the audited action was performed. It is optional and can be null.
### `updated_values`
- **Type:** `Json?`
- **Description:** This field stores the values of the row that were updated after the audited action was performed

367
docs/my-website/docs/proxy/pass_through.md Normal file
View file

@ -0,0 +1,367 @@
import Image from '@theme/IdealImage';
# ➡️ Create Pass Through Endpoints
Add pass through routes to LiteLLM Proxy
**Example:** Add a route `/v1/rerank` that forwards requests to `https://api.cohere.com/v1/rerank` through LiteLLM Proxy
💡 This allows making the following Request to LiteLLM Proxy
```shell
curl --request POST \
--url http://localhost:4000/v1/rerank \
--header 'accept: application/json' \
--header 'content-type: application/json' \
--data '{
"model": "rerank-english-v3.0",
"query": "What is the capital of the United States?",
"top_n": 3,
"documents": ["Carson City is the capital city of the American state of Nevada."]
}'
```
## Tutorial - Pass through Cohere Re-Rank Endpoint
**Step 1** Define pass through routes on [litellm config.yaml](configs.md)
```yaml
general_settings:
master_key: sk-1234
pass_through_endpoints:
- path: "/v1/rerank" # route you want to add to LiteLLM Proxy Server
target: "https://api.cohere.com/v1/rerank" # URL this route should forward requests to
headers: # headers to forward to this URL
Authorization: "bearer os.environ/COHERE_API_KEY" # (Optional) Auth Header to forward to your Endpoint
content-type: application/json # (Optional) Extra Headers to pass to this endpoint
accept: application/json
```
**Step 2** Start Proxy Server in detailed_debug mode
```shell
litellm --config config.yaml --detailed_debug
```
**Step 3** Make Request to pass through endpoint
Here `http://localhost:4000` is your litellm proxy endpoint
```shell
curl --request POST \
--url http://localhost:4000/v1/rerank \
--header 'accept: application/json' \
--header 'content-type: application/json' \
--data '{
"model": "rerank-english-v3.0",
"query": "What is the capital of the United States?",
"top_n": 3,
"documents": ["Carson City is the capital city of the American state of Nevada.",
"The Commonwealth of the Northern Mariana Islands is a group of islands in the Pacific Ocean. Its capital is Saipan.",
"Washington, D.C. (also known as simply Washington or D.C., and officially as the District of Columbia) is the capital of the United States. It is a federal district.",
"Capitalization or capitalisation in English grammar is the use of a capital letter at the start of a word. English usage varies from capitalization in other languages.",
"Capital punishment (the death penalty) has existed in the United States since beforethe United States was a country. As of 2017, capital punishment is legal in 30 of the 50 states."]
}'
```
🎉 **Expected Response**
This request got forwarded from LiteLLM Proxy -> Defined Target URL (with headers)
```shell
{
"id": "37103a5b-8cfb-48d3-87c7-da288bedd429",
"results": [
{
"index": 2,
"relevance_score": 0.999071
},
{
"index": 4,
"relevance_score": 0.7867867
},
{
"index": 0,
"relevance_score": 0.32713068
}
],
"meta": {
"api_version": {
"version": "1"
},
"billed_units": {
"search_units": 1
}
}
}
```
## Tutorial - Pass Through Langfuse Requests
**Step 1** Define pass through routes on [litellm config.yaml](configs.md)
```yaml
general_settings:
master_key: sk-1234
pass_through_endpoints:
- path: "/api/public/ingestion" # route you want to add to LiteLLM Proxy Server
target: "https://us.cloud.langfuse.com/api/public/ingestion" # URL this route should forward
headers:
LANGFUSE_PUBLIC_KEY: "os.environ/LANGFUSE_DEV_PUBLIC_KEY" # your langfuse account public key
LANGFUSE_SECRET_KEY: "os.environ/LANGFUSE_DEV_SK_KEY" # your langfuse account secret key
```
**Step 2** Start Proxy Server in detailed_debug mode
```shell
litellm --config config.yaml --detailed_debug
```
**Step 3** Make Request to pass through endpoint
Run this code to make a sample trace
```python
from langfuse import Langfuse
langfuse = Langfuse(
host="http://localhost:4000", # your litellm proxy endpoint
public_key="anything", # no key required since this is a pass through
secret_key="anything", # no key required since this is a pass through
)
print("sending langfuse trace request")
trace = langfuse.trace(name="test-trace-litellm-proxy-passthrough")
print("flushing langfuse request")
langfuse.flush()
print("flushed langfuse request")
```
🎉 **Expected Response**
On success
Expect to see the following Trace Generated on your Langfuse Dashboard
<Image img={require('../../img/proxy_langfuse.png')} />
You will see the following endpoint called on your litellm proxy server logs
```shell
POST /api/public/ingestion HTTP/1.1" 207 Multi-Status
```
## ✨ [Enterprise] - Use LiteLLM keys/authentication on Pass Through Endpoints
Use this if you want the pass through endpoint to honour LiteLLM keys/authentication
This also enforces the key's rpm limits on pass-through endpoints.
Usage - set `auth: true` on the config
```yaml
general_settings:
master_key: sk-1234
pass_through_endpoints:
- path: "/v1/rerank"
target: "https://api.cohere.com/v1/rerank"
auth: true # 👈 Key change to use LiteLLM Auth / Keys
headers:
Authorization: "bearer os.environ/COHERE_API_KEY"
content-type: application/json
accept: application/json
```
Test Request with LiteLLM Key
```shell
curl --request POST \
--url http://localhost:4000/v1/rerank \
--header 'accept: application/json' \
--header 'Authorization: Bearer sk-1234'\
--header 'content-type: application/json' \
--data '{
"model": "rerank-english-v3.0",
"query": "What is the capital of the United States?",
"top_n": 3,
"documents": ["Carson City is the capital city of the American state of Nevada.",
"The Commonwealth of the Northern Mariana Islands is a group of islands in the Pacific Ocean. Its capital is Saipan.",
"Washington, D.C. (also known as simply Washington or D.C., and officially as the District of Columbia) is the capital of the United States. It is a federal district.",
"Capitalization or capitalisation in English grammar is the use of a capital letter at the start of a word. English usage varies from capitalization in other languages.",
"Capital punishment (the death penalty) has existed in the United States since beforethe United States was a country. As of 2017, capital punishment is legal in 30 of the 50 states."]
}'
```
## `pass_through_endpoints` Spec on config.yaml
All possible values for `pass_through_endpoints` and what they mean
**Example config**
```yaml
general_settings:
pass_through_endpoints:
- path: "/v1/rerank" # route you want to add to LiteLLM Proxy Server
target: "https://api.cohere.com/v1/rerank" # URL this route should forward requests to
headers: # headers to forward to this URL
Authorization: "bearer os.environ/COHERE_API_KEY" # (Optional) Auth Header to forward to your Endpoint
content-type: application/json # (Optional) Extra Headers to pass to this endpoint
accept: application/json
```
**Spec**
* `pass_through_endpoints` *list*: A collection of endpoint configurations for request forwarding.
* `path` *string*: The route to be added to the LiteLLM Proxy Server.
* `target` *string*: The URL to which requests for this path should be forwarded.
* `headers` *object*: Key-value pairs of headers to be forwarded with the request. You can set any key value pair here and it will be forwarded to your target endpoint
* `Authorization` *string*: The authentication header for the target API.
* `content-type` *string*: The format specification for the request body.
* `accept` *string*: The expected response format from the server.
* `LANGFUSE_PUBLIC_KEY` *string*: Your Langfuse account public key - only set this when forwarding to Langfuse.
* `LANGFUSE_SECRET_KEY` *string*: Your Langfuse account secret key - only set this when forwarding to Langfuse.
* `<your-custom-header>` *string*: Pass any custom header key/value pair
## Custom Chat Endpoints (Anthropic/Bedrock/Vertex)
Allow developers to call the proxy with Anthropic/boto3/etc. client sdk's.
Test our [Anthropic Adapter](../anthropic_completion.md) for reference [**Code**](https://github.com/BerriAI/litellm/blob/fd743aaefd23ae509d8ca64b0c232d25fe3e39ee/litellm/adapters/anthropic_adapter.py#L50)
### 1. Write an Adapter
Translate the request/response from your custom API schema to the OpenAI schema (used by litellm.completion()) and back.
For provider-specific params 👉 [**Provider-Specific Params**](../completion/provider_specific_params.md)
```python
from litellm import adapter_completion
import litellm
from litellm import ChatCompletionRequest, verbose_logger
from litellm.integrations.custom_logger import CustomLogger
from litellm.types.llms.anthropic import AnthropicMessagesRequest, AnthropicResponse
import os
# What is this?
## Translates OpenAI call to Anthropic `/v1/messages` format
import json
import os
import traceback
import uuid
from typing import Literal, Optional
import dotenv
import httpx
from pydantic import BaseModel
###################
# CUSTOM ADAPTER ##
###################
class AnthropicAdapter(CustomLogger):
def __init__(self) -> None:
super().__init__()
def translate_completion_input_params(
self, kwargs
) -> Optional[ChatCompletionRequest]:
"""
- translate params, where needed
- pass rest, as is
"""
request_body = AnthropicMessagesRequest(**kwargs) # type: ignore
translated_body = litellm.AnthropicConfig().translate_anthropic_to_openai(
anthropic_message_request=request_body
)
return translated_body
def translate_completion_output_params(
self, response: litellm.ModelResponse
) -> Optional[AnthropicResponse]:
return litellm.AnthropicConfig().translate_openai_response_to_anthropic(
response=response
)
def translate_completion_output_params_streaming(self) -> Optional[BaseModel]:
return super().translate_completion_output_params_streaming()
anthropic_adapter = AnthropicAdapter()
###########
# TEST IT #
###########
## register CUSTOM ADAPTER
litellm.adapters = [{"id": "anthropic", "adapter": anthropic_adapter}]
## set ENV variables
os.environ["OPENAI_API_KEY"] = "your-openai-key"
os.environ["COHERE_API_KEY"] = "your-cohere-key"
messages = [{ "content": "Hello, how are you?","role": "user"}]
# openai call
response = adapter_completion(model="gpt-3.5-turbo", messages=messages, adapter_id="anthropic")
# cohere call
response = adapter_completion(model="command-nightly", messages=messages, adapter_id="anthropic")
print(response)
```
### 2. Create new endpoint
We pass the custom callback class defined in Step1 to the config.yaml. Set callbacks to python_filename.logger_instance_name
In the config below, we pass
python_filename: `custom_callbacks.py`
logger_instance_name: `anthropic_adapter`. This is defined in Step 1
`target: custom_callbacks.proxy_handler_instance`
```yaml
model_list:
- model_name: my-fake-claude-endpoint
litellm_params:
model: gpt-3.5-turbo
api_key: os.environ/OPENAI_API_KEY
general_settings:
master_key: sk-1234
pass_through_endpoints:
- path: "/v1/messages" # route you want to add to LiteLLM Proxy Server
target: custom_callbacks.anthropic_adapter # Adapter to use for this route
headers:
litellm_user_api_key: "x-api-key" # Field in headers, containing LiteLLM Key
```
### 3. Test it!
**Start proxy**
```bash
litellm --config /path/to/config.yaml
```
**Curl**
```bash
curl --location 'http://0.0.0.0:4000/v1/messages' \
-H 'x-api-key: sk-1234' \
-H 'anthropic-version: 2023-06-01' \ # ignored
-H 'content-type: application/json' \
-D '{
"model": "my-fake-claude-endpoint",
"max_tokens": 1024,
"messages": [
{"role": "user", "content": "Hello, world"}
]
}'
```

2
docs/my-website/docs/proxy/perf.md
View file

@ -1,3 +1,5 @@
import Image from '@theme/IdealImage';
# LiteLLM Proxy Performance
### Throughput - 30% Increase

56
docs/my-website/docs/proxy/pii_masking.md
View file

@ -180,3 +180,59 @@ chat_completion = client.chat.completions.create(
"_response_ms": 1753.426
}
```
## Turn on for logging only
Only apply PII Masking before logging to Langfuse, etc.
Not on the actual llm api request / response.
:::note
This is currently only applied for
- `/chat/completion` requests
- on 'success' logging
:::
1. Setup config.yaml
```yaml
litellm_settings:
presidio_logging_only: true
model_list:
- model_name: gpt-3.5-turbo
litellm_params:
model: gpt-3.5-turbo
api_key: os.environ/OPENAI_API_KEY
```
2. Start proxy
```bash
litellm --config /path/to/config.yaml
```
3. Test it!
```bash
curl -X POST 'http://0.0.0.0:4000/chat/completions' \
-H 'Content-Type: application/json' \
-H 'Authorization: Bearer sk-1234' \
-D '{
"model": "gpt-3.5-turbo",
"messages": [
{
"role": "user",
"content": "Hi, my name is Jane!"
}
]
}'
```
**Expected Logged Response**
```
Hi, my name is <PERSON>!
```

14
docs/my-website/docs/proxy/prod.md
View file

@ -21,6 +21,7 @@ general_settings:
litellm_settings:
set_verbose: False # Switch off Debug Logging, ensure your logs do not have any debugging on
json_logs: true # Get debug logs in json format
```
Set slack webhook url in your env
@ -28,6 +29,11 @@ Set slack webhook url in your env
export SLACK_WEBHOOK_URL="https://hooks.slack.com/services/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"
```
Turn off FASTAPI's default info logs
```bash
export LITELLM_LOG="ERROR"
```
:::info
Need Help or want dedicated support ? Talk to a founder [here]: (https://calendly.com/d/4mp-gd3-k5k/litellm-1-1-onboarding-chat)
@ -62,6 +68,14 @@ router_settings:
redis_host: os.environ/REDIS_HOST
redis_port: os.environ/REDIS_PORT
redis_password: os.environ/REDIS_PASSWORD
litellm_settings:
cache: True
cache_params:
type: redis
host: os.environ/REDIS_HOST
port: os.environ/REDIS_PORT
password: os.environ/REDIS_PASSWORD
```
## 4. Disable 'load_dotenv'

68
docs/my-website/docs/proxy/prometheus.md
View file

@ -1,4 +1,7 @@
# Grafana, Prometheus metrics [BETA]
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
# 📈 Prometheus metrics [BETA]
LiteLLM Exposes a `/metrics` endpoint for Prometheus to Poll
@ -54,6 +57,63 @@ http://localhost:4000/metrics
| `litellm_total_tokens` | input + output tokens per `"user", "key", "model", "team", "end-user"` |
| `litellm_llm_api_failed_requests_metric` | Number of failed LLM API requests per `"user", "key", "model", "team", "end-user"` |
### Budget Metrics
| Metric Name | Description |
|----------------------|--------------------------------------|
| `litellm_remaining_team_budget_metric` | Remaining Budget for Team (A team created on LiteLLM) |
| `litellm_remaining_api_key_budget_metric` | Remaining Budget for API Key (A key Created on LiteLLM)|
### ✨ (Enterprise) LLM Remaining Requests and Remaining Tokens
Set this on your config.yaml to allow you to track how close you are to hitting your TPM / RPM limits on each model group
```yaml
litellm_settings:
success_callback: ["prometheus"]
failure_callback: ["prometheus"]
return_response_headers: true # ensures the LLM API calls track the response headers
```
| Metric Name | Description |
|----------------------|--------------------------------------|
| `litellm_remaining_requests_metric` | Track `x-ratelimit-remaining-requests` returned from LLM API Deployment |
| `litellm_remaining_tokens` | Track `x-ratelimit-remaining-tokens` return from LLM API Deployment |
Example Metric
<Tabs>
<TabItem value="Remaining Requests" label="Remaining Requests">
```shell
litellm_remaining_requests
{
api_base="https://api.openai.com/v1",
api_provider="openai",
litellm_model_name="gpt-3.5-turbo",
model_group="gpt-3.5-turbo"
}
8998.0
```
</TabItem>
<TabItem value="Requests" label="Remaining Tokens">
```shell
litellm_remaining_tokens
{
api_base="https://api.openai.com/v1",
api_provider="openai",
litellm_model_name="gpt-3.5-turbo",
model_group="gpt-3.5-turbo"
}
999981.0
```
</TabItem>
</Tabs>
## Monitor System Health
To monitor the health of litellm adjacent services (redis / postgres), do:
@ -72,3 +132,9 @@ litellm_settings:
| `litellm_redis_latency` | histogram latency for redis calls |
| `litellm_redis_fails` | Number of failed redis calls |
| `litellm_self_latency` | Histogram latency for successful litellm api call |
## 🔥 Community Maintained Grafana Dashboards
Link to Grafana Dashboards made by LiteLLM community
https://github.com/BerriAI/litellm/tree/main/cookbook/litellm_proxy_server/grafana_dashboard

7
docs/my-website/docs/proxy/prompt_injection.md
View file

@ -1,12 +1,15 @@
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
# 🕵️ Prompt Injection Detection
LiteLLM Supports the following methods for detecting prompt injection attacks
- [Using Lakera AI API](#lakeraai)
- [Using Lakera AI API](#✨-enterprise-lakeraai)
- [Similarity Checks](#similarity-checking)
- [LLM API Call to check](#llm-api-checks)
## LakeraAI
## ✨ [Enterprise] LakeraAI
Use this if you want to reject /chat, /completions, /embeddings calls that have prompt injection attacks

9
docs/my-website/docs/proxy/quick_start.md
View file

@ -24,6 +24,15 @@ $ litellm --model huggingface/bigcode/starcoder
#INFO: Proxy running on http://0.0.0.0:4000
```
:::info
Run with `--detailed_debug` if you need detailed debug logs
```shell
$ litellm --model huggingface/bigcode/starcoder --detailed_debug
:::
### Test
In a new shell, run, this will make an `openai.chat.completions` request. Ensure you're using openai v1.0.0+
```shell

Some files were not shown because too many files have changed in this diff Show more