forked from phoenix/litellm-mirror
1051 lines
No EOL
37 KiB
YAML
1051 lines
No EOL
37 KiB
YAML
version: 2.1
|
|
orbs:
|
|
codecov: codecov/codecov@4.0.1
|
|
|
|
jobs:
|
|
local_testing:
|
|
docker:
|
|
- image: cimg/python:3.11
|
|
auth:
|
|
username: ${DOCKERHUB_USERNAME}
|
|
password: ${DOCKERHUB_PASSWORD}
|
|
working_directory: ~/project
|
|
|
|
steps:
|
|
- checkout
|
|
|
|
- run:
|
|
name: Show git commit hash
|
|
command: |
|
|
echo "Git commit hash: $CIRCLE_SHA1"
|
|
|
|
- restore_cache:
|
|
keys:
|
|
- v1-dependencies-{{ checksum ".circleci/requirements.txt" }}
|
|
- run:
|
|
name: Install Dependencies
|
|
command: |
|
|
python -m pip install --upgrade pip
|
|
python -m pip install -r .circleci/requirements.txt
|
|
pip install "pytest==7.3.1"
|
|
pip install "pytest-retry==1.6.3"
|
|
pip install "pytest-asyncio==0.21.1"
|
|
pip install "pytest-cov==5.0.0"
|
|
pip install mypy
|
|
pip install "google-generativeai==0.3.2"
|
|
pip install "google-cloud-aiplatform==1.43.0"
|
|
pip install pyarrow
|
|
pip install "boto3==1.34.34"
|
|
pip install "aioboto3==12.3.0"
|
|
pip install langchain
|
|
pip install lunary==0.2.5
|
|
pip install "azure-identity==1.16.1"
|
|
pip install "langfuse==2.45.0"
|
|
pip install "logfire==0.29.0"
|
|
pip install numpydoc
|
|
pip install traceloop-sdk==0.21.1
|
|
pip install opentelemetry-api==1.25.0
|
|
pip install opentelemetry-sdk==1.25.0
|
|
pip install opentelemetry-exporter-otlp==1.25.0
|
|
pip install openai==1.51.0
|
|
pip install prisma==0.11.0
|
|
pip install "detect_secrets==1.5.0"
|
|
pip install "httpx==0.24.1"
|
|
pip install "respx==0.21.1"
|
|
pip install fastapi
|
|
pip install "gunicorn==21.2.0"
|
|
pip install "anyio==4.2.0"
|
|
pip install "aiodynamo==23.10.1"
|
|
pip install "asyncio==3.4.3"
|
|
pip install "apscheduler==3.10.4"
|
|
pip install "PyGithub==1.59.1"
|
|
pip install argon2-cffi
|
|
pip install "pytest-mock==3.12.0"
|
|
pip install python-multipart
|
|
pip install google-cloud-aiplatform
|
|
pip install prometheus-client==0.20.0
|
|
pip install "pydantic==2.7.1"
|
|
pip install "diskcache==5.6.1"
|
|
pip install "Pillow==10.3.0"
|
|
pip install "jsonschema==4.22.0"
|
|
- save_cache:
|
|
paths:
|
|
- ./venv
|
|
key: v1-dependencies-{{ checksum ".circleci/requirements.txt" }}
|
|
- run:
|
|
name: Run prisma ./docker/entrypoint.sh
|
|
command: |
|
|
set +e
|
|
chmod +x docker/entrypoint.sh
|
|
./docker/entrypoint.sh
|
|
set -e
|
|
- run:
|
|
name: Black Formatting
|
|
command: |
|
|
cd litellm
|
|
python -m pip install black
|
|
python -m black .
|
|
cd ..
|
|
- run:
|
|
name: Linting Testing
|
|
command: |
|
|
cd litellm
|
|
python -m pip install types-requests types-setuptools types-redis types-PyYAML
|
|
if ! python -m mypy . --ignore-missing-imports; then
|
|
echo "mypy detected errors"
|
|
exit 1
|
|
fi
|
|
cd ..
|
|
|
|
# Run pytest and generate JUnit XML report
|
|
- run:
|
|
name: Run tests
|
|
command: |
|
|
pwd
|
|
ls
|
|
python -m pytest -vv tests/local_testing --cov=litellm --cov-report=xml -x --junitxml=test-results/junit.xml --durations=5 -k "not test_python_38.py and not router and not assistants"
|
|
no_output_timeout: 120m
|
|
- run:
|
|
name: Rename the coverage files
|
|
command: |
|
|
mv coverage.xml local_testing_coverage.xml
|
|
mv .coverage local_testing_coverage
|
|
|
|
# Store test results
|
|
- store_test_results:
|
|
path: test-results
|
|
- persist_to_workspace:
|
|
root: .
|
|
paths:
|
|
- local_testing_coverage.xml
|
|
- local_testing_coverage
|
|
ui_endpoint_testing:
|
|
docker:
|
|
- image: cimg/python:3.11
|
|
auth:
|
|
username: ${DOCKERHUB_USERNAME}
|
|
password: ${DOCKERHUB_PASSWORD}
|
|
working_directory: ~/project
|
|
|
|
steps:
|
|
- checkout
|
|
- run:
|
|
name: Install Dependencies
|
|
command: |
|
|
python -m pip install --upgrade pip
|
|
python -m pip install -r requirements.txt
|
|
pip install "pytest==7.3.1"
|
|
pip install "pytest-retry==1.6.3"
|
|
pip install "pytest-asyncio==0.21.1"
|
|
pip install "pytest-cov==5.0.0"
|
|
- save_cache:
|
|
paths:
|
|
- ./venv
|
|
key: v1-dependencies-{{ checksum ".circleci/requirements.txt" }}
|
|
- run:
|
|
name: Run prisma ./docker/entrypoint.sh
|
|
command: |
|
|
set +e
|
|
chmod +x docker/entrypoint.sh
|
|
./docker/entrypoint.sh
|
|
set -e
|
|
# Run pytest and generate JUnit XML report
|
|
- run:
|
|
name: Run tests
|
|
command: |
|
|
pwd
|
|
ls
|
|
python -m pytest -vv tests/proxy_admin_ui_tests -x --cov=litellm --cov-report=xml --junitxml=test-results/junit.xml --durations=5
|
|
no_output_timeout: 120m
|
|
|
|
- run:
|
|
name: Rename the coverage files
|
|
command: |
|
|
mv coverage.xml ui_endpoint_testing_coverage.xml
|
|
mv .coverage ui_endpoint_testing_coverage
|
|
|
|
# Store test results
|
|
- store_test_results:
|
|
path: test-results
|
|
|
|
- persist_to_workspace:
|
|
root: .
|
|
paths:
|
|
- ui_endpoint_testing_coverage.xml
|
|
- ui_endpoint_testing_coverage
|
|
litellm_router_testing: # Runs all tests with the "router" keyword
|
|
docker:
|
|
- image: cimg/python:3.11
|
|
auth:
|
|
username: ${DOCKERHUB_USERNAME}
|
|
password: ${DOCKERHUB_PASSWORD}
|
|
working_directory: ~/project
|
|
|
|
steps:
|
|
- checkout
|
|
- run:
|
|
name: Install Dependencies
|
|
command: |
|
|
python -m pip install --upgrade pip
|
|
python -m pip install -r requirements.txt
|
|
pip install "pytest==7.3.1"
|
|
pip install "respx==0.21.1"
|
|
pip install "pytest-cov==5.0.0"
|
|
pip install "pytest-retry==1.6.3"
|
|
pip install "pytest-asyncio==0.21.1"
|
|
# Run pytest and generate JUnit XML report
|
|
- run:
|
|
name: Run tests
|
|
command: |
|
|
pwd
|
|
ls
|
|
python -m pytest tests/local_testing --cov=litellm --cov-report=xml -vv -k "router" -x -s -v --junitxml=test-results/junit.xml --durations=5
|
|
no_output_timeout: 120m
|
|
- run:
|
|
name: Rename the coverage files
|
|
command: |
|
|
mv coverage.xml litellm_router_coverage.xml
|
|
mv .coverage litellm_router_coverage
|
|
# Store test results
|
|
- store_test_results:
|
|
path: test-results
|
|
|
|
- persist_to_workspace:
|
|
root: .
|
|
paths:
|
|
- litellm_router_coverage.xml
|
|
- litellm_router_coverage
|
|
litellm_assistants_api_testing: # Runs all tests with the "assistants" keyword
|
|
docker:
|
|
- image: cimg/python:3.11
|
|
auth:
|
|
username: ${DOCKERHUB_USERNAME}
|
|
password: ${DOCKERHUB_PASSWORD}
|
|
working_directory: ~/project
|
|
|
|
steps:
|
|
- checkout
|
|
- run:
|
|
name: Install Dependencies
|
|
command: |
|
|
python -m pip install --upgrade pip
|
|
python -m pip install -r requirements.txt
|
|
pip install "pytest==7.3.1"
|
|
pip install "respx==0.21.1"
|
|
pip install "pytest-retry==1.6.3"
|
|
pip install "pytest-asyncio==0.21.1"
|
|
pip install "pytest-cov==5.0.0"
|
|
# Run pytest and generate JUnit XML report
|
|
- run:
|
|
name: Run tests
|
|
command: |
|
|
pwd
|
|
ls
|
|
python -m pytest tests/local_testing/ -vv -k "assistants" --cov=litellm --cov-report=xml -x -s -v --junitxml=test-results/junit.xml --durations=5
|
|
no_output_timeout: 120m
|
|
- run:
|
|
name: Rename the coverage files
|
|
command: |
|
|
mv coverage.xml litellm_assistants_api_coverage.xml
|
|
mv .coverage litellm_assistants_api_coverage
|
|
# Store test results
|
|
- store_test_results:
|
|
path: test-results
|
|
- persist_to_workspace:
|
|
root: .
|
|
paths:
|
|
- litellm_assistants_api_coverage.xml
|
|
- litellm_assistants_api_coverage
|
|
load_testing:
|
|
docker:
|
|
- image: cimg/python:3.11
|
|
auth:
|
|
username: ${DOCKERHUB_USERNAME}
|
|
password: ${DOCKERHUB_PASSWORD}
|
|
working_directory: ~/project
|
|
|
|
steps:
|
|
- checkout
|
|
- run:
|
|
name: Install Dependencies
|
|
command: |
|
|
python -m pip install --upgrade pip
|
|
python -m pip install -r requirements.txt
|
|
pip install "pytest==7.3.1"
|
|
pip install "pytest-retry==1.6.3"
|
|
pip install "pytest-asyncio==0.21.1"
|
|
# Run pytest and generate JUnit XML report
|
|
- run:
|
|
name: Run tests
|
|
command: |
|
|
pwd
|
|
ls
|
|
python -m pytest -vv tests/load_tests -x -s -v --junitxml=test-results/junit.xml --durations=5
|
|
no_output_timeout: 120m
|
|
|
|
# Store test results
|
|
- store_test_results:
|
|
path: test-results
|
|
llm_translation_testing:
|
|
docker:
|
|
- image: cimg/python:3.11
|
|
auth:
|
|
username: ${DOCKERHUB_USERNAME}
|
|
password: ${DOCKERHUB_PASSWORD}
|
|
working_directory: ~/project
|
|
|
|
steps:
|
|
- checkout
|
|
- run:
|
|
name: Install Dependencies
|
|
command: |
|
|
python -m pip install --upgrade pip
|
|
python -m pip install -r requirements.txt
|
|
pip install "pytest==7.3.1"
|
|
pip install "pytest-retry==1.6.3"
|
|
pip install "pytest-cov==5.0.0"
|
|
pip install "pytest-asyncio==0.21.1"
|
|
pip install "respx==0.21.1"
|
|
# Run pytest and generate JUnit XML report
|
|
- run:
|
|
name: Run tests
|
|
command: |
|
|
pwd
|
|
ls
|
|
python -m pytest -vv tests/llm_translation --cov=litellm --cov-report=xml -x -s -v --junitxml=test-results/junit.xml --durations=5
|
|
no_output_timeout: 120m
|
|
- run:
|
|
name: Rename the coverage files
|
|
command: |
|
|
mv coverage.xml llm_translation_coverage.xml
|
|
mv .coverage llm_translation_coverage
|
|
|
|
# Store test results
|
|
- store_test_results:
|
|
path: test-results
|
|
- persist_to_workspace:
|
|
root: .
|
|
paths:
|
|
- llm_translation_coverage.xml
|
|
- llm_translation_coverage
|
|
installing_litellm_on_python:
|
|
docker:
|
|
- image: circleci/python:3.8
|
|
auth:
|
|
username: ${DOCKERHUB_USERNAME}
|
|
password: ${DOCKERHUB_PASSWORD}
|
|
working_directory: ~/project
|
|
|
|
steps:
|
|
- checkout
|
|
- run:
|
|
name: Install Dependencies
|
|
command: |
|
|
python -m pip install --upgrade pip
|
|
pip install python-dotenv
|
|
pip install pytest
|
|
pip install tiktoken
|
|
pip install aiohttp
|
|
pip install openai
|
|
pip install click
|
|
pip install "boto3==1.34.34"
|
|
pip install jinja2
|
|
pip install tokenizers
|
|
pip install jsonschema
|
|
- run:
|
|
name: Run tests
|
|
command: |
|
|
pwd
|
|
ls
|
|
python -m pytest -vv tests/local_testing/test_python_38.py
|
|
|
|
check_code_and_doc_quality:
|
|
docker:
|
|
- image: cimg/python:3.11
|
|
auth:
|
|
username: ${DOCKERHUB_USERNAME}
|
|
password: ${DOCKERHUB_PASSWORD}
|
|
working_directory: ~/project/litellm
|
|
|
|
steps:
|
|
- checkout
|
|
- run:
|
|
name: Install Dependencies
|
|
command: |
|
|
python -m pip install --upgrade pip
|
|
pip install ruff
|
|
pip install pylint
|
|
pip install pyright
|
|
pip install .
|
|
- run: python -c "from litellm import *" || (echo '🚨 import failed, this means you introduced unprotected imports! 🚨'; exit 1)
|
|
- run: ruff check ./litellm
|
|
- run: python ./tests/documentation_tests/test_general_setting_keys.py
|
|
- run: python ./tests/documentation_tests/test_env_keys.py
|
|
|
|
db_migration_disable_update_check:
|
|
machine:
|
|
image: ubuntu-2204:2023.10.1
|
|
resource_class: xlarge
|
|
working_directory: ~/project
|
|
steps:
|
|
- checkout
|
|
- run:
|
|
name: Build Docker image
|
|
command: |
|
|
docker build -t myapp . -f ./docker/Dockerfile.database
|
|
- run:
|
|
name: Run Docker container
|
|
command: |
|
|
docker run --name my-app \
|
|
-p 4000:4000 \
|
|
-e DATABASE_URL=$PROXY_DATABASE_URL \
|
|
-e DISABLE_SCHEMA_UPDATE="True" \
|
|
-v $(pwd)/litellm/proxy/example_config_yaml/bad_schema.prisma:/app/schema.prisma \
|
|
-v $(pwd)/litellm/proxy/example_config_yaml/bad_schema.prisma:/app/litellm/proxy/schema.prisma \
|
|
-v $(pwd)/litellm/proxy/example_config_yaml/disable_schema_update.yaml:/app/config.yaml \
|
|
myapp:latest \
|
|
--config /app/config.yaml \
|
|
--port 4000 > docker_output.log 2>&1 || true
|
|
- run:
|
|
name: Display Docker logs
|
|
command: cat docker_output.log
|
|
- run:
|
|
name: Check for expected error
|
|
command: |
|
|
if grep -q "prisma schema out of sync with db. Consider running these sql_commands to sync the two" docker_output.log; then
|
|
echo "Expected error found. Test passed."
|
|
else
|
|
echo "Expected error not found. Test failed."
|
|
cat docker_output.log
|
|
exit 1
|
|
fi
|
|
|
|
build_and_test:
|
|
machine:
|
|
image: ubuntu-2204:2023.10.1
|
|
resource_class: xlarge
|
|
working_directory: ~/project
|
|
steps:
|
|
- checkout
|
|
- run:
|
|
name: Install Docker CLI (In case it's not already installed)
|
|
command: |
|
|
sudo apt-get update
|
|
sudo apt-get install -y docker-ce docker-ce-cli containerd.io
|
|
- run:
|
|
name: Install Python 3.9
|
|
command: |
|
|
curl https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh --output miniconda.sh
|
|
bash miniconda.sh -b -p $HOME/miniconda
|
|
export PATH="$HOME/miniconda/bin:$PATH"
|
|
conda init bash
|
|
source ~/.bashrc
|
|
conda create -n myenv python=3.9 -y
|
|
conda activate myenv
|
|
python --version
|
|
- run:
|
|
name: Install Dependencies
|
|
command: |
|
|
pip install "pytest==7.3.1"
|
|
pip install "pytest-asyncio==0.21.1"
|
|
pip install aiohttp
|
|
python -m pip install --upgrade pip
|
|
python -m pip install -r .circleci/requirements.txt
|
|
pip install "pytest==7.3.1"
|
|
pip install "pytest-retry==1.6.3"
|
|
pip install "pytest-mock==3.12.0"
|
|
pip install "pytest-asyncio==0.21.1"
|
|
pip install mypy
|
|
pip install "google-generativeai==0.3.2"
|
|
pip install "google-cloud-aiplatform==1.43.0"
|
|
pip install pyarrow
|
|
pip install "boto3==1.34.34"
|
|
pip install "aioboto3==12.3.0"
|
|
pip install langchain
|
|
pip install "langfuse>=2.0.0"
|
|
pip install "logfire==0.29.0"
|
|
pip install numpydoc
|
|
pip install prisma
|
|
pip install fastapi
|
|
pip install jsonschema
|
|
pip install "httpx==0.24.1"
|
|
pip install "gunicorn==21.2.0"
|
|
pip install "anyio==3.7.1"
|
|
pip install "aiodynamo==23.10.1"
|
|
pip install "asyncio==3.4.3"
|
|
pip install "PyGithub==1.59.1"
|
|
pip install "openai==1.51.0"
|
|
# Run pytest and generate JUnit XML report
|
|
- run:
|
|
name: Build Docker image
|
|
command: docker build -t my-app:latest -f ./docker/Dockerfile.database .
|
|
- run:
|
|
name: Run Docker container
|
|
command: |
|
|
docker run -d \
|
|
-p 4000:4000 \
|
|
-e DATABASE_URL=$PROXY_DATABASE_URL \
|
|
-e AZURE_API_KEY=$AZURE_API_KEY \
|
|
-e REDIS_HOST=$REDIS_HOST \
|
|
-e REDIS_PASSWORD=$REDIS_PASSWORD \
|
|
-e REDIS_PORT=$REDIS_PORT \
|
|
-e AZURE_FRANCE_API_KEY=$AZURE_FRANCE_API_KEY \
|
|
-e AZURE_EUROPE_API_KEY=$AZURE_EUROPE_API_KEY \
|
|
-e MISTRAL_API_KEY=$MISTRAL_API_KEY \
|
|
-e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID \
|
|
-e GROQ_API_KEY=$GROQ_API_KEY \
|
|
-e ANTHROPIC_API_KEY=$ANTHROPIC_API_KEY \
|
|
-e COHERE_API_KEY=$COHERE_API_KEY \
|
|
-e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY \
|
|
-e AWS_REGION_NAME=$AWS_REGION_NAME \
|
|
-e AUTO_INFER_REGION=True \
|
|
-e OPENAI_API_KEY=$OPENAI_API_KEY \
|
|
-e LITELLM_LICENSE=$LITELLM_LICENSE \
|
|
-e LANGFUSE_PROJECT1_PUBLIC=$LANGFUSE_PROJECT1_PUBLIC \
|
|
-e LANGFUSE_PROJECT2_PUBLIC=$LANGFUSE_PROJECT2_PUBLIC \
|
|
-e LANGFUSE_PROJECT1_SECRET=$LANGFUSE_PROJECT1_SECRET \
|
|
-e LANGFUSE_PROJECT2_SECRET=$LANGFUSE_PROJECT2_SECRET \
|
|
--name my-app \
|
|
-v $(pwd)/proxy_server_config.yaml:/app/config.yaml \
|
|
my-app:latest \
|
|
--config /app/config.yaml \
|
|
--port 4000 \
|
|
--detailed_debug \
|
|
- run:
|
|
name: Install curl and dockerize
|
|
command: |
|
|
sudo apt-get update
|
|
sudo apt-get install -y curl
|
|
sudo wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz
|
|
sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz
|
|
sudo rm dockerize-linux-amd64-v0.6.1.tar.gz
|
|
- run:
|
|
name: Start outputting logs
|
|
command: docker logs -f my-app
|
|
background: true
|
|
- run:
|
|
name: Wait for app to be ready
|
|
command: dockerize -wait http://localhost:4000 -timeout 5m
|
|
- run:
|
|
name: Run tests
|
|
command: |
|
|
pwd
|
|
ls
|
|
python -m pytest -s -vv tests/*.py -x --junitxml=test-results/junit.xml --durations=5 --ignore=tests/otel_tests --ignore=tests/pass_through_tests --ignore=tests/proxy_admin_ui_tests --ignore=tests/load_tests --ignore=tests/llm_translation
|
|
no_output_timeout: 120m
|
|
|
|
# Store test results
|
|
- store_test_results:
|
|
path: test-results
|
|
proxy_logging_guardrails_model_info_tests:
|
|
machine:
|
|
image: ubuntu-2204:2023.10.1
|
|
resource_class: xlarge
|
|
working_directory: ~/project
|
|
steps:
|
|
- checkout
|
|
- run:
|
|
name: Install Docker CLI (In case it's not already installed)
|
|
command: |
|
|
sudo apt-get update
|
|
sudo apt-get install -y docker-ce docker-ce-cli containerd.io
|
|
- run:
|
|
name: Install Python 3.9
|
|
command: |
|
|
curl https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh --output miniconda.sh
|
|
bash miniconda.sh -b -p $HOME/miniconda
|
|
export PATH="$HOME/miniconda/bin:$PATH"
|
|
conda init bash
|
|
source ~/.bashrc
|
|
conda create -n myenv python=3.9 -y
|
|
conda activate myenv
|
|
python --version
|
|
- run:
|
|
name: Install Dependencies
|
|
command: |
|
|
pip install "pytest==7.3.1"
|
|
pip install "pytest-asyncio==0.21.1"
|
|
pip install aiohttp
|
|
python -m pip install --upgrade pip
|
|
python -m pip install -r .circleci/requirements.txt
|
|
pip install "pytest==7.3.1"
|
|
pip install "pytest-retry==1.6.3"
|
|
pip install "pytest-mock==3.12.0"
|
|
pip install "pytest-asyncio==0.21.1"
|
|
pip install mypy
|
|
pip install "google-generativeai==0.3.2"
|
|
pip install "google-cloud-aiplatform==1.43.0"
|
|
pip install pyarrow
|
|
pip install "boto3==1.34.34"
|
|
pip install "aioboto3==12.3.0"
|
|
pip install langchain
|
|
pip install "langfuse>=2.0.0"
|
|
pip install "logfire==0.29.0"
|
|
pip install numpydoc
|
|
pip install prisma
|
|
pip install fastapi
|
|
pip install jsonschema
|
|
pip install "httpx==0.24.1"
|
|
pip install "gunicorn==21.2.0"
|
|
pip install "anyio==3.7.1"
|
|
pip install "aiodynamo==23.10.1"
|
|
pip install "asyncio==3.4.3"
|
|
pip install "PyGithub==1.59.1"
|
|
pip install "openai==1.51.0"
|
|
- run:
|
|
name: Build Docker image
|
|
command: docker build -t my-app:latest -f ./docker/Dockerfile.database .
|
|
- run:
|
|
name: Run Docker container
|
|
# intentionally give bad redis credentials here
|
|
# the OTEL test - should get this as a trace
|
|
command: |
|
|
docker run -d \
|
|
-p 4000:4000 \
|
|
-e DATABASE_URL=$PROXY_DATABASE_URL \
|
|
-e REDIS_HOST=$REDIS_HOST \
|
|
-e REDIS_PASSWORD=$REDIS_PASSWORD \
|
|
-e REDIS_PORT=$REDIS_PORT \
|
|
-e LITELLM_MASTER_KEY="sk-1234" \
|
|
-e OPENAI_API_KEY=$OPENAI_API_KEY \
|
|
-e LITELLM_LICENSE=$LITELLM_LICENSE \
|
|
-e OTEL_EXPORTER="in_memory" \
|
|
-e APORIA_API_BASE_2=$APORIA_API_BASE_2 \
|
|
-e APORIA_API_KEY_2=$APORIA_API_KEY_2 \
|
|
-e APORIA_API_BASE_1=$APORIA_API_BASE_1 \
|
|
-e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID \
|
|
-e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY \
|
|
-e AWS_REGION_NAME=$AWS_REGION_NAME \
|
|
-e APORIA_API_KEY_1=$APORIA_API_KEY_1 \
|
|
-e COHERE_API_KEY=$COHERE_API_KEY \
|
|
--name my-app \
|
|
-v $(pwd)/litellm/proxy/example_config_yaml/otel_test_config.yaml:/app/config.yaml \
|
|
-v $(pwd)/litellm/proxy/example_config_yaml/custom_guardrail.py:/app/custom_guardrail.py \
|
|
my-app:latest \
|
|
--config /app/config.yaml \
|
|
--port 4000 \
|
|
--detailed_debug \
|
|
- run:
|
|
name: Install curl and dockerize
|
|
command: |
|
|
sudo apt-get update
|
|
sudo apt-get install -y curl
|
|
sudo wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz
|
|
sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz
|
|
sudo rm dockerize-linux-amd64-v0.6.1.tar.gz
|
|
- run:
|
|
name: Start outputting logs
|
|
command: docker logs -f my-app
|
|
background: true
|
|
- run:
|
|
name: Wait for app to be ready
|
|
command: dockerize -wait http://localhost:4000 -timeout 5m
|
|
- run:
|
|
name: Run tests
|
|
command: |
|
|
pwd
|
|
ls
|
|
python -m pytest -vv tests/otel_tests -x --junitxml=test-results/junit.xml --durations=5
|
|
no_output_timeout: 120m
|
|
|
|
# Store test results
|
|
- store_test_results:
|
|
path: test-results
|
|
|
|
proxy_pass_through_endpoint_tests:
|
|
machine:
|
|
image: ubuntu-2204:2023.10.1
|
|
resource_class: xlarge
|
|
working_directory: ~/project
|
|
steps:
|
|
- checkout
|
|
- run:
|
|
name: Install Docker CLI (In case it's not already installed)
|
|
command: |
|
|
sudo apt-get update
|
|
sudo apt-get install -y docker-ce docker-ce-cli containerd.io
|
|
- run:
|
|
name: Install Python 3.9
|
|
command: |
|
|
curl https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh --output miniconda.sh
|
|
bash miniconda.sh -b -p $HOME/miniconda
|
|
export PATH="$HOME/miniconda/bin:$PATH"
|
|
conda init bash
|
|
source ~/.bashrc
|
|
conda create -n myenv python=3.9 -y
|
|
conda activate myenv
|
|
python --version
|
|
- run:
|
|
name: Install Dependencies
|
|
command: |
|
|
pip install "pytest==7.3.1"
|
|
pip install "pytest-retry==1.6.3"
|
|
pip install "pytest-asyncio==0.21.1"
|
|
pip install "google-cloud-aiplatform==1.43.0"
|
|
pip install aiohttp
|
|
pip install "openai==1.51.0"
|
|
python -m pip install --upgrade pip
|
|
pip install "pydantic==2.7.1"
|
|
pip install "pytest==7.3.1"
|
|
pip install "pytest-mock==3.12.0"
|
|
pip install "pytest-asyncio==0.21.1"
|
|
pip install "boto3==1.34.34"
|
|
pip install mypy
|
|
pip install pyarrow
|
|
pip install numpydoc
|
|
pip install prisma
|
|
pip install fastapi
|
|
pip install jsonschema
|
|
pip install "httpx==0.24.1"
|
|
pip install "anyio==3.7.1"
|
|
pip install "asyncio==3.4.3"
|
|
pip install "PyGithub==1.59.1"
|
|
pip install "google-cloud-aiplatform==1.59.0"
|
|
- run:
|
|
name: Build Docker image
|
|
command: docker build -t my-app:latest -f ./docker/Dockerfile.database .
|
|
- run:
|
|
name: Run Docker container
|
|
command: |
|
|
docker run -d \
|
|
-p 4000:4000 \
|
|
-e DATABASE_URL=$PROXY_DATABASE_URL \
|
|
-e LITELLM_MASTER_KEY="sk-1234" \
|
|
-e OPENAI_API_KEY=$OPENAI_API_KEY \
|
|
-e LITELLM_LICENSE=$LITELLM_LICENSE \
|
|
--name my-app \
|
|
-v $(pwd)/litellm/proxy/example_config_yaml/pass_through_config.yaml:/app/config.yaml \
|
|
-v $(pwd)/litellm/proxy/example_config_yaml/custom_auth_basic.py:/app/custom_auth_basic.py \
|
|
my-app:latest \
|
|
--config /app/config.yaml \
|
|
--port 4000 \
|
|
--detailed_debug \
|
|
- run:
|
|
name: Install curl and dockerize
|
|
command: |
|
|
sudo apt-get update
|
|
sudo apt-get install -y curl
|
|
sudo wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz
|
|
sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz
|
|
sudo rm dockerize-linux-amd64-v0.6.1.tar.gz
|
|
- run:
|
|
name: Start outputting logs
|
|
command: docker logs -f my-app
|
|
background: true
|
|
- run:
|
|
name: Wait for app to be ready
|
|
command: dockerize -wait http://localhost:4000 -timeout 5m
|
|
- run:
|
|
name: Run tests
|
|
command: |
|
|
pwd
|
|
ls
|
|
python -m pytest -vv tests/pass_through_tests/ -x --junitxml=test-results/junit.xml --durations=5
|
|
no_output_timeout: 120m
|
|
|
|
# Store test results
|
|
- store_test_results:
|
|
path: test-results
|
|
|
|
upload-coverage:
|
|
docker:
|
|
- image: cimg/python:3.9
|
|
steps:
|
|
- checkout
|
|
- attach_workspace:
|
|
at: .
|
|
# Check file locations
|
|
- run:
|
|
name: Check coverage file location
|
|
command: |
|
|
echo "Current directory:"
|
|
ls -la
|
|
echo "\nContents of tests/llm_translation:"
|
|
ls -la tests/llm_translation
|
|
- run:
|
|
name: Combine Coverage
|
|
command: |
|
|
python -m venv venv
|
|
. venv/bin/activate
|
|
pip install coverage
|
|
coverage combine llm_translation_coverage litellm_router_coverage local_testing_coverage litellm_assistants_api_coverage ui_endpoint_testing_coverage
|
|
coverage xml
|
|
- codecov/upload:
|
|
file: ./coverage.xml
|
|
|
|
|
|
publish_to_pypi:
|
|
docker:
|
|
- image: cimg/python:3.8
|
|
working_directory: ~/project
|
|
|
|
environment:
|
|
TWINE_USERNAME: __token__
|
|
|
|
steps:
|
|
- checkout
|
|
|
|
- run:
|
|
name: Copy model_prices_and_context_window File to model_prices_and_context_window_backup
|
|
command: |
|
|
cp model_prices_and_context_window.json litellm/model_prices_and_context_window_backup.json
|
|
|
|
- run:
|
|
name: Check if litellm dir was updated or if pyproject.toml was modified
|
|
command: |
|
|
if [ -n "$(git diff --name-only $CIRCLE_SHA1^..$CIRCLE_SHA1 | grep -E 'pyproject\.toml|litellm/')" ]; then
|
|
echo "litellm updated"
|
|
else
|
|
echo "No changes to litellm or pyproject.toml. Skipping PyPI publish."
|
|
circleci step halt
|
|
fi
|
|
|
|
- run:
|
|
name: Checkout code
|
|
command: git checkout $CIRCLE_SHA1
|
|
|
|
# Check if setup.py is modified and publish to PyPI
|
|
- run:
|
|
name: PyPI publish
|
|
command: |
|
|
echo "Install TOML package."
|
|
python -m pip install toml
|
|
VERSION=$(python -c "import toml; print(toml.load('pyproject.toml')['tool']['poetry']['version'])")
|
|
PACKAGE_NAME=$(python -c "import toml; print(toml.load('pyproject.toml')['tool']['poetry']['name'])")
|
|
if ! pip show -v $PACKAGE_NAME | grep -q "Version: ${VERSION}"; then
|
|
echo "pyproject.toml modified"
|
|
echo -e "[pypi]\nusername = $PYPI_PUBLISH_USERNAME\npassword = $PYPI_PUBLISH_PASSWORD" > ~/.pypirc
|
|
python -m pip install --upgrade pip
|
|
pip install build
|
|
pip install wheel
|
|
pip install --upgrade twine setuptools
|
|
rm -rf build dist
|
|
|
|
echo "Building package"
|
|
python -m build
|
|
|
|
echo "Twine upload to dist"
|
|
echo "Contents of dist directory:"
|
|
ls dist/
|
|
twine upload --verbose dist/*
|
|
else
|
|
echo "Version ${VERSION} of package is already published on PyPI. Skipping PyPI publish."
|
|
circleci step halt
|
|
fi
|
|
- run:
|
|
name: Trigger Github Action for new Docker Container + Trigger Stable Release Testing
|
|
command: |
|
|
echo "Install TOML package."
|
|
python3 -m pip install toml
|
|
VERSION=$(python3 -c "import toml; print(toml.load('pyproject.toml')['tool']['poetry']['version'])")
|
|
echo "LiteLLM Version ${VERSION}"
|
|
curl -X POST \
|
|
-H "Accept: application/vnd.github.v3+json" \
|
|
-H "Authorization: Bearer $GITHUB_TOKEN" \
|
|
"https://api.github.com/repos/BerriAI/litellm/actions/workflows/ghcr_deploy.yml/dispatches" \
|
|
-d "{\"ref\":\"main\", \"inputs\":{\"tag\":\"v${VERSION}\", \"commit_hash\":\"$CIRCLE_SHA1\"}}"
|
|
echo "triggering stable release server for version ${VERSION} and commit ${CIRCLE_SHA1}"
|
|
curl -X POST "https://proxyloadtester-production.up.railway.app/start/load/test?version=${VERSION}&commit_hash=${CIRCLE_SHA1}"
|
|
|
|
e2e_ui_testing:
|
|
machine:
|
|
image: ubuntu-2204:2023.10.1
|
|
resource_class: xlarge
|
|
working_directory: ~/project
|
|
steps:
|
|
- checkout
|
|
- run:
|
|
name: Install Docker CLI (In case it's not already installed)
|
|
command: |
|
|
sudo apt-get update
|
|
sudo apt-get install -y docker-ce docker-ce-cli containerd.io
|
|
- run:
|
|
name: Install Python 3.9
|
|
command: |
|
|
curl https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh --output miniconda.sh
|
|
bash miniconda.sh -b -p $HOME/miniconda
|
|
export PATH="$HOME/miniconda/bin:$PATH"
|
|
conda init bash
|
|
source ~/.bashrc
|
|
conda create -n myenv python=3.9 -y
|
|
conda activate myenv
|
|
python --version
|
|
- run:
|
|
name: Install Dependencies
|
|
command: |
|
|
npm install -D @playwright/test
|
|
pip install "pytest==7.3.1"
|
|
pip install "pytest-retry==1.6.3"
|
|
pip install "pytest-asyncio==0.21.1"
|
|
pip install aiohttp
|
|
pip install "openai==1.51.0"
|
|
python -m pip install --upgrade pip
|
|
pip install "pydantic==2.7.1"
|
|
pip install "pytest==7.3.1"
|
|
pip install "pytest-mock==3.12.0"
|
|
pip install "pytest-asyncio==0.21.1"
|
|
pip install mypy
|
|
pip install pyarrow
|
|
pip install numpydoc
|
|
pip install prisma
|
|
pip install fastapi
|
|
pip install jsonschema
|
|
pip install "httpx==0.24.1"
|
|
pip install "anyio==3.7.1"
|
|
pip install "asyncio==3.4.3"
|
|
- run:
|
|
name: Install Playwright Browsers
|
|
command: |
|
|
npx playwright install
|
|
- run:
|
|
name: Build Docker image
|
|
command: docker build -t my-app:latest -f ./docker/Dockerfile.database .
|
|
- run:
|
|
name: Run Docker container
|
|
command: |
|
|
docker run -d \
|
|
-p 4000:4000 \
|
|
-e DATABASE_URL=$PROXY_DATABASE_URL \
|
|
-e LITELLM_MASTER_KEY="sk-1234" \
|
|
-e OPENAI_API_KEY=$OPENAI_API_KEY \
|
|
-e UI_USERNAME="admin" \
|
|
-e UI_PASSWORD="gm" \
|
|
-e LITELLM_LICENSE=$LITELLM_LICENSE \
|
|
--name my-app \
|
|
-v $(pwd)/litellm/proxy/example_config_yaml/simple_config.yaml:/app/config.yaml \
|
|
my-app:latest \
|
|
--config /app/config.yaml \
|
|
--port 4000 \
|
|
--detailed_debug
|
|
- run:
|
|
name: Install curl and dockerize
|
|
command: |
|
|
sudo apt-get update
|
|
sudo apt-get install -y curl
|
|
sudo wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz
|
|
sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz
|
|
sudo rm dockerize-linux-amd64-v0.6.1.tar.gz
|
|
- run:
|
|
name: Start outputting logs
|
|
command: docker logs -f my-app
|
|
background: true
|
|
- run:
|
|
name: Wait for app to be ready
|
|
command: dockerize -wait http://localhost:4000 -timeout 5m
|
|
- run:
|
|
name: Run Playwright Tests
|
|
command: |
|
|
npx playwright test --reporter=html --output=test-results
|
|
no_output_timeout: 120m
|
|
- store_test_results:
|
|
path: test-results
|
|
|
|
workflows:
|
|
version: 2
|
|
build_and_test:
|
|
jobs:
|
|
- local_testing:
|
|
filters:
|
|
branches:
|
|
only:
|
|
- main
|
|
- /litellm_.*/
|
|
- litellm_assistants_api_testing:
|
|
filters:
|
|
branches:
|
|
only:
|
|
- main
|
|
- /litellm_.*/
|
|
- litellm_router_testing:
|
|
filters:
|
|
branches:
|
|
only:
|
|
- main
|
|
- /litellm_.*/
|
|
- check_code_and_doc_quality:
|
|
filters:
|
|
branches:
|
|
only:
|
|
- main
|
|
- /litellm_.*/
|
|
- ui_endpoint_testing:
|
|
filters:
|
|
branches:
|
|
only:
|
|
- main
|
|
- /litellm_.*/
|
|
- e2e_ui_testing:
|
|
filters:
|
|
branches:
|
|
only:
|
|
- main
|
|
- /litellm_.*/
|
|
- build_and_test:
|
|
filters:
|
|
branches:
|
|
only:
|
|
- main
|
|
- /litellm_.*/
|
|
- proxy_logging_guardrails_model_info_tests:
|
|
filters:
|
|
branches:
|
|
only:
|
|
- main
|
|
- /litellm_.*/
|
|
- proxy_pass_through_endpoint_tests:
|
|
filters:
|
|
branches:
|
|
only:
|
|
- main
|
|
- /litellm_.*/
|
|
- llm_translation_testing:
|
|
filters:
|
|
branches:
|
|
only:
|
|
- main
|
|
- /litellm_.*/
|
|
- upload-coverage:
|
|
requires:
|
|
- llm_translation_testing
|
|
- litellm_router_testing
|
|
- local_testing
|
|
- litellm_assistants_api_testing
|
|
- ui_endpoint_testing
|
|
- db_migration_disable_update_check:
|
|
filters:
|
|
branches:
|
|
only:
|
|
- main
|
|
- /litellm_.*/
|
|
- installing_litellm_on_python:
|
|
filters:
|
|
branches:
|
|
only:
|
|
- main
|
|
- /litellm_.*/
|
|
- load_testing:
|
|
filters:
|
|
branches:
|
|
only:
|
|
- main
|
|
- /litellm_.*/
|
|
- publish_to_pypi:
|
|
requires:
|
|
- local_testing
|
|
- build_and_test
|
|
- load_testing
|
|
- llm_translation_testing
|
|
- litellm_router_testing
|
|
- litellm_assistants_api_testing
|
|
- ui_endpoint_testing
|
|
- db_migration_disable_update_check
|
|
- e2e_ui_testing
|
|
- installing_litellm_on_python
|
|
- proxy_logging_guardrails_model_info_tests
|
|
- proxy_pass_through_endpoint_tests
|
|
- check_code_and_doc_quality
|
|
filters:
|
|
branches:
|
|
only:
|
|
- main
|
|
|