diff --git a/.circleci/config.yml b/.circleci/config.yml index 5f3ed20ac1..440343b45e 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -8,6 +8,11 @@ jobs: steps: - checkout + - run: + name: Show git commit hash + command: | + echo "Git commit hash: $CIRCLE_SHA1" + - run: name: Check if litellm dir was updated or if pyproject.toml was modified command: | @@ -28,17 +33,20 @@ jobs: pip install "pytest==7.3.1" pip install "pytest-asyncio==0.21.1" pip install mypy - pip install "google-generativeai>=0.3.2" - pip install "google-cloud-aiplatform>=1.38.0" - pip install "boto3>=1.28.57" - pip install "aioboto3>=12.3.0" + pip install "google-generativeai==0.3.2" + pip install "google-cloud-aiplatform==1.43.0" + pip install pyarrow + pip install "boto3==1.34.34" + pip install "aioboto3==12.3.0" pip install langchain - pip install "langfuse>=2.0.0" + pip install lunary==0.2.5 + pip install "langfuse==2.7.3" pip install numpydoc pip install traceloop-sdk==0.0.69 pip install openai pip install prisma pip install "httpx==0.24.1" + pip install fastapi pip install "gunicorn==21.2.0" pip install "anyio==3.7.1" pip install "aiodynamo==23.10.1" @@ -46,7 +54,10 @@ jobs: pip install "apscheduler==3.10.4" pip install "PyGithub==1.59.1" pip install argon2-cffi + pip install "pytest-mock==3.12.0" pip install python-multipart + pip install google-cloud-aiplatform + pip install prometheus-client==0.20.0 - save_cache: paths: - ./venv @@ -69,7 +80,7 @@ jobs: name: Linting Testing command: | cd litellm - python -m pip install types-requests types-setuptools types-redis + python -m pip install types-requests types-setuptools types-redis types-PyYAML if ! python -m mypy . --ignore-missing-imports; then echo "mypy detected errors" exit 1 @@ -119,6 +130,7 @@ jobs: build_and_test: machine: image: ubuntu-2204:2023.10.1 + resource_class: xlarge working_directory: ~/project steps: - checkout @@ -148,12 +160,14 @@ jobs: python -m pip install --upgrade pip python -m pip install -r .circleci/requirements.txt pip install "pytest==7.3.1" + pip install "pytest-mock==3.12.0" pip install "pytest-asyncio==0.21.1" pip install mypy - pip install "google-generativeai>=0.3.2" - pip install "google-cloud-aiplatform>=1.38.0" - pip install "boto3>=1.28.57" - pip install "aioboto3>=12.3.0" + pip install "google-generativeai==0.3.2" + pip install "google-cloud-aiplatform==1.43.0" + pip install pyarrow + pip install "boto3==1.34.34" + pip install "aioboto3==12.3.0" pip install langchain pip install "langfuse>=2.0.0" pip install numpydoc @@ -176,12 +190,19 @@ jobs: -p 4000:4000 \ -e DATABASE_URL=$PROXY_DOCKER_DB_URL \ -e AZURE_API_KEY=$AZURE_API_KEY \ + -e REDIS_HOST=$REDIS_HOST \ + -e REDIS_PASSWORD=$REDIS_PASSWORD \ + -e REDIS_PORT=$REDIS_PORT \ -e AZURE_FRANCE_API_KEY=$AZURE_FRANCE_API_KEY \ -e AZURE_EUROPE_API_KEY=$AZURE_EUROPE_API_KEY \ -e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID \ -e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY \ -e AWS_REGION_NAME=$AWS_REGION_NAME \ -e OPENAI_API_KEY=$OPENAI_API_KEY \ + -e LANGFUSE_PROJECT1_PUBLIC=$LANGFUSE_PROJECT1_PUBLIC \ + -e LANGFUSE_PROJECT2_PUBLIC=$LANGFUSE_PROJECT2_PUBLIC \ + -e LANGFUSE_PROJECT1_SECRET=$LANGFUSE_PROJECT1_SECRET \ + -e LANGFUSE_PROJECT2_SECRET=$LANGFUSE_PROJECT2_SECRET \ --name my-app \ -v $(pwd)/proxy_server_config.yaml:/app/config.yaml \ my-app:latest \ @@ -286,7 +307,7 @@ jobs: -H "Accept: application/vnd.github.v3+json" \ -H "Authorization: Bearer $GITHUB_TOKEN" \ "https://api.github.com/repos/BerriAI/litellm/actions/workflows/ghcr_deploy.yml/dispatches" \ - -d "{\"ref\":\"main\", \"inputs\":{\"tag\":\"v${VERSION}\"}}" + -d "{\"ref\":\"main\", \"inputs\":{\"tag\":\"v${VERSION}\", \"commit_hash\":\"$CIRCLE_SHA1\"}}" workflows: version: 2 diff --git a/.circleci/requirements.txt b/.circleci/requirements.txt index 4730fc28b1..b505536e27 100644 --- a/.circleci/requirements.txt +++ b/.circleci/requirements.txt @@ -3,12 +3,10 @@ openai python-dotenv tiktoken importlib_metadata -baseten cohere redis anthropic -boto3 orjson -pydantic -google-cloud-aiplatform +pydantic==1.10.14 +google-cloud-aiplatform==1.43.0 redisvl==0.0.7 # semantic caching \ No newline at end of file diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000000..78833aa31d --- /dev/null +++ b/.dockerignore @@ -0,0 +1,5 @@ +docs +cookbook +.circleci +.github +tests diff --git a/.github/workflows/ghcr_deploy.yml b/.github/workflows/ghcr_deploy.yml index d7cf4271c3..58cda02c31 100644 --- a/.github/workflows/ghcr_deploy.yml +++ b/.github/workflows/ghcr_deploy.yml @@ -5,15 +5,24 @@ on: inputs: tag: description: "The tag version you want to build" + release_type: + description: "The release type you want to build. Can be 'latest', 'stable', 'dev'" + type: string + default: "latest" + commit_hash: + description: "Commit hash" + required: true # Defines two custom environment variables for the workflow. Used for the Container registry domain, and a name for the Docker image that this workflow builds. env: REGISTRY: ghcr.io IMAGE_NAME: ${{ github.repository }} + CHART_NAME: litellm-helm # There is a single job in this workflow. It's configured to run on the latest available version of Ubuntu. jobs: docker-hub-deploy: + if: github.repository == 'BerriAI/litellm' runs-on: ubuntu-latest steps: - @@ -41,6 +50,13 @@ jobs: push: true file: Dockerfile.database tags: litellm/litellm-database:${{ github.event.inputs.tag || 'latest' }} + - + name: Build and push litellm-spend-logs image + uses: docker/build-push-action@v5 + with: + push: true + file: ./litellm-js/spend-logs/Dockerfile + tags: litellm/litellm-spend_logs:${{ github.event.inputs.tag || 'latest' }} build-and-push-image: runs-on: ubuntu-latest @@ -76,9 +92,9 @@ jobs: - name: Build and push Docker image uses: docker/build-push-action@4976231911ebf5f32aad765192d35f942aa48cb8 with: - context: . + context: https://github.com/BerriAI/litellm.git#${{ github.event.inputs.commit_hash}} push: true - tags: ${{ steps.meta.outputs.tags }}-${{ github.event.inputs.tag || 'latest' }}, ${{ steps.meta.outputs.tags }}-latest # if a tag is provided, use that, otherwise use the release tag, and if neither is available, use 'latest' + tags: ${{ steps.meta.outputs.tags }}-${{ github.event.inputs.tag || 'latest' }}, ${{ steps.meta.outputs.tags }}-${{ github.event.inputs.release_type }} # if a tag is provided, use that, otherwise use the release tag, and if neither is available, use 'latest' labels: ${{ steps.meta.outputs.labels }} platforms: local,linux/amd64,linux/arm64,linux/arm64/v8 @@ -103,15 +119,111 @@ jobs: uses: docker/metadata-action@9ec57ed1fcdbf14dcef7dfbe97b2010124a938b7 with: images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-database + # Configure multi platform Docker builds + - name: Set up QEMU + uses: docker/setup-qemu-action@e0e4588fad221d38ee467c0bffd91115366dc0c5 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@edfb0fe6204400c56fbfd3feba3fe9ad1adfa345 - name: Build and push Database Docker image uses: docker/build-push-action@f2a1d5e99d037542a71f64918e516c093c6f3fc4 with: - context: . + context: https://github.com/BerriAI/litellm.git#${{ github.event.inputs.commit_hash}} file: Dockerfile.database push: true - tags: ${{ steps.meta-database.outputs.tags }}-${{ github.event.inputs.tag || 'latest' }}, ${{ steps.meta-database.outputs.tags }}-latest + tags: ${{ steps.meta-database.outputs.tags }}-${{ github.event.inputs.tag || 'latest' }}, ${{ steps.meta-database.outputs.tags }}-${{ github.event.inputs.release_type }} labels: ${{ steps.meta-database.outputs.labels }} + platforms: local,linux/amd64,linux/arm64,linux/arm64/v8 + + build-and-push-image-spend-logs: + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Log in to the Container registry + uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract metadata (tags, labels) for spend-logs Dockerfile + id: meta-spend-logs + uses: docker/metadata-action@9ec57ed1fcdbf14dcef7dfbe97b2010124a938b7 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-spend_logs + # Configure multi platform Docker builds + - name: Set up QEMU + uses: docker/setup-qemu-action@e0e4588fad221d38ee467c0bffd91115366dc0c5 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@edfb0fe6204400c56fbfd3feba3fe9ad1adfa345 + + - name: Build and push Database Docker image + uses: docker/build-push-action@f2a1d5e99d037542a71f64918e516c093c6f3fc4 + with: + context: https://github.com/BerriAI/litellm.git#${{ github.event.inputs.commit_hash}} + file: ./litellm-js/spend-logs/Dockerfile + push: true + tags: ${{ steps.meta-spend-logs.outputs.tags }}-${{ github.event.inputs.tag || 'latest' }}, ${{ steps.meta-spend-logs.outputs.tags }}-${{ github.event.inputs.release_type }} + platforms: local,linux/amd64,linux/arm64,linux/arm64/v8 + + build-and-push-helm-chart: + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Log in to the Container registry + uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: lowercase github.repository_owner + run: | + echo "REPO_OWNER=`echo ${{github.repository_owner}} | tr '[:upper:]' '[:lower:]'`" >>${GITHUB_ENV} + - name: Get LiteLLM Latest Tag + id: current_app_tag + uses: WyriHaximus/github-action-get-previous-tag@v1.3.0 + + - name: Get last published chart version + id: current_version + shell: bash + run: | + CHART_LIST=$(helm show chart oci://${{ env.REGISTRY }}/${{ env.REPO_OWNER }}/${{ env.CHART_NAME }} 2>/dev/null || true) + if [ -z "${CHART_LIST}" ]; then + echo "current-version=0.1.0" | tee -a $GITHUB_OUTPUT + else + printf '%s' "${CHART_LIST}" | grep '^version:' | awk 'BEGIN{FS=":"}{print "current-version="$2}' | tr -d " " | tee -a $GITHUB_OUTPUT + fi + env: + HELM_EXPERIMENTAL_OCI: '1' + + # Automatically update the helm chart version one "patch" level + - name: Bump release version + id: bump_version + uses: christian-draeger/increment-semantic-version@1.1.0 + with: + current-version: ${{ steps.current_version.outputs.current-version || '0.1.0' }} + version-fragment: 'bug' + + - uses: ./.github/actions/helm-oci-chart-releaser + with: + name: ${{ env.CHART_NAME }} + repository: ${{ env.REPO_OWNER }} + tag: ${{ github.event.inputs.chartVersion || steps.bump_version.outputs.next-version || '0.1.0' }} + app_version: ${{ steps.current_app_tag.outputs.tag || 'latest' }} + path: deploy/charts/${{ env.CHART_NAME }} + registry: ${{ env.REGISTRY }} + registry_username: ${{ github.actor }} + registry_password: ${{ secrets.GITHUB_TOKEN }} + update_dependencies: true + release: name: "New LiteLLM Release" needs: [docker-hub-deploy, build-and-push-image, build-and-push-image-database] @@ -130,17 +242,20 @@ jobs: with: github-token: "${{ secrets.GITHUB_TOKEN }}" script: | + const commitHash = "${{ github.event.inputs.commit_hash}}"; + console.log("Commit Hash:", commitHash); // Add this line for debugging try { const response = await github.rest.repos.createRelease({ draft: false, generate_release_notes: true, + target_commitish: commitHash, name: process.env.RELEASE_TAG, owner: context.repo.owner, prerelease: false, repo: context.repo.repo, tag_name: process.env.RELEASE_TAG, }); - + core.exportVariable('RELEASE_ID', response.data.id); core.exportVariable('RELEASE_UPLOAD_URL', response.data.upload_url); } catch (error) { @@ -171,15 +286,14 @@ jobs: RELEASE_NOTES: ${{ steps.release-notes.outputs.result }} run: | curl -H "Content-Type: application/json" -X POST -d '{ - "content": "||@everyone||", + "content": "New LiteLLM release ${{ env.RELEASE_TAG }}", "username": "Release Changelog", "avatar_url": "https://cdn.discordapp.com/avatars/487431320314576937/bd64361e4ba6313d561d54e78c9e7171.png", "embeds": [ { - "title": "Changelog for ${RELEASE_TAG}", - "description": "${RELEASE_NOTES}", + "title": "Changelog for LiteLLM ${{ env.RELEASE_TAG }}", + "description": "${{ env.RELEASE_NOTES }}", "color": 2105893 } ] }' $WEBHOOK_URL - diff --git a/.github/workflows/interpret_load_test.py b/.github/workflows/interpret_load_test.py new file mode 100644 index 0000000000..9d95c768fc --- /dev/null +++ b/.github/workflows/interpret_load_test.py @@ -0,0 +1,94 @@ +import csv +import os +from github import Github + + +def interpret_results(csv_file): + with open(csv_file, newline="") as csvfile: + csvreader = csv.DictReader(csvfile) + rows = list(csvreader) + """ + in this csv reader + - Create 1 new column "Status" + - if a row has a median response time < 300 and an average response time < 300, Status = "Passed ✅" + - if a row has a median response time >= 300 or an average response time >= 300, Status = "Failed ❌" + - Order the table in this order Name, Status, Median Response Time, Average Response Time, Requests/s,Failures/s, Min Response Time, Max Response Time, all other columns + """ + + # Add a new column "Status" + for row in rows: + median_response_time = float( + row["Median Response Time"].strip().rstrip("ms") + ) + average_response_time = float( + row["Average Response Time"].strip().rstrip("s") + ) + + request_count = int(row["Request Count"]) + failure_count = int(row["Failure Count"]) + + failure_percent = round((failure_count / request_count) * 100, 2) + + # Determine status based on conditions + if ( + median_response_time < 300 + and average_response_time < 300 + and failure_percent < 5 + ): + row["Status"] = "Passed ✅" + else: + row["Status"] = "Failed ❌" + + # Construct Markdown table header + markdown_table = "| Name | Status | Median Response Time (ms) | Average Response Time (ms) | Requests/s | Failures/s | Request Count | Failure Count | Min Response Time (ms) | Max Response Time (ms) |" + markdown_table += ( + "\n| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |" + ) + + # Construct Markdown table rows + for row in rows: + markdown_table += f"\n| {row['Name']} | {row['Status']} | {row['Median Response Time']} | {row['Average Response Time']} | {row['Requests/s']} | {row['Failures/s']} | {row['Request Count']} | {row['Failure Count']} | {row['Min Response Time']} | {row['Max Response Time']} |" + print("markdown table: ", markdown_table) + return markdown_table + + +if __name__ == "__main__": + csv_file = "load_test_stats.csv" # Change this to the path of your CSV file + markdown_table = interpret_results(csv_file) + + # Update release body with interpreted results + github_token = os.getenv("GITHUB_TOKEN") + g = Github(github_token) + repo = g.get_repo( + "BerriAI/litellm" + ) # Replace with your repository's username and name + latest_release = repo.get_latest_release() + print("got latest release: ", latest_release) + print("latest release body: ", latest_release.body) + print("markdown table: ", markdown_table) + + # check if "Load Test LiteLLM Proxy Results" exists + existing_release_body = latest_release.body + if "Load Test LiteLLM Proxy Results" in latest_release.body: + # find the "Load Test LiteLLM Proxy Results" section and delete it + start_index = latest_release.body.find("Load Test LiteLLM Proxy Results") + existing_release_body = latest_release.body[:start_index] + + new_release_body = ( + existing_release_body + + "\n\n" + + "### Don't want to maintain your internal proxy? get in touch 🎉" + + "\nHosted Proxy Alpha: https://calendly.com/d/4mp-gd3-k5k/litellm-1-1-onboarding-chat" + + "\n\n" + + "## Load Test LiteLLM Proxy Results" + + "\n\n" + + markdown_table + ) + print("new release body: ", new_release_body) + try: + latest_release.update_release( + name=latest_release.tag_name, + message=new_release_body, + ) + except Exception as e: + print(e) diff --git a/.github/workflows/load_test.yml b/.github/workflows/load_test.yml index ed0c34fbdd..ddf613fa66 100644 --- a/.github/workflows/load_test.yml +++ b/.github/workflows/load_test.yml @@ -1,6 +1,11 @@ name: Test Locust Load Test -on: [push] +on: + workflow_run: + workflows: ["Build, Publish LiteLLM Docker Image. New Release"] + types: + - completed + workflow_dispatch: jobs: build: @@ -8,15 +13,32 @@ jobs: steps: - name: Checkout uses: actions/checkout@v1 + - name: Setup Python + uses: actions/setup-python@v2 + with: + python-version: '3.x' + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install PyGithub - name: Run Load Test id: locust_run uses: BerriAI/locust-github-action@master with: LOCUSTFILE: ".github/workflows/locustfile.py" - URL: "https://litellm-api.up.railway.app/" + URL: "https://litellm-database-docker-build-production.up.railway.app/" USERS: "100" RATE: "10" - RUNTIME: "60s" + RUNTIME: "300s" + - name: Process Load Test Stats + run: | + echo "Current working directory: $PWD" + ls + python ".github/workflows/interpret_load_test.py" + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + working-directory: ${{ github.workspace }} - name: Upload CSV as Asset to Latest Release uses: xresloader/upload-to-github-release@v1 env: @@ -25,4 +47,4 @@ jobs: file: "load_test_stats.csv;load_test.html" update_latest_release: true tag_name: "load-test" - overwrite: true + overwrite: true \ No newline at end of file diff --git a/.github/workflows/locustfile.py b/.github/workflows/locustfile.py index 5efdca84da..5dce0bb02f 100644 --- a/.github/workflows/locustfile.py +++ b/.github/workflows/locustfile.py @@ -1,4 +1,6 @@ -from locust import HttpUser, task, between +from locust import HttpUser, task, between, events +import json +import time class MyUser(HttpUser): @@ -8,7 +10,7 @@ class MyUser(HttpUser): def chat_completion(self): headers = { "Content-Type": "application/json", - "Authorization": f"Bearer sk-1234", + "Authorization": f"Bearer sk-S2-EZTUUDY0EmM6-Fy0Fyw", # Include any additional headers you may need for authentication, etc. } @@ -26,3 +28,15 @@ class MyUser(HttpUser): response = self.client.post("chat/completions", json=payload, headers=headers) # Print or log the response if needed + + @task(10) + def health_readiness(self): + start_time = time.time() + response = self.client.get("health/readiness") + response_time = time.time() - start_time + + @task(10) + def health_liveliness(self): + start_time = time.time() + response = self.client.get("health/liveliness") + response_time = time.time() - start_time diff --git a/.gitignore b/.gitignore index b03bc895bf..abc4ecb0ce 100644 --- a/.gitignore +++ b/.gitignore @@ -45,3 +45,10 @@ deploy/charts/litellm/charts/* deploy/charts/*.tgz litellm/proxy/vertex_key.json **/.vim/ +/node_modules +kub.yaml +loadtest_kub.yaml +litellm/proxy/_new_secret_config.yaml +litellm/proxy/_new_secret_config.yaml +litellm/proxy/_super_secret_config.yaml +litellm/proxy/_super_secret_config.yaml diff --git a/Dockerfile b/Dockerfile index 7193c76e27..c8e9956b29 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,8 +1,8 @@ # Base image for building -ARG LITELLM_BUILD_IMAGE=python:3.9 +ARG LITELLM_BUILD_IMAGE=python:3.11.8-slim # Runtime image -ARG LITELLM_RUNTIME_IMAGE=python:3.9-slim +ARG LITELLM_RUNTIME_IMAGE=python:3.11.8-slim # Builder stage FROM $LITELLM_BUILD_IMAGE as builder @@ -38,6 +38,11 @@ RUN pip wheel --no-cache-dir --wheel-dir=/wheels/ -r requirements.txt # install semantic-cache [Experimental]- we need this here and not in requirements.txt because redisvl pins to pydantic 1.0 RUN pip install redisvl==0.0.7 --no-deps +# ensure pyjwt is used, not jwt +RUN pip uninstall jwt -y +RUN pip uninstall PyJWT -y +RUN pip install PyJWT --no-cache-dir + # Build Admin UI RUN chmod +x build_admin_ui.sh && ./build_admin_ui.sh @@ -56,6 +61,8 @@ COPY --from=builder /wheels/ /wheels/ # Install the built wheel using pip; again using a wildcard if it's the only file RUN pip install *.whl /wheels/* --no-index --find-links=/wheels/ && rm -f *.whl && rm -rf /wheels +# Generate prisma client +RUN prisma generate RUN chmod +x entrypoint.sh EXPOSE 4000/tcp @@ -63,5 +70,4 @@ EXPOSE 4000/tcp ENTRYPOINT ["litellm"] # Append "--detailed_debug" to the end of CMD to view detailed debug logs -# CMD ["--port", "4000", "--config", "./proxy_server_config.yaml", "--run_gunicorn", "--detailed_debug"] -CMD ["--port", "4000", "--config", "./proxy_server_config.yaml", "--run_gunicorn", "--num_workers", "1"] \ No newline at end of file +CMD ["--port", "4000"] diff --git a/Dockerfile.database b/Dockerfile.database index 9e2d1637b0..22084bab89 100644 --- a/Dockerfile.database +++ b/Dockerfile.database @@ -1,8 +1,8 @@ # Base image for building -ARG LITELLM_BUILD_IMAGE=python:3.9 +ARG LITELLM_BUILD_IMAGE=python:3.11.8-slim # Runtime image -ARG LITELLM_RUNTIME_IMAGE=python:3.9-slim +ARG LITELLM_RUNTIME_IMAGE=python:3.11.8-slim # Builder stage FROM $LITELLM_BUILD_IMAGE as builder @@ -53,6 +53,11 @@ RUN pip install *.whl /wheels/* --no-index --find-links=/wheels/ && rm -f *.whl # install semantic-cache [Experimental]- we need this here and not in requirements.txt because redisvl pins to pydantic 1.0 RUN pip install redisvl==0.0.7 --no-deps +# ensure pyjwt is used, not jwt +RUN pip uninstall jwt -y +RUN pip uninstall PyJWT -y +RUN pip install PyJWT --no-cache-dir + # Build Admin UI RUN chmod +x build_admin_ui.sh && ./build_admin_ui.sh @@ -67,5 +72,5 @@ EXPOSE 4000/tcp ENTRYPOINT ["litellm"] # Append "--detailed_debug" to the end of CMD to view detailed debug logs -# CMD ["--port", "4000","--run_gunicorn", "--detailed_debug"] -CMD ["--port", "4000", "--run_gunicorn"] +# CMD ["--port", "4000", "--detailed_debug"] +CMD ["--port", "4000"] diff --git a/README.md b/README.md index 6bdaa9d375..38a1669357 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@
Call all LLM APIs using the OpenAI format [Bedrock, Huggingface, VertexAI, TogetherAI, Azure, OpenAI, etc.]