forked from phoenix/litellm-mirror
Merge branch 'main' into litellm_auto_create_user_fix
This commit is contained in:
Krish Dholakia
GitHub
commit
7d3244c012
60 changed files with 1237 additions and 215 deletions
|
|
@ -28,8 +28,9 @@ jobs:
|
|||
pip install "pytest==7.3.1"
|
||||
pip install "pytest-asyncio==0.21.1"
|
||||
pip install mypy
|
||||
pip install "google-generativeai>=0.3.2"
|
||||
pip install "google-cloud-aiplatform>=1.38.0"
|
||||
pip install "google-generativeai==0.3.2"
|
||||
pip install "google-cloud-aiplatform==1.43.0"
|
||||
pip install pyarrow
|
||||
pip install "boto3>=1.28.57"
|
||||
pip install "aioboto3>=12.3.0"
|
||||
pip install langchain
|
||||
|
|
@ -48,6 +49,7 @@ jobs:
|
|||
pip install argon2-cffi
|
||||
pip install "pytest-mock==3.12.0"
|
||||
pip install python-multipart
|
||||
pip install google-cloud-aiplatform
|
||||
- save_cache:
|
||||
paths:
|
||||
- ./venv
|
||||
|
|
@ -152,8 +154,9 @@ jobs:
|
|||
pip install "pytest-mock==3.12.0"
|
||||
pip install "pytest-asyncio==0.21.1"
|
||||
pip install mypy
|
||||
pip install "google-generativeai>=0.3.2"
|
||||
pip install "google-cloud-aiplatform>=1.38.0"
|
||||
pip install "google-generativeai==0.3.2"
|
||||
pip install "google-cloud-aiplatform==1.43.0"
|
||||
pip install pyarrow
|
||||
pip install "boto3>=1.28.57"
|
||||
pip install "aioboto3>=12.3.0"
|
||||
pip install langchain
|
||||
|
|
|
|||
|
|
@ -10,5 +10,5 @@ anthropic
|
|||
boto3
|
||||
orjson
|
||||
pydantic
|
||||
google-cloud-aiplatform
|
||||
google-cloud-aiplatform==1.43.0
|
||||
redisvl==0.0.7 # semantic caching
|
||||
46
.github/workflows/ghcr_deploy.yml
vendored
46
.github/workflows/ghcr_deploy.yml
vendored
|
|
@ -43,6 +43,14 @@ jobs:
|
|||
push: true
|
||||
file: Dockerfile.database
|
||||
tags: litellm/litellm-database:${{ github.event.inputs.tag || 'latest' }}
|
||||
-
|
||||
name: Build and push litellm-database image
|
||||
uses: docker/build-push-action@v5
|
||||
with:
|
||||
push: true
|
||||
context: ./litellm-js/spend-logs
|
||||
file: Dockerfile
|
||||
tags: litellm/litellm-spend_logs:${{ github.event.inputs.tag || 'latest' }}
|
||||
|
||||
build-and-push-image:
|
||||
runs-on: ubuntu-latest
|
||||
|
|
@ -120,6 +128,44 @@ jobs:
|
|||
tags: ${{ steps.meta-database.outputs.tags }}-${{ github.event.inputs.tag || 'latest' }}, ${{ steps.meta-database.outputs.tags }}-latest
|
||||
labels: ${{ steps.meta-database.outputs.labels }}
|
||||
platforms: local,linux/amd64,linux/arm64,linux/arm64/v8
|
||||
|
||||
build-and-push-image-spend-logs:
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: read
|
||||
packages: write
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Log in to the Container registry
|
||||
uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1
|
||||
with:
|
||||
registry: ${{ env.REGISTRY }}
|
||||
username: ${{ github.actor }}
|
||||
password: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Extract metadata (tags, labels) for database Dockerfile
|
||||
id: meta-spend-logs
|
||||
uses: docker/metadata-action@9ec57ed1fcdbf14dcef7dfbe97b2010124a938b7
|
||||
with:
|
||||
images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-spend_logs
|
||||
# Configure multi platform Docker builds
|
||||
- name: Set up QEMU
|
||||
uses: docker/setup-qemu-action@e0e4588fad221d38ee467c0bffd91115366dc0c5
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@edfb0fe6204400c56fbfd3feba3fe9ad1adfa345
|
||||
|
||||
- name: Build and push Database Docker image
|
||||
uses: docker/build-push-action@f2a1d5e99d037542a71f64918e516c093c6f3fc4
|
||||
with:
|
||||
context: ./litellm-js/spend-logs
|
||||
file: Dockerfile
|
||||
push: true
|
||||
tags: ${{ steps.meta-spend-logs.outputs.tags }}-${{ github.event.inputs.tag || 'latest' }}, ${{ steps.meta-spend-logs.outputs.tags }}-latest
|
||||
labels: ${{ steps.meta-spend-logs.outputs.labels }}
|
||||
platforms: local,linux/amd64,linux/arm64,linux/arm64/v8
|
||||
|
||||
build-and-push-helm-chart:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@
|
|||
|
||||
## Pre-requisites
|
||||
* `pip install -q google-generativeai`
|
||||
* Get API Key - https://aistudio.google.com/
|
||||
|
||||
# Gemini-Pro
|
||||
## Sample Usage
|
||||
|
|
|
|||
|
|
@ -1,7 +1,10 @@
|
|||
import Image from '@theme/IdealImage';
|
||||
|
||||
# Modify / Reject Incoming Requests
|
||||
|
||||
- Modify data before making llm api calls on proxy
|
||||
- Reject data before making llm api calls / before returning the response
|
||||
- Enforce 'user' param for all openai endpoint calls
|
||||
|
||||
See a complete example with our [parallel request rate limiter](https://github.com/BerriAI/litellm/blob/main/litellm/proxy/hooks/parallel_request_limiter.py)
|
||||
|
||||
|
|
@ -173,3 +176,18 @@ curl --location 'http://0.0.0.0:4000/chat/completions' \
|
|||
],
|
||||
}'
|
||||
```
|
||||
|
||||
## Advanced - Enforce 'user' param
|
||||
|
||||
Set `enforce_user_param` to true, to require all calls to the openai endpoints to have the 'user' param.
|
||||
|
||||
[**See Code**](https://github.com/BerriAI/litellm/blob/4777921a31c4c70e4d87b927cb233b6a09cd8b51/litellm/proxy/auth/auth_checks.py#L72)
|
||||
|
||||
```yaml
|
||||
general_settings:
|
||||
enforce_user_param: True
|
||||
```
|
||||
|
||||
**Result**
|
||||
|
||||
<Image img={require('../../img/end_user_enforcement.png')}/>
|
||||
|
|
@ -602,6 +602,7 @@ general_settings:
|
|||
"disable_spend_logs": "boolean", # turn off writing each transaction to the db
|
||||
"disable_reset_budget": "boolean", # turn off reset budget scheduled task
|
||||
"enable_jwt_auth": "boolean", # allow proxy admin to auth in via jwt tokens with 'litellm_proxy_admin' in claims
|
||||
"enforce_user_param": "boolean", # requires all openai endpoint requests to have a 'user' param
|
||||
"allowed_routes": "list", # list of allowed proxy API routes - a user can access. (currently JWT-Auth only)
|
||||
"key_management_system": "google_kms", # either google_kms or azure_kms
|
||||
"master_key": "string",
|
||||
|
|
|
|||
|
|
@ -40,7 +40,13 @@ Use this Docker `CMD`. This will start the proxy with 1 Uvicorn Async Worker
|
|||
CMD ["--port", "4000", "--config", "./proxy_server_config.yaml"]
|
||||
```
|
||||
|
||||
## 3. Switch off spend logging and resetting budgets
|
||||
## 3. Move spend logs to separate server
|
||||
|
||||
Writing each spend log to the db can slow down your proxy. In testing we saw a 70% improvement in median response time, by moving writing spend logs to a separate server.
|
||||
|
||||
|
||||
|
||||
## 4. Switch off resetting budgets
|
||||
|
||||
Add this to your config.yaml. (Only spend per Key, User and Team will be tracked - spend per API Call will not be written to the LiteLLM Database)
|
||||
```yaml
|
||||
|
|
@ -49,7 +55,7 @@ general_settings:
|
|||
disable_reset_budget: true
|
||||
```
|
||||
|
||||
## 4. Switch of `litellm.telemetry`
|
||||
## 5. Switch of `litellm.telemetry`
|
||||
|
||||
Switch of all telemetry tracking done by litellm
|
||||
|
||||
|
|
|
|||
|
|
@ -47,8 +47,9 @@ Your Proxy Swagger is available on the root of the Proxy: e.g.: `http://localhos
|
|||
Set the following in your .env on the Proxy
|
||||
|
||||
```shell
|
||||
UI_USERNAME=ishaan-litellm
|
||||
UI_PASSWORD=langchain
|
||||
LITELLM_MASTER_KEY="sk-1234" # this is your master key for using the proxy server
|
||||
UI_USERNAME=ishaan-litellm # username to sign in on UI
|
||||
UI_PASSWORD=langchain # password to sign in on UI
|
||||
```
|
||||
|
||||
On accessing the LiteLLM UI, you will be prompted to enter your username, password
|
||||
|
|
|
|||
BIN
docs/my-website/img/end_user_enforcement.png
Normal file
BIN
docs/my-website/img/end_user_enforcement.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 180 KiB |
26
litellm-js/spend-logs/Dockerfile
Normal file
26
litellm-js/spend-logs/Dockerfile
Normal file
|
|
@ -0,0 +1,26 @@
|
|||
# Use the specific Node.js v20.11.0 image
|
||||
FROM node:20.11.0
|
||||
|
||||
# Set the working directory inside the container
|
||||
WORKDIR /usr/src/app
|
||||
|
||||
# Copy package.json and package-lock.json to the working directory
|
||||
COPY package*.json ./
|
||||
|
||||
# Install dependencies
|
||||
RUN npm install
|
||||
|
||||
# Install Prisma globally
|
||||
RUN npm install -g prisma
|
||||
|
||||
# Copy the rest of the application code
|
||||
COPY . .
|
||||
|
||||
# Generate Prisma client
|
||||
RUN npx prisma generate
|
||||
|
||||
# Expose the port that the Node.js server will run on
|
||||
EXPOSE 3000
|
||||
|
||||
# Command to run the Node.js app with npm run dev
|
||||
CMD ["npm", "run", "dev"]
|
||||
8
litellm-js/spend-logs/README.md
Normal file
8
litellm-js/spend-logs/README.md
Normal file
|
|
@ -0,0 +1,8 @@
|
|||
```
|
||||
npm install
|
||||
npm run dev
|
||||
```
|
||||
|
||||
```
|
||||
open http://localhost:3000
|
||||
```
|
||||
508
litellm-js/spend-logs/package-lock.json
generated
Normal file
508
litellm-js/spend-logs/package-lock.json
generated
Normal file
|
|
@ -0,0 +1,508 @@
|
|||
{
|
||||
"name": "spend-logs",
|
||||
"lockfileVersion": 3,
|
||||
"requires": true,
|
||||
"packages": {
|
||||
"": {
|
||||
"dependencies": {
|
||||
"@hono/node-server": "^1.9.0",
|
||||
"hono": "^4.1.5"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/node": "^20.11.17",
|
||||
"tsx": "^4.7.1"
|
||||
}
|
||||
},
|
||||
"node_modules/@esbuild/aix-ppc64": {
|
||||
"version": "0.19.12",
|
||||
"resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.19.12.tgz",
|
||||
"integrity": "sha512-bmoCYyWdEL3wDQIVbcyzRyeKLgk2WtWLTWz1ZIAZF/EGbNOwSA6ew3PftJ1PqMiOOGu0OyFMzG53L0zqIpPeNA==",
|
||||
"cpu": [
|
||||
"ppc64"
|
||||
],
|
||||
"dev": true,
|
||||
"optional": true,
|
||||
"os": [
|
||||
"aix"
|
||||
],
|
||||
"engines": {
|
||||
"node": ">=12"
|
||||
}
|
||||
},
|
||||
"node_modules/@esbuild/android-arm": {
|
||||
"version": "0.19.12",
|
||||
"resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.19.12.tgz",
|
||||
"integrity": "sha512-qg/Lj1mu3CdQlDEEiWrlC4eaPZ1KztwGJ9B6J+/6G+/4ewxJg7gqj8eVYWvao1bXrqGiW2rsBZFSX3q2lcW05w==",
|
||||
"cpu": [
|
||||
"arm"
|
||||
],
|
||||
"dev": true,
|
||||
"optional": true,
|
||||
"os": [
|
||||
"android"
|
||||
],
|
||||
"engines": {
|
||||
"node": ">=12"
|
||||
}
|
||||
},
|
||||
"node_modules/@esbuild/android-arm64": {
|
||||
"version": "0.19.12",
|
||||
"resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.19.12.tgz",
|
||||
"integrity": "sha512-P0UVNGIienjZv3f5zq0DP3Nt2IE/3plFzuaS96vihvD0Hd6H/q4WXUGpCxD/E8YrSXfNyRPbpTq+T8ZQioSuPA==",
|
||||
"cpu": [
|
||||
"arm64"
|
||||
],
|
||||
"dev": true,
|
||||
"optional": true,
|
||||
"os": [
|
||||
"android"
|
||||
],
|
||||
"engines": {
|
||||
"node": ">=12"
|
||||
}
|
||||
},
|
||||
"node_modules/@esbuild/android-x64": {
|
||||
"version": "0.19.12",
|
||||
"resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.19.12.tgz",
|
||||
"integrity": "sha512-3k7ZoUW6Q6YqhdhIaq/WZ7HwBpnFBlW905Fa4s4qWJyiNOgT1dOqDiVAQFwBH7gBRZr17gLrlFCRzF6jFh7Kew==",
|
||||
"cpu": [
|
||||
"x64"
|
||||
],
|
||||
"dev": true,
|
||||
"optional": true,
|
||||
"os": [
|
||||
"android"
|
||||
],
|
||||
"engines": {
|
||||
"node": ">=12"
|
||||
}
|
||||
},
|
||||
"node_modules/@esbuild/darwin-arm64": {
|
||||
"version": "0.19.12",
|
||||
"resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.19.12.tgz",
|
||||
"integrity": "sha512-B6IeSgZgtEzGC42jsI+YYu9Z3HKRxp8ZT3cqhvliEHovq8HSX2YX8lNocDn79gCKJXOSaEot9MVYky7AKjCs8g==",
|
||||
"cpu": [
|
||||
"arm64"
|
||||
],
|
||||
"dev": true,
|
||||
"optional": true,
|
||||
"os": [
|
||||
"darwin"
|
||||
],
|
||||
"engines": {
|
||||
"node": ">=12"
|
||||
}
|
||||
},
|
||||
"node_modules/@esbuild/darwin-x64": {
|
||||
"version": "0.19.12",
|
||||
"resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.19.12.tgz",
|
||||
"integrity": "sha512-hKoVkKzFiToTgn+41qGhsUJXFlIjxI/jSYeZf3ugemDYZldIXIxhvwN6erJGlX4t5h417iFuheZ7l+YVn05N3A==",
|
||||
"cpu": [
|
||||
"x64"
|
||||
],
|
||||
"dev": true,
|
||||
"optional": true,
|
||||
"os": [
|
||||
"darwin"
|
||||
],
|
||||
"engines": {
|
||||
"node": ">=12"
|
||||
}
|
||||
},
|
||||
"node_modules/@esbuild/freebsd-arm64": {
|
||||
"version": "0.19.12",
|
||||
"resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.19.12.tgz",
|
||||
"integrity": "sha512-4aRvFIXmwAcDBw9AueDQ2YnGmz5L6obe5kmPT8Vd+/+x/JMVKCgdcRwH6APrbpNXsPz+K653Qg8HB/oXvXVukA==",
|
||||
"cpu": [
|
||||
"arm64"
|
||||
],
|
||||
"dev": true,
|
||||
"optional": true,
|
||||
"os": [
|
||||
"freebsd"
|
||||
],
|
||||
"engines": {
|
||||
"node": ">=12"
|
||||
}
|
||||
},
|
||||
"node_modules/@esbuild/freebsd-x64": {
|
||||
"version": "0.19.12",
|
||||
"resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.19.12.tgz",
|
||||
"integrity": "sha512-EYoXZ4d8xtBoVN7CEwWY2IN4ho76xjYXqSXMNccFSx2lgqOG/1TBPW0yPx1bJZk94qu3tX0fycJeeQsKovA8gg==",
|
||||
"cpu": [
|
||||
"x64"
|
||||
],
|
||||
"dev": true,
|
||||
"optional": true,
|
||||
"os": [
|
||||
"freebsd"
|
||||
],
|
||||
"engines": {
|
||||
"node": ">=12"
|
||||
}
|
||||
},
|
||||
"node_modules/@esbuild/linux-arm": {
|
||||
"version": "0.19.12",
|
||||
"resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.19.12.tgz",
|
||||
"integrity": "sha512-J5jPms//KhSNv+LO1S1TX1UWp1ucM6N6XuL6ITdKWElCu8wXP72l9MM0zDTzzeikVyqFE6U8YAV9/tFyj0ti+w==",
|
||||
"cpu": [
|
||||
"arm"
|
||||
],
|
||||
"dev": true,
|
||||
"optional": true,
|
||||
"os": [
|
||||
"linux"
|
||||
],
|
||||
"engines": {
|
||||
"node": ">=12"
|
||||
}
|
||||
},
|
||||
"node_modules/@esbuild/linux-arm64": {
|
||||
"version": "0.19.12",
|
||||
"resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.19.12.tgz",
|
||||
"integrity": "sha512-EoTjyYyLuVPfdPLsGVVVC8a0p1BFFvtpQDB/YLEhaXyf/5bczaGeN15QkR+O4S5LeJ92Tqotve7i1jn35qwvdA==",
|
||||
"cpu": [
|
||||
"arm64"
|
||||
],
|
||||
"dev": true,
|
||||
"optional": true,
|
||||
"os": [
|
||||
"linux"
|
||||
],
|
||||
"engines": {
|
||||
"node": ">=12"
|
||||
}
|
||||
},
|
||||
"node_modules/@esbuild/linux-ia32": {
|
||||
"version": "0.19.12",
|
||||
"resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.19.12.tgz",
|
||||
"integrity": "sha512-Thsa42rrP1+UIGaWz47uydHSBOgTUnwBwNq59khgIwktK6x60Hivfbux9iNR0eHCHzOLjLMLfUMLCypBkZXMHA==",
|
||||
"cpu": [
|
||||
"ia32"
|
||||
],
|
||||
"dev": true,
|
||||
"optional": true,
|
||||
"os": [
|
||||
"linux"
|
||||
],
|
||||
"engines": {
|
||||
"node": ">=12"
|
||||
}
|
||||
},
|
||||
"node_modules/@esbuild/linux-loong64": {
|
||||
"version": "0.19.12",
|
||||
"resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.19.12.tgz",
|
||||
"integrity": "sha512-LiXdXA0s3IqRRjm6rV6XaWATScKAXjI4R4LoDlvO7+yQqFdlr1Bax62sRwkVvRIrwXxvtYEHHI4dm50jAXkuAA==",
|
||||
"cpu": [
|
||||
"loong64"
|
||||
],
|
||||
"dev": true,
|
||||
"optional": true,
|
||||
"os": [
|
||||
"linux"
|
||||
],
|
||||
"engines": {
|
||||
"node": ">=12"
|
||||
}
|
||||
},
|
||||
"node_modules/@esbuild/linux-mips64el": {
|
||||
"version": "0.19.12",
|
||||
"resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.19.12.tgz",
|
||||
"integrity": "sha512-fEnAuj5VGTanfJ07ff0gOA6IPsvrVHLVb6Lyd1g2/ed67oU1eFzL0r9WL7ZzscD+/N6i3dWumGE1Un4f7Amf+w==",
|
||||
"cpu": [
|
||||
"mips64el"
|
||||
],
|
||||
"dev": true,
|
||||
"optional": true,
|
||||
"os": [
|
||||
"linux"
|
||||
],
|
||||
"engines": {
|
||||
"node": ">=12"
|
||||
}
|
||||
},
|
||||
"node_modules/@esbuild/linux-ppc64": {
|
||||
"version": "0.19.12",
|
||||
"resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.19.12.tgz",
|
||||
"integrity": "sha512-nYJA2/QPimDQOh1rKWedNOe3Gfc8PabU7HT3iXWtNUbRzXS9+vgB0Fjaqr//XNbd82mCxHzik2qotuI89cfixg==",
|
||||
"cpu": [
|
||||
"ppc64"
|
||||
],
|
||||
"dev": true,
|
||||
"optional": true,
|
||||
"os": [
|
||||
"linux"
|
||||
],
|
||||
"engines": {
|
||||
"node": ">=12"
|
||||
}
|
||||
},
|
||||
"node_modules/@esbuild/linux-riscv64": {
|
||||
"version": "0.19.12",
|
||||
"resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.19.12.tgz",
|
||||
"integrity": "sha512-2MueBrlPQCw5dVJJpQdUYgeqIzDQgw3QtiAHUC4RBz9FXPrskyyU3VI1hw7C0BSKB9OduwSJ79FTCqtGMWqJHg==",
|
||||
"cpu": [
|
||||
"riscv64"
|
||||
],
|
||||
"dev": true,
|
||||
"optional": true,
|
||||
"os": [
|
||||
"linux"
|
||||
],
|
||||
"engines": {
|
||||
"node": ">=12"
|
||||
}
|
||||
},
|
||||
"node_modules/@esbuild/linux-s390x": {
|
||||
"version": "0.19.12",
|
||||
"resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.19.12.tgz",
|
||||
"integrity": "sha512-+Pil1Nv3Umes4m3AZKqA2anfhJiVmNCYkPchwFJNEJN5QxmTs1uzyy4TvmDrCRNT2ApwSari7ZIgrPeUx4UZDg==",
|
||||
"cpu": [
|
||||
"s390x"
|
||||
],
|
||||
"dev": true,
|
||||
"optional": true,
|
||||
"os": [
|
||||
"linux"
|
||||
],
|
||||
"engines": {
|
||||
"node": ">=12"
|
||||
}
|
||||
},
|
||||
"node_modules/@esbuild/linux-x64": {
|
||||
"version": "0.19.12",
|
||||
"resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.19.12.tgz",
|
||||
"integrity": "sha512-B71g1QpxfwBvNrfyJdVDexenDIt1CiDN1TIXLbhOw0KhJzE78KIFGX6OJ9MrtC0oOqMWf+0xop4qEU8JrJTwCg==",
|
||||
"cpu": [
|
||||
"x64"
|
||||
],
|
||||
"dev": true,
|
||||
"optional": true,
|
||||
"os": [
|
||||
"linux"
|
||||
],
|
||||
"engines": {
|
||||
"node": ">=12"
|
||||
}
|
||||
},
|
||||
"node_modules/@esbuild/netbsd-x64": {
|
||||
"version": "0.19.12",
|
||||
"resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.19.12.tgz",
|
||||
"integrity": "sha512-3ltjQ7n1owJgFbuC61Oj++XhtzmymoCihNFgT84UAmJnxJfm4sYCiSLTXZtE00VWYpPMYc+ZQmB6xbSdVh0JWA==",
|
||||
"cpu": [
|
||||
"x64"
|
||||
],
|
||||
"dev": true,
|
||||
"optional": true,
|
||||
"os": [
|
||||
"netbsd"
|
||||
],
|
||||
"engines": {
|
||||
"node": ">=12"
|
||||
}
|
||||
},
|
||||
"node_modules/@esbuild/openbsd-x64": {
|
||||
"version": "0.19.12",
|
||||
"resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.19.12.tgz",
|
||||
"integrity": "sha512-RbrfTB9SWsr0kWmb9srfF+L933uMDdu9BIzdA7os2t0TXhCRjrQyCeOt6wVxr79CKD4c+p+YhCj31HBkYcXebw==",
|
||||
"cpu": [
|
||||
"x64"
|
||||
],
|
||||
"dev": true,
|
||||
"optional": true,
|
||||
"os": [
|
||||
"openbsd"
|
||||
],
|
||||
"engines": {
|
||||
"node": ">=12"
|
||||
}
|
||||
},
|
||||
"node_modules/@esbuild/sunos-x64": {
|
||||
"version": "0.19.12",
|
||||
"resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.19.12.tgz",
|
||||
"integrity": "sha512-HKjJwRrW8uWtCQnQOz9qcU3mUZhTUQvi56Q8DPTLLB+DawoiQdjsYq+j+D3s9I8VFtDr+F9CjgXKKC4ss89IeA==",
|
||||
"cpu": [
|
||||
"x64"
|
||||
],
|
||||
"dev": true,
|
||||
"optional": true,
|
||||
"os": [
|
||||
"sunos"
|
||||
],
|
||||
"engines": {
|
||||
"node": ">=12"
|
||||
}
|
||||
},
|
||||
"node_modules/@esbuild/win32-arm64": {
|
||||
"version": "0.19.12",
|
||||
"resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.19.12.tgz",
|
||||
"integrity": "sha512-URgtR1dJnmGvX864pn1B2YUYNzjmXkuJOIqG2HdU62MVS4EHpU2946OZoTMnRUHklGtJdJZ33QfzdjGACXhn1A==",
|
||||
"cpu": [
|
||||
"arm64"
|
||||
],
|
||||
"dev": true,
|
||||
"optional": true,
|
||||
"os": [
|
||||
"win32"
|
||||
],
|
||||
"engines": {
|
||||
"node": ">=12"
|
||||
}
|
||||
},
|
||||
"node_modules/@esbuild/win32-ia32": {
|
||||
"version": "0.19.12",
|
||||
"resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.19.12.tgz",
|
||||
"integrity": "sha512-+ZOE6pUkMOJfmxmBZElNOx72NKpIa/HFOMGzu8fqzQJ5kgf6aTGrcJaFsNiVMH4JKpMipyK+7k0n2UXN7a8YKQ==",
|
||||
"cpu": [
|
||||
"ia32"
|
||||
],
|
||||
"dev": true,
|
||||
"optional": true,
|
||||
"os": [
|
||||
"win32"
|
||||
],
|
||||
"engines": {
|
||||
"node": ">=12"
|
||||
}
|
||||
},
|
||||
"node_modules/@esbuild/win32-x64": {
|
||||
"version": "0.19.12",
|
||||
"resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.19.12.tgz",
|
||||
"integrity": "sha512-T1QyPSDCyMXaO3pzBkF96E8xMkiRYbUEZADd29SyPGabqxMViNoii+NcK7eWJAEoU6RZyEm5lVSIjTmcdoB9HA==",
|
||||
"cpu": [
|
||||
"x64"
|
||||
],
|
||||
"dev": true,
|
||||
"optional": true,
|
||||
"os": [
|
||||
"win32"
|
||||
],
|
||||
"engines": {
|
||||
"node": ">=12"
|
||||
}
|
||||
},
|
||||
"node_modules/@hono/node-server": {
|
||||
"version": "1.9.0",
|
||||
"resolved": "https://registry.npmjs.org/@hono/node-server/-/node-server-1.9.0.tgz",
|
||||
"integrity": "sha512-oJjk7WXBlENeHhWiMqSyxPIZ3Kmf5ZYxqdlcSIXyN8Rn50bNJsPl99G4POBS03Jxh56FdfRJ0SEnC8mAVIiavQ==",
|
||||
"engines": {
|
||||
"node": ">=18.14.1"
|
||||
}
|
||||
},
|
||||
"node_modules/@types/node": {
|
||||
"version": "20.11.30",
|
||||
"resolved": "https://registry.npmjs.org/@types/node/-/node-20.11.30.tgz",
|
||||
"integrity": "sha512-dHM6ZxwlmuZaRmUPfv1p+KrdD1Dci04FbdEm/9wEMouFqxYoFl5aMkt0VMAUtYRQDyYvD41WJLukhq/ha3YuTw==",
|
||||
"dev": true,
|
||||
"dependencies": {
|
||||
"undici-types": "~5.26.4"
|
||||
}
|
||||
},
|
||||
"node_modules/esbuild": {
|
||||
"version": "0.19.12",
|
||||
"resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.19.12.tgz",
|
||||
"integrity": "sha512-aARqgq8roFBj054KvQr5f1sFu0D65G+miZRCuJyJ0G13Zwx7vRar5Zhn2tkQNzIXcBrNVsv/8stehpj+GAjgbg==",
|
||||
"dev": true,
|
||||
"hasInstallScript": true,
|
||||
"bin": {
|
||||
"esbuild": "bin/esbuild"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=12"
|
||||
},
|
||||
"optionalDependencies": {
|
||||
"@esbuild/aix-ppc64": "0.19.12",
|
||||
"@esbuild/android-arm": "0.19.12",
|
||||
"@esbuild/android-arm64": "0.19.12",
|
||||
"@esbuild/android-x64": "0.19.12",
|
||||
"@esbuild/darwin-arm64": "0.19.12",
|
||||
"@esbuild/darwin-x64": "0.19.12",
|
||||
"@esbuild/freebsd-arm64": "0.19.12",
|
||||
"@esbuild/freebsd-x64": "0.19.12",
|
||||
"@esbuild/linux-arm": "0.19.12",
|
||||
"@esbuild/linux-arm64": "0.19.12",
|
||||
"@esbuild/linux-ia32": "0.19.12",
|
||||
"@esbuild/linux-loong64": "0.19.12",
|
||||
"@esbuild/linux-mips64el": "0.19.12",
|
||||
"@esbuild/linux-ppc64": "0.19.12",
|
||||
"@esbuild/linux-riscv64": "0.19.12",
|
||||
"@esbuild/linux-s390x": "0.19.12",
|
||||
"@esbuild/linux-x64": "0.19.12",
|
||||
"@esbuild/netbsd-x64": "0.19.12",
|
||||
"@esbuild/openbsd-x64": "0.19.12",
|
||||
"@esbuild/sunos-x64": "0.19.12",
|
||||
"@esbuild/win32-arm64": "0.19.12",
|
||||
"@esbuild/win32-ia32": "0.19.12",
|
||||
"@esbuild/win32-x64": "0.19.12"
|
||||
}
|
||||
},
|
||||
"node_modules/fsevents": {
|
||||
"version": "2.3.3",
|
||||
"resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz",
|
||||
"integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==",
|
||||
"dev": true,
|
||||
"hasInstallScript": true,
|
||||
"optional": true,
|
||||
"os": [
|
||||
"darwin"
|
||||
],
|
||||
"engines": {
|
||||
"node": "^8.16.0 || ^10.6.0 || >=11.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/get-tsconfig": {
|
||||
"version": "4.7.3",
|
||||
"resolved": "https://registry.npmjs.org/get-tsconfig/-/get-tsconfig-4.7.3.tgz",
|
||||
"integrity": "sha512-ZvkrzoUA0PQZM6fy6+/Hce561s+faD1rsNwhnO5FelNjyy7EMGJ3Rz1AQ8GYDWjhRs/7dBLOEJvhK8MiEJOAFg==",
|
||||
"dev": true,
|
||||
"dependencies": {
|
||||
"resolve-pkg-maps": "^1.0.0"
|
||||
},
|
||||
"funding": {
|
||||
"url": "https://github.com/privatenumber/get-tsconfig?sponsor=1"
|
||||
}
|
||||
},
|
||||
"node_modules/hono": {
|
||||
"version": "4.1.5",
|
||||
"resolved": "https://registry.npmjs.org/hono/-/hono-4.1.5.tgz",
|
||||
"integrity": "sha512-3ChJiIoeCxvkt6vnkxJagplrt1YZg3NyNob7ssVeK2PUqEINp4q1F94HzFnvY9QE8asVmbW5kkTDlyWylfg2vg==",
|
||||
"engines": {
|
||||
"node": ">=16.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/resolve-pkg-maps": {
|
||||
"version": "1.0.0",
|
||||
"resolved": "https://registry.npmjs.org/resolve-pkg-maps/-/resolve-pkg-maps-1.0.0.tgz",
|
||||
"integrity": "sha512-seS2Tj26TBVOC2NIc2rOe2y2ZO7efxITtLZcGSOnHHNOQ7CkiUBfw0Iw2ck6xkIhPwLhKNLS8BO+hEpngQlqzw==",
|
||||
"dev": true,
|
||||
"funding": {
|
||||
"url": "https://github.com/privatenumber/resolve-pkg-maps?sponsor=1"
|
||||
}
|
||||
},
|
||||
"node_modules/tsx": {
|
||||
"version": "4.7.1",
|
||||
"resolved": "https://registry.npmjs.org/tsx/-/tsx-4.7.1.tgz",
|
||||
"integrity": "sha512-8d6VuibXHtlN5E3zFkgY8u4DX7Y3Z27zvvPKVmLon/D4AjuKzarkUBTLDBgj9iTQ0hg5xM7c/mYiRVM+HETf0g==",
|
||||
"dev": true,
|
||||
"dependencies": {
|
||||
"esbuild": "~0.19.10",
|
||||
"get-tsconfig": "^4.7.2"
|
||||
},
|
||||
"bin": {
|
||||
"tsx": "dist/cli.mjs"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=18.0.0"
|
||||
},
|
||||
"optionalDependencies": {
|
||||
"fsevents": "~2.3.3"
|
||||
}
|
||||
},
|
||||
"node_modules/undici-types": {
|
||||
"version": "5.26.5",
|
||||
"resolved": "https://registry.npmjs.org/undici-types/-/undici-types-5.26.5.tgz",
|
||||
"integrity": "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==",
|
||||
"dev": true
|
||||
}
|
||||
}
|
||||
}
|
||||
13
litellm-js/spend-logs/package.json
Normal file
13
litellm-js/spend-logs/package.json
Normal file
|
|
@ -0,0 +1,13 @@
|
|||
{
|
||||
"scripts": {
|
||||
"dev": "tsx watch src/index.ts"
|
||||
},
|
||||
"dependencies": {
|
||||
"@hono/node-server": "^1.9.0",
|
||||
"hono": "^4.1.5"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/node": "^20.11.17",
|
||||
"tsx": "^4.7.1"
|
||||
}
|
||||
}
|
||||
29
litellm-js/spend-logs/schema.prisma
Normal file
29
litellm-js/spend-logs/schema.prisma
Normal file
|
|
@ -0,0 +1,29 @@
|
|||
generator client {
|
||||
provider = "prisma-client-js"
|
||||
}
|
||||
|
||||
datasource client {
|
||||
provider = "postgresql"
|
||||
url = env("DATABASE_URL")
|
||||
}
|
||||
|
||||
model LiteLLM_SpendLogs {
|
||||
request_id String @id
|
||||
call_type String
|
||||
api_key String @default("")
|
||||
spend Float @default(0.0)
|
||||
total_tokens Int @default(0)
|
||||
prompt_tokens Int @default(0)
|
||||
completion_tokens Int @default(0)
|
||||
startTime DateTime
|
||||
endTime DateTime
|
||||
model String @default("")
|
||||
api_base String @default("")
|
||||
user String @default("")
|
||||
metadata Json @default("{}")
|
||||
cache_hit String @default("")
|
||||
cache_key String @default("")
|
||||
request_tags Json @default("[]")
|
||||
team_id String?
|
||||
end_user String?
|
||||
}
|
||||
32
litellm-js/spend-logs/src/_types.ts
Normal file
32
litellm-js/spend-logs/src/_types.ts
Normal file
|
|
@ -0,0 +1,32 @@
|
|||
export type LiteLLM_IncrementSpend = {
|
||||
key_transactions: Array<LiteLLM_IncrementObject>, // [{"key": spend},..]
|
||||
user_transactions: Array<LiteLLM_IncrementObject>,
|
||||
team_transactions: Array<LiteLLM_IncrementObject>,
|
||||
spend_logs_transactions: Array<LiteLLM_SpendLogs>
|
||||
}
|
||||
|
||||
export type LiteLLM_IncrementObject = {
|
||||
key: string,
|
||||
spend: number
|
||||
}
|
||||
|
||||
export type LiteLLM_SpendLogs = {
|
||||
request_id: string; // @id means it's a unique identifier
|
||||
call_type: string;
|
||||
api_key: string; // @default("") means it defaults to an empty string if not provided
|
||||
spend: number; // Float in Prisma corresponds to number in TypeScript
|
||||
total_tokens: number; // Int in Prisma corresponds to number in TypeScript
|
||||
prompt_tokens: number;
|
||||
completion_tokens: number;
|
||||
startTime: Date; // DateTime in Prisma corresponds to Date in TypeScript
|
||||
endTime: Date;
|
||||
model: string; // @default("") means it defaults to an empty string if not provided
|
||||
api_base: string;
|
||||
user: string;
|
||||
metadata: any; // Json type in Prisma is represented by any in TypeScript; could also use a more specific type if the structure of JSON is known
|
||||
cache_hit: string;
|
||||
cache_key: string;
|
||||
request_tags: any; // Similarly, this could be an array or a more specific type depending on the expected structure
|
||||
team_id?: string | null; // ? indicates it's optional and can be undefined, but could also be null if not provided
|
||||
end_user?: string | null;
|
||||
};
|
||||
84
litellm-js/spend-logs/src/index.ts
Normal file
84
litellm-js/spend-logs/src/index.ts
Normal file
|
|
@ -0,0 +1,84 @@
|
|||
import { serve } from '@hono/node-server'
|
||||
import { Hono } from 'hono'
|
||||
import { PrismaClient } from '@prisma/client'
|
||||
import {LiteLLM_SpendLogs, LiteLLM_IncrementSpend, LiteLLM_IncrementObject} from './_types'
|
||||
|
||||
const app = new Hono()
|
||||
const prisma = new PrismaClient()
|
||||
// In-memory storage for logs
|
||||
let spend_logs: LiteLLM_SpendLogs[] = [];
|
||||
const key_logs: LiteLLM_IncrementObject[] = [];
|
||||
const user_logs: LiteLLM_IncrementObject[] = [];
|
||||
const transaction_logs: LiteLLM_IncrementObject[] = [];
|
||||
|
||||
|
||||
app.get('/', (c) => {
|
||||
return c.text('Hello Hono!')
|
||||
})
|
||||
|
||||
const MIN_LOGS = 1; // Minimum number of logs needed to initiate a flush
|
||||
const FLUSH_INTERVAL = 5000; // Time in ms to wait before trying to flush again
|
||||
const BATCH_SIZE = 100; // Preferred size of each batch to write to the database
|
||||
const MAX_LOGS_PER_INTERVAL = 1000; // Maximum number of logs to flush in a single interval
|
||||
|
||||
const flushLogsToDb = async () => {
|
||||
if (spend_logs.length >= MIN_LOGS) {
|
||||
// Limit the logs to process in this interval to MAX_LOGS_PER_INTERVAL or less
|
||||
const logsToProcess = spend_logs.slice(0, MAX_LOGS_PER_INTERVAL);
|
||||
|
||||
for (let i = 0; i < logsToProcess.length; i += BATCH_SIZE) {
|
||||
// Create subarray for current batch, ensuring it doesn't exceed the BATCH_SIZE
|
||||
const batch = logsToProcess.slice(i, i + BATCH_SIZE);
|
||||
|
||||
// Convert datetime strings to Date objects
|
||||
const batchWithDates = batch.map(entry => ({
|
||||
...entry,
|
||||
startTime: new Date(entry.startTime),
|
||||
endTime: new Date(entry.endTime),
|
||||
// Repeat for any other DateTime fields you may have
|
||||
}));
|
||||
|
||||
await prisma.liteLLM_SpendLogs.createMany({
|
||||
data: batchWithDates,
|
||||
});
|
||||
|
||||
console.log(`Flushed ${batch.length} logs to the DB.`);
|
||||
}
|
||||
|
||||
// Remove the processed logs from spend_logs
|
||||
spend_logs = spend_logs.slice(logsToProcess.length);
|
||||
|
||||
console.log(`${logsToProcess.length} logs processed. Remaining in queue: ${spend_logs.length}`);
|
||||
} else {
|
||||
// This will ensure it doesn't falsely claim "No logs to flush." when it's merely below the MIN_LOGS threshold.
|
||||
if(spend_logs.length > 0) {
|
||||
console.log(`Accumulating logs. Currently at ${spend_logs.length}, waiting for at least ${MIN_LOGS}.`);
|
||||
} else {
|
||||
console.log("No logs to flush.");
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
// Setup interval for attempting to flush the logs
|
||||
setInterval(flushLogsToDb, FLUSH_INTERVAL);
|
||||
|
||||
// Route to receive log messages
|
||||
app.post('/spend/update', async (c) => {
|
||||
const incomingLogs = await c.req.json<LiteLLM_SpendLogs[]>();
|
||||
|
||||
spend_logs.push(...incomingLogs);
|
||||
|
||||
console.log(`Received and stored ${incomingLogs.length} logs. Total logs in memory: ${spend_logs.length}`);
|
||||
|
||||
return c.json({ message: `Successfully stored ${incomingLogs.length} logs` });
|
||||
});
|
||||
|
||||
|
||||
|
||||
const port = 3000
|
||||
console.log(`Server is running on port ${port}`)
|
||||
|
||||
serve({
|
||||
fetch: app.fetch,
|
||||
port
|
||||
})
|
||||
13
litellm-js/spend-logs/tsconfig.json
Normal file
13
litellm-js/spend-logs/tsconfig.json
Normal file
|
|
@ -0,0 +1,13 @@
|
|||
{
|
||||
"compilerOptions": {
|
||||
"target": "ESNext",
|
||||
"module": "ESNext",
|
||||
"moduleResolution": "Bundler",
|
||||
"strict": true,
|
||||
"types": [
|
||||
"node"
|
||||
],
|
||||
"jsx": "react-jsx",
|
||||
"jsxImportSource": "hono/jsx",
|
||||
}
|
||||
}
|
||||
38
litellm/llms/custom_httpx/httpx_handler.py
Normal file
38
litellm/llms/custom_httpx/httpx_handler.py
Normal file
|
|
@ -0,0 +1,38 @@
|
|||
from typing import Optional
|
||||
import httpx
|
||||
|
||||
|
||||
class HTTPHandler:
|
||||
def __init__(self, concurrent_limit=1000):
|
||||
# Create a client with a connection pool
|
||||
self.client = httpx.AsyncClient(
|
||||
limits=httpx.Limits(
|
||||
max_connections=concurrent_limit,
|
||||
max_keepalive_connections=concurrent_limit,
|
||||
)
|
||||
)
|
||||
|
||||
async def close(self):
|
||||
# Close the client when you're done with it
|
||||
await self.client.aclose()
|
||||
|
||||
async def get(
|
||||
self, url: str, params: Optional[dict] = None, headers: Optional[dict] = None
|
||||
):
|
||||
response = await self.client.get(url, params=params, headers=headers)
|
||||
return response
|
||||
|
||||
async def post(
|
||||
self,
|
||||
url: str,
|
||||
data: Optional[dict] = None,
|
||||
params: Optional[dict] = None,
|
||||
headers: Optional[dict] = None,
|
||||
):
|
||||
try:
|
||||
response = await self.client.post(
|
||||
url, data=data, params=params, headers=headers
|
||||
)
|
||||
return response
|
||||
except Exception as e:
|
||||
raise e
|
||||
|
|
@ -13,6 +13,7 @@ from functools import partial
|
|||
import dotenv, traceback, random, asyncio, time, contextvars
|
||||
from copy import deepcopy
|
||||
import httpx
|
||||
|
||||
import litellm
|
||||
from ._logging import verbose_logger
|
||||
from litellm import ( # type: ignore
|
||||
|
|
|
|||
File diff suppressed because one or more lines are too long
|
|
@ -1 +1 @@
|
|||
(self.webpackChunk_N_E=self.webpackChunk_N_E||[]).push([[185],{11837:function(n,e,t){Promise.resolve().then(t.t.bind(t,99646,23)),Promise.resolve().then(t.t.bind(t,63385,23))},63385:function(){},99646:function(n){n.exports={style:{fontFamily:"'__Inter_c23dc8', '__Inter_Fallback_c23dc8'",fontStyle:"normal"},className:"__className_c23dc8"}}},function(n){n.O(0,[971,69,744],function(){return n(n.s=11837)}),_N_E=n.O()}]);
|
||||
(self.webpackChunk_N_E=self.webpackChunk_N_E||[]).push([[185],{87421:function(n,e,t){Promise.resolve().then(t.t.bind(t,99646,23)),Promise.resolve().then(t.t.bind(t,63385,23))},63385:function(){},99646:function(n){n.exports={style:{fontFamily:"'__Inter_c23dc8', '__Inter_Fallback_c23dc8'",fontStyle:"normal"},className:"__className_c23dc8"}}},function(n){n.O(0,[971,69,744],function(){return n(n.s=87421)}),_N_E=n.O()}]);
|
||||
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
|
|
@ -1 +1 @@
|
|||
(self.webpackChunk_N_E=self.webpackChunk_N_E||[]).push([[744],{70377:function(e,n,t){Promise.resolve().then(t.t.bind(t,47690,23)),Promise.resolve().then(t.t.bind(t,48955,23)),Promise.resolve().then(t.t.bind(t,5613,23)),Promise.resolve().then(t.t.bind(t,11902,23)),Promise.resolve().then(t.t.bind(t,31778,23)),Promise.resolve().then(t.t.bind(t,77831,23))}},function(e){var n=function(n){return e(e.s=n)};e.O(0,[971,69],function(){return n(35317),n(70377)}),_N_E=e.O()}]);
|
||||
(self.webpackChunk_N_E=self.webpackChunk_N_E||[]).push([[744],{32028:function(e,n,t){Promise.resolve().then(t.t.bind(t,47690,23)),Promise.resolve().then(t.t.bind(t,48955,23)),Promise.resolve().then(t.t.bind(t,5613,23)),Promise.resolve().then(t.t.bind(t,11902,23)),Promise.resolve().then(t.t.bind(t,31778,23)),Promise.resolve().then(t.t.bind(t,77831,23))}},function(e){var n=function(n){return e(e.s=n)};e.O(0,[971,69],function(){return n(35317),n(32028)}),_N_E=e.O()}]);
|
||||
|
|
@ -1 +1 @@
|
|||
<!DOCTYPE html><html id="__next_error__"><head><meta charSet="utf-8"/><meta name="viewport" content="width=device-width, initial-scale=1"/><link rel="preload" as="script" fetchPriority="low" href="/ui/_next/static/chunks/webpack-3b0d290a8fe6941d.js" crossorigin=""/><script src="/ui/_next/static/chunks/fd9d1056-a85b2c176012d8e5.js" async="" crossorigin=""></script><script src="/ui/_next/static/chunks/69-e1b183dda365ec86.js" async="" crossorigin=""></script><script src="/ui/_next/static/chunks/main-app-096338c8e1915716.js" async="" crossorigin=""></script><title>🚅 LiteLLM</title><meta name="description" content="LiteLLM Proxy Admin UI"/><link rel="icon" href="/ui/favicon.ico" type="image/x-icon" sizes="16x16"/><meta name="next-size-adjust"/><script src="/ui/_next/static/chunks/polyfills-c67a75d1b6f99dc8.js" crossorigin="" noModule=""></script></head><body><script src="/ui/_next/static/chunks/webpack-3b0d290a8fe6941d.js" crossorigin="" async=""></script><script>(self.__next_f=self.__next_f||[]).push([0]);self.__next_f.push([2,null])</script><script>self.__next_f.push([1,"1:HL[\"/ui/_next/static/media/c9a5bc6a7c948fb0-s.p.woff2\",\"font\",{\"crossOrigin\":\"\",\"type\":\"font/woff2\"}]\n2:HL[\"/ui/_next/static/css/68a21c6e6697f7ca.css\",\"style\",{\"crossOrigin\":\"\"}]\n0:\"$L3\"\n"])</script><script>self.__next_f.push([1,"4:I[47690,[],\"\"]\n6:I[77831,[],\"\"]\n7:I[19914,[\"730\",\"static/chunks/730-1411b729a1c79695.js\",\"931\",\"static/chunks/app/page-144687b251040a22.js\"],\"\"]\n8:I[5613,[],\"\"]\n9:I[31778,[],\"\"]\nb:I[48955,[],\"\"]\nc:[]\n"])</script><script>self.__next_f.push([1,"3:[[[\"$\",\"link\",\"0\",{\"rel\":\"stylesheet\",\"href\":\"/ui/_next/static/css/68a21c6e6697f7ca.css\",\"precedence\":\"next\",\"crossOrigin\":\"\"}]],[\"$\",\"$L4\",null,{\"buildId\":\"aIO8mtlEIEUTmgL8cGjve\",\"assetPrefix\":\"/ui\",\"initialCanonicalUrl\":\"/\",\"initialTree\":[\"\",{\"children\":[\"__PAGE__\",{}]},\"$undefined\",\"$undefined\",true],\"initialSeedData\":[\"\",{\"children\":[\"__PAGE__\",{},[\"$L5\",[\"$\",\"$L6\",null,{\"propsForComponent\":{\"params\":{}},\"Component\":\"$7\",\"isStaticGeneration\":true}],null]]},[null,[\"$\",\"html\",null,{\"lang\":\"en\",\"children\":[\"$\",\"body\",null,{\"className\":\"__className_c23dc8\",\"children\":[\"$\",\"$L8\",null,{\"parallelRouterKey\":\"children\",\"segmentPath\":[\"children\"],\"loading\":\"$undefined\",\"loadingStyles\":\"$undefined\",\"loadingScripts\":\"$undefined\",\"hasLoading\":false,\"error\":\"$undefined\",\"errorStyles\":\"$undefined\",\"errorScripts\":\"$undefined\",\"template\":[\"$\",\"$L9\",null,{}],\"templateStyles\":\"$undefined\",\"templateScripts\":\"$undefined\",\"notFound\":[[\"$\",\"title\",null,{\"children\":\"404: This page could not be found.\"}],[\"$\",\"div\",null,{\"style\":{\"fontFamily\":\"system-ui,\\\"Segoe UI\\\",Roboto,Helvetica,Arial,sans-serif,\\\"Apple Color Emoji\\\",\\\"Segoe UI Emoji\\\"\",\"height\":\"100vh\",\"textAlign\":\"center\",\"display\":\"flex\",\"flexDirection\":\"column\",\"alignItems\":\"center\",\"justifyContent\":\"center\"},\"children\":[\"$\",\"div\",null,{\"children\":[[\"$\",\"style\",null,{\"dangerouslySetInnerHTML\":{\"__html\":\"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}\"}}],[\"$\",\"h1\",null,{\"className\":\"next-error-h1\",\"style\":{\"display\":\"inline-block\",\"margin\":\"0 20px 0 0\",\"padding\":\"0 23px 0 0\",\"fontSize\":24,\"fontWeight\":500,\"verticalAlign\":\"top\",\"lineHeight\":\"49px\"},\"children\":\"404\"}],[\"$\",\"div\",null,{\"style\":{\"display\":\"inline-block\"},\"children\":[\"$\",\"h2\",null,{\"style\":{\"fontSize\":14,\"fontWeight\":400,\"lineHeight\":\"49px\",\"margin\":0},\"children\":\"This page could not be found.\"}]}]]}]}]],\"notFoundStyles\":[],\"styles\":null}]}]}],null]],\"initialHead\":[false,\"$La\"],\"globalErrorComponent\":\"$b\",\"missingSlots\":\"$Wc\"}]]\n"])</script><script>self.__next_f.push([1,"a:[[\"$\",\"meta\",\"0\",{\"name\":\"viewport\",\"content\":\"width=device-width, initial-scale=1\"}],[\"$\",\"meta\",\"1\",{\"charSet\":\"utf-8\"}],[\"$\",\"title\",\"2\",{\"children\":\"🚅 LiteLLM\"}],[\"$\",\"meta\",\"3\",{\"name\":\"description\",\"content\":\"LiteLLM Proxy Admin UI\"}],[\"$\",\"link\",\"4\",{\"rel\":\"icon\",\"href\":\"/ui/favicon.ico\",\"type\":\"image/x-icon\",\"sizes\":\"16x16\"}],[\"$\",\"meta\",\"5\",{\"name\":\"next-size-adjust\"}]]\n5:null\n"])</script><script>self.__next_f.push([1,""])</script></body></html>
|
||||
<!DOCTYPE html><html id="__next_error__"><head><meta charSet="utf-8"/><meta name="viewport" content="width=device-width, initial-scale=1"/><link rel="preload" as="script" fetchPriority="low" href="/ui/_next/static/chunks/webpack-3b0d290a8fe6941d.js" crossorigin=""/><script src="/ui/_next/static/chunks/fd9d1056-a85b2c176012d8e5.js" async="" crossorigin=""></script><script src="/ui/_next/static/chunks/69-e1b183dda365ec86.js" async="" crossorigin=""></script><script src="/ui/_next/static/chunks/main-app-9b4fb13a7db53edf.js" async="" crossorigin=""></script><title>🚅 LiteLLM</title><meta name="description" content="LiteLLM Proxy Admin UI"/><link rel="icon" href="/ui/favicon.ico" type="image/x-icon" sizes="16x16"/><meta name="next-size-adjust"/><script src="/ui/_next/static/chunks/polyfills-c67a75d1b6f99dc8.js" crossorigin="" noModule=""></script></head><body><script src="/ui/_next/static/chunks/webpack-3b0d290a8fe6941d.js" crossorigin="" async=""></script><script>(self.__next_f=self.__next_f||[]).push([0]);self.__next_f.push([2,null])</script><script>self.__next_f.push([1,"1:HL[\"/ui/_next/static/media/c9a5bc6a7c948fb0-s.p.woff2\",\"font\",{\"crossOrigin\":\"\",\"type\":\"font/woff2\"}]\n2:HL[\"/ui/_next/static/css/68a21c6e6697f7ca.css\",\"style\",{\"crossOrigin\":\"\"}]\n0:\"$L3\"\n"])</script><script>self.__next_f.push([1,"4:I[47690,[],\"\"]\n6:I[77831,[],\"\"]\n7:I[92182,[\"730\",\"static/chunks/730-1411b729a1c79695.js\",\"931\",\"static/chunks/app/page-8672e345e79b8043.js\"],\"\"]\n8:I[5613,[],\"\"]\n9:I[31778,[],\"\"]\nb:I[48955,[],\"\"]\nc:[]\n"])</script><script>self.__next_f.push([1,"3:[[[\"$\",\"link\",\"0\",{\"rel\":\"stylesheet\",\"href\":\"/ui/_next/static/css/68a21c6e6697f7ca.css\",\"precedence\":\"next\",\"crossOrigin\":\"\"}]],[\"$\",\"$L4\",null,{\"buildId\":\"YrQPd5ySk25sMOIe_YoyJ\",\"assetPrefix\":\"/ui\",\"initialCanonicalUrl\":\"/\",\"initialTree\":[\"\",{\"children\":[\"__PAGE__\",{}]},\"$undefined\",\"$undefined\",true],\"initialSeedData\":[\"\",{\"children\":[\"__PAGE__\",{},[\"$L5\",[\"$\",\"$L6\",null,{\"propsForComponent\":{\"params\":{}},\"Component\":\"$7\",\"isStaticGeneration\":true}],null]]},[null,[\"$\",\"html\",null,{\"lang\":\"en\",\"children\":[\"$\",\"body\",null,{\"className\":\"__className_c23dc8\",\"children\":[\"$\",\"$L8\",null,{\"parallelRouterKey\":\"children\",\"segmentPath\":[\"children\"],\"loading\":\"$undefined\",\"loadingStyles\":\"$undefined\",\"loadingScripts\":\"$undefined\",\"hasLoading\":false,\"error\":\"$undefined\",\"errorStyles\":\"$undefined\",\"errorScripts\":\"$undefined\",\"template\":[\"$\",\"$L9\",null,{}],\"templateStyles\":\"$undefined\",\"templateScripts\":\"$undefined\",\"notFound\":[[\"$\",\"title\",null,{\"children\":\"404: This page could not be found.\"}],[\"$\",\"div\",null,{\"style\":{\"fontFamily\":\"system-ui,\\\"Segoe UI\\\",Roboto,Helvetica,Arial,sans-serif,\\\"Apple Color Emoji\\\",\\\"Segoe UI Emoji\\\"\",\"height\":\"100vh\",\"textAlign\":\"center\",\"display\":\"flex\",\"flexDirection\":\"column\",\"alignItems\":\"center\",\"justifyContent\":\"center\"},\"children\":[\"$\",\"div\",null,{\"children\":[[\"$\",\"style\",null,{\"dangerouslySetInnerHTML\":{\"__html\":\"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}\"}}],[\"$\",\"h1\",null,{\"className\":\"next-error-h1\",\"style\":{\"display\":\"inline-block\",\"margin\":\"0 20px 0 0\",\"padding\":\"0 23px 0 0\",\"fontSize\":24,\"fontWeight\":500,\"verticalAlign\":\"top\",\"lineHeight\":\"49px\"},\"children\":\"404\"}],[\"$\",\"div\",null,{\"style\":{\"display\":\"inline-block\"},\"children\":[\"$\",\"h2\",null,{\"style\":{\"fontSize\":14,\"fontWeight\":400,\"lineHeight\":\"49px\",\"margin\":0},\"children\":\"This page could not be found.\"}]}]]}]}]],\"notFoundStyles\":[],\"styles\":null}]}]}],null]],\"initialHead\":[false,\"$La\"],\"globalErrorComponent\":\"$b\",\"missingSlots\":\"$Wc\"}]]\n"])</script><script>self.__next_f.push([1,"a:[[\"$\",\"meta\",\"0\",{\"name\":\"viewport\",\"content\":\"width=device-width, initial-scale=1\"}],[\"$\",\"meta\",\"1\",{\"charSet\":\"utf-8\"}],[\"$\",\"title\",\"2\",{\"children\":\"🚅 LiteLLM\"}],[\"$\",\"meta\",\"3\",{\"name\":\"description\",\"content\":\"LiteLLM Proxy Admin UI\"}],[\"$\",\"link\",\"4\",{\"rel\":\"icon\",\"href\":\"/ui/favicon.ico\",\"type\":\"image/x-icon\",\"sizes\":\"16x16\"}],[\"$\",\"meta\",\"5\",{\"name\":\"next-size-adjust\"}]]\n5:null\n"])</script><script>self.__next_f.push([1,""])</script></body></html>
|
||||
|
|
@ -1,7 +1,7 @@
|
|||
2:I[77831,[],""]
|
||||
3:I[19914,["730","static/chunks/730-1411b729a1c79695.js","931","static/chunks/app/page-144687b251040a22.js"],""]
|
||||
3:I[92182,["730","static/chunks/730-1411b729a1c79695.js","931","static/chunks/app/page-8672e345e79b8043.js"],""]
|
||||
4:I[5613,[],""]
|
||||
5:I[31778,[],""]
|
||||
0:["aIO8mtlEIEUTmgL8cGjve",[[["",{"children":["__PAGE__",{}]},"$undefined","$undefined",true],["",{"children":["__PAGE__",{},["$L1",["$","$L2",null,{"propsForComponent":{"params":{}},"Component":"$3","isStaticGeneration":true}],null]]},[null,["$","html",null,{"lang":"en","children":["$","body",null,{"className":"__className_c23dc8","children":["$","$L4",null,{"parallelRouterKey":"children","segmentPath":["children"],"loading":"$undefined","loadingStyles":"$undefined","loadingScripts":"$undefined","hasLoading":false,"error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L5",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":"404"}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],"notFoundStyles":[],"styles":null}]}]}],null]],[[["$","link","0",{"rel":"stylesheet","href":"/ui/_next/static/css/68a21c6e6697f7ca.css","precedence":"next","crossOrigin":""}]],"$L6"]]]]
|
||||
0:["YrQPd5ySk25sMOIe_YoyJ",[[["",{"children":["__PAGE__",{}]},"$undefined","$undefined",true],["",{"children":["__PAGE__",{},["$L1",["$","$L2",null,{"propsForComponent":{"params":{}},"Component":"$3","isStaticGeneration":true}],null]]},[null,["$","html",null,{"lang":"en","children":["$","body",null,{"className":"__className_c23dc8","children":["$","$L4",null,{"parallelRouterKey":"children","segmentPath":["children"],"loading":"$undefined","loadingStyles":"$undefined","loadingScripts":"$undefined","hasLoading":false,"error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L5",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":"404"}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],"notFoundStyles":[],"styles":null}]}]}],null]],[[["$","link","0",{"rel":"stylesheet","href":"/ui/_next/static/css/68a21c6e6697f7ca.css","precedence":"next","crossOrigin":""}]],"$L6"]]]]
|
||||
6:[["$","meta","0",{"name":"viewport","content":"width=device-width, initial-scale=1"}],["$","meta","1",{"charSet":"utf-8"}],["$","title","2",{"children":"🚅 LiteLLM"}],["$","meta","3",{"name":"description","content":"LiteLLM Proxy Admin UI"}],["$","link","4",{"rel":"icon","href":"/ui/favicon.ico","type":"image/x-icon","sizes":"16x16"}],["$","meta","5",{"name":"next-size-adjust"}]]
|
||||
1:null
|
||||
|
|
|
|||
|
|
@ -14,9 +14,9 @@ model_list:
|
|||
# cache_params:
|
||||
# type: redis
|
||||
# callbacks: ["batch_redis_requests"]
|
||||
# success_callbacks: ["langfuse"]
|
||||
# # success_callbacks: ["langfuse"]
|
||||
|
||||
general_settings:
|
||||
master_key: sk-1234
|
||||
# disable_spend_logs: true
|
||||
disable_spend_logs: true
|
||||
database_url: "postgresql://neondb_owner:hz8tyUlJ5ivV@ep-cool-sunset-a5ywubeh.us-east-2.aws.neon.tech/neondb?sslmode=require"
|
||||
|
|
@ -698,6 +698,8 @@ class LiteLLM_VerificationTokenView(LiteLLM_VerificationToken):
|
|||
team_tpm_limit: Optional[int] = None
|
||||
team_rpm_limit: Optional[int] = None
|
||||
team_max_budget: Optional[float] = None
|
||||
team_models: List = []
|
||||
team_blocked: bool = False
|
||||
soft_budget: Optional[float] = None
|
||||
team_model_aliases: Optional[Dict] = None
|
||||
|
||||
|
|
|
|||
|
|
@ -15,7 +15,7 @@ from litellm.proxy._types import (
|
|||
LiteLLM_TeamTable,
|
||||
LiteLLMRoutes,
|
||||
)
|
||||
from typing import Optional, Literal
|
||||
from typing import Optional, Literal, Union
|
||||
from litellm.proxy.utils import PrismaClient
|
||||
from litellm.caching import DualCache
|
||||
|
||||
|
|
@ -26,6 +26,8 @@ def common_checks(
|
|||
request_body: dict,
|
||||
team_object: LiteLLM_TeamTable,
|
||||
end_user_object: Optional[LiteLLM_EndUserTable],
|
||||
general_settings: dict,
|
||||
route: str,
|
||||
) -> bool:
|
||||
"""
|
||||
Common checks across jwt + key-based auth.
|
||||
|
|
@ -34,6 +36,7 @@ def common_checks(
|
|||
2. If team can call model
|
||||
3. If team is in budget
|
||||
4. If end_user ('user' passed to /chat/completions, /embeddings endpoint) is in budget
|
||||
5. [OPTIONAL] If 'enforce_end_user' enabled - did developer pass in 'user' param for openai endpoints
|
||||
"""
|
||||
_model = request_body.get("model", None)
|
||||
if team_object.blocked == True:
|
||||
|
|
@ -65,6 +68,16 @@ def common_checks(
|
|||
raise Exception(
|
||||
f"End User={end_user_object.user_id} over budget. Spend={end_user_object.spend}, Budget={end_user_budget}"
|
||||
)
|
||||
# 5. [OPTIONAL] If 'enforce_user_param' enabled - did developer pass in 'user' param for openai endpoints
|
||||
if (
|
||||
general_settings.get("enforce_user_param", None) is not None
|
||||
and general_settings["enforce_user_param"] == True
|
||||
):
|
||||
if route in LiteLLMRoutes.openai_routes.value and "user" not in request_body:
|
||||
raise Exception(
|
||||
f"'user' param not passed in. 'enforce_user_param'={general_settings['enforce_user_param']}"
|
||||
)
|
||||
|
||||
return True
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -6,7 +6,6 @@ Currently only supports admin.
|
|||
JWT token must have 'litellm_proxy_admin' in scope.
|
||||
"""
|
||||
|
||||
import httpx
|
||||
import jwt
|
||||
import json
|
||||
import os
|
||||
|
|
@ -14,42 +13,10 @@ from litellm.caching import DualCache
|
|||
from litellm._logging import verbose_proxy_logger
|
||||
from litellm.proxy._types import LiteLLM_JWTAuth, LiteLLM_UserTable
|
||||
from litellm.proxy.utils import PrismaClient
|
||||
from litellm.llms.custom_httpx.httpx_handler import HTTPHandler
|
||||
from typing import Optional
|
||||
|
||||
|
||||
class HTTPHandler:
|
||||
def __init__(self, concurrent_limit=1000):
|
||||
# Create a client with a connection pool
|
||||
self.client = httpx.AsyncClient(
|
||||
limits=httpx.Limits(
|
||||
max_connections=concurrent_limit,
|
||||
max_keepalive_connections=concurrent_limit,
|
||||
)
|
||||
)
|
||||
|
||||
async def close(self):
|
||||
# Close the client when you're done with it
|
||||
await self.client.aclose()
|
||||
|
||||
async def get(
|
||||
self, url: str, params: Optional[dict] = None, headers: Optional[dict] = None
|
||||
):
|
||||
response = await self.client.get(url, params=params, headers=headers)
|
||||
return response
|
||||
|
||||
async def post(
|
||||
self,
|
||||
url: str,
|
||||
data: Optional[dict] = None,
|
||||
params: Optional[dict] = None,
|
||||
headers: Optional[dict] = None,
|
||||
):
|
||||
response = await self.client.post(
|
||||
url, data=data, params=params, headers=headers
|
||||
)
|
||||
return response
|
||||
|
||||
|
||||
class JWTHandler:
|
||||
"""
|
||||
- treat the sub id passed in as the user id
|
||||
|
|
|
|||
|
|
@ -21,8 +21,6 @@ telemetry = None
|
|||
def append_query_params(url, params):
|
||||
from litellm._logging import verbose_proxy_logger
|
||||
|
||||
from litellm._logging import verbose_proxy_logger
|
||||
|
||||
verbose_proxy_logger.debug(f"url: {url}")
|
||||
verbose_proxy_logger.debug(f"params: {params}")
|
||||
parsed_url = urlparse.urlparse(url)
|
||||
|
|
|
|||
|
|
@ -97,7 +97,6 @@ from litellm.proxy.utils import (
|
|||
_is_projected_spend_over_limit,
|
||||
_get_projected_spend_over_limit,
|
||||
update_spend,
|
||||
monitor_spend_list,
|
||||
)
|
||||
from litellm.proxy.secret_managers.google_kms import load_google_kms
|
||||
from litellm.proxy.secret_managers.aws_secret_manager import load_aws_secret_manager
|
||||
|
|
@ -118,6 +117,7 @@ from litellm.proxy.auth.auth_checks import (
|
|||
allowed_routes_check,
|
||||
get_actual_routes,
|
||||
)
|
||||
from litellm.llms.custom_httpx.httpx_handler import HTTPHandler
|
||||
|
||||
try:
|
||||
from litellm._version import version
|
||||
|
|
@ -130,7 +130,6 @@ from fastapi import (
|
|||
HTTPException,
|
||||
status,
|
||||
Depends,
|
||||
BackgroundTasks,
|
||||
Header,
|
||||
Response,
|
||||
Form,
|
||||
|
|
@ -305,6 +304,8 @@ proxy_logging_obj = ProxyLogging(user_api_key_cache=user_api_key_cache)
|
|||
async_result = None
|
||||
celery_app_conn = None
|
||||
celery_fn = None # Redis Queue for handling requests
|
||||
### DB WRITER ###
|
||||
db_writer_client: Optional[HTTPHandler] = None
|
||||
### logger ###
|
||||
|
||||
|
||||
|
|
@ -440,6 +441,8 @@ async def user_api_key_auth(
|
|||
request_body=request_data,
|
||||
team_object=team_object,
|
||||
end_user_object=end_user_object,
|
||||
general_settings=general_settings,
|
||||
route=route,
|
||||
)
|
||||
# save user object in cache
|
||||
await user_api_key_cache.async_set_cache(
|
||||
|
|
@ -867,6 +870,23 @@ async def user_api_key_auth(
|
|||
f"ExceededTokenBudget: Current Team Spend: {valid_token.team_spend}; Max Budget for Team: {valid_token.team_max_budget}"
|
||||
)
|
||||
|
||||
# Check 8: Additional Common Checks across jwt + key auth
|
||||
_team_obj = LiteLLM_TeamTable(
|
||||
team_id=valid_token.team_id,
|
||||
max_budget=valid_token.team_max_budget,
|
||||
spend=valid_token.team_spend,
|
||||
tpm_limit=valid_token.team_tpm_limit,
|
||||
rpm_limit=valid_token.team_rpm_limit,
|
||||
blocked=valid_token.team_blocked,
|
||||
models=valid_token.team_models,
|
||||
)
|
||||
_ = common_checks(
|
||||
request_body=request_data,
|
||||
team_object=_team_obj,
|
||||
end_user_object=None,
|
||||
general_settings=general_settings,
|
||||
route=route,
|
||||
)
|
||||
# Token passed all checks
|
||||
api_key = valid_token.token
|
||||
|
||||
|
|
@ -1233,6 +1253,7 @@ async def update_database(
|
|||
user_ids.append(litellm_proxy_budget_name)
|
||||
### KEY CHANGE ###
|
||||
for _id in user_ids:
|
||||
if _id is not None:
|
||||
prisma_client.user_list_transactons[_id] = (
|
||||
response_cost
|
||||
+ prisma_client.user_list_transactons.get(_id, 0)
|
||||
|
|
@ -1364,7 +1385,16 @@ async def update_database(
|
|||
)
|
||||
|
||||
payload["spend"] = response_cost
|
||||
if prisma_client is not None:
|
||||
if (
|
||||
os.getenv("SPEND_LOGS_URL", None) is not None
|
||||
and prisma_client is not None
|
||||
):
|
||||
if isinstance(payload["startTime"], datetime):
|
||||
payload["startTime"] = payload["startTime"].isoformat()
|
||||
if isinstance(payload["endTime"], datetime):
|
||||
payload["endTime"] = payload["endTime"].isoformat()
|
||||
prisma_client.spend_log_transactons.append(payload)
|
||||
elif prisma_client is not None:
|
||||
await prisma_client.insert_data(data=payload, table_name="spend")
|
||||
except Exception as e:
|
||||
verbose_proxy_logger.debug(
|
||||
|
|
@ -2615,11 +2645,7 @@ async def async_data_generator(response, user_api_key_dict):
|
|||
verbose_proxy_logger.debug(
|
||||
f"\033[1;31mAn error occurred: {e}\n\n Debug this by setting `--debug`, e.g. `litellm --model gpt-3.5-turbo --debug`"
|
||||
)
|
||||
router_model_names = (
|
||||
[m["model_name"] for m in llm_model_list]
|
||||
if llm_model_list is not None
|
||||
else []
|
||||
)
|
||||
router_model_names = llm_router.model_names if llm_router is not None else []
|
||||
if user_debug:
|
||||
traceback.print_exc()
|
||||
|
||||
|
|
@ -2678,7 +2704,7 @@ def on_backoff(details):
|
|||
|
||||
@router.on_event("startup")
|
||||
async def startup_event():
|
||||
global prisma_client, master_key, use_background_health_checks, llm_router, llm_model_list, general_settings, proxy_budget_rescheduler_min_time, proxy_budget_rescheduler_max_time, litellm_proxy_admin_name
|
||||
global prisma_client, master_key, use_background_health_checks, llm_router, llm_model_list, general_settings, proxy_budget_rescheduler_min_time, proxy_budget_rescheduler_max_time, litellm_proxy_admin_name, db_writer_client
|
||||
import json
|
||||
|
||||
### LOAD MASTER KEY ###
|
||||
|
|
@ -2711,6 +2737,8 @@ async def startup_event():
|
|||
## COST TRACKING ##
|
||||
cost_tracking()
|
||||
|
||||
db_writer_client = HTTPHandler()
|
||||
|
||||
proxy_logging_obj._init_litellm_callbacks() # INITIALIZE LITELLM CALLBACKS ON SERVER STARTUP <- do this to catch any logging errors on startup, not when calls are being made
|
||||
|
||||
## JWT AUTH ##
|
||||
|
|
@ -2821,7 +2849,7 @@ async def startup_event():
|
|||
update_spend,
|
||||
"interval",
|
||||
seconds=batch_writing_interval,
|
||||
args=[prisma_client],
|
||||
args=[prisma_client, db_writer_client],
|
||||
)
|
||||
scheduler.start()
|
||||
|
||||
|
|
@ -2881,7 +2909,6 @@ async def completion(
|
|||
fastapi_response: Response,
|
||||
model: Optional[str] = None,
|
||||
user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth),
|
||||
background_tasks: BackgroundTasks = BackgroundTasks(),
|
||||
):
|
||||
global user_temperature, user_request_timeout, user_max_tokens, user_api_base
|
||||
try:
|
||||
|
|
@ -2943,11 +2970,7 @@ async def completion(
|
|||
start_time = time.time()
|
||||
|
||||
### ROUTE THE REQUESTs ###
|
||||
router_model_names = (
|
||||
[m["model_name"] for m in llm_model_list]
|
||||
if llm_model_list is not None
|
||||
else []
|
||||
)
|
||||
router_model_names = llm_router.model_names if llm_router is not None else []
|
||||
# skip router if user passed their key
|
||||
if "api_key" in data:
|
||||
response = await litellm.atext_completion(**data)
|
||||
|
|
@ -3047,7 +3070,6 @@ async def chat_completion(
|
|||
fastapi_response: Response,
|
||||
model: Optional[str] = None,
|
||||
user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth),
|
||||
background_tasks: BackgroundTasks = BackgroundTasks(),
|
||||
):
|
||||
global general_settings, user_debug, proxy_logging_obj, llm_model_list
|
||||
try:
|
||||
|
|
@ -3161,11 +3183,8 @@ async def chat_completion(
|
|||
start_time = time.time()
|
||||
|
||||
### ROUTE THE REQUEST ###
|
||||
router_model_names = (
|
||||
[m["model_name"] for m in llm_model_list]
|
||||
if llm_model_list is not None
|
||||
else []
|
||||
)
|
||||
# Do not change this - it should be a constant time fetch - ALWAYS
|
||||
router_model_names = llm_router.model_names if llm_router is not None else []
|
||||
# skip router if user passed their key
|
||||
if "api_key" in data:
|
||||
tasks.append(litellm.acompletion(**data))
|
||||
|
|
@ -3238,11 +3257,7 @@ async def chat_completion(
|
|||
verbose_proxy_logger.debug(
|
||||
f"\033[1;31mAn error occurred: {e}\n\n Debug this by setting `--debug`, e.g. `litellm --model gpt-3.5-turbo --debug`"
|
||||
)
|
||||
router_model_names = (
|
||||
[m["model_name"] for m in llm_model_list]
|
||||
if llm_model_list is not None
|
||||
else []
|
||||
)
|
||||
router_model_names = llm_router.model_names if llm_router is not None else []
|
||||
if user_debug:
|
||||
traceback.print_exc()
|
||||
|
||||
|
|
@ -3284,7 +3299,6 @@ async def embeddings(
|
|||
request: Request,
|
||||
model: Optional[str] = None,
|
||||
user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth),
|
||||
background_tasks: BackgroundTasks = BackgroundTasks(),
|
||||
):
|
||||
global proxy_logging_obj
|
||||
try:
|
||||
|
|
@ -3350,11 +3364,7 @@ async def embeddings(
|
|||
if data["model"] in litellm.model_alias_map:
|
||||
data["model"] = litellm.model_alias_map[data["model"]]
|
||||
|
||||
router_model_names = (
|
||||
[m["model_name"] for m in llm_model_list]
|
||||
if llm_model_list is not None
|
||||
else []
|
||||
)
|
||||
router_model_names = llm_router.model_names if llm_router is not None else []
|
||||
if (
|
||||
"input" in data
|
||||
and isinstance(data["input"], list)
|
||||
|
|
@ -3460,7 +3470,6 @@ async def embeddings(
|
|||
async def image_generation(
|
||||
request: Request,
|
||||
user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth),
|
||||
background_tasks: BackgroundTasks = BackgroundTasks(),
|
||||
):
|
||||
global proxy_logging_obj
|
||||
try:
|
||||
|
|
@ -3526,11 +3535,7 @@ async def image_generation(
|
|||
if data["model"] in litellm.model_alias_map:
|
||||
data["model"] = litellm.model_alias_map[data["model"]]
|
||||
|
||||
router_model_names = (
|
||||
[m["model_name"] for m in llm_model_list]
|
||||
if llm_model_list is not None
|
||||
else []
|
||||
)
|
||||
router_model_names = llm_router.model_names if llm_router is not None else []
|
||||
|
||||
### CALL HOOKS ### - modify incoming data / reject request before calling the model
|
||||
data = await proxy_logging_obj.pre_call_hook(
|
||||
|
|
@ -3674,11 +3679,7 @@ async def audio_transcriptions(
|
|||
**data,
|
||||
} # add the team-specific configs to the completion call
|
||||
|
||||
router_model_names = (
|
||||
[m["model_name"] for m in llm_model_list]
|
||||
if llm_model_list is not None
|
||||
else []
|
||||
)
|
||||
router_model_names = llm_router.model_names if llm_router is not None else []
|
||||
|
||||
assert (
|
||||
file.filename is not None
|
||||
|
|
@ -3843,11 +3844,7 @@ async def moderations(
|
|||
**data,
|
||||
} # add the team-specific configs to the completion call
|
||||
|
||||
router_model_names = (
|
||||
[m["model_name"] for m in llm_model_list]
|
||||
if llm_model_list is not None
|
||||
else []
|
||||
)
|
||||
router_model_names = llm_router.model_names if llm_router is not None else []
|
||||
|
||||
### CALL HOOKS ### - modify incoming data / reject request before calling the model
|
||||
data = await proxy_logging_obj.pre_call_hook(
|
||||
|
|
@ -4353,7 +4350,7 @@ async def info_key_fn(
|
|||
|
||||
@router.get(
|
||||
"/spend/keys",
|
||||
tags=["budget & spend Tracking"],
|
||||
tags=["Budget & Spend Tracking"],
|
||||
dependencies=[Depends(user_api_key_auth)],
|
||||
)
|
||||
async def spend_key_fn():
|
||||
|
|
@ -4385,7 +4382,7 @@ async def spend_key_fn():
|
|||
|
||||
@router.get(
|
||||
"/spend/users",
|
||||
tags=["budget & spend Tracking"],
|
||||
tags=["Budget & Spend Tracking"],
|
||||
dependencies=[Depends(user_api_key_auth)],
|
||||
)
|
||||
async def spend_user_fn(
|
||||
|
|
@ -4437,7 +4434,7 @@ async def spend_user_fn(
|
|||
|
||||
@router.get(
|
||||
"/spend/tags",
|
||||
tags=["budget & spend Tracking"],
|
||||
tags=["Budget & Spend Tracking"],
|
||||
dependencies=[Depends(user_api_key_auth)],
|
||||
responses={
|
||||
200: {"model": List[LiteLLM_SpendLogs]},
|
||||
|
|
@ -4510,6 +4507,77 @@ async def view_spend_tags(
|
|||
)
|
||||
|
||||
|
||||
@router.post(
|
||||
"/spend/calculate",
|
||||
tags=["Budget & Spend Tracking"],
|
||||
dependencies=[Depends(user_api_key_auth)],
|
||||
responses={
|
||||
200: {
|
||||
"cost": {
|
||||
"description": "The calculated cost",
|
||||
"example": 0.0,
|
||||
"type": "float",
|
||||
}
|
||||
}
|
||||
},
|
||||
)
|
||||
async def calculate_spend(request: Request):
|
||||
"""
|
||||
Accepts all the params of completion_cost.
|
||||
|
||||
Calculate spend **before** making call:
|
||||
|
||||
```
|
||||
curl --location 'http://localhost:4000/spend/calculate'
|
||||
--header 'Authorization: Bearer sk-1234'
|
||||
--header 'Content-Type: application/json'
|
||||
--data '{
|
||||
"model": "anthropic.claude-v2",
|
||||
"messages": [{"role": "user", "content": "Hey, how'''s it going?"}]
|
||||
}'
|
||||
```
|
||||
|
||||
Calculate spend **after** making call:
|
||||
|
||||
```
|
||||
curl --location 'http://localhost:4000/spend/calculate'
|
||||
--header 'Authorization: Bearer sk-1234'
|
||||
--header 'Content-Type: application/json'
|
||||
--data '{
|
||||
"completion_response": {
|
||||
"id": "chatcmpl-123",
|
||||
"object": "chat.completion",
|
||||
"created": 1677652288,
|
||||
"model": "gpt-3.5-turbo-0125",
|
||||
"system_fingerprint": "fp_44709d6fcb",
|
||||
"choices": [{
|
||||
"index": 0,
|
||||
"message": {
|
||||
"role": "assistant",
|
||||
"content": "Hello there, how may I assist you today?"
|
||||
},
|
||||
"logprobs": null,
|
||||
"finish_reason": "stop"
|
||||
}]
|
||||
"usage": {
|
||||
"prompt_tokens": 9,
|
||||
"completion_tokens": 12,
|
||||
"total_tokens": 21
|
||||
}
|
||||
}
|
||||
}'
|
||||
```
|
||||
"""
|
||||
from litellm import completion_cost
|
||||
|
||||
data = await request.json()
|
||||
if "completion_response" in data:
|
||||
data["completion_response"] = litellm.ModelResponse(
|
||||
**data["completion_response"]
|
||||
)
|
||||
return {"cost": completion_cost(**data)}
|
||||
|
||||
|
||||
@router.get(
|
||||
"/spend/logs",
|
||||
tags=["Budget & Spend Tracking"],
|
||||
|
|
@ -5240,6 +5308,7 @@ async def user_info(
|
|||
user_info = {"spend": spend}
|
||||
|
||||
## REMOVE HASHED TOKEN INFO before returning ##
|
||||
returned_keys = []
|
||||
for key in keys:
|
||||
try:
|
||||
key = key.model_dump() # noqa
|
||||
|
|
@ -5248,10 +5317,24 @@ async def user_info(
|
|||
key = key.dict()
|
||||
key.pop("token", None)
|
||||
|
||||
if (
|
||||
"team_id" in key
|
||||
and key["team_id"] is not None
|
||||
and key["team_id"] != "litellm-dashboard"
|
||||
):
|
||||
team_info = await prisma_client.get_data(
|
||||
team_id=key["team_id"], table_name="team"
|
||||
)
|
||||
team_alias = getattr(team_info, "team_alias", None)
|
||||
key["team_alias"] = team_alias
|
||||
else:
|
||||
key["team_alias"] = "None"
|
||||
returned_keys.append(key)
|
||||
|
||||
response_data = {
|
||||
"user_id": user_id,
|
||||
"user_info": user_info,
|
||||
"keys": keys,
|
||||
"keys": returned_keys,
|
||||
"teams": team_list,
|
||||
}
|
||||
return response_data
|
||||
|
|
@ -5639,7 +5722,7 @@ async def new_team(
|
|||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail={
|
||||
"error": f"tpm limit higher than user max. User tpm limit={user_api_key_dict.tpm_limit}"
|
||||
"error": f"tpm limit higher than user max. User tpm limit={user_api_key_dict.tpm_limit}. User role={user_api_key_dict.user_role}"
|
||||
},
|
||||
)
|
||||
|
||||
|
|
@ -5651,7 +5734,7 @@ async def new_team(
|
|||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail={
|
||||
"error": f"rpm limit higher than user max. User rpm limit={user_api_key_dict.rpm_limit}"
|
||||
"error": f"rpm limit higher than user max. User rpm limit={user_api_key_dict.rpm_limit}. User role={user_api_key_dict.user_role}"
|
||||
},
|
||||
)
|
||||
|
||||
|
|
@ -5663,7 +5746,7 @@ async def new_team(
|
|||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail={
|
||||
"error": f"max budget higher than user max. User max budget={user_api_key_dict.max_budget}"
|
||||
"error": f"max budget higher than user max. User max budget={user_api_key_dict.max_budget}. User role={user_api_key_dict.user_role}"
|
||||
},
|
||||
)
|
||||
|
||||
|
|
@ -5673,7 +5756,7 @@ async def new_team(
|
|||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail={
|
||||
"error": f"Model not in allowed user models. User allowed models={user_api_key_dict.models}"
|
||||
"error": f"Model not in allowed user models. User allowed models={user_api_key_dict.models}. User id={user_api_key_dict.user_id}"
|
||||
},
|
||||
)
|
||||
|
||||
|
|
@ -6170,7 +6253,7 @@ async def block_team(
|
|||
raise Exception("No DB Connected.")
|
||||
|
||||
record = await prisma_client.db.litellm_teamtable.update(
|
||||
where={"team_id": data.team_id}, data={"blocked": True}
|
||||
where={"team_id": data.team_id}, data={"blocked": True} # type: ignore
|
||||
)
|
||||
|
||||
return record
|
||||
|
|
@ -6192,7 +6275,7 @@ async def unblock_team(
|
|||
raise Exception("No DB Connected.")
|
||||
|
||||
record = await prisma_client.db.litellm_teamtable.update(
|
||||
where={"team_id": data.team_id}, data={"blocked": False}
|
||||
where={"team_id": data.team_id}, data={"blocked": False} # type: ignore
|
||||
)
|
||||
|
||||
return record
|
||||
|
|
@ -6795,7 +6878,6 @@ async def async_queue_request(
|
|||
request: Request,
|
||||
model: Optional[str] = None,
|
||||
user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth),
|
||||
background_tasks: BackgroundTasks = BackgroundTasks(),
|
||||
):
|
||||
global general_settings, user_debug, proxy_logging_obj
|
||||
"""
|
||||
|
|
@ -7058,6 +7140,13 @@ async def login(request: Request):
|
|||
except ImportError:
|
||||
subprocess.run(["pip", "install", "python-multipart"])
|
||||
global master_key
|
||||
if master_key is None:
|
||||
raise ProxyException(
|
||||
message="Master Key not set for Proxy. Please set Master Key to use Admin UI. Set `LITELLM_MASTER_KEY` in .env or set general_settings:master_key in config.yaml. https://docs.litellm.ai/docs/proxy/virtual_keys. If set, use `--detailed_debug` to debug issue.",
|
||||
type="auth_error",
|
||||
param="master_key",
|
||||
code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
)
|
||||
form = await request.form()
|
||||
username = str(form.get("username"))
|
||||
password = str(form.get("password"))
|
||||
|
|
@ -7997,6 +8086,8 @@ async def shutdown_event():
|
|||
|
||||
await jwt_handler.close()
|
||||
|
||||
if db_writer_client is not None:
|
||||
await db_writer_client.close()
|
||||
## RESET CUSTOM VARIABLES ##
|
||||
cleanup_router_config_variables()
|
||||
|
||||
|
|
|
|||
|
|
@ -5,7 +5,6 @@ datasource client {
|
|||
|
||||
generator client {
|
||||
provider = "prisma-client-py"
|
||||
previewFeatures = ["jsonProtocol"]
|
||||
}
|
||||
|
||||
// Budget / Rate Limits for an org
|
||||
|
|
|
|||
|
|
@ -13,6 +13,7 @@ from litellm.proxy._types import (
|
|||
Member,
|
||||
)
|
||||
from litellm.caching import DualCache
|
||||
from litellm.llms.custom_httpx.httpx_handler import HTTPHandler
|
||||
from litellm.proxy.hooks.parallel_request_limiter import (
|
||||
_PROXY_MaxParallelRequestsHandler,
|
||||
)
|
||||
|
|
@ -298,6 +299,7 @@ class ProxyLogging:
|
|||
return
|
||||
else:
|
||||
user_info = str(user_info)
|
||||
|
||||
# percent of max_budget left to spend
|
||||
if user_max_budget > 0:
|
||||
percent_left = (user_max_budget - user_current_spend) / user_max_budget
|
||||
|
|
@ -317,22 +319,35 @@ class ProxyLogging:
|
|||
)
|
||||
return
|
||||
|
||||
## PREVENTITIVE ALERTING ## - https://github.com/BerriAI/litellm/issues/2727
|
||||
# - Alert once within 28d period
|
||||
# - Cache this information
|
||||
# - Don't re-alert, if alert already sent
|
||||
_cache: DualCache = self.call_details["user_api_key_cache"]
|
||||
|
||||
# check if 5% of max budget is left
|
||||
if percent_left <= 0.05:
|
||||
message = "5% budget left for" + user_info
|
||||
result = await _cache.async_get_cache(key=message)
|
||||
if result is None:
|
||||
await self.alerting_handler(
|
||||
message=message,
|
||||
level="Medium",
|
||||
)
|
||||
await _cache.async_set_cache(key=message, value="SENT", ttl=2419200)
|
||||
|
||||
return
|
||||
|
||||
# check if 15% of max budget is left
|
||||
if percent_left <= 0.15:
|
||||
message = "15% budget left for" + user_info
|
||||
result = await _cache.async_get_cache(key=message)
|
||||
if result is None:
|
||||
await self.alerting_handler(
|
||||
message=message,
|
||||
level="Low",
|
||||
)
|
||||
await _cache.async_set_cache(key=message, value="SENT", ttl=2419200)
|
||||
return
|
||||
|
||||
return
|
||||
|
|
@ -449,16 +464,15 @@ class ProxyLogging:
|
|||
Covers:
|
||||
1. /chat/completions
|
||||
"""
|
||||
new_response = copy.deepcopy(response)
|
||||
for callback in litellm.callbacks:
|
||||
try:
|
||||
if isinstance(callback, CustomLogger):
|
||||
await callback.async_post_call_success_hook(
|
||||
user_api_key_dict=user_api_key_dict, response=new_response
|
||||
user_api_key_dict=user_api_key_dict, response=response
|
||||
)
|
||||
except Exception as e:
|
||||
raise e
|
||||
return new_response
|
||||
return response
|
||||
|
||||
async def post_call_streaming_hook(
|
||||
self,
|
||||
|
|
@ -1013,6 +1027,8 @@ class PrismaClient:
|
|||
t.max_budget AS team_max_budget,
|
||||
t.tpm_limit AS team_tpm_limit,
|
||||
t.rpm_limit AS team_rpm_limit,
|
||||
t.models AS team_models,
|
||||
t.blocked AS team_blocked,
|
||||
m.aliases as team_model_aliases
|
||||
FROM "LiteLLM_VerificationToken" AS v
|
||||
LEFT JOIN "LiteLLM_TeamTable" AS t ON v.team_id = t.team_id
|
||||
|
|
@ -1023,6 +1039,10 @@ class PrismaClient:
|
|||
response = await self.db.query_first(query=sql_query)
|
||||
|
||||
if response is not None:
|
||||
if response["team_models"] is None:
|
||||
response["team_models"] = []
|
||||
if response["team_blocked"] is None:
|
||||
response["team_blocked"] = False
|
||||
response = LiteLLM_VerificationTokenView(**response)
|
||||
# for prisma we need to cast the expires time to str
|
||||
if response.expires is not None and isinstance(
|
||||
|
|
@ -1867,7 +1887,7 @@ async def reset_budget(prisma_client: PrismaClient):
|
|||
|
||||
|
||||
async def update_spend(
|
||||
prisma_client: PrismaClient,
|
||||
prisma_client: PrismaClient, db_writer_client: Optional[HTTPHandler]
|
||||
):
|
||||
"""
|
||||
Batch write updates to db.
|
||||
|
|
@ -1995,13 +2015,30 @@ async def update_spend(
|
|||
except Exception as e:
|
||||
raise e
|
||||
|
||||
### UPDATE SPEND LOGS ###
|
||||
base_url = os.getenv("SPEND_LOGS_URL", None)
|
||||
if (
|
||||
len(prisma_client.spend_log_transactons) > 0
|
||||
and base_url is not None
|
||||
and db_writer_client is not None
|
||||
):
|
||||
if not base_url.endswith("/"):
|
||||
base_url += "/"
|
||||
response = await db_writer_client.post(
|
||||
url=base_url + "spend/update",
|
||||
data=json.dumps(prisma_client.spend_log_transactons), # type: ignore
|
||||
headers={"Content-Type": "application/json"},
|
||||
)
|
||||
if response.status_code == 200:
|
||||
prisma_client.spend_log_transactons = []
|
||||
|
||||
async def monitor_spend_list(prisma_client: PrismaClient):
|
||||
"""
|
||||
Check the length of each spend list, if it exceeds a threshold (e.g. 100 items) - write to db
|
||||
"""
|
||||
if len(prisma_client.user_list_transactons) > 10000:
|
||||
await update_spend(prisma_client=prisma_client)
|
||||
|
||||
# async def monitor_spend_list(prisma_client: PrismaClient):
|
||||
# """
|
||||
# Check the length of each spend list, if it exceeds a threshold (e.g. 100 items) - write to db
|
||||
# """
|
||||
# if len(prisma_client.user_list_transactons) > 10000:
|
||||
# await update_spend(prisma_client=prisma_client)
|
||||
|
||||
|
||||
async def _read_request_body(request):
|
||||
|
|
|
|||
|
|
@ -11,6 +11,7 @@ from litellm import token_counter
|
|||
from litellm.caching import DualCache
|
||||
from litellm.integrations.custom_logger import CustomLogger
|
||||
from litellm._logging import verbose_router_logger
|
||||
from litellm.utils import print_verbose
|
||||
|
||||
|
||||
class LowestTPMLoggingHandler(CustomLogger):
|
||||
|
|
@ -153,27 +154,21 @@ class LowestTPMLoggingHandler(CustomLogger):
|
|||
# Find lowest used model
|
||||
# ----------------------
|
||||
lowest_tpm = float("inf")
|
||||
deployment = None
|
||||
|
||||
if tpm_dict is None: # base case - none of the deployments have been used
|
||||
# Return the 1st deployment where deployment["tpm"] >= input_tokens
|
||||
# initialize a tpm dict with {model_id: 0}
|
||||
tpm_dict = {}
|
||||
for deployment in healthy_deployments:
|
||||
_deployment_tpm = (
|
||||
deployment.get("tpm", None)
|
||||
or deployment.get("litellm_params", {}).get("tpm", None)
|
||||
or deployment.get("model_info", {}).get("tpm", None)
|
||||
or float("inf")
|
||||
)
|
||||
|
||||
if _deployment_tpm >= input_tokens:
|
||||
return deployment
|
||||
return None
|
||||
|
||||
all_deployments = tpm_dict
|
||||
tpm_dict[deployment["model_info"]["id"]] = 0
|
||||
else:
|
||||
for d in healthy_deployments:
|
||||
## if healthy deployment not yet used
|
||||
if d["model_info"]["id"] not in all_deployments:
|
||||
all_deployments[d["model_info"]["id"]] = 0
|
||||
if d["model_info"]["id"] not in tpm_dict:
|
||||
tpm_dict[d["model_info"]["id"]] = 0
|
||||
|
||||
all_deployments = tpm_dict
|
||||
|
||||
deployment = None
|
||||
for item, item_tpm in all_deployments.items():
|
||||
## get the item from model list
|
||||
_deployment = None
|
||||
|
|
@ -184,24 +179,27 @@ class LowestTPMLoggingHandler(CustomLogger):
|
|||
if _deployment is None:
|
||||
continue # skip to next one
|
||||
|
||||
_deployment_tpm = (
|
||||
_deployment.get("tpm", None)
|
||||
or _deployment.get("litellm_params", {}).get("tpm", None)
|
||||
or _deployment.get("model_info", {}).get("tpm", None)
|
||||
or float("inf")
|
||||
)
|
||||
_deployment_tpm = None
|
||||
if _deployment_tpm is None:
|
||||
_deployment_tpm = _deployment.get("tpm")
|
||||
if _deployment_tpm is None:
|
||||
_deployment_tpm = _deployment.get("litellm_params", {}).get("tpm")
|
||||
if _deployment_tpm is None:
|
||||
_deployment_tpm = _deployment.get("model_info", {}).get("tpm")
|
||||
if _deployment_tpm is None:
|
||||
_deployment_tpm = float("inf")
|
||||
|
||||
_deployment_rpm = (
|
||||
_deployment.get("rpm", None)
|
||||
or _deployment.get("litellm_params", {}).get("rpm", None)
|
||||
or _deployment.get("model_info", {}).get("rpm", None)
|
||||
or float("inf")
|
||||
)
|
||||
_deployment_rpm = None
|
||||
if _deployment_rpm is None:
|
||||
_deployment_rpm = _deployment.get("rpm")
|
||||
if _deployment_rpm is None:
|
||||
_deployment_rpm = _deployment.get("litellm_params", {}).get("rpm")
|
||||
if _deployment_rpm is None:
|
||||
_deployment_rpm = _deployment.get("model_info", {}).get("rpm")
|
||||
if _deployment_rpm is None:
|
||||
_deployment_rpm = float("inf")
|
||||
|
||||
if item_tpm == 0:
|
||||
deployment = _deployment
|
||||
break
|
||||
elif item_tpm + input_tokens > _deployment_tpm:
|
||||
if item_tpm + input_tokens > _deployment_tpm:
|
||||
continue
|
||||
elif (rpm_dict is not None and item in rpm_dict) and (
|
||||
rpm_dict[item] + 1 > _deployment_rpm
|
||||
|
|
@ -210,5 +208,5 @@ class LowestTPMLoggingHandler(CustomLogger):
|
|||
elif item_tpm < lowest_tpm:
|
||||
lowest_tpm = item_tpm
|
||||
deployment = _deployment
|
||||
verbose_router_logger.info("returning picked lowest tpm/rpm deployment.")
|
||||
print_verbose("returning picked lowest tpm/rpm deployment.")
|
||||
return deployment
|
||||
|
|
|
|||
|
|
@ -1107,6 +1107,7 @@ def test_completion_openai_litellm_key():
|
|||
# test_ completion_openai_litellm_key()
|
||||
|
||||
|
||||
@pytest.mark.skip(reason="Unresponsive endpoint.[TODO] Rehost this somewhere else")
|
||||
def test_completion_ollama_hosted():
|
||||
try:
|
||||
litellm.request_timeout = 20 # give ollama 20 seconds to response
|
||||
|
|
|
|||
|
|
@ -31,6 +31,8 @@ def test_image_generation_openai():
|
|||
except litellm.ContentPolicyViolationError:
|
||||
pass # OpenAI randomly raises these errors - skip when they occur
|
||||
except Exception as e:
|
||||
if "Connection error" in str(e):
|
||||
pass
|
||||
pytest.fail(f"An exception occurred - {str(e)}")
|
||||
|
||||
|
||||
|
|
@ -53,6 +55,8 @@ def test_image_generation_azure():
|
|||
except Exception as e:
|
||||
if "Your task failed as a result of our safety system." in str(e):
|
||||
pass
|
||||
if "Connection error" in str(e):
|
||||
pass
|
||||
else:
|
||||
pytest.fail(f"An exception occurred - {str(e)}")
|
||||
|
||||
|
|
@ -79,6 +83,8 @@ def test_image_generation_azure_dall_e_3():
|
|||
except Exception as e:
|
||||
if "Your task failed as a result of our safety system." in str(e):
|
||||
pass
|
||||
if "Connection error" in str(e):
|
||||
pass
|
||||
else:
|
||||
pytest.fail(f"An exception occurred - {str(e)}")
|
||||
|
||||
|
|
@ -97,6 +103,8 @@ async def test_async_image_generation_openai():
|
|||
except litellm.ContentPolicyViolationError:
|
||||
pass # openai randomly raises these errors - skip when they occur
|
||||
except Exception as e:
|
||||
if "Connection error" in str(e):
|
||||
pass
|
||||
pytest.fail(f"An exception occurred - {str(e)}")
|
||||
|
||||
|
||||
|
|
@ -117,6 +125,8 @@ async def test_async_image_generation_azure():
|
|||
except Exception as e:
|
||||
if "Your task failed as a result of our safety system." in str(e):
|
||||
pass
|
||||
if "Connection error" in str(e):
|
||||
pass
|
||||
else:
|
||||
pytest.fail(f"An exception occurred - {str(e)}")
|
||||
|
||||
|
|
|
|||
|
|
@ -590,6 +590,8 @@ async def test_aimg_gen_on_router():
|
|||
pass
|
||||
elif "Operation polling timed out" in str(e):
|
||||
pass
|
||||
elif "Connection error" in str(e):
|
||||
pass
|
||||
else:
|
||||
traceback.print_exc()
|
||||
pytest.fail(f"Error occurred: {e}")
|
||||
|
|
@ -671,6 +673,8 @@ def test_aembedding_on_router():
|
|||
pass
|
||||
elif "Operation polling timed out" in str(e):
|
||||
pass
|
||||
elif "Connection error" in str(e):
|
||||
pass
|
||||
else:
|
||||
traceback.print_exc()
|
||||
pytest.fail(f"Error occurred: {e}")
|
||||
|
|
|
|||
|
|
@ -264,7 +264,7 @@ def test_router_skip_rate_limited_deployments():
|
|||
end_time=end_time,
|
||||
)
|
||||
|
||||
## CHECK WHAT'S SELECTED ## - should skip 2, and pick 1
|
||||
## CHECK WHAT'S SELECTED ##
|
||||
# print(router.lowesttpm_logger.get_available_deployments(model_group="azure-model"))
|
||||
try:
|
||||
router.get_available_deployment(
|
||||
|
|
@ -273,7 +273,41 @@ def test_router_skip_rate_limited_deployments():
|
|||
)
|
||||
pytest.fail(f"Should have raised No Models Available error")
|
||||
except Exception as e:
|
||||
pass
|
||||
print(f"An exception occurred! {str(e)}")
|
||||
|
||||
|
||||
def test_single_deployment_tpm_zero():
|
||||
import litellm
|
||||
import os
|
||||
from datetime import datetime
|
||||
|
||||
model_list = [
|
||||
{
|
||||
"model_name": "gpt-3.5-turbo",
|
||||
"litellm_params": {
|
||||
"model": "gpt-3.5-turbo",
|
||||
"api_key": os.getenv("OPENAI_API_KEY"),
|
||||
"tpm": 0,
|
||||
},
|
||||
}
|
||||
]
|
||||
|
||||
router = litellm.Router(
|
||||
model_list=model_list,
|
||||
routing_strategy="usage-based-routing",
|
||||
cache_responses=True,
|
||||
)
|
||||
|
||||
model = "gpt-3.5-turbo"
|
||||
messages = [{"content": "Hello, how are you?", "role": "user"}]
|
||||
try:
|
||||
router.get_available_deployment(
|
||||
model=model,
|
||||
messages=[{"role": "user", "content": "Hey, how's it going?"}],
|
||||
)
|
||||
pytest.fail(f"Should have raised No Models Available error")
|
||||
except Exception as e:
|
||||
print(f"it worked - {str(e)}! \n{traceback.format_exc()}")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
|
|
|||
|
|
@ -92,4 +92,7 @@ async def test_batch_update_spend(prisma_client):
|
|||
setattr(litellm.proxy.proxy_server, "prisma_client", prisma_client)
|
||||
setattr(litellm.proxy.proxy_server, "master_key", "sk-1234")
|
||||
await litellm.proxy.proxy_server.prisma_client.connect()
|
||||
await update_spend(prisma_client=litellm.proxy.proxy_server.prisma_client)
|
||||
await update_spend(
|
||||
prisma_client=litellm.proxy.proxy_server.prisma_client,
|
||||
db_writer_client=None,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -2775,7 +2775,6 @@ def client(original_function):
|
|||
or isinstance(e, openai.Timeout)
|
||||
or isinstance(e, openai.APIConnectionError)
|
||||
):
|
||||
print_verbose(f"RETRY TRIGGERED!")
|
||||
kwargs["num_retries"] = num_retries
|
||||
return litellm.completion_with_retries(*args, **kwargs)
|
||||
elif (
|
||||
|
|
@ -3598,6 +3597,8 @@ def token_counter(
|
|||
raise ValueError("text and messages cannot both be None")
|
||||
elif isinstance(text, List):
|
||||
text = "".join(t for t in text if isinstance(t, str))
|
||||
elif isinstance(text, str):
|
||||
count_response_tokens = True # user just trying to count tokens for a text. don't add the chat_ml +3 tokens to this
|
||||
|
||||
if model is not None:
|
||||
tokenizer_json = _select_tokenizer(model=model)
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
[tool.poetry]
|
||||
name = "litellm"
|
||||
version = "1.34.7"
|
||||
version = "1.34.11"
|
||||
description = "Library to easily interface with LLM API providers"
|
||||
authors = ["BerriAI"]
|
||||
license = "MIT"
|
||||
|
|
@ -80,7 +80,7 @@ requires = ["poetry-core", "wheel"]
|
|||
build-backend = "poetry.core.masonry.api"
|
||||
|
||||
[tool.commitizen]
|
||||
version = "1.34.7"
|
||||
version = "1.34.11"
|
||||
version_files = [
|
||||
"pyproject.toml:^version"
|
||||
]
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@ fastapi>=0.109.1 # server dep
|
|||
pydantic>=2.5 # openai req.
|
||||
backoff==2.2.1 # server dep
|
||||
pyyaml>=6.0.1 # server dep
|
||||
uvicorn==0.22.0 # server dep
|
||||
uvicorn==0.29.0 # server dep
|
||||
gunicorn==21.2.0 # server dep
|
||||
boto3==1.34.34 # aws bedrock/sagemaker calls
|
||||
redis==5.0.0 # caching
|
||||
|
|
|
|||
|
|
@ -5,7 +5,6 @@ datasource client {
|
|||
|
||||
generator client {
|
||||
provider = "prisma-client-py"
|
||||
previewFeatures = ["jsonProtocol"]
|
||||
}
|
||||
|
||||
// Budget / Rate Limits for an org
|
||||
|
|
|
|||
|
|
@ -198,6 +198,10 @@ async def image_generation(session, key):
|
|||
print()
|
||||
|
||||
if status != 200:
|
||||
if (
|
||||
"Connection error" in response_text
|
||||
): # OpenAI endpoint returns a connection error
|
||||
return
|
||||
raise Exception(f"Request did not return a 200 status code: {status}")
|
||||
|
||||
|
||||
|
|
|
|||
File diff suppressed because one or more lines are too long
|
|
@ -0,0 +1 @@
|
|||
self.__BUILD_MANIFEST={__rewrites:{afterFiles:[],beforeFiles:[],fallback:[]},"/_error":["static/chunks/pages/_error-d6107f1aac0c574c.js"],sortedPages:["/_app","/_error"]},self.__BUILD_MANIFEST_CB&&self.__BUILD_MANIFEST_CB();
|
||||
|
|
@ -0,0 +1 @@
|
|||
self.__SSG_MANIFEST=new Set([]);self.__SSG_MANIFEST_CB&&self.__SSG_MANIFEST_CB()
|
||||
|
|
@ -0,0 +1 @@
|
|||
(self.webpackChunk_N_E=self.webpackChunk_N_E||[]).push([[185],{87421:function(n,e,t){Promise.resolve().then(t.t.bind(t,99646,23)),Promise.resolve().then(t.t.bind(t,63385,23))},63385:function(){},99646:function(n){n.exports={style:{fontFamily:"'__Inter_c23dc8', '__Inter_Fallback_c23dc8'",fontStyle:"normal"},className:"__className_c23dc8"}}},function(n){n.O(0,[971,69,744],function(){return n(n.s=87421)}),_N_E=n.O()}]);
|
||||
File diff suppressed because one or more lines are too long
|
|
@ -0,0 +1 @@
|
|||
(self.webpackChunk_N_E=self.webpackChunk_N_E||[]).push([[744],{32028:function(e,n,t){Promise.resolve().then(t.t.bind(t,47690,23)),Promise.resolve().then(t.t.bind(t,48955,23)),Promise.resolve().then(t.t.bind(t,5613,23)),Promise.resolve().then(t.t.bind(t,11902,23)),Promise.resolve().then(t.t.bind(t,31778,23)),Promise.resolve().then(t.t.bind(t,77831,23))}},function(e){var n=function(n){return e(e.s=n)};e.O(0,[971,69],function(){return n(35317),n(32028)}),_N_E=e.O()}]);
|
||||
|
|
@ -1 +1 @@
|
|||
<!DOCTYPE html><html id="__next_error__"><head><meta charSet="utf-8"/><meta name="viewport" content="width=device-width, initial-scale=1"/><link rel="preload" as="script" fetchPriority="low" href="/ui/_next/static/chunks/webpack-3b0d290a8fe6941d.js" crossorigin=""/><script src="/ui/_next/static/chunks/fd9d1056-a85b2c176012d8e5.js" async="" crossorigin=""></script><script src="/ui/_next/static/chunks/69-e1b183dda365ec86.js" async="" crossorigin=""></script><script src="/ui/_next/static/chunks/main-app-096338c8e1915716.js" async="" crossorigin=""></script><title>🚅 LiteLLM</title><meta name="description" content="LiteLLM Proxy Admin UI"/><link rel="icon" href="/ui/favicon.ico" type="image/x-icon" sizes="16x16"/><meta name="next-size-adjust"/><script src="/ui/_next/static/chunks/polyfills-c67a75d1b6f99dc8.js" crossorigin="" noModule=""></script></head><body><script src="/ui/_next/static/chunks/webpack-3b0d290a8fe6941d.js" crossorigin="" async=""></script><script>(self.__next_f=self.__next_f||[]).push([0]);self.__next_f.push([2,null])</script><script>self.__next_f.push([1,"1:HL[\"/ui/_next/static/media/c9a5bc6a7c948fb0-s.p.woff2\",\"font\",{\"crossOrigin\":\"\",\"type\":\"font/woff2\"}]\n2:HL[\"/ui/_next/static/css/68a21c6e6697f7ca.css\",\"style\",{\"crossOrigin\":\"\"}]\n0:\"$L3\"\n"])</script><script>self.__next_f.push([1,"4:I[47690,[],\"\"]\n6:I[77831,[],\"\"]\n7:I[19914,[\"730\",\"static/chunks/730-1411b729a1c79695.js\",\"931\",\"static/chunks/app/page-144687b251040a22.js\"],\"\"]\n8:I[5613,[],\"\"]\n9:I[31778,[],\"\"]\nb:I[48955,[],\"\"]\nc:[]\n"])</script><script>self.__next_f.push([1,"3:[[[\"$\",\"link\",\"0\",{\"rel\":\"stylesheet\",\"href\":\"/ui/_next/static/css/68a21c6e6697f7ca.css\",\"precedence\":\"next\",\"crossOrigin\":\"\"}]],[\"$\",\"$L4\",null,{\"buildId\":\"aIO8mtlEIEUTmgL8cGjve\",\"assetPrefix\":\"/ui\",\"initialCanonicalUrl\":\"/\",\"initialTree\":[\"\",{\"children\":[\"__PAGE__\",{}]},\"$undefined\",\"$undefined\",true],\"initialSeedData\":[\"\",{\"children\":[\"__PAGE__\",{},[\"$L5\",[\"$\",\"$L6\",null,{\"propsForComponent\":{\"params\":{}},\"Component\":\"$7\",\"isStaticGeneration\":true}],null]]},[null,[\"$\",\"html\",null,{\"lang\":\"en\",\"children\":[\"$\",\"body\",null,{\"className\":\"__className_c23dc8\",\"children\":[\"$\",\"$L8\",null,{\"parallelRouterKey\":\"children\",\"segmentPath\":[\"children\"],\"loading\":\"$undefined\",\"loadingStyles\":\"$undefined\",\"loadingScripts\":\"$undefined\",\"hasLoading\":false,\"error\":\"$undefined\",\"errorStyles\":\"$undefined\",\"errorScripts\":\"$undefined\",\"template\":[\"$\",\"$L9\",null,{}],\"templateStyles\":\"$undefined\",\"templateScripts\":\"$undefined\",\"notFound\":[[\"$\",\"title\",null,{\"children\":\"404: This page could not be found.\"}],[\"$\",\"div\",null,{\"style\":{\"fontFamily\":\"system-ui,\\\"Segoe UI\\\",Roboto,Helvetica,Arial,sans-serif,\\\"Apple Color Emoji\\\",\\\"Segoe UI Emoji\\\"\",\"height\":\"100vh\",\"textAlign\":\"center\",\"display\":\"flex\",\"flexDirection\":\"column\",\"alignItems\":\"center\",\"justifyContent\":\"center\"},\"children\":[\"$\",\"div\",null,{\"children\":[[\"$\",\"style\",null,{\"dangerouslySetInnerHTML\":{\"__html\":\"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}\"}}],[\"$\",\"h1\",null,{\"className\":\"next-error-h1\",\"style\":{\"display\":\"inline-block\",\"margin\":\"0 20px 0 0\",\"padding\":\"0 23px 0 0\",\"fontSize\":24,\"fontWeight\":500,\"verticalAlign\":\"top\",\"lineHeight\":\"49px\"},\"children\":\"404\"}],[\"$\",\"div\",null,{\"style\":{\"display\":\"inline-block\"},\"children\":[\"$\",\"h2\",null,{\"style\":{\"fontSize\":14,\"fontWeight\":400,\"lineHeight\":\"49px\",\"margin\":0},\"children\":\"This page could not be found.\"}]}]]}]}]],\"notFoundStyles\":[],\"styles\":null}]}]}],null]],\"initialHead\":[false,\"$La\"],\"globalErrorComponent\":\"$b\",\"missingSlots\":\"$Wc\"}]]\n"])</script><script>self.__next_f.push([1,"a:[[\"$\",\"meta\",\"0\",{\"name\":\"viewport\",\"content\":\"width=device-width, initial-scale=1\"}],[\"$\",\"meta\",\"1\",{\"charSet\":\"utf-8\"}],[\"$\",\"title\",\"2\",{\"children\":\"🚅 LiteLLM\"}],[\"$\",\"meta\",\"3\",{\"name\":\"description\",\"content\":\"LiteLLM Proxy Admin UI\"}],[\"$\",\"link\",\"4\",{\"rel\":\"icon\",\"href\":\"/ui/favicon.ico\",\"type\":\"image/x-icon\",\"sizes\":\"16x16\"}],[\"$\",\"meta\",\"5\",{\"name\":\"next-size-adjust\"}]]\n5:null\n"])</script><script>self.__next_f.push([1,""])</script></body></html>
|
||||
<!DOCTYPE html><html id="__next_error__"><head><meta charSet="utf-8"/><meta name="viewport" content="width=device-width, initial-scale=1"/><link rel="preload" as="script" fetchPriority="low" href="/ui/_next/static/chunks/webpack-3b0d290a8fe6941d.js" crossorigin=""/><script src="/ui/_next/static/chunks/fd9d1056-a85b2c176012d8e5.js" async="" crossorigin=""></script><script src="/ui/_next/static/chunks/69-e1b183dda365ec86.js" async="" crossorigin=""></script><script src="/ui/_next/static/chunks/main-app-9b4fb13a7db53edf.js" async="" crossorigin=""></script><title>🚅 LiteLLM</title><meta name="description" content="LiteLLM Proxy Admin UI"/><link rel="icon" href="/ui/favicon.ico" type="image/x-icon" sizes="16x16"/><meta name="next-size-adjust"/><script src="/ui/_next/static/chunks/polyfills-c67a75d1b6f99dc8.js" crossorigin="" noModule=""></script></head><body><script src="/ui/_next/static/chunks/webpack-3b0d290a8fe6941d.js" crossorigin="" async=""></script><script>(self.__next_f=self.__next_f||[]).push([0]);self.__next_f.push([2,null])</script><script>self.__next_f.push([1,"1:HL[\"/ui/_next/static/media/c9a5bc6a7c948fb0-s.p.woff2\",\"font\",{\"crossOrigin\":\"\",\"type\":\"font/woff2\"}]\n2:HL[\"/ui/_next/static/css/68a21c6e6697f7ca.css\",\"style\",{\"crossOrigin\":\"\"}]\n0:\"$L3\"\n"])</script><script>self.__next_f.push([1,"4:I[47690,[],\"\"]\n6:I[77831,[],\"\"]\n7:I[92182,[\"730\",\"static/chunks/730-1411b729a1c79695.js\",\"931\",\"static/chunks/app/page-8672e345e79b8043.js\"],\"\"]\n8:I[5613,[],\"\"]\n9:I[31778,[],\"\"]\nb:I[48955,[],\"\"]\nc:[]\n"])</script><script>self.__next_f.push([1,"3:[[[\"$\",\"link\",\"0\",{\"rel\":\"stylesheet\",\"href\":\"/ui/_next/static/css/68a21c6e6697f7ca.css\",\"precedence\":\"next\",\"crossOrigin\":\"\"}]],[\"$\",\"$L4\",null,{\"buildId\":\"YrQPd5ySk25sMOIe_YoyJ\",\"assetPrefix\":\"/ui\",\"initialCanonicalUrl\":\"/\",\"initialTree\":[\"\",{\"children\":[\"__PAGE__\",{}]},\"$undefined\",\"$undefined\",true],\"initialSeedData\":[\"\",{\"children\":[\"__PAGE__\",{},[\"$L5\",[\"$\",\"$L6\",null,{\"propsForComponent\":{\"params\":{}},\"Component\":\"$7\",\"isStaticGeneration\":true}],null]]},[null,[\"$\",\"html\",null,{\"lang\":\"en\",\"children\":[\"$\",\"body\",null,{\"className\":\"__className_c23dc8\",\"children\":[\"$\",\"$L8\",null,{\"parallelRouterKey\":\"children\",\"segmentPath\":[\"children\"],\"loading\":\"$undefined\",\"loadingStyles\":\"$undefined\",\"loadingScripts\":\"$undefined\",\"hasLoading\":false,\"error\":\"$undefined\",\"errorStyles\":\"$undefined\",\"errorScripts\":\"$undefined\",\"template\":[\"$\",\"$L9\",null,{}],\"templateStyles\":\"$undefined\",\"templateScripts\":\"$undefined\",\"notFound\":[[\"$\",\"title\",null,{\"children\":\"404: This page could not be found.\"}],[\"$\",\"div\",null,{\"style\":{\"fontFamily\":\"system-ui,\\\"Segoe UI\\\",Roboto,Helvetica,Arial,sans-serif,\\\"Apple Color Emoji\\\",\\\"Segoe UI Emoji\\\"\",\"height\":\"100vh\",\"textAlign\":\"center\",\"display\":\"flex\",\"flexDirection\":\"column\",\"alignItems\":\"center\",\"justifyContent\":\"center\"},\"children\":[\"$\",\"div\",null,{\"children\":[[\"$\",\"style\",null,{\"dangerouslySetInnerHTML\":{\"__html\":\"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}\"}}],[\"$\",\"h1\",null,{\"className\":\"next-error-h1\",\"style\":{\"display\":\"inline-block\",\"margin\":\"0 20px 0 0\",\"padding\":\"0 23px 0 0\",\"fontSize\":24,\"fontWeight\":500,\"verticalAlign\":\"top\",\"lineHeight\":\"49px\"},\"children\":\"404\"}],[\"$\",\"div\",null,{\"style\":{\"display\":\"inline-block\"},\"children\":[\"$\",\"h2\",null,{\"style\":{\"fontSize\":14,\"fontWeight\":400,\"lineHeight\":\"49px\",\"margin\":0},\"children\":\"This page could not be found.\"}]}]]}]}]],\"notFoundStyles\":[],\"styles\":null}]}]}],null]],\"initialHead\":[false,\"$La\"],\"globalErrorComponent\":\"$b\",\"missingSlots\":\"$Wc\"}]]\n"])</script><script>self.__next_f.push([1,"a:[[\"$\",\"meta\",\"0\",{\"name\":\"viewport\",\"content\":\"width=device-width, initial-scale=1\"}],[\"$\",\"meta\",\"1\",{\"charSet\":\"utf-8\"}],[\"$\",\"title\",\"2\",{\"children\":\"🚅 LiteLLM\"}],[\"$\",\"meta\",\"3\",{\"name\":\"description\",\"content\":\"LiteLLM Proxy Admin UI\"}],[\"$\",\"link\",\"4\",{\"rel\":\"icon\",\"href\":\"/ui/favicon.ico\",\"type\":\"image/x-icon\",\"sizes\":\"16x16\"}],[\"$\",\"meta\",\"5\",{\"name\":\"next-size-adjust\"}]]\n5:null\n"])</script><script>self.__next_f.push([1,""])</script></body></html>
|
||||
|
|
@ -1,7 +1,7 @@
|
|||
2:I[77831,[],""]
|
||||
3:I[19914,["730","static/chunks/730-1411b729a1c79695.js","931","static/chunks/app/page-144687b251040a22.js"],""]
|
||||
3:I[92182,["730","static/chunks/730-1411b729a1c79695.js","931","static/chunks/app/page-8672e345e79b8043.js"],""]
|
||||
4:I[5613,[],""]
|
||||
5:I[31778,[],""]
|
||||
0:["aIO8mtlEIEUTmgL8cGjve",[[["",{"children":["__PAGE__",{}]},"$undefined","$undefined",true],["",{"children":["__PAGE__",{},["$L1",["$","$L2",null,{"propsForComponent":{"params":{}},"Component":"$3","isStaticGeneration":true}],null]]},[null,["$","html",null,{"lang":"en","children":["$","body",null,{"className":"__className_c23dc8","children":["$","$L4",null,{"parallelRouterKey":"children","segmentPath":["children"],"loading":"$undefined","loadingStyles":"$undefined","loadingScripts":"$undefined","hasLoading":false,"error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L5",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":"404"}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],"notFoundStyles":[],"styles":null}]}]}],null]],[[["$","link","0",{"rel":"stylesheet","href":"/ui/_next/static/css/68a21c6e6697f7ca.css","precedence":"next","crossOrigin":""}]],"$L6"]]]]
|
||||
0:["YrQPd5ySk25sMOIe_YoyJ",[[["",{"children":["__PAGE__",{}]},"$undefined","$undefined",true],["",{"children":["__PAGE__",{},["$L1",["$","$L2",null,{"propsForComponent":{"params":{}},"Component":"$3","isStaticGeneration":true}],null]]},[null,["$","html",null,{"lang":"en","children":["$","body",null,{"className":"__className_c23dc8","children":["$","$L4",null,{"parallelRouterKey":"children","segmentPath":["children"],"loading":"$undefined","loadingStyles":"$undefined","loadingScripts":"$undefined","hasLoading":false,"error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L5",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":"404"}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],"notFoundStyles":[],"styles":null}]}]}],null]],[[["$","link","0",{"rel":"stylesheet","href":"/ui/_next/static/css/68a21c6e6697f7ca.css","precedence":"next","crossOrigin":""}]],"$L6"]]]]
|
||||
6:[["$","meta","0",{"name":"viewport","content":"width=device-width, initial-scale=1"}],["$","meta","1",{"charSet":"utf-8"}],["$","title","2",{"children":"🚅 LiteLLM"}],["$","meta","3",{"name":"description","content":"LiteLLM Proxy Admin UI"}],["$","link","4",{"rel":"icon","href":"/ui/favicon.ico","type":"image/x-icon","sizes":"16x16"}],["$","meta","5",{"name":"next-size-adjust"}]]
|
||||
1:null
|
||||
|
|
|
|||
|
|
@ -12,33 +12,32 @@ import {
|
|||
Select,
|
||||
message,
|
||||
} from "antd";
|
||||
import { keyCreateCall, slackBudgetAlertsHealthCheck } from "./networking";
|
||||
import { keyCreateCall, slackBudgetAlertsHealthCheck, modelAvailableCall } from "./networking";
|
||||
|
||||
const { Option } = Select;
|
||||
|
||||
interface CreateKeyProps {
|
||||
userID: string;
|
||||
teamID: string | null;
|
||||
team: any | null;
|
||||
userRole: string | null;
|
||||
accessToken: string;
|
||||
data: any[] | null;
|
||||
userModels: string[];
|
||||
setData: React.Dispatch<React.SetStateAction<any[] | null>>;
|
||||
}
|
||||
|
||||
const CreateKey: React.FC<CreateKeyProps> = ({
|
||||
userID,
|
||||
teamID,
|
||||
team,
|
||||
userRole,
|
||||
accessToken,
|
||||
data,
|
||||
userModels,
|
||||
setData,
|
||||
}) => {
|
||||
const [form] = Form.useForm();
|
||||
const [isModalVisible, setIsModalVisible] = useState(false);
|
||||
const [apiKey, setApiKey] = useState(null);
|
||||
const [softBudget, setSoftBudget] = useState(null);
|
||||
const [userModels, setUserModels] = useState([]);
|
||||
const handleOk = () => {
|
||||
setIsModalVisible(false);
|
||||
form.resetFields();
|
||||
|
|
@ -50,6 +49,29 @@ const CreateKey: React.FC<CreateKeyProps> = ({
|
|||
form.resetFields();
|
||||
};
|
||||
|
||||
useEffect(() => {
|
||||
const fetchUserModels = async () => {
|
||||
try {
|
||||
if (userID === null || userRole === null) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (accessToken !== null) {
|
||||
const model_available = await modelAvailableCall(accessToken, userID, userRole);
|
||||
let available_model_names = model_available["data"].map(
|
||||
(element: { id: string }) => element.id
|
||||
);
|
||||
console.log("available_model_names:", available_model_names);
|
||||
setUserModels(available_model_names);
|
||||
}
|
||||
} catch (error) {
|
||||
console.error("Error fetching user models:", error);
|
||||
}
|
||||
};
|
||||
|
||||
fetchUserModels();
|
||||
}, [accessToken, userID, userRole]);
|
||||
|
||||
const handleCreate = async (formValues: Record<string, any>) => {
|
||||
try {
|
||||
message.info("Making API Call");
|
||||
|
|
@ -105,12 +127,15 @@ const CreateKey: React.FC<CreateKeyProps> = ({
|
|||
<Form.Item label="Key Name" name="key_alias">
|
||||
<Input />
|
||||
</Form.Item>
|
||||
<Form.Item label="Team ID" name="team_id">
|
||||
<Input
|
||||
placeholder="ai_team"
|
||||
defaultValue={teamID ? teamID : ""}
|
||||
/>
|
||||
<Form.Item
|
||||
label="Team ID"
|
||||
name="team_id"
|
||||
initialValue={team ? team["team_id"] : null}
|
||||
valuePropName="team_id"
|
||||
>
|
||||
<Input value={team ? team["team_alias"] : ""} disabled />
|
||||
</Form.Item>
|
||||
|
||||
<Form.Item label="Models" name="models">
|
||||
<Select
|
||||
mode="multiple"
|
||||
|
|
@ -158,7 +183,7 @@ const CreateKey: React.FC<CreateKeyProps> = ({
|
|||
<Input />
|
||||
</Form.Item>
|
||||
<Form.Item label="Team ID (Contact Group)" name="team_id">
|
||||
<Input placeholder="ai_team" />
|
||||
<Input placeholder="default team (create a new team)" />
|
||||
</Form.Item>
|
||||
|
||||
<Form.Item label="Description" name="description">
|
||||
|
|
|
|||
|
|
@ -59,9 +59,6 @@ const Navbar: React.FC<NavbarProps> = ({
|
|||
target="_blank"
|
||||
className="mr-2"
|
||||
>
|
||||
<Button variant="primary" size="lg">
|
||||
Enable SSO
|
||||
</Button>
|
||||
</a>
|
||||
) : null}
|
||||
|
||||
|
|
|
|||
|
|
@ -102,7 +102,7 @@ const Team: React.FC<TeamProps> = ({
|
|||
const handleCreate = async (formValues: Record<string, any>) => {
|
||||
try {
|
||||
if (accessToken != null) {
|
||||
//message.info("Making API Call");
|
||||
message.info("Creating Team");
|
||||
const response: any = await teamCreateCall(accessToken, formValues);
|
||||
if (teams !== null) {
|
||||
setTeams([...teams, response]);
|
||||
|
|
@ -122,7 +122,7 @@ const Team: React.FC<TeamProps> = ({
|
|||
const handleMemberCreate = async (formValues: Record<string, any>) => {
|
||||
try {
|
||||
if (accessToken != null && teams != null) {
|
||||
message.info("Making API Call");
|
||||
message.info("Adding Member");
|
||||
const user_role: Member = {
|
||||
role: "user",
|
||||
user_email: formValues.user_email,
|
||||
|
|
@ -157,7 +157,7 @@ const Team: React.FC<TeamProps> = ({
|
|||
};
|
||||
console.log(`received teams ${teams}`);
|
||||
return (
|
||||
<div className="w-full">
|
||||
<div className="w-full mx-4">
|
||||
<Grid numItems={1} className="gap-2 p-2 h-[75vh] w-full">
|
||||
<Col numColSpan={1}>
|
||||
<Title level={4}>All Teams</Title>
|
||||
|
|
@ -168,6 +168,7 @@ const Team: React.FC<TeamProps> = ({
|
|||
<TableHeaderCell>Team Name</TableHeaderCell>
|
||||
<TableHeaderCell>Spend (USD)</TableHeaderCell>
|
||||
<TableHeaderCell>Budget (USD)</TableHeaderCell>
|
||||
<TableHeaderCell>Models</TableHeaderCell>
|
||||
<TableHeaderCell>TPM / RPM Limits</TableHeaderCell>
|
||||
</TableRow>
|
||||
</TableHead>
|
||||
|
|
@ -176,22 +177,25 @@ const Team: React.FC<TeamProps> = ({
|
|||
{teams && teams.length > 0
|
||||
? teams.map((team: any) => (
|
||||
<TableRow key={team.team_id}>
|
||||
<TableCell>{team["team_alias"]}</TableCell>
|
||||
<TableCell>{team["spend"]}</TableCell>
|
||||
<TableCell>
|
||||
<TableCell style={{ maxWidth: "4px", whiteSpace: "pre-wrap", overflow: "hidden" }}>{team["team_alias"]}</TableCell>
|
||||
<TableCell style={{ maxWidth: "4px", whiteSpace: "pre-wrap", overflow: "hidden" }}>{team["spend"]}</TableCell>
|
||||
<TableCell style={{ maxWidth: "4px", whiteSpace: "pre-wrap", overflow: "hidden" }}>
|
||||
{team["max_budget"] ? team["max_budget"] : "No limit"}
|
||||
</TableCell>
|
||||
<TableCell>
|
||||
<TableCell style={{ maxWidth: "4px", whiteSpace: "pre-wrap", overflow: "hidden" }}>
|
||||
<Text>{JSON.stringify(team["models"] ? team["models"] : [])}</Text>
|
||||
</TableCell>
|
||||
<TableCell style={{ maxWidth: "4px", whiteSpace: "pre-wrap", overflow: "hidden" }}>
|
||||
<Text>
|
||||
TPM Limit:{" "}
|
||||
{team.tpm_limit ? team.tpm_limit : "Unlimited"}{" "}
|
||||
<br></br> RPM Limit:{" "}
|
||||
<br></br>RPM Limit:{" "}
|
||||
{team.rpm_limit ? team.rpm_limit : "Unlimited"}
|
||||
</Text>
|
||||
</TableCell>
|
||||
<TableCell>
|
||||
{/* <TableCell>
|
||||
<Icon icon={CogIcon} size="sm" />
|
||||
</TableCell>
|
||||
</TableCell> */}
|
||||
</TableRow>
|
||||
))
|
||||
: null}
|
||||
|
|
@ -293,7 +297,7 @@ const Team: React.FC<TeamProps> = ({
|
|||
<TableRow>
|
||||
<TableHeaderCell>Member Name</TableHeaderCell>
|
||||
<TableHeaderCell>Role</TableHeaderCell>
|
||||
<TableHeaderCell>Action</TableHeaderCell>
|
||||
{/* <TableHeaderCell>Action</TableHeaderCell> */}
|
||||
</TableRow>
|
||||
</TableHead>
|
||||
|
||||
|
|
@ -310,9 +314,9 @@ const Team: React.FC<TeamProps> = ({
|
|||
: null}
|
||||
</TableCell>
|
||||
<TableCell>{member["role"]}</TableCell>
|
||||
<TableCell>
|
||||
{/* <TableCell>
|
||||
<Icon icon={CogIcon} size="sm" />
|
||||
</TableCell>
|
||||
</TableCell> */}
|
||||
</TableRow>
|
||||
)
|
||||
)
|
||||
|
|
|
|||
|
|
@ -203,6 +203,8 @@ const UserDashboard: React.FC<UserDashboardProps> = ({
|
|||
);
|
||||
}
|
||||
|
||||
console.log("inside user dashboard, selected team", selectedTeam);
|
||||
|
||||
return (
|
||||
<div>
|
||||
<Grid numItems={1} className="gap-0 p-10 h-[75vh] w-full">
|
||||
|
|
@ -220,10 +222,10 @@ const UserDashboard: React.FC<UserDashboardProps> = ({
|
|||
setData={setKeys}
|
||||
/>
|
||||
<CreateKey
|
||||
key={selectedTeam ? selectedTeam.team_id : null}
|
||||
userID={userID}
|
||||
teamID={selectedTeam ? selectedTeam["team_id"] : null}
|
||||
team={selectedTeam ? selectedTeam : null}
|
||||
userRole={userRole}
|
||||
userModels={userModels}
|
||||
accessToken={accessToken}
|
||||
data={keys}
|
||||
setData={setKeys}
|
||||
|
|
|
|||
|
|
@ -141,7 +141,7 @@ const ViewKeyTable: React.FC<ViewKeyTableProps> = ({
|
|||
/>
|
||||
</TableCell>
|
||||
<TableCell style={{ maxWidth: "4px", whiteSpace: "pre-wrap", overflow: "hidden" }}>
|
||||
<Text>{item.team_id}</Text>
|
||||
<Text>{item.team_alias && item.team_alias != "None" ? item.team_alias : item.team_id}</Text>
|
||||
</TableCell>
|
||||
<TableCell style={{ maxWidth: "4px", whiteSpace: "pre-wrap", overflow: "hidden" }}>
|
||||
<Text>{JSON.stringify(item.metadata).slice(0, 400)}</Text>
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue