phoenix/litellm-mirror
0
0
Fork 1
mirror of https://github.com/BerriAI/litellm.git synced 2025-04-25 02:34:29 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

UI (new_usage.tsx): Report 'total_tokens' + report success/failure calls (#9675)

Browse source 
* feat(internal_user_endpoints.py): return 'total_tokens' in `/user/daily/analytics`

* test(test_internal_user_endpoints.py): add unit test to assert spend metrics and dailyspend metadata always report the same fields

* build(schema.prisma): record success + failure calls to daily user table

allows understanding why model requests might exceed provider requests (e.g. user hit rate limit error)

* fix(internal_user_endpoints.py): report success / failure requests in API

* fix(proxy/utils.py): default to success

status can be missing or none at times for successful requests

* feat(new_usage.tsx): show success/failure calls on UI

* style(new_usage.tsx): ui cleanup

* fix: fix linting error

* fix: fix linting error

* feat(litellm-proxy-extras/): add new migration files
This commit is contained in:
Krish Dholakia 2025-03-31 22:48:43 -07:00 • committed by GitHub
parent f2a7edaddc
commit 62ad84fb64
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
16 changed files with 240 additions and 137 deletions
Download patch file Download diff file Expand all files Collapse all files

BIN
litellm-proxy-extras/dist/litellm_proxy_extras-0.1.2-py3-none-any.whl vendored Normal file
View file

Binary file not shown.

BIN
litellm-proxy-extras/dist/litellm_proxy_extras-0.1.2.tar.gz vendored Normal file
View file

Binary file not shown.

4
litellm-proxy-extras/litellm_proxy_extras/migrations/20250331215456_track_success_and_failed_requests_daily_agg_table/migration.sql Normal file
View file

@ -0,0 +1,4 @@
-- AlterTable
ALTER TABLE "LiteLLM_DailyUserSpend" ADD COLUMN "failed_requests" INTEGER NOT NULL DEFAULT 0,
ADD COLUMN "successful_requests" INTEGER NOT NULL DEFAULT 0;

7
litellm-proxy-extras/poetry.lock generated Normal file
View file

@ -0,0 +1,7 @@
# This file is automatically @generated by Poetry 1.8.3 and should not be changed by hand.
package = []
[metadata]
lock-version = "2.0"
python-versions = ">=3.8.1,<4.0, !=3.9.7"
content-hash = "2cf39473e67ff0615f0a61c9d2ac9f02b38cc08cbb1bdb893d89bee002646623"

4
litellm-proxy-extras/pyproject.toml
View file

@ -1,6 +1,6 @@
[tool.poetry] [tool.poetry]
name = "litellm-proxy-extras" name = "litellm-proxy-extras"
version = "0.1.1" version = "0.1.2"
description = "Additional files for the LiteLLM Proxy. Reduces the size of the main litellm package." description = "Additional files for the LiteLLM Proxy. Reduces the size of the main litellm package."
authors = ["BerriAI"] authors = ["BerriAI"]
readme = "README.md" readme = "README.md"
@ -22,7 +22,7 @@ requires = ["poetry-core"]
build-backend = "poetry.core.masonry.api" build-backend = "poetry.core.masonry.api"
[tool.commitizen] [tool.commitizen]
version = "0.1.1" version = "0.1.2"
version_files = [ version_files = [
"pyproject.toml:version", "pyproject.toml:version",
"../requirements.txt:litellm-proxy-extras==", "../requirements.txt:litellm-proxy-extras==",

10
litellm/proxy/_new_secret_config.yaml
View file

@ -32,8 +32,8 @@ litellm_settings:
callbacks: ["prometheus"] callbacks: ["prometheus"]
# json_logs: true # json_logs: true
# router_settings: router_settings:
# routing_strategy: usage-based-routing-v2 # 👈 KEY CHANGE routing_strategy: usage-based-routing-v2 # 👈 KEY CHANGE
# redis_host: os.environ/REDIS_HOST redis_host: os.environ/REDIS_HOST
# redis_password: os.environ/REDIS_PASSWORD redis_password: os.environ/REDIS_PASSWORD
# redis_port: os.environ/REDIS_PORT redis_port: os.environ/REDIS_PORT

2
litellm/proxy/_types.py
View file

@ -2736,6 +2736,8 @@ class DailyUserSpendTransaction(TypedDict):
completion_tokens: int completion_tokens: int
spend: float spend: float
api_requests: int api_requests: int
successful_requests: int
failed_requests: int
class DBSpendUpdateTransactions(TypedDict): class DBSpendUpdateTransactions(TypedDict):

22
litellm/proxy/management_endpoints/internal_user_endpoints.py
View file

@ -1259,6 +1259,8 @@ class SpendMetrics(BaseModel):
prompt_tokens: int = Field(default=0) prompt_tokens: int = Field(default=0)
completion_tokens: int = Field(default=0) completion_tokens: int = Field(default=0)
total_tokens: int = Field(default=0) total_tokens: int = Field(default=0)
successful_requests: int = Field(default=0)
failed_requests: int = Field(default=0)
api_requests: int = Field(default=0) api_requests: int = Field(default=0)
@ -1284,7 +1286,10 @@ class DailySpendMetadata(BaseModel):
total_spend: float = Field(default=0.0) total_spend: float = Field(default=0.0)
total_prompt_tokens: int = Field(default=0) total_prompt_tokens: int = Field(default=0)
total_completion_tokens: int = Field(default=0) total_completion_tokens: int = Field(default=0)
total_tokens: int = Field(default=0)
total_api_requests: int = Field(default=0) total_api_requests: int = Field(default=0)
total_successful_requests: int = Field(default=0)
total_failed_requests: int = Field(default=0)
page: int = Field(default=1) page: int = Field(default=1)
total_pages: int = Field(default=1) total_pages: int = Field(default=1)
has_more: bool = Field(default=False) has_more: bool = Field(default=False)
@ -1307,6 +1312,8 @@ class LiteLLM_DailyUserSpend(BaseModel):
completion_tokens: int = 0 completion_tokens: int = 0
spend: float = 0.0 spend: float = 0.0
api_requests: int = 0 api_requests: int = 0
successful_requests: int = 0
failed_requests: int = 0
class GroupedData(TypedDict): class GroupedData(TypedDict):
@ -1322,6 +1329,8 @@ def update_metrics(
group_metrics.completion_tokens += record.completion_tokens group_metrics.completion_tokens += record.completion_tokens
group_metrics.total_tokens += record.prompt_tokens + record.completion_tokens group_metrics.total_tokens += record.prompt_tokens + record.completion_tokens
group_metrics.api_requests += record.api_requests group_metrics.api_requests += record.api_requests
group_metrics.successful_requests += record.successful_requests
group_metrics.failed_requests += record.failed_requests
return group_metrics return group_metrics
@ -1443,6 +1452,10 @@ async def get_user_daily_activity(
take=page_size, take=page_size,
) )
daily_spend_data_pydantic_list = [
LiteLLM_DailyUserSpend(**record.model_dump()) for record in daily_spend_data
]
# Process results # Process results
results = [] results = []
total_metrics = SpendMetrics() total_metrics = SpendMetrics()
@ -1450,7 +1463,7 @@ async def get_user_daily_activity(
# Group data by date and other dimensions # Group data by date and other dimensions
grouped_data: Dict[str, Dict[str, Any]] = {} grouped_data: Dict[str, Dict[str, Any]] = {}
for record in daily_spend_data: for record in daily_spend_data_pydantic_list:
date_str = record.date date_str = record.date
if date_str not in grouped_data: if date_str not in grouped_data:
grouped_data[date_str] = { grouped_data[date_str] = {
@ -1474,7 +1487,9 @@ async def get_user_daily_activity(
total_metrics.total_tokens += ( total_metrics.total_tokens += (
record.prompt_tokens + record.completion_tokens record.prompt_tokens + record.completion_tokens
) )
total_metrics.api_requests += 1 total_metrics.api_requests += record.api_requests
total_metrics.successful_requests += record.successful_requests
total_metrics.failed_requests += record.failed_requests
# Convert grouped data to response format # Convert grouped data to response format
for date_str, data in grouped_data.items(): for date_str, data in grouped_data.items():
@ -1495,7 +1510,10 @@ async def get_user_daily_activity(
total_spend=total_metrics.spend, total_spend=total_metrics.spend,
total_prompt_tokens=total_metrics.prompt_tokens, total_prompt_tokens=total_metrics.prompt_tokens,
total_completion_tokens=total_metrics.completion_tokens, total_completion_tokens=total_metrics.completion_tokens,
total_tokens=total_metrics.total_tokens,
total_api_requests=total_metrics.api_requests, total_api_requests=total_metrics.api_requests,
total_successful_requests=total_metrics.successful_requests,
total_failed_requests=total_metrics.failed_requests,
page=page, page=page,
total_pages=-(-total_count // page_size), # Ceiling division total_pages=-(-total_count // page_size), # Ceiling division
has_more=(page * page_size) < total_count, has_more=(page * page_size) < total_count,

3
litellm/proxy/schema.prisma
View file

@ -327,6 +327,8 @@ model LiteLLM_DailyUserSpend {
completion_tokens Int @default(0) completion_tokens Int @default(0)
spend Float @default(0.0) spend Float @default(0.0)
api_requests Int @default(0) api_requests Int @default(0)
successful_requests Int @default(0)
failed_requests Int @default(0)
created_at DateTime @default(now()) created_at DateTime @default(now())
updated_at DateTime @updatedAt updated_at DateTime @updatedAt
@ -352,4 +354,3 @@ enum JobStatus {
INACTIVE INACTIVE
} }

73
litellm/proxy/utils.py
View file

@ -10,7 +10,17 @@ import traceback
from datetime import datetime, timedelta from datetime import datetime, timedelta
from email.mime.multipart import MIMEMultipart from email.mime.multipart import MIMEMultipart
from email.mime.text import MIMEText from email.mime.text import MIMEText
from typing import TYPE_CHECKING, Any, Dict, List, Literal, Optional, Union, overload from typing import (
TYPE_CHECKING,
Any,
Dict,
List,
Literal,
Optional,
Union,
cast,
overload,
)
from litellm.proxy._types import ( from litellm.proxy._types import (
DB_CONNECTION_ERROR_TYPES, DB_CONNECTION_ERROR_TYPES,
@ -18,6 +28,7 @@ from litellm.proxy._types import (
DailyUserSpendTransaction, DailyUserSpendTransaction,
ProxyErrorTypes, ProxyErrorTypes,
ProxyException, ProxyException,
SpendLogsMetadata,
SpendLogsPayload, SpendLogsPayload,
) )
from litellm.types.guardrails import GuardrailEventHooks from litellm.types.guardrails import GuardrailEventHooks
@ -1145,6 +1156,41 @@ class PrismaClient:
) # Client to connect to Prisma db ) # Client to connect to Prisma db
verbose_proxy_logger.debug("Success - Created Prisma Client") verbose_proxy_logger.debug("Success - Created Prisma Client")
def get_request_status(
self, payload: Union[dict, SpendLogsPayload]
) -> Literal["success", "failure"]:
"""
Determine if a request was successful or failed based on payload metadata.
Args:
payload (Union[dict, SpendLogsPayload]): Request payload containing metadata
Returns:
Literal["success", "failure"]: Request status
"""
try:
# Get metadata and convert to dict if it's a JSON string
payload_metadata: Union[Dict, SpendLogsMetadata, str] = payload.get(
"metadata", {}
)
if isinstance(payload_metadata, str):
payload_metadata_json: Union[Dict, SpendLogsMetadata] = cast(
Dict, json.loads(payload_metadata)
)
else:
payload_metadata_json = payload_metadata
# Check status in metadata dict
return (
"failure"
if payload_metadata_json.get("status") == "failure"
else "success"
)
except (json.JSONDecodeError, AttributeError):
# Default to success if metadata parsing fails
return "success"
def add_spend_log_transaction_to_daily_user_transaction( def add_spend_log_transaction_to_daily_user_transaction(
self, payload: Union[dict, SpendLogsPayload] self, payload: Union[dict, SpendLogsPayload]
): ):
@ -1156,12 +1202,15 @@ class PrismaClient:
If key exists, update the transaction with the new spend and usage If key exists, update the transaction with the new spend and usage
""" """
expected_keys = ["user", "startTime", "api_key", "model", "custom_llm_provider"] expected_keys = ["user", "startTime", "api_key", "model", "custom_llm_provider"]
if not all(key in payload for key in expected_keys): if not all(key in payload for key in expected_keys):
verbose_proxy_logger.debug( verbose_proxy_logger.debug(
f"Missing expected keys: {expected_keys}, in payload, skipping from daily_user_spend_transactions" f"Missing expected keys: {expected_keys}, in payload, skipping from daily_user_spend_transactions"
) )
return return
request_status = self.get_request_status(payload)
verbose_proxy_logger.info(f"Logged request status: {request_status}")
if isinstance(payload["startTime"], datetime): if isinstance(payload["startTime"], datetime):
start_time = payload["startTime"].isoformat() start_time = payload["startTime"].isoformat()
date = start_time.split("T")[0] date = start_time.split("T")[0]
@ -1174,6 +1223,7 @@ class PrismaClient:
return return
try: try:
daily_transaction_key = f"{payload['user']}_{date}_{payload['api_key']}_{payload['model']}_{payload['custom_llm_provider']}" daily_transaction_key = f"{payload['user']}_{date}_{payload['api_key']}_{payload['model']}_{payload['custom_llm_provider']}"
if daily_transaction_key in self.daily_user_spend_transactions: if daily_transaction_key in self.daily_user_spend_transactions:
daily_transaction = self.daily_user_spend_transactions[ daily_transaction = self.daily_user_spend_transactions[
daily_transaction_key daily_transaction_key
@ -1182,6 +1232,11 @@ class PrismaClient:
daily_transaction["prompt_tokens"] += payload["prompt_tokens"] daily_transaction["prompt_tokens"] += payload["prompt_tokens"]
daily_transaction["completion_tokens"] += payload["completion_tokens"] daily_transaction["completion_tokens"] += payload["completion_tokens"]
daily_transaction["api_requests"] += 1 daily_transaction["api_requests"] += 1
if request_status == "success":
daily_transaction["successful_requests"] += 1
else:
daily_transaction["failed_requests"] += 1
else: else:
daily_transaction = DailyUserSpendTransaction( daily_transaction = DailyUserSpendTransaction(
user_id=payload["user"], user_id=payload["user"],
@ -1194,6 +1249,8 @@ class PrismaClient:
completion_tokens=payload["completion_tokens"], completion_tokens=payload["completion_tokens"],
spend=payload["spend"], spend=payload["spend"],
api_requests=1, api_requests=1,
successful_requests=1 if request_status == "success" else 0,
failed_requests=1 if request_status != "success" else 0,
) )
self.daily_user_spend_transactions[ self.daily_user_spend_transactions[
@ -2603,6 +2660,12 @@ class ProxyUpdateSpend:
], ],
"spend": transaction["spend"], "spend": transaction["spend"],
"api_requests": transaction["api_requests"], "api_requests": transaction["api_requests"],
"successful_requests": transaction[
"successful_requests"
],
"failed_requests": transaction[
"failed_requests"
],
}, },
"update": { "update": {
"prompt_tokens": { "prompt_tokens": {
@ -2617,6 +2680,14 @@ class ProxyUpdateSpend:
"api_requests": { "api_requests": {
"increment": transaction["api_requests"] "increment": transaction["api_requests"]
}, },
"successful_requests": {
"increment": transaction[
"successful_requests"
]
},
"failed_requests": {
"increment": transaction["failed_requests"]
},
}, },
}, },
) )

87
poetry.lock generated
View file

@ -1151,69 +1151,6 @@ files = [
[package.extras] [package.extras]
protobuf = ["grpcio-tools (>=1.70.0)"] protobuf = ["grpcio-tools (>=1.70.0)"]
[[package]]
name = "grpcio"
version = "1.71.0"
description = "HTTP/2-based RPC framework"
optional = true
python-versions = ">=3.9"
files = [
{file = "grpcio-1.71.0-cp310-cp310-linux_armv7l.whl", hash = "sha256:c200cb6f2393468142eb50ab19613229dcc7829b5ccee8b658a36005f6669fdd"},
{file = "grpcio-1.71.0-cp310-cp310-macosx_12_0_universal2.whl", hash = "sha256:b2266862c5ad664a380fbbcdbdb8289d71464c42a8c29053820ee78ba0119e5d"},
{file = "grpcio-1.71.0-cp310-cp310-manylinux_2_17_aarch64.whl", hash = "sha256:0ab8b2864396663a5b0b0d6d79495657ae85fa37dcb6498a2669d067c65c11ea"},
{file = "grpcio-1.71.0-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c30f393f9d5ff00a71bb56de4aa75b8fe91b161aeb61d39528db6b768d7eac69"},
{file = "grpcio-1.71.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f250ff44843d9a0615e350c77f890082102a0318d66a99540f54769c8766ab73"},
{file = "grpcio-1.71.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:e6d8de076528f7c43a2f576bc311799f89d795aa6c9b637377cc2b1616473804"},
{file = "grpcio-1.71.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:9b91879d6da1605811ebc60d21ab6a7e4bae6c35f6b63a061d61eb818c8168f6"},
{file = "grpcio-1.71.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:f71574afdf944e6652203cd1badcda195b2a27d9c83e6d88dc1ce3cfb73b31a5"},
{file = "grpcio-1.71.0-cp310-cp310-win32.whl", hash = "sha256:8997d6785e93308f277884ee6899ba63baafa0dfb4729748200fcc537858a509"},
{file = "grpcio-1.71.0-cp310-cp310-win_amd64.whl", hash = "sha256:7d6ac9481d9d0d129224f6d5934d5832c4b1cddb96b59e7eba8416868909786a"},
{file = "grpcio-1.71.0-cp311-cp311-linux_armv7l.whl", hash = "sha256:d6aa986318c36508dc1d5001a3ff169a15b99b9f96ef5e98e13522c506b37eef"},
{file = "grpcio-1.71.0-cp311-cp311-macosx_10_14_universal2.whl", hash = "sha256:d2c170247315f2d7e5798a22358e982ad6eeb68fa20cf7a820bb74c11f0736e7"},
{file = "grpcio-1.71.0-cp311-cp311-manylinux_2_17_aarch64.whl", hash = "sha256:e6f83a583ed0a5b08c5bc7a3fe860bb3c2eac1f03f1f63e0bc2091325605d2b7"},
{file = "grpcio-1.71.0-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4be74ddeeb92cc87190e0e376dbc8fc7736dbb6d3d454f2fa1f5be1dee26b9d7"},
{file = "grpcio-1.71.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4dd0dfbe4d5eb1fcfec9490ca13f82b089a309dc3678e2edabc144051270a66e"},
{file = "grpcio-1.71.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:a2242d6950dc892afdf9e951ed7ff89473aaf744b7d5727ad56bdaace363722b"},
{file = "grpcio-1.71.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:0fa05ee31a20456b13ae49ad2e5d585265f71dd19fbd9ef983c28f926d45d0a7"},
{file = "grpcio-1.71.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:3d081e859fb1ebe176de33fc3adb26c7d46b8812f906042705346b314bde32c3"},
{file = "grpcio-1.71.0-cp311-cp311-win32.whl", hash = "sha256:d6de81c9c00c8a23047136b11794b3584cdc1460ed7cbc10eada50614baa1444"},
{file = "grpcio-1.71.0-cp311-cp311-win_amd64.whl", hash = "sha256:24e867651fc67717b6f896d5f0cac0ec863a8b5fb7d6441c2ab428f52c651c6b"},
{file = "grpcio-1.71.0-cp312-cp312-linux_armv7l.whl", hash = "sha256:0ff35c8d807c1c7531d3002be03221ff9ae15712b53ab46e2a0b4bb271f38537"},
{file = "grpcio-1.71.0-cp312-cp312-macosx_10_14_universal2.whl", hash = "sha256:b78a99cd1ece4be92ab7c07765a0b038194ded2e0a26fd654591ee136088d8d7"},
{file = "grpcio-1.71.0-cp312-cp312-manylinux_2_17_aarch64.whl", hash = "sha256:dc1a1231ed23caac1de9f943d031f1bc38d0f69d2a3b243ea0d664fc1fbd7fec"},
{file = "grpcio-1.71.0-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e6beeea5566092c5e3c4896c6d1d307fb46b1d4bdf3e70c8340b190a69198594"},
{file = "grpcio-1.71.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d5170929109450a2c031cfe87d6716f2fae39695ad5335d9106ae88cc32dc84c"},
{file = "grpcio-1.71.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:5b08d03ace7aca7b2fadd4baf291139b4a5f058805a8327bfe9aece7253b6d67"},
{file = "grpcio-1.71.0-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:f903017db76bf9cc2b2d8bdd37bf04b505bbccad6be8a81e1542206875d0e9db"},
{file = "grpcio-1.71.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:469f42a0b410883185eab4689060a20488a1a0a00f8bbb3cbc1061197b4c5a79"},
{file = "grpcio-1.71.0-cp312-cp312-win32.whl", hash = "sha256:ad9f30838550695b5eb302add33f21f7301b882937460dd24f24b3cc5a95067a"},
{file = "grpcio-1.71.0-cp312-cp312-win_amd64.whl", hash = "sha256:652350609332de6dac4ece254e5d7e1ff834e203d6afb769601f286886f6f3a8"},
{file = "grpcio-1.71.0-cp313-cp313-linux_armv7l.whl", hash = "sha256:cebc1b34ba40a312ab480ccdb396ff3c529377a2fce72c45a741f7215bfe8379"},
{file = "grpcio-1.71.0-cp313-cp313-macosx_10_14_universal2.whl", hash = "sha256:85da336e3649a3d2171e82f696b5cad2c6231fdd5bad52616476235681bee5b3"},
{file = "grpcio-1.71.0-cp313-cp313-manylinux_2_17_aarch64.whl", hash = "sha256:f9a412f55bb6e8f3bb000e020dbc1e709627dcb3a56f6431fa7076b4c1aab0db"},
{file = "grpcio-1.71.0-cp313-cp313-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:47be9584729534660416f6d2a3108aaeac1122f6b5bdbf9fd823e11fe6fbaa29"},
{file = "grpcio-1.71.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7c9c80ac6091c916db81131d50926a93ab162a7e97e4428ffc186b6e80d6dda4"},
{file = "grpcio-1.71.0-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:789d5e2a3a15419374b7b45cd680b1e83bbc1e52b9086e49308e2c0b5bbae6e3"},
{file = "grpcio-1.71.0-cp313-cp313-musllinux_1_1_i686.whl", hash = "sha256:1be857615e26a86d7363e8a163fade914595c81fec962b3d514a4b1e8760467b"},
{file = "grpcio-1.71.0-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:a76d39b5fafd79ed604c4be0a869ec3581a172a707e2a8d7a4858cb05a5a7637"},
{file = "grpcio-1.71.0-cp313-cp313-win32.whl", hash = "sha256:74258dce215cb1995083daa17b379a1a5a87d275387b7ffe137f1d5131e2cfbb"},
{file = "grpcio-1.71.0-cp313-cp313-win_amd64.whl", hash = "sha256:22c3bc8d488c039a199f7a003a38cb7635db6656fa96437a8accde8322ce2366"},
{file = "grpcio-1.71.0-cp39-cp39-linux_armv7l.whl", hash = "sha256:c6a0a28450c16809f94e0b5bfe52cabff63e7e4b97b44123ebf77f448534d07d"},
{file = "grpcio-1.71.0-cp39-cp39-macosx_10_14_universal2.whl", hash = "sha256:a371e6b6a5379d3692cc4ea1cb92754d2a47bdddeee755d3203d1f84ae08e03e"},
{file = "grpcio-1.71.0-cp39-cp39-manylinux_2_17_aarch64.whl", hash = "sha256:39983a9245d37394fd59de71e88c4b295eb510a3555e0a847d9965088cdbd033"},
{file = "grpcio-1.71.0-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9182e0063112e55e74ee7584769ec5a0b4f18252c35787f48738627e23a62b97"},
{file = "grpcio-1.71.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:693bc706c031aeb848849b9d1c6b63ae6bcc64057984bb91a542332b75aa4c3d"},
{file = "grpcio-1.71.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:20e8f653abd5ec606be69540f57289274c9ca503ed38388481e98fa396ed0b41"},
{file = "grpcio-1.71.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:8700a2a57771cc43ea295296330daaddc0d93c088f0a35cc969292b6db959bf3"},
{file = "grpcio-1.71.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:d35a95f05a8a2cbe8e02be137740138b3b2ea5f80bd004444e4f9a1ffc511e32"},
{file = "grpcio-1.71.0-cp39-cp39-win32.whl", hash = "sha256:f9c30c464cb2ddfbc2ddf9400287701270fdc0f14be5f08a1e3939f1e749b455"},
{file = "grpcio-1.71.0-cp39-cp39-win_amd64.whl", hash = "sha256:63e41b91032f298b3e973b3fa4093cbbc620c875e2da7b93e249d4728b54559a"},
{file = "grpcio-1.71.0.tar.gz", hash = "sha256:2b85f7820475ad3edec209d3d89a7909ada16caab05d3f2e08a7e8ae3200a55c"},
]
[package.extras]
protobuf = ["grpcio-tools (>=1.71.0)"]
[[package]] [[package]]
name = "grpcio-status" name = "grpcio-status"
version = "1.70.0" version = "1.70.0"
@ -1230,22 +1167,6 @@ googleapis-common-protos = ">=1.5.5"
grpcio = ">=1.70.0" grpcio = ">=1.70.0"
protobuf = ">=5.26.1,<6.0dev" protobuf = ">=5.26.1,<6.0dev"
[[package]]
name = "grpcio-status"
version = "1.71.0"
description = "Status proto mapping for gRPC"
optional = true
python-versions = ">=3.9"
files = [
{file = "grpcio_status-1.71.0-py3-none-any.whl", hash = "sha256:843934ef8c09e3e858952887467f8256aac3910c55f077a359a65b2b3cde3e68"},
{file = "grpcio_status-1.71.0.tar.gz", hash = "sha256:11405fed67b68f406b3f3c7c5ae5104a79d2d309666d10d61b152e91d28fb968"},
]
[package.dependencies]
googleapis-common-protos = ">=1.5.5"
grpcio = ">=1.71.0"
protobuf = ">=5.26.1,<6.0dev"
[[package]] [[package]]
name = "gunicorn" name = "gunicorn"
version = "23.0.0" version = "23.0.0"
@ -1678,13 +1599,13 @@ referencing = ">=0.31.0"
[[package]] [[package]]
name = "litellm-proxy-extras" name = "litellm-proxy-extras"
version = "0.1.1" version = "0.1.2"
description = "Additional files for the LiteLLM Proxy. Reduces the size of the main litellm package." description = "Additional files for the LiteLLM Proxy. Reduces the size of the main litellm package."
optional = true optional = true
python-versions = "!=2.7.*,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,!=3.7.*,>=3.8" python-versions = "!=2.7.*,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,!=3.7.*,>=3.8"
files = [ files = [
{file = "litellm_proxy_extras-0.1.1-py3-none-any.whl", hash = "sha256:2b3c4c5474bacbde2424c1cd13b21f85c65e9c4346f6159badd49a210eedef5c"}, {file = "litellm_proxy_extras-0.1.2-py3-none-any.whl", hash = "sha256:2caa7bdba5a533cd1781b55e3f7c581138d2a5b68a7e6d737327669dd21d5e08"},
{file = "litellm_proxy_extras-0.1.1.tar.gz", hash = "sha256:a1eb911ad2e3742238863d314a8bd6d02dd0cc213ba040b2c0593f132fbf3117"}, {file = "litellm_proxy_extras-0.1.2.tar.gz", hash = "sha256:218e97980ab5a34eed7dcd1564a910c9a790168d672cdec3c464eba9b7cb1518"},
] ]
[[package]] [[package]]
@ -4135,4 +4056,4 @@ proxy = ["PyJWT", "apscheduler", "backoff", "boto3", "cryptography", "fastapi",
[metadata] [metadata]
lock-version = "2.0" lock-version = "2.0"
python-versions = ">=3.8.1,<4.0, !=3.9.7" python-versions = ">=3.8.1,<4.0, !=3.9.7"
content-hash = "16cbf20784776377805f5e33c6bc97dce76303132aa3d81c7e6fe743f0ee3fc1" content-hash = "524b2f8276ba057f8dc8a79dd460c1a243ef4aece7c08a8bf344e029e07b8841"

2
pyproject.toml
View file

@ -55,7 +55,7 @@ websockets = {version = "^13.1.0", optional = true}
boto3 = {version = "1.34.34", optional = true} boto3 = {version = "1.34.34", optional = true}
redisvl = {version = "^0.4.1", optional = true, markers = "python_version >= '3.9' and python_version < '3.14'"} redisvl = {version = "^0.4.1", optional = true, markers = "python_version >= '3.9' and python_version < '3.14'"}
mcp = {version = "1.5.0", optional = true, python = ">=3.10"} mcp = {version = "1.5.0", optional = true, python = ">=3.10"}
litellm-proxy-extras = {version = "0.1.1", optional = true} litellm-proxy-extras = {version = "0.1.2", optional = true}
[tool.poetry.extras] [tool.poetry.extras]
proxy = [ proxy = [

2
requirements.txt
View file

@ -38,7 +38,7 @@ sentry_sdk==2.21.0 # for sentry error handling
detect-secrets==1.5.0 # Enterprise - secret detection / masking in LLM requests detect-secrets==1.5.0 # Enterprise - secret detection / masking in LLM requests
cryptography==43.0.1 cryptography==43.0.1
tzdata==2025.1 # IANA time zone database tzdata==2025.1 # IANA time zone database
litellm-proxy-extras==0.1.1 # for proxy extras - e.g. prisma migrations litellm-proxy-extras==0.1.2 # for proxy extras - e.g. prisma migrations
### LITELLM PACKAGE DEPENDENCIES ### LITELLM PACKAGE DEPENDENCIES
python-dotenv==1.0.0 # for env python-dotenv==1.0.0 # for env

3
schema.prisma
View file

@ -327,6 +327,8 @@ model LiteLLM_DailyUserSpend {
completion_tokens Int @default(0) completion_tokens Int @default(0)
spend Float @default(0.0) spend Float @default(0.0)
api_requests Int @default(0) api_requests Int @default(0)
successful_requests Int @default(0)
failed_requests Int @default(0)
created_at DateTime @default(now()) created_at DateTime @default(now())
updated_at DateTime @updatedAt updated_at DateTime @updatedAt
@ -351,3 +353,4 @@ enum JobStatus {
ACTIVE ACTIVE
INACTIVE INACTIVE
} }

27
tests/litellm/proxy/management_endpoints/test_internal_user_endpoints.py
View file

@ -55,3 +55,30 @@ async def test_ui_view_users_with_null_email(mocker, caplog):
assert response == [ assert response == [
LiteLLM_UserTableFiltered(user_id="test-user-null-email", user_email=None) LiteLLM_UserTableFiltered(user_id="test-user-null-email", user_email=None)
] ]
def test_user_daily_activity_types():
"""
Assert all fiels in SpendMetrics are reported in DailySpendMetadata as "total_"
"""
from litellm.proxy.management_endpoints.internal_user_endpoints import (
DailySpendMetadata,
SpendMetrics,
)
# Create a SpendMetrics instance
spend_metrics = SpendMetrics()
# Create a DailySpendMetadata instance
daily_spend_metadata = DailySpendMetadata()
# Assert all fields in SpendMetrics are reported in DailySpendMetadata as "total_"
for field in spend_metrics.__dict__:
if field.startswith("total_"):
assert hasattr(
daily_spend_metadata, field
), f"Field {field} is not reported in DailySpendMetadata"
else:
assert not hasattr(
daily_spend_metadata, field
), f"Field {field} is reported in DailySpendMetadata"

131
ui/litellm-dashboard/src/components/new_usage.tsx
View file

@ -33,6 +33,8 @@ interface SpendMetrics {
completion_tokens: number; completion_tokens: number;
total_tokens: number; total_tokens: number;
api_requests: number; api_requests: number;
successful_requests: number;
failed_requests: number;
} }
interface BreakdownMetrics { interface BreakdownMetrics {
@ -59,7 +61,7 @@ const NewUsagePage: React.FC<NewUsagePageProps> = ({
// Derived states from userSpendData // Derived states from userSpendData
const totalSpend = userSpendData.metadata?.total_spend || 0; const totalSpend = userSpendData.metadata?.total_spend || 0;
// Calculate top models from the breakdown data // Calculate top models from the breakdown data
const getTopModels = () => { const getTopModels = () => {
const modelSpend: { [key: string]: SpendMetrics } = {}; const modelSpend: { [key: string]: SpendMetrics } = {};
@ -71,7 +73,9 @@ const NewUsagePage: React.FC<NewUsagePageProps> = ({
prompt_tokens: 0, prompt_tokens: 0,
completion_tokens: 0, completion_tokens: 0,
total_tokens: 0, total_tokens: 0,
api_requests: 0 api_requests: 0,
successful_requests: 0,
failed_requests: 0
}; };
} }
modelSpend[model].spend += metrics.spend; modelSpend[model].spend += metrics.spend;
@ -79,6 +83,8 @@ const NewUsagePage: React.FC<NewUsagePageProps> = ({
modelSpend[model].completion_tokens += metrics.completion_tokens; modelSpend[model].completion_tokens += metrics.completion_tokens;
modelSpend[model].total_tokens += metrics.total_tokens; modelSpend[model].total_tokens += metrics.total_tokens;
modelSpend[model].api_requests += metrics.api_requests; modelSpend[model].api_requests += metrics.api_requests;
modelSpend[model].successful_requests += metrics.successful_requests || 0;
modelSpend[model].failed_requests += metrics.failed_requests || 0;
}); });
}); });
@ -87,6 +93,8 @@ const NewUsagePage: React.FC<NewUsagePageProps> = ({
key: model, key: model,
spend: metrics.spend, spend: metrics.spend,
requests: metrics.api_requests, requests: metrics.api_requests,
successful_requests: metrics.successful_requests,
failed_requests: metrics.failed_requests,
tokens: metrics.total_tokens tokens: metrics.total_tokens
})) }))
.sort((a, b) => b.spend - a.spend) .sort((a, b) => b.spend - a.spend)
@ -104,7 +112,9 @@ const NewUsagePage: React.FC<NewUsagePageProps> = ({
prompt_tokens: 0, prompt_tokens: 0,
completion_tokens: 0, completion_tokens: 0,
total_tokens: 0, total_tokens: 0,
api_requests: 0 api_requests: 0,
successful_requests: 0,
failed_requests: 0
}; };
} }
providerSpend[provider].spend += metrics.spend; providerSpend[provider].spend += metrics.spend;
@ -112,6 +122,8 @@ const NewUsagePage: React.FC<NewUsagePageProps> = ({
providerSpend[provider].completion_tokens += metrics.completion_tokens; providerSpend[provider].completion_tokens += metrics.completion_tokens;
providerSpend[provider].total_tokens += metrics.total_tokens; providerSpend[provider].total_tokens += metrics.total_tokens;
providerSpend[provider].api_requests += metrics.api_requests; providerSpend[provider].api_requests += metrics.api_requests;
providerSpend[provider].successful_requests += metrics.successful_requests || 0;
providerSpend[provider].failed_requests += metrics.failed_requests || 0;
}); });
}); });
@ -120,6 +132,8 @@ const NewUsagePage: React.FC<NewUsagePageProps> = ({
provider, provider,
spend: metrics.spend, spend: metrics.spend,
requests: metrics.api_requests, requests: metrics.api_requests,
successful_requests: metrics.successful_requests,
failed_requests: metrics.failed_requests,
tokens: metrics.total_tokens tokens: metrics.total_tokens
})); }));
}; };
@ -135,7 +149,9 @@ const NewUsagePage: React.FC<NewUsagePageProps> = ({
prompt_tokens: 0, prompt_tokens: 0,
completion_tokens: 0, completion_tokens: 0,
total_tokens: 0, total_tokens: 0,
api_requests: 0 api_requests: 0,
successful_requests: 0,
failed_requests: 0
}; };
} }
keySpend[key].spend += metrics.spend; keySpend[key].spend += metrics.spend;
@ -143,6 +159,8 @@ const NewUsagePage: React.FC<NewUsagePageProps> = ({
keySpend[key].completion_tokens += metrics.completion_tokens; keySpend[key].completion_tokens += metrics.completion_tokens;
keySpend[key].total_tokens += metrics.total_tokens; keySpend[key].total_tokens += metrics.total_tokens;
keySpend[key].api_requests += metrics.api_requests; keySpend[key].api_requests += metrics.api_requests;
keySpend[key].successful_requests += metrics.successful_requests;
keySpend[key].failed_requests += metrics.failed_requests;
}); });
}); });
@ -185,6 +203,7 @@ const NewUsagePage: React.FC<NewUsagePageProps> = ({
<Text className="text-tremor-default text-tremor-content dark:text-dark-tremor-content mb-2 mt-2 text-lg"> <Text className="text-tremor-default text-tremor-content dark:text-dark-tremor-content mb-2 mt-2 text-lg">
Project Spend {new Date().toLocaleString('default', { month: 'long' })} 1 - {new Date(new Date().getFullYear(), new Date().getMonth() + 1, 0).getDate()} Project Spend {new Date().toLocaleString('default', { month: 'long' })} 1 - {new Date(new Date().getFullYear(), new Date().getMonth() + 1, 0).getDate()}
</Text> </Text>
<ViewUserSpend <ViewUserSpend
userID={userID} userID={userID}
userRole={userRole} userRole={userRole}
@ -195,6 +214,44 @@ const NewUsagePage: React.FC<NewUsagePageProps> = ({
/> />
</Col> </Col>
<Col numColSpan={2}>
<Card>
<Title>Usage Metrics</Title>
<Grid numItems={5} className="gap-4 mt-4">
<Card>
<Title>Total Requests</Title>
<Text className="text-2xl font-bold mt-2">
{userSpendData.metadata?.total_api_requests?.toLocaleString() || 0}
</Text>
</Card>
<Card>
<Title>Successful Requests</Title>
<Text className="text-2xl font-bold mt-2 text-green-600">
{userSpendData.metadata?.total_successful_requests?.toLocaleString() || 0}
</Text>
</Card>
<Card>
<Title>Failed Requests</Title>
<Text className="text-2xl font-bold mt-2 text-red-600">
{userSpendData.metadata?.total_failed_requests?.toLocaleString() || 0}
</Text>
</Card>
<Card>
<Title>Total Tokens</Title>
<Text className="text-2xl font-bold mt-2">
{userSpendData.metadata?.total_tokens?.toLocaleString() || 0}
</Text>
</Card>
<Card>
<Title>Average Cost per Request</Title>
<Text className="text-2xl font-bold mt-2">
${((totalSpend || 0) / (userSpendData.metadata?.total_api_requests || 1)).toFixed(4)}
</Text>
</Card>
</Grid>
</Card>
</Col>
{/* Daily Spend Chart */} {/* Daily Spend Chart */}
<Col numColSpan={2}> <Col numColSpan={2}>
<Card> <Card>
@ -215,6 +272,8 @@ const NewUsagePage: React.FC<NewUsagePageProps> = ({
<p className="font-bold">{data.date}</p> <p className="font-bold">{data.date}</p>
<p className="text-cyan-500">Spend: ${data.metrics.spend.toFixed(2)}</p> <p className="text-cyan-500">Spend: ${data.metrics.spend.toFixed(2)}</p>
<p className="text-gray-600">Requests: {data.metrics.api_requests}</p> <p className="text-gray-600">Requests: {data.metrics.api_requests}</p>
<p className="text-gray-600">Successful: {data.metrics.successful_requests}</p>
<p className="text-gray-600">Failed: {data.metrics.failed_requests}</p>
<p className="text-gray-600">Tokens: {data.metrics.total_tokens}</p> <p className="text-gray-600">Tokens: {data.metrics.total_tokens}</p>
</div> </div>
); );
@ -240,7 +299,9 @@ const NewUsagePage: React.FC<NewUsagePageProps> = ({
{/* Top Models */} {/* Top Models */}
<Col numColSpan={1}> <Col numColSpan={1}>
<Card className="h-full"> <Card className="h-full">
<Title>Top Models</Title> <div className="flex justify-between items-center mb-4">
<Title>Top Models</Title>
</div>
<BarChart <BarChart
className="mt-4 h-40" className="mt-4 h-40"
data={getTopModels()} data={getTopModels()}
@ -258,7 +319,9 @@ const NewUsagePage: React.FC<NewUsagePageProps> = ({
<div className="bg-white p-4 shadow-lg rounded-lg border"> <div className="bg-white p-4 shadow-lg rounded-lg border">
<p className="font-bold">{data.key}</p> <p className="font-bold">{data.key}</p>
<p className="text-cyan-500">Spend: ${data.spend.toFixed(2)}</p> <p className="text-cyan-500">Spend: ${data.spend.toFixed(2)}</p>
<p className="text-gray-600">Requests: {data.requests.toLocaleString()}</p> <p className="text-gray-600">Total Requests: {data.requests.toLocaleString()}</p>
<p className="text-green-600">Successful: {data.successful_requests.toLocaleString()}</p>
<p className="text-red-600">Failed: {data.failed_requests.toLocaleString()}</p>
<p className="text-gray-600">Tokens: {data.tokens.toLocaleString()}</p> <p className="text-gray-600">Tokens: {data.tokens.toLocaleString()}</p>
</div> </div>
); );
@ -270,7 +333,9 @@ const NewUsagePage: React.FC<NewUsagePageProps> = ({
{/* Spend by Provider */} {/* Spend by Provider */}
<Col numColSpan={2}> <Col numColSpan={2}>
<Card className="h-full"> <Card className="h-full">
<Title>Spend by Provider</Title> <div className="flex justify-between items-center mb-4">
<Title>Spend by Provider</Title>
</div>
<Grid numItems={2}> <Grid numItems={2}>
<Col numColSpan={1}> <Col numColSpan={1}>
<DonutChart <DonutChart
@ -288,20 +353,28 @@ const NewUsagePage: React.FC<NewUsagePageProps> = ({
<TableRow> <TableRow>
<TableHeaderCell>Provider</TableHeaderCell> <TableHeaderCell>Provider</TableHeaderCell>
<TableHeaderCell>Spend</TableHeaderCell> <TableHeaderCell>Spend</TableHeaderCell>
<TableHeaderCell>Requests</TableHeaderCell> <TableHeaderCell className="text-green-600">Successful</TableHeaderCell>
<TableHeaderCell className="text-red-600">Failed</TableHeaderCell>
<TableHeaderCell>Tokens</TableHeaderCell> <TableHeaderCell>Tokens</TableHeaderCell>
</TableRow> </TableRow>
</TableHead> </TableHead>
<TableBody> <TableBody>
{getProviderSpend().map((provider) => ( {getProviderSpend()
<TableRow key={provider.provider}> .filter(provider => provider.spend > 0)
<TableCell>{provider.provider}</TableCell> .map((provider) => (
<TableCell> <TableRow key={provider.provider}>
${provider.spend < 0.00001 <TableCell>{provider.provider}</TableCell>
? "less than 0.00" <TableCell>
: provider.spend.toFixed(2)} ${provider.spend < 0.00001
? "less than 0.00001"
: provider.spend.toFixed(2)}
</TableCell>
<TableCell className="text-green-600">
{provider.successful_requests.toLocaleString()}
</TableCell>
<TableCell className="text-red-600">
{provider.failed_requests.toLocaleString()}
</TableCell> </TableCell>
<TableCell>{provider.requests.toLocaleString()}</TableCell>
<TableCell>{provider.tokens.toLocaleString()}</TableCell> <TableCell>{provider.tokens.toLocaleString()}</TableCell>
</TableRow> </TableRow>
))} ))}
@ -313,31 +386,7 @@ const NewUsagePage: React.FC<NewUsagePageProps> = ({
</Col> </Col>
{/* Usage Metrics */} {/* Usage Metrics */}
<Col numColSpan={2}>
<Card>
<Title>Usage Metrics</Title>
<Grid numItems={3} className="gap-4 mt-4">
<Card>
<Title>Total Requests</Title>
<Text className="text-2xl font-bold mt-2">
{userSpendData.metadata?.total_api_requests?.toLocaleString() || 0}
</Text>
</Card>
<Card>
<Title>Total Tokens</Title>
<Text className="text-2xl font-bold mt-2">
{userSpendData.metadata?.total_tokens?.toLocaleString() || 0}
</Text>
</Card>
<Card>
<Title>Average Cost per Request</Title>
<Text className="text-2xl font-bold mt-2">
${((totalSpend || 0) / (userSpendData.metadata?.total_api_requests || 1)).toFixed(4)}
</Text>
</Card>
</Grid>
</Card>
</Col>
</Grid> </Grid>
</TabPanel> </TabPanel>