phoenix/litellm-mirror
0
0
Fork 1
mirror of https://github.com/BerriAI/litellm.git synced 2025-04-27 11:43:54 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge branch 'BerriAI:main' into main

Browse source
This commit is contained in:
Shixian Sheng 2025-03-28 06:44:51 -04:00 • committed by GitHub
commit 79e98ef012
28 changed files with 1144 additions and 33 deletions
Download patch file Download diff file Expand all files Collapse all files

1
ci_cd/baseline_db.py
View file

@ -1,4 +1,3 @@
import os
import subprocess
from pathlib import Path
from datetime import datetime

3
deploy/migrations/20250327180120_add_api_requests_to_daily_user_table/migration.sql Normal file
View file

@ -0,0 +1,3 @@
-- AlterTable
ALTER TABLE "LiteLLM_DailyUserSpend" ADD COLUMN "api_requests" INTEGER NOT NULL DEFAULT 0;

14
docs/my-website/docs/set_keys.md
View file

@ -188,7 +188,13 @@ Currently implemented for:
- OpenAI (if OPENAI_API_KEY is set)
- Fireworks AI (if FIREWORKS_AI_API_KEY is set)
- LiteLLM Proxy (if LITELLM_PROXY_API_KEY is set)
- Gemini (if GEMINI_API_KEY is set)
- XAI (if XAI_API_KEY is set)
- Anthropic (if ANTHROPIC_API_KEY is set)
You can also specify a custom provider to check:
**All providers**:
```python
from litellm import get_valid_models
@ -196,6 +202,14 @@ valid_models = get_valid_models(check_provider_endpoint=True)
print(valid_models)
```
**Specific provider**:
```python
from litellm import get_valid_models
valid_models = get_valid_models(check_provider_endpoint=True, custom_llm_provider="openai")
print(valid_models)
```
### `validate_environment(model: str)`
This helper tells you if you have all the required environment variables for a model, and if not - what's missing.

3
litellm/__init__.py
View file

@ -813,6 +813,7 @@ from .llms.oobabooga.chat.transformation import OobaboogaConfig
from .llms.maritalk import MaritalkConfig
from .llms.openrouter.chat.transformation import OpenrouterConfig
from .llms.anthropic.chat.transformation import AnthropicConfig
from .llms.anthropic.common_utils import AnthropicModelInfo
from .llms.groq.stt.transformation import GroqSTTConfig
from .llms.anthropic.completion.transformation import AnthropicTextConfig
from .llms.triton.completion.transformation import TritonConfig
@ -848,6 +849,7 @@ from .llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import (
VertexGeminiConfig,
VertexGeminiConfig as VertexAIConfig,
)
from .llms.gemini.common_utils import GeminiModelInfo
from .llms.gemini.chat.transformation import (
GoogleAIStudioGeminiConfig,
GoogleAIStudioGeminiConfig as GeminiConfig, # aliased to maintain backwards compatibility
@ -984,6 +986,7 @@ from .llms.fireworks_ai.embed.fireworks_ai_transformation import (
from .llms.friendliai.chat.transformation import FriendliaiChatConfig
from .llms.jina_ai.embedding.transformation import JinaAIEmbeddingConfig
from .llms.xai.chat.transformation import XAIChatConfig
from .llms.xai.common_utils import XAIModelInfo
from .llms.volcengine import VolcEngineConfig
from .llms.codestral.completion.transformation import CodestralTextCompletionConfig
from .llms.azure.azure import (

51
litellm/llms/anthropic/common_utils.py
View file

@ -6,7 +6,10 @@ from typing import Optional, Union
import httpx
import litellm
from litellm.llms.base_llm.base_utils import BaseLLMModelInfo
from litellm.llms.base_llm.chat.transformation import BaseLLMException
from litellm.secret_managers.main import get_secret_str
class AnthropicError(BaseLLMException):
@ -19,6 +22,54 @@ class AnthropicError(BaseLLMException):
super().__init__(status_code=status_code, message=message, headers=headers)
class AnthropicModelInfo(BaseLLMModelInfo):
@staticmethod
def get_api_base(api_base: Optional[str] = None) -> Optional[str]:
return (
api_base
or get_secret_str("ANTHROPIC_API_BASE")
or "https://api.anthropic.com"
)
@staticmethod
def get_api_key(api_key: Optional[str] = None) -> Optional[str]:
return api_key or get_secret_str("ANTHROPIC_API_KEY")
@staticmethod
def get_base_model(model: Optional[str] = None) -> Optional[str]:
return model.replace("anthropic/", "") if model else None
def get_models(
self, api_key: Optional[str] = None, api_base: Optional[str] = None
) -> list[str]:
api_base = AnthropicModelInfo.get_api_base(api_base)
api_key = AnthropicModelInfo.get_api_key(api_key)
if api_base is None or api_key is None:
raise ValueError(
"ANTHROPIC_API_BASE or ANTHROPIC_API_KEY is not set. Please set the environment variable, to query Anthropic's `/models` endpoint."
)
response = litellm.module_level_client.get(
url=f"{api_base}/v1/models",
headers={"x-api-key": api_key, "anthropic-version": "2023-06-01"},
)
try:
response.raise_for_status()
except httpx.HTTPStatusError:
raise Exception(
f"Failed to fetch models from Anthropic. Status code: {response.status_code}, Response: {response.text}"
)
models = response.json()["data"]
litellm_model_names = []
for model in models:
stripped_model_name = model["id"]
litellm_model_name = "anthropic/" + stripped_model_name
litellm_model_names.append(litellm_model_name)
return litellm_model_names
def process_anthropic_headers(headers: Union[httpx.Headers, dict]) -> dict:
openai_headers = {}
if "anthropic-ratelimit-requests-limit" in headers:

12
litellm/llms/base_llm/base_utils.py
View file

@ -19,11 +19,19 @@ class BaseLLMModelInfo(ABC):
self,
model: str,
) -> Optional[ProviderSpecificModelInfo]:
"""
Default values all models of this provider support.
"""
return None
@abstractmethod
def get_models(self) -> List[str]:
pass
def get_models(
self, api_key: Optional[str] = None, api_base: Optional[str] = None
) -> List[str]:
"""
Returns a list of models supported by this provider.
"""
return []
@staticmethod
@abstractmethod

52
litellm/llms/gemini/common_utils.py Normal file
View file

@ -0,0 +1,52 @@
from typing import List, Optional
import litellm
from litellm.llms.base_llm.base_utils import BaseLLMModelInfo
from litellm.secret_managers.main import get_secret_str
class GeminiModelInfo(BaseLLMModelInfo):
@staticmethod
def get_api_base(api_base: Optional[str] = None) -> Optional[str]:
return (
api_base
or get_secret_str("GEMINI_API_BASE")
or "https://generativelanguage.googleapis.com/v1beta"
)
@staticmethod
def get_api_key(api_key: Optional[str] = None) -> Optional[str]:
return api_key or (get_secret_str("GEMINI_API_KEY"))
@staticmethod
def get_base_model(model: str) -> Optional[str]:
return model.replace("gemini/", "")
def get_models(
self, api_key: Optional[str] = None, api_base: Optional[str] = None
) -> List[str]:
api_base = GeminiModelInfo.get_api_base(api_base)
api_key = GeminiModelInfo.get_api_key(api_key)
if api_base is None or api_key is None:
raise ValueError(
"GEMINI_API_BASE or GEMINI_API_KEY is not set. Please set the environment variable, to query Gemini's `/models` endpoint."
)
response = litellm.module_level_client.get(
url=f"{api_base}/models?key={api_key}",
)
if response.status_code != 200:
raise ValueError(
f"Failed to fetch models from Gemini. Status code: {response.status_code}, Response: {response.json()}"
)
models = response.json()["models"]
litellm_model_names = []
for model in models:
stripped_model_name = model["name"].strip("models/")
litellm_model_name = "gemini/" + stripped_model_name
litellm_model_names.append(litellm_model_name)
return litellm_model_names

5
litellm/llms/mistral/mistral_chat_transformation.py
View file

@ -80,6 +80,7 @@ class MistralConfig(OpenAIGPTConfig):
"temperature",
"top_p",
"max_tokens",
"max_completion_tokens",
"tools",
"tool_choice",
"seed",
@ -105,6 +106,10 @@ class MistralConfig(OpenAIGPTConfig):
for param, value in non_default_params.items():
if param == "max_tokens":
optional_params["max_tokens"] = value
if (
param == "max_completion_tokens"
): # max_completion_tokens should take priority
optional_params["max_tokens"] = value
if param == "tools":
optional_params["tools"] = value
if param == "stream" and value is True:

4
litellm/llms/topaz/common_utils.py
View file

@ -11,7 +11,9 @@ class TopazException(BaseLLMException):
class TopazModelInfo(BaseLLMModelInfo):
def get_models(self) -> List[str]:
def get_models(
self, api_key: Optional[str] = None, api_base: Optional[str] = None
) -> List[str]:
return [
"topaz/Standard V2",
"topaz/Low Resolution V2",

51
litellm/llms/xai/common_utils.py Normal file
View file

@ -0,0 +1,51 @@
from typing import Optional
import httpx
import litellm
from litellm.llms.base_llm.base_utils import BaseLLMModelInfo
from litellm.secret_managers.main import get_secret_str
class XAIModelInfo(BaseLLMModelInfo):
@staticmethod
def get_api_base(api_base: Optional[str] = None) -> Optional[str]:
return api_base or get_secret_str("XAI_API_BASE") or "https://api.x.ai"
@staticmethod
def get_api_key(api_key: Optional[str] = None) -> Optional[str]:
return api_key or get_secret_str("XAI_API_KEY")
@staticmethod
def get_base_model(model: str) -> Optional[str]:
return model.replace("xai/", "")
def get_models(
self, api_key: Optional[str] = None, api_base: Optional[str] = None
) -> list[str]:
api_base = self.get_api_base(api_base)
api_key = self.get_api_key(api_key)
if api_base is None or api_key is None:
raise ValueError(
"XAI_API_BASE or XAI_API_KEY is not set. Please set the environment variable, to query XAI's `/models` endpoint."
)
response = litellm.module_level_client.get(
url=f"{api_base}/v1/models",
headers={"Authorization": f"Bearer {api_key}"},
)
try:
response.raise_for_status()
except httpx.HTTPStatusError:
raise Exception(
f"Failed to fetch models from XAI. Status code: {response.status_code}, Response: {response.text}"
)
models = response.json()["data"]
litellm_model_names = []
for model in models:
stripped_model_name = model["id"]
litellm_model_name = "xai/" + stripped_model_name
litellm_model_names.append(litellm_model_name)
return litellm_model_names

1
litellm/proxy/_types.py
View file

@ -2718,3 +2718,4 @@ class DailyUserSpendTransaction(TypedDict):
prompt_tokens: int
completion_tokens: int
spend: float
api_requests: int

94
litellm/proxy/db/prisma_client.py
View file

@ -3,6 +3,7 @@ This file contains the PrismaWrapper class, which is used to wrap the Prisma cli
"""
import asyncio
import glob
import os
import random
import subprocess
@ -178,6 +179,69 @@ class PrismaManager:
verbose_proxy_logger.warning(f"Error creating baseline migration: {e}")
return False
@staticmethod
def _copy_spend_tracking_migrations(prisma_dir: str) -> bool:
import shutil
from pathlib import Path
"""
Check for and copy over spend tracking migrations if they exist in the deploy directory.
Returns True if migrations were found and copied, False otherwise.
"""
try:
# Get the current file's directory
current_dir = Path(__file__).parent
# Check for migrations in the deploy directory (../../deploy/migrations)
deploy_migrations_dir = (
current_dir.parent.parent.parent / "deploy" / "migrations"
)
# Local migrations directory
local_migrations_dir = Path(prisma_dir + "/migrations")
if deploy_migrations_dir.exists():
# Create local migrations directory if it doesn't exist
local_migrations_dir.mkdir(parents=True, exist_ok=True)
# Copy all migration files
# Copy entire migrations folder recursively
shutil.copytree(
deploy_migrations_dir, local_migrations_dir, dirs_exist_ok=True
)
return True
return False
except Exception:
return False
@staticmethod
def _get_migration_names(migrations_dir: str) -> list:
"""Get all migration directory names from the migrations folder"""
migration_paths = glob.glob(f"{migrations_dir}/*/migration.sql")
return [Path(p).parent.name for p in migration_paths]
@staticmethod
def _resolve_all_migrations(migrations_dir: str):
"""Mark all existing migrations as applied"""
migration_names = PrismaManager._get_migration_names(migrations_dir)
for migration_name in migration_names:
try:
verbose_proxy_logger.info(f"Resolving migration: {migration_name}")
subprocess.run(
["prisma", "migrate", "resolve", "--applied", migration_name],
timeout=60,
check=True,
capture_output=True,
text=True,
)
verbose_proxy_logger.debug(f"Resolved migration: {migration_name}")
except subprocess.CalledProcessError as e:
if "is already recorded as applied in the database." not in e.stderr:
verbose_proxy_logger.warning(
f"Failed to resolve migration {migration_name}: {e.stderr}"
)
@staticmethod
def setup_database(use_migrate: bool = False) -> bool:
"""
@ -194,8 +258,10 @@ class PrismaManager:
os.chdir(prisma_dir)
try:
if use_migrate:
PrismaManager._copy_spend_tracking_migrations(
prisma_dir
) # place a migration in the migrations directory
verbose_proxy_logger.info("Running prisma migrate deploy")
# First try to run migrate deploy directly
try:
subprocess.run(
["prisma", "migrate", "deploy"],
@ -205,25 +271,31 @@ class PrismaManager:
text=True,
)
verbose_proxy_logger.info("prisma migrate deploy completed")
# Resolve all migrations in the migrations directory
migrations_dir = os.path.join(prisma_dir, "migrations")
PrismaManager._resolve_all_migrations(migrations_dir)
return True
except subprocess.CalledProcessError as e:
# Check if this is the non-empty schema error
verbose_proxy_logger.warning(
f"prisma db error: {e.stderr}, e: {e.stdout}"
)
if (
"P3005" in e.stderr
and "database schema is not empty" in e.stderr
):
# Create baseline migration
verbose_proxy_logger.info("Creating baseline migration")
if PrismaManager._create_baseline_migration(schema_path):
# Try migrate deploy again after baseline
subprocess.run(
["prisma", "migrate", "deploy"],
timeout=60,
check=True,
verbose_proxy_logger.info(
"Resolving all migrations after baseline"
)
# Resolve all migrations after baseline
migrations_dir = os.path.join(prisma_dir, "migrations")
PrismaManager._resolve_all_migrations(migrations_dir)
return True
else:
# If it's a different error, raise it
raise e
else:
# Use prisma db push with increased timeout
subprocess.run(

293
litellm/proxy/management_endpoints/internal_user_endpoints.py
View file

@ -14,8 +14,9 @@ These are members of a Team on LiteLLM
import asyncio
import traceback
import uuid
from datetime import datetime, timedelta, timezone
from typing import Any, List, Optional, Union, cast
from datetime import date, datetime, timedelta, timezone
from enum import Enum
from typing import Any, Dict, List, Optional, TypedDict, Union, cast
import fastapi
from fastapi import APIRouter, Depends, Header, HTTPException, Request, status
@ -1242,3 +1243,291 @@ async def ui_view_users(
except Exception as e:
verbose_proxy_logger.exception(f"Error searching users: {str(e)}")
raise HTTPException(status_code=500, detail=f"Error searching users: {str(e)}")
class GroupByDimension(str, Enum):
DATE = "date"
MODEL = "model"
API_KEY = "api_key"
TEAM = "team"
ORGANIZATION = "organization"
MODEL_GROUP = "model_group"
PROVIDER = "custom_llm_provider"
class SpendMetrics(BaseModel):
spend: float = Field(default=0.0)
prompt_tokens: int = Field(default=0)
completion_tokens: int = Field(default=0)
total_tokens: int = Field(default=0)
api_requests: int = Field(default=0)
class BreakdownMetrics(BaseModel):
"""Breakdown of spend by different dimensions"""
models: Dict[str, SpendMetrics] = Field(default_factory=dict) # model -> metrics
providers: Dict[str, SpendMetrics] = Field(
default_factory=dict
) # provider -> metrics
api_keys: Dict[str, SpendMetrics] = Field(
default_factory=dict
) # api_key -> metrics
class DailySpendData(BaseModel):
date: date
metrics: SpendMetrics
breakdown: BreakdownMetrics = Field(default_factory=BreakdownMetrics)
class DailySpendMetadata(BaseModel):
total_spend: float = Field(default=0.0)
total_prompt_tokens: int = Field(default=0)
total_completion_tokens: int = Field(default=0)
total_api_requests: int = Field(default=0)
page: int = Field(default=1)
total_pages: int = Field(default=1)
has_more: bool = Field(default=False)
class SpendAnalyticsPaginatedResponse(BaseModel):
results: List[DailySpendData]
metadata: DailySpendMetadata = Field(default_factory=DailySpendMetadata)
class LiteLLM_DailyUserSpend(BaseModel):
id: str
user_id: str
date: str
api_key: str
model: str
model_group: Optional[str] = None
custom_llm_provider: Optional[str] = None
prompt_tokens: int = 0
completion_tokens: int = 0
spend: float = 0.0
api_requests: int = 0
class GroupedData(TypedDict):
metrics: SpendMetrics
breakdown: BreakdownMetrics
def update_metrics(
group_metrics: SpendMetrics, record: LiteLLM_DailyUserSpend
) -> SpendMetrics:
group_metrics.spend += record.spend
group_metrics.prompt_tokens += record.prompt_tokens
group_metrics.completion_tokens += record.completion_tokens
group_metrics.total_tokens += record.prompt_tokens + record.completion_tokens
group_metrics.api_requests += record.api_requests
return group_metrics
def update_breakdown_metrics(
breakdown: BreakdownMetrics, record: LiteLLM_DailyUserSpend
) -> BreakdownMetrics:
"""Updates breakdown metrics for a single record using the existing update_metrics function"""
# Update model breakdown
if record.model not in breakdown.models:
breakdown.models[record.model] = SpendMetrics()
breakdown.models[record.model] = update_metrics(
breakdown.models[record.model], record
)
# Update provider breakdown
provider = record.custom_llm_provider or "unknown"
if provider not in breakdown.providers:
breakdown.providers[provider] = SpendMetrics()
breakdown.providers[provider] = update_metrics(
breakdown.providers[provider], record
)
# Update api key breakdown
if record.api_key not in breakdown.api_keys:
breakdown.api_keys[record.api_key] = SpendMetrics()
breakdown.api_keys[record.api_key] = update_metrics(
breakdown.api_keys[record.api_key], record
)
return breakdown
@router.get(
"/user/daily/activity",
tags=["Budget & Spend Tracking", "Internal User management"],
dependencies=[Depends(user_api_key_auth)],
response_model=SpendAnalyticsPaginatedResponse,
)
async def get_user_daily_activity(
start_date: Optional[str] = fastapi.Query(
default=None,
description="Start date in YYYY-MM-DD format",
),
end_date: Optional[str] = fastapi.Query(
default=None,
description="End date in YYYY-MM-DD format",
),
group_by: List[GroupByDimension] = fastapi.Query(
default=[GroupByDimension.DATE],
description="Dimensions to group by. Can combine multiple (e.g. date,team)",
),
view_by: Literal["team", "organization", "user"] = fastapi.Query(
default="user",
description="View spend at team/org/user level",
),
team_id: Optional[str] = fastapi.Query(
default=None,
description="Filter by specific team",
),
org_id: Optional[str] = fastapi.Query(
default=None,
description="Filter by specific organization",
),
model: Optional[str] = fastapi.Query(
default=None,
description="Filter by specific model",
),
api_key: Optional[str] = fastapi.Query(
default=None,
description="Filter by specific API key",
),
page: int = fastapi.Query(
default=1, description="Page number for pagination", ge=1
),
page_size: int = fastapi.Query(
default=50, description="Items per page", ge=1, le=100
),
user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth),
) -> SpendAnalyticsPaginatedResponse:
"""
[BETA] This is a beta endpoint. It will change.
Meant to optimize querying spend data for analytics for a user.
Returns:
(by date/team/org/user/model/api_key/model_group/provider)
- spend
- prompt_tokens
- completion_tokens
- total_tokens
- api_requests
- breakdown by team, organization, user, model, api_key, model_group, provider
"""
from litellm.proxy.proxy_server import prisma_client
if prisma_client is None:
raise HTTPException(
status_code=500,
detail={"error": CommonProxyErrors.db_not_connected_error.value},
)
if start_date is None or end_date is None:
raise HTTPException(
status_code=status.HTTP_400_BAD_REQUEST,
detail={"error": "Please provide start_date and end_date"},
)
try:
# Build filter conditions
where_conditions: Dict[str, Any] = {
"date": {
"gte": start_date,
"lte": end_date,
}
}
if team_id:
where_conditions["team_id"] = team_id
if org_id:
where_conditions["organization_id"] = org_id
if model:
where_conditions["model"] = model
if api_key:
where_conditions["api_key"] = api_key
# Get total count for pagination
total_count = await prisma_client.db.litellm_dailyuserspend.count(
where=where_conditions
)
# Fetch paginated results
daily_spend_data = await prisma_client.db.litellm_dailyuserspend.find_many(
where=where_conditions,
order=[
{"date": "desc"},
],
skip=(page - 1) * page_size,
take=page_size,
)
# Process results
results = []
total_metrics = SpendMetrics()
# Group data by date and other dimensions
grouped_data: Dict[str, Dict[str, Any]] = {}
for record in daily_spend_data:
date_str = record.date
if date_str not in grouped_data:
grouped_data[date_str] = {
"metrics": SpendMetrics(),
"breakdown": BreakdownMetrics(),
}
# Update metrics
grouped_data[date_str]["metrics"] = update_metrics(
grouped_data[date_str]["metrics"], record
)
# Update breakdowns
grouped_data[date_str]["breakdown"] = update_breakdown_metrics(
grouped_data[date_str]["breakdown"], record
)
# Update total metrics
total_metrics.spend += record.spend
total_metrics.prompt_tokens += record.prompt_tokens
total_metrics.completion_tokens += record.completion_tokens
total_metrics.total_tokens += (
record.prompt_tokens + record.completion_tokens
)
total_metrics.api_requests += 1
# Convert grouped data to response format
for date_str, data in grouped_data.items():
results.append(
DailySpendData(
date=datetime.strptime(date_str, "%Y-%m-%d").date(),
metrics=data["metrics"],
breakdown=data["breakdown"],
)
)
# Sort results by date
results.sort(key=lambda x: x.date, reverse=True)
return SpendAnalyticsPaginatedResponse(
results=results,
metadata=DailySpendMetadata(
total_spend=total_metrics.spend,
total_prompt_tokens=total_metrics.prompt_tokens,
total_completion_tokens=total_metrics.completion_tokens,
total_api_requests=total_metrics.api_requests,
page=page,
total_pages=-(-total_count // page_size), # Ceiling division
has_more=(page * page_size) < total_count,
),
)
except Exception as e:
verbose_proxy_logger.exception(
"/spend/daily/analytics: Exception occured - {}".format(str(e))
)
raise HTTPException(
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
detail={"error": f"Failed to fetch analytics: {str(e)}"},
)

4
litellm/proxy/proxy_server.py
View file

@ -509,9 +509,6 @@ async def proxy_startup_event(app: FastAPI):
if isinstance(worker_config, dict):
await initialize(**worker_config)
### LOAD MASTER KEY ###
# check if master key set in environment - load from there
master_key = get_secret("LITELLM_MASTER_KEY", None) # type: ignore
# check if DATABASE_URL in environment - load from there
if prisma_client is None:
_db_url: Optional[str] = get_secret("DATABASE_URL", None) # type: ignore
@ -1974,6 +1971,7 @@ class ProxyConfig:
if master_key and master_key.startswith("os.environ/"):
master_key = get_secret(master_key) # type: ignore
if not isinstance(master_key, str):
raise Exception(
"Master key must be a string. Current type - {}".format(

10
litellm/proxy/schema.prisma
View file

@ -314,19 +314,19 @@ model LiteLLM_AuditLog {
updated_values Json? // value of the row after change
}
// Track daily user spend metrics per model and key
model LiteLLM_DailyUserSpend {
id String @id @default(uuid())
user_id String
date String
api_key String // Hashed API Token
model String // The specific model used
model_group String? // public model_name / model_group
custom_llm_provider String? // The LLM provider (e.g., "openai", "anthropic")
api_key String
model String
model_group String?
custom_llm_provider String?
prompt_tokens Int @default(0)
completion_tokens Int @default(0)
spend Float @default(0.0)
api_requests Int @default(0)
created_at DateTime @default(now())
updated_at DateTime @updatedAt

2
litellm/proxy/spend_tracking/spend_management_endpoints.py
View file

@ -3,7 +3,7 @@ import collections
import os
from datetime import datetime, timedelta, timezone
from functools import lru_cache
from typing import TYPE_CHECKING, Any, List, Optional
from typing import TYPE_CHECKING, Any, Dict, List, Literal, Optional
import fastapi
from fastapi import APIRouter, Depends, HTTPException, status

6
litellm/proxy/utils.py
View file

@ -1179,6 +1179,7 @@ class PrismaClient:
daily_transaction["spend"] += payload["spend"]
daily_transaction["prompt_tokens"] += payload["prompt_tokens"]
daily_transaction["completion_tokens"] += payload["completion_tokens"]
daily_transaction["api_requests"] += 1
else:
daily_transaction = DailyUserSpendTransaction(
user_id=payload["user"],
@ -1190,6 +1191,7 @@ class PrismaClient:
prompt_tokens=payload["prompt_tokens"],
completion_tokens=payload["completion_tokens"],
spend=payload["spend"],
api_requests=1,
)
self.daily_user_spend_transactions[daily_transaction_key] = (
@ -2598,6 +2600,7 @@ class ProxyUpdateSpend:
"completion_tokens"
],
"spend": transaction["spend"],
"api_requests": transaction["api_requests"],
},
"update": {
"prompt_tokens": {
@ -2609,6 +2612,9 @@ class ProxyUpdateSpend:
]
},
"spend": {"increment": transaction["spend"]},
"api_requests": {
"increment": transaction["api_requests"]
},
},
},
)

24
litellm/utils.py
View file

@ -5744,13 +5744,15 @@ def trim_messages(
return messages
def get_valid_models(check_provider_endpoint: bool = False) -> List[str]:
def get_valid_models(
check_provider_endpoint: bool = False, custom_llm_provider: Optional[str] = None
) -> List[str]:
"""
Returns a list of valid LLMs based on the set environment variables
Args:
check_provider_endpoint: If True, will check the provider's endpoint for valid models.
custom_llm_provider: If provided, will only check the provider's endpoint for valid models.
Returns:
A list of valid LLMs
"""
@ -5762,6 +5764,9 @@ def get_valid_models(check_provider_endpoint: bool = False) -> List[str]:
valid_models = []
for provider in litellm.provider_list:
if custom_llm_provider and provider != custom_llm_provider:
continue
# edge case litellm has together_ai as a provider, it should be togetherai
env_provider_1 = provider.replace("_", "")
env_provider_2 = provider
@ -5783,10 +5788,17 @@ def get_valid_models(check_provider_endpoint: bool = False) -> List[str]:
provider=LlmProviders(provider),
)
if custom_llm_provider and provider != custom_llm_provider:
continue
if provider == "azure":
valid_models.append("Azure-LLM")
elif provider_config is not None and check_provider_endpoint:
valid_models.extend(provider_config.get_models())
try:
models = provider_config.get_models()
valid_models.extend(models)
except Exception as e:
verbose_logger.debug(f"Error getting valid models: {e}")
else:
models_for_provider = litellm.models_by_provider.get(provider, [])
valid_models.extend(models_for_provider)
@ -6400,10 +6412,16 @@ class ProviderConfigManager:
return litellm.FireworksAIConfig()
elif LlmProviders.OPENAI == provider:
return litellm.OpenAIGPTConfig()
elif LlmProviders.GEMINI == provider:
return litellm.GeminiModelInfo()
elif LlmProviders.LITELLM_PROXY == provider:
return litellm.LiteLLMProxyChatConfig()
elif LlmProviders.TOPAZ == provider:
return litellm.TopazModelInfo()
elif LlmProviders.ANTHROPIC == provider:
return litellm.AnthropicModelInfo()
elif LlmProviders.XAI == provider:
return litellm.XAIModelInfo()
return None

1
schema.prisma
View file

@ -326,6 +326,7 @@ model LiteLLM_DailyUserSpend {
prompt_tokens Int @default(0)
completion_tokens Int @default(0)
spend Float @default(0.0)
api_requests Int @default(0)
created_at DateTime @default(now())
updated_at DateTime @updatedAt

18
tests/litellm_utils_tests/test_utils.py
View file

@ -303,6 +303,24 @@ def test_aget_valid_models():
os.environ = old_environ
@pytest.mark.parametrize("custom_llm_provider", ["gemini", "anthropic", "xai"])
def test_get_valid_models_with_custom_llm_provider(custom_llm_provider):
from litellm.utils import ProviderConfigManager
from litellm.types.utils import LlmProviders
provider_config = ProviderConfigManager.get_provider_model_info(
model=None,
provider=LlmProviders(custom_llm_provider),
)
assert provider_config is not None
valid_models = get_valid_models(
check_provider_endpoint=True, custom_llm_provider=custom_llm_provider
)
print(valid_models)
assert len(valid_models) > 0
assert provider_config.get_models() == valid_models
# test_get_valid_models()

5
tests/proxy_unit_tests/test_key_generate_prisma.py
View file

@ -3883,8 +3883,11 @@ async def test_get_paginated_teams(prisma_client):
@pytest.mark.asyncio
@pytest.mark.flaky(reruns=3)
@pytest.mark.flaky(retries=3, delay=1)
@pytest.mark.parametrize("entity_type", ["key", "user", "team"])
@pytest.mark.skip(
reason="Skipping reset budget job test. Fails on ci/cd due to db timeout errors. Need to replace with mock db."
)
async def test_reset_budget_job(prisma_client, entity_type):
"""
Test that the ResetBudgetJob correctly resets budgets for keys, users, and teams.

1
tests/proxy_unit_tests/test_update_spend.py
View file

@ -28,6 +28,7 @@ class MockPrismaClient:
self.team_list_transactons = {}
self.team_member_list_transactons = {}
self.org_list_transactons = {}
self.daily_user_spend_transactions = {}
def jsonify_object(self, obj):
return obj

2
tests/test_end_users.py
View file

@ -160,7 +160,7 @@ async def test_end_user_new():
@pytest.mark.asyncio
async def test_end_user_specific_region():
async def test_aaaend_user_specific_region():
"""
- Specify region user can make calls in
- Make a generic call

10
ui/litellm-dashboard/src/app/page.tsx
View file

@ -20,6 +20,7 @@ import PassThroughSettings from "@/components/pass_through_settings";
import BudgetPanel from "@/components/budgets/budget_panel";
import SpendLogsTable from "@/components/view_logs";
import ModelHub from "@/components/model_hub";
import NewUsagePage from "@/components/new_usage";
import APIRef from "@/components/api_ref";
import ChatUI from "@/components/chat_ui";
import Sidebar from "@/components/leftnav";
@ -346,7 +347,14 @@ export default function CreateKeyPage() {
accessToken={accessToken}
allTeams={teams as Team[] ?? []}
/>
) : (
) : page == "new_usage" ? (
<NewUsagePage
userID={userID}
userRole={userRole}
accessToken={accessToken}
/>
) :
(
<Usage
userID={userID}
userRole={userRole}

1
ui/litellm-dashboard/src/components/leftnav.tsx
View file

@ -68,6 +68,7 @@ const menuItems: MenuItem[] = [
{ key: "9", page: "caching", label: "Caching", icon: <DatabaseOutlined />, roles: all_admin_roles },
{ key: "10", page: "budgets", label: "Budgets", icon: <BankOutlined />, roles: all_admin_roles },
{ key: "11", page: "guardrails", label: "Guardrails", icon: <SafetyOutlined />, roles: all_admin_roles },
{ key: "12", page: "new_usage", label: "New Usage", icon: <BarChartOutlined />, roles: all_admin_roles },
]
},
{

36
ui/litellm-dashboard/src/components/networking.tsx
View file

@ -1070,6 +1070,42 @@ export const organizationDeleteCall = async (
}
};
export const userDailyActivityCall = async (accessToken: String, startTime: Date, endTime: Date) => {
/**
* Get daily user activity on proxy
*/
try {
let url = proxyBaseUrl ? `${proxyBaseUrl}/user/daily/activity` : `/user/daily/activity`;
const queryParams = new URLSearchParams();
queryParams.append('start_date', startTime.toISOString());
queryParams.append('end_date', endTime.toISOString());
const queryString = queryParams.toString();
if (queryString) {
url += `?${queryString}`;
}
const response = await fetch(url, {
method: "GET",
headers: {
[globalLitellmHeaderName]: `Bearer ${accessToken}`,
"Content-Type": "application/json",
},
});
if (!response.ok) {
const errorData = await response.text();
handleError(errorData);
throw new Error("Network response was not ok");
}
const data = await response.json();
return data;
} catch (error) {
console.error("Failed to create key:", error);
throw error;
}
};
export const getTotalSpendCall = async (accessToken: String) => {
/**
* Get all models on proxy

471
ui/litellm-dashboard/src/components/new_usage.tsx Normal file
View file

@ -0,0 +1,471 @@
/**
* New Usage Page
*
* Uses the new `/user/daily/activity` endpoint to get daily activity data for a user.
*
* Works at 1m+ spend logs, by querying an aggregate table instead.
*/
import React, { useState, useEffect } from "react";
import {
BarChart, Card, Title, Text,
Grid, Col, TabGroup, TabList, Tab,
TabPanel, TabPanels, DonutChart,
Table, TableHead, TableRow,
TableHeaderCell, TableBody, TableCell,
Subtitle
} from "@tremor/react";
import { AreaChart } from "@tremor/react";
import { userDailyActivityCall } from "./networking";
import ViewUserSpend from "./view_user_spend";
import TopKeyView from "./top_key_view";
interface NewUsagePageProps {
accessToken: string | null;
userRole: string | null;
userID: string | null;
}
interface SpendMetrics {
spend: number;
prompt_tokens: number;
completion_tokens: number;
total_tokens: number;
api_requests: number;
}
interface BreakdownMetrics {
models: { [key: string]: SpendMetrics };
providers: { [key: string]: SpendMetrics };
api_keys: { [key: string]: SpendMetrics };
}
interface DailyData {
date: string;
metrics: SpendMetrics;
breakdown: BreakdownMetrics;
}
const NewUsagePage: React.FC<NewUsagePageProps> = ({
accessToken,
userRole,
userID,
}) => {
const [userSpendData, setUserSpendData] = useState<{
results: DailyData[];
metadata: any;
}>({ results: [], metadata: {} });
// Derived states from userSpendData
const totalSpend = userSpendData.metadata?.total_spend || 0;
// Calculate top models from the breakdown data
const getTopModels = () => {
const modelSpend: { [key: string]: SpendMetrics } = {};
userSpendData.results.forEach(day => {
Object.entries(day.breakdown.models || {}).forEach(([model, metrics]) => {
if (!modelSpend[model]) {
modelSpend[model] = {
spend: 0,
prompt_tokens: 0,
completion_tokens: 0,
total_tokens: 0,
api_requests: 0
};
}
modelSpend[model].spend += metrics.spend;
modelSpend[model].prompt_tokens += metrics.prompt_tokens;
modelSpend[model].completion_tokens += metrics.completion_tokens;
modelSpend[model].total_tokens += metrics.total_tokens;
modelSpend[model].api_requests += metrics.api_requests;
});
});
return Object.entries(modelSpend)
.map(([model, metrics]) => ({
key: model,
spend: metrics.spend,
requests: metrics.api_requests,
tokens: metrics.total_tokens
}))
.sort((a, b) => b.spend - a.spend)
.slice(0, 5);
};
// Calculate provider spend from the breakdown data
const getProviderSpend = () => {
const providerSpend: { [key: string]: SpendMetrics } = {};
userSpendData.results.forEach(day => {
Object.entries(day.breakdown.providers || {}).forEach(([provider, metrics]) => {
if (!providerSpend[provider]) {
providerSpend[provider] = {
spend: 0,
prompt_tokens: 0,
completion_tokens: 0,
total_tokens: 0,
api_requests: 0
};
}
providerSpend[provider].spend += metrics.spend;
providerSpend[provider].prompt_tokens += metrics.prompt_tokens;
providerSpend[provider].completion_tokens += metrics.completion_tokens;
providerSpend[provider].total_tokens += metrics.total_tokens;
providerSpend[provider].api_requests += metrics.api_requests;
});
});
return Object.entries(providerSpend)
.map(([provider, metrics]) => ({
provider,
spend: metrics.spend,
requests: metrics.api_requests,
tokens: metrics.total_tokens
}));
};
// Calculate top API keys from the breakdown data
const getTopKeys = () => {
const keySpend: { [key: string]: SpendMetrics } = {};
userSpendData.results.forEach(day => {
Object.entries(day.breakdown.api_keys || {}).forEach(([key, metrics]) => {
if (!keySpend[key]) {
keySpend[key] = {
spend: 0,
prompt_tokens: 0,
completion_tokens: 0,
total_tokens: 0,
api_requests: 0
};
}
keySpend[key].spend += metrics.spend;
keySpend[key].prompt_tokens += metrics.prompt_tokens;
keySpend[key].completion_tokens += metrics.completion_tokens;
keySpend[key].total_tokens += metrics.total_tokens;
keySpend[key].api_requests += metrics.api_requests;
});
});
return Object.entries(keySpend)
.map(([api_key, metrics]) => ({
api_key,
key_alias: api_key.substring(0, 10), // Using truncated key as alias
spend: metrics.spend,
}))
.sort((a, b) => b.spend - a.spend)
.slice(0, 5);
};
const fetchUserSpendData = async () => {
if (!accessToken) return;
const startTime = new Date(Date.now() - 28 * 24 * 60 * 60 * 1000);
const endTime = new Date();
const data = await userDailyActivityCall(accessToken, startTime, endTime);
setUserSpendData(data);
};
useEffect(() => {
fetchUserSpendData();
}, [accessToken]);
return (
<div style={{ width: "100%" }} className="p-8">
<Text>Experimental Usage page, using new `/user/daily/activity` endpoint.</Text>
<TabGroup>
<TabList variant="solid" className="mt-1">
<Tab>Cost</Tab>
<Tab>Activity</Tab>
</TabList>
<TabPanels>
{/* Cost Panel */}
<TabPanel>
<Grid numItems={2} className="gap-2 h-[100vh] w-full">
{/* Total Spend Card */}
<Col numColSpan={2}>
<Text className="text-tremor-default text-tremor-content dark:text-dark-tremor-content mb-2 mt-2 text-lg">
Project Spend {new Date().toLocaleString('default', { month: 'long' })} 1 - {new Date(new Date().getFullYear(), new Date().getMonth() + 1, 0).getDate()}
</Text>
<ViewUserSpend
userID={userID}
userRole={userRole}
accessToken={accessToken}
userSpend={totalSpend}
selectedTeam={null}
userMaxBudget={null}
/>
</Col>
{/* Daily Spend Chart */}
<Col numColSpan={2}>
<Card>
<Title>Daily Spend</Title>
<BarChart
data={userSpendData.results}
index="date"
categories={["metrics.spend"]}
colors={["cyan"]}
valueFormatter={(value) => `$${value.toFixed(2)}`}
yAxisWidth={100}
showLegend={false}
customTooltip={({ payload, active }) => {
if (!active || !payload?.[0]) return null;
const data = payload[0].payload;
return (
<div className="bg-white p-4 shadow-lg rounded-lg border">
<p className="font-bold">{data.date}</p>
<p className="text-cyan-500">Spend: ${data.metrics.spend.toFixed(2)}</p>
<p className="text-gray-600">Requests: {data.metrics.api_requests}</p>
<p className="text-gray-600">Tokens: {data.metrics.total_tokens}</p>
</div>
);
}}
/>
</Card>
</Col>
{/* Top API Keys */}
<Col numColSpan={1}>
<Card className="h-full">
<Title>Top API Keys</Title>
<TopKeyView
topKeys={getTopKeys()}
accessToken={accessToken}
userID={userID}
userRole={userRole}
teams={null}
/>
</Card>
</Col>
{/* Top Models */}
<Col numColSpan={1}>
<Card className="h-full">
<Title>Top Models</Title>
<BarChart
className="mt-4 h-40"
data={getTopModels()}
index="key"
categories={["spend"]}
colors={["cyan"]}
valueFormatter={(value) => `$${value.toFixed(2)}`}
layout="vertical"
yAxisWidth={200}
showLegend={false}
customTooltip={({ payload, active }) => {
if (!active || !payload?.[0]) return null;
const data = payload[0].payload;
return (
<div className="bg-white p-4 shadow-lg rounded-lg border">
<p className="font-bold">{data.key}</p>
<p className="text-cyan-500">Spend: ${data.spend.toFixed(2)}</p>
<p className="text-gray-600">Requests: {data.requests.toLocaleString()}</p>
<p className="text-gray-600">Tokens: {data.tokens.toLocaleString()}</p>
</div>
);
}}
/>
</Card>
</Col>
{/* Spend by Provider */}
<Col numColSpan={2}>
<Card className="h-full">
<Title>Spend by Provider</Title>
<Grid numItems={2}>
<Col numColSpan={1}>
<DonutChart
className="mt-4 h-40"
data={getProviderSpend()}
index="provider"
category="spend"
valueFormatter={(value) => `$${value.toFixed(2)}`}
colors={["cyan"]}
/>
</Col>
<Col numColSpan={1}>
<Table>
<TableHead>
<TableRow>
<TableHeaderCell>Provider</TableHeaderCell>
<TableHeaderCell>Spend</TableHeaderCell>
<TableHeaderCell>Requests</TableHeaderCell>
<TableHeaderCell>Tokens</TableHeaderCell>
</TableRow>
</TableHead>
<TableBody>
{getProviderSpend().map((provider) => (
<TableRow key={provider.provider}>
<TableCell>{provider.provider}</TableCell>
<TableCell>
${provider.spend < 0.00001
? "less than 0.00"
: provider.spend.toFixed(2)}
</TableCell>
<TableCell>{provider.requests.toLocaleString()}</TableCell>
<TableCell>{provider.tokens.toLocaleString()}</TableCell>
</TableRow>
))}
</TableBody>
</Table>
</Col>
</Grid>
</Card>
</Col>
{/* Usage Metrics */}
<Col numColSpan={2}>
<Card>
<Title>Usage Metrics</Title>
<Grid numItems={3} className="gap-4 mt-4">
<Card>
<Title>Total Requests</Title>
<Text className="text-2xl font-bold mt-2">
{userSpendData.metadata?.total_api_requests?.toLocaleString() || 0}
</Text>
</Card>
<Card>
<Title>Total Tokens</Title>
<Text className="text-2xl font-bold mt-2">
{userSpendData.metadata?.total_tokens?.toLocaleString() || 0}
</Text>
</Card>
<Card>
<Title>Average Cost per Request</Title>
<Text className="text-2xl font-bold mt-2">
${((totalSpend || 0) / (userSpendData.metadata?.total_api_requests || 1)).toFixed(4)}
</Text>
</Card>
</Grid>
</Card>
</Col>
</Grid>
</TabPanel>
{/* Activity Panel */}
<TabPanel>
<Grid numItems={1} className="gap-2 h-[75vh] w-full">
<Card>
<Title>All Up</Title>
<Grid numItems={2}>
<Col>
<Subtitle style={{ fontSize: "15px", fontWeight: "normal", color: "#535452"}}>
API Requests {valueFormatterNumbers(userSpendData.metadata?.total_api_requests || 0)}
</Subtitle>
<AreaChart
className="h-40"
data={[...userSpendData.results].reverse()}
valueFormatter={valueFormatterNumbers}
index="date"
colors={['cyan']}
categories={['metrics.api_requests']}
/>
</Col>
<Col>
<Subtitle style={{ fontSize: "15px", fontWeight: "normal", color: "#535452"}}>
Tokens {valueFormatterNumbers(userSpendData.metadata?.total_tokens || 0)}
</Subtitle>
<BarChart
className="h-40"
data={[...userSpendData.results].reverse()}
valueFormatter={valueFormatterNumbers}
index="date"
colors={['cyan']}
categories={['metrics.total_tokens']}
/>
</Col>
</Grid>
</Card>
{/* Per Model Activity */}
{Object.entries(getModelActivityData(userSpendData)).map(([model, data], index) => (
<Card key={index}>
<Title>{model}</Title>
<Grid numItems={2}>
<Col>
<Subtitle style={{ fontSize: "15px", fontWeight: "normal", color: "#535452"}}>
API Requests {valueFormatterNumbers(data.total_requests)}
</Subtitle>
<AreaChart
className="h-40"
data={[...data.daily_data].reverse()}
index="date"
colors={['cyan']}
categories={['api_requests']}
valueFormatter={valueFormatterNumbers}
/>
</Col>
<Col>
<Subtitle style={{ fontSize: "15px", fontWeight: "normal", color: "#535452"}}>
Tokens {valueFormatterNumbers(data.total_tokens)}
</Subtitle>
<BarChart
className="h-40"
data={data.daily_data}
index="date"
colors={['cyan']}
categories={['total_tokens']}
valueFormatter={valueFormatterNumbers}
/>
</Col>
</Grid>
</Card>
))}
</Grid>
</TabPanel>
</TabPanels>
</TabGroup>
</div>
);
};
// Add this helper function to process model-specific activity data
const getModelActivityData = (userSpendData: {
results: DailyData[];
metadata: any;
}) => {
const modelData: {
[key: string]: {
total_requests: number;
total_tokens: number;
daily_data: Array<{
date: string;
api_requests: number;
total_tokens: number;
}>;
};
} = {};
userSpendData.results.forEach((day: DailyData) => {
Object.entries(day.breakdown.models || {}).forEach(([model, metrics]) => {
if (!modelData[model]) {
modelData[model] = {
total_requests: 0,
total_tokens: 0,
daily_data: []
};
}
modelData[model].total_requests += metrics.api_requests;
modelData[model].total_tokens += metrics.total_tokens;
modelData[model].daily_data.push({
date: day.date,
api_requests: metrics.api_requests,
total_tokens: metrics.total_tokens
});
});
});
return modelData;
};
// Add this helper function for number formatting
function valueFormatterNumbers(number: number) {
const formatter = new Intl.NumberFormat('en-US', {
maximumFractionDigits: 0,
notation: 'compact',
compactDisplay: 'short',
});
return formatter.format(number);
}
export default NewUsagePage;