Litellm dev 04 18 2025 p2 (#10157)

* fix(proxy/_types.py): allow internal user to call api playground

* fix(new_usage.tsx): cleanup tag based usage - only show for proxy admin

not clear what tags internal user should be allowed to see

* fix(team_endpoints.py): allow internal user view spend for teams they belong to

* fix(team_endpoints.py): return team alias on `/team/daily/activity` API

allows displaying team alias on ui

* fix: fix linting error

* fix(entity_usage.tsx): allow viewing top keys by team

* fix(entity_usage.tsx): show alias, if available in breakdown

allows entity alias to be easily displayed

* Show usage by key (on all up, team, and tag usage dashboards)  (#10152)

* fix(entity_usage.tsx): allow user to select team in team usage tab

* fix(new_usage.tsx): load all tags for filtering

* fix(tag_management_endpoints.py): return dynamic tags from db on `/tag/list`

* fix(litellm_pre_call_utils.py): support x-litellm-tags even if tag based routing not enabled

* fix(new_usage.tsx): show breakdown of usage by api key on dashboard

helpful when looking at spend by team

* fix(networking.tsx): exclude litellm-dashboard team id's from calls

adds noisy ui tokens to key activity

* fix(new_usage.tsx): allow user to see activity by key on main tab

* feat(internal_user_endpoints.py): refactor to use common_daily_activity function

reuses same logic across teams/keys/tags

Allows returning team_alias in api_keys consistently

* fix(leftnav.tsx): swap old usage with new usage tab

* fix(entity_usage.tsx): show breakdown of teams in daily spend chart

* style(new_usage.tsx): show global usage tab if user is admin / has admin view

* fix(new_usage.tsx): add disclaimer for new usage dashboard

* fix(new_usage.tsx): fix linting error

* Allow filtering usage dashboard by team + tag (#10150)

* fix(entity_usage.tsx): allow user to select team in team usage tab

* fix(new_usage.tsx): load all tags for filtering

* fix(tag_management_endpoints.py): return dynamic tags from db on `/tag/list`

* fix(litellm_pre_call_utils.py): support x-litellm-tags even if tag based routing not enabled

* fix: fix linting error
This commit is contained in:
Krish Dholakia 2025-04-19 07:32:23 -07:00 committed by GitHub
parent b9756bf006
commit ef6ac42658
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
16 changed files with 429 additions and 287 deletions

View file

@ -370,6 +370,7 @@ export default function CreateKeyPage() {
userID={userID}
userRole={userRole}
accessToken={accessToken}
teams={teams as Team[] ?? []}
/>
) :
(

View file

@ -1,7 +1,7 @@
import React from 'react';
import { Card, Grid, Text, Title, Accordion, AccordionHeader, AccordionBody } from '@tremor/react';
import { AreaChart, BarChart } from '@tremor/react';
import { SpendMetrics, DailyData, ModelActivityData } from './usage/types';
import { SpendMetrics, DailyData, ModelActivityData, MetricWithMetadata, KeyMetricWithMetadata } from './usage/types';
import { Collapse } from 'antd';
interface ActivityMetricsProps {
@ -224,7 +224,7 @@ export const ActivityMetrics: React.FC<ActivityMetricsProps> = ({ modelMetrics }
key={modelName}
header={
<div className="flex justify-between items-center w-full">
<Title>{modelName || 'Unknown Model'}</Title>
<Title>{modelMetrics[modelName].label || 'Unknown Item'}</Title>
<div className="flex space-x-4 text-sm text-gray-500">
<span>${modelMetrics[modelName].total_spend.toFixed(2)}</span>
<span>{modelMetrics[modelName].total_requests.toLocaleString()} requests</span>
@ -243,14 +243,24 @@ export const ActivityMetrics: React.FC<ActivityMetricsProps> = ({ modelMetrics }
);
};
// Helper function to format key label
const formatKeyLabel = (modelData: KeyMetricWithMetadata, model: string): string => {
const keyAlias = modelData.metadata.key_alias || `key-hash-${model}`;
const teamId = modelData.metadata.team_id;
return teamId ? `${keyAlias} (team_id: ${teamId})` : keyAlias;
};
// Process data function
export const processActivityData = (dailyActivity: { results: DailyData[] }): Record<string, ModelActivityData> => {
export const processActivityData = (dailyActivity: { results: DailyData[] }, key: "models" | "api_keys"): Record<string, ModelActivityData> => {
const modelMetrics: Record<string, ModelActivityData> = {};
dailyActivity.results.forEach((day) => {
Object.entries(day.breakdown.models || {}).forEach(([model, modelData]) => {
Object.entries(day.breakdown[key] || {}).forEach(([model, modelData]) => {
if (!modelMetrics[model]) {
modelMetrics[model] = {
label: key === 'api_keys'
? formatKeyLabel(modelData as KeyMetricWithMetadata, model)
: model,
total_requests: 0,
total_successful_requests: 0,
total_failed_requests: 0,

View file

@ -8,8 +8,9 @@ import {
} from "@tremor/react";
import { Select } from 'antd';
import { ActivityMetrics, processActivityData } from './activity_metrics';
import { SpendMetrics, DailyData } from './usage/types';
import { DailyData, KeyMetricWithMetadata, EntityMetricWithMetadata } from './usage/types';
import { tagDailyActivityCall, teamDailyActivityCall } from './networking';
import TopKeyView from "./top_key_view";
interface EntityMetrics {
metrics: {
@ -48,16 +49,27 @@ interface EntitySpendData {
};
}
export interface EntityList {
label: string;
value: string;
}
interface EntityUsageProps {
accessToken: string | null;
entityType: 'tag' | 'team';
entityId?: string | null;
userID: string | null;
userRole: string | null;
entityList: EntityList[] | null;
}
const EntityUsage: React.FC<EntityUsageProps> = ({
accessToken,
entityType,
entityId
entityId,
userID,
userRole,
entityList
}) => {
const [spendData, setSpendData] = useState<EntitySpendData>({
results: [],
@ -70,8 +82,8 @@ const EntityUsage: React.FC<EntityUsageProps> = ({
}
});
const modelMetrics = processActivityData(spendData);
const modelMetrics = processActivityData(spendData, "models");
const keyMetrics = processActivityData(spendData, "api_keys");
const [selectedTags, setSelectedTags] = useState<string[]>([]);
const [dateValue, setDateValue] = useState<DateRangePickerValue>({
from: new Date(Date.now() - 28 * 24 * 60 * 60 * 1000),
@ -144,29 +156,46 @@ const EntityUsage: React.FC<EntityUsageProps> = ({
.slice(0, 5);
};
const getTopApiKeys = () => {
const apiKeySpend: { [key: string]: any } = {};
const getTopAPIKeys = () => {
const keySpend: { [key: string]: KeyMetricWithMetadata } = {};
spendData.results.forEach(day => {
Object.entries(day.breakdown.api_keys || {}).forEach(([key, metrics]) => {
if (!apiKeySpend[key]) {
apiKeySpend[key] = {
key: key,
spend: 0,
requests: 0,
successful_requests: 0,
failed_requests: 0,
tokens: 0
if (!keySpend[key]) {
keySpend[key] = {
metrics: {
spend: 0,
prompt_tokens: 0,
completion_tokens: 0,
total_tokens: 0,
api_requests: 0,
successful_requests: 0,
failed_requests: 0,
cache_read_input_tokens: 0,
cache_creation_input_tokens: 0
},
metadata: {
key_alias: metrics.metadata.key_alias
}
};
}
apiKeySpend[key].spend += metrics.metrics.spend;
apiKeySpend[key].requests += metrics.metrics.api_requests;
apiKeySpend[key].successful_requests += metrics.metrics.successful_requests;
apiKeySpend[key].failed_requests += metrics.metrics.failed_requests;
apiKeySpend[key].tokens += metrics.metrics.total_tokens;
keySpend[key].metrics.spend += metrics.metrics.spend;
keySpend[key].metrics.prompt_tokens += metrics.metrics.prompt_tokens;
keySpend[key].metrics.completion_tokens += metrics.metrics.completion_tokens;
keySpend[key].metrics.total_tokens += metrics.metrics.total_tokens;
keySpend[key].metrics.api_requests += metrics.metrics.api_requests;
keySpend[key].metrics.successful_requests += metrics.metrics.successful_requests;
keySpend[key].metrics.failed_requests += metrics.metrics.failed_requests;
keySpend[key].metrics.cache_read_input_tokens += metrics.metrics.cache_read_input_tokens || 0;
keySpend[key].metrics.cache_creation_input_tokens += metrics.metrics.cache_creation_input_tokens || 0;
});
});
return Object.values(apiKeySpend)
return Object.entries(keySpend)
.map(([api_key, metrics]) => ({
api_key,
key_alias: metrics.metadata.key_alias || "-", // Using truncated key as alias
spend: metrics.metrics.spend,
}))
.sort((a, b) => b.spend - a.spend)
.slice(0, 5);
};
@ -203,47 +232,49 @@ const EntityUsage: React.FC<EntityUsageProps> = ({
};
const getAllTags = () => {
const tags = new Set<string>();
spendData.results.forEach(day => {
Object.keys(day.breakdown.entities || {}).forEach(tag => {
tags.add(tag);
});
});
return Array.from(tags).map(tag => ({
label: tag,
value: tag
}));
if (entityList) {
return entityList;
}
};
const filterDataByTags = (data: any[]) => {
const filterDataByTags = (data: EntityMetricWithMetadata[]) => {
if (selectedTags.length === 0) return data;
return data.filter(item => selectedTags.includes(item.entity));
return data.filter(item => selectedTags.includes(item.metadata.id));
};
const getEntityBreakdown = () => {
const entitySpend: { [key: string]: any } = {};
const entitySpend: { [key: string]: EntityMetricWithMetadata } = {};
spendData.results.forEach(day => {
Object.entries(day.breakdown.entities || {}).forEach(([entity, data]) => {
if (!entitySpend[entity]) {
entitySpend[entity] = {
entity,
spend: 0,
requests: 0,
successful_requests: 0,
failed_requests: 0,
tokens: 0
metrics: {
spend: 0,
prompt_tokens: 0,
completion_tokens: 0,
total_tokens: 0,
api_requests: 0,
successful_requests: 0,
failed_requests: 0,
cache_read_input_tokens: 0,
cache_creation_input_tokens: 0
},
metadata: {
alias: data.metadata.team_alias || entity,
id: entity
}
};
}
entitySpend[entity].spend += data.metrics.spend;
entitySpend[entity].requests += data.metrics.api_requests;
entitySpend[entity].successful_requests += data.metrics.successful_requests;
entitySpend[entity].failed_requests += data.metrics.failed_requests;
entitySpend[entity].tokens += data.metrics.total_tokens;
entitySpend[entity].metrics.spend += data.metrics.spend;
entitySpend[entity].metrics.api_requests += data.metrics.api_requests;
entitySpend[entity].metrics.successful_requests += data.metrics.successful_requests;
entitySpend[entity].metrics.failed_requests += data.metrics.failed_requests;
entitySpend[entity].metrics.total_tokens += data.metrics.total_tokens;
});
});
const result = Object.values(entitySpend)
.sort((a, b) => b.spend - a.spend);
.sort((a, b) => b.metrics.spend - a.metrics.spend);
return filterDataByTags(result);
};
@ -261,9 +292,10 @@ const EntityUsage: React.FC<EntityUsageProps> = ({
onValueChange={setDateValue}
/>
</Col>
<Col>
<Text>Filter by {entityType === 'tag' ? 'Tags' : 'Teams'}</Text>
<Select
{entityList && entityList.length > 0 && (
<Col>
<Text>Filter by {entityType === 'tag' ? 'Tags' : 'Teams'}</Text>
<Select
mode="multiple"
style={{ width: '100%' }}
placeholder={`Select ${entityType === 'tag' ? 'tags' : 'teams'} to filter...`}
@ -272,13 +304,15 @@ const EntityUsage: React.FC<EntityUsageProps> = ({
options={getAllTags()}
className="mt-2"
allowClear
/>
</Col>
/>
</Col>
)}
</Grid>
<TabGroup>
<TabList variant="solid" className="mt-1">
<Tab>Cost</Tab>
<Tab>Activity</Tab>
<Tab>Model Activity</Tab>
<Tab>Key Activity</Tab>
</TabList>
<TabPanels>
<TabPanel>
@ -324,20 +358,45 @@ const EntityUsage: React.FC<EntityUsageProps> = ({
{/* Daily Spend Chart */}
<Col numColSpan={2}>
<Card>
<Title>Daily Spend</Title>
<BarChart
data={[...spendData.results].sort((a, b) =>
new Date(a.date).getTime() - new Date(b.date).getTime()
)}
index="date"
categories={["metrics.spend"]}
colors={["cyan"]}
valueFormatter={(value) => `$${value.toFixed(2)}`}
yAxisWidth={100}
showLegend={false}
/>
</Card>
<Card>
<Title>Daily Spend</Title>
<BarChart
data={[...spendData.results].sort((a, b) =>
new Date(a.date).getTime() - new Date(b.date).getTime()
)}
index="date"
categories={["metrics.spend"]}
colors={["cyan"]}
valueFormatter={(value) => `$${value.toFixed(2)}`}
yAxisWidth={100}
showLegend={false}
customTooltip={({ payload, active }) => {
if (!active || !payload?.[0]) return null;
const data = payload[0].payload;
return (
<div className="bg-white p-4 shadow-lg rounded-lg border">
<p className="font-bold">{data.date}</p>
<p className="text-cyan-500">Total Spend: ${data.metrics.spend.toFixed(2)}</p>
<p className="text-gray-600">Total Requests: {data.metrics.api_requests}</p>
<p className="text-gray-600">Successful: {data.metrics.successful_requests}</p>
<p className="text-gray-600">Failed: {data.metrics.failed_requests}</p>
<p className="text-gray-600">Total Tokens: {data.metrics.total_tokens}</p>
<div className="mt-2 border-t pt-2">
<p className="font-semibold">Spend by {entityType === 'tag' ? 'Tag' : 'Team'}:</p>
{Object.entries(data.breakdown.entities || {}).map(([entity, entityData]) => {
const metrics = entityData as EntityMetrics;
return (
<p key={entity} className="text-sm text-gray-600">
{metrics.metadata.team_alias || entity}: ${metrics.metrics.spend.toFixed(2)}
</p>
);
})}
</div>
</div>
);
}}
/>
</Card>
</Col>
{/* Entity Breakdown Section */}
@ -353,13 +412,13 @@ const EntityUsage: React.FC<EntityUsageProps> = ({
</a>
</div>
</div>
<Grid numItems={2}>
<Col numColSpan={1}>
<Grid numItems={2}>
<Col numColSpan={1}>
<BarChart
className="mt-4 h-52"
data={getEntityBreakdown()}
index="entity"
categories={["spend"]}
index="metadata.alias"
categories={["metrics.spend"]}
colors={["cyan"]}
valueFormatter={(value) => `$${value.toFixed(4)}`}
layout="vertical"
@ -380,18 +439,18 @@ const EntityUsage: React.FC<EntityUsageProps> = ({
</TableHead>
<TableBody>
{getEntityBreakdown()
.filter(entity => entity.spend > 0)
.filter(entity => entity.metrics.spend > 0)
.map((entity) => (
<TableRow key={entity.entity}>
<TableCell>{entity.entity}</TableCell>
<TableCell>${entity.spend.toFixed(4)}</TableCell>
<TableRow key={entity.metadata.id}>
<TableCell>{entity.metadata.alias}</TableCell>
<TableCell>${entity.metrics.spend.toFixed(4)}</TableCell>
<TableCell className="text-green-600">
{entity.successful_requests.toLocaleString()}
{entity.metrics.successful_requests.toLocaleString()}
</TableCell>
<TableCell className="text-red-600">
{entity.failed_requests.toLocaleString()}
{entity.metrics.failed_requests.toLocaleString()}
</TableCell>
<TableCell>{entity.tokens.toLocaleString()}</TableCell>
<TableCell>{entity.metrics.total_tokens.toLocaleString()}</TableCell>
</TableRow>
))}
</TableBody>
@ -407,17 +466,13 @@ const EntityUsage: React.FC<EntityUsageProps> = ({
<Col numColSpan={1}>
<Card>
<Title>Top API Keys</Title>
<BarChart
className="mt-4 h-40"
data={getTopApiKeys()}
index="key"
categories={["spend"]}
colors={["cyan"]}
valueFormatter={(value) => `$${value.toFixed(2)}`}
layout="vertical"
yAxisWidth={200}
showLegend={false}
/>
<TopKeyView
topKeys={getTopAPIKeys()}
accessToken={accessToken}
userID={userID}
userRole={userRole}
teams={null}
/>
</Card>
</Col>
@ -494,6 +549,9 @@ const EntityUsage: React.FC<EntityUsageProps> = ({
<TabPanel>
<ActivityMetrics modelMetrics={modelMetrics} />
</TabPanel>
<TabPanel>
<ActivityMetrics modelMetrics={keyMetrics} />
</TabPanel>
</TabPanels>
</TabGroup>
</div>

View file

@ -57,7 +57,7 @@ const Sidebar: React.FC<SidebarProps> = ({
{ key: "1", page: "api-keys", label: "Virtual Keys", icon: <KeyOutlined /> },
{ key: "3", page: "llm-playground", label: "Test Key", icon: <PlayCircleOutlined />, roles: rolesWithWriteAccess },
{ key: "2", page: "models", label: "Models", icon: <BlockOutlined />, roles: rolesWithWriteAccess },
{ key: "4", page: "usage", label: "Usage", icon: <BarChartOutlined /> },
{ key: "12", page: "new_usage", label: "Usage", icon: <BarChartOutlined />, roles: [...all_admin_roles, ...internalUserRoles] },
{ key: "6", page: "teams", label: "Teams", icon: <TeamOutlined /> },
{ key: "17", page: "organizations", label: "Organizations", icon: <BankOutlined />, roles: all_admin_roles },
{ key: "5", page: "users", label: "Internal Users", icon: <UserOutlined />, roles: all_admin_roles },
@ -73,7 +73,7 @@ const Sidebar: React.FC<SidebarProps> = ({
{ key: "9", page: "caching", label: "Caching", icon: <DatabaseOutlined />, roles: all_admin_roles },
{ key: "10", page: "budgets", label: "Budgets", icon: <BankOutlined />, roles: all_admin_roles },
{ key: "11", page: "guardrails", label: "Guardrails", icon: <SafetyOutlined />, roles: all_admin_roles },
{ key: "12", page: "new_usage", label: "New Usage", icon: <BarChartOutlined />, roles: [...all_admin_roles, ...internalUserRoles] },
{ key: "4", page: "usage", label: "Old Usage", icon: <BarChartOutlined /> },
{ key: "20", page: "transform-request", label: "API Playground", icon: <ApiOutlined />, roles: [...all_admin_roles, ...internalUserRoles] },
{ key: "18", page: "mcp-tools", label: "MCP Tools", icon: <ToolOutlined />, roles: all_admin_roles },
{ key: "19", page: "tag-management", label: "Tag Management", icon: <TagsOutlined />, roles: all_admin_roles },

View file

@ -1200,6 +1200,7 @@ export const teamDailyActivityCall = async (accessToken: String, startTime: Date
if (teamIds) {
queryParams.append('team_ids', teamIds.join(','));
}
queryParams.append('exclude_team_ids', 'litellm-dashboard');
const queryString = queryParams.toString();
if (queryString) {
url += `?${queryString}`;

View file

@ -17,23 +17,29 @@ import {
} from "@tremor/react";
import { AreaChart } from "@tremor/react";
import { userDailyActivityCall } from "./networking";
import { userDailyActivityCall, tagListCall } from "./networking";
import { Tag } from "./tag_management/types";
import ViewUserSpend from "./view_user_spend";
import TopKeyView from "./top_key_view";
import { ActivityMetrics, processActivityData } from './activity_metrics';
import { SpendMetrics, DailyData, ModelActivityData, MetricWithMetadata, KeyMetricWithMetadata } from './usage/types';
import EntityUsage from './entity_usage';
import { old_admin_roles, v2_admin_role_names, all_admin_roles, rolesAllowedToSeeUsage, rolesWithWriteAccess, internalUserRoles } from '../utils/roles';
import { Team } from "./key_team_helpers/key_list";
import { EntityList } from "./entity_usage";
interface NewUsagePageProps {
accessToken: string | null;
userRole: string | null;
userID: string | null;
teams: Team[];
}
const NewUsagePage: React.FC<NewUsagePageProps> = ({
accessToken,
userRole,
userID,
teams
}) => {
const [userSpendData, setUserSpendData] = useState<{
results: DailyData[];
@ -46,6 +52,23 @@ const NewUsagePage: React.FC<NewUsagePageProps> = ({
to: new Date(),
});
const [allTags, setAllTags] = useState<EntityList[]>([]);
const getAllTags = async () => {
if (!accessToken) {
return;
}
const tags = await tagListCall(accessToken);
setAllTags(Object.values(tags).map((tag: Tag) => ({
label: tag.name,
value: tag.name
})));
};
useEffect(() => {
getAllTags();
}, [accessToken]);
// Derived states from userSpendData
const totalSpend = userSpendData.metadata?.total_spend || 0;
@ -227,16 +250,19 @@ const NewUsagePage: React.FC<NewUsagePageProps> = ({
fetchUserSpendData();
}, [accessToken, dateValue]);
const modelMetrics = processActivityData(userSpendData);
const modelMetrics = processActivityData(userSpendData, "models");
const keyMetrics = processActivityData(userSpendData, "api_keys");
return (
<div style={{ width: "100%" }} className="p-8">
<Text>Usage Analytics Dashboard</Text>
<Text className="text-sm text-gray-500 mb-4">
This is the new usage dashboard. <br/> You may see empty data, as these use <a href="https://github.com/BerriAI/litellm/blob/6de348125208dd4be81ff0e5813753df2fbe9735/schema.prisma#L320" className="text-blue-500 hover:text-blue-700 ml-1">new aggregate tables</a> to allow UI to work at 1M+ spend logs. To access the old dashboard, go to Experimental {'>'} Old Usage.
</Text>
<TabGroup>
<TabList variant="solid" className="mt-1">
<Tab>Your Usage</Tab>
<Tab>Tag Usage</Tab>
{all_admin_roles.includes(userRole || "") ? <Tab>Global Usage</Tab> : <Tab>Your Usage</Tab>}
<Tab>Team Usage</Tab>
{all_admin_roles.includes(userRole || "") ? <Tab>Tag Usage</Tab> : <></>}
</TabList>
<TabPanels>
{/* Your Usage Panel */}
@ -256,7 +282,8 @@ const NewUsagePage: React.FC<NewUsagePageProps> = ({
<TabGroup>
<TabList variant="solid" className="mt-1">
<Tab>Cost</Tab>
<Tab>Activity</Tab>
<Tab>Model Activity</Tab>
<Tab>Key Activity</Tab>
</TabList>
<TabPanels>
{/* Cost Panel */}
@ -459,25 +486,38 @@ const NewUsagePage: React.FC<NewUsagePageProps> = ({
<TabPanel>
<ActivityMetrics modelMetrics={modelMetrics} />
</TabPanel>
<TabPanel>
<ActivityMetrics modelMetrics={keyMetrics} />
</TabPanel>
</TabPanels>
</TabGroup>
</TabPanel>
{/* Tag Usage Panel */}
<TabPanel>
<EntityUsage
accessToken={accessToken}
entityType="tag"
/>
</TabPanel>
{/* Team Usage Panel */}
<TabPanel>
<EntityUsage
accessToken={accessToken}
entityType="team"
userID={userID}
userRole={userRole}
entityList={teams?.map(team => ({
label: team.team_alias,
value: team.team_id
})) || null}
/>
</TabPanel>
{/* Tag Usage Panel */}
<TabPanel>
<EntityUsage
accessToken={accessToken}
entityType="tag"
userID={userID}
userRole={userRole}
entityList={allTags}
/>
</TabPanel>
</TabPanels>
</TabGroup>
</div>

View file

@ -31,10 +31,12 @@ export interface KeyMetricWithMetadata {
metrics: SpendMetrics;
metadata: {
key_alias: string | null;
team_id?: string | null;
};
}
export interface ModelActivityData {
label: string;
total_requests: number;
total_successful_requests: number;
total_failed_requests: number;
@ -62,11 +64,17 @@ export interface ModelActivityData {
export interface KeyMetadata {
key_alias: string | null;
team_id: string | null;
}
export interface KeyMetricWithMetadata {
export interface EntityMetadata {
alias: string;
id: string;
}
export interface EntityMetricWithMetadata {
metrics: SpendMetrics;
metadata: KeyMetadata;
metadata: EntityMetadata;
}
export interface MetricWithMetadata {