phoenix/litellm-mirror
0
0
Fork 1
mirror of https://github.com/BerriAI/litellm.git synced 2025-04-25 02:34:29 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

[Feat - Cost Tracking improvement] Track prompt caching metrics in DailyUserSpendTransactions (#10029)

Browse source 
* stash changes

* emit cache read/write tokens to daily spend update

* emit cache read/write tokens on daily activity

* update types.ts

* docs prompt caching

* undo ui change

* fix activity metrics

* fix prompt caching metrics

* fix typed dict fields

* fix get_aggregated_daily_spend_update_transactions

* fix aggregating cache tokens

* test_cache_token_fields_aggregation

* daily_transaction

* add cache_creation_input_tokens and cache_read_input_tokens to LiteLLM_DailyUserSpend

* test_daily_spend_update_queue.py
This commit is contained in:
Ishaan Jaff 2025-04-15 21:40:57 -07:00 • committed by GitHub
parent d32d6fe03e
commit bd88263b29
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
12 changed files with 197 additions and 35 deletions
Download patch file Download diff file Expand all files Collapse all files

35
ui/litellm-dashboard/src/components/activity_metrics.tsx
View file

@ -79,6 +79,21 @@ const ModelSection = ({ modelName, metrics }: { modelName: string; metrics: Mode
stack
/>
</Card>
<Card>
<Title>Prompt Caching Metrics</Title>
<div className="mb-2">
<Text>Cache Read: {metrics.total_cache_read_input_tokens?.toLocaleString() || 0} tokens</Text>
<Text>Cache Creation: {metrics.total_cache_creation_input_tokens?.toLocaleString() || 0} tokens</Text>
</div>
<AreaChart
data={metrics.daily_data}
index="date"
categories={["metrics.cache_read_input_tokens", "metrics.cache_creation_input_tokens"]}
colors={["cyan", "purple"]}
valueFormatter={(number: number) => number.toLocaleString()}
/>
</Card>
</Grid>
</div>
);
@ -97,6 +112,8 @@ export const ActivityMetrics: React.FC<ActivityMetricsProps> = ({ modelMetrics }
total_successful_requests: 0,
total_tokens: 0,
total_spend: 0,
total_cache_read_input_tokens: 0,
total_cache_creation_input_tokens: 0,
daily_data: {} as Record<string, {
prompt_tokens: number;
completion_tokens: number;
@ -105,6 +122,8 @@ export const ActivityMetrics: React.FC<ActivityMetricsProps> = ({ modelMetrics }
spend: number;
successful_requests: number;
failed_requests: number;
cache_read_input_tokens: number;
cache_creation_input_tokens: number;
}>
};
@ -114,6 +133,8 @@ export const ActivityMetrics: React.FC<ActivityMetricsProps> = ({ modelMetrics }
totalMetrics.total_successful_requests += model.total_successful_requests;
totalMetrics.total_tokens += model.total_tokens;
totalMetrics.total_spend += model.total_spend;
totalMetrics.total_cache_read_input_tokens += model.total_cache_read_input_tokens || 0;
totalMetrics.total_cache_creation_input_tokens += model.total_cache_creation_input_tokens || 0;
// Aggregate daily data
model.daily_data.forEach(day => {
@ -125,7 +146,9 @@ export const ActivityMetrics: React.FC<ActivityMetricsProps> = ({ modelMetrics }
api_requests: 0,
spend: 0,
successful_requests: 0,
failed_requests: 0
failed_requests: 0,
cache_read_input_tokens: 0,
cache_creation_input_tokens: 0
};
}
totalMetrics.daily_data[day.date].prompt_tokens += day.metrics.prompt_tokens;
@ -135,6 +158,8 @@ export const ActivityMetrics: React.FC<ActivityMetricsProps> = ({ modelMetrics }
totalMetrics.daily_data[day.date].spend += day.metrics.spend;
totalMetrics.daily_data[day.date].successful_requests += day.metrics.successful_requests;
totalMetrics.daily_data[day.date].failed_requests += day.metrics.failed_requests;
totalMetrics.daily_data[day.date].cache_read_input_tokens += day.metrics.cache_read_input_tokens || 0;
totalMetrics.daily_data[day.date].cache_creation_input_tokens += day.metrics.cache_creation_input_tokens || 0;
});
});
@ -233,6 +258,8 @@ export const processActivityData = (dailyActivity: { results: DailyData[] }): Re
prompt_tokens: 0,
completion_tokens: 0,
total_spend: 0,
total_cache_read_input_tokens: 0,
total_cache_creation_input_tokens: 0,
daily_data: []
};
}
@ -245,6 +272,8 @@ export const processActivityData = (dailyActivity: { results: DailyData[] }): Re
modelMetrics[model].total_spend += modelData.metrics.spend;
modelMetrics[model].total_successful_requests += modelData.metrics.successful_requests;
modelMetrics[model].total_failed_requests += modelData.metrics.failed_requests;
modelMetrics[model].total_cache_read_input_tokens += modelData.metrics.cache_read_input_tokens || 0;
modelMetrics[model].total_cache_creation_input_tokens += modelData.metrics.cache_creation_input_tokens || 0;
// Add daily data
modelMetrics[model].daily_data.push({
@ -256,7 +285,9 @@ export const processActivityData = (dailyActivity: { results: DailyData[] }): Re
api_requests: modelData.metrics.api_requests,
spend: modelData.metrics.spend,
successful_requests: modelData.metrics.successful_requests,
failed_requests: modelData.metrics.failed_requests
failed_requests: modelData.metrics.failed_requests,
cache_read_input_tokens: modelData.metrics.cache_read_input_tokens || 0,
cache_creation_input_tokens: modelData.metrics.cache_creation_input_tokens || 0
}
});
});

17
ui/litellm-dashboard/src/components/new_usage.tsx
View file

@ -62,7 +62,9 @@ const NewUsagePage: React.FC<NewUsagePageProps> = ({
total_tokens: 0,
api_requests: 0,
successful_requests: 0,
failed_requests: 0
failed_requests: 0,
cache_read_input_tokens: 0,
cache_creation_input_tokens: 0
},
metadata: {}
};
@ -74,6 +76,8 @@ const NewUsagePage: React.FC<NewUsagePageProps> = ({
modelSpend[model].metrics.api_requests += metrics.metrics.api_requests;
modelSpend[model].metrics.successful_requests += metrics.metrics.successful_requests || 0;
modelSpend[model].metrics.failed_requests += metrics.metrics.failed_requests || 0;
modelSpend[model].metrics.cache_read_input_tokens += metrics.metrics.cache_read_input_tokens || 0;
modelSpend[model].metrics.cache_creation_input_tokens += metrics.metrics.cache_creation_input_tokens || 0;
});
});
@ -104,7 +108,9 @@ const NewUsagePage: React.FC<NewUsagePageProps> = ({
total_tokens: 0,
api_requests: 0,
successful_requests: 0,
failed_requests: 0
failed_requests: 0,
cache_read_input_tokens: 0,
cache_creation_input_tokens: 0
},
metadata: {}
};
@ -116,6 +122,8 @@ const NewUsagePage: React.FC<NewUsagePageProps> = ({
providerSpend[provider].metrics.api_requests += metrics.metrics.api_requests;
providerSpend[provider].metrics.successful_requests += metrics.metrics.successful_requests || 0;
providerSpend[provider].metrics.failed_requests += metrics.metrics.failed_requests || 0;
providerSpend[provider].metrics.cache_read_input_tokens += metrics.metrics.cache_read_input_tokens || 0;
providerSpend[provider].metrics.cache_creation_input_tokens += metrics.metrics.cache_creation_input_tokens || 0;
});
});
@ -145,6 +153,8 @@ const NewUsagePage: React.FC<NewUsagePageProps> = ({
api_requests: 0,
successful_requests: 0,
failed_requests: 0,
cache_read_input_tokens: 0,
cache_creation_input_tokens: 0
},
metadata: {
key_alias: metrics.metadata.key_alias
@ -158,6 +168,8 @@ const NewUsagePage: React.FC<NewUsagePageProps> = ({
keySpend[key].metrics.api_requests += metrics.metrics.api_requests;
keySpend[key].metrics.successful_requests += metrics.metrics.successful_requests;
keySpend[key].metrics.failed_requests += metrics.metrics.failed_requests;
keySpend[key].metrics.cache_read_input_tokens += metrics.metrics.cache_read_input_tokens || 0;
keySpend[key].metrics.cache_creation_input_tokens += metrics.metrics.cache_creation_input_tokens || 0;
});
});
@ -325,7 +337,6 @@ const NewUsagePage: React.FC<NewUsagePageProps> = ({
/>
</Card>
</Col>
{/* Top API Keys */}
<Col numColSpan={1}>
<Card className="h-full">

6
ui/litellm-dashboard/src/components/usage/types.ts
View file

@ -6,6 +6,8 @@ export interface SpendMetrics {
api_requests: number;
successful_requests: number;
failed_requests: number;
cache_read_input_tokens: number;
cache_creation_input_tokens: number;
}
export interface DailyData {
@ -36,6 +38,8 @@ export interface ModelActivityData {
total_requests: number;
total_successful_requests: number;
total_failed_requests: number;
total_cache_read_input_tokens: number;
total_cache_creation_input_tokens: number;
total_tokens: number;
prompt_tokens: number;
completion_tokens: number;
@ -50,6 +54,8 @@ export interface ModelActivityData {
spend: number;
successful_requests: number;
failed_requests: number;
cache_read_input_tokens: number;
cache_creation_input_tokens: number;
};
}[];
}