phoenix/litellm
1
0
Fork 0
forked from phoenix/litellm-mirror
Code Pull requests Activity

fix - rate lmit error dashboard

Browse source
This commit is contained in:
Ishaan Jaff 2024-05-30 20:56:05 -07:00
parent 1f41fda21d
commit 9a55365791
1 changed files with 25 additions and 19 deletions
Download patch file Download diff file Expand all files Collapse all files

44
litellm/proxy/proxy_server.py
View file

@ -6890,14 +6890,14 @@ async def get_global_activity_exceptions_per_deployment(
const chartdata = [ const chartdata = [
{ {
date: 'Jan 22', date: 'Jan 22',
num_exceptions: 10 num_rate_limit_exceptions: 10
}, },
{ {
date: 'Jan 23', date: 'Jan 23',
num_exceptions: 12 num_rate_limit_exceptions: 12
}, },
], ],
"sum_num_exceptions": 20, "sum_num_rate_limit_exceptions": 20,
}, },
{ {
@ -6906,14 +6906,14 @@ async def get_global_activity_exceptions_per_deployment(
const chartdata = [ const chartdata = [
{ {
date: 'Jan 22', date: 'Jan 22',
num_exceptions: 10, num_rate_limit_exceptions: 10,
}, },
{ {
date: 'Jan 23', date: 'Jan 23',
num_exceptions: 12 num_rate_limit_exceptions: 12
}, },
], ],
"sum_num_exceptions": 20, "sum_num_rate_limit_exceptions": 20,
}, },
] ]
@ -6940,13 +6940,14 @@ async def get_global_activity_exceptions_per_deployment(
SELECT SELECT
api_base, api_base,
date_trunc('day', "startTime")::date AS date, date_trunc('day', "startTime")::date AS date,
COUNT(*) AS num_exceptions COUNT(*) AS num_rate_limit_exceptions
FROM FROM
"LiteLLM_ErrorLogs" "LiteLLM_ErrorLogs"
WHERE WHERE
"startTime" >= $1::date "startTime" >= $1::date
AND "startTime" < ($2::date + INTERVAL '1 day') AND "startTime" < ($2::date + INTERVAL '1 day')
AND model_group = $3 AND model_group = $3
AND status_code = '429'
GROUP BY GROUP BY
api_base, api_base,
date_trunc('day', "startTime") date_trunc('day', "startTime")
@ -6968,19 +6969,21 @@ async def get_global_activity_exceptions_per_deployment(
if _model not in model_ui_data: if _model not in model_ui_data:
model_ui_data[_model] = { model_ui_data[_model] = {
"daily_data": [], "daily_data": [],
"sum_num_exceptions": 0, "sum_num_rate_limit_exceptions": 0,
} }
_date_obj = datetime.fromisoformat(row["date"]) _date_obj = datetime.fromisoformat(row["date"])
row["date"] = _date_obj.strftime("%b %d") row["date"] = _date_obj.strftime("%b %d")
model_ui_data[_model]["daily_data"].append(row) model_ui_data[_model]["daily_data"].append(row)
model_ui_data[_model]["sum_num_exceptions"] += row.get("num_exceptions", 0) model_ui_data[_model]["sum_num_rate_limit_exceptions"] += row.get(
"num_rate_limit_exceptions", 0
)
# sort mode ui data by sum_api_requests -> get top 10 models # sort mode ui data by sum_api_requests -> get top 10 models
model_ui_data = dict( model_ui_data = dict(
sorted( sorted(
model_ui_data.items(), model_ui_data.items(),
key=lambda x: x[1]["sum_num_exceptions"], key=lambda x: x[1]["sum_num_rate_limit_exceptions"],
reverse=True, reverse=True,
)[:10] )[:10]
) )
@ -6993,7 +6996,9 @@ async def get_global_activity_exceptions_per_deployment(
{ {
"api_base": model, "api_base": model,
"daily_data": _sort_daily_data, "daily_data": _sort_daily_data,
"sum_num_exceptions": data["sum_num_exceptions"], "sum_num_rate_limit_exceptions": data[
"sum_num_rate_limit_exceptions"
],
} }
) )
@ -7035,11 +7040,11 @@ async def get_global_activity_exceptions(
const chartdata = [ const chartdata = [
{ {
date: 'Jan 22', date: 'Jan 22',
num_exceptions: 10, num_rate_limit_exceptions: 10,
}, },
{ {
date: 'Jan 23', date: 'Jan 23',
num_exceptions: 10, num_rate_limit_exceptions: 10,
}, },
], ],
"sum_api_exceptions": 20, "sum_api_exceptions": 20,
@ -7066,13 +7071,14 @@ async def get_global_activity_exceptions(
sql_query = """ sql_query = """
SELECT SELECT
date_trunc('day', "startTime")::date AS date, date_trunc('day', "startTime")::date AS date,
COUNT(*) AS num_exceptions COUNT(*) AS num_rate_limit_exceptions
FROM FROM
"LiteLLM_ErrorLogs" "LiteLLM_ErrorLogs"
WHERE WHERE
"startTime" >= $1::date "startTime" >= $1::date
AND "startTime" < ($2::date + INTERVAL '1 day') AND "startTime" < ($2::date + INTERVAL '1 day')
AND model_group = $3 AND model_group = $3
AND status_code = '429'
GROUP BY GROUP BY
date_trunc('day', "startTime") date_trunc('day', "startTime")
ORDER BY ORDER BY
@ -7085,7 +7091,7 @@ async def get_global_activity_exceptions(
if db_response is None: if db_response is None:
return [] return []
sum_num_exceptions = 0 sum_num_rate_limit_exceptions = 0
daily_data = [] daily_data = []
for row in db_response: for row in db_response:
# cast date to datetime # cast date to datetime
@ -7093,14 +7099,14 @@ async def get_global_activity_exceptions(
row["date"] = _date_obj.strftime("%b %d") row["date"] = _date_obj.strftime("%b %d")
daily_data.append(row) daily_data.append(row)
sum_num_exceptions += row.get("num_exceptions", 0) sum_num_rate_limit_exceptions += row.get("num_rate_limit_exceptions", 0)
# sort daily_data by date # sort daily_data by date
daily_data = sorted(daily_data, key=lambda x: x["date"]) daily_data = sorted(daily_data, key=lambda x: x["date"])
data_to_return = { data_to_return = {
"daily_data": daily_data, "daily_data": daily_data,
"sum_num_exceptions": sum_num_exceptions, "sum_num_rate_limit_exceptions": sum_num_rate_limit_exceptions,
} }
return data_to_return return data_to_return
@ -10830,7 +10836,7 @@ async def model_metrics_exceptions(
SELECT SELECT
CASE WHEN api_base = '' THEN litellm_model_name ELSE CONCAT(litellm_model_name, '-', api_base) END AS combined_model_api_base, CASE WHEN api_base = '' THEN litellm_model_name ELSE CONCAT(litellm_model_name, '-', api_base) END AS combined_model_api_base,
exception_type, exception_type,
COUNT(*) AS num_exceptions COUNT(*) AS num_rate_limit_exceptions
FROM "LiteLLM_ErrorLogs" FROM "LiteLLM_ErrorLogs"
WHERE "startTime" >= $1::timestamp AND "endTime" <= $2::timestamp AND model_group = $3 WHERE "startTime" >= $1::timestamp AND "endTime" <= $2::timestamp AND model_group = $3
GROUP BY combined_model_api_base, exception_type GROUP BY combined_model_api_base, exception_type
@ -10838,7 +10844,7 @@ async def model_metrics_exceptions(
SELECT SELECT
combined_model_api_base, combined_model_api_base,
COUNT(*) AS total_exceptions, COUNT(*) AS total_exceptions,
json_object_agg(exception_type, num_exceptions) AS exception_counts json_object_agg(exception_type, num_rate_limit_exceptions) AS exception_counts
FROM cte FROM cte
GROUP BY combined_model_api_base GROUP BY combined_model_api_base
ORDER BY total_exceptions DESC ORDER BY total_exceptions DESC