phoenix/litellm-mirror
0
0
Fork 1
mirror of https://github.com/BerriAI/litellm.git synced 2025-04-27 11:43:54 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

feat(proxy_server.py): give request-level breakdown if ttft metric is selected for ju

Browse source 
st that day
This commit is contained in:
Krrish Dholakia 2024-05-28 18:09:22 -07:00
parent 7b565271e2
commit 4dd7b05406
1 changed files with 66 additions and 28 deletions
Download patch file Download diff file Expand all files Collapse all files

44
litellm/proxy/proxy_server.py
View file

@ -493,7 +493,7 @@ async def user_api_key_auth(
if route in LiteLLMRoutes.public_routes.value: if route in LiteLLMRoutes.public_routes.value:
# check if public endpoint # check if public endpoint
return UserAPIKeyAuth() return UserAPIKeyAuth(user_role="app_owner")
if general_settings.get("enable_jwt_auth", False) == True: if general_settings.get("enable_jwt_auth", False) == True:
is_jwt = jwt_handler.is_jwt(token=api_key) is_jwt = jwt_handler.is_jwt(token=api_key)
@ -1385,7 +1385,9 @@ async def user_api_key_auth(
api_key=api_key, user_role="app_owner", **valid_token_dict api_key=api_key, user_role="app_owner", **valid_token_dict
) )
else: else:
return UserAPIKeyAuth(api_key=api_key, **valid_token_dict) return UserAPIKeyAuth(
api_key=api_key, user_role="app_owner", **valid_token_dict
)
else: else:
raise Exception() raise Exception()
except Exception as e: except Exception as e:
@ -9579,6 +9581,32 @@ async def model_streaming_metrics(
startTime = startTime or datetime.now() - timedelta(days=7) # show over past week startTime = startTime or datetime.now() - timedelta(days=7) # show over past week
endTime = endTime or datetime.now() endTime = endTime or datetime.now()
is_same_day = startTime.date() == endTime.date()
if is_same_day:
sql_query = """
SELECT
api_base,
model_group,
model,
"startTime",
request_id,
EXTRACT(epoch FROM ("completionStartTime" - "startTime")) AS time_to_first_token
FROM
"LiteLLM_SpendLogs"
WHERE
"model_group" = $1 AND "cache_hit" != 'True'
AND "completionStartTime" IS NOT NULL
AND "completionStartTime" != "endTime"
AND DATE("startTime") = DATE($2::timestamp)
GROUP BY
api_base,
model_group,
model,
request_id
ORDER BY
time_to_first_token DESC;
"""
else:
sql_query = """ sql_query = """
SELECT SELECT
api_base, api_base,
@ -9611,7 +9639,16 @@ async def model_streaming_metrics(
for model_data in db_response: for model_data in db_response:
_api_base = model_data["api_base"] _api_base = model_data["api_base"]
_model = model_data["model"] _model = model_data["model"]
time_to_first_token = model_data["time_to_first_token"]
unique_key = ""
if is_same_day:
_request_id = model_data["request_id"]
unique_key = _request_id
if _request_id not in _daily_entries:
_daily_entries[_request_id] = {}
else:
_day = model_data["day"] _day = model_data["day"]
unique_key = _day
time_to_first_token = model_data["time_to_first_token"] time_to_first_token = model_data["time_to_first_token"]
if _day not in _daily_entries: if _day not in _daily_entries:
_daily_entries[_day] = {} _daily_entries[_day] = {}
@ -9622,7 +9659,8 @@ async def model_streaming_metrics(
_combined_model_name = _combined_model_name.split("/openai/")[0] _combined_model_name = _combined_model_name.split("/openai/")[0]
_all_api_bases.add(_combined_model_name) _all_api_bases.add(_combined_model_name)
_daily_entries[_day][_combined_model_name] = time_to_first_token
_daily_entries[unique_key][_combined_model_name] = time_to_first_token
""" """
each entry needs to be like this: each entry needs to be like this: