phoenix/litellm-mirror
0
0
Fork 1
mirror of https://github.com/BerriAI/litellm.git synced 2025-04-27 19:54:13 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

fix better debugging for latency

Browse source
This commit is contained in:
Ishaan Jaff 2024-04-25 11:35:08 -07:00
parent 4931514330
commit 3ab5e687f6
1 changed files with 8 additions and 6 deletions
Download patch file Download diff file Expand all files Collapse all files

14
litellm/router_strategy/lowest_latency.py
View file

@ -339,18 +339,20 @@ class LowestLatencyLoggingHandler(CustomLogger):
item_rpm = item_map.get(precise_minute, {}).get("rpm", 0) item_rpm = item_map.get(precise_minute, {}).get("rpm", 0)
item_tpm = item_map.get(precise_minute, {}).get("tpm", 0) item_tpm = item_map.get(precise_minute, {}).get("tpm", 0)
# _latency_per_deployment is used for debuggig
_deployment_api_base = _deployment.get("litellm_params", {}).get(
"api_base", ""
)
# get average latency # get average latency
total: float = 0.0 total: float = 0.0
for _call_latency in item_latency: for _call_latency in item_latency:
if isinstance(_call_latency, float): if isinstance(_call_latency, float):
total += _call_latency total += _call_latency
item_latency = total / len(item_latency) item_latency = total / len(item_latency)
print("item_latency=", item_latency, "deployment=", deployment) # noqa print("item_latency=", item_latency, "deployment=", _deployment) # noqa
# Debugging Logic #
_deployment_api_base = _deployment.get("litellm_params", {}).get(
"api_base", ""
)
# End of Debugging Logic #
if _deployment_api_base is not None: if _deployment_api_base is not None:
_latency_per_deployment[_deployment_api_base] = item_latency _latency_per_deployment[_deployment_api_base] = item_latency
if item_latency == 0: if item_latency == 0: