phoenix/litellm-mirror
0
0
Fork 1
mirror of https://github.com/BerriAI/litellm.git synced 2025-04-27 11:43:54 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

fix debugging lowest latency router

Browse source
This commit is contained in:
Ishaan Jaff 2024-04-25 19:34:28 -07:00
parent 3ab5e687f6
commit 7306072d33
1 changed files with 9 additions and 4 deletions
Download patch file Download diff file Expand all files Collapse all files

13
litellm/router_strategy/lowest_latency.py
View file

@ -345,16 +345,21 @@ class LowestLatencyLoggingHandler(CustomLogger):
if isinstance(_call_latency, float): if isinstance(_call_latency, float):
total += _call_latency total += _call_latency
item_latency = total / len(item_latency) item_latency = total / len(item_latency)
print("item_latency=", item_latency, "deployment=", _deployment) # noqa
# Debugging Logic # # -------------- #
# Debugging Logic
# -------------- #
# We use _latency_per_deployment to log to langfuse, slack - this is not used to make a decision on routing
# this helps a user to debug why the router picked a specfic deployment #
_deployment_api_base = _deployment.get("litellm_params", {}).get( _deployment_api_base = _deployment.get("litellm_params", {}).get(
"api_base", "" "api_base", ""
) )
# End of Debugging Logic #
if _deployment_api_base is not None: if _deployment_api_base is not None:
_latency_per_deployment[_deployment_api_base] = item_latency _latency_per_deployment[_deployment_api_base] = item_latency
# -------------- #
# End of Debugging Logic
# -------------- #
if item_latency == 0: if item_latency == 0:
deployment = _deployment deployment = _deployment
break break