phoenix/litellm-mirror
0
0
Fork 1
mirror of https://github.com/BerriAI/litellm.git synced 2025-04-25 02:34:29 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Revert "Fix latency redis"

Browse source
This commit is contained in:
Krish Dholakia 2025-03-19 18:11:22 -07:00 • committed by GitHub
parent 57503954fe
commit 0de015b4c5
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
1 changed files with 5 additions and 17 deletions
Download patch file Download diff file Expand all files Collapse all files

22
litellm/router_strategy/lowest_latency.py
View file

@ -124,9 +124,7 @@ class LowestLatencyLoggingHandler(CustomLogger):
len(request_count_dict[id].get("latency", []))
< self.routing_args.max_latency_list_size
):
latency_value = final_value.total_seconds() if isinstance(final_value, timedelta) else final_value
request_count_dict[id].setdefault("latency", []).append(latency_value)
request_count_dict[id].setdefault("latency", []).append(final_value)
else:
request_count_dict[id]["latency"] = request_count_dict[id][
"latency"
@ -334,9 +332,7 @@ class LowestLatencyLoggingHandler(CustomLogger):
len(request_count_dict[id].get("latency", []))
< self.routing_args.max_latency_list_size
):
latency_value = final_value.total_seconds() if isinstance(final_value, timedelta) else final_value
request_count_dict[id].setdefault("latency", []).append(latency_value)
request_count_dict[id].setdefault("latency", []).append(final_value)
else:
request_count_dict[id]["latency"] = request_count_dict[id][
"latency"
@ -476,15 +472,11 @@ class LowestLatencyLoggingHandler(CustomLogger):
for _call_latency in item_ttft_latency:
if isinstance(_call_latency, float):
total += _call_latency
total = total / len(item_ttft_latency) if total > 0 else float('inf')
elif len(item_latency) > 0 and isinstance(item_latency[0], timedelta):
for _call_latency in item_latency:
total += _call_latency.total_seconds()
else:
for _call_latency in item_latency:
if isinstance(_call_latency, float):
total += _call_latency
item_latency = total / len(item_latency) if total > 0 else float('inf')
item_latency = total / len(item_latency)
# -------------- #
# Debugging Logic
@ -511,9 +503,6 @@ class LowestLatencyLoggingHandler(CustomLogger):
if len(potential_deployments) == 0:
return None
if not healthy_deployments:
return None
# Sort potential deployments by latency
sorted_deployments = sorted(potential_deployments, key=lambda x: x[1])
@ -523,10 +512,9 @@ class LowestLatencyLoggingHandler(CustomLogger):
# Find deployments within buffer of lowest latency
buffer = self.routing_args.lowest_latency_buffer * lowest_latency
# If no deployments within buffer, fall back to all sorted deployments
valid_deployments = [
x for x in sorted_deployments if x[1] <= lowest_latency + buffer
] or sorted_deployments
x for x in sorted_deployments if x[1] <= lowest_latency + buffer
]
# Pick a random deployment from valid deployments
random_valid_deployment = random.choice(valid_deployments)