feat(lowest_latency.py): support expanded time window for latency based routing

uses a 1hr avg. of latency for deployments, to determine which to route to

https://github.com/BerriAI/litellm/issues/1361
This commit is contained in:
Krrish Dholakia 2024-01-08 22:52:32 +05:30 committed by ishaan-jaff
parent 9dc2bc227b
commit 832c10b402

View file

@ -142,7 +142,7 @@ class LowestLatencyLoggingHandler(CustomLogger):
continue # skip to next one
# get average latency
total = 0.0
total = 0
for _call_latency in item_latency:
if isinstance(_call_latency, timedelta):
total += float(_call_latency.total_seconds())