phoenix/litellm-mirror
0
0
Fork 1
mirror of https://github.com/BerriAI/litellm.git synced 2025-04-27 11:43:54 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

fix(lowest_latency.py): set default none value for time_to_first_token in sync log success event

Browse source
This commit is contained in:
Krrish Dholakia 2024-05-21 18:42:15 -07:00
parent bfde1357dd
commit f19d7327ca
1 changed files with 2 additions and 1 deletions
Download patch file Download diff file Expand all files Collapse all files

3
litellm/router_strategy/lowest_latency.py
View file

@ -84,6 +84,7 @@ class LowestLatencyLoggingHandler(CustomLogger):
response_ms: timedelta = end_time - start_time
time_to_first_token_response_time: Optional[timedelta] = None
if kwargs.get("stream", None) is not None and kwargs["stream"] == True:
# only log ttft for streaming request
time_to_first_token_response_time = (
@ -91,7 +92,7 @@ class LowestLatencyLoggingHandler(CustomLogger):
)
final_value = response_ms
time_to_first_token: Optional[float] = None
total_tokens = 0
if isinstance(response_obj, ModelResponse):