phoenix/litellm
1
0
Fork 0
forked from phoenix/litellm-mirror
Code Pull requests Activity

fix(lowest_tpm_rpm.py): handle async scenarios

Browse source
This commit is contained in:
Krrish Dholakia 2024-03-06 21:38:30 -08:00
parent ec79482612
commit 0273410310
1 changed files with 2 additions and 3 deletions
Download patch file Download diff file Expand all files Collapse all files

5
litellm/router_strategy/lowest_tpm_rpm.py
View file

@ -200,9 +200,8 @@ class LowestTPMLoggingHandler(CustomLogger):
if item_tpm == 0:
deployment = _deployment
break
elif (
item_tpm + input_tokens > _deployment_tpm
or rpm_dict[item] + 1 > _deployment_rpm
elif item_tpm + input_tokens > _deployment_tpm or (
item in rpm_dict and rpm_dict[item] + 1 > _deployment_rpm
): # if user passed in tpm / rpm in the model_list
continue
elif item_tpm < lowest_tpm: