phoenix/litellm
1
0
Fork 0
forked from phoenix/litellm-mirror
Code Pull requests Activity

fix(lowest_tpm_rpm.py): handle async scenarios

Browse source
This commit is contained in:
Krrish Dholakia 2024-03-06 21:38:30 -08:00
parent ec79482612
commit 0273410310
1 changed files with 2 additions and 3 deletions
Download patch file Download diff file Expand all files Collapse all files

5
litellm/router_strategy/lowest_tpm_rpm.py
View file

@ -200,9 +200,8 @@ class LowestTPMLoggingHandler(CustomLogger):
if item_tpm == 0: if item_tpm == 0:
deployment = _deployment deployment = _deployment
break break
elif ( elif item_tpm + input_tokens > _deployment_tpm or (
item_tpm + input_tokens > _deployment_tpm item in rpm_dict and rpm_dict[item] + 1 > _deployment_rpm
or rpm_dict[item] + 1 > _deployment_rpm
): # if user passed in tpm / rpm in the model_list ): # if user passed in tpm / rpm in the model_list
continue continue
elif item_tpm < lowest_tpm: elif item_tpm < lowest_tpm: