From 0273410310a45a87285bae16dc4561bfabd138e1 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Wed, 6 Mar 2024 21:38:30 -0800 Subject: [PATCH] fix(lowest_tpm_rpm.py): handle async scenarios --- litellm/router_strategy/lowest_tpm_rpm.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/litellm/router_strategy/lowest_tpm_rpm.py b/litellm/router_strategy/lowest_tpm_rpm.py index e97d81aa1..9f0b9eb22 100644 --- a/litellm/router_strategy/lowest_tpm_rpm.py +++ b/litellm/router_strategy/lowest_tpm_rpm.py @@ -200,9 +200,8 @@ class LowestTPMLoggingHandler(CustomLogger): if item_tpm == 0: deployment = _deployment break - elif ( - item_tpm + input_tokens > _deployment_tpm - or rpm_dict[item] + 1 > _deployment_rpm + elif item_tpm + input_tokens > _deployment_tpm or ( + item in rpm_dict and rpm_dict[item] + 1 > _deployment_rpm ): # if user passed in tpm / rpm in the model_list continue elif item_tpm < lowest_tpm: