phoenix/litellm
1
0
Fork 0
forked from phoenix/litellm-mirror
Code Pull requests Activity

fix(router.py): make sure pre call rpm check runs even when model not in model cost map

Browse source
This commit is contained in:
Krrish Dholakia 2024-04-11 09:27:26 -07:00
parent 240aaf7af8
commit 84d43484c6
3 changed files with 52 additions and 11 deletions
Download patch file Download diff file Expand all files Collapse all files

22
litellm/router.py
View file

@ -2357,20 +2357,20 @@ class Router:
"model", None
)
model_info = litellm.get_model_info(model=model)
except:
continue
if (
isinstance(model_info, dict)
and model_info.get("max_input_tokens", None) is not None
):
if (
isinstance(model_info["max_input_tokens"], int)
and input_tokens > model_info["max_input_tokens"]
isinstance(model_info, dict)
and model_info.get("max_input_tokens", None) is not None
):
invalid_model_indices.append(idx)
_context_window_error = True
continue
if (
isinstance(model_info["max_input_tokens"], int)
and input_tokens > model_info["max_input_tokens"]
):
invalid_model_indices.append(idx)
_context_window_error = True
continue
except Exception as e:
verbose_router_logger.debug("An error occurs - {}".format(str(e)))
## RPM CHECK ##
_litellm_params = deployment.get("litellm_params", {})