phoenix/litellm-mirror
0
0
Fork 1
mirror of https://github.com/BerriAI/litellm.git synced 2025-04-26 11:14:04 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

fix

Browse source
This commit is contained in:
sumanth 2023-12-19 13:15:48 +05:30
parent 121149fed4
commit 64dfd0d9a6
2 changed files with 53 additions and 2 deletions
Download patch file Download diff file Expand all files Collapse all files

14
litellm/router.py
View file

@ -770,10 +770,22 @@ class Router:
# ----------------------
lowest_tpm = float("inf")
deployment = None
# load model context map
models_context_map = litellm.model_cost
# return deployment with lowest tpm usage
for item in potential_deployments:
item_tpm, item_rpm = self._get_deployment_usage(deployment_name=item["litellm_params"]["model"])
deployment_name=item["litellm_params"]["model"]
custom_llm_provider = item["litellm_params"].get("custom_llm_provider", None)
if custom_llm_provider is not None:
deployment_name = f"{custom_llm_provider}/{deployment_name}"
else:
litellm_provider = models_context_map.get(deployment_name, {}).get("litellm_provider", None)
if litellm_provider is not None:
deployment_name = f"{litellm_provider}/{deployment_name}"
item_tpm, item_rpm = self._get_deployment_usage(deployment_name=deployment_name)
if item_tpm == 0:
return item