phoenix/litellm-mirror
0
0
Fork 1
mirror of https://github.com/BerriAI/litellm.git synced 2025-04-25 10:44:24 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge pull request #1182 from sumanth13131/usage-based-routing-fix

Browse source 
usage_based_routing_fix
This commit is contained in:
Krish Dholakia 2023-12-23 11:50:34 +05:30 • committed by GitHub
commit 2df5ce4b7c
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
2 changed files with 53 additions and 2 deletions
Download patch file Download diff file Expand all files Collapse all files

14
litellm/router.py
View file

@ -860,10 +860,22 @@ class Router:
# ----------------------
lowest_tpm = float("inf")
deployment = None
# load model context map
models_context_map = litellm.model_cost
# return deployment with lowest tpm usage
for item in potential_deployments:
item_tpm, item_rpm = self._get_deployment_usage(deployment_name=item["litellm_params"]["model"])
deployment_name=item["litellm_params"]["model"]
custom_llm_provider = item["litellm_params"].get("custom_llm_provider", None)
if custom_llm_provider is not None:
deployment_name = f"{custom_llm_provider}/{deployment_name}"
else:
litellm_provider = models_context_map.get(deployment_name, {}).get("litellm_provider", None)
if litellm_provider is not None:
deployment_name = f"{litellm_provider}/{deployment_name}"
item_tpm, item_rpm = self._get_deployment_usage(deployment_name=deployment_name)
if item_tpm == 0:
return item