phoenix/litellm
1
0
Fork 0
forked from phoenix/litellm-mirror
Code Pull requests Activity

fix(router.py): handle initial scenario for tpm/rpm routing

Browse source
This commit is contained in:
Krrish Dholakia 2023-12-30 07:28:13 +05:30
parent 05795ead59
commit a34de56289
3 changed files with 9 additions and 4 deletions
Download patch file Download diff file Expand all files Collapse all files

7
litellm/router.py
View file

@ -1606,7 +1606,12 @@ class Router:
self.routing_strategy == "usage-based-routing"
and self.lowesttpm_logger is not None
):
return self.lowesttpm_logger.get_available_deployments(model_group=model)
min_deployment = self.lowesttpm_logger.get_available_deployments(
model_group=model
)
if min_deployment is None:
min_deployment = random.choice(healthy_deployments)
return min_deployment
raise ValueError("No models available.")