phoenix/litellm-mirror
0
0
Fork 1
mirror of https://github.com/BerriAI/litellm.git synced 2025-04-25 10:44:24 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

fix(router.py): correctly raise no model available error

Browse source 
https://github.com/BerriAI/litellm/issues/1289
This commit is contained in:
Krrish Dholakia 2024-01-01 21:22:42 +05:30
parent 91e6b2e79a
commit a83e2e07cf
4 changed files with 86 additions and 21 deletions
Download patch file Download diff file Expand all files Collapse all files

21
litellm/router.py
View file

@ -1705,7 +1705,6 @@ class Router:
deployment = self.leastbusy_logger.get_available_deployments(
model_group=model, healthy_deployments=healthy_deployments
)
return deployment
elif self.routing_strategy == "simple-shuffle":
# if users pass rpm or tpm, we do a random weighted pick - based on rpm/tpm
############## Check if we can do a RPM/TPM based weighted pick #################
@ -1744,24 +1743,24 @@ class Router:
self.routing_strategy == "latency-based-routing"
and self.lowestlatency_logger is not None
):
min_deployment = self.lowestlatency_logger.get_available_deployments(
deployment = self.lowestlatency_logger.get_available_deployments(
model_group=model, healthy_deployments=healthy_deployments
)
if min_deployment is None:
min_deployment = random.choice(healthy_deployments)
return min_deployment
elif (
self.routing_strategy == "usage-based-routing"
and self.lowesttpm_logger is not None
):
min_deployment = self.lowesttpm_logger.get_available_deployments(
model_group=model, healthy_deployments=healthy_deployments
deployment = self.lowesttpm_logger.get_available_deployments(
model_group=model,
healthy_deployments=healthy_deployments,
messages=messages,
input=input,
)
if min_deployment is None:
min_deployment = random.choice(healthy_deployments)
return min_deployment
raise ValueError("No models available.")
if deployment is None:
raise ValueError("No models available.")
return deployment
def flush_cache(self):
litellm.cache = None