phoenix/litellm-mirror
0
0
Fork 1
mirror of https://github.com/BerriAI/litellm.git synced 2025-04-25 02:34:29 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

fix(router.py): adding health checks

Browse source
This commit is contained in:
Krrish Dholakia 2023-11-06 18:26:41 -08:00
parent 6a45879932
commit 2b48a1fc3e
2 changed files with 15 additions and 16 deletions
Download patch file Download diff file Expand all files Collapse all files

27
litellm/router.py
View file

@ -34,7 +34,7 @@ class Router:
self.set_model_list(model_list)
self.healthy_deployments = []
### HEALTH CHECK THREAD ### - commenting out as further testing required
# self._start_health_check_thread()
self._start_health_check_thread()
### CACHING ###
if redis_host is not None and redis_port is not None and redis_password is not None:
@ -105,19 +105,18 @@ class Router:
Returns the deployment with the shortest queue
"""
### COMMENTING OUT AS IT NEEDS FURTHER TESTING
# logging.debug(f"self.healthy_deployments: {self.healthy_deployments}")
# if len(self.healthy_deployments) > 0:
# for item in self.healthy_deployments:
# print(f"item: {item}")
# if item[0]["model_name"] == model: # first one in queue will be the one with the most availability
# return item
# else:
potential_deployments = []
for item in self.model_list:
if item["model_name"] == model:
potential_deployments.append(item)
item = random.choice(potential_deployments)
return item
logging.debug(f"self.healthy_deployments: {self.healthy_deployments}")
if len(self.healthy_deployments) > 0:
for item in self.healthy_deployments:
if item[0]["model_name"] == model: # first one in queue will be the one with the most availability
return item
else:
potential_deployments = []
for item in self.model_list:
if item["model_name"] == model:
potential_deployments.append(item)
item = random.choice(potential_deployments)
return item
raise ValueError("No models available.")