phoenix/litellm-mirror
0
0
Fork 1
mirror of https://github.com/BerriAI/litellm.git synced 2025-04-25 18:54:30 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

fix(router.py): adding health checks

Browse source
This commit is contained in:
Krrish Dholakia 2023-11-06 18:26:41 -08:00
parent 6a45879932
commit 2b48a1fc3e
2 changed files with 15 additions and 16 deletions
Download patch file Download diff file Expand all files Collapse all files

27
litellm/router.py
View file

@ -34,7 +34,7 @@ class Router:
self.set_model_list(model_list) self.set_model_list(model_list)
self.healthy_deployments = [] self.healthy_deployments = []
### HEALTH CHECK THREAD ### - commenting out as further testing required ### HEALTH CHECK THREAD ### - commenting out as further testing required
# self._start_health_check_thread() self._start_health_check_thread()
### CACHING ### ### CACHING ###
if redis_host is not None and redis_port is not None and redis_password is not None: if redis_host is not None and redis_port is not None and redis_password is not None:
@ -105,19 +105,18 @@ class Router:
Returns the deployment with the shortest queue Returns the deployment with the shortest queue
""" """
### COMMENTING OUT AS IT NEEDS FURTHER TESTING ### COMMENTING OUT AS IT NEEDS FURTHER TESTING
# logging.debug(f"self.healthy_deployments: {self.healthy_deployments}") logging.debug(f"self.healthy_deployments: {self.healthy_deployments}")
# if len(self.healthy_deployments) > 0: if len(self.healthy_deployments) > 0:
# for item in self.healthy_deployments: for item in self.healthy_deployments:
# print(f"item: {item}") if item[0]["model_name"] == model: # first one in queue will be the one with the most availability
# if item[0]["model_name"] == model: # first one in queue will be the one with the most availability return item
# return item else:
# else: potential_deployments = []
potential_deployments = [] for item in self.model_list:
for item in self.model_list: if item["model_name"] == model:
if item["model_name"] == model: potential_deployments.append(item)
potential_deployments.append(item) item = random.choice(potential_deployments)
item = random.choice(potential_deployments) return item
return item
raise ValueError("No models available.") raise ValueError("No models available.")

4
pyproject.toml
View file

@ -1,6 +1,6 @@
[tool.poetry] [tool.poetry]
name = "litellm" name = "litellm"
version = "0.13.6" version = "0.13.7"
description = "Library to easily interface with LLM API providers" description = "Library to easily interface with LLM API providers"
authors = ["BerriAI"] authors = ["BerriAI"]
license = "MIT License" license = "MIT License"
@ -26,7 +26,7 @@ requires = ["poetry-core"]
build-backend = "poetry.core.masonry.api" build-backend = "poetry.core.masonry.api"
[tool.commitizen] [tool.commitizen]
version = "0.13.6" version = "0.13.7"
version_files = [ version_files = [
"pyproject.toml:^version" "pyproject.toml:^version"
] ]