phoenix/litellm-mirror
0
0
Fork 1
mirror of https://github.com/BerriAI/litellm.git synced 2025-04-25 18:54:30 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

fix(router.py): adding health checks

Browse source
This commit is contained in:
Krrish Dholakia 2023-11-06 18:26:41 -08:00
parent 6a45879932
commit 2b48a1fc3e
2 changed files with 15 additions and 16 deletions
Download patch file Download diff file Expand all files Collapse all files

15
litellm/router.py
View file

@ -34,7 +34,7 @@ class Router:
self.set_model_list(model_list)
self.healthy_deployments = []
### HEALTH CHECK THREAD ### - commenting out as further testing required
# self._start_health_check_thread()
self._start_health_check_thread()
### CACHING ###
if redis_host is not None and redis_port is not None and redis_password is not None:
@ -105,13 +105,12 @@ class Router:
Returns the deployment with the shortest queue
"""
### COMMENTING OUT AS IT NEEDS FURTHER TESTING
# logging.debug(f"self.healthy_deployments: {self.healthy_deployments}")
# if len(self.healthy_deployments) > 0:
# for item in self.healthy_deployments:
# print(f"item: {item}")
# if item[0]["model_name"] == model: # first one in queue will be the one with the most availability
# return item
# else:
logging.debug(f"self.healthy_deployments: {self.healthy_deployments}")
if len(self.healthy_deployments) > 0:
for item in self.healthy_deployments:
if item[0]["model_name"] == model: # first one in queue will be the one with the most availability
return item
else:
potential_deployments = []
for item in self.model_list:
if item["model_name"] == model:

4
pyproject.toml
View file

@ -1,6 +1,6 @@
[tool.poetry]
name = "litellm"
version = "0.13.6"
version = "0.13.7"
description = "Library to easily interface with LLM API providers"
authors = ["BerriAI"]
license = "MIT License"
@ -26,7 +26,7 @@ requires = ["poetry-core"]
build-backend = "poetry.core.masonry.api"
[tool.commitizen]
version = "0.13.6"
version = "0.13.7"
version_files = [
"pyproject.toml:^version"
]