phoenix/litellm-mirror
0
0
Fork 1
mirror of https://github.com/BerriAI/litellm.git synced 2025-04-24 18:24:20 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

fix(proxy): allow background health check interval to be any number

Browse source 
At the moment the code will only work when the interval is a float, otherwise it will perform health check requests in a tight loop.
This commit is contained in:
Florian Greinacher 2024-10-22 10:58:57 +02:00
parent a5fe87d6a5
commit a4d3307a84
No known key found for this signature in database
1 changed files with 10 additions and 5 deletions
Download patch file Download diff file Expand all files Collapse all files

15
litellm/proxy/proxy_server.py
View file

@ -1341,6 +1341,14 @@ async def _run_background_health_check():
if _llm_model_list is None:
return
try:
health_check_interval = float(health_check_interval) # type: ignore Type errors are handled in the exception handler below
except (ValueError, TypeError):
verbose_proxy_logger.error(
f"Not running background health checks due to invalid health_check_interval '{health_check_interval}'."
)
return
while True:
healthy_endpoints, unhealthy_endpoints = await perform_health_check(
model_list=_llm_model_list, details=health_check_details
@ -1352,10 +1360,7 @@ async def _run_background_health_check():
health_check_results["healthy_count"] = len(healthy_endpoints)
health_check_results["unhealthy_count"] = len(unhealthy_endpoints)
if health_check_interval is not None and isinstance(
health_check_interval, float
):
await asyncio.sleep(health_check_interval)
await asyncio.sleep(health_check_interval)
class ProxyConfig:
@ -1543,7 +1548,7 @@ class ProxyConfig:
## INIT PROXY REDIS USAGE CLIENT ##
redis_usage_cache = litellm.cache.cache
async def get_config(self, config_file_path: Optional[str] = None) -> dict:
"""
Load config file