phoenix/litellm-mirror
0
0
Fork 1
mirror of https://github.com/BerriAI/litellm.git synced 2025-04-26 03:04:13 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

fix(proxy): allow background health check interval to be any number

Browse source 
At the moment the code will only work when the interval is a float, otherwise it will perform health check requests in a tight loop.
This commit is contained in:
Florian Greinacher 2024-10-22 10:58:57 +02:00
parent a5fe87d6a5
commit a4d3307a84
No known key found for this signature in database
1 changed files with 10 additions and 5 deletions
Download patch file Download diff file Expand all files Collapse all files

11
litellm/proxy/proxy_server.py
View file

@ -1341,6 +1341,14 @@ async def _run_background_health_check():
if _llm_model_list is None: if _llm_model_list is None:
return return
try:
health_check_interval = float(health_check_interval) # type: ignore Type errors are handled in the exception handler below
except (ValueError, TypeError):
verbose_proxy_logger.error(
f"Not running background health checks due to invalid health_check_interval '{health_check_interval}'."
)
return
while True: while True:
healthy_endpoints, unhealthy_endpoints = await perform_health_check( healthy_endpoints, unhealthy_endpoints = await perform_health_check(
model_list=_llm_model_list, details=health_check_details model_list=_llm_model_list, details=health_check_details
@ -1352,9 +1360,6 @@ async def _run_background_health_check():
health_check_results["healthy_count"] = len(healthy_endpoints) health_check_results["healthy_count"] = len(healthy_endpoints)
health_check_results["unhealthy_count"] = len(unhealthy_endpoints) health_check_results["unhealthy_count"] = len(unhealthy_endpoints)
if health_check_interval is not None and isinstance(
health_check_interval, float
):
await asyncio.sleep(health_check_interval) await asyncio.sleep(health_check_interval)