phoenix/litellm-mirror
0
0
Fork 1
mirror of https://github.com/BerriAI/litellm.git synced 2025-04-26 03:04:13 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

(fix) proxy - health checks support cli model

Browse source
This commit is contained in:
ishaan-jaff 2023-12-19 12:55:20 +05:30
parent b82fcd51d7
commit ce1b0b89ba
2 changed files with 15 additions and 3 deletions
Download patch file Download diff file Expand all files Collapse all files

5
litellm/proxy/health_check.py
View file

@ -96,7 +96,7 @@ async def _perform_health_check(model_list: list):
async def perform_health_check(model_list: list, model: Optional[str] = None): async def perform_health_check(model_list: list, model: Optional[str] = None, cli_model: Optional[str] = None):
""" """
Perform a health check on the system. Perform a health check on the system.
@ -104,6 +104,9 @@ async def perform_health_check(model_list: list, model: Optional[str] = None):
(bool): True if the health check passes, False otherwise. (bool): True if the health check passes, False otherwise.
""" """
if not model_list: if not model_list:
if cli_model:
model_list = [{"model_name": cli_model, "litellm_params": {"model": cli_model}}]
else:
return [], [] return [], []
if model is not None: if model is not None:

11
litellm/proxy/proxy_server.py
View file

@ -1514,9 +1514,18 @@ async def health_endpoint(request: Request, model: Optional[str] = fastapi.Query
``` ```
else, the health checks will be run on models when /health is called. else, the health checks will be run on models when /health is called.
""" """
global health_check_results, use_background_health_checks global health_check_results, use_background_health_checks, user_model
if llm_model_list is None: if llm_model_list is None:
# if no router set, check if user set a model using litellm --model ollama/llama2
if user_model is not None:
healthy_endpoints, unhealthy_endpoints = await perform_health_check(model_list=[], cli_model=user_model)
return {
"healthy_endpoints": healthy_endpoints,
"unhealthy_endpoints": unhealthy_endpoints,
"healthy_count": len(healthy_endpoints),
"unhealthy_count": len(unhealthy_endpoints),
}
raise HTTPException( raise HTTPException(
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
detail={"error": "Model list not initialized"}, detail={"error": "Model list not initialized"},