phoenix/litellm
1
0
Fork 0
forked from phoenix/litellm-mirror
Code Pull requests Activity

Merge pull request #2493 from BerriAI/litellm_return_429_no_models_available

Browse source 
[Proxy] return 429 when no models available
This commit is contained in:
Ishaan Jaff 2024-03-13 13:33:43 -07:00 • committed by GitHub
commit d82be720d2
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
2 changed files with 18 additions and 12 deletions
Download patch file Download diff file Expand all files Collapse all files

24
litellm/proxy/proxy_server.py
View file

@ -171,6 +171,15 @@ class ProxyException(Exception):
self.param = param self.param = param
self.code = code self.code = code
# rules for proxyExceptions
# Litellm router.py returns "No healthy deployment available" when there are no deployments available
# Should map to 429 errors https://github.com/BerriAI/litellm/issues/2487
if (
"No healthy deployment available" in self.message
or "No deployments available" in self.message
):
self.code = 429
def to_dict(self) -> dict: def to_dict(self) -> dict:
"""Converts the ProxyException instance to a dictionary.""" """Converts the ProxyException instance to a dictionary."""
return { return {
@ -2919,10 +2928,7 @@ async def chat_completion(
param=getattr(e, "param", "None"), param=getattr(e, "param", "None"),
code=getattr(e, "status_code", status.HTTP_400_BAD_REQUEST), code=getattr(e, "status_code", status.HTTP_400_BAD_REQUEST),
) )
else: error_msg = f"{str(e)}"
error_traceback = traceback.format_exc()
error_msg = f"{str(e)}\n\n{error_traceback}"
raise ProxyException( raise ProxyException(
message=getattr(e, "message", error_msg), message=getattr(e, "message", error_msg),
type=getattr(e, "type", "None"), type=getattr(e, "type", "None"),
@ -3105,8 +3111,7 @@ async def embeddings(
code=getattr(e, "status_code", status.HTTP_400_BAD_REQUEST), code=getattr(e, "status_code", status.HTTP_400_BAD_REQUEST),
) )
else: else:
error_traceback = traceback.format_exc() error_msg = f"{str(e)}"
error_msg = f"{str(e)}\n\n{error_traceback}"
raise ProxyException( raise ProxyException(
message=getattr(e, "message", error_msg), message=getattr(e, "message", error_msg),
type=getattr(e, "type", "None"), type=getattr(e, "type", "None"),
@ -3256,8 +3261,7 @@ async def image_generation(
code=getattr(e, "status_code", status.HTTP_400_BAD_REQUEST), code=getattr(e, "status_code", status.HTTP_400_BAD_REQUEST),
) )
else: else:
error_traceback = traceback.format_exc() error_msg = f"{str(e)}"
error_msg = f"{str(e)}\n\n{error_traceback}"
raise ProxyException( raise ProxyException(
message=getattr(e, "message", error_msg), message=getattr(e, "message", error_msg),
type=getattr(e, "type", "None"), type=getattr(e, "type", "None"),
@ -3420,7 +3424,7 @@ async def audio_transcriptions(
) )
else: else:
error_traceback = traceback.format_exc() error_traceback = traceback.format_exc()
error_msg = f"{str(e)}\n\n{error_traceback}" error_msg = f"{str(e)}"
raise ProxyException( raise ProxyException(
message=getattr(e, "message", error_msg), message=getattr(e, "message", error_msg),
type=getattr(e, "type", "None"), type=getattr(e, "type", "None"),
@ -3573,7 +3577,7 @@ async def moderations(
) )
else: else:
error_traceback = traceback.format_exc() error_traceback = traceback.format_exc()
error_msg = f"{str(e)}\n\n{error_traceback}" error_msg = f"{str(e)}"
raise ProxyException( raise ProxyException(
message=getattr(e, "message", error_msg), message=getattr(e, "message", error_msg),
type=getattr(e, "type", "None"), type=getattr(e, "type", "None"),

6
litellm/router.py
View file

@ -2204,7 +2204,7 @@ class Router:
f"healthy deployments: length {len(healthy_deployments)} {healthy_deployments}" f"healthy deployments: length {len(healthy_deployments)} {healthy_deployments}"
) )
if len(healthy_deployments) == 0: if len(healthy_deployments) == 0:
raise ValueError("No models available") raise ValueError(f"No healthy deployment available, passed model={model}")
if litellm.model_alias_map and model in litellm.model_alias_map: if litellm.model_alias_map and model in litellm.model_alias_map:
model = litellm.model_alias_map[ model = litellm.model_alias_map[
model model
@ -2275,7 +2275,9 @@ class Router:
verbose_router_logger.info( verbose_router_logger.info(
f"get_available_deployment for model: {model}, No deployment available" f"get_available_deployment for model: {model}, No deployment available"
) )
raise ValueError("No models available.") raise ValueError(
f"No deployments available for selected model, passed model={model}"
)
verbose_router_logger.info( verbose_router_logger.info(
f"get_available_deployment for model: {model}, Selected deployment: {self.print_deployment(deployment)} for model: {model}" f"get_available_deployment for model: {model}, Selected deployment: {self.print_deployment(deployment)} for model: {model}"
) )