phoenix/litellm
1
0
Fork 0
forked from phoenix/litellm-mirror
Code Pull requests Activity

fix(router.py): improve error message returned for fallbacks

Browse source
This commit is contained in:
Krrish Dholakia 2024-06-25 11:26:56 -07:00
parent 2bd993039b
commit cccc55213b
3 changed files with 85 additions and 63 deletions
Download patch file Download diff file Expand all files Collapse all files

2
litellm/proxy/_new_secret_config.yaml
View file

@ -45,7 +45,7 @@ litellm_settings:
request_timeout: 120 request_timeout: 120
allowed_fails: 3 allowed_fails: 3
# fallbacks: [{"summarize": ["summarize-l", "summarize-xl"]}, {"summarize-l": ["summarize-xl"]}] # fallbacks: [{"summarize": ["summarize-l", "summarize-xl"]}, {"summarize-l": ["summarize-xl"]}]
context_window_fallbacks: [{"summarize": ["summarize-l", "summarize-xl"]}, {"summarize-l": ["summarize-xl"]}] # context_window_fallbacks: [{"summarize": ["summarize-l", "summarize-xl"]}, {"summarize-l": ["summarize-xl"]}]

38
litellm/router.py
View file

@ -2175,10 +2175,8 @@ class Router:
) )
): # don't retry a malformed request ): # don't retry a malformed request
raise e raise e
if ( if isinstance(e, litellm.ContextWindowExceededError):
isinstance(e, litellm.ContextWindowExceededError) if context_window_fallbacks is not None:
and context_window_fallbacks is not None
):
fallback_model_group = None fallback_model_group = None
for ( for (
item item
@ -2208,10 +2206,20 @@ class Router:
return response return response
except Exception as e: except Exception as e:
pass pass
elif ( else:
isinstance(e, litellm.ContentPolicyViolationError) error_message = "model={}. context_window_fallbacks={}. fallbacks={}.\n\nSet 'context_window_fallback' - https://docs.litellm.ai/docs/routing#fallbacks".format(
and content_policy_fallbacks is not None model_group, context_window_fallbacks, fallbacks
): )
verbose_router_logger.info(
msg="Got 'ContextWindowExceededError'. No context_window_fallback set. Defaulting \
to fallbacks, if available.{}".format(
error_message
)
)
e.message += "\n{}".format(error_message)
elif isinstance(e, litellm.ContentPolicyViolationError):
if content_policy_fallbacks is not None:
fallback_model_group = None fallback_model_group = None
for ( for (
item item
@ -2241,7 +2249,19 @@ class Router:
return response return response
except Exception as e: except Exception as e:
pass pass
elif fallbacks is not None: else:
error_message = "model={}. content_policy_fallback={}. fallbacks={}.\n\nSet 'content_policy_fallback' - https://docs.litellm.ai/docs/routing#fallbacks".format(
model_group, content_policy_fallbacks, fallbacks
)
verbose_router_logger.info(
msg="Got 'ContentPolicyViolationError'. No content_policy_fallback set. Defaulting \
to fallbacks, if available.{}".format(
error_message
)
)
e.message += "\n{}".format(error_message)
if fallbacks is not None:
verbose_router_logger.debug(f"inside model fallbacks: {fallbacks}") verbose_router_logger.debug(f"inside model fallbacks: {fallbacks}")
generic_fallback_idx: Optional[int] = None generic_fallback_idx: Optional[int] = None
## check for specific model group-specific fallbacks ## check for specific model group-specific fallbacks

4
litellm/tests/test_router_fallbacks.py
View file

@ -1129,7 +1129,9 @@ async def test_router_content_policy_fallbacks(
mock_response = Exception("content filtering policy") mock_response = Exception("content filtering policy")
else: else:
mock_response = litellm.ModelResponse( mock_response = litellm.ModelResponse(
choices=[litellm.Choices(finish_reason="content_filter")] choices=[litellm.Choices(finish_reason="content_filter")],
model="gpt-3.5-turbo",
usage=litellm.Usage(prompt_tokens=10, completion_tokens=0, total_tokens=10),
) )
router = Router( router = Router(
model_list=[ model_list=[