phoenix/litellm
1
0
Fork 0
forked from phoenix/litellm-mirror
Code Pull requests Activity

fix(router.py): improve error message returned for fallbacks

Browse source
This commit is contained in:
Krrish Dholakia 2024-06-25 11:26:56 -07:00
parent 2bd993039b
commit cccc55213b
3 changed files with 85 additions and 63 deletions
Download patch file Download diff file Expand all files Collapse all files

2
litellm/proxy/_new_secret_config.yaml
View file

@ -45,7 +45,7 @@ litellm_settings:
request_timeout: 120
allowed_fails: 3
# fallbacks: [{"summarize": ["summarize-l", "summarize-xl"]}, {"summarize-l": ["summarize-xl"]}]
context_window_fallbacks: [{"summarize": ["summarize-l", "summarize-xl"]}, {"summarize-l": ["summarize-xl"]}]
# context_window_fallbacks: [{"summarize": ["summarize-l", "summarize-xl"]}, {"summarize-l": ["summarize-xl"]}]

38
litellm/router.py
View file

@ -2175,10 +2175,8 @@ class Router:
)
): # don't retry a malformed request
raise e
if (
isinstance(e, litellm.ContextWindowExceededError)
and context_window_fallbacks is not None
):
if isinstance(e, litellm.ContextWindowExceededError):
if context_window_fallbacks is not None:
fallback_model_group = None
for (
item
@ -2208,10 +2206,20 @@ class Router:
return response
except Exception as e:
pass
elif (
isinstance(e, litellm.ContentPolicyViolationError)
and content_policy_fallbacks is not None
):
else:
error_message = "model={}. context_window_fallbacks={}. fallbacks={}.\n\nSet 'context_window_fallback' - https://docs.litellm.ai/docs/routing#fallbacks".format(
model_group, context_window_fallbacks, fallbacks
)
verbose_router_logger.info(
msg="Got 'ContextWindowExceededError'. No context_window_fallback set. Defaulting \
to fallbacks, if available.{}".format(
error_message
)
)
e.message += "\n{}".format(error_message)
elif isinstance(e, litellm.ContentPolicyViolationError):
if content_policy_fallbacks is not None:
fallback_model_group = None
for (
item
@ -2241,7 +2249,19 @@ class Router:
return response
except Exception as e:
pass
elif fallbacks is not None:
else:
error_message = "model={}. content_policy_fallback={}. fallbacks={}.\n\nSet 'content_policy_fallback' - https://docs.litellm.ai/docs/routing#fallbacks".format(
model_group, content_policy_fallbacks, fallbacks
)
verbose_router_logger.info(
msg="Got 'ContentPolicyViolationError'. No content_policy_fallback set. Defaulting \
to fallbacks, if available.{}".format(
error_message
)
)
e.message += "\n{}".format(error_message)
if fallbacks is not None:
verbose_router_logger.debug(f"inside model fallbacks: {fallbacks}")
generic_fallback_idx: Optional[int] = None
## check for specific model group-specific fallbacks

4
litellm/tests/test_router_fallbacks.py
View file

@ -1129,7 +1129,9 @@ async def test_router_content_policy_fallbacks(
mock_response = Exception("content filtering policy")
else:
mock_response = litellm.ModelResponse(
choices=[litellm.Choices(finish_reason="content_filter")]
choices=[litellm.Choices(finish_reason="content_filter")],
model="gpt-3.5-turbo",
usage=litellm.Usage(prompt_tokens=10, completion_tokens=0, total_tokens=10),
)
router = Router(
model_list=[