fix(router.py): support context window fallbacks for pre-call checks

This commit is contained in:
Krrish Dholakia 2024-04-01 10:51:54 -07:00
parent f46a9d09a5
commit 52b1538b2e
2 changed files with 61 additions and 1 deletions

View file

@ -2210,6 +2210,18 @@ class Router:
):
invalid_model_indices.append(idx)
if len(invalid_model_indices) == len(_returned_deployments):
"""
- no healthy deployments available b/c context window checks
"""
raise litellm.ContextWindowExceededError(
message="Context Window exceeded for given call",
model=model,
llm_provider="",
response=httpx.Response(
status_code=400, request=httpx.Request("GET", "https://example.com")
),
)
if len(invalid_model_indices) > 0:
for idx in reversed(invalid_model_indices):
_returned_deployments.pop(idx)