phoenix/litellm-mirror
0
0
Fork 1
mirror of https://github.com/BerriAI/litellm.git synced 2025-04-26 11:14:04 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge pull request #3460 from BerriAI/litellm_use_retry_policy_per_mg

Browse source 
[Feat] Set a Retry Policy per model group
This commit is contained in:
Ishaan Jaff 2024-05-04 20:42:40 -07:00 • committed by GitHub
commit 040bf1f15e
2 changed files with 24 additions and 12 deletions
Download patch file Download diff file Expand all files Collapse all files

21
litellm/router.py
View file

@ -86,6 +86,9 @@ class Router:
retry_policy: Optional[
RetryPolicy
] = None, # set custom retries for different exceptions
model_group_retry_policy: Optional[
dict[str, RetryPolicy]
] = {}, # set custom retry policies based on model group
allowed_fails: Optional[
int
] = None, # Number of times a deployment can failbefore being added to cooldown
@ -308,6 +311,9 @@ class Router:
) # noqa
self.routing_strategy_args = routing_strategy_args
self.retry_policy: Optional[RetryPolicy] = retry_policy
self.model_group_retry_policy: Optional[dict[str, RetryPolicy]] = (
model_group_retry_policy
)
def routing_strategy_init(self, routing_strategy: str, routing_strategy_args: dict):
if routing_strategy == "least-busy":
@ -1509,11 +1515,13 @@ class Router:
)
await asyncio.sleep(_timeout)
## LOGGING
if self.retry_policy is not None or kwargs.get("retry_policy") is not None:
if (
self.retry_policy is not None
or self.model_group_retry_policy is not None
):
# get num_retries from retry policy
_retry_policy_retries = self.get_num_retries_from_retry_policy(
exception=original_exception,
dynamic_retry_policy=kwargs.get("retry_policy"),
exception=original_exception, model_group=kwargs.get("model")
)
if _retry_policy_retries is not None:
num_retries = _retry_policy_retries
@ -3273,7 +3281,7 @@ class Router:
verbose_router_logger.error(f"Error in _track_deployment_metrics: {str(e)}")
def get_num_retries_from_retry_policy(
self, exception: Exception, dynamic_retry_policy: Optional[RetryPolicy] = None
self, exception: Exception, model_group: Optional[str] = None
):
"""
BadRequestErrorRetries: Optional[int] = None
@ -3284,8 +3292,9 @@ class Router:
"""
# if we can find the exception then in the retry policy -> return the number of retries
retry_policy = self.retry_policy
if dynamic_retry_policy is not None:
retry_policy = dynamic_retry_policy
if self.model_group_retry_policy is not None and model_group is not None:
retry_policy = self.model_group_retry_policy.get(model_group, None)
if retry_policy is None:
return None
if (