phoenix/litellm-mirror
0
0
Fork 1
mirror of https://github.com/BerriAI/litellm.git synced 2025-04-26 03:04:13 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge pull request #3716 from BerriAI/litellm_set_cooldown_time_based_on_exception_header

Browse source 
[Feat] Router/ Proxy - set cooldown_time based on Azure exception headers
This commit is contained in:
Ishaan Jaff 2024-05-20 16:34:12 -07:00 • committed by GitHub
commit 25fae37efc
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
2 changed files with 45 additions and 10 deletions
Download patch file Download diff file Expand all files Collapse all files

17
litellm/utils.py
View file

@ -8071,11 +8071,8 @@ def _should_retry(status_code: int):
return False
def _calculate_retry_after(
remaining_retries: int,
max_retries: int,
def _get_retry_after_from_exception_header(
response_headers: Optional[httpx.Headers] = None,
min_timeout: int = 0,
):
"""
Reimplementation of openai's calculate retry after, since that one can't be imported.
@ -8101,10 +8098,20 @@ def _calculate_retry_after(
retry_after = int(retry_date - time.time())
else:
retry_after = -1
return retry_after
except Exception:
except Exception as e:
retry_after = -1
def _calculate_retry_after(
remaining_retries: int,
max_retries: int,
response_headers: Optional[httpx.Headers] = None,
min_timeout: int = 0,
):
retry_after = _get_retry_after_from_exception_header(response_headers)
# If the API asks us to wait a certain amount of time (and it's a reasonable amount), just do what it says.
if 0 < retry_after <= 60:
return retry_after