phoenix/litellm
1
0
Fork 0
forked from phoenix/litellm-mirror
Code Pull requests Activity

fix(parallel_request_limiter.py): only update hidden params, don't set new (can lead to errors for responses where attribute can't be set)

Browse source
This commit is contained in:
Krrish Dholakia 2024-09-28 19:41:35 -07:00
parent fa64b6ca24
commit 6c7d1d5c96
1 changed files with 32 additions and 24 deletions
Download patch file Download diff file Expand all files Collapse all files

10
litellm/proxy/hooks/parallel_request_limiter.py
View file

@ -791,10 +791,18 @@ class _PROXY_MaxParallelRequestsHandler(CustomLogger):
) )
key_tpm_limit = user_api_key_dict.tpm_limit key_tpm_limit = user_api_key_dict.tpm_limit
_hidden_params = getattr(response, "_hidden_params", {}) or {} if hasattr(response, "_hidden_params"):
_hidden_params = getattr(response, "_hidden_params")
else:
_hidden_params = None
if _hidden_params is not None and (
isinstance(_hidden_params, BaseModel) or isinstance(_hidden_params, dict)
):
if isinstance(_hidden_params, BaseModel): if isinstance(_hidden_params, BaseModel):
_hidden_params = _hidden_params.model_dump() _hidden_params = _hidden_params.model_dump()
_additional_headers = _hidden_params.get("additional_headers", {}) or {} _additional_headers = _hidden_params.get("additional_headers", {}) or {}
if key_remaining_rpm_limit is not None: if key_remaining_rpm_limit is not None:
_additional_headers["x-ratelimit-remaining-requests"] = ( _additional_headers["x-ratelimit-remaining-requests"] = (
key_remaining_rpm_limit key_remaining_rpm_limit