phoenix/litellm-mirror
0
0
Fork 1
mirror of https://github.com/BerriAI/litellm.git synced 2025-04-24 18:24:20 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge f449857d57 into b82af5b826

Browse source
This commit is contained in:
Vedran Vidovic 2025-04-24 06:57:43 +00:00 • committed by GitHub
commit 91c1d33870
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
2 changed files with 30 additions and 1 deletions
Download patch file Download diff file Expand all files Collapse all files

2
litellm/proxy/hooks/parallel_request_limiter.py
View file

@ -222,7 +222,7 @@ class _PROXY_MaxParallelRequestsHandler(CustomLogger):
)
# check if below limit
if current_global_requests is None:
current_global_requests = 1
current_global_requests = 0
# if above -> raise error
if current_global_requests >= global_max_parallel_requests:
return self.raise_rate_limit_error(

29
tests/local_testing/test_parallel_request_limiter.py
View file

@ -67,6 +67,35 @@ async def test_global_max_parallel_requests():
except Exception as e:
print(e)
# Test: n requests (up to global_max_parallel_requests) must succeed
# and (n+1)th request must fail.
global_max_parallel_requests = 4
for _ in range(global_max_parallel_requests):
await parallel_request_handler.async_pre_call_hook(
user_api_key_dict=user_api_key_dict,
cache=local_cache,
data={
"metadata": {
"global_max_parallel_requests": global_max_parallel_requests
}
},
call_type="",
)
try:
await parallel_request_handler.async_pre_call_hook(
user_api_key_dict=user_api_key_dict,
cache=local_cache,
data={
"metadata": {
"global_max_parallel_requests": global_max_parallel_requests
}
},
call_type="",
)
pytest.fail("Expected call to fail")
except Exception as e:
print(e)
@pytest.mark.flaky(retries=6, delay=1)
@pytest.mark.asyncio