mirror of
https://github.com/BerriAI/litellm.git
synced 2025-04-25 18:54:30 +00:00
Don't fail for global_max_parallel_requests
= 1
When `global_max_parallel_requests` was set to `1`, all requests failed.
This commit is contained in:
parent
f670ebeb2f
commit
f449857d57
2 changed files with 30 additions and 1 deletions
|
@ -222,7 +222,7 @@ class _PROXY_MaxParallelRequestsHandler(CustomLogger):
|
||||||
)
|
)
|
||||||
# check if below limit
|
# check if below limit
|
||||||
if current_global_requests is None:
|
if current_global_requests is None:
|
||||||
current_global_requests = 1
|
current_global_requests = 0
|
||||||
# if above -> raise error
|
# if above -> raise error
|
||||||
if current_global_requests >= global_max_parallel_requests:
|
if current_global_requests >= global_max_parallel_requests:
|
||||||
return self.raise_rate_limit_error(
|
return self.raise_rate_limit_error(
|
||||||
|
|
|
@ -67,6 +67,35 @@ async def test_global_max_parallel_requests():
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
print(e)
|
print(e)
|
||||||
|
|
||||||
|
# Test: n requests (up to global_max_parallel_requests) must succeed
|
||||||
|
# and (n+1)th request must fail.
|
||||||
|
global_max_parallel_requests = 4
|
||||||
|
for _ in range(global_max_parallel_requests):
|
||||||
|
await parallel_request_handler.async_pre_call_hook(
|
||||||
|
user_api_key_dict=user_api_key_dict,
|
||||||
|
cache=local_cache,
|
||||||
|
data={
|
||||||
|
"metadata": {
|
||||||
|
"global_max_parallel_requests": global_max_parallel_requests
|
||||||
|
}
|
||||||
|
},
|
||||||
|
call_type="",
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
await parallel_request_handler.async_pre_call_hook(
|
||||||
|
user_api_key_dict=user_api_key_dict,
|
||||||
|
cache=local_cache,
|
||||||
|
data={
|
||||||
|
"metadata": {
|
||||||
|
"global_max_parallel_requests": global_max_parallel_requests
|
||||||
|
}
|
||||||
|
},
|
||||||
|
call_type="",
|
||||||
|
)
|
||||||
|
pytest.fail("Expected call to fail")
|
||||||
|
except Exception as e:
|
||||||
|
print(e)
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.flaky(retries=6, delay=1)
|
@pytest.mark.flaky(retries=6, delay=1)
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
|
|
Loading…
Add table
Add a link
Reference in a new issue