phoenix/litellm-mirror
0
0
Fork 1
mirror of https://github.com/BerriAI/litellm.git synced 2025-04-25 18:54:30 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

fix(test_parallel_request_limiter.py): use mock responses for streaming

Browse source
This commit is contained in:
Krrish Dholakia 2024-02-08 21:45:24 -08:00
parent 1ef7ad3416
commit b9393fb769
5 changed files with 35 additions and 5 deletions
Download patch file Download diff file Expand all files Collapse all files

4
litellm/proxy/hooks/parallel_request_limiter.py
View file

@ -125,7 +125,7 @@ class _PROXY_MaxParallelRequestsHandler(CustomLogger):
# ------------
new_val = {
"current_requests": current["current_requests"] - 1,
"current_requests": max(current["current_requests"] - 1, 0),
"current_tpm": current["current_tpm"] + total_tokens,
"current_rpm": current["current_rpm"] + 1,
}
@ -183,7 +183,7 @@ class _PROXY_MaxParallelRequestsHandler(CustomLogger):
}
new_val = {
"current_requests": current["current_requests"] - 1,
"current_requests": max(current["current_requests"] - 1, 0),
"current_tpm": current["current_tpm"],
"current_rpm": current["current_rpm"],
}