phoenix/litellm-mirror
0
0
Fork 1
mirror of https://github.com/BerriAI/litellm.git synced 2025-04-26 11:14:04 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

test(test_router_max_parallel_requests.py): more extensive testing for setting max parallel requests

Browse source
This commit is contained in:
Krrish Dholakia 2024-04-20 12:56:54 -07:00
parent a9108cbdc2
commit 9f6e90e17d
3 changed files with 69 additions and 4 deletions
Download patch file Download diff file Expand all files Collapse all files

2
litellm/router.py
View file

@ -2562,7 +2562,7 @@ class Router:
"""
model_id = deployment["model_info"]["id"]
if client_type == "max_parallel_requests":
cache_key = "{}_max_parallel_requests".format(model_id)
cache_key = "{}_max_parallel_requests_client".format(model_id)
client = self.cache.get_cache(key=cache_key, local_only=True)
return client
elif client_type == "async":