phoenix/litellm
1
0
Fork 0
forked from phoenix/litellm-mirror
Code Pull requests Activity

fix(router.py): fix high-traffic bug for usage-based-routing-v2

Browse source
This commit is contained in:
Krrish Dholakia 2024-04-29 16:47:30 -07:00
parent 09bae3d8ad
commit e7b4882e97
1 changed files with 4 additions and 1 deletions
Download patch file Download diff file Expand all files Collapse all files

5
litellm/router.py
View file

@ -2779,7 +2779,10 @@ class Router:
self.cache.get_cache(key=model_id, local_only=True) or 0
)
### get usage based cache ###
if isinstance(model_group_cache, dict):
if (
isinstance(model_group_cache, dict)
and self.routing_strategy != "usage-based-routing-v2"
):
model_group_cache[model_id] = model_group_cache.get(model_id, 0)
current_request = max(