phoenix/litellm-mirror
0
0
Fork 1
mirror of https://github.com/BerriAI/litellm.git synced 2025-04-25 10:44:24 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge pull request #3412 from sumanth13131/usage-based-routing-ttl-on-cache

Browse source 
usage-based-routing-ttl-on-cache
This commit is contained in:
Krish Dholakia 2024-05-21 07:58:41 -07:00 • committed by GitHub
commit 2cda5a2bc3
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
4 changed files with 98 additions and 14 deletions
Download patch file Download diff file Expand all files Collapse all files

8
litellm/router.py
View file

@ -374,13 +374,17 @@ class Router:
litellm.callbacks.append(self.leastbusy_logger) # type: ignore
elif routing_strategy == "usage-based-routing":
self.lowesttpm_logger = LowestTPMLoggingHandler(
router_cache=self.cache, model_list=self.model_list
router_cache=self.cache,
model_list=self.model_list,
routing_args=routing_strategy_args
)
if isinstance(litellm.callbacks, list):
litellm.callbacks.append(self.lowesttpm_logger) # type: ignore
elif routing_strategy == "usage-based-routing-v2":
self.lowesttpm_logger_v2 = LowestTPMLoggingHandler_v2(
router_cache=self.cache, model_list=self.model_list
router_cache=self.cache,
model_list=self.model_list,
routing_args=routing_strategy_args
)
if isinstance(litellm.callbacks, list):
litellm.callbacks.append(self.lowesttpm_logger_v2) # type: ignore