phoenix/litellm-mirror
0
0
Fork 1
mirror of https://github.com/BerriAI/litellm.git synced 2025-04-24 18:24:20 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

(feat) proxy+ router: support 1k request/second

Browse source
This commit is contained in:
ishaan-jaff 2023-11-29 16:22:04 -08:00
parent da75b15176
commit 3c6764efef
1 changed files with 9 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

9
litellm/router.py
View file

@ -106,6 +106,14 @@ class Router:
### HEALTH CHECK THREAD ### ### HEALTH CHECK THREAD ###
if self.routing_strategy == "least-busy": if self.routing_strategy == "least-busy":
self._start_health_check_thread() self._start_health_check_thread()
if self.routing_strategy == "simple-shuffle":
# use rpm based shuffle if user provided values of rpm
try:
rpm = self.model_list[0]["litellm_params"].get("rpm", None)
if rpm is not None:
self.routing_strategy = "weighted-shuffle"
except:
pass
### CACHING ### ### CACHING ###
redis_cache = None redis_cache = None
@ -135,6 +143,7 @@ class Router:
litellm.failure_callback.append(self.deployment_callback_on_failure) litellm.failure_callback.append(self.deployment_callback_on_failure)
else: else:
litellm.failure_callback = [self.deployment_callback_on_failure] litellm.failure_callback = [self.deployment_callback_on_failure]
self.print_verbose(f"Intialized router with Routing strategy: {self.routing_strategy}\n")
### COMPLETION + EMBEDDING FUNCTIONS ### COMPLETION + EMBEDDING FUNCTIONS