From 3c6764efef2e2dff5610b3fbf00d49224f4e380c Mon Sep 17 00:00:00 2001
From: ishaan-jaff <ishaanjaffer0324@gmail.com>
Date: Wed, 29 Nov 2023 16:22:04 -0800
Subject: [PATCH] (feat) proxy+ router: support 1k request/second

---
 litellm/router.py | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/litellm/router.py b/litellm/router.py
index 5e0605da59..ac0058713e 100644
--- a/litellm/router.py
+++ b/litellm/router.py
@@ -106,6 +106,14 @@ class Router:
         ### HEALTH CHECK THREAD ###
         if self.routing_strategy == "least-busy":
             self._start_health_check_thread()
+        if self.routing_strategy == "simple-shuffle":
+            # use rpm based shuffle if user provided values of rpm
+            try:
+                rpm = self.model_list[0]["litellm_params"].get("rpm", None)
+                if rpm is not None:
+                    self.routing_strategy = "weighted-shuffle"
+            except:
+                pass
 
         ### CACHING ###
         redis_cache = None
@@ -135,6 +143,7 @@ class Router:
             litellm.failure_callback.append(self.deployment_callback_on_failure)
         else:
             litellm.failure_callback = [self.deployment_callback_on_failure]
+        self.print_verbose(f"Intialized router with Routing strategy: {self.routing_strategy}\n")
 
     
     ### COMPLETION + EMBEDDING FUNCTIONS