From 040dc1f23e7a29147a18221e66f18eaf0c58b49c Mon Sep 17 00:00:00 2001
From: Ishaan Jaff <ishaanjaffer0324@gmail.com>
Date: Mon, 6 May 2024 16:43:42 -0700
Subject: [PATCH 1/7] fix retry policy

---
 litellm/router.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/litellm/router.py b/litellm/router.py
index 303fab128..9fff3015c 100644
--- a/litellm/router.py
+++ b/litellm/router.py
@@ -3268,6 +3268,8 @@ class Router:
 
         if retry_policy is None:
             return None
+        if isinstance(retry_policy, dict):
+            retry_policy = RetryPolicy(**retry_policy)
         if (
             isinstance(exception, litellm.BadRequestError)
             and retry_policy.BadRequestErrorRetries is not None

From 57a78703481f3bd80274ee5befbcf2e0bb54b114 Mon Sep 17 00:00:00 2001
From: Ishaan Jaff <ishaanjaffer0324@gmail.com>
Date: Mon, 6 May 2024 17:23:51 -0700
Subject: [PATCH 2/7] ui - show retry policy per model group

---
 .../src/components/model_dashboard.tsx        | 52 ++++++++++++-------
 1 file changed, 34 insertions(+), 18 deletions(-)

diff --git a/ui/litellm-dashboard/src/components/model_dashboard.tsx b/ui/litellm-dashboard/src/components/model_dashboard.tsx
index 43ca9fdff..1297db295 100644
--- a/ui/litellm-dashboard/src/components/model_dashboard.tsx
+++ b/ui/litellm-dashboard/src/components/model_dashboard.tsx
@@ -1275,24 +1275,40 @@ const handleEditSubmit = async (formValues: Record<string, any>) => {
               <Text className="mb-6">How many retries should be attempted based on the Exception</Text>
               {retry_policy_map &&
   <table>
-    <tbody>
-  {Object.keys(retry_policy_map).map((key, idx) => (
-    <tr key={idx} className="flex justify-between items-center mt-2">
-      <td>
-        <Text>{key}</Text>
-      </td>
-      <td>
-        <InputNumber
-          className="ml-5"
-          defaultValue={defaultRetry}
-          min={0}
-          step={1}
-        />
-      </td>
-    </tr>
-  ))}
-</tbody>
-  </table>
+  <tbody>
+    {Object.entries(retry_policy_map).map(([exceptionType, retryPolicyKey], idx) => {
+      let retryCount = (modelGroupRetryPolicy[selectedModelGroup] ?? {})[retryPolicyKey];
+      if (retryCount == null) {
+        retryCount = defaultRetry;
+      }
+      
+      return (
+        <tr key={idx} className="flex justify-between items-center mt-2">
+          <td>
+            <Text>{exceptionType}</Text>
+          </td>
+          <td>
+            <InputNumber
+              className="ml-5"
+              value={retryCount}
+              min={0}
+              step={1}
+              onChange={(value) => {
+                setModelGroupRetryPolicy({
+                  ...modelGroupRetryPolicy,
+                  [selectedModelGroup]: {
+                    ...modelGroupRetryPolicy[selectedModelGroup],
+                    [retryPolicyKey]: value,
+                  },
+                });
+              }}
+            />
+          </td>
+        </tr>
+      );
+    })}
+  </tbody>
+</table>
 }
 <Button className="mt-6 mr-8">
   Save

From 116e555a03fd98ae2e345816c6143ad0553a1c7f Mon Sep 17 00:00:00 2001
From: Ishaan Jaff <ishaanjaffer0324@gmail.com>
Date: Mon, 6 May 2024 17:43:43 -0700
Subject: [PATCH 3/7] fix - setting model_group_retry_policy in db

---
 litellm/types/router.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/litellm/types/router.py b/litellm/types/router.py
index dbfc549cd..4b54d8f41 100644
--- a/litellm/types/router.py
+++ b/litellm/types/router.py
@@ -55,6 +55,7 @@ class UpdateRouterConfig(BaseModel):
 
     routing_strategy_args: Optional[dict] = None
     routing_strategy: Optional[str] = None
+    model_group_retry_policy: Optional[dict] = None
     allowed_fails: Optional[int] = None
     cooldown_time: Optional[float] = None
     num_retries: Optional[int] = None

From 4fdf1147d39f830267bf51d69c64f8853aa0a36c Mon Sep 17 00:00:00 2001
From: Ishaan Jaff <ishaanjaffer0324@gmail.com>
Date: Mon, 6 May 2024 17:44:55 -0700
Subject: [PATCH 4/7] ui - get/set model retry policy

---
 .../src/components/model_dashboard.tsx        | 30 +++++++++++++++++--
 1 file changed, 27 insertions(+), 3 deletions(-)

diff --git a/ui/litellm-dashboard/src/components/model_dashboard.tsx b/ui/litellm-dashboard/src/components/model_dashboard.tsx
index 1297db295..a62365474 100644
--- a/ui/litellm-dashboard/src/components/model_dashboard.tsx
+++ b/ui/litellm-dashboard/src/components/model_dashboard.tsx
@@ -18,7 +18,7 @@ import {
 } from "@tremor/react";
 import { TabPanel, TabPanels, TabGroup, TabList, Tab, TextInput, Icon, DateRangePicker } from "@tremor/react";
 import { Select, SelectItem, MultiSelect, MultiSelectItem, DateRangePickerValue } from "@tremor/react";
-import { modelInfoCall, userGetRequesedtModelsCall, modelCreateCall, Model, modelCostMap, modelDeleteCall, healthCheckCall, modelUpdateCall, modelMetricsCall, modelExceptionsCall, modelMetricsSlowResponsesCall, getCallbacksCall } from "./networking";
+import { modelInfoCall, userGetRequesedtModelsCall, modelCreateCall, Model, modelCostMap, modelDeleteCall, healthCheckCall, modelUpdateCall, modelMetricsCall, modelExceptionsCall, modelMetricsSlowResponsesCall, getCallbacksCall, setCallbacksCall } from "./networking";
 import { BarChart, AreaChart } from "@tremor/react";
 import {
   Button as Button2,
@@ -443,6 +443,29 @@ const handleEditSubmit = async (formValues: Record<string, any>) => {
     setLastRefreshed(currentDate.toLocaleString());
   };
 
+  const handleSaveRetrySettings = async () => {
+    if (!accessToken) {
+      console.error("Access token is missing");
+      return;
+    }
+
+    console.log("new modelGroupRetryPolicy:", modelGroupRetryPolicy);
+  
+    try {
+      const payload = {
+          router_settings: {
+            model_group_retry_policy: modelGroupRetryPolicy
+          }
+      };
+  
+      await setCallbacksCall(accessToken, payload);
+      message.success("Retry settings saved successfully");
+    } catch (error) {
+      console.error("Failed to save retry settings:", error);
+      message.error("Failed to save retry settings");
+    }
+  };
+
 
   useEffect(() => {
     if (!accessToken || !token || !userRole || !userID) {
@@ -1277,7 +1300,8 @@ const handleEditSubmit = async (formValues: Record<string, any>) => {
   <table>
   <tbody>
     {Object.entries(retry_policy_map).map(([exceptionType, retryPolicyKey], idx) => {
-      let retryCount = (modelGroupRetryPolicy[selectedModelGroup] ?? {})[retryPolicyKey];
+
+      let retryCount = modelGroupRetryPolicy?.[selectedModelGroup]?.[retryPolicyKey]
       if (retryCount == null) {
         retryCount = defaultRetry;
       }
@@ -1310,7 +1334,7 @@ const handleEditSubmit = async (formValues: Record<string, any>) => {
   </tbody>
 </table>
 }
-<Button className="mt-6 mr-8">
+<Button className="mt-6 mr-8" onClick={handleSaveRetrySettings}>
   Save
 </Button>
 

From eb5dc38fc9200b360be0cefbc17a1bfefa136ac2 Mon Sep 17 00:00:00 2001
From: Ishaan Jaff <ishaanjaffer0324@gmail.com>
Date: Mon, 6 May 2024 17:57:07 -0700
Subject: [PATCH 5/7] fix - set retry policy on ui

---
 .../src/components/model_dashboard.tsx        | 23 ++++++++++++-------
 1 file changed, 15 insertions(+), 8 deletions(-)

diff --git a/ui/litellm-dashboard/src/components/model_dashboard.tsx b/ui/litellm-dashboard/src/components/model_dashboard.tsx
index a62365474..2b7cf923d 100644
--- a/ui/litellm-dashboard/src/components/model_dashboard.tsx
+++ b/ui/litellm-dashboard/src/components/model_dashboard.tsx
@@ -60,6 +60,10 @@ interface EditModelModalProps {
   onSubmit: (data: FormData) => void; // Assuming FormData is the type of data to be submitted
 }
 
+interface RetryPolicyObject {
+  [key: string]: { [retryPolicyKey: string]: number } | undefined;
+}
+
 //["OpenAI", "Azure OpenAI", "Anthropic", "Gemini (Google AI Studio)", "Amazon Bedrock", "OpenAI-Compatible Endpoints (Groq, Together AI, Mistral AI, etc.)"]
 
 enum Providers {
@@ -222,7 +226,7 @@ const ModelDashboard: React.FC<ModelDashboardProps> = ({
     to: new Date(),
   });
 
-  const [modelGroupRetryPolicy, setModelGroupRetryPolicy] = useState<Record<string, number>>({});
+  const [modelGroupRetryPolicy, setModelGroupRetryPolicy] = useState<RetryPolicyObject | null>(null);
   const [defaultRetry, setDefaultRetry] = useState<number>(0);
 
 
@@ -1301,7 +1305,7 @@ const handleEditSubmit = async (formValues: Record<string, any>) => {
   <tbody>
     {Object.entries(retry_policy_map).map(([exceptionType, retryPolicyKey], idx) => {
 
-      let retryCount = modelGroupRetryPolicy?.[selectedModelGroup]?.[retryPolicyKey]
+      let retryCount = modelGroupRetryPolicy?.[selectedModelGroup!]?.[retryPolicyKey]
       if (retryCount == null) {
         retryCount = defaultRetry;
       }
@@ -1318,12 +1322,15 @@ const handleEditSubmit = async (formValues: Record<string, any>) => {
               min={0}
               step={1}
               onChange={(value) => {
-                setModelGroupRetryPolicy({
-                  ...modelGroupRetryPolicy,
-                  [selectedModelGroup]: {
-                    ...modelGroupRetryPolicy[selectedModelGroup],
-                    [retryPolicyKey]: value,
-                  },
+                setModelGroupRetryPolicy(prevModelGroupRetryPolicy => {
+                  const prevRetryPolicy = prevModelGroupRetryPolicy?.[selectedModelGroup!] ?? {};
+                  return {
+                    ...prevModelGroupRetryPolicy ?? {},
+                    [selectedModelGroup!]: {
+                      ...prevRetryPolicy,
+                      [retryPolicyKey!]: value,
+                    },
+                  } as RetryPolicyObject;
                 });
               }}
             />

From f04f45a80d751dc0bda274c9b00e3bf1a44e379c Mon Sep 17 00:00:00 2001
From: Ishaan Jaff <ishaanjaffer0324@gmail.com>
Date: Mon, 6 May 2024 17:58:56 -0700
Subject: [PATCH 6/7] ui - fix show num retries for 500 error

---
 ui/litellm-dashboard/src/components/model_dashboard.tsx | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/ui/litellm-dashboard/src/components/model_dashboard.tsx b/ui/litellm-dashboard/src/components/model_dashboard.tsx
index 2b7cf923d..6e207b430 100644
--- a/ui/litellm-dashboard/src/components/model_dashboard.tsx
+++ b/ui/litellm-dashboard/src/components/model_dashboard.tsx
@@ -93,7 +93,8 @@ const retry_policy_map: Record <string, string> = {
   "AuthenticationError  (401)": "AuthenticationErrorRetries",
   "TimeoutError (408)": "TimeoutErrorRetries",
   "RateLimitError (429)": "RateLimitErrorRetries",
-  "ContentPolicyViolationError (400)": "ContentPolicyViolationErrorRetries"
+  "ContentPolicyViolationError (400)": "ContentPolicyViolationErrorRetries",
+  "InternalServerError (500)": "InternalServerErrorRetries"
 };
 
 

From 6a056c0c1724b881b897091099bebda548d87024 Mon Sep 17 00:00:00 2001
From: Ishaan Jaff <ishaanjaffer0324@gmail.com>
Date: Mon, 6 May 2024 17:59:38 -0700
Subject: [PATCH 7/7] fix support InternalServerErrorRetries

---
 litellm/types/router.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/litellm/types/router.py b/litellm/types/router.py
index 4b54d8f41..4a62a267e 100644
--- a/litellm/types/router.py
+++ b/litellm/types/router.py
@@ -345,3 +345,4 @@ class RetryPolicy(BaseModel):
     TimeoutErrorRetries: Optional[int] = None
     RateLimitErrorRetries: Optional[int] = None
     ContentPolicyViolationErrorRetries: Optional[int] = None
+    InternalServerErrorRetries: Optional[int] = None