diff --git a/litellm/router.py b/litellm/router.py index 303fab128..9fff3015c 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -3268,6 +3268,8 @@ class Router: if retry_policy is None: return None + if isinstance(retry_policy, dict): + retry_policy = RetryPolicy(**retry_policy) if ( isinstance(exception, litellm.BadRequestError) and retry_policy.BadRequestErrorRetries is not None diff --git a/litellm/types/router.py b/litellm/types/router.py index dbfc549cd..4a62a267e 100644 --- a/litellm/types/router.py +++ b/litellm/types/router.py @@ -55,6 +55,7 @@ class UpdateRouterConfig(BaseModel): routing_strategy_args: Optional[dict] = None routing_strategy: Optional[str] = None + model_group_retry_policy: Optional[dict] = None allowed_fails: Optional[int] = None cooldown_time: Optional[float] = None num_retries: Optional[int] = None @@ -344,3 +345,4 @@ class RetryPolicy(BaseModel): TimeoutErrorRetries: Optional[int] = None RateLimitErrorRetries: Optional[int] = None ContentPolicyViolationErrorRetries: Optional[int] = None + InternalServerErrorRetries: Optional[int] = None diff --git a/ui/litellm-dashboard/src/components/model_dashboard.tsx b/ui/litellm-dashboard/src/components/model_dashboard.tsx index 43ca9fdff..6e207b430 100644 --- a/ui/litellm-dashboard/src/components/model_dashboard.tsx +++ b/ui/litellm-dashboard/src/components/model_dashboard.tsx @@ -18,7 +18,7 @@ import { } from "@tremor/react"; import { TabPanel, TabPanels, TabGroup, TabList, Tab, TextInput, Icon, DateRangePicker } from "@tremor/react"; import { Select, SelectItem, MultiSelect, MultiSelectItem, DateRangePickerValue } from "@tremor/react"; -import { modelInfoCall, userGetRequesedtModelsCall, modelCreateCall, Model, modelCostMap, modelDeleteCall, healthCheckCall, modelUpdateCall, modelMetricsCall, modelExceptionsCall, modelMetricsSlowResponsesCall, getCallbacksCall } from "./networking"; +import { modelInfoCall, userGetRequesedtModelsCall, modelCreateCall, Model, modelCostMap, modelDeleteCall, healthCheckCall, modelUpdateCall, modelMetricsCall, modelExceptionsCall, modelMetricsSlowResponsesCall, getCallbacksCall, setCallbacksCall } from "./networking"; import { BarChart, AreaChart } from "@tremor/react"; import { Button as Button2, @@ -60,6 +60,10 @@ interface EditModelModalProps { onSubmit: (data: FormData) => void; // Assuming FormData is the type of data to be submitted } +interface RetryPolicyObject { + [key: string]: { [retryPolicyKey: string]: number } | undefined; +} + //["OpenAI", "Azure OpenAI", "Anthropic", "Gemini (Google AI Studio)", "Amazon Bedrock", "OpenAI-Compatible Endpoints (Groq, Together AI, Mistral AI, etc.)"] enum Providers { @@ -89,7 +93,8 @@ const retry_policy_map: Record = { "AuthenticationError (401)": "AuthenticationErrorRetries", "TimeoutError (408)": "TimeoutErrorRetries", "RateLimitError (429)": "RateLimitErrorRetries", - "ContentPolicyViolationError (400)": "ContentPolicyViolationErrorRetries" + "ContentPolicyViolationError (400)": "ContentPolicyViolationErrorRetries", + "InternalServerError (500)": "InternalServerErrorRetries" }; @@ -222,7 +227,7 @@ const ModelDashboard: React.FC = ({ to: new Date(), }); - const [modelGroupRetryPolicy, setModelGroupRetryPolicy] = useState>({}); + const [modelGroupRetryPolicy, setModelGroupRetryPolicy] = useState(null); const [defaultRetry, setDefaultRetry] = useState(0); @@ -443,6 +448,29 @@ const handleEditSubmit = async (formValues: Record) => { setLastRefreshed(currentDate.toLocaleString()); }; + const handleSaveRetrySettings = async () => { + if (!accessToken) { + console.error("Access token is missing"); + return; + } + + console.log("new modelGroupRetryPolicy:", modelGroupRetryPolicy); + + try { + const payload = { + router_settings: { + model_group_retry_policy: modelGroupRetryPolicy + } + }; + + await setCallbacksCall(accessToken, payload); + message.success("Retry settings saved successfully"); + } catch (error) { + console.error("Failed to save retry settings:", error); + message.error("Failed to save retry settings"); + } + }; + useEffect(() => { if (!accessToken || !token || !userRole || !userID) { @@ -1275,26 +1303,46 @@ const handleEditSubmit = async (formValues: Record) => { How many retries should be attempted based on the Exception {retry_policy_map && - - {Object.keys(retry_policy_map).map((key, idx) => ( - - - - - ))} - -
- {key} - - -
+ + {Object.entries(retry_policy_map).map(([exceptionType, retryPolicyKey], idx) => { + + let retryCount = modelGroupRetryPolicy?.[selectedModelGroup!]?.[retryPolicyKey] + if (retryCount == null) { + retryCount = defaultRetry; + } + + return ( + + + {exceptionType} + + + { + setModelGroupRetryPolicy(prevModelGroupRetryPolicy => { + const prevRetryPolicy = prevModelGroupRetryPolicy?.[selectedModelGroup!] ?? {}; + return { + ...prevModelGroupRetryPolicy ?? {}, + [selectedModelGroup!]: { + ...prevRetryPolicy, + [retryPolicyKey!]: value, + }, + } as RetryPolicyObject; + }); + }} + /> + + + ); + })} + + } -