diff --git a/litellm/router.py b/litellm/router.py index fbb245a3d9..303fab1283 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -2631,6 +2631,7 @@ class Router: "retry_after", "fallbacks", "context_window_fallbacks", + "model_group_retry_policy", ] for var in vars_to_include: @@ -2656,6 +2657,7 @@ class Router: "retry_after", "fallbacks", "context_window_fallbacks", + "model_group_retry_policy", ] _int_settings = [ diff --git a/ui/litellm-dashboard/src/components/model_dashboard.tsx b/ui/litellm-dashboard/src/components/model_dashboard.tsx index b223e91281..43ca9fdff3 100644 --- a/ui/litellm-dashboard/src/components/model_dashboard.tsx +++ b/ui/litellm-dashboard/src/components/model_dashboard.tsx @@ -18,7 +18,7 @@ import { } from "@tremor/react"; import { TabPanel, TabPanels, TabGroup, TabList, Tab, TextInput, Icon, DateRangePicker } from "@tremor/react"; import { Select, SelectItem, MultiSelect, MultiSelectItem, DateRangePickerValue } from "@tremor/react"; -import { modelInfoCall, userGetRequesedtModelsCall, modelCreateCall, Model, modelCostMap, modelDeleteCall, healthCheckCall, modelUpdateCall, modelMetricsCall, modelExceptionsCall, modelMetricsSlowResponsesCall } from "./networking"; +import { modelInfoCall, userGetRequesedtModelsCall, modelCreateCall, Model, modelCostMap, modelDeleteCall, healthCheckCall, modelUpdateCall, modelMetricsCall, modelExceptionsCall, modelMetricsSlowResponsesCall, getCallbacksCall } from "./networking"; import { BarChart, AreaChart } from "@tremor/react"; import { Button as Button2, @@ -83,6 +83,17 @@ const provider_map: Record = { }; + +const retry_policy_map: Record = { + "BadRequestError (400)": "BadRequestErrorRetries", + "AuthenticationError (401)": "AuthenticationErrorRetries", + "TimeoutError (408)": "TimeoutErrorRetries", + "RateLimitError (429)": "RateLimitErrorRetries", + "ContentPolicyViolationError (400)": "ContentPolicyViolationErrorRetries" +}; + + + const handleSubmit = async (formValues: Record, accessToken: string, form: any) => { try { /** @@ -211,6 +222,10 @@ const ModelDashboard: React.FC = ({ to: new Date(), }); + const [modelGroupRetryPolicy, setModelGroupRetryPolicy] = useState>({}); + const [defaultRetry, setDefaultRetry] = useState(0); + + const EditModelModal: React.FC = ({ visible, onCancel, model, onSubmit }) => { const [form] = Form.useForm(); let litellm_params_to_edit: Record = {} @@ -514,6 +529,22 @@ const handleEditSubmit = async (formValues: Record) => { setSlowResponsesData(slowResponses); + const routerSettingsInfo = await getCallbacksCall(accessToken, userID, userRole); + + let router_settings = routerSettingsInfo.router_settings; + + console.log("routerSettingsInfo:", router_settings) + + let model_group_retry_policy = router_settings.model_group_retry_policy; + let default_retries = router_settings.num_retries; + + console.log("model_group_retry_policy:", model_group_retry_policy) + console.log("default_retries:", default_retries) + setModelGroupRetryPolicy(model_group_retry_policy); + setDefaultRetry(default_retries); + + + } catch (error) { console.error("There was an error fetching the model data", error); } @@ -795,6 +826,7 @@ const handleEditSubmit = async (formValues: Record) => { Add Model
/health Models
Model Analytics + Model Retry Settings
@@ -816,7 +848,7 @@ const handleEditSubmit = async (formValues: Record) => {
- Filter by Public Model Name + Filter by Public Model Name setSelectedModelGroup(value)} + > + {availableModelGroups.map((group, idx) => ( + setSelectedModelGroup(group)} + > + {group} + + ))} + +
+ + + Retry Policy for {selectedModelGroup} + + How many retries should be attempted based on the Exception + {retry_policy_map && + + + {Object.keys(retry_policy_map).map((key, idx) => ( + + + + + ))} + +
+ {key} + + +
+} + + +