build(add-fallbacks-on-UI): allows admin to add fallbacks on the UI

.gitignore

@@ -50,3 +50,4 @@ kub.yaml
 loadtest_kub.yaml
 litellm/proxy/_new_secret_config.yaml
 litellm/proxy/_new_secret_config.yaml
+litellm/proxy/_super_secret_config.yaml

litellm/router.py

@@ -2533,6 +2533,8 @@ class Router:
             "timeout",
             "max_retries",
             "retry_after",
+            "fallbacks",
+            "context_window_fallbacks",
         ]
 
         for var in vars_to_include:

ui/litellm-dashboard/package-lock.json

@@ -36,6 +36,7 @@
         "eslint": "^8",
         "eslint-config-next": "14.1.0",
         "postcss": "^8.4.33",
+        "prettier": "3.2.5",
         "tailwindcss": "^3.4.1",
         "typescript": "5.3.3"
       }
@@ -5515,6 +5516,21 @@
         "node": ">= 0.8.0"
       }
     },
+    "node_modules/prettier": {
+      "version": "3.2.5",
+      "resolved": "https://registry.npmjs.org/prettier/-/prettier-3.2.5.tgz",
+      "integrity": "sha512-3/GWa9aOC0YeD7LUfvOG2NiDyhOWRvt1k+rcKhOuYnMY24iiCphgneUfJDyFXd6rZCAnuLBv6UeAULtrhT/F4A==",
+      "dev": true,
+      "bin": {
+        "prettier": "bin/prettier.cjs"
+      },
+      "engines": {
+        "node": ">=14"
+      },
+      "funding": {
+        "url": "https://github.com/prettier/prettier?sponsor=1"
+      }
+    },
     "node_modules/prismjs": {
       "version": "1.29.0",
       "resolved": "https://registry.npmjs.org/prismjs/-/prismjs-1.29.0.tgz",

ui/litellm-dashboard/package.json

@@ -37,6 +37,7 @@
     "eslint": "^8",
     "eslint-config-next": "14.1.0",
     "postcss": "^8.4.33",
+    "prettier": "3.2.5",
     "tailwindcss": "^3.4.1",
     "typescript": "5.3.3"
   }

ui/litellm-dashboard/src/app/page.tsx

@@ -24,7 +24,7 @@ const CreateKeyPage = () => {
   const [keys, setKeys] = useState<null | any[]>(null);
   const [showSSOBanner, setShowSSOBanner] = useState<boolean>(true);
   const searchParams = useSearchParams();
-
+  const [modelData, setModelData] = useState<any>({ data: [] });
   const userID = searchParams.get("userID");
   const token = searchParams.get("token");
 
@@ -132,6 +132,8 @@ const CreateKeyPage = () => {
               userRole={userRole}
               token={token}
               accessToken={accessToken}
+              modelData={modelData}
+              setModelData={setModelData}
             />
           ) : page == "llm-playground" ? (
             <ChatUI
@@ -179,6 +181,7 @@ const CreateKeyPage = () => {
                 userID={userID}
                 userRole={userRole}
                 accessToken={accessToken}
+                modelData={modelData}
               />
           ) : (
             <Usage

ui/litellm-dashboard/src/components/add_fallbacks.tsx

@@ -0,0 +1,164 @@
+/**
+ * Modal to add fallbacks to the proxy router config
+ */
+
+
+
+import React, { useState, useEffect, useRef } from "react";
+import { Button, TextInput, Grid, Col } from "@tremor/react";
+import { Select, SelectItem, MultiSelect, MultiSelectItem, Card, Metric, Text, Title, Subtitle, Accordion, AccordionHeader, AccordionBody, } from "@tremor/react";
+import { CopyToClipboard } from 'react-copy-to-clipboard';
+import { setCallbacksCall } from "./networking";
+import {
+  Button as Button2,
+  Modal,
+  Form,
+  Input,
+  InputNumber,
+  Select as Select2,
+  message,
+} from "antd";
+import { keyCreateCall, slackBudgetAlertsHealthCheck, modelAvailableCall } from "./networking";
+import { list } from "postcss";
+
+const { Option } = Select2;
+
+interface AddFallbacksProps {
+  models: string[] | undefined; 
+  accessToken: string;
+  routerSettings: { [key: string]: any; }
+  setRouterSettings: React.Dispatch<React.SetStateAction<{ [key: string]: any }>>;
+}
+
+const AddFallbacks: React.FC<AddFallbacksProps> = ({
+    models, 
+    accessToken,
+    routerSettings,
+    setRouterSettings
+}) => {
+  const [form] = Form.useForm();
+  const [isModalVisible, setIsModalVisible] = useState(false);
+  const [selectedModel, setSelectedModel] = useState("");
+  const handleOk = () => {
+    setIsModalVisible(false);
+    form.resetFields();
+  };
+
+  const handleCancel = () => {
+    setIsModalVisible(false);
+    form.resetFields();
+  };
+
+  const updateFallbacks = (formValues: Record<string, any>) => {
+    // Print the received value
+    console.log(formValues);
+
+    // Extract model_name and models from formValues
+    const { model_name, models } = formValues;
+
+    // Create new fallback
+    const newFallback = { [model_name]: models };
+
+    // Get current fallbacks, or an empty array if it's null
+    const currentFallbacks = routerSettings.fallbacks || [];
+
+    // Add new fallback to the current fallbacks
+    const updatedFallbacks = [...currentFallbacks, newFallback];
+
+    // Create a new routerSettings object with updated fallbacks
+    const updatedRouterSettings = { ...routerSettings, fallbacks: updatedFallbacks };
+
+    // Print updated routerSettings
+    console.log(updatedRouterSettings);
+
+    const payload = {
+        router_settings: updatedRouterSettings
+    };
+
+    try {
+        setCallbacksCall(accessToken, payload);
+        // Update routerSettings state
+        setRouterSettings(updatedRouterSettings);
+    } catch (error) {
+        message.error("Failed to update router settings: " + error, 20);
+    }
+
+    message.success("router settings updated successfully");
+
+    setIsModalVisible(false)
+    form.resetFields();
+  };
+
+
+  return (
+    <div>
+      <Button className="mx-auto" onClick={() => setIsModalVisible(true)}>
+        + Add Fallbacks
+      </Button>
+      <Modal
+        title="Add Fallbacks"
+        visible={isModalVisible}
+        width={800}
+        footer={null}
+        onOk={handleOk}
+        onCancel={handleCancel}
+      >
+        <Form
+          form={form}
+          onFinish={updateFallbacks}
+          labelCol={{ span: 8 }}
+          wrapperCol={{ span: 16 }}
+          labelAlign="left"
+        >
+            <>
+              <Form.Item 
+                label="Public Model Name" 
+                name="model_name"
+                rules={[{ required: true, message: 'Set the model to fallback for' }]}
+                help="required"
+              >
+                <Select defaultValue={selectedModel}>
+                {models && models.map((model: string, index) => (
+                    <SelectItem
+                        key={index}
+                        value={model}
+                        onClick={() => setSelectedModel(model)}
+                    >
+                        {model}
+                    </SelectItem>
+                    ))
+                }
+                </Select>
+              </Form.Item>
+
+              <Form.Item 
+                label="Fallback Models" 
+                name="models"
+                rules={[{ required: true, message: 'Please select a model' }]}
+                help="required"
+              >
+                <MultiSelect value={models}>
+                    {models &&
+                      models.filter(data => data != selectedModel).map((model: string) => (
+                        (
+                          <MultiSelectItem key={model} value={model}>
+                            {model}
+                          </MultiSelectItem>
+                        )
+                      ))}
+
+                </MultiSelect>
+              </Form.Item>
+            </>
+          
+          <div style={{ textAlign: "right", marginTop: "10px" }}>
+            <Button2 htmlType="submit">Add Fallbacks</Button2>
+          </div>
+        </Form>
+      </Modal>
+
+    </div>
+  );
+};
+
+export default AddFallbacks;

ui/litellm-dashboard/src/components/general_settings.tsx

@@ -17,20 +17,24 @@ import {
   TextInput,
   Col,
 } from "@tremor/react";
+import { TabPanel, TabPanels, TabGroup, TabList, Tab, Icon } from "@tremor/react";
 import { getCallbacksCall, setCallbacksCall, serviceHealthCheck } from "./networking";
 import { Modal, Form, Input, Select, Button as Button2, message } from "antd";
 import StaticGenerationSearchParamsBailoutProvider from "next/dist/client/components/static-generation-searchparams-bailout-provider";
+import AddFallbacks from "./add_fallbacks"
 
 interface GeneralSettingsPageProps {
   accessToken: string | null;
   userRole: string | null;
   userID: string | null;
+  modelData: any
 }
 
 const GeneralSettings: React.FC<GeneralSettingsPageProps> = ({
   accessToken,
   userRole,
   userID,
+  modelData
 }) => {
   const [routerSettings, setRouterSettings] = useState<{ [key: string]: any }>({});
   const [isModalVisible, setIsModalVisible] = useState(false);
@@ -103,6 +107,13 @@ const GeneralSettings: React.FC<GeneralSettingsPageProps> = ({
 
   return (
     <div className="w-full mx-4">
+      <TabGroup className="gap-2 p-8 h-[75vh] w-full mt-2">
+        <TabList variant="line" defaultValue="1">
+          <Tab value="1">General Settings</Tab>
+          <Tab value="2">Fallbacks</Tab>
+        </TabList>
+        <TabPanels>
+          <TabPanel>
       <Grid numItems={1} className="gap-2 p-8 w-full mt-2">
       <Title>Router Settings</Title>
         <Card >
@@ -129,8 +140,8 @@ const GeneralSettings: React.FC<GeneralSettingsPageProps> = ({
                     />
                   </TableCell>
                 </TableRow>
-))}
+              ))}
-</TableBody>
+            </TableBody>
         </Table>
         </Card>
         <Col>
@@ -139,7 +150,34 @@ const GeneralSettings: React.FC<GeneralSettingsPageProps> = ({
             </Button>
         </Col>
       </Grid>
+      </TabPanel>
+      <TabPanel>
+      <Table>
+      <TableHead>
+        <TableRow>
+          <TableHeaderCell>Model Name</TableHeaderCell>
+          <TableHeaderCell>Fallbacks</TableHeaderCell>
+        </TableRow>
+      </TableHead>
 
+        <TableBody>
+          {
+            routerSettings["fallbacks"] &&
+            routerSettings["fallbacks"].map((item: Object, index: number) =>
+              Object.entries(item).map(([key, value]) => (
+                <TableRow key={index.toString() + key}>
+                  <TableCell>{key}</TableCell>
+                  <TableCell>{Array.isArray(value) ? value.join(', ') : value}</TableCell>
+                </TableRow>
+              ))
+            )
+          }
+        </TableBody>
+      </Table>
+      <AddFallbacks models={modelData?.data ? modelData.data.map((data: any) => data.model_name) : []} accessToken={accessToken} routerSettings={routerSettings} setRouterSettings={setRouterSettings}/>
+      </TabPanel>
+      </TabPanels>
+    </TabGroup>
     </div>
   );
 };

ui/litellm-dashboard/src/components/leftnav.tsx

@@ -118,7 +118,7 @@ const Sidebar: React.FC<SidebarProps> = ({
             userRole == "Admin" ? (
               <Menu.Item key="9" onClick={() => setPage("general-settings")}>
               <Text>
-                Settings
+                Router Settings
               </Text>
               </Menu.Item>
             ) : null

ui/litellm-dashboard/src/components/model_dashboard.tsx

@@ -49,6 +49,8 @@ interface ModelDashboardProps {
   token: string | null;
   userRole: string | null;
   userID: string | null;
+  modelData: any, 
+  setModelData: any
 }
 
 interface EditModelModalProps {
@@ -176,8 +178,9 @@ const ModelDashboard: React.FC<ModelDashboardProps> = ({
   token,
   userRole,
   userID,
+  modelData = { data: [] },
+  setModelData
 }) => {
-  const [modelData, setModelData] = useState<any>({ data: [] });
   const [pendingRequests, setPendingRequests] = useState<any[]>([]);
   const [form] = Form.useForm();
   const [modelMap, setModelMap] = useState<any>(null);