diff --git a/.gitignore b/.gitignore index 309f726fe..357f3e1bf 100644 --- a/.gitignore +++ b/.gitignore @@ -50,3 +50,4 @@ kub.yaml loadtest_kub.yaml litellm/proxy/_new_secret_config.yaml litellm/proxy/_new_secret_config.yaml +litellm/proxy/_super_secret_config.yaml diff --git a/litellm/router.py b/litellm/router.py index 3416a8495..da871daba 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -2533,6 +2533,8 @@ class Router: "timeout", "max_retries", "retry_after", + "fallbacks", + "context_window_fallbacks", ] for var in vars_to_include: diff --git a/ui/litellm-dashboard/package-lock.json b/ui/litellm-dashboard/package-lock.json index fca6aae6e..846189b27 100644 --- a/ui/litellm-dashboard/package-lock.json +++ b/ui/litellm-dashboard/package-lock.json @@ -36,6 +36,7 @@ "eslint": "^8", "eslint-config-next": "14.1.0", "postcss": "^8.4.33", + "prettier": "3.2.5", "tailwindcss": "^3.4.1", "typescript": "5.3.3" } @@ -5515,6 +5516,21 @@ "node": ">= 0.8.0" } }, + "node_modules/prettier": { + "version": "3.2.5", + "resolved": "https://registry.npmjs.org/prettier/-/prettier-3.2.5.tgz", + "integrity": "sha512-3/GWa9aOC0YeD7LUfvOG2NiDyhOWRvt1k+rcKhOuYnMY24iiCphgneUfJDyFXd6rZCAnuLBv6UeAULtrhT/F4A==", + "dev": true, + "bin": { + "prettier": "bin/prettier.cjs" + }, + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/prettier/prettier?sponsor=1" + } + }, "node_modules/prismjs": { "version": "1.29.0", "resolved": "https://registry.npmjs.org/prismjs/-/prismjs-1.29.0.tgz", diff --git a/ui/litellm-dashboard/package.json b/ui/litellm-dashboard/package.json index c79e3fe37..108d70d98 100644 --- a/ui/litellm-dashboard/package.json +++ b/ui/litellm-dashboard/package.json @@ -37,6 +37,7 @@ "eslint": "^8", "eslint-config-next": "14.1.0", "postcss": "^8.4.33", + "prettier": "3.2.5", "tailwindcss": "^3.4.1", "typescript": "5.3.3" } diff --git a/ui/litellm-dashboard/src/app/page.tsx b/ui/litellm-dashboard/src/app/page.tsx index f20f0aee7..96731f7f5 100644 --- a/ui/litellm-dashboard/src/app/page.tsx +++ b/ui/litellm-dashboard/src/app/page.tsx @@ -24,7 +24,7 @@ const CreateKeyPage = () => { const [keys, setKeys] = useState(null); const [showSSOBanner, setShowSSOBanner] = useState(true); const searchParams = useSearchParams(); - + const [modelData, setModelData] = useState({ data: [] }); const userID = searchParams.get("userID"); const token = searchParams.get("token"); @@ -132,6 +132,8 @@ const CreateKeyPage = () => { userRole={userRole} token={token} accessToken={accessToken} + modelData={modelData} + setModelData={setModelData} /> ) : page == "llm-playground" ? ( { userID={userID} userRole={userRole} accessToken={accessToken} + modelData={modelData} /> ) : ( >; +} + +const AddFallbacks: React.FC = ({ + models, + accessToken, + routerSettings, + setRouterSettings +}) => { + const [form] = Form.useForm(); + const [isModalVisible, setIsModalVisible] = useState(false); + const [selectedModel, setSelectedModel] = useState(""); + const handleOk = () => { + setIsModalVisible(false); + form.resetFields(); + }; + + const handleCancel = () => { + setIsModalVisible(false); + form.resetFields(); + }; + + const updateFallbacks = (formValues: Record) => { + // Print the received value + console.log(formValues); + + // Extract model_name and models from formValues + const { model_name, models } = formValues; + + // Create new fallback + const newFallback = { [model_name]: models }; + + // Get current fallbacks, or an empty array if it's null + const currentFallbacks = routerSettings.fallbacks || []; + + // Add new fallback to the current fallbacks + const updatedFallbacks = [...currentFallbacks, newFallback]; + + // Create a new routerSettings object with updated fallbacks + const updatedRouterSettings = { ...routerSettings, fallbacks: updatedFallbacks }; + + // Print updated routerSettings + console.log(updatedRouterSettings); + + const payload = { + router_settings: updatedRouterSettings + }; + + try { + setCallbacksCall(accessToken, payload); + // Update routerSettings state + setRouterSettings(updatedRouterSettings); + } catch (error) { + message.error("Failed to update router settings: " + error, 20); + } + + message.success("router settings updated successfully"); + + setIsModalVisible(false) + form.resetFields(); + }; + + + return ( +
+ + +
+ <> + + + + + + + {models && + models.filter(data => data != selectedModel).map((model: string) => ( + ( + + {model} + + ) + ))} + + + + + +
+ Add Fallbacks +
+
+
+ +
+ ); +}; + +export default AddFallbacks; diff --git a/ui/litellm-dashboard/src/components/general_settings.tsx b/ui/litellm-dashboard/src/components/general_settings.tsx index e55bda48e..e227568d0 100644 --- a/ui/litellm-dashboard/src/components/general_settings.tsx +++ b/ui/litellm-dashboard/src/components/general_settings.tsx @@ -17,20 +17,24 @@ import { TextInput, Col, } from "@tremor/react"; +import { TabPanel, TabPanels, TabGroup, TabList, Tab, Icon } from "@tremor/react"; import { getCallbacksCall, setCallbacksCall, serviceHealthCheck } from "./networking"; import { Modal, Form, Input, Select, Button as Button2, message } from "antd"; import StaticGenerationSearchParamsBailoutProvider from "next/dist/client/components/static-generation-searchparams-bailout-provider"; +import AddFallbacks from "./add_fallbacks" interface GeneralSettingsPageProps { accessToken: string | null; userRole: string | null; userID: string | null; + modelData: any } const GeneralSettings: React.FC = ({ accessToken, userRole, userID, + modelData }) => { const [routerSettings, setRouterSettings] = useState<{ [key: string]: any }>({}); const [isModalVisible, setIsModalVisible] = useState(false); @@ -103,6 +107,13 @@ const GeneralSettings: React.FC = ({ return (
+ + + General Settings + Fallbacks + + + Router Settings @@ -114,23 +125,23 @@ const GeneralSettings: React.FC = ({ - {Object.entries(routerSettings).map(([param, value]) => ( - - - {param} -

{paramExplanation[param]}

-
- - - -
-))} -
+ {Object.entries(routerSettings).map(([param, value]) => ( + + + {param} +

{paramExplanation[param]}

+
+ + + +
+ ))} +
@@ -139,7 +150,34 @@ const GeneralSettings: React.FC = ({
+
+ + + + + Model Name + Fallbacks + + + + { + routerSettings["fallbacks"] && + routerSettings["fallbacks"].map((item: Object, index: number) => + Object.entries(item).map(([key, value]) => ( + + {key} + {Array.isArray(value) ? value.join(', ') : value} + + )) + ) + } + +
+ data.model_name) : []} accessToken={accessToken} routerSettings={routerSettings} setRouterSettings={setRouterSettings}/> +
+
+
); }; diff --git a/ui/litellm-dashboard/src/components/leftnav.tsx b/ui/litellm-dashboard/src/components/leftnav.tsx index 461c360f9..c4296f18e 100644 --- a/ui/litellm-dashboard/src/components/leftnav.tsx +++ b/ui/litellm-dashboard/src/components/leftnav.tsx @@ -118,7 +118,7 @@ const Sidebar: React.FC = ({ userRole == "Admin" ? ( setPage("general-settings")}> - Settings + Router Settings ) : null diff --git a/ui/litellm-dashboard/src/components/model_dashboard.tsx b/ui/litellm-dashboard/src/components/model_dashboard.tsx index 40c355747..89b573f77 100644 --- a/ui/litellm-dashboard/src/components/model_dashboard.tsx +++ b/ui/litellm-dashboard/src/components/model_dashboard.tsx @@ -49,6 +49,8 @@ interface ModelDashboardProps { token: string | null; userRole: string | null; userID: string | null; + modelData: any, + setModelData: any } interface EditModelModalProps { @@ -176,8 +178,9 @@ const ModelDashboard: React.FC = ({ token, userRole, userID, + modelData = { data: [] }, + setModelData }) => { - const [modelData, setModelData] = useState({ data: [] }); const [pendingRequests, setPendingRequests] = useState([]); const [form] = Form.useForm(); const [modelMap, setModelMap] = useState(null);