diff --git a/litellm/router.py b/litellm/router.py index 2a710f5cfc..e6db92397b 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -1309,12 +1309,18 @@ class Router: Try calling the function_with_retries If it fails after num_retries, fall back to another model group """ + mock_testing_fallbacks = kwargs.get("mock_testing_fallbacks", None) model_group = kwargs.get("model") fallbacks = kwargs.get("fallbacks", self.fallbacks) context_window_fallbacks = kwargs.get( "context_window_fallbacks", self.context_window_fallbacks ) try: + if mock_testing_fallbacks is not None and mock_testing_fallbacks == True: + raise Exception( + f"This is a mock exception for model={model_group}, to trigger a fallback. Fallbacks={fallbacks}" + ) + response = await self.async_function_with_retries(*args, **kwargs) verbose_router_logger.debug(f"Async Response: {response}") return response diff --git a/ui/litellm-dashboard/src/components/general_settings.tsx b/ui/litellm-dashboard/src/components/general_settings.tsx index ace755a2cd..cc88bb1ee6 100644 --- a/ui/litellm-dashboard/src/components/general_settings.tsx +++ b/ui/litellm-dashboard/src/components/general_settings.tsx @@ -20,8 +20,10 @@ import { import { TabPanel, TabPanels, TabGroup, TabList, Tab, Icon } from "@tremor/react"; import { getCallbacksCall, setCallbacksCall, serviceHealthCheck } from "./networking"; import { Modal, Form, Input, Select, Button as Button2, message } from "antd"; +import { InformationCircleIcon, PencilAltIcon, PencilIcon, StatusOnlineIcon, TrashIcon, RefreshIcon } from "@heroicons/react/outline"; import StaticGenerationSearchParamsBailoutProvider from "next/dist/client/components/static-generation-searchparams-bailout-provider"; import AddFallbacks from "./add_fallbacks" +import openai from "openai"; interface GeneralSettingsPageProps { accessToken: string | null; @@ -30,6 +32,44 @@ interface GeneralSettingsPageProps { modelData: any } +async function testFallbackModelResponse( + selectedModel: string, + accessToken: string +) { + // base url should be the current base_url + const isLocal = process.env.NODE_ENV === "development"; + console.log("isLocal:", isLocal); + const proxyBaseUrl = isLocal + ? "http://localhost:4000" + : window.location.origin; + const client = new openai.OpenAI({ + apiKey: accessToken, // Replace with your OpenAI API key + baseURL: proxyBaseUrl, // Replace with your OpenAI API base URL + dangerouslyAllowBrowser: true, // using a temporary litellm proxy key + }); + + try { + const response = await client.chat.completions.create({ + model: selectedModel, + messages: [ + { + role: "user", + content: "Hi, this is a test message", + }, + ], + }); + + message.success( + + Test model={selectedModel}, received model={responseModel}. + See window.open('https://docs.litellm.ai/docs/proxy/reliability', '_blank')} style={{ textDecoration: 'underline', color: 'blue' }}>curl + + ); + } catch (error) { + message.error(`Error occurred while generating model response. Please try again. Error: ${error}`, 20); + } +} + const GeneralSettings: React.FC = ({ accessToken, userRole, @@ -73,6 +113,38 @@ const GeneralSettings: React.FC = ({ setSelectedCallback(null); }; + const deleteFallbacks = async (key: string) => { + /** + * pop the key from the Object, if it exists + */ + if (!accessToken) { + return; + } + + console.log(`received key: ${key}`) + console.log(`routerSettings['fallbacks']: ${routerSettings['fallbacks']}`) + + routerSettings["fallbacks"].map((dict: { [key: string]: any }) => { + // Check if the dictionary has the specified key and delete it if present + if (key in dict) { + delete dict[key]; + } + return dict; // Return the updated dictionary + }); + + const payload = { + router_settings: routerSettings + }; + + try { + await setCallbacksCall(accessToken, payload); + setRouterSettings({ ...routerSettings }); + message.success("Router settings updated successfully"); + } catch (error) { + message.error("Failed to update router settings: " + error, 20); + } + } + const handleSaveChanges = (router_settings: any) => { if (!accessToken) { return; @@ -81,7 +153,13 @@ const GeneralSettings: React.FC = ({ console.log("router_settings", router_settings); const updatedVariables = Object.fromEntries( - Object.entries(router_settings).map(([key, value]) => [key, (document.querySelector(`input[name="${key}"]`) as HTMLInputElement)?.value || value]) + Object.entries(router_settings).map(([key, value]) => { + if (key === 'routing_strategy_args' && typeof value === 'string') { + return [key, JSON.parse(value as string)]; + } else { + return [key, (document.querySelector(`input[name="${key}"]`) as HTMLInputElement)?.value || value]; + } + }) ); console.log("updatedVariables", updatedVariables); @@ -168,6 +246,18 @@ const GeneralSettings: React.FC = ({ {key} {Array.isArray(value) ? value.join(', ') : value} + + + + + deleteFallbacks(key)} + /> + )) )