diff --git a/litellm/router.py b/litellm/router.py
index 2a710f5cf..e6db92397 100644
--- a/litellm/router.py
+++ b/litellm/router.py
@@ -1309,12 +1309,18 @@ class Router:
Try calling the function_with_retries
If it fails after num_retries, fall back to another model group
"""
+ mock_testing_fallbacks = kwargs.get("mock_testing_fallbacks", None)
model_group = kwargs.get("model")
fallbacks = kwargs.get("fallbacks", self.fallbacks)
context_window_fallbacks = kwargs.get(
"context_window_fallbacks", self.context_window_fallbacks
)
try:
+ if mock_testing_fallbacks is not None and mock_testing_fallbacks == True:
+ raise Exception(
+ f"This is a mock exception for model={model_group}, to trigger a fallback. Fallbacks={fallbacks}"
+ )
+
response = await self.async_function_with_retries(*args, **kwargs)
verbose_router_logger.debug(f"Async Response: {response}")
return response
diff --git a/ui/litellm-dashboard/src/components/general_settings.tsx b/ui/litellm-dashboard/src/components/general_settings.tsx
index ace755a2c..cc88bb1ee 100644
--- a/ui/litellm-dashboard/src/components/general_settings.tsx
+++ b/ui/litellm-dashboard/src/components/general_settings.tsx
@@ -20,8 +20,10 @@ import {
import { TabPanel, TabPanels, TabGroup, TabList, Tab, Icon } from "@tremor/react";
import { getCallbacksCall, setCallbacksCall, serviceHealthCheck } from "./networking";
import { Modal, Form, Input, Select, Button as Button2, message } from "antd";
+import { InformationCircleIcon, PencilAltIcon, PencilIcon, StatusOnlineIcon, TrashIcon, RefreshIcon } from "@heroicons/react/outline";
import StaticGenerationSearchParamsBailoutProvider from "next/dist/client/components/static-generation-searchparams-bailout-provider";
import AddFallbacks from "./add_fallbacks"
+import openai from "openai";
interface GeneralSettingsPageProps {
accessToken: string | null;
@@ -30,6 +32,44 @@ interface GeneralSettingsPageProps {
modelData: any
}
+async function testFallbackModelResponse(
+ selectedModel: string,
+ accessToken: string
+) {
+ // base url should be the current base_url
+ const isLocal = process.env.NODE_ENV === "development";
+ console.log("isLocal:", isLocal);
+ const proxyBaseUrl = isLocal
+ ? "http://localhost:4000"
+ : window.location.origin;
+ const client = new openai.OpenAI({
+ apiKey: accessToken, // Replace with your OpenAI API key
+ baseURL: proxyBaseUrl, // Replace with your OpenAI API base URL
+ dangerouslyAllowBrowser: true, // using a temporary litellm proxy key
+ });
+
+ try {
+ const response = await client.chat.completions.create({
+ model: selectedModel,
+ messages: [
+ {
+ role: "user",
+ content: "Hi, this is a test message",
+ },
+ ],
+ });
+
+ message.success(
+
+ Test model={selectedModel}, received model={responseModel}.
+ See window.open('https://docs.litellm.ai/docs/proxy/reliability', '_blank')} style={{ textDecoration: 'underline', color: 'blue' }}>curl
+
+ );
+ } catch (error) {
+ message.error(`Error occurred while generating model response. Please try again. Error: ${error}`, 20);
+ }
+}
+
const GeneralSettings: React.FC = ({
accessToken,
userRole,
@@ -73,6 +113,38 @@ const GeneralSettings: React.FC = ({
setSelectedCallback(null);
};
+ const deleteFallbacks = async (key: string) => {
+ /**
+ * pop the key from the Object, if it exists
+ */
+ if (!accessToken) {
+ return;
+ }
+
+ console.log(`received key: ${key}`)
+ console.log(`routerSettings['fallbacks']: ${routerSettings['fallbacks']}`)
+
+ routerSettings["fallbacks"].map((dict: { [key: string]: any }) => {
+ // Check if the dictionary has the specified key and delete it if present
+ if (key in dict) {
+ delete dict[key];
+ }
+ return dict; // Return the updated dictionary
+ });
+
+ const payload = {
+ router_settings: routerSettings
+ };
+
+ try {
+ await setCallbacksCall(accessToken, payload);
+ setRouterSettings({ ...routerSettings });
+ message.success("Router settings updated successfully");
+ } catch (error) {
+ message.error("Failed to update router settings: " + error, 20);
+ }
+ }
+
const handleSaveChanges = (router_settings: any) => {
if (!accessToken) {
return;
@@ -81,7 +153,13 @@ const GeneralSettings: React.FC = ({
console.log("router_settings", router_settings);
const updatedVariables = Object.fromEntries(
- Object.entries(router_settings).map(([key, value]) => [key, (document.querySelector(`input[name="${key}"]`) as HTMLInputElement)?.value || value])
+ Object.entries(router_settings).map(([key, value]) => {
+ if (key === 'routing_strategy_args' && typeof value === 'string') {
+ return [key, JSON.parse(value as string)];
+ } else {
+ return [key, (document.querySelector(`input[name="${key}"]`) as HTMLInputElement)?.value || value];
+ }
+ })
);
console.log("updatedVariables", updatedVariables);
@@ -168,6 +246,18 @@ const GeneralSettings: React.FC = ({
{key}
{Array.isArray(value) ? value.join(', ') : value}
+
+
+
+
+ deleteFallbacks(key)}
+ />
+
))
)