diff --git a/litellm/router.py b/litellm/router.py index 8ea1a124a..9fd07d2d7 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -326,9 +326,9 @@ class Router: litellm.failure_callback.append(self.deployment_callback_on_failure) else: litellm.failure_callback = [self.deployment_callback_on_failure] - verbose_router_logger.info( + print( # noqa f"Intialized router with Routing strategy: {self.routing_strategy}\n\nRouting fallbacks: {self.fallbacks}\n\nRouting context window fallbacks: {self.context_window_fallbacks}\n\nRouter Redis Caching={self.cache.redis_cache}" - ) + ) # noqa self.routing_strategy_args = routing_strategy_args def print_deployment(self, deployment: dict): @@ -2614,6 +2614,11 @@ class Router: for var in vars_to_include: if var in _all_vars: _settings_to_return[var] = _all_vars[var] + if ( + var == "routing_strategy_args" + and self.routing_strategy == "latency-based-routing" + ): + _settings_to_return[var] = self.lowestlatency_logger.routing_args return _settings_to_return def update_settings(self, **kwargs): diff --git a/ui/litellm-dashboard/src/components/general_settings.tsx b/ui/litellm-dashboard/src/components/general_settings.tsx index b9721820d..c2013b157 100644 --- a/ui/litellm-dashboard/src/components/general_settings.tsx +++ b/ui/litellm-dashboard/src/components/general_settings.tsx @@ -15,7 +15,13 @@ import { Grid, Button, TextInput, + Select as Select2, + SelectItem, Col, + Accordion, + AccordionBody, + AccordionHeader, + AccordionList, } from "@tremor/react"; import { TabPanel, TabPanels, TabGroup, TabList, Tab, Icon } from "@tremor/react"; import { getCallbacksCall, setCallbacksCall, serviceHealthCheck } from "./networking"; @@ -24,6 +30,7 @@ import { InformationCircleIcon, PencilAltIcon, PencilIcon, StatusOnlineIcon, Tra import StaticGenerationSearchParamsBailoutProvider from "next/dist/client/components/static-generation-searchparams-bailout-provider"; import AddFallbacks from "./add_fallbacks" import openai from "openai"; +import Paragraph from "antd/es/skeleton/Paragraph"; interface GeneralSettingsPageProps { accessToken: string | null; @@ -72,6 +79,62 @@ async function testFallbackModelResponse( } } +interface AccordionHeroProps { + selectedStrategy: string | null; + strategyArgs: routingStrategyArgs; + paramExplanation: { [key: string]: string } +} + +interface routingStrategyArgs { + ttl?: number; + lowest_latency_buffer?: number; +} + +const defaultLowestLatencyArgs: routingStrategyArgs = { + "ttl": 3600, + "lowest_latency_buffer": 0 +} + +export const AccordionHero: React.FC = ({ selectedStrategy, strategyArgs, paramExplanation }) => ( + + Routing Strategy Specific Args + + { + selectedStrategy == "latency-based-routing" ? + + + + + Setting + Value + + + + {Object.entries(strategyArgs).map(([param, value]) => ( + + + {param} +

{paramExplanation[param]}

+
+ + + +
+ ))} +
+
+
+ : No specific settings + } +
+
+); + const GeneralSettings: React.FC = ({ accessToken, userRole, @@ -82,6 +145,8 @@ const GeneralSettings: React.FC = ({ const [isModalVisible, setIsModalVisible] = useState(false); const [form] = Form.useForm(); const [selectedCallback, setSelectedCallback] = useState(null); + const [selectedStrategy, setSelectedStrategy] = useState(null) + const [strategySettings, setStrategySettings] = useState(null); let paramExplanation: { [key: string]: string } = { "routing_strategy_args": "(dict) Arguments to pass to the routing strategy", @@ -91,6 +156,8 @@ const GeneralSettings: React.FC = ({ "num_retries": "(int) Number of retries for failed requests. Defaults to 0.", "timeout": "(float) Timeout for requests. Defaults to None.", "retry_after": "(int) Minimum time to wait before retrying a failed request", + "ttl": "(int) Sliding window to look back over when calculating the average latency of a deployment. Default - 1 hour (in seconds).", + "lowest_latency_buffer": "(float) Shuffle between deployments within this % of the lowest latency. Default - 0 (i.e. always pick lowest latency)." } useEffect(() => { @@ -141,6 +208,7 @@ const GeneralSettings: React.FC = ({ try { await setCallbacksCall(accessToken, payload); setRouterSettings({ ...routerSettings }); + setSelectedStrategy(routerSettings["routing_strategy"]) message.success("Router settings updated successfully"); } catch (error) { message.error("Failed to update router settings: " + error, 20); @@ -156,11 +224,33 @@ const GeneralSettings: React.FC = ({ const updatedVariables = Object.fromEntries( Object.entries(router_settings).map(([key, value]) => { - if (key !== 'routing_strategy_args') { + if (key !== 'routing_strategy_args' && key !== "routing_strategy") { return [key, (document.querySelector(`input[name="${key}"]`) as HTMLInputElement)?.value || value]; } + else if (key == "routing_strategy") { + return [key, selectedStrategy] + } + else if (key == "routing_strategy_args" && selectedStrategy == "latency-based-routing") { + let setRoutingStrategyArgs: routingStrategyArgs = {} + + const lowestLatencyBufferElement = document.querySelector(`input[name="lowest_latency_buffer"]`) as HTMLInputElement; + const ttlElement = document.querySelector(`input[name="ttl"]`) as HTMLInputElement; + + if (lowestLatencyBufferElement?.value) { + setRoutingStrategyArgs["lowest_latency_buffer"] = Number(lowestLatencyBufferElement.value) + } + + if (ttlElement?.value) { + setRoutingStrategyArgs["ttl"] = Number(ttlElement.value) + } + + console.log(`setRoutingStrategyArgs: ${setRoutingStrategyArgs}`) + return [ + "routing_strategy_args", setRoutingStrategyArgs + ] + } return null; - }).filter(entry => entry !== null) as Iterable<[string, unknown]> + }).filter(entry => entry !== null && entry !== undefined) as Iterable<[string, unknown]> ); console.log("updatedVariables", updatedVariables); @@ -183,6 +273,7 @@ const GeneralSettings: React.FC = ({ return null; } + return (
@@ -203,24 +294,41 @@ const GeneralSettings: React.FC = ({ - {Object.entries(routerSettings).filter(([param, value]) => param != "fallbacks" && param != "context_window_fallbacks").map(([param, value]) => ( + {Object.entries(routerSettings).filter(([param, value]) => param != "fallbacks" && param != "context_window_fallbacks" && param != "routing_strategy_args").map(([param, value]) => ( {param}

{paramExplanation[param]}

- + { + param == "routing_strategy" ? + + usage-based-routing + latency-based-routing + simple-shuffle + : + + }
))}
+ 0 + ? routerSettings['routing_strategy_args'] + : defaultLowestLatencyArgs // default value when keys length is 0 + } + paramExplanation={paramExplanation} + />