diff --git a/docs/my-website/docs/proxy/reliability.md b/docs/my-website/docs/proxy/reliability.md index 51e90fe39..bd04216dd 100644 --- a/docs/my-website/docs/proxy/reliability.md +++ b/docs/my-website/docs/proxy/reliability.md @@ -136,6 +136,21 @@ curl --location 'http://0.0.0.0:4000/chat/completions' \ ' ``` +### Test it! + + +```bash +curl --location 'http://0.0.0.0:4000/chat/completions' \ + --header 'Content-Type: application/json' \ + --data-raw '{ + "model": "zephyr-beta", # 👈 MODEL NAME to fallback from + "messages": [ + {"role": "user", "content": "what color is red"} + ], + "mock_testing_fallbacks": true + }' +``` + ## Advanced - Context Window Fallbacks **Before call is made** check if a call is within model context window with **`enable_pre_call_checks: true`**. diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index d840b29c0..9bcc280b3 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -2521,9 +2521,10 @@ class ProxyConfig: # decode base64 decoded_b64 = base64.b64decode(v) # decrypt value - _litellm_params[k] = decrypt_value( - value=decoded_b64, master_key=master_key - ) + _value = decrypt_value(value=decoded_b64, master_key=master_key) + # sanity check if string > size 0 + if len(_value) > 0: + _litellm_params[k] = _value _litellm_params = LiteLLM_Params(**_litellm_params) else: verbose_proxy_logger.error( diff --git a/litellm/router.py b/litellm/router.py index e6db92397..672eb097d 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -1309,7 +1309,7 @@ class Router: Try calling the function_with_retries If it fails after num_retries, fall back to another model group """ - mock_testing_fallbacks = kwargs.get("mock_testing_fallbacks", None) + mock_testing_fallbacks = kwargs.pop("mock_testing_fallbacks", None) model_group = kwargs.get("model") fallbacks = kwargs.get("fallbacks", self.fallbacks) context_window_fallbacks = kwargs.get( @@ -1369,7 +1369,10 @@ class Router: elif fallbacks is not None: verbose_router_logger.debug(f"inside model fallbacks: {fallbacks}") for item in fallbacks: - if list(item.keys())[0] == model_group: + key_list = list(item.keys()) + if len(key_list) == 0: + continue + if key_list[0] == model_group: fallback_model_group = item[model_group] break if fallback_model_group is None: diff --git a/ui/litellm-dashboard/src/components/general_settings.tsx b/ui/litellm-dashboard/src/components/general_settings.tsx index cc88bb1ee..f43730610 100644 --- a/ui/litellm-dashboard/src/components/general_settings.tsx +++ b/ui/litellm-dashboard/src/components/general_settings.tsx @@ -57,11 +57,12 @@ async function testFallbackModelResponse( content: "Hi, this is a test message", }, ], + mock_testing_fallbacks: true }); message.success( - Test model={selectedModel}, received model={responseModel}. + Test model={selectedModel}, received model={response.model}. See window.open('https://docs.litellm.ai/docs/proxy/reliability', '_blank')} style={{ textDecoration: 'underline', color: 'blue' }}>curl ); @@ -154,14 +155,12 @@ const GeneralSettings: React.FC = ({ const updatedVariables = Object.fromEntries( Object.entries(router_settings).map(([key, value]) => { - if (key === 'routing_strategy_args' && typeof value === 'string') { - return [key, JSON.parse(value as string)]; - } else { + if (key !== 'routing_strategy_args') { return [key, (document.querySelector(`input[name="${key}"]`) as HTMLInputElement)?.value || value]; } - }) + return null; + }).filter(entry => entry !== null) as Iterable<[string, unknown]> ); - console.log("updatedVariables", updatedVariables); const payload = { diff --git a/ui/litellm-dashboard/src/components/model_dashboard.tsx b/ui/litellm-dashboard/src/components/model_dashboard.tsx index 89b573f77..c3ea6f5aa 100644 --- a/ui/litellm-dashboard/src/components/model_dashboard.tsx +++ b/ui/litellm-dashboard/src/components/model_dashboard.tsx @@ -106,6 +106,9 @@ const handleSubmit = async (formValues: Record, accessToken: string litellmParamsObj["model"] = litellm_model let modelName: string = ""; for (const [key, value] of Object.entries(formValues)) { + if (value === '') { + continue; + } if (key == "model_name") { modelName = modelName + value }