diff --git a/litellm/proxy/_super_secret_config.yaml b/litellm/proxy/_super_secret_config.yaml index 42f9e3be50..98191aa043 100644 --- a/litellm/proxy/_super_secret_config.yaml +++ b/litellm/proxy/_super_secret_config.yaml @@ -18,8 +18,6 @@ model_list: model: azure/chatgpt-v-2 api_key: os.environ/AZURE_API_KEY api_base: os.environ/AZURE_API_BASE - input_cost_per_token: 0.0 - output_cost_per_token: 0.0 - model_name: bert-classifier litellm_params: model: huggingface/text-classification/shahrukhx01/question-vs-statement-classifier @@ -31,8 +29,7 @@ router_settings: enable_pre_call_checks: true litellm_settings: - set_verbose: True - fallbacks: [{"gpt-3.5-turbo-012": ["gpt-3.5-turbo-0125-preview"]}] + fallbacks: [{"gpt-3.5-turbo-012": ["azure-gpt-3.5-turbo"]}] # service_callback: ["prometheus_system"] # success_callback: ["prometheus"] # failure_callback: ["prometheus"] diff --git a/litellm/router.py b/litellm/router.py index c4f7ed0f2c..5a47e0dbec 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -262,13 +262,23 @@ class Router: self.retry_after = retry_after self.routing_strategy = routing_strategy - self.fallbacks = fallbacks or litellm.fallbacks + + ## SETTING FALLBACKS ## + ### validate if it's set + in correct format + _fallbacks = fallbacks or litellm.fallbacks + + self.validate_fallbacks(fallback_param=_fallbacks) + ### set fallbacks + self.fallbacks = _fallbacks + if default_fallbacks is not None or litellm.default_fallbacks is not None: _fallbacks = default_fallbacks or litellm.default_fallbacks + self.validate_fallbacks(fallback_param=_fallbacks) if self.fallbacks is not None: self.fallbacks.append({"*": _fallbacks}) else: self.fallbacks = [{"*": _fallbacks}] + self.context_window_fallbacks = ( context_window_fallbacks or litellm.context_window_fallbacks ) @@ -336,6 +346,21 @@ class Router: if self.alerting_config is not None: self._initialize_alerting() + def validate_fallbacks(self, fallback_param: Optional[List]): + if fallback_param is None: + return + if len(fallback_param) > 0: # if set + ## for dictionary in list, check if only 1 key in dict + for _dict in fallback_param: + assert isinstance(_dict, dict), "Item={}, not a dictionary".format( + _dict + ) + assert ( + len(_dict.keys()) == 1 + ), "Only 1 key allows in dictionary. You set={} for dict={}".format( + len(_dict.keys()), _dict + ) + def routing_strategy_init(self, routing_strategy: str, routing_strategy_args: dict): if routing_strategy == "least-busy": self.leastbusy_logger = LeastBusyLoggingHandler(