phoenix/litellm-mirror
0
0
Fork 1
mirror of https://github.com/BerriAI/litellm.git synced 2025-04-26 11:14:04 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

fix(router.py): add validation for how router fallbacks are setup

Browse source 
prevent user errors
This commit is contained in:
Krrish Dholakia 2024-05-15 10:43:46 -07:00
parent efca96baf8
commit dba713ea43
2 changed files with 27 additions and 5 deletions
Download patch file Download diff file Expand all files Collapse all files

5
litellm/proxy/_super_secret_config.yaml
View file

@ -18,8 +18,6 @@ model_list:
model: azure/chatgpt-v-2 model: azure/chatgpt-v-2
api_key: os.environ/AZURE_API_KEY api_key: os.environ/AZURE_API_KEY
api_base: os.environ/AZURE_API_BASE api_base: os.environ/AZURE_API_BASE
input_cost_per_token: 0.0
output_cost_per_token: 0.0
- model_name: bert-classifier - model_name: bert-classifier
litellm_params: litellm_params:
model: huggingface/text-classification/shahrukhx01/question-vs-statement-classifier model: huggingface/text-classification/shahrukhx01/question-vs-statement-classifier
@ -31,8 +29,7 @@ router_settings:
enable_pre_call_checks: true enable_pre_call_checks: true
litellm_settings: litellm_settings:
set_verbose: True fallbacks: [{"gpt-3.5-turbo-012": ["azure-gpt-3.5-turbo"]}]
fallbacks: [{"gpt-3.5-turbo-012": ["gpt-3.5-turbo-0125-preview"]}]
# service_callback: ["prometheus_system"] # service_callback: ["prometheus_system"]
# success_callback: ["prometheus"] # success_callback: ["prometheus"]
# failure_callback: ["prometheus"] # failure_callback: ["prometheus"]

27
litellm/router.py
View file

@ -262,13 +262,23 @@ class Router:
self.retry_after = retry_after self.retry_after = retry_after
self.routing_strategy = routing_strategy self.routing_strategy = routing_strategy
self.fallbacks = fallbacks or litellm.fallbacks
## SETTING FALLBACKS ##
### validate if it's set + in correct format
_fallbacks = fallbacks or litellm.fallbacks
self.validate_fallbacks(fallback_param=_fallbacks)
### set fallbacks
self.fallbacks = _fallbacks
if default_fallbacks is not None or litellm.default_fallbacks is not None: if default_fallbacks is not None or litellm.default_fallbacks is not None:
_fallbacks = default_fallbacks or litellm.default_fallbacks _fallbacks = default_fallbacks or litellm.default_fallbacks
self.validate_fallbacks(fallback_param=_fallbacks)
if self.fallbacks is not None: if self.fallbacks is not None:
self.fallbacks.append({"*": _fallbacks}) self.fallbacks.append({"*": _fallbacks})
else: else:
self.fallbacks = [{"*": _fallbacks}] self.fallbacks = [{"*": _fallbacks}]
self.context_window_fallbacks = ( self.context_window_fallbacks = (
context_window_fallbacks or litellm.context_window_fallbacks context_window_fallbacks or litellm.context_window_fallbacks
) )
@ -336,6 +346,21 @@ class Router:
if self.alerting_config is not None: if self.alerting_config is not None:
self._initialize_alerting() self._initialize_alerting()
def validate_fallbacks(self, fallback_param: Optional[List]):
if fallback_param is None:
return
if len(fallback_param) > 0: # if set
## for dictionary in list, check if only 1 key in dict
for _dict in fallback_param:
assert isinstance(_dict, dict), "Item={}, not a dictionary".format(
_dict
)
assert (
len(_dict.keys()) == 1
), "Only 1 key allows in dictionary. You set={} for dict={}".format(
len(_dict.keys()), _dict
)
def routing_strategy_init(self, routing_strategy: str, routing_strategy_args: dict): def routing_strategy_init(self, routing_strategy: str, routing_strategy_args: dict):
if routing_strategy == "least-busy": if routing_strategy == "least-busy":
self.leastbusy_logger = LeastBusyLoggingHandler( self.leastbusy_logger = LeastBusyLoggingHandler(