diff --git a/litellm/router.py b/litellm/router.py index a7c75945d..cd7c72c4a 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -72,7 +72,9 @@ class Router: ## RELIABILITY ## num_retries: Optional[int] = None, timeout: Optional[float] = None, - default_litellm_params={}, # default params for Router.chat.completion.create + default_litellm_params: Optional[ + dict + ] = None, # default params for Router.chat.completion.create default_max_parallel_requests: Optional[int] = None, set_verbose: bool = False, debug_level: Literal["DEBUG", "INFO"] = "INFO", @@ -158,6 +160,7 @@ class Router: router = Router(model_list=model_list, fallbacks=[{"azure-gpt-3.5-turbo": "openai-gpt-3.5-turbo"}]) ``` """ + if semaphore: self.semaphore = semaphore self.set_verbose = set_verbose @@ -260,6 +263,7 @@ class Router: ) # dict to store aliases for router, ex. {"gpt-4": "gpt-3.5-turbo"}, all requests with gpt-4 -> get routed to gpt-3.5-turbo group # make Router.chat.completions.create compatible for openai.chat.completions.create + default_litellm_params = default_litellm_params or {} self.chat = litellm.Chat(params=default_litellm_params, router_obj=self) # default litellm args @@ -475,6 +479,7 @@ class Router: ) kwargs["model_info"] = deployment.get("model_info", {}) data = deployment["litellm_params"].copy() + model_name = data["model"] for k, v in self.default_litellm_params.items(): if ( diff --git a/litellm/tests/test_router_timeout.py b/litellm/tests/test_router_timeout.py index dff30113b..4f99d1a99 100644 --- a/litellm/tests/test_router_timeout.py +++ b/litellm/tests/test_router_timeout.py @@ -89,15 +89,15 @@ def test_router_timeouts(): @pytest.mark.asyncio async def test_router_timeouts_bedrock(): - import openai + import openai, uuid # Model list for OpenAI and Anthropic models - model_list = [ + _model_list = [ { "model_name": "bedrock", "litellm_params": { "model": "bedrock/anthropic.claude-instant-v1", - "timeout": 0.001, + "timeout": 0.00001, }, "tpm": 80000, }, @@ -105,17 +105,18 @@ async def test_router_timeouts_bedrock(): # Configure router router = Router( - model_list=model_list, + model_list=_model_list, routing_strategy="usage-based-routing", debug_level="DEBUG", set_verbose=True, + num_retries=0, ) litellm.set_verbose = True try: response = await router.acompletion( model="bedrock", - messages=[{"role": "user", "content": "hello, who are u"}], + messages=[{"role": "user", "content": f"hello, who are u {uuid.uuid4()}"}], ) print(response) pytest.fail("Did not raise error `openai.APITimeoutError`")