fix(completion()): add request_timeout as a param, fix claude error when request_timeout set

litellm/main.py

@@ -168,6 +168,7 @@ def completion(
     logit_bias: dict = {},
     user: str = "",
     deployment_id = None,
+    request_timeout: Optional[int] = None,
     # Optional liteLLM function params
     **kwargs,
 ) -> ModelResponse:
@@ -220,7 +221,7 @@ def completion(
     metadata = kwargs.get('metadata', None)
     fallbacks = kwargs.get('fallbacks', [])
     ######## end of unpacking kwargs ###########
-    openai_params = ["functions", "function_call", "temperature", "temperature", "top_p", "n", "stream", "stop", "max_tokens", "presence_penalty", "frequency_penalty", "logit_bias", "user"]
+    openai_params = ["functions", "function_call", "temperature", "temperature", "top_p", "n", "stream", "stop", "max_tokens", "presence_penalty", "frequency_penalty", "logit_bias", "user", "request_timeout"]
     litellm_params = ["metadata", "acompletion", "caching", "return_async", "mock_response", "api_key", "api_version", "api_base", "force_timeout", "logger_fn", "verbose", "custom_llm_provider", "litellm_logging_obj", "litellm_call_id", "use_client", "id", "metadata", "fallbacks"]
     default_params = openai_params + litellm_params
     non_default_params = {k: v for k,v in kwargs.items() if k not in default_params} # model-specific params - pass them straight to the model/provider
@@ -260,6 +261,7 @@ def completion(
             frequency_penalty=frequency_penalty,
             logit_bias=logit_bias,
             user=user,
+            request_timeout=request_timeout,
             deployment_id=deployment_id,
             # params to identify the model
             model=model,

litellm/tests/test_completion.py

@@ -40,7 +40,7 @@ def test_completion_claude():
     try:
         # test without max tokens
         response = completion(
-            model="claude-instant-1", messages=messages
+            model="claude-instant-1", messages=messages, request_timeout=10,
         )
         # Add any assertions here to check the response
         print(response)
@@ -48,6 +48,8 @@ def test_completion_claude():
     except Exception as e:
         pytest.fail(f"Error occurred: {e}")
 
+test_completion_claude()
+
 def test_completion_claude_max_tokens():
     try:
         litellm.set_verbose = True
@@ -339,13 +341,13 @@ def test_completion_cohere(): # commenting for now as the cohere endpoint is bei
     except Exception as e:
         pytest.fail(f"Error occurred: {e}")
 
-test_completion_cohere()
+# test_completion_cohere()
 
 
 def test_completion_openai():
     try:
         litellm.api_key = os.environ['OPENAI_API_KEY']
-        response = completion(model="gpt-3.5-turbo", messages=messages, max_tokens=10)
+        response = completion(model="gpt-3.5-turbo", messages=messages, max_tokens=10, request_timeout=10)
         print("This is the response object\n", response)
         print("\n\nThis is response ms:", response.response_ms)
 
@@ -362,7 +364,7 @@ def test_completion_openai():
         litellm.api_key = None
     except Exception as e:
         pytest.fail(f"Error occurred: {e}")
-# test_completion_openai()
+test_completion_openai()
 
 
 def test_completion_openai_prompt():
@@ -1018,7 +1020,7 @@ def test_completion_with_fallbacks():
 def test_completion_ai21():
     model_name = "j2-light"
     try:
-        response = completion(model=model_name, messages=messages, max_tokens=100, temperature=0.8, logger_fn=logger_fn)
+        response = completion(model=model_name, messages=messages, max_tokens=100, temperature=0.8)
         # Add any assertions here to check the response
         print(response)
         print(response.response_ms)

litellm/timeout.py

@@ -64,6 +64,8 @@ def timeout(timeout_duration: float = 0.0, exception_to_raise=Timeout):
             local_timeout_duration = timeout_duration
             if "force_timeout" in kwargs:
                 local_timeout_duration = kwargs["force_timeout"]
+            elif "request_timeout" in kwargs and kwargs["request_timeout"] is not None:
+                local_timeout_duration = kwargs["request_timeout"]
             try:
                 value = await asyncio.wait_for(
                     func(*args, **kwargs), timeout=timeout_duration

litellm/utils.py

@@ -947,6 +947,7 @@ def get_optional_params(  # use the openai defaults
     frequency_penalty=0,
     logit_bias={},
     user="",
+    request_timeout=None,
     deployment_id=None,
     model=None,
     custom_llm_provider="",
@@ -971,6 +972,7 @@ def get_optional_params(  # use the openai defaults
         "logit_bias":{},
         "user":"",
         "deployment_id":None,
+        "request_timeout":None,
         "model":None,
         "custom_llm_provider":"",
     }
@@ -991,6 +993,8 @@ def get_optional_params(  # use the openai defaults
             if k not in supported_params:
                 if k == "n" and n == 1: # langchain sends n=1 as a default value
                     pass
+                if k == "request_timeout": # litellm handles request time outs
+                    pass
                 else: 
                     unsupported_params.append(k)
         if unsupported_params and not litellm.drop_params:
@@ -1273,7 +1277,7 @@ def get_optional_params(  # use the openai defaults
         if stream:
             optional_params["stream"] = stream
     else:  # assume passing in params for openai/azure openai
-        supported_params = ["functions", "function_call", "temperature", "top_p", "n", "stream", "stop", "max_tokens", "presence_penalty", "frequency_penalty", "logit_bias", "user", "deployment_id"]
+        supported_params = ["functions", "function_call", "temperature", "top_p", "n", "stream", "stop", "max_tokens", "presence_penalty", "frequency_penalty", "logit_bias", "user", "deployment_id", "request_timeout"]
         _check_valid_arg(supported_params=supported_params)
         optional_params = non_default_params
     # if user passed in non-default kwargs for specific providers/models, pass them along 

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "litellm"
-version = "0.1.821"
+version = "0.1.822"
 description = "Library to easily interface with LLM API providers"
 authors = ["BerriAI"]
 license = "MIT License"