forked from phoenix/litellm-mirror
Merge pull request #3433 from BerriAI/litellm_timeout_fix
fix(bedrock.py): convert httpx.timeout to boto3 valid timeout
This commit is contained in:
Krish Dholakia
GitHub
commit
7d2aa2f645
8 changed files with 146 additions and 89 deletions
|
|
@ -151,7 +151,7 @@ class AzureChatCompletion(BaseLLM):
|
|||
api_type: str,
|
||||
azure_ad_token: str,
|
||||
print_verbose: Callable,
|
||||
timeout,
|
||||
timeout: Union[float, httpx.Timeout],
|
||||
logging_obj,
|
||||
optional_params,
|
||||
litellm_params,
|
||||
|
|
|
|||
|
|
@ -533,7 +533,7 @@ def init_bedrock_client(
|
|||
aws_session_name: Optional[str] = None,
|
||||
aws_profile_name: Optional[str] = None,
|
||||
aws_role_name: Optional[str] = None,
|
||||
timeout: Optional[int] = None,
|
||||
timeout: Optional[Union[float, httpx.Timeout]] = None,
|
||||
):
|
||||
# check for custom AWS_REGION_NAME and use it if not passed to init_bedrock_client
|
||||
litellm_aws_region_name = get_secret("AWS_REGION_NAME", None)
|
||||
|
|
@ -592,7 +592,14 @@ def init_bedrock_client(
|
|||
|
||||
import boto3
|
||||
|
||||
if isinstance(timeout, float):
|
||||
config = boto3.session.Config(connect_timeout=timeout, read_timeout=timeout)
|
||||
elif isinstance(timeout, httpx.Timeout):
|
||||
config = boto3.session.Config(
|
||||
connect_timeout=timeout.connect, read_timeout=timeout.read
|
||||
)
|
||||
else:
|
||||
config = boto3.session.Config()
|
||||
|
||||
### CHECK STS ###
|
||||
if aws_role_name is not None and aws_session_name is not None:
|
||||
|
|
|
|||
|
|
@ -246,7 +246,7 @@ class OpenAIChatCompletion(BaseLLM):
|
|||
def completion(
|
||||
self,
|
||||
model_response: ModelResponse,
|
||||
timeout: float,
|
||||
timeout: Union[float, httpx.Timeout],
|
||||
model: Optional[str] = None,
|
||||
messages: Optional[list] = None,
|
||||
print_verbose: Optional[Callable] = None,
|
||||
|
|
@ -271,9 +271,12 @@ class OpenAIChatCompletion(BaseLLM):
|
|||
if model is None or messages is None:
|
||||
raise OpenAIError(status_code=422, message=f"Missing model or messages")
|
||||
|
||||
if not isinstance(timeout, float):
|
||||
if not isinstance(timeout, float) and not isinstance(
|
||||
timeout, httpx.Timeout
|
||||
):
|
||||
raise OpenAIError(
|
||||
status_code=422, message=f"Timeout needs to be a float"
|
||||
status_code=422,
|
||||
message=f"Timeout needs to be a float or httpx.Timeout",
|
||||
)
|
||||
|
||||
if custom_llm_provider != "openai":
|
||||
|
|
@ -425,7 +428,7 @@ class OpenAIChatCompletion(BaseLLM):
|
|||
self,
|
||||
data: dict,
|
||||
model_response: ModelResponse,
|
||||
timeout: float,
|
||||
timeout: Union[float, httpx.Timeout],
|
||||
api_key: Optional[str] = None,
|
||||
api_base: Optional[str] = None,
|
||||
organization: Optional[str] = None,
|
||||
|
|
@ -480,7 +483,7 @@ class OpenAIChatCompletion(BaseLLM):
|
|||
def streaming(
|
||||
self,
|
||||
logging_obj,
|
||||
timeout: float,
|
||||
timeout: Union[float, httpx.Timeout],
|
||||
data: dict,
|
||||
model: str,
|
||||
api_key: Optional[str] = None,
|
||||
|
|
@ -524,7 +527,7 @@ class OpenAIChatCompletion(BaseLLM):
|
|||
async def async_streaming(
|
||||
self,
|
||||
logging_obj,
|
||||
timeout: float,
|
||||
timeout: Union[float, httpx.Timeout],
|
||||
data: dict,
|
||||
model: str,
|
||||
api_key: Optional[str] = None,
|
||||
|
|
|
|||
|
|
@ -39,6 +39,7 @@ from litellm.utils import (
|
|||
Usage,
|
||||
get_optional_params_embeddings,
|
||||
get_optional_params_image_gen,
|
||||
supports_httpx_timeout,
|
||||
)
|
||||
from .llms import (
|
||||
anthropic_text,
|
||||
|
|
@ -450,7 +451,7 @@ def completion(
|
|||
model: str,
|
||||
# Optional OpenAI params: see https://platform.openai.com/docs/api-reference/chat/create
|
||||
messages: List = [],
|
||||
timeout: Optional[Union[float, int]] = None,
|
||||
timeout: Optional[Union[float, str, httpx.Timeout]] = None,
|
||||
temperature: Optional[float] = None,
|
||||
top_p: Optional[float] = None,
|
||||
n: Optional[int] = None,
|
||||
|
|
@ -648,11 +649,21 @@ def completion(
|
|||
non_default_params = {
|
||||
k: v for k, v in kwargs.items() if k not in default_params
|
||||
} # model-specific params - pass them straight to the model/provider
|
||||
if timeout is None:
|
||||
timeout = (
|
||||
kwargs.get("request_timeout", None) or 600
|
||||
) # set timeout for 10 minutes by default
|
||||
timeout = float(timeout)
|
||||
|
||||
### TIMEOUT LOGIC ###
|
||||
timeout = timeout or kwargs.get("request_timeout", 600) or 600
|
||||
# set timeout for 10 minutes by default
|
||||
|
||||
if (
|
||||
timeout is not None
|
||||
and isinstance(timeout, httpx.Timeout)
|
||||
and supports_httpx_timeout(custom_llm_provider) == False
|
||||
):
|
||||
read_timeout = timeout.read or 600
|
||||
timeout = read_timeout # default 10 min timeout
|
||||
elif timeout is not None and not isinstance(timeout, httpx.Timeout):
|
||||
timeout = float(timeout) # type: ignore
|
||||
|
||||
try:
|
||||
if base_url is not None:
|
||||
api_base = base_url
|
||||
|
|
@ -873,7 +884,7 @@ def completion(
|
|||
logger_fn=logger_fn,
|
||||
logging_obj=logging,
|
||||
acompletion=acompletion,
|
||||
timeout=timeout,
|
||||
timeout=timeout, # type: ignore
|
||||
client=client, # pass AsyncAzureOpenAI, AzureOpenAI client
|
||||
)
|
||||
|
||||
|
|
@ -1014,7 +1025,7 @@ def completion(
|
|||
optional_params=optional_params,
|
||||
litellm_params=litellm_params,
|
||||
logger_fn=logger_fn,
|
||||
timeout=timeout,
|
||||
timeout=timeout, # type: ignore
|
||||
custom_prompt_dict=custom_prompt_dict,
|
||||
client=client, # pass AsyncOpenAI, OpenAI client
|
||||
organization=organization,
|
||||
|
|
@ -1099,7 +1110,7 @@ def completion(
|
|||
optional_params=optional_params,
|
||||
litellm_params=litellm_params,
|
||||
logger_fn=logger_fn,
|
||||
timeout=timeout,
|
||||
timeout=timeout, # type: ignore
|
||||
)
|
||||
|
||||
if (
|
||||
|
|
@ -1473,7 +1484,7 @@ def completion(
|
|||
acompletion=acompletion,
|
||||
logging_obj=logging,
|
||||
custom_prompt_dict=custom_prompt_dict,
|
||||
timeout=timeout,
|
||||
timeout=timeout, # type: ignore
|
||||
)
|
||||
if (
|
||||
"stream" in optional_params
|
||||
|
|
@ -1566,7 +1577,7 @@ def completion(
|
|||
logger_fn=logger_fn,
|
||||
logging_obj=logging,
|
||||
acompletion=acompletion,
|
||||
timeout=timeout,
|
||||
timeout=timeout, # type: ignore
|
||||
)
|
||||
## LOGGING
|
||||
logging.post_call(
|
||||
|
|
@ -1893,7 +1904,7 @@ def completion(
|
|||
logger_fn=logger_fn,
|
||||
encoding=encoding,
|
||||
logging_obj=logging,
|
||||
timeout=timeout,
|
||||
timeout=timeout, # type: ignore
|
||||
)
|
||||
if (
|
||||
"stream" in optional_params
|
||||
|
|
|
|||
|
|
@ -375,7 +375,9 @@ class Router:
|
|||
except Exception as e:
|
||||
raise e
|
||||
|
||||
def _completion(self, model: str, messages: List[Dict[str, str]], **kwargs):
|
||||
def _completion(
|
||||
self, model: str, messages: List[Dict[str, str]], **kwargs
|
||||
) -> Union[ModelResponse, CustomStreamWrapper]:
|
||||
model_name = None
|
||||
try:
|
||||
# pick the one that is available (lowest TPM/RPM)
|
||||
|
|
@ -438,7 +440,9 @@ class Router:
|
|||
)
|
||||
raise e
|
||||
|
||||
async def acompletion(self, model: str, messages: List[Dict[str, str]], **kwargs):
|
||||
async def acompletion(
|
||||
self, model: str, messages: List[Dict[str, str]], **kwargs
|
||||
) -> Union[ModelResponse, CustomStreamWrapper]:
|
||||
try:
|
||||
kwargs["model"] = model
|
||||
kwargs["messages"] = messages
|
||||
|
|
@ -454,7 +458,9 @@ class Router:
|
|||
except Exception as e:
|
||||
raise e
|
||||
|
||||
async def _acompletion(self, model: str, messages: List[Dict[str, str]], **kwargs):
|
||||
async def _acompletion(
|
||||
self, model: str, messages: List[Dict[str, str]], **kwargs
|
||||
) -> Union[ModelResponse, CustomStreamWrapper]:
|
||||
"""
|
||||
- Get an available deployment
|
||||
- call it with a semaphore over the call
|
||||
|
|
|
|||
|
|
@ -5,74 +5,59 @@ plugins: timeout-2.2.0, asyncio-0.23.2, anyio-3.7.1, xdist-3.3.1
|
|||
asyncio: mode=Mode.STRICT
|
||||
collected 1 item
|
||||
|
||||
test_custom_logger.py Chunks have a created at hidden param
|
||||
Chunks sorted
|
||||
token_counter messages received: [{'role': 'user', 'content': 'write a one sentence poem about: 73348'}]
|
||||
Token Counter - using OpenAI token counter, for model=gpt-3.5-turbo
|
||||
LiteLLM: Utils - Counting tokens for OpenAI model=gpt-3.5-turbo
|
||||
Logging Details LiteLLM-Success Call: None
|
||||
success callbacks: []
|
||||
Token Counter - using OpenAI token counter, for model=gpt-3.5-turbo
|
||||
LiteLLM: Utils - Counting tokens for OpenAI model=gpt-3.5-turbo
|
||||
Logging Details LiteLLM-Success Call streaming complete
|
||||
Looking up model=gpt-3.5-turbo in model_cost_map
|
||||
Success: model=gpt-3.5-turbo in model_cost_map
|
||||
prompt_tokens=17; completion_tokens=0
|
||||
Returned custom cost for model=gpt-3.5-turbo - prompt_tokens_cost_usd_dollar: 2.55e-05, completion_tokens_cost_usd_dollar: 0.0
|
||||
final cost: 2.55e-05; prompt_tokens_cost_usd_dollar: 2.55e-05; completion_tokens_cost_usd_dollar: 0.0
|
||||
. [100%]
|
||||
test_image_generation.py . [100%]
|
||||
|
||||
=============================== warnings summary ===============================
|
||||
../../../../../../opt/homebrew/lib/python3.11/site-packages/pydantic/_internal/_config.py:284: 18 warnings
|
||||
../../../../../../opt/homebrew/lib/python3.11/site-packages/pydantic/_internal/_config.py:284: 23 warnings
|
||||
/opt/homebrew/lib/python3.11/site-packages/pydantic/_internal/_config.py:284: PydanticDeprecatedSince20: Support for class-based `config` is deprecated, use ConfigDict instead. Deprecated in Pydantic V2.0 to be removed in V3.0. See Pydantic V2 Migration Guide at https://errors.pydantic.dev/2.7/migration/
|
||||
warnings.warn(DEPRECATION_MESSAGE, DeprecationWarning)
|
||||
|
||||
../proxy/_types.py:218
|
||||
/Users/krrishdholakia/Documents/litellm/litellm/proxy/_types.py:218: PydanticDeprecatedSince20: Pydantic V1 style `@root_validator` validators are deprecated. You should migrate to Pydantic V2 style `@model_validator` validators, see the migration guide for more details. Deprecated in Pydantic V2.0 to be removed in V3.0. See Pydantic V2 Migration Guide at https://errors.pydantic.dev/2.7/migration/
|
||||
../proxy/_types.py:219
|
||||
/Users/krrishdholakia/Documents/litellm/litellm/proxy/_types.py:219: PydanticDeprecatedSince20: Pydantic V1 style `@root_validator` validators are deprecated. You should migrate to Pydantic V2 style `@model_validator` validators, see the migration guide for more details. Deprecated in Pydantic V2.0 to be removed in V3.0. See Pydantic V2 Migration Guide at https://errors.pydantic.dev/2.7/migration/
|
||||
@root_validator(pre=True)
|
||||
|
||||
../proxy/_types.py:305
|
||||
/Users/krrishdholakia/Documents/litellm/litellm/proxy/_types.py:305: PydanticDeprecatedSince20: `pydantic.config.Extra` is deprecated, use literal values instead (e.g. `extra='allow'`). Deprecated in Pydantic V2.0 to be removed in V3.0. See Pydantic V2 Migration Guide at https://errors.pydantic.dev/2.7/migration/
|
||||
../proxy/_types.py:306
|
||||
/Users/krrishdholakia/Documents/litellm/litellm/proxy/_types.py:306: PydanticDeprecatedSince20: `pydantic.config.Extra` is deprecated, use literal values instead (e.g. `extra='allow'`). Deprecated in Pydantic V2.0 to be removed in V3.0. See Pydantic V2 Migration Guide at https://errors.pydantic.dev/2.7/migration/
|
||||
extra = Extra.allow # Allow extra fields
|
||||
|
||||
../proxy/_types.py:308
|
||||
/Users/krrishdholakia/Documents/litellm/litellm/proxy/_types.py:308: PydanticDeprecatedSince20: Pydantic V1 style `@root_validator` validators are deprecated. You should migrate to Pydantic V2 style `@model_validator` validators, see the migration guide for more details. Deprecated in Pydantic V2.0 to be removed in V3.0. See Pydantic V2 Migration Guide at https://errors.pydantic.dev/2.7/migration/
|
||||
../proxy/_types.py:309
|
||||
/Users/krrishdholakia/Documents/litellm/litellm/proxy/_types.py:309: PydanticDeprecatedSince20: Pydantic V1 style `@root_validator` validators are deprecated. You should migrate to Pydantic V2 style `@model_validator` validators, see the migration guide for more details. Deprecated in Pydantic V2.0 to be removed in V3.0. See Pydantic V2 Migration Guide at https://errors.pydantic.dev/2.7/migration/
|
||||
@root_validator(pre=True)
|
||||
|
||||
../proxy/_types.py:337
|
||||
/Users/krrishdholakia/Documents/litellm/litellm/proxy/_types.py:337: PydanticDeprecatedSince20: Pydantic V1 style `@root_validator` validators are deprecated. You should migrate to Pydantic V2 style `@model_validator` validators, see the migration guide for more details. Deprecated in Pydantic V2.0 to be removed in V3.0. See Pydantic V2 Migration Guide at https://errors.pydantic.dev/2.7/migration/
|
||||
../proxy/_types.py:338
|
||||
/Users/krrishdholakia/Documents/litellm/litellm/proxy/_types.py:338: PydanticDeprecatedSince20: Pydantic V1 style `@root_validator` validators are deprecated. You should migrate to Pydantic V2 style `@model_validator` validators, see the migration guide for more details. Deprecated in Pydantic V2.0 to be removed in V3.0. See Pydantic V2 Migration Guide at https://errors.pydantic.dev/2.7/migration/
|
||||
@root_validator(pre=True)
|
||||
|
||||
../proxy/_types.py:384
|
||||
/Users/krrishdholakia/Documents/litellm/litellm/proxy/_types.py:384: PydanticDeprecatedSince20: Pydantic V1 style `@root_validator` validators are deprecated. You should migrate to Pydantic V2 style `@model_validator` validators, see the migration guide for more details. Deprecated in Pydantic V2.0 to be removed in V3.0. See Pydantic V2 Migration Guide at https://errors.pydantic.dev/2.7/migration/
|
||||
../proxy/_types.py:385
|
||||
/Users/krrishdholakia/Documents/litellm/litellm/proxy/_types.py:385: PydanticDeprecatedSince20: Pydantic V1 style `@root_validator` validators are deprecated. You should migrate to Pydantic V2 style `@model_validator` validators, see the migration guide for more details. Deprecated in Pydantic V2.0 to be removed in V3.0. See Pydantic V2 Migration Guide at https://errors.pydantic.dev/2.7/migration/
|
||||
@root_validator(pre=True)
|
||||
|
||||
../proxy/_types.py:450
|
||||
/Users/krrishdholakia/Documents/litellm/litellm/proxy/_types.py:450: PydanticDeprecatedSince20: Pydantic V1 style `@root_validator` validators are deprecated. You should migrate to Pydantic V2 style `@model_validator` validators, see the migration guide for more details. Deprecated in Pydantic V2.0 to be removed in V3.0. See Pydantic V2 Migration Guide at https://errors.pydantic.dev/2.7/migration/
|
||||
../proxy/_types.py:454
|
||||
/Users/krrishdholakia/Documents/litellm/litellm/proxy/_types.py:454: PydanticDeprecatedSince20: Pydantic V1 style `@root_validator` validators are deprecated. You should migrate to Pydantic V2 style `@model_validator` validators, see the migration guide for more details. Deprecated in Pydantic V2.0 to be removed in V3.0. See Pydantic V2 Migration Guide at https://errors.pydantic.dev/2.7/migration/
|
||||
@root_validator(pre=True)
|
||||
|
||||
../proxy/_types.py:462
|
||||
/Users/krrishdholakia/Documents/litellm/litellm/proxy/_types.py:462: PydanticDeprecatedSince20: Pydantic V1 style `@root_validator` validators are deprecated. You should migrate to Pydantic V2 style `@model_validator` validators, see the migration guide for more details. Deprecated in Pydantic V2.0 to be removed in V3.0. See Pydantic V2 Migration Guide at https://errors.pydantic.dev/2.7/migration/
|
||||
../proxy/_types.py:466
|
||||
/Users/krrishdholakia/Documents/litellm/litellm/proxy/_types.py:466: PydanticDeprecatedSince20: Pydantic V1 style `@root_validator` validators are deprecated. You should migrate to Pydantic V2 style `@model_validator` validators, see the migration guide for more details. Deprecated in Pydantic V2.0 to be removed in V3.0. See Pydantic V2 Migration Guide at https://errors.pydantic.dev/2.7/migration/
|
||||
@root_validator(pre=True)
|
||||
|
||||
../proxy/_types.py:502
|
||||
/Users/krrishdholakia/Documents/litellm/litellm/proxy/_types.py:502: PydanticDeprecatedSince20: Pydantic V1 style `@root_validator` validators are deprecated. You should migrate to Pydantic V2 style `@model_validator` validators, see the migration guide for more details. Deprecated in Pydantic V2.0 to be removed in V3.0. See Pydantic V2 Migration Guide at https://errors.pydantic.dev/2.7/migration/
|
||||
../proxy/_types.py:509
|
||||
/Users/krrishdholakia/Documents/litellm/litellm/proxy/_types.py:509: PydanticDeprecatedSince20: Pydantic V1 style `@root_validator` validators are deprecated. You should migrate to Pydantic V2 style `@model_validator` validators, see the migration guide for more details. Deprecated in Pydantic V2.0 to be removed in V3.0. See Pydantic V2 Migration Guide at https://errors.pydantic.dev/2.7/migration/
|
||||
@root_validator(pre=True)
|
||||
|
||||
../proxy/_types.py:536
|
||||
/Users/krrishdholakia/Documents/litellm/litellm/proxy/_types.py:536: PydanticDeprecatedSince20: Pydantic V1 style `@root_validator` validators are deprecated. You should migrate to Pydantic V2 style `@model_validator` validators, see the migration guide for more details. Deprecated in Pydantic V2.0 to be removed in V3.0. See Pydantic V2 Migration Guide at https://errors.pydantic.dev/2.7/migration/
|
||||
../proxy/_types.py:546
|
||||
/Users/krrishdholakia/Documents/litellm/litellm/proxy/_types.py:546: PydanticDeprecatedSince20: Pydantic V1 style `@root_validator` validators are deprecated. You should migrate to Pydantic V2 style `@model_validator` validators, see the migration guide for more details. Deprecated in Pydantic V2.0 to be removed in V3.0. See Pydantic V2 Migration Guide at https://errors.pydantic.dev/2.7/migration/
|
||||
@root_validator(pre=True)
|
||||
|
||||
../proxy/_types.py:823
|
||||
/Users/krrishdholakia/Documents/litellm/litellm/proxy/_types.py:823: PydanticDeprecatedSince20: Pydantic V1 style `@root_validator` validators are deprecated. You should migrate to Pydantic V2 style `@model_validator` validators, see the migration guide for more details. Deprecated in Pydantic V2.0 to be removed in V3.0. See Pydantic V2 Migration Guide at https://errors.pydantic.dev/2.7/migration/
|
||||
../proxy/_types.py:840
|
||||
/Users/krrishdholakia/Documents/litellm/litellm/proxy/_types.py:840: PydanticDeprecatedSince20: Pydantic V1 style `@root_validator` validators are deprecated. You should migrate to Pydantic V2 style `@model_validator` validators, see the migration guide for more details. Deprecated in Pydantic V2.0 to be removed in V3.0. See Pydantic V2 Migration Guide at https://errors.pydantic.dev/2.7/migration/
|
||||
@root_validator(pre=True)
|
||||
|
||||
../proxy/_types.py:850
|
||||
/Users/krrishdholakia/Documents/litellm/litellm/proxy/_types.py:850: PydanticDeprecatedSince20: Pydantic V1 style `@root_validator` validators are deprecated. You should migrate to Pydantic V2 style `@model_validator` validators, see the migration guide for more details. Deprecated in Pydantic V2.0 to be removed in V3.0. See Pydantic V2 Migration Guide at https://errors.pydantic.dev/2.7/migration/
|
||||
../proxy/_types.py:867
|
||||
/Users/krrishdholakia/Documents/litellm/litellm/proxy/_types.py:867: PydanticDeprecatedSince20: Pydantic V1 style `@root_validator` validators are deprecated. You should migrate to Pydantic V2 style `@model_validator` validators, see the migration guide for more details. Deprecated in Pydantic V2.0 to be removed in V3.0. See Pydantic V2 Migration Guide at https://errors.pydantic.dev/2.7/migration/
|
||||
@root_validator(pre=True)
|
||||
|
||||
../proxy/_types.py:869
|
||||
/Users/krrishdholakia/Documents/litellm/litellm/proxy/_types.py:869: PydanticDeprecatedSince20: Pydantic V1 style `@root_validator` validators are deprecated. You should migrate to Pydantic V2 style `@model_validator` validators, see the migration guide for more details. Deprecated in Pydantic V2.0 to be removed in V3.0. See Pydantic V2 Migration Guide at https://errors.pydantic.dev/2.7/migration/
|
||||
../proxy/_types.py:886
|
||||
/Users/krrishdholakia/Documents/litellm/litellm/proxy/_types.py:886: PydanticDeprecatedSince20: Pydantic V1 style `@root_validator` validators are deprecated. You should migrate to Pydantic V2 style `@model_validator` validators, see the migration guide for more details. Deprecated in Pydantic V2.0 to be removed in V3.0. See Pydantic V2 Migration Guide at https://errors.pydantic.dev/2.7/migration/
|
||||
@root_validator(pre=True)
|
||||
|
||||
../../../../../../opt/homebrew/lib/python3.11/site-packages/pkg_resources/__init__.py:121
|
||||
|
|
@ -126,30 +111,33 @@ final cost: 2.55e-05; prompt_tokens_cost_usd_dollar: 2.55e-05; completion_tokens
|
|||
Implementing implicit namespace packages (as specified in PEP 420) is preferred to `pkg_resources.declare_namespace`. See https://setuptools.pypa.io/en/latest/references/keywords.html#keyword-namespace-packages
|
||||
declare_namespace(pkg)
|
||||
|
||||
test_custom_logger.py::test_redis_cache_completion_stream
|
||||
/opt/homebrew/lib/python3.11/site-packages/_pytest/unraisableexception.py:78: PytestUnraisableExceptionWarning: Exception ignored in: <function StreamWriter.__del__ at 0x1019c28e0>
|
||||
test_image_generation.py::test_aimage_generation_bedrock_with_optional_params
|
||||
/opt/homebrew/lib/python3.11/site-packages/_pytest/threadexception.py:73: PytestUnhandledThreadExceptionWarning: Exception in thread Thread-1 (success_handler)
|
||||
|
||||
Traceback (most recent call last):
|
||||
File "/opt/homebrew/Cellar/python@3.11/3.11.6_1/Frameworks/Python.framework/Versions/3.11/lib/python3.11/asyncio/streams.py", line 395, in __del__
|
||||
self.close()
|
||||
File "/opt/homebrew/Cellar/python@3.11/3.11.6_1/Frameworks/Python.framework/Versions/3.11/lib/python3.11/asyncio/streams.py", line 343, in close
|
||||
return self._transport.close()
|
||||
^^^^^^^^^^^^^^^^^^^^^^^
|
||||
File "/opt/homebrew/Cellar/python@3.11/3.11.6_1/Frameworks/Python.framework/Versions/3.11/lib/python3.11/asyncio/sslproto.py", line 112, in close
|
||||
self._ssl_protocol._start_shutdown()
|
||||
File "/opt/homebrew/Cellar/python@3.11/3.11.6_1/Frameworks/Python.framework/Versions/3.11/lib/python3.11/asyncio/sslproto.py", line 620, in _start_shutdown
|
||||
self._shutdown_timeout_handle = self._loop.call_later(
|
||||
^^^^^^^^^^^^^^^^^^^^^^
|
||||
File "/opt/homebrew/Cellar/python@3.11/3.11.6_1/Frameworks/Python.framework/Versions/3.11/lib/python3.11/asyncio/base_events.py", line 727, in call_later
|
||||
timer = self.call_at(self.time() + delay, callback, *args,
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
File "/opt/homebrew/Cellar/python@3.11/3.11.6_1/Frameworks/Python.framework/Versions/3.11/lib/python3.11/asyncio/base_events.py", line 740, in call_at
|
||||
self._check_closed()
|
||||
File "/opt/homebrew/Cellar/python@3.11/3.11.6_1/Frameworks/Python.framework/Versions/3.11/lib/python3.11/asyncio/base_events.py", line 519, in _check_closed
|
||||
raise RuntimeError('Event loop is closed')
|
||||
RuntimeError: Event loop is closed
|
||||
File "/Users/krrishdholakia/Documents/litellm/litellm/utils.py", line 1412, in _success_handler_helper_fn
|
||||
litellm.completion_cost(
|
||||
File "/Users/krrishdholakia/Documents/litellm/litellm/utils.py", line 4442, in completion_cost
|
||||
raise e
|
||||
File "/Users/krrishdholakia/Documents/litellm/litellm/utils.py", line 4405, in completion_cost
|
||||
raise Exception(
|
||||
Exception: Model=1024-x-1024/stability.stable-diffusion-xl-v1 not found in completion cost model map
|
||||
|
||||
warnings.warn(pytest.PytestUnraisableExceptionWarning(msg))
|
||||
During handling of the above exception, another exception occurred:
|
||||
|
||||
Traceback (most recent call last):
|
||||
File "/opt/homebrew/Cellar/python@3.11/3.11.6_1/Frameworks/Python.framework/Versions/3.11/lib/python3.11/threading.py", line 1045, in _bootstrap_inner
|
||||
self.run()
|
||||
File "/opt/homebrew/Cellar/python@3.11/3.11.6_1/Frameworks/Python.framework/Versions/3.11/lib/python3.11/threading.py", line 982, in run
|
||||
self._target(*self._args, **self._kwargs)
|
||||
File "/Users/krrishdholakia/Documents/litellm/litellm/utils.py", line 1465, in success_handler
|
||||
start_time, end_time, result = self._success_handler_helper_fn(
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
File "/Users/krrishdholakia/Documents/litellm/litellm/utils.py", line 1459, in _success_handler_helper_fn
|
||||
raise Exception(f"[Non-Blocking] LiteLLM.Success_Call Error: {str(e)}")
|
||||
Exception: [Non-Blocking] LiteLLM.Success_Call Error: Model=1024-x-1024/stability.stable-diffusion-xl-v1 not found in completion cost model map
|
||||
|
||||
warnings.warn(pytest.PytestUnhandledThreadExceptionWarning(msg))
|
||||
|
||||
-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html
|
||||
======================== 1 passed, 56 warnings in 2.43s ========================
|
||||
======================== 1 passed, 61 warnings in 3.00s ========================
|
||||
|
|
|
|||
|
|
@ -10,7 +10,37 @@ sys.path.insert(
|
|||
import time
|
||||
import litellm
|
||||
import openai
|
||||
import pytest, uuid
|
||||
import pytest, uuid, httpx
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model, provider",
|
||||
[
|
||||
("gpt-3.5-turbo", "openai"),
|
||||
("anthropic.claude-instant-v1", "bedrock"),
|
||||
("azure/chatgpt-v-2", "azure"),
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("sync_mode", [True, False])
|
||||
@pytest.mark.asyncio
|
||||
async def test_httpx_timeout(model, provider, sync_mode):
|
||||
"""
|
||||
Test if setting httpx.timeout works for completion calls
|
||||
"""
|
||||
timeout_val = httpx.Timeout(10.0, connect=60.0)
|
||||
|
||||
messages = [{"role": "user", "content": "Hey, how's it going?"}]
|
||||
|
||||
if sync_mode:
|
||||
response = litellm.completion(
|
||||
model=model, messages=messages, timeout=timeout_val
|
||||
)
|
||||
else:
|
||||
response = await litellm.acompletion(
|
||||
model=model, messages=messages, timeout=timeout_val
|
||||
)
|
||||
|
||||
print(f"response: {response}")
|
||||
|
||||
|
||||
def test_timeout():
|
||||
|
|
|
|||
|
|
@ -4453,7 +4453,19 @@ def completion_cost(
|
|||
raise e
|
||||
|
||||
|
||||
def supports_function_calling(model: str):
|
||||
def supports_httpx_timeout(custom_llm_provider: str) -> bool:
|
||||
"""
|
||||
Helper function to know if a provider implementation supports httpx timeout
|
||||
"""
|
||||
supported_providers = ["openai", "azure", "bedrock"]
|
||||
|
||||
if custom_llm_provider in supported_providers:
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
|
||||
def supports_function_calling(model: str) -> bool:
|
||||
"""
|
||||
Check if the given model supports function calling and return a boolean value.
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue