(fix) router - timeout exception mapping

This commit is contained in:
ishaan-jaff 2024-01-19 20:30:31 -08:00
parent 7cf0bb475f
commit 84684c50fa
2 changed files with 19 additions and 11 deletions

View file

@ -39,6 +39,8 @@ def test_timeout():
def test_hanging_request_azure():
litellm.set_verbose = True
import asyncio
try:
router = litellm.Router(
model_list=[
@ -58,13 +60,20 @@ def test_hanging_request_azure():
)
encoded = litellm.utils.encode(model="gpt-3.5-turbo", text="blue")[0]
response = router.completion(
model="azure-gpt",
messages=[{"role": "user", "content": f"what color is red {uuid.uuid4()}"}],
logit_bias={encoded: 100},
timeout=0.01,
)
print(response)
async def _test():
response = await router.acompletion(
model="azure-gpt",
messages=[
{"role": "user", "content": f"what color is red {uuid.uuid4()}"}
],
logit_bias={encoded: 100},
timeout=0.01,
)
print(response)
return response
response = asyncio.run(_test())
if response.choices[0].message.content is not None:
pytest.fail("Got a response, expected a timeout")