phoenix/litellm-mirror
0
0
Fork 1
mirror of https://github.com/BerriAI/litellm.git synced 2025-04-27 11:43:54 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

fix(router.py): fix client init for streaming timeouts

Browse source
This commit is contained in:
Krrish Dholakia 2024-04-05 12:06:46 -07:00 • committed by Ishaan Jaff
parent a8d0aace83
commit bf8097e961
4 changed files with 29 additions and 7 deletions
Download patch file Download diff file Expand all files Collapse all files

21
litellm/tests/test_router_init.py
View file

@ -252,24 +252,31 @@ def test_stream_timeouts_router():
"api_version": os.getenv("AZURE_API_VERSION"),
"api_base": os.getenv("AZURE_API_BASE"),
"timeout": 200, # regular calls will not timeout, stream calls will
"stream_timeout": 0.000_001,
"stream_timeout": 10,
},
},
]
router = Router(model_list=model_list)
print("PASSED !")
data = {
"model": "gpt-3.5-turbo",
"messages": [{"role": "user", "content": "hello, write a 20 pg essay"}],
"stream": True,
}
selected_client = router._get_client(
deployment=router.model_list[0],
kwargs={
"model": "gpt-3.5-turbo",
"messages": [{"role": "user", "content": "hello, write a 20 pg essay"}],
"stream": True,
},
kwargs=data,
client_type=None,
)
print("Select client timeout", selected_client.timeout)
assert selected_client.timeout == 0.000_001
assert selected_client.timeout == 10
# make actual call
response = router.completion(**data)
for chunk in response:
print(f"chunk: {chunk}")
except openai.APITimeoutError as e:
print(
"Passed: Raised correct exception. Got openai.APITimeoutError\nGood Job", e