phoenix/litellm-mirror
0
0
Fork 1
mirror of https://github.com/BerriAI/litellm.git synced 2025-04-26 03:04:13 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

docs(routing.md): update tutorial on deploying router

Browse source
This commit is contained in:
Krrish Dholakia 2023-11-16 21:46:35 -08:00
parent c4f51594d5
commit d9123ea2e8
4 changed files with 39 additions and 40 deletions
Download patch file Download diff file Expand all files Collapse all files

7
litellm/router.py
View file

@ -153,7 +153,10 @@ class Router:
for current_attempt in range(self.num_retries):
try:
# if the function call is successful, no exception will be raised and we'll break out of the loop
return await original_function(*args, **kwargs)
response = await original_function(*args, **kwargs)
if isinstance(response, asyncio.coroutines.Coroutine): # async errors are often returned as coroutines
response = await response
return response
except openai.RateLimitError as e:
# on RateLimitError we'll wait for an exponential time before trying again
@ -231,6 +234,8 @@ class Router:
deployment = self.get_available_deployment(model=model, messages=messages)
data = deployment["litellm_params"]
response = await litellm.acompletion(**{**data, "messages": messages, "caching": self.cache_responses, **kwargs})
if isinstance(response, asyncio.coroutines.Coroutine): # async errors are often returned as coroutines
response = await response
return response
except Exception as e:
kwargs["model"] = model