phoenix/litellm
1
0
Fork 0
forked from phoenix/litellm-mirror
Code Pull requests Activity

adding tenacity retries

Browse source
This commit is contained in:
Krrish Dholakia 2023-08-03 10:06:31 -07:00
parent 10832be1e4
commit a8b3fc6c2d
12 changed files with 37 additions and 11 deletions
Download patch file Download diff file Expand all files Collapse all files

6
litellm/main.py
View file

@ -7,6 +7,11 @@ import traceback
import litellm
from litellm import client, logging, exception_type, timeout, success_callback, failure_callback
import random
from tenacity import (
retry,
stop_after_attempt,
wait_random_exponential,
) # for exponential backoff
####### ENVIRONMENT VARIABLES ###################
dotenv.load_dotenv() # Loading env variables using dotenv
@ -55,6 +60,7 @@ def get_optional_params(
####### COMPLETION ENDPOINTS ################
#############################################
@client
@retry(wait=wait_random_exponential(min=1, max=60), stop=stop_after_attempt(2), reraise=True, retry_error_callback=lambda retry_state: setattr(retry_state.outcome, 'retry_variable', litellm.retry)) # retry call, turn this off by setting `litellm.retry = False`
@timeout(60) ## set timeouts, in case calls hang (e.g. Azure) - default is 60s, override with `force_timeout`
def completion(
model, messages, # required params