From 0558ac2345dd264aba81cc575f0d8a036adabac6 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Thu, 21 Mar 2024 08:32:42 -0700 Subject: [PATCH] (fix) don't run .completion retries if using router / proxy --- litellm/utils.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/litellm/utils.py b/litellm/utils.py index a8c003181..1b72ee42e 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -2752,7 +2752,12 @@ def client(original_function): "context_window_fallback_dict", {} ) - if num_retries: + _is_litellm_router_call = "model_group" in kwargs.get( + "metadata", {} + ) # check if call from litellm.router/proxy + if ( + num_retries and not _is_litellm_router_call + ): # only enter this if call is not from litellm router/proxy. router has it's own logic for retrying if ( isinstance(e, openai.APIError) or isinstance(e, openai.Timeout) @@ -3222,7 +3227,12 @@ def client(original_function): "context_window_fallback_dict", {} ) - if num_retries: + _is_litellm_router_call = "model_group" in kwargs.get( + "metadata", {} + ) # check if call from litellm.router/proxy + if ( + num_retries and not _is_litellm_router_call + ): # only enter this if call is not from litellm router/proxy. router has it's own logic for retrying try: kwargs["num_retries"] = num_retries kwargs["original_function"] = original_function