phoenix/litellm-mirror
0
0
Fork 1
mirror of https://github.com/BerriAI/litellm.git synced 2025-04-24 10:14:26 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

fix(router.py): check if fallbacks is none

Browse source
This commit is contained in:
Krrish Dholakia 2023-11-25 14:58:07 -08:00
parent 95579fda7d
commit ab0bc87427
4 changed files with 11 additions and 5 deletions
Download patch file Download diff file Expand all files Collapse all files

10
litellm/router.py
View file

@ -281,7 +281,7 @@ class Router:
original_exception = e
try:
self.print_verbose(f"Trying to fallback b/w models")
if isinstance(e, litellm.ContextWindowExceededError):
if isinstance(e, litellm.ContextWindowExceededError) and self.context_window_fallbacks is not None:
fallback_model_group = None
for item in self.context_window_fallbacks: # [{"gpt-3.5-turbo": ["gpt-4"]}]
if list(item.keys())[0] == model_group:
@ -302,6 +302,8 @@ class Router:
except Exception as e:
pass
else:
if self.fallbacks is None:
raise original_exception
self.print_verbose(f"inside model fallbacks: {self.fallbacks}")
for item in self.fallbacks:
if list(item.keys())[0] == model_group:
@ -374,9 +376,10 @@ class Router:
self.print_verbose(f"An exception occurs {original_exception}")
try:
self.print_verbose(f"Trying to fallback b/w models. Initial model group: {model_group}")
if isinstance(e, litellm.ContextWindowExceededError):
if isinstance(e, litellm.ContextWindowExceededError) and self.context_window_fallbacks is not None:
self.print_verbose(f"inside context window fallbacks: {self.context_window_fallbacks}")
fallback_model_group = None
for item in self.context_window_fallbacks: # [{"gpt-3.5-turbo": ["gpt-4"]}]
if list(item.keys())[0] == model_group:
fallback_model_group = item[model_group]
@ -396,6 +399,9 @@ class Router:
except Exception as e:
pass
else:
if self.fallbacks is None:
raise original_exception
self.print_verbose(f"inside model fallbacks: {self.fallbacks}")
fallback_model_group = None
for item in self.fallbacks:

2
litellm/tests/test_acooldowns_router.py
View file

@ -93,7 +93,7 @@ def test_multiple_deployments_parallel():
del futures[future] # Remove the done future with exception
print(f"Remaining futures: {len(futures)}")
router.reset()
end_time = time.time()
print(results)
print(f"ELAPSED TIME: {end_time - start_time}")

2
litellm/tests/test_router.py
View file

@ -116,7 +116,7 @@ def test_reading_key_from_model_list():
except Exception as e:
os.environ["AZURE_API_KEY"] = old_api_key
print(f"FAILED TEST")
pytest.fail("Got unexpected exception on router!", e)
pytest.fail(f"Got unexpected exception on router! - {e}")
# test_reading_key_from_model_list()

2
litellm/utils.py
View file

@ -1991,7 +1991,7 @@ def get_optional_params( # use the openai defaults
optional_params["temperature"] = temperature
if max_tokens is not None:
optional_params["max_tokens"] = max_tokens
if logit_bias != {}:
if logit_bias is not None:
optional_params["logit_bias"] = logit_bias
if top_p is not None:
optional_params["p"] = top_p