phoenix/litellm-mirror
0
0
Fork 1
mirror of https://github.com/BerriAI/litellm.git synced 2025-04-26 03:04:13 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

fixing exception mapping

Browse source
This commit is contained in:
Krrish Dholakia 2023-08-05 09:52:01 -07:00
parent 9b0e9bf57c
commit 92a13958ce
8 changed files with 188 additions and 115 deletions
Download patch file Download diff file Expand all files Collapse all files

93
litellm/tests/test_exceptions.py
View file

@ -8,6 +8,7 @@ from litellm import embedding, completion
from concurrent.futures import ThreadPoolExecutor
import pytest
litellm.failure_callback = ["sentry"]
# litellm.set_verbose = True
#### What this tests ####
# This tests exception mapping -> trigger an exception from an llm provider -> assert if output is of the expected type
@ -22,11 +23,16 @@ import pytest
# models = ["gpt-3.5-turbo", "chatgpt-test", "claude-instant-1", "command-nightly"]
models = ["command-nightly"]
def logging_fn(model_call_dict):
print(f"model_call_dict: {model_call_dict['model']}")
if "model" in model_call_dict:
print(f"model_call_dict: {model_call_dict['model']}")
else:
print(f"model_call_dict: {model_call_dict}")
# Test 1: Context Window Errors
@pytest.mark.parametrize("model", models)
def test_context_window(model):
sample_text = "how does a court case get to the Supreme Court?" * 100000
sample_text = "how does a court case get to the Supreme Court?" * 5000
messages = [{"content": sample_text, "role": "user"}]
try:
azure = model == "chatgpt-test"
@ -41,44 +47,61 @@ def test_context_window(model):
return
except Exception as e:
print("Uncaught Error in test_context_window")
# print(f"Error Type: {type(e).__name__}")
print(f"Error Type: {type(e).__name__}")
print(f"Uncaught Exception - {e}")
pytest.fail(f"Error occurred: {e}")
return
test_context_window("command-nightly")
# # Test 2: InvalidAuth Errors
# def logger_fn(model_call_object: dict):
# print(f"model call details: {model_call_object}")
# @pytest.mark.parametrize("model", models)
# def invalid_auth(model): # set the model key to an invalid key, depending on the model
# messages = [{ "content": "Hello, how are you?","role": "user"}]
# try:
# azure = False
# if model == "gpt-3.5-turbo":
# os.environ["OPENAI_API_KEY"] = "bad-key"
# elif model == "chatgpt-test":
# os.environ["AZURE_API_KEY"] = "bad-key"
# azure = True
# elif model == "claude-instant-1":
# os.environ["ANTHROPIC_API_KEY"] = "bad-key"
# elif model == "command-nightly":
# os.environ["COHERE_API_KEY"] = "bad-key"
# elif model == "replicate/llama-2-70b-chat:2c1608e18606fad2812020dc541930f2d0495ce32eee50074220b87300bc16e1":
# os.environ["REPLICATE_API_KEY"] = "bad-key"
# os.environ["REPLICATE_API_TOKEN"] = "bad-key"
# print(f"model: {model}")
# response = completion(model=model, messages=messages, azure=azure)
# print(f"response: {response}")
# except AuthenticationError as e:
# return
# except OpenAIError: # is at least an openai error -> in case of random model errors - e.g. overloaded server
# return
# except Exception as e:
# print(f"Uncaught Exception - {e}")
# pytest.fail(f"Error occurred: {e}")
# return
# Test 2: InvalidAuth Errors
@pytest.mark.parametrize("model", models)
def invalid_auth(model): # set the model key to an invalid key, depending on the model
messages = [{ "content": "Hello, how are you?","role": "user"}]
temporary_key = None
try:
azure = False
if model == "gpt-3.5-turbo":
temporary_key = os.environ["OPENAI_API_KEY"]
os.environ["OPENAI_API_KEY"] = "bad-key"
elif model == "chatgpt-test":
temporary_key = os.environ["AZURE_API_KEY"]
os.environ["AZURE_API_KEY"] = "bad-key"
azure = True
elif model == "claude-instant-1":
temporary_key = os.environ["ANTHROPIC_API_KEY"]
os.environ["ANTHROPIC_API_KEY"] = "bad-key"
elif model == "command-nightly":
temporary_key = os.environ["COHERE_API_KEY"]
os.environ["COHERE_API_KEY"] = "bad-key"
elif model == "replicate/llama-2-70b-chat:2c1608e18606fad2812020dc541930f2d0495ce32eee50074220b87300bc16e1":
temporary_key = os.environ["REPLICATE_API_KEY"]
os.environ["REPLICATE_API_KEY"] = "bad-key"
print(f"model: {model}")
response = completion(model=model, messages=messages, azure=azure)
print(f"response: {response}")
except AuthenticationError as e:
print(f"AuthenticationError Caught Exception - {e}")
except OpenAIError: # is at least an openai error -> in case of random model errors - e.g. overloaded server
print(f"OpenAIError Caught Exception - {e}")
except Exception as e:
print(type(e))
print(e.__class__.__name__)
print(f"Uncaught Exception - {e}")
pytest.fail(f"Error occurred: {e}")
if temporary_key != None: # reset the key
if model == "gpt-3.5-turbo":
os.environ["OPENAI_API_KEY"] = temporary_key
elif model == "chatgpt-test":
os.environ["AZURE_API_KEY"] = temporary_key
azure = True
elif model == "claude-instant-1":
os.environ["ANTHROPIC_API_KEY"] = temporary_key
elif model == "command-nightly":
os.environ["COHERE_API_KEY"] = temporary_key
elif model == "replicate/llama-2-70b-chat:2c1608e18606fad2812020dc541930f2d0495ce32eee50074220b87300bc16e1":
os.environ["REPLICATE_API_KEY"] = temporary_key
return
invalid_auth("command-nightly")
# # Test 3: Rate Limit Errors
# def test_model(model):
# try: