From 28a2e31edea08d4e54e7dcc987104619aa31ee41 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Sat, 7 Oct 2023 18:23:54 -0700 Subject: [PATCH] ci(test_logging): running just this test on circle ci --- .circleci/config.yml | 2 +- litellm/tests/test_logging.py | 626 +++++++++++++++++----------------- 2 files changed, 314 insertions(+), 314 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 31cea53a5..43ffb9ddd 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -56,7 +56,7 @@ jobs: command: | pwd ls - python -m pytest -vv tests/test_logging.py --junitxml=test-results/junit.xml + python -m pytest -vv litellm/tests/test_logging.py --junitxml=test-results/junit.xml no_output_timeout: 30m # Store test results diff --git a/litellm/tests/test_logging.py b/litellm/tests/test_logging.py index c8a353075..d0f1e8650 100644 --- a/litellm/tests/test_logging.py +++ b/litellm/tests/test_logging.py @@ -40,343 +40,343 @@ messages = [{"content": user_message, "role": "user"}] # 1. On Call Success # normal completion +# test on openai completion call +def test_logging_success_completion(): + global score + try: + # Redirect stdout + old_stdout = sys.stdout + sys.stdout = new_stdout = io.StringIO() + + response = completion(model="gpt-3.5-turbo", messages=messages) + # Restore stdout + sys.stdout = old_stdout + output = new_stdout.getvalue().strip() + + if "Logging Details Pre-API Call" not in output: + raise Exception("Required log message not found!") + elif "Logging Details Post-API Call" not in output: + raise Exception("Required log message not found!") + elif "Logging Details LiteLLM-Success Call" not in output: + raise Exception("Required log message not found!") + score += 1 + except Exception as e: + pytest.fail(f"Error occurred: {e}") + pass + +## test on non-openai completion call +def test_logging_success_completion_non_openai(): + global score + try: + # Redirect stdout + old_stdout = sys.stdout + sys.stdout = new_stdout = io.StringIO() + + response = completion(model="claude-instant-1", messages=messages) + + # Restore stdout + sys.stdout = old_stdout + output = new_stdout.getvalue().strip() + + if "Logging Details Pre-API Call" not in output: + raise Exception("Required log message not found!") + elif "Logging Details Post-API Call" not in output: + raise Exception("Required log message not found!") + elif "Logging Details LiteLLM-Success Call" not in output: + raise Exception("Required log message not found!") + score += 1 + except Exception as e: + pytest.fail(f"Error occurred: {e}") + pass + +# streaming completion ## test on openai completion call -# def test_logging_success_completion(): -# global score +def test_logging_success_streaming_openai(): + global score + try: + # litellm.set_verbose = False + def custom_callback( + kwargs, # kwargs to completion + completion_response, # response from completion + start_time, end_time # start/end time + ): + if "complete_streaming_response" in kwargs: + print(f"Complete Streaming Response: {kwargs['complete_streaming_response']}") + + # Assign the custom callback function + litellm.success_callback = [custom_callback] + + # Redirect stdout + old_stdout = sys.stdout + sys.stdout = new_stdout = io.StringIO() + + response = completion(model="gpt-3.5-turbo", messages=messages, stream=True) + for chunk in response: + pass + + # Restore stdout + sys.stdout = old_stdout + output = new_stdout.getvalue().strip() + + if "Logging Details Pre-API Call" not in output: + raise Exception("Required log message not found!") + elif "Logging Details Post-API Call" not in output: + raise Exception("Required log message not found!") + elif "Logging Details LiteLLM-Success Call" not in output: + raise Exception("Required log message not found!") + elif "Complete Streaming Response:" not in output: + raise Exception("Required log message not found!") + score += 1 + except Exception as e: + pytest.fail(f"Error occurred: {e}") + pass + +# test_logging_success_streaming_openai() + +## test on non-openai completion call +def test_logging_success_streaming_non_openai(): + global score + try: + # litellm.set_verbose = False + def custom_callback( + kwargs, # kwargs to completion + completion_response, # response from completion + start_time, end_time # start/end time + ): + # print(f"streaming response: {completion_response}") + if "complete_streaming_response" in kwargs: + print(f"Complete Streaming Response: {kwargs['complete_streaming_response']}") + + # Assign the custom callback function + litellm.success_callback = [custom_callback] + + # Redirect stdout + old_stdout = sys.stdout + sys.stdout = new_stdout = io.StringIO() + + response = completion(model="claude-instant-1", messages=messages, stream=True) + for idx, chunk in enumerate(response): + pass + + # Restore stdout + sys.stdout = old_stdout + output = new_stdout.getvalue().strip() + + if "Logging Details Pre-API Call" not in output: + raise Exception("Required log message not found!") + elif "Logging Details Post-API Call" not in output: + raise Exception("Required log message not found!") + elif "Logging Details LiteLLM-Success Call" not in output: + raise Exception("Required log message not found!") + elif "Complete Streaming Response:" not in output: + raise Exception(f"Required log message not found! {output}") + score += 1 + except Exception as e: + pytest.fail(f"Error occurred: {e}") + pass + +# test_logging_success_streaming_non_openai() +# embedding + +def test_logging_success_embedding_openai(): + try: + # Redirect stdout + old_stdout = sys.stdout + sys.stdout = new_stdout = io.StringIO() + + response = embedding(model="text-embedding-ada-002", input=["good morning from litellm"]) + + # Restore stdout + sys.stdout = old_stdout + output = new_stdout.getvalue().strip() + + if "Logging Details Pre-API Call" not in output: + raise Exception("Required log message not found!") + elif "Logging Details Post-API Call" not in output: + raise Exception("Required log message not found!") + elif "Logging Details LiteLLM-Success Call" not in output: + raise Exception("Required log message not found!") + except Exception as e: + pytest.fail(f"Error occurred: {e}") + +# ## 2. On LiteLLM Call failure +# ## TEST BAD KEY + +# # normal completion +# ## test on openai completion call +# try: +# temporary_oai_key = os.environ["OPENAI_API_KEY"] +# os.environ["OPENAI_API_KEY"] = "bad-key" + +# temporary_anthropic_key = os.environ["ANTHROPIC_API_KEY"] +# os.environ["ANTHROPIC_API_KEY"] = "bad-key" + + +# # Redirect stdout +# old_stdout = sys.stdout +# sys.stdout = new_stdout = io.StringIO() + # try: -# # Redirect stdout -# old_stdout = sys.stdout -# sys.stdout = new_stdout = io.StringIO() - # response = completion(model="gpt-3.5-turbo", messages=messages) -# # Restore stdout -# sys.stdout = old_stdout -# output = new_stdout.getvalue().strip() - -# if "Logging Details Pre-API Call" not in output: -# raise Exception("Required log message not found!") -# elif "Logging Details Post-API Call" not in output: -# raise Exception("Required log message not found!") -# elif "Logging Details LiteLLM-Success Call" not in output: -# raise Exception("Required log message not found!") -# score += 1 -# except Exception as e: -# pytest.fail(f"Error occurred: {e}") +# except AuthenticationError: +# print(f"raised auth error") # pass +# # Restore stdout +# sys.stdout = old_stdout +# output = new_stdout.getvalue().strip() + +# print(output) + +# if "Logging Details Pre-API Call" not in output: +# raise Exception("Required log message not found!") +# elif "Logging Details Post-API Call" not in output: +# raise Exception("Required log message not found!") +# elif "Logging Details LiteLLM-Failure Call" not in output: +# raise Exception("Required log message not found!") + +# os.environ["OPENAI_API_KEY"] = temporary_oai_key +# os.environ["ANTHROPIC_API_KEY"] = temporary_anthropic_key + +# score += 1 +# except Exception as e: +# print(f"exception type: {type(e).__name__}") +# pytest.fail(f"Error occurred: {e}") +# pass # ## test on non-openai completion call -# def test_logging_success_completion_non_openai(): -# global score +# try: +# temporary_oai_key = os.environ["OPENAI_API_KEY"] +# os.environ["OPENAI_API_KEY"] = "bad-key" + +# temporary_anthropic_key = os.environ["ANTHROPIC_API_KEY"] +# os.environ["ANTHROPIC_API_KEY"] = "bad-key" +# # Redirect stdout +# old_stdout = sys.stdout +# sys.stdout = new_stdout = io.StringIO() + # try: -# # Redirect stdout -# old_stdout = sys.stdout -# sys.stdout = new_stdout = io.StringIO() - # response = completion(model="claude-instant-1", messages=messages) - -# # Restore stdout -# sys.stdout = old_stdout -# output = new_stdout.getvalue().strip() - -# if "Logging Details Pre-API Call" not in output: -# raise Exception("Required log message not found!") -# elif "Logging Details Post-API Call" not in output: -# raise Exception("Required log message not found!") -# elif "Logging Details LiteLLM-Success Call" not in output: -# raise Exception("Required log message not found!") -# score += 1 -# except Exception as e: -# pytest.fail(f"Error occurred: {e}") +# except AuthenticationError: # pass +# if "Logging Details Pre-API Call" not in output: +# raise Exception("Required log message not found!") +# elif "Logging Details Post-API Call" not in output: +# raise Exception("Required log message not found!") +# elif "Logging Details LiteLLM-Failure Call" not in output: +# raise Exception("Required log message not found!") +# os.environ["OPENAI_API_KEY"] = temporary_oai_key +# os.environ["ANTHROPIC_API_KEY"] = temporary_anthropic_key +# score += 1 +# except Exception as e: +# print(f"exception type: {type(e).__name__}") +# # Restore stdout +# sys.stdout = old_stdout +# output = new_stdout.getvalue().strip() + +# print(output) +# pytest.fail(f"Error occurred: {e}") + + # # streaming completion # ## test on openai completion call -# def test_logging_success_streaming_openai(): -# global score +# try: +# temporary_oai_key = os.environ["OPENAI_API_KEY"] +# os.environ["OPENAI_API_KEY"] = "bad-key" + +# temporary_anthropic_key = os.environ["ANTHROPIC_API_KEY"] +# os.environ["ANTHROPIC_API_KEY"] = "bad-key" +# # Redirect stdout +# old_stdout = sys.stdout +# sys.stdout = new_stdout = io.StringIO() + # try: -# # litellm.set_verbose = False -# def custom_callback( -# kwargs, # kwargs to completion -# completion_response, # response from completion -# start_time, end_time # start/end time -# ): -# if "complete_streaming_response" in kwargs: -# print(f"Complete Streaming Response: {kwargs['complete_streaming_response']}") - -# # Assign the custom callback function -# litellm.success_callback = [custom_callback] - -# # Redirect stdout -# old_stdout = sys.stdout -# sys.stdout = new_stdout = io.StringIO() - -# response = completion(model="gpt-3.5-turbo", messages=messages, stream=True) -# for chunk in response: -# pass - -# # Restore stdout -# sys.stdout = old_stdout -# output = new_stdout.getvalue().strip() - -# if "Logging Details Pre-API Call" not in output: -# raise Exception("Required log message not found!") -# elif "Logging Details Post-API Call" not in output: -# raise Exception("Required log message not found!") -# elif "Logging Details LiteLLM-Success Call" not in output: -# raise Exception("Required log message not found!") -# elif "Complete Streaming Response:" not in output: -# raise Exception("Required log message not found!") -# score += 1 -# except Exception as e: -# pytest.fail(f"Error occurred: {e}") +# response = completion(model="gpt-3.5-turbo", messages=messages) +# except AuthenticationError: # pass -# # test_logging_success_streaming_openai() +# # Restore stdout +# sys.stdout = old_stdout +# output = new_stdout.getvalue().strip() + +# print(output) + +# if "Logging Details Pre-API Call" not in output: +# raise Exception("Required log message not found!") +# elif "Logging Details Post-API Call" not in output: +# raise Exception("Required log message not found!") +# elif "Logging Details LiteLLM-Failure Call" not in output: +# raise Exception("Required log message not found!") + +# os.environ["OPENAI_API_KEY"] = temporary_oai_key +# os.environ["ANTHROPIC_API_KEY"] = temporary_anthropic_key +# score += 1 +# except Exception as e: +# print(f"exception type: {type(e).__name__}") +# pytest.fail(f"Error occurred: {e}") # ## test on non-openai completion call -# def test_logging_success_streaming_non_openai(): -# global score +# try: +# temporary_oai_key = os.environ["OPENAI_API_KEY"] +# os.environ["OPENAI_API_KEY"] = "bad-key" + +# temporary_anthropic_key = os.environ["ANTHROPIC_API_KEY"] +# os.environ["ANTHROPIC_API_KEY"] = "bad-key" +# # Redirect stdout +# old_stdout = sys.stdout +# sys.stdout = new_stdout = io.StringIO() + # try: -# # litellm.set_verbose = False -# def custom_callback( -# kwargs, # kwargs to completion -# completion_response, # response from completion -# start_time, end_time # start/end time -# ): -# # print(f"streaming response: {completion_response}") -# if "complete_streaming_response" in kwargs: -# print(f"Complete Streaming Response: {kwargs['complete_streaming_response']}") - -# # Assign the custom callback function -# litellm.success_callback = [custom_callback] - -# # Redirect stdout -# old_stdout = sys.stdout -# sys.stdout = new_stdout = io.StringIO() - -# response = completion(model="claude-instant-1", messages=messages, stream=True) -# for idx, chunk in enumerate(response): -# pass - -# # Restore stdout -# sys.stdout = old_stdout -# output = new_stdout.getvalue().strip() - -# if "Logging Details Pre-API Call" not in output: -# raise Exception("Required log message not found!") -# elif "Logging Details Post-API Call" not in output: -# raise Exception("Required log message not found!") -# elif "Logging Details LiteLLM-Success Call" not in output: -# raise Exception("Required log message not found!") -# elif "Complete Streaming Response:" not in output: -# raise Exception(f"Required log message not found! {output}") -# score += 1 -# except Exception as e: -# pytest.fail(f"Error occurred: {e}") +# response = completion(model="claude-instant-1", messages=messages) +# except AuthenticationError: # pass + +# # Restore stdout +# sys.stdout = old_stdout +# output = new_stdout.getvalue().strip() + +# print(output) + +# if "Logging Details Pre-API Call" not in output: +# raise Exception("Required log message not found!") +# elif "Logging Details Post-API Call" not in output: +# raise Exception("Required log message not found!") +# elif "Logging Details LiteLLM-Failure Call" not in output: +# raise Exception("Required log message not found!") +# score += 1 +# except Exception as e: +# print(f"exception type: {type(e).__name__}") +# pytest.fail(f"Error occurred: {e}") -# # test_logging_success_streaming_non_openai() # # embedding -# def test_logging_success_embedding_openai(): +# try: +# temporary_oai_key = os.environ["OPENAI_API_KEY"] +# os.environ["OPENAI_API_KEY"] = "bad-key" + +# temporary_anthropic_key = os.environ["ANTHROPIC_API_KEY"] +# os.environ["ANTHROPIC_API_KEY"] = "bad-key" +# # Redirect stdout +# old_stdout = sys.stdout +# sys.stdout = new_stdout = io.StringIO() + # try: -# # Redirect stdout -# old_stdout = sys.stdout -# sys.stdout = new_stdout = io.StringIO() - # response = embedding(model="text-embedding-ada-002", input=["good morning from litellm"]) +# except AuthenticationError: +# pass -# # Restore stdout -# sys.stdout = old_stdout -# output = new_stdout.getvalue().strip() +# # Restore stdout +# sys.stdout = old_stdout +# output = new_stdout.getvalue().strip() -# if "Logging Details Pre-API Call" not in output: -# raise Exception("Required log message not found!") -# elif "Logging Details Post-API Call" not in output: -# raise Exception("Required log message not found!") -# elif "Logging Details LiteLLM-Success Call" not in output: -# raise Exception("Required log message not found!") -# except Exception as e: -# pytest.fail(f"Error occurred: {e}") +# print(output) -# # ## 2. On LiteLLM Call failure -# # ## TEST BAD KEY - -# # # normal completion -# # ## test on openai completion call -# # try: -# # temporary_oai_key = os.environ["OPENAI_API_KEY"] -# # os.environ["OPENAI_API_KEY"] = "bad-key" - -# # temporary_anthropic_key = os.environ["ANTHROPIC_API_KEY"] -# # os.environ["ANTHROPIC_API_KEY"] = "bad-key" - - -# # # Redirect stdout -# # old_stdout = sys.stdout -# # sys.stdout = new_stdout = io.StringIO() - -# # try: -# # response = completion(model="gpt-3.5-turbo", messages=messages) -# # except AuthenticationError: -# # print(f"raised auth error") -# # pass -# # # Restore stdout -# # sys.stdout = old_stdout -# # output = new_stdout.getvalue().strip() - -# # print(output) - -# # if "Logging Details Pre-API Call" not in output: -# # raise Exception("Required log message not found!") -# # elif "Logging Details Post-API Call" not in output: -# # raise Exception("Required log message not found!") -# # elif "Logging Details LiteLLM-Failure Call" not in output: -# # raise Exception("Required log message not found!") - -# # os.environ["OPENAI_API_KEY"] = temporary_oai_key -# # os.environ["ANTHROPIC_API_KEY"] = temporary_anthropic_key - -# # score += 1 -# # except Exception as e: -# # print(f"exception type: {type(e).__name__}") -# # pytest.fail(f"Error occurred: {e}") -# # pass - -# # ## test on non-openai completion call -# # try: -# # temporary_oai_key = os.environ["OPENAI_API_KEY"] -# # os.environ["OPENAI_API_KEY"] = "bad-key" - -# # temporary_anthropic_key = os.environ["ANTHROPIC_API_KEY"] -# # os.environ["ANTHROPIC_API_KEY"] = "bad-key" -# # # Redirect stdout -# # old_stdout = sys.stdout -# # sys.stdout = new_stdout = io.StringIO() - -# # try: -# # response = completion(model="claude-instant-1", messages=messages) -# # except AuthenticationError: -# # pass - -# # if "Logging Details Pre-API Call" not in output: -# # raise Exception("Required log message not found!") -# # elif "Logging Details Post-API Call" not in output: -# # raise Exception("Required log message not found!") -# # elif "Logging Details LiteLLM-Failure Call" not in output: -# # raise Exception("Required log message not found!") -# # os.environ["OPENAI_API_KEY"] = temporary_oai_key -# # os.environ["ANTHROPIC_API_KEY"] = temporary_anthropic_key -# # score += 1 -# # except Exception as e: -# # print(f"exception type: {type(e).__name__}") -# # # Restore stdout -# # sys.stdout = old_stdout -# # output = new_stdout.getvalue().strip() - -# # print(output) -# # pytest.fail(f"Error occurred: {e}") - - -# # # streaming completion -# # ## test on openai completion call -# # try: -# # temporary_oai_key = os.environ["OPENAI_API_KEY"] -# # os.environ["OPENAI_API_KEY"] = "bad-key" - -# # temporary_anthropic_key = os.environ["ANTHROPIC_API_KEY"] -# # os.environ["ANTHROPIC_API_KEY"] = "bad-key" -# # # Redirect stdout -# # old_stdout = sys.stdout -# # sys.stdout = new_stdout = io.StringIO() - -# # try: -# # response = completion(model="gpt-3.5-turbo", messages=messages) -# # except AuthenticationError: -# # pass - -# # # Restore stdout -# # sys.stdout = old_stdout -# # output = new_stdout.getvalue().strip() - -# # print(output) - -# # if "Logging Details Pre-API Call" not in output: -# # raise Exception("Required log message not found!") -# # elif "Logging Details Post-API Call" not in output: -# # raise Exception("Required log message not found!") -# # elif "Logging Details LiteLLM-Failure Call" not in output: -# # raise Exception("Required log message not found!") - -# # os.environ["OPENAI_API_KEY"] = temporary_oai_key -# # os.environ["ANTHROPIC_API_KEY"] = temporary_anthropic_key -# # score += 1 -# # except Exception as e: -# # print(f"exception type: {type(e).__name__}") -# # pytest.fail(f"Error occurred: {e}") - -# # ## test on non-openai completion call -# # try: -# # temporary_oai_key = os.environ["OPENAI_API_KEY"] -# # os.environ["OPENAI_API_KEY"] = "bad-key" - -# # temporary_anthropic_key = os.environ["ANTHROPIC_API_KEY"] -# # os.environ["ANTHROPIC_API_KEY"] = "bad-key" -# # # Redirect stdout -# # old_stdout = sys.stdout -# # sys.stdout = new_stdout = io.StringIO() - -# # try: -# # response = completion(model="claude-instant-1", messages=messages) -# # except AuthenticationError: -# # pass - -# # # Restore stdout -# # sys.stdout = old_stdout -# # output = new_stdout.getvalue().strip() - -# # print(output) - -# # if "Logging Details Pre-API Call" not in output: -# # raise Exception("Required log message not found!") -# # elif "Logging Details Post-API Call" not in output: -# # raise Exception("Required log message not found!") -# # elif "Logging Details LiteLLM-Failure Call" not in output: -# # raise Exception("Required log message not found!") -# # score += 1 -# # except Exception as e: -# # print(f"exception type: {type(e).__name__}") -# # pytest.fail(f"Error occurred: {e}") - -# # # embedding - -# # try: -# # temporary_oai_key = os.environ["OPENAI_API_KEY"] -# # os.environ["OPENAI_API_KEY"] = "bad-key" - -# # temporary_anthropic_key = os.environ["ANTHROPIC_API_KEY"] -# # os.environ["ANTHROPIC_API_KEY"] = "bad-key" -# # # Redirect stdout -# # old_stdout = sys.stdout -# # sys.stdout = new_stdout = io.StringIO() - -# # try: -# # response = embedding(model="text-embedding-ada-002", input=["good morning from litellm"]) -# # except AuthenticationError: -# # pass - -# # # Restore stdout -# # sys.stdout = old_stdout -# # output = new_stdout.getvalue().strip() - -# # print(output) - -# # if "Logging Details Pre-API Call" not in output: -# # raise Exception("Required log message not found!") -# # elif "Logging Details Post-API Call" not in output: -# # raise Exception("Required log message not found!") -# # elif "Logging Details LiteLLM-Failure Call" not in output: -# # raise Exception("Required log message not found!") -# # except Exception as e: -# # print(f"exception type: {type(e).__name__}") -# # pytest.fail(f"Error occurred: {e}") \ No newline at end of file +# if "Logging Details Pre-API Call" not in output: +# raise Exception("Required log message not found!") +# elif "Logging Details Post-API Call" not in output: +# raise Exception("Required log message not found!") +# elif "Logging Details LiteLLM-Failure Call" not in output: +# raise Exception("Required log message not found!") +# except Exception as e: +# print(f"exception type: {type(e).__name__}") +# pytest.fail(f"Error occurred: {e}") \ No newline at end of file