From fc10cf5eebe962c81ef7f2c0dd13a570ec396674 Mon Sep 17 00:00:00 2001 From: ishaan-jaff Date: Tue, 19 Sep 2023 21:29:51 -0700 Subject: [PATCH 1/5] exception_type work --- litellm/__init__.py | 1 + litellm/main.py | 2 +- litellm/tests/test_completion.py | 34 +++++++++++++++---------------- litellm/utils.py | 35 +++++++++++++++++++++++++++++++- 4 files changed, 53 insertions(+), 19 deletions(-) diff --git a/litellm/__init__.py b/litellm/__init__.py index cc6f88c0c..9b8082d39 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -38,6 +38,7 @@ cache: Optional[Cache] = None # cache object model_alias_map: Dict[str, str] = {} max_budget: float = 0.0 # set the max budget across all providers _current_cost = 0 # private variable, used if max budget is set +error_logs: Dict = {} ############################################# def get_model_cost_map(): diff --git a/litellm/main.py b/litellm/main.py index b9acd6e4a..78a0a8af0 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -1046,7 +1046,7 @@ def completion( except Exception as e: ## Map to OpenAI Exception raise exception_type( - model=model, custom_llm_provider=custom_llm_provider, original_exception=e + model=model, custom_llm_provider=custom_llm_provider, original_exception=e, completion_kwargs=args, ) diff --git a/litellm/tests/test_completion.py b/litellm/tests/test_completion.py index 3bfd3956d..0a01affa7 100644 --- a/litellm/tests/test_completion.py +++ b/litellm/tests/test_completion.py @@ -134,24 +134,24 @@ def test_completion_with_litellm_call_id(): # pytest.fail(f"Error occurred: {e}") # using Non TGI or conversational LLMs -# def hf_test_completion(): -# try: -# # litellm.set_verbose=True -# user_message = "My name is Merve and my favorite" -# messages = [{ "content": user_message,"role": "user"}] -# response = completion( -# model="huggingface/roneneldan/TinyStories-3M", -# messages=messages, -# api_base="https://p69xlsj6rpno5drq.us-east-1.aws.endpoints.huggingface.cloud", -# task=None, -# ) -# # Add any assertions here to check the response -# print(response) +def hf_test_completion(): + try: + # litellm.set_verbose=True + user_message = "My name is Merve and my favorite" + messages = [{ "content": user_message,"role": "user"}] + response = completion( + model="huggingface/roneneldan/TinyStories-3M", + messages=messages, + api_base="https://p69xlsj6rpno5drq.us-east-1.aws.endpoints.huggingface.cloud", -# except Exception as e: -# pytest.fail(f"Error occurred: {e}") + ) + # Add any assertions here to check the response + print(response) -# hf_test_completion() + except Exception as e: + pytest.fail(f"Error occurred: {e}") + +hf_test_completion() def test_completion_cohere(): # commenting for now as the cohere endpoint is being flaky try: @@ -352,7 +352,7 @@ def test_completion_azure(): try: print("azure gpt-3.5 test\n\n") response = completion( - model="azure/chatgpt-v-2", + model="chatgpt-v-2", messages=messages, ) # Add any assertions here to check the response diff --git a/litellm/utils.py b/litellm/utils.py index ec525f536..e1257a8c5 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -198,6 +198,7 @@ class Logging: def pre_call(self, input, api_key, model=None, additional_args={}): # Log the exact input to the LLM API print_verbose(f"Logging Details Pre-API Call for call id {self.litellm_call_id}") + litellm.error_logs['PRE_CALL'] = locals() try: # print_verbose(f"logging pre call for model: {self.model} with call type: {self.call_type}") self.model_call_details["input"] = input @@ -280,6 +281,7 @@ class Logging: def post_call(self, original_response, input=None, api_key=None, additional_args={}): # Log the exact result from the LLM API, for streaming - log the type of response received + litellm.error_logs['POST_CALL'] = locals() try: self.model_call_details["input"] = input self.model_call_details["api_key"] = api_key @@ -1870,9 +1872,40 @@ def get_model_list(): ) ####### EXCEPTION MAPPING ################ -def exception_type(model, original_exception, custom_llm_provider): +def exception_type( + model, + original_exception, + custom_llm_provider, + completion_kwargs={}, + ): global user_logger_fn, liteDebuggerClient exception_mapping_worked = False + + litellm.error_logs['EXCEPTION'] = original_exception + litellm.error_logs['KWARGS'] = completion_kwargs + + import urllib.parse + import json + for log_key in litellm.error_logs: + current_logs = litellm.error_logs[log_key] + if type(current_logs) == dict: + filtered_error_logs = {key: value for key, value in current_logs.items() if isinstance(value, (str, int, float, bool, list, dict))} + litellm.error_logs[log_key] = filtered_error_logs + else: + litellm.error_logs[log_key] = str(current_logs) + + # Convert the filtered_error_logs dictionary to a JSON string + error_logs_json = json.dumps(litellm.error_logs) + # URL-encode the JSON data + encoded_data = urllib.parse.quote(error_logs_json) + print(encoded_data) + # Print the encoded data (this is what you can include in a URL) + print("\033[91m" + str(litellm.error_logs) + "\033[0m") + + decoded_data = urllib.parse.unquote(encoded_data) + + # Print the decoded data + print(decoded_data) try: if isinstance(original_exception, OriginalError): # Handle the OpenAIError From 2ba8e9bc0223efcaefa410a868ad584401a58722 Mon Sep 17 00:00:00 2001 From: ishaan-jaff Date: Wed, 20 Sep 2023 20:09:02 -0700 Subject: [PATCH 2/5] only show error log dashboard on fail --- litellm/utils.py | 44 ++++++++++++++++++++++---------------------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/litellm/utils.py b/litellm/utils.py index e1257a8c5..998c3c18c 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -1881,31 +1881,31 @@ def exception_type( global user_logger_fn, liteDebuggerClient exception_mapping_worked = False - litellm.error_logs['EXCEPTION'] = original_exception - litellm.error_logs['KWARGS'] = completion_kwargs + if litellm.set_verbose == True: + litellm.error_logs['EXCEPTION'] = original_exception + litellm.error_logs['KWARGS'] = completion_kwargs + try: + # code to show users their litellm error dashboard + import urllib.parse + import json + for log_key in litellm.error_logs: + current_logs = litellm.error_logs[log_key] + if type(current_logs) == dict: + filtered_error_logs = {key: str(value) for key, value in current_logs.items()} + litellm.error_logs[log_key] = filtered_error_logs + else: + litellm.error_logs[log_key] = str(current_logs) - import urllib.parse - import json - for log_key in litellm.error_logs: - current_logs = litellm.error_logs[log_key] - if type(current_logs) == dict: - filtered_error_logs = {key: value for key, value in current_logs.items() if isinstance(value, (str, int, float, bool, list, dict))} - litellm.error_logs[log_key] = filtered_error_logs - else: - litellm.error_logs[log_key] = str(current_logs) + # Convert the filtered_error_logs dictionary to a JSON string + error_logs_json = json.dumps(litellm.error_logs) + # URL-encode the JSON data + encoded_data = urllib.parse.quote(error_logs_json) - # Convert the filtered_error_logs dictionary to a JSON string - error_logs_json = json.dumps(litellm.error_logs) - # URL-encode the JSON data - encoded_data = urllib.parse.quote(error_logs_json) - print(encoded_data) - # Print the encoded data (this is what you can include in a URL) - print("\033[91m" + str(litellm.error_logs) + "\033[0m") + print("👉 view error logs:") + print("\033[91m" + '\033[4m' + 'https://logs.litellm.ai/?data=' + str(encoded_data) + "\033[0m") - decoded_data = urllib.parse.unquote(encoded_data) - - # Print the decoded data - print(decoded_data) + except: + pass try: if isinstance(original_exception, OriginalError): # Handle the OpenAIError From 7b386ee4d87781055820904770cd4643e129e5a9 Mon Sep 17 00:00:00 2001 From: ishaan-jaff Date: Wed, 20 Sep 2023 20:11:32 -0700 Subject: [PATCH 3/5] fix test_completion --- litellm/tests/test_completion.py | 46 +++++++++++++++++++++----------- 1 file changed, 31 insertions(+), 15 deletions(-) diff --git a/litellm/tests/test_completion.py b/litellm/tests/test_completion.py index 0a01affa7..ba013223d 100644 --- a/litellm/tests/test_completion.py +++ b/litellm/tests/test_completion.py @@ -134,24 +134,40 @@ def test_completion_with_litellm_call_id(): # pytest.fail(f"Error occurred: {e}") # using Non TGI or conversational LLMs -def hf_test_completion(): - try: - # litellm.set_verbose=True - user_message = "My name is Merve and my favorite" - messages = [{ "content": user_message,"role": "user"}] - response = completion( - model="huggingface/roneneldan/TinyStories-3M", - messages=messages, - api_base="https://p69xlsj6rpno5drq.us-east-1.aws.endpoints.huggingface.cloud", +# def hf_test_completion(): +# try: +# # litellm.set_verbose=True +# user_message = "My name is Merve and my favorite" +# messages = [{ "content": user_message,"role": "user"}] +# response = completion( +# model="huggingface/roneneldan/TinyStories-3M", +# messages=messages, +# api_base="https://p69xlsj6rpno5drq.us-east-1.aws.endpoints.huggingface.cloud", +# task=None, +# ) +# # Add any assertions here to check the response +# print(response) - ) - # Add any assertions here to check the response - print(response) - except Exception as e: - pytest.fail(f"Error occurred: {e}") +# this should throw an exception, to trigger https://logs.litellm.ai/ +# def hf_test_error_logs(): +# try: +# litellm.set_verbose=True +# user_message = "My name is Merve and my favorite" +# messages = [{ "content": user_message,"role": "user"}] +# response = completion( +# model="huggingface/roneneldan/TinyStories-3M", +# messages=messages, +# api_base="https://p69xlsj6rpno5drq.us-east-1.aws.endpoints.huggingface.cloud", -hf_test_completion() +# ) +# # Add any assertions here to check the response +# print(response) + +# except Exception as e: +# pytest.fail(f"Error occurred: {e}") + +# hf_test_error_logs() def test_completion_cohere(): # commenting for now as the cohere endpoint is being flaky try: From c844d26ff1f8c6b40a04ce05f45f724e3ced39eb Mon Sep 17 00:00:00 2001 From: ishaan-jaff Date: Wed, 20 Sep 2023 20:12:26 -0700 Subject: [PATCH 4/5] fix azure --- litellm/tests/test_completion.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/litellm/tests/test_completion.py b/litellm/tests/test_completion.py index ba013223d..d7cda7378 100644 --- a/litellm/tests/test_completion.py +++ b/litellm/tests/test_completion.py @@ -434,7 +434,7 @@ def test_completion_azure_with_litellm_key(): def test_completion_azure_deployment_id(): try: response = completion( - deployment_id="chatgpt-v-2", + deployment_id="azure/chatgpt-v-2", model="gpt-3.5-turbo", messages=messages, ) From 20e86e6a05232380ac87a01d6330a28d3da565a8 Mon Sep 17 00:00:00 2001 From: ishaan-jaff Date: Wed, 20 Sep 2023 20:14:18 -0700 Subject: [PATCH 5/5] fix test completion --- litellm/tests/test_completion.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/litellm/tests/test_completion.py b/litellm/tests/test_completion.py index d7cda7378..c1433d325 100644 --- a/litellm/tests/test_completion.py +++ b/litellm/tests/test_completion.py @@ -147,6 +147,10 @@ def test_completion_with_litellm_call_id(): # ) # # Add any assertions here to check the response # print(response) +# except Exception as e: +# pytest.fail(f"Error occurred: {e}") + +# hf_test_completion() # this should throw an exception, to trigger https://logs.litellm.ai/ @@ -368,7 +372,7 @@ def test_completion_azure(): try: print("azure gpt-3.5 test\n\n") response = completion( - model="chatgpt-v-2", + model="azure/chatgpt-v-2", messages=messages, ) # Add any assertions here to check the response @@ -434,7 +438,7 @@ def test_completion_azure_with_litellm_key(): def test_completion_azure_deployment_id(): try: response = completion( - deployment_id="azure/chatgpt-v-2", + deployment_id="chatgpt-v-2", model="gpt-3.5-turbo", messages=messages, )