fix error tracking

docs/my-website/docs/observability/llmonitor_integration.md

@@ -4,15 +4,16 @@
 
 ## Use LLMonitor to log requests across all LLM Providers (OpenAI, Azure, Anthropic, Cohere, Replicate, PaLM)
 
-liteLLM provides `success_callbacks` and `failure_callbacks`, making it easy for you to send data to a particular provider depending on the status of your responses.
+liteLLM provides `callbacks`, making it easy for you to log data depending on the status of your responses.
 
 ### Using Callbacks
 
 Use just 2 lines of code, to instantly log your responses **across all providers** with llmonitor:
 
 ```
-litellm.success_callback=["llmonitor"]
+litellm.success_callback = ["llmonitor"]
-litellm.error_callback=["llmonitor"]
+litellm.failure_callback = ["llmonitor"]
+
 ```
 
 Complete code
@@ -27,8 +28,8 @@ os.environ["LLMONITOR_APP_ID"] = "your-llmonitor-app-id"
 os.environ["OPENAI_API_KEY"], os.environ["COHERE_API_KEY"] = "", ""
 
 # set callbacks
-litellm.success_callback=["llmonitor"]
+litellm.success_callback = ["llmonitor"]
-litellm.error_callback=["llmonitor"]
+litellm.failure_callback = ["llmonitor"]
 
 #openai call
 response = completion(model="gpt-3.5-turbo", messages=[{"role": "user", "content": "Hi 👋 - i'm openai"}])

litellm/integrations/llmonitor.py

@@ -57,14 +57,14 @@ class LLMonitorLogger:
             messages=None,
             user_id=None,
             response_obj=None,
-            time=datetime.datetime.now(),
+            start_time=datetime.datetime.now(),
+            end_time=datetime.datetime.now(),
             error=None,
     ):
         # Method definition
         try:
             print_verbose(
-                f"LLMonitor Logging - Enters logging function for model {model}"
+                f"LLMonitor Logging - Logging request for model {model}")
-            )
 
             if response_obj:
                 usage = parse_usage(response_obj['usage'])
@@ -73,27 +73,41 @@ class LLMonitorLogger:
                 usage = None
                 output = None
 
-            print(type, run_id, model, messages, usage, output, time, user_id,
+            if error:
-                  error)
+                error_obj = {'stack': error}
 
-            headers = {'Content-Type': 'application/json'}
+            else:
+                error_obj = None
 
-            data = {
+            data = [{
                 "type": "llm",
                 "name": model,
                 "runId": run_id,
                 "app": self.app_id,
-                "error": error,
+                'event': 'start',
-                "event": type,
+                "timestamp": start_time.isoformat(),
-                "timestamp": time.isoformat(),
                 "userId": user_id,
                 "input": parse_messages(messages),
-                "usage": usage,
+            }, {
+                "type": "llm",
+                "runId": run_id,
+                "app": self.app_id,
+                "event": type,
+                "error": error_obj,
+                "timestamp": end_time.isoformat(),
+                "userId": user_id,
                 "output": parse_messages(output),
-            }
+                "tokensUsage": usage,
+            }]
 
-            print_verbose(f"LLMonitor Logging - final data object: {data}")
+            # print_verbose(f"LLMonitor Logging - final data object: {data}")
-            # response = requests.post(url, headers=headers, json=data)
+
+            response = requests.post(
+                self.api_url + '/api/report',
+                headers={'Content-Type': 'application/json'},
+                json={'events': data})
+
+            print_verbose(f"LLMonitor Logging - response: {response}")
         except:
             # traceback.print_exc()
             print_verbose(

litellm/tests/test_llmonitor_integration.py

@@ -9,37 +9,16 @@ sys.path.insert(0, os.path.abspath('../..'))
 from litellm import completion
 import litellm
 
-litellm.input_callback = ["llmonitor"]
 litellm.success_callback = ["llmonitor"]
-litellm.error_callback = ["llmonitor"]
+litellm.failure_callback = ["llmonitor"]
 
 litellm.set_verbose = True
 
-os.environ[
+# openai call
-    "OPENAI_API_KEY"] = "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"
+first_success_test = completion(model="gpt-3.5-turbo",
-
-print(os.environ["OPENAI_API_KEY"])
-
-# def my_custom_logging_fn(model_call_dict):
-#     print(f"model call details: {model_call_dict}")
-
-# # openai call
-# response = completion(model="gpt-3.5-turbo",
-#                       messages=[{
-#                           "role": "user",
-#                           "content": "Hi 👋 - i'm openai"
-#                       }],
-#                       logger_fn=my_custom_logging_fn)
-
-# print(response)
-
-# #bad request call
-# response = completion(model="chatgpt-test", messages=[{"role": "user", "content": "Hi 👋 - i'm a bad request"}])
-
-# cohere call
-response = completion(model="command-nightly",
                                 messages=[{
                                     "role": "user",
-                          "content": "Hi 👋 - i'm cohere"
+                                    "content": "Hi 👋 - i'm openai"
                                 }])
-print(response)
+
+print(first_success_test)

litellm/utils.py

@@ -209,19 +209,7 @@ class Logging:
                             litellm_params["litellm_call_id"],
                             print_verbose=print_verbose,
                         )
-                    elif callback == "llmonitor":
+
-                        print_verbose("reaches llmonitor for logging!")
-                        model = self.model
-                        messages = self.messages
-                        print(f"liteDebuggerClient: {liteDebuggerClient}")
-                        llmonitorLogger.log_event(
-                            type="start",
-                            model=model,
-                            messages=messages,
-                            user_id=litellm._thread_context.user,
-                            run_id=self.litellm_params["litellm_call_id"],
-                            print_verbose=print_verbose,
-                        )
                     elif callback == "lite_debugger":
                         print_verbose("reaches litedebugger for logging!")
                         model = self.model
@@ -426,6 +414,7 @@ def client(original_function):
                 add_cache(result, *args, **kwargs)
             # LOG SUCCESS
             crash_reporting(*args, **kwargs)
+
             my_thread = threading.Thread(
                 target=handle_success,
                 args=(args, kwargs, result, start_time,
@@ -433,6 +422,7 @@ def client(original_function):
             my_thread.start()
             return result
         except Exception as e:
+
             traceback_exception = traceback.format_exc()
             crash_reporting(*args, **kwargs, exception=traceback_exception)
             end_time = datetime.datetime.now()
@@ -855,21 +845,15 @@ def handle_failure(exception, traceback_exception, start_time, end_time, args,
                     print_verbose("reaches llmonitor for logging!")
                     model = args[0] if len(args) > 0 else kwargs["model"]
                     messages = args[1] if len(args) > 1 else kwargs["messages"]
-                    usage = {
+
-                        "prompt_tokens":
-                        prompt_token_calculator(model, messages=messages),
-                        "completion_tokens":
-                        0,
-                    }
                     llmonitorLogger.log_event(
                         type="error",
                         user_id=litellm._thread_context.user,
                         model=model,
                         error=traceback_exception,
-                        response_obj=result,
                         run_id=kwargs["litellm_call_id"],
-                        timestamp=end_time,
+                        start_time=start_time,
-                        usage=usage,
+                        end_time=end_time,
                         print_verbose=print_verbose,
                     )
                 elif callback == "supabase":
@@ -992,7 +976,8 @@ def handle_success(args, kwargs, result, start_time, end_time):
                         messages=messages,
                         user_id=litellm._thread_context.user,
                         response_obj=result,
-                        time=end_time,
+                        start_time=start_time,
+                        end_time=end_time,
                         run_id=kwargs["litellm_call_id"],
                         print_verbose=print_verbose,
                     )