adding helicone integration

litellm/integrations/helicone.py

@@ -0,0 +1,49 @@
+#### What this does ####
+#    On success, logs events to Helicone
+import dotenv, os
+import requests
+dotenv.load_dotenv() # Loading env variables using dotenv
+import traceback
+class HeliconeLogger:
+    # Class variables or attributes
+    helicone_model_list = ["gpt", "claude"]
+    def __init__(self):
+        # Instance variables
+        self.provider_url = "https://api.openai.com/v1"
+        self.key = os.getenv('HELICONE_API_KEY')
+        
+    def log_success(self, model, messages, response_obj, start_time, end_time):
+        # Method definition
+        try:
+            model = model if any(accepted_model in model for accepted_model in self.helicone_model_list) else "gpt-3.5-turbo"
+            provider_request = {"model": model, "messages": messages}
+
+            providerResponse = {
+                "json": response_obj, 
+                "headers": {"openai-version": "2020-10-01"}, 
+                "status": 200
+            }
+
+            # Code to be executed
+            url = "https://api.hconeai.com/oai/v1/log"
+            headers = {
+                'Authorization': f'Bearer {self.key}',
+                'Content-Type': 'application/json'
+            }
+            start_time_seconds = int(start_time.timestamp())
+            start_time_milliseconds = int((start_time.timestamp() - start_time_seconds) * 1000)
+            end_time_seconds = int(end_time.timestamp())
+            end_time_milliseconds = int((end_time.timestamp() - end_time_seconds) * 1000)
+            data = {
+                "providerRequest": {"url": self.provider_url, "json": provider_request, "meta": {"Helicone-Auth": f"Bearer {self.key}"}},
+                "providerResponse": providerResponse,
+                "timing": {"startTime": {"seconds": start_time_seconds, "milliseconds": start_time_milliseconds}, "endTime": {"seconds": end_time_seconds, "milliseconds": end_time_milliseconds}} # {"seconds": .., "milliseconds": ..}
+            }
+            response = requests.post(url, headers=headers, json=data)
+            # if response.status_code == 200:
+            #     print("Success!")
+            # else:
+            #     print("Request was not successful. Status Code:", response.status_code)
+        except:
+            # traceback.print_exc()
+            pass

litellm/tests/test_client.py

@@ -9,10 +9,10 @@ sys.path.insert(0, os.path.abspath('../..'))  # Adds the parent directory to the
 import litellm
 from litellm import embedding, completion
 
-litellm.success_callback = ["posthog"]
+litellm.success_callback = ["posthog", "helicone"]
 litellm.failure_callback = ["slack", "sentry", "posthog"]
 
-# litellm.set_verbose = True
+litellm.set_verbose = True
 
 def logger_fn(model_call_object: dict):
     # print(f"model call details: {model_call_object}")
@@ -23,11 +23,14 @@ messages = [{ "content": user_message,"role": "user"}]
 
 def test_completion_openai():
     try:
+        print("running query")
         response = completion(model="gpt-3.5-turbo", messages=messages, logger_fn=logger_fn)
+        print(f"response: {response}")
         # Add any assertions here to check the response
     except Exception as e:
+        traceback.print_exc()
         pytest.fail(f"Error occurred: {e}")
-
+test_completion_openai()
 def test_completion_non_openai():
     try:
         response = completion(model="claude-instant-1", messages=messages, logger_fn=logger_fn)

litellm/utils.py

@@ -2,6 +2,7 @@ import dotenv, json, traceback, threading
 import subprocess, os 
 import litellm, openai 
 import random, uuid, requests
+import datetime
 from openai.error import AuthenticationError, InvalidRequestError, RateLimitError, ServiceUnavailableError, OpenAIError
 ####### ENVIRONMENT VARIABLES ###################
 dotenv.load_dotenv() # Loading env variables using dotenv
@@ -11,6 +12,7 @@ add_breadcrumb = None
 posthog = None
 slack_app = None
 alerts_channel = None
+heliconeLogger = None
 callback_list = []
 user_logger_fn = None
 additional_details = {}
@@ -68,7 +70,7 @@ def client(original_function):
         global callback_list, add_breadcrumb
         if (len(litellm.success_callback) > 0 or len(litellm.failure_callback) > 0) and len(callback_list) == 0: 
           callback_list = list(set(litellm.success_callback + litellm.failure_callback))
-          set_callbacks(callback_list=callback_list)
+          set_callbacks(callback_list=callback_list,)
         if add_breadcrumb:
           add_breadcrumb(
                 category="litellm.llm_call",
@@ -83,9 +85,11 @@ def client(original_function):
         try:
           function_setup(args, kwargs)
           ## MODEL CALL
+          start_time = datetime.datetime.now()
           result = original_function(*args, **kwargs)
+          end_time = datetime.datetime.now()
           ## LOG SUCCESS 
-          my_thread = threading.Thread(target=handle_success, args=(args, kwargs)) # don't interrupt execution of main thread
+          my_thread = threading.Thread(target=handle_success, args=(args, kwargs, result, start_time, end_time)) # don't interrupt execution of main thread
           my_thread.start()
           return result
         except Exception as e:
@@ -97,7 +101,7 @@ def client(original_function):
 
 ####### HELPER FUNCTIONS ################
 def set_callbacks(callback_list):
-  global sentry_sdk_instance, capture_exception, add_breadcrumb, posthog, slack_app, alerts_channel
+  global sentry_sdk_instance, capture_exception, add_breadcrumb, posthog, slack_app, alerts_channel, heliconeLogger
   try:
     for callback in callback_list:
       if callback == "sentry":
@@ -134,6 +138,10 @@ def set_callbacks(callback_list):
         )
         alerts_channel = os.environ["SLACK_API_CHANNEL"]
         print_verbose(f"Initialized Slack App: {slack_app}")
+      elif callback == "helicone":
+        from .integrations.helicone import HeliconeLogger
+
+        heliconeLogger = HeliconeLogger()
   except:
     pass
 
@@ -200,7 +208,8 @@ def handle_failure(exception, traceback_exception, args, kwargs):
     except:
       pass
 
-def handle_success(*args, **kwargs):
+def handle_success(args, kwargs, result, start_time, end_time):
+  global heliconeLogger
   try:
     success_handler = additional_details.pop("success_handler", None)
     failure_handler = additional_details.pop("failure_handler", None)
@@ -223,6 +232,11 @@ def handle_success(*args, **kwargs):
           for detail in additional_details: 
             slack_msg += f"{detail}: {additional_details[detail]}\n"
           slack_app.client.chat_postMessage(channel=alerts_channel, text=slack_msg)
+        elif callback == "helicone":
+          print_verbose("reaches helicone for logging!")
+          model = args[0] if len(args) > 0 else kwargs["model"]
+          messages = args[1] if len(args) > 1 else kwargs["messages"]
+          heliconeLogger.log_success(model=model, messages=messages, response_obj=result, start_time=start_time, end_time=end_time)
       except:
         pass
 

setup.py

@@ -2,7 +2,7 @@ from setuptools import setup, find_packages
 
 setup(
     name='litellm',
-    version='0.1.220',
+    version='0.1.221',
     description='Library to easily interface with LLM API providers',
     author='BerriAI',
     packages=[