From aa598d506f38ae4df7011f43a1107b985d57c6c8 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Wed, 2 Aug 2023 22:25:08 -0700 Subject: [PATCH] adding helicone integration --- litellm/__pycache__/__init__.cpython-311.pyc | Bin 932 -> 956 bytes litellm/__pycache__/main.cpython-311.pyc | Bin 11596 -> 11621 bytes litellm/__pycache__/utils.cpython-311.pyc | Bin 17317 -> 18290 bytes .../__pycache__/helicone.cpython-311.pyc | Bin 0 -> 3384 bytes litellm/integrations/helicone.py | 49 ++++++++++++++++++ litellm/tests/test_client.py | 9 ++-- litellm/utils.py | 22 ++++++-- setup.py | 2 +- 8 files changed, 74 insertions(+), 8 deletions(-) create mode 100644 litellm/integrations/__pycache__/helicone.cpython-311.pyc create mode 100644 litellm/integrations/helicone.py diff --git a/litellm/__pycache__/__init__.cpython-311.pyc b/litellm/__pycache__/__init__.cpython-311.pyc index 19b37aa2f7de3b2dee19b2a9772c500a6445d365..e552de8a2935b63d29e218e34cec4773f7d70460 100644 GIT binary patch delta 252 zcmZ3&zK5N6IWI340}y-?Kb>-NBClk96Oc2VA)O%#gc(y9qL@+`qnJ~eqF7Rxqga8M zEuA5SWf5Z(dkSkhV-!aU8<6BoVNc;mXNuxVWl!a(0-BJ@2`0HfWE6J_Cr}Mf3Renu z4sR}R6fYwKkj<0Bm&+f;pDPe0kSiD^2o~eb5fBPtNMR0U(ByLqdC9=Yz%X&m6-MF7 zjf}S3T)BxA@g@1$sd>ed_cG2FX91dhi?y^QGpD$S7051P0}t0aU}C!kNOA!;{Ms z#ly$|WOL{6=JG}H%n1Vb27*qMSEG##!Rb zK%;K4mX>7Z6c@1o*+r}%f^BjOQv{bJkjV(d#iu7fW0GV2z{bEMFqwyW4U0HX5CEoC BFlhh) diff --git a/litellm/__pycache__/main.cpython-311.pyc b/litellm/__pycache__/main.cpython-311.pyc index 7b8de5bd0ad442322e44255e306d9df97cfb7019..fb94555c74afae389f4c7b21a6d9152c751b539a 100644 GIT binary patch delta 531 zcmX>T^)!liIWI340}xyhKb^8;Bd+FwlN)$dCO2@a zvD7FmP?*ffEs@v$>)u F4*)=?blv~} delta 555 zcmaDFbtZ~;IWI340}v#LoK9J}k=LD#F=TTn+XqHg!y1Mf#>wxv#U>|k@H0M|T)?3y z3KmP@oWoQjyo9lju~uXPOWyI#%Q$?ImGg1&Gjby-=R;D?ocCdK2-i_YM&8N$xD6)H z6BJ^Zz?AoDa{`Yjlk)u#_V>2*h$xUVvG-lMD+`y|cxq(}a zrAA?a{A5NYk;w)EEI==*Gb&6zCg{rOJefyGTH6(&F-5ONaW=zTCRD?l7;8j=8I&1H z+$IMJ>3Zld0|vorcrY+Blz1Xc07i+@8iQp_Kovj?0Zoi447EyiO682@3>D0g4CRcP zMt+(+lOIahY-Sc#V`k3OVce`PX2-~=Iyp`J{pKi%sf>(vn?FdVurOYj94dF3`GLIB lW;J30RT@jeMA5N diff --git a/litellm/__pycache__/utils.cpython-311.pyc b/litellm/__pycache__/utils.cpython-311.pyc index 61ebdb12983b63afe91019a4352499e0c5716c5b..76bf2d548a06d35fbbfd4bf9564ed1b819af0526 100644 GIT binary patch delta 3141 zcmaKuYj9J?6@Yi|m1ODaEm;rS*p`v}kdZO8F#!jRdXKmy3kOP&NwY~=|Ecsb=vetXLzKLOdlh=>fq7TZT{p(ravl3e~@IR(>+(V zY%)#O+V7s-J$ugHyJye(r{BX5&tlWhj7AR8^1-G*@;lqlncNJA&ozxS;;25P=Mm3D z4KZWL7&C=TF>}ZqvxF=$YsiWbg9LQYMp~`zDISOIw3gHApw&sMi&i(S6%#02$>UH} z1clb{>?stg=2a2IV;+yIc=ah9s^K-Xx0WxX^zd5BI$lS)mggwFyph*aEgx^7t$H5s zMSjyKu;4+zI&TjimJSG+E%+mh%QKa1vj(rp^y+S?aSwT9dX;ONy}_aZNn3@ej;NZggu zkV`GTo>bXa__D32%Nn^1{Ee%=-V-NlzJqy`(!8c2d#u$KF0bXok|0H5LY|E$j*x_P zx1SR`sbw)pk>`?9Sd>N-1&t^q4@afER*3UT51F@a)j&*nJj7;eH9tee1lj^=qs z^PD4)a|EWFK6cjToDIK--|YFIAv^SZHpFMgH^$(XM^ zqy(U7;j8#HQAf;~Qp4$z)w6W=)snriE6to9;t|9IM#BiCSJ`271pBKqJKTT6^?B_9 zAsQJ=#Dz~_A%2`U?O5vAnV6UmM88{nfz(w_TSn=y7zP{wOpxDI)=cFck+>vGh+!#` zh$q)C+Y%?KO5U{GT(YDQpMY-dSU4JeDLi&ad;tt8CMQ6GNl6WbM=8pe6+2)A04hmL zzYdKEm7moul-Ct>s6pNQ0FY^^`U$gjPP=tMZ%-ea*VkqBbqm@p*~fn$RWx3#nXB;U zD*V%?&kDORomqhCj|zk2pViJOH#)1bGvCJpZbX78=v3dbGgQ9iz#u!j%4p+Gji$Sv zx#I(+JM}@Pr&;}eBWS(v#~?QcZM5-+ZJIt6bC(6JyQ(15uT$UCfYv=N23gl-q>T^V z%s@3eMt55&8L4KX43Cq)*KgN6PkY2sQrGZje1`nHp{eB!x}|86L_jGjrWVGG8&|1yk;pL--@7=c6FlbLHlhf-Rd4$ zS3zwTi>#G(#L!ZUITC2u#d5Ni$Sr%Yp8Tc7`?OxB?OD2rtFREIU=;t5debg>8{l;* z!)n{qn>(eG4YKZS#B-}hsGrXD4%TqGg6wt%rX=^WNt#^^*9>3I>G5OKjd971Bvq|w z^QKu$mqcr^(EMM$EJLP#e3UmIQubD)HwSMnz4!!*m(f#fDRrQf@oUXkSDCS{%u=k_ zR@y0(RI{>}84IcNIi5AjCVF+Svg(?BmTvLY(l%q1#;2pj%s{1=i;1a&!KIiYhSrT!-=%ZJQNU@yIN%iEb-){dX+WIfShXmG#|{Wd&vK6N zj3-1-l%||We4?XpkBqk{G+E_8plYVuv_g&~U&|74ZrX67<@zOzYY8R+KP2snR zXFxs+fRrk>14aP^kOrIwTmX~~mUs@D%2_glv;~l&$m{9KMv{le#^^f4b+pHCUUjer zL2t$X>VpO&Zv!p?-U0jw@B@JIg77Lya97NFB^J})0O$e!3TmzNyFmI|%jy=LP4}IF zd1oNo(s5HY=jnAR2;7m03oIF;r0E*KpbE9Z^gtkJv3xw7sj=eW%| zZgZB~yjbm-DgWW{JHuD3m-b)WpZ29+KHr#byso)^}Wsq?V!k8qY+7nMq?wCLQIgvqvE&7 zQ2RHqFY|i)7mP;P`wV$wTUfsm25Ite+Yru>-j0sCJ=97Pa^UiI6nEnWcmM=4mpg`; zZ4G%1pOA$3E8=fqBmhuWA%fHZP~xv1Bm+Qk_j{0fNn3F9)GHt!1#Ab50+e`Eq7Y)Q zcxjprCk1f?&If&r$BO~31nEZ5c@Cglm5|SsfKk4VxE7kqvY!RX0)YQmKG$xICHTXx zsL&z)iMAo1C4WO%U@^uE1a}2UbJuX0`GoREPIkRW7PkFs^8T(+ogh4Xe-TObFyvud~ delta 2116 zcmZ{lYfMx}6o6;$Yu~W!F3bDAtv0dU_FsQM5$!X?3Y9Ce)@)&$-A#O`I?v=gc{G&di)S zvky+v%fmEg!eTKJXjz|qB@a9|7L&;ROU4R!XK-DnZgargZ3$SqV*)YVv4PlbYrskg zM_S3=IQFxY*e26v8+1E#2lRO8PUs2kq&ZQhfus%+NS65_5=fDS4kA;Twh6L0L<6a^ z1fyxP4wx?Mff=#^m?;~9S+YeofmgO{1}!ICMjugX3+MAi5={;_7(NzhEfZn~j1_l` z6?4XlSz`rzEB4G_9VO%pca9ifjmIRFsPqa)$TjGT4cC55r;-k^T%spwi0gwdmfGH^ z+$SuJ5rVU{ao3-kDa5P%IKS+rN$iHzUPy_`y-0FMpMKezOO&l4q0h8DvR7iw)`SA_ z8S!J5Jyk&!*;rhHs6}KKtl4G=AsnSL2??(Y?1#8Cs%LuJcDr7A9zIjtfO$!24fc2S zu_Lx>0T;riZKbh(5Q0;Hg~KGmnHjY^ihr`_&f0Uw)iwb9jpYC0%$RfLf7mnW z`67R^X4dJyNaGkW*aKy6|PkZycQrk0lIIHl803S$`$zJ1V&1 zm7VORlo9JL4Ag{>5iRVGl=PQfqEZ3cyuPKmt7}hlOQ*6k>ed2MPy}RRQ}BS5nk8t) zIk+ZpEYUQPUPe zP6~GJGzEdViQ6eo**TD>;wj3`S{*3U>C&b=ZaNp8rt{pKr%1e6fL=FU6y+kf4V0M; zl271n@#u9+aC0e}4dQJH-EQkC$_95OC|?q8ix~bVW1o)hVZ8+#r5(7XUF?&BZ|M*_ zT39Fralp@R6guPjP*ec(LTBrNAh>;BSe9Imqt7Go;wbwO1BiZ>0vY)D)lG2xQIy+fX%p5!gtfSmzu+%>}S9ZRKip z00i+TK`Rvvi}kV@t%$GD;}SfV_>QFU23Ef|fo)q`WwDEv2my@fBb<$$UHibaun^&> z&T?089%1Q~Tk;CL)nuw@(<-l%n~A5O%s4r3vN)5NjdC^TqkId;#;GGbURlS}L*aYt zpxB04Sydr@mo--TjTdnhh~LAXR+aK5?HIf`=wtUcB+xpht64*x;i{UyIY~R73AWzd zY+634VK(9R(+T$2U6ZvHw}LO^(qn7;%KcrfHOiz7bVv*dUIj*YO6grT=`CjWyp62E vm(5=FNi67F!#?nNHfB(Jghb(?1=J8BQS6}A)D$66s5BkzQPi@M&EEe2&nC-w diff --git a/litellm/integrations/__pycache__/helicone.cpython-311.pyc b/litellm/integrations/__pycache__/helicone.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..731a7e377f6c8f4435cc6788ec5940a8f60a11ad GIT binary patch literal 3384 zcmbVOO>Eo96&_O5uc@E-C))ftru>t&;#hLx7;$VTyN;v8=?31QPQ9^!hGj%XqAiJ9 zl5!kDDhxD0oy8t}NZ~_Mz^Bwj_OQo2?xny2?SV81L?B=wKv5ujQ{i2}$f<8AQ7gr5 zfh{?FZ+_mq_vX!;kMn20--qD&`<1`(lU{`WfgAQiH7W-Wpt6m4#1mPRtaBnsSZy*% zl1M-+4tvr|sJyZph^R|bNs2_b5KlfuyyGPa67bcZq?0EWQIz@~Hy2WkUW?f>2mR38 z1}oTRlH>`VOp{3m?|6!m6i)$l@=nmgtin9-Eu5tw%@lul$+wV+JJ<6J?ENd+O# ziLsQFi?3ZUL-UC{x92`xNU*baZ?m5ymZOwO7i4Ko`f_HoY+RG$Qy&x;DvV(W>E|K8fyyu0Z;^w^-5AOq=&0WKO@Etr2 z(q0vk-+s!}j?=EU^8e*Q#rN|ylzk-#H198gwf6N?I@+{3{||rYpRb`z|5tzf5Vz(o zmtB0|Xe1%h!55CQyX-4>@SzuEnJ)V==a0eH2|EyI&4Y4)?>f;8Zu{yX8-mRtwBoVr zWlz~#4t&-95_a%aa~CVYHod`>=1#t^wH#crzz*x<-K*BgmXEd0=6l|sgHLvw#PL=w z?NElnZV?Rt=&51bXFw3zuD9n8ftD@#b;mM#Ohm&ArQfzIm{7=IaS9GFfl7QaJ>|B|pIP=y+^mR4vL6r6?&MhhEc@gJpARK`9055IVXB zxCba!@?9LeI5v7=Y;^3x_uqjiER9-Gq`c|Q2pqsD;HRQ;YEdcmw*qkn{}CEUEO8>B zaU7OYa$grXS&*4hh$|G}w_zb=9P5jcrhm4mW+YiGS-mBCP69N|tE2ax6a?`tM6Kkv zBQ%Od!4(I9xO@_?fXWFf2TI(E0H%4xq;q06D>hm%-~~5j^`4j$Ot+BdaRn3v?cc#K zh0lwEqMD(fI+OySxL~?eF(>BJ@>#rkQa%ODGFkF0*3%_Y8g9&H9&-wlmsC5E9u>1O zX3+`{GY5$_74@6Wf((BN)4}DRm|gZSqMjfu_^Zk|cTESD@q%UWiOUsae85bPTz672 zse%N)QmU}7njQ7{vnenDQ)DhBJmgZVrq?pW3hSw8NcLl|@m9+pU`@0v)K|w!4_8c| z6&e;>Gd&QSdaW4=wqaHt!t>8yf`edo{xprXqaJSQYux}dWb18;_B4Ci1Sy_VIlR=m zg9^@C<~K~+V;vo{qve20?^w#oY-YP;`3S zpvN_Oe7~1zJuly>#;0lM(3y+@v^Qh)X0-0gR$?ombf-JNX& z)$ZNbhwtly%f{fcMn~R^#Ep>|?dFok6>1PM;uZ<4Ld=psV3xqGMrVU@#60dCyxpMH z!AsRkH?{kcHe7%LL%)E#DzGd$>j3s!;(7}^HHRCRx~Xm)s_ww%jURo;-?=rowf;P& z(-DJ?fMfgNzUP&l#jij7meRrrJ)AJYiOqS~#I51yqdLtPG^5c>H5fL613SuI@SGMr zw;$@+TG2xhBNWl>^3PC@cKqXShmGq?dgwDF^qFRt!x~B8wCFMDzNT5QTrY=*he{{p z(C-pw5>x1}13j2d_1x@qEWrI2ZgzPtC$OyPWAQOAW^tWn+0Tnywmu?9VYZCZP@cgA zL$`&-Rvydv&TkPnRFtbgHk$97Q_Rk=c5+0c!Zp(2ya3r^6W^e-zlM&Ix@N2tymRpq zzTC?ACoG=_f=>;{vyJXC+-=!!_vxKyroNPG@^uiwmr3O@keY)a2uO9JQ$xp&mMRKv pSZ@_|ZnzB8siSTKb!(`*ib9%QYUFtWJU>eQh5LT^4+GYc{tf2UVF3UD literal 0 HcmV?d00001 diff --git a/litellm/integrations/helicone.py b/litellm/integrations/helicone.py new file mode 100644 index 000000000..559481daa --- /dev/null +++ b/litellm/integrations/helicone.py @@ -0,0 +1,49 @@ +#### What this does #### +# On success, logs events to Helicone +import dotenv, os +import requests +dotenv.load_dotenv() # Loading env variables using dotenv +import traceback +class HeliconeLogger: + # Class variables or attributes + helicone_model_list = ["gpt", "claude"] + def __init__(self): + # Instance variables + self.provider_url = "https://api.openai.com/v1" + self.key = os.getenv('HELICONE_API_KEY') + + def log_success(self, model, messages, response_obj, start_time, end_time): + # Method definition + try: + model = model if any(accepted_model in model for accepted_model in self.helicone_model_list) else "gpt-3.5-turbo" + provider_request = {"model": model, "messages": messages} + + providerResponse = { + "json": response_obj, + "headers": {"openai-version": "2020-10-01"}, + "status": 200 + } + + # Code to be executed + url = "https://api.hconeai.com/oai/v1/log" + headers = { + 'Authorization': f'Bearer {self.key}', + 'Content-Type': 'application/json' + } + start_time_seconds = int(start_time.timestamp()) + start_time_milliseconds = int((start_time.timestamp() - start_time_seconds) * 1000) + end_time_seconds = int(end_time.timestamp()) + end_time_milliseconds = int((end_time.timestamp() - end_time_seconds) * 1000) + data = { + "providerRequest": {"url": self.provider_url, "json": provider_request, "meta": {"Helicone-Auth": f"Bearer {self.key}"}}, + "providerResponse": providerResponse, + "timing": {"startTime": {"seconds": start_time_seconds, "milliseconds": start_time_milliseconds}, "endTime": {"seconds": end_time_seconds, "milliseconds": end_time_milliseconds}} # {"seconds": .., "milliseconds": ..} + } + response = requests.post(url, headers=headers, json=data) + # if response.status_code == 200: + # print("Success!") + # else: + # print("Request was not successful. Status Code:", response.status_code) + except: + # traceback.print_exc() + pass \ No newline at end of file diff --git a/litellm/tests/test_client.py b/litellm/tests/test_client.py index eb9963401..1b95914cb 100644 --- a/litellm/tests/test_client.py +++ b/litellm/tests/test_client.py @@ -9,10 +9,10 @@ sys.path.insert(0, os.path.abspath('../..')) # Adds the parent directory to the import litellm from litellm import embedding, completion -litellm.success_callback = ["posthog"] +litellm.success_callback = ["posthog", "helicone"] litellm.failure_callback = ["slack", "sentry", "posthog"] -# litellm.set_verbose = True +litellm.set_verbose = True def logger_fn(model_call_object: dict): # print(f"model call details: {model_call_object}") @@ -23,11 +23,14 @@ messages = [{ "content": user_message,"role": "user"}] def test_completion_openai(): try: + print("running query") response = completion(model="gpt-3.5-turbo", messages=messages, logger_fn=logger_fn) + print(f"response: {response}") # Add any assertions here to check the response except Exception as e: + traceback.print_exc() pytest.fail(f"Error occurred: {e}") - +test_completion_openai() def test_completion_non_openai(): try: response = completion(model="claude-instant-1", messages=messages, logger_fn=logger_fn) diff --git a/litellm/utils.py b/litellm/utils.py index 67a3d6df7..1cbbb37cf 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -2,6 +2,7 @@ import dotenv, json, traceback, threading import subprocess, os import litellm, openai import random, uuid, requests +import datetime from openai.error import AuthenticationError, InvalidRequestError, RateLimitError, ServiceUnavailableError, OpenAIError ####### ENVIRONMENT VARIABLES ################### dotenv.load_dotenv() # Loading env variables using dotenv @@ -11,6 +12,7 @@ add_breadcrumb = None posthog = None slack_app = None alerts_channel = None +heliconeLogger = None callback_list = [] user_logger_fn = None additional_details = {} @@ -68,7 +70,7 @@ def client(original_function): global callback_list, add_breadcrumb if (len(litellm.success_callback) > 0 or len(litellm.failure_callback) > 0) and len(callback_list) == 0: callback_list = list(set(litellm.success_callback + litellm.failure_callback)) - set_callbacks(callback_list=callback_list) + set_callbacks(callback_list=callback_list,) if add_breadcrumb: add_breadcrumb( category="litellm.llm_call", @@ -83,9 +85,11 @@ def client(original_function): try: function_setup(args, kwargs) ## MODEL CALL + start_time = datetime.datetime.now() result = original_function(*args, **kwargs) + end_time = datetime.datetime.now() ## LOG SUCCESS - my_thread = threading.Thread(target=handle_success, args=(args, kwargs)) # don't interrupt execution of main thread + my_thread = threading.Thread(target=handle_success, args=(args, kwargs, result, start_time, end_time)) # don't interrupt execution of main thread my_thread.start() return result except Exception as e: @@ -97,7 +101,7 @@ def client(original_function): ####### HELPER FUNCTIONS ################ def set_callbacks(callback_list): - global sentry_sdk_instance, capture_exception, add_breadcrumb, posthog, slack_app, alerts_channel + global sentry_sdk_instance, capture_exception, add_breadcrumb, posthog, slack_app, alerts_channel, heliconeLogger try: for callback in callback_list: if callback == "sentry": @@ -134,6 +138,10 @@ def set_callbacks(callback_list): ) alerts_channel = os.environ["SLACK_API_CHANNEL"] print_verbose(f"Initialized Slack App: {slack_app}") + elif callback == "helicone": + from .integrations.helicone import HeliconeLogger + + heliconeLogger = HeliconeLogger() except: pass @@ -200,7 +208,8 @@ def handle_failure(exception, traceback_exception, args, kwargs): except: pass -def handle_success(*args, **kwargs): +def handle_success(args, kwargs, result, start_time, end_time): + global heliconeLogger try: success_handler = additional_details.pop("success_handler", None) failure_handler = additional_details.pop("failure_handler", None) @@ -223,6 +232,11 @@ def handle_success(*args, **kwargs): for detail in additional_details: slack_msg += f"{detail}: {additional_details[detail]}\n" slack_app.client.chat_postMessage(channel=alerts_channel, text=slack_msg) + elif callback == "helicone": + print_verbose("reaches helicone for logging!") + model = args[0] if len(args) > 0 else kwargs["model"] + messages = args[1] if len(args) > 1 else kwargs["messages"] + heliconeLogger.log_success(model=model, messages=messages, response_obj=result, start_time=start_time, end_time=end_time) except: pass diff --git a/setup.py b/setup.py index a33431fb0..e2d3a25be 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from setuptools import setup, find_packages setup( name='litellm', - version='0.1.220', + version='0.1.221', description='Library to easily interface with LLM API providers', author='BerriAI', packages=[