From 75057c72d2345c72cefa4b3a54d27f3bbedee992 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Sat, 5 Aug 2023 19:57:33 -0700 Subject: [PATCH] adding support for supabase integration --- docs/supabase_integration.md | 75 +++++++++++++ litellm/__init__.py | 27 ++--- litellm/__pycache__/__init__.cpython-311.pyc | Bin 1656 -> 2166 bytes litellm/__pycache__/main.cpython-311.pyc | Bin 14268 -> 13828 bytes litellm/__pycache__/utils.cpython-311.pyc | Bin 24502 -> 27052 bytes .../__pycache__/__init__.cpython-311.pyc | Bin 201 -> 201 bytes .../__pycache__/aispend.cpython-311.pyc | Bin 4923 -> 4923 bytes .../__pycache__/berrispend.cpython-311.pyc | Bin 4682 -> 4690 bytes .../__pycache__/supabase.cpython-311.pyc | Bin 0 -> 5615 bytes litellm/integrations/supabase.py | 104 ++++++++++++++++++ litellm/tests/test_berrispend_integration.py | 2 +- litellm/tests/test_supabase_integration.py | 27 +++++ litellm/utils.py | 44 +++++++- mkdocs.yml | 1 + pyproject.toml | 2 +- 15 files changed, 259 insertions(+), 23 deletions(-) create mode 100644 docs/supabase_integration.md create mode 100644 litellm/integrations/__pycache__/supabase.cpython-311.pyc create mode 100644 litellm/integrations/supabase.py create mode 100644 litellm/tests/test_supabase_integration.py diff --git a/docs/supabase_integration.md b/docs/supabase_integration.md new file mode 100644 index 000000000..ff9348332 --- /dev/null +++ b/docs/supabase_integration.md @@ -0,0 +1,75 @@ +# Supabase Tutorial +[Supabase](https://supabase.com/) is an open source Firebase alternative. +Start your project with a Postgres database, Authentication, instant APIs, Edge Functions, Realtime subscriptions, Storage, and Vector embeddings. + +## Use Supabase to see total spend across all LLM Providers (OpenAI, Azure, Anthropic, Cohere, Replicate, PaLM) +liteLLM provides `success_callbacks` and `failure_callbacks`, making it easy for you to send data to a particular provider depending on the status of your responses. + +In this case, we want to log requests to Supabase in both scenarios - when it succeeds and fails. + +### Create a supabase table + +Go to your Supabase project > go to the [Supabase SQL Editor](https://supabase.com/dashboard/projects) and create a new table with this configuration. + +Note: You can change the table name. Just don't change the column names. + +```sql +create table + public.request_logs ( + id bigint generated by default as identity, + created_at timestamp with time zone null default now(), + model text null default ''::text, + messages json null default '{}'::json, + response json null default '{}'::json, + end_user text null default ''::text, + error json null default '{}'::json, + response_time real null default '0'::real, + total_cost real null, + additional_details json null default '{}'::json, + constraint request_logs_pkey primary key (id) + ) tablespace pg_default; +``` + +### Use Callbacks +Use just 2 lines of code, to instantly see costs and log your responses **across all providers** with Supabase: + +``` +litellm.success_callback=["supabase"] +litellm.failure_callback=["supabase"] +``` + +Complete code +```python +from litellm import completion + +## set env variables +os.environ["SUPABASE_URL"] = "your-supabase-url" +os.environ["SUPABASE_key"] = "your-supabase-key" +os.environ["OPENAI_API_KEY"] = "" + +# set callbacks +litellm.success_callback=["supabase"] +litellm.failure_callback=["supabase"] + +#openai call +response = completion(model="gpt-3.5-turbo", messages=[{"role": "user", "content": "Hi 👋 - i'm openai"}]) + +#bad call +response = completion(model="chatgpt-test", messages=[{"role": "user", "content": "Hi 👋 - i'm a bad call to test error logging"}]) +``` + +### Additional Controls + +**Different Table name** +If you modified your table name, here's how to pass the new name. + +```python +litellm.modify_integration("supabase",{"table_name": "litellm_logs"}) +``` + +**Identify end-user** +Here's how to map your llm call to an end-user + +```python +litellm.identify({"end_user": "krrish@berri.ai"}) +``` \ No newline at end of file diff --git a/litellm/__init__.py b/litellm/__init__.py index 2d8bbff11..8b1e88e1f 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -1,3 +1,4 @@ +import threading success_callback = [] failure_callback = [] set_verbose=False @@ -9,20 +10,16 @@ azure_key = None anthropic_key = None replicate_key = None cohere_key = None -MAX_TOKENS = { - 'gpt-3.5-turbo': 4000, - 'gpt-3.5-turbo-0613': 4000, - 'gpt-3.5-turbo-0301': 4000, - 'gpt-3.5-turbo-16k': 16000, - 'gpt-3.5-turbo-16k-0613': 16000, - 'gpt-4': 8000, - 'gpt-4-0613': 8000, - 'gpt-4-32k': 32000, - 'claude-instant-1': 100000, - 'claude-2': 100000, - 'command-nightly': 4096, - 'replicate/llama-2-70b-chat:2c1608e18606fad2812020dc541930f2d0495ce32eee50074220b87300bc16e1': 4096, -} +####### THREAD-SPECIFIC DATA ################### +class MyLocal(threading.local): + def __init__(self): + self.user = "Hello World" + +_thread_context = MyLocal() +def identify(event_details): + # Store user in thread local data + if "user" in event_details: + _thread_context.user = event_details["user"] ####### PROXY PARAMS ################### configurable params if you use proxy models like Helicone api_base = None headers = None @@ -71,6 +68,6 @@ open_ai_embedding_models = [ 'text-embedding-ada-002' ] from .timeout import timeout -from .utils import client, logging, exception_type, get_optional_params # Import all the symbols from main.py +from .utils import client, logging, exception_type, get_optional_params, modify_integration from .main import * # Import all the symbols from main.py from .integrations import * \ No newline at end of file diff --git a/litellm/__pycache__/__init__.cpython-311.pyc b/litellm/__pycache__/__init__.cpython-311.pyc index c047a814425c1dfb3cbdc7d5600ae36110a1c407..3cf0db7d0abadfd2ae22a7ac90f8fd92502223f2 100644 GIT binary patch literal 2166 zcma)6&5zqe6dybB#`$nQHrefVyIEd{uORdoQA8DvOw>iD0;E#a%+Z{gr+HI@afvEs0Z0cb0+oQu zD*6;r?LMMQ41HV496!k+^b{yLv?dD4v;jK^ZyhM-^OA1*7Iccw3Fg!*K zl0;>a6xc?Ns<1d06+R!`^en=yY_eRT0?Ay=7E>AA8o&VQ%2>b2|Vh&V0n!cI^P; zy}>BbGa;0Hc3!S@iNi3CN*FsH9dgN6F#c{x zcxn-{#C}Jj41*|2kd>7{6&U{lz|N^wLviSxG){?~%h&ti@@bWHc(Qt02h$R7LYAyp zm9PR3*5$@KlpNWvWp^%JI%sQQ&GH)QDLSCjm*NOC2VG*!>B!UI){+4 zh4Ppa(PgPpw>TM6*0~baXPoW&0da%Q<*+2!fZG-c*nYU0gu|ReMyc&~JA@Jl>dH%b zjBp1Ef$cDF7__x$$>KJgMO5Hkw+mCFDm%8=Ah10b2crRtnq3xPFHsPV2SgCZkCvfo z;rQS-1gtBFoDFW3_u3btVp|Ms~^?xLMaRdTWHEYxIbcWB0ht) zj5CJ?Vic7ea*PA7&s;y!M5^)z%q2GNvr$wep>!iz1Ge82-oUm}ah`@mELmQU&1Py` zz9i{QYRIFLl^5w`VB) z3bMhtAIsNXPQOqGfgQOF}M97TFZVvKUev9z>kHzw2?Lp;F^hb zxv_H+HY99H*aC?14zb;+@}DE~g?y+MZ_J7QAvlEB=p*qy7-X^dKLEx#MNy`xKFRvD zxdQwQD~)k^TB?qVQ=>jEP4&uHn;H$^Ds|A+TVo>ysf+Wf0uR-j@W6iPy^{KW7DgR4 z*B`y}Q~T%kxE8B9aHpUeW?NF z48}&hjB>_QZN*xy1rzFV1!)bKoeDatUYzXRd}iGI8|h(jbW8ybf>JO{?( z7+4%E@hYVz;YJD~ZHy64zGl=6P8p1vH4sk2DTgySYeZB5=RnPq49Sk50g?+Tgv?W_ z1uVdgd7Q@ut7sS1B7-2Gw@P+dE!zv~f?ZK7_M*BN>P2g5m)48cCD@}os8$-^-u;Rg zgtp9+EJEgF190Oq06hkrJY^uuAyi8RY2(Nh>nlys9Xj29kiT#grIuVD{j1d_IS?-1 zlUs*D=`YC(qRDZ=nz5c{ntL00eEw${U9 z&omBP>(S_eLyjy{*IaVVvNT&06>(MSiu!@(t|_|QlG;RWx1`p-hLyIgNQ#6X>dl7S zsZ09`mKvR=PU;FFq$x?O4MmZUMY*OCs52y$W+alk0^r_Hc!CrT?E1xqnU z5o%$^c>-JJoD`^DhrN|>lC)0e&pT$D4C{ShTYQa~GFNNMUX6QJP29VAe{XmD?&j9c zG^QPydtGfnru+fXFmVRn5ANj}JR6#FL)Wwh4!jg{1d2RGftFZXGkepN*>j2EXs+4s z4c?+Ns37k>bA368t=oM(w8(}dfd-?12HU_{lwlYz%6Ha37c2f^m3pP>SFiYK!Oxd` zq2w=A{anc}FZsGLtE4lL83IfavnopECNV#eneg5V{`uyeQ}iGAQQ`F5eYA9X(VZ1& F>L>Dk^``&; diff --git a/litellm/__pycache__/main.cpython-311.pyc b/litellm/__pycache__/main.cpython-311.pyc index e1b1f2e550d0dd88b0b39aec11a88a18f4cf293e..6329538c6b08672e0b5e0af973b62d5de3c75508 100644 GIT binary patch delta 3153 zcmbtWeP~y;lS``P(ajzt|;CGA27Wx)!?# zr{QE_y&KN*jw^(dOp=)i8zmt@B+E^HjU-6CN3!m6i+)*TsxHa)CgHfZ*thbmaZ;0H zS3k`XVH&+sOT7m3$0bmUtO^p-k!&h9OX5wvf^M*DZd-z+Y+93*rL4R{WhH+{^D5ExumC-9S^FE!`6igx zWqBg;FmF1=TNEpV^&`XzJNf`=fDcR_w-M99CpDp{+gDkwo8Un2Bqo^Ai^6>!H}eoP z1k5##;)Kr}ez-uec}hU%ogdMWk& zH|ch3u2t!~?|WUW(h#s%g*pkGv`qWzI8|NAJHbeG2flCFW;OV%S+$bE{0CMaA^q?N zoBxorrl#q7PX3vud7(K#f>#b5VJ!u9YbJ9x%I9p`J*hr;re%b; zvTk{&Wscxsh+q+Wnaf+|9(Y>(>b;bHC9;&Tg)q|%=WC<|%&w=%5UjXn^+T)>8iKc7 z-TISAJ`Ve?W6l|sL^#DDsG$`G^tl%VH{)bA3_3b!H@x5ulO1@^-5d3zCa-Fi^%E;uhCi7&LP?}k@#IpRu4QQmMHr7_Wcmau z8Q36-E3!mqV6Anl?Q1AFjc|rREjeU-9WJzWi*Fz~kAPWHHI*QYcshg_TL%eM;|Jh^ z=Ykz?1U-)IN-=Z+ZhMq3jA8x<+Erq}i*3;p^>xuFQH-^ejq-CVat2HMVP#{>_B2hb zC9vX}h@OUjw|U7dc-uRI)Q)!McqXy5CDSakqyws__~lJ1FI|)`QS5no3Y7NCCQUk9 zHpr>11cguA-Q+yzyx(ZXI?Lu2b}C63LuY9_q`YUyIk@HRm+znV2>OX2L=i?2jv$O7 zv>>1>dKBRp!Z-r%7saNa{Ro&H9Ym*XvfyVdg19r@yvSZ z%=*>U-SmOSU+6tn^o*B0<5yQpde^?uc9+u`V*8%1mwI0eT?-wU?Im;DU5+=z4m@2Y z&tQf9iO0W}E_%jFo-q(Qe>rrt-k==~j;aPjHG@L$SkV(Jd1A~W0#jYz{r{};=I%){ zB!%6b)cwuV37+oku7b=VcZ(kueQS?rhyp#DCdIW7SvM;t>GhH*O zrKp)j4#c*S7(nJHqOZ^~Togx2;z<6(zQ2-iJRFa3A4g2fLH^^3zU3bNk3Bj@ujtq& zR-Xn&@a+5dJs$4#i=MosQK~Zbe~DBViVlVc0%p`zAGQwbJ@DB;qgnd^Z|5Gb7DEwW zFyx1U!A5iKjlS2wUK=7jHR!O`UL|bQHYg(G2B&ppHX9vK)T=M+8?ZkZApEz7oON+# zjFs5`OPrq%Jv0sngg*w3i5ypz`}tX)FEW^c6~_Unh;^=$*w`%x%BI^ox$Bj?o1 zKFL(*6v8yZ2?lJ|sRQODqc7X2oXxNXl$W;R$xV5@XeTTfO zZL5a3jxKyKKkR75R%3%}V_9a?IkiS_vUMxF0NJ0y&TwacSD`yvbd8o=qYrJk183)6 zxaf?OoDul_@KsZ1p(|2!MoZ2pTlT|iBOi*hj#)SNk=ryU@E>*i=NkB5H0b7FGx9mP z2ud_&#RTY$Dm(`7M^~MGwhZ>uE##_O@H|rIAvU_;UV_VnKZK7+S32yF7YB#Z? z>(|g$Nd{88RcKp3PP>XKTZ2G@uJn&COp_=QLI`bwX^oMA$`G-MmPsH=5kja%0`7UP zod#{8LgHHf`Mz_`@7#0Fz3*HfAOAs0`nDul1U~*a{~>+rjN~J~KNJ3wNhc^$q8hH* zJTDM|iqt{{YNe%dLZvzV7>SeE8d`Q-7_3p;+|)~LZxTVc!oT@3CQv&q*FFwPxJ4~> zuC(C(gg9D}QHgRJYSYj`U2`VtKCXY0JGz3-;5uKS*L5DT97d zw%Q8b6LyG+{@ulxsCiC5?fDjYmCVSeOOn^rl7gimBpzZFq_lHdm?l)5(>=s`4Spt8 z7}(#b1wJ-zg|(K3d{VyUz;~+JNy)oOIN%#*nKVp zVxaoWw6G!~XCc8U;WS~h|L;tsVr8gp-lZKCn>9@Q(CVs%FIXk1xIrwcnqZIQ(#_ks znH%PFMEQ9uzCC$Xu{1ZNI8SXuS(^A4u9iJT zNDthz)h*WusnBHaUnyDmvV9}ryoK^^qSfxKqh6~}6rOY3C#PVyGhlt12d1C4&(tQ@ z!zrf+UUB+;RV%|RoQwyr^gGT?B)?-NuKbQo@UUKY@v!iXtAjGn^L#eud7U`HDDNfB zuwUM0YUXZO6fVm(rpIu)1%57X2n=XbgvU8VbgU@Ikr2G$TMu9Ix+E{xrF9%aA+`p- z>TM-ofotCS7iy8C*El)TBuh<9sw@2L;ddqeJT;?w64FnXqi*Z@rxAqiuP5T?aY56@Q&Ib3MbgW`OWY!AF&p+3qo z*1sV&zYM(S?+6dzb1^;k96n-7IZ-(>qD*3@mwEN!BfppIgnRxV*$UpuP*+ZuO6N+H z8Z{rWV~3eCd`LOUu*Ay#=zmY6a*9v|WLK~2s$ zI-ZKDY%lz)^7Bq@AFLyM$Ho<0JIyNLU|<({2Hp)cS_ja{7KF9%Ng!at=ESe69ABJev6g}`k;zshP$;JH>nO({3uH=c)=-W2V31!Kv^oBlPg)W6(vvSrEY$XYA!3i^_bOa3)k ze{)`ZyFmX=YsTM`_4mM?P&9@Z;cxvctLyMJd8L2%hwciy?C0ocl?0d)FoqcW!>u* zUJBnK+eiATPPnDB&<*-qr46)Gf2-5T=Oad5`oh50sOR6mh75vmK@j<0pc&(|j*>(8 zP73UIR0n-c9Jgpf1}8!;co;ZL^llBZVh6e`&t#y4JgGSg4ziAtl@tTP=_yb?g?;VMc<{)vSq?eF zNNVyZ!z@_>e$@W#UMug`beN`8-uu}>l-D2}MbMffCao=#zwvMxtuXh^y%Q;VcwFgY qSGbIqMM$gsCB3UBgxrKP9T7O);aK>7M>#Q+3dFhaac31VApPIl7~N(7 diff --git a/litellm/__pycache__/utils.cpython-311.pyc b/litellm/__pycache__/utils.cpython-311.pyc index 5f984ca5341fef0b25244bd7f2e34dc2a8c1680b..999be9cf9d9afcbbd4bc3ad222a5d7f73369483e 100644 GIT binary patch delta 5661 zcmb7I3ve69dA_{^KoB4v6bTaG6MPEbOQc9q;zNu`@ga&bMT?>$QKDc7calKip)5|; z3o@0&na~Pd(_Y(>Tgz4IsbW=Zngn?$<4kS2vXfM?1k!NLBGpVft&?V^)1f9-D!bFi z|K9-=rMT1S;_kbD|Nq~;y}N(Ee-}S|i_CpM%-=GZ3>=jE`rG2^eb1Zi2O z$#_Yybi9;sJSTCJPIigUdx2=D%0%KVhqD6CN;s?FtcG(noGv)saMm2+ytQzx^K;|% zqTy+7yg?-JGx|Ba9@L3hPm}RAq6sXGhd9v;L&Tp^Byh7P(Q=t^+*LT$0)ykAr5Re9 zdPOTVT`Oio(`IoMV2hXoxK7LkY!z*Q>%~044Pt?q558@p9ct~O1F%CRqnx{tT_aBZ zw}d@kZmV~bq^Wyw^r#dPw}lTKk|-#wJrbq8}{u6d;r+&H-gyo`WG;$e6*EV?%8QYW<-@v<+(2v0H{YZAmMLr!0Gi!=d_~ zK-hN#%o9$xiMl~gt?Yu;MLOAaYnKt9fVLp4V}|TPVq?|W&15t-p4}smE$m|M7gjev zZ~-Z4KI{#N0m*aF>kk~GlG{MHpu->&Z1rRVYqB}WRyJg-hOVfsib$~wwn5_FgFfR3 zTLB^hz!T0*RCxfMFW{F#GTjHt9n6s5t<48185_&@6Wauerh~^qKG`1*d3@eLfNI$X zb~oA2{?mSt9e31aQ$IM-NrWQ^0d~pZDpmTbt-&1hgAr`9qD`jV91EU!m>i?;|Gyd!d>Tm>h$V_#~ikjT~zlKB64SR zPw9@6a3u8)L9>lBBjnmwK+YY&FUw`*l}mjPPFOq+hL?rnKf1B_bc~23l_QoAcD7Y zsznJ4+J`2DZ=sn&$eiVtGz~)5Vs6+voLmw47Qnmc@-_cM8g1 zsd=?#y5!TPFB0foxzGK3y9sB=S>lM0^#B$P#Ylnc67LFK)A zU4IMzUNd^V*V4@ov}u3Zie5k6Ku~VmY=X*7y~{A<5N@vGhjN9Rxmu8YIv7g2U{=5h z_q$t$*lVs#B4{RNVt?WODiMUxtE{oM7Or25?XPw40v|fgPSw9;;X}1iy_la0-^;O* zhAqT8OF~nib*8kh$K*A=q$u=vpsP&j;xf#nbP=mqUBg`qhFz+N)v}Jo8*|z2ZMnR> zmX)lj~G$+bO1zD4>bKKnO%Mcw`F+!9=xLM7NuYx1oL$`Cu>(d?_H(7*v zik0f~tl<0&bv8vaXr`k@A&3c>4>M<}=1?FJc8Jqltbe-emO%LR#ZLWsB&{R)q z+1pK2Hw6|t!8WYjsQVI%C)w$>UnS45|6W_AQUI^ zv55DObVa99{1tS`qEh5&I24gQ;RBO&C-%4tL1E`3O6heDmEQ)qqE%ae2NjR_ zp99@z#c)sGVJYHFPe>&2sU`+!{8};eBqERosip|S29+NOAmzO zITnF&TOYj>yVu;X7xP_ml~)w~DS{Ox8PY=J3Qt>cWN3Gn(I26ka5Q{WeIP5EdJr%j zB{QYav7kiZ6(xm;OesUlKK4{gUiOP9zJN}eBhtwTyyib|X`4E)w1Q{xNKazkcp1Hv zY-)Mrap(m!DLJ79rF95v5tO9XhSC6lJ5NpByHWArb*hOCb*~~^M0kZ`8`t%eD4zqr z9UTKmW+AtEQcq7eu#4+Ti!lGt(KIAMh^&Tbi6nNIEv}>15t3oK%OOs?miX-?wx_jO zAa~ibZ8w}t9B0|iFIw_1v@cky;+CpA=E8VU$Go{SVeX8ZJK3ELr+vBc+`%7YBpL_c z3dg*8Yr?!WZr*ywT=bE-dcj=%#@MypAC7!6F>)v|5`ZqFq=$#gc76|DL_WaZ10_Ew z#Ld-r;f`B{PZY$vZxuehzocPf?cb~4*~Yz-@p$oQHUB{+50ww9t9F{T31c0|i8jMn zrSPGUH&!ZqSgIYXjJ0&^B=)`VxNbhlH#Ab5Q4vg>PA9doOP%eouJ~nVBc|Ssbp=WY zK%Du)A}mspIz^J)5Z`Ra#x)k@CB6fVDMCM+-FTu1R|mcoJi{#$+zvf^&DFs?T`vp5 z#7OLZ*G2-A2J>{&#cV!gie`wW6j4RxLgp_nW^eV(u;;o5k=woj=G>HdH~V$>94hNU znWn+h>{8DsMWH!RIZ}R?Weo9D5ewM2doC8@6QwnUD`{-f#zwXk@p2v8wz+cZVN1Hj zBtD!{kOZR|d6^c028;fH289^{D~ZN*8;%1O>pf$UF}zbn#>h{VsAaDBGJJ}wJjZTt z&a;$!E@L{1El*iyg_*6e7y;?QVkD}ig+QO1dOPa>U$Fa%tq(w`u_h@cQOesKB{8|lxR`WqCH zpAVA%J6X6#K?;@zX z5ap$8VE@qXP&w#vG$D)rav)K+8Ln{58~PK5{s`uzw%CH?`f)U9TPLc+ADWU(G}1 zeV1XlRr~(hw&5o2ty&#uZZ&B^bF0+}@@>0eq*}OL-ZoMu#LKiG`;cMM1!ZN$gDc$g zu&ftJYQow@K1!a14+pDr9^*(YG8?KlPI=({s0Ekk>C-I?D?yYr$le36@dY* zmiv`1??TOjt}?ExT+Fn^9kuaHJQiK;*E$ofEzjB$nN@M+SkiJCRf-!rP~?)I|?Itg_!uYDBN>jlfh<2BU6BTxdmz<52K!FceEJ`;W0LBtO=> z>&v`OVT(`L&E30<4~XMUR=4Ld@(DY&r>jhPMt!PVd``9N-(lAXi?P4o)6VboEk`2g zFp`yM&IRpxPKIi_OfLJ0+!6GFP4?&8FY)Y~n{wFCd$M;-Dvs!UhQKsPH_VO_8Ap~X zlgk-4%5{P+3V)bR9#!y&+f=Q;+&jK+=G_ zcGZ;3O1V7g4IGo`!@F5nVHi_pWtQpG%F3BVjqr*T_zZ+d5L3%p?v6Gqp)J28E@*4w z+M0Vh&XSif*M4MfT`;$P=k&aJa{`Z%v%41c=Gn?~=4tb<%3aU$GltoYd41`k#WrpF z*kqX*e{TP|{TGJjO|FE=71y~wHkxMIpX)l;b)k0NSe-Cd$2HZ;>;n!N9(H#^U~Ue} zQs_x~ABLeO03|qQu&O*8z7|@Ckb|H^(SM=z7{YagBzt0CS;t2pCT%O|BGPbJ3ix3m zFHxLC$=ru*sXe^1qi6qCQ80y{`jPf&GE=!ND-8cPc>N8^u0@^Ik4$JJf}}AJhW&%6 zKGrN~lm$EOM@h-045b$lUP4e->*qQ64r*q^*itFvM<^*7RiTxASPdeC5&95D5pX4_ zp5OncI~xi7d9Vxz z$s{4hC2j@o7BWfh61NNmZ6tGvTZR)vLk7tbw*s>m^|Y4sk}R@%iTguPUcmnVZyjH1 delta 4066 zcmb_feNbH06@T~bH_LtiyTI-SvIqoLSVDw^5JLonBv3#lN&?px;W!z2m? zHEPsUliW1bR@4+Qwu?ZA%jCFi-8IPMxXabY>mxG}@2pId_*N*nfN3 z``dHRIrsbg?pYSjke6O3&ZiuX90AMwzn9MTA95Cx1Ba>}FB2qVJU>w|RFLotc@o|s zZ=!IhFi|vALmOw3qx*BQ_YT39Dxdgt;W5Q5{ zWZEYTRZ0Z@%rODK$IOyt9~oLBS;0~@E=V>hhyR%+0y9-h_9uxTJPS2bBta6Orww{K zq}+X^aVR7?pzEcQ6R<{d0oF=xz_64DSSNX;e3)RZQ~<4diR=(UUUrTIwNddCV#BR9om-PXA$Do?4g zq(U9+PdOz)K2|aMdl7u-Zi^?!$7MP)rqDtbuvZtjQENeMOc|$*&??gg>kjgCPg406n#{U|uz8VqJM#s&E-}EkZp&NYsrHI-b(ale_UdC4w>- zq;B?HUIXdVKF(`5kSkfG_X>O6g^n!gEi_+=?GARoHymgHQ7=$wBr4w&iB68l=eO3$ z&Un`lH~ZRKNk+9`VHXKqi~X)c=mba^07iqYl<&+OjmKm~r8_~{%MKT{<>Z5;3!lPW zYD+~i;@yR9$Jn$I#iJX+{S}1dvj+n{_q<6Ez9-m(8KFZsC=Nfby%PAqIF~Q2l1kzb!?R|L-x>3h(C=BO znGz{FC?2?DhDfE8cw_=a|JCv0gm2cu{Vm_{_k;iU!9SSwcY}XfJdjv5YvumdmHw+D zCD7rRB3R`wUtLI+{q5Ye66<#;TbROB8WEV?BUOHIj& z1SUsY*A5i3mYkbpN;5{*Dtf)8bP9&h?OlnfBvm^pO;Vc2=E~bgt8q&qgiF!G7*k>@ zjiHzqjZCT2RKDnv+t6wvP1R+$LhnXbN4AySP0jES zDC*~n#`RKM5TA+s+!I9Nc@_9$d5sm#j^TD?dxcl@B={c0JY zq1RhJg0jCY9~vjZ=BpQTi@7XPM|o= z_B7r}?qUCGtTgjievnnIzkD@{x+O~Gh$>6E38qPp_|7!^c=a<#Pbfm?&2~kBW=@+4URhNFlp#gMS>815Z<-5~_i$8>W&|nGx-1 zYC*tEq1$9-Iw1r5WZjTbDLsT1e2sNCFjpO!N~Y9aIfZMahtYNrZN^D?Zwj_)Z)(2k zAyi=A8j)|or~D{-ozFKkj7^?O8c}Ko2o+{h=nhm|*g!VFXifcte1FfN`Dy??7hnAm zNme7Vc#2~3Nns9NX6nlC3~&3eGs64mT?;{hyxqla+(=!m#Nzr~K-^!Q7O&9GZfr1+ zH1ljeZv5(8T4e9H91CAlCj7?IXS&)cF6N5RSaeoiy}`8DPz&;@vYh^0!>O9Wev9F> z#nhjxy}0FCQak`NL3$PI>ggt&6w?IHtYxj6NF%$UwI1@%cUo)x{NSy!5=}~wwRJO> z<%WLr*UNAj%WK=S3bSJLqR58qm8H^`X7*Y2L@3XkIs}a&vf4QJy#>f^ROUJBBtyyjhf?9h)o^dVs*i+j}T>k%IM^T<)MTTLXQCGb|hA6IvNEIW~lW#+K_bLU=?s~4wUt1LDGfXG{Y!V zAdN1f+9TL}6yafnw-AmaoIp5<@CL#K9&*xQWUeF|#y2|la3}!00)Dxrmf)2~zO6IZ zSba|sTi3J6QDJg?E&%Merp0n?PtQKW{?nJk%zaVd-pzeS*sE9ly}c(Syjjzg+uba_ z-6TTe?dF^wi|HLhbB}0xXR{eJ?}{eSylV-f9Lni!GQ8W++5ELOpM$bL)Q3<*AoLDNRq|}HKl_~O! zzRb?( zAWUn?U0)jJia@*;R=5p&;AQG&DIzD53cM9Ol)BOIEi~lCW2zjFCq@9}gsjrNv={Ut zV`d4sGyZReZj>TwBxQs)&YYRI9IXh?tSS*>L|B6E+yrR7^x=HK}8T$JALhE27NmW*Ep?^a= z&lY?y_hGXdfzzi7B_{%(^b3@>BfN(2M>Z~(&7B5O&rQOw-bk#D=hV!teB(m!B9ToI zX(T)K131P(gf;}uQqD-u!y~9Wiom(ngc84e4JFP}9uU{<#C8l}5}_3V6Hn#|=fuD( z<>$*!f?}ee`3T6JudSOCNohJRZ=nV_e)tHfLH6O;nZpySnR~*1+Cpg{UnMvV%@Uku-ox+3Jb3nV-g_p* zGZmV;jUSuDCT*?MOhVI=WvV7s)uK}E57VZp+kWgCJ&C6$MasUkRns(Asnn>ycJ6z& zu_00aRF!spIQQIh?>+aN`*F^B{M_SlBPedypV=SpL+Ed$Q7R2Z;$Z`Y&|8Q_EG3|* zSyNHUXj`Hd3*yk6wKVAt5bu^6h%!gpqBdi+J!&^V0}}A2KFiV5nUQ@d81AXI)&Ih zZz6W@9Sc|me`blgSZWG|p8l9LC;2rmLW$S@?z=8`eK?%>{(tVRefIl};gD5#Cz-{V zBF=LuS#RW1>5LMKi?R|+bCSu{_lOx~H=AD`E=&AFNg^KRUupq1-+b|$FXJ645-lVA z+IJ|F_~EAO(?4DQ^>E@}!}DXW`nO8NKZg^4dH%DJ7uu*19t9ENk*_S|zr!JK){{sp z;enpPu#%Bx#4K$zORa{g&T!w*;eqT<-ay~sY<=~(!$b2~U!|Zh;q2Wt$P{QN{3BVn zQJE~4Ne}dcoVdVbST4+`WQ9p7;lo*Hk<*{8Q@BMXJj2UkD%=NZ)Wv6+%Cbsn@}oqt`bF_0t-(pVXs~fo^o-G zX_T_vK#SICbc4!KYZa@NWFVA7b7gygE^Au-kihEN(i|4S4m&4xdg|i&(U(W3#$y*R zOjNX98K0Jj$gCxt)U9bgt=AY=LJ)MPTs)N!ty@G{cO*E4OD*Vhi66^ILPcwyyD8Oy zD82SfGA&BVxFm^^?v^t%X-SN8vaGw~vs`>W7H2@ZRlW%fZjp;;6lO-?bWdF37=??) z1)fVOy0^TF$p~3=o6HHE4SnC&|6@lElli?5rp-^E}f#CdM;K5F+;q zyut}WvRv~ig-b|`!iy=nx3oe}`ljxT#rPDj#A4a{siN6EB^LMxB>xo9GWv%SzjtJP&xY%x zdHRw{U&8dIin%M1f*oc)di1E^K*7$fmch-I!S`5o=+sAz>X~T1<%-&Jh3MhO z2FTzv(XVXB(1(Xd!`9z)Q-I@k;FSuYjc2qBTL{sITC&5RBS$Hv5l}B6X}q$XqngUS ztYw?dL6q5Uv9@hG3+pV|pD4q=J#vQ($99h)!;$)!t;tcK(Vmu3Xjps<9RN8)Mzy>jlZ> z=ZH|{Dr@Aj?kd$0z1k@l7S+P~U#>*DQmZO^zTz{@*>jF-7H~}NNyGU_HrO8Bo{^aZ z_yO=1$+S|8@DjPOdObv(w7`*PTZ;THiAxK796UxMe8T_lzYfifd`vdlNDxGp6O5!l zw+dWJw}EStLmp|ru}qTml1L+`!3%IPab^y@l)^}`CZFVVCzoPPs<}!PGt0=xY>X8J zfsypqU0UrLQfDj{lQN(;kqw~}f?I`*x{3rw&LnkjW$Hp0b<9EnTvI8X(Y>TyF-Rss zC|FsfXNZ};w8Y23n=(Q?BQT06Wt+;5ZD&^(Opw0|QF9rAn+bJcXNT4u#?Fvd>-*7J zTpIwm-#zqRM(utP55J~%U&NPL%n9n{Bu*`Dzk%t{{l4StgKFO>etBB$i{h(1 zp1ZDIl?w12s8g0t5&AlH+VTlPhfiBD9R|=m``#YMo>l-@H*oN-JKx;DGN}c+?=Iv6 zgDVlO`}uVy-#r4twZ4&!)_mUyb0!(yLPoZ%MD91WgJ<>-|x-$pHTZxnXT5j!gUI`l{E>0>$auc*q}q2fA21bZXc8F>ijf#M|N zDHW1k7;gKIhTC=_3j!wydBQu0wd8E?pr6>PT?hra!dhu|Rh^^2k-`JK1CQ=q^6-rE zI>=peg41*6Tv^~OxpR)3ow(bzayC^QYE;$kYpIsC=4{}k-^kVGAno6F;yHH?^s5RJ z70#2baQAFY#Vjl9yyaM;4J%za8tmJVrxE+MbD^~gN4c#fj40bzy*q2ftL!Q{ddDco zt|7pvw%ph^&r%)9vskyW2f`Efz+Giu?f;>DU}HOcd>W@hDt=^r-o!c~g zyiOJK+>mE7J6Li{!3i;u;8TfUI5-Yznj{C|Bfwx>%cSDO_Xn?uQqb_I!K|-4Sn0u! z2lblxtOzbomJWheMqJTrl8|Y0-$Z5TsGdw znqdnI17D z9EjpmmkAC;^S&#p?+W%^(E@F1pnofHbTe>ty-_{(YCdp54P4j?Om7CJadbW(5Y&LM z6;L({hL$=vSmmVGn8ufg+Cz`>d||2j*c--s9Dw zv-zP3b!Ym|Oqcuwk>?_v0rU$b4G@~-K@y3a1|nnRbAXo1 zzX$s}vF+)C#btjTl7|Yscfb8Sa4PW8HhX;m?MCKGWRZ|tcd#P)LYBy#C*H&OSB9+D z3L-<2Mx*b9(FZRCvRQCHjnI_#64XwRp)FBz+k(%XFPNqeOBbMnL}U5efD2ZNq6$t# zHDFY~8#LryHhvmvSaztWA&(kW)QC}|hJ3gL8Vces0?4&GLK`7`Ers`p09yN@jV8<` zabpTV3xzjs;&~bGR{(ajI>Q@_cuvAiGJtlddt(;!*YQ3HKnwSObQ+5{a9|NY^9R>E z@l_UkH~_7+V|@|N@VH?PKnosNPvCDPaQ!TRb|}1&#liyKcLP9c=~&NVHidm6fChRV wC3@}^0d}+wc5R%-2@yY?1}L=n>cBMsT6Px>Oi&aY?&I+%2LID1^cu(XKVZ4Jc>n+a literal 0 HcmV?d00001 diff --git a/litellm/integrations/supabase.py b/litellm/integrations/supabase.py new file mode 100644 index 000000000..1ac28763f --- /dev/null +++ b/litellm/integrations/supabase.py @@ -0,0 +1,104 @@ +#### What this does #### +# On success + failure, log events to Supabase + +import dotenv, os +import requests +dotenv.load_dotenv() # Loading env variables using dotenv +import traceback +import datetime, subprocess, sys + +model_cost = { + "gpt-3.5-turbo": {"max_tokens": 4000, "input_cost_per_token": 0.0000015, "output_cost_per_token": 0.000002}, + "gpt-35-turbo": {"max_tokens": 4000, "input_cost_per_token": 0.0000015, "output_cost_per_token": 0.000002}, # azure model name + "gpt-3.5-turbo-0613": {"max_tokens": 4000, "input_cost_per_token": 0.0000015, "output_cost_per_token": 0.000002}, + "gpt-3.5-turbo-0301": {"max_tokens": 4000, "input_cost_per_token": 0.0000015, "output_cost_per_token": 0.000002}, + "gpt-3.5-turbo-16k": {"max_tokens": 16000, "input_cost_per_token": 0.000003, "output_cost_per_token": 0.000004}, + "gpt-35-turbo-16k": {"max_tokens": 16000, "input_cost_per_token": 0.000003, "output_cost_per_token": 0.000004}, # azure model name + "gpt-3.5-turbo-16k-0613": {"max_tokens": 16000, "input_cost_per_token": 0.000003, "output_cost_per_token": 0.000004}, + "gpt-4": {"max_tokens": 8000, "input_cost_per_token": 0.000003, "output_cost_per_token": 0.00006}, + "gpt-4-0613": {"max_tokens": 8000, "input_cost_per_token": 0.000003, "output_cost_per_token": 0.00006}, + "gpt-4-32k": {"max_tokens": 8000, "input_cost_per_token": 0.00006, "output_cost_per_token": 0.00012}, + "claude-instant-1": {"max_tokens": 100000, "input_cost_per_token": 0.00000163, "output_cost_per_token": 0.00000551}, + "claude-2": {"max_tokens": 100000, "input_cost_per_token": 0.00001102, "output_cost_per_token": 0.00003268}, + "text-bison-001": {"max_tokens": 8192, "input_cost_per_token": 0.000004, "output_cost_per_token": 0.000004}, + "chat-bison-001": {"max_tokens": 4096, "input_cost_per_token": 0.000002, "output_cost_per_token": 0.000002}, + "command-nightly": {"max_tokens": 4096, "input_cost_per_token": 0.000015, "output_cost_per_token": 0.000015}, +} + +class Supabase: + # Class variables or attributes + supabase_table_name = "request_logs" + def __init__(self): + # Instance variables + self.supabase_url = os.getenv("SUPABASE_URL") + self.supabase_key = os.getenv("SUPABASE_KEY") + try: + import supabase + except ImportError: + subprocess.check_call([sys.executable, '-m', 'pip', 'install', 'supabase']) + import supabase + self.supabase_client = supabase.create_client(self.supabase_url, self.supabase_key) + + def price_calculator(self, model, response_obj, start_time, end_time): + # try and find if the model is in the model_cost map + # else default to the average of the costs + prompt_tokens_cost_usd_dollar = 0 + completion_tokens_cost_usd_dollar = 0 + if model in model_cost: + prompt_tokens_cost_usd_dollar = model_cost[model]["input_cost_per_token"] * response_obj["usage"]["prompt_tokens"] + completion_tokens_cost_usd_dollar = model_cost[model]["output_cost_per_token"] * response_obj["usage"]["completion_tokens"] + elif "replicate" in model: + # replicate models are charged based on time + # llama 2 runs on an nvidia a100 which costs $0.0032 per second - https://replicate.com/replicate/llama-2-70b-chat + model_run_time = end_time - start_time # assuming time in seconds + cost_usd_dollar = model_run_time * 0.0032 + prompt_tokens_cost_usd_dollar = cost_usd_dollar / 2 + completion_tokens_cost_usd_dollar = cost_usd_dollar / 2 + else: + # calculate average input cost + input_cost_sum = 0 + output_cost_sum = 0 + for model in model_cost: + input_cost_sum += model_cost[model]["input_cost_per_token"] + output_cost_sum += model_cost[model]["output_cost_per_token"] + avg_input_cost = input_cost_sum / len(model_cost.keys()) + avg_output_cost = output_cost_sum / len(model_cost.keys()) + prompt_tokens_cost_usd_dollar = model_cost[model]["input_cost_per_token"] * response_obj["usage"]["prompt_tokens"] + completion_tokens_cost_usd_dollar = model_cost[model]["output_cost_per_token"] * response_obj["usage"]["completion_tokens"] + return prompt_tokens_cost_usd_dollar, completion_tokens_cost_usd_dollar + + def log_event(self, model, messages, end_user, response_obj, start_time, end_time, print_verbose): + try: + print_verbose(f"Supabase Logging - Enters logging function for model {model}, response_obj: {response_obj}") + + prompt_tokens_cost_usd_dollar, completion_tokens_cost_usd_dollar = self.price_calculator(model, response_obj, start_time, end_time) + total_cost = prompt_tokens_cost_usd_dollar + completion_tokens_cost_usd_dollar + + response_time = (end_time-start_time).total_seconds() + if "choices" in response_obj: + supabase_data_obj = { + "response_time": response_time, + "model": response_obj["model"], + "total_cost": total_cost, + "messages": messages, + "response": response_obj['choices'][0]['message']['content'], + "end_user": end_user + } + print_verbose(f"Supabase Logging - final data object: {supabase_data_obj}") + data, count = self.supabase_client.table(self.supabase_table_name).insert(supabase_data_obj).execute() + elif "error" in response_obj: + supabase_data_obj = { + "response_time": response_time, + "model": response_obj["model"], + "total_cost": total_cost, + "messages": messages, + "error": response_obj['error'], + "end_user": end_user + } + print_verbose(f"Supabase Logging - final data object: {supabase_data_obj}") + data, count = self.supabase_client.table(self.supabase_table_name).insert(supabase_data_obj).execute() + + except: + # traceback.print_exc() + print_verbose(f"Supabase Logging Error - {traceback.format_exc()}") + pass diff --git a/litellm/tests/test_berrispend_integration.py b/litellm/tests/test_berrispend_integration.py index c93b4de5c..122c9201d 100644 --- a/litellm/tests/test_berrispend_integration.py +++ b/litellm/tests/test_berrispend_integration.py @@ -1,6 +1,6 @@ #### What this tests #### # This tests if logging to the helicone integration actually works - +# pytest mistakes intentional bad calls as failed tests -> [TODO] fix this # import sys, os # import traceback # import pytest diff --git a/litellm/tests/test_supabase_integration.py b/litellm/tests/test_supabase_integration.py new file mode 100644 index 000000000..7923b967d --- /dev/null +++ b/litellm/tests/test_supabase_integration.py @@ -0,0 +1,27 @@ +#### What this tests #### +# This tests if logging to the helicone integration actually works +# pytest mistakes intentional bad calls as failed tests -> [TODO] fix this +import sys, os +import traceback +import pytest + +sys.path.insert(0, os.path.abspath('../..')) # Adds the parent directory to the system path +import litellm +from litellm import embedding, completion + +litellm.success_callback = ["supabase"] +litellm.failure_callback = ["supabase"] + +litellm.modify_integration("supabase",{"table_name": "litellm_logs"}) + +litellm.set_verbose = True + +user_message = "Hello, how are you?" +messages = [{ "content": user_message,"role": "user"}] + + +#openai call +response = completion(model="gpt-3.5-turbo", messages=[{"role": "user", "content": "Hi 👋 - i'm openai"}]) + +#bad request call +response = completion(model="chatgpt-test", messages=[{"role": "user", "content": "Hi 👋 - i'm a bad request"}]) \ No newline at end of file diff --git a/litellm/utils.py b/litellm/utils.py index 076378b1d..6e1026c1f 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -6,6 +6,10 @@ import datetime, time from anthropic import Anthropic import tiktoken encoding = tiktoken.get_encoding("cl100k_base") +from .integrations.helicone import HeliconeLogger +from .integrations.aispend import AISpendLogger +from .integrations.berrispend import BerriSpendLogger +from .integrations.supabase import Supabase from openai.error import AuthenticationError, InvalidRequestError, RateLimitError, ServiceUnavailableError, OpenAIError ####### ENVIRONMENT VARIABLES ################### dotenv.load_dotenv() # Loading env variables using dotenv @@ -18,6 +22,7 @@ alerts_channel = None heliconeLogger = None aispendLogger = None berrispendLogger = None +supabaseClient = None callback_list = [] user_logger_fn = None additional_details = {} @@ -160,7 +165,7 @@ def get_optional_params( return optional_params def set_callbacks(callback_list): - global sentry_sdk_instance, capture_exception, add_breadcrumb, posthog, slack_app, alerts_channel, heliconeLogger, aispendLogger, berrispendLogger + global sentry_sdk_instance, capture_exception, add_breadcrumb, posthog, slack_app, alerts_channel, heliconeLogger, aispendLogger, berrispendLogger, supabaseClient try: for callback in callback_list: if callback == "sentry": @@ -199,16 +204,15 @@ def set_callbacks(callback_list): alerts_channel = os.environ["SLACK_API_CHANNEL"] print_verbose(f"Initialized Slack App: {slack_app}") elif callback == "helicone": - from .integrations.helicone import HeliconeLogger heliconeLogger = HeliconeLogger() elif callback == "aispend": - from .integrations.aispend import AISpendLogger aispendLogger = AISpendLogger() elif callback == "berrispend": - from .integrations.berrispend import BerriSpendLogger berrispendLogger = BerriSpendLogger() - except: - pass + elif callback == "supabase": + supabaseClient = Supabase() + except Exception as e: + raise e def handle_failure(exception, traceback_exception, start_time, end_time, args, kwargs): @@ -287,6 +291,22 @@ def handle_failure(exception, traceback_exception, start_time, end_time, args, k } } aispendLogger.log_event(model=model, response_obj=result, start_time=start_time, end_time=end_time, print_verbose=print_verbose) + elif callback == "supabase": + print_verbose("reaches supabase for logging!") + model = args[0] if len(args) > 0 else kwargs["model"] + messages = args[1] if len(args) > 1 else kwargs["messages"] + result = { + "model": model, + "created": time.time(), + "error": traceback_exception, + "usage": { + "prompt_tokens": prompt_token_calculator(model, messages=messages), + "completion_tokens": 0 + } + } + print(f"litellm._thread_context: {litellm._thread_context}") + supabaseClient.log_event(model=model, messages=messages, end_user=litellm._thread_context.user, response_obj=result, start_time=start_time, end_time=end_time, print_verbose=print_verbose) + except: print_verbose(f"Error Occurred while logging failure: {traceback.format_exc()}") pass @@ -354,6 +374,12 @@ def handle_success(args, kwargs, result, start_time, end_time): model = args[0] if len(args) > 0 else kwargs["model"] messages = args[1] if len(args) > 1 else kwargs["messages"] berrispendLogger.log_event(model=model, messages=messages, response_obj=result, start_time=start_time, end_time=end_time, print_verbose=print_verbose) + elif callback == "supabase": + print_verbose("reaches supabase for logging!") + model = args[0] if len(args) > 0 else kwargs["model"] + messages = args[1] if len(args) > 1 else kwargs["messages"] + print(f"litellm._thread_context: {litellm._thread_context}") + supabaseClient.log_event(model=model, messages=messages, end_user=litellm._thread_context.user, response_obj=result, start_time=start_time, end_time=end_time, print_verbose=print_verbose) except Exception as e: ## LOGGING logging(logger_fn=user_logger_fn, exception=e) @@ -369,6 +395,12 @@ def handle_success(args, kwargs, result, start_time, end_time): print_verbose(f"[Non-Blocking] Success Callback Error - {traceback.format_exc()}") pass +# integration helper function +def modify_integration(integration_name, integration_params): + global supabaseClient + if integration_name == "supabase": + if "table_name" in integration_params: + Supabase.supabase_table_name = integration_params["table_name"] def exception_type(model, original_exception): global user_logger_fn diff --git a/mkdocs.yml b/mkdocs.yml index 2091774ae..7498ecc38 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -14,6 +14,7 @@ nav: - Quick Start: advanced.md - Output Integrations: client_integrations.md - Helicone Tutorial: helicone_integration.md + - Supabase Tutorial: supabase_integration.md - BerriSpend Tutorial: berrispend_integration.md - 💡 Support: - Troubleshooting & Help: troubleshoot.md diff --git a/pyproject.toml b/pyproject.toml index dda6e0581..4369eb619 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "litellm" -version = "0.1.345" +version = "0.1.346" description = "Library to easily interface with LLM API providers" authors = ["BerriAI"] license = "MIT License"