From bc767cc42a9c1d5205751bf6c43fef01155eee66 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Sat, 12 Aug 2023 16:34:32 -0700 Subject: [PATCH] adding anthropic llm class - handles sync + stream --- .DS_Store | Bin 8196 -> 6148 bytes litellm/__init__.py | 2 +- litellm/__pycache__/__init__.cpython-311.pyc | Bin 3847 -> 3888 bytes litellm/__pycache__/main.cpython-311.pyc | Bin 22254 -> 21031 bytes litellm/__pycache__/timeout.cpython-311.pyc | Bin 5193 -> 5193 bytes litellm/__pycache__/utils.cpython-311.pyc | Bin 38107 -> 39112 bytes litellm/llms/__init__.py | 1 + litellm/llms/anthropic.py | 97 +++++++++++ litellm/main.py | 104 ++++------- litellm/tests/test_logging.py | 39 +++-- litellm/tests/test_streaming.py | 28 +++ litellm/utils.py | 47 ++++- old-docs/advanced.md | 29 ---- old-docs/berrispend_integration.md | 34 ---- old-docs/client_integrations.md | 12 -- old-docs/contact.md | 6 - old-docs/contributing.md | 34 ---- old-docs/helicone_integration.md | 55 ------ old-docs/index.md | 43 ----- old-docs/input.md | 172 ------------------- old-docs/output.md | 12 -- old-docs/secret.md | 33 ---- old-docs/stream.md | 33 ---- old-docs/supported.md | 72 -------- old-docs/supported_embedding.md | 5 - old-docs/token_usage.md | 45 ----- old-docs/troubleshoot.md | 9 - 27 files changed, 219 insertions(+), 693 deletions(-) create mode 100644 litellm/llms/__init__.py create mode 100644 litellm/llms/anthropic.py create mode 100644 litellm/tests/test_streaming.py delete mode 100644 old-docs/advanced.md delete mode 100644 old-docs/berrispend_integration.md delete mode 100644 old-docs/client_integrations.md delete mode 100644 old-docs/contact.md delete mode 100644 old-docs/contributing.md delete mode 100644 old-docs/helicone_integration.md delete mode 100644 old-docs/index.md delete mode 100644 old-docs/input.md delete mode 100644 old-docs/output.md delete mode 100644 old-docs/secret.md delete mode 100644 old-docs/stream.md delete mode 100644 old-docs/supported.md delete mode 100644 old-docs/supported_embedding.md delete mode 100644 old-docs/token_usage.md delete mode 100644 old-docs/troubleshoot.md diff --git a/.DS_Store b/.DS_Store index 5819da711ed4e7e5053d6fb809bdcddfa3f56a49..d1cbcc87c8e898f83b116798f666d20bbac51e8e 100644 GIT binary patch delta 186 zcmZp1XfcprU|?W$DortDU=RQ@Ie-{Mvv5r;6q~50C<+o_1dI7G1T(mL=Hw?QAQ0dN60RWYHx_z#C$I7@um(b)xlSc{JG{QOftxZ`eX)`GTzDFs=z}xsevj-*CU5Pah)(92@eF3%y zzP*km{F|~U-_xvoa$@YYEg*Z4n9?>vu$LimN=Yd#i=8p^cdMgT8YZp|xxHxOrz)#U RU3p{)fvRxlDL=Iy)o(I`g((04 diff --git a/litellm/__init__.py b/litellm/__init__.py index 93d74ab91..4d118bd9a 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -113,7 +113,7 @@ open_ai_embedding_models = [ ] from .timeout import timeout -from .utils import client, logging, exception_type, get_optional_params, modify_integration, token_counter, cost_per_token, completion_cost, load_test_model +from .utils import client, logging, exception_type, get_optional_params, modify_integration, token_counter, cost_per_token, completion_cost, load_test_model, get_litellm_params from .main import * # Import all the symbols from main.py from .integrations import * from openai.error import AuthenticationError, InvalidRequestError, RateLimitError, ServiceUnavailableError, OpenAIError \ No newline at end of file diff --git a/litellm/__pycache__/__init__.cpython-311.pyc b/litellm/__pycache__/__init__.cpython-311.pyc index cd671c282785c606b05934d159f31165a41dc887..84bb43b8c2b03a60ef8cc3046cc1ec803c8ed0cb 100644 GIT binary patch delta 337 zcmZpd+aSlgoR^o20SKzuZ>022Zr!wK%#(ZV9EQmc-{|mZaw7dB8)(UF^G@>5fUIm3`qRqu*uC&Da}c> iD~bSe8G*RiZ}VR6TBiC!rVl`Y0V8>kt)IM;&kq2R9#SU& delta 296 zcmdlW*DlAqoR^o20SFwgUrVW+$ScXnvQhmiqYfiOib{%lj(!M)HUQELDH=J3xkgb& zxyDh(xh7F2xu#L3xn@yjn{}8N*%&z|>v42XKFg88XujEu(}8JnHMg>j9?;|>eGp*) zA`C%<5r~ih5fUIm3`qRqu*uC&Da}c>D+&j48G*RiY4a`aTBiC~rVl`Y0V5|r;`0Ll DUx`Fg diff --git a/litellm/__pycache__/main.cpython-311.pyc b/litellm/__pycache__/main.cpython-311.pyc index 1884f5ce9e4485f93a3161c20d641cef69ed81f8..c0ed4c412b3ab41afdd4de74d27497ccf3fdaf60 100644 GIT binary patch delta 8764 zcmc&(Yj7Lab>3YN06`EW_yF-B!1oJ$i+b=OlA=Ue4_Z%BbjF<65XcLXpn(N-v6LnG zu0XkJV%cdT&rPOz$QdRRGuDV6J5hgl(sZn5+!>E&(g9Uw%3$KD+m7pY+O*WfnsL(B z({mSCfD2Kw$Y_i8{@ctWEbjD zm}>lH9WfNs#`-DC0m~e!h6He$iqTG-xTs~2*@bDq`gd2EG zAy2qsqJrlHF5sT4R3)RlWuD`1z%O}BRKd8B?yYgeV7eLQ!&D0@2=b`#nr?;<^8OlR zz06J21`2_(7!_%!&v2+16kvZPhzE{3RI23wd0dcj@zWN@gNoqC&OEYRGorG~x@FMh z2FKA?`7&cYvcqBx^k@8DOCxeYy9t#8^-`C z^t-}g+rO~8^74uNy`nUWBlE$jBS((X=Y-mu9|_;!_4P37p_g(Zc?s@7LKhULuN!HB zzOlpeCp!3KjloE1dGrPTbUr2(O>8r zV|IzLr4Awx9myejVjxD6OH3q>m`OgdkOIPy(iMIgfW4uD%gGMOq|W3xQb??%h!m3& zQc7$q!ZOIck(MnxNh#7jBQXA}Ja!?Sq_|HkDZpn?1xW>Ji{s{U)%K;5p4b(`&h=X51j9rGTZ= zpw3ysujF*c%%Gbu+wU*m4_=saSmk_}{r;Ia$9``RZjGQ5{4IS$n|G?l{ahNRwELmLB^%`} zr3RL>2v7gRQQ6|o+T-r~?{Nmc3%6~SJy)hGvt$LyHa$n&$t~TwQnU=2;ReeN$|1YN z>IY4Y!PJ%FWeDLnlrYXF$u8nTCC;P(pr|y{mOb}qUJ5D219?+`7*&=XqlkytHL~m( zS;*n!QmFzKarVPfu&^t`08?E`F9)si)VNzS?pAv35Wh@fA@UF|6@)zLAoNqIBYh_v z@@RJ^ALEc6aUq9Xp^4KKTDEArX0%c}iYp&9Dp!(9sVk$x9OUfefhzjh{fph;-Bw66 z_hkZ}dd)seCkX7^y8l74m_Mp2a5T?iHlQliF~{Agnz)0!T!m_g<1B~s0Ng56t5}(} z>R2nw1*#zE*AP>X-_ijpmfXxo!O^N#>a!fJ8Z{`ZR1gPh$l!_^ljz}MX?I#vj#LAB z_c-bphnm1)tM_wH8RZ({A~hE*%!R*_OI(|s6+#zeJ~xEeP2NgZoqKpUJz3=8YiV^+ z2k&9+mb%aKT2(3Jq}yt|v3uRLmedjWT>wVpg+Oda8AmmWhqA(IjVccej)@g=pEWBt z)~XzGy_5;9so>m{9Gqjdq;in@mF6rHf`uI8*Lr7Z2^)=(6>stS$I}8u5Q!Jm|%DH1VxNtDoGc8STs(Z6b|}0@@}w zk|sEJ$xRPYh(N612~0rSuL+tuMlZ%Pp2Vy@*R0xHsYtLU*6+}&0*{g`)VIi9h_^ao zm7B#TP4sM4WesA>)&nr5P;<9CwM)p&#JkkSwwbLtYosMp2JBL!ZS3AMfED`P%Q=va z;g>wv(=ut9Ygez865fr9GgbH_?NnE0r>24c%lEKVU1(Eoo=Ydq%|SF7L#5NIDsY{N zraga;O#fk%T5FKJqO|SB#W94ubEZaW=mhBG#i&1sYtjcD) z3k`3XKDQ@v+%}TI?!MKurb_>V;LD5sS>bu7x-+4&`A`X<~A^Fx2$2>Qp&aj%w`*^3#uD$5emW;xTN?_)B&!4X_PM zf3n}I?YBto%?F{JQeLS1A3rClYa}0&q+z(GeKl2W_m-Ye zaf#W!$}4Xcy6Kl|T4H6=B~=&{r^iVd+gMsKp66LYsO6;RGn{oCZj*bn_5jFLD}U0v za$*@?@Z3nhS{5WZ?3tEzAv-B#bhzexS{4Y3C3^6Ic6(lC!B;du!kWPO6t zhsU9f*?W&fjubd2L3a;Z zG=^fE2PX|hf7SFxQcU%BjhO43#V8Zn=-XazzFCUEOZQoLN$wBQkGx+>PBtIOOs3y! zt^)jj*8G#ySaj3an=Shr8SHoq#1f{!xv79+gzMl4w7k{B57XY(YTitrXl*FTXGS#p z=Yz~-QSd(8LciAfN-I2ia(IFb3Xy2S7!c0}aYRg5RA>XfX9DLFI=mp_0_tn)s!dq@ z>URM)gxRZo#lvs~YXyMNe{P#7!R)>h`Lhc%;B_;;Y5!D!?rX2-Wjfp5V7SP5WAy9o z*wU9l8F$fw4v&F7?BgD~t;0hnJ8CPd*d=0QnvoeGiCoo*@r(4&J3NmkZ=ockHM>*A ztCGedY*G$<>)%ludz1Xuf{+jv>CCr+>18{@l=#Gjms&xsox7vqlzALJLs8~ZlQ2D{-7 z2jpW$7LVVxa%D~HC6%vr*l$@t(o%PPXl%W#e#59MYrMk&`IwQ#V;gx~ZuvK7ZX4@Y zjrAWmD(T`58$I5It4~K^g(2av04L*!a7UpZD>T`2^LCe=FOHAT#w9s!xVUP#xMsKr zjhnyQUc*;EeFnHL@&}dahm`4u1UCIps)x4i#C`FB6Y(?80q2-7&HAVKnN6ptKon+K znJz2Sv7UMJNwM>>K$*HZ&wemRu9vmU2mIg->i(An&6)nbh}&P{m0{zVtoEW z+|QOdB%D;{k17&J6^Wy%9y;7D_kM1L=+6fzeW}M3i<_F)tD8Wpt20+;{-SZM@AzW) zeP{h&b*~MN$9V?>gd>^LIIJuW*d1b6SY7(~$0% z4o(xgDP7#&1rRup4r~Z?QCv3_UG>FHb@2aM7joAN?JpFr7gbWv?rRSL>wCLL09F9zi-EXddetz!W|)Qsts6MQzfS+M`(6Hh z>Knw+zCG9tG;hy8ncPd3+orZvQya}0nt(pv&?)})n}0U6xTU21`$mR7x1og%1J{3? zr=2~W^uPC6i{pDn<4@1T4YR9;*)_u~G^l5IsEf&;ihy&)CkdGFu%N*FwBTcQ)t_eg z0|jFOP4>{GPad=lLOyY%^S-GZL`g<*duoPODTs1#2BkUugAr(c^7_QL6| zK5wC4cek)-1*1~J5M7uI2PGv!Byv!23cCFBkkXJh4_5&nu4c>ylYVJx))x*$S(c6_%u`S! zgaRx-M(KYYXyV_Y^#|MBEZ8KhN!|MW10mxAQ zdZg>=LDr$wJ@uA2Gtdr-5~{7G|$Ol4Z+}S*wbXTR@_Wy()Dm!46O1tZp&<32vhx?| z`BT?B0&(sU{@3ZHaTopCxT$!ZYl>_C>Bx8+{m%H4`i)$!u!#O{y#DcNJ>S1kz}eva zz?;!qD1Pe6c!Z^!1Hy=~QK*b^+4PQ%6ACW7@8~%p_j1{WK^Z?`rt?o+$YI+558~~% A>;M1& literal 22254 zcmeHvYj7J^mR>g=jWqWnloTa)ZgDz&L@Zf}h0DyCSqrAj8LO6Yi| zYRmH{=X9e1G(_5x*SmIAw%I&=`}TeHx#ygF?!6!SPi!_b0f%nmztV5KNDzOEH_B0# z4SfDTNrHHnpb44`6O;0loFt`t&7=mdny@yao79o`OdHlms7Xo+lVQW8K}u(wG)h;~ zq)ED(C(Y8;GHHRUE^Lk1CT-a~C6gr(rw^AzN+(MrWs_wQ`=mYMm~=#(lg@0K^2u^Y zLxn3Mm6Meup&^3hb5%;iX~Rd*;!ogTJ|buDXx(eFSm~DqkT?e$nNxPs1 z9bJ8wROeLRO*YYPD5sgO0op>7mx*@IZ;%IL?WCw3j^2TbVLrelLV+*@jdU`$LiW7y~7bEW{JB$mIkS6uHU-=I4WqScawgr()6R(2PI8%)~{*)NCv?6^x5I zNIfs=m{>R{nga272uh45+O?u>Ius4XXZ?^h9*c_lP?Qed77dYLJRX<{PUWNm0($s= z8~#845s@viodbkKL?7<7yw5Wf?h3dO_{{f7#pW3- zQ#4P-BJ<%Oc9U4jEJXdGs6UuxKX6NYkpBUH zDPqG^CYb7YQ@voSPc{9HBFy%mxH(hTDh)(F)6?=b)wD$sW#yliHm;X8rjM^R@ufXN zX%A<5>eTdn4iylg7lCI>wwd`v*PiZuU5N$eMoc7rgPnCk)P|zd?H17rVz$6U{SyCx zjq@S@&EOr7jTkc(^efV=4=gP(AU-JeLB>BF6%C6)C@mfot+NY~D)^@ZQ;>oQ&WA%& zfkaTWBx2APvyeOx5}i|$(E5Qz{&^<07@|RFOS6$!DGL8oDYhC83;dhlUkm@Azwbp; zY*Gnak(jv?m@<1L3dBflB9b);gOZ9MK%v0Avzk99X$`9ds%3RRb*vt!o~3|NtO2Nj zH3BuVCZHzP4Ajh8fLd59P%CQ#YGX@)martyQnnIk8CwC=&Q|>~inoUa19?*t;kbcy z(M~zMLNadJnK0&C<-O={l7u~LS4WumlUe z!#W>SeFVJ!B*%Y}P_@IRIN~8mGd$MFQ;5k9U$3RnO)6zsO4D;-6n)zN5WeXnk{%`FwIr*QLu99v5C| z-(7+wjuYxpOf7@Sx*{u8_ncO$t5EXiDwO)uIV+{a;1%puovNR6*6u<5M=-X2k{e%> z-e~>vmi(YW?Z73yte0-g8)3RBA5Qw7dT$47VtsToSU~54mSP*t`q&y(S*`i9=(d7z z`w+gTm)@0+gPgmhSQ0`d+2g3`@t37( zR>^@|DUGS#KK{$f+EM?xApM(_+hiv&~4dT1)gWI~Y&#*r8-D4`J~ z0X)A?B<~O7QT{Iz^j^3&9w)9Fl1;3aZCV1IOM34aS?}}eK;lrow#4ClIPJ@a^C7lQ z%|Z2DviUhn^QrTz?~*Ogr=N+Dd^_jrm9W}QZlkr$YzqtjOOS!y2P0(-o!47qlqnkb z8kBOSaqpKZo!zf)zofJM;l@Z)vNh72Y>Tuc+ezX}wP--nG_@ABDm4~qQ7(Q5Et1M& z+kQ$U>u|idKAhY|A7FQZo7e$ZO=dbDe-|ulCHXY4e-4EOP=?A zMgC)Qez`Oa!M3XVCfV^!+Nxzu$5px6p=;xppOQkq=(ScG3rP zo>SLzX>AuvZl|?u*W50pw&>wjmU_u?{>yv8V=V;N>9=1EBp@);*Y!6(IC3~K27Q{k&dgQ)_R<~rP zNFsZb{*Xq6h-y?gu5L-zliQu#4I_b;Elc)B+f?3BuTmCc+ddw0l&Ux}s;(i~%kED1 zDWw!%+1{d*eLH$tC!WR2Dv}Mbwcs6{{0P?2KFO`4;iduGH@8P=hqS&?M>pixB%aH% zdUvp_Ub?_{pWCa{_Pkd1IsKRazq}!B^nbAxik~X26`jgi-UHJ7WM6U*eVQe497*m~ z$>^CP$=gStrO&bZFeUiu`*)PX^WU5t(qqz`pr757-ml0<;g#LX?q~ZSTqvHiZX@L; z)|;2H9cP!`biv#sUz6JFDU@6KV&3lp$^9qKd}q&G6aA2_Qput{CyOsDvZ#{47m6fs zK$5_LoCID<9@tR|FMo4VxTEwPz;zUQoSs0R^Fxx||KLiIBp%#W5)VFuBpMV>70oUW z()Mlr)Pv6?xmWYjy5lN}dUfWXBzk9wCZj_|Ud4c7Ghoj4dzDY|qFR=-KE;9LAV~hT z@WIGnawsyCJOs1N!E^!7mGWS9qi}^XH|tl$9#K+1m{iLM)V9;wg^&y|p{>L*4>%YBctQZSn1iAoa0vc+k&KE|TmmmJxJx6NBZ1rgMC@h|vX{*>!FVtVJJ0jM zXds-pBig5#;6Gb{s5`kRGwec#68;;Z0Dv17U|U|a(Xi_ryMsIN{tzwdBQZJ{W^g9O z;6#%_-+r9IdAnFLJUMY`)IWUjwEx)fGJ6dW1=k|b$0YxyPiQcLU_bH z;5teJG#$cs1yl#Z1*B3LoYn)Pf(k|f6%rJUSqufBFlGUl0lyXR%i-xo~lGZ1}X)Y;2P@Um9}~izIzhpi;R5 zU3c8x`5b{h?-Dbx-EiHUBw6Ubyutsm=APv;@iBm105_5V-hE7btPwTcyBUNP#c{dP z=VNdc?hEr!7+@oaWH`K3n+G&>4~1jU&+)?$P@dm~^q+fyqzKM+Ts|KfR;$w0LUl(j zhG~Qp+1C&$KmB7a+BZ_P?D0hm0Vw=VCyn;hL{&)bfI7{VB;FeEP)Ld5{^oeGadjtrL47krd?(UQgEARK3@Q+aJ*S{DF*;AOzLb{8Z83)3U@LcB*bo;u-|mB2Q< zkeCHkTMD45TQVFA27obN+s6ETHQ^=Ii;I+GiqWP@wof;Q@HGH_MRBAF(tb1;qJ8|r z$Ha|$K^j1p^1>ERX&`I65?=1D?So3Y^Iu8H&u3d741JRiUuu)x7-{iYarC5O7 zf}t^4*Fk_S5mF*KqTZLTr~F}OU- zoW^(~g4tv!Xx7xoQHXD2V22MB6jMOI#DTvI?s@eKZjq7) zFR>P_AwcT2Gg2EI%%x)@?W)oeMCpH!}XIvfimL@fX-8AKO|S{R!|BUqXkjTl+*iXaq(fgG^0 zVp(p8$+noefwfE_p^*f!2wOfCvPByEE%}#?Gc}mXIu%<0moMu)!eAGII|Z0%&kq|4 zZ%PWHB$T6U-<5_$uuQ;;qRp4yj2e;QzDC8iFjp{%F{^jre$L0hl|{W1ZL+E|a`O^7AgQWQ-r&E6{tDV9!yujGf~5)2?r!Iv=u zoR@*_1Zfbh;5*a)tn(?FuLQyi!BK_*jIn7{!6`EcHf$f3JW*LOa9x+d!$*bg}OZ;aJ;w^4$Gq}ac;G~qnc_-6~1SOfljkvr=%yiGFx`^m^Uz!Y zyn;k6aI%*uN#YDXmBh?4fyySCBmjfo0fnKN+n5a0gq{bzL;VTH85A2PFBE1;$t4zvI;o4vV0;+TASn6o zbx3}_^qLZBhBD)Czy`oQp@^aLBsX|Yxr5(ekleK?<@P2SBe}V#a{GPqoMww4q>Gf@OxPUD zqgxg%%1qR@u3l5Z`;U&aA-b9JoUT6gW|UHdm( zbw8NC8{6pIBXkaNou@u92+&Md+h5Ib*FyZ*oG>;gJ?}dp^o??TFMQB1)O2$--G6g6 zz};BjuPzE#7k@An(A1KdlITSlU^cFUyPF1y}v{jP+7e9kt(*|SCH%FUUwicE!fOAkTm z9y-D1*)k}%%f>g1z)W-LPg(`X0B4fUp$pvQ8|y*EFzdTs3lsuVIQz4X_kAB0=E2TLfv?vr*eD)b_na3C5a?%%=f0pCFmm zFd*@%E5|3UWj|j7{O&Jg)ZWJn+=~H@y1`R71nLF^cHmR?eK*Q$-noE$f|o{O#Fqv> z$+zDGpLo+imN!pH5jPepJ{>UjJ1o44h;_oD&0@~-0o(9%(%bTl=+>pl(7S%Q7>Fma|%LHht>sSa)`yfHN37& z2wmkVZ^qZQRwej)m0NpVYWB6+jH!H8&zZb%rmNTJ_4+;z&i$u<{nGmW7y11!KA~Kz zH9XZUP|Y0GoS`Z?<%Az^l(;xo7jNqpY~5fY4I|{A9pnzb!0)*v?71XIY}B@K?L&O+ zA))pV2U6a7~y71qf`Gy53H{namh%nGz>-ZJGBoLQ_`pTO_Gz zFc)UWik~wz!MRrV$R)HNdrYslpW)ii0E^07(~E1b^3FlQIk;?urY)Cd>`pKb4i~;% zwM3SiGRCr%M%#`cp``GC?>;XrlU`mgI3R_vDjCxauyGq8C&_vm3#g)wQg(SY%Lk9W7We|4yJFd z-4R+3KnPCWI=Fmd`NSqXtL7>P(kItW3r+hW1SfADke->#R#>k5K>F(1HKDm5LU8it z1IsPTEg8zXa$u!|r)r@gj+&MBA9uXdkqSL6n025I9yp)WwFq^+7?4K5%C&{ZESL!H z#4B7FgmICl76ob%0-MgBl>^E!riWn+ERVA~Q@;<4I*w#WvV&jtHZ`|p-C0(jI9t+7 z5BKp-DD*&9Qa3#y@2rSNO`kLgn66B;#)U)V+J%y_@Sj&bvni z_b3S96Uxbz53U^s0w+%m3Dgir4P|yRcHe+pZki;y?BGn&*>mi1c)jNm-*YKDcs(h0 zfw-}7;o!ByAS4gxL+atY9;J9l8qRwQhVz4b&7e>-SU8;T?Dtd``904wp4YXndHLFY zp|&4vF_`9pyG=_iXKBg!_Ht&g5^jgEG!V+Zk}2maQABNT#sxmF)&g!85)(Y!B9N4! zBVku=CGV*`>pit{CeYhf)O{CEHNddNQ4JZ&nTp_F)(hy%RCt!n%VtTnCI6>yqrxLp zv|{9wiaO4>o3H23^`~YwzhYS5q4U7!Pgio4Z6HAQ>UX{W!n=Cwq2W zT+b1~a}@gXyVoD(tGacQhlA6%=V5Z)cbxYf&-KTKyNPQa;N1rV_W`iBQYReYIuCIT zNAN5=nwrVj_d)M`MOT1dV<>_78IDdWP0Coy-#-85`PDX@jFr1H?!7P??ZVLrNOp<= zAuC~bZP^H$`!m8|GyWDyYU+26T%l4=-4PbQ$DQ~-oX{=jG!s%^T#B`gxt=XvKr!Fe#Jesw=P_5P{%&fGtfzAbomKdcr!gO8vS4?kKIJSV=B zeJBk`Xe{AGW9jCpCV>K{uqoG*8R`XYyE8+T{4nx*P!k+A z0j8|X`7BeT;$Gc!dOme_t~)!|CV1x_!MO)K33UEHwKc5U8hBfiU~5Xz8Bf>halzBI zHqCiX!TG3Ez5>6JZ|Txb3AoUpw2;H z!{&VB#d|NN=nZQnSM@S?WfE!C%e?ivV7<;!*E1Gp>cZb`m|&Ko(+=gy6P;zHdG)r+ zIRhy@rWrw*)E`GF1qwC{?xTWarF-ym?SC4|2M}zk_4T1O>x2z899HUY}@P zauR>D>!cel8~QVyaQW|jXIkL$f2&U(fXf$7^Q&g<7v8p4_1fR-^$^B|z1)H8>9{8S z`G0{O1@%V@SVE0Lp~;&%%*Pm6_+5B@n>62W> zeMC~nYT$PqY*Jr{_t>!|ZH2#AFHV)DSooDXP2yrK;9_)hQtPp@L=G)Z;D;6{{LsRW zh@>I9D}g%@+2bh`juFZ;(weU*kf8+jB_oFj>?@6KeWU4z;=16sH7^-xRj?JpM-Yz3P=0-t;beH(Z|8@%PO4&C9k-P3y94B>?LNFqE&j-l_Z0&Dn?6B0%JG znOdfvxNBDz-|M{J2`djReaoe*X5QFXkp9FP&Dl@L=fjwMU8XXW{Zq=d4x>*cPx%DO z$4Tdw6WgVQW=d_9$Y_jkfBM8{V=*+gz2?R>WNJqK4U050dikR1t{2`EbyIj!^F#;V4c}Mn53`x7c=mx+} zvr-T97C!koMrvV~oLR<@3%5@ZzL0BO*$Gz6a5s|0)f zr}nmWd)wN=qY2L5#@j~(`$%ebLt4=LFupeM^W-m*4=+Ew@$fRPt*Y1eaEVV|hU?=@ zcxKn$GOj(UrM9YiuAz^w+9On9{4+LZRBFyK!G7#HnzO^3swWI*9v|nb&cMlA&kEMF z9CdaxKVQ&maR)L>^LoetT-)jL0`UpqVhyF@fX$R zlQ!*NG&Y|!YX8!xhwxKgAR5ol|0NJ`{edb+{(EK2LDldDgTm7$0btPu4(=S5DIIF# zLThe+`1A20w=F)6Lf8y zu5H6$yPJIN`***;+PH4;at7}PW#O!cdFqHj9pRL7%Ya451#L>nBoB8A`K25<)}xmz zePV8>?xKn8_a0gcnY?}1sz<^kS=foR27Wg{`~s!~IET4R6#kNwF8X!C>twrjZ0QxS zqw?qV0B)9;^#P+heSl*HSQmzU0oBumINI4)6tHRbyL?i{L%vqL_9S9Ga&$Ms7h7ME zQD*vVKNjR*%77~}QUk3cSa-?42Q*afV}FJ75fMPX(t|ukhWT5__j~NuQxH)4b@lkV z-pA>E8;*t#`-R5+yrW-m^ry}~u~cxDZa7ydAn6YIhSPOFF_&`{w|Mhy!F-$3-IgaQ zt;{>vJRE_*%HN5L6oB90@Nf0V~271Lwpn+1EBe$Xae-oRPaV%>L&BwFgyBD z%zezE!!IpL5Iczz424fn{iFebF<}VjAEGf4x+wv_L@85BGJ?3{lOg z&kRwLlKwKp3*57v8KN{L{bh(=PJL!9zU2YI;!AsZOOIgbNf|SQpZltFlW?RcfpG9d zxj>Y2MER!CzUghuR8(%3RjyjLv>I0lKz(qF1&P)OyDM6AnK5PE@)$?}z2u04&=@yK zW2$P|`I_ghXS2Le-*T0pH7 Mc?V*)r;>jDKiiPnNdN!< diff --git a/litellm/__pycache__/timeout.cpython-311.pyc b/litellm/__pycache__/timeout.cpython-311.pyc index 14a356abc300ab281fe54fc47cfb5e804879a83a..09f976993936f4b5c3eb904c22c7dadb0be4e9a9 100644 GIT binary patch delta 19 ZcmX@9aZ-b8IWI340}w>q+sI`v0suDb1pEL1 delta 19 ZcmX@9aZ-b8IWI340}!;{-NMdQ#4;jlgHa3=#@e4Z;FJQn>?2r`mETpuFDm2$vSeDG(Yh$}I zCIk{5$-*+}2TikiwAm(XHf{LSeqZ`b`bWq%U(?!@ChG0BB}w~rY16E;X?DA9)9g8O zZDVM9?fbiP9`iag=gb-X?1JUrFIwuJ^7-5nJm*4rI&hyPy@kZ=qs$cE8@X6FMq-5e zk`g79^g5{jDA{uhRgOw~meT;>3fe#mfOjixq($&;cS#@hN1JE}r2nu_qJ`8Z{(O{J zBy}Y%;a5r7L}9N&7l6hp+DuE;RvL!44Bm2hE7a9=p}M-)ODm7cdpc+p^t*;G0$fY0 z0oTzQz)rdta6PRB+(27s1X}wTtpjQ!T>{u;k-8-sJu1=qFIyz(8F;ur4C-<*A_XbQ>ft(W}f}_*%H)n@r(X4 ztAo1vkNjmp2lb$K`Z)DwrD2KH@E4Rueo_g5hKH|Gw(z%S%KxlvX;i*Uzzp!3540v~ z>lL#cNuq(Q`~`^y^-5mZ73OV$Vp(Q)nv|Q};gT#;R?a&3fk2(&)NAHdEZ|QCI(*JU zPAct?s5A~G{%)Z5qqI71&^kyCN+Z@!N(V`_u*Z-GGE_|&a(rxvsYYR9NYgU|u~cdx zHpnu=Nvg8k$>3fRXoap>8^SFJ%lTgl2I^d_0f`obCWJ>14Qh%F&V| zMf)wQYr0ycmm;2xZPH6^?yjZQmpW>?BGyY)HsD{1$iTU@)QaU+cXz$@((0P-TI!#!p$G9i!ftX_Hi8>4~Q}rR1j>Wanbb>918iubo!xCytPY$S=A)R$2&oQjBy%|jv zJJwKUc6@v=IT70%2Q~wxsF$6Ck7$IQ$CB9DSf70!!0`4BiLDFkl~CCVWO?+=K2`67 zJs(e+-taK8{i!7E5cH=wiw|g;7nrkdfM3%U3ZUw)*o~^}QBeIcyc%{6Zc#3k*5(Vt z`O-+fcp(-lTk~c0(~2`>pOyd)_e>W_-teTYlr4Z7hBcY)O(>j<7V!`2T~;OQ;|Iz@ zymG0hM45@Ib}IE_D`A1nF_3_6VO3Znfb*u{Y^A8(}8x^XE)(H*F)g@3j`1ktkn6-m# z-dt@^qd?TS(H=KhlNRzV4UH4C<8{s(uM=7*5-oh>EVAz8^>uNBZn4ny%p0`_xFNyy z&Rat-a7zRi?GJBs0bkqH7Jdx(gL@!8GVA2HW!WPD{NASRg*yGgNgVcL^E|>tly1#mk}^1 zFazPIym*~nB1Y_Bn}K6E(F}M>uvhux>q_c{?oUw{2ckIuX2Oq%JZu+=5bn;svF+)B}{xQ%na##>$YeOaxe(bOg^)B|8=9V z5^UxT{>Zjy@xt{=01(!=g#2nx?TKjLh>C`R&`=V7(4)5BG|w zjOTK65pzOQhIKGAXm%LHu67`-1Tb7G>=Tv6b=9zHuoeCjB^pWsiN3fV8_a0>W>wST zkhxq#*>4ak5iaw;ZV!`7ymUv!#M@W{6P%bjj9cn=s8EPZC8qZ=HBMt~xf*rxd-%PYj;9@X0f>?*`s3>~&BhQPVAOsNR)`Izv z6y_J323L;#7QpbJ$+e+G0+z@$dI759atQOYLL}!*K5>(guV-6g2_U+eDHc{oOU^U= zd~<`K6x&^Z5Jo6NC_-q!YQjdsxdK4WI2U|)K=r!&Py^wETtwYvFZ?DHd(0fe!oG#V zt^9O<7o=0K^{<>5LdGz{L4c^+uw(dgt4=>@gOlQAw%W4{n-qRG#QoYA z$%-#s)JT03s6hrO_(DTCWzcXZwPae;<7r5h6s+mxKhtY!ZpY%C0EVM4uElk5NU`9} zI0g?2xAH|pg=`jYZY}{4+zp}?Y*Y~iO9~o~TYx!5l zmylIFKfcAi5(kKIHn;51<5thSfY1Nqo;vp@P>%3C_Z>My8u+6}>hP)zS5-K>9sK)8 zeo}y^G>ls8HQ`9>`E&QylDqh8_jZt1c+t@Zd^@lihI)kE&7VEGf?VOhI=aX$0}Io; z_rA-OV@>>tu~NS4Sir_wb(zQgMXoGlSyVb^Ddkg<68pF{Ykl0tzt>(oVWaYmcX+sC z$}l2(&!6NWy|=M_Or~)?zVaIOz&OGgfT(QPshWT*rQwDJXS#MU2@$aqhG)o-Qcz7C zbYR-M1uM}vlf!jbL3_BOW`Z1=xgW*T&|YRRnTUzHWA*&> zePt6jabJf-LPO_1EaiitbLD5t&+nTGMy>=Shx_ub;OnmP%dYbCyQf^WS6sEzlEty? z=b`XJ`yXHM^rEM$pRRtgd@9s*CDe30nh%E0Rh+FjpM84!RIueruw~NSlJ_acw~GWj z0o{m)J4b!of9#D_ECJIRb5`r>h+e(2ddyNie(gheOygLYinF%$Ceu`SZPYI8!1{pA zm)&=g{DOb`zH{UqzVrTDNF)E{@hFk_bCn^k^apa^x&O1I7>&%HN5DWLE*|U=zWGEI z?F8DerJ-YX7c|HE5i|h98H*u178AK0!H4X%Cw4jYEapM}W zQ9gBIk>eT~*FpIIooG|)UypQNj�QI~BR@O5`@)a`J~(C%cuCQwwj6;Q-sCry9tf+%u<|2%MB9~wY z6N}C4>FQyh194YR7d$;ZQMm`7s1;!oi+~8Wdj`_VVbQ&Lz=*)I7+V$2E@OA-AQ1$l1}ygu(Q z%m<3|3(E6F3#Lm-2>h6IBee57cw0rde0tHR2`QbH{syxa{se!2Nh@iemi`vrMcyB- L3)1|%4^{j>VDw!L delta 4865 zcmb7{32Wgl5$e` z3wdZhD9xd@v_Ng5h42=^TMTcBI+spR=Y}$A=^o3*Ryq-k&!c6)`LrCkfK~t((n-KJ z>IJsbI$8)>eCO(nGT}x zC!D!UPMRd?uw;|M(hemlF-uZPDwMQBy`&5|8FI47Ss-VDoJr&e@NgsYQ(dA)1mr5al4B^#1NT}ksj5_M~3{9Lk_U+2!V zm|3@xv$Lygk}M_7Nrm6zu69{9@3@K{{;GSP)4J12r7np|J0QhNJd^*MR?RoG2Fakb z&9p%pB);sGx}`rx)rf8h3?-PF{;g*-aW_IMtO?PKn9k?s^iRrQwU}u@)FGw;bXz3W z)1xwfD9Y;jD>)_R43P3qatgASLt+uU@jGFjZ;~#^)x0owTKc-&UU~L}B~zsr%N!lG z(u=kBC6%TZXL^^EnT{8kL4Lf<0+QpECQMJYFZG&^&+smtXgV>`vecWtn14(zS^{Q= zT-U6OMeEulv0z^~+H+mG&&g)SMm5GD6H8mYBgC74BpXC*L)-|^GlQ{2RP$@GJ~bML!5^z^aDP*UCHOxpD=jI| zekffr`BoW5`&jjdmSGS)nf_7DF|uL@Bz>a}*$Zj54+S#AIt=xER^bv|8DO#g0Zq3D z;U}VM;aHT7RbV%x`hNbvlqw5{)Q=(Sg6pn_19)75{lz8}Kwce{dqkC7OQXlMW zr~`X8=^5R_GBu4!Wsr*9n>3A+nm}r7U=N>AH@V=Fy(?<+?z#q>hg!u}hxY68R$bEc zbV-LuG(YL48GRxEv9_cUJ3Q1T^slh>UA^EN>*~5LwPB<7v7XzYj}8$yUP(G>=4dMo z5N`gb(V^XxhdPCYyOUDSWlu?9IZO1BP}f*{1f=f&r{_Y~Jgzrpu;3BWQBM?p$24V+ zcFu5Y#<)RefQcMo;=j5gX%Al>h8=Wsg|1`VFdQJw6Vl9aiz5@H`9ca4mxR|>z~66Z z_S^w$N4NI}wv8_THN1J+YBIw2O$=kK~;BXu^v_Hg$Ni3>#o=UYztR35b+NL zSfD>1&xRU&cx?fvjrpEOAoD7`@t2K=Si=uAK4)_uT724Cc*-R?@{eIYhTfD zs(vlpuf`Hudgt`xqyWcqRMyQb7EyI;Fc#G?#N5EInYo_K;^~=-9FKwq`x)XfPFiM> zdOoM6X)T64h^@i_mo>0O_H&^N(CwNU*Nn#5F_dmWM6k%o)c8Ow8aKMbmSMru{5LJN zmKPw|nEtfIWg{oJ+%~^`9+u5VECA@{a5Sv3Bbf9A0|OdFSiib0sG^gjY#r*IMEoA{ zI^s3Ht4%8qJ+`u|L851%LpBG3ee48xwih^r?rW&Kj5oCB+1Fwcu{gc5eUnQ$jrkG& z)T$N-Mn-lVagu+&YBza{53O#<#nJhQ7$ihVBLMPeR{OkIpl6N^Hnw@oWhbx#G;@aA zx_njtz~mQzqZY%d|3bl-zYNv0yt}KTSeVREqtu^(NoL~|`O{s^Fh3u46^HOJWO%mu zDwqtH6Wufr8!)VRuxK@6HbA$jutHQ8&{W+NhtuU9EWtBIcL#d|ntvb`*H)evo`LbL*h(Ga)wabdo zW2_GVha*h9pQ;)h&2gCQU27{L20@5AwEZ(MO;LDjgVkwn2C3)(^5D|%#0B8vkHqv%NZzB z5H^GzF*Zlcidiu&Vlm;2vUdTx16>|Z1cNYjhSM`q6{kbEpE)r*=JM1PPQHAd5mVrE zk8>}2a@^7!=WSse?Peo#5qXF#L@kyIAK`9~y8-2658n~WwD+I};(VHi)+sqpL2@O# z0SKGwmsr@zzltm&<$O{9oRo%w1Y#>{*md)kSUAe2jEM&^HFhy#Lzoo~+K#Ctz&Cn* z6}=KWdnXoMQ4!mS;|=ywZ^eDg!5la$#Kg2UnnlQ2o*&aJl_021kHj7{!@)CMYjxm_ zn2G(U@$K6EWX1y%6C}F^)Zm6C59sIK}WYnrC&0DnvCxxLpBFV;o`>MhO--!SNICG2yK|{o&Rc^W1CEW&q%` z80$|(K?tCGB4JI%!w0DLs~WqJ^`fHLXuO`U*j{SC9@P;8X>I#EvLcr641Vp-{f^5k zS>d1TtcCmMgj5Zg!8=l&_UYJF#J2RksryWhd1z)nVgX;iyV~A~<%oOv(C$N|iaYjH zo8bna@YX$VWIqgw&|k6E#71i4-o2B_dcJ(`Jn}fddGD>x6=*;#D#E&X`M#OtSA6}x z3Vu(mjVJe?Yag;Cv{2owp^QnIx=oujXVQ>7X~&MhTo53zh7Ko7_L?Db~mp$m|M096>Qj9KMiOB@!5b= zY)o(;Uvsd+`V!9WR}xPiY&K7Qy=o?Z_TbAVD{JH4{Av{DA8vbWJ2&y8skgE89AX_pgza}QbsmA?*oeUIV@ibFt1-16A>zK+ z+~t@RfnJ0#5d#>?Qix#$M;t*s0>BR`jlCej9ut=pafpfdX@n+>M8^IWTTyH*v3ta1 zqL;6j% zA8z)#^WMxUd^5Y?!sP1-DY_tiiwjMB`0!^nQ^~9g(zo$fl>Pm3!6yFbU8Vm4hW?I{ diff --git a/litellm/llms/__init__.py b/litellm/llms/__init__.py new file mode 100644 index 000000000..b9742821a --- /dev/null +++ b/litellm/llms/__init__.py @@ -0,0 +1 @@ +from . import * \ No newline at end of file diff --git a/litellm/llms/anthropic.py b/litellm/llms/anthropic.py new file mode 100644 index 000000000..0e59c6de9 --- /dev/null +++ b/litellm/llms/anthropic.py @@ -0,0 +1,97 @@ +import os, json, sseclient +from enum import Enum +import requests +from litellm import logging +import time +from typing import Callable +class AnthropicConstants(Enum): + HUMAN_PROMPT = "\n\nHuman:" + AI_PROMPT = "\n\nAssistant:" + +class AnthropicLLM: + + def __init__(self, default_max_tokens_to_sample, api_key=None): + self.default_max_tokens_to_sample = default_max_tokens_to_sample + self.completion_url = "https://api.anthropic.com/v1/complete" + self.validate_environment(api_key=api_key) + + def validate_environment(self, api_key): # set up the environment required to run the model + # set the api key + try: + self.api_key = os.getenv("ANTHROPIC_API_KEY") if "ANTHROPIC_API_KEY" in os.environ else api_key + if self.api_key == None: + raise Exception + + self.headers = { + "accept": "application/json", + "anthropic-version": "2023-06-01", + "content-type": "application/json", + "x-api-key": self.api_key + } + + except: + raise ValueError("Missing Anthropic API Key - A call is being made to anthropic but no key is set either in the environment variables or via params") + pass + + def _stream(self): # logic for handling streaming with the LLM API + pass + + def completion(self, model: str, messages: list, model_response: dict, print_verbose: Callable, optional_params=None, litellm_params=None, logger_fn=None): # logic for parsing in - calling - parsing out model completion calls + model = model + prompt = f"{AnthropicConstants.HUMAN_PROMPT.value}" + for message in messages: + if "role" in message: + if message["role"] == "user": + prompt += f"{AnthropicConstants.HUMAN_PROMPT.value}{message['content']}" + else: + prompt += f"{AnthropicConstants.AI_PROMPT.value}{message['content']}" + else: + prompt += f"{AnthropicConstants.HUMAN_PROMPT.value}{message['content']}" + prompt += f"{AnthropicConstants.AI_PROMPT.value}" + if "max_tokens" in optional_params and optional_params["max_tokens"] != float('inf'): + max_tokens = optional_params["max_tokens"] + else: + max_tokens = self.default_max_tokens_to_sample + data = { + "model": model, + "prompt": prompt, + "max_tokens_to_sample": max_tokens, + **optional_params + } + + ## LOGGING + logging(model=model, input=prompt, additional_args={"litellm_params": litellm_params, "optional_params": optional_params}, logger_fn=logger_fn) + ## COMPLETION CALL + response = requests.post(self.completion_url, headers=self.headers, data=json.dumps(data)) + if "stream" in optional_params and optional_params["stream"] == True: + return response.iter_lines() + else: + ## LOGGING + logging(model=model, input=prompt, additional_args={"litellm_params": litellm_params, "optional_params": optional_params, "original_response": response.text}, logger_fn=logger_fn) + print_verbose(f"raw model_response: {response.text}") + ## RESPONSE OBJECT + completion_response = response.json() + print(f"completion_response: {completion_response}") + if "error" in completion_response: + raise Exception(completion_response["error"]) + else: + model_response["choices"][0]["message"]["content"] = completion_response["completion"] + ## CALCULATING USAGE + prompt_tokens = 0 + completion_tokens = 0 + + + model_response["created"] = time.time() + model_response["model"] = model + model_response["usage"] = { + "prompt_tokens": prompt_tokens, + "completion_tokens": completion_tokens, + "total_tokens": prompt_tokens + completion_tokens + } + return model_response + + def embedding(): # logic for parsing in - calling - parsing out model embedding calls + pass + + def stream(): # logic for how to parse in-out model completion streams + pass \ No newline at end of file diff --git a/litellm/main.py b/litellm/main.py index 4c3d75bf5..c1103e40b 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -4,11 +4,12 @@ from functools import partial import dotenv, traceback, random, asyncio, time from copy import deepcopy import litellm -from litellm import client, logging, exception_type, timeout, get_optional_params +from litellm import client, logging, exception_type, timeout, get_optional_params, get_litellm_params +from litellm.utils import get_secret, install_and_import, CustomStreamWrapper, read_config_args +from .llms.anthropic import AnthropicLLM import tiktoken from concurrent.futures import ThreadPoolExecutor encoding = tiktoken.get_encoding("cl100k_base") -from litellm.utils import get_secret, install_and_import, CustomStreamWrapper, read_config_args ####### ENVIRONMENT VARIABLES ################### dotenv.load_dotenv() # Loading env variables using dotenv new_response = { @@ -38,14 +39,13 @@ async def acompletion(*args, **kwargs): # @retry(wait=wait_random_exponential(min=1, max=60), stop=stop_after_attempt(2), reraise=True, retry_error_callback=lambda retry_state: setattr(retry_state.outcome, 'retry_variable', litellm.retry)) # retry call, turn this off by setting `litellm.retry = False` @timeout(600) ## set timeouts, in case calls hang (e.g. Azure) - default is 60s, override with `force_timeout` def completion( - messages, model="gpt-3.5-turbo",# required params + model, messages,# required params # Optional OpenAI params: see https://platform.openai.com/docs/api-reference/chat/create functions=[], function_call="", # optional params temperature=1, top_p=1, n=1, stream=False, stop=None, max_tokens=float('inf'), presence_penalty=0, frequency_penalty=0, logit_bias={}, user="", deployment_id=None, # Optional liteLLM function params - *, return_async=False, api_key=None, force_timeout=600, azure=False, logger_fn=None, verbose=False, - hugging_face = False, replicate=False,together_ai = False, custom_llm_provider=None, custom_api_base=None + *, return_async=False, api_key=None, force_timeout=600, logger_fn=None, verbose=False, custom_llm_provider=None, custom_api_base=None ): try: global new_response @@ -57,9 +57,15 @@ def completion( temperature=temperature, top_p=top_p, n=n, stream=stream, stop=stop, max_tokens=max_tokens, presence_penalty=presence_penalty, frequency_penalty=frequency_penalty, logit_bias=logit_bias, user=user, deployment_id=deployment_id, # params to identify the model - model=model, replicate=replicate, hugging_face=hugging_face, together_ai=together_ai + model=model, custom_llm_provider=custom_llm_provider ) - if azure == True or custom_llm_provider == "azure": # [TODO]: remove azure=True flag, move to 'custom_llm_provider' approach + # For logging - save the values of the litellm-specific params passed in + litellm_params = get_litellm_params( + return_async=return_async, api_key=api_key, force_timeout=force_timeout, + logger_fn=logger_fn, verbose=verbose, custom_llm_provider=custom_llm_provider, + custom_api_base=custom_api_base) + + if custom_llm_provider == "azure": # azure configs openai.api_type = "azure" openai.api_base = litellm.api_base if litellm.api_base is not None else get_secret("AZURE_API_BASE") @@ -72,7 +78,7 @@ def completion( else: openai.api_key = get_secret("AZURE_API_KEY") ## LOGGING - logging(model=model, input=messages, additional_args=optional_params, azure=azure, logger_fn=logger_fn) + logging(model=model, input=messages, additional_args=optional_params, custom_llm_provider=custom_llm_provider, logger_fn=logger_fn) ## COMPLETION CALL if litellm.headers: response = openai.ChatCompletion.create( @@ -102,7 +108,7 @@ def completion( else: openai.api_key = get_secret("OPENAI_API_KEY") ## LOGGING - logging(model=model, input=messages, additional_args=args, azure=azure, logger_fn=logger_fn) + logging(model=model, input=messages, additional_args=args, custom_llm_provider=custom_llm_provider, logger_fn=logger_fn) ## COMPLETION CALL if litellm.headers: response = openai.ChatCompletion.create( @@ -131,7 +137,7 @@ def completion( openai.organization = litellm.organization prompt = " ".join([message["content"] for message in messages]) ## LOGGING - logging(model=model, input=prompt, additional_args=optional_params, azure=azure, logger_fn=logger_fn) + logging(model=model, input=prompt, additional_args=optional_params, custom_llm_provider=custom_llm_provider, logger_fn=logger_fn) ## COMPLETION CALL if litellm.headers: response = openai.Completion.create( @@ -146,14 +152,14 @@ def completion( ) completion_response = response["choices"]["text"] ## LOGGING - logging(model=model, input=prompt, azure=azure, additional_args={"max_tokens": max_tokens, "original_response": completion_response}, logger_fn=logger_fn) + logging(model=model, input=prompt, custom_llm_provider=custom_llm_provider, additional_args={"max_tokens": max_tokens, "original_response": completion_response}, logger_fn=logger_fn) ## RESPONSE OBJECT model_response["choices"][0]["message"]["content"] = completion_response model_response["created"] = response["created"] model_response["model"] = model model_response["usage"] = response["usage"] response = model_response - elif "replicate" in model or replicate == True or custom_llm_provider == "replicate": + elif "replicate" in model or custom_llm_provider == "replicate": # import replicate/if it fails then pip install replicate install_and_import("replicate") import replicate @@ -168,11 +174,11 @@ def completion( os.environ["REPLICATE_API_TOKEN"] = litellm.replicate_key prompt = " ".join([message["content"] for message in messages]) input = {"prompt": prompt} - if max_tokens != float('inf'): + if "max_tokens" in optional_params: input["max_length"] = max_tokens # for t5 models input["max_new_tokens"] = max_tokens # for llama2 models ## LOGGING - logging(model=model, input=input, azure=azure, additional_args={"max_tokens": max_tokens}, logger_fn=logger_fn) + logging(model=model, input=input, custom_llm_provider=custom_llm_provider, additional_args={"max_tokens": max_tokens}, logger_fn=logger_fn) ## COMPLETION CALL output = replicate.run( model, @@ -187,7 +193,7 @@ def completion( response += item completion_response = response ## LOGGING - logging(model=model, input=prompt, azure=azure, additional_args={"max_tokens": max_tokens, "original_response": completion_response}, logger_fn=logger_fn) + logging(model=model, input=prompt, custom_llm_provider=custom_llm_provider, additional_args={"max_tokens": max_tokens, "original_response": completion_response}, logger_fn=logger_fn) prompt_tokens = len(encoding.encode(prompt)) completion_tokens = len(encoding.encode(completion_response)) ## RESPONSE OBJECT @@ -201,59 +207,13 @@ def completion( } response = model_response elif model in litellm.anthropic_models: - # import anthropic/if it fails then pip install anthropic - install_and_import("anthropic") - from anthropic import Anthropic, HUMAN_PROMPT, AI_PROMPT - - #anthropic defaults to os.environ.get("ANTHROPIC_API_KEY") - if api_key: - os.environ["ANTHROPIC_API_KEY"] = api_key - elif litellm.anthropic_key: - os.environ["ANTHROPIC_API_KEY"] = litellm.anthropic_key - prompt = f"{HUMAN_PROMPT}" - for message in messages: - if "role" in message: - if message["role"] == "user": - prompt += f"{HUMAN_PROMPT}{message['content']}" - else: - prompt += f"{AI_PROMPT}{message['content']}" - else: - prompt += f"{HUMAN_PROMPT}{message['content']}" - prompt += f"{AI_PROMPT}" - anthropic = Anthropic() - if max_tokens != float('inf'): - max_tokens_to_sample = max_tokens - else: - max_tokens_to_sample = litellm.max_tokens # default in Anthropic docs https://docs.anthropic.com/claude/reference/client-libraries - ## LOGGING - logging(model=model, input=prompt, azure=azure, additional_args={"max_tokens": max_tokens}, logger_fn=logger_fn) - ## COMPLETION CALL - completion = anthropic.completions.create( - model=model, - prompt=prompt, - max_tokens_to_sample=max_tokens_to_sample, - **optional_params - ) + anthropic_key = api_key if api_key is not None else litellm.anthropic_key + anthropic_client = AnthropicLLM(default_max_tokens_to_sample=litellm.max_tokens, api_key=anthropic_key) + model_response = anthropic_client.completion(model=model, messages=messages, model_response=model_response, print_verbose=print_verbose, optional_params=optional_params, litellm_params=litellm_params, logger_fn=logger_fn) if 'stream' in optional_params and optional_params['stream'] == True: # don't try to access stream object, - response = CustomStreamWrapper(completion, model) + response = CustomStreamWrapper(model_response, model) return response - - completion_response = completion.completion - ## LOGGING - logging(model=model, input=prompt, azure=azure, additional_args={"max_tokens": max_tokens, "original_response": completion_response}, logger_fn=logger_fn) - prompt_tokens = anthropic.count_tokens(prompt) - completion_tokens = anthropic.count_tokens(completion_response) - ## RESPONSE OBJECT - print_verbose(f"raw model_response: {model_response}") - model_response["choices"][0]["message"]["content"] = completion_response - model_response["created"] = time.time() - model_response["model"] = model - model_response["usage"] = { - "prompt_tokens": prompt_tokens, - "completion_tokens": completion_tokens, - "total_tokens": prompt_tokens + completion_tokens - } response = model_response elif model in litellm.openrouter_models or custom_llm_provider == "openrouter": @@ -270,7 +230,7 @@ def completion( else: openai.api_key = get_secret("OPENROUTER_API_KEY") ## LOGGING - logging(model=model, input=messages, additional_args=optional_params, azure=azure, logger_fn=logger_fn) + logging(model=model, input=messages, additional_args=optional_params, custom_llm_provider=custom_llm_provider, logger_fn=logger_fn) ## COMPLETION CALL if litellm.headers: response = openai.ChatCompletion.create( @@ -311,7 +271,7 @@ def completion( co = cohere.Client(cohere_key) prompt = " ".join([message["content"] for message in messages]) ## LOGGING - logging(model=model, input=prompt, azure=azure, logger_fn=logger_fn) + logging(model=model, input=prompt, custom_llm_provider=custom_llm_provider, logger_fn=logger_fn) ## COMPLETION CALL response = co.generate( model=model, @@ -364,7 +324,7 @@ def completion( "total_tokens": prompt_tokens + completion_tokens } response = model_response - elif together_ai == True or custom_llm_provider == "together_ai": + elif custom_llm_provider == "together_ai": import requests TOGETHER_AI_TOKEN = get_secret("TOGETHER_AI_TOKEN") headers = {"Authorization": f"Bearer {TOGETHER_AI_TOKEN}"} @@ -410,7 +370,7 @@ def completion( prompt = " ".join([message["content"] for message in messages]) ## LOGGING - logging(model=model, input=prompt, azure=azure, logger_fn=logger_fn) + logging(model=model, input=prompt, custom_llm_provider=custom_llm_provider, logger_fn=logger_fn) chat_model = ChatModel.from_pretrained(model) @@ -419,7 +379,7 @@ def completion( completion_response = chat.send_message(prompt, **optional_params) ## LOGGING - logging(model=model, input=prompt, azure=azure, additional_args={"max_tokens": max_tokens, "original_response": completion_response}, logger_fn=logger_fn) + logging(model=model, input=prompt, custom_llm_provider=custom_llm_provider, additional_args={"max_tokens": max_tokens, "original_response": completion_response}, logger_fn=logger_fn) ## RESPONSE OBJECT model_response["choices"][0]["message"]["content"] = completion_response @@ -428,13 +388,13 @@ def completion( response = model_response else: ## LOGGING - logging(model=model, input=messages, azure=azure, logger_fn=logger_fn) + logging(model=model, input=messages, custom_llm_provider=custom_llm_provider, logger_fn=logger_fn) args = locals() raise ValueError(f"Invalid completion model args passed in. Check your input - {args}") return response except Exception as e: ## LOGGING - logging(model=model, input=messages, azure=azure, additional_args={"max_tokens": max_tokens}, logger_fn=logger_fn, exception=e) + logging(model=model, input=messages, custom_llm_provider=custom_llm_provider, additional_args={"max_tokens": max_tokens}, logger_fn=logger_fn, exception=e) ## Map to OpenAI Exception raise exception_type(model=model, original_exception=e) diff --git a/litellm/tests/test_logging.py b/litellm/tests/test_logging.py index d7b0d828d..3174083ef 100644 --- a/litellm/tests/test_logging.py +++ b/litellm/tests/test_logging.py @@ -28,31 +28,32 @@ except: # test on non-openai completion call try: response = completion(model="claude-instant-1", messages=messages, logger_fn=logger_fn) + print(f"claude response: {response}") score +=1 except: print(f"error occurred: {traceback.format_exc()}") pass -# test on openai embedding call -try: - response = embedding(model='text-embedding-ada-002', input=[user_message], logger_fn=logger_fn) - score +=1 -except: - traceback.print_exc() +# # test on openai embedding call +# try: +# response = embedding(model='text-embedding-ada-002', input=[user_message], logger_fn=logger_fn) +# score +=1 +# except: +# traceback.print_exc() -# test on bad azure openai embedding call -> missing azure flag and this isn't an embedding model -try: - response = embedding(model='chatgpt-test', input=[user_message], logger_fn=logger_fn) -except: - score +=1 # expect this to fail - traceback.print_exc() +# # test on bad azure openai embedding call -> missing azure flag and this isn't an embedding model +# try: +# response = embedding(model='chatgpt-test', input=[user_message], logger_fn=logger_fn) +# except: +# score +=1 # expect this to fail +# traceback.print_exc() -# test on good azure openai embedding call -try: - response = embedding(model='azure-embedding-model', input=[user_message], azure=True, logger_fn=logger_fn) - score +=1 -except: - traceback.print_exc() +# # test on good azure openai embedding call +# try: +# response = embedding(model='azure-embedding-model', input=[user_message], azure=True, logger_fn=logger_fn) +# score +=1 +# except: +# traceback.print_exc() -print(f"Score: {score}, Overall score: {score/5}") \ No newline at end of file +# print(f"Score: {score}, Overall score: {score/5}") \ No newline at end of file diff --git a/litellm/tests/test_streaming.py b/litellm/tests/test_streaming.py new file mode 100644 index 000000000..b7332772f --- /dev/null +++ b/litellm/tests/test_streaming.py @@ -0,0 +1,28 @@ +#### What this tests #### +# This tests streaming for the completion endpoint + +import sys, os +import traceback +sys.path.insert(0, os.path.abspath('../..')) # Adds the parent directory to the system path +import litellm +from litellm import completion + +litellm.set_verbose = False + +score = 0 + +def logger_fn(model_call_object: dict): + print(f"model call details: {model_call_object}") + +user_message = "Hello, how are you?" +messages = [{ "content": user_message,"role": "user"}] + +# test on anthropic completion call +try: + response = completion(model="claude-instant-1", messages=messages, stream=True, logger_fn=logger_fn) + for chunk in response: + print(chunk['choices'][0]['delta']) + score +=1 +except: + print(f"error occurred: {traceback.format_exc()}") + pass \ No newline at end of file diff --git a/litellm/utils.py b/litellm/utils.py index 65cd96a8e..fc7600cb7 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -64,13 +64,15 @@ def install_and_import(package: str): ####### LOGGING ################### #Logging function -> log the exact model details + what's being sent | Non-Blocking -def logging(model=None, input=None, azure=False, additional_args={}, logger_fn=None, exception=None): +def logging(model=None, input=None, custom_llm_provider=None, azure=False, additional_args={}, logger_fn=None, exception=None): try: model_call_details = {} if model: model_call_details["model"] = model if azure: model_call_details["azure"] = azure + if custom_llm_provider: + model_call_details["custom_llm_provider"] = custom_llm_provider if exception: model_call_details["exception"] = exception @@ -206,6 +208,32 @@ def completion_cost(model="gpt-3.5-turbo", prompt="", completion=""): return prompt_tokens_cost_usd_dollar + completion_tokens_cost_usd_dollar ####### HELPER FUNCTIONS ################ +def get_litellm_params( + return_async=False, + api_key=None, + force_timeout=600, + azure=False, + logger_fn=None, + verbose=False, + hugging_face=False, + replicate=False, + together_ai=False, + custom_llm_provider=None, + custom_api_base=None +): + litellm_params = { + "return_async": return_async, + "api_key": api_key, + "force_timeout": force_timeout, + "logger_fn": logger_fn, + "verbose": verbose, + "custom_llm_provider": custom_llm_provider, + "custom_api_base": custom_api_base + } + + return litellm_params + + def get_optional_params( # 12 optional params functions = [], @@ -222,9 +250,7 @@ def get_optional_params( user = "", deployment_id = None, model = None, - replicate = False, - hugging_face = False, - together_ai = False, + custom_llm_provider = "" ): optional_params = {} if model in litellm.anthropic_models: @@ -247,13 +273,13 @@ def get_optional_params( if max_tokens != float('inf'): optional_params["max_tokens"] = max_tokens return optional_params - elif replicate == True: + elif custom_llm_provider == "replicate": # any replicate models # TODO: handle translating remaining replicate params if stream: optional_params["stream"] = stream return optional_params - elif together_ai == True: + elif custom_llm_provider == "together_ai": if stream: optional_params["stream_tokens"] = stream if temperature != 1: @@ -698,6 +724,13 @@ class CustomStreamWrapper: def __iter__(self): return self + def handle_anthropic_chunk(self, chunk): + str_line = chunk.decode('utf-8') # Convert bytes to string + if str_line.startswith('data:'): + data_json = json.loads(str_line[5:]) + return data_json.get("completion", "") + return "" + def handle_together_ai_chunk(self, chunk): chunk = chunk.decode("utf-8") text_index = chunk.find('"text":"') # this checks if text: exists @@ -713,7 +746,7 @@ class CustomStreamWrapper: completion_obj ={ "role": "assistant", "content": ""} if self.model in litellm.anthropic_models: chunk = next(self.completion_stream) - completion_obj["content"] = chunk.completion + completion_obj["content"] = self.handle_anthropic_chunk(chunk) elif self.model == "replicate": chunk = next(self.completion_stream) completion_obj["content"] = chunk diff --git a/old-docs/advanced.md b/old-docs/advanced.md deleted file mode 100644 index aa3b22389..000000000 --- a/old-docs/advanced.md +++ /dev/null @@ -1,29 +0,0 @@ -# Advanced - Callbacks - -## Use Callbacks to send Output Data to Posthog, Sentry etc -liteLLM provides `success_callbacks` and `failure_callbacks`, making it easy for you to send data to a particular provider depending on the status of your responses. - -liteLLM supports: - -- [Helicone](https://docs.helicone.ai/introduction) -- [Sentry](https://docs.sentry.io/platforms/python/) -- [PostHog](https://posthog.com/docs/libraries/python) -- [Slack](https://slack.dev/bolt-python/concepts) - -### Quick Start -```python -from litellm import completion - -# set callbacks -litellm.success_callback=["posthog", "helicone"] -litellm.failure_callback=["sentry"] - -## set env variables -os.environ['SENTRY_API_URL'], os.environ['SENTRY_API_TRACE_RATE']= "" -os.environ['POSTHOG_API_KEY'], os.environ['POSTHOG_API_URL'] = "api-key", "api-url" -os.environ["HELICONE_API_KEY"] = "" - -response = completion(model="gpt-3.5-turbo", messages=messages) -``` - - diff --git a/old-docs/berrispend_integration.md b/old-docs/berrispend_integration.md deleted file mode 100644 index cecd90eba..000000000 --- a/old-docs/berrispend_integration.md +++ /dev/null @@ -1,34 +0,0 @@ -# BerriSpend Tutorial -BerriSpend is a free dashboard to monitor your cost and logs across llm providers. - -## Use BerriSpend to see total spend across all LLM Providers (OpenAI, Azure, Anthropic, Cohere, Replicate, PaLM) -liteLLM provides `success_callbacks` and `failure_callbacks`, making it easy for you to send data to a particular provider depending on the status of your responses. - -In this case, we want to log requests to BerriSpend when a request succeeds. - -### Use Callbacks -Use just 2 lines of code, to instantly see costs and log your responses **across all providers** with BerriSpend: - -``` -litellm.success_callback=["berrispend"] -litellm.failure_callback=["berrispend"] -``` - -Complete code -```python -from litellm import completion - -## set env variables -os.environ["BERRISPEND_ACCOUNT_ID"] = "your-email-id" -os.environ["OPENAI_API_KEY"] = "" - -# set callbacks -litellm.success_callback=["berrispend"] -litellm.failure_callback=["berrispend"] - -#openai call -response = completion(model="gpt-3.5-turbo", messages=[{"role": "user", "content": "Hi 👋 - i'm openai"}]) - -#bad call -response = completion(model="chatgpt-test", messages=[{"role": "user", "content": "Hi 👋 - i'm a bad call to test error logging"}]) -``` \ No newline at end of file diff --git a/old-docs/client_integrations.md b/old-docs/client_integrations.md deleted file mode 100644 index a7ebc6969..000000000 --- a/old-docs/client_integrations.md +++ /dev/null @@ -1,12 +0,0 @@ -# Data Logging Integrations - -| Integration | Required OS Variables | How to Use with callbacks | -|-----------------|--------------------------------------------|-------------------------------------------| -| Sentry | `SENTRY_API_URL` | `litellm.success_callback=["sentry"], litellm.failure_callback=["sentry"]` | -| Posthog | `POSTHOG_API_KEY`,
`POSTHOG_API_URL` | `litellm.success_callback=["posthog"], litellm.failure_callback=["posthog"]` | -| Slack | `SLACK_API_TOKEN`,
`SLACK_API_SECRET`,
`SLACK_API_CHANNEL` | `litellm.success_callback=["slack"], litellm.failure_callback=["slack"]` | -| Helicone | `HELICONE_API_TOKEN` | `litellm.success_callback=["helicone"]` | - - - - diff --git a/old-docs/contact.md b/old-docs/contact.md deleted file mode 100644 index d5309cd73..000000000 --- a/old-docs/contact.md +++ /dev/null @@ -1,6 +0,0 @@ -# Contact Us - -[![](https://dcbadge.vercel.app/api/server/wuPM9dRgDw)](https://discord.gg/wuPM9dRgDw) - -* [Meet with us 👋](https://calendly.com/d/4mp-gd3-k5k/berriai-1-1-onboarding-litellm-hosted-version) -* Contact us at ishaan@berri.ai / krrish@berri.ai diff --git a/old-docs/contributing.md b/old-docs/contributing.md deleted file mode 100644 index 1c831e204..000000000 --- a/old-docs/contributing.md +++ /dev/null @@ -1,34 +0,0 @@ -## Contributing to Documentation -Clone litellm -``` -git clone https://github.com/BerriAI/litellm.git -``` - -### Local setup for locally running docs - -#### Installation -``` -pip install mkdocs -``` - -#### Locally Serving Docs -``` -mkdocs serve -``` -If you see `command not found: mkdocs` try running the following -``` -python3 -m mkdocs serve -``` - -This command builds your Markdown files into HTML and starts a development server to browse your documentation. Open up [http://127.0.0.1:8000/](http://127.0.0.1:8000/) in your web browser to see your documentation. You can make changes to your Markdown files and your docs will automatically rebuild. - -[Full tutorial here](https://docs.readthedocs.io/en/stable/intro/getting-started-with-mkdocs.html) - -### Making changes to Docs -- All the docs are placed under the `docs` directory -- If you are adding a new `.md` file or editing the hierarchy edit `mkdocs.yml` in the root of the project -- After testing your changes, make a change to the `main` branch of [github.com/BerriAI/litellm](https://github.com/BerriAI/litellm) - - - - diff --git a/old-docs/helicone_integration.md b/old-docs/helicone_integration.md deleted file mode 100644 index 273d22d4f..000000000 --- a/old-docs/helicone_integration.md +++ /dev/null @@ -1,55 +0,0 @@ -# Helicone Tutorial -[Helicone](https://helicone.ai/) is an open source observability platform that proxies your OpenAI traffic and provides you key insights into your spend, latency and usage. - -## Use Helicone to log requests across all LLM Providers (OpenAI, Azure, Anthropic, Cohere, Replicate, PaLM) -liteLLM provides `success_callbacks` and `failure_callbacks`, making it easy for you to send data to a particular provider depending on the status of your responses. - -In this case, we want to log requests to Helicone when a request succeeds. - -### Approach 1: Use Callbacks -Use just 1 line of code, to instantly log your responses **across all providers** with helicone: -``` -litellm.success_callback=["helicone"] -``` - -Complete code -```python -from litellm import completion - -## set env variables -os.environ["HELICONE_API_KEY"] = "your-helicone-key" -os.environ["OPENAI_API_KEY"], os.environ["COHERE_API_KEY"] = "", "" - -# set callbacks -litellm.success_callback=["helicone"] - -#openai call -response = completion(model="gpt-3.5-turbo", messages=[{"role": "user", "content": "Hi 👋 - i'm openai"}]) - -#cohere call -response = completion(model="command-nightly", messages=[{"role": "user", "content": "Hi 👋 - i'm cohere"}]) -``` - -### Approach 2: [OpenAI + Azure only] Use Helicone as a proxy -Helicone provides advanced functionality like caching, etc. Helicone currently supports this for Azure and OpenAI. - -If you want to use Helicone to proxy your OpenAI/Azure requests, then you can - - -- Set helicone as your base url via: `litellm.api_url` -- Pass in helicone request headers via: `litellm.headers` - -Complete Code -``` -import litellm -from litellm import completion - -litellm.api_base = "https://oai.hconeai.com/v1" -litellm.headers = {"Helicone-Auth": f"Bearer {os.getenv('HELICONE_API_KEY')}"} - -response = litellm.completion( - model="gpt-3.5-turbo", - messages=[{"role": "user", "content": "how does a court case get to the Supreme Court?"}] -) - -print(response) -``` diff --git a/old-docs/index.md b/old-docs/index.md deleted file mode 100644 index 9e3f1dde4..000000000 --- a/old-docs/index.md +++ /dev/null @@ -1,43 +0,0 @@ -# *🚅 litellm* -a light 100 line package to simplify calling OpenAI, Azure, Cohere, Anthropic APIs - -###### litellm manages: -* Calling all LLM APIs using the OpenAI format - `completion(model, messages)` -* Consistent output for all LLM APIs, text responses will always be available at `['choices'][0]['message']['content']` -* Consistent Exceptions for all LLM APIs, we map RateLimit, Context Window, and Authentication Error exceptions across all providers to their OpenAI equivalents. [see Code](https://github.com/BerriAI/litellm/blob/ba1079ff6698ef238c5c7f771dd2b698ec76f8d9/litellm/utils.py#L250) - -###### observability: -* Logging - see exactly what the raw model request/response is by plugging in your own function `completion(.., logger_fn=your_logging_fn)` and/or print statements from the package `litellm.set_verbose=True` -* Callbacks - automatically send your data to Helicone, Sentry, Posthog, Slack - `litellm.success_callbacks`, `litellm.failure_callbacks` [see Callbacks](https://litellm.readthedocs.io/en/latest/advanced/) - -## Quick Start -Go directly to code: [Getting Started Notebook](https://colab.research.google.com/drive/1gR3pY-JzDZahzpVdbGBtrNGDBmzUNJaJ?usp=sharing) -### Installation -``` -pip install litellm -``` - -### Usage -```python -from litellm import completion - -## set ENV variables -os.environ["OPENAI_API_KEY"] = "openai key" -os.environ["COHERE_API_KEY"] = "cohere key" - -messages = [{ "content": "Hello, how are you?","role": "user"}] - -# openai call -response = completion(model="gpt-3.5-turbo", messages=messages) - -# cohere call -response = completion("command-nightly", messages) -``` -Need Help / Support : [see troubleshooting](https://litellm.readthedocs.io/en/latest/troubleshoot) - -## Why did we build liteLLM -- **Need for simplicity**: Our code started to get extremely complicated managing & translating calls between Azure, OpenAI, Cohere - -## Support -* [Meet with us 👋](https://calendly.com/d/4mp-gd3-k5k/berriai-1-1-onboarding-litellm-hosted-version) -* Contact us at ishaan@berri.ai / krrish@berri.ai diff --git a/old-docs/input.md b/old-docs/input.md deleted file mode 100644 index 5d3ba77a5..000000000 --- a/old-docs/input.md +++ /dev/null @@ -1,172 +0,0 @@ -# Completion Function - completion() -The Input params are **exactly the same** as the -OpenAI Create chat completion, and let you call **Azure OpenAI, Anthropic, Cohere, Replicate, OpenRouter** models in the same format. - -In addition, liteLLM allows you to pass in the following **Optional** liteLLM args:
-`forceTimeout`, `azure`, `logger_fn`, `verbose` - - -## Input - Request Body - -**`model`** -string Required
-ID of the model to use. See the model endpoint compatibility table for details on which models work with the Chat API. - ---- - -**`messages`** -array Required
- -A list of messages comprising the conversation so far. Example Python Code - -```python -from litellm import completion - -messages= - [ - {"role": "system", "content": "You are a helpful assistant."}, - {"role": "user", "content": "Knock knock."}, - {"role": "assistant", "content": "Who's there?"}, - {"role": "user", "content": "Orange."}, - ] - -# openai call -response = completion(model="gpt-3.5-turbo", messages=messages, temperature=0) - -# cohere call -response = completion(model="command-nightly", messages=messages, temperature=0) -``` - - ---- ->> **`role`** ->> string Required
->> The role of the messages author. One of system, user, assistant, or function. ->>
->> ->> --- - ->> **`content`** ->> string Required
->> The contents of the message. content is required for all messages, and may be null for assistant messages with function calls. ->>
->> ->> --- - ->> **`name`** ->> string Optional
->> The name of the author of this message. name is required if role is function, and it should be the name of the function whose response is in the content. May contain a-z, A-Z, 0-9, and underscores, with a maximum length of 64 characters. ->>
->> ->> --- - ->> **`function_call`** ->> object Optional
->> The name and arguments of a function that should be called, as generated by the model. ->>
->> ->> --- - -**`functions`** -array Optional
-A list of functions the model may generate JSON inputs for. -
- ---- ->> **`name`** ->> string Required
->> The name of the function to be called. Must be a-z, A-Z, 0-9, or contain underscores and dashes, with a maximum length of 64. ->>
->> ->> --- - ->> **`description`** ->> string Optional
->> A description of what the function does, used by the model to choose when and how to call the function. ->>
->> ->> --- - ->> **`parameters`** ->> object Required
->> The parameters the functions accept, described as a JSON Schema object. ->> To describe a function that accepts no parameters, provide the value `{"type": "object", "properties": {}}`. ->>
->> ->> --- - - -**`function_call`** -string or object Optional
-Controls how the model responds to function calls. "none" means the model does not call a function, and responds to the end-user. "auto" means the model can pick between an end-user or calling a function. Specifying a particular function via `{"name": "my_function"}` forces the model to call that function. "none" is the default when no functions are present. "auto" is the default if functions are present. -
- ---- - -**`temperature`** -number Optional, Defaults to 1
-What sampling temperature to use, between 0 and 2. Higher values like 0.8 will make the output more random, while lower values like 0.2 will make it more focused and deterministic. We generally recommend altering this or `top_p` but not both. -
- ---- - -**`top_p`** -number Optional, Defaults to 1
-An alternative to sampling with temperature, called nucleus sampling, where the model considers the results of the tokens with top_p probability mass. So 0.1 means only the tokens comprising the top 10% probability mass are considered. We generally recommend altering this or 1temperature` but not both. -
- ---- - -**`n`** -integer Optional, Defaults to 1
-How many chat completion choices to generate for each input message. -
- ---- - -**`stream`** -boolean Optional, Defaults to false
-If set, partial message deltas will be sent, like in ChatGPT. Tokens will be sent as data-only server-sent events as they become available, with the stream terminated by a `data: [DONE]` message. -
- ---- - -**`stop`** -string or array Optional, Defaults to null
-Up to 4 sequences where the API will stop generating further tokens. -
- ---- - -**`max_tokens`** -integer Optional, Defaults to inf
-The maximum number of tokens to generate in the chat completion. The total length of input tokens and generated tokens is limited by the model's context length -
- ---- - -**`presence_penalty`** -number Optional, Defaults to 0
-Number between -2.0 and 2.0. Positive values penalize new tokens based on whether they appear in the text so far, increasing the model's likelihood to talk about new topics. -
- ---- - -**`frequency_penalty`** -number Optional, Defaults to 0
-Number between -2.0 and 2.0. Positive values penalize new tokens based on their existing frequency in the text so far, decreasing the model's likelihood to repeat the same line verbatim. -
- ---- - -**`logit_bias`** -map Optional, Defaults to null
-Modify the likelihood of specified tokens appearing in the completion. Accepts a JSON object that maps tokens (specified by their token ID in the tokenizer) to an associated bias value from -100 to 100. Mathematically, the bias is added to the logits generated by the model prior to sampling. The exact effect will vary per model, but values between -1 and 1 should decrease or increase the likelihood of selection; values like -100 or 100 should result in a ban or exclusive selection of the relevant token. -
- ---- - -**`user`** -string Optional
-A unique identifier representing your end-user, which can help liteLLM to monitor and detect abuse. - diff --git a/old-docs/output.md b/old-docs/output.md deleted file mode 100644 index 9b67c44b3..000000000 --- a/old-docs/output.md +++ /dev/null @@ -1,12 +0,0 @@ -# Completion Function - completion() -Here's the exact json output you can expect from a `litellm` completion call: - -```python -{'choices': [{'finish_reason': 'stop', - 'index': 0, - 'message': {'role': 'assistant', - 'content': " I'm doing well, thank you for asking. I am Claude, an AI assistant created by Anthropic."}}], - 'created': 1691429984.3852863, - 'model': 'claude-instant-1', - 'usage': {'prompt_tokens': 18, 'completion_tokens': 23, 'total_tokens': 41}} -``` \ No newline at end of file diff --git a/old-docs/secret.md b/old-docs/secret.md deleted file mode 100644 index e6dd96ad1..000000000 --- a/old-docs/secret.md +++ /dev/null @@ -1,33 +0,0 @@ -# Supported Secret Managers -liteLLM reads secrets from yoour secret manager, .env file - -- [Infisical Secret Manager](#infisical-secret-manager) -- [.env Files](#env-files) - -For expected format of secrets see [supported LLM models](https://litellm.readthedocs.io/en/latest/supported) - -## Infisical Secret Manager -Integrates with [Infisical's Secret Manager](https://infisical.com/) for secure storage and retrieval of API keys and sensitive data. - -### Usage -liteLLM manages reading in your LLM API secrets/env variables from Infisical for you - -``` -import litellm -from infisical import InfisicalClient - -litellm.secret_manager = InfisicalClient(token="your-token") - -messages = [ - {"role": "system", "content": "You are a helpful assistant."}, - {"role": "user", "content": "What's the weather like today?"}, -] - -response = litellm.completion(model="gpt-3.5-turbo", messages=messages) - -print(response) -``` - - -## .env Files -If no secret manager client is specified, Litellm automatically uses the `.env` file to manage sensitive data. diff --git a/old-docs/stream.md b/old-docs/stream.md deleted file mode 100644 index 5e8cc32ca..000000000 --- a/old-docs/stream.md +++ /dev/null @@ -1,33 +0,0 @@ -# Streaming Responses & Async Completion - -- [Streaming Responses](#streaming-responses) -- [Async Completion](#async-completion) - -## Streaming Responses -LiteLLM supports streaming the model response back by passing `stream=True` as an argument to the completion function -### Usage -```python -response = completion(model="gpt-3.5-turbo", messages=messages, stream=True) -for chunk in response: - print(chunk['choices'][0]['delta']) - -``` - -## Async Completion -Asynchronous Completion with LiteLLM -LiteLLM provides an asynchronous version of the completion function called `acompletion` -### Usage -``` -from litellm import acompletion -import asyncio - -async def test_get_response(): - user_message = "Hello, how are you?" - messages = [{"content": user_message, "role": "user"}] - response = await acompletion(model="gpt-3.5-turbo", messages=messages) - return response - -response = asyncio.run(test_get_response()) -print(response) - -``` \ No newline at end of file diff --git a/old-docs/supported.md b/old-docs/supported.md deleted file mode 100644 index 8b2aeca22..000000000 --- a/old-docs/supported.md +++ /dev/null @@ -1,72 +0,0 @@ -## Generation/Completion/Chat Completion Models - -### OpenAI Chat Completion Models - -| Model Name | Function Call | Required OS Variables | -|------------------|----------------------------------------|--------------------------------------| -| gpt-3.5-turbo | `completion('gpt-3.5-turbo', messages)` | `os.environ['OPENAI_API_KEY']` | -| gpt-3.5-turbo-16k | `completion('gpt-3.5-turbo-16k', messages)` | `os.environ['OPENAI_API_KEY']` | -| gpt-3.5-turbo-16k-0613 | `completion('gpt-3.5-turbo-16k-0613', messages)` | `os.environ['OPENAI_API_KEY']` | -| gpt-4 | `completion('gpt-4', messages)` | `os.environ['OPENAI_API_KEY']` | - -## Azure OpenAI Chat Completion Models - -| Model Name | Function Call | Required OS Variables | -|------------------|-----------------------------------------|-------------------------------------------| -| gpt-3.5-turbo | `completion('gpt-3.5-turbo', messages, azure=True)` | `os.environ['AZURE_API_KEY']`,
`os.environ['AZURE_API_BASE']`,
`os.environ['AZURE_API_VERSION']` | -| gpt-4 | `completion('gpt-4', messages, azure=True)` | `os.environ['AZURE_API_KEY']`,
`os.environ['AZURE_API_BASE']`,
`os.environ['AZURE_API_VERSION']` | - -### OpenAI Text Completion Models - -| Model Name | Function Call | Required OS Variables | -|------------------|--------------------------------------------|--------------------------------------| -| text-davinci-003 | `completion('text-davinci-003', messages)` | `os.environ['OPENAI_API_KEY']` | - -### Cohere Models - -| Model Name | Function Call | Required OS Variables | -|------------------|--------------------------------------------|--------------------------------------| -| command-nightly | `completion('command-nightly', messages)` | `os.environ['COHERE_API_KEY']` | - - -### Anthropic Models - -| Model Name | Function Call | Required OS Variables | -|------------------|--------------------------------------------|--------------------------------------| -| claude-instant-1 | `completion('claude-instant-1', messages)` | `os.environ['ANTHROPIC_API_KEY']` | -| claude-2 | `completion('claude-2', messages)` | `os.environ['ANTHROPIC_API_KEY']` | - -### Hugging Face Inference API - -All [`text2text-generation`](https://huggingface.co/models?library=transformers&pipeline_tag=text2text-generation&sort=downloads) and [`text-generation`](https://huggingface.co/models?library=transformers&pipeline_tag=text-generation&sort=downloads) models are supported by liteLLM. You can use any text model from Hugging Face with the following steps: - -* Copy the `model repo` URL from Hugging Face and set it as the `model` parameter in the completion call. -* Set `hugging_face` parameter to `True`. -* Make sure to set the hugging face API key - -Here are some examples of supported models: -**Note that the models mentioned in the table are examples, and you can use any text model available on Hugging Face by following the steps above.** - -| Model Name | Function Call | Required OS Variables | -|------------------|-------------------------------------------------------------------------------------|--------------------------------------| -| [stabilityai/stablecode-completion-alpha-3b-4k](https://huggingface.co/stabilityai/stablecode-completion-alpha-3b-4k) | `completion(model="stabilityai/stablecode-completion-alpha-3b-4k", messages=messages, hugging_face=True)` | `os.environ['HF_TOKEN']` | -| [bigcode/starcoder](https://huggingface.co/bigcode/starcoder) | `completion(model="bigcode/starcoder", messages=messages, hugging_face=True)` | `os.environ['HF_TOKEN']` | -| [google/flan-t5-xxl](https://huggingface.co/google/flan-t5-xxl) | `completion(model="google/flan-t5-xxl", messages=messages, hugging_face=True)` | `os.environ['HF_TOKEN']` | -| [google/flan-t5-large](https://huggingface.co/google/flan-t5-large) | `completion(model="google/flan-t5-large", messages=messages, hugging_face=True)` | `os.environ['HF_TOKEN']` | - -### OpenRouter Completion Models - -All the text models from [OpenRouter](https://openrouter.ai/docs) are supported by liteLLM. - -| Model Name | Function Call | Required OS Variables | -|------------------|--------------------------------------------|--------------------------------------| -| openai/gpt-3.5-turbo | `completion('openai/gpt-3.5-turbo', messages)` | `os.environ['OR_SITE_URL']`,
`os.environ['OR_APP_NAME']`,
`os.environ['OR_API_KEY']` | -| openai/gpt-3.5-turbo-16k | `completion('openai/gpt-3.5-turbo-16k', messages)` | `os.environ['OR_SITE_URL']`,
`os.environ['OR_APP_NAME']`,
`os.environ['OR_API_KEY']` | -| openai/gpt-4 | `completion('openai/gpt-4', messages)` | `os.environ['OR_SITE_URL']`,
`os.environ['OR_APP_NAME']`,
`os.environ['OR_API_KEY']` | -| openai/gpt-4-32k | `completion('openai/gpt-4-32k', messages)` | `os.environ['OR_SITE_URL']`,
`os.environ['OR_APP_NAME']`,
`os.environ['OR_API_KEY']` | -| anthropic/claude-2 | `completion('anthropic/claude-2', messages)` | `os.environ['OR_SITE_URL']`,
`os.environ['OR_APP_NAME']`,
`os.environ['OR_API_KEY']` | -| anthropic/claude-instant-v1 | `completion('anthropic/claude-instant-v1', messages)` | `os.environ['OR_SITE_URL']`,
`os.environ['OR_APP_NAME']`,
`os.environ['OR_API_KEY']` | -| google/palm-2-chat-bison | `completion('google/palm-2-chat-bison', messages)` | `os.environ['OR_SITE_URL']`,
`os.environ['OR_APP_NAME']`,
`os.environ['OR_API_KEY']` | -| google/palm-2-codechat-bison | `completion('google/palm-2-codechat-bison', messages)` | `os.environ['OR_SITE_URL']`,
`os.environ['OR_APP_NAME']`,
`os.environ['OR_API_KEY']` | -| meta-llama/llama-2-13b-chat | `completion('meta-llama/llama-2-13b-chat', messages)` | `os.environ['OR_SITE_URL']`,
`os.environ['OR_APP_NAME']`,
`os.environ['OR_API_KEY']` | -| meta-llama/llama-2-70b-chat | `completion('meta-llama/llama-2-70b-chat', messages)` | `os.environ['OR_SITE_URL']`,
`os.environ['OR_APP_NAME']`,
`os.environ['OR_API_KEY']` | \ No newline at end of file diff --git a/old-docs/supported_embedding.md b/old-docs/supported_embedding.md deleted file mode 100644 index d509adc58..000000000 --- a/old-docs/supported_embedding.md +++ /dev/null @@ -1,5 +0,0 @@ -## Embedding Models - -| Model Name | Function Call | Required OS Variables | -|----------------------|---------------------------------------------|--------------------------------------| -| text-embedding-ada-002 | `embedding('text-embedding-ada-002', input)` | `os.environ['OPENAI_API_KEY']` | \ No newline at end of file diff --git a/old-docs/token_usage.md b/old-docs/token_usage.md deleted file mode 100644 index 5bf2fbd3d..000000000 --- a/old-docs/token_usage.md +++ /dev/null @@ -1,45 +0,0 @@ -# Token Usage -By default LiteLLM returns token usage in all completion requests ([See here](https://litellm.readthedocs.io/en/latest/output/)) - -However, we also expose 3 public helper functions to calculate token usage across providers: - -- `token_counter`: This returns the number of tokens for a given input - it uses the tokenizer based on the model, and defaults to tiktoken if no model-specific tokenizer is available. - -- `cost_per_token`: This returns the cost (in USD) for prompt (input) and completion (output) tokens. It utilizes our model_cost map which can be found in `__init__.py` and also as a [community resource](https://github.com/BerriAI/litellm/blob/main/cookbook/community-resources/max_tokens.json). - -- `completion_cost`: This returns the overall cost (in USD) for a given LLM API Call. It combines `token_counter` and `cost_per_token` to return the cost for that query (counting both cost of input and output). - -## Example Usage - -1. `token_counter` - -```python -from litellm import token_counter - -messages = [{"user": "role", "content": "Hey, how's it going"}] -print(token_counter(model="gpt-3.5-turbo", messages=messages)) -``` - -2. `cost_per_token` - -```python -from litellm import cost_per_token - -prompt_tokens = 5 -completion_tokens = 10 -prompt_tokens_cost_usd_dollar, completion_tokens_cost_usd_dollar = cost_per_token(model="gpt-3.5-turbo", prompt_tokens=prompt_tokens, completion_tokens=completion_tokens)) - -print(prompt_tokens_cost_usd_dollar, completion_tokens_cost_usd_dollar) -``` - -3. `completion_cost` - -```python -from litellm import completion_cost - -prompt = "Hey, how's it going" -completion = "Hi, I'm gpt - I am doing well" -cost_of_query = completion_cost(model="gpt-3.5-turbo", prompt=prompt, completion=completion)) - -print(cost_of_query) -``` diff --git a/old-docs/troubleshoot.md b/old-docs/troubleshoot.md deleted file mode 100644 index 3dc4a2662..000000000 --- a/old-docs/troubleshoot.md +++ /dev/null @@ -1,9 +0,0 @@ -## Stable Version - -If you're running into problems with installation / Usage -Use the stable version of litellm - -``` -pip install litellm==0.1.1 -``` -