From 3ca79a88bb0d351648f3f7a3d1fdfeb5371ecc8a Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Sat, 30 Sep 2023 18:09:08 -0700 Subject: [PATCH] improvements to proxy cli and finish reason mapping for anthropic --- litellm/{proxy => }/.env.template | 0 litellm/__pycache__/utils.cpython-311.pyc | Bin 137059 -> 137527 bytes litellm/llms/prompt_templates/factory.py | 1 - litellm/proxy/proxy_cli.py | 72 +++++++++++++++++----- litellm/proxy/proxy_server.py | 11 ++-- litellm/utils.py | 11 +++- poetry.lock | 13 +++- pyproject.toml | 1 + 8 files changed, 84 insertions(+), 25 deletions(-) rename litellm/{proxy => }/.env.template (100%) diff --git a/litellm/proxy/.env.template b/litellm/.env.template similarity index 100% rename from litellm/proxy/.env.template rename to litellm/.env.template diff --git a/litellm/__pycache__/utils.cpython-311.pyc b/litellm/__pycache__/utils.cpython-311.pyc index d3a54a2d4ce6f9c1df91226ee7d4874596cac59a..a7ff83ae40d7c5bec621f224ff630f63c67b8c6f 100644 GIT binary patch delta 15144 zcmb7r3wTu3wSV@!@*)osLJ}T%0SN;JNC3qE2?0SA2;mI@87Fg2l7VCKRpu{c`h#b@uZ}L82UZ-T9)p=W*B(X|+Dcw=2teNDZHVQM!~kI?ITQ8#ee zNGiK#pN;s?AM5A!`(reri8{MhyhYTk(nb;2)!JynMr~qPE1}FLtwkHFPthl5ch!m^ z8OLpL)&#UQ+Oj4ri_ z`vl2Cf2<$%C5EJNXcLH%WFnkIX;Iftn@k)++B&UVU#wM-D2scD;?Sm$C}FLV%BKEX zS^Kmtj+z!!B$;oZ^qMY7<72e8PKJxM=~Q_J{kc(Z)n@9gb$zs(i0N8x@~Y;voAq^d z4s8~dwo&P966|(u4ykLsRz>&&Z7$&kt(vf1o3|yVy=EgfGM@x+XbVVyP1-^#+^pTs z`HS>*+G1*W36;lbONs3diS4p2!n;MQA-W&=W9o#qoSL~qTd%F4zqe|)Q2sV;CE=Z# zhwv`8?yY29@730=xm#VL-6Vh*M(E~o+bl}r1OlPz7 z2f}V4i%GN)yf06VBsBXrd7C{~i6BcU-wU$Q;|Mp#{MiwIJYn1;w-C_(CmiEXI>sM& zj6WUQ@BWl`V~dW*7M)JX*tX_$V%Bj-mK>qx{rDqU~im>Et|zfaC>kL}9S$R*a> ziR!8xauPX)NJv%OPdBCAah3{ZQ`%AiEX^KaI{g7nU&|UaW1B4Fbm;{yBj*CD0jw9n zis>}ww1%MHr-!1dX(O`G8Zu@{;+g!?XD6S?uRNJwsZOTM{?GbifdK1BU$eyQj=NJw ziWzLRSWDyuXqc_mEzCmXLCgOxCl__wPn4;?X*pu68k1Ic!#vc8l~YlB8sG*%E}+{t zrsK1>Sy>3c9m-5wu6{M3uev2SzGH6s!#1%%eU>ps%vO2*C;QmXmVg5!b|+Zgg%KrJ z0yID!pq{{Vg~Bv?o6JP1hnjrrJ$FJcm9V;*)qm6m zcMz~c%^jeNCF7r4;n6ht1b^3IaGpS2+-S(Eyx5suPO%Dh!!>QciF0A@bdm1 z$}>R{28m;;Ft<=_ z>X?&zT!liyzi)tP*^IHf%K^_g2ic_%>#VLh+2gBHbU6c2Gby$b*hNVH<6H3c=j(c|+{EY|d}*WVbT z=-MsH{h(w z%V1M;nDn`N^sIbzg4jnpX~yXR&BMZR|2C*oqie^ooM@ART$EHk4gO3OEw}oyL;1%v ziE4Ff%zig_3`C}@F-V;XN%n&T>U~gp8iVyAxd(i9sr@CBT&&`I)h8uodECyAQN(IV z_6M3tbaXYGY}Z*rE26x(BnK9z?B&0L^DBS3(q3;OTcr=oIEtK1Ib?zwAHHoQRV8!@?jMb`gEts=TQWm$U^{D1}lyI;sxZr z2zUwbGT?WB*VU8d*=4MFc1dpb4V1h|Kp~hl!2(7*Y4zwRh>of6%1gzpj*<#*V(MPx z@I>}FQqQQpHxC!TP`|&qvKTft`*f+q6RcfjIsflL$hK!$EfLi9v#LkKB&O5n^)=|0 zn?kY~3z^tVv7&Lej#nDz9ues&RYXI}r)ul3mCBqoIpZ{a^k@w2wB5?uTQz%#h^Xze z7Z2dZF&fP{iimpH-=v%NK(JMM)OWM{PkRL{STl!_dKK^+z;6NU9Iqn<@y&R0MvuNm z4}|3rWU_DFr?1Mv7r3+SyLYw&mJTOL)3Rh43E$DFF~;@Y19=_vqwkk%d;7!rYw273+C%aL|i z9m70mI(eA!Oh_{&cbF^j(Cww*-qr-eX9?N{hA1ZZ1M29K2U+dY)RRloCq`8-&w}zC z0B&ri)=|uAk$U%7WgTA_oon4u4XM8n!!Ps z{g8Ed38Y8WmSq*_c$KUlnaS3Maosjq06sebP4yXQv3i8Xn!YO##9RLbL> zN1YX?VtW{#2LoH0*|DbYRo66(F5wXLFEHUwAjp_5Z*w!*UYUf16NeGhiEF8GyCgYoI_!?mj#=Oovy}eDE9-%g9~} z;mt&}L(?qBB5_m=^*4!e9XtKA9MhN_iZ#=HEwnXj@-#L!d1$^{RQ(YRhGzg2mGK^cMyEh&0bQ1b%m!mjkdP_eq>uE0mJiL6FgNY<&)n(5*d zb!bgd9)w(Yz-GE=frelKX8wzIqGFR*?`*xM{VXb-0~7*^0K);_0`38@IXHg(2A=}} z$;67fMxhn`Z}o>8tauTUAEG!9Z~*`dn(@>+jX}iWzaW!s3x_w=ldU5L97Y}%lfA~5 z{8D|^I=CHU)=bj8p@!O^SJIL|Hd^61g?nVFhld~!Jk|wk%LXBnxLpl#9)S;|)QoRh z=Lt8^VqB9v1$}|G?L;k|00%)Rh4S+7Q+KZ|5gR&QUi*!GG)m+kK$JSep|f54VUQZiZ4hrDZIsWtk&0sx`KNR<;dl#-#srmYp7yD z$9wm<;|A3u*D68^ZP1Lt)|#dU>>iTO$!-F4pMn{hme2~#rYm>%UyM#mu%$i6NTj*i zez4?me&Fh67Jy_S!GGFw{1&u;@{Xgs!=m&dO44CXQ2KNaX7ON4m}8q6+qGH1h&AK9 z&3;du0tN#c#(6~EQ)}lS|kv@<*y=^oJSpiZ*MxTZjD{$0#eY#wUti*7T z1_=!ikJry3LF!;)IWtoAfChk}lJ}H48!6e|F?~-(AJL$G^jI-%K{_5Q6)V++$3}^O z%Gg&d#;9rg%EfB6dEZJpBssV5H)U`U>RU^|7xo7OJT3C#f`*sg+!$QfM6T=cYwkpA z&|i<{UPV8~t3N)zYtGB0hZ=HHf z2+}km$!5AVdg%vZ)0sQK`=_NaTKx zN47@=p3l7QXGlIoKrH;FC0(KJ5#%Jp2f})wEAvs%?giYcUV47;AWSM&9K~!yJNNFM zA5~vIKg9VfqJ6ew@S#tgE}R(4$5ra#e#P9gzLixAXU?BnebY2gW%W&-S<{zU9Z9OZ zskEddn)jrdcQ`u}9WqmMV0#K$hZvFD^JesR?O6=ie8@}K3BKCp3&|@ zZ4O+iwTQJIA8kGxX`vnrSb`<>Filh>c>;p-Fir^TA#2+gvQ{@%D2CigUCU&S7}lh6 ziYkVZsFdaq`l&Bo&vBlkyeS=-ZzLrRd<=YfrDP+Tg_3RmJTmEI^Suc-UO@6MfQtb5 zgFFP-4|o{>tI1aZ2MElxxz*FF=FeTUaQghHd7n_PyytF*`QJm{RTx`4lqPnom>8v;{+EtR7h7m!!cfP=kvm3oO!}0pTwIJd*0I zp@i>e0o}tXtuqiJ4t5QoUlNI#)*~Et6=1l@=KzlqbR8A^KVpm>pF)pY%`o9>jq7j`B}_-`8br8&t}b z`7Rr%_Nw|T*CkGau-GxlE9$_NABq%J{Ozy`9wFU}^G{L!H^65E2V9o>^9VYP+^Bn7 z%kX4%-?u|YG7k;Kn92`yp4a>*`abIHw<9vK1E9TMQ?0JiioLN(UPj&fRQ`AAjn_b6 zX2|;ll+XsXB|tM!@5Y}%ttSCb5ghopv!7l9@U#glw-xfCTHahWZ&d%Ko!h~(Jle?c z5_kxh*{qov8SVm!EYsc`Y_@vM?*4bmvz9pVIz=d*EyKpmMi&OFF%tM7gA(e~N1*8Tebp$B{KFRJC-L=UVkO%*fUjt&O33ry& zlP@5553LEv@GPx9vvro^1K6!Lki6~6S1RKDu2fboW z+5oQH?Lu7HYQT6SO_aEYL4F=DY+TDF(M%x zLSWhvLtsK$OF}X^7Qjt&gUq^@#m9k!$90nNJh2}85n4?~?kK=$K(`mMcHcwi;7@Wq zO0PAN`-w5_-y-7zfK_@Oir5RzBai!5hP*<2cH7nR1aQ0_rR)tX&mj8bt)eH?;-lM> zkY!{x{1NbAS#nSbe$dWm(U#@e-uWoG+UPwt@@YNxKy1(MQQNJs9xi|er+{Mxpb{_@ zFcAQ+NU>CoeNiZlK%Lv4nJ6>RF+g0TDMLK!%5S408bweHEsDCQQp&Sj&lo*OWIJyr zwm*r?86=v77-PIRSllq^=Tzl?*qZD!2Ih#-;sIk;j@VagZBEcI_r3}e&IMExkhca~ zXt5d&uGRw~YqvTTwAT?_bvDeg@CFp|9NLqO%KaoGfyW}?{)o&KABlF@qb+eJ-9QHO zmIP&^CgCU)CRm|7a`qAlZQ`rZ70x|qwaR|dJ^Nu!H=T2Eif+b9+JXD&P|w<-q@(Hr z>PXMMnVGo8+i1L#FXEj`sOSmfOul$hJY?KiAYRm2tbQ}~b=TxI1Y3(jdH!%-JzXBu zSFn7S5{M(6BXkS-5SqYxeBq4F^98GESk+qbbW$@R_v@%}V2Bvvtf9saMNSM6$wE9~ zoE<8{&RZ$#NMzYCaj!kiOKDjP@DacZjBkdE0jb<{k3wUpTU_I;BbsB8MQ*XwA=Vnt z6pIm}&iJ5Ml(+Nnzyxf%Xz55R32U73)QL&d>_=C@VNab-t2JHoc*9s((#nwJhdSxz zI}{F?j%I8?Clec40f4>fnlr4^?HW()b;ykdQHm)rmTa;ayGMx$iEE*aN$AWrstHaBiZEoXFI1Vpv{cQ;#0dE_aGp zVRS4M8&k0;lnYVQkJLV6+9GjH;$BemNT@d2771S&J29&S_Ve!`T0gc#{-*(75Rmt5 z1)HlWW@1A)Oz$ni-f&Au^5{Kicvpym0aHPl1p!%J9$jq8t;T~Z#L)IM zuSd6RA{=a&Tu!(Slk@Y zD53D!A|-b7#*|y=wb4;JGZ>sGe-3Vd8IiYd5i7DY9`H?+>G_shVW(DLwd8E zWbC~uy*qLYy^>qWaGb1&4%_Co(@zsytYTI0w@QQi;Zjp9u+{l)#I!5lHNHi?{7 z4QK}^G$HKgqQlEfm0myHw9|1;r|oFpzys+Z@h}scSJ&eX7?sS}X79SjAe{+fw1u$9 zK@U5-_&Ayb{4?^%Ch?Vs;|Va9Lf^wb6a#Z!L-Tz?x{so~hk|&E&OPtkSs15(D2ft5 zBLUfw_$?yc#wTpnS&vN^!|xE+7xGO2yB_?JL@kFlj-YM!1~z~V=fAEpDrp;Jh;5>O zto5{Jgz@qn;;|$M#`7y(%aj|zt>RiS-q^QQj8BY~$fL%Etzv>}LL!mAWenXWg4x_3 zo5YzZUr3Oex$$YnQ`^LNF)?y}n|M}K>?Q6qzH#Exzw8n(1UFL-U8dtTlJz)gAp z{B);SF0{y3JH>RH58|4M-LWDhDWh}pKp#8R0!_)&1P3hL^k^0h(r%Eg&kdddZBPB+ z@Y(3tC9W6iBcJXPH6o9dbdl(KvzPyk%AXqbhFBuDMh+XIR*YckzoLNtSVy-1Q@vSx zsBb$XkME&{&=jt>m#SDEAT4u&MYKCKW>}xyTIA4+K9LkVe=6p=9we_1J!3qzM-)X; zeP3hx|l9~$33B65w>4~r`Cs!{ZaIF#F6X(6gl1VCbI5&aD+G>IdT zrbk6-r?+woQ6KH$mZ)a-v92IGd&|LaGuW{-eJu6yCa9M??k2WU0a`)MVT$#N<%${4 z4-eBk`dS|a?HQtw6L9Gl2)Dho2(-o1@-$Kvas0kv|-uy{2t`UsY0=6Z!fHp;8{m=D4^RCk++7h64~B3_mc@cA7tXZV1Q58ZG;C$~e% zd>Z#SK6e?#FVV)71IH7{;3L30k;1C$QVwyfe_#)O2P&-N>@6t7ye_XrbCU?jDl8b! zA-t?`BX^|a=L69q(-D9rc+U9h@!m}C^|ZkR6%X9gNF6l}zbpodFN_ag7Uixh zh!!nI?ki$o++th&T5;JnCo=UF5r~^$>pS91`P}2xD^FI}9j|XVS?xbr-Fl*Y?aA`B zJE~6h8?nFSWWVB*{U)9s=sw=h{rBaLIkuz`Uw^s6_}vllZ2O(m*8Ot^+?p8k_SnjP zbK_&)iM7$kJMjtCS@w6*%c|4u$CF%?b3ENiImfdykRDYyuYb&mexN+jKVg27{X|~D z{5bo&Hc-AB=cJr>lWrbKAAh`lY9IRRV^<}J&&D$*4yY}(pB+|tZEaG%{ro^dU+43k zwS~rauZzY)UIW-=0#w(Bz)XX2Dnsi6zHSR0HFmrqCWzg}n{SA!_J2|@=xBH<6gLfU z1E37>dzkVV;5dMtsoQzrHuA# z6vIWvQk#=(6m!*IVdKnEkt1F+t{fG=tK<>!9{CL|3(mlO*wfg};X@R2lF|31rl4dYdV2P4YEO&n#Dm5Ir-i?r)rB(!Gro@A58|O*h>b_@%y=Tf;FsvLHKlN9;Dgf( zDB!?>8zSrKmyd#m!YX`!-YMi}P?o|ZQ<1ukQf9Wb@-a4lAo{y zrF03@b+J$YvLVK)55!~_)GTX^0j4M%@gTU~4*nxRF%p1tF*BJaIcs<8;fwrhj8&$X z?Yy1{IwCKc;+!itdOSeu=z8L5x;W^BWB~lyjTg^}k?kCyF&DT~dK?f$H!qg^@S(;A zh{AIStEW2>MmLN-PB>h0B1A>AmiRo!PeR^gf{=@t(Z(jv(9eq;XE|l1Mz)<7lblIj zDj|Wb>c)w`h~4h}=&&oaE<^_^e9&aw{X$P&GYTIa{ma;KQ53X;n>A(dA^{dPUA!R* z^@z*7Y~Y?;LC0V@>(ZqbpKSci5GBD9#cBY8qy%ZVCg}|g{WAf6XYnyAOzrBLX_Q{wmHIMPG`7Jk|?)Y#ob0+Z%c{8htuk zZTP-I4iq9MHP`|pW(xzlN3VHgp_!}SnQ{0?T zgIvJRjMA^fU&n$4MNy`^T9{rZ1S?Sy@RU*gwP-C`0H+t$;D$^g`i|+^YqMa0Uq=4+ zwb(0e#9C@0mK-rvZMIRPs6{5Jr3@>0V|dV`1;}A0Sx3Si#5CG>LNnHyd4+N68{thr zkGb0iBUAq+2D%1t2ls*u%OuHGK#OvL(JE{=dKaQXRLi8et9wd^&Fmt4!_6Y`YZV@` zFqwq@Szvlw##LFQ0CI!O+Q_xp2IO%w+=JwMi$?PvGO}Fj$6zL3;7ck%CW@ILe-d^giQEoNah<88p!cZy4wwR0ZjkX}!dSf|=%T zY7WXU&I{>fo|pF+ha#o%whAE(LnhePiDcuy6KrKSW}@vk!SW3NUlnB|ZvX)IK=L9A z@{!^TpTS7+?H^wT@^#^g1wr-$?qD~tq!+>k>$?M#4NX-Py1}p&ZV!j0_ zIH2jkeWh&2_a?v=z}JAm;9mq71z?}8LF%_CZR7_zuOj(h$ov)H7=S0`S`?haC+`P8 zN9udz@qJ+iK5_FS|AD+ENYwyvi6H9$JZ<3++d7iu1H=eE`J9ao$!NF1OFJyHtYrdb zSIN^0!aD~}**(5EGI5mQ(7<8MQA;-G!6M0idVT%a+@-I*I;I_IcEnXG@xzq;=ksq~+rYfyX#)sXYf>9Tbpb7`4c1Hbv7MU-B~{zBf+1TRm9<(#8%AA% zX-UohY4zHbR1;Fg_FPqJx8q;8v?OhfHj=9ARwil1nw|d}t_fR`evLLtt+02D4yStJ zgK)G)E!DZ&Rn%WGjBp!kbUA6%bGb$@l9a@!MqBlSuEo5lZeAIS}rcv)(qy8jP&2VivajesXw`tW>y;ECBc)M0ZxJ&aB zDs2_v9olNb-CBU~&Sv3i8)$Wx%PmOi4e3U;Q>&v@-lbhb{qNSqyd?L=b1#z~eb6#)oce-U=%TQ4XSOKU4_y|mAI6{i4HC<8<*M)*%-BTNGA$J@kY%4j5%TdO>s)!&F9ofp}8& z&p(vULgOg|)Fa%A({J+oisw~+w|>2X7#{+ZqF;;ce+?=DPpCk*N^yY^DGSgo-508@3+NGlC|FJ|qm^9H zMMeSGXtD^vu4fZUfY;TW!ro$2V?*I7(aBF0rrjUWYr{4w78Y8aSF060+~TCVxyQrVOOBs?ZaWl1P_lBjW4)wgM7rQaGtnHx7 z0>1^N9e`T_x2g1A&3?Bmv~s0CxH7(Z|JXRK*TceT5m4V<(Wg*h2%b%QeOQ5!W;z1;8a=Rnxf;A=oGKf#+|6skAk!HL`MiO!WYbBfg@>dk5Ly|QcY^LN z^`{|YoGjfPs{PQ>h0NkMG(qlWmOofmAMp@{K$X|GTCP)}p~DMUIN-)?7a*IMq*v9i zT&c^60lyxM$XnEFL*4l-$~_>o7r=Dy!4RR#9V&g;AUh^7y4!6X)VyI`#pjK6!w#qQ zjE^wehNPc#>J5;yx4enOczLgyJFd+6YZMQu2gVJ)@(~o9Bz&^D+$17A^6DN%=`q0L z1nax0(}QzWw-T503DiBQ29?xApQfU_o%|hYATId~faS#UWNV3`?peTdfad|P0A3?7 zbLv82OI#4LhqOWhVZE{OGAl>X`*i~H+AJkzdQVagRkN&kna9xUxN?^c5fdA)F7>*y z_n`)+Cm#SDQYU8i6_2QY%^W`fHezNp%gGa}T4n8%H_(x7hZkV+ptQ2tbD%=g;q&@x zbj!@a+LZa+*=%F^@r2;vfu#)qbCZJr(+S*yE9g(9a?Bc=^Jg^h4&|9KsT#DTRQ)h3 zTU8Eemm(qJ_2q1)Kg`OBGFRwtrjqj|UsbD{$-&TC%UZL;^*%b0!wK_y0rXjjFQW7k z;AOyJ0NdDWC}E*yT0#S_q7qBC)|c+B^Uv}Iy~Sj-rmMaVb9;QGIe7+spm8&KrQW2u zd8p$PexQ1}yvNv9%4bblC!^^k71ReJEky7p2LB##9B`}3ujnL>so@plMYOS@;-}Q; zA=JMOfFV=(gMGomCg-vcZc_gTr6Ck&!Y+|^GEZnAEW(p39|w$ML|5R z_6xbUk!nq4ul5N`IgVbR0gkE1D@(+Q#;+>xvrQR;nnX}D437rj4}iA-p9B60a07|} z{Q%cB!*TV_vPZ;qjW;Zh3Fp_Sy+@U<=sErritr;8ez74Yw)Q$S#rHRPZr+l-|Cs>$ zgk$P~6}=r7scTx}$rUG3#4>enO}~zZFcy2oZIeAvz76oW`l!ZTVL2N-GcA^+1CK07 z_I zd2v{P>7p%(bDBNWF;%guR`hRtY1K@-by9z=vO-;Y@IV%`onj)l-EszL5uH*Av?tfe zOKN7QtNSO+#M50qAPj*b!fc9BlGM&nFX3zaUFgFU7faBFkyYx~YbT2r)Te9v6oR+; z2W_U4;y-MAkoJF0ftzGD%2dCG{v#86p*P0&0rUl20^AH>RlI=G%YZz9OWoGcxrqHr zA)e0ypeZwrrXumf1AmH2*5T*st%iYlFXGt?XeID}s=Rgi(bq6MQ}c#vszP2#frJcL zez*;@Zb^kV7jG$EGz;lx9>{ZQ@b2(c@YH5n?K)4ShJtrZ^1*Wfb0?Abr41km!fmJy zVl7n9tQ#!WG=9GBQt}{y=FWh4@35UZt|%-m<$Y*6eO6eIN1fb_Z8sejVxKx4eYp?3 zx7%T+*M>Ab;0bzbb(v3n%#1p(^wx$wwcd!YMn0vU**w9~iyGc+{BiRF;k*Kqtx$`v z?<4lB&DYOxT3$q*x_-E$AGLj~TsO>i?!yAMH+pZFX77rV%JL5PQ>j&WdUR`V@s0ZX z*1x#6p_`>g_4ZAJ#U3I>&UH>C48|H>Na<3S0Cn!J9ysf5DScc@VSlmfnB)ytC0|4na7L8lYeP$$rKw z7lb|iFnR-qH(oR%V#r=9(ltCJeYywdb*MhV&df|{x{^rjS~Hc7SWlIgBJEq%-n)mc zeh*~eq21ZigBd|bu0U6ZcYVFoEsy9$MMe#UlkgJNUY{=<0$Bwj zY}r1PG=L7c!#K5E69xfrZ69y&4_mO~4y5BhL*V*kjY18q2xbZA{Ig>)Xj=5|@zeK_WXDa#=$ z<=}4pPCOfcARq*|9RMdIrva|P<4(Y4fCOv-+yUS)*E>b0M(5LqlV{#cjkNVbB(f$EribWAkfbh(dD2K%WXureR&;cE z1A1MJ#~Y}t@yZ)8#V){Zz`n-pXNv?0?W*VIE#cJ;LG{%H6r;y!He27_XxxiNm)AdJ zaC7y&sL6ssMD$>D<*(6qAK+^Bx99S^^0CW15N?lh00{{kQ-#m>bUaCYA8MTT{Fja{ zzs0Zz2+XXC*^|r1Pxp+UJ>4^N@)CKon*B=0_Pnml@zaM49vp9ZNNs$jbAPPQY%^i} zyvg$?m&e;4WKt6=rcItZIbPYIj=a)&Fl1r2EiaqxnOIh_U{bu5Ewk<1$+OF*PaHo# zF{(znjtp?ZEy%rU+>r-dPZ0sx2-vH>Jkrf^oQi*I%>F-vlG|~way0Rh1)WQCjutph zP|aXfe{@aR1z5al%kvDcoZ;+y552%4RGrJ0In4;t6WP-(L zW((cKS(JaoLU>cBNA#wPP}mA?;<28YNe7RW9U}jcs#^!n+p6uG{T(0D_>qkh-ptJG zaz9AnDCbyezGQv>4wcnv)jJ~xo<;Ffz+VBdcliteij>a-pda}XV81%`PA73doqNX} zZ9v5f05*{fm`GfuHK>0X@DSix0PICR2jETdB9$meK8Om|-osQfZK1H4q6gRbWhf}W zXPMGrff(!%Iz9s6%`-5*d*q|2Ozf9-O>PffuX%?&rVhW`sXI20d>nus)0{~71Qq`U zu&eLho#yzOidBu1&%Bo<9&If6+eINv)&5*hTK1@OT`4=Z@Z4DE1_<;Ub@#c7VdpXO z3jjhAO72j?+Hz#Cqtj8q_h`NZptm7(#H+67b@5kq{rTz9BWOoRz!zUj5_WXT1hfi~ zII_;9&Sr+Mp0d2PWfTX}C5CN2wMCeG90b3i3C*^2J)vOF6Oj;yr1Q;UgkB`kB_T;Y zQJL$KKcG86H2S|d-qvYd2Gr;u=Q@7|*Zb72A4j-KQN*1`ey`sC@dlBh7XNfbNfj#C zz<6Jr$Kw}(zY`pETK3INe@1QGeyuRurC$80S3jn)0?jzg4>o0ZeyM+&a{XKsMTAec zx7sRQqli3EE5F28oMP^D3NO~xQ9{qk8Mv`bCXX@Pk8YRsK{|O_vvL3(9s(RBIQVb5 zqXauH#Cq1Dxj`NNc_43o*%PB|*7*x>;O@ayW^!Gq&RQ&+`9Cn0Lz^_6Jdw^SVLf*M zuL$qVY+sEx;z16i=|za_h&DE;3TOcaWv2MNbrBr-y8|LU%C6=s5YI>7k5TH163kN4 zi^i6h%Oj|J4)8o6abvrP#~%sYovfgpBiMw}61N4*M$jF`X70WyO8Qpek#+qUN-Xk0 zG&7kiWxhO`9wGl2mdA{%h3K80i7(= z;gMH30(Gsnf80RiaI~^bFh8B?kuQawrc{rm?~g(4RRHtS1%qG`QOOYk8v(1n8(QE* zWL*CV>!4I#LhG#8ggI1UObb2X5#}Jrg5ns!I6w(tBmmZtVu}2i5$qre)7wErfTY-6 z9mIXk)HPJ3OaD@1e7?x(6`#B1g~Lqo`skKL&TJ|X-4ScZ7q!ARKp6L&5fi)K2M%zC z%liphj+4JNrWA-lVwbU|Kw-dw&?GsVR?<=erfZEikmc8`lTps6=KGD* zp5l<$YdqakydZWPb-hI13>faYbfQ3s1D>9l+6$71yXQ}7N+?)UwL?9hLQTE7BM&)d zyxmLmbS$EYo{as_OJoUgz;N{z5yvvBdO5bGx412N%yKGA4`2lWv`-YxcGfB9v0egr zG`}y>%ub5|Mn#e6@9@#M*JGQD#A3UM81D@fMZ#y;2Z?d{8^9cnY12uU9||t4y~YQu z-S7+&c~jv$A--$sDclO1cDP&j1z2rt8zz=kaa2)<3JAgq z1~@EWH@_AQyoV9hN$9r)x4d4q$pnXXi+TfkGJn?^Q?3*}9Gj@3L#*~naYdoa--7RZ z7l=|Z-}rEW*qpr|Yny|S^!-T^ZK2>o(ccC8wCugicyOWcjb=+_IaE=T9~Uz}?q7by zXMO}Dq`NI3b9qnN=o1!+HR4ysbBn|Xaieizk?2A;lDSy;#C66si$zF0Xne9*+$kE2 zh9zQQ`veipWXXQ3adwHQ$WD;nMH9*~#)L|7t&_viFO8#>;wkY^tYN8GD;9I+@E(lk zITUW(FmP0{*IjlvJV1IGYBaY<3u>j9^*pt;ZNsmx%$Lux8U$ z6N%J$!VzzzJ}h}RK4t7!CVJ$J0onFokoo1^!&=*7ys=F5n$!m5TKQu(S2ha{HnTCh zEdbUfbErfq0hY(DsIVKw%SFemTc~V0E#Bp5W5IHCzFYcz=mr=U`);|oy0g~- zYAgbWwU+oCW?Q_UqCm@&$R6^HO}m{X$TuR`7ALNd{IC+pDIg1YE%v(x@maEa5S2}_ z)}3i>N_(A`Qf6?)8{`Xr% zmu^QOiHxvLXBoYU#;aTwNz--0NWM|@Nk<CGR# zfUBtwt~OK2X?p_xpw13kzKy;!jEy_UkBo?Yyh9AOWt>DEGayTi{M$swDSx7x_0FY} zD#|A>n{OqSaGM}s^$xX|cH%`#|>i|S$~_@DW=6-J4Kh|s}i%UPhr+Z z7s^u@aT=4ku=2uH^x+Lv+v9|;g`#H|UM1SknTBG(=a?okjLn8r6C- zc@Ct`TPwI{he$OZyGvZYs`F;hTFbOz z#L`YNKB2&tivY#A5NMmRQWC0QhYd+Ib9?$8F;?u3wYgWkE|#*fLG7*hr;N^GHa-5Kb|p) z?iU5dnEOO~WBL7}7nj7d#@zeFa|Mavvq9)e0C=|o>TfZ$R=ga$>wYmLIm&Y4x#H&C z!lE(E%&^`|G_9MxXcXj_uMA6ud>?9M*ow%?odCRPv)mgC7eS4g#^39-^XLseIt5zT zow0Pkm@Qt5J+fbHN{&KOrlV>dWzno%Z>C#}%*QJT@IiO&cuHg`jqLd%SACzGWz@eYx~Ji6dIJEu|G&qF&CmIk;H60LU zmSy(MIa>KHI`GE*GfKydS%)dgy@hbr`7IlIzLn~rnzu4% z^rMFlMoq||KOZ@BfO#14QP({0h~$rkk00(Gl9BvLe?d>53~_iz7*D+^0=@b8PnIc= zX&Qmq4iX<9UKjKw#Li(uj^&nU{pXa) z|C|v6#hXUadtzkP5*kGB_^m8ol~MPe=`mRwCbiW zRN;$Fn=a?4ju?_;>)tvu43rX^x#~U8&T`q1AvO9IQULrSbb3SVZLFQ~D9- zwp_k3{C&}(R~L}~08F&AUi^C2`XhKz0~uT1YP(VQzG&|(K+i&B`}<<76H=55jX%CG zdKc}%#0{9D4|?<=sZN4@8+`ICXe97WzPZ;w*^Y{wY@0h^FbZ zn2@VMZ=(@9EBZy*y@sg5-Nt&O#P@%=_W%3U|16kF__6pUu|?K`PB5^FX)X5oa2|!a z(F9>9ktn2}`r!Ihqx+|#gX1cyPKk~ARE%+CE~8ci_)NmM@tnBR{U8?Y46h4QqQuI4 z^1EBes`*UV4z|8CCY={OmV%n)^!bE`VofLCg2EP;>|XiIKZ{}C1E4_b^~!QQvJF7T zblGE8nkG1MUbJ^XtFi>Mj5akF~l(Y^|eMO1Na07`oSRmQhpi=Gqy z2W1=xmKAak!W^ulI((c~8=~u9d}`j8X0lh)$ocb}4aU_M#Nfgm=$;k-{*pDzQ(9(q zjK6tK?+*q6Gh%fJtFN$H7xm6k0UKF*C1=M`4@zq7~h4_o{ z{=Yf+{Q@<7M#HgN_0&mwzFBK@yCNSIh zyg?5wqj@OZI~JpnPdJOdvFkfABYhES0m}I7JMqO(kRVUU{Vr#wRW4vD1_FL-*e{8- zeP)w3DLCbCF~C!pp2_B{!~rt~cqn%NC2_x)idb|$t^-Ns>um!E(u~}N4S^l-25O=Y z3c!b%Y2|G_K^+0MC2|Z~KF3vn&;655^y$EJ4@POGyzhv9g z=o}1)OPK_B`AR7x%of2fp_wIK6@mI6R;jK>_ zk8ec=l$FgQaV3&U>|1#i4`XB@f$6BHZ{Vc--uODj*2#GmJpbLuO0{iwY@q2fV!uwc zbrI1ks4IyCbCs7SI$4lVT8l##a$T^ zv}`S!CJ{aQUtZ*rRm97ba`ztDjyjr7EyQ_VJ~}y4V^5e(ZPsdejhs49qr{esghKRM ziaiA@HZOaM#)Q~)^8=KuQv=4aaRqET){%O=2efQx`UFxCsu55N|? z2&Kbl^`rDs6vf}5_$1&sfKOdEa(rrFoz7pQ^aDEMogTl@;kPrA-+s&o!;1jR06qX8 zO2|NSBFVc85ymBO$&4#3$d`QU&rXSdm6cd=h$AH$A;otnLhl*_w9 zZZK}ivgN1WNh67NQg}t|=`7o=$+2baY!!~;I)}L8>!e@C3tazA%hu}H2OVuo#s30D Cw4Vq7 diff --git a/litellm/llms/prompt_templates/factory.py b/litellm/llms/prompt_templates/factory.py index 381e8b8fd..d24548488 100644 --- a/litellm/llms/prompt_templates/factory.py +++ b/litellm/llms/prompt_templates/factory.py @@ -217,5 +217,4 @@ def prompt_factory(model: str, messages: list): else: return hf_chat_template(original_model_name, messages) except: - traceback.print_exc() return default_pt(messages=messages) # default that covers Bloom, T-5, any non-chat tuned model (e.g. base Llama2) \ No newline at end of file diff --git a/litellm/proxy/proxy_cli.py b/litellm/proxy/proxy_cli.py index dc6778e6d..28f252fe4 100644 --- a/litellm/proxy/proxy_cli.py +++ b/litellm/proxy/proxy_cli.py @@ -1,9 +1,55 @@ import click import subprocess -import os +import os, appdirs from dotenv import load_dotenv load_dotenv() +from importlib import resources +import shutil + +config_filename = ".env.litellm" + +# Using appdirs to determine user-specific config path +config_dir = appdirs.user_config_dir("litellm") +user_config_path = os.path.join(config_dir, config_filename) + +def load_config(): + try: + if not os.path.exists(user_config_path): + # If user's config doesn't exist, copy the default config from the package + here = os.path.abspath(os.path.dirname(__file__)) + parent_dir = os.path.dirname(here) + default_config_path = os.path.join(parent_dir, '.env.template') + # Ensure the user-specific directory exists + os.makedirs(config_dir, exist_ok=True) + # Copying the file using shutil.copy + shutil.copy(default_config_path, user_config_path) + + # As the .env file is typically much simpler in structure, we use load_dotenv here directly + load_dotenv(dotenv_path=user_config_path) + except: + pass + +def open_config(): + # Create the .env file if it doesn't exist + if not os.path.exists(user_config_path): + # If user's env doesn't exist, copy the default env from the package + here = os.path.abspath(os.path.dirname(__file__)) + parent_dir = os.path.dirname(here) + default_env_path = os.path.join(parent_dir, '.env.template') + # Ensure the user-specific directory exists + os.makedirs(config_dir, exist_ok=True) + # Copying the file using shutil.copy + try: + shutil.copy(default_env_path, user_config_path) + except Exception as e: + print(f"Failed to copy .env.template: {e}") + + # Open the .env file in the default editor + if os.name == 'nt': # For Windows + os.startfile(user_config_path) + elif os.name == 'posix': # For MacOS, Linux, and anything using Bash + subprocess.call(('open', '-t', user_config_path)) @click.command() @click.option('--port', default=8000, help='Port to bind the server to.') @@ -16,22 +62,17 @@ load_dotenv() @click.option('--telemetry', default=True, type=bool, help='Helps us know if people are using this feature. Turn this off by doing `--telemetry False`') @click.option('--config', is_flag=True, help='Create and open .env file from .env.template') @click.option('--test', default=None, help='proxy chat completions url to make a test request to') -def run_server(port, api_base, model, deploy, debug, temperature, max_tokens, telemetry, config, test): +@click.option('--local', is_flag=True, default=False, help='for local debugging') +def run_server(port, api_base, model, deploy, debug, temperature, max_tokens, telemetry, config, test, local): if config: - if os.path.exists('.env.template'): - if not os.path.exists('.env'): - with open('.env.template', 'r') as source: - data = source.read() - with open('.env', 'w') as destination: - destination.write(data) - - click.echo('Opening .env file...') - subprocess.call(['open', '.env']) # replace `open` with `start` on Windows - else: - click.echo('No .env.template file found.') + open_config() - from .proxy_server import app, initialize, deploy_proxy - # from proxy_server import app, initialize, deploy_proxy + if local: + from proxy_server import app, initialize, deploy_proxy + debug = True + else: + from .proxy_server import app, initialize, deploy_proxy + if deploy == True: print(f"\033[32mLiteLLM: Deploying your proxy to api.litellm.ai\033[0m\n") print(f"\033[32mLiteLLM: Deploying proxy for model: {model}\033[0m\n") @@ -57,6 +98,7 @@ def run_server(port, api_base, model, deploy, debug, temperature, max_tokens, te click.echo(f'LiteLLM: response from proxy {response}') return else: + load_config() initialize(model, api_base, debug, temperature, max_tokens, telemetry) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 313ca43bd..5f771d489 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -120,9 +120,8 @@ def model_list(): async def completion(request: Request): data = await request.json() print_verbose(f"data passed in: {data}") - if (user_model is None): - raise ValueError("Proxy model needs to be set") - data["model"] = user_model + if user_model: + data["model"] = user_model if user_api_base: data["api_base"] = user_api_base ## check for custom prompt template ## @@ -154,9 +153,8 @@ async def completion(request: Request): async def chat_completion(request: Request): data = await request.json() print_verbose(f"data passed in: {data}") - if (user_model is None): - raise ValueError("Proxy model needs to be set") - data["model"] = user_model + if user_model: + data["model"] = user_model # override with user settings if user_temperature: data["temperature"] = user_temperature @@ -186,7 +184,6 @@ async def chat_completion(request: Request): ) response = litellm.completion(**data) if 'stream' in data and data['stream'] == True: # use generate_responses to stream responses - print("reaches stream") return StreamingResponse(data_generator(response), media_type='text/event-stream') print_verbose(f"response: {response}") return response \ No newline at end of file diff --git a/litellm/utils.py b/litellm/utils.py index 49bece66a..30d1870de 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -94,6 +94,14 @@ last_fetched_at_keys = None def _generate_id(): # private helper function return 'chatcmpl-' + str(uuid.uuid4()) +def map_finish_reason(finish_reason: str): # openai supports 5 stop sequences - 'stop', 'length', 'function_call', 'content_filter', 'null' + # anthropic mapping + print(f"receives finish reason: {finish_reason}") + if finish_reason == "stop_sequence": + return "stop" + return finish_reason + + class Message(OpenAIObject): def __init__(self, content="default", role="assistant", logprobs=None, **params): super(Message, self).__init__(**params) @@ -114,7 +122,7 @@ class Choices(OpenAIObject): def __init__(self, finish_reason=None, index=0, message=None, **params): super(Choices, self).__init__(**params) if finish_reason: - self.finish_reason = finish_reason + self.finish_reason = map_finish_reason(finish_reason) else: self.finish_reason = "stop" self.index = index @@ -3200,6 +3208,7 @@ class CustomStreamWrapper: model_response.choices[0].delta = Delta(**completion_obj) return model_response elif model_response.choices[0].finish_reason: + model_response.choices[0].finish_reason = map_finish_reason(model_response.choices[0].finish_reason) # ensure consistent output to openai return model_response except StopIteration: raise StopIteration diff --git a/poetry.lock b/poetry.lock index dc313bd6f..4943d7f52 100644 --- a/poetry.lock +++ b/poetry.lock @@ -122,6 +122,17 @@ files = [ [package.dependencies] frozenlist = ">=1.1.0" +[[package]] +name = "appdirs" +version = "1.4.4" +description = "A small Python module for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." +optional = false +python-versions = "*" +files = [ + {file = "appdirs-1.4.4-py2.py3-none-any.whl", hash = "sha256:a841dacd6b99318a741b166adb07e19ee71a274450e68237b4650ca1055ab128"}, + {file = "appdirs-1.4.4.tar.gz", hash = "sha256:7d5d0167b2b1ba821647616af46a749d1c653740dd0d2415100fe26e27afdf41"}, +] + [[package]] name = "async-timeout" version = "4.0.3" @@ -1074,4 +1085,4 @@ testing = ["big-O", "jaraco.functools", "jaraco.itertools", "more-itertools", "p [metadata] lock-version = "2.0" python-versions = "^3.8" -content-hash = "0fa234d1342838a6cc444dd996dbe404ca2cd6c872dcf560dbe420a2956aaecd" +content-hash = "c8cae152cee4eda56560529476234bc5e91171c6207641af797e7bebf720a499" diff --git a/pyproject.toml b/pyproject.toml index e857833f6..e3f3c556d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -14,6 +14,7 @@ tiktoken = ">=0.4.0" importlib-metadata = ">=6.8.0" tokenizers = "*" click = "*" +appdirs = "^1.4.4" [tool.poetry.scripts] litellm = 'litellm:run_server'