From 4b0f8825f321c63cac4bfceefc00d26d7bdfabb1 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Wed, 11 Oct 2023 21:08:34 -0700 Subject: [PATCH] fix(proxy_server): fix prompt template for proxy server --- litellm/__pycache__/main.cpython-311.pyc | Bin 51977 -> 51977 bytes litellm/__pycache__/utils.cpython-311.pyc | Bin 160439 -> 159321 bytes litellm/proxy/proxy_server.py | 23 +--------------------- litellm/tests/test_prompt_factory.py | 23 ++++++++++++++++++++++ 4 files changed, 24 insertions(+), 22 deletions(-) create mode 100644 litellm/tests/test_prompt_factory.py diff --git a/litellm/__pycache__/main.cpython-311.pyc b/litellm/__pycache__/main.cpython-311.pyc index e2293109d120fa25422da1e5f026feaeb6b65b2d..d8a3221f4ae243a077eaa1c0082f819e28bc7f63 100644 GIT binary patch delta 22 ccmeC2#@soLnR_`eFBbz49N|{q$o=mW07a|@l>h($ delta 22 ccmeC2#@soLnR_`eFBbz4s7tGDFo6JJ4Lczr5C{ng`{ppD2M8oHp=Sb$6XJH=y`l}2 zUPUABii$)T5P2ea-&3!$2?j^OUF515F6dQpdFNEm(%E>v|KBJl)m3%YI(6z)SGD`g zkPkiziFh(RJXC`}t0~X=T*r$ME6R*f_-Mp`7k>1(CU^N+M>%Z>5^Q*V&)ow zx=UX;f&MB#F-y)Uh%pfo|26Aen$VVIg=gVscV`Vo%Jh2s%9Z#xL#ii~%>*7|)qP4AXj0N2 z9NVGMCsx6!v~dT@j;2k=|4*gowi;RuU#1rv9W?(~`hpYb3y!8QIAt1qbjZvTrpi`A z%*bn9|5)UG`cqk^r}9r^jkquPRQ`|`@}J2+I-=}N;aL8R6Ztdli$t6klAcLAI=uA2 z@?)m*6Q=U}B2Fi?3bmYSk8 zBr)j{fer#c#G=Bj!N=lFC*n;VnxMF((+Sz_34`Q27Q7W{705mv+)6)ntdlJZi>6IM zqux_mG&f4~_t5ESvvY+bCcuXy151kV;k|4DAKuFioom#;H>5afu0elNW5BAD20d1t zG-|0Fl{2@f?Zm>Zdc!#yqJPSb=4o(V-fLb7m*k9Pr9!OGwsP4?A?QbH)_`16dskvJ z=%YJuUUq>SwetDe_+8>5fZydetXCO-C!AIS$Cza|#K@CY#OQIVYi6}ASn+Cbk^zg- zP1OE2fo}+WN8s1CLH1YxM_WnbEFBWt>`YJP#G(|8_PU#vcNIvFF zF&T;6S;FVu=SOhpu`Zo{QX#d5%YB=&;hDCvO)G+pgjDyn7OQ`kcQh9!DQ2K_uTr44#u`gcO?P@wcYtw0rc~*VR)N-cajc*N%xZ2geegxWdNciYh!&~t;K0I zH`qmMy+b0N5|J~7N+aY!59Jw_Qq5KJ;)gPzxvlx3*8rmB@vW7M8mRtpB5K2uM{n8C zBw3{gslrYbVXK;WATqD8)LNxFsxmt5E3M8|R>^FslgJUIMgrWxYpEm=aL8Y_j?%Bk z;_Gc0kIV=cqCt*6lOW&x{4!*H_W4ypluoXG!41x~ke$UsR5Kw35#JlBbf>&%*Szq% zuxQdNuDL|szpEtbFVx7H^091|quQ1kIqszw9pi+u;QxsL|+k7mUb^s;>}z&E*(eVl5dpH?;bHL(9HFuKAsSVWu>*j zvc@V&6G%j@N+;eSXt`CC?6qsq!W@ee;Y3}s!<|= zitDM*b#mW5!_opR0S(mHtM-@{1Z4B;Z;FtvA^UD15WwVC;r3BXL#+X7T#1KiY;T~3 z8|8C*Y%NQ$XiED3qQ?D|ljJVRos*)UC&JNat*x`vn;R{XrNJR_-@Tc5^R;+0mZGIi z^n43}dkM($*}dcShp_lXo9UHF5e8*~wb|c%Q#b53>Tx@PEd=gE@C%GNH;HuOQ!~`w zXthc9CMS;hPU`Zwy!zcF<6bK6Be1LO?sr=us)WiD39Kd%Bu_k2k^CAKJrR_PIIT8c zBsoCsuggyyndX&5zI0@={sXM7ZyS5`2}6rtCQn4wtSObn<>um9<>nbv7IGOq(Z!W> zrqxuJg-@xO;;(I$8`={SNn1~JWyLIWNk!G%Qh#fm{BV0btdrgC?FJ9_kv^0+pUTo- z#NxiTS5FPs!eaTthXV{hVuisVXMVKIKryk_z^>m0v27226eB>gyz|TiqcSE$!Y2QG zrU)0tp&yTr3)H?q1M+XB{ictH!Ef@Oj|Uop1dS%#(Du)d!@`sQPK`X;VWnn{#I1Om zDxGrlcSTdfuv!Wy5J7+s_Zey?#ih>((11%{5_l7Vrw>`brfTk-DK&nLcgmZ-Gqto* zpU(+!%ZHKW{aq+{Nna75A&@>HKqi+yCBPl$3oM!9q<>HaE%?$qSn>#VhbPEtTURI9 zZ9HS}7_3*L>Ue|zceA*FpB<&jV+4-NGtVcCc@K-8KC0!!=XKFc080%8R5Oy@36KgG*)sr1C8=2DiFVLhe~sBG;UZ|3aG_Nwn5h3T zr0NJpkna5{$xR4JPvCubkN_V?`w}O1|#u;Z)CpQ<1-5d)y!Uz@`2g8j& zQPs}`eqrTtkPX9`Eev>JP>IGyDq+>}kXooE#83pL z1c?jG(=wj59j9)AOwOD4vWMfL+Q>6HuJn9%IUYubW)K|$1?~w6@K})c?O+YN=TpdF z&u4+zZB2(vfNb}z888u`kiD4++q7W&)V(STFcSis*%czjdH2nSoN4m@_mkK|xlryt zoegOaVcUxB{k-kNSy(Qth_Wrgx(sjKJj6NkRSIj(g})BpB;0ut_n!k%0q+m@pQX*w z&cVKCjm9}7PLlEh4o;2U7`;h*XFuF#Vrf0B*Icb#s96uL zASu~7ij9kbWY%JW^n(8zb}TO+5+IjN%7uAb=%U|-~12#LF(A$aSw4q=Y_Z~&JL{_sy{qvbX9@=NS*;} zjuhyz=E%@=EFX;xtub0Kjb=ZChPEES4mA<*xyD%PACr!1ZKG38VMbM zaqX2N$cU%K88P`t1{Ras*uRRP7?!$+kA@o{b!lZZR+&tYzO#fszDT zQalFS@8D9WNHmgA3)r+W7(8GqX;V&sE4QCYeEnHVCBI|fF3iKWm%(-Lq&sUWJf(}{ zE7ew_#I^L&{A32CMg?ecpPb3UE1(SCch9LnorImq?Ae*%>Jy@I;$GSJbAeOGUfTzW zK{ty!dbKA{`aCgbs0&u37b!nc?+sOH&TwBHhM`Edwi;4wqG3nS6G-q`KhYaNG`PS_ z3mgC0;#`6l?4=Pb26t);5<}F_VkmMB@nNphNZHOf-;&eQ-i7l`4V-$6>HVg!m8>Fa~y zC6dw9BC6Y68ZoSsaTX#bS|?K7$xkH1M^>Xzw|67LX><`Zie0FInCPqk2{9Up^pT-3 zWdap^4PqbfIHYjFRXoKQHP;6;+t4sB&5l@hY$?Qy4G{3HbqWE$JvEx{1QgM`66o7W zf;^vOf1ZHP9shR{LemB6TP}>G{&?JR9I_MTL*`-I1iSIUgDK0%9vm_cb-+XNG5fW2XXsQV%deLyC zk~(=RBHjQlgFkao7^+A6ikM;==f}1$ggz}jisqB7t0Q-raX)l2Tkp=B(#?7OhF~X` zLS%Jd^t}%3Fm@uH-f4h4E2^6EWMjoxf8MRT$<{N0-rf321bT7nZYUzRM^~YHckA8= zT+yw2M(XC)=+eIIH&x73d8XRzS&BoWtunfLCOfha^uzWcnQZT<^-d31a%nUJ)1aRoY{=Eod@>9t5Qm9>vtn@g;lyDod^P)S_%2P0 zIJmp~|HF)3+-X|%JlTeOZPz`5cNQXqRw1qhcih>5N!+{RcBbGb;0uX9Gi(a7T_=uk z;TxKj7^nXvk~mUjJM~ciS^QBdeAOaSB>qVPU2V^zmt0|$>6vKv>4-@&I&_iW*J8xc zTu#NMRX*a{o z*D%+6&Z1CI5l|HDN}%_g^$-HlX&hiMcy8sAgu+q)O2C@2KbjZpQjZq$2{aU}vR0Ueag zRU;+yaW<*Oxf=oGU)TKK5c_#av=df79XMd$w z%Ks|4qS`a2_*b^cJ@`YJ3QH9`L?4!3BO)4~9%EI)iwXW7)S;V$cXbZ-ngwHJz?QzH zDWj7WrYdSV`gW@I=4O97XPkoA{>j4Q(qttJviFukv?>|(I%OBa(-a}Rb4xD3P zG!@x$uNIwi-07Xjs&`)9$fEy}&1vtaGMU%?x=1_2OSZdHb|Z^KyE$biLwnbCH3xJ* zvBJwD^NJqa13SfoJ3DYEhBJG#RHe#NIXb6msYajcGPGV(QS>&e+_MvxZb%~eL$~O? zNwB?(Q>#khj#=MQah9m5k8j2UAXm`p1bJy;LR53GSnUeNxS1!`;GL~%R&0fs67_VO zP1S+hRo+ls)x*^D5t<0CXTKuA56dmw~nyOKon{DfNm-w4g; zHZmcZ`2kai3%moH${p!XQSKVVdtU`+_uv|+BPY;HF3=4{vG0|0fu51Nc@uilLcb>& z#YH^Kc3-v^ca>UPs0=a1ef3MdB}4!>N=sFQ)E4ZOP-i+_<8tRkN_yFU*N$ETidWG* z`ro|9>P3pKqu6^&cqIb8jN+A0RIko@-?a2b;MzJ!9%c6WY|;N6rwMFJL~pTp8O4ap`4C4sN2oH4T2{Ck;vx%FRSC*V;AGag8gzBt91u^kMh1H0 zN!BgRAk)bmP}vSS>`KY9vQOQ|9W`^>{!{^JzKMVJ>|X0N$l=55R@2# z+b7&Y;W1mt#*n} zD58$;bkxyFecf}^?M?)EN-HW{1<_Y;s7n6e$O19@gfBOd~(I z8&0*E>_aCEXp!io!lSRZTSSMHL=8p6;2@R!=eiO<+6WxXry`EjkHEv!IUP%QI>HZd zo}&u7MB$09wKp`@Tb*@wn|aN8i?q@q?Z8GnoR%cJgc3@g2rE0)1mP1EAtICp$?9md z+ZRDL({An-f7hhs@R%UK8G+;^>qtoSrqVE)5tXwqqZ(1os#)BX@K zSy&&X{dbfeA8NHM<0*&@y&XF-jSFu`(^*6^>w7m8gowfpEq8Z0sxFP~JPuI{=oYm} zS1I*DedgdHQxd66r`aBZXt6ffZFs4HA4~cT;)$-W!<&@#4d%doN^TzS`4%6-X!iQu z5FZ&!)CjCGq8b(K?A3fsPBJiD9T=|k)y>} zchx4iMr#^J^1ymixWtdSbd470Dt^Pz6J%NLpeK*kVQrMk*aaEx)xV8)bGUEa44>$N zY3y;}-e!&WLqBk^yYGh~@FqKOKMV?eg^WyKi~FbhAxTI)M|{E@)>_HxG_SMNH(5DD zDV7a*5C(=)vqWHz+wveJ2+B>PFxK)Al%|l|p{)j^?9vBBnEML9YZJykeh5aOr^h`E zU&9IZ`@?v;f0|8bg(5>c8bC&CIxDv?Z{P@-+d?$)clWw!qb*kJgc{J_J~cS2ugdltgn<|pBjP#BPHLs0Fi zCfk}Z>C#t3>@0!}M$zK5q_1|^ZCK6kzRYAdJp=tKNM4Cc7{n!1&f&kKTE!YvdKcx9 zxV(m5<&`=FIr?Y}Gd>GbgS25NV{WOzd`y>E5=bV zBwkOed3BH&Dg((;M`Ky!^Dx?wOsrgFjzMnM^NAI04fDJL4Tsx~O0kt)j}WR5Z29F!?F@W|GRc*pl6N!e8m$ zvl~hwiJOz`kGE&J!M#)Fyug<1g?_AI55%#%_u|JGR01=*aSu+R2)1}Hs%dq18F&m)#cC)U+EnR!3t8Mw3^?9Tl#3)Z+# z?}v?AIM0^64kM4n|D1)^?Zx54b0`!;_(gH0qkczWo$$BAsUX&!7lKu?e5y{WoBDeTEkR&X8!I z68M||w?RN@(pS-jtI5UXcHs&~Q!BR%S0R@kO$5l)e2G*tg?6G$;7tmxVY{cCt8$Vm62 z>ol0pNb}kJ4v=v*{-y(MwvuIEfGqE2ox9+#to;J)UqEw9$vlXZ6ia~kN&F5UpF-Rx z=oYNIvV?VLh17YuNP1_lma>7tL)oxZfG$j8{&sb(wUK9dHVa-jEM^~HgbIBnc9`QHatXdOfW*+Y;~|kPxeRG@xOMnd zo12I;pO{?QfY13eO6d@Fc>@6&dKU5IpCoJ})i+X8HPzK9G5@p6&{sbjD_(T}b{R_a z1_gnYf$%85WfZtJk9i!Qg#0>_f$0Syb0lFZ6R?>mlNCJ%p5r(aF}UOND9*+GhI+lq zmI^}B0M1DvMre`tb(qK2iB_}s>yoq7cRrVuZ5M=<`lVQY+Z~}5HXEWQWBFVG^>~~> zBij`sSdw3*VGAZe^HK?ke8^+9Y*47+g30XJP$AP;N>mA43U{9h6>irW=26XMwkkr{ zievUigdpi{ST&HXixj>QE()wRO30+ZOlA#zgl~osHclnpg`0(6{U?+2&u=J~ zS0<;E5DC<5>GMcolEZ;Rtnu~Y!VsA6HpSxFX;do6E^l9Y?P*xlS$ zNH#=qhLp@nGYHIMPo)UemethFFAJ*KSvcQawZ$aGDvn1b?x$oziLAvj-Axwhh0Vn#B1 zg8kRJX?waMq?kVDDX6HJX)<~fZkhpnqVqZa4q+A`@aUTw8`1Z^Ve2!5R9xY=WC(ZY zHzTKg?wm~FBVj%xuB4kM(rmWaY3#x#Bk%x$2MKH=@DPD6v2`oe z5%91DIl^i zR{9e9r%0x!2|Po9Ud!QxduOmvDqM_k_ZcEg)*7m5NP^vqh6zDXRe_>c;)DpqKE6=A zfz`f5P6=lujFBXuy)kK6gEd`0w|j~094@56V0Lu4aJ1xM;?6a1rBW;bJt@t@7^#n& zK*u$<+B&;OU)^NIJsTf3Zn53e`z1CuM@WIMw~iFPXZlbH-r8!e18yhh~%Y|m(62o$k%qlE@I>z+SG_#p@$V>2fUi78~$r}e7wdz?_n{*r^; zFj=ri1uV#o_Y8v);d*>ga$|`wDT_S7W5n-Q>S{~)!s*eKSC&;t+$sJ}RJOCPN`%~H zJVx^IejRH)TKwQvI!NsY2pl57h2`vDrbn){q6|H9k+_?2L-5heq=x?_u#44~3N?l| zsC)?%o4(9bDr7;a`}a~ITaOt{)^uTb5?6pnZqAi6e2Y-tbFZB)vNE9sI(!%@(G|@yT$6nz{JiVfb~nkPcf} z>Kq}ng-f4CJ+>1#Mu6LePjo(|k5gS>2*`|>lHdo*_?@BC;#ebb<+Cml|5mU`I;r` z+9oS{Iv+GXUVI>Uu?*wpcq*HON@*y`t>r67 zWhEBolf+0*0j9{ZN@)#Nl~tDEuaf^HVB?nyMJ+cXiLy%5jM-vH3{f9YRD@FJ&5k0k`A5GW(?4{A6}Ae#V>3p{>?vcFgb(aPU@=8*vl2^10F z$q9XoE4@WH^hvCAhyZ;>CtW1KlWiW1=_4^^AQPz+LaiAD_}2jqRQDUJT!Ax~zslq< z0_nXRf8WMmqVY|OL^}>?H~~tjC7z`&rBW$@Kvu$2FelaVo!fqk1MsEWM@6f2= iaxD|>!ruqgg+NY+rW;%oV}KgK&mB5L_xwg-neg9ypt%ar<_BQHza{%65mV&;>1DLT~YKC zeQK+V!g?PlV30xaUj5cxe>K7EinFk?9^ZN!L=?f}tLisbF4q0OM^fom-Cb2(UDaLv zUOnDB8R+~tF!bS&kRTOacO_(*uleRs=*G(6V7#T@v+037j>?+_wR&A*PsD<6HG-{z zy)6g{YNM@EhQ(=eeC@=^!i`5VRJJOjp%T+51gd4Q-myHR+ zdXNJ}W6ujKk|CrdB>#*G!R~+oVNTB%E58$jbD{FT*1R9u4u~G0y7L2b1vzo!v{}

G3nX&8&w3%DYZDyOaP0}IS6=kj8Xf`!U_10#SWbSB}Y%P{1 zqgDzcF8`w5tpvPutXjNHrAsQL3WLnHX2Q)q&u!fp5KNs~K2cM9Iscl%cts!7Y$kzO z1d0jhdO`?c#u4x~uxhaVhSa%px#%ukV{41qVw1A4(ej?y>uwRi zC_i@na@|BkoD+JqH!K4vlvnTS(N05*p=ZpEr!|n;b9Q$efUYOxRspoju_CS~N{-in zPM*Nt(KaHYvPXYsTL8?EJNIpe-^$<^P>7GkAQFWpLFQ%{LFDf1GJ%5QCH zthbp>ZSA7D)hZEBiO3m)q$%=&KV<9H5@wxz@(*dy(DU;jUIYl2U%0^$t-Z-?Yc@-!`W9{4!h-CZf1g*UDfoScOSo2WNeGe=QK^eX}v<+eNQ_aFTwS4?3Cs=!K=qVM# z?bKaQV(g&OF8S|IuL!vji)dfmfY-=#j+BL8OO<4!%GBEBTaIiKeo^-roRu)9i||(w z=tf`!>1J6xIwO;#2_tRyHgHhsY6A0!r(J&eXlx9P)~J&#gzzmAN z^_$I-G@Im5y9JkfpN$IV&3G?P9yIN3thuI>vciM114^Wc_ zwYS@XF*0U0hpHQ@X%ZTD!osuws>To80Y+S^WS}4JlnPsj$#n z^E%-oORFntt4yWUwN<6wnm zsjfAZRWDv%?yYT>Q~w?Vo$~a*zpHx>+ej`s`ecUo8!YbYnSFAK8dl5F`?+Q{``;<`+7Bng zkMfKU^K?2vrTRwOv-v|qNW!aB$=4=Cs+LJyWhV(~m2VxGK0g@Il7T=70Y0~367Pr)$Ps6Zj=j|80|H#}22#AY2`x3!#{_5wr1uGs z#--B)xRrc}C1bSo7ede`F1>*zm(Xr?>CKiaTcmaij{;me^H#JHQF@aazD0oBRJ6~i z-X>%(fj;@IGqE#|W6?FnZ!hqe7}=fl4goSYT(7yu1d=L(2!zY2-&AR1v6$G?_RZ1= zNMtXoVXC)9ZYKd`^VLMBXRU&lfX6tX#SFbgiQt~f{! zuO;?e6Aw_+3v7QJh73^D*)$X$~>TVs-H_wz7oEz0{PJKZ!Kz(l^$3*l2_Ig;Oom;zt4%)Zt4il~d_W zD$Qe0#6wac`i@EpL|}|n!Z{v2@!;w>HS+~r?l_fQjEA~l9(eKLu3#$@U`9|X(IGI# zd3ORlpcgI=a?Z_w`vE?8ev}C%0Owg|7Tlv2ERQ%}%>sZ3!IEVU6r(qzNZ9vPTZ%l# z74IC#hGq!g4VJZ7ne47CVjGRHaSUS95S!-4uEZ{;&l2f;VPHRJ!z@;k1M|ZzuVR}_ zcbgY%DJ&%tlH|Dyf~ifrI@S(y;e#|j2`Iuv)I&TEMv_ZZHw6mMO@t?f-FYiDDl*=m*M zQ3LKAWmMuG@&=K5g@D&Tq(9Q_s{~#~U>xJ;8r&Ov!)sOH(j{^4@HkCf0c61nwzUAp z#Uzth1fIp6#`?ZirP{6<&^XxP0yq}d41v*SR0MZc4G2xnt0%x!0xpji3ZYtkMx|pn zrobF_-6Y7>52$p}V+TUn+%!mFk4%JQ;hc`KRLGcgbVYCNlHMsx@ux3oS%1>90lg|Z z>0FR1I=wf&;_3AW@TV`TvOlV_H>&bfR66@E4XT85QO+ZiKn*2J6I6eUtQlLEEA$x! zyy?pgS~5f1S2SbE6z%Z>9b%48(IV#fj8rVYlMuAbsD5YMjAfbX{!HyMBRfAC)<6(j zHU-jQj&sWtXaiveu%AAMMz&xo?oC=S_Z7gjXWM$EcCoLo}7Xe14+rAVspppw_5@G$ot>zohU z;a=x=^WhQAJicY#K`r@+6qKdLnXZF%wB{;6;#RW z>Kd>MHi1QyL3Ef#Ss!)m(Zc6}VSR<7;xt`)t8I~-f}H(v2s^0}0z=&uTLhI|w<$W+AfG;ZXy99ZY6$odJ&JtzI!nOSx0($Ua68YZAQ->G4+-0@DK*T8*T~? zx2v#^m)rhOmRt)7K=LI-7={crhYwX9Z42SzZyM_%*d1aK#4+6=C~BnFNN%uA@YJJ) zfEYD+)0E1^Xk;DhX@la%kq-Su_OC}}MGjJy1AC&j#`-t%5}E8FORRlNh7sX3h6oxn zSWvY03|ooJWAIT!y@*MJ$j$H*{y!#%)I2u)>x6R*o+hqgpL9sh6QL>ODEM?=!^iq7 zG?y}HsIyfnvI~CwT5??FC7xc~V7x9q3XRl3Nh2Km!J1~dg1J+4RQnIwgc=T;yAY+VjROv$jxx%u40xS zL;uORq?ng*8CqtfbAx8w?HCz|^!oegDj3w&?HJbJLD=j;lkV>>!+7{c?eJe^2ay}y zT`oaxRBEHSi$jHzI79Qpx?dRPsDjG-599Z`)Tw%(Q%WuQaiM zcw05%gi&Oe=;A1X>O^fbArl3>c7Oy7;0zY)Dah(nSTBK1sc{L2Q<|^ActS9a*1VtR)IAh z(<2)bM|R~|Zfy?VT!smu45H?e8|mogJqkH?a1}&5itQn|RuA8iM(H$E1aY>XBK=MH zSN3;^bNmS4Cg!=sUrFHKOho!6opy%ZF=ScdnjW^v+3`GtmRRDkL@837%f(haz|Vl} z1~izG5j>#Ouf3n1D(wcbWF!myH{*I?kb$|knR&yEe?-!Qx{e-y4Wl;l$b8grnY~2( z^&gYFWWY3x8fwuoLaceo=^u%RYX7j=A4)?^p;@*I`^yFgLk?~vD009_C?CPR=3_gDH^lIg^0TyGOKOPFL(T-!^S=4U35xSy!`Rwp z2yqOPSS%mOz6X&RJq#H}ZRGL$sF4wl9whz8gPK~2#@F5BhYIL?M z@vLh@qU;e!ptgpY;BJlOK3sqncx~$~@iM-k)QAiDUnST^%i*yOF%zr3d+UM`3}%sE zH-9U+bgv;p5mN?nA%AcTIvrfvnDDUB;+r`X*|Slm4Z_waqqfq4*LOZ~2b)9VW6|Ft z{U1G`q(j&G1^?DC-S$0%$ztWW`~O3*);*TNQGMMYq5V&-!+4xS&3^U_F%BNt1ENuvjrEV6N-gQS{H-(YK=&AolmjidbhK%nNP0#3* z-G-@*{}K_;s-X+c$m1Vm=cAu_hf#BgBhS*1HF^;FVMI6$k43x9l7}%SuQ3OU8~9$K z_?XWi@l;+B@NOcAG3?Hbkfar9h%yu=j`&dWSJED& zrqP4q|CkzD$t9lq{m0~x+UN(pk*KLLC}ux5K|+(+ZTZs&t5F;p;bjj}h2Gsunf)@i ziQpe`G3{pI%|`8;vmABznxBK<`&Q*QP8_$r>uQ z`e);A8tyh9dl9J^E{(MI_w@RU+~WoN}Z$waTk{tHW1H3CcvQBp## zLndA6=B8`YL`;+WwUi=S9Zdh z8HH^LYo@d{+iV@ynG+@uzM!=QQ*T?_3hG-Xl!*1V`q^EYgf$b}TJAw1c?OY$FT61Q zh;DbXuQ%ZfNPKC!8O%jISBhSgw(Q2rmOBYNLCtm{z{K8MtdjBwcw0)%?AT_w5)QFR ztuPG^vQ4e{e&_)ET`M%ge)d@_v;$+S+u#n^%g(fcSxLxxfUUK_GNpW!9kIX;9nVj? zz{=X;DR`ZI*A9E&X(o5TKTSUo$6X|IEXk3tyr$$YF|VPjmk_uNol#u;sSjt#(1(PC8R$INT(<+hT@g3@) zu7GqM*$XBDe0v4tI$YY;_IlAO#S?BCiS-JVe6RKSn^pd{mA{uwM9NYUfxXl?1xv}w!qg=b&A2Oa zV@oS$E}2CWHxC6F<|~^vS=%ixgW1xE8NZ^5jDqc!@D=|-3C4mi(l%6=_hiEZ_@V%#NRoqFChCw_H^ZDdS5{10KV{zIBiJj z_57SL#Oz#1|4t>O_a|T$ye_016u4V|DkZBwrSM>Of6C0=hEtI-cN+R4$Mr{!dm`$1 zWPWdC{t50OudjXAvB-3Mk*RM{egC3*^qIb#hJLJy%04lv;`aIdQCa)8^hHhRjhgWB zq@shZeUr-iCzTDt59m}W#yiD5o1Td2H_qxyF78h*-nsa6LfW3W>W(yFU?AY`SfEsvq)#u9~g> zVd^~Ls_E*B)3tchNQ;T*qNm_=e|9J9z5(h2_Peu*G@tM*;As^2)Xcf)B-03{Y+g2KBj>~=^7i51)q6XCDSbUPFT{fSJFz=KY9JH!id zpAa8|)!ZnVZKf;hTRY91A!h&mg+X2rRZ9e3KXNC;3QDGGFiXB0$`fe?z??#Q(JcKB z(dAhn?`N~y-+BK z-;fZGBDG4Mz0nnrZo^#4^qJ{hLVEY7zu+Z(hfTV4YswebRIFPj%_Gq-5POaE1J<}S z8(T2*Qb&AbLI)~RT_nP8j-~g(?YduRI$i9=$6+iBc^C}NpYMZjg+iYIUHZo6PRr(* zsnS{M`z?a5U|JedX|+H!&z!7g84tnug(Q?BhMtR|q*`4dtTJRO@gzVllx~<%R9`}4 z`H9`}5UkNh8c@6%U@tum(N2^vM`$F`T*1C|&?+GU)3piE$lS`{qMXO7M-`3u2|&Cb z_2>N)u~oD}r4sp;ccR#nkHQRHF7daMdJ3Egk3lx1a?J#y=MCi}g&Ed>sGi1(Lfv!#b55(B?;n7Es8V4&%<88sG&3&re%9khPOZ$7mx#wd{4MC>AxySf z?_Bv5ltVn%A1NF^ir^Y{JHkbQT|5lQ%y1Z@S;5ng$xG1A(hs9!B(MvIafF@D=BHtj zI-U;{Y2XJo2)VU+4D9exNHcQ!24$q&xQE!*5$e^0fc3K}`V$Qdphm&LG z@!V`bYa&KSz@4DY96&p+pX{{t=q z+(sf!!|TbEVv6gvK<&9k_;jpBYUFI}`_oXXoq-H$oYnt?O)#ByQZ&gi*q-P8HEt25 z3Yr0Ky7Q^@3U$`e@@U!E(q!U$?HKme2T-G(iT>2p7XB*k8UqFUCZ}KL%mu>kHp1+*?7R{?fq_~$U)6R3jxesxppwA_g z&&h73#ueDu6+|DAxy^tUL1_gInup8B*o|L78kDdnzrsDWj7XQWFTR4=m@Jv|H58>T zBvds4%CvWd`p!?5dQ4Pb#2jBkwRSZ&5S_hW!LkoKOA?x9P%bu>5I0wRn&~J6oa^ z>JyHUta<|Elge`FUP;+oT0HUAu!TAyJ-C*r63~P?x9Wr&)Vc=3Twnn~!W}qWdxHc? z+l8oNHa1v5UsE%+LCB(sGzcZytFflaX)y?6V9I>zxD8pNdTdngMBoZ*thbozMbXrT z`O)~*8GfADO_kjJyV>CoVR4Y15CoR6kWk_C$`WRXB;DuyCG?;ngGsC(q@%mMD-4CDImGjnSZV_V&%pUTQ4Z*;cA2@UG%i^P^>b_IAp{h_0N zS70f9t(jk4y~r5s4xq{SxsP-AeuH}EA#iCsJ37$zK4URSLJ~x=q9kFLb{}$j!}(g0 za8g)#KXIk8xx!4QdW)qUPt3RxSWPDB0jheC06j*M9wzVzfkz2EMqtRYdLQ8ke8$eF z3Qxd&&WF;3X24@XScVV-XPwy@!WCMW!R{I_{7o%AA~2g#sDeE9yiwSqe}o2cOwG0% zg<>dRTXTdG2j2poC50a#+6M{nA+}QES4eI>p0O?Xsj4E|4rbr%fmw2eazP#DJe(`cQ|oGJLiElb3j{ql7NY1i=nlR`kM9tDi1tKqd>x}s zu3(Y?pJPZg!aBqZ{hInk4Zd~$P$Znw;}fT^XA5zO z^ab&wTEF=|L|tgUB#C9t5!%Cj8t2MB!tS3VTm#RraV5gs46+zkFn<2n(pbiKSeIr& zP32;V8&NM&d6Mlc5wbS$6_wk^|3j=x-PU4}-lY0B2)spr3(MKRM7LZM%3$c0i^R>6 zYlqK%0af%7c%FsM6_)7Urt(<|fyOfXTp$=i%PPLMq(HUR*4s7jW@&sRf_NlT_l0;p+ll0ZtK4nFAJ9D~w&kN6QCF zj^)yj|GBh{wiev!xsyCkJ;>jTiAr$(0U3p^ZPb?yH<#)Ag#I# z6cgkBHNXvAnIt|xeBvVTk(n92*k;1>bc?mwEGpkU@U`OiWY?!zZk=!ysM*7HxYHeV zzEvme)CvdH@_#kOIy+Vh&jAm!r2ioq>}=X9AsMb?D^>{^MbDAr^1yE{mTT z4~DZyffET#BfvufdNm>anflQGav;4z-~<8sP+8*9Hs6kP#EMU3ER_PN7T<~}Uwlbz zg!_>V)C=hjdUDGj=1KIljXy=>kH+|^MWQ2z65>-#E%8{DVn`nO`Nbv_gYkIE7r;q> zLNQ&M2AmxpZAcUi@J(A;0=OZ~X16q;n#VdHYY;YQ3zujC|L2jB;Tewq*1xtj$T_Q7 uI3h&XYD)zO8BqDbDh<{9&`OLHJ*Gu%%4^ diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 0ff9280c4..f71218ef0 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -245,9 +245,8 @@ def deploy_proxy(model, api_base, debug, temperature, max_tokens, telemetry, dep # for streaming def data_generator(response): - print("inside generator") + print_verbose("inside generator") for chunk in response: - print(f"chunk: {chunk}") print_verbose(f"returned chunk: {chunk}") yield f"data: {json.dumps(chunk)}\n\n" @@ -302,26 +301,6 @@ def litellm_completion(data, type): data["max_tokens"] = user_max_tokens if user_api_base: data["api_base"] = user_api_base - ## CUSTOM PROMPT TEMPLATE ## - run `litellm --config` to set this - litellm.register_prompt_template( - model=user_model, - roles={ - "system": { - "pre_message": os.getenv("MODEL_SYSTEM_MESSAGE_START_TOKEN", ""), - "post_message": os.getenv("MODEL_SYSTEM_MESSAGE_END_TOKEN", ""), - }, - "assistant": { - "pre_message": os.getenv("MODEL_ASSISTANT_MESSAGE_START_TOKEN", ""), - "post_message": os.getenv("MODEL_ASSISTANT_MESSAGE_END_TOKEN", "") - }, - "user": { - "pre_message": os.getenv("MODEL_USER_MESSAGE_START_TOKEN", ""), - "post_message": os.getenv("MODEL_USER_MESSAGE_END_TOKEN", "") - } - }, - initial_prompt_value=os.getenv("MODEL_PRE_PROMPT", ""), - final_prompt_value=os.getenv("MODEL_POST_PROMPT", "") - ) if type == "completion": response = litellm.text_completion(**data) elif type == "chat_completion": diff --git a/litellm/tests/test_prompt_factory.py b/litellm/tests/test_prompt_factory.py new file mode 100644 index 000000000..11ebbb424 --- /dev/null +++ b/litellm/tests/test_prompt_factory.py @@ -0,0 +1,23 @@ +#### What this tests #### +# This tests if prompts are being correctly formatted +import sys +import os +import io + +sys.path.insert(0, os.path.abspath('../..')) + +# from litellm.llms.prompt_templates.factory import prompt_factory +from litellm import completion + +def codellama_prompt_format(): + model = "huggingface/codellama/CodeLlama-7b-Instruct-hf" + messages = [{"role": "system", "content": "You are a good bot"}, {"role": "user", "content": "Hey, how's it going?"}] + expected_response = """[INST] <> +You are a good bot +<> + [/INST] +[INST] Hey, how's it going? [/INST]""" + response = completion(model=model, messages=messages) + print(response) + +# codellama_prompt_format() \ No newline at end of file