From b44299cce726a74f5163a2bd4aa591a2e67f3eeb Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Thu, 31 Aug 2023 16:42:40 -0700 Subject: [PATCH] update model split tests with ui --- litellm/__init__.py | 1 + litellm/__pycache__/__init__.cpython-311.pyc | Bin 7745 -> 7793 bytes litellm/__pycache__/main.cpython-311.pyc | Bin 28135 -> 28175 bytes litellm/__pycache__/utils.cpython-311.pyc | Bin 76900 -> 80294 bytes litellm/main.py | 4 +- litellm/tests/test_split_test.py | 24 +++++++++ litellm/utils.py | 52 +++++++++++++++++++ pyproject.toml | 2 +- 8 files changed, 80 insertions(+), 3 deletions(-) create mode 100644 litellm/tests/test_split_test.py diff --git a/litellm/__init__.py b/litellm/__init__.py index 1602f8ffd..57ba28b99 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -286,6 +286,7 @@ from .utils import ( Logging, acreate, get_model_list, + completion_with_split_tests ) from .main import * # type: ignore from .integrations import * diff --git a/litellm/__pycache__/__init__.cpython-311.pyc b/litellm/__pycache__/__init__.cpython-311.pyc index 61bd76f5967023caceffc0483334eae34518abad..8740f2ca7b72e8f0af98a0f93e487158a0b01b7c 100644 GIT binary patch delta 333 zcmX?T^U;QPIWI340}z}L`0$*06* jxw?Q%Mj$Q@+Wb*0oUNXhk?8}&z!AL2w#!UDC0h;v)!kN4 delta 286 zcmexpbI^u&IWI340}$vWyib`mkyn!G56ec46D$Ud3@ITgVLARG5IO)%N8|+N21Nzs z21fqlmIDBEyG6hN diff --git a/litellm/__pycache__/main.cpython-311.pyc b/litellm/__pycache__/main.cpython-311.pyc index 0d73a670dab2bc0d8122036236963c215104ee24..d9973c270e607d3b26d039dfc4f8e3de9df5f56d 100644 GIT binary patch delta 4760 zcmcIndrX_x75}b{ZNNOj!#IQhHpYMYCJ-#OpM$A4sx{fQNSFF83;!k^E2RlfF+B)w0Q=r29Oe17+H#qTg9 zle1)LM3T)jDGMo@AI7m1cu9XstMSON(@>j|g4#*=Xz}^{BRFNS*z*!3ksZtzxg*jz zi6x$-s3Y-`Bu{%Jkq2)Zb{X(Au{E|~ao3QOh`I#(mrJL|1vqsq9YEBmX(^v&hB+qCIMIk(wm<2s4 zpGM5>OkJldlpQk~C{G$Omd+2T$Vk)KeA$W#SVkuJGg8fVx#sH_*P#u%e1@Odz?Es&6^SBY0cTO^ z7WAn;n^K_ughj5W>|sDHa;D1J)%x^_JflunnSI;BypxliLtN7I`^j53ikO z8-Zn4m^Y$>9kX$G4SXWks1Oy9T98v=QSOS$FpO9dB4e&|)_}VBM072&SI)QGqy{iq z3vXvXwOmwYcsgf%PXyb(HYF!|EV{gNmZ2ehB8DnsR&QIeS|uv)*T}aAD@7HKy}>GQ ztuHp?Oul1ABVSiR(bz@{@#6YERt?3u`?NKp8jk0h(`r^6T?r8|NfY=AQmhCPc`u&azXEvSR!KEIZVj+H zxNOZf*RA3~y{K0`Sk3;803A~69qr3Q;LOWd^5CwUdT_d+*wi5FWS6&jq9wRxVq0)4 z)_%hoC$NM}_^jaGC>*dUN;kqmY=L=uv93`xp6O6`94kvH2sR-~n#MY#!4dYu3}0B` z0$3`09Udr(Agw4rttb|YYOm}y-B{}svKI@B)5u}j+da_}Y>shu9~`hR;rB}%ZYGDV zE$Cs(*!KU6hv$ld+HJ~{DzLV#l2)g1e&xQi5>|(An$&lT+gPhw&&?%E@i1V#pP2Nz z6_4LHIPCIz2k{H@Ncs8XeWa)1132)5P_Z#9ub{6Qj-qplOc7aO0)@0@2UT6>nZgyLP3os ztFXPM&Uu^;qbPFKQ2cdqA`tNhiJTx3HcMPJ63hj?Lir%PQuD^GEw%UFn?|^g533eJ@+ww?!Pc-}e zTPySV4%L=#YGUw^+cgp3J4yXEBDW*KN?HdmZ7n%QAIaQ?BxLaVN3c9b!ns^ZV<>*9 zTGWeh(#^Q8{FauTbkqeRgtU;}Kc!%pLUA6&J;8${zmLf4w7ACQQ%1Re5?qb7sV7PN zCW$q}e*PAmZEQOBTatyZNDGO>qb_A|!Y{kMp*Z6CX%Z*h?dDia;RXHmnoZ50IVZ`lobC=@h{4&i`4!Htu#f^<4tn>^D`Fpg&+yRQX#bN=u$qAE9Iyl#q zX?iP5lJ1xOekYTX(!W)Ivi|<|Ma_|gUp0NevM%LY7fr@>iB}~geA*o3KCpsHMee3nU3UkrZPRLiO!x%?S-D~bZ1W)j%7NU4`Yhc_I#gp!;klLvwwkh zZ>>&)<^29at6;@?DHi&}QwrjAMKdz?IoLJW z)K{thIV$m&VZ84s)5Ax7?YeY4{x1s^2hQtu@4~e^c`a$&Nn{U^y-2V}E|&yV13y4g zjf$9X*H-I>ijKRlTk3`Kp~_v4k|ffj>FuNT7DpO?p7`a0Q)^8Ht7<}RTQ8tWD4}EG zkXx1s(277r`kP=+@&oO$(#) zA;#w6O?PwR30fo4wUBZ69#*L4NXO?<5;6|^eTO|GgOe^Sp;S`LBl024kJRbxc+&0g z>BvFT&qzo?32D(Q{sB$@3uLTfk$lpL8}?65a~;j+;rY=k*nnm{8=Qn0 zPMPI&RZf0}+dSC`zp|)Hs_&|6@r_uZQ(kv7uftXRlLCKONEmxr!v^4&UR&KIgK<$W r8Ivz%nl7cRS?pp1i7atZx{h`;NuROU6T1M`i7bPvw9N#~6VLnycUEuZ delta 4560 zcmcH+YiwKP^?r8j#Lm-s*h%xS<2X&6(B}PWUUgb04UdMto2DUga-GDf6PwRBg|g2J z6eb>JFm$x>gG_5BR>Yc46mOa|ZFCXp#q7=1V2tL)E7U^ z9%Fl;HpK!TCpWQTs7`S+Yvj(9A(p%WkQcVWuMPF)MayY}B%Q%G_Jm7C$w>q3D#+8? zL}{eSc!IGKSW3NQu+fQO`)mP>q-}&>rRhv@ikv2FMMibAF znd%d^3;Q0VpET3a(2<_c>~LR}8O~;>!$qS33eyTtxCdwl187!tWANFMyurdf_ z2G*-L%hd}S@n7rLF<}z+$b8m2OzRNjo`pGJKKlm<$*9|MOlaOnPKCuGH+&&Q4pId7 zWe>yt+-y8EU@b5;6(ph}h08^`up+f;i?GYBs|;b4TRTt;V`-IhRm?88t>d?;RBPA{ z|H$oT8^Pq<1dDl@N#P1n0e9tjGAoq66-5%#-p!k=aLAp);bY-SIYUjFQ5Zy}+@&5x zShtF)5z#qBg{Xul%mvzwCmonJOL66*>$C>bif`-*JJ%<_-6`w;Q}X2=ECT0tX*fAt zCG4W=9<1fC{m~R*$0)=LBbjuTqt+oyALinUrI=O2lBEVdEwLuWLp8AJ>ey6^n*_f1 zpjGm=1ue#GH9C*yAJcA94#2A=W=Jhhhs^~>%O*6^eL4wU!8h*8X0aIt3JPni%HaeJ zMm3LX@=I~ z8@ILr&J}ki3ccGQ>KAWWpGGEWB+)S4l=wnEhW>oP(yv23!oIj`{TyHt*12xAj;EUw zI>Y6Z=>JvH-?#ou+o*(Km(yZt#AFYyHzV96nkd;h7I9U&!Bm=)nn-Xd4z(-JvQkX2 zydTdQ`b^Qp#u_zYs5GQ)R_;|Gh4%3Fx0JZ$^Jwl@fg}u<$0_PPL);Y+C}?GH!xi;EFa9UhdVI;9n-yuTnNVm z^I`3)6Mv;Sbq-k)<1?mp zl4+$~XiL}h&Hoo$&Tb5AyOpQZBz0dSNn6F%&yl3qy|=bRCa#;L_bRqA7nsZi&|A6U zdoo5i82ZAViC?yRoUC9cnbifh*1Y_@VXQAc<)+$D;k zZY0lzzeLI;<-4mKhr})6KIPmh1@mAQeT#Rm&&6>jOX0pR_d)2UKj_;~+G&8L_6^|Q zl$q3pTYybxT>FTw;X}A^x=HprSv1ujn4I*@PL2e|rkzQAka)ESqNV`%A;aSz;oeX% zFdOpnLomPjNF|-Z4}q14FR%io@b?Sp{A#Q<&$WNC+Hhdjs?B(kE8k^ z56lFWb>9c^z-aRLRKPdx4Mmd_@7)SNLP{p#_FAQAg3u`fJ_I4U@@+QmB!VAUb6M6M zc){=)+E)_@LQY*v$__-Gj1R)@x^Vh1ZIa{W!^OHX{p}>V3v~7S*dPqm52da|g5Lp4 z^`ke`s((k-3KI=2u6a6)qR3T6DQn_Hw+g?Hz;OaGwS?6pz~nR$J`8U(ymVuYE_9vG z!268{Z!9|;lYOXZ_{Or2$7HLT&)-nC_J^u08BMW%PgcE%YUjA0A0u2F)%#|Hb4p0v z@||$7Wf$8C-)SjkQ}9wt5$l3?TKe)vQI-b?P-XEn0{g+$TEh0hXzMrGR`|5Fkb2Un067aDkXvo>^sz2ZFxwk#Rpf*in=D2=RVSJk5B3zYG^T zI_F;@TFf_M7{{kP%E(MW_WGkqg!89}oOaa9u~}jaevwG29$$SOp6v6^D*O~t43pkj zFZU<`&i_dJj4@Bht9WM}Y z9o$<@;{>On9#cH$SFjJKDFXV5Ib{w>SmSF(OzohPeFBFDYxPS==P$t52Jc}>ka}pl zE(_27OJwh%GrBzk$lJ>sNn$sFJp}e5z{$5-==+I!kf@s2>{W;JC{f1crtVmyx^YH-htd1V<#KK~vO=^BdW2ZC4u z9^Z&J!HgAAh%b`N+XTK(;4)n%oc`T#rp7u;os+7Yb;N2R&`N*?8h@9-p9xTF@lT-J zQZ&^88d&s5X&4bS#9e;nYNvc);)FlO%c=`sE_b&V+2&ZNLxOBVU!r7&{Bg z-Y(;Dk|5xYv`*a39BTS>F<(;01G5vp$&sK38!M9p)mia2{ARL6SB59u3RzQk^ zqX?o}RL|d{?Hh3aRHLl~Wq1+oUnF29nWXVR@EF(8{w(}{>OCAqOTL?|R7aeW&dDcE zcE;v9>ETn5Hl1&5LKOd`(C;Ffr+=bhL-2vWY}c}0N=<()H}AEKtYtSFGBV?`v?8rQxCF8%h9ph~?u85)7;@&47zBsZsHsbh zp0sG+_-oQ?G@90B)Tq^JG+MP9AreEcP1P=+R^Kk9Nv&F!_j%5pC1BdWei!CD=brPN z=RD^*=RD^*%lu(ncjSW3{9r;toB&UL(RuOQ>buP8WbNZ6R?@8%P2Pg;!q!3}h=Q2l zE$S|AE#YCax3t^Q>VU9Tw0O(9%UjF4D_SeMovqI9%GOGtEAFmpt>Uy+Z*}+d*6H0f ztu@^0ItBMqsaBdU zEz3X|`4bIESBQ!8Ar2_7^a!n20cDM}Oj_2ViPA(vK%IotKza3lp$q^CkS2#1F9+t$ z8ey3rrtB5ORLOBfD+rIkM}1mbcv>2yrGG1J1y9R>w9Id%trWBN3azWeY!p|_*{f?^ z4Qig|UDMIMwskG)1&YjttafSblp@#hv^+@5KcWF);G;h7^&ZfLNIJA)!Cun3K`ewK zt`Um>H;OC85{PRRSBVbz-ozGZQ`t&wOklNG0q>hT1+f&`6aUIY0fyP=ONfuW? zWS>aZ2#yV`Rd;W!dAYl%b7`MnVxQ=4Bj(^vy_b+_?DvMM_+n5CEkP({IWe2liXhl; zZI=9gcc)b8a(Q|@vdh)N_Q$L+_K>?MRxeB2gfeVD$=yZe{fJ^1&k|3;OH^;3}<*f~V5VwwyIjqh!GrIuN z9I>j{^N<7Q#j{Ogx0s4o@d`9xN+UuO!eWGIEn?u6*J4sNQk#9E;7I?m`HeO54^0I2F)JWnYKat4Z&wp&XAfZP22$~v=advi*n8I&M4t4!aeN8v^-TxiG2{45&S0Y z7E+4TM~R}}>Gt+XM>X^ccvo~DS?a!27V=~%v^>+*Sel(_X+(h^F2ls2(XGHWw==iyz(kL+ZGK7^fYc9uPx*8-cO z-@4q_+3D%&jP&_KOwRhA7V64A*p|k!v(Fk-Hth54lFWW&aXrFr1XRD;G5u_APG&kU zHA3UpLNatuTEV518HLW3(w~51OkUX!~w#JQG#{>V?rq=(ba5U z(ei3u9&&XUqoi~n5^$lC!sm%~n%$}a!KH1d{eWA_W zzMZbPyJSAOp4FE)H}Xp0Bu9r3cf5r4;+SYVNoJM$76Tl!xopa0Bzr*j9HNzNApD8Llnfq_<=4I}pBya3=z4jP6JH zJ~P#3T+0>C)yn(TK}b=wosvxFA&qLnFfP_Dil7V>@(1t^){g6zS177ru7zRDc@M%) zc4KWZSfqz*^N2S1R_(qR@?ExWekr+?-7vp_>|-y@FU{rc&V`PO@&7P4R;s^+#TGAM zOX`yXysfSNzIL$c{%C9SoZm+5t$JEb0(GE6yuz{{DU4PF%kuZm)zlE2R@ z(+83GAp~9rtc@S0m{=j)H`s^uPSPASFL+uT_%6~tfPl?>lu$lIvDv^@EWr4M7}lv> zuMxtE9!3O=Z8ZOmxT%BS2xf^&a2ee=%)AX1Ylqw8P45eJQQq;Rs%T=H8VicKEDs~c zDO!kV$`L~y?Ec2Wyl9cU(vLtUe++ES+Ln(e}_mgnJQwf^ZDs zI6?t}6M-8l4~7u#V>=e#r|>9vtRl_qgOwF|naDCGECG&2 z#o+Gk1%pYCBHt%i#VVQ!EX8{`*?k}V+0Ey zMVOqyKUvH4T~@iFkep|0HvEoM24`L~u4#yaT%o5cAsiBQ+fma3)eqBbVz0wqp(J_1mGFAI!`;#4YwO}O!rG%6#UMjdg8sSQdGpL+JGYdPEOvNH zK6Is%Tk`E_uxSReiUrb;h32F0Fs8iD$hDQ$J1`jOEA%*9cx_JI_c2p~==N`G^SLQZ z)zr(U8Za=?hURtwt$}JjPqVwOO)5g;sKj>fcFEhQiOc|!QF&`$(Y(ohR^vJWn11Rrn9B!M<8<08VR z2>(L(3_vlp`})A6m3`Z#9zT5(qreMl_laO=6ax=QU~+NvC^}iXPFCW2`ntpPlbP`X zCfi`lmLU6Vz}>}5f|=xNh+M`ti8q5c_B-(+DQ!loIPjMxFA(`%rNmk5A=IWKGDhQ1SYq&1>VrLcs@AUm`KML^C{v~qIsv|!#u{ktc0G=gWF+(r) zb@$4u(TY$~9dM<3G_;fE6%@eI4jH@>H_KdP3y|%=4zmf8huG(yO#vr_mVSHUDdAOW`t4#p3f2h6b0)wDR`6%&=@KHB4Q z`*-)W(;>*IBy{*_yX2BR-IA{lEFQ+igjXhD6xnE3((0WUYf)9}_IliYSGT)YNxW2g zT#tA}dLK}#-O@r6vEoA*kpv+{@9u|5Tb)UgS^oC20NPxdf>c-lO~p_e0{Uitw1J^n zIQ*q{=0qItVe`@JrtWS(&4z5(qvCMV8WJY*g^Z%{q|EV*;_;Me7|5*|&ngT3Ppcl! z%9%(@Hl-RS1c0HIiEP1?Iiky;&ft67WAsF3zwLFB9A?^)ZOBB4vRLJgVN%Cl+u_3iG4u?4k<`M8-?)5gGb%u<7phmem@iNI&YI><>2@O1_PY_|y% z_TOMpkAQqBY@dbZ8m=SQg%h}o>`#Q$ur{LKs$P;%FFu&$ZN472*A>#t9LRexsPWIx z6=7FW`*BqBW=K|IaWLYlz)zEct-IbJ87KufayY-JUUvb6U=&=z=I_on;OK#s-?h6- zNhP~ww=)|#U%CdO(E^g#$nM$P#Ze6Cxk;cVMNeVFMWR9E!^f?)%`LZYgK--k58Qos zBwS8hIv3T!`&67P!3b6*=hHnm+uTeifLF`+4zO<%7$gorke_!Cm^$x+=yik1MRnLJ$)NFAY2V*+n=!tCTOrzy5&S~9ZxTs5UwP#SCjL?mBCrJ zv}luCPuogHY$fONsz&pskL69@Z@$aS_T2h_ZH0=UO)Fwu-=f;gGIKb3bXG zFFep7G?eO36loxQqJ&_$v@ru-PHGH`V>Ks@NOUr`L9@iHe>EPdUQHkvHa8@~%WHXt zrKOtJN|5Tc(gw{kr~a2^NcGDKf??;<1bBISeq2j}_LM=>V$_~8>NPD1!9Uz~y~%J4 zD|wtr_pCNtgQ_sG7w(xAk25G;j_<44H}~8~BzEJyMGN`l3R(!QCVqyYM-d(aIBHNw zJ|EOm)`olmAhJK*TUgC`G-9U3m?^Xib2)l96yIOS7Cf30u=90j9>w(wEK0lEBvH|V zD@q?nMmQBY;=oJU-77&VUrP_7?7S?#=0Q4GHTU)4j2E#uYHz*?dEAU}3&7EDZT1`y zk{~UbiO+ni*(X1&RF{ESm`kJ1l21X{kxwy7(f0a!L*oTr=@SUt&x)16cuTwset?l( zDC>F5Vr}2%2KNShoX~bP-HvH|te{)O$BGiu?(PMrK%#*UG43M-ZkM0L5XBr1V`vA4 z4q>PVLkvTFuHxe->SN!I2{>x_0+YT2AxApJs;HiFwCYi3n|B+-aiqD2Wgl!(+m*LH zcfq;VZ-ipO(L_f>lw_Dm{OU+chivSwgQBl7Fu=!059HU!T+^j$*kX0C_(5;njZC^uxG>)1yR=fVW{(a}71 z?BR;EyioiGITx6R9xqMb8PVv#}U}3djvk}Ltt9E{cSMIGs7`_ zNw|2>CZhusdgQs`kRS}$j+L>7pXOM~<@#uBS1#MLv?|ATgKfX=_Eb?AgwQUsOSn$E zQP@QWVrZ7!z@Gi-Jm>$Ha@fGn((@}?)Xlvq!=E>C$b3Bgv+0hV*vYst=YvnRCK$I1 z;W`9Zpy2FaeYC)E0|wn}&ZC7_?!n-902D25E8zG7{~tRK-HQ+%#+>mKTX7STa0AoE zjyzf#7=`R~3Jp(g2g3dwSOvm#<+q1tK0A2R_R#i59u<*U0AUCrY8v>sS%7h9Bt2ce@MA!rdr(Hv&Vr9leQ7it*f{@uKqa0_S+1 z<6QBKiFhH!d0sH2q+UdrK;W+v)?mU@4Vr~(O~NZlD`Hk>Yffis;N^67+?q1|>EbzS ziuCX18X)G~B0a>sTV{vwm?>^urgkiK&bk!snH2rHOm@fdC9|;+m3Vj5?S*w^#4KKV zbOJ6_J9IYiGY*1QMp@i5d9X&sh53?awvgOn;M1b6yVR2!zM&Cez)n0nmz-r^K3mO~ zlj;DxfZDo<@J|33BReqM7K!~7L;pg+B|iNe0cRZg1;W1pCLLw)N&YKLnvxNVh#WE` zV~eS+^(4|FoMA6McO-nKw=zAM1%5uU9*fgyINfIH^)3F=Qd1a{+W?9%lrH=vSMfTb+nqcc8?!{ZQhUby{3vBr1pORFz zc2eURF#@4`O%(${uw;#M7*x5M+8Jxif-h7R~V%+;yExDN;e(MQq zS7d(N2`g$jiEVs)kR-FiXEU@ZavGa{F;mlJVwO``u#s4Psvyq{h4BZ-GepgxR!$1X zz=TP<%*=Nx7ijN1m210n%C|t8l|5dNty2T1)7Wp{u@WqDNLIr+XVPp%(Q-tUZu&?;?y#6LpaWU5xGV*ip_#+Xqa|iBgOjDHBC{Wj_*f%E z59r10kk64UmoUp{G4qTjfdU3c^K6%L%jJ?iI_k7tQeRPx{p)DeB zrwdRcr}LmzkMoL~^hKQCuOY?CzB-LHEqWJgJ+-D`Iy+Le7E~VY$x63{sI4f`zbdmjje1QQyM|wXi z`L%~+vb%o$H<)0Cls$$l%>OuxdH?y$-AJ(qf&25^+@L9OT6Y3Txl!Xr^)kjNJa4^^{LUh|!e0MPRWWxk(1uKQEB=6F2uxM~ZZV zI`s$#SeeiKplNL6#^(~54?(XfkhPBt%=MgJqI9ec7 zA-iDf5c`0l>vh9n4Z1eqLbdtvP0{+LUe@?YN#2JLt-8wCUlsi>3bQe6Pa5HtORr1x zP&V{Q>1JqkwMVFa%Nv*p;aYGo{ojK&84}J}Y`5>abKus2`x*}}`_YOYuJ~!=v8AKg zGsd!KJhpDsGH=W>Z|I71wxk1T!`XLd-?MF`bnd8a-k5D(@a<35lC7=4?S#YHq!&)i znb9~$`?Avn;g{#=8tY?TuGd2N6@ftb6~e>%`HO3{CrRq!0{zJ}6NU?Pi%VipmS{0t zMlf8?!_@?0;HK2{XObEF%Wo3U5rJLY58;bG%Le|sU?zG8;B-C%UwmGNy)Irs6iG3{ zA*fq)%Wi%o;78v_^#a($U(>7k-2E@esKoTTcjI8>bNZAX`X}U!uroOOZ_g6r4}kx(9jlqF;UBqW4tzIa zE@OF(%6JVll*|1SU<6v{ zGNb!+saHy0V?!71~|>s zA!MwByA9e;9dPRH{B#+4n)Q9!rn=V}ZjR1>`ny>+kgMOAk4JI&bMyN(`GX&S1CLsW zmL@~8qU-R$0vcU@xQM_-O=WSPH2~Y?pIuE}U_bgIn?3oNo6DHNsy=_O1-a5hgd~KM zC^lA4Wvk&tp_nmzR8g{lTjg~X7YB@l%B510p*8WF)F&BNaO z%1snj@%4kHk(`PiP8^{vig`f@Wmp)~!o6;>MEJoV~pJaRH4yoPbSvRF@wby8~+ ztq8B-OiY-MP=Gnm?^Nd*J;os%9EBUL+%?j1YkUXt;{Fo$0fzy{O;3lX6OIPKXuLf9hK-cixYGlsN}b|h*bS3JlB!<} z1=k$UP9$q|bGbpqgF+Y@eBkol%lF`T2jMF@I~Er0Dh>GR96p*#>H>Vxh#g2V!u(1> zpSx8Hqxbj1>`l*y<)++*MZ|{7fnK;E>o6-ucdy6Q2A-qZu}~Hz#t(iaPD#fxI~o8* zhi86@&g(stDpW`zU34~l(_y%zkwz+p??@wdJ>0Yx79W2i zjcm|f#p}cSI<|&GijFT*}vZ2y;18=fYvX zAnIWk88|!=;d0C{`8NcE8rc%&1xuuu0Zo@tUAAft>ET9Eej>VUqA?>vG^k@Ij-*bL zQzJ2g7<(*kvPXu#SRt5#0Ue$U!U5-{g~ZEQ%$AqTJT+ATQwx0pgzGQeCd<8ZtE#Fi z`QO~y>gwvM=#rjy0)7%pBO4YlIi^8Ye*am5oqpW9ps!<$ElRF#3Ha546W!9Vgmx|h zn1^@&|GITSBcAwWw=49t6&g{!mk`~u*o56+vSX+uPpMKst8^?~Q$N85%+e#!c{ zKW0ul=(xB1VEI1FIb*_zdBy`(0PsI*tQ|Afju>mtC1o9`|Ij*jJf&cyXx2#b>~pr1 z1DPW;mW-_0Fp_-@{Eyl;j@dR&#Ojmd&InM^YOeD_#Ah6Tabp)Yv#? zY#cE*j$6|19K3b#yMfyS`*a^>+eh+h$Fl22j2VA4+Qy9O511aS88OZrHO?F}&OB?J zf7&>I)VN^GxM0M%VBBIKG1~8bYvF1WRO361a~kIhFV8P(DuhK&w6h3MPXXQe3-^HX zhw*nR8m};@8_>`i5&GIr;x|H%(?)yTkVV!GZ%rndE59?T@56)+ zdV4-tbQ#W~fi9nKmR8ue`F7deRI=~(^)1Y=$frK9gsFHTJ%4ZuXL88|iTtwy;QUmsu#r_w~dgX7*w68`=>P;ArM;hpeqP&Ij^N3IlPWA zihv9ilOOJ|w#$;}f=zL}#w}pq3+=rtSXk=8v|bht|Jt zcwK(WG-6vfYFjsETer_Jq0{L*G~;PS`*$9=;Q{fXnUBtUc;3VfXm(ZK#+oOCRQr$h`n?8IC zhp%FAF+tJI=B_d2b`Gr!#V$Q%-SdTUK3HA&&D#RH2U3B#;LitZG@&KOZ!IDliFNvj zFzJ8zqhj)v&XzJ{K7Ou*yhIXLX-Lw9a0&JdKk6Wz8RKbL6FJjJ=7caAmJU~xk>_T& z5MrGWF2Od9q#+3t!X>y;e-)3q1Y3uvm6N-z))WGV#1W_-eyNMs_P2qr3;z$XN;r!E delta 13233 zcmb_?d0>>))%V=lvhNF7$-Yb$LLjV#K*FXH0s$0^$T;Mggn>yW-kGSR!JtybSFq|; zr`6Z0ZKYD&a8$G|1rb_{3yUm1+WPsv?b~W=jW2btuiyEdCo@Bke&6@U_rUyemvhhE z&N=toW#<{|A5UA8f02|FZ=%ol`N!Ihg|{c?hzWbDQiaQ;C3?$x%R4KCNi%6l-pbyp z&MFF9v}A8}ucOn^Thm$7TiaRNTi01fbmhJEo%Kwc;+@dj(Am&Cv2$W?V`pRUq|QlP zF4a4^x2dy9n9L@(*f1sNDi%4hns7~}-!%1ts2lDvx#qbixf|T`bI~OJq9OMJEo~O1 z5&1%osdEugPIS+A&+j%zX`&&bP6stnUhsdV%wWF5!gV1rzsPJ_WYRLXnzSr;)lQ4a zw3~j$*SVPUvMF!eck&{$}q+oF|j6`d=!GOFTYt(@=@ZHZP%X{)q$ zt(yL>R?93o!*M2Usa8wTOM6UO70o36IW$4@(W%w#6vUi<#z(EAp`-jNR6$tZou*Bo zvX{k~!vC~o8cmf(lb|)yZxa0`yVqzCD7fGSVY+fM>RenrE zLIrhPRsyQj#W8Efl~Zsqb&1>WclEd@IGvt8Pr&JHQ@@Q_Y99}>B2^e$I}v1hlGEAc zb@}~WZm-wr+$CfUrTQ_Lb7d`P3&XHjT@$;|UdK`OO6+IpB@}enWf@W?1IpEFaTN{K z2u=k|155`v0LrGQobY3X!A^PVSZe+Q2jcwatgP8 zE^=A`t$=xeXcuBA%3X-|AulOktWi^w?i6+EaMJstR6UwJLDZ^uljnFEnNuX4+EKEGRbP?}U9rge+DP)B-!MRcgX ztYwpygVf&Z(+0e5KUBN8n5R@v_w@M#E)pkKpXT)R_WNW&x+s4rG?H~fOgd*)-QeC0 zz}!e~6$_q*DBy1O`M45eKDE&JoCicTD0d0nc9qLJ;NE4Hizu#JJpp&`*$t5ws$26M z;(nEyS4Q*Fl-JDDFE@g5FTr_;A5hOS@w<5wa!|K!-Qe@|>G7AKrUg>^q-<5C`AvB& zCoCNj?EHWLU_kZf7Z-33Fdu`dZN8o!PhU?&4}YQ_$-kAm(`8X5U72cS!701JNi`N$ zb!-BcLBKTt9{or~dO8ib+w1JY_-gKe%j5NvL~&~(1^tEyjZQ~_db6-V9W1OA539cw zdTA0b9p7gct?IYsr%DE>3Wv=Qb96+Hbj$qu!zqHQ9#2J9WNQBrvm2$(98 zZ?ra`W>r3z6>T_=%lAQjjr#A(iP>!w)RR4Z{R08+$U0ZoM%k{WSIrWGYM`oq26yLL z6c`3@@3)|kaO<(?I+s9u{jmgwGD7^+J5@#EsnFL|r{gC@%Ij%9>7glbd7aYj@Avgl z@4!u72e=+^17Mr7O>P{%8NtzExhEVR?H8h$Y2DW4>RRuXJmN@LxKdB?2c+B88?9n4 zsLcR2fo!B0(!odkMDzt3-sq|@u`ak{T^_XP|I;*n?BycdL)9(O>_M7hZT7*svZrD-$^ z=+0dfC*#PY&s~oUp)p=}qfpRfvOvn8Z>lU~YY`WzeK0L7>yac8vQF6+Betm>v#Q0- z>X}(B;wJUQtm+~|y1+2n#`_ROzX#h@OJ%d#JUc_QsoQ5~*YE(KuTg3}F51^SkfQ(t z7@sEYh@Cs4+iA`RyaBlr*}ni#4@?q$@!JUu;UUqbIBumzc)Naz~oB`>MDn_*L ztxC0)RdQQ)f#X@-Ml>>zju=_3-fb-}iB`#d{v}cQqpKJ6rDR^c@P?Mo%M=UGvCA-C zatQDvz)u0Y0nY$R0a07M9#O!(YS;Yx3|qZE)V5%+7=8*_TL5F#jg~L&*)7Q04rmAL z0NhGIR%r{uEcVQVRe|TtMj1SB#)6aPzNX9FAMp74ApbD7@>!y|RZw?B*DpE{o5T}} z`6GqPRm+zxxnMd)beorS#cNd2N(s^m-~ndQhvuwY7`6sX!I`X4Jc|MK=Vc8fq~kj3 zN^-$6Cfow3r*3!k_me@D4}rFFQM@GxtLcLTftH-N{Gha@@%ak|wdd;Ge6*+Ooy#YSJ)v8dFSE|$Mq;@1jIIHHz}M^adV8Jy(sz|dbIV;8g9VT0 zNuqLC4Y`X8xV`RPvJRW%tEyyGf#Xdy=?EagQKEoOX`v)VKPy zIe1&K$TA1UtD7#X6{*3JT=IsF{LWrlW(t%NgVC?tBu{LP3si*TCewxt)v;!r%3QNr zWUAY{vefl!az=U8s6A`yM|oL=DNwI$mlx&3GaUV7ec-Ivh8#8L@^QnM5B{{+vAN;& zdrkM52b0JFGCL!*Lvhy&&9pi0+PJI5)uwAKmzu5?j<|M~0X@S@W*Xx;I>}Dox(&P_ ztlVYR?E#u18bKa88ep=L)79zAYeb&1tSu!Osa#u{4>K%t2=q9j9ksixRu8E4YbOk| zB1Gn&bxluRPmYw=?SyF{%b>@scllj`fHaJMlIHTSU*~g4TDGN^*BUY=(dlCw0L?d+ z>Z7PTiS50HwF;&YR#A`Z-Q*19uctLkW8)~FMww%Pw*bf0MCUk?wF{gTcC(oT;U(pF zE)@--1I{{O`BP3wD8;o{bQYTqk256xSjFUZApdhQ=)b^RI)M|C~I&|&8G za1rzhO|ctw7`7-vDJ{fRlBS)rT%bfs7d6W3W^0SK$Ydhh0D-uSvbU(s8!j8JqtL<+ zR#RdZsaNDnpb%WsGsOf!(~xOM44H?lL$;x~q4=SMp+uYt{CoW7q@g6*?3&vh@p__k z2L@!H)8*gP*TvqTp49D=U2bQ<)9dyP1lV}%G2y)mOq9MfWz_V!`aRBdE^_x^tMwFP z*j-+a%kS)U^~-;!R6`h(Wg2?S-ex+5ben6CRLxi!GSmwjYqG#uW`YXLWEMh$!N>B% zxV1bn>Coq}edu>dSFc~@5m5j`GnI;LF^v?Paw|?} zAZ&Cg;DMm?G?Uz?5K@4c4DwvRH&N4`T}r1dk4d-lrd51 z%0D=r9T43PN;1TYdxuKSghF_aU`9uxpF%C%@PtHaF06O ze#2vkI9qyBc$)zso#8gjX)~4dLmSv}8HG_hm*VxgG$%7;8I>0h@wiae_njj3A9mk9pep=bs6EIpK7pMo*b243Jw^_Ew3<<5Aj-BX}9RZ;l z?b?k5rDW%2h=*%>S3=s>A%omP<96xZo9BB#;oyMDRnTLPZ#@bg_S+-^D7v zO*P$EHoT8W95se#xfpW=qa@c5=<&40>DtJ$Cf6d>iPQ`~7MM*=zqXOi$>`jougfj> zBPYh)ca=x_`sDqHS#gLar-0>MDEvBL8KQIuYLcq~tAsjvm!Q#l+GwNb+=MKhbOq=T#;AbE?k3vrEIjfxZ~(NDh4tso zpoNqSj`bp8OFf-aF=ARI;uecfOp8Jl+uAMamRlZ7TWaRC(X1~%U8ZJluN$s3Jz%Xc z|6DAoG(FT}YN@vEuP{@1f0aPEx;2*~ubA!g2SDce0ah;{1bdn)F%B9%Hk>bgIQ&|`qd z33k~HMP`TjA*4nW)esYx`skkWTISJ$Y$(>Q(Ib)b(L2?wduxWVKcG!t?>e`pTgWAp zPoON823obOfld8x%H@stCN!7X^PU29v?(3vBlCQ2Qy}2-2Y?$1c75ltCEA<>_-w{X z)S*W1onUOU@=(eg-7+~!C6aVYzpvluY#WO00kF3d=O**(W&!&H5_{0r5fn-7TJIu1 zgBJMFb7Z+5c`W;IS6KG-m@Zd8t

b{2tQ&7l18m2%$cd;GJqOLeL|*5urN};$?-k zV+W$Q0&W37CwK!V-=L5qM?QswrvcHai#oV0DfVZO9aN?F&8z0I=ILg)mq&e=swH>t ztWz_2x7u-ENeZtgAEGee_Rx#>-5C=zKb~#w9lM6qmS0}196JwNESJ9@`h4eXVKMjp zT}^xVe({sgsfR5>6olT{b+fRo8j8~j)O)+L)U~_UsAZ2>8Xk%eM{ zx_x(v`r{*+<4Shg=*EJ6#wSdDzozn~D&^5EQKmk5_!jlVqXo8dEm^BjM<2~dX$(`+ z6;fqjlIkcc5|fx)(_`}&RPBr-wUVF!Z2Pn7B|GCU7JSy-cd!5XNfuL`5)!0Wseuv|9|1FzIZ&Rw5HuS zBs)imdDcdaz@#S{9TIZG_LlX@un9=R6@Ux?t^%x8+n*>)7(~!XFgRYd+*`P~L~cgP zwFJ5a8*)QYCB#B*0Ynv)`A0R47415ZbkNwyD^%&8>P1HoJcov}T_F5EUk+y^J-uhbaIvumVC{gm%B{#_MSU1i?(|JaV-<}mT0~jpjmTyF`l65PpkJ0fnCJim;L#G|6 zW;+1qG4h9iZ2;ELVL^7ARFNFXLRQ7UpdbJ%nEy@E5iDcZ3RN2VRZ5SD%lgCw+;+i; zxjXdN&|!1PvUj$5&WbeCfsFRVjuP|HA~Qvfmc%b_upO%0B)B4}pCa@b02^2N1ppRYehK)BV9e1xuao~o);R^wP+H{7JQH51aSg}z8n938 zd0{8*R#(3mCniE@2NN!!BM)EkDxdUf5)w=+49_-CU;+{>#u0a`y8We0I_TK>(&hEt zs5}+$6QYQ8o)Yu~y0-CXPQ%lMJNag4;;)m1*sA92yKFq9fNwcG-J8fG3%Gk|d7`5| z`3OgHD-62fP5_ zej5E4pz}p(^y9x!@Xr8l;0FjryA>n%T#B;$0o<)u5IRVpXZCvhesZSA92LqwV(=Y5 zQ%}Z5&)7(NJvhH0205YNYlnm|tF>=5#a*YFH(Q2G0jv5XHB&wQ#@+U-=$18*rVhVR ztv0`Tu}Ba4tRmNv706PqH}lLJ5>&#GeA@O^9VshGqB{AbGPz(BNDHUX{h`>9q&6NY zBHD(Ng&CnhVKh%mq1+TzGO57If#H+m(n`+CPUFJ$$BK>YFW)M&8A(fzRwSf^2{nsq zJX$D{6s0<%shOioc|NzOEKE&{ahPqHN_yg}V_Je8Wd`*^Xs@U22(-t$(O zF*G*+dB>oF`T0&n#-NIdmz+4e|Cwqpu}V_D6OF>59y|etRYwXP6a}V-#&UHQKVnC(*KmlcmWh7G|Sms+&)uaV@7B=p6F8 z_Dpr@U8(A>Q)Sfml+)wYJEybNN2f|?z*A2D>Hlf^8o0?Nr;F9>w-bscgll4d>88yw zfyQVGd0jnkPZ7l|yy}^^ljmTq=Fgw3O$|&9$H;Li2#0B$CXaQg!a0FvWzikPZuEHE zs#fTY3HL)KvjIO-Kh*2WVO%9I4e-_m(~ZNrVvoN#;PVxGeRR=rM!lyu+mS6FP-X9U zM2_0=&fjT#hTgr-o{REND9i6($h!s2CkXD%vXhIsBF&HnpZlPli|2h7>+y~12 zfLgZ4=dokqwe2MModoF#^~N6>D(|LXWHpN}W`6(~poQEA|8G>;dq1}yNm)YwLr31L zipzFfTyHv2AHS;Ea$;JmdDRTdsTsCa%_`}GU${B*jPHq z=GBmgmD>jE$_Lre7QGF&fEJbY;fm@y3U$#h^2Jby_U)!K4^!D7oq}$r6Q6(x?#U2E zLa6J0nWN_ZE;)9ydB`lYhRmv^J{t-4&E~$CfL$H?YfG9v!VKR2I$=`v3(Ku!L6g+l zYx62Mo8cpezae@bHTL<@6w_ugB$PNaF8E@iNKsGzZJymsr?A%cJ3h)4v1+4 zl4;o=n@GX^nbww^nEg2xg!2W$1spD(H7~{TazS=$v+duD#ko@FX z5qmdq7(vcD$eO8}(50#MgYR|Yk z?lP+*Ntvqo^Nt$so>_9Ieo7o^nz%#oa?W-huetS=p85WBxzBpy_ceYuXxrG53au&bo6#}&cEl9a?zn{;cp zhqi{VqL}e`fLPA^x`oRA;OoWodgkcYo4L-9+wJH=3?LS80M()DRuT|T(Q%cdyC`)-Is>~V7 z>u2B27B8yzzdh457rB%<+Oz@+()BmWkOv=W%h{u=mjzpdC=0GJi}K)7A+p55pj(Jv zR7Y6pHhO(T7oFJSuw2L@RF7J)l?XPOg-aX`-fb2Y+>gAyC(Yu7d6)-`+w!kQO;0qu zO1jgbcR_}B@w9Tg6OAFo+?+G6@)XUWZq+nTF1>HcqB2-P-5A zCSYh|G7%UWn@u4-*6Bo^(`hV@EHt^4F)XV=SP1wjxECW-28bwyp0pz1>%WlpoxCH0 zY@>&CB^bWC;0tL;m>w5EEwE?j7^GUtJ&*neXC;dS@!{SJl0~|uk=4;6Yfyb1*rlW9 z7F@zbsvo1>!CTTqy~qkamnK#YvwHw<%6PX4izDfwut{0~2NC5_!GK6~-spC;$2=5h zsB8q3q6F4VW3T!Zh6k{ntcm|WDSb=i--tQZ7TlgLPR20(S7NUxgBm9OGZ@Si4YT$W zv2Ld=OxM+#{1f=FV`63Ny&W~Mw*nDz*!6@iUthPUhpw{%>*fBSIZIU4vCBdAlDOkD z?x|C8Yfd*vOwA89W>C8e_g#>oG&`4YK7(X2h zVU-J^_|w6mJW-y^q9ZoQdTPGBn{PKoZSYW@D7IZhNez4dnkQCS8dyytCct6AV}?FR zCB1;+-hfZkz6GxwFS2b*D6J#7eZ070?{KxSSww5_2eqPoPAT<_UY#9Ak9qS8yNzdC zy*|3t<@sns9vcI6PPfpLHCjTCAa#E*y-qZi?nHPg)nI5M&r&R<(G>AVrpa9=ifoq< z$?Cl~*NIf~@c*C;#SNfEfD^cH07ufoc9}wfh3yOJ<0rQ^W5yyh(c;oj8NAE2$eC!! z_W-j999QyvlWy010lHH%WMGG>k+S~_wXXr}2XH@shWS{E=z9QmLf%K{$>2o|q96|^ zKJ>n4Q#7I|rKOM`ebxN)M$&rJkTqK_UQxjYk$r)k)|$H*)jxW_FoO6u06e4MmkfMoZ`||Y`dYWp5fop?8aHeo zg7_~09Q{e0m`NOA@#&Ot@PZQ&-hHscZ+MvRBaK};HeS^6&}5z|??%r=*k&ccPkiPOa$sp**_d&Crh3BhC2#Z86D NwSr#ed>7bm|8FPva6SM4 diff --git a/litellm/main.py b/litellm/main.py index 1b9b7ca15..62a890093 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -94,7 +94,7 @@ def completion( custom_api_base=None, litellm_call_id=None, litellm_logging_obj=None, - completion_call_id=None, # this is an optional param to tag individual completion calls + id=None, # this is an optional param to tag individual completion calls # model specific optional params # used by text-bison only top_k=40, @@ -154,7 +154,7 @@ def completion( custom_api_base=custom_api_base, litellm_call_id=litellm_call_id, model_alias_map=litellm.model_alias_map, - completion_call_id=completion_call_id + completion_call_id=id ) logging.update_environment_variables(optional_params=optional_params, litellm_params=litellm_params) if custom_llm_provider == "azure": diff --git a/litellm/tests/test_split_test.py b/litellm/tests/test_split_test.py new file mode 100644 index 000000000..35ff007be --- /dev/null +++ b/litellm/tests/test_split_test.py @@ -0,0 +1,24 @@ +#### What this tests #### +# This tests the 'completion_with_split_tests' function to enable a/b testing between llm models + +import sys, os +import traceback + +sys.path.insert( + 0, os.path.abspath("../..") +) # Adds the parent directory to the system path +import litellm +from litellm import completion_with_split_tests +litellm.set_verbose = True +split_per_model = { + "gpt-4": 0.7, + "claude-instant-1.2": 0.3 +} + +messages = [{ "content": "Hello, how are you?","role": "user"}] + +# print(completion_with_split_tests(models=split_per_model, messages=messages)) + +# test with client + +print(completion_with_split_tests(models=split_per_model, messages=messages, use_client=True, id=1234)) \ No newline at end of file diff --git a/litellm/utils.py b/litellm/utils.py index def92ec09..3aaf00b9e 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -1898,6 +1898,58 @@ async def stream_to_string(generator): return response +########## experimental completion variants ############################ + +def get_model_split_test(models, completion_call_id): + global last_fetched_at + try: + # make the api call + last_fetched_at = time.time() + print(f"last_fetched_at: {last_fetched_at}") + response = requests.post( + #http://api.litellm.ai + url="http://api.litellm.ai/get_model_split_test", # get the updated dict from table or update the table with the dict + headers={"content-type": "application/json"}, + data=json.dumps({"completion_call_id": completion_call_id, "models": models}), + ) + print_verbose(f"get_model_list response: {response.text}") + data = response.json() + # update model list + split_test_models = data["split_test_models"] + # update environment - if required + threading.Thread(target=get_all_keys, args=()).start() + return split_test_models + except: + print_verbose( + f"[Non-Blocking Error] get_all_keys error - {traceback.format_exc()}" + ) + + +def completion_with_split_tests(models={}, messages=[], use_client=False, **kwargs): + """ + Example Usage: + + models = { + "gpt-4": 0.7, + "huggingface/wizard-coder": 0.3 + } + messages = [{ "content": "Hello, how are you?","role": "user"}] + completion_with_split_tests(models=models, messages=messages) + """ + import random + if use_client: + if "id" not in kwargs or kwargs["id"] is None: + raise ValueError("Please tag this completion call, if you'd like to update it's split test values through the UI. - eg. `completion_with_split_tests(.., id=1234)`.") + # get the most recent model split list from server + models = get_model_split_test(models=models, completion_call_id=kwargs["id"]) + + try: + selected_llm = random.choices(list(models.keys()), weights=list(models.values()))[0] + except: + traceback.print_exc() + raise ValueError("""models does not follow the required format - {'model_name': 'split_percentage'}, e.g. {'gpt-4': 0.7, 'huggingface/wizard-coder': 0.3}""") + return litellm.completion(model=selected_llm, messages=messages, **kwargs) + def completion_with_fallbacks(**kwargs): response = None rate_limited_models = set() diff --git a/pyproject.toml b/pyproject.toml index eb5c9be7b..d226ddfeb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "litellm" -version = "0.1.511" +version = "0.1.512" description = "Library to easily interface with LLM API providers" authors = ["BerriAI"] license = "MIT License"