From 5a19ee1a71753cc4bc2937b0c43bd683de84c691 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Mon, 2 Oct 2023 12:02:45 -0700 Subject: [PATCH] fix get optional params --- litellm/__pycache__/main.cpython-311.pyc | Bin 52775 -> 52793 bytes litellm/__pycache__/utils.cpython-311.pyc | Bin 139442 -> 138655 bytes litellm/llms/anthropic.py | 16 ++- litellm/llms/huggingface_restapi.py | 3 +- litellm/llms/petals.py | 8 ++ litellm/main.py | 6 +- litellm/tests/test_completion.py | 2 +- litellm/tests/test_stream_chunk_builder.py | 3 +- litellm/utils.py | 128 ++++++++++----------- pyproject.toml | 2 +- 10 files changed, 93 insertions(+), 75 deletions(-) diff --git a/litellm/__pycache__/main.cpython-311.pyc b/litellm/__pycache__/main.cpython-311.pyc index ee29db1470c862a0691b5f5ae80e587230d2d036..8fc681ffa25cabdf5f34c506bdfd7786f23a6014 100644 GIT binary patch delta 1984 zcmb7ET})I*6yDjpyZo#xNLdS36ifxLe-vqzRFR5MY*i$%AiEG&*qQy~vUiQNpLA$&34j4_698$ zzY<}uV_Vs-e4HvDxM+2ERnL!Uk%(74Gv2T;pKe?<-n3}EneC>fLgzL1W^g4fJ8u6@ z=$wOp^Il%%tD#$tTYR-4O9-n=lWzefRtwuj%fX|Rubr=Q7gdFKJ*yTKRnZN?-Y9;O z-S}Qvm1Ep&wAv6h3ovboq#oz(!i0to{db zI=94hQ5M(q-X1|I6nGs*qX=;Usxws-g^!x@96{-Zk>&yy>YlZwvJb({W}eG1h#6-U z(;kV%k0iw^-fdBxbT+^leAu#l1(KUiO%H`4v5;TyimEpL3Fc6kXz3?GC_U)TmQr%& zXBcAOIrv0`8d?h+DLzQFjunNGDxJjwIu%@8Mh1jWwQ~I9QP|@BfP~?qw}eob^7;i< z_%wmv`g)yGV}@RB+ac(mcH}vclaC=#__VEjh0HpI<35Cb89@jezqPp(!TzYDoP>=$ z{}gfJwx+u7gC~b{(U0!Vk(7|!A8KTGeN0HZaWe3W^#%Imb{|9LDujinWUKKg*rB+R z%CyC@dDcJUAnuOrB|%S}v0jnjkYUZ_e( z>h!?S7k`m9fqM!>!cDM4_9RDCKgMgP{d&}TnKHDM;eSI zc+#v@1ggjbQoCWdAPiyi4Y7;s`YS`F*7(z#PwK$C1gpWkM!t#T`)MllQ84X_|5cexkpE_ z3cRyC4BwnzpSr9J?wxNC$GvbuoxL{)Cf*ju^2>?UaP5lSkvf%dePVo*+<1A~C6y$V zeBF{%fSDHP421EfMd8GSZH{hSW4Celg4a$oSa+$Mc%c2#ij^KLKV97LIM=%poMYOE zn6Emx2E&&Ih!<8}-s?Dw{=^VO3`4ex2gpM)VkeARMYP z``LJSWew>y>{stwNk2UJ@ervr-uh|6)+fJ^NmQvv7TMFiXx&AKi@={x#gi(q1%0~k z30_*3USh-$OpIto!Y}-x_uMjuxR>L&}v6mCBU>LQ+fi9M~mh~&=(h=&gB&C7vNx^!pdVX5}5Deebd`q z2XD9pZUt_WUbx!S42K8iJ0B481dI%LNCd*oo~mBV*@KkMEG0w7ZUhaX2SEmPqqPUd zn#pHRg=yccSN2Z;t>1eX#9w}N4T{<=s``U-PpU*)xA`)5qNb=o@HusCHE^)d%OI`#)?H{O2dFEU%f zVy@DGGJBREei~RyYQO~>>~+LqNiK%TvxRVDcRuuihwLzpLQ}5QOQ1IFfxCmvWGB4% z{$Sl!5z03RG1H#Zd5Tef9hH5;tva|DZFv{FFfNGDfFR358NL%LhAK(OXc^j)Yu||; zFN~bbWLnXb2~Q!3O)Skk7T?>)cqGL*!R-g)QjgaQpXz!<)Xb@G^!vA)N~3>^nbYVG z_?tM&n0cHf6M8(!lCmUI(P(<6E*Kv55^DTB`n4lpzOXb_fsc=CaPG`#a3X{3eRK5cVTv9}HS}7<0}xo#N$cdYu0v9I7(?xbJ@UBuN{OU%YN58Mt%lRZ?R# zem-u?$azkp%64RtO>IEydjwr<{Qi&lvIVxFFD`t77Z1Q+LI>8a=44;SY)ts?*0jYV zqU{)O=0$k7n1OgI$+tmf(piB`=f8<9wTRcCJzG&ad^)*_1mX5%B{>46mnrEt4qv{f zkPbNTO?ul7%nKn<1bOZ9^3u5c0PD;M%e2L`WQzBrb2U2s2x|r0bJXL6-OLla7+r1d5+hykiHLe)2* zf8i>O!b3#LqJ^M;75YYJ9*E0G<5wgTK}7k!L2&^b6?`MUb8eEB3ZmclUPynOd(ORQ zyXS7%`$pKN{b5nhM@AY{_|e3r*&`m^6ZPDN2wk{JwJLH|(yHWDDRN6jy5`}O_7q+d z3(0)>I*3unxMSo~9ZB-XwNY^sRCR_sC6#Jf?6Np_TziT;R<77+HYTl3ng=$`%CPp7 zP*=IDBPQzMWSh1WqlFA+LPL%gN8!yk?q^nD?MOUKStz%lZ&YfV35iJQ@61HMRON}ixw&Fxf zk1aj6;;FS(?YJ4OBm}LLpq0d{tz?C)^w%5Glq45F9X~k1mK@ZWiW*ab8YiO0)czV* zXqn!vMGpd=C(Kg0^(@{tpgmG#BaQxNI^knFX>I}Cw+5ZeBC|FVQUiqpT1;P`^C6Qr2w5m00awGYXBdtfL^5=z4c*BO zx>Jen3^w%73(e<(1ltgy>Uyq)%p{4P;bgQpCBUrA# zI{j?RN(mR7CEtZ`_Ya5PJCTREdlGx82Mo?lKAq?bSZJ;vfb?J4y9w}r$ z_gd8+DW9uNio<0$74??&&&G)NZCY)tJGMQkmzBR5n30a7)R2QJ zcY<4-43B2J;(M7YTa7zG_WnBokC9m_^El7RZQG;)*|>t%saL7E%GJA;a6g20iJwn_ zMBbVRMvYBs*Ie0Dx2s8!VYMoF^r`903^>@;ksPVL!l;&S?t~<;@i!iaWV!XB7&&Xl zpx^1IFHLBeZ*N%8jQL`)5 zw@)lj*x`X=^2Hqsv$U)j4YOGo2>TOKejTb@Xmzpwz=l~Xuh^Md%;u3G4pmx$DlGwv z{>XsNVADx4)SKGcWY5lN(>>U@c(F}&A}q2yDu45>2>A7e74(D^bcYoPcJbzd?x*v~?E=$Y2 zT7S5$#p1LsY;Cr)BH=5ux7uAzt|oN3)$Xu1yY6MpIL@q-V|J%{zS+G41}@E**MyQ# zb{bnKt*#c+!WNsod9=xFkT<=UEWiF@wO*vioGM#iER+j72g&uF0|u-jS2)^Y4-n-h z_#?qX@|w;>L%k{Z87sfgnGsEU?iRI59qI3In{5#I5~Fjq^ajr z5MJsl8nJptcY3}zqQGZNBGxEhLHX(#Pu}E>$nzOvh*RLp$w$s)Z$!4wm`I!&@QoZx z<4U{{!!OBu_tqp|HB@vglaoAIa7nK4OxB)LX~M_IYdwYLtNLDTCYt(`%3pX=^|jq; z^p$6L2ITz~4It3m^Y%xXB43fm6yCU>gpr~0+-)?sjA{P z*o&n;!+g1gXm5%)VsJ2?YWe1ur{$q6a*S_8_1freQ;|2O*cUy3SXI8_%C*s(BfT+& zzUTzv6w5s?+bfA0?He`Qyp0?|6NI^>AaR1 z-_RmduJ^{|`l17>D9*@g-|#AwMSEk4e$jBbZ+Ka6pP=21uS`O_5MCjF{feo9 zs9&j8F}dTxS2B(!x6m6g?AmjLH3Q*;eS@+wv*W!Hrr@lV%I;Th5f;njxsL8xJGNUy>-6D+T1$diu<6|X?J0-EhOLSS^XD3q&-jF zqJk2O*wmbCJ9X%gm%U-t?nm;F=hHXzy2)hGA6qlMcFHX?W=rKVTXDHk7}09!nZ59>j9fDCoX556@>dlX5nXXjh> zYV9#3XLw@%E=ypOw;p*9e6q8<96pkFb?0l(pz!ye6Wx)KNxNw@9VUz(dbMmhwg2+? zM^%%*B2|8(*u?GH${c|JZl zD^|J;a_RZ$x_^=ITeFxz8Mq;;XGFhlR(W|wD2G#@o%)yfH#}jI+zaee47qxVL1OE z9qfST_%=P{f{7o|Lt4y0GWIrQMBFIU{0BV@Ow1wfY2pqc>Yqeq^P({Dz$kt?493A? zB_|x7j5FtxT!b;4sBnU3se1uYg#<+ear{gg4APAtI-bX-L-y1%qRSEdYP-Y69>-B; z0~6w%g2BWS$rIPLgT&j9AhBHjLviGm$U{9yPIpp(buz|KUmk`7ScS{ijL!SA^@2g+iE=t^!%E@Z(AN_j4v131pj zc`z`J7Q8tukcq%D;CJRhHq7Do=Rt&lm7`@4rZGB5OAA3OKT`pDG4~Kn z`w?>zxcCi~a7Uq7eD@MV)Ih*CH5-A9z0@2rIE7%{$$7)z+rANk2jxIz06 zGDax6D#$aTKgD(E7UQjT(59npjNQ(^tAh$y%}efrg>Zmxz6-L9kC2j?q~rj9`z~mM zr}&K)D9d;fM=VX8Wo@)Cv@Wo-iAx&nt#lVQ6`0t1zR7}Xd7YxJhdM~?Qz6!e$jS!3 z$O^SaL5uBfqS9l9Qh*l~Ljybn;NXwhpd^CUmS1CQYH+b_{D=+8QfQy^8|S-Rtrn-t z>RRMv^p-;JGVSR6Bm8zdWRFQDuSKa{A|X(CB5{G8jG74qi3Bt!EEy?tOkf4C<6U-0 zhbOq-4%-adNlb93^6)%3kP4qDliT2&8ve!0S3;+DKZ-tAzFY}Cngp6Wv&0tDkZRIs zVEg&ecF5NSXGi}qWQ=+h#eQS!f=0_?J9A>v{Q6ewz0ECF8>2ZBZya&AXG6*RourK* zLaBQg21u#zpk{xB)84@BF3VzT^CCM7#KqS>3i$>SGlF5tp+{k^6iG?MeZHEMz0FJ4 zzyuh}m#%>`a3gP63uE9t{_I*P*G)HKhi-mqEp)-n%BIJlN`g)NgbYLBLmvJF%!gYQ z_5>75@B!bt9&SkUqvEByJ0{GSKC!M=Jd4H`wct~+U|zG40%$AZx&c8qvFQfahVT~^cG&K3u<#oG9^79)Y z{YEk#FsElHh!%U@M?_KT*(RdIIY5t2vn~+-6&{hifO2w*-?kC%&@aXCozsDLJ_Cu$ zk&Tc?asKc$3@H<)j${80jI6%_(%o^T?o%WWhCKTjxJCCksde-FH$$Ov;u**RlL-Dx z>=(>guX=W#24CRWn_)IQt++SCT*wo>&yfKCuv!s*X_?Nq3 zCa(7C7ht&>PV%ljFgll_!#G4;^_7`O8^zgMW;|;zB;v9u-3zx%myOC}dtp(WQJizZ z;)p#683V+nk1mMpak4LNgDa@Hg-ON`A2GgPVg0hPL1#lQhq(&`&$>eoPnzNo`fnAio=F( zi0oV9iAj8ysCNkH>fw)^0;NF^|nCwT-z2@aDMf-$sYZo*jGLjRQS--q7;9OmPC;BeV-6yc_J znzA8kN945@bzmvPqh^2P!j^^w7G`(0wm6*jz%eqO*Bpg1Z3fCc%Ce(y58#pU^#?Ev z|I0cC)3w~ zMDPyw_D9)6XS>B=U1(>;yz^6-rme;fZz*sZPU*6GFD`iY%||!+eit)JIZQ} zl&3QarzDY`$Fk4N^{k4f|o^0gA@C%TB2G-q3-4PL`#M69p4-+J)(UOM-EkrVx$c) zgCRX>x_Ki@$FTf5TZ_1Diyc;6sl|0|0LlVa9ieSGm0EZcZa}VsF9xDgS4JXd%tU`GkWBAiR(&rnk=R(eV^6+sF?5&^x8*f-R#j;MM9dYmwN za|AAr&=w0DmRB3a2Q$(j}ede*qqKd+GoH delta 9079 zcmc&(d3cjm*3WsfG;Pu*UD6f0(3X-GN(-eeSe60>D!bsLKqaJ2D3vxXHz^7eDBwb2 z0IRQjx3b8h0&Nw6g0G{93j+#@Hi$DZD4;T$-uDlCZ&bX1B8{`Pa z`QDEuY8S|4ZS1f2Bp5KV!NJ}qgC|{l4_%Bg8G5SO_X;X4KiJR{^wBKa8 zkPG*?P)!P3I$P{yUj$ATK}u%?-T)eI&=@_O0jA zN#`=%64Y>=dhljJZRzW;5;aqvfodFlFL`T-@%s54F(TC@w>`na&h&+S7YjQp5cbny zU2KXQwyAst-=F(;Gx_KiOBc1w3BS|0xZWxY?h;Bc;||~!`O0NuEG&Jv$9SSx45L^K z*}fS1St2Zvl|t8jT6z<%GHnq|+d!W-+*CP%jNq=nsY1ARp>I45xqNj5+cDw-Q)E4EVQ2UK&bK@B&0x`TccwFUG=IP!@2SJjT$|D|l+~U0{Jn{) z>iKqP0$a>0mSNnv(*H}YZn=qE6>-^~a|O0V&QJ%tZ23oQ-l@|Q!Wi1*o8b;{qU$nc9&wfV}0S4uy9LmK3wmH;Kzt>L)`R{VvAs=#|2FL4;7uS_&~NM zmi1qHy5k#&#>OsWtn9ATWf9JZH8Fl(pT$5l*yqLhX0blZ;N(5lx-=*2GK}NfUw&ej z5cND8-w3N^RMseC{T17J+w*(ZY5)U9xcHH`fJ`xP?rrqT+tSn;L#JSqn_AI^XsdtO%Ch54o{H9>M&PT+r^1(@u>v6R!;=6)>$1@jw)ui z&T2DPJC=}#QRgEJ+LyGQgk+- zneBFeETVpILi~fMilQ8bOq>W0FtEqb{TCoUbJQr3g}NgvtqxOd9lBwz_61J%GQpiJ zdixI6qO>>dO@KtX@=tI@_B*H*-+Ol4oTj|BMU6%7oV>=V9liRyv;*9l=;oBunuHEb zf_p$=*4wuad8}+p*=e1zLuYj7 zjcC$!B~wlaj6)7(0XgPxdVqXDUD0t z|97TFHaXTh8=daF!hnrVbq5X>-Pujuu(2qX@6s1^=nF0k8{YZtceML7xwCSZ7eOw4 zKex_jP%J!jOKCUdgPFSRI+s4@ceHi-I=H|u*XiosL*u$9^6H`FhU=_N*DV&Pt3ji& zuCiU)fxox(zUp(CCkkTW>2{jTS%ctUcvmpy+*4y(o7!||$SJXE)c z(;4|BlDpa$9ri#%3!^<@Y3lnMJlBKeV`XP;RcDci@p@anLTt9~%o#wwsOxf$C+{`HEP5 zJXPstWKjE~$LGnF9!5@Yzj{IxV7|Eit+(NaYA48@#jq_^Hs)=zTbw zUuAe7%_Ph5`)lIop5f!KAo(T2R|x1OIg0RSgx3*F2qzKtGw?)AomMjCwyC$zEV<3^ zaI;u*He2-u%4`vLoQ=tO6UhSz#}L{PjwA5wonVyFKn^0qgcsK#@w2mu4bS0s31JvQ zKEgbN=Mnyb@B#x*#DedwQ@lkLRkkXJ_gX%Q znoc07SYAmGLKs4*xc$;mRXCFIG&~+6Md#)0Bw*h?;dZOdL6(>t1f3^lht)eTR1hDU z9j77eAt&RI(*DKe)Cka0=Eyiu(K`jW58~(t0#raO9VLe;5KSMF!%WDh7v<0jyXiv; z=+{3LttK)p9(~0EGiz{a?T*P-yWL!AC22Uuy`F()RP+-Cq!==hdlADhB85?s3>v3| zb||8!lrS3ZmGmlTjxY|yaUN?vQd)#3P&@~zT!g_02I`81R8;}e(e#H{NGlzN^eqfL za;wck9$~YLA*2Yo!x8wZJdV1!>i3aqLU;>d1d1$2Du!0ZLBH%+WE3I{MQFpJrx6}Q zIFHdigVabI9YtS?gOP^a$mQks2@df%#_;8tis%>dkPXRFuLL-& zQqEulPfI_gz|Q~&=ntvTX5hQ>Fv{M6v+hKgMPEpR0zDt<%7t`Df_+j-I$VJ~Ohk*| zO)j^EtJNYW(@+;rIUX!VwKpx^{YOhOAPvgs-5JnZIggooO)6^!MQ zRv1}1DytZMPEu#VLvqdC=oLO`NHyJ=4Sla+yqgXEm9=c}pORuAysn7FUFA`+-DbNPqSX)#b0vEP9FBvFQqD5CB8Ts2%1YR$Ji_MxTl(8dIHQQdl`;xs391}{HbTe| zdgMXquktO$l?S0W9HwEbAwCfo!;be_GJg0ghwR0f2w~DVCL_eX&9r#5c54NDs+ks> ztLv@A8x>u$2KtBKFhR(bo?QdwLMWz;T@xcP5vOS4Iv5M1X~jDD3?|b_>tQ6g=(_b# zsG6o>0`Jn3>)|DsAvJD*;R5WW$D1Gv&e5-$U;*4A&3PDd1$dvX7oiV)NcV{l2N&o` z5fU^Xqn~5Z_&It-gpH~NOrPCDZCly7ez+O5(ld|3H^R{OQ4Y85xTQfA3+inPhb0pa z^3F0?s4U7W-50ho)6G<+*W(OJzf9-PvSW+=hPKHUU;3%HTLWD#BJV!S8-6lL4M8f zm%(?)<7My;Qg74TqY$h55$P~G^(c(i2jK*MJ^TkPw$U9&A(Z0+ zA)|tPN`G~MA>P1Lc|vPyD;AoF)m~R?vs=9@Fpg%Qh5}_08$Tr7eHs=4yDmO>7Y4F_ zv}a(7vM(DsyxVjJ0Mey-@4*B${4Z_!5dH;9y73&mbSJOkzUUi@l1=C!&pU5^dDZe& z!B-SPhsk;#4SE!UZ#iEsS0NW~1u`0`-L&;Qq(C}->pZ&!jHo)3hFkz!I`0i~kueD2 zO(yLLv))@_<-fU^Y~~s(8BDicfJw?xOyEuF(gpZPmE`YG*uA%ax#3ay9+LN<&DHeM zr;ro#Cq^4}UD^}hbvWcR$jI^E$Xz-m6&a0Aph6O8TE-x6EM4~*WGYJ--A{V)GZ@2q zeERDbu!psHdoHnqt%Uyb5+oaV9r9C;C+lZ#^a+(k(O^E7w)5HzlCvOc{qb_F-(sLX;C5h$U;~sP5uvr%2gAPabB8t zm9-2?6Z`Twwf(@>^;0_gM`oykaayJ0KSBg#4nWa579-o!vyh(6z@w`$+e~JQ#Z=>c z_^`XLoC-g~tPnHGARMIDpP9;8bowvQFT{#$gul}Jet~;3$}rpDEebSYXWve5!p0D^%*~&Y>Osg+kKDy{vRX)D(lu%!PF04ia=K0}WQAlQjc`eNRV|duSuc^M719lXE{&Hquj)f| zu2#r~t8~3q__Oi>Hfey=J5<;NA^7w9xGBcXB#FKnCM0C`#xmn7_{8N{g2C)3XyT}? zWj|-|HO{~=mQ$5Zm=6Vv0^gkFI3It{@NQy!qkA99u}DQD;H!yzjdJ*8 zAr%PtIwAPz@V2bjsgpJY4bH%>)7vfb7LcD`N%Dz|$&U-%p#--G!NTVCNdha!>xfs4 tzv}pJ0v<&wlo=`9p^CLAAmNJaH#kXpEKX1<{tH-hFR1_k diff --git a/litellm/llms/anthropic.py b/litellm/llms/anthropic.py index a5e50a654..a1d11a797 100644 --- a/litellm/llms/anthropic.py +++ b/litellm/llms/anthropic.py @@ -18,6 +18,13 @@ class AnthropicError(Exception): self.message ) # Call the base class constructor with the parameters it needs + +# contains any default values we need to pass to the provider +AnthropicConfig = { + "max_tokens_to_sample": 256 # override by setting - completion(..,max_tokens=300) +} + + # makes headers for API call def validate_environment(api_key): if api_key is None: @@ -63,13 +70,16 @@ def completion( else: prompt += f"{AnthropicConstants.HUMAN_PROMPT.value}{message['content']}" prompt += f"{AnthropicConstants.AI_PROMPT.value}" - max_tokens_to_sample = optional_params.get("max_tokens_to_sample", 256) # required anthropic param, default to 256 if user does not provide an input - if max_tokens_to_sample != 256: # not default - print for testing + + ## Load Config + for k, v in AnthropicConfig.items(): + if k not in optional_params: + optional_params[k] = v + if optional_params["max_tokens_to_sample"] != 256: # not default - print for testing print_verbose(f"LiteLLM.Anthropic: Max Tokens Set") data = { "model": model, "prompt": prompt, - "max_tokens_to_sample": max_tokens_to_sample, **optional_params, } diff --git a/litellm/llms/huggingface_restapi.py b/litellm/llms/huggingface_restapi.py index 6c0231312..90abdb2d2 100644 --- a/litellm/llms/huggingface_restapi.py +++ b/litellm/llms/huggingface_restapi.py @@ -19,7 +19,8 @@ class HuggingfaceError(Exception): # contains any default values we need to pass to the provider HuggingfaceConfig = { - "return_full_text": False # override by setting - completion(..,return_full_text=True) + "return_full_text": False, # override by setting - completion(..,return_full_text=True) + "details": True # needed for getting logprobs etc. for tgi models. override by setting - completion(..., details=False) } def validate_environment(api_key): diff --git a/litellm/llms/petals.py b/litellm/llms/petals.py index 666602b2d..b11fbe06d 100644 --- a/litellm/llms/petals.py +++ b/litellm/llms/petals.py @@ -14,6 +14,10 @@ class PetalsError(Exception): self.message ) # Call the base class constructor with the parameters it needs +PetalsConfig = { + "max_new_tokens": 256 +} + def completion( model: str, messages: list, @@ -54,6 +58,10 @@ def completion( else: prompt += f"{message['content']}" + ## Load Config + for k, v in PetalsConfig.items(): + if k not in optional_params: + optional_params[k] = v ## LOGGING logging_obj.pre_call( diff --git a/litellm/main.py b/litellm/main.py index 0e595adf1..ee9c9a423 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -157,9 +157,9 @@ def completion( temperature: Optional[float] = None, top_p: Optional[float] = None, n: Optional[int] = None, - stream: bool = False, + stream: Optional[bool] = None, stop=None, - max_tokens: float = float("inf"), + max_tokens: Optional[float] = None, presence_penalty: Optional[float] = None, frequency_penalty: Optional[float]=None, logit_bias: dict = {}, @@ -218,7 +218,7 @@ def completion( ######## end of unpacking kwargs ########### args = locals() openai_params = ["functions", "function_call", "temperature", "temperature", "top_p", "n", "stream", "stop", "max_tokens", "presence_penalty", "frequency_penalty", "logit_bias", "user", "metadata"] - litellm_params = ["return_async", "mock_response", "api_key", "api_version", "api_base", "force_timeout", "logger_fn", "verbose", "custom_llm_provider", "litellm_logging_obj", "litellm_call_id", "id", "metadata", "fallbacks"] + litellm_params = ["return_async", "mock_response", "api_key", "api_version", "api_base", "force_timeout", "logger_fn", "verbose", "custom_llm_provider", "litellm_logging_obj", "litellm_call_id", "use_client", "id", "metadata", "fallbacks"] default_params = openai_params + litellm_params non_default_params = {k: v for k,v in kwargs.items() if k not in default_params} # model-specific params - pass them straight to the model/provider if mock_response: diff --git a/litellm/tests/test_completion.py b/litellm/tests/test_completion.py index 6377e8e8b..050152708 100644 --- a/litellm/tests/test_completion.py +++ b/litellm/tests/test_completion.py @@ -47,7 +47,7 @@ def test_completion_claude(): print(response.response_ms) except Exception as e: pytest.fail(f"Error occurred: {e}") -# test_completion_claude() +test_completion_claude() def test_completion_claude_max_tokens(): try: diff --git a/litellm/tests/test_stream_chunk_builder.py b/litellm/tests/test_stream_chunk_builder.py index 23dcf57a8..465ee4a52 100644 --- a/litellm/tests/test_stream_chunk_builder.py +++ b/litellm/tests/test_stream_chunk_builder.py @@ -1,6 +1,7 @@ from litellm import completion, stream_chunk_builder import litellm -import os +import os, dotenv +dotenv.load_dotenv() user_message = "What is the current weather in Boston?" messages = [{"content": user_message, "role": "user"}] diff --git a/litellm/utils.py b/litellm/utils.py index 1398d056d..9db751674 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -977,6 +977,9 @@ def get_optional_params( # use the openai defaults raise ValueError("LiteLLM.Exception: Function calling is not supported by this provider") def _check_valid_arg(supported_params): + print(f"checking params for {model}") + print(f"params passed in {passed_params}") + print(f"non-default params passed in {non_default_params}") unsupported_params = [k for k in non_default_params.keys() if k not in supported_params] if unsupported_params: raise ValueError("LiteLLM.Exception: Unsupported parameters passed: {}".format(', '.join(unsupported_params))) @@ -990,15 +993,14 @@ def get_optional_params( # use the openai defaults # handle anthropic params if stream: optional_params["stream"] = stream - if stop != None: + if stop: optional_params["stop_sequences"] = stop - if temperature != 1: + if temperature: optional_params["temperature"] = temperature - if top_p != 1: + if top_p: optional_params["top_p"] = top_p - if max_tokens != float("inf"): + if max_tokens: optional_params["max_tokens_to_sample"] = max_tokens - return optional_params elif custom_llm_provider == "cohere": ## check if unsupported param passed in supported_params = ["stream", "temperature", "max_tokens", "logit_bias"] @@ -1006,13 +1008,12 @@ def get_optional_params( # use the openai defaults # handle cohere params if stream: optional_params["stream"] = stream - if temperature != 1: + if temperature: optional_params["temperature"] = temperature - if max_tokens != float("inf"): + if max_tokens: optional_params["max_tokens"] = max_tokens if logit_bias != {}: optional_params["logit_bias"] = logit_bias - return optional_params elif custom_llm_provider == "replicate": ## check if unsupported param passed in supported_params = ["stream", "temperature", "max_tokens", "top_p", "stop"] @@ -1021,39 +1022,37 @@ def get_optional_params( # use the openai defaults if stream: optional_params["stream"] = stream return optional_params - if max_tokens != float("inf"): + if max_tokens: if "vicuna" in model or "flan" in model: optional_params["max_length"] = max_tokens else: optional_params["max_new_tokens"] = max_tokens - if temperature != 1: + if temperature: optional_params["temperature"] = temperature - if top_p != 1: + if top_p: optional_params["top_p"] = top_p - if stop != None: + if stop: optional_params["stop_sequences"] = stop elif custom_llm_provider == "huggingface": ## check if unsupported param passed in - supported_params = ["stream", "temperature", "max_tokens", "top_p", "stop", "return_full_text", "details"] + supported_params = ["stream", "temperature", "max_tokens", "top_p", "stop",] _check_valid_arg(supported_params=supported_params) - if temperature != 1: + if temperature: optional_params["temperature"] = temperature - if top_p != 1: + if top_p: optional_params["top_p"] = top_p - if n != 1: + if n: optional_params["best_of"] = n optional_params["do_sample"] = True # need to sample if you want best of for hf inference endpoints if stream: optional_params["stream"] = stream - if stop != None: + if stop: optional_params["stop"] = stop - if max_tokens != float("inf"): + if max_tokens: optional_params["max_new_tokens"] = max_tokens - if presence_penalty != 0: + if presence_penalty: optional_params["repetition_penalty"] = presence_penalty - optional_params["return_full_text"] = return_full_text - optional_params["details"] = True elif custom_llm_provider == "together_ai": ## check if unsupported param passed in supported_params = ["stream", "temperature", "max_tokens", "top_p", "stop", "frequency_penalty"] @@ -1061,24 +1060,24 @@ def get_optional_params( # use the openai defaults if stream: optional_params["stream_tokens"] = stream - if temperature != 1: + if temperature: optional_params["temperature"] = temperature - if top_p != 1: + if top_p: optional_params["top_p"] = top_p - if max_tokens != float("inf"): + if max_tokens: optional_params["max_tokens"] = max_tokens - if frequency_penalty != 0: + if frequency_penalty: optional_params["frequency_penalty"] = frequency_penalty # TODO: Check if should be repetition penalty - if stop != None: + if stop: optional_params["stop"] = stop #TG AI expects a list, example ["\n\n\n\n","<|endoftext|>"] elif custom_llm_provider == "palm": ## check if unsupported param passed in supported_params = ["temperature", "top_p"] _check_valid_arg(supported_params=supported_params) - if temperature != 1: + if temperature: optional_params["temperature"] = temperature - if top_p != 1: + if top_p: optional_params["top_p"] = top_p elif ( custom_llm_provider == "vertex_ai" @@ -1087,13 +1086,13 @@ def get_optional_params( # use the openai defaults supported_params = ["temperature", "top_p", "max_tokens", "stream"] _check_valid_arg(supported_params=supported_params) - if temperature != 1: + if temperature: optional_params["temperature"] = temperature - if top_p != 1: + if top_p: optional_params["top_p"] = top_p if stream: optional_params["stream"] = stream - if max_tokens != float("inf"): + if max_tokens: optional_params["max_output_tokens"] = max_tokens elif custom_llm_provider == "sagemaker": if "llama-2" in model: @@ -1108,11 +1107,11 @@ def get_optional_params( # use the openai defaults supported_params = ["temperature", "max_tokens"] _check_valid_arg(supported_params=supported_params) - if max_tokens != float("inf"): + if max_tokens: optional_params["max_new_tokens"] = max_tokens - if temperature != 1: + if temperature: optional_params["temperature"] = temperature - if top_p != 1: + if top_p: optional_params["top_p"] = top_p else: ## check if unsupported param passed in @@ -1124,92 +1123,90 @@ def get_optional_params( # use the openai defaults _check_valid_arg(supported_params=supported_params) # params "maxTokens":200,"temperature":0,"topP":250,"stop_sequences":[], # https://us-west-2.console.aws.amazon.com/bedrock/home?region=us-west-2#/providers?model=j2-ultra - if max_tokens != float("inf"): + if max_tokens: optional_params["maxTokens"] = max_tokens - if temperature != 1: + if temperature: optional_params["temperature"] = temperature - if stop != None: + if stop: optional_params["stop_sequences"] = stop - if top_p != 1: + if top_p: optional_params["topP"] = top_p elif "anthropic" in model: supported_params = ["max_tokens", "temperature", "stop", "top_p"] _check_valid_arg(supported_params=supported_params) # anthropic params on bedrock # \"max_tokens_to_sample\":300,\"temperature\":0.5,\"top_p\":1,\"stop_sequences\":[\"\\\\n\\\\nHuman:\"]}" - if max_tokens != float("inf"): + if max_tokens: optional_params["max_tokens_to_sample"] = max_tokens else: optional_params["max_tokens_to_sample"] = 256 # anthropic fails without max_tokens_to_sample - if temperature != 1: + if temperature: optional_params["temperature"] = temperature - if top_p != 1: + if top_p: optional_params["top_p"] = top_p - if stop != None: + if stop: optional_params["stop_sequences"] = stop elif "amazon" in model: # amazon titan llms supported_params = ["max_tokens", "temperature", "stop", "top_p"] _check_valid_arg(supported_params=supported_params) # see https://us-west-2.console.aws.amazon.com/bedrock/home?region=us-west-2#/providers?model=titan-large - if max_tokens != float("inf"): + if max_tokens: optional_params["maxTokenCount"] = max_tokens - if temperature != 1: + if temperature: optional_params["temperature"] = temperature - if stop != None: + if stop: optional_params["stopSequences"] = stop - if top_p != 1: + if top_p: optional_params["topP"] = top_p elif model in litellm.aleph_alpha_models: supported_params = ["max_tokens", "stream", "top_p", "temperature", "presence_penalty", "frequency_penalty", "n", "stop"] _check_valid_arg(supported_params=supported_params) - if max_tokens != float("inf"): + if max_tokens: optional_params["maximum_tokens"] = max_tokens if stream: optional_params["stream"] = stream - if temperature != 1: + if temperature: optional_params["temperature"] = temperature - if top_p != 1: + if top_p: optional_params["top_p"] = top_p - if presence_penalty != 0: + if presence_penalty: optional_params["presence_penalty"] = presence_penalty - if frequency_penalty != 0: + if frequency_penalty: optional_params["frequency_penalty"] = frequency_penalty - if n != 1: + if n: optional_params["n"] = n - if stop != None: + if stop: optional_params["stop_sequences"] = stop elif model in litellm.nlp_cloud_models or custom_llm_provider == "nlp_cloud": supported_params = ["max_tokens", "stream", "temperature", "top_p", "presence_penalty", "frequency_penalty", "n", "stop"] _check_valid_arg(supported_params=supported_params) - if max_tokens != float("inf"): + if max_tokens: optional_params["max_length"] = max_tokens if stream: optional_params["stream"] = stream - if temperature != 1: + if temperature: optional_params["temperature"] = temperature - if top_p != 1: + if top_p: optional_params["top_p"] = top_p - if presence_penalty != 0: + if presence_penalty: optional_params["presence_penalty"] = presence_penalty - if frequency_penalty != 0: + if frequency_penalty: optional_params["frequency_penalty"] = frequency_penalty - if n != 1: + if n: optional_params["num_return_sequences"] = n - if stop != None: + if stop: optional_params["stop_sequences"] = stop elif model in litellm.petals_models or custom_llm_provider == "petals": supported_params = ["max_tokens", "temperature", "top_p"] _check_valid_arg(supported_params=supported_params) # max_new_tokens=1,temperature=0.9, top_p=0.6 - if max_tokens != float("inf"): + if max_tokens: optional_params["max_new_tokens"] = max_tokens - else: - optional_params["max_new_tokens"] = 256 # petals always needs max_new_tokens - if temperature != 1: + if temperature: optional_params["temperature"] = temperature - if top_p != 1: + if top_p: optional_params["top_p"] = top_p else: # assume passing in params for openai/azure openai supported_params = ["functions", "function_call", "temperature", "top_p", "n", "stream", "stop", "max_tokens", "presence_penalty", "logit_bias", "user", "deployment_id"] @@ -1219,6 +1216,7 @@ def get_optional_params( # use the openai defaults for k in passed_params.keys(): if k not in default_params.keys(): optional_params[k] = passed_params[k] + print(f"final params going to model: {optional_params}") return optional_params def get_llm_provider(model: str, custom_llm_provider: Optional[str] = None): diff --git a/pyproject.toml b/pyproject.toml index 129acb1c2..cc845f058 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "litellm" -version = "0.1.809" +version = "0.1.810" description = "Library to easily interface with LLM API providers" authors = ["BerriAI"] license = "MIT License"