From 2f44191642c43027e744d237fcae9b9fb4855b5c Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Thu, 21 Sep 2023 14:00:07 -0700 Subject: [PATCH] add completion configs --- litellm/__init__.py | 3 +- litellm/__pycache__/__init__.cpython-311.pyc | Bin 10998 -> 11116 bytes litellm/__pycache__/main.cpython-311.pyc | Bin 47407 -> 47622 bytes litellm/__pycache__/utils.cpython-311.pyc | Bin 117011 -> 119632 bytes litellm/main.py | 7 +-- litellm/tests/test_config.py | 42 ++++++++++++++ litellm/utils.py | 56 +++++++++++++++++++ 7 files changed, 102 insertions(+), 6 deletions(-) create mode 100644 litellm/tests/test_config.py diff --git a/litellm/__init__.py b/litellm/__init__.py index 68de303b6..2d5618bcd 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -281,7 +281,8 @@ from .utils import ( register_prompt_template, validate_environment, check_valid_key, - get_llm_provider + get_llm_provider, + completion_with_config ) from .main import * # type: ignore from .integrations import * diff --git a/litellm/__pycache__/__init__.cpython-311.pyc b/litellm/__pycache__/__init__.cpython-311.pyc index 77b9f46da503d120e7d4b6c03603a3b946a1365c..0bee58842316ba1befa0ac49a468adbc926a2b4a 100644 GIT binary patch delta 839 zcmc(Z&1(}u7{+%~o0v5lu}$ko5>47d5-*nE&7de=M3J;@06u|uyL zc}=nsRYdp~Wh_V6&>HQbUX*E(b-IBzgq9mpWvRcX{647!aY77gR4W<2bFO#Z7T*Zs zUF;jg3zCerjW%PJ5yTn9IL)5Drn+?za%TXufHt55FagVe1^@#z09@y9E0-bMyzce^2i)XW6#+jr<4sVSg`wH+EHxUN&^fOf3GSL{%rY=erCp z>;hH*u0W9RvFF>46%4QXHt{$s+)lZV!F~dG%0CzKe77)Kbe}{20`O8GNViu_3;Q}X z9K*t{`-&eg=9O(>$zppy`>!-eddc!hLIc_L~5E(R;DL^q_bf*ofC$@cWLDHO`#wsFo@Iy_bQh+M`8B=aO+>a JUgr4Xn*)~(--ZAH delta 751 zcmaD8_AOL>IWI340}xE!#hsd{!octt#DM``DC6^0cDBGm{e5Nrd~lM<577!?Ut5DMnoq=e-}@4 zk17Z2iOQ+St&FM!vKfOJG^02F;`+kRq$xc4hg79l2GA+D*a}ih5_5`gapvbIB_`$P zr%&#embBt23IIwL1%ikm5RnKXl0ZZlb=gRF$!*Wl2K=z z+$g)0v1+rroF9`6S8jevYEFDkW^qYTE=XY>h{y*K1(O#jh;SBxn8iS1@@56^$-IhE zCe0v`77)=2BHBPiJBa825uHHd7GF|nN_uKZd~RZ1VtQ&(QP<>XMOopsKq*EbF5b_{ z!0>^Yk&*Gnf%XnKHwx%_8@0ryxK@UiAF zcG)qBSsNToOpp&4J5Dy6*z8+pz0UIne5`rx=S|42<7aR3GRB)M55JAK>bGuV#2)=L z)%CBc`c?JoSG8Wcrg`8$G)X^5N=gv$fzyfZp+}Q)#HY`fC1}KDP^WoBpD)=?=SIuqc7_kHJJun@d*CDoGA)ImmmsH3SkkpJeLW}o=S!yW0_tOC5;E#jKNH;tUsd> zgmd^)p96xVg?}`bWa^|zBP>xefEbr28lweqZQ5kT4N_9EwfUs(lBrq!d)}bFf7~|O1>Z~hTd^{_Ci$X1xe`~cwrSWB zBGV2tGOCR2i0-v^!_jmL+?BClCAC=x0h+ANT!5I8>;hz-876d1}+i$}#i2xePg zcJ^$s9Cl{6ixtuH*;dhFm+dS;N|5cl1ZI#-k#s3Bk|W#U)0_vzN(kgG(O1c~lZkN3 znhBS3b=pd~3O>wj*3XpFgSn7u%YyhqGqmJAE!yB}-Vw`8sr-adt|X3;YAH>wM7q=D zO1M~AVy%`1DN(i!Rh-eGOW;p^BGqzrpCH@d8Ec+c4J-34a3eood^>6>NEbQTTG&|F zZyhG;?uQcT!Ae+_MNY?K!3@s^a^RweAI?4$Q4p9 z$JDVmO47^~(qi6I5ve|w?03V}l6tWoGD`#6`fxXhZ^<*!e*!{fhr~JXby;5OH?#}(^6Az&NMZRI6Y3Oy>T@%l zO5L~QWb)sLzE&PpPe?YrV9$c@R_rtmpAsG82bZ4YNZp8PRNF&(u zXJpIRl-8cmNPQ=?BOZ$&{?6IY&W-Kca9f!M`0ovLDjU4G-Lw z9p~PrPB-3ir}6&F5MTAFI1j$4Dk%AePA7RRo-wGMS!TM4mr9$b;3L4fnfB%iIeqe} zkhGJy*oIuUVZr`e@{9AI0Y~-swN2ssNt?}6?X2mh+T1=8W2NnIwR9StMoF52uNI!G zX%*+gXEpUn3w%o>3w@oDMWP_j2kR^!e5bZ_B*bA@`{k?>xL%tKb#>*&#r{lr;mK}( z8!VokqF*d8KG_4!4;Al-G|NU-q7mfg9lawCBiC*fJ;JbaRfMZY`Y~9MkiKsdl2aKC zOB1Nn@i>ox5)D!Z#(HaN-5Jb6&f!meczCRU>N(SMz8M%PC>Z2VMO9`G4NA-fsopoJY0o z;a6|#(6-;y%_dys+{uF9W^V}XEYSfhoKquX82W;S=%+fSL}p~ zO$7zZ5QX0v**q$a z@^t>lle{*RGBI(ok(~!`X=l;D0qwjW{!n5`r_`T6m{M<$_D6EU?~%k^Q%QW26ZUQL zLf1lD14*_2R#N$Yk1}l%wUb$D&s#8J+#P1oNZVrD4 zpDf&fnQqTXFBwqZOWp;JMJ4(b@`{r_x=jp`l~}&4gqIf8>Q~9D#yn!dO=G%0E&Sn# zZqUvT%jHiAf32AqG4_j?Sl_icv#KWi@z`8sog>%C8#5sc=M|e-rO*J;jq?o*cZMbHeBhbIYrehC}EK*Vf z%!32Y#Bmk%A^57>Rl^ArO`B1blNaNoy(-TmM2L(25FG7s7IsmaQjN13RzvtQ1R=6a zQ4y_JZRuuZq^^b4+s0Pj~yUD22|qQ zepf(39jQAzihZa(dNBuVTHz?2NR6GOk%=_eDfrEbVgncMD8#MYWEoYoO*>wRAg{dB zG&;r}fsa>is2`)xElDEZoCysK^Vqq{I&K|b!WT~xj?d<9dWsewjs9#^wwV7N!pT_d zNd%)-qK@P6MPG^OPiQ#yU23<2bv2uMK_QkaBMUxSUD!XirCaZYYdISk*%HR7={D9> z!iGCKrj#r1)Rjb+w1kQ-H^Wa&I)chewfH!O1yLL z0CPKby7wq?K}0BcsNOFFtlcWEfE#OV1$0Cd-45K1lF&N(#f~*$H|vJQ>ncoh2t7&Q z6b!6;P-O65>+TX?fxvo~_&U6|eun8MH260JehRt`uNvMU^dh{z;m&DA{5zRrn>*|X zc)d=abM(WTBNS$Crw=kV?oXo&n9(;lTi>w#;DwFZ^>kKYl()X+=Vy#z+XFB_yL z52{Px`bJ-ym)dp$^!)_Yqy{fW4xuCs-vSuoP&MODAdf{YG(~i?r{I}QIVt}}G=4|m za{|AIw>HhR#|XWKKrx2{xUubS=D-j{#3%z}`m%7k0=R)vNIfDuy%84%0JMuw+NVMq8ScvuEy_xiw>5+rOoyE{4*uvwjn?XQCR;YATitD569KXw|j2VQKyyMcl=Hg4{;*X2V-!JZbtNhaHb(c0cFBkW}U)+Dp ze8n>3Qp$`^vJ0bayL7tg^kMk9^yXEf@RQCodcB?4YN7W#rR6HS3` zpLQ)*$E?D&LSi9Yv!=2d?X{X#E4BWd(qf?Zb&GCyzHmKHY$pEK^HXw?PW2)(;cPCp)Hi3+$i=q!<|dAG=xfz?Lpzu3;_{;AYTnb6(ptljRboQ( zsuJzRRVGwNA-)Us)OMk+>51;WCtl1W*HnxQE$0cjZF(gxj5m_Y6W+sCz|(TEjjV%L zCV3)TK%fluOAEqmkDG-sFz_$Wc)NWj7wGLcI3G-rhvCA33=>ZeC@e5X^p(hyn#?_f zk7g^Z5SunLyu2~9^LEdQ8dr!6(M7>{2fv$Nq2B%Q^L>9`5RYQmw2Ib?hm>V2L zhB3g=#jY5(4&Zlm_oQ3nr~7TV($}pkH7~o{lK&`E@%n16+>vC z-{%SQbD$WULCgjuThX`}QhGajffo;Cr;O7zahea84@@)D{Yxd2>Y&9m6Vif~F>mgPTmk#-p+Hh3e@jg^z*WpHS2o4@D z5nJK;!<8FI>1-|m@)_J)0HcM3*<9G|3$ZykV2c~)N6TpDasmqpa62{-#<|2H3A7eH zi;-9Lkpg`$4n7iHd8El`;*)6}3ijmVgG;&y<+Mnxj3iOw-Tna|1thnk=b6u#Y+M*N z4`-o?#yoo&Y11z3PoQWQRtWHy6QzSM(CBdDb%~}^E^JF<|4cY-L8UZ_9bLZc3IZj*-PiBN^c!B4Z24664*NcHs;Q~~>lX0PDa+OG-aS7V~8Kc4pSeRJJxp>ZbxuA7~N z(Iv*xh!D3;24XkSI8H(vj|jmwuqnvh}FVNsv$RHYK%CX0v6&G?Wg@dR1ZFJbVeO$Uy{bD8^x%aHqf=lE)fu$A3X8lTFQ2EOIX;p&KhoJRwwdkcH7%#Hp_PqYGdFK1(nfsPrF{SV8WvvuY z(_zl(E$Q5iOd(9EMoN)BID5LlOd&~4)J`I{06sXqKx~bsKbI=1F<7rUpve{RZ}n_* z1fAjSESc7j@fbNCT5|#Pp2;rY(Jgg&;#dL7{5Yu@4xU+K;3lh$UOn@ouGod^6$6%r zZdaHsCt^IlP-Vq_L7*SbpUdV)nmLJ$mUZAut+yC;=o!&qLF%1EfYhVt{7wu-%Lt9f zDi-6CALB8VYDkOcsuYaaRqDuOwS<3zU=DWnH)#LYjLfbJFb%xM#E#MrQ*2@Lu z%+j_F_+4aEwcZ|O&8$S1*NX<%_JGHQ*&NZM3cVi-YKaQ$5z*lCP)!k6V}_dOI$YZa z{5u`}10F9r0h4iMS9ID7Z)v#q+cX`l4W(x@X$w8>X+*Lek5&@*e6m zFsEM%_}DD!?A4dqX}5YJ0*t{z#L_lnjuqs zQFQ%*fEWM8?L435;m$y52&21<(WSvCF7p7v=*)80RUM7|LD6|VxJBBKS`Bj7psnAC zn~xqp;B_x#M{pGXgiruuSxhU6;uR1t}YCU+o^el9Cx)C;F1b zgqSb}ub7f!THZ7W=Jc46qj5q?Ml7DA2||V?X5wg~kYJL}Ny3`K5 zd@Xsg?sB!&BMRx3Sg+bxFA9eA*ao#Rq7)Ec%QA4q+p>+1VKhCK#+G`dH!UBy z$MP8*V}qSXz#q#7vtSc~39yr6;oAhfOJFxkFcAC&noVHgOLj?iKgNXJ@WGF4)oUFi zAtdKZf7A8qT~+m}>eZ{)^Nr68&wOgIJz=xOaPahv$9l6b+S2(`&lSWN_;y-pxK9ut zPNT006`?;FF7bKIr?@-Xr51o2q*j0%r8aNJUW2s$l<|&EiQmkL8$v%f))-<;K<++x z|G1RKTDDT3WjKXbO;-&Z_Y}PPbCQ#I`Ugv1g+UT#_K;Y~c#2camE2Vr`qbnEZZot2 z4^yZvs#%~HtzEnr`oMaL&nkx@n$_*`504Ca6`x-w4bZX*-oT6WoNaM(*=6G!)-wI~ zU9(=&U)cVUFAklJyKagrhM`+T18IelG}9f)C6;DD?{aq1>vjiyJb7&o)=3Kj9ITJ5 z1x$+@_DkL&7x4y0{Bpoc>Y*`U0g$6Vgh>}a?Dy<^-@)B@U*)?+S^3~M-!j-)y0i)kY0^5Zlo?NQ@Mt79HS- zS+WN0Qfx3?EuzkhyZK`JL`IviOf96hGBSl?bs3G%Y~V|1Pv)z9Aw9m@LCdoaI7+0# zhb?OHk-shK3garqos6)e-qfG9>NmWcZYmS3&4CUqcd7&Iw!s-p>Dox0B(1R5= z`oZF8`d;p4z8uG9z>4hBs)tSUXl3;RS}A8*Nz6^1Am724Qh!x?;+piS^#?1|3RQ&G zA_FM!Kk^*(p}ggGQK|zob1J8bQU=4YPmC9(dR7yIr5EDlHkzJa#aGhKe7{hsbkawc zrPqndrYYK-3g%W*mEMS!N)WmED#VVbSp2}Apg2>jOGMyXC^cMbnbY9N+a^CwT}#dj zRm$$$;#zfkT&1?JRjAH2mfpAYu&`Y5+!oj6x5;(=yj)}Dqx7kQdxRB=54onysbM-? z;h;wgW1K61R>xH<=oeo7nM=P?=e#_v^1p{ZDGW_+p%;oyY@OUbO-kk@VUClwT;+ik zyc4We&gk7btG8NRdCVa71y`!oP;LuW(~HFgDNvKPKWvaTKP*i8`owm8p~#V@mR{R& z6@t=JEHEJ(p1TkuZ=m*N_wY6Jk!6_+`drOz+F{qcI*FD;w4vlbg;mNCw8Ij+I;SNj zGy#RW3YOt3EwIkv-_o9Wc}2+|(vM1u8${JUdp}9StQ}Q&&2t;}?q7i48Tw$^W5R0X zG_I{-p_O6a`RXnnoDAF2AayU!#eFPE^YXRGzoogA8T3!(B40~$E2?a3hSvwz4z~x_ z@tj&q-4z!n10_U7TV(~mhPqc2nCj#db*+x6b(g#7bxSgR=}g?dV7+P~c?M3c-`729 z2(^4K59gQezoJq$*$xIm0f)a%Kt^RMn9ZP4q5G$)9E?#x>#CfI6*E1ZMbz%}uZB?! zBW8J!{;H~pEx06J9@;#-CD<^$HP|?@dvfLwunpb61iL-3WNn`=;M*2ui!HNaBtCnkHvyVdh zWz7;^q`9l?C102E;0gu#duqCLU_Z-@$tL9|sNKI}y%V`s74C=w^R7;iw*cpZI_C>K zSwkzta-(Vww$O&vYbV_{8n%XXSZ(p~v?t3>Us|2&Xn|9`d}bsMwt|{5=U85wWpCZg zNk@WhYHKsM+ZkN1wyEneS)poemZj}n4%9JC>8_aAR8mmo?gFb#W+=)(Pa2WqA~TN5 z{gls6p=+G+^xft0neFqBv$W~Prf)8HWN%R0RrVqu=_nje5%~}O$(k14L(ea%rmNQ$ z3LBMwMH37xWb}=5Ik@s~)3WcwRGmHQEXNBZ%*>N=(OBd28HPUYiT<)K&(x)MT^^b|0iuH4aF^+p&d|{+ z2VGO2X4<4~(jBK@&S)$rDK{q-1V%m!_naH`DaCNJ_{udQEm)?!H$4^vNLR~Xe0)QS z0GEVNLBkg@E2l1c8rcG%n#9Z zyVFXMfMyIR{t+^Y;|SIh_Y&{um{;~3c8z#t_mFa!1h7+~w>s?YIt{E#2-t-WtNuIh;J}GD>pFA?AkUsi( zr*k{I+TM**8FfYp2WwI85uZyrJmSR*GIG(ZL;isQpFH65_wN&pGAF| z{s;8A?xih_&?Fi)yC3!GbNR`DTlU@MW*-8kI1?lF8#-$jhsM$7SW=>ynYE(q$~40 zw1M)lAbaSWn_T(K%v03JBiJ*O2DwbP^ei?r&77xd&u+(*o<$*2i8S2PXPxRJ_tKu; zomJEHnSN2^xtU|6i~6URH2-RtmeYWd7!#qU+n7^KZ)|N{P_E3& z({ce5G_^G12VZCZNTD~k?J={Z@1d)< z7e}FrMVa)_q3s#`L3;Q0z9MEMwr|ihTD1GFG2(UY_a4@wMgZ}+hlX@VKSC`#_VV3S z-XUh=ZqSVTKyWRlY5KbvelsH-^aneNtjiH)w!ch2-f=fSN=NS)>KSBlrVOU?GM0#}qGd1$>IvHAaT8k&f?4>i-0X-NIq|ZA~+OYnsclHda{QB&#>I%-s$t_k z%`g%t_vs?W^;2geDHlF1Z*t;@|4V2zWBIwmcqoheJd+LiJS*`~iST(zQ#5vcS>3b_ z%RjGdU_`G26UOFxV^u>6_FQj{Zj5J0YkCVvR18fWhK4fM)LGe7&XAtGn1)`HaJ{F* z-q6bl*Lyh=)amG>4(e6EWLEm5dGeXt$gFR zPNo@gGELZ$N-Iy+7PG*^Yi9V)JvO93Xu=~|Gw&UPuh-sL@&3|Dkv~oU>7*#JqdAtm z31i7G5xj-qZ3NK>aFtqOzgzJPMtmF(5GHaScA5iY{lh+mxwmF^kH8r(i8-@FNx^Ms zC{pR>J;;4E{r;)k_~XdNqOU7JO6hM-)kH4`Jq2>012$QX0aVj)I@MGMjSZn)r<*Ks zsBO*6Hgf=apFSts8gL36a$L?aXem5j!#_dY=kxeRdiwmbov3)S5&^mntjZ4?UV;=% z@eT(_6?BME5$@E6Bij+wB4EZvnF!+&gU!i4jMLgBlryhb}+{3s$#;j)iDjQ#ovj8m9EL?L`NS;NnyH; z_D=EL3H|Sd!rw^}To1u`@U_DM@UkZjnvwX2yn0fLdm9wQe1iouA;Q+!IUN>|Mm*-k z5j&LJ2~3>S{dfIF+~TxbOm-0;42}|{(%xy0sk}X z(z9GNzGlIT0vy%MA0SJ73mWzAUn3S_<#DXD)Cyfoi|K@GAZ!}^hY_ti;{;$nP!FvE zBLLi;`agd$7$qhqNwB^G6AVHBJjQ(xmvpS{9ZSN6);BB}@OvjI?Na*NNqJ{1A0#-! zoRCOI9`8zjJHFt%$#2BhJevvte#h3oW3?aahCKS_yv!} zJR;ozP--vs?m=M$(gcNm3-e=xi6B`rjIFb14nv1#l-&@}+5z1RDKHs;@Fpt(=IlUh z70$(&Mm_*cOd+L+&qWbs)8as^NV)21^3%E9Om@aBY44cqK{sI8wQGts$wIQYP|fTa z^!q$sGK%z)Kxh3v*(;$|Cnck&o-WT}n>QIwW3PMIm$?rnHUv*cg2xaG=>($?ixLE#UiUf{qq7pNZWtZIUe4|561B^VQw0}V~q<7tZL@s?O_W;qqv0Ra5O_> zxMWAz%FtLY$q}|OG>(f+497Dxfs3++6B(Ms*%HHchNg0fsbL30(>O;~I33aPa2A)C z63%ALi#XuyWN0CmoOP+`{A1#Z5w#bYQIo*t|YusZvd4#GQ*uvX@I47@toZe?$*1u^PJfp-l10_ tlyb{rFc?1L0NfHdL)59%a1>+K7kcN%KjPyP_~bA*1q$i&&$qKE{{Px)F)siB diff --git a/litellm/__pycache__/utils.cpython-311.pyc b/litellm/__pycache__/utils.cpython-311.pyc index de1a3929e58f3916ea70fd4a1bb4c66c06e76989..7c768446dbbe5382e38572b490e2fc1303ab43d6 100644 GIT binary patch delta 2840 zcmZ`(du&tJ89&Fb>ql&dJnYzca2*KXEC~=oD1p%?Ku9SBO4wLZ*39@^CkA6@#|{bX z8mDb0xgF4iTY5tWQY2RD7TTsO%$<0CK*FR6 zo`WPp*1$7pLi0Z|B>hfXhkq#Lv`=CxwdwZYsPvLGo8yt$-Z&yJ4t%R zQC}j60+{RvFJoj_;`L7?giV4d>rpb#R5g|p!0$hf{wA|bGNh2S17 z@{YW4Q=h)6$4{MJPe0ZmE-KHnwUwx=*R9=&trn(pd#9Z77*PLNCY9O0g6`C zj@g0q+%sS9*uH_ujni#q}t0>vm|!Y#N8bE1YZ=0{v# zu-|4}WN%#1=j}}k>$pMGi@IZI0@J2K?0HY3-X*>J=~C%C9~SFqFR@6|-*#}MXdH=# zZBqN48q7)jaz{esu39fc%dR|XTihrbiESao;&h39VLxsXO%y7h4~P&c{%=qD-kJX= ztw)4u4oNpYEtQVU6gSbb|8GZ@EQ)G2p;)OjIa9Vm#6cu#eumFrmU0hThGND?zR_|- z55_Gqmt5t>?uNtGc}=_s4%8|ZiRNR7)`(`YXpwW9Xq)Dw%roW7N(j0PhkPY}(2NTl zE3Fi*?7TL^x{yS}LK1#WF9O$=la?MNTJp7nJc38isJ0i4-~u7BK zaDX53#RmDK!_gRjNZ@^ZG)9Br0X{-OCP9BJ7zz96@um~*#pdBi40zF~K=@#|Y1SAR z4(Bat6cRkKvOB{2BS(iq0%L>7;%o*>J~k*kT=IcnNT5D2r>s)SUQ(I>m@N(q0*Uq? zjS$c?2ebxD?aopuQNQ}!dFXF+IbAa#V;8hhouLE)en!3<%^z{r^6eKBFyCQuro{e!+R2|@E_Z2?~>bjasFlCL7e{VQl@t&0j2Xq4T2(!JB= z3CE(ii;jXm0Fug@|6>rwP?obQoNLZfJ`uR+&sb_zOKlG6O}p`2dEIm%Q{JeSHzqri zopV;_l;L$<##*OZ>vBj77TNMjrunF(QmJ|>Q?g4f*`*Zi`pD)~Dq1oL+j`ZuUNNrE z8IY@5(UsrjtjWVF=bAcroAWB1H(OP!7+rU{3Wek67^gjB>ribSim~Iat3I_sb*;PO zdhE9Av5c!tb+suv$DGqW)iQbfmE+TE&&DU>$!!x$PVGqUNUfN0UR^!5ea>AyMb3LK zc&B}@HBKcOld*~RQzOZd)OTlgs*R72b!E9CrMOn%YBOBzT}#!}0o77BEvS};JC-%K zEo+puTQinzs%0CizZ+-EoO0LMhcDy@qwr#8s=8FbvM#l(E4h8nS))3=GkV3@nsK(O z&emjC)=@KcTy=O;9@UX(NcF0Y4apr@yCa9NwHwblmMZ+NjAOUz*sT=rhQ7FKlD6zO zCDNX;bf}gN;F+Gr+3Kd$h+4hj1HD?kMR|OWTD>>fozq!OWrY+{EsC>krWF7tBvD$K z-1GTd!d{U>W>eV$%vF1)J?9TxIG|QHCcCrM9sqmwbd6$P4KvF(rB15+mI>=!PtzSw z^KDPFvTp0I_h&piRnN`|8<^&h#=0q6yJo^LF*0fYhTuwZ+oSotf|M;OP44&{My?!^ zf#=^VP_a|Bd!}0ydvmH4fK6j_lzo9W9VMM!bkl3@vTJXyYqxiqwYM}F7PpKVCY$wL z_H?m?|Dkbz!M=*^>;uhzDIIM2C4+y;{HwU$@D<2Kmh|3NajzbKA;0xg{Ac`?Jb3}{ zfz`bWc)0;-^VXJ&c)(x)d4(lCdJ%u9HyD}1C*9juDId6mEA@6*?@fqNb?}|Yx?YN4sVIcngZG6sL!iWs6rVqb^8w|Kq zmapNq6)Yvna>+Bjmi(1nxW-G_ZWDvm@`G!5wV{m3Zn@z)-l%T@`48#C*YQf7;R%q9 zMQQn$_%&>7ghgk!_nf@(241T{+H~j!eorf~3xu-vQr0!fx(#<>r! delta 745 zcmYL`O-NKx6vyv*U!!mQXqOh*M1C|PrPE4V6oEk)N=+Lrnx1o>y%{}Q(nNlkS6E-IEyh1VW`W%TeS56 zBjg>$+r%++h!3%oIF3=`1S0Vfy2Qu(jTAB*z?hrBZg-Nh#4~2c6yBi-4XjPo&dHxjR6v;K-gS_e=f#aorSsPI+cn>afQwN{Sw zCtF5mq1}Y6c=H0jKu$cLgEm_IoP$bh_kVQ7JjATuq;s|$n};=1`=g1LMQAcXXRJ() zF2V;5e#o0Ip#;oA@-@rk5`@7lCVf@DTZSsA&`EQHuw5E;`XUZuq}NTQoR}N*@giZb zK)dZ%F$yWw^R&w1D45C{P&gs%bvR>LN*9Uq>(FRclb)7? z>ri7@A<|oZx$qWdpx`(yF0}@qh$|b=pzUfhy#Y&lqZ%<+ljCYq{Ge?9qWNnbjK%0o fCKFth;33hs3CHXf(&}kb6LNYJo)(DkNBH{>4hzRd diff --git a/litellm/main.py b/litellm/main.py index 538c60b8d..150fa31f1 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -1321,11 +1321,8 @@ def text_completion(*args, **kwargs): return completion(*args, **kwargs) ##### Moderation ####################### -def moderation(*args, **kwargs): +def moderation(input: str, api_key: Optional[str]=None): # only supports open ai for now - api_key = None - if "api_key" in kwargs: - api_key = kwargs["api_key"] api_key = ( api_key or litellm.api_key or @@ -1336,7 +1333,7 @@ def moderation(*args, **kwargs): openai.api_type = "open_ai" openai.api_version = None openai.api_base = "https://api.openai.com/v1" - response = openai.Moderation.create(*args, **kwargs) + response = openai.Moderation.create(input) return response ####### HELPER FUNCTIONS ################ diff --git a/litellm/tests/test_config.py b/litellm/tests/test_config.py new file mode 100644 index 000000000..0df5928f9 --- /dev/null +++ b/litellm/tests/test_config.py @@ -0,0 +1,42 @@ +import sys, os +import traceback +from dotenv import load_dotenv + +load_dotenv() +import os + +sys.path.insert( + 0, os.path.abspath("../..") +) # Adds the parent directory to the system path +import pytest +import litellm +from litellm import completion_with_config + +config = { + "function": "completion", + "model": { + "claude-instant-1": { + "needs_moderation": True + }, + "gpt-3.5-turbo": { + "error_handling": { + "ContextWindowExceededError": {"fallback_model": "gpt-3.5-turbo-16k"} + } + } + } +} + +def test_config(): + try: + sample_text = "how does a court case get to the Supreme Court?" * 1000 + messages = [{"content": sample_text, "role": "user"}] + response = completion_with_config(model="gpt-3.5-turbo", messages=messages, config=config) + print(response) + messages=[{"role": "user", "content": "I want to kill them."}] + response = completion_with_config(model="claude-instant-1", messages=messages, config=config) + print(response) + except Exception as e: + print(f"Exception: {e}") + pytest.fail(f"An exception occurred: {e}") + +# test_config() \ No newline at end of file diff --git a/litellm/utils.py b/litellm/utils.py index 7e168d1ba..bb88101c1 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -2772,6 +2772,62 @@ def read_config_args(config_path): ########## experimental completion variants ############################ +def completion_with_config(*args, config: Union[dict, str], **kwargs): + if config is not None: + if isinstance(config, str): + config = read_config_args(config) + elif isinstance(config, dict): + config = config + else: + raise Exception("Config path must be a string or a dictionary.") + else: + raise Exception("Config path not passed in.") + + ## load the completion config + completion_config = None + + if config["function"] == "completion": + completion_config = config + + if completion_config is None: + raise Exception("No completion config in the config file") + + models_with_config = completion_config["model"].keys() + model = args[0] if len(args) > 0 else kwargs["model"] + messages = args[1] if len(args) > 1 else kwargs["messages"] + if model in models_with_config: + ## Moderation check + if completion_config["model"][model].get("needs_moderation"): + input = " ".join(message["content"] for message in messages) + response = litellm.moderation(input=input) + flagged = response["results"][0]["flagged"] + if flagged: + raise Exception("This response was flagged as inappropriate") + + ## Load Error Handling Logic + error_handling = None + if completion_config["model"][model].get("error_handling"): + error_handling = completion_config["model"][model]["error_handling"] + + try: + response = litellm.completion(*args, **kwargs) + return response + except Exception as e: + exception_name = type(e).__name__ + fallback_model = None + if exception_name in error_handling: + error_handler = error_handling[exception_name] + # either switch model or api key + fallback_model = error_handler.get("fallback_model", None) + if fallback_model: + kwargs["model"] = fallback_model + return litellm.completion(*args, **kwargs) + raise e + else: + return litellm.completion(*args, **kwargs) + + + def get_model_split_test(models, completion_call_id): global last_fetched_at try: