From b1fbe515fe45efed74ebe9d21cf68735d5b01986 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Wed, 13 Sep 2023 09:48:45 -0700 Subject: [PATCH] return model name as part of streaming object --- litellm/__pycache__/utils.cpython-311.pyc | Bin 99701 -> 101068 bytes litellm/tests/test_streaming.py | 4 +-- litellm/utils.py | 34 +++++++++++++--------- pyproject.toml | 2 +- 4 files changed, 24 insertions(+), 16 deletions(-) diff --git a/litellm/__pycache__/utils.cpython-311.pyc b/litellm/__pycache__/utils.cpython-311.pyc index ea63bbb384b0f62f53bcf6905ef1dfbcb4f96158..b26f9b7a7d6a1e7bd9a4ce02e76e4b202f37e0ac 100644 GIT binary patch delta 12767 zcmb7K3w)H-m7hDWJSGW&1mqR+;x$PKZwMBV5FP?ycq0U1GMR6Z3`}Oioe6;yoPel^ z&;nQebXP38qEfY@qZ_TYt*h3GbPERZow5(;By07Fo2B`z%^f zpz0Bu#qv1)nV*LB+%}5ZioevhK`Ys3X=uj#-Km*<6E#fXwifwnr+(HeWJmujU%>=+IT{r7SJlFu1Raws_5IVwP@A!tz}ty z|7oq-M5+m>Zrkv9mPHF{Q|Pg|#iCVeHvTtB6IM&0MVtDFpmFKX{E*xh8rVj3G&te3 z=7HLD>f5Sq(PjiZ+RVVU0ne~ra!a#atJ&vh2%@dos?FMGZ`jf+1dY3u>f`9-(WsmD zHTs(!*s5I@*xH<<&7qDV?RuI~8!?|tgM_tOdWvZC2-~$f!fo1oLa8kvjIsa=X&}3{ zhz5#jH&AVd7UKHFf$iE7;=h#IGqrl+-AP=R?Gtg|4$VV^JGJG6U0R#Af`77cN29kq9BHxlj%T1bnNv`Hkydg8NJ+d#FqXbpt>w2g$f zYBv$yrfnj;UE92GvZZ0a=B0-_`Y0x>`AGRqfdj`MiCS#zQljEFf+n3c~+i%$+8tg>oFdXv(p_osWlQlg4lzY;XgZ^Cf_XkbY zX3x#AJuJQ z>d>((kL9TLdV}F$%l~ zGity6V9tf1CG?fMpZ>|o)Z1+}XC|l6gPaMbaMQWwz$`YP0S(s?&}hB?E?aGm(#w{w zfa0&JL>&()XC+?FvI;RxMaGZih<0Mh|w z0M_FSl={m%8MtC`E-H80vF&Xf+1;Z_Mt&yNV5S5sSL=;fFG7esA8p>%J zst!#&KuR#fp}_;(PSakgdbxlEE{_lE&pu_MAn}UU60nJ?4>{h>8+F`%E_2Lz{I5Qn zSwsI%4Kcq7FBDbmx0C%Y3?5E4%1(am&)%PHn#w8rX#lB(KeUdDg45epq=nXIXD^;y|7fVxJn7KcxoZxNW8*ucoq0Rv^w8LNjBFjUG~w z1yo(d>Nf+5#a{JT{`|PZL!X7@>-}7fmjv_t%7w=|1VrdBx{iQ+zW3i{YrZ4QaEXZ|Jq;ek^GgYeH92`^V+N>^C&QULwZq8?cm~JO} zbg0U*NkcYJ(Z~yi+dE=jzb_PO^7*&QjcQxj3?~cRsh%twD%#bL%0`v#Lbu%j9{gr> z6#*kN)i10+=%N5HqI{%yCQ(!VUPgJUi!m@FgDt_ZFXWZUkRriA##K#ay|Wv|Th)Qe z$)&fUnAFETz2)?TwG`x?*JSC+zG%87AxG(zJyg$+T%K*cLFY#bTR-s@XGFY=XWGhu`OK4VZocSr)EgVq<_A zkkZQIs0G)T*qqFaq?}481()4|>*))GU5)Go2W#@x^qK;3yIMZ4P93Z%h_ejvawAit z2npc~TR76mS~K#a9e&zsqiN4LjIm6a(J3pf!f&F5T?V!@vVHBrWxg;iF=@Cu+QHoG zC(X;_po7^ATT7rVsd@w*-tsv&FtU7_Mq@$|{fx<$G2^!3Bn@|jV&)<}1`Z0q!w(7v zI=vk1(;7Q!dgt+36=F)_%~{{ch~EnOe+0nkNONJ7q12OBxftaoDBDec#};8YXpK`4 zSx#klcK=0qKe}+tU~wQxU&p1#uEzRcEKpy+thzqZqApcVRh9F~*&gdAr)(kr2^}8> z@bb)W_60*7k`&mxK-ml)_0GKVsVwp%puO4xX)Aq6hpMWsa&THq*4pFXkkrtgx(cx^ z@kHHn>%x9f+YPENz+Hg*08av*2lR{N8tMRdtJ;N+IZ^W4)T!SUCTxoy5YB%_^Udnf z8_H{S6uSv%%VI60*D*5W^kSwE$w7k`J2QYM$)n1(c&y_^Dn6Q+vG~s!gV?aJBW1>; zKq1xKmCIIQr5KJ7t@)4{qdAV*LBML{WzMV43_uxu42)`X-Aa^r4Mf!YD<_u0M)C&$ zmK~wSaQfQYDT2!rs7|Pft8R$1DjV_K1ZV~DF35W7=jWKVJI4$gcAspC8qTeq_?A36 z%grLN+>p(1(ORSc+rshYC~5}yr7zk_3!ptBW9m<O;5#*yyR)F|Hsq&P}+NF9BW zyYf%e*!82W!kVaGpOxX_WxPbN~UJ7)wgj4Zg{uFwy0E$d->5`4PaRR#k!Fj*sd2^~BP^VQb-U zsc7W1`=qZe>TUDI{H^k3^9-)wZi!)bYG^91a~@F=wm8>%M8cgskT!s z-LgVlO5DHYvM7HDWLU-&|JI<46%&Q6(h1g6p|Rpl^-icU9-?}R)nS@dq&?_2;|wf< zoz5T8$QcbMmjV*!1nB@c35*j#v)q-;b=0OG3>Fxju@{*BRkyDmTZ+7=yi4f zXeN!Ko@*;|{|mnqrIVeF_CagH;TyH+uCEQcH2-j zW7~a>5yanGg%%JcYs?(ec!E!J`G>aPG7x<`(j-ifHs(mrBsAHR$vKwr2 znzU#{?>V zn0TslbB0)<^5PRjwVE9tDMqOc@#;YnsnKx6BJJL-vI>2Q)P3==xK3qu`v+fw1b@-2j*Zm zQj^Pdhg?LVh4pCX1O_S~LftA}SAPJe{7kkm?c zgu=cg3sBmVMd~MyhJNOP(C!OPnaoo(T)t4Cz18arwYU1@9E^5Joj>pqP3omPSG$&= zVkw|ljW}2->eZryv&1sB=V0MA9P92NA?b*i7Wk-o;$XSMM_ryvoIg0hHiM(#Ah2Ls z!L)oSWQJrT5Am?gx1)EF`HA*7YUbgUL)eTZ^k`&Z5hCY^$}x!t4!Bt8pL zUcvcI2w{{gwzY>SA&rF1DdhHd} zYv@*W@!{*7n;`XmHTuNzY2A=y58x!mwBv$LUP1d2z>fik(EJU+-GIB)qbFXv=@44- z0N8WPee7N|KLjvP_Y1(gfc=0R0*au#b5j~g+A~#@nap)S!%5wK^}|Q5aq)n6ff?s_ zL$Y570bpvv`skYj2R?~sWN^kn(=2M>*}>`$&rfkG^tz-fUzq7~p?E9ckZOD37V)6E z^1_(w+01tIXInpo(kZ~x1Sg%Q&A6#wZp@lHuGW5c^d#o744n{pPbN>nUqye6hVL4! z4xBBDmvA0lDv`2LQr^}?w>vhO&iM2_5ii=8Se*>oyS5SrlTW7N7x(G z`sC~^*B%h;1>8b#@=J&9`z;03mU1vN=xtK#U#w>Dmc{60HRC7T-Cf32hOIr)ZYGJm zJ`)&$H>s=uEwzB8&B8P|er1p!Q?rrpZ}r8zII}J@Z!F{!Xk(u$KnXrY7hJyf7(#YD z?JB&Yp2H9c(D4!B8I<6k@+p+Qi4q4ucESUAyc57~hR7k2RLeIA+(RX%Vx}$WsdKW% z$q;*y6K+H&;{)HTGwsT@=M;&x{u-psK}_*#ff&VrD6huoQ0k_{ONA=`CA&J@>2S#p z!3c05G3KRL1`TJ+r9F+gnU1ThXD7R#$9#Bwv7JrF1&bHZ@Lj<70Qak#&XzlWh~hDI z>)FwZUqSIz0>g%c-Siv@IpvQ4X^&yX_k(A8`mck8edYo6`?KSRvnKk{aPzm+v~%Uv z$3gxw;9mfHz>NKjjnIlZc%1w(N^huJ&rOQMKTMOstK@04u=nuMf@k~_)b$G#DG!oo z(8^B3f_#S_`K;hg9_~$7=y!rVg-%Zs#3lO-&LfhYgk9rVG+*r>eGX}Z`lQHg`n0~% z+B9tkwj8Q{299q5&I8^C{3}6@8CcjJM`=eccFBwA1elWe_^m%UYk1di`O+H(Wos$7 z?3=tA89wtOSiVo3Ur!VoZGyI|y}wvVKV+!4f3ZOPM2&d&TB}v48{fS?ZX+T}9FgUk zIVex~>O@p#6Onle$wVm&efq`=4x2e>$^cy5o732miP4Z^APr9k_xN zAkF-95O{n@?Y(MK@`FK+kEr53b;k#fJ9ksN->M$mQm8z?d`Rq2SAIEs2?Y8$n_B3c zn3f8pavC+QS(rROnnMgkJv7-%&4fIWqiXHNa>pms=Tc(##XKSQsDrHe6HqQn&q<6 zc}e~h&#mgVOH&>B^jwzs{-sR2>o2JN4DeTV?s7%`-%$KJ;B&wi%5`O9(^m?EW{&8& z;Lgj@iBbk26OaYq*|5*0g|nN?^U%(+6{sIyDRUIk^UB13U3sT)*lKY9cR&RI4$?Jt zO=~c!znmqC2b2V(>}1l4)>=I$Lktxe`j`wc$5jh+^5i{wI718{JOsn^sYUxJ{aA+h zj##DFXNomquKr-Am?qZh?_`Qu`DxNBaDPTG&Ju=Lr+=0ung(~EH{ba1-oxs0$Hk>2JRPgzC01wAn8lqp$7Dd`pj0y$P z#ZbhjMddP7u!65ahkj9xU4t#ex!wmL`6U2WmcOD>@A4moigRcx(zm(9M3KzC#-bju zv*%@(_-VfHEzn#*Plp-BQ%sG4A@Vt_=Jd@A50PAGtP~ysc8y|pL@+TU$$78^OzgQb zQvA-A1E~#ryN{0RA^npQQRDgrNZth)dUdJrJHAG2X7=1uDpuPI5LS3O&`E;{tycYS zW5jjk@K;)T^iqVWIP!k<>$?N#H;ol#o|JS$!N`*uS2 z5-!?r7H)6`Z0I>NRy-#%-^J(|a)W-ycro3Tw!}&Ojqze!_I%I-w)cECUKCj?nLc?_ z%c?qqdELj82C#qds$Z(ls}!U2>%oEJm`Sd0uN2e8?fMgy_@TahLu;6^#zw z=B=kkKeAL5=vS)57^j{|wJY?I6GUVfk8YKzF-dO|Fse&`cmi?#haO{sm?Zf3a(PM5 zpD2dAcrlNL5GVEN6X}=I2YS9UQEV2pCUPf>Yl>kPGRX#_Fzn4i%2|*i&;KydVH!_FHY&iZe-7= zQ^nUH$E4|Eh?DK-((9*->4TX~ii3F}*Q(z$T`Upb?YTHzED*&k$yV^BzvLZg2#;-m zR*W-D-#k-XFaAY;V5YcKUJo*+Wr0CUzh<;iE$x2ExjtD#lRT}Dm?d5)PSY*buvGiO)sn^_Y(S&T2bz@Cuy=Y zhrXj${9~5MN`z1A;kja%`uedfea<|XR_~f8%JsHditZ236KkpBGpeAcQFUS?n(L_f z*gTP`pQxiqPR(_qR4=Ndit_oQN@VLR=8M6@vWN9bV%d`2Tiiu&N$EGw7jf$Lx%qYT z0^t`f{Z|Xbw0LZCFQLVvCgP^w(k|0WNwLU|5hxbzLqL+QjEwLHB z&lbyQIxX*Z#A?ztdxd7%?YP;|DLO4XY#S_{!ky=l6>*|4-37N>$%`nI<^uSNW-Ur1 zN!_MbB4V#0kHj65yU2_f97k|{W7xyE%j8(H8C^J*G{Ye>yq^)Hr)&{&SLHn|Jhp|=q&#vshPHo7zHPfcBNh^`T1_0vm*C;kGB;2!vu zFGws0n^k}{05%^iCxZakN3uoO7#(2S_2NwR z6i3NrOA1s+c~6Yf z3GJlmGs_S3D;q>Xoa1ylHv)5WS#cCEU}oF#LP^w=cP9uquJgQJ!Q}XyeuYYgEl4jI zaA0VHWZ9-%q`>DLX=d%X!SG(}& zfAWg8H9x{2d?3J>_$|B*eAwox!8my}Z1ih7C8f+YPrs#6OwQ*6`2!e%CvX1ltN&Z0 z80k1cwR?L$YZT|K&bvU-s-N=bVf_5O%rX7C#m@<{R2&WA{zBK1EQ>i zXJwK=sys?h=!K8eyEe6YZnGHUc$)gX*fXF>3OPo{zeC74WiwM6}r%V*IM@V5TfR?#^QCtr%_ zx%4FyRNZ?Do`exuFo4l>Ur;YtXiO=Q9Q5ir^d9I3K}sH;<>fOCEFz&^mX@SR ztCzXHil*0?zaE#O!uf5KPw7udG0Sn9%3t*SR*H$XKoHE4J<)bmyJE3->4FV>Htfo( z#ZN;M=nV$qju>Q~hp{M3Ctr`!W%P7(&~M;UVmTOt%;4c|+L`*9nbO-m+l6wxL&IL% zGptjL5b-}jhc+JAO9cl_UdGGtJPUw)Q`P}Ev4Ddac3f1;&8TbyXn+?19|N$tnZI+gPvXZ; zSqhi{Ksb~hlzad`fE_r7(k+13c-=I05)HVJx+2gM>v&;b27bNIEGLt7@Az@9D(djkw{2lUC4LzBfCVA>vvS9 zLFotCtezKki5qQ|d3HhXPk$Nq*Iz;Hr|G6VePFk6ug!A^dU5y5aCp}h)P9<7${Ql+ roysr6VPZ|SpQf9p^#r=b&uo)4tDxlJtMLlgf7-WoMNjKKkt_ZOJTpG@ delta 11644 zcmb7Kd01ewTo74)wV`aOY7JAo!@y$CZ_g}Zy>+9@7#0G zJ@?#m&pr3PdGS`yr$;=!9>~Z@bI@n_6Aokh3&(p66giKMIqz)Mha7NJY|tJ0(D26k zii!&1a5~By`mh5IeYjC^zsun`MSu3E;u@~YrMeOKJE?~L>`!o0$e{~eRJ-)N1ES(< zdOk_rtdArN=s|rn(YEN7`dIo7>Q(wU`mXdk=-)zsL)Y~JB2_VoDgRm04Sgc<=rs;~ zjPBxp<8|S57=}Jc{mnI?bqo}sy2;gj^g`mS)~od?M!7!Km}Qg?>eSRx?bfFq&?;(B zSMAiNA8=RHCVSJFPHoEd8T5Ar{S_G@eWnqrPSQnVaB)pNtEH*Q75c+@Xi`Wz)V-*S$y9P3+fRbqT*(@BN-C=~b$xq@NZ=>i6lBe6y)m&H)swK^fD; zCRLuXqG$>TQvuTec>q6PI$#DtOb}ctC*!%O6RuGcG6$&-GfGH!`oKZz+nIez-Pb#+ zh3-7yt#EP4bs)8ZXgfrOM%rms#??{X2(L0?4be!e z?9%QNHwod`C8TqghzyCRbn-eJaj)t#ec;H`;(pTY6!hayjo2Y}IRagBQC5%lnL5;Y zr;Ey|oqW23^l_J;6tb8|N@SG`DWQTrZXP1cV5L;mMnhFbOl`^<=ueRoz_Jif3Md0C z0!#!KNi1bSE8)15~5N4rr(q-^x}8PvGgUE0v4*_*|}n=x-xsDh$$ocs#bS7 zef%ETWBz#RxXD+}9}g2yMSrn%1ef++uh35ojHDSL=DTS84y6jAi-B~992KgSv3JtE%(-FC}O3p3x*qwg;GW(Pn4Bf zZ^UB38Y5(%_h#lv|Ht0A}s5CYT!wgT$ZU4x6o;l!JR9}*s$RoyUj%*PgI6#LXluN zAUlGCL|eH-U7EDa(~ROCwQzFb=vEXvtaEc`IXOI@#gt15<_m(_B9xqSQ zJ~Z49xE62#fPu?{fWrh<--c++wm={ptqB<0F-Ep0yGRWfj}UZ51=gi&Jht*j9d6>f zLG3S`B0fw!R`?yS*rTqQJyx`;1GDE%!8}>%o#PEeE4Q-e*;5@4)f@ICvf6g1>hH5x z48k;6T2-*B*09|Nx|FVwTh*31**=a=SU?ow_s;3(mGh|V7Zg6eJ*QgiRq165)Z*g4 zYgsBB-AdJiaRc31u1K_r?Ow)O8Y#5%ib^Hfbua63auTX$;5h<#-#U-T>IDCe0lvj*=hsSDWYh zQ}QYJ$Nu3`(M5w3C+FUn(z*v+w*cTQ6ykY#!zL%4az5w_P;p@6-_uMuoAptx=jkzwrp8O3u4)_t0yjt)~HA3OjXOvGS%hMzHIPyg; z0Pr^}vzo#}qcnQhxlF0D{Hd(=x53-Z#CS}4U{C9*_sR;gld7`4?n4vXTqTSAqB0R$ zwA|_M5uW{EiUDo}d>wE<;870Qn@|KCQvY7^pxBgnU}>B1JOXNy(yz{+cML@=Q`(Ps zw8ffY;_x_0y|IYu_i<+Y^_rzMPW9lZZLRftEHIO5S4M2Sied9Fp;@osoUO*9#O$fgY#GtTk-{T>9#$~ z8#ls6J*~fHc~;%EX^8(hs+7+IzT6IN z4}&#BCu>$lgOavV8RpcFHix8Tc;)bV;c>H}e#8wqSn1fD5RniEEN^{t!0uXNuH<(C zKL-31@DqYqZvwWh(y9xEmbjw&gliIN^=2uqCTET-FnQ~ z?_DZd=?y^{td9ligYl|b`2%%2JYV}8Rjf>03a=HO52+|iRe7Y@W5)pXQe=kq8BuFh zYIKFTnAjL?7x{O9Z43d0Lu6fShaM?ac-v^POQ~&JT6N;EyukRJX&W&VjU@p<&GOW;&z6V7HO>SsFe0tsLV?O4J#JMG?DV&O6a9@LAO|0c&LBq)qmrcsTWHI% z!{=}i(rYP^!y>E>RvB^tNWOTK798!*fnbO`khu`T%VGpdc>v@?NgwYV1tN)BzH6X1 zns{DKgm-OD5jE=kwG+e`^^a?Zhykkqfr(jTsnXKo(S|^s90g(h)w%-_F;o5MKvls- z5=?_`j8w%#(FpI3oMO-lCfi4w>nR%xgmjvlm1_0DQR02od~o8rDO4o>%!QSnn}qUg|{G&}abV5VOo1|{a+ zkm6bf-=#GhC3AI{FD*)Yhoq@8q~)b~BAByMW|ELNb z&)4W8$#H+;`0)v@ERMTANMv3?LH)ul!?jIB`HlMImX-Z^k_Vv&FV-fq_Lv-&xaQWM z3C=qyVidqK;{fbkF1btySm<-mdzSegN4n=LWilB&7xN96qW}`kD=3b zfF^?xIzgNPD=+HIN<_9m6|I{v<)qPwt!PHKL3ODpzlMRZN2kS&j$}P%Cwq4ILuVOX z`86$`RKpI{RqEL7a&)NGLAfhx$hV=KzOLeI4K?z>P|O=ruHSVcv#W&T%PmLB0tN+wMV>P68exIO(x%&(nD; zs9m>@b%A7|pEI=T>ES{w#qLQx{c55AQH+Dv1<$7Kkr44cR3re80d7@;Ud#79f#Pjy{%a!_KZ)X5 z0?UP5-1ZL%{p8btq+hV$fb5H$FeJcrV=YX0k!Tj2%vRKa)TbEsh-;Zp;T_{X62Sm3swpv8|+%f7;jJV1|p z3gMQQ9Ym`=kYaKz`!>UtRJP!5XyBB_lc|TwUJSJ?XSkf>o5q z(^C2sNq#)>^6%<}xU5S4u+sTYhx+;-7K*3T8-JMP{M@1Xy;TzUUy@77o{sJu(1!>7 z86b1p@u3J(XF)|Re50oo=p7mt>1~saD*1$A|o7*=B|&*urA>h z6f%;l2${=C7pK&;yZ+&i{Z_F*tU@QN0ZRxhjqky&R7ppu5XCt=N6f-t-lY03*~Pa# z7!K+7PA}i0+Mlb^cTUlk_3=AD6x-Cp7ju?Ci}%u~HJ$#Fs&K|PfvVP&cAN_BZu+1b zn5&bU8LL4(sW=mtaYp@5`-riaDof+*VS57}0p6})NUrcZ5dN;dXrJl(9L1jkwx|UkP1U?khof&| z=SQh--^ZX{0(_!w`gC08rzm~~_&eaT`st_FRDPv0sOE6(A!(A=+!vtHYdVLd*I(pc z03HoHR#G{)Ej$JFtXsM|_*tHoLC^COr$2kW-=JlX^jkna08Y{}dQEL8X71@FhL6n+ z#UwmSehw%mXvwl|O*OgkXf!t*jnqho`L<8Y^`YK2N10iWE^@MaqvLK=)DJK#(#6AK zsp-uSYs5@*bB35E%1xCaW@aW?M?srY=9?M963flDUZOHPg4TS8!&?m-^ks8mZ!yBY zM_EHnEMILr-VlhzgYm|gWWVMR_&al3Z;_Y70fTQ-ehXpT9owz9d8W4*s$Yd(VN$y4 zi3atUECqp0J^&5&1G+XnJBQ*qTgcW6qEQGkc-3EJelbam%v=Zw9LH>SbLwOIHmz)Aq;{~ReWtrRaIM6L#OsX<$bEeOTO0C?#ogF#f<4Q-9h@6Hv&&BMi_T>Q{{qgXsK zqN|am(VH>4K^zAu7`#1&b!bvA&b19I5mQ{PtQ7b0-)v^3+m}Fe(5ClQc4+O=oi1N_ zJU>(G1O>8ueU`W-^PBR;uP8XZX;GJUr@ zj(Fek*-zZN-Fw_kqRFwtwb9We{F&u)B-Ps9fPBVw4qQfcTo0i%6TlZGDZRt$&@^TT* z;2`Ug+hKxcbK43rP?VWRR)``o%zSQzC~y59iSzgQ%3mogLZhXCs{uSCm<-8=hs`Bh zne7|JBToj5A$J4Dz;>F%GwcK_`7i>4{R*7LAa8?Pm z<<@>}_pTDzLX0w>SS=>a;Z4~?_n%cwx*P|Yh@O_4cgX$dh*J>+4ZjD$H&Pv{3GgP~ zY>r$b#)zZlvNd9~c95ElZfjaY(M)_|KD1s;9{wr(wqv)(9jfK&cqS8; zJM+JY;LrU3rJN_AT+$J{@}F*rvPWJ=TG{uv*zN78c|A&p31Sqm%zHP8eC-H5&1gHn zL9Es?+AvgSM@zF{v$(Bw5Gx;Rj?phR_!wkgy5hV+`(g4F@;Fq&Jo)?5y-hHpN9((1 zC8$MD_RnfK7^kyjI`uRfFXOS;&V57?}xfrTl6$|&H&ISUPjkL z49|R5U-sH|(d%r=Fj;DfxyulRnS7`=(F0q8U%#5K7-Fz?7ZF?9E*auwr{_j61kD#} zMNq6Y$A`o;59~*|Om#@qYxrf~1Lhkc@qyTAKEGAu`FKnnEFWzZ{Y0_psS~5Lhln%L zHo1-{bD4*GooCO*-RR{W0(zYoj0EuRz2m1al*OGyB`4q=V)uu|67$Ul;T31i*Tdq2 zsc25g3G;O8pN!HYQ9J`tfS1fq>P6ESoMb5}@L)uziQN5ED_JSPFWbHz5#JZnnQ#mu zc*DkV!0+Q;tql7O62)hOetT$Jlv9VR%$*G)n1-?O0AFu=w?X_~vyJ$sd49Vnt3H8B zdu$Ntvac+XgjCyee9)4+XMOoJ0{cstmbE2qlBQ*;XHoSO%Qts4iE*BLQ9fxt*d%6Z z&r$jBZSOXTNv>s;5PlJElp2_~VsT*U!i{{+Yw^y*tC$J&%K?&}%%rgbY_(RR^d4Gj zjr2ZR$|ud>P71Ln%$A)(X}_ROrnF`45`#tShxYK$4gD<4B~#Fp>TU3ASQqO4qUNY`wpBzsbnGg;+#;Wua+H7b9S7)s0iN9Ell&hqF{dOxw< z&Sd!{%wg#+D8&QiHH34ny%6~{!-tbQq2L+wpS#5^8vvUCd{TWIrN03_1F)Cjm7&Z5 zj0PZ>$ug8G0Gj~ETFI=k9>raNXTb9&zyjc|OyY)$Z-(SMc*Jdu#LP?Fwb+N?3_Rk1 z%coEKY?y~f9MtT66PuMCiV+PZ;*_Knu)JdlJcwsr;=EEhp13Qc(XfQ4OL$_B?f9Rm zb7*gC3>$OhpQx5}al{@=ZF_Q`Snm><=G=qAUy#{X&@T%A6OJwW2-HuKRTJBuJSbjq bO|El_QSFYe#*di(N%z{#ZBHK-8RGu{3m-TZ diff --git a/litellm/tests/test_streaming.py b/litellm/tests/test_streaming.py index d37cc1c4e..c577567b7 100644 --- a/litellm/tests/test_streaming.py +++ b/litellm/tests/test_streaming.py @@ -82,7 +82,7 @@ def ai21_completion_call(): print(f"error occurred: {traceback.format_exc()}") pass - +ai21_completion_call() # test on openai completion call def test_openai_chat_completion_call(): try: @@ -122,7 +122,7 @@ async def completion_call(): print(f"error occurred: {traceback.format_exc()}") pass -asyncio.run(completion_call()) +# asyncio.run(completion_call()) # # test on azure completion call # try: diff --git a/litellm/utils.py b/litellm/utils.py index e28e9e29a..d91723eed 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -87,6 +87,12 @@ class Message(OpenAIObject): self.role = role self.logprobs = logprobs +class Delta(OpenAIObject): + def __init__(self, content="default", logprobs=None, **params): + super(Delta, self).__init__(**params) + self.content = content + self.logprobs = logprobs + class Choices(OpenAIObject): def __init__(self, finish_reason="stop", index=0, message=Message(), **params): @@ -95,11 +101,20 @@ class Choices(OpenAIObject): self.index = index self.message = message +class StreamingChoices(OpenAIObject): + def __init__(self, finish_reason="stop", index=0, delta=Delta(), **params): + super(StreamingChoices, self).__init__(**params) + self.finish_reason = finish_reason + self.index = index + self.delta = delta class ModelResponse(OpenAIObject): - def __init__(self, choices=None, created=None, model=None, usage=None, **params): + def __init__(self, choices=None, created=None, model=None, usage=None, stream=False, **params): super(ModelResponse, self).__init__(**params) - self.choices = self.choices = choices if choices else [Choices(message=Message())] + if stream: + self.choices = self.choices = choices if choices else [StreamingChoices()] + else: + self.choices = self.choices = choices if choices else [Choices()] self.created = created self.model = model self.usage = ( @@ -2274,7 +2289,7 @@ class CustomStreamWrapper: def __next__(self): try: - completion_obj = {"role": "assistant", "content": ""} + completion_obj = {"content": ""} if self.model in litellm.anthropic_models: chunk = next(self.completion_stream) completion_obj["content"] = self.handle_anthropic_chunk(chunk) @@ -2315,19 +2330,12 @@ class CustomStreamWrapper: return chunk # open ai returns finish_reason, we should just return the openai chunk #completion_obj["content"] = self.handle_openai_chat_completion_chunk(chunk) - # LOGGING threading.Thread(target=self.logging_obj.success_handler, args=(completion_obj,)).start() # return this for all models - return { - "choices": - [ - { - "delta": completion_obj, - "finish_reason": None - }, - ] - } + model_response = ModelResponse(stream=True) + model_response.choices[0].delta = completion_obj + return model_response except Exception as e: raise StopIteration diff --git a/pyproject.toml b/pyproject.toml index 9f73e68f1..6347dbbe8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "litellm" -version = "0.1.609" +version = "0.1.610" description = "Library to easily interface with LLM API providers" authors = ["BerriAI"] license = "MIT License"