From dcb866b3539a0d5b299b249a7f452ce150aff036 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Tue, 17 Oct 2023 09:22:25 -0700 Subject: [PATCH] docs(proxy_server.md): update proxy server docs to include multi-agent autogen tutorial --- docs/my-website/docs/proxy_server.md | 79 ++++++++++++++++++++++ litellm/__pycache__/main.cpython-311.pyc | Bin 53849 -> 53755 bytes litellm/__pycache__/utils.cpython-311.pyc | Bin 170871 -> 170877 bytes litellm/llms/baseten.py | 2 +- litellm/llms/huggingface_restapi.py | 16 +++-- litellm/main.py | 1 - litellm/proxy/llm.py | 3 +- litellm/proxy/proxy_server.py | 38 ++++++++--- litellm/utils.py | 2 +- 9 files changed, 122 insertions(+), 19 deletions(-) diff --git a/docs/my-website/docs/proxy_server.md b/docs/my-website/docs/proxy_server.md index d2a9d521f..37e009e4c 100644 --- a/docs/my-website/docs/proxy_server.md +++ b/docs/my-website/docs/proxy_server.md @@ -208,6 +208,85 @@ user_proxy.initiate_chat(assistant, message="Plot a chart of META and TESLA stoc Credits [@victordibia](https://github.com/microsoft/autogen/issues/45#issuecomment-1749921972) for this tutorial. + +```python +from autogen import AssistantAgent, GroupChatManager, UserProxyAgent +from autogen.agentchat import GroupChat + +config_list = [ + { + "model": "ollama/mistralorca", + "api_base": "http://localhost:8000", # litellm compatible endpoint + "api_type": "open_ai", + "api_key": "NULL", # just a placeholder + } +] +llm_config = {"config_list": config_list, "seed": 42} + +code_config_list = [ + { + "model": "ollama/phind-code", + "api_base": "http://localhost:8000", # litellm compatible endpoint + "api_type": "open_ai", + "api_key": "NULL", # just a placeholder + } +] + +code_config = {"config_list": code_config_list, "seed": 42} + +admin = UserProxyAgent( + name="Admin", + system_message="A human admin. Interact with the planner to discuss the plan. Plan execution needs to be approved by this admin.", + llm_config=llm_config, + code_execution_config=False, +) + + +engineer = AssistantAgent( + name="Engineer", + llm_config=code_config, + system_message="""Engineer. You follow an approved plan. You write python/shell code to solve tasks. Wrap the code in a code block that specifies the script type. The user can't modify your code. So do not suggest incomplete code which requires others to modify. Don't use a code block if it's not intended to be executed by the executor. +Don't include multiple code blocks in one response. Do not ask others to copy and paste the result. Check the execution result returned by the executor. +If the result indicates there is an error, fix the error and output the code again. Suggest the full code instead of partial code or code changes. If the error can't be fixed or if the task is not solved even after the code is executed successfully, analyze the problem, revisit your assumption, collect additional info you need, and think of a different approach to try. +""", +) +planner = AssistantAgent( + name="Planner", + system_message="""Planner. Suggest a plan. Revise the plan based on feedback from admin and critic, until admin approval. +The plan may involve an engineer who can write code and a scientist who doesn't write code. +Explain the plan first. Be clear which step is performed by an engineer, and which step is performed by a scientist. +""", + llm_config=llm_config, +) +executor = UserProxyAgent( + name="Executor", + system_message="Executor. Execute the code written by the engineer and report the result.", + human_input_mode="NEVER", + llm_config=llm_config, + code_execution_config={"last_n_messages": 3, "work_dir": "paper"}, +) +critic = AssistantAgent( + name="Critic", + system_message="Critic. Double check plan, claims, code from other agents and provide feedback. Check whether the plan includes adding verifiable info such as source URL.", + llm_config=llm_config, +) +groupchat = GroupChat( + agents=[admin, engineer, planner, executor, critic], + messages=[], + max_round=50, +) +manager = GroupChatManager(groupchat=groupchat, llm_config=llm_config) + + +admin.initiate_chat( + manager, + message=""" +""", +) +``` + +Credits [@Nathan](https://gist.github.com/CUexter) for this tutorial. + ```python diff --git a/litellm/__pycache__/main.cpython-311.pyc b/litellm/__pycache__/main.cpython-311.pyc index a153c493047a2ac9ad2786b83f4eff8ea5320a8f..f990b64c1e8cfc9b6e7c262a15a810970e0b2f4a 100644 GIT binary patch delta 2737 zcmZux3s6+o89wJO%ktWXEbQ`hfdyfKWdQ*}SYel4Ku|y-!WgI&FKSSM+}&ss?}oIc zK@4M7f2v?n(HKleYISZqsgoK*E44}6shxJjQfr-d`be9J*G{Z6?TqxC!<+WZ-19%r z`M>{h&u=d&{_~DPb6KqpL1Z1%XLU@6U)Qu7v0r-G_#uZcOW83mVnPOEzpB8OL1Vh3 z2=96<-8Xd;BN1bz-ARQQtEEx%0H?exBGCpPr5Gj0x-kxSN`FjqV|+(SOSfTclbSPP zF}@&qGaXnF+BGZvz-q;KAN(cj4vBEpmWb6JF^1o}pQB)epTL9c2k>657XFcw7g|Mi zbWLF4UYHvN<=%MufU^xb2Kd+7Kw&GG|KVIB0;S=2Ey_U zu%#gO#lv$sd&2pzh%_&ah*sd=vVh_o zC@)G=IKAJY08~!VA(|(?#>U|5EPVOKaC6gA2w{-#V`V8=abGiJ=WM{=fNL8qSYoN@ ziz^^yV}drDr26LksC9b3Pi)1wA8r@dEKBswlXjE#BjR{;-m%sD{=DO*61(ZfLU@G4 z)~P3#ygR=PqopR5TXZ)d?<9PwGy^^;t-!}fQ=-Co@KuTxF4U^_ zRf&~i72gY)Wj6KeODKtEUs(q|&PG1+JJ zb6v%xMyFt(!(=)=U(GT8)c@%<&w4$~2DirhGuCZ~)1dyvGVptWch=xjaLsAPBk*~e zk-So$IW5W&^3?pAJT=jFo0}V zS-`hd+{y>c**$@Thk0MEcX|l{KMHxS4Du_sxl+lqM)N=GT4y`QpC2cW;}!NmQg422 zRLNf$PcbA3@nHB zF5Gt8B7eUGluSj8m?cI`j6+aGB7RRwukb}kQ#%g?;6BLR^>f@U&F=b?lODH!TY)t& zvZpyvPaaCa2+#JM!!y#Uy{#A@k-pgXT+6M^kwRY`Q1_bAl({;^(PBn-gIy*{e`9e( z?cmTojHz^w3u#g+?uF$wA;mOO5`J2lN)6N23U=(FzLKNzpAAkUrGL$Gsww@n#F9MP0M(^chBI$5TiyKT~`ca}1Z zTkm8pla*G7jcGQ=JCYbvgB+2J**Fims3YtGaXX%2C; zDixD0R=uMYDJENylE~7{{x*z@VC_Jj(oU-QU8!l{NdW%4wE1`(#{ZV~yq$ot3;O+= zaW(wPze(9loNh=FCwwZd0>g*Lqu|PL1KtUl)m2k&C;V>27T8YM*#Jp*rc@QUld3D=g$sSX?0|*( z{)=u;$u^_x5nk%@3X6eREa4e3S;AHYoKaS8ce`t66x-dd8Rd3&gS%0nx6cF%9iE=< zE>DMN4o)HsxroUVMlMSr3wB&A!by_<;`BH_ib@z$4)w~h zOpHToBIS6-u0;f(iLpt@uy%O&P^)ZaVj9vK{hq;tvW1E16rFh}Ud~`*Cej%Ey9Zlk qD--ijpkB^rEJ-_YA^RsPj^n;SWP74OTu`4u4q`s-lKjh;xc>t0xXTRy delta 2731 zcmZuydr(tX8b9ZT1O)Pckc0$7!pjI4;v>-@n2?YV6(9JhwN+e|8=o{_h%Rey7Q5Ao zmDT8%uBfQky3~%0#ol(TyVc6Hj&5f=b!WOhn6CRz_hGlwz3y(?o#~9b=Ny8H(>uA} z>wMpLe&2V#d-B5@vb$$wsyCI&OhlfGBlfUfdqK6{fZr9j8Ln{ntT@%Q3KP~-@RAIl z1(oHxOdG-C9ZMToBx-@!QtZaqBKDQ+gSXKzeGSgP)D9%;+MdX z-yq7%i!r_{POflZ>=Rd37GNxju_-<*%Z}uT@7wJd?}WcrT_-!9cNAge9um26D@Vaa z@4y#RzcBGzgqq{%1UG;YI!E5o2cy^_*i~(W>(iV$^Tw=$TL+ZHO9@w}*Fb1`sk$|l zYB}E;H^p8f5hh4ZFT*du)oLGZg^0xnx*2+~J11li-05WZ2-MD4hT9-9;{;v+RZat; zL(W1?#2G6Y+Xo@O9Uj`vcpDt3m;~=T53$~uRD85gCfLs412}Gv)I-8$hJ!AZB{m*> ze2!4Ni{rP)YlS(4?|=_nC2HDAED#I-(>{)m!Z)sGvO7oNcjWRv6X1b6&%SUB!44GN z&v(X6LT=dcL8Z5K)S%P3A8HxlOFF=1) z8SZ4A=$e&T5Kncin`AbPtN7+h1b%7dF8BL*n+8bRiri9do?Z>2UU1ec-MM~^dX zX@O2}81ILBUSm!&9ugV?s971GD};o(WEL{&98dKyP&W;~20Q9(n1^Y}Jb0(BdQ>{> zvs->ezM4|Lsg~pyO5^qf7pWmFIWpS34h^iCp||qAsb2Q;M}Fk2!|be&va@cCeag<- z?AI$ldjj(&sPi}CBdqUl*DK`ovN=YtDZCkO_{-(T$Q|)V5@`f=eJMT;&iYwTRY7O` zSSmhK+m}N0cYZ#f*19#mAeI3`XLSylplQc^GQLn)$fu=>P!Ktp@>j?1K1Ri#08OA0 zpM=_gjoeGLH=O~S;}qXNK(4`a>?)+*{OLY9|9T%c_{Mm9e37uIKSJV)K?MZ?xj8?4cAN6fmeee-Oq`113uY2?$A*8Co~iImF;{M$!;S# zk#Eawbm6wWfrZ`EhB&JKMejFI{lJfht{vyi1F7)du|HkEGT*{u!WLgh&1yar%w;ZX%w}g)po+ zkPnT$%kU~V)f>QO__FsIe-k~UIx-C-ijKBuEX+sm5hXgGn9DPw4O9+MvX=U-qhvWH zi>a-VN~_?-qYgz25s_RRI_gbVtfJ-+eA~aflm(Pi-|r~-mJ&H#eza|LKAbwObJO%A z^3c*?aM6ftX)rJ%Um9E%Tpp#lMkbKKYj2BmggZtzFQyqr5y01{gSbST{`y4~Ho{j! zYjtM2D5Hddw-TvC3%+Z5*nUBcmxy~V45bs-nV05cjrh`~BOJDa@q>q04*8cWu}2JC zj$*QY@k$N$!@sYr#zC>^!v;BN>yeK);_2|ek9Xn);?8Rlhjp;^Mi4I+FWit}9ESIP zw>OKPN|bgz+9HNNnUdDESc5YqMCqL@l_X=T0;#o9IukRHCQq8c#7vZDkg}MVjdD%B zWl4*qWU2~f=OuzYUP;YV4a(3YWV_>14pVcHQkPiXvq;i1RgZFvy}G1YGElWk%0~uE z@AhPyWMs=GlyB{qC%dEqrcOjTI?2q$LPP*sm{^STg}p7wW~qd!labnx=;(PtDrag1 zMOTtEOO;HWf>ioMOHZ?8XQ~q^w33U7r0k-x>K~+Y9QPkY$O9STGIr~w4Ccd&hdx?{#S-+b@k_x)DadbT=s z>eS75kEH+fNP5=YSy|~_=-*|z!}KQ}%<7%3r8$iu+1=C4R9~R3K?Z7r#_VjF-o4Kz z+l|?q=s)8~_N-xjPj#^+|HojQ*Y(8UYFb4O1CUDs0uz1Cs_F6gD2o&}c+PJ@{yRo!uZX}pHM2H2(uDlT$i!f#|z#8+9yaVquDYsT>t^gKPc|+o zJzC7GuBL`y5n6Yiz(@eEu@R+>#?HQj#75{fa=hXN)8-B7 z^+An_Wo1_9M&pV8zwNdeL$(t9OPj;ka@K+Yn9{VZ40!!!`XZOFQJ*GdK(dI;)OtM_ zbk*rW={Hi#FDzwcY{P^soE<1_2PnWTk*4x9!ftgkng^aW1Z^be-=P)NrmZokOL;qf z?*y~}c*5<*3j<4Ycn(!IVkBW$(v999x**uM%_E&vFNNzHQjlN(o z_n(GAAj6St*$ieiQ+2<`$!ju&EynKAwIg_W%oa*fjI+ly7h7Cm zPW!4gE?F03+Se{KD)os*%Gt{t+!w2uc3;5l@&%=iAL+zzKsw1M1m!NOkUNbnXHT%R z{I(k}pFKWU) zkUlov$IJ~#Z=Kig@;Rj*YzX*C);uDsArK5XT@79*(dCo(V7|S^;VE46 zU|qHW-ihoyU+ZdTzTYx-FE}@6AFA1%K7dlx_-esG@u*Q;GwGbTHIa3|Cd|xsVLAh~ zD=l+>3xgo5j`A^Xs+l(fqs?Tu%e_LkOc-LDUc;TuuECJhUG)j=crXRK;e}Lk2rkqA zJjQ^9V@eOA^-}=vF$kG@$CHodiL(X{)WIYe~GvF_P z7XVBo6f2=VGcBRhmr<#V@r!#EvfMw%Z{D9vQCeX577rM96vcA@;{oRa*nVC^sgn`C zX*{(!e||z>2hrzaz}o;5@D2c?F!TKS8fSgLqj%Uc36Zi7U~1Zo$M*G(OuZ;QQ`}=z zdHRQ08Bxsh2>_dCudqBBT<~5^UW$4b$~H>?IL^#ub_cQ^jR{Lj@9c+uRc#m(vws^E zTaDA6>EcUL=2RoB_e&_WD(vW%P^LVEabE*=0k9>_oaHXBuTkpk5NciSRlM&kD!k~> z&GN9$P+*nrM=y7y&lRF`ft~efOl0W}r5I6EdJ{jNVFURU<uB93Dk3yp7m~*nF-=Q``~J3BtaE9TiWo3+-*jg!*!k9&y*dk|OGi zF^z+}u>nBe4o&t$pIZQr8EYCH)sLeH*Jzz|Y*i`v$r9X+68i?r=caV90N!aZPis1s z?sns=#^J-DJDCOcnE`f~-=OaAfMjYglYF}0qGg>iXLWHHZqm%g5eQdWNB>*ymlf!a zgWn1=wxHw$uoye-Cm9k0KR-KA)Ye6+gNW*u^n zrJ1kp%o6!$?M<>cX89Hq!ERDX9e2bT^Z*qoK z(0Svr4nf{;1>mg+P#Bqb?K!r=g(wdN#G8UP8h&ZnWGien?QWkJ$M3Mne_ww{h<(QL z8=oBTXX@feHq$NF>37xZawJV+W;VE_t3K$gcZJ+5Ur$WTo-N zmI2}+qj2kNyXB*mcu5o(OoOr2(-zV%4^c>!e@l~C8rgiyd&v&yvz=x)g+vmnHf^i)rXbxu zSe%zq7wIy#-Tt^(6`6lWFx6H`eT-8*ij4Q}?)PsgGJ4%}aZh&a)+Cq-bJAJBQ`Z>A zJ^jKRRzU|yg#sAnV<{vw4#0%wNWH=5b<;`QZef$qwhANCcgn_0^P(ycpj%sRKsSyY8NwQvM>e8svTH3JEtcC`jf%__jh07P?sDt00aY2H z0BsH07EYI!7m#Z(3ETDMC^Z4r0&oBaa|p^h{4-wK*ITTQe7SE@mN;k(Ywb;jIk&Zk z@EhLN>EaIKf!6*hES+MZ#A2=3_*?5Gd9ezblL@!UVC2$277EcES^Ze0meYdO*hOED zl3_gi#G<@yRHOre{jD3-x-tC8i8(i-2*Fr7Xo;+QGAeq)=g3OH)o8i~uz`R=I%Ui| zINHG-xDC~EQ@qzkl&%F-Vt9*j-@)Q;kQ5zOod#WFynFCBC7mlf488EFp?%i<`p|f< zjmEC0di3W>nC%2jOmroPuxK*gf2y`5!A@cb_t{`v^7NQ+qM6x3A2SUX$9mT)3V5c0 zm?TRki4G~Xx<>}wt4O+B$&>J;Oaf10ch$Sr2K;7Lz3cKCgd$S|js6hD5lluq(G&i6 zNmAGNik3cj5KEf*YcVc3)UWW@`fN0=IaE=aXh|^Har7I997_`9Cn!(z=z4?Kzg)WL z>`V-0VKecT_((FA+L;)(hdUtq>(Fq$vG|#B7hj3uRe=BNG&Giocp6$sY6vu#$($-6 zOFsFGE9^#ViTwYp!rO%@PVBIQSS>r)pu4?vAGMAJ$-8DVSmDt6zo?We6@G64+zPnO zFb|(6UWgPQIhbwb19+RkJ-Cz9gHO?^f9Cbd=o6 zN#di(spFMN;W$fXcJ;hzRh6@xmGfpfFPOH3Su}Git7>LmSUqpn)Oh{9pdhPq)~Jyq z<8`m2F28clw0Sd~m2>9JtcEKUVFFk#7Et` zLs({R)tq_GsdK6qo*!>*FuwYzn^{WKj{tj&lP7v>U9?Er z$4$0T+@%)vkOp0pAPvrqS*a>B8*vitaqGSjlu3~TpSd*QHj2I|>$d_6AmT7>wSj<7 zuH&M|>ke_GMX|TdSn|arkrTP~i_P}JAeu$+m*IkO?%$ibKMQVB0}UZ>fRZFXh#{tE zmUfC9>3m;UEQ_cliwXW^sKaod9x0wRwx9lOFHD~>P3X{ zis^KhYS3lKs|OuPrj4VeKD3OP&f9{_N(SWsYuYjp>gT=C1OIP z`TM_Vxrj{6G=F0~Uo(TUF%tgqdC?sXj)J1un2K{Sd>SXopyB$dk7zWm`Dtk(Zx;?~ z;=7-ud-|twVZ{HY9rst=Lm_CgbouF4gG*~HY^p`<>2yfT!?iR0BQj2u|7vTHlO?ofWuyX19*+V%!3EGuzF$5vu>9iVhy~7%JZbyiZ`}#R5IVdF}r2I5%_%_u3oaSLFu)I8%xO zSm*N~pZS1dwKGf16w9I~vc$1$(HcF}Q=As!3q`?RKM_&KdW)VSs($D#&a-oPR;$h{ z71g7`h+Kjha1W!rxZfMHGE$tKcoiJip?L)$0Pw4Wr9v0~P~~M}R`@7- z8aZGl=aD|*!(96rERdp3juICZ)MF6em8j6=2W@0U^*$10)gbwOi8X9rq~)$IaggeVbJMXwiQ#PqGs2xajPS>Fn1AtWl4T z7K6gOsT*7j#d?&Q@YPEjmgEp;8m2+OPq!KNlgU<|cRN_)g)o>0n0DFdcX~aYsfwV( z0l>q6uL#zbPea1JtlI5vlv3$qM6Wq;8L~eCd7Ygoqry?fRq`1S)Ph=8bTWSOy)aED z&!w}TSm!OEYnVDZMihlNpv>lk@QKaIOmVv!*e8$2WSpgBuVhD$8!+HB8m>o)GX$GZ zVq=S^o~*nkj7X+qST2F1_+Juc-O(YD7qAQi{ohenoGqpfW$CdlIa|PTWD#d_PTix2 z$bAMaqbp{&y<@DftCxCe$tiLmCK?#cA1lH+C7rB>XNuY3y)(o}$5!ybyNT77Fw2|J zPyo0Yuv*z?ihlMTD6dh&XNvv{ZbkVv0y7Ct=2vY({ueqWjDZ*FWNtgrg%xLahSa0{ESJZ>AU#hGH$@_eUqzJ*)li_{n_iLibLa+)_Qp-ho!uKCifuesaRg z8EkaB>9I_*Qd!a0p(|@UuFo(lG?8g!b<ARd z);gd2>{9Zmv*k~KDs#5TwwHsE@@Uy?u`_uzxHmIgiQ|%@mN+>b10gfT<)&;Bd94Sc zr>ezeLaP*pt4F@-U80TXc^`<$wh&D`*Dt4uc~FaC%YpLlBZzieAdZVLtZZ$@Ox@=Tlr0WO zpGQ{G^5hp1E5NH#c5*Q2#viTNR%` zthze7Oq5sw*e`p?De9+XVyJi_I@l>Lvx)WUcWyCAT&|vTi-{$?ANcGwc^VMM2c}eI z9#Jw4z7T@CtdRn(plQPspR7XHc>vbyVU+mj<{VJmEz$`ACCJxQ*dxx#SdY2?8-#CB zhdiP}d=UN4BQk9nM^X0@;ANHV6@zkKLHSj{Yk=3)Os}}4_E)JyJNxVRP`VmRm?*si zNYKe_GO-EjY}V}WqbJk$kxE`E`ic|Lfh)z&WjP4w>I=oE7Q4Ct++wx zW0*T=+zYev-U7G}eGLLq?;w!F78&m;8n!H8q5O=G#E#guu2hZd0Ra3T#8tb~oA>kf%=T@;UgM&w+lTKabxJf)DURUql zBwEB1QP*!poe(dn=WiDC7VSh2ej?$Mm8D(=LI+rK>rYddS=F(}Xg-gv-Ggw^%vcc$ zH8_JIy1)b_Zx!~opQ?Z%`j)Wks=xw#p<$lc&|B0#L-Yzm6K1k6;PM2mOAH^SB^a;{ z62^W=oSl{hCL+TAG+Leo_yC;;P^mrhdkPh}Y?!$TXXuQ*^?wD9(kPLjW^5DPikOIO z`bjVQ?Uw`DrWz`=V*B@)=lkf*+eB86aCe%ohPGNCED5)2+C474KH#Ttm${A;Waj!P zD?<^IGx7exn8V$U)r0tnH*;ZWYCL-X>}4%T`fRf_)QA zrg_0ojH5G{>`b)pABCok*FkDwP^a?;Spq*L7Ns-xEb%Rt$c(j-hi%lE$=C%zEqh!H zu%nOsHhRP3qL*gv0v2SxiaaUCm-E9SD+9JiiWvrGOYl+zX00r5J|3K+<$n?v2+>>7 zJ?cDBrXDybR_Enoa?C*@;E^@aaZiblMLuLU{fA_Ha5VlT*pr*o$U~x_7~aFo@CKdB z>1q;OLFuE3Xtk-Ohs3N=>}pPfeYR|an)X^>IM8XLMnzi;z&abYTwzNi8@*-f-CWlgsX`}T$9P4tXs{Mng5txTP z+~DY&M?{)!WMbyEDNF+0HU^;AKrD~}4$D>s1AeOJds2a#PefsK-aBH7)-SOLYmDck@BXMC zMRB|VN2nj)6QwHaJyEEJzc2cu^ipYJT!J`uYaF#+7^N}0VvI5iaXw=<4dT(?%(Nao zV}fS8+XO0^l1z)UZ1nEL9!^E;$U+{@a{sKxgv}>q#yEf-jEa_jDDrLnp$#*+wuur{ z*7i2jt+i|Lz{yKH)Y4Bxx4A4}sFxJUa!eQv%NvUBxUbA~-px+_CepHhEX)Szh?2~Y z7A*3ER`!FePL>!)-ktih%o@Np2N8Fp#Ca#=sN|R6{dJT$V}DScI3vc?aaNu)V6A9+ zl;C7vPc9-6Mkyzu}*@6hR6dx=hhOVUlqHP)4B|;RdiY)E93AixPaW)I)#gb$-LMpTcbQ()9 zhxEjl2-4Veevj^-sj=DGC=rOR%+~y39AYGTYvlk5$#(Y|5645+xls*?^Y*cu8RYPm zd+X>;i=wEvRZfmJK(z5^3~Ek}mM2z6ojICK*c)kiw@?R%(n+x{S35f;_GPSUE!Gz1 z;6CT=l1kGq!`%CfOM4Xj2UqF!-L<-MKaHSITdcc2I5Knm*nqKnvPacs02+Q^3B*uFXK#Xg3uKM4&>!3baTn<07rI3z|5 zdntrh)~83$sQY_peI`Ld@*wEXW1kwdzNfL|YwdE}X3eh4DJ_q$)Y?J)Rz7Gq^_F}My9zz71|SV$X6u)`b$<78^7G3z>dv0p9C1_h%bwZ^dujZd zNHJU!&1c#zdgSZi>qGTTKW%UrHyrsERXDnKxG4VnJ0eRcv4!K8jzkKGZ0=@|iDfO) z`9v{Lw~ZiZC;AZ=tDXI|-eP<7vHsfRcj2Wfr7+pz*~7e0=lP}9x0k^iZWvUf}wQ_BYrY@_{iqu_$iMQLMhX-q0>?PcjH);!Y zKRFZw_W;(Y`cc|%7F~__iQi`p<&8*N#k zPyK1ERxkFeq2si##2D4<9IbB#i-e8Tr)Hm{jTe#VhI6zck+%-BTtys^cJ`wD2X)_g zty|7RC=UjYu~Dy&*Jj%vMD2sB|GC;%!_k|(9{1@mJq_t}FMJq-9#&H(XwChQkD~8T zc#~r3j=3po{k_rS6SPOg7$%1G%PzHr=5=K7YGxLH4;pmB-O#;-mBevDt$J~y=Cc0* z3;rQGdXiRZFJbaIV3-YVSZdt4RnqJtEs1Kc|hR1~a*lzK@pHV_Lh5t8H@9 zd+=)78PW>UE>`nqYd45DqVLYuj%y`vfe5f`c6B;ket&=xC3JfZI-TmBxmsz-`)KcU zRJNh6O?9o(9u#fSr>e9SqBri|4q;M|zT8D01v=?Xj=tY*l0QIx`=T@EX};v1mDH(f zZT?j2u~hez#qx7y{p3FN(n9TUw;;N6Dw*WbrTu!|sP-+=c6U34b-yO~g)wpC5Owon zZE~1ngvT+*N9cP3z(U#&@gAlIGll+~!cQMl%42A}8h!c1VU_ddW|rSYD-+M2=o9)$ z59st%O<$V3XhN z4VcOE8+FQkuqCkgzQd5yfKJBpA?g6-Ii$D`XqGG0k!9M);=|~DPVE~Z{-k!fw6imK zDi(=Xec;mO*nh;pGpeFin_%x^qq0zqwc71sV6?!k{V9dUy}nW#b2fBr=35_*>wb5@ zgAZl-)P+gzP3~X`A7e&nk`D7ptF#rIXQB&xLM5z?&()Kww5!CIYO+t8O!v8~d|Ef_ zPfzS1<3sg(pH?2`6M^kTqgpem-s_i1OeVkvV5)i1Z?P!L$>J(b2%9O(j#>AhAq8Xq zMy2+MIulOC6{3T-Ar!QZla#dB){)25{CaJWJsE?3;J+Lb`=htkYn91X%J(yMTxz3B zn6o|5rihc4j#5E%SWs(A5oyZata)?zA&vV%e3V%WH*144GSM>&kgXnW*7}LA>g{Ij zNC|Hr-gG%=WHw`ZYrR4%3$s!oB>sCbDTkB&n(6rZCbo^3e~$jyPVz7dOP2i-Z=`(G zB~*vqO+hig4LL)u;3~=DW65FXnl@~3N?y^|6P3EP4Ixa49qs68-FS;JXLrE=E*G?x zs9V=*v(9A$frQOmtVmD8&N`hEV*DoxP@I{G`yIvo{M=yK0h@rPdab8JmcusoM#o43 zvPhl3UTYS!qX*Y(`;#eg-_WAHAsl=b^~Aa@kbV3M4?b>AQhn?@o5z*^TCKO6gOJ1M z$p`05D$x$K7->|6O{GO-w&LoT>rgJu-Tq!x6}?`zclJ>C?3)zJ2&{0|CMrgH!Q delta 15964 zcmbt*3w%`7wLbe~=H&e*gak+e;Wa#zrwAetAc0UwFocI!bezmgk|C3suxA1ippo)W ziimnB+iJzC_yAj9<7Gg(qQytO6yKsff?8C(Xf3uyZS?|oed{DMA!`5s-@WOtpL_OW z?X}lhd+oK?KAhc~^3C3q^as<^Q+m)}=%bO|1COT{Wr!qWWJZBq{)8%x`57{$cgc;m zo6>Kj{~3SCs2tz_%N{oS|2N3dL`KdusQUha_yA1+KY^)-!qV$%HdCcH)DjGYyw2v3oEs@Ad_~Ja ze@B8GfzHbT?%Vp6h~-BAemCm;`x4`xlAbzGw$ymJpHD0`#umLXtqy&c5tufQE9~Og z9LfKh5!sAE0l-|fbBWf=xT^TA-aJbyRfZA>N=mMmqtIiyF{`9jUy0&%%t9NAZN}3j z*Jy1$jMvZP8W(S|8)N%lB|OHq{^er5@pk_$;u^y>U_kLT=z9ZTGJws2scyp0O~$MfTtC|Mw8QnNP{ay58Evf0QOTvNLMFyK&!tw@V+_;|%=lC6BZUMIlumz;I^GS+cN*I-nXI!ww;M+1*%>Tz+Tg zZjwAAy(Jh5J6$b4mcD!t^d2$}PoJyrMscsvZ^r0RQ4~8w`F>~FQX<;|FJd1W_5+?W zemY}d>eDD5AZV*IUL8Nh_VI+0ks4U!Daf7+ZefUeDOgvdn~@_^p3ui0xR1cECqEwCn;+bNj)b^=p(wY1Q# zksjl{MI$D@g!JN937EACG^nMrcM}P^~VVqmsJ1;Ig z-o!lhQRBIXiz1avQqshOMy+RHJBxiU2Au@J=692eK^fk4wpdh-O(O97KomaSL&-1#C)S@SyA4vQT-m^1mHV>Zd|b}SKo}{W@Fj1 zy<&5ue^XTGO#6Q0eSc}$hiKeQK%og+E}I0iv<^*izUHs^}u zh_CteL{V>C);c_&)eNmVG+7Ed3hY(B)=|Y`OT+I=YT-nJZKVi!I0 z=HZ>k<94=>#z?hUDu; z(~5#T9Nboruob0i0GqK#ra&T&G**ad;}hv18kreh26M#A0IOyyZBaTC8rZ)bjeNFZ zzs)+u&l$T{_i5lEEXsUh&|$OAvE;DV?``&m6lTr6nlCM3Ii% zY-*G||AllycAB~be^#`T8yUVPAt{x`293xjW8#K#@siQLVMrgaS-V>^bqduQXigCI z{Z|6@I%hn$VRU=k{JVJ$uqJNGZCN{c4u5eiLE;Q3U zB&fQeOHvdf{nj~<*+p2qn|KegL|Aq?Ae+t{cyN5f_r1fMHLr4p8!1Tk@OhVQZOB}# zJrB?`GIP^JTUJk$M*yrQW9!cg#mLA*KR+bI9>agrGeh9Y9dd6Dd_o(zu!DWJZ|*aHb=MofYG$=5;4WNcUx`p z9taE8rur|{dPVYXDboAaqHZ$aw*(i4rIG3#Hj&B!kfk}}8Z}y^8@H&nX^?|y3wy5% zQ*cyp`zo<3sNAC?QA9?=Bsnj&mBhT!(*F1$28E4BTI%b2k6K! zy4qcBbevf3uoV?)jTDYz>(sm4UKvDHYB)&Sgtmy&<>LwD3J_rB);wLNWvxdG`}55x-DVv5k3~5K70G9@ zbFM?R#~A<2l&mc%GASmyKC=FqsOSr)B+CHn(Ec;P1_Fxbb{LDE9k+x#a2u-S4Y6LE zP}&G6!*H--W(HcDoekaq*2fYMbd{k7I`F#D`I&Lz*`6_t{{$em8z$yqVXJ zK4voRlg+Ls3YKOP3v_pn&%qrM>%1NrbT^S2coa7?QD&jp`22VMDtWs2z#o$=ttB?I zr5_%{lILZtH?BBTn)gF}HW`}_jV+G1@IdCE>*)6mIj)RzX)(yF^mx53zCgWn(K#F+ z$}(l%Eh&)zEHhwccojP#;v3Ph*|_@oOaHAM#p?n8TQW43n0O9sCA0)vOgkqE$g)pA z?`p3@YoYwV%);A;Igal*-pv`I7O&e!H%;sCki3g-!cu;PMczTBY`Fu!w*qbl{L=XF zh0Dd^NYRVWW@N?b!H2jLRR8%DOlohJ*{|M)&bI*W1neaER!llI4TRYbr3C4&BFX2G zq3uPNhY8G7m*3mc=yds88eL{0yk3}k9Yp(cfPH{Hkzbp`1f^vTzqcl<5;f(37C^eO z_}EWS{R2l2`OPz0G9c9yra8P-7i<=7a8eD$$wI zW!XmB@e$&vG4c2iaWdjMzCA(vNT>tPir$gTKMhTo6_bLQQ9ZxBs;tskHowyOlkzKB zDrRiLy3V)ZPA^s>q^qesW;-ZXytaY4mf;5>7B*@E)g@~T)H1Z!r@sGeP3 zQy!~)0F}A37tWehSv9MoY(~80pmF@;`~p@$c1`*Gxs@}@YU8bIjPy@N>No@CE@Rp! zyHYWPF5~nkMcR20$vnBrmcw0KsE0`C_SIzE@maCZB2Rth(nOun_luI!29QC# zV%qA0LBCwXMUT&I-7_hmPcWQcOcj}tyT90~=Seh+(1nqPQFdlkuNSeJq+koYuFLcJa?5lymxHpMPTBA(iF=8o zqNkbR3$(O`ot@{g9Ey%$%i`!Lbz|CQGOACFmK1V=-qXr_-qgL_`JcY_Om769mopl8W*8mm(*gJiO zcI+hi7XY@0{43xM0y76&x2AewZFx;>hwW9jr3ptnHkv#Q;C-8heH-h86JGue@Fw8T z0BmCU1%Q3pmsE1(S^DNZbrenS5}1kJzzUxX23XCeP60JiJ0{xS1HkX~>f-p0q4GH3 zeKkK_^q%wqmCc+k(O#!NgL9N00^kwp7-xT04Qb2=%}USIZh0k%ED#? zWn=DvZUUeY?A9%1aMPz)f@SO|u~e^9p>bm1C??`VEv`MgJ5=~X16B}UZ3ES$*`i;2 z2`9k%%ll{yoCuwBl+Exr%LL5NferkNl`l!LJfYpn3~t9dc~%VO0hHZq4LE(C?kvJU zWCZXe;4cJigUczWuH56rpmxqv45UiBoyBrMjuMF~c?gW~LRrr?{6vPpO!kr|LgekW zRK6bkj6_S8yU`VPA~`Mt_?;dlQArdz|mXsiq4}puB>=c#j+N3hw+(e(#SAL;l<^3SO3-AEoLBOv8tY$vekM5Ww zO7uZk@u2A7IpSCLLM)YnUN^mo`6>YV)9OM-ov#tSblyMqXrBe*glLDww587W`u)L@#X;%!$Z0egZS*)j_{!Z0 zoKCZ95n|Getn>KkhGW~b&%P6ySi?^P$y{kr)GzM1=36=Q5B@Ohb+Ax)wJe5NIDRmm> z6a@HoT3gYPQnlvqiF(Mivp08SE%Uiyqoxx2v-rxSY^1x%xk9xpCf*K4+ZKx)A)ZiM zmx!=M}7vd>p-KnK)OHh0ss#M{65O+tlPOVua2|{hcbXMR>>a zI$LFcZV?;AfoQ|6qCtpP)T_6N`IGnnz8~HA5M!kjgZm+tFiVMqnO@!T zB8@{_T08Zay3!DR#HXs&5CaO?yNv}IzS8iTd7G?Pj~k+&6PhsX{-DbfvaT_F!WLq{ z8b}zsDSo6{78ws5OKG$m2>1}42ZO(kl<{d)T6HlFt>R&vH)RhW?u zkHO;O`wwg>bb$|$g&+@DlSXZGsl)X>KFscQJUZ|(VJ3{5j)}!eN4gdG+ge=oQ~>vF zCc5fE_=QKN7C9IFZ2XAOOO1I_Y!oTceNT$@wtTiq_z8Ajv>*<0)~cHQVoE!95}hh< zW8n}E$u1><$`_ufX0optZ$f@w9VdS|2_2#XDZZi1Ea~#m$32v>c2e{oZpF7BymgX@ zo|)Rx)Zip1jX`E&i)$71E#EJodAmO)eFt8?iw=ECXqxb-;Ri&YqOY*Jw2;?L-rh;h zyw%I%=(sy5&jB$+M<4lB^wtBSpJr_W7I}`^|BRS4_yh#NtAafeZ~XpQfVHw$xB5m4 z4vK4p=%=oGR$MNM)!t{tikut}#&{AGk6I9&@|-v+rb4n6-`epB(xg*%OmG8%nM%&M zh0Mzp3I=NFID>C7Q+*+4J>73YNWbtFu2E7JdJa zxI^;{0+&1)(`ri^fG3mzlLoYD1J<58Pk+)uUE1`kW>!~~UtKH9zzgDbax^0#ej82g zu}<>asIBu`sf$6lW2$2>ie%g9_*89)?#9+Y6Wv(`qt_5jkcy?2ETeT%HBJS&K=nE- zhB`65HDL-*Xa#s9QQLieV2!sld)v3bO~(fyUAV!(Dd;&GrK8ci4vWNuYrveD8V`p; z{G@ML8CKD;aAsyW_1$f)z6%RjJYAz$i##3E?~s7@s`#xaVF_GLqq|d+W#DCc^wrnI zQ(_o*n}!+^gQfT%8xVOzBx=**lI@T<#_}3t{E}5dTV42NL4j$S_uIEbZ!MF}z|<)< zN@*Qgq58ig@+LslYyuL?rY+8<6dT-W?cD{8SM})i?}+IM?OnKwt2>hdEYi0-7@+h* zm{jftC%oP(QK|%B3ANVb%m#CiY%Qdl0jxuR%gRxT8GyVfTK1k;td+*;vr3qh;Or`e ziI&oo_&^j#&Eq0T6vw;A*iX-Br7_d7m37!Iq=x=xn)PDZAyoES*vXbO z)2y|mqj$Fj;RLkKETrNr_rltWEzOaT*45ggBR&$jwst7OwAZbo)r;2F4hC6dLWXTSG42jFKU0(G({G0@s0$2=~L0~3YN(fmSbQbDs03J~02ZD0^Oy!N$ z2gO2_bx!oIt3>%~09y%i2c|vbr&lB^q^L&II#4ME>m07|slwMpgbMuOu4FTwj(#x% zvSLJC{rsHhGvXAQco%<)5(1Y_{f2yPUdqUNS2|hW@;ttA?HxUJPCS`fd@bg$BQSFo z(AiWLS`~0tQnrNODrG-)AVs^OJ(1XO^y>aq@JLKlgz^0Wr2v*^T!4_BWXZ6|SQsp0 zUN%dx7F}4-ETsngTn6BsIvR7NjsoVRGCz7GRcjI=UrkKcem@!4L^{Z3!?IYuZuEq# zXb*U3fSKM*Pmb|`j*Z}6bUy_br{-s9#Uc=G$dn$@LWHBsvbE~Oj<06b!QR@!>14^(8n8e-R)4A zb>IJgQmhyEsdoxBpY?GA?x6HUPw9GmOct_sO^#$2HW31Ch3L~EHMVa~kFkeg^ViU@ z6bs_qK^w%*ADVRV#7+yLm1RJOsALvc-b>9GqvcL_x8y6eEH-tn6ECRTYu{i9SHZh4t^sKSxix+X9y)&wAx8Kmf5`aJXzne}-y^ek`hT$ZTvJ1xek(uvxDRF(@{YLl8jQJW;5h;Esv^%4`;fXel>3R2NS zDDNgPGu^I$lUQ|je6ddl_~U3Cj)ixSF(+wr^v6*9m>PDOcKTurC%4Fby39vIy4^6J z#GohD?8(~dQl!S{3m4wV;H7Jt%wp)F=*h|2K9R-TvDVq=u2+vt(F(156~_lI_2v}K zrGtODJ348qR;(8?s~kA+!OSA#&ihoTT$|v6TPLL08bg~_()rV6b- zp}h>21kTN**b^O6J8j8j_-bn8+`01{sa@`z>>b(Ij)K{-1g5=}zOa_olU%2Rt6PPv z+zQ$k(hAqEgaULkt>XcYvuI|fYMP_15N}0K%+Ws7ir)ceSihO+bh-k8Al;|w-W+l| z<-63Rj?L4G^WrBbADl-q=BUc6(jFH_qc2ryje;JPM$Omq^?yQsd!qB^YkoWBil17j z6^cFT?SDEpn0^pl>`>EW8bX?M}1U;?C!@6OCj z>pOPRsB;!>qJEZ%G(q5x7-sk2-&u5CasLbTrvRS;J_j%{J~ps)xi(6n(y9b~G3J;T z?YC4*6xF%3f-2fx^H9l|O=MW4jmCFgr`U00keLcbDD8pQbGd|Sy5}^`sL<7#*$xAv z5A()i)^ULGfDgbE%i}|o_z1S6fgLr3Lb)DN;(w7!oaXYX=i)oViRJFFsaTOHlNH1e7{RCI8SR?^P;gGBh> zkV$zF8LpXvkDWWV4lkB(0BjQ3pu%EgHv@+vb5Ivo9HdUsFJ-XkBiFDi)FfF}EGFz$ z(}qn=|4xQJOfB&Uc# z0w0sJs6+<^=PH;PHh>zm^MpLtwM4w9qC^jSu4JZeJrjqlAmkXM6$3Rws`g({jrLd0qkBXAsBWqYy?H<+|6s8r8(u< z==GcF^Xhi|aA;N5*U$-kJXTfFsm=EqqGjcR9GMkWayju)QB^^IRs1LUHfiTET_83{ zI%qfrv>Yrj%twW-N*EQJ^E0U!J#m`q7{ z8K$qvLRQ?#d2Tj~)9Axq<04u!c~f7oF&u8OwiPy$nZmlIJ2ZW|MZYPtAbO~0wrBSuC0Hy#q?!qVS);H;Exdb=~I1k_pVJ6XQmzRMmTi9Hb z*ut(xDH9!s0Qg<|QPlC9@Bx$#0$uV(uTUX0PI28H=b2DZ=-ujMf5k@G~G6s{>isp VbcY-Le?{$?SX1;}L%Tome*i;h%qRc= diff --git a/litellm/llms/baseten.py b/litellm/llms/baseten.py index 8f24e129e..aecacd84f 100644 --- a/litellm/llms/baseten.py +++ b/litellm/llms/baseten.py @@ -121,7 +121,7 @@ def completion( sum_logprob = 0 for token in completion_response[0]["details"]["tokens"]: sum_logprob += token["logprob"] - model_response["choices"][0]["message"]["logprobs"] = sum_logprob + model_response["choices"][0]["message"]._logprobs = sum_logprob else: raise BasetenError( message=f"Unable to parse response. Original response: {response.text}", diff --git a/litellm/llms/huggingface_restapi.py b/litellm/llms/huggingface_restapi.py index c63c31150..b3c3e5e38 100644 --- a/litellm/llms/huggingface_restapi.py +++ b/litellm/llms/huggingface_restapi.py @@ -141,7 +141,6 @@ def completion( litellm_params=None, logger_fn=None, ): - print(f'headers inside hf rest api: {headers}') headers = validate_environment(api_key, headers) task = get_hf_task_for_model(model) print_verbose(f"{model}, {task}") @@ -254,8 +253,6 @@ def completion( ## Some servers might return streaming responses even though stream was not set to true. (e.g. Baseten) is_streamed = False - print(f"response keys: {response.__dict__.keys()}") - print(f"response keys: {response.__dict__['headers']}") if response.__dict__['headers']["Content-Type"] == "text/event-stream": is_streamed = True @@ -313,7 +310,7 @@ def completion( sum_logprob = 0 for token in completion_response[0]["details"]["tokens"]: sum_logprob += token["logprob"] - model_response["choices"][0]["message"]["logprobs"] = sum_logprob + model_response["choices"][0]["message"]._logprob = sum_logprob if "best_of" in optional_params and optional_params["best_of"] > 1: if "details" in completion_response[0] and "best_of_sequences" in completion_response[0]["details"]: choices_list = [] @@ -337,9 +334,14 @@ def completion( prompt_tokens = len( encoding.encode(input_text) ) ##[TODO] use the llama2 tokenizer here - completion_tokens = len( - encoding.encode(model_response["choices"][0]["message"].get("content", "")) - ) ##[TODO] use the llama2 tokenizer here + print_verbose(f'output: {model_response["choices"][0]["message"]}') + output_text = model_response["choices"][0]["message"].get("content", "") + if output_text is not None and len(output_text) > 0: + completion_tokens = len( + encoding.encode(model_response["choices"][0]["message"].get("content", "")) + ) ##[TODO] use the llama2 tokenizer here + else: + completion_tokens = 0 model_response["created"] = time.time() model_response["model"] = model diff --git a/litellm/main.py b/litellm/main.py index e24c5af73..6512ded0a 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -729,7 +729,6 @@ def completion( headers or litellm.headers ) - print(f'headers before hf rest api: {hf_headers}') model_response = huggingface_restapi.completion( model=model, messages=messages, diff --git a/litellm/proxy/llm.py b/litellm/proxy/llm.py index 878131697..85d34d06c 100644 --- a/litellm/proxy/llm.py +++ b/litellm/proxy/llm.py @@ -9,6 +9,7 @@ import backoff import openai.error import litellm +from litellm.utils import trim_messages import litellm.exceptions cost_dict: Dict[str, Dict[str, float]] = defaultdict(dict) @@ -113,7 +114,7 @@ def litellm_completion(data: Dict, user_api_base: Optional[str], user_headers: Optional[dict], user_debug: bool) -> litellm.ModelResponse: - try: + try: global debug debug = user_debug if user_model: diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 90a967921..6d67ccdcc 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -1,6 +1,7 @@ import sys, os, platform, time, copy import threading import shutil, random, traceback +messages = [] sys.path.insert( 0, os.path.abspath("../..") ) # Adds the parent directory to the system path - for litellm local dev @@ -70,6 +71,7 @@ print() import litellm from fastapi import FastAPI, Request from fastapi.routing import APIRouter +from fastapi.encoders import jsonable_encoder from fastapi.responses import StreamingResponse, FileResponse from fastapi.middleware.cors import CORSMiddleware import json @@ -106,6 +108,12 @@ def print_verbose(print_statement): if user_debug: print(print_statement) +def find_avatar_url(role): + role = role.replace(" ", "%20") + avatar_filename = f"avatars/{role}.png" + avatar_url = f"/static/{avatar_filename}" + return avatar_url + def usage_telemetry(feature: str): # helps us know if people are using this feature. Set `litellm --telemetry False` to your cli call to turn this off if user_telemetry: data = { @@ -161,7 +169,14 @@ def save_params_to_config(data: dict): # Write config to file with open(user_config_path, 'wb') as f: tomli_w.dump(config, f) - + +def print_cost_logs(): + with open('costs.json', 'r') as f: + # print this in green + print("\033[1;32m") + print(f.read()) + print("\033[0m") + return def load_config(): try: @@ -469,13 +484,20 @@ async def v1_chat_completion(request: Request): response = litellm_completion(data, type="chat_completion") return response -def print_cost_logs(): - with open('costs.json', 'r') as f: - # print this in green - print("\033[1;32m") - print(f.read()) - print("\033[0m") - return +@router.post("/send_message") +async def send_message(request: Request): + try: + data = await request.json() + role = data.get("role") + text = data.get("text") + + avatarUrl = find_avatar_url(role) + + message = {"role": role, "text": text, "avatarUrl": avatarUrl} + messages.append(message) + return jsonable_encoder(messages) + except: + return "An error occurred", 500 @router.get("/ollama_logs") async def retrieve_server_log(request: Request): diff --git a/litellm/utils.py b/litellm/utils.py index 478eeda36..a077dbc7e 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -119,7 +119,7 @@ class Message(OpenAIObject): super(Message, self).__init__(**params) self.content = content self.role = role - self.logprobs = logprobs + self._logprobs = logprobs class Delta(OpenAIObject): def __init__(self, content=None, logprobs=None, role=None, **params):