From e2c143dfbcd511fe81b423a6acb648cc07732c79 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Mon, 4 Sep 2023 15:42:22 -0700 Subject: [PATCH] test async streaming --- docs/my-website/docs/completion/stream.md | 38 +++++++++++++++++++ litellm/__pycache__/__init__.cpython-311.pyc | Bin 7852 -> 7908 bytes litellm/__pycache__/main.cpython-311.pyc | Bin 29166 -> 29129 bytes litellm/__pycache__/utils.cpython-311.pyc | Bin 84544 -> 84544 bytes litellm/tests/test_streaming.py | 24 +++++++++++- 5 files changed, 61 insertions(+), 1 deletion(-) diff --git a/docs/my-website/docs/completion/stream.md b/docs/my-website/docs/completion/stream.md index e5fa50ab5a..b954e60ad1 100644 --- a/docs/my-website/docs/completion/stream.md +++ b/docs/my-website/docs/completion/stream.md @@ -30,4 +30,42 @@ async def test_get_response(): response = asyncio.run(test_get_response()) print(response) +``` + +## Async Streaming +We've implemented an `__anext__()` function in the streaming object returned. This +### Usage +``` +from litellm import acompletion +import asyncio + +def logger_fn(model_call_object: dict): + print(f"LOGGER FUNCTION: {model_call_object}") + + +user_message = "Hello, how are you?" +messages = [{"content": user_message, "role": "user"}] + +# # test on ai21 completion call +async def ai21_async_completion_call(): + try: + response = completion( + model="j2-ultra", messages=messages, stream=True, logger_fn=logger_fn + ) + print(f"response: {response}") + complete_response = "" + start_time = time.time() + # Change for loop to async for loop + async for chunk in response: + chunk_time = time.time() + print(f"time since initial request: {chunk_time - start_time:.5f}") + print(chunk["choices"][0]["delta"]) + complete_response += chunk["choices"][0]["delta"]["content"] + if complete_response == "": + raise Exception("Empty response received") + except: + print(f"error occurred: {traceback.format_exc()}") + pass + +asyncio.run(ai21_async_completion_call()) ``` \ No newline at end of file diff --git a/litellm/__pycache__/__init__.cpython-311.pyc b/litellm/__pycache__/__init__.cpython-311.pyc index 1417494660b33c0cd7f2b9aba4fdda053f1d6b88..a95ce65c852fe2b43aaf76a12b70f58f1bfa0917 100644 GIT binary patch delta 725 zcmX|;O=}ZD7{_OJQ%zPvp)WC^v}#dfqG{8-w&tZtZIkxm#X@x2WD_%E8rf~iW&@&N zFF9+=cnByIibSM>$bJI9LA`j$!IORfFFknjnQe67|9j?n{?9Wr%YMrI%(y?e-3o#g zDxJ~Xq1%UFe(@hk`|JW!*zG5hPk8oXm*{jFb`mNgI@3k;3PVrt3{BLR5jp{#FA|av z3-uv6`By?@j$Z8|GEXrHgBCy$P!tpc#T$rTJH{jdarT%;^LKlL20VOg7Ln;DzSHgh&tY~}^YJ(v%}{3=VZq+MTROYAPo zvK(7sDVAo#dAjgUA!{_!z@$K_W>af(;>}A&nfJWY`HJXO1*!ryfyV-Mfuz9y&->#{ x>n+;RSk`zA1_rgiT4!Ee^4{>DqOpTmPws8@TyK_75#;`F$ZzdT1Ix7XHs<1y*ELR$~S$u`)ZoPqQyIvPfsTm?V~Hj^4IS$0R03Vn1s)??ct?W#C9A6Rp z;aQLMz3rXuFu3b&+kJOcH2L1B!H-9;8ZAh-0Snj!I=~|!5B&Sj-=T`rqq{bncVCDD ZEWZ9ne=P1O!I9V(blHD3Ru8BU{{xTRnLYpj diff --git a/litellm/__pycache__/main.cpython-311.pyc b/litellm/__pycache__/main.cpython-311.pyc index 3b9ce290a9c5fc50c6bcf91f41ed5d6f0e9c05c4..d7f053f664c2c135a849439ad735c7af7c343ae0 100644 GIT binary patch delta 1860 zcmZ8gdrVVT7(d^=6brPLcUz!^f;Hf0-&Q@5!z?z$N_7bTmU&gpQn=-4Am+~%TFAyH#uw(OjuiSFc{-|u^#^PO|P zd+REky$ag>T5Tku!`5(5SbapB5B+8_)0+uIm?G%>Sz@*~o61B0e~veYak;hWm5MuTLAqVK39Nv5y8|ycSMz2v!@+OJN-el3{wTUU-l+b!{)K}mw2(1&* zQTQltAHgWKMomNFHxuu4{1&V*1Mh(cY*@x+Xp00Bwc7g~}yDI>lK+ z21SSRC07CZsAj*fr>&{k-_zD?g|W!$=3ZaG*QF$f{miRJG6@+V0l?`kMl`6?V1wdR z4*;c$5ubCg0e5Kr;dNd2l^xo{%vz~i46shw5q*h^*dP-*TbMD(;So){G8z{S@U-$% z`~VLxDZPd`;3K*o;1*+Y+{bv)!PWnZaLaDI zom&VJrsi#hUObRzfdFo@CPuFz(mCO|pobPo{N7*<+YZ*CCAK0>AhJhTE36yj4w{}^ zkDjAusF7X0)Qt6aAFyM*OQrL<2OP_)_aDLf}@VDr-NN?kzvdypui^@G<#C7E^sKrmq zufrMKTVaVf&qBVVfLAN>`gbzz5`{6YwOhK(1YMwel`qidlbV{_n>u~#r7;#Wxia%d zwzf3~n#69w=N}7Wvb2YpHEVoQz_)JFA^pj6-edNNRlY8t)EwxRq-#u{+A{f*HnsZO zeKB1D=|dKJokDLd1$?K{gTGfMCcLgEWFxt|6o@8vi{n+tM)x>Zt>~)wsNB9cXN@Ko zjYRi-+%e)>daN-ez1#Z; z{0nd@eJi$y8(*UqD=$6693L^5%?z>9F-#ZI6OU76EW#tU_ynQ7YnAjFi#Up2!JMEl z@g{>43{Ei^#%)4D^lc{IVvxmP9M1_kut>QhH1e=hsc&lmIEqSpEet9T+WVQ(*O?4A zaiFu5cHyPY<$9-dnq{t}Fvj<;@0I>$>R(vnuSh({#B}BfYwhk?FR7S*6nFW>ILW}w z8oyucEqK^9m@n>z66K6I2GOPw^2CQj-SPNU_rK7Ar+dn)JE9)u9kCx>tYjALu+IYXATM delta 2015 zcmZ8g3rt&87(Rb5g#s<*RiKPATcD*tdD90JSb>5Y+i=Eb*@7Ln@>p+y%XK=vm6`O9_|k+#<)*7%`ZiSj1viPtk8&Anrk4c zxHRKH>2%_2Dv08C?O$AKBvIL}JHpgPeHXx1WqayJw5S9#?gZ#n&S#EuFs_W`WI$C? zL@N_1ZsAP`8LU`{HxhA|oH|^xs9K48b6j}RsK-ByDR?+rizkY71=R~Za+PeJ@=wzF z4*xr6v|6s7aA44yg_mqunyL251X@l&hw)bBncS@TUFYJ3yi%w^oyn9|E7xpR@mL$TfmKPm@k0`k60Qg>Wd9K3nW85 zujYf{R-rB0Ao!zRAaeagJ6_Oa;+}%T(11Nwo4QeMoLCXlwifEuO?15`+*5ctjrH>3 zRl@3M^P;|+G0A!ZG_&4>Wz@ZFunYGVp1_-IGUt;H&<%OOr|63Z09s-%w$t85qqBB% zb_@M1wn>h@AbsXXv{i0JeaSYiO*(@6va@pKHhL-}|1COSXZC_m`Tv zQ7I06+Dv@CRGlnE+v%oBI1ue%E5jYQG`&f~6SB-2F17DA#*r6&Ue!Lj&x$=7x7ah+LGJ(Jq~{QBnxUyM8( znehicaCV<@cF%D{m-m~7SSmg>nm={ao^dpv%dwryE0{}ETT(6)3coU#(dN&kko?NC zruq*}^|)h&0Uy3GPTz2uX!5Cp!UL=4GI6@ggrBb{QeNu(GI`8E_Ak%(bsEUKT$_&Z zsWjVa4>{$ASmH_Q@^YuS+AbG&+NEZ(*UAbr1?^tOXRm2DGyY8FCyXz0Ek%sKvMto!oTi(sW;E73C^5 z&2d#q9(05a|wh4sJ`R<7l|yj%=G&>z*vwUvv}%nJZnZw5R7 delta 24 ecmX>wh4sJ`R<7l|yj%=GusEQRYbzI{nHK