From fb285c8c9f4cb6fa2f442bef46dde1858bf6fa64 Mon Sep 17 00:00:00 2001
From: Krrish Dholakia <krrishdholakia@gmail.com>
Date: Fri, 11 Aug 2023 09:51:14 -0700
Subject: [PATCH] update

---
 litellm/__pycache__/__init__.cpython-311.pyc | Bin 3787 -> 3787 bytes
 litellm/__pycache__/main.cpython-311.pyc     | Bin 20230 -> 21560 bytes
 litellm/main.py                              |  22 +++++++++++++++++--
 litellm/tests/test_batch_completions.py      |  17 ++++++++++++++
 pyproject.toml                               |   2 +-
 5 files changed, 38 insertions(+), 3 deletions(-)
 create mode 100644 litellm/tests/test_batch_completions.py

diff --git a/litellm/__pycache__/__init__.cpython-311.pyc b/litellm/__pycache__/__init__.cpython-311.pyc
index 79e48505e865a81f9a38d72d55b063579cbc0b70..55f1f9c754599041caa844a00ae1a6ebe43445cb 100644
GIT binary patch
delta 20
acmX>tds>!zIWI340}v#qUfalhfDZsXP6f{Z

delta 20
acmX>tds>!zIWI340}vQbyt<M503QH60R_qc

diff --git a/litellm/__pycache__/main.cpython-311.pyc b/litellm/__pycache__/main.cpython-311.pyc
index e145581291976b83a534062c78303ece8240b6fc..e9b3e7f1c89dd265680095af8df8a7a917a635be 100644
GIT binary patch
delta 3356
zcma)8eQ*=U72nlKC&`v9%LXCKAIR9)mOn(m0pma*31ESte45N)#wbF2C<1+T?*!P9
z$L_=-BB2Dj$zW19VcLem@DHilDa}mV$z(d6PTGv!(#E}n=C4lM{8M|<PD%didnaRK
zPdjw)?ssqBd;9k7-h2D*+*|a{4O;dCr?Z4W`!V|?`^VwSWnTK{%gt}}Fv8@-@}#HV
zLy3tf%TBI9C2Y)kof2{jo?`3wg03x5mGt%dlGXjy$(sHeV~q7=ttgi~Ba|>ZD*>fa
z=7>|~oRDHPP9t@!Y>f1;R9s-bg1HtOjS*JP+{RVGC}{Xu<zfx6N8+eOPdFLqFqeT9
zfOt{3mQ9qh<zrM5AJN)CJ@deHtN!P7-~yO3vlZeubfvR_c|p68(L*FqCGJzj6QFuU
z-}79Fu>I+DV#iq}n$^<0Fkt!<wba8-)D$+E>nS}}7_dy5s6zy8>*=guwg8n27j5UM
z_`RoBDoxFTptJC93G-67@H7m#gD{qNAglrC<E>(>VI$ote%4SUw<5bueAdt)Z$+|4
z*cx}17D?=f#L>osP7jc(3Bd2?e(_%88agIEZ)_S_1+0L?QC*iaJe|yFJcK-x!Q+R4
z=H)0gYsy&-*xlH1r+GXUPel@8uBe%GN>u`u;(%>9!0>51Qf&x0FGmmfJ^<Ysg&ooq
zrdzV&wbe?^UX<)dH~=v3B{B{P8faL46UjaVbhqeJli1m`N;YO|6q%+WX9bAnR>BV#
z{@rAvk2GAr5dw6<@J;j5CA5hnEf*eX#&>j)8C{O2GFeT9y$Xpff%A_XU#cyP7w%TS
z-}rdA^SXm8U(YJ47S;wc3K-OI{ULFn^`%E!{q{pv{cW938CL(x=h5xT*{G7y;_1}Q
z;#b!z=0ESei0>Dot%>du@3qypIMKJ#NG2XWt#B0v@)qRSBSTrPgik4h+*)*0YR}7M
zNCw5__IBzLKX30WKaPwj0z*&)?Wn9Y!m2b10hmzG6_xu$bH^c?5HEDBYJC<Ib=%x#
zsJc0mRyAWYbSHQKkr_^=nUYZXkoZ+cll%hO&518Mw9rB#bRzR{g!Kp;5V{aHBDfLI
z3H}7aCWOrhPa<p)h2RExEs{0j^WYl!Wh4)am1~0C#+D93UpL1!CCS4mdJf^CHDcGu
z0Aukn?E1vHHNn0FlHUW+P0=*Jj)e8;{ky`45ANX^lwmgNP9vnknaE%w9bx=5@@z_q
zWzzAKrc027d<d227H)eiT;s>S?*Xg^LOlREGD>EAI|?s8W|!!B@y*V4;>tQ#;nPl=
zYovN`J^4dj$Oo@ayL+49^;dr{_GUMQy5aTZ`jVk`>C3GxL(S4x%@&|>ee(^!xC@>y
zK7zQrLNKNuv>aJ-EYnkFlyO^*Fj`eqv%@R3EJv?VF;!dXBh0jT(6A{+h9%8H#AUi-
zWIY)+=S<tlKyKLbe@)e*p@Va6sYAnZ&XSWE#Sd;GIrB-wJyh0=x>p<)qdEDz^dPxN
zhL2iPtr}**T+7iV&f37BjhP;4kf6ah$=SY7FVbOaPP%5r<)=t)Nd=|k!mu668D7`n
zK)nS|u~CvLZ6IrjYCdB+D;*(cC?!V-g^NjJBqwbr7*)3(sFPuP&JGpJyq9B8@E!mW
zY4o@;A+C@Z$&mriFoh3+j)-wu{^VuTg%yX2M>JG88KNzlz*2A!M(Fa1tOi-X_}B))
zP~-847JWV}K5VQT`87m?>Id#9nRIj&?`xj?v#$-jGH^b8Aw17CPWXV0UqRDSOwo8L
zQszWl)pS|S9#6(KBdT>Xi${SK$z&i80wuf#^mr{QmlY>YDykZZL1ui1i~!fB%#~x^
zF((NdMx7BJQ}OtSpNK1r{}L^@PMulA)vd)ft15<rf4*@)YL^%1TR6M;6tI7aOYQ*}
zCHF?^>aR6kjZegH)W4Y=PhNU_oW9<5sq41qC%zy0?(|Nr4c%QE8VydDSB$pZvsO)6
ztM6K?XGn=uKYn)F+cZP$mU;v7j(c`TzVCe1g{n&jUa8I3-t*Rs_gyWSD9JmgtDE!A
zOOYv?ciQd0<b8eRJHAQx<{L?X5qRLv%X#@PzPj<#SKBAr^RDTpb$Qo#$&{_(p#eQN
z*rdCsc;8MJ-+6gjc7H5;e<yp#JyWv(uI!&Q?wLw-Oaea*XAPJ|xBrQ-z#CW~7pVL9
z?=N&7qI|@;5#r&-CV(kN*Ih@~X!Eqad^D(5!nVFopCQ}Iqz}r>+bXOdET@14&$eGk
zhm3!Yr$B>0jqnUY8UW(dIH7j{JyZ_g(fC~npN=H5$`k&-uS30E2)ziq0UnIlDq2G8
zjx8yh#t+RXs(u4i-NmslAp_3?{xU)y0ZXD$2X*_7<Z*?ucq+zUMdsT8x&#%BpA&aN
zb<e$s%-aa>BG>>R1(r-4K#?)~kB~CL{e7TRRDTeW#elRc^YeWS!rvA)?EK6`&k4`&
zZkHbzh64utOB~yM!nq1~fz|w5DB2~ac6V2Qf}|JWO#s~-O=kwV1^eg3>OIXP714Am
zn&n(cX~DVtFD9x3QZ}rlqL5Co+HoB0Oh8#oB$C0b7KaLL?2_?MfYm}T#ex=QX-!F;
z=3_WRJ;ECZM)ks6;<${W*T#Qh9eQl6^()(ENZo8g*~0$@JPME6CqCTsiJPpQTzrdn
zzE+_7#F4!XoA=3d^9)&vvl1~m$Es&7#AF-u%*clPxWi4|Gh`lG#Lc}&%sBsl0K}>n
A3IG5A

delta 2040
zcmah}Yiv|S6uxtJ_jcdgEp*%Mwsg0<-FDmVmb5@xLn^401&j2BiH&kuc5chUzL<NL
zw1o)K7D21A9t=^N8ckFX6OH@FMB^hd29@|r3@$!M{9*iIQv8MeVVrZfJQLEnx!;`m
z&Y3f3&YYP~&XJRg#QC1XVHJ2g_xfe}!pOX{mRy=|dG!HNP}Ux-Mg=x1y-Nh)0{_cx
z%tHlQk*>+qj@4$oW8O^NSRE1goQi`!c-iEM6M>qkg&SF^Ek&q(!7xEmBvDTtGs0Md
z;^L-N)Ookl34yw3rT)1o;TG#?73?F`K^G<C4$3*|)X~imKf=E?3pToT#sJqyL$L9_
ze%*W~88^~u_>D9={M5sv2v9O2L~6h)D#cmxXJYd6z4?ksV}MjlHVDF5!2;2yb;a|h
z4@BaHE!Hp`GrP!tFlUaCHE_*ZTl~R%n2=_!ii_Es7WVpMUS_BDnt3Hxypt<#%EiyE
zN-e2`Q1B^tl#{FjA;w{pMc|cS59x#NgY{Ab={EQ~7?k=@+y<Ud+*THG+z+YHumjIW
z6*>6*tPw7TI>{{D2!*=?oQfD2nwO+J%VqK!>qk!1IX22==0dGeQzkV|_rOrIH>CT2
z2$lN!52MtEfNL;ZjXef0HEZ>|P~43$$l+E9NDRaENU-{S6h{y+mhytl@OZdM(%lE)
z>F~6}%~dO5veBZcMI@^_4)pRGyzThBvAm(%F}J})>)fht>YlvIO-V{+^94=iiRg!c
z$mvz*+X}bmLt~rYuV)U^TCy|ca6wTuS)0l$+@Xs5?+3YUc6F}{cX*ApcW=4hd6X>5
zWKzj%sa*Da`AADuIhLA8WfN)nmWu6$x7x!b4j0=4t#*vfmdK~%BMMXbMAm|gB{5xK
zihM|!VpyX*P<O{E$$?@N-t6cgPPo+3y(xjjeuN|f<xp}Yvf2Tb%cqk1^d97FNnUGd
z1p`x=7dA#m$RRi#ZSw3vt$8IiYRQ<-sTxz^tLPds1(%~?X&U_=1rpQlsPZm28f%a$
zQPl&Bu?7-`&tjX14{pRdrISdHLbNm1s~`U)E@4b*N`~!4m5gwA%;gf+H#3e}FP!O&
zjUGbrZ4M<dnPcyuP_cDzhrDNafaOq!SEuCAOGM5mrqa0tWd-ESN|xqxsjOBqFhylk
zXuPr!&w9DO5y#$}5aNO=R|&!iVX1C=aelqUKu*CM-Cfm7R>A6B5=h0SD|H)Rl;&Lj
zUa>ffKXzBTx>~12;d1--I^O*%?F{qo&*)AZ{bSzV%e!UKI^Annwzp1iFf4B{ak*%T
ze_;sd_xC|udJDo<ge(VMkb1HnfxjO0)x?qBfw1!qT_5-cy7t~vd-^kQ98FK4shjC@
zXOO_sV)|)3hZ5G6UQ8v+w#<Ho(o}YWJ&)ufhmwIGJe!3J1MBuZgXD(@A0gm_88P2`
zp<7X<uU<sShwvo_70vxCjbW<!t+=(`H+=kQF|zA-kw1x+!Cses78rmY#UC&<IPM7W
zVWf%8BO8aW2YY?rp;&|P8i$fGnafWx6OL!0cBtjJeL~S>C7aCYn>vd&jx;a1bUG6&
zXsNVXPKJJeoT#;7vB)&1DcK|JS#;z_ID?>91_r8!RDMn|Bz-f>=FOQLEu@t`_7mp_
z|J82zW#}q=JNz*jf@34W=#WG<FA4YJb%P)}W_;I8f@q%cEJ?clfE})mJYhul{{t8q
B)4u=!

diff --git a/litellm/main.py b/litellm/main.py
index fa88e3ca82..e5291a9b73 100644
--- a/litellm/main.py
+++ b/litellm/main.py
@@ -6,6 +6,7 @@ from copy import deepcopy
 import litellm
 from litellm import client, logging, exception_type, timeout, get_optional_params
 import tiktoken
+from concurrent.futures import ThreadPoolExecutor
 encoding = tiktoken.get_encoding("cl100k_base")
 from litellm.utils import get_secret, install_and_import, CustomStreamWrapper
 ####### ENVIRONMENT VARIABLES ###################
@@ -116,8 +117,6 @@ def completion(
           messages = messages,
           **optional_params
         )
-      if custom_api_base: # reset after call, if a dynamic api base was passsed
-        openai.api_base = "https://api.openai.com/v1"
     elif model in litellm.open_ai_text_completion_models:
       openai.api_type = "openai"
       openai.api_base = litellm.api_base if litellm.api_base is not None else "https://api.openai.com/v1"
@@ -439,6 +438,25 @@ def completion(
     ## Map to OpenAI Exception
     raise exception_type(model=model, original_exception=e)
 
+def batch_completion(*args, **kwargs):
+  batch_messages = args[1] if len(args) > 1 else kwargs.get("messages")
+  completions = []
+  with ThreadPoolExecutor() as executor:
+    for message_list in batch_messages:
+      if len(args) > 1:
+        args_modified = list(args)
+        args_modified[1] = message_list
+        future = executor.submit(completion, *args_modified)
+      else:
+        kwargs_modified = dict(kwargs)
+        kwargs_modified["messages"] = message_list
+        future = executor.submit(completion, *args, **kwargs_modified)
+      completions.append(future)
+  
+  # Retrieve the results from the futures
+  results = [future.result() for future in completions]
+  return results
+
 ### EMBEDDING ENDPOINTS ####################
 @client
 @timeout(60) ## set timeouts, in case calls hang (e.g. Azure) - default is 60s, override with `force_timeout`
diff --git a/litellm/tests/test_batch_completions.py b/litellm/tests/test_batch_completions.py
new file mode 100644
index 0000000000..ca041ec95b
--- /dev/null
+++ b/litellm/tests/test_batch_completions.py
@@ -0,0 +1,17 @@
+#### What this tests ####
+#    This tests calling batch_completions by running 100 messages together
+
+import sys, os
+import traceback
+sys.path.insert(0, os.path.abspath('../..'))  # Adds the parent directory to the system path
+import litellm
+from litellm import batch_completion
+
+messages = [[{"role": "user", "content": "Hey, how's it going"}] for _ in range(100)]
+print(messages[0:5])
+print(len(messages))
+model = "gpt-3.5-turbo"
+
+result = batch_completion(model=model, messages=messages)
+print(result)
+print(len(result))
\ No newline at end of file
diff --git a/pyproject.toml b/pyproject.toml
index d5eb3177c3..9fe3e3433e 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "litellm"
-version = "0.1.380"
+version = "0.1.381"
 description = "Library to easily interface with LLM API providers"
 authors = ["BerriAI"]
 license = "MIT License"