From fd953b7ab4bd933cfd950d5ddc693fed9f3b9441 Mon Sep 17 00:00:00 2001
From: Krrish Dholakia <krrishdholakia@gmail.com>
Date: Tue, 26 Sep 2023 06:56:26 -0700
Subject: [PATCH] allow backup api keys and api bases

---
 litellm/__pycache__/__init__.cpython-311.pyc | Bin 11762 -> 11773 bytes
 litellm/__pycache__/main.cpython-311.pyc     | Bin 48499 -> 49494 bytes
 litellm/__pycache__/utils.cpython-311.pyc    | Bin 133042 -> 133445 bytes
 litellm/tests/test_completion.py             |  15 ++++++++++++++-
 litellm/utils.py                             |  10 +++++++++-
 pyproject.toml                               |   2 +-
 6 files changed, 24 insertions(+), 3 deletions(-)

diff --git a/litellm/__pycache__/__init__.cpython-311.pyc b/litellm/__pycache__/__init__.cpython-311.pyc
index d18aa4f2253b5fe4e54a54771ed1d84f4a660744..871c98cbb7ad5b991028f5c90bac5ecd1c9ee84b 100644
GIT binary patch
delta 195
zcmewq{WqF-IWI340}#Yz38e;Y<Q0=<6rHRhJ(H6&B{j7mGcT=Z^KNM#c8vm{lA=No
zQ3N82K|~3VxW!(QT3nKumwt=2v?Md9c=8veQ0by}kPJ(1VrJegp3J<G)bygnlFa<P
z;v((IDawlCMSUQ7zNFHW^wg60+{C=Z^wgrF{>jsoWf?bb-m3hOT}e!w=>r3dXyENA
U0rD6@LX1ow!7QWAC$zPg0G!A?djJ3c

delta 172
zcmewx{VAGvIWI340}$Nz7fN;9$SWqzC^A_^dgkU6(p>B;`9O)u7nMXrZ?Tu87MEn^
zrQc#LEy>I&E-D4d{89>?9HK16*giQ+S<&DYUs7pGdTL30Zem_ydTLQoA5cw^9f+_3
z5r#nG7l%!5eoARhs$J2h%?Fe}vMY&-Gksuy5e>W@B|sh{NQjZ?Bba5d`Leba69Dh;
BG!6g&

diff --git a/litellm/__pycache__/main.cpython-311.pyc b/litellm/__pycache__/main.cpython-311.pyc
index 396f89a3bdd6ec714cf99a706ba6191a51d0a792..4ae00fb4ec3afab7e7b7afa0f5c86649973c55a8 100644
GIT binary patch
delta 3024
zcmaJ@eNYo;8s9g|CO`;D2oU*5AV7c!Aw-mK#1NtaRzCa$iwJCxC<MAm5p9&B=hqz;
z=)=uu$KI(^KhAQa-qt#f?aXo08>!cdTHA8DbF|vK8Sn61M&&$bO7DF)=-u2Ow}1Se
z-}C+Kv+r*5<b~v$-E{bAg(8%~*TyXoPUhK#@UOIz<z{%Gbs^wioA({U!V<eif&eD;
zad0etl-9V%`D$Gs@l5Dz5gp;)$R{VtBn<c{pA`%l(Yzz+CZbjDRz4=hj%cO(w|tY)
zOh`IEm>PrVaQA!shiP_%p7W2>&4J>vkeBfmZo_9Z(qZm4UXz)DlydikFNSU0Dj_3s
zk3p%op^Z?h&x{<cJB5uOiVosLlur$Y`O%UP3WhTb8Dtp5HnCy82zJv^#+Ug4dI#;a
zxC@dgZ$3!~TqzWn3F?Hx;L-Y1G_FTOX95q4%}Ltktyrfhj}#BkGB>~_wgtACt7NUG
z5ROG~$e4#%Kv{jr3H@0%=&|VGUY1neHW$B|ZG-77JK79~(ru^>3}&4=#C;A|Y!5_u
zB3Kukv?QZ;_}r3#w!n0n4sC_+Qq!O?yH4KW%j}^zgD;cqfLe<YN8t+K&u)n83`{V0
z7u0UlOV?A|5G!NdP;Fg=x?nHHZn$DKqc`B8)dW+ET3{+$4t)zQ!2R5KenrkPguL)*
zq3iz?dlTMRR2wLOdZ5d&7&3A%p>1HziQ{K;#R9*~x1e4?r5d=OpO3bSvzHa@hVml4
zEXaKT-|G%|Sl~c!fu*n+?SSFJWV!F<e!QQ;@yr6o<dco=gdYl*As<W^B{1VSYZthR
zNY=R`8`_1f$@0EH))3+vz-r2d>S7&wn+z4i(riDRD?WmDLq&-W?SaiDY3L^~Sh5}M
zg^<OU@O_Rgjtbc~w{#T@>!QQh{c}?sWB1|ttIz>)c5mrfw2$mrhTI$Q90$R^#K;`n
z8RPwm%u1U^--2H)S%VJZn0Tps3Wqd<!DiO%*;(hSW9hjbQq1>2vn>k!6t>wi(9iIo
zMi{rHhYTEL*h5Dpqwm1KY$a$A-X4vXutQjAp{=YJ4a40sJ9-zA?FKZ$JM2a3h7i{2
z>~^`^xMsh)+~KIHSh=*$)?}-xXsUA5??`U<db>S^7E5!tE9+HcR%=(MrN?UN>UO%D
zT_%Qe;IqOlB~VY`5P|i)Z*`GGVxr;l+DE8~KfNvnp{M-VdILf%|De8rMjwN&u@zl|
z{>D`K6_WD_fluK#jlYv#C3GBiY@9;3_{~kJNcE6d{!ZXAfoXVuQ<n4@q5p)No1$W`
z5&B<ci24J8*>{_x7kx(9?+FAdW9AR*adKW~Z&PO%>+Imd@O$T%b52iplgG=s+?%;C
zp{hAws={_I5)L-MFB#WCddmWdGn{|DWk!)dsYpClbxUEnp|A*2N<B}YXm$1-m3Bla
z(3t&2rM<065*SRMVRzh(xU8Eh%Wahf)SILG1cp-RrYTwx{*(aD#SlvAjfa*FtP~_F
z#k@OlhTC!3x8w2zN){h68JjSvGfwKt2qtC-a%>YFLKvk@6%;Skq2Qt5=g(ABqz-!(
zc&3+dPo@1<Ww@m>OvWZn#^)s8Nj04@9B=!eO^{LQq-iQh9r2WaAVMgmoi`+VjxRxp
zD-el~c4sVvw^<;+Jq4-w)!jMaeH-z1PHJrRrPSq$2qLb9*|kJ|nUcOzL0u0+M7gdA
ztya<3wF|4|^n{$jd_t8*925DD6e7RYIpT=)7nF(FchqGHqTfv@GZ1+nEl(%%Z}V-@
zL{6dj4m~x6e=0;xAKDp7Po<}ID(Oc`2J=UeV);kK_8by0oo$OD@=0h}5|Pi$V$~UC
zP){T^qY;bDM5=p?^o+4w)Lx|7^GW1vV$d!lHJc(DXN~Gzx%6zVJ%qT%$|y%`w#b%Q
zZAOt)XakMLDy_CC(Xv8jD-y{{+8U6H9OA8vt=8KtqGfHkJy52;G}P8$pvM|QG`5C#
zEaMrh^OyTJA!LTw{)LPcuX~VR)&Fbq_Mp9=qagUh-U@VskJ_h2=nk0oXRE#?r~Hou
zCJFeVbH4#c5AMH=@}S~Cen<%};z`6l7;8=h{RJtUKM=tz$JQhK)dMv$NdW@e84c8|
zWxzVL235lTp>p&id@)p05rm_-QUbZSfM4qA!mkkPXIx%qr-v)X4pT5!O!}4(s3DPc
z1j-4B9}3c*s|4q;g;|5?eg4Ss5}CA;SPS6p;XYL@p<+@psSyP3Bhl+$^%R?MOK^-S
z?0-7>nPqhj$4Wn4=P36xb&gez)f}no4<WC0m)qm?h;#U97u!2G3|LRZ!pxC0X%h})
z7(V&ERWilbq;C@jQv~-X*mFE7`VLn7A&$-#C(Dv=!(9?F30IEmqr`njk(uuiV-11t
z;UCBI<yvACml1>87#SApIsAqbIW($;!ISxn3mdlaUz{94a`9QMBe@S?>uD3Z3qL=d
z#dP9;J{Uher6GR5+~f2%;jdOpm&eJ~!__kxXo!DuW-UUKP<b{EMM2xyJjRQYM&QKR
z*HIq-<JoIL((9ybJ4~F@={IAaiRM;d!BrA-FEKD&E8#s5dcH7~td{#L)=g1d9^nBW
zVJk?K_z>G+%lTY%9gd$*(in-W0=xXu*7hz}tCQOS-<)Te-PryMzVyPMB~eo((MLjP
z&vp-Yi>PYI{-_2I-v7}#v<|Enm!a=r`^D5ca>hAw8aZ-2IPzRLvN?`yiz6H1j*)zF
zoBp5<7yeq4PMn1p?F1HJcoAHTzw2*yI2UukPhb3u#{VQfzPzfhSBXLe>OVLs4;N@r
zl~PKLAQNdYrHT|nL>fv(Y6ZDS!>9<Ypb%*|6{Z%HB2`hrazQQ92uh(AG$M_r)X{=g
zr1L0kybvSOc&bkwCFn%1r*P*4k)~6T@rUgrC4xy*=TqU)L#+diLWZbjQW3F-lSg!d
zSyZzqrFO_O&?#6%6~9+l_YSxP2NDhVz{@p5IRj>)Dqz6RS{o<47EqE`B1)qbssmaB
bqNEyOLqK`86u9|GRA3K13H;J4X34(+K+Jt+

delta 2610
zcmZ8ieNa@_6@TYFc41i-SQc18J_5+cy5KGl1O?P(*$5hukBBCItiW4k{aEzw#-gqv
zA=+vs!OkHwaYhp}EwmBTX`h`)(@qB4rfSj}$7<3|ZAOz$n^ecAwwadL+I!wIX{PUw
zbAIQX-#zD^bMJfa`k?e+T1@<*$rLZ}$M<uy@7$dqCtkIR8|?V0)epnUidi>-D@W!k
z5r*+{&TKr9H6f)0t|}|D2Pkqmrwt@?z^@cM7Axw8!$P<m^X%!0C;tjaNr8ZpF~<#3
zV&FZ+TUbV*QW-AF0Lc^>RmO|m0FrXIq%4{|6RYiec^SpNUNQ!DDwa|^V2LB&x_+xf
z8w~2h8S;j$B4PHHyaBr$%fXAM9QCjfZx$6}igOKg;Z~;$Heg;^c3OPEz*lt7tawW9
z-j@*?z&A=W@n6nD=)v4_7fv{H@VRomQI4{LDY6fT%L_n8rQ8mi@Lq8?Y}T^Rsc1L$
zhD&z|EH7Lt_u>l`g`Cw@o{R5RtN}l|sxx2<E~u=+hviAQud*1n;b>(UY{yg8HV9x}
z-Xg4h{1fQodz+<HR(&5}2Yxu;KeKcvuB~p1lJQ|>H`dgQU}ja8^4FRZL_D<60U?}R
zxE!9<c2inA$Zg)GIVNzBce!h(%Z;J8d4~%m+r+mpqpk~f5hpe#kk2{&_<G&a*xh6N
zxX0r$w=NCE`cmlEvPgjooUvynOKWI~vKV-lF#~5CaHq2t_Gr1J0OQQP_^5t0gsH0-
zS(5kR(M3mdpBv+cGp-#F9g=LDNhgL^bO`bQwk<A(=drCW9fue9zzg_b@mtm%$E5IF
zc}X4Thv)8d91#O%o_-LgmgE{=oLPT@+kacK9QJFiZ*ZN2L2WA#Sj)Kwu&kj_cYq)6
z)vsH}hE;HYbF%b-+nll<jqWnrz`piyyDUYM^4rud56l!u3Vlc$jz03YvCD0T@9^FV
z@h9#Q!@*-Z`Oq<O;=A~%yB-c>+X5>LahZ-Q8+XG?c(Kt9FQdtm2S*f#r`D`o=&TmS
zM<T-0cc4nS`*a4tCB?8d4`88UU$;Plv*`2ozy-YMEiz8>s_bVZ-o|O~dHt^mzKiEK
z-i8m9GhIcH{1@_kPT~%UyZGnsc>@m!{)3B1QmAjUFVyQ}-Y&m)t8X{EK%sA>CG#s1
z4~ZV%)7usDZts(Q+afW9*?Efnl0>vNu@ZLqSjg8O<*|6+E+g{=cX)##<_~OU@E&4M
zjXseFSPEY0IWDft!X0vjxYwloK>pGss(K|ZRGC=l;EYj=t217hY)&C{DZy<e{o6Su
zEhgc23BV29IA|T<zRNY9ITX8=?U_wFvY<iZ-Y_?q$o?d^A&>MeXeuH7#awqX={vQq
z4AOVg8uCfstI=rpGh<k;a6gAi2={Z%%qiV>Hfq+F7491HPTOPlIE85kxTyZL)4Ug?
zufT1f*nf;3C+R77ovTm-p8RN?sVC}OX_}`g#Z{|;Wr^jQUM_gFtxdJsTn^3Cp5czB
z>5Pweb(Ts~ovC~t#atKb$nN3u<`4enM@fqEVl#jp|Gocl9SzQ>%s+67uFGc!FM^1t
z4>iM>^3Ow700ol|JCm>Rb&ReqyH4UGY(1RE+3fHnRAK2*jlPyA2|k7(7vM`nvvf~z
zvrjoW)EX;R1HSysEKD6)1x>hRq!F&+nUU6Jkuz95iTONZL?7(q*FcWw{2|}=AX~@-
zwm3#FE2}5bLd;eY4J16|qnfM<n_h9~IygM1gkM<_t8XH2HJ(2<ki3$hwy1#kBDNh*
zU-NZMtq5DhIktrVs}#{KZTEOuB2v4jF`{etba<X(ls96atJxO_`hwa%B9dS3pV<t)
zbKIq0&q*K0DEikrVog`6rk4v_3i|_Y8qH7tm>Us;XM49#mg#QaAjWlkZ8RrM8?c4;
z`zLa?lDLIeMr({|<kSXX<s~w4-k4F>&3y)C(O9JfE0Mifqub0KE0i;Dj(|}+pw+bg
zCbo>*;0E@O&(m$?gjF~`e%nIvh%x93dHI{y-52z+)p&A(|ALh36P*CpvF!A0uwe7)
zY8~T?w%~!&>-fjz*6B+zuJ<YJChq<G5q~14P5yvC*z0A!u3%q)bx_pDqc({x<BBaK
z$1ZZ{7+pRV!aK^sA{sgSjN7&}Rz)}(BlIM(w8PzmE6&b`_i_K(d`kg^nt3Rq@9FLH
z_xM;pPM(!@&vSoB$^Ype#k4PIVVEdVaCeY>KxPtJ&b9J}<a1|W6{b!ug<H68vZ$Rd
zJfj=R=-MzkGe#3-G(|>}Vkc-lwI7Mu=I37`s>B@Rbd#vyVmhvlf7_nj=3B(Rr~H2M
zR}!?~(O+~7v?M{iD*R7=WK2{g&D0A?7ByCbaYAycYS3W3kZM(p8cYyUtg1<ai9&)|
zP10bp5NA})8cY#PX4Rs>bitgiS~WOJux6<l8q5*~%xP-2hI0g7Iah-vLTc8l?vZ-c
zrkQhv#PlOQL+e$$W|j&mnXeX%WUFPGIZsHk9tjR@R~?$!2xNzb0;&fz2R~Y)<w)gF
znYujccoGEbY;{G{Sj^v4i&b42wVnY%Z&BAqjc*L)UaTmJe;2s8CkbNQ{&Y1?%iW!$
J*xo4<{|g_y__qK6

diff --git a/litellm/__pycache__/utils.cpython-311.pyc b/litellm/__pycache__/utils.cpython-311.pyc
index e0f896c0d1acdd820d33975b96f08de290ce94e2..368e334d699a30e4488bcac03a9beb30a4d28fc0 100644
GIT binary patch
delta 1158
zcmYLHOKclO7@mK<>)mx5KkD5)aqOx|YX_yFQV<ZFLnGUzs0po7gdB;wZtQ}|!_j0*
z+G-6Al@N#s9Gq%QrAn<-9twwsN3}VWLq)@hO0g6P%Thv|P|_$uEa5;oR4`+gG8%pJ
z&o|%ynwhUZk`_Le?4Q@v*bp5L%Da;5!;|)#cKBx5_KpQ1f)XeLxnrG;<lZuLaT$TQ
zR$z~6)lq`$Mmi+W5XW4zfY0R0w_3=^3fO&gH=tWXlzsK-j3^^bV4CXCMa@dgPoO~v
z>EcyBcWt7s34KO`d|3&^L^L=bWGg$552?AI{&CqUDO5-yBWgQxxATooK%>37sB>>3
zd|0<!<w>~?)ZC?QCz)RXKZvBf0xG#PLdltJr;W;CYF0<ppyb3PIBeKL<3cym9Jym}
zD8Yo3qS>!Q0(3D2*Wp9RK&44~7YJcqhd~ZdKbl(GgS1_YtBw|gUm)k&p_vDyb2%v1
zvW(>J&#=!co~Ih!qKmBhD7#SUEh!6A9aW3CRzZcP$Cq~^3v&ZH^rL{>hfh%#?4`*!
z+nF`Z;AKGQylZ?Sr47EGO!(qsDc?}?-1sI7j{_6}JMeGJjxl(b-YUMxpuo0i^8PCL
zKqj-R5cHPpX`Fg3H5wm|jieLF;YzA%(wkIGc3%WH5zFA<JRGSnL#w1V<qyroF7%OL
z8J>T;;0+elW)tm|)jt@#7dH8p8ugyrlTUtnyx?wIa<`et>DpcNx8IOw<+;H8@q)i+
z$={QW<ew`jt;N9p8x6A!b6xXVArM*$gec!tQ1%v|Xv>C|m4?N}{RO3CN$FUu%vH*G
zo?2AcG_8nSDZ^a7SfiTgKUj9E+4!_HBV{Erum;b#ruheSm<}pn>p>vb*1&UsR&E{z
zv{Sk8(zutldi%+q*09XoadlE$kZqABVZq%UsTF>x6{tOE$$jNh49hxv2f~Z=J!E6y
z7w89}kJ_r?`4yVM8lpB1lY<*j2fZY^0a4o#MltY`TN|(iA@aq2@Y{|unn4FC-G>=l
z#gjp&ads2F1*&V`f=jkA(=m9~_;Cw#D@4fQZRk45D!?o?W=?^fc#^$s<+Ge*Y@9)Y
z{*#h25<eT$(nHD7@!07!e)P-{c7BBVl<b3NFiwtYk2LrYJLx6s+t3M7qyA5Lg%`#s
zmp8P(;4(nWaQqEEzU}`CDlJwTs`gLCCH_=8J&bEvtT7UO2p;&rn0N>&*>M0A6Fr90
J<aaCg{l7>BN-Y2Y

delta 852
zcmYLGT}V@582;X~owMz@pCpgdfo`Ii3pF9bGQB8gAX!U<%8NQp&k9>^@Xa*htg<T!
zq1CDngg_$bqKoX0b2G{^@47JQA|%bt{?v(sgnlln?>myd7tj0h{ygvdeBZ<k^W$q~
z`+b{DMAQb8xxwt20sAvMJed)PIfMvOPz(}nwJ!2`2I|T7S&$@NMGDu5R8UZe_8=4o
zY)Pb&Wh6EW_5!@3@;-F@O-!&LRZzJ;gj-a@2uD&*kP@eo4iftaUP&lqqq>E7=5W5s
zk=!20PXv>jL_lE}I;^4*E)fVxx96aMa8n>z6}}s(=pHDb3S{>j6gdY=QLlB4h+!GR
zyJ99v^{8r%iu~|^gN#vz9mWf^K{cpC1N9wDo9UuyXkyLxLq9ri=s^9jp$xm$b=8cn
zsL+nQ!XeyAr?K0B&_WfB78n`!Qt+DbGK*yfZ^@k$c)(1ar=Y>4x8u%)-0mWkX>f24
zfi$LJ3kjv6-Z|-RcrO)ak=r8aFxZ?m7TF{s&BF_uV57<ZKr-`CR~R?`q?TnR=69l;
z&2eQ)bWM`$X~?UlblH7?ta+FHQ8&HiWyxM^Q#tplx|ZU2x#+9l$7`y6rTpts-dCYr
z`ULNQZ=rfXJN5<Ifj>xNn>P9tilMlP#@I(;+R+sV%K=4^dxFtu;B?Sy!mT9p4LZaF
ztcQV<_!nRan#i6-s1W@u&0s6(S%jOSPQ{>7%UOb_K>Nlra7$$D`idG&UZ(#F&E&&(
zs4rnBVRj2<LtZN$U=ORmzRa>GN!=>AJ<Tj`rKM?m*Ex)X;a<5n5(<W+c)wmDeXCFl
zZQ6rXIBMi0G#A%QzhD^Pgm(8gc#PXN7U@d#yEk+jJ0p>^IG=GvNaha|!BtKC3*8on
P7xJ=b9gdJIW^Vi+A93qm

diff --git a/litellm/tests/test_completion.py b/litellm/tests/test_completion.py
index 7007e923a..61f5ef375 100644
--- a/litellm/tests/test_completion.py
+++ b/litellm/tests/test_completion.py
@@ -15,7 +15,6 @@ from litellm import embedding, completion, text_completion, completion_cost
 user_message = "Write a short poem about the sky"
 messages = [{"content": user_message, "role": "user"}]
 
-print(f"os path: {os.path.isfile('litellm_uuid.txt')}")
 def logger_fn(user_model_dict):
     print(f"user_model_dict: {user_model_dict}")
 
@@ -774,6 +773,20 @@ def test_completion_with_fallbacks():
     except Exception as e:
         pytest.fail(f"Error occurred: {e}")
 
+
+def test_completion_with_fallbacks_multiple_keys():
+    backup_keys = [{"api_key": os.getenv("BACKUP_OPENAI_API_KEY_1")}, {"api_key": os.getenv("BACKUP_OPENAI_API_KEY_2")}]
+    try:
+        api_key = "bad-key"
+        response = completion(
+            model="gpt-3.5-turbo", messages=messages, force_timeout=120, fallbacks=backup_keys, api_key=api_key
+        )
+        # Add any assertions here to check the response
+        print(response)
+    except Exception as e:
+        pytest.fail(f"Error occurred: {e}")
+
+# test_completion_with_fallbacks_multiple_keys() 
 # def test_petals():
 #     try:
 #         response = completion(model="petals-team/StableBeluga2", messages=messages)
diff --git a/litellm/utils.py b/litellm/utils.py
index 7f21e9450..5016eb4db 100644
--- a/litellm/utils.py
+++ b/litellm/utils.py
@@ -3288,6 +3288,7 @@ def completion_with_fallbacks(**kwargs):
     rate_limited_models = set()
     model_expiration_times = {}
     start_time = time.time()
+    original_model = kwargs["model"]
     fallbacks = [kwargs["model"]] + kwargs["fallbacks"]
     del kwargs["fallbacks"]  # remove fallbacks so it's not recursive
 
@@ -3295,7 +3296,13 @@ def completion_with_fallbacks(**kwargs):
         for model in fallbacks:
             # loop thru all models
             try:
-                if (
+                # check if it's dict or new model string 
+                if isinstance(model, dict): # completion(model="gpt-4", fallbacks=[{"api_key": "", "api_base": ""}, {"api_key": "", "api_base": ""}])
+                    kwargs["api_key"] = model.get("api_key", None)
+                    kwargs["api_base"] = model.get("api_base", None)
+                    model = original_model
+                    print(f"switched api keys")
+                elif (
                     model in rate_limited_models
                 ):  # check if model is currently cooling down
                     if (
@@ -3318,6 +3325,7 @@ def completion_with_fallbacks(**kwargs):
                     return response
 
             except Exception as e:
+                print(e)
                 rate_limited_models.add(model)
                 model_expiration_times[model] = (
                     time.time() + 60
diff --git a/pyproject.toml b/pyproject.toml
index 7de1bddf9..b10ad4bf5 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "litellm"
-version = "0.1.754"
+version = "0.1.755"
 description = "Library to easily interface with LLM API providers"
 authors = ["BerriAI"]
 license = "MIT License"