From 6e05fca7146d9b20cfb2971f5b3661a24f78c3c8 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Tue, 26 Sep 2023 21:47:20 -0700 Subject: [PATCH] add switching api keys + api bases to docs --- .../docs/completion/reliable_completions.md | 21 +++++++++++----- litellm/__pycache__/main.cpython-311.pyc | Bin 50033 -> 50311 bytes litellm/main.py | 23 ++++++++++++------ litellm/tests/test_bad_params.py | 11 +++++++-- 4 files changed, 39 insertions(+), 16 deletions(-) diff --git a/docs/my-website/docs/completion/reliable_completions.md b/docs/my-website/docs/completion/reliable_completions.md index a58975c1b..7aecbed0c 100644 --- a/docs/my-website/docs/completion/reliable_completions.md +++ b/docs/my-website/docs/completion/reliable_completions.md @@ -1,12 +1,9 @@ # Reliability - - - ## Helper utils LiteLLM supports the following functions for reliability: * `litellm.longer_context_model_fallback_dict`: Dictionary which has a mapping for those models which have larger equivalents * `completion_with_retries`: use tenacity retries -* `completion()` with fallback models: set `fallback_models=['gpt-3.5-turbo', 'command-nightly', 'llama2`]. If primary model fails try fallback models +* `completion()` with fallbacks: switch between models/keys/api bases in case of errors. ## Context Window Errors @@ -55,7 +52,8 @@ def test_completion_custom_provider_model_name(): printf"Error occurred: {e}") ``` -## Specify fallback models +## Switch Models/API Keys/API Bases + LLM APIs can be unstable, completion() with fallbacks ensures you'll always get a response from your calls ## Usage @@ -63,8 +61,19 @@ To use fallback models with `completion()`, specify a list of models in the `fal The `fallbacks` list should include the primary model you want to use, followed by additional models that can be used as backups in case the primary model fails to provide a response. +### switch models ```python -response = completion(model="bad-model", fallbacks=["gpt-3.5-turbo" "command-nightly"], messages=messages) +response = completion(model="bad-model", messages=messages, + fallbacks=["gpt-3.5-turbo" "command-nightly"]) +``` + +### switch api keys/bases (E.g. azure deployment) +Switch between different keys for the same azure deployment, or use another deployment as well. + +```python +api_key="bad-key" +response = completion(model="azure/gpt-4", messages=messages, api_key=api_key, + fallbacks=[{"api_key": "good-key-1"}, {"api_key": "good-key-2", "api_base": "good-api-base-2"}]) ``` ### Output from calls diff --git a/litellm/__pycache__/main.cpython-311.pyc b/litellm/__pycache__/main.cpython-311.pyc index 9773545ee7aae3a4828ffb2bf8a1721c5a7d9c3f..f1f66b5eb7a8b6fae23ede28ecd1b375f7391a76 100644 GIT binary patch delta 4216 zcmbVPe^gV~9e+2E1R;cw00{v?1VjiL0s;+!{0I;b`HlD+2FQyNOdjmz)hc}{+Syta zE$(YuH|<8xHtmjBX`i-LPitw<+O=n`wLex*R;%sWu3go}we{3>9(V5x(PBOOWA7y2 z`@P@$`To4$dtY+pL3-;pI{KhW6-nU-CSskrKRg}%4?TJwuIl^IKCvg^V}ysQ#3eGc z5AxeFlCr)=vagUIy~O6IpiThRIx0Z9a~K4dT37WseP&KIc8KI=GoJ6YNbV9{gGwhz9 z5$<5)1$&T}I|RpA3w&IduB;KlxpWyN)bzK)uCc7TB zV|O}ev>WMKJ|x*}>s2olY6a;#gtKM@_{NqBxkY!V(o)DP`kz&b!&UORmwA@^>O3oa zix4%HWE|`(Qo)Nw5m0APDH}t@ol5(@>1kuvB<;GXwCi!&i-Nqeek$4e{*_X)!=YsB z`&a#+$^M}vHN7Jg5N@>*W|P#PcTL;FPIw~2hBiyB*1*3YwORvwJ=XzO%ThF9>_fO6 z@1Hs?-TW!CPMgPn0fnW_cWB1@Ni$-tG_18HbjJ8eq9r#A;o}cLPks`lltrUgz*1&J zTO>Tq{u$#RoWi&8eA*s9+HNuSogW{ndmpCPi z*(Nl?bLFY}*Fu@&8-oe)c+FH1pVzdQ;(xsEytOV+Lf!#<6CILnBO>=T3v*#ILr-g& z?a*9dh;3d>F*|%MLUT3MY4WuSEkY~U(+y|@p0?>Chg4XlWn`w%0!J$?Z#e`b?im#WbITs@8J_iyWPIU zuz98)nyMA%9lmyk^QpqM{j$PEfJQC4Uk(Xd}#zFEM*>p&G~w?7gJrBZ<(-cYHy3< z$ZoLOY3_dopE<1YAKwZH`!oo-Le}`N!CaGy_KWp3KFzRyW?lo!FX<~-<5^ocbeGGq z%(kSZrLC`)sjP0gd(no%jYGyho*(oUTdh5Vt{lec9CWd3xSpKe0grWko^>e2ByV}y**TY--)(j~ZI^FJpnfDBEZsw0Ne;7C5c5)3kF0+iQNx~yn zi^-6N;JdKVLAR544Y0*C#lJ7Bm!T%GuDp!4h$*WK2pPnpj#PxU!OG5;(9gxBpEb!8 z@3Lc47s9>?S9>lhKOk&4Cg$bhrPawO z<}wik2>F(ft6*Sq^#8`JUwvCt+;wPVa+jYbu$Kr4dLSH)FbTuNA1Nt$1{kN?A4YiY zbpm*UkYE^?^GB_Ba=ddxmxSZAMEf0hpD9x42%8A8PHWh541`u^19Us1;BjXh{)zRB zGauE7pF3};&>8sLTNo8ZnX1vwnvv!UQL$j+XLVJOS<~5+QX&k<;#CRf(z4!5vwe_O z_D)*a*~H?rnU?1*Z>KCIA7foWPNkP$q3Cq&w}gzUznej6bZ_fQp0r|uKWAbVyce_J zZRzv1)PnX?ERIRYZ`J_JlCI-H69zNj_;eYV7tLdeWX#6=CmiVMKM2BUgeh zGTSpGra>KaJK1Qq$4io?a(juyw3OsYW2?m>B{sQC7tCyOg`z5Gro$F#?=;Ypoha7c z8HdL>Ii8DS4|F3`2xx18d_L}sqhj&a!!&O7FSfsqBH@`Gb?7_sKRfhDCX++_PHW7E zq|H7e*hj0;ED#@U zsf#2nSWAcv+wd#A1GrTgzud(;JzlOFFPLX=rNn$0AG{wV-?|wHX^mdtG^i@UO`w%X%@i~m;MMRd$-5Q_3%#a2O_)~g5;B1h%x4<+Z_Al zwQUYZlb>#LRQu&^j>V34j=1qha8B>w0PA&nrLg?8i`j51HhA^7W%4x`mn}w&wTG+x z#9uciW-WIC`VXeneS#P-eiF9I zeMZo&gnS9_A1qQDiBKw;2@;;nkgvn@h_A#Iu>MeyoW-mAMe)!c zq?C4%Tm;+~(0162J_X@$j-11Q-@}o^7vrRelwK#_g)fxV171=(IPz3BIwty`T8hv) z$eTz)Nzgc9lW)XY$6?z<2PzjopLiq8{w}e543jT!;K#38<6>RjzAnz$;~ikRcCt!> zGt*o?`m^kWpXJzI$0g@>L^ zi?xuoMR?7x=c;Q(_z7@+~5obU5p)CFi(ex1r?cM0*q)S^5`V%cUqrMYI zQ5P6b)}k-Ld2)6EIT#%2XpZz6N4AwC1?ET{IZ`X`31W`q?+hAJ@thxF0}dn!@E-}dXoB~Tv};Nz-K3N!{M=}m}=)d!k`Gkh`?v4K^=$<3m`>6hablCAs1s>vMLe*ud9 Bi30!t delta 3922 zcmbVPeNa@_6@Pc%egMnDvdiweEFYpGEFUfeD9D$)>@LVR2q+P(0&fuz7V>sYjJ_vL z8WU1cF-H?;k~BDNZLCB~UZ*A#n}l?1CQj7Yn8t3B$>=nuO*DCqNn_e{(tGbh4UT{H z&2aAToO|v$_k7#~JoY$y<54!|*E(GkgTMCuM&J0iGcli9P&53~GKdaJ-HR?GytotE z73d)3WhH==cmmZ+y-8zOP6X0_l8X>)@L!RN7HhDQv}(!ch}HW)m2#GO5%NfZGzUV{ z(l3_V5K5L_OLrqANUS|4Z15nq@@CcsHroH0l%Ab~;#vhOw_%Hd+=u^Ch#OrF_vM>G z%qfPf#6XP zS|ZpN=vU02oc0}w#%H48a9%0ufeR^CYo_R!jPmcnKKtf;wsC!;H8=*{6Mi3B_aX5;~}*aqthv@w&2|0?%-Afj?VebLSP5qf(oB4~Eb~r9w@@+F02gvh zs!G8<9e})*b`*q%3cKSx^Ex<4v#@Hak!XSAxL)J&n*`@=p7jVGxKU^Z`RSNw(JN^9 zr3yyy4mQI`i3uu73}`E~l-OABU<;O#i^)`Wab z%y1OH{2jFO>VmJ(&a17X7htj_U zzmJ{suL(Ammnj9!^cZZZu(Pua11|R>CGNwW)W_QuQQDu(={5)di0@E}L`YiYf_uu7 z<5)kBZ9Np;$4$B(9$IXMPJ4ngN)J#qvOyJK|Vk&~Q#?!F44v6bO= ziA_SIhuM%KHVaKcGq{#o(W8J0%u(BQ*iF;KVxb8xlyn%l#{`D-n~=#Rl;L$@SKisD zs0n6@l4I^@R7a&Lay;xlpeMa|PEvX&ycjA2Sh8efTI54k;pJykx8_&e|NOe-gBat{8_$T~{nZY|P zTA|UCjCK>NWWM^ZVVm7^x4cM)%ocGCzP!c33dT`{GuSG$a_n4KYK(WyX~zctI`0SP zeF8r5WT8E{;A!ldDLlt=OgXKjvYU|*DJTV%kizYKM#b%WMln6Pux4VL&^G1AC*sKv zi4fZ(8KIrq52)CJp1{0>=OiRSwg~x37YoT7_I8WBLH>uS!Kv zN>Y^=XS1vLDy+j-6SAI=y@cG0iFBr;PJyby-1!9(B~_OdA%m2%Ar;~Nb^Z8tG%6`J zHY#9UcP2Us4|n&Yx8d{d<=Q`x^RE)}8YK3-p?-_NGw`>bYubwhYA~^{lRoH8MEcK( z;5s2U2)PLw-wO3N1bz$kzTA$t3H*UU!hTN(gMW{PqI&zggByoOINwl6NihE>viBw- zVXqj4qVMxtMgmU+3x;Ob(TGtq7fS7ttS>^Vz8la@WNRbhNypH4Q;7kihH^}QsojjW-SmGtySag{p4f1Fa8O4ciAbp}~K z%XFE^dJQEGB{SEOsP${fQDejL?6r*a;aK*1tP11n@znhFGIu^vzme;*lJ#a(*25bfr;HXA8TXtuJ($sH9QrDy5)VlNVlBdTC&@YnQkjRMKH^ zhs|`cQLe5ucD74rbagGp>tYq=;qozy^!}qY2<1ZRc!8<_SN{cR{rC$k?%s%rw~!kC zJW+$bkP`P;kmBD;DBSDNlb6hYKuCy?4JFr^W<+n zzLjXU6H-kGeVk5$8er&NMe@GD=!iey3($*& z*a2?a{O#b%lvCA%wU$chhubu|_sJzTVPZG%AHn$ZOUz^rg(AJfJwA>b@NecX6O9nO z{(Q2D=GcM{^H+$lg^;Uo^Z8<}l?Z8OY;buh0h*6$RhzM_McQ;UnKd6(gp~ZqkdLn? zN5}xPEBH2;m@ZZgVXcGGThj-SmKLsqXnX?w$L;7cJaK%5Y7{Fx3#X4?izoXbZNL}Y zi04GlNWjN;Avk>^6CIc26KhdYc)xeV-#4&%<7juVpN}ENNG*8_^v)Ei9>BIJ;2%o(NT9kRCt!P(y@CJ*9GJIgoWg|8#>9YmnwIf5Sq&B+q{uM38M$GqLdmk=D@ zBdCFB(H6kw`cD?(QS|eZON<$0uNLox)V=*91HC?8gm+JJs$E!qN-BHlLxt%Ja#AEJ zY~aBFPbaSta$adhA=vrK@6dY4KUIyc!j4nR*N~^clOE%DymiP|!9OY8JoOhAwZoa$TgTe7(28*ntFoubalVd5=GZ9)jjzR;Yi<6BMnwdDW zY@u)wV@Z^)6ecobaV9y5;$#MUPN6V^iBCM_K2R>(DVxc}m?wKDHpp3&&1MV_t>iix{nOGJhgRt_ut55o58*4PmCD5iv%K+!W^4 nBSvkMJHyNml5#_%P$<4)FuB1piik(das;(KD&70W3dMf`yU0H% diff --git a/litellm/main.py b/litellm/main.py index 20e554c76..761080b35 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -672,22 +672,29 @@ def completion( litellm.openai_key or get_secret("DEEPINFRA_API_KEY") ) + openai.api_base = "https://api.deepinfra.com/v1/openai" ## LOGGING logging.pre_call( input=messages, api_key=api_key, ) ## COMPLETION CALL - openai.api_key = api_key # set key for deep infra + openai.api_key = "1LntUh4fmg5z6iEW7UPPRhGdBDNuJx5y" + openai.api_base = "https://api.deepinfra.com/v1/openai" try: - response = openai.ChatCompletion.create( - model=model, - messages=messages, - api_base="https://api.deepinfra.com/v1/openai", # use the deepinfra api base - api_type="openai", - api_version=api_version, # default None - **optional_params, + chat_completion = openai.ChatCompletion.create( + model="meta-llama/Llama-2-70b-chat-hf", + messages=[{"role": "user", "content": "Hello world"}] ) + print(f"chat_completion: {chat_completion}") + # response = openai.ChatCompletion.create( + # model=model, + # messages=messages, + # api_base="https://api.deepinfra.com/v1/openai", # use the deepinfra api base + # api_type="openai", + # api_version=api_version, # default None + # **optional_params, + # ) except Exception as e: ## LOGGING - log the original exception returned logging.post_call( diff --git a/litellm/tests/test_bad_params.py b/litellm/tests/test_bad_params.py index 4296d997b..576cc18f9 100644 --- a/litellm/tests/test_bad_params.py +++ b/litellm/tests/test_bad_params.py @@ -12,7 +12,7 @@ import litellm from litellm import embedding, completion -litellm.set_verbose = True +# litellm.set_verbose = True user_message = "Hello, how are you?" messages = [{"content": user_message, "role": "user"}] model_val = None @@ -41,7 +41,7 @@ def test_completion_with_no_provider(): print(f"error occurred: {e}") pass -test_completion_with_no_provider() +# test_completion_with_no_provider() # # bad key # temp_key = os.environ.get("OPENAI_API_KEY") # os.environ["OPENAI_API_KEY"] = "bad-key" @@ -53,3 +53,10 @@ test_completion_with_no_provider() # print(f"error occurred: {traceback.format_exc()}") # pass # os.environ["OPENAI_API_KEY"] = str(temp_key) # this passes linting#5 + +def logger_fn(model_details): + print(model_details) +os.environ['OPENAI_API_KEY'] = "1LntUh4fmg5z6iEW7UPPRhGdBDNuJx5y" +messages = [{"role":"user","content":"Request boss to grant me 1 day leave"}] +litellm.api_base = "https://api.deepinfra.com/v1/" +response = litellm.completion(model="meta-llama/Llama-2-70b-chat-hf", messages=messages, custom_llm_provider="openai", logger_fn=logger_fn) \ No newline at end of file