From 9513d6b862a43e3cc231d5972d07186404f39902 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Fri, 13 Oct 2023 22:43:32 -0700 Subject: [PATCH] fix(utils.py): read env variables for known openai-compatible api's (e.g. perplexity), dynamically from th eenvironment --- litellm/__pycache__/main.cpython-311.pyc | Bin 52540 -> 52588 bytes litellm/__pycache__/utils.cpython-311.pyc | Bin 163527 -> 163534 bytes litellm/main.py | 5 +++-- litellm/utils.py | 11 ++++++----- 4 files changed, 9 insertions(+), 7 deletions(-) diff --git a/litellm/__pycache__/main.cpython-311.pyc b/litellm/__pycache__/main.cpython-311.pyc index 42ce552ba1bd011afe2d03517e8a7ffbc5bf63b3..521efeb3222d76679c268be4eacdba713b229a42 100644 GIT binary patch delta 3684 zcma)932aoy8J^j<%dU^TZ13f@yuc9u+t~O(0n!vqOL-VK7)Z@4PQpCw zB1%!>z?Lz^2}GFE*v1$HdX-wWNx`aGr8I{ThM-naqX^PQLe+E2*+cETw>Dk_Rx;Ae z{Qvwj^S=Mw*>~Tf?z}}Q$CS!sKw7}#*>dlqa`tT$T?`6$0PnKq#O-Jo|FZDcjHiOod$xXt{E*}*7;d*kiJ)Oo76 z5;^e2BL@7)oBdf$y`YDQYQY)ugB$fHGVGp zCgnzM{F2FG>{M-#5I`Q*i#*7&8}`bO7demz_v9Qkxai^&a^(0KJOo)EZ6R{dYb76# z71*pk1ZWe=n=BcX0&te}cw>FY_Z&cZBRP1}37rh0G~~mFaxD0$B?EqjCoEO)xA=ms z8ar}VQ!Zr3BiT+Iw3p&*xq+_(ssit^e%s_gxyU$KJxc6;RTTEP&t9U<1smIgDhogWd|(KMxb%F4RCj-k%rB_V0tVl=biR_rX5!GW7!JgNqf1 zz9Tk6(jqP8qLy?m(Nm8nigeson-9(_le3hCD+!E2HLeZlF>l`{3lK0y!DdtdrIA$w zKAhhT1LRHvRy!2hrM=Mjm0ln>5--H>)ZzCXN_@#tt_?El7Yu@YbQ6B>m=WhXlhT*7 z1x#mxtbkrA%GNjyumJZsvtc2A&1nr4GMg6dT4j37qUCh!tAs;^lWn6Us1w;|y>PHgoho~`yBhGqC8pPL|hNN|s+zTl@+8xqbfC68W54@t=YgSdRY{ z%+^+*@&O6m#g?NAGH+!oNK8A7h_H0`2?@RV1T~+LkHdG*awDoe!9)WSz7)|sLyHi;Tuci4YB^qC5wIh@uEFIjl2ey8eC9X8fvKSNSa5XH}BVzp0IS5LHlZ7Hp3 z%tldtJ?K=j4Je8l=>7$4uPVupQgl#-qIe+W!gp(S$^-F=hc6w|OV~zSTpN@}7nX9^ zCe(<;F4Ru?D9Lpj-62*(!Mb;9-=S7Ae_X=LD~Xpg@aJLgqWlO&5t{Yq2oVAf`Ee|2pvZ4GHxZrI!dY(5^A`)jAW_wWETO4=o4D(jHU zisM~zys9WVD4SO2jkI(B?2bswq+0(?jfJPkd|P7}Qu8uG-CG8<#u-f>FBcdE(3+>s z-m4nljK;^O0A($;;~Pw8YKdM8$=Ak{8wW{Da>CwB35C(qiAvo^WB25UYPGI zvUcZ#`*!j6`~CUJeP-%@(AlS_9_VF+eqa`_e@NAdX#Q;rn}lQksfy$Y^Z#<=*H485 zks4uPsy+}AcT6=Z!a;GRsXQ$1ooY6Rb>d1(L7iPB+Vx?XNOWXH1vnlswYDfC6_=hb zuaC>tS0{%z-pGRRF1O~;IS4sU5(_Gz z9v2Q@fseTV4)0UIEN&ECgsZolOkP82P>?EaOoH(~21EX;*r6EUS z<6fx3?~cDtu8obGb*51{Dd(ie8>`xp%M=O!4r*B~Q!>9yay}DjkAt@*i(f|jAX14* zP`a>_JYn0|G!bP8rs33yAQ>wsoK)K0G_GsHB7y1nz+@zEKRMDXJbin!$V;nNCJYS% zYa!IGbV89NNXHK*)sAKvS$T!LFetN@SM-bviE>Nd?ek@JS+>3^ELSib$q{w9MiB zAq%e*Srtglh(!l$c(urCfI<_a_OrZJWYd6B7h5yf#;1#{9%xO&I+B?Nf$iswz+f5v z{?Ja|B#JYEF?U2Z)X$qmHVbHVe6~pD00Ph=l6gR%GrW1IledYi3#bjTU4z?sx5#<~ zbhe>v-Yc>`pwh=S4|ei?ku3(x()kjRBx+mdwV#zsB$B&;46_uFDE8}lg}7%OckRk| GCI1Hj>*m$~ delta 3499 zcmahL3v5%@^ z0(Ch-d`btXC26uiXU$WCOM_(IK_c*k~owQOZQs?Y@_oI1OP}dhgtG z&$;KG|Gi(FWIj5{D25e^WJCv;3RbUrPtl`Gt}jC9K-V$EVR(K-FC5YR2{T;>yM%Ry zVVodCh(~-D=(8=lJd)R!%<4$q4%8RV>UkAw;Il{`EY5x*+s-A5Qm4j-)ab0QhS0UEiR44k2HxH4 zgv+_LE)_{<4MffQXB6vgvLUWx#?n00SH*kaIjakM;8m;7=-G?eINr0%(~tYnPG&jk z$Fq#db`wOhh;__66h{xG-_+nAnJ3iSmOsf^H}VePLg=y>m_m{cxAL^`+n*`m-TWjN zwLe8E;ar^|ov7HRQ}V~UQ`CTMo>zlxITcG?JtMeFS;Vb~>Pds^hE#`5UCfto-7%{r zQ12);m9VXbM~h?aLzvn~zp25OLfB#Pm2#`1D^ywu+cC3)6tkT#QX(lG>l$L{UiwW9 zzKj%|DTT>%8dKS9sTd9HAVtK+w$z9_X_7d*X_AO3deoUiXYV^Ja5?GW?=M)po3YxW9@Jz8* zMtyd&jqiJAX1kxGFXouT~vp>N!D__L$nJ z|9JS zNP>{-H$H~aAS&+n;|2)(b#AGFu)v2%LkM*%_(l>UjqL6jEnZa6>n1diMnZ5a^b8c# z`{7}k5_Z-raT8S67vm6YsL#zIO{7`;#Ma0jr4?*r7fa1f>`hG!G3D#^V| zlx^{(4K6gSrMF7!nN4EmG+l_;Y5HU4%EQn`I*Wyf&y4;XgD%g$Vy? z3tB|@g}lWn!h1yltqAYSb9PwKeTyXh{%&h>zn;18ar9>}-(<+?{2RR_{sC$*iR=%} z0fPv?tqxj6_+R6c`0;={=oc5pL++r&G2Sc>6iX{D-hjkA-lh*|q?Puf2Fb?pMHvCP zw6fUHD5_I{#+Q}^M7LADK1RxKY{T(!iIx)c)73yM(#>^zqWz2^oQ1tm`o>p?k6Z$5?K9Wq3@Ru8YArecJeCD z8)czIn?+P_pf_c+UoLwwQNZ6vG>rO@N;dfAB_#w#Em#dbqYg&BOD!B4HOX)~oF5Cq zy)i>eIu&c4Gy24S(6@!YwWMFG9HDJH=<9M1`nmvF%P!g{syZeuomVZLlX1Cw8Yfb6 zh2`g0#p#c>S9hq;C#vL5C-X^GAgR;F{N5(-bP9ibFPWi5a`S^R8BP&i{dob#Il@mr zynwM;V1Kbc9v29AF8?QSce4s7N09jcOG;XVk+dGWWU_mR{_ZkJmbgzF ONtAe&2+YUpW&Z;_!=;J< diff --git a/litellm/__pycache__/utils.cpython-311.pyc b/litellm/__pycache__/utils.cpython-311.pyc index 4c39a1c48348d94578c59ab8b65746b507dd9bdb..af8acfef82538ce0d755c75f370bde75097aba97 100644 GIT binary patch delta 1377 zcmZuveN5D49DhE~{eJfXcN{ObL+5r#OZS!UDB7eo&y@p5?FNZO%j zF0GucJ=ISuO{Y#aXDO#XW`?0$%YXdiLz|c|Ong`M&P? zVbstyYA_uznWBhTyEC0tokvV_yTm(LiH z^NF8&r}?O|L8cZ*30a!iBKajhZ6e+@x6}f^{fRSbEsAptH!WdsufXeky@yceTmFfX-jg`8d08zne0o9 z@YwRpZe{r)B(dx21mPeDQ`t9nj4Ur`Wlx2tu!f)&X0}<}z8#EV$AcDjXvD~Rf>veg zMTo4yM0QIUl0z026N+Ij!8wAL3x2rXz*<5{$Y#Ni9oZihz{Zv~#;~iQIOMY1p(*KO zvq&SpyeHKVl1{w37Oh{q5n_nib=tu6i$Mra!{riPpTpBoRbQbDbs&M&ULHu*d258H zHy#4k2!*<3y5P|dg5~i#glb1A&o8#A+01#=R%IX zVsmUU4@Nt}1cst33FA~-R@-_t!KE5q>}=0#8Bau5ON^E<(HiEUKRNAEa#5Jbra2fx zR@j?m-EO{OPf;g6eR#cUFI6p%cc8U5iyGUCRCB(n&wtP#ZBcSfNG*1X$~WrElE{0- z=>j`*rU^VVxinYM`J|{ar{;VbEz8iH&ydBr&sbYqSJB}0G*!E+yxxA%Ng$(dSkewf zdhch}fyqzz;oZ0fMNcw9jHpB6!A27)}w_achu?AwF{rN;E>Y`mjjIJ|Er^k}q3}R3L%3 zHlyE$4!-O|9FTC1Yj?mS<0fy~h1C?%d|)@eqDWSD?8Qw?RyOX#S~!(|_Tdb&c*lN( zD02A+ZE$Ktf2R(@CSd{Z{Sx;n>iIA25dWx|e|{7DQp6-H}(-8+94|FNr6X;G{0$}fdG%D9rxBC*ESG+#$9APMpPs}gJb5O~2RwXX0qu}*kXwssHsI!^ zv*=BY$dX-3Ez$Uvm&~JCaY*Oh6|_h%(zL%$_W;AnE4B2Pm`GF|ohoCH&+t-JSNtLn n8zvFl{@fTJUVWnRXx-RZU9e28SaE5=$}#yr=;q~b(iZ7otUAzS delta 1371 zcmZuwZBSHI7(UOr_wKT=%kr_ax~z)`qqqzpen3eiqTq-QOpKFeOzt4iV2aARnb?w+ zKh#P_=Tui`lA)Brv1ZH`J(yF9lOLq9cQ-+!QP9d9ZDk=oKq$xE|ZB){aRn~68OPHur;s#DfdJ-Ij4XHu~UHnu02Fx@xq z={_MILzD@Lma3&7%gDR4{Y2!g(Z9GDbh7vn6SHaxiv;Z~`-vVqS)~@o@->HQu7ruL z(DEhO=TL`};bcd(Sc~Jov?8(0dbGLGeNxE5=7r)|W@rMdjKs0>P$Kh(;#ot;#@YnG zE%>b9PX(v`Ze~|PcD76Kh~R1Ac$O5lvBiSt2>xrEiB*Li>iueJVw=Ja%w$c@iEJor zWnJN);f^r9um$Q3L%^FBto{62(5sUFdUZ}Au#1$PoAiyt`nmLCB2VP8z zh7kiX+aknZD!4HzNO9Pu9r6l7X-xq@fJ4*P+sW9vM*h~Z!RP$|5VGO9@Epssq@bJTh zNTL{4c@b8iiCdq=D6pMN#n=h#;qT1HJQ+jkH>Ie@`#i4<<%-z7bqTiPH2-r6@)VIg z`#IcC6ZXU=xG~HZG@(=ya;Fa~g}mj%4k3B{7EA$>_|9hBNWqu9{ylWcIL|EyutC8u zyrm5d6tVnHJ5EtJ)%HWUi)rfa!`K41N;_~4+5CJ5!W4OY{}E(Kh7z;5XBWh^WSN!@)=E(p$VA!G?>&J}`0xDu)y^WvZ@VWZx2!22culfxg z6sP#(2Y3ja<-d-i+=LOeU<~(R;^}|m1&ZK_*Xb3bIL+c1nqWj3UumV6jM%`_C(t$- zoxI0MR{#O-aM3o1QFW(M7x1k*Fqs}inETS`GKv9yDV;7g;xV^n(P|k5d}A)1rH|my zO!{>URJEayet`o#;~82AtmmtW=vOk1a#snR0o3w}1@x{YsuV1uwm6*OORH#3A~JaM zYFZp4%A9(Iegxc8tJl#JVj_u+G*`h8pYNrbv7{16h!7q3FO2eM4c(i%8l$nce3j-| SeRX+lRQ?aH=2cs1i~JACq03SL diff --git a/litellm/main.py b/litellm/main.py index b02e8cebd..3f37ace14 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -257,7 +257,7 @@ def completion( if deployment_id != None: # azure llms model=deployment_id custom_llm_provider="azure" - model, custom_llm_provider = get_llm_provider(model=model, custom_llm_provider=custom_llm_provider, api_base=api_base) + model, custom_llm_provider, dynamic_api_key = get_llm_provider(model=model, custom_llm_provider=custom_llm_provider, api_base=api_base) model_api_key = get_api_key(llm_provider=custom_llm_provider, dynamic_api_key=api_key) # get the api key from the environment if required for the model if model_api_key and "sk-litellm" in model_api_key: api_base = "https://proxy.litellm.ai" @@ -391,6 +391,7 @@ def completion( # set API KEY api_key = ( api_key or + dynamic_api_key or # allows us to read env variables for compatible openai api's like perplexity litellm.api_key or litellm.openai_key or get_secret("OPENAI_API_KEY") @@ -1371,7 +1372,7 @@ def embedding( caching=False, custom_llm_provider=None, ): - model, custom_llm_provider = get_llm_provider(model, custom_llm_provider) + model, custom_llm_provider, dynamic_api_key = get_llm_provider(model, custom_llm_provider) try: response = None logging = litellm_logging_obj diff --git a/litellm/utils.py b/litellm/utils.py index 74e81fa8f..08d9614f8 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -1405,15 +1405,16 @@ def get_optional_params( # use the openai defaults def get_llm_provider(model: str, custom_llm_provider: Optional[str] = None, api_base: Optional[str] = None): try: + dynamic_api_key = None # check if llm provider provided if custom_llm_provider: - return model, custom_llm_provider + return model, custom_llm_provider, dynamic_api_key # check if llm provider part of model name if model.split("/",1)[0] in litellm.provider_list: custom_llm_provider = model.split("/", 1)[0] model = model.split("/", 1)[1] - return model, custom_llm_provider + return model, custom_llm_provider, dynamic_api_key # check if api base is a known openai compatible endpoint if api_base: @@ -1421,8 +1422,8 @@ def get_llm_provider(model: str, custom_llm_provider: Optional[str] = None, api_ if endpoint in api_base: custom_llm_provider = "custom_openai" if endpoint == "api.perplexity.ai": - litellm.api_key = os.getenv("PERPLEXITYAI_API_KEY") - return model, custom_llm_provider + dynamic_api_key = os.getenv("PERPLEXITYAI_API_KEY") + return model, custom_llm_provider, dynamic_api_key # check if model in known model provider list -> for huggingface models, raise exception as they don't have a fixed provider (can be togetherai, anyscale, baseten, runpod, et.) ## openai - chatcompletion + text completion @@ -1479,7 +1480,7 @@ def get_llm_provider(model: str, custom_llm_provider: Optional[str] = None, api_ print("\033[1;31mProvider List: https://docs.litellm.ai/docs/providers\033[0m") print() raise ValueError(f"LLM Provider NOT provided. Pass in the LLM provider you are trying to call. E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/{model}',..)` Learn more: https://docs.litellm.ai/docs/providers") - return model, custom_llm_provider + return model, custom_llm_provider, dynamic_api_key except Exception as e: raise e