From 00c2384a2da9bf6cc2b78b71ba186d966b64e919 Mon Sep 17 00:00:00 2001 From: Zakhar Kogan <36503576+zaharkogan@users.noreply.github.com> Date: Fri, 28 Jul 2023 11:44:08 +0300 Subject: [PATCH] Added openrouter.ai support --- .env.example | 5 ++ .gitignore | 1 + build/lib/litellm/main.py | 62 ++++++++++++++++++- completion_test.py | 38 +++++++----- litellm/__pycache__/__init__.cpython-311.pyc | Bin 0 -> 199 bytes litellm/__pycache__/main.cpython-311.pyc | Bin 0 -> 6147 bytes litellm/main.py | 33 +++++++++- testos.py | 0 8 files changed, 118 insertions(+), 21 deletions(-) create mode 100644 .env.example create mode 100644 .gitignore create mode 100644 litellm/__pycache__/__init__.cpython-311.pyc create mode 100644 litellm/__pycache__/main.cpython-311.pyc create mode 100644 testos.py diff --git a/.env.example b/.env.example new file mode 100644 index 0000000000..7889a204c8 --- /dev/null +++ b/.env.example @@ -0,0 +1,5 @@ +OPENAI_API_KEY = "" +COHERE_API_KEY = "" +OPENROUTER_API_KEY = "" +OR_SITE_URL = "" +OR_APP_NAME = "LiteLLM Example app" \ No newline at end of file diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000000..2eea525d88 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +.env \ No newline at end of file diff --git a/build/lib/litellm/main.py b/build/lib/litellm/main.py index cbefea5a9c..053d796e1d 100644 --- a/build/lib/litellm/main.py +++ b/build/lib/litellm/main.py @@ -1,4 +1,7 @@ -import os, openai, cohere +import os, openai, cohere, dotenv + +# Loading env variables using dotenv +dotenv.load_dotenv() ####### COMPLETION MODELS ################### open_ai_chat_completion_models = [ @@ -13,6 +16,18 @@ cohere_models = [ 'command-nightly', ] +openrouter_models = [ + 'google/palm-2-codechat-bison', + 'google/palm-2-chat-bison', + 'openai/gpt-3.5-turbo', + 'openai/gpt-3.5-turbo-16k', + 'openai/gpt-4-32k', + 'anthropic/claude-2', + 'anthropic/claude-instant-v1', + 'meta-llama/llama-2-13b-chat', + 'meta-llama/llama-2-70b-chat' +] + ####### EMBEDDING MODELS ################### open_ai_embedding_models = [ 'text-embedding-ada-002' @@ -34,6 +49,32 @@ def completion(model, messages, azure=False): engine=model, messages = messages ) + elif "replicate" in model: + prompt = " ".join([message["content"] for message in messages]) + output = replicate.run( + model, + input={ + "prompt": prompt, + }) + print(f"output: {output}") + response = "" + for item in output: + print(f"item: {item}") + response += item + new_response = { + "choices": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": response, + "role": "assistant" + } + } + ] + } + print(f"new response: {new_response}") + response = new_response elif model in cohere_models: cohere_key = os.environ.get("COHERE_API_KEY") co = cohere.Client(cohere_key) @@ -76,6 +117,22 @@ def completion(model, messages, azure=False): model=model, prompt = prompt ) + + elif model in openrouter_models: + openai.api_base = "https://openrouter.ai/api/v1" + openai.api_key = os.environ.get("OPENROUTER_API_KEY") + + prompt = " ".join([message["content"] for message in messages]) + + response = openai.ChatCompletion.create( + model=model, + messages=messages, + headers={ + "HTTP-Referer": os.environ.get("OR_SITE_URL"), # To identify your app + "X-Title": os.environ.get("OR_APP_NAME") + }, + ) + reply = response.choices[0].message return response @@ -99,5 +156,4 @@ def embedding(model, input=[], azure=False): ############################################# -############################################# - +############################################# \ No newline at end of file diff --git a/completion_test.py b/completion_test.py index 4c552ab673..5bb31928b9 100644 --- a/completion_test.py +++ b/completion_test.py @@ -1,4 +1,4 @@ -from main import completion +from litellm.main import completion import os ## Configs for Models ## @@ -8,23 +8,27 @@ import os messages = [{ "content": "Hello, how are you?","role": "user"}] -# openai call -response = completion(model="gpt-3.5-turbo", messages=messages) -print("\nOpenAI call") -print(response) +# # openai call +# response = completion(model="gpt-3.5-turbo", messages=messages) +# print("\nOpenAI call") +# print(response) -# azure openai call -response = completion("chatgpt-test", messages, azure=True) -print("\nAzure call") -print(response) +# # azure openai call +# response = completion("chatgpt-test", messages, azure=True) +# print("\nAzure call") +# print(response) -# text davinci openai call -response = completion("text-davinci-003", messages) -print("\nDavinci call") -print(response) +# # text davinci openai call +# response = completion("text-davinci-003", messages) +# print("\nDavinci call") +# print(response) -# cohere call -response = completion("command-nightly", messages) -print("\nCohere call") -print(response) +# # cohere call +# response = completion("command-nightly", messages) +# print("\nCohere call") +# print(response) +# openrouter call +response = completion("google/palm-2-codechat-bison", messages) +print("\OpenRouter call") +print(response) \ No newline at end of file diff --git a/litellm/__pycache__/__init__.cpython-311.pyc b/litellm/__pycache__/__init__.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..9e55d25fcb94ea0e4d10ebbfb5a4e85d88ac6f99 GIT binary patch literal 199 zcmZ3^%ge<81kcM3r&s~$#~=<2FhLog`GAb+3@HpLj5!P;3@J>(44TX@8G(YDjJFuI z{D34&ZenKMN`}uM9lvb#LyJ?3iuJ1!vojKl^j%VmvrF;|^po>bQuPZeOEU8F^m8&x zQgd>0;dFd_W?p7Ve7s&k6Y#^9j#|aR+(3BD&H0*8@HiQ+8%36bGa3(loYi1HQ z#-VPtDWw-Lg5GOd)0@&tZd|p(MYKxRr}C4uh@MkPd#TG-$Ga*YPEf+ zH9qrQ&v)+UeDg=Q+d<%`U;VT2Qy(G!z)tx`Z#15!p|L^)B2Wo3U3yc~l-Z}JY4goY zGc*yISxb5IJ!m{EcL+rUT43Vzv{kUgscBZQ0_5UEVE;f(+XN1JcEKswKc=P~FlxI< z3l3;I1?mh5x&DcG#?$V2N(=Ar9|~*p%7tXwiroY022K1(3k&>$B*&!iz`*{rHku^6(a1My@so)9A`J~0>G7mg(bF*d_%;R}+Ql+$gSW~(#J$&@JbQl$Rm zwyj3Pg9k6A8){|_g!k{ely2l@ZAM9^q*x@D;O7M~yf5AP+?XV*8VrUP2GgB$qQ-|4 z34V@`m=Ne4+Z-P@tqc(y8H+?ca&Dr+s&!UZ39gLwx8!$pJ$`Ub_Y#w_ zo*?Jjm)N@Xo(#Ly`V5m{({2Jh-slj#LWAJTFhZlybe9&I?=o>pqL#QiJS`b+OFWz@ zrHdJ&1*%knyngmt7UHkxXUV4RsA8LKD7Q1VP3!D0)Jp5CXeDD`g^>CD53IrRsBM!T zn2MZy?UgN?WtKsCQ1ahF9=(-CXg?{Tz1AM9zA1FnjaNKOAa�^OMecdDUUuzMlj( zO9nU=f#cnE9RKQnm*W>&P1mbfO|2AXo69`EFs^K2N%9vUt-a-)1nuK5La-L^HY4qh zSJC$ix$k~$?q4CQzE&LQ*+hR$hI(n<+F8F>c;hbpsP`UZyN8vG2QiYFtwds>#*b%P z%i36SWEdz}c8yONRz5wih=ylm`t0Orbma7@=&_MAqlT~A|6p|T%&CcS!(APEcl0MA z+F(UFF3F-{n-f))kBh2dSHx68it(EGjOvb6(kmQ`Og6#)X%N5)iIWUD?^vQTP@h!; z&9CVj&NJizWpJ^itckMrfIOfLx_`jn%12=0>h6AaC~{U66*ZFPFU{~uiRYsn|0%EW!1#oF0%CJguYFpfR z(Bfh;=vJ%l1zvp#&BPhWJ#TgDk2Z{AL13nkCaAO zlJ>~r07X=s2|;TznH$P3_bp*H0WKy286 z;i$Hl5)C@38k{IEND7!J6Bo78+y!104JWQzfNkJk28W$X;v(28+!2mf9M525VBnf) zSZ0%wtYH0VFv>jGdrFaH&2Yz(GeB6>v?LW)n_YuFo{%6y7`C`5iwf=!qDrE?6vaD8 zwCdUAot1gKfNU}EsG-SMRqw`jr2_?u&KGi41*2eGGi+r~EE~G%`Gy6(-f)4df(w^i z)o^TZgMy%ANyCZ}b5V5yn(pe)^f+8wj!w#t@L>3>zlT7dCA!sdP58z6?D=(n*C!`G zo>=p@-JV^MZ^^k`LwWzfg8!g7+TMA$c)@)tjf%O3!MCvOQBEWw;5sC#|cF}=ZGB($~F1lKHUtn=E$^3=77YZG}k zP+$W&HlVYf<=#B&&(36L)>-d*XOG_0yx!iU`+JI3#_uW;1RQ5~6bS;x-e@Z_&Sp0I z-ZyRL*`YhbIY$Vdyk&2}vNvbht9v_xx9=?2cV>^hOdP<_%`4Zh=q{h`Z9&R$H`ny+@S57c=} z>+R{44{v>#>phTf87j04J*RX}lit+%0;RjRh5ltCF>S8o?5wN z$3QYRIKC&x;iLNxAX_!;yq#}3RA@Pb?5w}UBEPJ)anwWWmu($)9iMqV^{g)C+lC5l zLu*Yfw-2uzxpibUoNpQ^G!1|bykI&AIMW|E^alVySp#vm6kLHj?-yJ<3a-BFsBZUV zBWCWa`B?IO!U*~iL=X%h7(}oSfMFjkf3{N&VBh@2h$*-7Id3CdhAgEVX`YYfuN(!S zVhgieDZkk(yOeif;+MGMuK@mQ>21do@9xLm-S^LaK9uwB&U;4+-jVDWq!|pKdvY8; zuN^+CNOze?j~2W~0c9l;gs77tS4pm>m=6Jm1}h}tezRaW6G>i(mO9EX%qTxV@HT=U zB6tS@CVx{zrP3U0Yg2;HY{O6$%_qbo$~cUnVXI#NC^8g9>7*z3I!}=$R8y8Mkj6Y| zE|BINX@+#qH0rb+V&G0XM}iQFAr~`rrQ?T=kIuS_Hb)CxBxR5zzG8q_yI?i|``U_) aQHp{@_8c5zNFca+wn#iU{Q_Xd(|-f&4f|mL literal 0 HcmV?d00001 diff --git a/litellm/main.py b/litellm/main.py index 5f325f2590..14de0b02ed 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -1,4 +1,7 @@ -import os, openai, cohere +import os, openai, cohere, dotenv + +# Loading env variables using dotenv +dotenv.load_dotenv() ####### COMPLETION MODELS ################### open_ai_chat_completion_models = [ @@ -13,6 +16,18 @@ cohere_models = [ 'command-nightly', ] +openrouter_models = [ + 'google/palm-2-codechat-bison', + 'google/palm-2-chat-bison', + 'openai/gpt-3.5-turbo', + 'openai/gpt-3.5-turbo-16k', + 'openai/gpt-4-32k', + 'anthropic/claude-2', + 'anthropic/claude-instant-v1', + 'meta-llama/llama-2-13b-chat', + 'meta-llama/llama-2-70b-chat' +] + ####### EMBEDDING MODELS ################### open_ai_embedding_models = [ 'text-embedding-ada-002' @@ -102,6 +117,22 @@ def completion(model, messages, azure=False): model=model, prompt = prompt ) + + elif model in openrouter_models: + openai.api_base = "https://openrouter.ai/api/v1" + openai.api_key = os.environ.get("OPENROUTER_API_KEY") + + prompt = " ".join([message["content"] for message in messages]) + + response = openai.ChatCompletion.create( + model=model, + messages=messages, + headers={ + "HTTP-Referer": os.environ.get("OR_SITE_URL"), # To identify your app + "X-Title": os.environ.get("OR_APP_NAME") + }, + ) + reply = response.choices[0].message return response diff --git a/testos.py b/testos.py new file mode 100644 index 0000000000..e69de29bb2