From fe8d59f5eb78ca88c35f80d3076953f5f5df23a6 Mon Sep 17 00:00:00 2001 From: wslee Date: Mon, 10 Jun 2024 17:27:15 +0900 Subject: [PATCH] add friendli_ai provider --- README.md | 1 + docs/my-website/docs/providers/friendli_ai.md | 53 +++++++++++++++++++ docs/my-website/sidebars.js | 1 + litellm/__init__.py | 3 ++ litellm/main.py | 6 +-- litellm/utils.py | 10 ++++ 6 files changed, 71 insertions(+), 3 deletions(-) create mode 100644 docs/my-website/docs/providers/friendli_ai.md diff --git a/README.md b/README.md index fe7d56b6c..83dd68553 100644 --- a/README.md +++ b/README.md @@ -256,6 +256,7 @@ curl 'http://0.0.0.0:4000/key/generate' \ | [IBM - watsonx.ai](https://docs.litellm.ai/docs/providers/watsonx) | ✅ | ✅ | ✅ | ✅ | ✅ | | | [voyage ai](https://docs.litellm.ai/docs/providers/voyage) | | | | | ✅ | | | [xinference [Xorbits Inference]](https://docs.litellm.ai/docs/providers/xinference) | | | | | ✅ | | +| [FriendliAI](https://docs.litellm.ai/docs/providers/friendli) | ✅ | ✅ | ✅ | ✅ | | | [**Read the Docs**](https://docs.litellm.ai/docs/) diff --git a/docs/my-website/docs/providers/friendli_ai.md b/docs/my-website/docs/providers/friendli_ai.md new file mode 100644 index 000000000..5f92c8a58 --- /dev/null +++ b/docs/my-website/docs/providers/friendli_ai.md @@ -0,0 +1,53 @@ +# FriendliAI +https://friendli.ai/ + +**We support ALL FriendliAI models, just set `friendli_ai/` as a prefix when sending completion requests** + +## API Key +```python +# env variable +os.environ['FRIENDLI_AI_API_KEY'] +``` + +## Sample Usage +```python +from litellm import completion +import os + +os.environ['FRIENDLI_AI_API_KEY'] = "" +response = completion( + model="friendli_ai/mixtral-8x7b-instruct-v0-1", + messages=[ + {"role": "user", "content": "hello from litellm"} + ], +) +print(response) +``` + +## Sample Usage - Streaming +```python +from litellm import completion +import os + +os.environ['FRIENDLI_AI_API_KEY'] = "" +response = completion( + model="friendli_ai/mixtral-8x7b-instruct-v0-1", + messages=[ + {"role": "user", "content": "hello from litellm"} + ], + stream=True +) + +for chunk in response: + print(chunk) +``` + + +## Supported Models - ALL FriendliAI Models Supported! +We support ALL FriendliAI AI models, just set `friendli_ai/` as a prefix when sending completion requests + +| Model Name | Function Call | +|--------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| mixtral-8x7b-instruct | `completion(model="friendli_ai/mixtral-8x7b-instruct-v0-1", messages)` | +| llama3-8b-instruct | `completion(model="friendli_ai/meta-llama-3-8b-instruct", messages)` | +| llama3-70b-instruct | `completion(model="friendli_ai/meta-llama-3-70b-instruct", messages)` | diff --git a/docs/my-website/sidebars.js b/docs/my-website/sidebars.js index b6b597d30..2ed1388d5 100644 --- a/docs/my-website/sidebars.js +++ b/docs/my-website/sidebars.js @@ -150,6 +150,7 @@ const sidebars = { "providers/groq", "providers/deepseek", "providers/fireworks_ai", + "providers/friendli_ai", "providers/vllm", "providers/xinference", "providers/cloudflare_workers", diff --git a/litellm/__init__.py b/litellm/__init__.py index e92ae355e..cbe82f96c 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -392,6 +392,7 @@ openai_compatible_endpoints: List = [ "api.groq.com/openai/v1", "api.deepseek.com/v1", "api.together.xyz/v1", + "inference.friendli.ai/v1", ] # this is maintained for Exception Mapping @@ -405,6 +406,7 @@ openai_compatible_providers: List = [ "xinference", "together_ai", "fireworks_ai", + "friendli_ai", ] @@ -628,6 +630,7 @@ provider_list: List = [ "cloudflare", "xinference", "fireworks_ai", + "friendli_ai", "watsonx", "triton", "predibase", diff --git a/litellm/main.py b/litellm/main.py index 2c906e990..3c3034cdd 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -1051,7 +1051,7 @@ def completion( # note: if a user sets a custom base - we should ensure this works # allow for the setting of dynamic and stateful api-bases api_base = ( - api_base # for deepinfra/perplexity/anyscale/groq we check in get_llm_provider and pass in the api base from there + api_base # for deepinfra/perplexity/anyscale/groq/friendli_ai we check in get_llm_provider and pass in the api base from there or litellm.api_base or get_secret("OPENAI_API_BASE") or "https://api.openai.com/v1" @@ -1065,7 +1065,7 @@ def completion( # set API KEY api_key = ( api_key - or litellm.api_key # for deepinfra/perplexity/anyscale we check in get_llm_provider and pass in the api key from there + or litellm.api_key # for deepinfra/perplexity/anyscale/friendli_ai we check in get_llm_provider and pass in the api key from there or litellm.openai_key or get_secret("OPENAI_API_KEY") ) @@ -4288,7 +4288,7 @@ def speech( response: Optional[HttpxBinaryResponseContent] = None if custom_llm_provider == "openai": api_base = ( - api_base # for deepinfra/perplexity/anyscale/groq we check in get_llm_provider and pass in the api base from there + api_base # for deepinfra/perplexity/anyscale/groq/friendli we check in get_llm_provider and pass in the api base from there or litellm.api_base or get_secret("OPENAI_API_BASE") or "https://api.openai.com/v1" diff --git a/litellm/utils.py b/litellm/utils.py index ed405419b..f98e904ff 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -6607,6 +6607,11 @@ def get_llm_provider( or get_secret("TOGETHERAI_API_KEY") or get_secret("TOGETHER_AI_TOKEN") ) + elif custom_llm_provider == "friendli_ai": + api_base = "https://inference.friendli.ai/v1" + dynamic_api_key = get_secret("FRIENDLI_AI_API_KEY") or get_secret( + "FRIENDLI_TOKEN" + ) if api_base is not None and not isinstance(api_base, str): raise Exception( "api base needs to be a string. api_base={}".format(api_base) @@ -6654,6 +6659,11 @@ def get_llm_provider( elif endpoint == "api.deepseek.com/v1": custom_llm_provider = "deepseek" dynamic_api_key = get_secret("DEEPSEEK_API_KEY") + elif endpoint == "inference.friendli.ai/v1": + custom_llm_provider = "friendli_ai" + dynamic_api_key = get_secret( + "FRIENDLI_AI_API_KEY" + ) or get_secret("FRIENDLI_TOKEN") if api_base is not None and not isinstance(api_base, str): raise Exception(