phoenix/litellm
1
0
Fork 0
forked from phoenix/litellm-mirror
Code Pull requests Activity

add friendli_ai provider

Browse source
This commit is contained in:
wslee 2024-06-10 17:27:15 +09:00
parent 90ae7f040a
commit fe8d59f5eb
6 changed files with 71 additions and 3 deletions
Download patch file Download diff file Expand all files Collapse all files

1
README.md
View file

@ -256,6 +256,7 @@ curl 'http://0.0.0.0:4000/key/generate' \
| [IBM - watsonx.ai](https://docs.litellm.ai/docs/providers/watsonx) | ✅ | ✅ | ✅ | ✅ | ✅ | |
| [voyage ai](https://docs.litellm.ai/docs/providers/voyage) | | | | | ✅ | |
| [xinference [Xorbits Inference]](https://docs.litellm.ai/docs/providers/xinference) | | | | | ✅ | |
| [FriendliAI](https://docs.litellm.ai/docs/providers/friendli) | ✅ | ✅ | ✅ | ✅ | | |
[**Read the Docs**](https://docs.litellm.ai/docs/)

53
docs/my-website/docs/providers/friendli_ai.md Normal file
View file

@ -0,0 +1,53 @@
# FriendliAI
https://friendli.ai/
**We support ALL FriendliAI models, just set `friendli_ai/` as a prefix when sending completion requests**
## API Key
```python
# env variable
os.environ['FRIENDLI_AI_API_KEY']
```
## Sample Usage
```python
from litellm import completion
import os
os.environ['FRIENDLI_AI_API_KEY'] = ""
response = completion(
model="friendli_ai/mixtral-8x7b-instruct-v0-1",
messages=[
{"role": "user", "content": "hello from litellm"}
],
)
print(response)
```
## Sample Usage - Streaming
```python
from litellm import completion
import os
os.environ['FRIENDLI_AI_API_KEY'] = ""
response = completion(
model="friendli_ai/mixtral-8x7b-instruct-v0-1",
messages=[
{"role": "user", "content": "hello from litellm"}
],
stream=True
)
for chunk in response:
print(chunk)
```
## Supported Models - ALL FriendliAI Models Supported!
We support ALL FriendliAI AI models, just set `friendli_ai/` as a prefix when sending completion requests
| Model Name | Function Call |
|--------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------|
| mixtral-8x7b-instruct | `completion(model="friendli_ai/mixtral-8x7b-instruct-v0-1", messages)` |
| llama3-8b-instruct | `completion(model="friendli_ai/meta-llama-3-8b-instruct", messages)` |
| llama3-70b-instruct | `completion(model="friendli_ai/meta-llama-3-70b-instruct", messages)` |

1
docs/my-website/sidebars.js
View file

@ -150,6 +150,7 @@ const sidebars = {
"providers/groq",
"providers/deepseek",
"providers/fireworks_ai",
"providers/friendli_ai",
"providers/vllm",
"providers/xinference",
"providers/cloudflare_workers",

3
litellm/__init__.py
View file

@ -392,6 +392,7 @@ openai_compatible_endpoints: List = [
"api.groq.com/openai/v1",
"api.deepseek.com/v1",
"api.together.xyz/v1",
"inference.friendli.ai/v1",
]
# this is maintained for Exception Mapping
@ -405,6 +406,7 @@ openai_compatible_providers: List = [
"xinference",
"together_ai",
"fireworks_ai",
"friendli_ai",
]
@ -628,6 +630,7 @@ provider_list: List = [
"cloudflare",
"xinference",
"fireworks_ai",
"friendli_ai",
"watsonx",
"triton",
"predibase",

6
litellm/main.py
View file

@ -1051,7 +1051,7 @@ def completion(
# note: if a user sets a custom base - we should ensure this works
# allow for the setting of dynamic and stateful api-bases
api_base = (
api_base # for deepinfra/perplexity/anyscale/groq we check in get_llm_provider and pass in the api base from there
api_base # for deepinfra/perplexity/anyscale/groq/friendli_ai we check in get_llm_provider and pass in the api base from there
or litellm.api_base
or get_secret("OPENAI_API_BASE")
or "https://api.openai.com/v1"
@ -1065,7 +1065,7 @@ def completion(
# set API KEY
api_key = (
api_key
or litellm.api_key # for deepinfra/perplexity/anyscale we check in get_llm_provider and pass in the api key from there
or litellm.api_key # for deepinfra/perplexity/anyscale/friendli_ai we check in get_llm_provider and pass in the api key from there
or litellm.openai_key
or get_secret("OPENAI_API_KEY")
)
@ -4288,7 +4288,7 @@ def speech(
response: Optional[HttpxBinaryResponseContent] = None
if custom_llm_provider == "openai":
api_base = (
api_base # for deepinfra/perplexity/anyscale/groq we check in get_llm_provider and pass in the api base from there
api_base # for deepinfra/perplexity/anyscale/groq/friendli we check in get_llm_provider and pass in the api base from there
or litellm.api_base
or get_secret("OPENAI_API_BASE")
or "https://api.openai.com/v1"

10
litellm/utils.py
View file

@ -6607,6 +6607,11 @@ def get_llm_provider(
or get_secret("TOGETHERAI_API_KEY")
or get_secret("TOGETHER_AI_TOKEN")
)
elif custom_llm_provider == "friendli_ai":
api_base = "https://inference.friendli.ai/v1"
dynamic_api_key = get_secret("FRIENDLI_AI_API_KEY") or get_secret(
"FRIENDLI_TOKEN"
)
if api_base is not None and not isinstance(api_base, str):
raise Exception(
"api base needs to be a string. api_base={}".format(api_base)
@ -6654,6 +6659,11 @@ def get_llm_provider(
elif endpoint == "api.deepseek.com/v1":
custom_llm_provider = "deepseek"
dynamic_api_key = get_secret("DEEPSEEK_API_KEY")
elif endpoint == "inference.friendli.ai/v1":
custom_llm_provider = "friendli_ai"
dynamic_api_key = get_secret(
"FRIENDLI_AI_API_KEY"
) or get_secret("FRIENDLI_TOKEN")
if api_base is not None and not isinstance(api_base, str):
raise Exception(