diff --git a/README.md b/README.md index 6d26e92c2..7df894ea1 100644 --- a/README.md +++ b/README.md @@ -238,6 +238,7 @@ curl 'http://0.0.0.0:4000/key/generate' \ | [cloudflare AI Workers](https://docs.litellm.ai/docs/providers/cloudflare_workers) | ✅ | ✅ | ✅ | ✅ | | | | [cohere](https://docs.litellm.ai/docs/providers/cohere) | ✅ | ✅ | ✅ | ✅ | ✅ | | | [anthropic](https://docs.litellm.ai/docs/providers/anthropic) | ✅ | ✅ | ✅ | ✅ | | | +| [empower](https://docs.litellm.ai/docs/providers/empower) | ✅ | ✅ | ✅ | ✅ | | [huggingface](https://docs.litellm.ai/docs/providers/huggingface) | ✅ | ✅ | ✅ | ✅ | ✅ | | | [replicate](https://docs.litellm.ai/docs/providers/replicate) | ✅ | ✅ | ✅ | ✅ | | | | [together_ai](https://docs.litellm.ai/docs/providers/togetherai) | ✅ | ✅ | ✅ | ✅ | | | diff --git a/docs/my-website/docs/providers/empower.md b/docs/my-website/docs/providers/empower.md new file mode 100644 index 000000000..59df44cc9 --- /dev/null +++ b/docs/my-website/docs/providers/empower.md @@ -0,0 +1,89 @@ +# Empower +LiteLLM supports all models on Empower. + +## API Keys + +```python +import os +os.environ["EMPOWER_API_KEY"] = "your-api-key" +``` +## Example Usage + +```python +from litellm import completion +import os + +os.environ["EMPOWER_API_KEY"] = "your-api-key" + +messages = [{"role": "user", "content": "Write me a poem about the blue sky"}] + +response = completion(model="empower/empower-functions", messages=messages) +print(response) +``` + +## Example Usage - Streaming +```python +from litellm import completion +import os + +os.environ["EMPOWER_API_KEY"] = "your-api-key" + +messages = [{"role": "user", "content": "Write me a poem about the blue sky"}] + +response = completion(model="empower/empower-functions", messages=messages, streaming=True) +for chunk in response: + print(chunk['choices'][0]['delta']) + +``` + +## Example Usage - Automatic Tool Calling + +```python +from litellm import completion +import os + +os.environ["EMPOWER_API_KEY"] = "your-api-key" + +messages = [{"role": "user", "content": "What's the weather like in San Francisco, Tokyo, and Paris?"}] +tools = [ + { + "type": "function", + "function": { + "name": "get_current_weather", + "description": "Get the current weather in a given location", + "parameters": { + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "The city and state, e.g. San Francisco, CA", + }, + "unit": {"type": "string", "enum": ["celsius", "fahrenheit"]}, + }, + "required": ["location"], + }, + }, + } +] + +response = completion( + model="empower/empower-functions-small", + messages=messages, + tools=tools, + tool_choice="auto", # auto is default, but we'll be explicit +) +print("\nLLM Response:\n", response) +``` + +## Empower Models +liteLLM supports `non-streaming` and `streaming` requests to all models on https://empower.dev/ + +Example Empower Usage - Note: liteLLM supports all models deployed on Empower + + +### Empower LLMs - Automatic Tool Using models +| Model Name | Function Call | Required OS Variables | +|-----------------------------------|------------------------------------------------------------------------|---------------------------------| +| empower/empower-functions | `completion('empower/empower-functions', messages)` | `os.environ['TOGETHERAI_API_KEY']` | +| empower/empower-functions-small | `completion('empower/empower-functions-small', messages)` | `os.environ['TOGETHERAI_API_KEY']` | + diff --git a/litellm/__init__.py b/litellm/__init__.py index d018e7e6c..f0dab5e29 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -334,6 +334,7 @@ cohere_models: List = [] cohere_chat_models: List = [] mistral_chat_models: List = [] anthropic_models: List = [] +empower_models: List = [] openrouter_models: List = [] vertex_language_models: List = [] vertex_vision_models: List = [] @@ -363,7 +364,9 @@ for key, value in model_cost.items(): elif value.get("litellm_provider") == "mistral": mistral_chat_models.append(key) elif value.get("litellm_provider") == "anthropic": - anthropic_models.append(key) + anthropic_models.append(key) + elif value.get("litellm_provider") == "empower": + empower_models.append(key) elif value.get("litellm_provider") == "openrouter": openrouter_models.append(key) elif value.get("litellm_provider") == "vertex_ai-text-models": @@ -411,6 +414,7 @@ openai_compatible_endpoints: List = [ "https://integrate.api.nvidia.com/v1", "api.deepseek.com/v1", "api.together.xyz/v1", + "app.empower.dev/api/v1", "inference.friendli.ai/v1", ] @@ -428,6 +432,7 @@ openai_compatible_providers: List = [ "xinference", "together_ai", "fireworks_ai", + "empower", "friendliai", "azure_ai", ] @@ -530,6 +535,10 @@ huggingface_models: List = [ "meta-llama/Llama-2-70b", "meta-llama/Llama-2-70b-chat", ] # these have been tested on extensively. But by default all text2text-generation and text-generation models are supported by liteLLM. - https://docs.litellm.ai/docs/providers +empower_models = [ + "empower/empower-functions", + "empower/empower-functions-small", +] together_ai_models: List = [ # llama llms - chat @@ -665,6 +674,7 @@ provider_list: List = [ "triton", "predibase", "databricks", + "empower", "custom", # custom apis ] diff --git a/litellm/tests/test_completion.py b/litellm/tests/test_completion.py index 0598c52df..59d5db034 100644 --- a/litellm/tests/test_completion.py +++ b/litellm/tests/test_completion.py @@ -281,6 +281,34 @@ def test_completion_claude(): # test_completion_claude() +def test_completion_empower(): + litellm.set_verbose = True + messages = [ + { + "role": "user", + "content": "\nWhat is the query for `console.log` => `console.error`\n", + }, + { + "role": "assistant", + "content": "\nThis is the GritQL query for the given before/after examples:\n\n`console.log` => `console.error`\n\n", + }, + { + "role": "user", + "content": "\nWhat is the query for `console.info` => `consdole.heaven`\n", + }, + ] + try: + # test without max tokens + response = completion( + model="empower/empower-functions-small", + messages=messages, + ) + # Add any assertions, here to check response args + print(response) + except Exception as e: + pytest.fail(f"Error occurred: {e}") + + def test_completion_claude_3_empty_response(): litellm.set_verbose = True diff --git a/litellm/utils.py b/litellm/utils.py index 2d680feb2..cf2c679a8 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -4262,6 +4262,9 @@ def get_llm_provider( # deepinfra is openai compatible, we just need to set this to custom_openai and have the api_base be https://api.endpoints.anyscale.com/v1 api_base = "https://api.deepinfra.com/v1/openai" dynamic_api_key = get_secret("DEEPINFRA_API_KEY") + elif custom_llm_provider == "empower": + api_base = "https://app.empower.dev/api/v1" + dynamic_api_key = get_secret("EMPOWER_API_KEY") elif custom_llm_provider == "groq": # groq is openai compatible, we just need to set this to custom_openai and have the api_base be https://api.groq.com/openai/v1 api_base = "https://api.groq.com/openai/v1" @@ -4384,6 +4387,9 @@ def get_llm_provider( elif endpoint == "https://codestral.mistral.ai/v1": custom_llm_provider = "text-completion-codestral" dynamic_api_key = get_secret("CODESTRAL_API_KEY") + elif endpoint == "app.empower.dev/api/v1": + custom_llm_provider = "empower" + dynamic_api_key = get_secret("EMPOWER_API_KEY") elif endpoint == "api.deepseek.com/v1": custom_llm_provider = "deepseek" dynamic_api_key = get_secret("DEEPSEEK_API_KEY") @@ -4480,6 +4486,8 @@ def get_llm_provider( # openai embeddings elif model in litellm.open_ai_embedding_models: custom_llm_provider = "openai" + elif model in litellm.empower_models: + custom_llm_provider = "empower" if custom_llm_provider is None or custom_llm_provider == "": if litellm.suppress_debug_info == False: print() # noqa @@ -8118,7 +8126,7 @@ class CustomStreamWrapper: if chunk.startswith(self.complete_response): # Remove last_sent_chunk only if it appears at the start of the new chunk - chunk = chunk[len(self.complete_response) :] + chunk = chunk[len(self.complete_response):] self.complete_response += chunk return chunk @@ -10116,7 +10124,7 @@ def mock_completion_streaming_obj( model_response, mock_response, model, n: Optional[int] = None ): for i in range(0, len(mock_response), 3): - completion_obj = Delta(role="assistant", content=mock_response[i : i + 3]) + completion_obj = Delta(role="assistant", content=mock_response[i: i + 3]) if n is None: model_response.choices[0].delta = completion_obj else: @@ -10125,7 +10133,7 @@ def mock_completion_streaming_obj( _streaming_choice = litellm.utils.StreamingChoices( index=j, delta=litellm.utils.Delta( - role="assistant", content=mock_response[i : i + 3] + role="assistant", content=mock_response[i: i + 3] ), ) _all_choices.append(_streaming_choice) @@ -10137,7 +10145,7 @@ async def async_mock_completion_streaming_obj( model_response, mock_response, model, n: Optional[int] = None ): for i in range(0, len(mock_response), 3): - completion_obj = Delta(role="assistant", content=mock_response[i : i + 3]) + completion_obj = Delta(role="assistant", content=mock_response[i: i + 3]) if n is None: model_response.choices[0].delta = completion_obj else: @@ -10146,7 +10154,7 @@ async def async_mock_completion_streaming_obj( _streaming_choice = litellm.utils.StreamingChoices( index=j, delta=litellm.utils.Delta( - role="assistant", content=mock_response[i : i + 3] + role="assistant", content=mock_response[i: i + 3] ), ) _all_choices.append(_streaming_choice)