diff --git a/litellm/__init__.py b/litellm/__init__.py index dcce13c97e..3ba2ffc18d 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -159,7 +159,8 @@ for key, value in model_cost.items(): openai_compatible_endpoints: List = [ "api.perplexity.ai", "api.endpoints.anyscale.com/v1", - "api.deepinfra.com/v1/openai" + "api.deepinfra.com/v1/openai", + "api.mistral.ai/v1" ] @@ -292,6 +293,7 @@ provider_list: List = [ "deepinfra", "perplexity", "anyscale", + "mistral", "maritalk", "custom", # custom apis ] diff --git a/litellm/main.py b/litellm/main.py index 7bc5f92962..973d213959 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -172,6 +172,7 @@ async def acompletion(*args, **kwargs): or custom_llm_provider == "azure" or custom_llm_provider == "custom_openai" or custom_llm_provider == "anyscale" + or custom_llm_provider == "mistral" or custom_llm_provider == "openrouter" or custom_llm_provider == "deepinfra" or custom_llm_provider == "perplexity" @@ -548,6 +549,7 @@ def completion( or custom_llm_provider == "deepinfra" or custom_llm_provider == "perplexity" or custom_llm_provider == "anyscale" + or custom_llm_provider == "mistral" or custom_llm_provider == "openai" or "ft:gpt-3.5-turbo" in model # finetune gpt-3.5-turbo ): # allow user to make an openai call with a custom base diff --git a/litellm/utils.py b/litellm/utils.py index af3b9f447f..f85f9fd5e4 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -2681,6 +2681,20 @@ def get_optional_params( # use the openai defaults optional_params["stream"] = stream if max_tokens: optional_params["max_tokens"] = max_tokens + elif custom_llm_provider == "mistral": + supported_params = ["temperature", "top_p", "stream", "max_tokens", "safe_mode", "random_seed"] + _check_valid_arg(supported_params=supported_params) + optional_params = non_default_params + if temperature is not None: + if temperature == 0 and model == "mistralai/Mistral-7B-Instruct-v0.1": # this model does no support temperature == 0 + temperature = 0.0001 # close to 0 + optional_params["temperature"] = temperature + if top_p is not None: + optional_params["top_p"] = top_p + if stream is not None: + optional_params["stream"] = stream + if max_tokens is not None: + optional_params["max_tokens"] = max_tokens else: # assume passing in params for openai/azure openai supported_params = ["functions", "function_call", "temperature", "top_p", "n", "stream", "stop", "max_tokens", "presence_penalty", "frequency_penalty", "logit_bias", "user", "response_format", "seed", "tools", "tool_choice", "max_retries"] _check_valid_arg(supported_params=supported_params) @@ -2751,6 +2765,10 @@ def get_llm_provider(model: str, custom_llm_provider: Optional[str] = None, api_ # deepinfra is openai compatible, we just need to set this to custom_openai and have the api_base be https://api.endpoints.anyscale.com/v1 api_base = "https://api.deepinfra.com/v1/openai" dynamic_api_key = get_secret("DEEPINFRA_API_KEY") + elif custom_llm_provider == "mistral": + # mistral is openai compatible, we just need to set this to custom_openai and have the api_base be https://api.mistral.ai + api_base = "https://api.mistral.ai/v1" + dynamic_api_key = get_secret("MISTRAL_API_KEY") return model, custom_llm_provider, dynamic_api_key, api_base # check if api base is a known openai compatible endpoint @@ -2766,6 +2784,9 @@ def get_llm_provider(model: str, custom_llm_provider: Optional[str] = None, api_ elif endpoint == "api.deepinfra.com/v1/openai": custom_llm_provider = "deepinfra" dynamic_api_key = get_secret("DEEPINFRA_API_KEY") + elif endpoint == "api.mistral.ai/v1": + custom_llm_provider = "mistral" + dynamic_api_key = get_secret("MISTRAL_API_KEY") return model, custom_llm_provider, dynamic_api_key, api_base # check if model in known model provider list -> for huggingface models, raise exception as they don't have a fixed provider (can be togetherai, anyscale, baseten, runpod, et.)