diff --git a/docs/my-website/docs/completion/input.md b/docs/my-website/docs/completion/input.md index e844c541c..6ad412af8 100644 --- a/docs/my-website/docs/completion/input.md +++ b/docs/my-website/docs/completion/input.md @@ -39,38 +39,34 @@ This is a list of openai params we translate across providers. Use `litellm.get_supported_openai_params()` for an updated list of params for each model + provider -| Provider | temperature | max_tokens | top_p | stream | stop | n | presence_penalty | frequency_penalty | functions | function_call | logit_bias | user | response_format | seed | tools | tool_choice | logprobs | top_logprobs | extra_headers | -|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|--| -|Anthropic| ✅ | ✅ | ✅ | ✅ | ✅ | | | | | | |✅ | ✅ | ✅ | ✅ | ✅ | | | ✅ -|OpenAI| ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |✅ | ✅ | ✅ | ✅ |✅ | ✅ | ✅ | ✅ | ✅ | -|Azure OpenAI| ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |✅ | ✅ | ✅ | ✅ |✅ | ✅ | | | ✅ | +| Provider | temperature | max_tokens | top_p | stream | stream_options | stop | n | presence_penalty | frequency_penalty | functions | function_call | logit_bias | user | response_format | seed | tools | tool_choice | logprobs | top_logprobs | extra_headers | +|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---| +|Anthropic| ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | | | | | | |✅ | ✅ | ✅ | ✅ | ✅ | | | ✅ | +|OpenAI| ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |✅ | ✅ | ✅ | ✅ |✅ | ✅ | ✅ | ✅ | ✅ | +|Azure OpenAI| ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |✅ | ✅ | ✅ | ✅ |✅ | ✅ | | | ✅ | |Replicate | ✅ | ✅ | ✅ | ✅ | ✅ | | | | | | -|Anyscale | ✅ | ✅ | ✅ | ✅ | +|Anyscale | ✅ | ✅ | ✅ | ✅ | ✅ | |Cohere| ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | | | -|Huggingface| ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | | | | | +|Huggingface| ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | | | | |Openrouter| ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | | | | | ✅ | | | | | |AI21| ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | | | -|VertexAI| ✅ | ✅ | | ✅ | | | | | | | | | | | ✅ | | | +|VertexAI| ✅ | ✅ | | ✅ | ✅ | | | | | | | | | | ✅ | | | |Bedrock| ✅ | ✅ | ✅ | ✅ | ✅ | | | | | | | | | | ✅ (for anthropic) | | -|Sagemaker| ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | | | | | +|Sagemaker| ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | | | | |TogetherAI| ✅ | ✅ | ✅ | ✅ | ✅ | | | | | | ✅ | -|AlephAlpha| ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | | | | | -|Palm| ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | | | | | -|NLP Cloud| ✅ | ✅ | ✅ | ✅ | ✅ | | | | | | -|Petals| ✅ | ✅ | | ✅ | | | | | | | +|AlephAlpha| ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | | | | +|Palm| ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | | | | +|NLP Cloud| ✅ | ✅ | ✅ | ✅ | ✅ | | | | | | +|Petals| ✅ | ✅ | | ✅ | ✅ | | | | | | |Ollama| ✅ | ✅ | ✅ | ✅ | ✅ | | | ✅ | | | | | ✅ | | | |Databricks| ✅ | ✅ | ✅ | ✅ | ✅ | | | | | | | | | | | -|ClarifAI| ✅ | ✅ | | | | | | | | | | | | | | - +|ClarifAI| ✅ | ✅ | |✅ | ✅ | | | | | | | | | | | :::note By default, LiteLLM raises an exception if the openai param being passed in isn't supported. -To drop the param instead, set `litellm.drop_params = True`. +To drop the param instead, set `litellm.drop_params = True` or `completion(..drop_params=True)`. -**For function calling:** - -Add to prompt for non-openai models, set: `litellm.add_function_to_prompt = True`. ::: ## Input Params diff --git a/docs/my-website/docs/enterprise.md b/docs/my-website/docs/enterprise.md index 84804aa5f..0d57b4c25 100644 --- a/docs/my-website/docs/enterprise.md +++ b/docs/my-website/docs/enterprise.md @@ -36,7 +36,11 @@ Includes all enterprise features. Professional Support can assist with LLM/Provider integrations, deployment, upgrade management, and LLM Provider troubleshooting. We can’t solve your own infrastructure-related issues but we will guide you to fix them. -We offer custom SLAs based on your needs and the severity of the issue. The standard SLA is 6 hours for Sev0-Sev1 severity and 24h for Sev2-Sev3 between 7am – 7pm PT (Monday through Saturday). +- 1 hour for Sev0 issues +- 6 hours for Sev1 +- 24h for Sev2-Sev3 between 7am – 7pm PT (Monday through Saturday) + +**We can offer custom SLAs** based on your needs and the severity of the issue ### What’s the cost of the Self-Managed Enterprise edition? diff --git a/docs/my-website/docs/proxy/configs.md b/docs/my-website/docs/proxy/configs.md index 5eeb05f36..2552c2004 100644 --- a/docs/my-website/docs/proxy/configs.md +++ b/docs/my-website/docs/proxy/configs.md @@ -80,6 +80,13 @@ For more provider-specific info, [go here](../providers/) $ litellm --config /path/to/config.yaml ``` +:::tip + +Run with `--detailed_debug` if you need detailed debug logs + +```shell +$ litellm --config /path/to/config.yaml --detailed_debug +::: ### Using Proxy - Curl Request, OpenAI Package, Langchain, Langchain JS Calling a model group diff --git a/docs/my-website/docs/proxy/quick_start.md b/docs/my-website/docs/proxy/quick_start.md index 050d9b598..4ee4d8831 100644 --- a/docs/my-website/docs/proxy/quick_start.md +++ b/docs/my-website/docs/proxy/quick_start.md @@ -24,6 +24,15 @@ $ litellm --model huggingface/bigcode/starcoder #INFO: Proxy running on http://0.0.0.0:4000 ``` + +:::info + +Run with `--detailed_debug` if you need detailed debug logs + +```shell +$ litellm --model huggingface/bigcode/starcoder --detailed_debug +::: + ### Test In a new shell, run, this will make an `openai.chat.completions` request. Ensure you're using openai v1.0.0+ ```shell diff --git a/litellm/llms/azure.py b/litellm/llms/azure.py index f0279d0d7..834fcbea9 100644 --- a/litellm/llms/azure.py +++ b/litellm/llms/azure.py @@ -144,7 +144,7 @@ class AzureOpenAIConfig: "tools", "tool_choice", "top_p", - "log_probs", + "logprobs", "top_logprobs", "response_format", "seed", @@ -157,6 +157,7 @@ class AzureOpenAIConfig: optional_params: dict, model: str, api_version: str, # Y-M-D-{optional} + drop_params, ) -> dict: supported_openai_params = self.get_supported_openai_params() @@ -181,7 +182,11 @@ class AzureOpenAIConfig: and api_version_day < "01" ) ): - if litellm.drop_params == False: + if litellm.drop_params == True or ( + drop_params is not None and drop_params == True + ): + pass + else: raise UnsupportedParamsError( status_code=400, message=f"""Azure does not support 'tool_choice', for api_version={api_version}. Bump your API version to '2023-12-01-preview' or later. This parameter requires 'api_version="2023-12-01-preview"' or later. Azure API Reference: https://learn.microsoft.com/en-us/azure/ai-services/openai/reference#chat-completions""", @@ -189,7 +194,11 @@ class AzureOpenAIConfig: elif value == "required" and ( api_version_year == "2024" and api_version_month <= "05" ): ## check if tool_choice value is supported ## - if litellm.drop_params == False: + if litellm.drop_params == True or ( + drop_params is not None and drop_params == True + ): + pass + else: raise UnsupportedParamsError( status_code=400, message=f"Azure does not support '{value}' as a {param} param, for api_version={api_version}. To drop 'tool_choice=required' for calls with this Azure API version, set `litellm.drop_params=True` or for proxy:\n\n`litellm_settings:\n drop_params: true`\nAzure API Reference: https://learn.microsoft.com/en-us/azure/ai-services/openai/reference#chat-completions", diff --git a/litellm/proxy/_super_secret_config.yaml b/litellm/proxy/_super_secret_config.yaml index fd40b6544..3bc516604 100644 --- a/litellm/proxy/_super_secret_config.yaml +++ b/litellm/proxy/_super_secret_config.yaml @@ -28,7 +28,8 @@ model_list: api_key: os.environ/AZURE_API_KEY api_version: '2023-05-15' model: azure/chatgpt-v-2 - model_name: gpt-3.5-turbo + drop_params: True + model_name: gpt-3.5-turbo-drop-params - model_name: tts litellm_params: model: openai/tts-1 diff --git a/litellm/tests/test_completion.py b/litellm/tests/test_completion.py index 4aedaeb39..47c55ca4f 100644 --- a/litellm/tests/test_completion.py +++ b/litellm/tests/test_completion.py @@ -2160,6 +2160,7 @@ def test_completion_azure_key_completion_arg(): model="azure/chatgpt-v-2", messages=messages, api_key=old_key, + logprobs=True, max_tokens=10, ) print(f"response: {response}") diff --git a/litellm/tests/test_optional_params.py b/litellm/tests/test_optional_params.py index 737d488fd..0741b6fbe 100644 --- a/litellm/tests/test_optional_params.py +++ b/litellm/tests/test_optional_params.py @@ -9,6 +9,7 @@ from litellm.utils import get_optional_params_embeddings, get_optional_params from litellm.llms.prompt_templates.factory import ( map_system_message_pt, ) +from unittest.mock import patch, MagicMock from litellm.types.completion import ( ChatCompletionUserMessageParam, ChatCompletionSystemMessageParam, @@ -243,3 +244,45 @@ def test_azure_tool_choice(api_version): ), "tool_choice={} for api version={}".format( optional_params["tool_choice"], api_version ) + + +@pytest.mark.parametrize("drop_params", [True, False, None]) +def test_dynamic_drop_params(drop_params): + """ + Make a call to cohere w/ drop params = True vs. false. + """ + if drop_params == True: + optional_params = litellm.utils.get_optional_params( + model="command-r", + custom_llm_provider="cohere", + response_format="json", + drop_params=drop_params, + ) + else: + try: + optional_params = litellm.utils.get_optional_params( + model="command-r", + custom_llm_provider="cohere", + response_format="json", + drop_params=drop_params, + ) + pytest.fail("Expected to fail") + except Exception as e: + pass + + +def test_dynamic_drop_params_e2e(): + with patch("requests.post", new=MagicMock()) as mock_response: + try: + response = litellm.completion( + model="command-r", + messages=[{"role": "user", "content": "Hey, how's it going?"}], + response_format={"key": "value"}, + drop_params=True, + ) + except Exception as e: + pass + + mock_response.assert_called_once() + print(mock_response.call_args.kwargs["data"]) + assert "response_format" not in mock_response.call_args.kwargs["data"] diff --git a/litellm/types/router.py b/litellm/types/router.py index 4398ab24f..3dbc5e75d 100644 --- a/litellm/types/router.py +++ b/litellm/types/router.py @@ -283,12 +283,6 @@ class updateDeployment(BaseModel): class LiteLLMParamsTypedDict(TypedDict, total=False): - """ - [TODO] - - allow additional params (not in list) - - set value to none if not set -> don't raise error if value not set - """ - model: str custom_llm_provider: Optional[str] tpm: Optional[int] @@ -300,6 +294,8 @@ class LiteLLMParamsTypedDict(TypedDict, total=False): stream_timeout: Optional[Union[float, str]] max_retries: Optional[int] organization: Optional[str] # for openai orgs + ## DROP PARAMS ## + drop_params: Optional[bool] ## UNIFIED PROJECT/REGION ## region_name: Optional[str] ## VERTEX AI ## diff --git a/litellm/utils.py b/litellm/utils.py index 6ae62b417..7aea67341 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -5173,6 +5173,7 @@ def get_optional_params( top_logprobs=None, extra_headers=None, api_version=None, + drop_params=None, **kwargs, ): # retrieve all parameters passed to the function @@ -5244,6 +5245,7 @@ def get_optional_params( "top_logprobs": None, "extra_headers": None, "api_version": None, + "drop_params": None, } # filter out those parameters that were passed with non-default values non_default_params = { @@ -5253,6 +5255,7 @@ def get_optional_params( k != "model" and k != "custom_llm_provider" and k != "api_version" + and k != "drop_params" and k in default_params and v != default_params[k] ) @@ -5335,11 +5338,16 @@ def get_optional_params( # Always keeps this in elif code blocks else: unsupported_params[k] = non_default_params[k] - if unsupported_params and not litellm.drop_params: - raise UnsupportedParamsError( - status_code=500, - message=f"{custom_llm_provider} does not support parameters: {unsupported_params}, for model={model}. To drop these, set `litellm.drop_params=True` or for proxy:\n\n`litellm_settings:\n drop_params: true`\n", - ) + if unsupported_params: + if litellm.drop_params == True or ( + drop_params is not None and drop_params == True + ): + pass + else: + raise UnsupportedParamsError( + status_code=500, + message=f"{custom_llm_provider} does not support parameters: {unsupported_params}, for model={model}. To drop these, set `litellm.drop_params=True` or for proxy:\n\n`litellm_settings:\n drop_params: true`\n", + ) def _map_and_modify_arg(supported_params: dict, provider: str, model: str): """ @@ -6040,6 +6048,7 @@ def get_optional_params( optional_params=optional_params, model=model, api_version=api_version, # type: ignore + drop_params=drop_params, ) else: # assume passing in params for text-completion openai supported_params = get_supported_openai_params(