Merge pull request #1441 from BerriAI/litellm_azure_vision_support_vision_enhancements

[Feat] Azure GPT vision support vision enhancements

litellm/tests/test_optional_params.py

@@ -36,3 +36,85 @@ def test_azure_optional_params_embeddings():
     )
     assert len(optional_params) == 1
     assert optional_params["user"] == "John"
+
+
+def test_azure_gpt_optional_params_gpt_vision():
+    # for OpenAI, Azure all extra params need to get passed as extra_body to OpenAI python. We assert we actually set extra_body here
+    optional_params = litellm.utils.get_optional_params(
+        user="John",
+        custom_llm_provider="azure",
+        max_tokens=10,
+        temperature=0.2,
+        enhancements={"ocr": {"enabled": True}, "grounding": {"enabled": True}},
+        dataSources=[
+            {
+                "type": "AzureComputerVision",
+                "parameters": {
+                    "endpoint": "<your_computer_vision_endpoint>",
+                    "key": "<your_computer_vision_key>",
+                },
+            }
+        ],
+    )
+
+    print(optional_params)
+    assert optional_params["max_tokens"] == 10
+    assert optional_params["temperature"] == 0.2
+    assert optional_params["extra_body"] == {
+        "enhancements": {"ocr": {"enabled": True}, "grounding": {"enabled": True}},
+        "dataSources": [
+            {
+                "type": "AzureComputerVision",
+                "parameters": {
+                    "endpoint": "<your_computer_vision_endpoint>",
+                    "key": "<your_computer_vision_key>",
+                },
+            }
+        ],
+    }
+
+
+# test_azure_gpt_optional_params_gpt_vision()
+
+
+def test_azure_gpt_optional_params_gpt_vision_with_extra_body():
+    # if user passes extra_body, we should not over write it, we should pass it along to OpenAI python
+    optional_params = litellm.utils.get_optional_params(
+        user="John",
+        custom_llm_provider="azure",
+        max_tokens=10,
+        temperature=0.2,
+        extra_body={
+            "meta": "hi",
+        },
+        enhancements={"ocr": {"enabled": True}, "grounding": {"enabled": True}},
+        dataSources=[
+            {
+                "type": "AzureComputerVision",
+                "parameters": {
+                    "endpoint": "<your_computer_vision_endpoint>",
+                    "key": "<your_computer_vision_key>",
+                },
+            }
+        ],
+    )
+
+    print(optional_params)
+    assert optional_params["max_tokens"] == 10
+    assert optional_params["temperature"] == 0.2
+    assert optional_params["extra_body"] == {
+        "enhancements": {"ocr": {"enabled": True}, "grounding": {"enabled": True}},
+        "dataSources": [
+            {
+                "type": "AzureComputerVision",
+                "parameters": {
+                    "endpoint": "<your_computer_vision_endpoint>",
+                    "key": "<your_computer_vision_key>",
+                },
+            }
+        ],
+        "meta": "hi",
+    }
+
+
+# test_azure_gpt_optional_params_gpt_vision_with_extra_body()

litellm/utils.py

@@ -4046,10 +4046,18 @@ def get_optional_params(
             optional_params["logprobs"] = logprobs
         if top_logprobs is not None:
             optional_params["top_logprobs"] = top_logprobs
-    # if user passed in non-default kwargs for specific providers/models, pass them along
+    if custom_llm_provider in ["openai", "azure"] + litellm.openai_compatible_providers:
-    for k in passed_params.keys():
+        # for openai, azure we should pass the extra/passed params within `extra_body` https://github.com/openai/openai-python/blob/ac33853ba10d13ac149b1fa3ca6dba7d613065c9/src/openai/resources/models.py#L46
-        if k not in default_params.keys():
+        extra_body = passed_params.pop("extra_body", {})
-            optional_params[k] = passed_params[k]
+        for k in passed_params.keys():
+            if k not in default_params.keys():
+                extra_body[k] = passed_params[k]
+        optional_params["extra_body"] = extra_body
+    else:
+        # if user passed in non-default kwargs for specific providers/models, pass them along
+        for k in passed_params.keys():
+            if k not in default_params.keys():
+                optional_params[k] = passed_params[k]
     return optional_params