(patch) using image_urls with vertex/anthropic models (#6775)

* fix is_pdf_used check anthropic

* fix validate_environment

* add comment on is_vertex_request

litellm/llms/anthropic/chat/handler.py

@@ -60,6 +60,7 @@ def validate_environment(
     user_headers,
     model,
     messages: List[AllMessageValues],
+    is_vertex_request: bool,
     tools: Optional[List[AllAnthropicToolsValues]],
     anthropic_version: Optional[str] = None,
 ):
@@ -80,6 +81,7 @@ def validate_environment(
         prompt_caching_set=prompt_caching_set,
         pdf_used=pdf_used,
         api_key=api_key,
+        is_vertex_request=is_vertex_request,
     )
 
     if user_headers is not None and isinstance(user_headers, dict):
@@ -486,19 +488,20 @@ class AnthropicChatCompletion(BaseLLM):
         headers={},
         client=None,
     ):
+        optional_params = copy.deepcopy(optional_params)
+        stream = optional_params.pop("stream", None)
+        json_mode: bool = optional_params.pop("json_mode", False)
+        is_vertex_request: bool = optional_params.pop("is_vertex_request", False)
+        _is_function_call = False
+        messages = copy.deepcopy(messages)
         headers = validate_environment(
             api_key,
             headers,
             model,
             messages=messages,
             tools=optional_params.get("tools"),
+            is_vertex_request=is_vertex_request,
         )
-        _is_function_call = False
-        messages = copy.deepcopy(messages)
-        optional_params = copy.deepcopy(optional_params)
-        stream = optional_params.pop("stream", None)
-        json_mode: bool = optional_params.pop("json_mode", False)
-        is_vertex_request: bool = optional_params.pop("is_vertex_request", False)
 
         data = AnthropicConfig()._transform_request(
             model=model,

litellm/llms/anthropic/chat/transformation.py

@@ -107,6 +107,7 @@ class AnthropicConfig:
         computer_tool_used: bool = False,
         prompt_caching_set: bool = False,
         pdf_used: bool = False,
+        is_vertex_request: bool = False,
     ) -> dict:
         import json
 
@@ -123,8 +124,13 @@ class AnthropicConfig:
             "accept": "application/json",
             "content-type": "application/json",
         }
-        if len(betas) > 0:
+
+        # Don't send any beta headers to Vertex, Vertex has failed requests when they are sent
+        if is_vertex_request is True:
+            pass
+        elif len(betas) > 0:
             headers["anthropic-beta"] = ",".join(betas)
+
         return headers
 
     def _map_tool_choice(
@@ -403,6 +409,7 @@ class AnthropicConfig:
     def is_pdf_used(self, messages: List[AllMessageValues]) -> bool:
         """
         Set to true if media passed into messages.
+
         """
         for message in messages:
             if (