Support 'file' message type for VLLM video url's + Anthropic redacted message thinking support (#10129)

* feat(hosted_vllm/chat/transformation.py): support calling vllm video url with openai 'file' message type allows switching between gemini/vllm easily * [WIP] redacted thinking tests (#9044) * WIP: redacted thinking tests * test: add test for redacted thinking in assistant message --------- Co-authored-by: Krish Dholakia <krrishdholakia@gmail.com> * fix(anthropic/chat/transformation.py): support redacted thinking block on anthropic completion Fixes https://github.com/BerriAI/litellm/issues/9058 * fix(anthropic/chat/handler.py): transform anthropic redacted messages on streaming Fixes https://github.com/BerriAI/litellm/issues/9058 * fix(bedrock/): support redacted text on streaming + non-streaming Fixes https://github.com/BerriAI/litellm/issues/9058 * feat(litellm_proxy/chat/transformation.py): support 'reasoning_effort' param for proxy allows using reasoning effort with thinking models on proxy * test: update tests * fix(utils.py): fix linting error * fix: fix linting errors * fix: fix linting errors * fix: fix linting error * fix: fix linting errors * fix(anthropic/chat/transformation.py): fix returning citations in chat completion --------- Co-authored-by: Johann Miller <22018973+johannkm@users.noreply.github.com>
2025-04-26 03:04:13 +00:00 · 2025-04-19 11:16:37 -07:00 · 2025-04-19 11:16:37 -07:00 · f08a4e3c06
commit f08a4e3c06
parent 3c463f6715
20 changed files with 638 additions and 109 deletions
--- a/litellm/llms/hosted_vllm/chat/transformation.py
+++ b/litellm/llms/hosted_vllm/chat/transformation.py
@ -2,9 +2,19 @@
 Translate from OpenAI's `/v1/chat/completions` to VLLM's `/v1/chat/completions`
 """

-from typing import Optional, Tuple
+from typing import List, Optional, Tuple, cast

+from litellm.litellm_core_utils.prompt_templates.common_utils import (
+    _get_image_mime_type_from_url,
+)
+from litellm.litellm_core_utils.prompt_templates.factory import _parse_mime_type
 from litellm.secret_managers.main import get_secret_str
+from litellm.types.llms.openai import (
+    AllMessageValues,
+    ChatCompletionFileObject,
+    ChatCompletionVideoObject,
+    ChatCompletionVideoUrlObject,
+)

 from ....utils import _remove_additional_properties, _remove_strict_from_schema
 from ...openai.chat.gpt_transformation import OpenAIGPTConfig
@ -38,3 +48,71 @@ class HostedVLLMChatConfig(OpenAIGPTConfig):
            api_key or get_secret_str("HOSTED_VLLM_API_KEY") or "fake-api-key"
        )  # vllm does not require an api key
        return api_base, dynamic_api_key
+
+    def _is_video_file(self, content_item: ChatCompletionFileObject) -> bool:
+        """
+        Check if the file is a video
+
+        - format: video/<extension>
+        - file_data: base64 encoded video data
+        - file_id: infer mp4 from extension
+        """
+        file = content_item.get("file", {})
+        format = file.get("format")
+        file_data = file.get("file_data")
+        file_id = file.get("file_id")
+        if content_item.get("type") != "file":
+            return False
+        if format and format.startswith("video/"):
+            return True
+        elif file_data:
+            mime_type = _parse_mime_type(file_data)
+            if mime_type and mime_type.startswith("video/"):
+                return True
+        elif file_id:
+            mime_type = _get_image_mime_type_from_url(file_id)
+            if mime_type and mime_type.startswith("video/"):
+                return True
+        return False
+
+    def _convert_file_to_video_url(
+        self, content_item: ChatCompletionFileObject
+    ) -> ChatCompletionVideoObject:
+        file = content_item.get("file", {})
+        file_id = file.get("file_id")
+        file_data = file.get("file_data")
+
+        if file_id:
+            return ChatCompletionVideoObject(
+                type="video_url", video_url=ChatCompletionVideoUrlObject(url=file_id)
+            )
+        elif file_data:
+            return ChatCompletionVideoObject(
+                type="video_url", video_url=ChatCompletionVideoUrlObject(url=file_data)
+            )
+        raise ValueError("file_id or file_data is required")
+
+    def _transform_messages(
+        self, messages: List[AllMessageValues], model: str
+    ) -> List[AllMessageValues]:
+        """
+        Support translating video files from file_id or file_data to video_url
+        """
+        for message in messages:
+            if message["role"] == "user":
+                message_content = message.get("content")
+                if message_content and isinstance(message_content, list):
+                    replaced_content_items: List[
+                        Tuple[int, ChatCompletionFileObject]
+                    ] = []
+                    for idx, content_item in enumerate(message_content):
+                        if content_item.get("type") == "file":
+                            content_item = cast(ChatCompletionFileObject, content_item)
+                            if self._is_video_file(content_item):
+                                replaced_content_items.append((idx, content_item))
+                    for idx, content_item in replaced_content_items:
+                        message_content[idx] = self._convert_file_to_video_url(
+                            content_item
+                        )
+        transformed_messages = super()._transform_messages(messages, model)
+        return transformed_messages