diff --git a/litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py b/litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py index a0a99f580b..5a8319a747 100644 --- a/litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py +++ b/litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py @@ -14,6 +14,7 @@ from litellm.types.llms.openai import ChatCompletionThinkingBlock from litellm.types.utils import ( ChatCompletionDeltaToolCall, ChatCompletionMessageToolCall, + ChatCompletionRedactedThinkingBlock, Choices, Delta, EmbeddingResponse, @@ -486,7 +487,14 @@ def convert_to_model_response_object( # noqa: PLR0915 ) # Handle thinking models that display `thinking_blocks` within `content` - thinking_blocks: Optional[List[ChatCompletionThinkingBlock]] = None + thinking_blocks: Optional[ + List[ + Union[ + ChatCompletionThinkingBlock, + ChatCompletionRedactedThinkingBlock, + ] + ] + ] = None if "thinking_blocks" in choice["message"]: thinking_blocks = choice["message"]["thinking_blocks"] provider_specific_fields["thinking_blocks"] = thinking_blocks diff --git a/litellm/litellm_core_utils/prompt_templates/common_utils.py b/litellm/litellm_core_utils/prompt_templates/common_utils.py index 2fe99fb27b..40cd4e286b 100644 --- a/litellm/litellm_core_utils/prompt_templates/common_utils.py +++ b/litellm/litellm_core_utils/prompt_templates/common_utils.py @@ -471,3 +471,59 @@ def unpack_defs(schema, defs): unpack_defs(ref, defs) value["items"] = ref continue + + +def _get_image_mime_type_from_url(url: str) -> Optional[str]: + """ + Get mime type for common image URLs + See gemini mime types: https://cloud.google.com/vertex-ai/generative-ai/docs/multimodal/image-understanding#image-requirements + + Supported by Gemini: + application/pdf + audio/mpeg + audio/mp3 + audio/wav + image/png + image/jpeg + image/webp + text/plain + video/mov + video/mpeg + video/mp4 + video/mpg + video/avi + video/wmv + video/mpegps + video/flv + """ + url = url.lower() + + # Map file extensions to mime types + mime_types = { + # Images + (".jpg", ".jpeg"): "image/jpeg", + (".png",): "image/png", + (".webp",): "image/webp", + # Videos + (".mp4",): "video/mp4", + (".mov",): "video/mov", + (".mpeg", ".mpg"): "video/mpeg", + (".avi",): "video/avi", + (".wmv",): "video/wmv", + (".mpegps",): "video/mpegps", + (".flv",): "video/flv", + # Audio + (".mp3",): "audio/mp3", + (".wav",): "audio/wav", + (".mpeg",): "audio/mpeg", + # Documents + (".pdf",): "application/pdf", + (".txt",): "text/plain", + } + + # Check each extension group against the URL + for extensions, mime_type in mime_types.items(): + if any(url.endswith(ext) for ext in extensions): + return mime_type + + return None diff --git a/litellm/litellm_core_utils/prompt_templates/factory.py b/litellm/litellm_core_utils/prompt_templates/factory.py index aa5dc0d49a..5b11b224bb 100644 --- a/litellm/litellm_core_utils/prompt_templates/factory.py +++ b/litellm/litellm_core_utils/prompt_templates/factory.py @@ -2258,6 +2258,14 @@ def _parse_content_type(content_type: str) -> str: return m.get_content_type() +def _parse_mime_type(base64_data: str) -> Optional[str]: + mime_type_match = re.match(r"data:(.*?);base64", base64_data) + if mime_type_match: + return mime_type_match.group(1) + else: + return None + + class BedrockImageProcessor: """Handles both sync and async image processing for Bedrock conversations.""" diff --git a/litellm/llms/anthropic/chat/handler.py b/litellm/llms/anthropic/chat/handler.py index ebb8650044..397aa1e047 100644 --- a/litellm/llms/anthropic/chat/handler.py +++ b/litellm/llms/anthropic/chat/handler.py @@ -29,6 +29,7 @@ from litellm.types.llms.anthropic import ( UsageDelta, ) from litellm.types.llms.openai import ( + ChatCompletionRedactedThinkingBlock, ChatCompletionThinkingBlock, ChatCompletionToolCallChunk, ) @@ -501,18 +502,19 @@ class ModelResponseIterator: ) -> Tuple[ str, Optional[ChatCompletionToolCallChunk], - List[ChatCompletionThinkingBlock], + List[Union[ChatCompletionThinkingBlock, ChatCompletionRedactedThinkingBlock]], Dict[str, Any], ]: """ Helper function to handle the content block delta """ - text = "" tool_use: Optional[ChatCompletionToolCallChunk] = None provider_specific_fields = {} content_block = ContentBlockDelta(**chunk) # type: ignore - thinking_blocks: List[ChatCompletionThinkingBlock] = [] + thinking_blocks: List[ + Union[ChatCompletionThinkingBlock, ChatCompletionRedactedThinkingBlock] + ] = [] self.content_blocks.append(content_block) if "text" in content_block["delta"]: @@ -541,20 +543,25 @@ class ModelResponseIterator: ) ] provider_specific_fields["thinking_blocks"] = thinking_blocks + return text, tool_use, thinking_blocks, provider_specific_fields def _handle_reasoning_content( - self, thinking_blocks: List[ChatCompletionThinkingBlock] + self, + thinking_blocks: List[ + Union[ChatCompletionThinkingBlock, ChatCompletionRedactedThinkingBlock] + ], ) -> Optional[str]: """ Handle the reasoning content """ reasoning_content = None for block in thinking_blocks: + thinking_content = cast(Optional[str], block.get("thinking")) if reasoning_content is None: reasoning_content = "" - if "thinking" in block: - reasoning_content += block["thinking"] + if thinking_content is not None: + reasoning_content += thinking_content return reasoning_content def chunk_parser(self, chunk: dict) -> ModelResponseStream: @@ -567,7 +574,13 @@ class ModelResponseIterator: usage: Optional[Usage] = None provider_specific_fields: Dict[str, Any] = {} reasoning_content: Optional[str] = None - thinking_blocks: Optional[List[ChatCompletionThinkingBlock]] = None + thinking_blocks: Optional[ + List[ + Union[ + ChatCompletionThinkingBlock, ChatCompletionRedactedThinkingBlock + ] + ] + ] = None index = int(chunk.get("index", 0)) if type_chunk == "content_block_delta": @@ -605,6 +618,15 @@ class ModelResponseIterator: }, "index": self.tool_index, } + elif ( + content_block_start["content_block"]["type"] == "redacted_thinking" + ): + thinking_blocks = [ + ChatCompletionRedactedThinkingBlock( + type="redacted_thinking", + data=content_block_start["content_block"]["data"], + ) + ] elif type_chunk == "content_block_stop": ContentBlockStop(**chunk) # type: ignore # check if tool call content block diff --git a/litellm/llms/anthropic/chat/transformation.py b/litellm/llms/anthropic/chat/transformation.py index 2bf9d0d992..06e0553f8d 100644 --- a/litellm/llms/anthropic/chat/transformation.py +++ b/litellm/llms/anthropic/chat/transformation.py @@ -30,6 +30,7 @@ from litellm.types.llms.openai import ( REASONING_EFFORT, AllMessageValues, ChatCompletionCachedContent, + ChatCompletionRedactedThinkingBlock, ChatCompletionSystemMessage, ChatCompletionThinkingBlock, ChatCompletionToolCallChunk, @@ -575,13 +576,21 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): ) -> Tuple[ str, Optional[List[Any]], - Optional[List[ChatCompletionThinkingBlock]], + Optional[ + List[ + Union[ChatCompletionThinkingBlock, ChatCompletionRedactedThinkingBlock] + ] + ], Optional[str], List[ChatCompletionToolCallChunk], ]: text_content = "" citations: Optional[List[Any]] = None - thinking_blocks: Optional[List[ChatCompletionThinkingBlock]] = None + thinking_blocks: Optional[ + List[ + Union[ChatCompletionThinkingBlock, ChatCompletionRedactedThinkingBlock] + ] + ] = None reasoning_content: Optional[str] = None tool_calls: List[ChatCompletionToolCallChunk] = [] for idx, content in enumerate(completion_response["content"]): @@ -600,20 +609,30 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): index=idx, ) ) - ## CITATIONS - if content.get("citations", None) is not None: - if citations is None: - citations = [] - citations.append(content["citations"]) - if content.get("thinking", None) is not None: + + elif content.get("thinking", None) is not None: if thinking_blocks is None: thinking_blocks = [] thinking_blocks.append(cast(ChatCompletionThinkingBlock, content)) + elif content["type"] == "redacted_thinking": + if thinking_blocks is None: + thinking_blocks = [] + thinking_blocks.append( + cast(ChatCompletionRedactedThinkingBlock, content) + ) + + ## CITATIONS + if content.get("citations") is not None: + if citations is None: + citations = [] + citations.append(content["citations"]) if thinking_blocks is not None: reasoning_content = "" for block in thinking_blocks: - if "thinking" in block: - reasoning_content += block["thinking"] + thinking_content = cast(Optional[str], block.get("thinking")) + if thinking_content is not None: + reasoning_content += thinking_content + return text_content, citations, thinking_blocks, reasoning_content, tool_calls def calculate_usage( @@ -703,7 +722,13 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): else: text_content = "" citations: Optional[List[Any]] = None - thinking_blocks: Optional[List[ChatCompletionThinkingBlock]] = None + thinking_blocks: Optional[ + List[ + Union[ + ChatCompletionThinkingBlock, ChatCompletionRedactedThinkingBlock + ] + ] + ] = None reasoning_content: Optional[str] = None tool_calls: List[ChatCompletionToolCallChunk] = [] diff --git a/litellm/llms/bedrock/chat/converse_transformation.py b/litellm/llms/bedrock/chat/converse_transformation.py index 76ea51f435..297dbab27b 100644 --- a/litellm/llms/bedrock/chat/converse_transformation.py +++ b/litellm/llms/bedrock/chat/converse_transformation.py @@ -22,6 +22,7 @@ from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMExcepti from litellm.types.llms.bedrock import * from litellm.types.llms.openai import ( AllMessageValues, + ChatCompletionRedactedThinkingBlock, ChatCompletionResponseMessage, ChatCompletionSystemMessage, ChatCompletionThinkingBlock, @@ -627,9 +628,11 @@ class AmazonConverseConfig(BaseConfig): def _transform_thinking_blocks( self, thinking_blocks: List[BedrockConverseReasoningContentBlock] - ) -> List[ChatCompletionThinkingBlock]: + ) -> List[Union[ChatCompletionThinkingBlock, ChatCompletionRedactedThinkingBlock]]: """Return a consistent format for thinking blocks between Anthropic and Bedrock.""" - thinking_blocks_list: List[ChatCompletionThinkingBlock] = [] + thinking_blocks_list: List[ + Union[ChatCompletionThinkingBlock, ChatCompletionRedactedThinkingBlock] + ] = [] for block in thinking_blocks: if "reasoningText" in block: _thinking_block = ChatCompletionThinkingBlock(type="thinking") @@ -640,6 +643,11 @@ class AmazonConverseConfig(BaseConfig): if _signature is not None: _thinking_block["signature"] = _signature thinking_blocks_list.append(_thinking_block) + elif "redactedContent" in block: + _redacted_block = ChatCompletionRedactedThinkingBlock( + type="redacted_thinking", data=block["redactedContent"] + ) + thinking_blocks_list.append(_redacted_block) return thinking_blocks_list def _transform_usage(self, usage: ConverseTokenUsageBlock) -> Usage: diff --git a/litellm/llms/bedrock/chat/invoke_handler.py b/litellm/llms/bedrock/chat/invoke_handler.py index 09bdd63572..dfd1658543 100644 --- a/litellm/llms/bedrock/chat/invoke_handler.py +++ b/litellm/llms/bedrock/chat/invoke_handler.py @@ -50,6 +50,7 @@ from litellm.llms.custom_httpx.http_handler import ( ) from litellm.types.llms.bedrock import * from litellm.types.llms.openai import ( + ChatCompletionRedactedThinkingBlock, ChatCompletionThinkingBlock, ChatCompletionToolCallChunk, ChatCompletionToolCallFunctionChunk, @@ -1255,19 +1256,33 @@ class AWSEventStreamDecoder: def translate_thinking_blocks( self, thinking_block: BedrockConverseReasoningContentBlockDelta - ) -> Optional[List[ChatCompletionThinkingBlock]]: + ) -> Optional[ + List[Union[ChatCompletionThinkingBlock, ChatCompletionRedactedThinkingBlock]] + ]: """ Translate the thinking blocks to a string """ - thinking_blocks_list: List[ChatCompletionThinkingBlock] = [] - _thinking_block = ChatCompletionThinkingBlock(type="thinking") + thinking_blocks_list: List[ + Union[ChatCompletionThinkingBlock, ChatCompletionRedactedThinkingBlock] + ] = [] + _thinking_block: Optional[ + Union[ChatCompletionThinkingBlock, ChatCompletionRedactedThinkingBlock] + ] = None + if "text" in thinking_block: + _thinking_block = ChatCompletionThinkingBlock(type="thinking") _thinking_block["thinking"] = thinking_block["text"] elif "signature" in thinking_block: + _thinking_block = ChatCompletionThinkingBlock(type="thinking") _thinking_block["signature"] = thinking_block["signature"] _thinking_block["thinking"] = "" # consistent with anthropic response - thinking_blocks_list.append(_thinking_block) + elif "redactedContent" in thinking_block: + _thinking_block = ChatCompletionRedactedThinkingBlock( + type="redacted_thinking", data=thinking_block["redactedContent"] + ) + if _thinking_block is not None: + thinking_blocks_list.append(_thinking_block) return thinking_blocks_list def converse_chunk_parser(self, chunk_data: dict) -> ModelResponseStream: @@ -1279,31 +1294,44 @@ class AWSEventStreamDecoder: usage: Optional[Usage] = None provider_specific_fields: dict = {} reasoning_content: Optional[str] = None - thinking_blocks: Optional[List[ChatCompletionThinkingBlock]] = None + thinking_blocks: Optional[ + List[ + Union[ + ChatCompletionThinkingBlock, ChatCompletionRedactedThinkingBlock + ] + ] + ] = None index = int(chunk_data.get("contentBlockIndex", 0)) if "start" in chunk_data: start_obj = ContentBlockStartEvent(**chunk_data["start"]) self.content_blocks = [] # reset - if ( - start_obj is not None - and "toolUse" in start_obj - and start_obj["toolUse"] is not None - ): - ## check tool name was formatted by litellm - _response_tool_name = start_obj["toolUse"]["name"] - response_tool_name = get_bedrock_tool_name( - response_tool_name=_response_tool_name - ) - tool_use = { - "id": start_obj["toolUse"]["toolUseId"], - "type": "function", - "function": { - "name": response_tool_name, - "arguments": "", - }, - "index": index, - } + if start_obj is not None: + if "toolUse" in start_obj and start_obj["toolUse"] is not None: + ## check tool name was formatted by litellm + _response_tool_name = start_obj["toolUse"]["name"] + response_tool_name = get_bedrock_tool_name( + response_tool_name=_response_tool_name + ) + tool_use = { + "id": start_obj["toolUse"]["toolUseId"], + "type": "function", + "function": { + "name": response_tool_name, + "arguments": "", + }, + "index": index, + } + elif ( + "reasoningContent" in start_obj + and start_obj["reasoningContent"] is not None + ): # redacted thinking can be in start object + thinking_blocks = self.translate_thinking_blocks( + start_obj["reasoningContent"] + ) + provider_specific_fields = { + "reasoningContent": start_obj["reasoningContent"], + } elif "delta" in chunk_data: delta_obj = ContentBlockDeltaEvent(**chunk_data["delta"]) self.content_blocks.append(delta_obj) diff --git a/litellm/llms/databricks/chat/transformation.py b/litellm/llms/databricks/chat/transformation.py index 6f5738fb4b..7eb3d82963 100644 --- a/litellm/llms/databricks/chat/transformation.py +++ b/litellm/llms/databricks/chat/transformation.py @@ -37,6 +37,7 @@ from litellm.types.llms.databricks import ( ) from litellm.types.llms.openai import ( AllMessageValues, + ChatCompletionRedactedThinkingBlock, ChatCompletionThinkingBlock, ChatCompletionToolChoiceFunctionParam, ChatCompletionToolChoiceObjectParam, @@ -314,13 +315,24 @@ class DatabricksConfig(DatabricksBase, OpenAILikeChatConfig, AnthropicConfig): @staticmethod def extract_reasoning_content( content: Optional[AllDatabricksContentValues], - ) -> Tuple[Optional[str], Optional[List[ChatCompletionThinkingBlock]]]: + ) -> Tuple[ + Optional[str], + Optional[ + List[ + Union[ChatCompletionThinkingBlock, ChatCompletionRedactedThinkingBlock] + ] + ], + ]: """ Extract and return the reasoning content and thinking blocks """ if content is None: return None, None - thinking_blocks: Optional[List[ChatCompletionThinkingBlock]] = None + thinking_blocks: Optional[ + List[ + Union[ChatCompletionThinkingBlock, ChatCompletionRedactedThinkingBlock] + ] + ] = None reasoning_content: Optional[str] = None if isinstance(content, list): for item in content: diff --git a/litellm/llms/hosted_vllm/chat/transformation.py b/litellm/llms/hosted_vllm/chat/transformation.py index 9332e98789..e328bf2881 100644 --- a/litellm/llms/hosted_vllm/chat/transformation.py +++ b/litellm/llms/hosted_vllm/chat/transformation.py @@ -2,9 +2,19 @@ Translate from OpenAI's `/v1/chat/completions` to VLLM's `/v1/chat/completions` """ -from typing import Optional, Tuple +from typing import List, Optional, Tuple, cast +from litellm.litellm_core_utils.prompt_templates.common_utils import ( + _get_image_mime_type_from_url, +) +from litellm.litellm_core_utils.prompt_templates.factory import _parse_mime_type from litellm.secret_managers.main import get_secret_str +from litellm.types.llms.openai import ( + AllMessageValues, + ChatCompletionFileObject, + ChatCompletionVideoObject, + ChatCompletionVideoUrlObject, +) from ....utils import _remove_additional_properties, _remove_strict_from_schema from ...openai.chat.gpt_transformation import OpenAIGPTConfig @@ -38,3 +48,71 @@ class HostedVLLMChatConfig(OpenAIGPTConfig): api_key or get_secret_str("HOSTED_VLLM_API_KEY") or "fake-api-key" ) # vllm does not require an api key return api_base, dynamic_api_key + + def _is_video_file(self, content_item: ChatCompletionFileObject) -> bool: + """ + Check if the file is a video + + - format: video/ + - file_data: base64 encoded video data + - file_id: infer mp4 from extension + """ + file = content_item.get("file", {}) + format = file.get("format") + file_data = file.get("file_data") + file_id = file.get("file_id") + if content_item.get("type") != "file": + return False + if format and format.startswith("video/"): + return True + elif file_data: + mime_type = _parse_mime_type(file_data) + if mime_type and mime_type.startswith("video/"): + return True + elif file_id: + mime_type = _get_image_mime_type_from_url(file_id) + if mime_type and mime_type.startswith("video/"): + return True + return False + + def _convert_file_to_video_url( + self, content_item: ChatCompletionFileObject + ) -> ChatCompletionVideoObject: + file = content_item.get("file", {}) + file_id = file.get("file_id") + file_data = file.get("file_data") + + if file_id: + return ChatCompletionVideoObject( + type="video_url", video_url=ChatCompletionVideoUrlObject(url=file_id) + ) + elif file_data: + return ChatCompletionVideoObject( + type="video_url", video_url=ChatCompletionVideoUrlObject(url=file_data) + ) + raise ValueError("file_id or file_data is required") + + def _transform_messages( + self, messages: List[AllMessageValues], model: str + ) -> List[AllMessageValues]: + """ + Support translating video files from file_id or file_data to video_url + """ + for message in messages: + if message["role"] == "user": + message_content = message.get("content") + if message_content and isinstance(message_content, list): + replaced_content_items: List[ + Tuple[int, ChatCompletionFileObject] + ] = [] + for idx, content_item in enumerate(message_content): + if content_item.get("type") == "file": + content_item = cast(ChatCompletionFileObject, content_item) + if self._is_video_file(content_item): + replaced_content_items.append((idx, content_item)) + for idx, content_item in replaced_content_items: + message_content[idx] = self._convert_file_to_video_url( + content_item + ) + transformed_messages = super()._transform_messages(messages, model) + return transformed_messages diff --git a/litellm/llms/litellm_proxy/chat/transformation.py b/litellm/llms/litellm_proxy/chat/transformation.py index 6699ef70d4..22013198ba 100644 --- a/litellm/llms/litellm_proxy/chat/transformation.py +++ b/litellm/llms/litellm_proxy/chat/transformation.py @@ -13,6 +13,7 @@ class LiteLLMProxyChatConfig(OpenAIGPTConfig): def get_supported_openai_params(self, model: str) -> List: list = super().get_supported_openai_params(model) list.append("thinking") + list.append("reasoning_effort") return list def _map_openai_params( diff --git a/litellm/llms/vertex_ai/gemini/transformation.py b/litellm/llms/vertex_ai/gemini/transformation.py index 0afad13feb..f4a05d9104 100644 --- a/litellm/llms/vertex_ai/gemini/transformation.py +++ b/litellm/llms/vertex_ai/gemini/transformation.py @@ -12,6 +12,9 @@ from pydantic import BaseModel import litellm from litellm._logging import verbose_logger +from litellm.litellm_core_utils.prompt_templates.common_utils import ( + _get_image_mime_type_from_url, +) from litellm.litellm_core_utils.prompt_templates.factory import ( convert_to_anthropic_image_obj, convert_to_gemini_tool_call_invoke, @@ -99,62 +102,6 @@ def _process_gemini_image(image_url: str, format: Optional[str] = None) -> PartT raise e -def _get_image_mime_type_from_url(url: str) -> Optional[str]: - """ - Get mime type for common image URLs - See gemini mime types: https://cloud.google.com/vertex-ai/generative-ai/docs/multimodal/image-understanding#image-requirements - - Supported by Gemini: - application/pdf - audio/mpeg - audio/mp3 - audio/wav - image/png - image/jpeg - image/webp - text/plain - video/mov - video/mpeg - video/mp4 - video/mpg - video/avi - video/wmv - video/mpegps - video/flv - """ - url = url.lower() - - # Map file extensions to mime types - mime_types = { - # Images - (".jpg", ".jpeg"): "image/jpeg", - (".png",): "image/png", - (".webp",): "image/webp", - # Videos - (".mp4",): "video/mp4", - (".mov",): "video/mov", - (".mpeg", ".mpg"): "video/mpeg", - (".avi",): "video/avi", - (".wmv",): "video/wmv", - (".mpegps",): "video/mpegps", - (".flv",): "video/flv", - # Audio - (".mp3",): "audio/mp3", - (".wav",): "audio/wav", - (".mpeg",): "audio/mpeg", - # Documents - (".pdf",): "application/pdf", - (".txt",): "text/plain", - } - - # Check each extension group against the URL - for extensions, mime_type in mime_types.items(): - if any(url.endswith(ext) for ext in extensions): - return mime_type - - return None - - def _gemini_convert_messages_with_history( # noqa: PLR0915 messages: List[AllMessageValues], ) -> List[ContentType]: diff --git a/litellm/types/llms/bedrock.py b/litellm/types/llms/bedrock.py index fe3f2e1b5f..6e02ff2ab7 100644 --- a/litellm/types/llms/bedrock.py +++ b/litellm/types/llms/bedrock.py @@ -179,6 +179,7 @@ class ToolUseBlockStartEvent(TypedDict): class ContentBlockStartEvent(TypedDict, total=False): toolUse: Optional[ToolUseBlockStartEvent] + reasoningContent: BedrockConverseReasoningContentBlockDelta class ContentBlockDeltaEvent(TypedDict, total=False): diff --git a/litellm/types/llms/openai.py b/litellm/types/llms/openai.py index 10766b65a6..f3b470cb98 100644 --- a/litellm/types/llms/openai.py +++ b/litellm/types/llms/openai.py @@ -468,6 +468,12 @@ class ChatCompletionThinkingBlock(TypedDict, total=False): cache_control: Optional[Union[dict, ChatCompletionCachedContent]] +class ChatCompletionRedactedThinkingBlock(TypedDict, total=False): + type: Required[Literal["redacted_thinking"]] + data: str + cache_control: Optional[Union[dict, ChatCompletionCachedContent]] + + class WebSearchOptionsUserLocationApproximate(TypedDict, total=False): city: str """Free text input for the city of the user, e.g. `San Francisco`.""" @@ -797,7 +803,9 @@ class ChatCompletionResponseMessage(TypedDict, total=False): function_call: Optional[ChatCompletionToolCallFunctionChunk] provider_specific_fields: Optional[dict] reasoning_content: Optional[str] - thinking_blocks: Optional[List[ChatCompletionThinkingBlock]] + thinking_blocks: Optional[ + List[Union[ChatCompletionThinkingBlock, ChatCompletionRedactedThinkingBlock]] + ] class ChatCompletionUsageBlock(TypedDict): diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 88f9638438..533ffaa64a 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -29,6 +29,7 @@ from .guardrails import GuardrailEventHooks from .llms.openai import ( Batch, ChatCompletionAnnotation, + ChatCompletionRedactedThinkingBlock, ChatCompletionThinkingBlock, ChatCompletionToolCallChunk, ChatCompletionUsageBlock, @@ -552,7 +553,9 @@ class Message(OpenAIObject): function_call: Optional[FunctionCall] audio: Optional[ChatCompletionAudioResponse] = None reasoning_content: Optional[str] = None - thinking_blocks: Optional[List[ChatCompletionThinkingBlock]] = None + thinking_blocks: Optional[ + List[Union[ChatCompletionThinkingBlock, ChatCompletionRedactedThinkingBlock]] + ] = None provider_specific_fields: Optional[Dict[str, Any]] = Field( default=None, exclude=True ) @@ -567,7 +570,11 @@ class Message(OpenAIObject): audio: Optional[ChatCompletionAudioResponse] = None, provider_specific_fields: Optional[Dict[str, Any]] = None, reasoning_content: Optional[str] = None, - thinking_blocks: Optional[List[ChatCompletionThinkingBlock]] = None, + thinking_blocks: Optional[ + List[ + Union[ChatCompletionThinkingBlock, ChatCompletionRedactedThinkingBlock] + ] + ] = None, annotations: Optional[List[ChatCompletionAnnotation]] = None, **params, ): @@ -650,7 +657,9 @@ class Message(OpenAIObject): class Delta(OpenAIObject): reasoning_content: Optional[str] = None - thinking_blocks: Optional[List[ChatCompletionThinkingBlock]] = None + thinking_blocks: Optional[ + List[Union[ChatCompletionThinkingBlock, ChatCompletionRedactedThinkingBlock]] + ] = None provider_specific_fields: Optional[Dict[str, Any]] = Field(default=None) def __init__( @@ -661,7 +670,11 @@ class Delta(OpenAIObject): tool_calls=None, audio: Optional[ChatCompletionAudioResponse] = None, reasoning_content: Optional[str] = None, - thinking_blocks: Optional[List[ChatCompletionThinkingBlock]] = None, + thinking_blocks: Optional[ + List[ + Union[ChatCompletionThinkingBlock, ChatCompletionRedactedThinkingBlock] + ] + ] = None, annotations: Optional[List[ChatCompletionAnnotation]] = None, **params, ): diff --git a/tests/litellm/llms/anthropic/chat/test_anthropic_chat_handler.py b/tests/litellm/llms/anthropic/chat/test_anthropic_chat_handler.py new file mode 100644 index 0000000000..25ec01167e --- /dev/null +++ b/tests/litellm/llms/anthropic/chat/test_anthropic_chat_handler.py @@ -0,0 +1,38 @@ +import json +import os +import sys +from unittest.mock import MagicMock + +import pytest +from fastapi.testclient import TestClient + +sys.path.insert( + 0, os.path.abspath("../../../../..") +) # Adds the parent directory to the system path + +from litellm.llms.anthropic.chat.handler import ModelResponseIterator + + +def test_redacted_thinking_content_block_delta(): + chunk = { + "type": "content_block_start", + "index": 58, + "content_block": { + "type": "redacted_thinking", + "data": "EuoBCoYBGAIiQJ/SxkPAgqxhKok29YrpJHRUJ0OT8ahCHKAwyhmRuUhtdmDX9+mn4gDzKNv3fVpQdB01zEPMzNY3QuTCd+1bdtEqQK6JuKHqdndbwpr81oVWb4wxd1GqF/7Jkw74IlQa27oobX+KuRkopr9Dllt/RDe7Se0sI1IkU7tJIAQCoP46OAwSDF51P09q67xhHlQ3ihoM2aOVlkghq/X0w8NlIjBMNvXYNbjhyrOcIg6kPFn2ed/KK7Cm5prYAtXCwkb4Wr5tUSoSHu9T5hKdJRbr6WsqEc7Lle7FULqMLZGkhqXyc3BA", + }, + } + model_response_iterator = ModelResponseIterator( + streaming_response=MagicMock(), sync_stream=False, json_mode=False + ) + model_response = model_response_iterator.chunk_parser(chunk=chunk) + print(f"\n\nmodel_response: {model_response}\n\n") + assert model_response.choices[0].delta.thinking_blocks is not None + assert len(model_response.choices[0].delta.thinking_blocks) == 1 + print( + f"\n\nmodel_response.choices[0].delta.thinking_blocks[0]: {model_response.choices[0].delta.thinking_blocks[0]}\n\n" + ) + assert ( + model_response.choices[0].delta.thinking_blocks[0]["type"] + == "redacted_thinking" + ) diff --git a/tests/litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py b/tests/litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py index 9f672110a4..6c98c87cd7 100644 --- a/tests/litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py +++ b/tests/litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py @@ -56,3 +56,58 @@ def test_calculate_usage(): assert usage.prompt_tokens_details.cached_tokens == 0 assert usage._cache_creation_input_tokens == 12304 assert usage._cache_read_input_tokens == 0 + + +def test_extract_response_content_with_citations(): + config = AnthropicConfig() + + completion_response = { + "id": "msg_01XrAv7gc5tQNDuoADra7vB4", + "type": "message", + "role": "assistant", + "model": "claude-3-5-sonnet-20241022", + "content": [ + {"type": "text", "text": "According to the documents, "}, + { + "citations": [ + { + "type": "char_location", + "cited_text": "The grass is green. ", + "document_index": 0, + "document_title": "My Document", + "start_char_index": 0, + "end_char_index": 20, + } + ], + "type": "text", + "text": "the grass is green", + }, + {"type": "text", "text": " and "}, + { + "citations": [ + { + "type": "char_location", + "cited_text": "The sky is blue.", + "document_index": 0, + "document_title": "My Document", + "start_char_index": 20, + "end_char_index": 36, + } + ], + "type": "text", + "text": "the sky is blue", + }, + {"type": "text", "text": "."}, + ], + "stop_reason": "end_turn", + "stop_sequence": None, + "usage": { + "input_tokens": 610, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "output_tokens": 51, + }, + } + + _, citations, _, _, _ = config.extract_response_content(completion_response) + assert citations is not None diff --git a/tests/litellm/llms/bedrock/chat/test_converse_transformation.py b/tests/litellm/llms/bedrock/chat/test_converse_transformation.py index 5390daa1a0..cc0dd495e3 100644 --- a/tests/litellm/llms/bedrock/chat/test_converse_transformation.py +++ b/tests/litellm/llms/bedrock/chat/test_converse_transformation.py @@ -40,3 +40,22 @@ def test_transform_usage(): ) assert openai_usage._cache_creation_input_tokens == usage["cacheWriteInputTokens"] assert openai_usage._cache_read_input_tokens == usage["cacheReadInputTokens"] + + +def test_transform_thinking_blocks_with_redacted_content(): + thinking_blocks = [ + { + "reasoningText": { + "text": "This is a test", + "signature": "test_signature", + } + }, + { + "redactedContent": "This is a redacted content", + }, + ] + config = AmazonConverseConfig() + transformed_thinking_blocks = config._transform_thinking_blocks(thinking_blocks) + assert len(transformed_thinking_blocks) == 2 + assert transformed_thinking_blocks[0]["type"] == "thinking" + assert transformed_thinking_blocks[1]["type"] == "redacted_thinking" diff --git a/tests/litellm/llms/bedrock/chat/test_invoke_handler.py b/tests/litellm/llms/bedrock/chat/test_invoke_handler.py new file mode 100644 index 0000000000..ed94c784c9 --- /dev/null +++ b/tests/litellm/llms/bedrock/chat/test_invoke_handler.py @@ -0,0 +1,22 @@ +import json +import os +import sys + +import pytest +from fastapi.testclient import TestClient + +sys.path.insert( + 0, os.path.abspath("../../../../..") +) # Adds the parent directory to the system path +from unittest.mock import MagicMock, patch + +from litellm.llms.bedrock.chat.invoke_handler import AWSEventStreamDecoder + + +def test_transform_thinking_blocks_with_redacted_content(): + thinking_block = {"redactedContent": "This is a redacted content"} + decoder = AWSEventStreamDecoder(model="test") + transformed_thinking_blocks = decoder.translate_thinking_blocks(thinking_block) + assert len(transformed_thinking_blocks) == 1 + assert transformed_thinking_blocks[0]["type"] == "redacted_thinking" + assert transformed_thinking_blocks[0]["data"] == "This is a redacted content" diff --git a/tests/litellm/llms/hosted_vllm/chat/test_hosted_vllm_chat_transformation.py b/tests/litellm/llms/hosted_vllm/chat/test_hosted_vllm_chat_transformation.py new file mode 100644 index 0000000000..7885195b5b --- /dev/null +++ b/tests/litellm/llms/hosted_vllm/chat/test_hosted_vllm_chat_transformation.py @@ -0,0 +1,45 @@ +import json +import os +import sys +from unittest.mock import AsyncMock, MagicMock, patch + +import httpx +import pytest + +sys.path.insert( + 0, os.path.abspath("../../../../..") +) # Adds the parent directory to the system path + +from litellm.llms.hosted_vllm.chat.transformation import HostedVLLMChatConfig + + +def test_hosted_vllm_chat_transformation_file_url(): + config = HostedVLLMChatConfig() + video_url = "https://example.com/video.mp4" + video_data = f"data:video/mp4;base64,{video_url}" + messages = [ + { + "role": "user", + "content": [ + { + "type": "file", + "file": { + "file_data": video_data, + }, + } + ], + } + ] + transformed_response = config.transform_request( + model="hosted_vllm/llama-3.1-70b-instruct", + messages=messages, + optional_params={}, + litellm_params={}, + headers={}, + ) + assert transformed_response["messages"] == [ + { + "role": "user", + "content": [{"type": "video_url", "video_url": {"url": video_data}}], + } + ] diff --git a/tests/llm_translation/test_anthropic_completion.py b/tests/llm_translation/test_anthropic_completion.py index 078467f588..73d0bec5bf 100644 --- a/tests/llm_translation/test_anthropic_completion.py +++ b/tests/llm_translation/test_anthropic_completion.py @@ -968,6 +968,107 @@ def test_anthropic_citations_api_streaming(): assert has_citations +@pytest.mark.parametrize( + "model", + [ + "anthropic/claude-3-7-sonnet-20250219", + "bedrock/us.anthropic.claude-3-7-sonnet-20250219-v1:0", + ], +) +def test_anthropic_thinking_output(model): + from litellm import completion + + litellm._turn_on_debug() + + resp = completion( + model=model, + messages=[{"role": "user", "content": "What is the capital of France?"}], + thinking={"type": "enabled", "budget_tokens": 1024}, + ) + + print(resp) + assert resp.choices[0].message.reasoning_content is not None + assert isinstance(resp.choices[0].message.reasoning_content, str) + assert resp.choices[0].message.thinking_blocks is not None + assert isinstance(resp.choices[0].message.thinking_blocks, list) + assert len(resp.choices[0].message.thinking_blocks) > 0 + + assert resp.choices[0].message.thinking_blocks[0]["type"] == "thinking" + assert resp.choices[0].message.thinking_blocks[0]["signature"] is not None + + +@pytest.mark.parametrize( + "model", + [ + "anthropic/claude-3-7-sonnet-20250219", + # "bedrock/us.anthropic.claude-3-7-sonnet-20250219-v1:0", + ], +) +def test_anthropic_redacted_thinking_output(model): + from litellm import completion + + litellm._turn_on_debug() + + resp = completion( + model=model, + messages=[{"role": "user", "content": "ANTHROPIC_MAGIC_STRING_TRIGGER_REDACTED_THINKING_46C9A13E193C177646C7398A98432ECCCE4C1253D5E2D82641AC0E52CC2876CB"}], + thinking={"type": "enabled", "budget_tokens": 1024}, + ) + + print(resp) + assert resp.choices[0].message.thinking_blocks is not None + assert isinstance(resp.choices[0].message.thinking_blocks, list) + assert len(resp.choices[0].message.thinking_blocks) > 0 + assert resp.choices[0].message.thinking_blocks[0]["type"] == "redacted_thinking" + assert resp.choices[0].message.thinking_blocks[0]["data"] is not None + + + + +@pytest.mark.parametrize( + "model", + [ + "anthropic/claude-3-7-sonnet-20250219", + # "bedrock/us.anthropic.claude-3-7-sonnet-20250219-v1:0", + # "bedrock/invoke/us.anthropic.claude-3-7-sonnet-20250219-v1:0", + ], +) +def test_anthropic_thinking_output_stream(model): + litellm.set_verbose = True + try: + # litellm._turn_on_debug() + resp = litellm.completion( + model=model, + messages=[{"role": "user", "content": "Tell me a joke."}], + stream=True, + thinking={"type": "enabled", "budget_tokens": 1024}, + timeout=10, + ) + + reasoning_content_exists = False + signature_block_exists = False + for chunk in resp: + print(f"chunk 2: {chunk}") + if ( + hasattr(chunk.choices[0].delta, "thinking_blocks") + and chunk.choices[0].delta.thinking_blocks is not None + and chunk.choices[0].delta.reasoning_content is not None + and isinstance(chunk.choices[0].delta.thinking_blocks, list) + and len(chunk.choices[0].delta.thinking_blocks) > 0 + and isinstance(chunk.choices[0].delta.reasoning_content, str) + ): + reasoning_content_exists = True + print(chunk.choices[0].delta.thinking_blocks[0]) + if chunk.choices[0].delta.thinking_blocks[0].get("signature"): + signature_block_exists = True + assert chunk.choices[0].delta.thinking_blocks[0]["type"] == "thinking" + assert reasoning_content_exists + assert signature_block_exists + except litellm.Timeout: + pytest.skip("Model is timing out") + + + def test_anthropic_custom_headers(): from litellm import completion from litellm.llms.custom_httpx.http_handler import HTTPHandler @@ -1045,3 +1146,37 @@ def test_anthropic_thinking_in_assistant_message(model): +@pytest.mark.parametrize( + "model", + [ + "anthropic/claude-3-7-sonnet-20250219", + # "bedrock/us.anthropic.claude-3-7-sonnet-20250219-v1:0", + ], +) +def test_anthropic_redacted_thinking_in_assistant_message(model): + litellm._turn_on_debug() + params = { + "model": model, + "messages": [ + { + "role": "assistant", + "content": [ + { + "type": "redacted_thinking", + "data": "EqkBCkYIARgCKkAflgFkky5bvpaXt2GnDYgbA8QOCr+BF53t+UmiRA22Z7Ply9z2xfTGYSqvjlhIEsV6WDPdVoXndztvhKCzE2PUEgxwXpRD1hBLUSajVWoaDEftxmhqdg0mRwPUGCIwcht1EH91+gznPoaMNquU4sGeaOLFaeyNeG4dJXsYT/Jc4OG3453LN5ra4uVxC/GgKhGMQ1A9aO2Ac0O5M+bOdp1RFw==Eo0CCkYIARgCKkCcHATldbjR0vfU1DlNaQr3J2GKem6OjFybQyshp4C9XnysT/6y1CNcI+VGsbX99GfKLGqcsGYr81WlM+d7NscJEgxzkyZuwL3QnnxFiUUaDIA3nZpQa15D5XD72yIwyIGpJwhdavzXvE1bQLZj43aNtznG6Uwsxx4ZlLv83SUqH7GqzMxvm3stLj3cYmKMKnUqqhpeluvoxODUY/fhhF6Bjsj9C1MIRL+9urDH2EtAmZ+BrvLoXjRlbEH9+DtzLE57I1ShMDbUqLJXxXTcjhPkmu3JscBYf0waXfUgrQl2Pnv5dAxM2S3ZASk8di7ak0XcRknVBhhaR2ykdDbVyxzFzyZo8Fc=EtcBCkYIARgCKkCl6nQeKqHIBgdZ1EByLfEwnlZxsZWoDwablEKqRAIrKvB10ccs6RZqrTMZgcMLaW3QpWwnI4fC/WiOe811B94JEgyvTK4+E/zB+a42bYcaDOPesimKdlIPLT7VQiIwplWjvDcbe16vZSJ0OezjHCHEvML4QJPyvGE3NRHcLzC9UiGYriFys5zgv0O7qKr5Kj/56IL1BbaFqSANA7vjGoW+GSlv294L4LzqNWCD0ANzDnEjlXlVeibNM74v+KKXRVwn/IInHPog4hJA0/3GQyA=EtwBCkYIARgCKkBda4XEzq+PTfE7niGdYVzvAXRTb+3ujsDVGhVNtFnPx6K/I6ORfxOWmwEuk7iXygehQA18p0CVYLsCU4AHFvtjEgzYH2JNCxa8F07pGioaDOA635mdHKbyiecBJSIwshUavES7HZBnA4l3k8l92LAhuJQV1C5tUgKkk0pHRT+/OzDfXvxsZSx7AmR7J3QXKkQwHL6K9yZEWdeh/B22ft/GxyRViO7nZrT95PAAux31u++rYQyeFJ+rv0Yrs/KoBnlNUg9YFOpDMo1bMWV9n4CGwq92bw==EtEBCkYIARgCKkCZdn2NBzxiOEJt/E8VOs6YLbYjRaCkvhEdz5apcEZlBQJpulvgv1JvamrMZD0FCJZVTwxd/65M9Ady/LbtYTh7EgwtL7W9DXSFjxPErCIaDGk0e/bXY8yJdjk3CSIwYS0TtiaFK8tJrREBFA9IOp+q+tnE8Wl338CbbskRvF5topYmtofuBIG4GQkHvbQjKjn2BmwrEic/CdSEVbvEix7AWEsw92DabVmseTQhUbbuYRa4Ou6jXMW2pMJFUBjMr95gF6BlVFr4iEA=EsUBCkYIARgCKkAsEmKjMN9TVYLyBdo1+0uopommcjQx8Fu65+mje5Ft05KOnyKAzuUyORtk5r73glan8L+WlygaOOrZ1hi81219EgwpdTA6qbcaggIWeTIaDDrJ0eTbsqku4VSY8CIw3mJfRyv7ISHih4mpAVioGuuduXbaie5eKn5a+WgQiOmm22uZ4Gv72uluCSGGriHnKi28bHMomrytYLvKNvhL51yf5/Tgm/lIgQ9gyTJLqVzVjGn6ng1sN8vUti/tuGw=EsoBCkYIARgCKkB+jJBrxqqpzyGt5RXDKTBVxTnE8IrYRysAL2U/H171INDMCxrDHxfts3M0wuQirXN/2fZXwmQJIZRzzumA+I2sEgw0ySDeyTfHgTiafo8aDKOTl485koQiPwXipyIwG9n/zWUZ+tgfFELW2rV5/yo6Pq/r9bJdrd2b25qCATwX2gd54gsjWhSvLDkD7pLJKjL6ZuiW4N6hVo6JIR4UL8LxcsP9tET0ElIgQZ/h8HOIi18fQKsEdtseWCFnuXse21KIeg==EtwBCkYIARgCKkDWMlgTA+iKsScbpNtZab6dgMKRZYpQSoJ274+n0TqvLAqHL8GxLm1sMVom81LcVWCZZeIVQFbkmbJxyBovvLoUEgxy6YGb0EeJW10P8XEaDKowL3qI/z000pgR2SIwZIczlDKkqw75UYcEOC6Cx9yc0CdYjJnmQOa4Ezni20SANA8YnBMIYJqW4osO/KalKkTLmgvJRQE1Hk8Bn3af9fIYt+vITYEY4Wr7/UVNBtSXBOMP0YoSgNyzjX/pu2N3oy2Blv/YAgtHIJ3Xwd43clN5F2wU+Q==EtQBCkYIARgCKkD3vxW2GsLyEGtmBpI6NdNyh4i/ea7E9rp5puSHdk/dSCpW5G1wI3nrFIS2bUqZsvsDu3YgcDixG8eeDnzacC/qEgzilh/V8vaE1X9lRlIaDAa17eq6kSgaRrsAfSIwFAXgLu5BUKldMeQdcomRqgmY9hDzkDlRnBrbO9GxXsrmpGTU9iqVZQ7z9OVW522bKjyB/GeuNlv4V8a8uricx1InN8q94coWGCRPvAJVAvhP/YMCcNlvrgoN8C2RGc13e88uDq01r6gpkWTlVDY=EssBCkYIARgCKkAOhKBpvfqIElQ1mlG7NiCiolHnqagXryuwNsODnttLBeVMGBsZ8DgpSGWonVE/22MQgciWLY7WaaeoDcpL3X/pEgx4xuL/KqOgxrBnau4aDH3pQ/Sqr1aHa68YiiIwR6+w9QOWFfut8ZG8z+QkAO/kZVePcELKabHp7ikY+DOjvOt4FfnaChwQFTSGzZhaKjPK4MwQukuZIT1PFGFIh20Hi6wMQlHvsChIF88nUV2EAz4Sgb/vWPiQBbWP3gT3hJBehQY=EtMBCkYIARgCKkCT0yD5m4Rvs3KBNkAC2g7aprLTzKRqF+vdHAeYte9KngJZhThexj65o+q9HOGhIIAsboRhz70xkAybdQdsrg8OEgzQm1M980FeZMCi1XsaDJSFOpIuOhUOkPIs+iIw62jO5yY9ZETmrYtEb+pYN5Cyf467YVOOv7FBo44gIFgUvFklU5+y09k3MGzrBNViKjvkopPoFbpYI9ilB3dN6pAzrzhDzOum+Rsx1N25+UYvdT+yYBilrIPW1XmLmzT+ZMs4eV5caG35ZsNsjQ==EtwBCkYIARgCKkCOShz0/2ZO3u0WH8PBN63fAwKo4TcNFM3axUJL9dK9JJDLtC0XwP9Ee4vqPZyLBao4RyAefbYmY3TJ1As/AbuvEgxbYiyN4UcjaJU9mwkaDP9L3FACdMRQ+UFOSSIwQ0btU6cKIRsSNzvBsP8Fa4Ab7vOnlo4YSAv2lD7ZdDKVcQaWQZHYsQb/QQDfIGKGKkRXhNoET9KyQkb/x8lVpUR1d2u/sHTdgKEjkUdQop88SUFHvkGcJrMUTvnuvUdO4MdHwKnN0IINbDHTEUjUXSQPkpfTTA==EtwBCkYIARgCKkCIwQCFJUrhd1aT8hGMNcPIl+CaSZWsqerPDUGzZnS2tt2+tAs+TAPcKVHC07BdEXj6aKSbrOb8b7OQ/KFbrWJ4Egz980omEnE4djm8t5UaDDXrDJWgFSuZ+LWFmSIw/RzMo5ncKnqvf0TZ1krxMi4/DpAZb0Lgmc1XxGT2JPA4At9EEHNVPrWLXwGM3vUYKkQltG8EJFOWL1In5541dca1pnRDyBg4JVRQ5CuvA/pUCI2e9ARiODI7D+ydZorcnWQ7j2Qc1DguMQVHMbPLyGbQx9vqgQ==EtsBCkYIARgCKkDiH+ww5G0OgaW7zSQD7ZKYdViZfi+KO+TkA/k4rlTKsIwpUILZZ/53ppu93xaEazsD92GXKKSG3B/jBCqjQRg7EgzR3K/BJFTt359xPOgaDEHyoGVloiLS71ufAiIwO77B26VivdVgd2Dmv3DOtUAFs/jDwLM9EmNCBeoivwJPD2hYEKNm6TUWTinGfO2jKkNbrYgpA5esB0y1iXA0qGwRAmnD8ykZc0DT40vvd9EDvb5gHCd7RyjEU9BKnXBPWpGdTi4U+LZKYQ9LEE6sJ8vBm8w3EtUBCkYIARgCKkBbxQIjnTzzKf8Qhfcu+so91+MMbpJNyga27D9tZBtTexYLMJtzDWux4urfCc5TjjX0MvK62lKkhcPLuJE7KiI8EgzFF+TlNgPNp6RoyQgaDBAUDEAsqBMj7z4kciIwUWEZMGkG8ZnjltVpuffHxw5Rqyc+Smh1MnqnWxo0JlCOC43W5JH5KoJ/4RDxX7IjKj2fs5F6eiRMEi+L4KyjDBIvoPoE/wrdC+Fo6c8lMJiYw0MJ/lXgJQv6p0GRe251X+pcfN+2lx067/GLP6qjEtsBCkYIARgCKkCItf9nN0FKJsetom0ZoZvccwboNM2erGP7tIAYsOzsA9lmh7rFI2mFbOOC2WZ1v+QkvxppQ2wO+N35t29LC7RPEgzyJgiM1GHTVN+VPPwaDOXyzSg9BQ85oi58DCIwu/JxKJwVECkbru1d05yhwMYDsJrSJW1BO2ZBrg8Tb48S+dpD6hEPd1itq8cSM3ChKkNv83rGY8Gjg2DiTWDsIqUCD0pb2drrwnjkherr5/EQWdhHC7MijF8zyvqU4tBZrxP+64GcII7P87ja8B4YxGUIw9J7Et0BCkYIARgCKkCInOjYRgGSjcV/WHJ6HjB983rvz/nrOZ9xZMdrTYdHURtXN4zMAjZYQ8ZBk31n4aFGv5PAtDfbjqcytZUaCKicEgwXQrjgS0FHWq/2PwAaDKjYgoXuPPq+RNJUvCIwh1VmSiLGu+3pl7RcCBxnH/ue38EUDZAIRYiDI59h8CVdZpDSqaH8yJvFlR5Jxc8xKkXcEPduWcuONY+vatnIo5AQeSh9HM4oM4DoDma1OvVfdPUpbvaTP3ZhEv4iOMjvwzHBBkvc8b9jV2oTb8Xe50COLFJvURk=EtcBCkYIARgCKkDM4CyfgVBHhusU4C0tg/RwXiAbNtjOoYfcufGUnFlQKcpuJnekvb61EAerBrELguIrvNJIbyqy0Kcd/r64hu1UEgyITWjG3/cVsm/o0JkaDKm1/y0HF1YpqoiFoCIwqImOpk6SngP99aXE4p5c7y9rOvVo3lmKidTUdi1lmtoEZ9sXdY49nLsGeCuCjPJKKj976uFmgrZWIEZIL+HQGVjDOJ7mK8NzAxjX3m0AELsWN5FgbGOHus/S4o2EKi43/MLaRervgaFdrxK9BKGE6LY=EtMBCkYIARgCKkDvEoH/lv1fRxN+JaknzdY53WmQrEGJ7yupv22X2TdxN2+GmY8l1KYONWboOxalfoSbSlp3+zVJXdvTCa60CYnnEgyUslgNTFL5iGt+aq0aDESsIoNRuPYqDc5fbCIw9gHGejHXKw9GMR0sw1RnIF2FBI5Zo5/4EK2AFZ8BU5yAYgJw0wTc16ZVEFEraKS+KjtqVPmiodedFzc+f4kr+U8dy+xQtcsmTe9KcvAYmskvZ6Kl6iCitm/PZdjl/7COePcTVu32QnxZuG4Mpw==EtEBCkYIARgCKkB/SdSv2Jo8DJ4pOOK4mYXhSsPrnf6/ESHL7voj6FbdYPsgg2f3XQByQV93Menel5tgcx0jvNfY7Z9nx4Rz3iTvEgxN/mWUwb6Lb/1BfkAaDBONEsjWD1fKeK8H/iIwy+yJUFPTde2wxI/j6em5uS8HWGsfX9pUB4u/K4QHAd85bn63rrXSxbe2DHIG620UKjk+C6q3aXztOAGAyvhjiN9lnNAFPv93GTnwj+14n07c/xPdHBQyXXi742UBjFdQkmwp3m6RWf5psYU=EuQBCkYIARgCKkBxavD9zRmeX22ltvtCNzZzXTpsAHmNwSuejX7ibJueaDQaSOykBjNJavdMn6yQ8mAxCpNrNmhtBhGxHBGZE668EgzFNqHVE2WctK5ZiN0aDGNFTI5T3/0vDCtFXiIwRDXV5+9nWYGzuih8cG8h4dCs+n90rcL/Tz78QKsfpZeLNpr4aZSU8KHO2OmcmFoOKkxdgzKPy/gOfcCELsudlawbVyobU4CIhOYacIPhi+0XvgjXpqP0JIANaOdawb2zWrKhBKNA4VCHzbFkDm9cV1WrGIw0cEJ3oRU7idRgEsEBCkYIARgCKkDJUpJz2Ct4ZZJlWkAGg1Lc/rVqCd/V5rq01yehv9GkTIaq9H2jgjVKnUV1e4o9F1cUxmMk6fn4XK01sp/szP2GEgyvuemo2Di0USGKingaDCAMXK1kWRk6KofoyyIwxr/Jdwz2RrUytRWMGjrs4MkcQ2rhrVL/00Ktebga9cwrqeDOq+7nN8L64V+XEwsJKimHdmpCQPqYz8rIX25+v2XqcBDXzoBW8+eqdJKRhKcYooLbBXK3DUgRVQ==", + }, + { + "type": "text", + "text": "I'm not able to respond to special commands or trigger phrases like the one you've shared. Those types of strings don't activate any special modes or features in my system. Is there something specific I can help you with today? I'm happy to assist with questions, have a conversation, provide information, or help with various tasks within my normal capabilities.", + }, + ], + }, + {"role": "user", "content": [{"type": "text", "text": "Who do you know?"}]}, + ], + "max_tokens": 32768, + "thinking": {"type": "enabled", "budget_tokens": 30720}, + } + + response = litellm.completion(**params) + + assert response is not None