From e7ef14398f92dd2315fa0bd9dd509c3f41dcf7cd Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Fri, 21 Mar 2025 10:20:21 -0700 Subject: [PATCH 1/5] fix(anthropic/chat/transformation.py): correctly update response_format to tool call transformation Fixes https://github.com/BerriAI/litellm/issues/9411 --- litellm/llms/anthropic/chat/transformation.py | 2 +- tests/llm_translation/base_llm_unit_tests.py | 72 ++++++++++++++++++- .../test_anthropic_completion.py | 4 +- 3 files changed, 74 insertions(+), 4 deletions(-) diff --git a/litellm/llms/anthropic/chat/transformation.py b/litellm/llms/anthropic/chat/transformation.py index 383c1cd3e5..aff70a6e62 100644 --- a/litellm/llms/anthropic/chat/transformation.py +++ b/litellm/llms/anthropic/chat/transformation.py @@ -387,7 +387,7 @@ class AnthropicConfig(BaseConfig): _input_schema["additionalProperties"] = True _input_schema["properties"] = {} else: - _input_schema["properties"] = {"values": json_schema} + _input_schema.update(json_schema) _tool = AnthropicMessagesTool( name=RESPONSE_FORMAT_TOOL_NAME, input_schema=_input_schema diff --git a/tests/llm_translation/base_llm_unit_tests.py b/tests/llm_translation/base_llm_unit_tests.py index 32f631daad..f3614fdb4c 100644 --- a/tests/llm_translation/base_llm_unit_tests.py +++ b/tests/llm_translation/base_llm_unit_tests.py @@ -20,6 +20,7 @@ from litellm.utils import ( get_optional_params, ProviderConfigManager, ) +from litellm.main import stream_chunk_builder from typing import Union # test_example.py @@ -338,7 +339,7 @@ class BaseLLMChatTest(ABC): @pytest.mark.flaky(retries=6, delay=1) def test_json_response_pydantic_obj(self): - litellm.set_verbose = True + litellm._turn_on_debug() from pydantic import BaseModel from litellm.utils import supports_response_schema @@ -995,3 +996,72 @@ class BaseOSeriesModelsTest(ABC): # test across azure/openai ), "temperature should not be in the request body" except Exception as e: pytest.fail(f"Error occurred: {e}") + + +class BaseAnthropicChatTest(ABC): + """ + Ensures consistent result across anthropic model usage + """ + + @abstractmethod + def get_base_completion_call_args(self) -> dict: + """Must return the base completion call args""" + pass + + @property + def completion_function(self): + return litellm.completion + + def test_anthropic_response_format_streaming_vs_non_streaming(self): + litellm.set_verbose = True + args = { + "messages": [ + { + "content": "Your goal is to summarize the previous agent's thinking process into short descriptions to let user better understand the research progress. If no information is available, just say generic phrase like 'Doing some research...' with the given output format. Make sure to adhere to the output format no matter what, even if you don't have any information or you are not allowed to respond to the given input information (then just say generic phrase like 'Doing some research...').", + "role": "system", + }, + { + "role": "user", + "content": "Here is the input data (previous agent's output): \n\n Let's try to refine our search further, focusing more on the technical aspects of home automation and home energy system management:", + }, + ], + "response_format": { + "type": "json_schema", + "json_schema": { + "name": "final_output", + "strict": True, + "schema": { + "description": 'Progress report for the thinking process\n\nThis model represents a snapshot of the agent\'s current progress during\nthe thinking process, providing a brief description of the current activity.\n\nAttributes:\n agent_doing: Brief description of what the agent is currently doing.\n Should be kept under 10 words. Example: "Learning about home automation"', + "properties": { + "agent_doing": {"title": "Agent Doing", "type": "string"} + }, + "required": ["agent_doing"], + "title": "ThinkingStep", + "type": "object", + "additionalProperties": False, + }, + }, + }, + } + + base_completion_call_args = self.get_base_completion_call_args() + + response = self.completion_function( + **base_completion_call_args, **args, stream=True + ) + + chunks = [] + for chunk in response: + print(f"chunk: {chunk}") + chunks.append(chunk) + + print(f"chunks: {chunks}") + built_response = stream_chunk_builder(chunks=chunks) + + non_stream_response = self.completion_function( + **base_completion_call_args, **args, stream=False + ) + + assert json.loads(built_response.choices[0].message.content) == json.loads( + non_stream_response.choices[0].message.content + ), f"Got={json.loads(built_response.choices[0].message.content)}, Expected={json.loads(non_stream_response.choices[0].message.content)}" diff --git a/tests/llm_translation/test_anthropic_completion.py b/tests/llm_translation/test_anthropic_completion.py index da47e745e7..8f8f4084bb 100644 --- a/tests/llm_translation/test_anthropic_completion.py +++ b/tests/llm_translation/test_anthropic_completion.py @@ -36,7 +36,7 @@ from litellm.types.llms.openai import ChatCompletionToolCallFunctionChunk from litellm.llms.anthropic.common_utils import process_anthropic_headers from litellm.llms.anthropic.chat.handler import AnthropicChatCompletion from httpx import Headers -from base_llm_unit_tests import BaseLLMChatTest +from base_llm_unit_tests import BaseLLMChatTest, BaseAnthropicChatTest def streaming_format_tests(chunk: dict, idx: int): @@ -462,7 +462,7 @@ def test_create_json_tool_call_for_response_format(): from litellm import completion -class TestAnthropicCompletion(BaseLLMChatTest): +class TestAnthropicCompletion(BaseLLMChatTest, BaseAnthropicChatTest): def get_base_completion_call_args(self) -> dict: return {"model": "anthropic/claude-3-5-sonnet-20240620"} From 81a1494a51afe1302a76c5585722543b4c061790 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Fri, 21 Mar 2025 10:35:36 -0700 Subject: [PATCH 2/5] test: add unit testing --- .../test_anthropic_chat_transformation.py | 35 +++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 tests/litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py diff --git a/tests/litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py b/tests/litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py new file mode 100644 index 0000000000..04f2728284 --- /dev/null +++ b/tests/litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py @@ -0,0 +1,35 @@ +import json +import os +import sys + +import pytest +from fastapi.testclient import TestClient + +sys.path.insert( + 0, os.path.abspath("../../../../..") +) # Adds the parent directory to the system path +from unittest.mock import MagicMock, patch + +from litellm.llms.anthropic.chat.transformation import AnthropicConfig + + +def test_response_format_transformation_unit_test(): + config = AnthropicConfig() + + response_format_json_schema = { + "description": 'Progress report for the thinking process\n\nThis model represents a snapshot of the agent\'s current progress during\nthe thinking process, providing a brief description of the current activity.\n\nAttributes:\n agent_doing: Brief description of what the agent is currently doing.\n Should be kept under 10 words. Example: "Learning about home automation"', + "properties": {"agent_doing": {"title": "Agent Doing", "type": "string"}}, + "required": ["agent_doing"], + "title": "ThinkingStep", + "type": "object", + "additionalProperties": False, + } + + result = config._create_json_tool_call_for_response_format( + json_schema=response_format_json_schema + ) + + assert result["input_schema"]["properties"] == { + "agent_doing": {"title": "Agent Doing", "type": "string"} + } + print(result) From a1b716c1efaebde057220da349cb172982530483 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Fri, 21 Mar 2025 10:51:34 -0700 Subject: [PATCH 3/5] test: fix test - handle llm api inconsistency --- tests/llm_translation/base_llm_unit_tests.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/llm_translation/base_llm_unit_tests.py b/tests/llm_translation/base_llm_unit_tests.py index f3614fdb4c..82a1ef40fb 100644 --- a/tests/llm_translation/base_llm_unit_tests.py +++ b/tests/llm_translation/base_llm_unit_tests.py @@ -1062,6 +1062,7 @@ class BaseAnthropicChatTest(ABC): **base_completion_call_args, **args, stream=False ) - assert json.loads(built_response.choices[0].message.content) == json.loads( - non_stream_response.choices[0].message.content + assert ( + json.loads(built_response.choices[0].message.content).keys() + == json.loads(non_stream_response.choices[0].message.content).keys() ), f"Got={json.loads(built_response.choices[0].message.content)}, Expected={json.loads(non_stream_response.choices[0].message.content)}" From 86be28b640f746c2190196ea1c96ad18b93b0c00 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Fri, 21 Mar 2025 12:20:21 -0700 Subject: [PATCH 4/5] fix: fix linting error --- litellm/llms/anthropic/chat/transformation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/litellm/llms/anthropic/chat/transformation.py b/litellm/llms/anthropic/chat/transformation.py index aff70a6e62..1a77c453f4 100644 --- a/litellm/llms/anthropic/chat/transformation.py +++ b/litellm/llms/anthropic/chat/transformation.py @@ -387,7 +387,7 @@ class AnthropicConfig(BaseConfig): _input_schema["additionalProperties"] = True _input_schema["properties"] = {} else: - _input_schema.update(json_schema) + _input_schema.update(cast(AnthropicInputSchema, json_schema)) _tool = AnthropicMessagesTool( name=RESPONSE_FORMAT_TOOL_NAME, input_schema=_input_schema From 8265a88e0a423b019c7be25d3ed165c47da4c52f Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Fri, 21 Mar 2025 15:10:30 -0700 Subject: [PATCH 5/5] test: update tests --- tests/llm_translation/test_anthropic_completion.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/llm_translation/test_anthropic_completion.py b/tests/llm_translation/test_anthropic_completion.py index 8f8f4084bb..a83d1d69e9 100644 --- a/tests/llm_translation/test_anthropic_completion.py +++ b/tests/llm_translation/test_anthropic_completion.py @@ -455,7 +455,8 @@ def test_create_json_tool_call_for_response_format(): _input_schema = tool.get("input_schema") assert _input_schema is not None assert _input_schema.get("type") == "object" - assert _input_schema.get("properties") == {"values": custom_schema} + assert _input_schema.get("name") == custom_schema["name"] + assert _input_schema.get("age") == custom_schema["age"] assert "additionalProperties" not in _input_schema