From 7ef3b680a25f893a7db3e5170d2a64beb6657bbc Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Tue, 12 Nov 2024 14:37:55 -0800 Subject: [PATCH 1/5] add support for response_format=json anthropic --- litellm/llms/anthropic/chat/transformation.py | 58 ++++++++++++++----- litellm/types/llms/anthropic.py | 8 ++- 2 files changed, 49 insertions(+), 17 deletions(-) diff --git a/litellm/llms/anthropic/chat/transformation.py b/litellm/llms/anthropic/chat/transformation.py index 18c53b696..47264b8f0 100644 --- a/litellm/llms/anthropic/chat/transformation.py +++ b/litellm/llms/anthropic/chat/transformation.py @@ -7,6 +7,7 @@ from litellm.types.llms.anthropic import ( AllAnthropicToolsValues, AnthropicComputerTool, AnthropicHostedTools, + AnthropicInputSchema, AnthropicMessageRequestBase, AnthropicMessagesRequest, AnthropicMessagesTool, @@ -159,15 +160,19 @@ class AnthropicConfig: returned_tool: Optional[AllAnthropicToolsValues] = None if tool["type"] == "function" or tool["type"] == "custom": + _input_function_parameters: dict = ( + tool["function"].get("parameters", None) or {} + ) + _tool_input_schema: AnthropicInputSchema = AnthropicInputSchema( + type=_input_function_parameters.get("type", "object"), + properties=_input_function_parameters.get("properties", {}), + additionalProperties=_input_function_parameters.get( + "additionalProperties", True + ), + ) _tool = AnthropicMessagesTool( name=tool["function"]["name"], - input_schema=tool["function"].get( - "parameters", - { - "type": "object", - "properties": {}, - }, - ), + input_schema=_tool_input_schema, ) _description = tool["function"].get("description") @@ -304,17 +309,10 @@ class AnthropicConfig: - You should set tool_choice (see Forcing tool use) to instruct the model to explicitly use that tool - Remember that the model will pass the input to the tool, so the name of the tool and description should be from the model’s perspective. """ - _tool_choice = None _tool_choice = {"name": "json_tool_call", "type": "tool"} - - _tool = AnthropicMessagesTool( - name="json_tool_call", - input_schema={ - "type": "object", - "properties": {"values": json_schema}, # type: ignore - }, + _tool = self._create_json_tool_call_for_response_format( + json_schema=json_schema, ) - optional_params["tools"] = [_tool] optional_params["tool_choice"] = _tool_choice optional_params["json_mode"] = True @@ -341,6 +339,34 @@ class AnthropicConfig: return optional_params + def _create_json_tool_call_for_response_format( + self, + json_schema: Optional[dict] = None, + ) -> AnthropicMessagesTool: + """ + Handles creating a tool call for getting responses in JSON format. + + Args: + json_schema (Optional[dict]): The JSON schema the response should be in + + Returns: + AnthropicMessagesTool: The tool call to send to Anthropic API to get responses in JSON format + """ + _input_schema: AnthropicInputSchema = AnthropicInputSchema( + type="object", + ) + + if json_schema is None: + # Anthropic raises a 400 BadRequest error if properties is passed as None + # see usage with additionalProperties (Example 5) https://github.com/anthropics/anthropic-cookbook/blob/main/tool_use/extracting_structured_json.ipynb + _input_schema["additionalProperties"] = True + _input_schema["properties"] = {} + else: + _input_schema["properties"] = json_schema + + _tool = AnthropicMessagesTool(name="json_tool_call", input_schema=_input_schema) + return _tool + def is_cache_control_set(self, messages: List[AllMessageValues]) -> bool: """ Return if {"cache_control": ..} in message content block diff --git a/litellm/types/llms/anthropic.py b/litellm/types/llms/anthropic.py index b0a3780b8..55e37ad97 100644 --- a/litellm/types/llms/anthropic.py +++ b/litellm/types/llms/anthropic.py @@ -12,10 +12,16 @@ class AnthropicMessagesToolChoice(TypedDict, total=False): disable_parallel_tool_use: bool # default is false +class AnthropicInputSchema(TypedDict, total=False): + type: Optional[str] + properties: Optional[dict] + additionalProperties: Optional[bool] + + class AnthropicMessagesTool(TypedDict, total=False): name: Required[str] description: str - input_schema: Required[dict] + input_schema: Optional[AnthropicInputSchema] type: Literal["custom"] cache_control: Optional[Union[dict, ChatCompletionCachedContent]] From 368959f9f318f3eec2789e246b60dcc289ad65bb Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Tue, 12 Nov 2024 14:40:11 -0800 Subject: [PATCH 2/5] add test_json_response_format to baseLLM ChatTest --- tests/llm_translation/base_llm_unit_tests.py | 26 ++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/tests/llm_translation/base_llm_unit_tests.py b/tests/llm_translation/base_llm_unit_tests.py index 18ac7216f..acb764ba1 100644 --- a/tests/llm_translation/base_llm_unit_tests.py +++ b/tests/llm_translation/base_llm_unit_tests.py @@ -53,6 +53,32 @@ class BaseLLMChatTest(ABC): response = litellm.completion(**base_completion_call_args, messages=messages) assert response is not None + def test_json_response_format(self): + """ + Test that the JSON response format is supported by the LLM API + """ + base_completion_call_args = self.get_base_completion_call_args() + litellm.set_verbose = True + + messages = [ + { + "role": "system", + "content": "Your output should be a JSON object with no additional properties. ", + }, + { + "role": "user", + "content": "Respond with this in json. city=San Francisco, state=CA, weather=sunny, temp=60", + }, + ] + + response = litellm.completion( + **base_completion_call_args, + messages=messages, + response_format={"type": "json_object"}, + ) + + print(response) + @pytest.fixture def pdf_messages(self): import base64 From 186a6792432aabb361d49c06b01f6c2f8a1ffadc Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Tue, 12 Nov 2024 15:01:03 -0800 Subject: [PATCH 3/5] fix test_litellm_anthropic_prompt_caching_tools --- litellm/llms/anthropic/chat/transformation.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/litellm/llms/anthropic/chat/transformation.py b/litellm/llms/anthropic/chat/transformation.py index 47264b8f0..10f1fbab1 100644 --- a/litellm/llms/anthropic/chat/transformation.py +++ b/litellm/llms/anthropic/chat/transformation.py @@ -164,11 +164,7 @@ class AnthropicConfig: tool["function"].get("parameters", None) or {} ) _tool_input_schema: AnthropicInputSchema = AnthropicInputSchema( - type=_input_function_parameters.get("type", "object"), - properties=_input_function_parameters.get("properties", {}), - additionalProperties=_input_function_parameters.get( - "additionalProperties", True - ), + **_input_function_parameters ) _tool = AnthropicMessagesTool( name=tool["function"]["name"], From 3ccbd5bb7bcd9bb7597d90ae706217812413574c Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Tue, 12 Nov 2024 18:02:24 -0800 Subject: [PATCH 4/5] fix test_anthropic_function_call_with_no_schema --- litellm/llms/anthropic/chat/transformation.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/litellm/llms/anthropic/chat/transformation.py b/litellm/llms/anthropic/chat/transformation.py index 10f1fbab1..e222d8721 100644 --- a/litellm/llms/anthropic/chat/transformation.py +++ b/litellm/llms/anthropic/chat/transformation.py @@ -160,15 +160,17 @@ class AnthropicConfig: returned_tool: Optional[AllAnthropicToolsValues] = None if tool["type"] == "function" or tool["type"] == "custom": - _input_function_parameters: dict = ( - tool["function"].get("parameters", None) or {} - ) - _tool_input_schema: AnthropicInputSchema = AnthropicInputSchema( - **_input_function_parameters + _input_schema: dict = tool["function"].get( + "parameters", + { + "type": "object", + "properties": {}, + }, ) + input_schema: AnthropicInputSchema = AnthropicInputSchema(**_input_schema) _tool = AnthropicMessagesTool( name=tool["function"]["name"], - input_schema=_tool_input_schema, + input_schema=input_schema, ) _description = tool["function"].get("description") From 06b3cfb5fbb8a104acfae63112bd5e53582cad7a Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Tue, 12 Nov 2024 18:09:06 -0800 Subject: [PATCH 5/5] test test_create_json_tool_call_for_response_format --- .../test_anthropic_completion.py | 32 +++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/tests/llm_translation/test_anthropic_completion.py b/tests/llm_translation/test_anthropic_completion.py index 9d7c9af73..c399c3a47 100644 --- a/tests/llm_translation/test_anthropic_completion.py +++ b/tests/llm_translation/test_anthropic_completion.py @@ -627,6 +627,38 @@ def test_anthropic_tool_helper(cache_control_location): assert tool["cache_control"] == {"type": "ephemeral"} +def test_create_json_tool_call_for_response_format(): + """ + tests using response_format=json with anthropic + + A tool call to anthropic is made when response_format=json is used. + + """ + # Initialize AnthropicConfig + config = AnthropicConfig() + + # Test case 1: No schema provided + # See Anthropics Example 5 on how to handle cases when no schema is provided https://github.com/anthropics/anthropic-cookbook/blob/main/tool_use/extracting_structured_json.ipynb + tool = config._create_json_tool_call_for_response_format() + assert tool["name"] == "json_tool_call" + _input_schema = tool.get("input_schema") + assert _input_schema is not None + assert _input_schema.get("type") == "object" + assert _input_schema.get("additionalProperties") is True + assert _input_schema.get("properties") == {} + + # Test case 2: With custom schema + # reference: https://github.com/anthropics/anthropic-cookbook/blob/main/tool_use/extracting_structured_json.ipynb + custom_schema = {"name": {"type": "string"}, "age": {"type": "integer"}} + tool = config._create_json_tool_call_for_response_format(json_schema=custom_schema) + assert tool["name"] == "json_tool_call" + _input_schema = tool.get("input_schema") + assert _input_schema is not None + assert _input_schema.get("type") == "object" + assert _input_schema.get("properties") == custom_schema + assert "additionalProperties" not in _input_schema + + from litellm import completion