From 743f4348603d4934df5c2352a6e3304b462de33c Mon Sep 17 00:00:00 2001 From: Yuan Tang Date: Sat, 15 Feb 2025 00:19:16 -0500 Subject: [PATCH] fix: Ensure a tool call can be converted before adding to buffer (#1119) # What does this PR do? This fixes an issue when running the e2e agent example: https://github.com/meta-llama/llama-stack-apps/blob/main/examples/agents/e2e_loop_with_client_tools.py ``` | File "/home/yutang/repos/llama-stack/llama_stack/providers/remote/inference/vllm/vllm.py", line 175, in _process_vllm_chat_completion_stream_response | tool_call = convert_tool_call(choice.delta.tool_calls[0]) | File "/home/yutang/repos/llama-stack/llama_stack/providers/utils/inference/openai_compat.py", line 441, in convert_tool_call | return ToolCall( | File "/home/yutang/.conda/envs/distribution-myenv/lib/python3.10/site-packages/pydantic/main.py", line 214, in __init__ | validated_self = self.__pydantic_validator__.validate_python(data, self_instance=self) | pydantic_core._pydantic_core.ValidationError: 4 validation errors for ToolCall | call_id | Input should be a valid string [type=string_type, input_value=None, input_type=NoneType] | For further information visit https://errors.pydantic.dev/2.10/v/string_type | tool_name.enum[BuiltinTool] | Input should be 'brave_search', 'wolfram_alpha', 'photogen' or 'code_interpreter' [type=enum, input_value=None, input_type=NoneType] | For further information visit https://errors.pydantic.dev/2.10/v/enum | tool_name.str | Input should be a valid string [type=string_type, input_value=None, input_type=NoneType] | For further information visit https://errors.pydantic.dev/2.10/v/string_type | arguments | Input should be a valid dictionary [type=dict_type, input_value=202, input_type=int] | For further information visit https://errors.pydantic.dev/2.10/v/dict_type ``` This issue happened because not all arguments have been appended to the tool call buffer yet. The current code assumes that we are ready to convert the tool call whenever args can be converted to JSON successfully. In this case, `json.loads("202")` would succeed but the rest of the arguments have not been properly parsed yet. [//]: # (If resolving an issue, uncomment and update the line below) [//]: # (Closes #[issue-number]) ## Test Plan The e2e example worked successfully (although note that I ran the script twice with each function call separately due to https://github.com/meta-llama/llama-stack/issues/1120): ``` tool_execution> Tool:get_ticker_data Args:{'ticker_symbol': 'GOOG', 'start': '2023-01-01', 'end': '2023-12-31'} tool_execution> Tool:get_ticker_data Response:"[{\"('Year', '')\":2023,\"('Close', 'GOOG')\":140.4254455566}]" tool_execution> Tool:web_search Args:{'query': '42nd president of the United States'} tool_execution> Tool:web_search Response:"{\"query\": \"42nd president of the United States\", \"top_k\": [{\"title\": \"William J. Clinton | whitehouse.gov\", \"url\": \"https://obamawhitehouse.archives.gov/1600/presidents/williamjclinton\", \"description\": \"Bill Clinton is an American politician from Arkansas who served as the 42nd President of the United States (1993-2001). He took office at the end of the Cold War, and was the first baby-boomer generation President.\", \"type\": \"search_result\"}, {\"title\": \"Bill Clinton - Wikipedia\", \"url\": \"https://en.wikipedia.org/wiki/Bill_Clinton\", \"description\": \"William Jefferson Clinton (n\\u00e9 Blythe; born August 19, 1946) is an American politician and lawyer who served as the 42nd president of the United States from 1993 to 2001. A member of the Democratic Party, he previously served as the attorney general of Arkansas from 1977 to 1979 and as the ...\", \"type\": \"search_result\"}, [{\"type\": \"video_result\", \"url\": \"https://www.youtube.com/watch?v=eR2z_1-v87Y\", \"title\": \"A Conversation with Bill Clinton, 42nd President of the United ...\", \"description\": \"William Jefferson Clinton, the first Democratic president in six decades to be elected twice, led the United States to the longest economic expansion in Amer...\"}, {\"type\": \"video_result\", \"url\": \"https://www.facebook.com/clintoncenter/videos/january-20-1993-president-clinton-was-sworn-in-as-the-42nd-president-of-the-unit/448417409677375/\", \"title\": \"January 20, 1993, President Clinton was sworn in as the 42nd ...\", \"description\": \"WATCH: On January 20, 1993, President Bill Clinton was sworn in as the 42nd President of the United States. #InaugurationDay Video courtesy of the...\"}, {\"type\": \"video_result\", \"url\": \"https://www.youtube.com/watch?v=vI0HGQqEJh0\", \"title\": \"42nd President of the United States, Bill Clinton, shared thoughts ...\", \"description\": \"AboutPressCopyrightContact usCreatorsAdvertiseDevelopersTermsPrivacyPolicy & SafetyHow YouTube worksTest new features \\u00b7 \\u00a9 2024 Google LLC\"}, {\"type\": \"video_result\", \"url\": \"https://www.youtube.com/shorts/vI0HGQqEJh0\", \"title\": \"42nd President of the United States, Bill Clinton, shared ...\", \"description\": \"Enjoy the videos and music you love, upload original content, and share it all with friends, family, and the world on YouTube.\"}, {\"type\": \"video_result\", \"url\": \"https://www.youtube.com/watch?v=PHihhihVth0\", \"title\": \"Bill & Hillary Clinton returning to Little Rock for 20th ...\", \"description\": \"Enjoy the videos and music you love, upload original content, and share it all with friends, family, and the world on YouTube.\"}]]}" ``` All text inference tests passed. [//]: # (## Documentation) Signed-off-by: Yuan Tang --- .../providers/utils/inference/openai_compat.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/llama_stack/providers/utils/inference/openai_compat.py b/llama_stack/providers/utils/inference/openai_compat.py index da8e3ce2d..def7e8f37 100644 --- a/llama_stack/providers/utils/inference/openai_compat.py +++ b/llama_stack/providers/utils/inference/openai_compat.py @@ -427,10 +427,14 @@ def convert_tool_call( """ Convert a ChatCompletionMessageToolCall tool call to either a ToolCall or UnparseableToolCall. Returns an UnparseableToolCall - if the tool call is not valid JSON. + if the tool call is not valid ToolCall. """ try: - arguments = json.loads(tool_call.function.arguments) + valid_tool_call = ToolCall( + call_id=tool_call.id, + tool_name=tool_call.function.name, + arguments=json.loads(tool_call.function.arguments), + ) except Exception as e: return UnparseableToolCall( call_id=tool_call.id or "", @@ -438,8 +442,4 @@ def convert_tool_call( arguments=tool_call.function.arguments or "", ) - return ToolCall( - call_id=tool_call.id, - tool_name=tool_call.function.name, - arguments=arguments, - ) + return valid_tool_call