From ee57e58f2995f4f3ac9f0792daaa06243d215f1f Mon Sep 17 00:00:00 2001 From: Ben Browning Date: Tue, 10 Jun 2025 08:45:55 -0400 Subject: [PATCH] fix: loosen tool call checks in inference store (#2420) # What does this PR do? This loosens up the tool call function name and arguments checks in `tests/integration/inference/test_openai_completion.py::test_inference_store_tool_calls` because the small models we use in CI cannot reliably get the tool call function name or arguments exactly right. Closes #2345 ## Test Plan I ran this flaking test in a loop, let it run many dozens of times, and didn't observe any flakes after the changes. Previously it flaked quite regularly. ``` while uv run pytest -s -v \ 'tests/integration/inference/test_openai_completion.py::test_inference_store_tool_calls[llama_stack_client-txt=3B-False]' \ --stack-config=http://localhost:8321 \ --text-model="meta-llama/Llama-3.2-3B-Instruct" \ --embedding-model=all-MiniLM-L6-v2; do; sleep 0.1; done ``` Signed-off-by: Ben Browning --- tests/integration/inference/test_openai_completion.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/tests/integration/inference/test_openai_completion.py b/tests/integration/inference/test_openai_completion.py index 190840f70..461527d18 100644 --- a/tests/integration/inference/test_openai_completion.py +++ b/tests/integration/inference/test_openai_completion.py @@ -372,10 +372,14 @@ def test_inference_store_tool_calls(compat_client, client_with_models, text_mode ) assert input_content == message, retrieved_response tool_calls = retrieved_response.choices[0].message.tool_calls - # sometimes model doesn't ouptut tool calls, but we still want to test that the tool was called + # sometimes model doesn't output tool calls, but we still want to test that the tool was called if tool_calls: + # because we test with small models, just check that we retrieved + # a tool call with a name and arguments string, but ignore contents assert len(tool_calls) == 1 - assert tool_calls[0].function.name == "get_weather" - assert "tokyo" in tool_calls[0].function.arguments.lower() + assert tool_calls[0].function.name + assert tool_calls[0].function.arguments else: + # failed tool call parses show up as a message with content, so ensure + # that the retrieve response content matches the original request assert retrieved_response.choices[0].message.content == content