From 7918188f1ef42c52fa1d4f67811d85129892cb56 Mon Sep 17 00:00:00 2001 From: Ashwin Bharambe Date: Wed, 22 Oct 2025 14:59:42 -0700 Subject: [PATCH] fix(ci): enable responses tests in CI; suppress expected MCP auth error logs (#3889) Let us enable responses suite in CI now. Also a minor fix: MCP tool tests intentionally trigger authentication failures to verify error handling, but the resulting error logs clutter test output. --- .github/workflows/integration-tests.yml | 2 +- .../responses/test_tool_responses.py | 21 ++++++++++++------- 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index 274c779ba..d38e8337b 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -61,7 +61,7 @@ jobs: && fromJSON('[{"setup": "vllm", "suite": "base"}]') || github.event.inputs.test-setup == 'ollama-vision' && fromJSON('[{"setup": "ollama-vision", "suite": "vision"}]') - || fromJSON('[{"setup": "ollama", "suite": "base"}, {"setup": "ollama-vision", "suite": "vision"}]') + || fromJSON('[{"setup": "ollama", "suite": "base"}, {"setup": "ollama-vision", "suite": "vision"}, {"setup": "gpt", "suite": "responses"}]') }} steps: diff --git a/tests/integration/responses/test_tool_responses.py b/tests/integration/responses/test_tool_responses.py index 2cff4d27d..3f1c35214 100644 --- a/tests/integration/responses/test_tool_responses.py +++ b/tests/integration/responses/test_tool_responses.py @@ -5,6 +5,7 @@ # the root directory of this source tree. import json +import logging # allow-direct-logging import os import httpx @@ -198,7 +199,7 @@ def test_response_sequential_file_search( @pytest.mark.parametrize("case", mcp_tool_test_cases) -def test_response_non_streaming_mcp_tool(compat_client, text_model_id, case): +def test_response_non_streaming_mcp_tool(compat_client, text_model_id, case, caplog): if not isinstance(compat_client, LlamaStackAsLibraryClient): pytest.skip("in-process MCP server is only supported in library client") @@ -245,13 +246,17 @@ def test_response_non_streaming_mcp_tool(compat_client, text_model_id, case): if isinstance(compat_client, LlamaStackAsLibraryClient) else (httpx.HTTPStatusError, openai.AuthenticationError) ) - with pytest.raises(exc_type): - compat_client.responses.create( - model=text_model_id, - input=case.input, - tools=tools, - stream=False, - ) + # Suppress expected auth error logs only for the failing auth attempt + with caplog.at_level( + logging.CRITICAL, logger="llama_stack.providers.inline.agents.meta_reference.responses.streaming" + ): + with pytest.raises(exc_type): + compat_client.responses.create( + model=text_model_id, + input=case.input, + tools=tools, + stream=False, + ) for tool in tools: if tool["type"] == "mcp":