From 7b90e0e9c827aefee30e84d9d963974ea644292e Mon Sep 17 00:00:00 2001 From: Ashwin Bharambe Date: Wed, 22 Oct 2025 14:34:32 -0700 Subject: [PATCH] test: suppress expected error logs in SSE test (#3886) Our unit test outputs are filled with all kinds of obscene logs. This makes it really hard to spot real issues quickly. The problem is that these logs are necessary to output at the given logging level when the server is operating normally. It's just that we don't want to see some of them (especially the noisy ones) during tests. This PR begins the cleanup. We pytest's caplog fixture to for suppression. --- tests/unit/conftest.py | 9 ++++++++ tests/unit/server/test_auth.py | 32 ++++++++++++++++++--------- tests/unit/server/test_auth_github.py | 10 ++++++++- tests/unit/server/test_quota.py | 15 +++++++++---- tests/unit/server/test_sse.py | 11 ++++++++- 5 files changed, 60 insertions(+), 17 deletions(-) diff --git a/tests/unit/conftest.py b/tests/unit/conftest.py index 1ae96d448..893cc4a7d 100644 --- a/tests/unit/conftest.py +++ b/tests/unit/conftest.py @@ -4,9 +4,12 @@ # This source code is licensed under the terms described in the LICENSE file in # the root directory of this source tree. +import logging # allow-direct-logging import os import warnings +import pytest + def pytest_sessionstart(session) -> None: if "LLAMA_STACK_LOGGING" not in os.environ: @@ -17,4 +20,10 @@ def pytest_sessionstart(session) -> None: warnings.filterwarnings("ignore", category=PendingDeprecationWarning) +@pytest.fixture(autouse=True) +def suppress_httpx_logs(caplog): + """Suppress httpx INFO logs for all unit tests""" + caplog.set_level(logging.WARNING, logger="httpx") + + pytest_plugins = ["tests.unit.fixtures"] diff --git a/tests/unit/server/test_auth.py b/tests/unit/server/test_auth.py index 75cbf518b..cc9397f07 100644 --- a/tests/unit/server/test_auth.py +++ b/tests/unit/server/test_auth.py @@ -6,6 +6,7 @@ import base64 import json +import logging # allow-direct-logging from unittest.mock import AsyncMock, Mock, patch import pytest @@ -27,6 +28,13 @@ from llama_stack.core.server.auth_providers import ( ) +@pytest.fixture +def suppress_auth_errors(caplog): + """Suppress expected ERROR/WARNING logs for tests that deliberately trigger authentication errors""" + caplog.set_level(logging.CRITICAL, logger="llama_stack.core.server.auth") + caplog.set_level(logging.CRITICAL, logger="llama_stack.core.server.auth_providers") + + class MockResponse: def __init__(self, status_code, json_data): self.status_code = status_code @@ -237,20 +245,20 @@ def test_valid_http_authentication(http_client, valid_api_key): @patch("httpx.AsyncClient.post", new=mock_post_failure) -def test_invalid_http_authentication(http_client, invalid_api_key): +def test_invalid_http_authentication(http_client, invalid_api_key, suppress_auth_errors): response = http_client.get("/test", headers={"Authorization": f"Bearer {invalid_api_key}"}) assert response.status_code == 401 assert "Authentication failed" in response.json()["error"]["message"] @patch("httpx.AsyncClient.post", new=mock_post_exception) -def test_http_auth_service_error(http_client, valid_api_key): +def test_http_auth_service_error(http_client, valid_api_key, suppress_auth_errors): response = http_client.get("/test", headers={"Authorization": f"Bearer {valid_api_key}"}) assert response.status_code == 401 assert "Authentication service error" in response.json()["error"]["message"] -def test_http_auth_request_payload(http_client, valid_api_key, mock_auth_endpoint): +def test_http_auth_request_payload(http_client, valid_api_key, mock_auth_endpoint, suppress_auth_errors): with patch("httpx.AsyncClient.post") as mock_post: mock_response = MockResponse(200, {"message": "Authentication successful"}) mock_post.return_value = mock_response @@ -420,7 +428,7 @@ def test_valid_oauth2_authentication(oauth2_client, jwt_token_valid, mock_jwks_u @patch("httpx.AsyncClient.get", new=mock_jwks_response) -def test_invalid_oauth2_authentication(oauth2_client, invalid_token): +def test_invalid_oauth2_authentication(oauth2_client, invalid_token, suppress_auth_errors): response = oauth2_client.get("/test", headers={"Authorization": f"Bearer {invalid_token}"}) assert response.status_code == 401 assert "Invalid JWT token" in response.json()["error"]["message"] @@ -465,7 +473,7 @@ def oauth2_client_with_jwks_token(oauth2_app_with_jwks_token): @patch("httpx.AsyncClient.get", new=mock_auth_jwks_response) -def test_oauth2_with_jwks_token_expected(oauth2_client, jwt_token_valid): +def test_oauth2_with_jwks_token_expected(oauth2_client, jwt_token_valid, suppress_auth_errors): response = oauth2_client.get("/test", headers={"Authorization": f"Bearer {jwt_token_valid}"}) assert response.status_code == 401 @@ -726,21 +734,21 @@ def test_valid_introspection_authentication(introspection_client, valid_api_key) @patch("httpx.AsyncClient.post", new=mock_introspection_inactive) -def test_inactive_introspection_authentication(introspection_client, invalid_api_key): +def test_inactive_introspection_authentication(introspection_client, invalid_api_key, suppress_auth_errors): response = introspection_client.get("/test", headers={"Authorization": f"Bearer {invalid_api_key}"}) assert response.status_code == 401 assert "Token not active" in response.json()["error"]["message"] @patch("httpx.AsyncClient.post", new=mock_introspection_invalid) -def test_invalid_introspection_authentication(introspection_client, invalid_api_key): +def test_invalid_introspection_authentication(introspection_client, invalid_api_key, suppress_auth_errors): response = introspection_client.get("/test", headers={"Authorization": f"Bearer {invalid_api_key}"}) assert response.status_code == 401 assert "Not JSON" in response.json()["error"]["message"] @patch("httpx.AsyncClient.post", new=mock_introspection_failed) -def test_failed_introspection_authentication(introspection_client, invalid_api_key): +def test_failed_introspection_authentication(introspection_client, invalid_api_key, suppress_auth_errors): response = introspection_client.get("/test", headers={"Authorization": f"Bearer {invalid_api_key}"}) assert response.status_code == 401 assert "Token introspection failed: 500" in response.json()["error"]["message"] @@ -957,20 +965,22 @@ def test_valid_kubernetes_auth_authentication(kubernetes_auth_client, valid_toke @patch("httpx.AsyncClient.post", new=mock_kubernetes_selfsubjectreview_failure) -def test_invalid_kubernetes_auth_authentication(kubernetes_auth_client, invalid_token): +def test_invalid_kubernetes_auth_authentication(kubernetes_auth_client, invalid_token, suppress_auth_errors): response = kubernetes_auth_client.get("/test", headers={"Authorization": f"Bearer {invalid_token}"}) assert response.status_code == 401 assert "Invalid token" in response.json()["error"]["message"] @patch("httpx.AsyncClient.post", new=mock_kubernetes_selfsubjectreview_http_error) -def test_kubernetes_auth_http_error(kubernetes_auth_client, valid_token): +def test_kubernetes_auth_http_error(kubernetes_auth_client, valid_token, suppress_auth_errors): response = kubernetes_auth_client.get("/test", headers={"Authorization": f"Bearer {valid_token}"}) assert response.status_code == 401 assert "Token validation failed" in response.json()["error"]["message"] -def test_kubernetes_auth_request_payload(kubernetes_auth_client, valid_token, mock_kubernetes_api_server): +def test_kubernetes_auth_request_payload( + kubernetes_auth_client, valid_token, mock_kubernetes_api_server, suppress_auth_errors +): with patch("httpx.AsyncClient.post") as mock_post: mock_response = MockResponse( 200, diff --git a/tests/unit/server/test_auth_github.py b/tests/unit/server/test_auth_github.py index d87643579..f458f9a94 100644 --- a/tests/unit/server/test_auth_github.py +++ b/tests/unit/server/test_auth_github.py @@ -4,6 +4,7 @@ # This source code is licensed under the terms described in the LICENSE file in # the root directory of this source tree. +import logging # allow-direct-logging from unittest.mock import AsyncMock, patch import httpx @@ -15,6 +16,13 @@ from llama_stack.core.datatypes import AuthenticationConfig, AuthProviderType, G from llama_stack.core.server.auth import AuthenticationMiddleware +@pytest.fixture +def suppress_auth_errors(caplog): + """Suppress expected ERROR logs for tests that deliberately trigger authentication errors""" + caplog.set_level(logging.CRITICAL, logger="llama_stack.core.server.auth") + caplog.set_level(logging.CRITICAL, logger="llama_stack.core.server.auth_providers") + + class MockResponse: def __init__(self, status_code, json_data): self.status_code = status_code @@ -119,7 +127,7 @@ def test_authenticated_endpoint_with_valid_github_token(mock_client_class, githu @patch("llama_stack.core.server.auth_providers.httpx.AsyncClient") -def test_authenticated_endpoint_with_invalid_github_token(mock_client_class, github_token_client): +def test_authenticated_endpoint_with_invalid_github_token(mock_client_class, github_token_client, suppress_auth_errors): """Test accessing protected endpoint with invalid GitHub token""" # Mock the GitHub API to return 401 Unauthorized mock_client = AsyncMock() diff --git a/tests/unit/server/test_quota.py b/tests/unit/server/test_quota.py index 16b1772ce..0939414dd 100644 --- a/tests/unit/server/test_quota.py +++ b/tests/unit/server/test_quota.py @@ -4,6 +4,7 @@ # This source code is licensed under the terms described in the LICENSE file in # the root directory of this source tree. +import logging # allow-direct-logging from uuid import uuid4 import pytest @@ -17,6 +18,12 @@ from llama_stack.core.storage.datatypes import KVStoreReference, SqliteKVStoreCo from llama_stack.providers.utils.kvstore import register_kvstore_backends +@pytest.fixture +def suppress_quota_warnings(caplog): + """Suppress expected WARNING logs for SQLite backend and quota exceeded""" + caplog.set_level(logging.CRITICAL, logger="llama_stack.core.server.quota") + + class InjectClientIDMiddleware(BaseHTTPMiddleware): """ Middleware that injects 'authenticated_client_id' to mimic AuthenticationMiddleware. @@ -70,13 +77,13 @@ def auth_app(tmp_path, request): return app -def test_authenticated_quota_allows_up_to_limit(auth_app): +def test_authenticated_quota_allows_up_to_limit(auth_app, suppress_quota_warnings): client = TestClient(auth_app) assert client.get("/test").status_code == 200 assert client.get("/test").status_code == 200 -def test_authenticated_quota_blocks_after_limit(auth_app): +def test_authenticated_quota_blocks_after_limit(auth_app, suppress_quota_warnings): client = TestClient(auth_app) client.get("/test") client.get("/test") @@ -85,7 +92,7 @@ def test_authenticated_quota_blocks_after_limit(auth_app): assert resp.json()["error"]["message"] == "Quota exceeded" -def test_anonymous_quota_allows_up_to_limit(tmp_path, request): +def test_anonymous_quota_allows_up_to_limit(tmp_path, request, suppress_quota_warnings): inner_app = FastAPI() @inner_app.get("/test") @@ -107,7 +114,7 @@ def test_anonymous_quota_allows_up_to_limit(tmp_path, request): assert client.get("/test").status_code == 200 -def test_anonymous_quota_blocks_after_limit(tmp_path, request): +def test_anonymous_quota_blocks_after_limit(tmp_path, request, suppress_quota_warnings): inner_app = FastAPI() @inner_app.get("/test") diff --git a/tests/unit/server/test_sse.py b/tests/unit/server/test_sse.py index 54afe4ee4..f36c8c181 100644 --- a/tests/unit/server/test_sse.py +++ b/tests/unit/server/test_sse.py @@ -5,12 +5,21 @@ # the root directory of this source tree. import asyncio +import logging # allow-direct-logging from unittest.mock import AsyncMock, MagicMock +import pytest + from llama_stack.apis.common.responses import PaginatedResponse from llama_stack.core.server.server import create_dynamic_typed_route, create_sse_event, sse_generator +@pytest.fixture +def suppress_sse_errors(caplog): + """Suppress expected ERROR logs for tests that deliberately trigger SSE errors""" + caplog.set_level(logging.CRITICAL, logger="llama_stack.core.server.server") + + async def test_sse_generator_basic(): # An AsyncIterator wrapped in an Awaitable, just like our web methods async def async_event_gen(): @@ -70,7 +79,7 @@ async def test_sse_generator_client_disconnected_before_response_starts(): assert len(seen_events) == 0 -async def test_sse_generator_error_before_response_starts(): +async def test_sse_generator_error_before_response_starts(suppress_sse_errors): # Raise an error before the response starts async def async_event_gen(): raise Exception("Test error")