feat(tests): introduce a test "suite" concept to encompass dirs, options (#3339)

Our integration tests need to be 'grouped' because each group often
needs a specific set of models it works with. We separated vision tests
due to this, and we have a separate set of tests which test "Responses"
API.

This PR makes this system a bit more official so it is very easy to
target these groups and apply all testing infrastructure towards all the
groups (for example, record-replay) uniformly.

There are three suites declared:
- base
- vision
- responses

Note that our CI currently runs the "base" and "vision" suites.

You can use the `--suite` option when running pytest (or any of the
testing scripts or workflows.) For example:
```
OLLAMA_URL=http://localhost:11434 \
  pytest -s -v tests/integration/ --stack-config starter --suite vision
```

tests/integration/README.md

@@ -42,6 +42,27 @@ Model parameters can be influenced by the following options:
 Each of these are comma-separated lists and can be used to generate multiple parameter combinations. Note that tests will be skipped
 if no model is specified.
 
+### Suites (fast selection + sane defaults)
+
+- `--suite`: comma-separated list of named suites that both narrow which tests are collected and prefill common model options (unless you pass them explicitly).
+- Available suites:
+  - `responses`: collects tests under `tests/integration/responses`; this is a separate suite because it needs a strong tool-calling model.
+  - `vision`: collects only `tests/integration/inference/test_vision_inference.py`; defaults `--vision-model=ollama/llama3.2-vision:11b`, `--embedding-model=sentence-transformers/all-MiniLM-L6-v2`.
+- Explicit flags always win. For example, `--suite=responses --text-model=<X>` overrides the suite’s text model.
+
+Examples:
+
+```bash
+# Fast responses run with defaults
+pytest -s -v tests/integration --stack-config=server:starter --suite=responses
+
+# Fast single-file vision run with defaults
+pytest -s -v tests/integration --stack-config=server:starter --suite=vision
+
+# Combine suites and override a default
+pytest -s -v tests/integration --stack-config=server:starter --suite=responses,vision --embedding-model=text-embedding-3-small
+```
+
 ## Examples
 
 ### Testing against a Server

tests/integration/conftest.py

@@ -6,15 +6,17 @@
 import inspect
 import itertools
 import os
-import platform
 import textwrap
 import time
+from pathlib import Path
 
 import pytest
 from dotenv import load_dotenv
 
 from llama_stack.log import get_logger
 
+from .suites import SUITE_DEFINITIONS
+
 logger = get_logger(__name__, category="tests")
 
 
@@ -61,9 +63,22 @@ def pytest_configure(config):
         key, value = env_var.split("=", 1)
         os.environ[key] = value
 
-    if platform.system() == "Darwin":  # Darwin is the system name for macOS
-        os.environ["DISABLE_CODE_SANDBOX"] = "1"
-        logger.info("Setting DISABLE_CODE_SANDBOX=1 for macOS")
+    suites_raw = config.getoption("--suite")
+    suites: list[str] = []
+    if suites_raw:
+        suites = [p.strip() for p in str(suites_raw).split(",") if p.strip()]
+        unknown = [p for p in suites if p not in SUITE_DEFINITIONS]
+        if unknown:
+            raise pytest.UsageError(
+                f"Unknown suite(s): {', '.join(unknown)}. Available: {', '.join(sorted(SUITE_DEFINITIONS.keys()))}"
+            )
+    for suite in suites:
+        suite_def = SUITE_DEFINITIONS.get(suite, {})
+        defaults: dict = suite_def.get("defaults", {})
+        for dest, value in defaults.items():
+            current = getattr(config.option, dest, None)
+            if not current:
+                setattr(config.option, dest, value)
 
 
 def pytest_addoption(parser):
@@ -105,16 +120,21 @@ def pytest_addoption(parser):
         default=384,
         help="Output dimensionality of the embedding model to use for testing. Default: 384",
     )
-    parser.addoption(
-        "--record-responses",
-        action="store_true",
-        help="Record new API responses instead of using cached ones.",
-    )
     parser.addoption(
         "--report",
         help="Path where the test report should be written, e.g. --report=/path/to/report.md",
     )
 
+    available_suites = ", ".join(sorted(SUITE_DEFINITIONS.keys()))
+    suite_help = (
+        "Comma-separated integration test suites to narrow collection and prefill defaults. "
+        "Available: "
+        f"{available_suites}. "
+        "Explicit CLI flags (e.g., --text-model) override suite defaults. "
+        "Examples: --suite=responses or --suite=responses,vision."
+    )
+    parser.addoption("--suite", help=suite_help)
+
 
 MODEL_SHORT_IDS = {
     "meta-llama/Llama-3.2-3B-Instruct": "3B",
@@ -197,3 +217,40 @@ def pytest_generate_tests(metafunc):
 
 
 pytest_plugins = ["tests.integration.fixtures.common"]
+
+
+def pytest_ignore_collect(path: str, config: pytest.Config) -> bool:
+    """Skip collecting paths outside the selected suite roots for speed."""
+    suites_raw = config.getoption("--suite")
+    if not suites_raw:
+        return False
+
+    names = [p.strip() for p in str(suites_raw).split(",") if p.strip()]
+    roots: list[str] = []
+    for name in names:
+        suite_def = SUITE_DEFINITIONS.get(name)
+        if suite_def:
+            roots.extend(suite_def.get("roots", []))
+    if not roots:
+        return False
+
+    p = Path(str(path)).resolve()
+
+    # Only constrain within tests/integration to avoid ignoring unrelated tests
+    integration_root = (Path(str(config.rootpath)) / "tests" / "integration").resolve()
+    if not p.is_relative_to(integration_root):
+        return False
+
+    for r in roots:
+        rp = (Path(str(config.rootpath)) / r).resolve()
+        if rp.is_file():
+            # Allow the exact file and any ancestor directories so pytest can walk into it.
+            if p == rp:
+                return False
+            if p.is_dir() and rp.is_relative_to(p):
+                return False
+        else:
+            # Allow anything inside an allowed directory
+            if p.is_relative_to(rp):
+                return False
+    return True

tests/integration/suites.py

@@ -0,0 +1,53 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the terms described in the LICENSE file in
+# the root directory of this source tree.
+
+# Central definition of integration test suites. You can use these suites by passing --suite=name to pytest.
+# For example:
+#
+# ```bash
+# pytest tests/integration/ --suite=vision
+# ```
+#
+# Each suite can:
+# - restrict collection to specific roots (dirs or files)
+# - provide default CLI option values (e.g. text_model, embedding_model, etc.)
+
+from pathlib import Path
+
+this_dir = Path(__file__).parent
+default_roots = [
+    str(p)
+    for p in this_dir.glob("*")
+    if p.is_dir()
+    and p.name not in ("__pycache__", "fixtures", "test_cases", "recordings", "responses", "post_training")
+]
+
+SUITE_DEFINITIONS: dict[str, dict] = {
+    "base": {
+        "description": "Base suite that includes most tests but runs them with a text Ollama model",
+        "roots": default_roots,
+        "defaults": {
+            "text_model": "ollama/llama3.2:3b-instruct-fp16",
+            "embedding_model": "sentence-transformers/all-MiniLM-L6-v2",
+        },
+    },
+    "responses": {
+        "description": "Suite that includes only the OpenAI Responses tests; needs a strong tool-calling model",
+        "roots": ["tests/integration/responses"],
+        "defaults": {
+            "text_model": "openai/gpt-4o",
+            "embedding_model": "sentence-transformers/all-MiniLM-L6-v2",
+        },
+    },
+    "vision": {
+        "description": "Suite that includes only the vision tests",
+        "roots": ["tests/integration/inference/test_vision_inference.py"],
+        "defaults": {
+            "vision_model": "ollama/llama3.2-vision:11b",
+            "embedding_model": "sentence-transformers/all-MiniLM-L6-v2",
+        },
+    },
+}