diff --git a/.github/actions/run-and-record-tests/action.yml b/.github/actions/run-and-record-tests/action.yml index 60550cfdc..7f028b104 100644 --- a/.github/actions/run-and-record-tests/action.yml +++ b/.github/actions/run-and-record-tests/action.yml @@ -2,13 +2,6 @@ name: 'Run and Record Tests' description: 'Run integration tests and handle recording/artifact upload' inputs: - test-subdirs: - description: 'Comma-separated list of test subdirectories to run' - required: true - test-pattern: - description: 'Regex pattern to pass to pytest -k' - required: false - default: '' stack-config: description: 'Stack configuration to use' required: true @@ -18,10 +11,18 @@ inputs: inference-mode: description: 'Inference mode (record or replay)' required: true - run-vision-tests: - description: 'Whether to run vision tests' + test-suite: + description: 'Test suite to use: base, responses, vision, etc.' required: false - default: 'false' + default: '' + test-subdirs: + description: 'Comma-separated list of test subdirectories to run; overrides test-suite' + required: false + default: '' + test-pattern: + description: 'Regex pattern to pass to pytest -k' + required: false + default: '' runs: using: 'composite' @@ -42,7 +43,7 @@ runs: --test-subdirs '${{ inputs.test-subdirs }}' \ --test-pattern '${{ inputs.test-pattern }}' \ --inference-mode '${{ inputs.inference-mode }}' \ - ${{ inputs.run-vision-tests == 'true' && '--run-vision-tests' || '' }} \ + --test-suite '${{ inputs.test-suite }}' \ | tee pytest-${{ inputs.inference-mode }}.log @@ -57,12 +58,7 @@ runs: echo "New recordings detected, committing and pushing" git add tests/integration/recordings/ - if [ "${{ inputs.run-vision-tests }}" == "true" ]; then - git commit -m "Recordings update from CI (vision)" - else - git commit -m "Recordings update from CI" - fi - + git commit -m "Recordings update from CI (test-suite: ${{ inputs.test-suite }})" git fetch origin ${{ github.ref_name }} git rebase origin/${{ github.ref_name }} echo "Rebased successfully" diff --git a/.github/actions/setup-ollama/action.yml b/.github/actions/setup-ollama/action.yml index e57876cb0..dc2f87e8c 100644 --- a/.github/actions/setup-ollama/action.yml +++ b/.github/actions/setup-ollama/action.yml @@ -1,17 +1,17 @@ name: Setup Ollama description: Start Ollama inputs: - run-vision-tests: - description: 'Run vision tests: "true" or "false"' + test-suite: + description: 'Test suite to use: base, responses, vision, etc.' required: false - default: 'false' + default: '' runs: using: "composite" steps: - name: Start Ollama shell: bash run: | - if [ "${{ inputs.run-vision-tests }}" == "true" ]; then + if [ "${{ inputs.test-suite }}" == "vision" ]; then image="ollama-with-vision-model" else image="ollama-with-models" diff --git a/.github/actions/setup-test-environment/action.yml b/.github/actions/setup-test-environment/action.yml index d830e3d13..3be76f009 100644 --- a/.github/actions/setup-test-environment/action.yml +++ b/.github/actions/setup-test-environment/action.yml @@ -12,10 +12,10 @@ inputs: description: 'Provider to setup (ollama or vllm)' required: true default: 'ollama' - run-vision-tests: - description: 'Whether to setup provider for vision tests' + test-suite: + description: 'Test suite to use: base, responses, vision, etc.' required: false - default: 'false' + default: '' inference-mode: description: 'Inference mode (record or replay)' required: true @@ -33,7 +33,7 @@ runs: if: ${{ inputs.provider == 'ollama' && inputs.inference-mode == 'record' }} uses: ./.github/actions/setup-ollama with: - run-vision-tests: ${{ inputs.run-vision-tests }} + test-suite: ${{ inputs.test-suite }} - name: Setup vllm if: ${{ inputs.provider == 'vllm' && inputs.inference-mode == 'record' }} diff --git a/.github/workflows/README.md b/.github/workflows/README.md index 8344d12a4..2e0df58b8 100644 --- a/.github/workflows/README.md +++ b/.github/workflows/README.md @@ -8,7 +8,7 @@ Llama Stack uses GitHub Actions for Continuous Integration (CI). Below is a tabl | Installer CI | [install-script-ci.yml](install-script-ci.yml) | Test the installation script | | Integration Auth Tests | [integration-auth-tests.yml](integration-auth-tests.yml) | Run the integration test suite with Kubernetes authentication | | SqlStore Integration Tests | [integration-sql-store-tests.yml](integration-sql-store-tests.yml) | Run the integration test suite with SqlStore | -| Integration Tests (Replay) | [integration-tests.yml](integration-tests.yml) | Run the integration test suite from tests/integration in replay mode | +| Integration Tests (Replay) | [integration-tests.yml](integration-tests.yml) | Run the integration test suites from tests/integration in replay mode | | Vector IO Integration Tests | [integration-vector-io-tests.yml](integration-vector-io-tests.yml) | Run the integration test suite with various VectorIO providers | | Pre-commit | [pre-commit.yml](pre-commit.yml) | Run pre-commit checks | | Test Llama Stack Build | [providers-build.yml](providers-build.yml) | Test llama stack build | diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index 57e582b20..bb53eea2f 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -1,6 +1,6 @@ name: Integration Tests (Replay) -run-name: Run the integration test suite from tests/integration in replay mode +run-name: Run the integration test suites from tests/integration in replay mode on: push: @@ -32,14 +32,6 @@ on: description: 'Test against a specific provider' type: string default: 'ollama' - test-subdirs: - description: 'Comma-separated list of test subdirectories to run' - type: string - default: '' - test-pattern: - description: 'Regex pattern to pass to pytest -k' - type: string - default: '' concurrency: # Skip concurrency for pushes to main - each commit should be tested independently @@ -50,7 +42,7 @@ jobs: run-replay-mode-tests: runs-on: ubuntu-latest - name: ${{ format('Integration Tests ({0}, {1}, {2}, client={3}, vision={4})', matrix.client-type, matrix.provider, matrix.python-version, matrix.client-version, matrix.run-vision-tests) }} + name: ${{ format('Integration Tests ({0}, {1}, {2}, client={3}, {4})', matrix.client-type, matrix.provider, matrix.python-version, matrix.client-version, matrix.test-suite) }} strategy: fail-fast: false @@ -61,7 +53,7 @@ jobs: # Use Python 3.13 only on nightly schedule (daily latest client test), otherwise use 3.12 python-version: ${{ github.event.schedule == '0 0 * * *' && fromJSON('["3.12", "3.13"]') || fromJSON('["3.12"]') }} client-version: ${{ (github.event.schedule == '0 0 * * *' || github.event.inputs.test-all-client-versions == 'true') && fromJSON('["published", "latest"]') || fromJSON('["latest"]') }} - run-vision-tests: [true, false] + test-suite: [base, vision] steps: - name: Checkout repository @@ -73,15 +65,13 @@ jobs: python-version: ${{ matrix.python-version }} client-version: ${{ matrix.client-version }} provider: ${{ matrix.provider }} - run-vision-tests: ${{ matrix.run-vision-tests }} + test-suite: ${{ matrix.test-suite }} inference-mode: 'replay' - name: Run tests uses: ./.github/actions/run-and-record-tests with: - test-subdirs: ${{ inputs.test-subdirs }} - test-pattern: ${{ inputs.test-pattern }} stack-config: ${{ matrix.client-type == 'library' && 'ci-tests' || 'server:ci-tests' }} provider: ${{ matrix.provider }} inference-mode: 'replay' - run-vision-tests: ${{ matrix.run-vision-tests }} + test-suite: ${{ matrix.test-suite }} diff --git a/.github/workflows/record-integration-tests.yml b/.github/workflows/record-integration-tests.yml index d4f5586e2..01797a54b 100644 --- a/.github/workflows/record-integration-tests.yml +++ b/.github/workflows/record-integration-tests.yml @@ -10,18 +10,18 @@ run-name: Run the integration test suite from tests/integration on: workflow_dispatch: inputs: - test-subdirs: - description: 'Comma-separated list of test subdirectories to run' - type: string - default: '' test-provider: description: 'Test against a specific provider' type: string default: 'ollama' - run-vision-tests: - description: 'Whether to run vision tests' - type: boolean - default: false + test-suite: + description: 'Test suite to use: base, responses, vision, etc.' + type: string + default: '' + test-subdirs: + description: 'Comma-separated list of test subdirectories to run; overrides test-suite' + type: string + default: '' test-pattern: description: 'Regex pattern to pass to pytest -k' type: string @@ -38,11 +38,11 @@ jobs: - name: Echo workflow inputs run: | echo "::group::Workflow Inputs" - echo "test-subdirs: ${{ inputs.test-subdirs }}" - echo "test-provider: ${{ inputs.test-provider }}" - echo "run-vision-tests: ${{ inputs.run-vision-tests }}" - echo "test-pattern: ${{ inputs.test-pattern }}" echo "branch: ${{ github.ref_name }}" + echo "test-provider: ${{ inputs.test-provider }}" + echo "test-suite: ${{ inputs.test-suite }}" + echo "test-subdirs: ${{ inputs.test-subdirs }}" + echo "test-pattern: ${{ inputs.test-pattern }}" echo "::endgroup::" - name: Checkout repository @@ -56,15 +56,15 @@ jobs: python-version: "3.12" # Use single Python version for recording client-version: "latest" provider: ${{ inputs.test-provider || 'ollama' }} - run-vision-tests: ${{ inputs.run-vision-tests }} + test-suite: ${{ inputs.test-suite }} inference-mode: 'record' - name: Run and record tests uses: ./.github/actions/run-and-record-tests with: - test-pattern: ${{ inputs.test-pattern }} - test-subdirs: ${{ inputs.test-subdirs }} stack-config: 'server:ci-tests' # recording must be done with server since more tests are run provider: ${{ inputs.test-provider || 'ollama' }} inference-mode: 'record' - run-vision-tests: ${{ inputs.run-vision-tests }} + test-suite: ${{ inputs.test-suite }} + test-subdirs: ${{ inputs.test-subdirs }} + test-pattern: ${{ inputs.test-pattern }} diff --git a/scripts/github/schedule-record-workflow.sh b/scripts/github/schedule-record-workflow.sh index e381b60b6..09e055611 100755 --- a/scripts/github/schedule-record-workflow.sh +++ b/scripts/github/schedule-record-workflow.sh @@ -15,7 +15,7 @@ set -euo pipefail BRANCH="" TEST_SUBDIRS="" TEST_PROVIDER="ollama" -RUN_VISION_TESTS=false +TEST_SUITE="base" TEST_PATTERN="" # Help function @@ -27,9 +27,9 @@ Trigger the integration test recording workflow remotely. This way you do not ne OPTIONS: -b, --branch BRANCH Branch to run the workflow on (defaults to current branch) - -s, --test-subdirs DIRS Comma-separated list of test subdirectories to run (REQUIRED) -p, --test-provider PROVIDER Test provider to use: vllm or ollama (default: ollama) - -v, --run-vision-tests Include vision tests in the recording + -t, --test-suite SUITE Test suite to use: base, responses, vision, etc. (default: base) + -s, --test-subdirs DIRS Comma-separated list of test subdirectories to run (overrides suite) -k, --test-pattern PATTERN Regex pattern to pass to pytest -k -h, --help Show this help message @@ -38,7 +38,7 @@ EXAMPLES: $0 --test-subdirs "agents" # Record tests for specific branch with vision tests - $0 -b my-feature-branch --test-subdirs "inference" --run-vision-tests + $0 -b my-feature-branch --test-suite vision # Record multiple test subdirectories with specific provider $0 --test-subdirs "agents,inference" --test-provider vllm @@ -71,9 +71,9 @@ while [[ $# -gt 0 ]]; do TEST_PROVIDER="$2" shift 2 ;; - -v|--run-vision-tests) - RUN_VISION_TESTS=true - shift + -t|--test-suite) + TEST_SUITE="$2" + shift 2 ;; -k|--test-pattern) TEST_PATTERN="$2" @@ -92,11 +92,11 @@ while [[ $# -gt 0 ]]; do done # Validate required parameters -if [[ -z "$TEST_SUBDIRS" ]]; then - echo "Error: --test-subdirs is required" - echo "Please specify which test subdirectories to run, e.g.:" +if [[ -z "$TEST_SUBDIRS" && -z "$TEST_SUITE" ]]; then + echo "Error: --test-subdirs or --test-suite is required" + echo "Please specify which test subdirectories to run or test suite to use, e.g.:" echo " $0 --test-subdirs \"agents,inference\"" - echo " $0 --test-subdirs \"inference\" --run-vision-tests" + echo " $0 --test-suite vision" echo "" exit 1 fi @@ -239,17 +239,19 @@ echo "Triggering integration test recording workflow..." echo "Branch: $BRANCH" echo "Test provider: $TEST_PROVIDER" echo "Test subdirs: $TEST_SUBDIRS" -echo "Run vision tests: $RUN_VISION_TESTS" +echo "Test suite: $TEST_SUITE" echo "Test pattern: ${TEST_PATTERN:-"(none)"}" echo "" # Prepare inputs for gh workflow run -INPUTS="-f test-subdirs='$TEST_SUBDIRS'" +if [[ -n "$TEST_SUBDIRS" ]]; then + INPUTS="-f test-subdirs='$TEST_SUBDIRS'" +fi if [[ -n "$TEST_PROVIDER" ]]; then INPUTS="$INPUTS -f test-provider='$TEST_PROVIDER'" fi -if [[ "$RUN_VISION_TESTS" == "true" ]]; then - INPUTS="$INPUTS -f run-vision-tests=true" +if [[ -n "$TEST_SUITE" ]]; then + INPUTS="$INPUTS -f test-suite='$TEST_SUITE'" fi if [[ -n "$TEST_PATTERN" ]]; then INPUTS="$INPUTS -f test-pattern='$TEST_PATTERN'" diff --git a/scripts/integration-tests.sh b/scripts/integration-tests.sh index 104ba5cf3..ab7e37579 100755 --- a/scripts/integration-tests.sh +++ b/scripts/integration-tests.sh @@ -16,7 +16,7 @@ STACK_CONFIG="" PROVIDER="" TEST_SUBDIRS="" TEST_PATTERN="" -RUN_VISION_TESTS="false" +TEST_SUITE="base" INFERENCE_MODE="replay" EXTRA_PARAMS="" @@ -28,12 +28,16 @@ Usage: $0 [OPTIONS] Options: --stack-config STRING Stack configuration to use (required) --provider STRING Provider to use (ollama, vllm, etc.) (required) - --test-subdirs STRING Comma-separated list of test subdirectories to run (default: 'inference') - --run-vision-tests Run vision tests instead of regular tests + --test-suite STRING Comma-separated list of test suites to run (default: 'base') --inference-mode STRING Inference mode: record or replay (default: replay) + --test-subdirs STRING Comma-separated list of test subdirectories to run (overrides suite) --test-pattern STRING Regex pattern to pass to pytest -k --help Show this help message +Suites are defined in tests/integration/suites.py. They are used to narrow the collection of tests and provide default model options. + +You can also specify subdirectories (of tests/integration) to select tests from, which will override the suite. + Examples: # Basic inference tests with ollama $0 --stack-config server:ci-tests --provider ollama @@ -42,7 +46,7 @@ Examples: $0 --stack-config server:ci-tests --provider vllm --test-subdirs 'inference,agents' # Vision tests with ollama - $0 --stack-config server:ci-tests --provider ollama --run-vision-tests + $0 --stack-config server:ci-tests --provider ollama --test-suite vision # Record mode for updating test recordings $0 --stack-config server:ci-tests --provider ollama --inference-mode record @@ -64,9 +68,9 @@ while [[ $# -gt 0 ]]; do TEST_SUBDIRS="$2" shift 2 ;; - --run-vision-tests) - RUN_VISION_TESTS="true" - shift + --test-suite) + TEST_SUITE="$2" + shift 2 ;; --inference-mode) INFERENCE_MODE="$2" @@ -92,22 +96,25 @@ done # Validate required parameters if [[ -z "$STACK_CONFIG" ]]; then echo "Error: --stack-config is required" - usage exit 1 fi if [[ -z "$PROVIDER" ]]; then echo "Error: --provider is required" - usage + exit 1 +fi + +if [[ -z "$TEST_SUITE" && -z "$TEST_SUBDIRS" ]]; then + echo "Error: --test-suite or --test-subdirs is required" exit 1 fi echo "=== Llama Stack Integration Test Runner ===" echo "Stack Config: $STACK_CONFIG" echo "Provider: $PROVIDER" -echo "Test Subdirs: $TEST_SUBDIRS" -echo "Vision Tests: $RUN_VISION_TESTS" echo "Inference Mode: $INFERENCE_MODE" +echo "Test Suite: $TEST_SUITE" +echo "Test Subdirs: $TEST_SUBDIRS" echo "Test Pattern: $TEST_PATTERN" echo "" @@ -194,84 +201,46 @@ if [[ -n "$TEST_PATTERN" ]]; then PYTEST_PATTERN="${PYTEST_PATTERN} and $TEST_PATTERN" fi -# Run vision tests if specified -if [[ "$RUN_VISION_TESTS" == "true" ]]; then - echo "Running vision tests..." - set +e - pytest -s -v tests/integration/inference/test_vision_inference.py \ - --stack-config="$STACK_CONFIG" \ - -k "$PYTEST_PATTERN" \ - --vision-model=ollama/llama3.2-vision:11b \ - --embedding-model=sentence-transformers/all-MiniLM-L6-v2 \ - --color=yes $EXTRA_PARAMS \ - --capture=tee-sys - exit_code=$? - set -e - - if [ $exit_code -eq 0 ]; then - echo "✅ Vision tests completed successfully" - elif [ $exit_code -eq 5 ]; then - echo "⚠️ No vision tests collected (pattern matched no tests)" - else - echo "❌ Vision tests failed" - exit 1 - fi - exit 0 -fi - -# Run regular tests -if [[ -z "$TEST_SUBDIRS" ]]; then - TEST_SUBDIRS=$(find tests/integration -maxdepth 1 -mindepth 1 -type d | - sed 's|tests/integration/||' | - grep -Ev "^(__pycache__|fixtures|test_cases|recordings|non_ci|post_training)$" | - sort) -fi echo "Test subdirs to run: $TEST_SUBDIRS" -# Collect all test files for the specified test types -TEST_FILES="" -for test_subdir in $(echo "$TEST_SUBDIRS" | tr ',' '\n'); do - # Skip certain test types for vllm provider - if [[ "$PROVIDER" == "vllm" ]]; then - if [[ "$test_subdir" == "safety" ]] || [[ "$test_subdir" == "post_training" ]] || [[ "$test_subdir" == "tool_runtime" ]]; then - echo "Skipping $test_subdir for vllm provider" - continue +if [[ -n "$TEST_SUBDIRS" ]]; then + # Collect all test files for the specified test types + TEST_FILES="" + for test_subdir in $(echo "$TEST_SUBDIRS" | tr ',' '\n'); do + if [[ -d "tests/integration/$test_subdir" ]]; then + # Find all Python test files in this directory + test_files=$(find tests/integration/$test_subdir -name "test_*.py" -o -name "*_test.py") + if [[ -n "$test_files" ]]; then + TEST_FILES="$TEST_FILES $test_files" + echo "Added test files from $test_subdir: $(echo $test_files | wc -w) files" + fi + else + echo "Warning: Directory tests/integration/$test_subdir does not exist" fi + done + + if [[ -z "$TEST_FILES" ]]; then + echo "No test files found for the specified test types" + exit 1 fi - if [[ "$STACK_CONFIG" != *"server:"* ]] && [[ "$test_subdir" == "batches" ]]; then - echo "Skipping $test_subdir for library client until types are supported" - continue - fi + echo "" + echo "=== Running all collected tests in a single pytest command ===" + echo "Total test files: $(echo $TEST_FILES | wc -w)" - if [[ -d "tests/integration/$test_subdir" ]]; then - # Find all Python test files in this directory - test_files=$(find tests/integration/$test_subdir -name "test_*.py" -o -name "*_test.py") - if [[ -n "$test_files" ]]; then - TEST_FILES="$TEST_FILES $test_files" - echo "Added test files from $test_subdir: $(echo $test_files | wc -w) files" - fi - else - echo "Warning: Directory tests/integration/$test_subdir does not exist" - fi -done - -if [[ -z "$TEST_FILES" ]]; then - echo "No test files found for the specified test types" - exit 1 + PYTEST_TARGET="$TEST_FILES" + EXTRA_PARAMS="$EXTRA_PARAMS --text-model=$TEXT_MODEL --embedding-model=sentence-transformers/all-MiniLM-L6-v2" +else + PYTEST_TARGET="tests/integration/" + EXTRA_PARAMS="$EXTRA_PARAMS --suite=$TEST_SUITE" fi -echo "" -echo "=== Running all collected tests in a single pytest command ===" -echo "Total test files: $(echo $TEST_FILES | wc -w)" - set +e -pytest -s -v $TEST_FILES \ +pytest -s -v $PYTEST_TARGET \ --stack-config="$STACK_CONFIG" \ -k "$PYTEST_PATTERN" \ - --text-model="$TEXT_MODEL" \ - --embedding-model=sentence-transformers/all-MiniLM-L6-v2 \ - --color=yes $EXTRA_PARAMS \ + $EXTRA_PARAMS \ + --color=yes \ --capture=tee-sys exit_code=$? set -e @@ -294,7 +263,13 @@ df -h # stop server if [[ "$STACK_CONFIG" == *"server:"* ]]; then echo "Stopping Llama Stack Server..." - kill $(lsof -i :8321 | awk 'NR>1 {print $2}') + pids=$(lsof -i :8321 | awk 'NR>1 {print $2}') + if [[ -n "$pids" ]]; then + echo "Killing Llama Stack Server processes: $pids" + kill -9 $pids + else + echo "No Llama Stack Server processes found ?!" + fi echo "Llama Stack Server stopped" fi diff --git a/tests/README.md b/tests/README.md index 81f025f86..c00829d3e 100644 --- a/tests/README.md +++ b/tests/README.md @@ -77,7 +77,7 @@ You must be careful when re-recording. CI workflows assume a specific setup for ./scripts/github/schedule-record-workflow.sh --test-subdirs "agents,inference" # Record with vision tests enabled -./scripts/github/schedule-record-workflow.sh --test-subdirs "inference" --run-vision-tests +./scripts/github/schedule-record-workflow.sh --test-suite vision # Record with specific provider ./scripts/github/schedule-record-workflow.sh --test-subdirs "agents" --test-provider vllm diff --git a/tests/integration/README.md b/tests/integration/README.md index d177cbebf..b05beeb98 100644 --- a/tests/integration/README.md +++ b/tests/integration/README.md @@ -42,6 +42,27 @@ Model parameters can be influenced by the following options: Each of these are comma-separated lists and can be used to generate multiple parameter combinations. Note that tests will be skipped if no model is specified. +### Suites (fast selection + sane defaults) + +- `--suite`: comma-separated list of named suites that both narrow which tests are collected and prefill common model options (unless you pass them explicitly). +- Available suites: + - `responses`: collects tests under `tests/integration/responses`; this is a separate suite because it needs a strong tool-calling model. + - `vision`: collects only `tests/integration/inference/test_vision_inference.py`; defaults `--vision-model=ollama/llama3.2-vision:11b`, `--embedding-model=sentence-transformers/all-MiniLM-L6-v2`. +- Explicit flags always win. For example, `--suite=responses --text-model=` overrides the suite’s text model. + +Examples: + +```bash +# Fast responses run with defaults +pytest -s -v tests/integration --stack-config=server:starter --suite=responses + +# Fast single-file vision run with defaults +pytest -s -v tests/integration --stack-config=server:starter --suite=vision + +# Combine suites and override a default +pytest -s -v tests/integration --stack-config=server:starter --suite=responses,vision --embedding-model=text-embedding-3-small +``` + ## Examples ### Testing against a Server diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index fd9a54d04..96260fdb7 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -6,15 +6,17 @@ import inspect import itertools import os -import platform import textwrap import time +from pathlib import Path import pytest from dotenv import load_dotenv from llama_stack.log import get_logger +from .suites import SUITE_DEFINITIONS + logger = get_logger(__name__, category="tests") @@ -61,9 +63,22 @@ def pytest_configure(config): key, value = env_var.split("=", 1) os.environ[key] = value - if platform.system() == "Darwin": # Darwin is the system name for macOS - os.environ["DISABLE_CODE_SANDBOX"] = "1" - logger.info("Setting DISABLE_CODE_SANDBOX=1 for macOS") + suites_raw = config.getoption("--suite") + suites: list[str] = [] + if suites_raw: + suites = [p.strip() for p in str(suites_raw).split(",") if p.strip()] + unknown = [p for p in suites if p not in SUITE_DEFINITIONS] + if unknown: + raise pytest.UsageError( + f"Unknown suite(s): {', '.join(unknown)}. Available: {', '.join(sorted(SUITE_DEFINITIONS.keys()))}" + ) + for suite in suites: + suite_def = SUITE_DEFINITIONS.get(suite, {}) + defaults: dict = suite_def.get("defaults", {}) + for dest, value in defaults.items(): + current = getattr(config.option, dest, None) + if not current: + setattr(config.option, dest, value) def pytest_addoption(parser): @@ -105,16 +120,21 @@ def pytest_addoption(parser): default=384, help="Output dimensionality of the embedding model to use for testing. Default: 384", ) - parser.addoption( - "--record-responses", - action="store_true", - help="Record new API responses instead of using cached ones.", - ) parser.addoption( "--report", help="Path where the test report should be written, e.g. --report=/path/to/report.md", ) + available_suites = ", ".join(sorted(SUITE_DEFINITIONS.keys())) + suite_help = ( + "Comma-separated integration test suites to narrow collection and prefill defaults. " + "Available: " + f"{available_suites}. " + "Explicit CLI flags (e.g., --text-model) override suite defaults. " + "Examples: --suite=responses or --suite=responses,vision." + ) + parser.addoption("--suite", help=suite_help) + MODEL_SHORT_IDS = { "meta-llama/Llama-3.2-3B-Instruct": "3B", @@ -197,3 +217,40 @@ def pytest_generate_tests(metafunc): pytest_plugins = ["tests.integration.fixtures.common"] + + +def pytest_ignore_collect(path: str, config: pytest.Config) -> bool: + """Skip collecting paths outside the selected suite roots for speed.""" + suites_raw = config.getoption("--suite") + if not suites_raw: + return False + + names = [p.strip() for p in str(suites_raw).split(",") if p.strip()] + roots: list[str] = [] + for name in names: + suite_def = SUITE_DEFINITIONS.get(name) + if suite_def: + roots.extend(suite_def.get("roots", [])) + if not roots: + return False + + p = Path(str(path)).resolve() + + # Only constrain within tests/integration to avoid ignoring unrelated tests + integration_root = (Path(str(config.rootpath)) / "tests" / "integration").resolve() + if not p.is_relative_to(integration_root): + return False + + for r in roots: + rp = (Path(str(config.rootpath)) / r).resolve() + if rp.is_file(): + # Allow the exact file and any ancestor directories so pytest can walk into it. + if p == rp: + return False + if p.is_dir() and rp.is_relative_to(p): + return False + else: + # Allow anything inside an allowed directory + if p.is_relative_to(rp): + return False + return True diff --git a/tests/integration/non_ci/responses/__init__.py b/tests/integration/responses/__init__.py similarity index 100% rename from tests/integration/non_ci/responses/__init__.py rename to tests/integration/responses/__init__.py diff --git a/tests/integration/non_ci/responses/fixtures/__init__.py b/tests/integration/responses/fixtures/__init__.py similarity index 100% rename from tests/integration/non_ci/responses/fixtures/__init__.py rename to tests/integration/responses/fixtures/__init__.py diff --git a/tests/integration/non_ci/responses/fixtures/fixtures.py b/tests/integration/responses/fixtures/fixtures.py similarity index 100% rename from tests/integration/non_ci/responses/fixtures/fixtures.py rename to tests/integration/responses/fixtures/fixtures.py diff --git a/tests/integration/non_ci/responses/fixtures/images/vision_test_1.jpg b/tests/integration/responses/fixtures/images/vision_test_1.jpg similarity index 100% rename from tests/integration/non_ci/responses/fixtures/images/vision_test_1.jpg rename to tests/integration/responses/fixtures/images/vision_test_1.jpg diff --git a/tests/integration/non_ci/responses/fixtures/images/vision_test_2.jpg b/tests/integration/responses/fixtures/images/vision_test_2.jpg similarity index 100% rename from tests/integration/non_ci/responses/fixtures/images/vision_test_2.jpg rename to tests/integration/responses/fixtures/images/vision_test_2.jpg diff --git a/tests/integration/non_ci/responses/fixtures/images/vision_test_3.jpg b/tests/integration/responses/fixtures/images/vision_test_3.jpg similarity index 100% rename from tests/integration/non_ci/responses/fixtures/images/vision_test_3.jpg rename to tests/integration/responses/fixtures/images/vision_test_3.jpg diff --git a/tests/integration/non_ci/responses/fixtures/pdfs/llama_stack_and_models.pdf b/tests/integration/responses/fixtures/pdfs/llama_stack_and_models.pdf similarity index 100% rename from tests/integration/non_ci/responses/fixtures/pdfs/llama_stack_and_models.pdf rename to tests/integration/responses/fixtures/pdfs/llama_stack_and_models.pdf diff --git a/tests/integration/non_ci/responses/fixtures/test_cases.py b/tests/integration/responses/fixtures/test_cases.py similarity index 100% rename from tests/integration/non_ci/responses/fixtures/test_cases.py rename to tests/integration/responses/fixtures/test_cases.py diff --git a/tests/integration/non_ci/responses/helpers.py b/tests/integration/responses/helpers.py similarity index 100% rename from tests/integration/non_ci/responses/helpers.py rename to tests/integration/responses/helpers.py diff --git a/tests/integration/non_ci/responses/streaming_assertions.py b/tests/integration/responses/streaming_assertions.py similarity index 100% rename from tests/integration/non_ci/responses/streaming_assertions.py rename to tests/integration/responses/streaming_assertions.py diff --git a/tests/integration/non_ci/responses/test_basic_responses.py b/tests/integration/responses/test_basic_responses.py similarity index 100% rename from tests/integration/non_ci/responses/test_basic_responses.py rename to tests/integration/responses/test_basic_responses.py diff --git a/tests/integration/non_ci/responses/test_file_search.py b/tests/integration/responses/test_file_search.py similarity index 100% rename from tests/integration/non_ci/responses/test_file_search.py rename to tests/integration/responses/test_file_search.py diff --git a/tests/integration/non_ci/responses/test_tool_responses.py b/tests/integration/responses/test_tool_responses.py similarity index 100% rename from tests/integration/non_ci/responses/test_tool_responses.py rename to tests/integration/responses/test_tool_responses.py diff --git a/tests/integration/suites.py b/tests/integration/suites.py new file mode 100644 index 000000000..602855055 --- /dev/null +++ b/tests/integration/suites.py @@ -0,0 +1,53 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the terms described in the LICENSE file in +# the root directory of this source tree. + +# Central definition of integration test suites. You can use these suites by passing --suite=name to pytest. +# For example: +# +# ```bash +# pytest tests/integration/ --suite=vision +# ``` +# +# Each suite can: +# - restrict collection to specific roots (dirs or files) +# - provide default CLI option values (e.g. text_model, embedding_model, etc.) + +from pathlib import Path + +this_dir = Path(__file__).parent +default_roots = [ + str(p) + for p in this_dir.glob("*") + if p.is_dir() + and p.name not in ("__pycache__", "fixtures", "test_cases", "recordings", "responses", "post_training") +] + +SUITE_DEFINITIONS: dict[str, dict] = { + "base": { + "description": "Base suite that includes most tests but runs them with a text Ollama model", + "roots": default_roots, + "defaults": { + "text_model": "ollama/llama3.2:3b-instruct-fp16", + "embedding_model": "sentence-transformers/all-MiniLM-L6-v2", + }, + }, + "responses": { + "description": "Suite that includes only the OpenAI Responses tests; needs a strong tool-calling model", + "roots": ["tests/integration/responses"], + "defaults": { + "text_model": "openai/gpt-4o", + "embedding_model": "sentence-transformers/all-MiniLM-L6-v2", + }, + }, + "vision": { + "description": "Suite that includes only the vision tests", + "roots": ["tests/integration/inference/test_vision_inference.py"], + "defaults": { + "vision_model": "ollama/llama3.2-vision:11b", + "embedding_model": "sentence-transformers/all-MiniLM-L6-v2", + }, + }, +}