feat(tests): introduce a test "suite" concept to encompass dirs, options (#3339)

Our integration tests need to be 'grouped' because each group often
needs a specific set of models it works with. We separated vision tests
due to this, and we have a separate set of tests which test "Responses"
API.

This PR makes this system a bit more official so it is very easy to
target these groups and apply all testing infrastructure towards all the
groups (for example, record-replay) uniformly.

There are three suites declared:
- base
- vision
- responses

Note that our CI currently runs the "base" and "vision" suites.

You can use the `--suite` option when running pytest (or any of the
testing scripts or workflows.) For example:
```
OLLAMA_URL=http://localhost:11434 \
  pytest -s -v tests/integration/ --stack-config starter --suite vision
```

scripts/github/schedule-record-workflow.sh

@@ -15,7 +15,7 @@ set -euo pipefail
 BRANCH=""
 TEST_SUBDIRS=""
 TEST_PROVIDER="ollama"
-RUN_VISION_TESTS=false
+TEST_SUITE="base"
 TEST_PATTERN=""
 
 # Help function
@@ -27,9 +27,9 @@ Trigger the integration test recording workflow remotely. This way you do not ne
 
 OPTIONS:
     -b, --branch BRANCH         Branch to run the workflow on (defaults to current branch)
-    -s, --test-subdirs DIRS     Comma-separated list of test subdirectories to run (REQUIRED)
     -p, --test-provider PROVIDER Test provider to use: vllm or ollama (default: ollama)
-    -v, --run-vision-tests      Include vision tests in the recording
+    -t, --test-suite SUITE      Test suite to use: base, responses, vision, etc. (default: base)
+    -s, --test-subdirs DIRS     Comma-separated list of test subdirectories to run (overrides suite)
     -k, --test-pattern PATTERN  Regex pattern to pass to pytest -k
     -h, --help                  Show this help message
 
@@ -38,7 +38,7 @@ EXAMPLES:
     $0 --test-subdirs "agents"
 
     # Record tests for specific branch with vision tests
-    $0 -b my-feature-branch --test-subdirs "inference" --run-vision-tests
+    $0 -b my-feature-branch --test-suite vision
 
     # Record multiple test subdirectories with specific provider
     $0 --test-subdirs "agents,inference" --test-provider vllm
@@ -71,9 +71,9 @@ while [[ $# -gt 0 ]]; do
             TEST_PROVIDER="$2"
             shift 2
             ;;
-        -v|--run-vision-tests)
-            RUN_VISION_TESTS=true
-            shift
+        -t|--test-suite)
+            TEST_SUITE="$2"
+            shift 2
             ;;
         -k|--test-pattern)
             TEST_PATTERN="$2"
@@ -92,11 +92,11 @@ while [[ $# -gt 0 ]]; do
 done
 
 # Validate required parameters
-if [[ -z "$TEST_SUBDIRS" ]]; then
-    echo "Error: --test-subdirs is required"
-    echo "Please specify which test subdirectories to run, e.g.:"
+if [[ -z "$TEST_SUBDIRS" && -z "$TEST_SUITE" ]]; then
+    echo "Error: --test-subdirs or --test-suite is required"
+    echo "Please specify which test subdirectories to run or test suite to use, e.g.:"
     echo "  $0 --test-subdirs \"agents,inference\""
-    echo "  $0 --test-subdirs \"inference\" --run-vision-tests"
+    echo "  $0 --test-suite vision"
     echo ""
     exit 1
 fi
@@ -239,17 +239,19 @@ echo "Triggering integration test recording workflow..."
 echo "Branch: $BRANCH"
 echo "Test provider: $TEST_PROVIDER"
 echo "Test subdirs: $TEST_SUBDIRS"
-echo "Run vision tests: $RUN_VISION_TESTS"
+echo "Test suite: $TEST_SUITE"
 echo "Test pattern: ${TEST_PATTERN:-"(none)"}"
 echo ""
 
 # Prepare inputs for gh workflow run
-INPUTS="-f test-subdirs='$TEST_SUBDIRS'"
+if [[ -n "$TEST_SUBDIRS" ]]; then
+    INPUTS="-f test-subdirs='$TEST_SUBDIRS'"
+fi
 if [[ -n "$TEST_PROVIDER" ]]; then
     INPUTS="$INPUTS -f test-provider='$TEST_PROVIDER'"
 fi
-if [[ "$RUN_VISION_TESTS" == "true" ]]; then
-    INPUTS="$INPUTS -f run-vision-tests=true"
+if [[ -n "$TEST_SUITE" ]]; then
+    INPUTS="$INPUTS -f test-suite='$TEST_SUITE'"
 fi
 if [[ -n "$TEST_PATTERN" ]]; then
     INPUTS="$INPUTS -f test-pattern='$TEST_PATTERN'"

scripts/integration-tests.sh

@@ -16,7 +16,7 @@ STACK_CONFIG=""
 PROVIDER=""
 TEST_SUBDIRS=""
 TEST_PATTERN=""
-RUN_VISION_TESTS="false"
+TEST_SUITE="base"
 INFERENCE_MODE="replay"
 EXTRA_PARAMS=""
 
@@ -28,12 +28,16 @@ Usage: $0 [OPTIONS]
 Options:
     --stack-config STRING    Stack configuration to use (required)
     --provider STRING        Provider to use (ollama, vllm, etc.) (required)
-    --test-subdirs STRING    Comma-separated list of test subdirectories to run (default: 'inference')
-    --run-vision-tests       Run vision tests instead of regular tests
+    --test-suite STRING      Comma-separated list of test suites to run (default: 'base')
     --inference-mode STRING  Inference mode: record or replay (default: replay)
+    --test-subdirs STRING    Comma-separated list of test subdirectories to run (overrides suite)
     --test-pattern STRING    Regex pattern to pass to pytest -k
     --help                   Show this help message
 
+Suites are defined in tests/integration/suites.py. They are used to narrow the collection of tests and provide default model options.
+
+You can also specify subdirectories (of tests/integration) to select tests from, which will override the suite.
+
 Examples:
     # Basic inference tests with ollama
     $0 --stack-config server:ci-tests --provider ollama
@@ -42,7 +46,7 @@ Examples:
     $0 --stack-config server:ci-tests --provider vllm --test-subdirs 'inference,agents'
 
     # Vision tests with ollama
-    $0 --stack-config server:ci-tests --provider ollama --run-vision-tests
+    $0 --stack-config server:ci-tests --provider ollama --test-suite vision
 
     # Record mode for updating test recordings
     $0 --stack-config server:ci-tests --provider ollama --inference-mode record
@@ -64,9 +68,9 @@ while [[ $# -gt 0 ]]; do
             TEST_SUBDIRS="$2"
             shift 2
             ;;
-        --run-vision-tests)
-            RUN_VISION_TESTS="true"
-            shift
+        --test-suite)
+            TEST_SUITE="$2"
+            shift 2
             ;;
         --inference-mode)
             INFERENCE_MODE="$2"
@@ -92,22 +96,25 @@ done
 # Validate required parameters
 if [[ -z "$STACK_CONFIG" ]]; then
     echo "Error: --stack-config is required"
-    usage
     exit 1
 fi
 
 if [[ -z "$PROVIDER" ]]; then
     echo "Error: --provider is required"
-    usage
+    exit 1
+fi
+
+if [[ -z "$TEST_SUITE" && -z "$TEST_SUBDIRS" ]]; then
+    echo "Error: --test-suite or --test-subdirs is required"
     exit 1
 fi
 
 echo "=== Llama Stack Integration Test Runner ==="
 echo "Stack Config: $STACK_CONFIG"
 echo "Provider: $PROVIDER"
-echo "Test Subdirs: $TEST_SUBDIRS"
-echo "Vision Tests: $RUN_VISION_TESTS"
 echo "Inference Mode: $INFERENCE_MODE"
+echo "Test Suite: $TEST_SUITE"
+echo "Test Subdirs: $TEST_SUBDIRS"
 echo "Test Pattern: $TEST_PATTERN"
 echo ""
 
@@ -194,84 +201,46 @@ if [[ -n "$TEST_PATTERN" ]]; then
     PYTEST_PATTERN="${PYTEST_PATTERN} and $TEST_PATTERN"
 fi
 
-# Run vision tests if specified
-if [[ "$RUN_VISION_TESTS" == "true" ]]; then
-    echo "Running vision tests..."
-    set +e
-    pytest -s -v tests/integration/inference/test_vision_inference.py \
-        --stack-config="$STACK_CONFIG" \
-        -k "$PYTEST_PATTERN" \
-        --vision-model=ollama/llama3.2-vision:11b \
-        --embedding-model=sentence-transformers/all-MiniLM-L6-v2 \
-        --color=yes $EXTRA_PARAMS \
-        --capture=tee-sys
-    exit_code=$?
-    set -e
-
-    if [ $exit_code -eq 0 ]; then
-        echo "✅ Vision tests completed successfully"
-    elif [ $exit_code -eq 5 ]; then
-        echo "⚠️ No vision tests collected (pattern matched no tests)"
-    else
-        echo "❌ Vision tests failed"
-        exit 1
-    fi
-    exit 0
-fi
-
-# Run regular tests
-if [[ -z "$TEST_SUBDIRS" ]]; then
-   TEST_SUBDIRS=$(find tests/integration -maxdepth 1 -mindepth 1 -type d |
-            sed 's|tests/integration/||' |
-            grep -Ev "^(__pycache__|fixtures|test_cases|recordings|non_ci|post_training)$" |
-            sort)
-fi
 echo "Test subdirs to run: $TEST_SUBDIRS"
 
-# Collect all test files for the specified test types
-TEST_FILES=""
-for test_subdir in $(echo "$TEST_SUBDIRS" | tr ',' '\n'); do
-    # Skip certain test types for vllm provider
-    if [[ "$PROVIDER" == "vllm" ]]; then
-        if [[ "$test_subdir" == "safety" ]] || [[ "$test_subdir" == "post_training" ]] || [[ "$test_subdir" == "tool_runtime" ]]; then
-            echo "Skipping $test_subdir for vllm provider"
-            continue
+if [[ -n "$TEST_SUBDIRS" ]]; then
+    # Collect all test files for the specified test types
+    TEST_FILES=""
+    for test_subdir in $(echo "$TEST_SUBDIRS" | tr ',' '\n'); do
+        if [[ -d "tests/integration/$test_subdir" ]]; then
+            # Find all Python test files in this directory
+            test_files=$(find tests/integration/$test_subdir -name "test_*.py" -o -name "*_test.py")
+            if [[ -n "$test_files" ]]; then
+                TEST_FILES="$TEST_FILES $test_files"
+                echo "Added test files from $test_subdir: $(echo $test_files | wc -w) files"
+            fi
+        else
+            echo "Warning: Directory tests/integration/$test_subdir does not exist"
         fi
+    done
+
+    if [[ -z "$TEST_FILES" ]]; then
+        echo "No test files found for the specified test types"
+        exit 1
     fi
 
-    if [[ "$STACK_CONFIG" != *"server:"* ]] && [[ "$test_subdir" == "batches" ]]; then
-        echo "Skipping $test_subdir for library client until types are supported"
-        continue
-    fi
+    echo ""
+    echo "=== Running all collected tests in a single pytest command ==="
+    echo "Total test files: $(echo $TEST_FILES | wc -w)"
 
-    if [[ -d "tests/integration/$test_subdir" ]]; then
-        # Find all Python test files in this directory
-        test_files=$(find tests/integration/$test_subdir -name "test_*.py" -o -name "*_test.py")
-        if [[ -n "$test_files" ]]; then
-            TEST_FILES="$TEST_FILES $test_files"
-            echo "Added test files from $test_subdir: $(echo $test_files | wc -w) files"
-        fi
-    else
-        echo "Warning: Directory tests/integration/$test_subdir does not exist"
-    fi
-done
-
-if [[ -z "$TEST_FILES" ]]; then
-    echo "No test files found for the specified test types"
-    exit 1
+    PYTEST_TARGET="$TEST_FILES"
+    EXTRA_PARAMS="$EXTRA_PARAMS --text-model=$TEXT_MODEL --embedding-model=sentence-transformers/all-MiniLM-L6-v2"
+else
+    PYTEST_TARGET="tests/integration/"
+    EXTRA_PARAMS="$EXTRA_PARAMS --suite=$TEST_SUITE"
 fi
 
-echo ""
-echo "=== Running all collected tests in a single pytest command ==="
-echo "Total test files: $(echo $TEST_FILES | wc -w)"
-
 set +e
-pytest -s -v $TEST_FILES \
+pytest -s -v $PYTEST_TARGET \
     --stack-config="$STACK_CONFIG" \
     -k "$PYTEST_PATTERN" \
-    --text-model="$TEXT_MODEL" \
-    --embedding-model=sentence-transformers/all-MiniLM-L6-v2 \
-    --color=yes $EXTRA_PARAMS \
+    $EXTRA_PARAMS \
+    --color=yes \
     --capture=tee-sys
 exit_code=$?
 set -e
@@ -294,7 +263,13 @@ df -h
 # stop server
 if [[ "$STACK_CONFIG" == *"server:"* ]]; then
     echo "Stopping Llama Stack Server..."
-    kill $(lsof -i :8321 | awk 'NR>1 {print $2}')
+    pids=$(lsof -i :8321 | awk 'NR>1 {print $2}')
+    if [[ -n "$pids" ]]; then
+        echo "Killing Llama Stack Server processes: $pids"
+        kill -9 $pids
+    else
+        echo "No Llama Stack Server processes found ?!"
+    fi
     echo "Llama Stack Server stopped"
 fi