mirror of
https://github.com/meta-llama/llama-stack.git
synced 2025-10-04 04:04:14 +00:00
feat(tests): migrate to global "setups" system for test configuration (#3390)
This PR refactors the integration test system to use global "setups" which provides better separation of concerns: **suites = what to test, setups = how to configure.** NOTE: if you naming suggestions, please provide feedback Changes: - New `tests/integration/setups.py` with global, reusable configurations (ollama, vllm, gpt, claude) - Modified `scripts/integration-tests.sh` options to match with the underlying pytest options - Updated documentation to reflect the new global setup system The main benefit is that setups can be reused across multiple suites (e.g., use "gpt" with any suite) even though sometimes they could specifically tailored for a suite (vision <> ollama-vision). It is now easier to add new configurations without modifying existing suites. Usage examples: - `pytest tests/integration --suite=responses --setup=gpt` - `pytest tests/integration --suite=vision` # auto-selects "ollama-vision" setup - `pytest tests/integration --suite=base --setup=vllm`
This commit is contained in:
parent
28696c3f30
commit
a8aa815b6a
11 changed files with 385 additions and 206 deletions
|
@ -14,7 +14,7 @@ set -euo pipefail
|
|||
# Default values
|
||||
BRANCH=""
|
||||
TEST_SUBDIRS=""
|
||||
TEST_PROVIDER="ollama"
|
||||
TEST_SETUP="ollama"
|
||||
TEST_SUITE="base"
|
||||
TEST_PATTERN=""
|
||||
|
||||
|
@ -27,24 +27,24 @@ Trigger the integration test recording workflow remotely. This way you do not ne
|
|||
|
||||
OPTIONS:
|
||||
-b, --branch BRANCH Branch to run the workflow on (defaults to current branch)
|
||||
-p, --test-provider PROVIDER Test provider to use: vllm or ollama (default: ollama)
|
||||
-t, --test-suite SUITE Test suite to use: base, responses, vision, etc. (default: base)
|
||||
-s, --test-subdirs DIRS Comma-separated list of test subdirectories to run (overrides suite)
|
||||
-k, --test-pattern PATTERN Regex pattern to pass to pytest -k
|
||||
-t, --suite SUITE Test suite to use: base, responses, vision, etc. (default: base)
|
||||
-p, --setup SETUP Test setup to use: vllm, ollama, gpt, etc. (default: ollama)
|
||||
-s, --subdirs DIRS Comma-separated list of test subdirectories to run (overrides suite)
|
||||
-k, --pattern PATTERN Regex pattern to pass to pytest -k
|
||||
-h, --help Show this help message
|
||||
|
||||
EXAMPLES:
|
||||
# Record tests for current branch with agents subdirectory
|
||||
$0 --test-subdirs "agents"
|
||||
$0 --subdirs "agents"
|
||||
|
||||
# Record tests for specific branch with vision tests
|
||||
$0 -b my-feature-branch --test-suite vision
|
||||
$0 -b my-feature-branch --suite vision
|
||||
|
||||
# Record multiple test subdirectories with specific provider
|
||||
$0 --test-subdirs "agents,inference" --test-provider vllm
|
||||
# Record multiple test subdirectories with specific setup
|
||||
$0 --subdirs "agents,inference" --setup vllm
|
||||
|
||||
# Record tests matching a specific pattern
|
||||
$0 --test-subdirs "inference" --test-pattern "test_streaming"
|
||||
$0 --subdirs "inference" --pattern "test_streaming"
|
||||
|
||||
EOF
|
||||
}
|
||||
|
@ -63,19 +63,19 @@ while [[ $# -gt 0 ]]; do
|
|||
BRANCH="$2"
|
||||
shift 2
|
||||
;;
|
||||
-s|--test-subdirs)
|
||||
-s|--subdirs)
|
||||
TEST_SUBDIRS="$2"
|
||||
shift 2
|
||||
;;
|
||||
-p|--test-provider)
|
||||
TEST_PROVIDER="$2"
|
||||
-p|--setup)
|
||||
TEST_SETUP="$2"
|
||||
shift 2
|
||||
;;
|
||||
-t|--test-suite)
|
||||
-t|--suite)
|
||||
TEST_SUITE="$2"
|
||||
shift 2
|
||||
;;
|
||||
-k|--test-pattern)
|
||||
-k|--pattern)
|
||||
TEST_PATTERN="$2"
|
||||
shift 2
|
||||
;;
|
||||
|
@ -93,21 +93,16 @@ done
|
|||
|
||||
# Validate required parameters
|
||||
if [[ -z "$TEST_SUBDIRS" && -z "$TEST_SUITE" ]]; then
|
||||
echo "Error: --test-subdirs or --test-suite is required"
|
||||
echo "Error: --subdirs or --suite is required"
|
||||
echo "Please specify which test subdirectories to run or test suite to use, e.g.:"
|
||||
echo " $0 --test-subdirs \"agents,inference\""
|
||||
echo " $0 --test-suite vision"
|
||||
echo " $0 --subdirs \"agents,inference\""
|
||||
echo " $0 --suite vision"
|
||||
echo ""
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Validate test provider
|
||||
if [[ "$TEST_PROVIDER" != "vllm" && "$TEST_PROVIDER" != "ollama" ]]; then
|
||||
echo "❌ Error: Invalid test provider '$TEST_PROVIDER'"
|
||||
echo " Supported providers: vllm, ollama"
|
||||
echo " Example: $0 --test-subdirs \"agents\" --test-provider vllm"
|
||||
exit 1
|
||||
fi
|
||||
# Validate test setup (optional - setups are validated by the workflow itself)
|
||||
# Common setups: ollama, vllm, gpt, etc.
|
||||
|
||||
# Check if required tools are installed
|
||||
if ! command -v gh &> /dev/null; then
|
||||
|
@ -237,7 +232,7 @@ fi
|
|||
# Build the workflow dispatch command
|
||||
echo "Triggering integration test recording workflow..."
|
||||
echo "Branch: $BRANCH"
|
||||
echo "Test provider: $TEST_PROVIDER"
|
||||
echo "Test setup: $TEST_SETUP"
|
||||
echo "Test subdirs: $TEST_SUBDIRS"
|
||||
echo "Test suite: $TEST_SUITE"
|
||||
echo "Test pattern: ${TEST_PATTERN:-"(none)"}"
|
||||
|
@ -245,16 +240,16 @@ echo ""
|
|||
|
||||
# Prepare inputs for gh workflow run
|
||||
if [[ -n "$TEST_SUBDIRS" ]]; then
|
||||
INPUTS="-f test-subdirs='$TEST_SUBDIRS'"
|
||||
INPUTS="-f subdirs='$TEST_SUBDIRS'"
|
||||
fi
|
||||
if [[ -n "$TEST_PROVIDER" ]]; then
|
||||
INPUTS="$INPUTS -f test-provider='$TEST_PROVIDER'"
|
||||
if [[ -n "$TEST_SETUP" ]]; then
|
||||
INPUTS="$INPUTS -f test-setup='$TEST_SETUP'"
|
||||
fi
|
||||
if [[ -n "$TEST_SUITE" ]]; then
|
||||
INPUTS="$INPUTS -f test-suite='$TEST_SUITE'"
|
||||
INPUTS="$INPUTS -f suite='$TEST_SUITE'"
|
||||
fi
|
||||
if [[ -n "$TEST_PATTERN" ]]; then
|
||||
INPUTS="$INPUTS -f test-pattern='$TEST_PATTERN'"
|
||||
INPUTS="$INPUTS -f pattern='$TEST_PATTERN'"
|
||||
fi
|
||||
|
||||
# Run the workflow
|
||||
|
|
Loading…
Add table
Add a link
Reference in a new issue