mirror of
https://github.com/meta-llama/llama-stack.git
synced 2025-10-06 20:44:58 +00:00
feat(tests): migrate to global "setups" system for test configuration
This PR refactors the integration test system from suite-specific "contexts" to global "setups". This provides better separation of concerns: **suites = what to test, setups = how to configure.** Key changes: - New `tests/integration/setups.py` with global, reusable configurations (ollama, vllm, gpt, claude) - Simplified `tests/integration/suites.py` to only define test collection roots + default setup references - Updated CLI from `--context` to `--setup` parameter that works with any suite - Modified `scripts/integration-tests.sh` to use `--test-setup` instead of `--test-context` - Updated documentation to reflect the new global setup system Benefits: - Setups can be reused across multiple suites (e.g., use "gpt" with any suite) - Clear separation between test selection (suites) and configuration (setups) - Easier to add new configurations without modifying existing suites - Centralized configuration management Usage examples: - `pytest tests/integration --suite=responses --setup=gpt` - `pytest tests/integration --suite=vision --setup=ollama` - `pytest tests/integration --suite=base --setup=vllm`
This commit is contained in:
Ashwin Bharambe
parent
47b640370e
commit
c662d8aa31
10 changed files with 272 additions and 178 deletions
|
|
@ -14,7 +14,7 @@ set -euo pipefail
|
|||
# Default values
|
||||
BRANCH=""
|
||||
TEST_SUBDIRS=""
|
||||
TEST_PROVIDER="ollama"
|
||||
TEST_SETUP="ollama"
|
||||
TEST_SUITE="base"
|
||||
TEST_PATTERN=""
|
||||
|
||||
|
|
@ -27,7 +27,7 @@ Trigger the integration test recording workflow remotely. This way you do not ne
|
|||
|
||||
OPTIONS:
|
||||
-b, --branch BRANCH Branch to run the workflow on (defaults to current branch)
|
||||
-p, --test-provider PROVIDER Test provider to use: vllm or ollama (default: ollama)
|
||||
-p, --test-setup SETUP Test setup to use: vllm, ollama, gpt, etc. (default: ollama)
|
||||
-t, --test-suite SUITE Test suite to use: base, responses, vision, etc. (default: base)
|
||||
-s, --test-subdirs DIRS Comma-separated list of test subdirectories to run (overrides suite)
|
||||
-k, --test-pattern PATTERN Regex pattern to pass to pytest -k
|
||||
|
|
@ -40,8 +40,8 @@ EXAMPLES:
|
|||
# Record tests for specific branch with vision tests
|
||||
$0 -b my-feature-branch --test-suite vision
|
||||
|
||||
# Record multiple test subdirectories with specific provider
|
||||
$0 --test-subdirs "agents,inference" --test-provider vllm
|
||||
# Record multiple test subdirectories with specific setup
|
||||
$0 --test-subdirs "agents,inference" --test-setup vllm
|
||||
|
||||
# Record tests matching a specific pattern
|
||||
$0 --test-subdirs "inference" --test-pattern "test_streaming"
|
||||
|
|
@ -67,8 +67,8 @@ while [[ $# -gt 0 ]]; do
|
|||
TEST_SUBDIRS="$2"
|
||||
shift 2
|
||||
;;
|
||||
-p|--test-provider)
|
||||
TEST_PROVIDER="$2"
|
||||
-p|--test-setup)
|
||||
TEST_SETUP="$2"
|
||||
shift 2
|
||||
;;
|
||||
-t|--test-suite)
|
||||
|
|
@ -101,13 +101,8 @@ if [[ -z "$TEST_SUBDIRS" && -z "$TEST_SUITE" ]]; then
|
|||
exit 1
|
||||
fi
|
||||
|
||||
# Validate test provider
|
||||
if [[ "$TEST_PROVIDER" != "vllm" && "$TEST_PROVIDER" != "ollama" ]]; then
|
||||
echo "❌ Error: Invalid test provider '$TEST_PROVIDER'"
|
||||
echo " Supported providers: vllm, ollama"
|
||||
echo " Example: $0 --test-subdirs \"agents\" --test-provider vllm"
|
||||
exit 1
|
||||
fi
|
||||
# Validate test setup (optional - setups are validated by the workflow itself)
|
||||
# Common setups: ollama, vllm, gpt, etc.
|
||||
|
||||
# Check if required tools are installed
|
||||
if ! command -v gh &> /dev/null; then
|
||||
|
|
@ -237,7 +232,7 @@ fi
|
|||
# Build the workflow dispatch command
|
||||
echo "Triggering integration test recording workflow..."
|
||||
echo "Branch: $BRANCH"
|
||||
echo "Test provider: $TEST_PROVIDER"
|
||||
echo "Test setup: $TEST_SETUP"
|
||||
echo "Test subdirs: $TEST_SUBDIRS"
|
||||
echo "Test suite: $TEST_SUITE"
|
||||
echo "Test pattern: ${TEST_PATTERN:-"(none)"}"
|
||||
|
|
@ -245,16 +240,16 @@ echo ""
|
|||
|
||||
# Prepare inputs for gh workflow run
|
||||
if [[ -n "$TEST_SUBDIRS" ]]; then
|
||||
INPUTS="-f test-subdirs='$TEST_SUBDIRS'"
|
||||
INPUTS="-f subdirs='$TEST_SUBDIRS'"
|
||||
fi
|
||||
if [[ -n "$TEST_PROVIDER" ]]; then
|
||||
INPUTS="$INPUTS -f test-provider='$TEST_PROVIDER'"
|
||||
if [[ -n "$TEST_SETUP" ]]; then
|
||||
INPUTS="$INPUTS -f test-setup='$TEST_SETUP'"
|
||||
fi
|
||||
if [[ -n "$TEST_SUITE" ]]; then
|
||||
INPUTS="$INPUTS -f test-suite='$TEST_SUITE'"
|
||||
INPUTS="$INPUTS -f suite='$TEST_SUITE'"
|
||||
fi
|
||||
if [[ -n "$TEST_PATTERN" ]]; then
|
||||
INPUTS="$INPUTS -f test-pattern='$TEST_PATTERN'"
|
||||
INPUTS="$INPUTS -f pattern='$TEST_PATTERN'"
|
||||
fi
|
||||
|
||||
# Run the workflow
|
||||
|
|
|
|||
|
|
@ -13,10 +13,10 @@ set -euo pipefail
|
|||
|
||||
# Default values
|
||||
STACK_CONFIG=""
|
||||
PROVIDER=""
|
||||
TEST_SUITE="base"
|
||||
TEST_SETUP=""
|
||||
TEST_SUBDIRS=""
|
||||
TEST_PATTERN=""
|
||||
TEST_SUITE="base"
|
||||
INFERENCE_MODE="replay"
|
||||
EXTRA_PARAMS=""
|
||||
|
||||
|
|
@ -27,29 +27,30 @@ Usage: $0 [OPTIONS]
|
|||
|
||||
Options:
|
||||
--stack-config STRING Stack configuration to use (required)
|
||||
--provider STRING Provider to use (ollama, vllm, etc.) (required)
|
||||
--test-suite STRING Comma-separated list of test suites to run (default: 'base')
|
||||
--suite STRING Test suite to run (default: 'base')
|
||||
--setup STRING Test setup (models, env) to use (e.g., 'ollama', 'ollama-vision', 'gpt', 'vllm')
|
||||
--inference-mode STRING Inference mode: record or replay (default: replay)
|
||||
--test-subdirs STRING Comma-separated list of test subdirectories to run (overrides suite)
|
||||
--test-pattern STRING Regex pattern to pass to pytest -k
|
||||
--subdirs STRING Comma-separated list of test subdirectories to run (overrides suite)
|
||||
--pattern STRING Regex pattern to pass to pytest -k
|
||||
--help Show this help message
|
||||
|
||||
Suites are defined in tests/integration/suites.py. They are used to narrow the collection of tests and provide default model options.
|
||||
Suites are defined in tests/integration/suites.py and define which tests to run.
|
||||
Setups are defined in tests/integration/setups.py and provide global configuration (models, env).
|
||||
|
||||
You can also specify subdirectories (of tests/integration) to select tests from, which will override the suite.
|
||||
|
||||
Examples:
|
||||
# Basic inference tests with ollama
|
||||
$0 --stack-config server:ci-tests --provider ollama
|
||||
$0 --stack-config server:ci-tests --suite base --setup ollama
|
||||
|
||||
# Multiple test directories with vllm
|
||||
$0 --stack-config server:ci-tests --provider vllm --test-subdirs 'inference,agents'
|
||||
$0 --stack-config server:ci-tests --subdirs 'inference,agents' --setup vllm
|
||||
|
||||
# Vision tests with ollama
|
||||
$0 --stack-config server:ci-tests --provider ollama --test-suite vision
|
||||
$0 --stack-config server:ci-tests --suite vision # default setup for this suite is ollama-vision
|
||||
|
||||
# Record mode for updating test recordings
|
||||
$0 --stack-config server:ci-tests --provider ollama --inference-mode record
|
||||
$0 --stack-config server:ci-tests --suite base --inference-mode record
|
||||
EOF
|
||||
}
|
||||
|
||||
|
|
@ -60,15 +61,15 @@ while [[ $# -gt 0 ]]; do
|
|||
STACK_CONFIG="$2"
|
||||
shift 2
|
||||
;;
|
||||
--provider)
|
||||
PROVIDER="$2"
|
||||
--setup)
|
||||
TEST_SETUP="$2"
|
||||
shift 2
|
||||
;;
|
||||
--test-subdirs)
|
||||
--subdirs)
|
||||
TEST_SUBDIRS="$2"
|
||||
shift 2
|
||||
;;
|
||||
--test-suite)
|
||||
--suite)
|
||||
TEST_SUITE="$2"
|
||||
shift 2
|
||||
;;
|
||||
|
|
@ -76,7 +77,7 @@ while [[ $# -gt 0 ]]; do
|
|||
INFERENCE_MODE="$2"
|
||||
shift 2
|
||||
;;
|
||||
--test-pattern)
|
||||
--pattern)
|
||||
TEST_PATTERN="$2"
|
||||
shift 2
|
||||
;;
|
||||
|
|
@ -96,11 +97,13 @@ done
|
|||
# Validate required parameters
|
||||
if [[ -z "$STACK_CONFIG" ]]; then
|
||||
echo "Error: --stack-config is required"
|
||||
usage
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [[ -z "$PROVIDER" ]]; then
|
||||
echo "Error: --provider is required"
|
||||
if [[ -z "$TEST_SETUP" && -n "$TEST_SUBDIRS" ]]; then
|
||||
echo "Error: --test-setup is required when --test-subdirs is provided"
|
||||
usage
|
||||
exit 1
|
||||
fi
|
||||
|
||||
|
|
@ -111,7 +114,7 @@ fi
|
|||
|
||||
echo "=== Llama Stack Integration Test Runner ==="
|
||||
echo "Stack Config: $STACK_CONFIG"
|
||||
echo "Provider: $PROVIDER"
|
||||
echo "Setup: $TEST_SETUP"
|
||||
echo "Inference Mode: $INFERENCE_MODE"
|
||||
echo "Test Suite: $TEST_SUITE"
|
||||
echo "Test Subdirs: $TEST_SUBDIRS"
|
||||
|
|
@ -129,18 +132,10 @@ echo ""
|
|||
|
||||
# Set environment variables
|
||||
export LLAMA_STACK_CLIENT_TIMEOUT=300
|
||||
export LLAMA_STACK_TEST_INFERENCE_MODE="$INFERENCE_MODE"
|
||||
|
||||
# Configure provider-specific settings
|
||||
if [[ "$PROVIDER" == "ollama" ]]; then
|
||||
export OLLAMA_URL="http://0.0.0.0:11434"
|
||||
export TEXT_MODEL="ollama/llama3.2:3b-instruct-fp16"
|
||||
export SAFETY_MODEL="ollama/llama-guard3:1b"
|
||||
EXTRA_PARAMS="--safety-shield=llama-guard"
|
||||
else
|
||||
export VLLM_URL="http://localhost:8000/v1"
|
||||
export TEXT_MODEL="vllm/meta-llama/Llama-3.2-1B-Instruct"
|
||||
EXTRA_PARAMS=""
|
||||
# Setup-specific configuration is now handled by pytest via --setup
|
||||
if [[ -n "$TEST_SETUP" ]]; then
|
||||
EXTRA_PARAMS="--setup=$TEST_SETUP"
|
||||
fi
|
||||
|
||||
THIS_DIR=$(dirname "$0")
|
||||
|
|
@ -191,8 +186,8 @@ fi
|
|||
echo "=== Running Integration Tests ==="
|
||||
EXCLUDE_TESTS="builtin_tool or safety_with_image or code_interpreter or test_rag"
|
||||
|
||||
# Additional exclusions for vllm provider
|
||||
if [[ "$PROVIDER" == "vllm" ]]; then
|
||||
# Additional exclusions for vllm setup
|
||||
if [[ "$TEST_SETUP" == "vllm" ]]; then
|
||||
EXCLUDE_TESTS="${EXCLUDE_TESTS} or test_inference_store_tool_calls"
|
||||
fi
|
||||
|
||||
|
|
@ -229,21 +224,23 @@ if [[ -n "$TEST_SUBDIRS" ]]; then
|
|||
echo "Total test files: $(echo $TEST_FILES | wc -w)"
|
||||
|
||||
PYTEST_TARGET="$TEST_FILES"
|
||||
EXTRA_PARAMS="$EXTRA_PARAMS --text-model=$TEXT_MODEL --embedding-model=sentence-transformers/all-MiniLM-L6-v2"
|
||||
else
|
||||
PYTEST_TARGET="tests/integration/"
|
||||
EXTRA_PARAMS="$EXTRA_PARAMS --suite=$TEST_SUITE"
|
||||
fi
|
||||
|
||||
set +e
|
||||
set -x
|
||||
pytest -s -v $PYTEST_TARGET \
|
||||
--stack-config="$STACK_CONFIG" \
|
||||
--inference-mode="$INFERENCE_MODE" \
|
||||
-k "$PYTEST_PATTERN" \
|
||||
$EXTRA_PARAMS \
|
||||
--color=yes \
|
||||
--capture=tee-sys
|
||||
exit_code=$?
|
||||
set -e
|
||||
set +x
|
||||
|
||||
if [ $exit_code -eq 0 ]; then
|
||||
echo "✅ All tests completed successfully"
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue