mirror of
https://github.com/meta-llama/llama-stack.git
synced 2025-10-14 06:23:45 +00:00
feat(tests): migrate to global "setups" system for test configuration
This PR refactors the integration test system from suite-specific "contexts" to global "setups". This provides better separation of concerns: **suites = what to test, setups = how to configure.** Key changes: - New `tests/integration/setups.py` with global, reusable configurations (ollama, vllm, gpt, claude) - Simplified `tests/integration/suites.py` to only define test collection roots + default setup references - Updated CLI from `--context` to `--setup` parameter that works with any suite - Modified `scripts/integration-tests.sh` to use `--test-setup` instead of `--test-context` - Updated documentation to reflect the new global setup system Benefits: - Setups can be reused across multiple suites (e.g., use "gpt" with any suite) - Clear separation between test selection (suites) and configuration (setups) - Easier to add new configurations without modifying existing suites - Centralized configuration management Usage examples: - `pytest tests/integration --suite=responses --setup=gpt` - `pytest tests/integration --suite=vision --setup=ollama` - `pytest tests/integration --suite=base --setup=vllm`
This commit is contained in:
parent
47b640370e
commit
c662d8aa31
10 changed files with 272 additions and 178 deletions
42
.github/actions/run-and-record-tests/action.yml
vendored
42
.github/actions/run-and-record-tests/action.yml
vendored
|
@ -5,21 +5,22 @@ inputs:
|
|||
stack-config:
|
||||
description: 'Stack configuration to use'
|
||||
required: true
|
||||
provider:
|
||||
description: 'Provider to use for tests'
|
||||
required: true
|
||||
setup:
|
||||
description: 'Setup to use for tests (e.g., ollama, gpt, vllm)'
|
||||
required: false
|
||||
default: ''
|
||||
inference-mode:
|
||||
description: 'Inference mode (record or replay)'
|
||||
required: true
|
||||
test-suite:
|
||||
suite:
|
||||
description: 'Test suite to use: base, responses, vision, etc.'
|
||||
required: false
|
||||
default: ''
|
||||
test-subdirs:
|
||||
description: 'Comma-separated list of test subdirectories to run; overrides test-suite'
|
||||
subdirs:
|
||||
description: 'Comma-separated list of test subdirectories to run; overrides suite'
|
||||
required: false
|
||||
default: ''
|
||||
test-pattern:
|
||||
pattern:
|
||||
description: 'Regex pattern to pass to pytest -k'
|
||||
required: false
|
||||
default: ''
|
||||
|
@ -37,14 +38,23 @@ runs:
|
|||
- name: Run Integration Tests
|
||||
shell: bash
|
||||
run: |
|
||||
uv run --no-sync ./scripts/integration-tests.sh \
|
||||
--stack-config '${{ inputs.stack-config }}' \
|
||||
--provider '${{ inputs.provider }}' \
|
||||
--test-subdirs '${{ inputs.test-subdirs }}' \
|
||||
--test-pattern '${{ inputs.test-pattern }}' \
|
||||
--inference-mode '${{ inputs.inference-mode }}' \
|
||||
--test-suite '${{ inputs.test-suite }}' \
|
||||
| tee pytest-${{ inputs.inference-mode }}.log
|
||||
SCRIPT_ARGS="--stack-config '${{ inputs.stack-config }}' --inference-mode '${{ inputs.inference-mode }}'"
|
||||
|
||||
# Add optional arguments only if they are provided
|
||||
if [ -n '${{ inputs.setup }}' ]; then
|
||||
SCRIPT_ARGS="$SCRIPT_ARGS --setup '${{ inputs.setup }}'"
|
||||
fi
|
||||
if [ -n '${{ inputs.suite }}' ]; then
|
||||
SCRIPT_ARGS="$SCRIPT_ARGS --suite '${{ inputs.suite }}'"
|
||||
fi
|
||||
if [ -n '${{ inputs.subdirs }}' ]; then
|
||||
SCRIPT_ARGS="$SCRIPT_ARGS --subdirs '${{ inputs.subdirs }}'"
|
||||
fi
|
||||
if [ -n '${{ inputs.pattern }}' ]; then
|
||||
SCRIPT_ARGS="$SCRIPT_ARGS --pattern '${{ inputs.pattern }}'"
|
||||
fi
|
||||
|
||||
uv run --no-sync ./scripts/integration-tests.sh $SCRIPT_ARGS | tee pytest-${{ inputs.inference-mode }}.log
|
||||
|
||||
|
||||
- name: Commit and push recordings
|
||||
|
@ -58,7 +68,7 @@ runs:
|
|||
echo "New recordings detected, committing and pushing"
|
||||
git add tests/integration/recordings/
|
||||
|
||||
git commit -m "Recordings update from CI (test-suite: ${{ inputs.test-suite }})"
|
||||
git commit -m "Recordings update from CI (suite: ${{ inputs.suite }})"
|
||||
git fetch origin ${{ github.ref_name }}
|
||||
git rebase origin/${{ github.ref_name }}
|
||||
echo "Rebased successfully"
|
||||
|
|
Loading…
Add table
Add a link
Reference in a new issue