mirror of
https://github.com/meta-llama/llama-stack.git
synced 2025-10-13 14:08:39 +00:00
feat(tests): migrate to global "setups" system for test configuration
This PR refactors the integration test system from suite-specific "contexts" to global "setups". This provides better separation of concerns: **suites = what to test, setups = how to configure.** Key changes: - New `tests/integration/setups.py` with global, reusable configurations (ollama, vllm, gpt, claude) - Simplified `tests/integration/suites.py` to only define test collection roots + default setup references - Updated CLI from `--context` to `--setup` parameter that works with any suite - Modified `scripts/integration-tests.sh` to use `--test-setup` instead of `--test-context` - Updated documentation to reflect the new global setup system Benefits: - Setups can be reused across multiple suites (e.g., use "gpt" with any suite) - Clear separation between test selection (suites) and configuration (setups) - Easier to add new configurations without modifying existing suites - Centralized configuration management Usage examples: - `pytest tests/integration --suite=responses --setup=gpt` - `pytest tests/integration --suite=vision --setup=ollama` - `pytest tests/integration --suite=base --setup=vllm`
This commit is contained in:
parent
47b640370e
commit
c662d8aa31
10 changed files with 272 additions and 178 deletions
32
.github/workflows/record-integration-tests.yml
vendored
32
.github/workflows/record-integration-tests.yml
vendored
|
@ -10,19 +10,19 @@ run-name: Run the integration test suite from tests/integration
|
|||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
test-provider:
|
||||
description: 'Test against a specific provider'
|
||||
test-setup:
|
||||
description: 'Test against a specific setup'
|
||||
type: string
|
||||
default: 'ollama'
|
||||
test-suite:
|
||||
suite:
|
||||
description: 'Test suite to use: base, responses, vision, etc.'
|
||||
type: string
|
||||
default: ''
|
||||
test-subdirs:
|
||||
description: 'Comma-separated list of test subdirectories to run; overrides test-suite'
|
||||
subdirs:
|
||||
description: 'Comma-separated list of test subdirectories to run; overrides suite'
|
||||
type: string
|
||||
default: ''
|
||||
test-pattern:
|
||||
pattern:
|
||||
description: 'Regex pattern to pass to pytest -k'
|
||||
type: string
|
||||
default: ''
|
||||
|
@ -39,10 +39,10 @@ jobs:
|
|||
run: |
|
||||
echo "::group::Workflow Inputs"
|
||||
echo "branch: ${{ github.ref_name }}"
|
||||
echo "test-provider: ${{ inputs.test-provider }}"
|
||||
echo "test-suite: ${{ inputs.test-suite }}"
|
||||
echo "test-subdirs: ${{ inputs.test-subdirs }}"
|
||||
echo "test-pattern: ${{ inputs.test-pattern }}"
|
||||
echo "test-setup: ${{ inputs.test-setup }}"
|
||||
echo "suite: ${{ inputs.suite }}"
|
||||
echo "subdirs: ${{ inputs.subdirs }}"
|
||||
echo "pattern: ${{ inputs.pattern }}"
|
||||
echo "::endgroup::"
|
||||
|
||||
- name: Checkout repository
|
||||
|
@ -55,16 +55,16 @@ jobs:
|
|||
with:
|
||||
python-version: "3.12" # Use single Python version for recording
|
||||
client-version: "latest"
|
||||
provider: ${{ inputs.test-provider || 'ollama' }}
|
||||
test-suite: ${{ inputs.test-suite }}
|
||||
setup: ${{ inputs.test-setup || 'ollama' }}
|
||||
suite: ${{ inputs.suite }}
|
||||
inference-mode: 'record'
|
||||
|
||||
- name: Run and record tests
|
||||
uses: ./.github/actions/run-and-record-tests
|
||||
with:
|
||||
stack-config: 'server:ci-tests' # recording must be done with server since more tests are run
|
||||
provider: ${{ inputs.test-provider || 'ollama' }}
|
||||
setup: ${{ inputs.test-setup || 'ollama' }}
|
||||
inference-mode: 'record'
|
||||
test-suite: ${{ inputs.test-suite }}
|
||||
test-subdirs: ${{ inputs.test-subdirs }}
|
||||
test-pattern: ${{ inputs.test-pattern }}
|
||||
suite: ${{ inputs.suite }}
|
||||
subdirs: ${{ inputs.subdirs }}
|
||||
pattern: ${{ inputs.pattern }}
|
||||
|
|
Loading…
Add table
Add a link
Reference in a new issue