mirror of
https://github.com/meta-llama/llama-stack.git
synced 2025-10-04 04:04:14 +00:00
feat(tests): migrate to global "setups" system for test configuration (#3390)
This PR refactors the integration test system to use global "setups" which provides better separation of concerns: **suites = what to test, setups = how to configure.** NOTE: if you naming suggestions, please provide feedback Changes: - New `tests/integration/setups.py` with global, reusable configurations (ollama, vllm, gpt, claude) - Modified `scripts/integration-tests.sh` options to match with the underlying pytest options - Updated documentation to reflect the new global setup system The main benefit is that setups can be reused across multiple suites (e.g., use "gpt" with any suite) even though sometimes they could specifically tailored for a suite (vision <> ollama-vision). It is now easier to add new configurations without modifying existing suites. Usage examples: - `pytest tests/integration --suite=responses --setup=gpt` - `pytest tests/integration --suite=vision` # auto-selects "ollama-vision" setup - `pytest tests/integration --suite=base --setup=vllm`
This commit is contained in:
parent
28696c3f30
commit
a8aa815b6a
11 changed files with 385 additions and 206 deletions
42
.github/actions/run-and-record-tests/action.yml
vendored
42
.github/actions/run-and-record-tests/action.yml
vendored
|
@ -5,21 +5,22 @@ inputs:
|
|||
stack-config:
|
||||
description: 'Stack configuration to use'
|
||||
required: true
|
||||
provider:
|
||||
description: 'Provider to use for tests'
|
||||
required: true
|
||||
setup:
|
||||
description: 'Setup to use for tests (e.g., ollama, gpt, vllm)'
|
||||
required: false
|
||||
default: ''
|
||||
inference-mode:
|
||||
description: 'Inference mode (record or replay)'
|
||||
required: true
|
||||
test-suite:
|
||||
suite:
|
||||
description: 'Test suite to use: base, responses, vision, etc.'
|
||||
required: false
|
||||
default: ''
|
||||
test-subdirs:
|
||||
description: 'Comma-separated list of test subdirectories to run; overrides test-suite'
|
||||
subdirs:
|
||||
description: 'Comma-separated list of test subdirectories to run; overrides suite'
|
||||
required: false
|
||||
default: ''
|
||||
test-pattern:
|
||||
pattern:
|
||||
description: 'Regex pattern to pass to pytest -k'
|
||||
required: false
|
||||
default: ''
|
||||
|
@ -37,14 +38,23 @@ runs:
|
|||
- name: Run Integration Tests
|
||||
shell: bash
|
||||
run: |
|
||||
uv run --no-sync ./scripts/integration-tests.sh \
|
||||
--stack-config '${{ inputs.stack-config }}' \
|
||||
--provider '${{ inputs.provider }}' \
|
||||
--test-subdirs '${{ inputs.test-subdirs }}' \
|
||||
--test-pattern '${{ inputs.test-pattern }}' \
|
||||
--inference-mode '${{ inputs.inference-mode }}' \
|
||||
--test-suite '${{ inputs.test-suite }}' \
|
||||
| tee pytest-${{ inputs.inference-mode }}.log
|
||||
SCRIPT_ARGS="--stack-config ${{ inputs.stack-config }} --inference-mode ${{ inputs.inference-mode }}"
|
||||
|
||||
# Add optional arguments only if they are provided
|
||||
if [ -n '${{ inputs.setup }}' ]; then
|
||||
SCRIPT_ARGS="$SCRIPT_ARGS --setup ${{ inputs.setup }}"
|
||||
fi
|
||||
if [ -n '${{ inputs.suite }}' ]; then
|
||||
SCRIPT_ARGS="$SCRIPT_ARGS --suite ${{ inputs.suite }}"
|
||||
fi
|
||||
if [ -n '${{ inputs.subdirs }}' ]; then
|
||||
SCRIPT_ARGS="$SCRIPT_ARGS --subdirs ${{ inputs.subdirs }}"
|
||||
fi
|
||||
if [ -n '${{ inputs.pattern }}' ]; then
|
||||
SCRIPT_ARGS="$SCRIPT_ARGS --pattern ${{ inputs.pattern }}"
|
||||
fi
|
||||
|
||||
uv run --no-sync ./scripts/integration-tests.sh $SCRIPT_ARGS | tee pytest-${{ inputs.inference-mode }}.log
|
||||
|
||||
|
||||
- name: Commit and push recordings
|
||||
|
@ -58,7 +68,7 @@ runs:
|
|||
echo "New recordings detected, committing and pushing"
|
||||
git add tests/integration/recordings/
|
||||
|
||||
git commit -m "Recordings update from CI (test-suite: ${{ inputs.test-suite }})"
|
||||
git commit -m "Recordings update from CI (suite: ${{ inputs.suite }})"
|
||||
git fetch origin ${{ github.ref_name }}
|
||||
git rebase origin/${{ github.ref_name }}
|
||||
echo "Rebased successfully"
|
||||
|
|
4
.github/actions/setup-ollama/action.yml
vendored
4
.github/actions/setup-ollama/action.yml
vendored
|
@ -1,7 +1,7 @@
|
|||
name: Setup Ollama
|
||||
description: Start Ollama
|
||||
inputs:
|
||||
test-suite:
|
||||
suite:
|
||||
description: 'Test suite to use: base, responses, vision, etc.'
|
||||
required: false
|
||||
default: ''
|
||||
|
@ -11,7 +11,7 @@ runs:
|
|||
- name: Start Ollama
|
||||
shell: bash
|
||||
run: |
|
||||
if [ "${{ inputs.test-suite }}" == "vision" ]; then
|
||||
if [ "${{ inputs.suite }}" == "vision" ]; then
|
||||
image="ollama-with-vision-model"
|
||||
else
|
||||
image="ollama-with-models"
|
||||
|
|
|
@ -8,11 +8,11 @@ inputs:
|
|||
client-version:
|
||||
description: 'Client version (latest or published)'
|
||||
required: true
|
||||
provider:
|
||||
description: 'Provider to setup (ollama or vllm)'
|
||||
required: true
|
||||
setup:
|
||||
description: 'Setup to configure (ollama, vllm, gpt, etc.)'
|
||||
required: false
|
||||
default: 'ollama'
|
||||
test-suite:
|
||||
suite:
|
||||
description: 'Test suite to use: base, responses, vision, etc.'
|
||||
required: false
|
||||
default: ''
|
||||
|
@ -30,13 +30,13 @@ runs:
|
|||
client-version: ${{ inputs.client-version }}
|
||||
|
||||
- name: Setup ollama
|
||||
if: ${{ inputs.provider == 'ollama' && inputs.inference-mode == 'record' }}
|
||||
if: ${{ (inputs.setup == 'ollama' || inputs.setup == 'ollama-vision') && inputs.inference-mode == 'record' }}
|
||||
uses: ./.github/actions/setup-ollama
|
||||
with:
|
||||
test-suite: ${{ inputs.test-suite }}
|
||||
suite: ${{ inputs.suite }}
|
||||
|
||||
- name: Setup vllm
|
||||
if: ${{ inputs.provider == 'vllm' && inputs.inference-mode == 'record' }}
|
||||
if: ${{ inputs.setup == 'vllm' && inputs.inference-mode == 'record' }}
|
||||
uses: ./.github/actions/setup-vllm
|
||||
|
||||
- name: Build Llama Stack
|
||||
|
|
Loading…
Add table
Add a link
Reference in a new issue