mirror of
https://github.com/meta-llama/llama-stack.git
synced 2025-10-04 12:07:34 +00:00
a8aa815b6a
This PR refactors the integration test system to use global "setups"
which provides better separation of concerns:
**suites = what to test, setups = how to configure.**
NOTE: if you naming suggestions, please provide feedback
Changes:
- New `tests/integration/setups.py` with global, reusable configurations
(ollama, vllm, gpt, claude)
- Modified `scripts/integration-tests.sh` options to match with the
underlying pytest options
- Updated documentation to reflect the new global setup system
The main benefit is that setups can be reused across multiple suites
(e.g., use "gpt" with any suite) even though sometimes they could
specifically tailored for a suite (vision <> ollama-vision). It is now
easier to add new configurations without modifying existing suites.
Usage examples:
- `pytest tests/integration --suite=responses --setup=gpt`
- `pytest tests/integration --suite=vision` # auto-selects
"ollama-vision" setup
- `pytest tests/integration --suite=base --setup=vllm`
70 lines
2.3 KiB
YAML
70 lines
2.3 KiB
YAML
# This workflow should be run manually when needing to re-record tests. This happens when you have
|
|
# - added a new test
|
|
# - or changed an existing test such that a new inference call is made
|
|
# You should make a PR and then run this workflow on that PR branch. The workflow will re-record the
|
|
# tests and commit the recordings to the PR branch.
|
|
name: Integration Tests (Record)
|
|
|
|
run-name: Run the integration test suite from tests/integration
|
|
|
|
on:
|
|
workflow_dispatch:
|
|
inputs:
|
|
test-setup:
|
|
description: 'Test against a specific setup'
|
|
type: string
|
|
default: 'ollama'
|
|
suite:
|
|
description: 'Test suite to use: base, responses, vision, etc.'
|
|
type: string
|
|
default: ''
|
|
subdirs:
|
|
description: 'Comma-separated list of test subdirectories to run; overrides suite'
|
|
type: string
|
|
default: ''
|
|
pattern:
|
|
description: 'Regex pattern to pass to pytest -k'
|
|
type: string
|
|
default: ''
|
|
|
|
jobs:
|
|
record-tests:
|
|
runs-on: ubuntu-latest
|
|
|
|
permissions:
|
|
contents: write
|
|
|
|
steps:
|
|
- name: Echo workflow inputs
|
|
run: |
|
|
echo "::group::Workflow Inputs"
|
|
echo "branch: ${{ github.ref_name }}"
|
|
echo "test-setup: ${{ inputs.test-setup }}"
|
|
echo "suite: ${{ inputs.suite }}"
|
|
echo "subdirs: ${{ inputs.subdirs }}"
|
|
echo "pattern: ${{ inputs.pattern }}"
|
|
echo "::endgroup::"
|
|
|
|
- name: Checkout repository
|
|
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
|
|
with:
|
|
fetch-depth: 0
|
|
|
|
- name: Setup test environment
|
|
uses: ./.github/actions/setup-test-environment
|
|
with:
|
|
python-version: "3.12" # Use single Python version for recording
|
|
client-version: "latest"
|
|
setup: ${{ inputs.test-setup || 'ollama' }}
|
|
suite: ${{ inputs.suite }}
|
|
inference-mode: 'record'
|
|
|
|
- name: Run and record tests
|
|
uses: ./.github/actions/run-and-record-tests
|
|
with:
|
|
stack-config: 'server:ci-tests' # recording must be done with server since more tests are run
|
|
setup: ${{ inputs.test-setup || 'ollama' }}
|
|
inference-mode: 'record'
|
|
suite: ${{ inputs.suite }}
|
|
subdirs: ${{ inputs.subdirs }}
|
|
pattern: ${{ inputs.pattern }}
|