mirror of
https://github.com/meta-llama/llama-stack.git
synced 2025-10-12 13:57:57 +00:00
f4489eeb83
We are going to split record and replay workflows completely to simplify the concurrency key design. We can add vision tests by just adding to our matrix.
94 lines
3.6 KiB
YAML
94 lines
3.6 KiB
YAML
name: Integration Tests (Replay)
|
|
|
|
run-name: Run the integration test suite from tests/integration in replay mode
|
|
|
|
on:
|
|
push:
|
|
branches: [ main ]
|
|
pull_request:
|
|
branches: [ main ]
|
|
types: [opened, synchronize, reopened]
|
|
paths:
|
|
- 'llama_stack/**'
|
|
- 'tests/**'
|
|
- 'uv.lock'
|
|
- 'pyproject.toml'
|
|
- '.github/workflows/integration-tests.yml' # This workflow
|
|
- '.github/actions/setup-ollama/action.yml'
|
|
- '.github/actions/setup-test-environment/action.yml'
|
|
- '.github/actions/run-and-record-tests/action.yml'
|
|
schedule:
|
|
# If changing the cron schedule, update the provider in the test-matrix job
|
|
- cron: '0 0 * * *' # (test latest client) Daily at 12 AM UTC
|
|
- cron: '1 0 * * 0' # (test vllm) Weekly on Sunday at 1 AM UTC
|
|
workflow_dispatch:
|
|
inputs:
|
|
test-all-client-versions:
|
|
description: 'Test against both the latest and published versions'
|
|
type: boolean
|
|
default: false
|
|
test-provider:
|
|
description: 'Test against a specific provider'
|
|
type: string
|
|
default: 'ollama'
|
|
|
|
concurrency:
|
|
# Skip concurrency for pushes to main - each commit should be tested independently
|
|
group: ${{ github.workflow }}-${{ github.ref == 'refs/heads/main' && github.run_id || github.ref }}
|
|
cancel-in-progress: true
|
|
|
|
jobs:
|
|
discover-tests:
|
|
runs-on: ubuntu-latest
|
|
outputs:
|
|
test-types: ${{ steps.generate-test-types.outputs.test-types }}
|
|
|
|
steps:
|
|
- name: Checkout repository
|
|
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
|
|
|
|
- name: Generate test types
|
|
id: generate-test-types
|
|
run: |
|
|
# Get test directories dynamically, excluding non-test directories
|
|
# NOTE: we are excluding post_training since the tests take too long
|
|
TEST_TYPES=$(find tests/integration -maxdepth 1 -mindepth 1 -type d -printf "%f\n" |
|
|
grep -Ev "^(__pycache__|fixtures|test_cases|recordings|post_training)$" |
|
|
sort | jq -R -s -c 'split("\n")[:-1]')
|
|
echo "test-types=$TEST_TYPES" >> $GITHUB_OUTPUT
|
|
|
|
run-replay-mode-tests:
|
|
needs: discover-tests
|
|
runs-on: ubuntu-latest
|
|
|
|
strategy:
|
|
fail-fast: false
|
|
matrix:
|
|
client-type: [library, server]
|
|
# Use vllm on weekly schedule, otherwise use test-provider input (defaults to ollama)
|
|
provider: ${{ (github.event.schedule == '1 0 * * 0') && fromJSON('["vllm"]') || fromJSON(format('["{0}"]', github.event.inputs.test-provider || 'ollama')) }}
|
|
python-version: ["3.12", "3.13"]
|
|
client-version: ${{ (github.event.schedule == '0 0 * * 0' || github.event.inputs.test-all-client-versions == 'true') && fromJSON('["published", "latest"]') || fromJSON('["latest"]') }}
|
|
run-vision-tests: ['true', 'false']
|
|
|
|
steps:
|
|
- name: Checkout repository
|
|
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
|
|
|
|
- name: Setup test environment
|
|
uses: ./.github/actions/setup-test-environment
|
|
with:
|
|
python-version: ${{ matrix.python-version }}
|
|
client-version: ${{ matrix.client-version }}
|
|
provider: ${{ matrix.provider }}
|
|
run-vision-tests: ${{ matrix.run-vision-tests }}
|
|
inference-mode: 'replay'
|
|
|
|
- name: Run tests
|
|
uses: ./.github/actions/run-and-record-tests
|
|
with:
|
|
test-types: ${{ needs.discover-tests.outputs.test-types }}
|
|
stack-config: ${{ matrix.client-type == 'library' && 'ci-tests' || 'server:ci-tests' }}
|
|
provider: ${{ matrix.provider }}
|
|
inference-mode: 'replay'
|
|
run-vision-tests: ${{ matrix.run-vision-tests }}
|