mirror of
https://github.com/meta-llama/llama-stack.git
synced 2025-10-03 19:57:35 +00:00
746e9c91a9
Add vLLM provider support to integration test CI workflows alongside existing Ollama support. Configure provider-specific test execution where vLLM runs only inference specific tests (excluding vision tests) while Ollama continues to run the full test suite. This enables comprehensive CI testing of both inference providers but keeps the vLLM footprint small, this can be expanded later if it proves to not be too disruptive. Signed-off-by: Derek Higgins <derekh@redhat.com>
90 lines
3.3 KiB
YAML
90 lines
3.3 KiB
YAML
name: Integration Tests (Replay)
|
|
|
|
run-name: Run the integration test suites from tests/integration in replay mode
|
|
|
|
on:
|
|
push:
|
|
branches: [ main ]
|
|
pull_request:
|
|
branches: [ main ]
|
|
types: [opened, synchronize, reopened]
|
|
paths:
|
|
- 'llama_stack/**'
|
|
- '!llama_stack/ui/**'
|
|
- 'tests/**'
|
|
- 'uv.lock'
|
|
- 'pyproject.toml'
|
|
- '.github/workflows/integration-tests.yml' # This workflow
|
|
- '.github/actions/setup-ollama/action.yml'
|
|
- '.github/actions/setup-test-environment/action.yml'
|
|
- '.github/actions/run-and-record-tests/action.yml'
|
|
schedule:
|
|
# If changing the cron schedule, update the provider in the test-matrix job
|
|
- cron: '0 0 * * *' # (test latest client) Daily at 12 AM UTC
|
|
workflow_dispatch:
|
|
inputs:
|
|
test-all-client-versions:
|
|
description: 'Test against both the latest and published versions'
|
|
type: boolean
|
|
default: false
|
|
test-setup:
|
|
description: 'Test against a specific setup'
|
|
type: string
|
|
default: 'ollama'
|
|
|
|
concurrency:
|
|
# Skip concurrency for pushes to main - each commit should be tested independently
|
|
group: ${{ github.workflow }}-${{ github.ref == 'refs/heads/main' && github.run_id || github.ref }}
|
|
cancel-in-progress: true
|
|
|
|
jobs:
|
|
|
|
run-replay-mode-tests:
|
|
runs-on: ubuntu-latest
|
|
name: ${{ format('Integration Tests ({0}, {1}, {2}, client={3}, {4})', matrix.client-type, matrix.setup, matrix.python-version, matrix.client-version, matrix.suite) }}
|
|
|
|
strategy:
|
|
fail-fast: false
|
|
matrix:
|
|
client-type: [library, server]
|
|
# Use Python 3.13 only on nightly schedule (daily latest client test), otherwise use 3.12
|
|
python-version: ${{ github.event.schedule == '0 0 * * *' && fromJSON('["3.12", "3.13"]') || fromJSON('["3.12"]') }}
|
|
client-version: ${{ (github.event.schedule == '0 0 * * *' || github.event.inputs.test-all-client-versions == 'true') && fromJSON('["published", "latest"]') || fromJSON('["latest"]') }}
|
|
setup: [ollama, vllm]
|
|
suite: [base, vision]
|
|
exclude:
|
|
- setup: vllm
|
|
suite: vision
|
|
|
|
|
|
steps:
|
|
- name: Checkout repository
|
|
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
|
|
|
|
# This could in theory be done in the matrix, but it was getting too complex
|
|
- name: Update Matrix
|
|
id: update-matrix
|
|
run: |
|
|
REWRITTEN_SUITE="${{ matrix.suite }}"
|
|
if [[ "${{ matrix.setup }}" == "vllm" && "${{ matrix.suite }}" == "base" ]]; then
|
|
REWRITTEN_SUITE="base-vllm-subset"
|
|
fi
|
|
echo "suite=${REWRITTEN_SUITE}" >> $GITHUB_OUTPUT
|
|
echo "Rewritten suite: ${REWRITTEN_SUITE}"
|
|
|
|
- name: Setup test environment
|
|
uses: ./.github/actions/setup-test-environment
|
|
with:
|
|
python-version: ${{ matrix.python-version }}
|
|
client-version: ${{ matrix.client-version }}
|
|
setup: ${{ matrix.setup }}
|
|
suite: ${{ steps.update-matrix.outputs.suite }}
|
|
inference-mode: 'replay'
|
|
|
|
- name: Run tests
|
|
uses: ./.github/actions/run-and-record-tests
|
|
with:
|
|
stack-config: ${{ matrix.client-type == 'library' && 'ci-tests' || 'server:ci-tests' }}
|
|
setup: ${{ matrix.setup }}
|
|
inference-mode: 'replay'
|
|
suite: ${{ steps.update-matrix.outputs.suite }}
|