mirror of
https://github.com/meta-llama/llama-stack.git
synced 2025-08-15 22:18:00 +00:00
feat(ci): add support for running vision inference tests (#2972)
This PR significantly refactors the Integration Tests workflow. The main goal behind the PR was to enable recording of vision tests which were never run as part of our CI ever before. During debugging, I ended up making several other changes refactoring and hopefully increasing the robustness of the workflow. After doing the experiments, I have updated the trigger event to be `pull_request_target` so this workflow can get write permissions by default but it will run with source code from the base (main) branch in the source repository only. If you do change the workflow, you'd need to experiment using the `workflow_dispatch` triggers. This should not be news to anyone using Github Actions (except me!) It is likely to be a little rocky though while I learn more about GitHub Actions, etc. Please be patient :) --------- Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>
This commit is contained in:
Ashwin Bharambe
GitHub
parent
709c974bd8
commit
27d866795c
108 changed files with 13985 additions and 15254 deletions
113
.github/workflows/integration-tests.yml
vendored
113
.github/workflows/integration-tests.yml
vendored
|
|
@ -5,7 +5,7 @@ run-name: Run the integration test suite from tests/integration
|
|||
on:
|
||||
push:
|
||||
branches: [ main ]
|
||||
pull_request:
|
||||
pull_request_target:
|
||||
branches: [ main ]
|
||||
types: [opened, synchronize, labeled]
|
||||
paths:
|
||||
|
|
@ -13,10 +13,10 @@ on:
|
|||
- 'tests/**'
|
||||
- 'uv.lock'
|
||||
- 'pyproject.toml'
|
||||
- 'requirements.txt'
|
||||
- '.github/workflows/integration-tests.yml' # This workflow
|
||||
- '.github/actions/setup-ollama/action.yml'
|
||||
- '.github/actions/run-integration-tests/action.yml'
|
||||
- '.github/actions/setup-test-environment/action.yml'
|
||||
- '.github/actions/run-and-record-tests/action.yml'
|
||||
schedule:
|
||||
# If changing the cron schedule, update the provider in the test-matrix job
|
||||
- cron: '0 0 * * *' # (test latest client) Daily at 12 AM UTC
|
||||
|
|
@ -31,6 +31,10 @@ on:
|
|||
description: 'Test against a specific provider'
|
||||
type: string
|
||||
default: 'ollama'
|
||||
force-inference-mode:
|
||||
description: 'Force inference mode (record or replay)'
|
||||
type: string
|
||||
default: ''
|
||||
|
||||
concurrency:
|
||||
# This creates three concurrency groups:
|
||||
|
|
@ -73,10 +77,16 @@ jobs:
|
|||
- name: Check if re-record-tests label exists
|
||||
id: check-rerecord-tests
|
||||
run: |
|
||||
if [[ "${{ contains(github.event.pull_request.labels.*.name, 're-record-tests') }}" == "true" ]]; then
|
||||
if [[ "${{ inputs.force-inference-mode }}" == "record" ]]; then
|
||||
echo "rerecord-tests=true" >> $GITHUB_OUTPUT
|
||||
else
|
||||
elif [[ "${{ inputs.force-inference-mode }}" == "replay" ]]; then
|
||||
echo "rerecord-tests=false" >> $GITHUB_OUTPUT
|
||||
else
|
||||
if [[ "${{ contains(github.event.pull_request.labels.*.name, 're-record-tests') }}" == "true" ]]; then
|
||||
echo "rerecord-tests=true" >> $GITHUB_OUTPUT
|
||||
else
|
||||
echo "rerecord-tests=false" >> $GITHUB_OUTPUT
|
||||
fi
|
||||
fi
|
||||
|
||||
record-tests:
|
||||
|
|
@ -92,65 +102,26 @@ jobs:
|
|||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
|
||||
with:
|
||||
ref: ${{ github.event.pull_request.head.ref }}
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Install dependencies
|
||||
uses: ./.github/actions/setup-runner
|
||||
- name: Setup test environment
|
||||
uses: ./.github/actions/setup-test-environment
|
||||
with:
|
||||
python-version: "3.12" # Use single Python version for recording
|
||||
client-version: "latest"
|
||||
provider: ${{ inputs.test-provider || 'ollama' }}
|
||||
|
||||
- name: Setup ollama
|
||||
if: ${{ inputs.test-provider == 'ollama' }}
|
||||
uses: ./.github/actions/setup-ollama
|
||||
|
||||
- name: Setup vllm
|
||||
if: ${{ inputs.test-provider == 'vllm' }}
|
||||
uses: ./.github/actions/setup-vllm
|
||||
|
||||
- name: Build Llama Stack
|
||||
run: |
|
||||
uv run llama stack build --template ci-tests --image-type venv
|
||||
|
||||
- name: Configure git for commits
|
||||
run: |
|
||||
git config --local user.email "github-actions[bot]@users.noreply.github.com"
|
||||
git config --local user.name "github-actions[bot]"
|
||||
|
||||
- name: Run Integration Tests for All Types (Recording Mode)
|
||||
uses: ./.github/actions/run-integration-tests
|
||||
- name: Run and record tests
|
||||
uses: ./.github/actions/run-and-record-tests
|
||||
with:
|
||||
test-types: ${{ needs.discover-tests.outputs.test-types }}
|
||||
stack-config: 'server:ci-tests' # recording must be done with server since more tests are run
|
||||
provider: ${{ inputs.test-provider }}
|
||||
provider: ${{ inputs.test-provider || 'ollama' }}
|
||||
inference-mode: 'record'
|
||||
|
||||
- name: Commit and push recordings
|
||||
run: |
|
||||
if ! git diff --quiet tests/integration/recordings/; then
|
||||
echo "Committing recordings"
|
||||
git add tests/integration/recordings/
|
||||
git commit -m "Update recordings"
|
||||
echo "Pushing all recording commits to PR"
|
||||
git push origin HEAD:${{ github.head_ref }}
|
||||
else
|
||||
echo "No recording changes"
|
||||
fi
|
||||
|
||||
- name: Write inference logs to file
|
||||
if: ${{ always() }}
|
||||
run: |
|
||||
sudo docker logs ollama > ollama-recording.log || true
|
||||
|
||||
- name: Upload recording logs
|
||||
if: ${{ always() }}
|
||||
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
|
||||
with:
|
||||
name: recording-logs-${{ github.run_id }}
|
||||
path: |
|
||||
*.log
|
||||
retention-days: 1
|
||||
|
||||
run-tests:
|
||||
run-replay-mode-tests:
|
||||
# Skip this job if we're in recording mode (handled by record-tests job)
|
||||
if: ${{ needs.discover-tests.outputs.rerecord-tests != 'true' }}
|
||||
needs: discover-tests
|
||||
|
|
@ -169,41 +140,17 @@ jobs:
|
|||
- name: Checkout repository
|
||||
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
|
||||
|
||||
- name: Install dependencies
|
||||
uses: ./.github/actions/setup-runner
|
||||
- name: Setup test environment
|
||||
uses: ./.github/actions/setup-test-environment
|
||||
with:
|
||||
python-version: ${{ matrix.python-version }}
|
||||
client-version: ${{ matrix.client-version }}
|
||||
provider: ${{ matrix.provider }}
|
||||
|
||||
- name: Build Llama Stack
|
||||
run: |
|
||||
uv run llama stack build --template ci-tests --image-type venv
|
||||
|
||||
- name: Check Storage and Memory Available Before Tests
|
||||
if: ${{ always() }}
|
||||
run: |
|
||||
free -h
|
||||
df -h
|
||||
|
||||
- name: Run Integration Tests (Replay Mode)
|
||||
uses: ./.github/actions/run-integration-tests
|
||||
- name: Run and record tests
|
||||
uses: ./.github/actions/run-and-record-tests
|
||||
with:
|
||||
test-types: ${{ needs.discover-tests.outputs.test-types }}
|
||||
stack-config: ${{ matrix.client-type == 'library' && 'ci-tests' || 'server:ci-tests' }}
|
||||
provider: ${{ matrix.provider }}
|
||||
inference-mode: 'replay'
|
||||
|
||||
- name: Check Storage and Memory Available After Tests
|
||||
if: ${{ always() }}
|
||||
run: |
|
||||
free -h
|
||||
df -h
|
||||
|
||||
- name: Upload test logs on failure
|
||||
if: ${{ failure() }}
|
||||
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
|
||||
with:
|
||||
name: test-logs-${{ github.run_id }}-${{ github.run_attempt }}-${{ matrix.provider }}-${{ matrix.client-type }}-${{ matrix.python-version }}-${{ matrix.client-version }}
|
||||
path: |
|
||||
*.log
|
||||
retention-days: 1
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue