mirror of
https://github.com/meta-llama/llama-stack.git
synced 2025-12-07 18:57:21 +00:00
0a6d122623
Add vLLM provider support to integration test CI workflows alongside existing Ollama support. Configure provider-specific test execution where vLLM runs only inference specific tests (excluding vision tests) while Ollama continues to run the full test suite. This enables comprehensive CI testing of both inference providers but keeps the vLLM footprint small, this can be expanded later if it proves to not be too disruptive. Also updated test skips that were marked with "inline::vllm", this should be "remote::vllm". This causes some failing log probs tests to be skipped and should be revisted. Signed-off-by: Derek Higgins <derekh@redhat.com>
103 lines
3.4 KiB
YAML
103 lines
3.4 KiB
YAML
name: 'Run and Record Tests'
|
|
description: 'Run integration tests and handle recording/artifact upload'
|
|
|
|
inputs:
|
|
stack-config:
|
|
description: 'Stack configuration to use'
|
|
required: true
|
|
setup:
|
|
description: 'Setup to use for tests (e.g., ollama, gpt, vllm)'
|
|
required: false
|
|
default: ''
|
|
inference-mode:
|
|
description: 'Inference mode (record or replay)'
|
|
required: true
|
|
suite:
|
|
description: 'Test suite to use: base, responses, vision, etc.'
|
|
required: false
|
|
default: ''
|
|
subdirs:
|
|
description: 'Comma-separated list of test subdirectories to run; overrides suite'
|
|
required: false
|
|
default: ''
|
|
pattern:
|
|
description: 'Regex pattern to pass to pytest -k'
|
|
required: false
|
|
default: ''
|
|
|
|
runs:
|
|
using: 'composite'
|
|
steps:
|
|
- name: Check Storage and Memory Available Before Tests
|
|
if: ${{ always() }}
|
|
shell: bash
|
|
run: |
|
|
free -h
|
|
df -h
|
|
|
|
- name: Run Integration Tests
|
|
shell: bash
|
|
run: |
|
|
SCRIPT_ARGS="--stack-config ${{ inputs.stack-config }} --inference-mode ${{ inputs.inference-mode }}"
|
|
|
|
# Add optional arguments only if they are provided
|
|
if [ -n '${{ inputs.setup }}' ]; then
|
|
SCRIPT_ARGS="$SCRIPT_ARGS --setup ${{ inputs.setup }}"
|
|
fi
|
|
if [ -n '${{ inputs.suite }}' ]; then
|
|
SCRIPT_ARGS="$SCRIPT_ARGS --suite ${{ inputs.suite }}"
|
|
fi
|
|
if [ -n '${{ inputs.subdirs }}' ]; then
|
|
SCRIPT_ARGS="$SCRIPT_ARGS --subdirs ${{ inputs.subdirs }}"
|
|
fi
|
|
if [ -n '${{ inputs.pattern }}' ]; then
|
|
SCRIPT_ARGS="$SCRIPT_ARGS --pattern ${{ inputs.pattern }}"
|
|
fi
|
|
|
|
echo "=== Running command ==="
|
|
echo "uv run --no-sync ./scripts/integration-tests.sh $SCRIPT_ARGS"
|
|
echo ""
|
|
|
|
uv run --no-sync ./scripts/integration-tests.sh $SCRIPT_ARGS | tee pytest-${{ inputs.inference-mode }}.log
|
|
|
|
|
|
- name: Commit and push recordings
|
|
if: ${{ inputs.inference-mode == 'record' }}
|
|
shell: bash
|
|
run: |
|
|
echo "Checking for recording changes"
|
|
git status --porcelain tests/integration/
|
|
|
|
if [[ -n $(git status --porcelain tests/integration/) ]]; then
|
|
echo "New recordings detected, committing and pushing"
|
|
git add tests/integration/
|
|
|
|
git commit -m "Recordings update from CI (setup: ${{ inputs.setup }}, suite: ${{ inputs.suite }})"
|
|
|
|
git fetch origin ${{ github.ref_name }}
|
|
git rebase origin/${{ github.ref_name }}
|
|
echo "Rebased successfully"
|
|
git push origin HEAD:${{ github.ref_name }}
|
|
echo "Pushed successfully"
|
|
else
|
|
echo "No recording changes"
|
|
fi
|
|
|
|
- name: Write docker logs to file
|
|
if: ${{ always() }}
|
|
shell: bash
|
|
run: |
|
|
# Ollama logs (if ollama container exists)
|
|
sudo docker logs ollama > ollama-${{ inputs.inference-mode }}.log 2>&1 || true
|
|
# vllm logs (if vllm container exists)
|
|
sudo docker logs vllm > vllm-${{ inputs.inference-mode }}.log 2>&1 || true
|
|
# Note: distro container logs are now dumped in integration-tests.sh before container is removed
|
|
|
|
- name: Upload logs
|
|
if: ${{ always() }}
|
|
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
|
|
with:
|
|
name: logs-${{ github.run_id }}-${{ github.run_attempt || '1' }}-${{ strategy.job-index || github.job }}-${{ github.action }}
|
|
path: |
|
|
*.log
|
|
retention-days: 1
|