mirror of
https://github.com/meta-llama/llama-stack.git
synced 2025-07-23 21:04:29 +00:00
01c222e12f
# What does this PR do? We are now automatically building the list of integration test to run. In that process, eval and files and being tested now. This is pending https://github.com/meta-llama/llama-stack/pull/2628 Signed-off-by: Sébastien Han <seb@redhat.com>
114 lines
3.8 KiB
YAML
114 lines
3.8 KiB
YAML
name: Integration Tests
|
|
|
|
on:
|
|
push:
|
|
branches: [ main ]
|
|
pull_request:
|
|
branches: [ main ]
|
|
paths:
|
|
- 'llama_stack/**'
|
|
- 'tests/integration/**'
|
|
- 'uv.lock'
|
|
- 'pyproject.toml'
|
|
- 'requirements.txt'
|
|
- '.github/workflows/integration-tests.yml' # This workflow
|
|
|
|
concurrency:
|
|
group: ${{ github.workflow }}-${{ github.ref }}
|
|
cancel-in-progress: true
|
|
|
|
jobs:
|
|
discover-tests:
|
|
runs-on: ubuntu-latest
|
|
outputs:
|
|
test-type: ${{ steps.generate-matrix.outputs.test-type }}
|
|
steps:
|
|
- name: Checkout repository
|
|
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
|
|
|
|
- name: Generate test matrix
|
|
id: generate-matrix
|
|
run: |
|
|
# Get test directories dynamically, excluding non-test directories
|
|
TEST_TYPES=$(find tests/integration -maxdepth 1 -mindepth 1 -type d -printf "%f\n" |
|
|
grep -Ev "^(__pycache__|fixtures|test_cases)$" |
|
|
sort | jq -R -s -c 'split("\n")[:-1]')
|
|
echo "test-type=$TEST_TYPES" >> $GITHUB_OUTPUT
|
|
|
|
test-matrix:
|
|
needs: discover-tests
|
|
runs-on: ubuntu-latest
|
|
|
|
strategy:
|
|
fail-fast: false
|
|
matrix:
|
|
test-type: ${{ fromJson(needs.discover-tests.outputs.test-type) }}
|
|
client-type: [library, server]
|
|
python-version: ["3.12", "3.13"]
|
|
|
|
steps:
|
|
- name: Checkout repository
|
|
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
|
|
|
|
- name: Install dependencies
|
|
uses: ./.github/actions/setup-runner
|
|
with:
|
|
python-version: ${{ matrix.python-version }}
|
|
|
|
- name: Setup ollama
|
|
uses: ./.github/actions/setup-ollama
|
|
|
|
- name: Build Llama Stack
|
|
run: |
|
|
uv run llama stack build --template starter --image-type venv
|
|
|
|
- name: Check Storage and Memory Available Before Tests
|
|
if: ${{ always() }}
|
|
run: |
|
|
free -h
|
|
df -h
|
|
|
|
- name: Run Integration Tests
|
|
env:
|
|
OLLAMA_INFERENCE_MODEL: "llama3.2:3b-instruct-fp16" # for server tests
|
|
ENABLE_OLLAMA: "ollama" # for server tests
|
|
OLLAMA_URL: "http://0.0.0.0:11434"
|
|
SAFETY_MODEL: "llama-guard3:1b"
|
|
LLAMA_STACK_CLIENT_TIMEOUT: "300" # Increased timeout for eval operations
|
|
# Use 'shell' to get pipefail behavior
|
|
# https://docs.github.com/en/actions/reference/workflow-syntax-for-github-actions#exit-codes-and-error-action-preference
|
|
# TODO: write a precommit hook to detect if a test contains a pipe but does not use 'shell: bash'
|
|
shell: bash
|
|
run: |
|
|
if [ "${{ matrix.client-type }}" == "library" ]; then
|
|
stack_config="starter"
|
|
else
|
|
stack_config="server:starter"
|
|
fi
|
|
uv run pytest -s -v tests/integration/${{ matrix.test-type }} --stack-config=${stack_config} \
|
|
-k "not(builtin_tool or safety_with_image or code_interpreter or test_rag)" \
|
|
--text-model="ollama/llama3.2:3b-instruct-fp16" \
|
|
--embedding-model=all-MiniLM-L6-v2 \
|
|
--safety-shield=ollama \
|
|
--color=yes \
|
|
--capture=tee-sys | tee pytest-${{ matrix.test-type }}.log
|
|
|
|
- name: Check Storage and Memory Available After Tests
|
|
if: ${{ always() }}
|
|
run: |
|
|
free -h
|
|
df -h
|
|
|
|
- name: Write ollama logs to file
|
|
if: ${{ always() }}
|
|
run: |
|
|
sudo docker logs ollama > ollama.log
|
|
|
|
- name: Upload all logs to artifacts
|
|
if: ${{ always() }}
|
|
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
|
|
with:
|
|
name: logs-${{ github.run_id }}-${{ github.run_attempt }}-${{ matrix.client-type }}-${{ matrix.test-type }}-${{ matrix.python-version }}
|
|
path: |
|
|
*.log
|
|
retention-days: 1
|