mirror of
https://github.com/meta-llama/llama-stack.git
synced 2025-12-24 22:39:59 +00:00
11c912da0a
We are now testing the safety capability with the starter image. This includes a few changes: * Enable the safety integration test * Relax the shield model requirements from llama-guard to make it work with llama-guard3:8b coming from Ollama * Expose a shield for each inference provider in the starter distro. The shield will only be registered if the provider is enabled. Shields will be added if the provider claims to support a safety model * Missing providers models have been added too * Pointers to official documentation pages for provider models support have been added Closes: https://github.com/meta-llama/llama-stack/issues/2528 Signed-off-by: Sébastien Han <seb@redhat.com>
107 lines
3.7 KiB
YAML
107 lines
3.7 KiB
YAML
name: Integration Tests
|
|
|
|
on:
|
|
push:
|
|
branches: [ main ]
|
|
pull_request:
|
|
branches: [ main ]
|
|
paths:
|
|
- 'llama_stack/**'
|
|
- 'tests/integration/**'
|
|
- 'uv.lock'
|
|
- 'pyproject.toml'
|
|
- 'requirements.txt'
|
|
- '.github/workflows/integration-tests.yml' # This workflow
|
|
|
|
concurrency:
|
|
group: ${{ github.workflow }}-${{ github.ref }}
|
|
cancel-in-progress: true
|
|
|
|
jobs:
|
|
test-matrix:
|
|
runs-on: ubuntu-latest
|
|
strategy:
|
|
matrix:
|
|
# Listing tests manually since some of them currently fail
|
|
# TODO: generate matrix list from tests/integration when fixed
|
|
test-type: [agents, inference, datasets, inspect, safety, scoring, post_training, providers, tool_runtime, vector_io]
|
|
client-type: [library, server]
|
|
python-version: ["3.12", "3.13"]
|
|
fail-fast: false # we want to run all tests regardless of failure
|
|
|
|
steps:
|
|
- name: Checkout repository
|
|
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
|
|
|
|
- name: Install dependencies
|
|
uses: ./.github/actions/setup-runner
|
|
with:
|
|
python-version: ${{ matrix.python-version }}
|
|
|
|
- name: Setup ollama
|
|
uses: ./.github/actions/setup-ollama
|
|
|
|
- name: Build Llama Stack
|
|
run: |
|
|
uv run llama stack build --template starter --image-type venv
|
|
|
|
- name: Check Storage and Memory Available Before Tests
|
|
if: ${{ always() }}
|
|
run: |
|
|
free -h
|
|
df -h
|
|
|
|
- name: Verify Ollama status is OK
|
|
if: matrix.client-type == 'http'
|
|
run: |
|
|
echo "Verifying Ollama status..."
|
|
ollama_status=$(curl -s -L http://127.0.0.1:8321/v1/providers/ollama|jq --raw-output .health.status)
|
|
echo "Ollama status: $ollama_status"
|
|
if [ "$ollama_status" != "OK" ]; then
|
|
echo "Ollama health check failed"
|
|
exit 1
|
|
fi
|
|
|
|
- name: Run Integration Tests
|
|
env:
|
|
OLLAMA_INFERENCE_MODEL: "llama3.2:3b-instruct-fp16" # for server tests
|
|
ENABLE_OLLAMA: "ollama" # for server tests
|
|
OLLAMA_URL: "http://0.0.0.0:11434"
|
|
SAFETY_MODEL: "llama-guard3:1b"
|
|
# Use 'shell' to get pipefail behavior
|
|
# https://docs.github.com/en/actions/reference/workflow-syntax-for-github-actions#exit-codes-and-error-action-preference
|
|
# TODO: write a precommit hook to detect if a test contains a pipe but does not use 'shell: bash'
|
|
shell: bash
|
|
run: |
|
|
if [ "${{ matrix.client-type }}" == "library" ]; then
|
|
stack_config="starter"
|
|
else
|
|
stack_config="server:starter"
|
|
fi
|
|
uv run pytest -s -v tests/integration/${{ matrix.test-type }} --stack-config=${stack_config} \
|
|
-k "not(builtin_tool or safety_with_image or code_interpreter or test_rag)" \
|
|
--text-model="ollama/llama3.2:3b-instruct-fp16" \
|
|
--embedding-model=all-MiniLM-L6-v2 \
|
|
--safety-shield=ollama \
|
|
--color=yes \
|
|
--capture=tee-sys | tee pytest-${{ matrix.test-type }}.log
|
|
|
|
- name: Check Storage and Memory Available After Tests
|
|
if: ${{ always() }}
|
|
run: |
|
|
free -h
|
|
df -h
|
|
|
|
- name: Write ollama logs to file
|
|
if: ${{ always() }}
|
|
run: |
|
|
sudo docker logs ollama > ollama.log
|
|
|
|
- name: Upload all logs to artifacts
|
|
if: ${{ always() }}
|
|
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
|
|
with:
|
|
name: logs-${{ github.run_id }}-${{ github.run_attempt }}-${{ matrix.client-type }}-${{ matrix.test-type }}-${{ matrix.python-version }}
|
|
path: |
|
|
*.log
|
|
retention-days: 1
|