mirror of
https://github.com/meta-llama/llama-stack.git
synced 2025-07-24 05:14:30 +00:00
6b8a8c1be9
- fireworks, together do not support Llama-guard 3 8b model anymore - Need to default to ollama - current safety shields logic was not correct since the shield_id was the provider ( which had duplicates ) - Followed similar logic to models Note: Seems a bit over-engineered but this can now be extended to other providers and fits in the overall mechanism of how env_vars are used to manage starter. ### How to test ``` ENABLE_OLLAMA=ollama ENABLE_FIREWORKS=fireworks SAFETY_MODEL=llama-guard3:1b pytest -s -v tests/integration/ --stack-config starter -k 'not(supervised_fine_tune or builtin_tool_code or safety_with_image or code_interpreter_for or rag_and_code or truncation or register_and_unregister)' --text-model fireworks/meta-llama/Llama-3.3-70B-Instruct --vision-model fireworks/meta-llama/Llama-4-Scout-17B-16E-Instruct --safety-shield llama-guard3:1b --embedding-model all-MiniLM-L6-v2 ``` ### Related but not obvious in this PR In the llama-stack-ops repo, we run tests before publishing packages and docker containers. The actions in that repo were using the fireworks / together distros ( which are non-existent ) So need to update that to run with `starter` and use `ollama` specifically for safety.
114 lines
3.8 KiB
YAML
114 lines
3.8 KiB
YAML
name: Integration Tests
|
|
|
|
on:
|
|
push:
|
|
branches: [ main ]
|
|
pull_request:
|
|
branches: [ main ]
|
|
paths:
|
|
- 'llama_stack/**'
|
|
- 'tests/integration/**'
|
|
- 'uv.lock'
|
|
- 'pyproject.toml'
|
|
- 'requirements.txt'
|
|
- '.github/workflows/integration-tests.yml' # This workflow
|
|
|
|
concurrency:
|
|
group: ${{ github.workflow }}-${{ github.ref }}
|
|
cancel-in-progress: true
|
|
|
|
jobs:
|
|
discover-tests:
|
|
runs-on: ubuntu-latest
|
|
outputs:
|
|
test-type: ${{ steps.generate-matrix.outputs.test-type }}
|
|
steps:
|
|
- name: Checkout repository
|
|
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
|
|
|
|
- name: Generate test matrix
|
|
id: generate-matrix
|
|
run: |
|
|
# Get test directories dynamically, excluding non-test directories
|
|
TEST_TYPES=$(find tests/integration -maxdepth 1 -mindepth 1 -type d -printf "%f\n" |
|
|
grep -Ev "^(__pycache__|fixtures|test_cases)$" |
|
|
sort | jq -R -s -c 'split("\n")[:-1]')
|
|
echo "test-type=$TEST_TYPES" >> $GITHUB_OUTPUT
|
|
|
|
test-matrix:
|
|
needs: discover-tests
|
|
runs-on: ubuntu-latest
|
|
|
|
strategy:
|
|
fail-fast: false
|
|
matrix:
|
|
test-type: ${{ fromJson(needs.discover-tests.outputs.test-type) }}
|
|
client-type: [library, server]
|
|
python-version: ["3.12", "3.13"]
|
|
|
|
steps:
|
|
- name: Checkout repository
|
|
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
|
|
|
|
- name: Install dependencies
|
|
uses: ./.github/actions/setup-runner
|
|
with:
|
|
python-version: ${{ matrix.python-version }}
|
|
|
|
- name: Setup ollama
|
|
uses: ./.github/actions/setup-ollama
|
|
|
|
- name: Build Llama Stack
|
|
run: |
|
|
uv run llama stack build --template starter --image-type venv
|
|
|
|
- name: Check Storage and Memory Available Before Tests
|
|
if: ${{ always() }}
|
|
run: |
|
|
free -h
|
|
df -h
|
|
|
|
- name: Run Integration Tests
|
|
env:
|
|
OLLAMA_INFERENCE_MODEL: "llama3.2:3b-instruct-fp16" # for server tests
|
|
ENABLE_OLLAMA: "ollama" # for server tests
|
|
OLLAMA_URL: "http://0.0.0.0:11434"
|
|
SAFETY_MODEL: "llama-guard3:1b"
|
|
LLAMA_STACK_CLIENT_TIMEOUT: "300" # Increased timeout for eval operations
|
|
# Use 'shell' to get pipefail behavior
|
|
# https://docs.github.com/en/actions/reference/workflow-syntax-for-github-actions#exit-codes-and-error-action-preference
|
|
# TODO: write a precommit hook to detect if a test contains a pipe but does not use 'shell: bash'
|
|
shell: bash
|
|
run: |
|
|
if [ "${{ matrix.client-type }}" == "library" ]; then
|
|
stack_config="starter"
|
|
else
|
|
stack_config="server:starter"
|
|
fi
|
|
uv run pytest -s -v tests/integration/${{ matrix.test-type }} --stack-config=${stack_config} \
|
|
-k "not(builtin_tool or safety_with_image or code_interpreter or test_rag)" \
|
|
--text-model="ollama/llama3.2:3b-instruct-fp16" \
|
|
--embedding-model=all-MiniLM-L6-v2 \
|
|
--safety-shield=$SAFETY_MODEL \
|
|
--color=yes \
|
|
--capture=tee-sys | tee pytest-${{ matrix.test-type }}.log
|
|
|
|
- name: Check Storage and Memory Available After Tests
|
|
if: ${{ always() }}
|
|
run: |
|
|
free -h
|
|
df -h
|
|
|
|
- name: Write ollama logs to file
|
|
if: ${{ always() }}
|
|
run: |
|
|
sudo docker logs ollama > ollama.log
|
|
|
|
- name: Upload all logs to artifacts
|
|
if: ${{ always() }}
|
|
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
|
|
with:
|
|
name: logs-${{ github.run_id }}-${{ github.run_attempt }}-${{ matrix.client-type }}-${{ matrix.test-type }}-${{ matrix.python-version }}
|
|
path: |
|
|
*.log
|
|
retention-days: 1
|