name: Integration Tests on: push: branches: [ main ] pull_request: branches: [ main ] paths: - 'llama_stack/**' - 'tests/integration/**' - 'uv.lock' - 'pyproject.toml' - 'requirements.txt' - '.github/workflows/integration-tests.yml' # This workflow concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: test-matrix: runs-on: ubuntu-latest strategy: matrix: # Listing tests manually since some of them currently fail # TODO: generate matrix list from tests/integration when fixed test-type: [agents, inference, datasets, inspect, scoring, post_training, providers, tool_runtime, vector_io] client-type: [library, server] python-version: ["3.12", "3.13"] fail-fast: false # we want to run all tests regardless of failure steps: - name: Checkout repository uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - name: Install dependencies uses: ./.github/actions/setup-runner with: python-version: ${{ matrix.python-version }} - name: Setup ollama uses: ./.github/actions/setup-ollama - name: Build Llama Stack run: | uv run llama stack build --template starter --image-type venv - name: Check Storage and Memory Available Before Tests if: ${{ always() }} run: | free -h df -h - name: Run Integration Tests env: OLLAMA_INFERENCE_MODEL: "meta-llama/Llama-3.2-3B-Instruct" # for server tests ENABLE_OLLAMA: "ollama" # for server tests OLLAMA_URL: "http://0.0.0.0:11434" # Use 'shell' to get pipefail behavior # https://docs.github.com/en/actions/reference/workflow-syntax-for-github-actions#exit-codes-and-error-action-preference # TODO: write a precommit hook to detect if a test contains a pipe but does not use 'shell: bash' shell: bash run: | if [ "${{ matrix.client-type }}" == "library" ]; then stack_config="starter" else stack_config="server:starter" fi uv run pytest -s -v tests/integration/${{ matrix.test-type }} --stack-config=${stack_config} \ -k "not(builtin_tool or safety_with_image or code_interpreter or test_rag)" \ --text-model="ollama/meta-llama/Llama-3.2-3B-Instruct" \ --embedding-model=all-MiniLM-L6-v2 \ --color=yes \ --capture=tee-sys | tee pytest-${{ matrix.test-type }}.log - name: Check Storage and Memory Available After Tests if: ${{ always() }} run: | free -h df -h - name: Write ollama logs to file if: ${{ always() }} run: | sudo docker logs ollama > ollama.log - name: Upload all logs to artifacts if: ${{ always() }} uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: name: logs-${{ github.run_id }}-${{ github.run_attempt }}-${{ matrix.client-type }}-${{ matrix.test-type }}-${{ matrix.python-version }} path: | *.log retention-days: 1