name: Integration Tests

run-name: Run the integration test suite with Ollama

on:
  push:
    branches: [ main ]
  pull_request:
    branches: [ main ]
    types: [opened, synchronize, reopened, labeled, unlabeled]
    paths:
      - 'llama_stack/**'
      - 'tests/**'
      - 'uv.lock'
      - 'pyproject.toml'
      - 'requirements.txt'
      - '.github/workflows/integration-tests.yml' # This workflow
      - '.github/actions/setup-ollama/action.yml'
  schedule:
    # If changing the cron schedule, update the provider in the test-matrix job
    - cron: '0 0 * * *'  # (test latest client) Daily at 12 AM UTC
    - cron: '1 0 * * 0'  # (test vllm) Weekly on Sunday at 1 AM UTC
  workflow_dispatch:
    inputs:
      test-all-client-versions:
        description: 'Test against both the latest and published versions'
        type: boolean
        default: false
      test-provider:
        description: 'Test against a specific provider'
        type: string
        default: 'ollama'

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

jobs:
  discover-tests:
    runs-on: ubuntu-latest
    outputs:
      test-type: ${{ steps.generate-matrix.outputs.test-type }}
      rerecord-tests: ${{ steps.check-rerecord-tests.outputs.rerecord-tests }}

    steps:
      - name: Checkout repository
        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2

      - name: Generate test matrix
        id: generate-matrix
        run: |
          # Get test directories dynamically, excluding non-test directories
          TEST_TYPES=$(find tests/integration -maxdepth 1 -mindepth 1 -type d -printf "%f\n" |
            grep -Ev "^(__pycache__|fixtures|test_cases|recordings)$" |
            sort | jq -R -s -c 'split("\n")[:-1]')
          echo "test-type=$TEST_TYPES" >> $GITHUB_OUTPUT

      - name: Check if re-record-tests label exists
        id: check-rerecord-tests
        run: |
          if [[ "${{ contains(github.event.pull_request.labels.*.name, 're-record-tests') }}" == "true" ]]; then
            echo "rerecord-tests=true" >> $GITHUB_OUTPUT
          else
            echo "rerecord-tests=false" >> $GITHUB_OUTPUT
          fi

  test-matrix:
    needs: discover-tests
    runs-on: ubuntu-latest

    permissions:
      # Set write permissions since we might need to commit recordings
      contents: write
      pull-requests: write

    env:
      # Create reusable variable for the re-record tests condition
      SHOULD_RECORD: ${{ needs.discover-tests.outputs.rerecord-tests == 'true' }}
      # TODO: set up another var to track whether we need ollama or not
      # not every matrix type needs ollama

    strategy:
      fail-fast: false
      matrix:
        test-type: ${{ fromJson(needs.discover-tests.outputs.test-type) }}
        client-type: [library, server]
        # Use vllm on weekly schedule, otherwise use test-provider input (defaults to ollama)
        provider: ${{ (github.event.schedule == '1 0 * * 0') && fromJSON('["vllm"]') || fromJSON(format('["{0}"]', github.event.inputs.test-provider || 'ollama')) }}
        python-version: ["3.12", "3.13"]
        client-version: ${{ (github.event.schedule == '0 0 * * 0' || github.event.inputs.test-all-client-versions == 'true') && fromJSON('["published", "latest"]') || fromJSON('["latest"]') }}
        exclude: # TODO: look into why these tests are failing and fix them
          - provider: vllm
            test-type: safety
          - provider: vllm
            test-type: post_training
          - provider: vllm
            test-type: tool_runtime

    steps:
      - name: Debug
        run: |
          echo "test-type: ${{ matrix.test-type }}"
          echo "client-type: ${{ matrix.client-type }}"
          echo "provider: ${{ matrix.provider }}"
          echo "python-version: ${{ matrix.python-version }}"
          echo "client-version: ${{ matrix.client-version }}"
          echo "SHOULD_RECORD: ${{ env.SHOULD_RECORD }}"
          echo "rerecord-tests: ${{ needs.discover-tests.outputs.rerecord-tests }}"

      - name: Checkout repository
        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2

      - name: Install dependencies
        uses: ./.github/actions/setup-runner
        with:
          python-version: ${{ matrix.python-version }}
          client-version: ${{ matrix.client-version }}

      - name: Setup ollama
        if: ${{ matrix.provider == 'ollama' && env.SHOULD_RECORD == 'true' }}
        uses: ./.github/actions/setup-ollama

      - name: Setup vllm
        if: ${{ matrix.provider == 'vllm' }}
        uses: ./.github/actions/setup-vllm

      - name: Build Llama Stack
        run: |
          uv run llama stack build --template ci-tests --image-type venv

      - name: Check Storage and Memory Available Before Tests
        if: ${{ always() }}
        run: |
          free -h
          df -h

      - name: Run Integration Tests
        env:
          LLAMA_STACK_CLIENT_TIMEOUT: "300" # Increased timeout for eval operations
        # Use 'shell' to get pipefail behavior
        # https://docs.github.com/en/actions/reference/workflow-syntax-for-github-actions#exit-codes-and-error-action-preference
        # TODO: write a precommit hook to detect if a test contains a pipe but does not use 'shell: bash'
        shell: bash
        run: |
          if [ "${{ matrix.client-type }}" == "library" ]; then
            stack_config="ci-tests"
          else
            stack_config="server:ci-tests"
          fi

          EXCLUDE_TESTS="builtin_tool or safety_with_image or code_interpreter or test_rag"
          export LLAMA_STACK_TEST_RECORDING_DIR="tests/integration/recordings"

          if [ "$SHOULD_RECORD" == "true" ]; then
            export LLAMA_STACK_TEST_INFERENCE_MODE="record"
          else
            export LLAMA_STACK_TEST_INFERENCE_MODE="replay"
          fi

          if [ "${{ matrix.provider }}" == "ollama" ]; then
            export OLLAMA_URL="http://0.0.0.0:11434"
            export TEXT_MODEL=ollama/llama3.2:3b-instruct-fp16
            export SAFETY_MODEL="ollama/llama-guard3:1b"
            EXTRA_PARAMS="--safety-shield=llama-guard"
          else
            export VLLM_URL="http://localhost:8000/v1"
            export TEXT_MODEL=vllm/meta-llama/Llama-3.2-1B-Instruct
            # TODO: remove the not(test_inference_store_tool_calls) once we can get the tool called consistently
            EXTRA_PARAMS=
            EXCLUDE_TESTS="${EXCLUDE_TESTS} or test_inference_store_tool_calls"
          fi

          uv run pytest -s -v tests/integration/${{ matrix.test-type }} --stack-config=${stack_config} \
            -k "not( ${EXCLUDE_TESTS} )" \
            --text-model=$TEXT_MODEL \
            --embedding-model=sentence-transformers/all-MiniLM-L6-v2 \
            --color=yes ${EXTRA_PARAMS} \
            --capture=tee-sys | tee pytest-${{ matrix.test-type }}.log

      - name: Update the PR if tests/integration/recordings/ has changed
        if: ${{ env.SHOULD_RECORD == 'true' }}
        run: |
          if ! git diff --quiet tests/integration/recordings/; then
            echo "Updating PR with updated recordings"
            git config --local user.email "github-actions[bot]@users.noreply.github.com"
            git config --local user.name "github-actions[bot]"
            git add tests/integration/recordings/
            git commit -m "Update recordings from integration tests"
            git push origin HEAD:${{ github.head_ref }}
          else
            echo "No changes to recordings detected"
          fi

      - name: Check Storage and Memory Available After Tests
        if: ${{ always() }}
        run: |
          free -h
          df -h

      - name: Write inference logs to file
        if: ${{ env.SHOULD_RECORD == 'true' }}
        run: |
          sudo docker logs ollama > ollama.log || true
          sudo docker logs vllm > vllm.log || true

      - name: Upload all logs to artifacts
        if: ${{ env.SHOULD_RECORD == 'true' }}
        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
        with:
          name: logs-${{ github.run_id }}-${{ github.run_attempt }}-${{ matrix.provider }}-${{ matrix.client-type }}-${{ matrix.test-type }}-${{ matrix.python-version }}-${{ matrix.client-version }}
          path: |
            *.log
          retention-days: 1