feat(ci): use replay mode, setup ollama if specific label exists on PR (#2955)

This PR makes setting up Ollama optional for CI. By default, we use
`replay` mode for inference requests and use the stored results from the
`tests/integration/recordings/` directory.

Every so often, users will update tests which will need us to re-record.
To do this, we check for the existence of a label `re-record-tests` on
the PR. If detected,
- ollama is spun up
- inference mode is set to record
- after the tests are done, if any new changes are detected, they are
pushed back to the PR

## Test Plan

This is GitHub CI. Gotta test it live.

.github/workflows/integration-tests.yml

@@ -7,6 +7,7 @@ on:
     branches: [ main ]
   pull_request:
     branches: [ main ]
+    types: [opened, synchronize, reopened, labeled, unlabeled]
     paths:
       - 'llama_stack/**'
       - 'tests/**'
@@ -39,6 +40,8 @@ jobs:
     runs-on: ubuntu-latest
     outputs:
       test-type: ${{ steps.generate-matrix.outputs.test-type }}
+      rerecord-tests: ${{ steps.check-rerecord-tests.outputs.rerecord-tests }}
+
     steps:
       - name: Checkout repository
         uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
@@ -52,10 +55,30 @@ jobs:
             sort | jq -R -s -c 'split("\n")[:-1]')
           echo "test-type=$TEST_TYPES" >> $GITHUB_OUTPUT
 
+      - name: Check if re-record-tests label exists
+        id: check-rerecord-tests
+        run: |
+          if [[ "${{ contains(github.event.pull_request.labels.*.name, 're-record-tests') }}" == "true" ]]; then
+            echo "rerecord-tests=true" >> $GITHUB_OUTPUT
+          else
+            echo "rerecord-tests=false" >> $GITHUB_OUTPUT
+          fi
+
   test-matrix:
     needs: discover-tests
     runs-on: ubuntu-latest
 
+    permissions:
+      # Set write permissions since we might need to commit recordings
+      contents: write
+      pull-requests: write
+
+    env:
+      # Create reusable variable for the re-record tests condition
+      SHOULD_RECORD: ${{ needs.discover-tests.outputs.rerecord-tests == 'true' }}
+      # TODO: set up another var to track whether we need ollama or not
+      # not every matrix type needs ollama
+
     strategy:
       fail-fast: false
       matrix:
@@ -74,6 +97,16 @@ jobs:
             test-type: tool_runtime
 
     steps:
+      - name: Debug
+        run: |
+          echo "test-type: ${{ matrix.test-type }}"
+          echo "client-type: ${{ matrix.client-type }}"
+          echo "provider: ${{ matrix.provider }}"
+          echo "python-version: ${{ matrix.python-version }}"
+          echo "client-version: ${{ matrix.client-version }}"
+          echo "SHOULD_RECORD: ${{ env.SHOULD_RECORD }}"
+          echo "rerecord-tests: ${{ needs.discover-tests.outputs.rerecord-tests }}"
+
       - name: Checkout repository
         uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
 
@@ -84,7 +117,7 @@ jobs:
           client-version: ${{ matrix.client-version }}
 
       - name: Setup ollama
-        if: ${{ matrix.provider == 'ollama' }}
+        if: ${{ matrix.provider == 'ollama' && env.SHOULD_RECORD == 'true' }}
         uses: ./.github/actions/setup-ollama
 
       - name: Setup vllm
@@ -116,6 +149,14 @@ jobs:
           fi
 
           EXCLUDE_TESTS="builtin_tool or safety_with_image or code_interpreter or test_rag"
+          export LLAMA_STACK_TEST_RECORDING_DIR="tests/integration/recordings"
+
+          if [ "$SHOULD_RECORD" == "true" ]; then
+            export LLAMA_STACK_TEST_INFERENCE_MODE="record"
+          else
+            export LLAMA_STACK_TEST_INFERENCE_MODE="replay"
+          fi
+
           if [ "${{ matrix.provider }}" == "ollama" ]; then
             export OLLAMA_URL="http://0.0.0.0:11434"
             export TEXT_MODEL=ollama/llama3.2:3b-instruct-fp16
@@ -129,7 +170,6 @@ jobs:
             EXCLUDE_TESTS="${EXCLUDE_TESTS} or test_inference_store_tool_calls"
           fi
 
-
           uv run pytest -s -v tests/integration/${{ matrix.test-type }} --stack-config=${stack_config} \
             -k "not( ${EXCLUDE_TESTS} )" \
             --text-model=$TEXT_MODEL \
@@ -137,6 +177,20 @@ jobs:
             --color=yes ${EXTRA_PARAMS} \
             --capture=tee-sys | tee pytest-${{ matrix.test-type }}.log
 
+      - name: Update the PR if tests/integration/recordings/ has changed
+        if: ${{ env.SHOULD_RECORD == 'true' }}
+        run: |
+          if ! git diff --quiet tests/integration/recordings/; then
+            echo "Updating PR with updated recordings"
+            git config --local user.email "github-actions[bot]@users.noreply.github.com"
+            git config --local user.name "github-actions[bot]"
+            git add tests/integration/recordings/
+            git commit -m "Update recordings from integration tests"
+            git push origin HEAD:${{ github.head_ref }}
+          else
+            echo "No changes to recordings detected"
+          fi
+
       - name: Check Storage and Memory Available After Tests
         if: ${{ always() }}
         run: |
@@ -144,13 +198,13 @@ jobs:
           df -h
 
       - name: Write inference logs to file
-        if: ${{ always() }}
+        if: ${{ env.SHOULD_RECORD == 'true' }}
         run: |
           sudo docker logs ollama > ollama.log || true
           sudo docker logs vllm > vllm.log || true
 
       - name: Upload all logs to artifacts
-        if: ${{ always() }}
+        if: ${{ env.SHOULD_RECORD == 'true' }}
         uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
         with:
           name: logs-${{ github.run_id }}-${{ github.run_attempt }}-${{ matrix.provider }}-${{ matrix.client-type }}-${{ matrix.test-type }}-${{ matrix.python-version }}-${{ matrix.client-version }}