kill separate ollama setup action

refactor and add setup-test-environment, run-and-record-tests actions

.github/actions/run-and-record-tests/action.yml

@@ -0,0 +1,77 @@
+name: 'Run and Record Tests'
+description: 'Run integration tests and handle recording/artifact upload'
+
+inputs:
+  test-types:
+    description: 'JSON array of test types to run'
+    required: true
+  stack-config:
+    description: 'Stack configuration to use'
+    required: true
+  provider:
+    description: 'Provider to use for tests'
+    required: true
+  inference-mode:
+    description: 'Inference mode (record or replay)'
+    required: true
+  run-vision-tests:
+    description: 'Whether to run vision tests'
+    required: false
+    default: 'false'
+
+runs:
+  using: 'composite'
+  steps:
+    - name: Check Storage and Memory Available Before Tests
+      if: ${{ always() }}
+      run: |
+        free -h
+        df -h
+
+    - name: Run Integration Tests
+      uses: ./.github/actions/run-integration-tests
+      with:
+        test-types: ${{ inputs.test-types }}
+        stack-config: ${{ inputs.stack-config }}
+        provider: ${{ inputs.provider }}
+        inference-mode: ${{ inputs.inference-mode }}
+        run-vision-tests: ${{ inputs.run-vision-tests }}
+
+    - name: Check Storage and Memory Available After Tests
+      if: ${{ always() }}
+      run: |
+        free -h
+        df -h
+
+    - name: Commit and push recordings
+      if: ${{ inputs.inference-mode == 'record' }}
+      shell: bash
+      run: |
+        git status --porcelain tests/integration/recordings/
+        if [[ -n $(git status --porcelain tests/integration/recordings/) ]]; then
+          echo "New recordings detected, committing and pushing"
+          git add tests/integration/recordings/
+          git commit -m "Recordings update from CI"
+
+          git fetch origin ${{ github.event.pull_request.head.ref }}
+          git rebase origin/${{ github.event.pull_request.head.ref }}
+          git push origin HEAD:${{ github.event.pull_request.head.ref }}
+          fi
+        else
+          echo "No recording changes"
+        fi
+
+    - name: Write inference logs to file
+      if: ${{ always() }}
+      shell: bash
+      run: |
+        sudo docker logs ollama > ollama-${{ inputs.inference-mode }}.log || true
+
+    - name: Upload logs
+      if: ${{ always() }}
+      uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
+      with:
+        name: ${{ inputs.inference-mode }}-logs-${{ github.run_id }}-${{ github.run_attempt || '' }}-${{ inputs.provider }}
+        path: |
+          *.log
+        retention-days: 1 

.github/actions/setup-ollama/action.yml

@@ -1,11 +1,23 @@
 name: Setup Ollama
 description: Start Ollama
+inputs:
+  run-vision-tests:
+    description: 'Run vision tests: "true" or "false"'
+    required: false
+    default: 'false'
 runs:
   using: "composite"
   steps:
     - name: Start Ollama
       shell: bash
       run: |
-        docker run -d --name ollama -p 11434:11434 docker.io/leseb/ollama-with-models
+        if [ "${{ inputs.run-vision-tests }}" == "true" ]; then
+          image="ollama/ollama-with-vision-model"
+        else
+          image="ollama-with-models"
+        fi
+
+        echo "Starting Ollama with image: $image"
+        docker run -d --name ollama -p 11434:11434 docker.io/llamastack/$image
         echo "Verifying Ollama status..."
         timeout 30 bash -c 'while ! curl -s -L http://127.0.0.1:11434; do sleep 1 && echo "."; done'

.github/actions/setup-test-environment/action.yml

@@ -0,0 +1,57 @@
+name: 'Setup Test Environment'
+description: 'Common setup steps for integration tests including dependencies, providers, and build'
+
+inputs:
+  python-version:
+    description: 'Python version to use'
+    required: true
+  client-version:
+    description: 'Client version (latest or published)'
+    required: true
+  provider:
+    description: 'Provider to setup (ollama or vllm)'
+    required: true
+    default: 'ollama'
+  run-vision-tests:
+    description: 'Whether to setup provider for vision tests'
+    required: false
+    default: 'false'
+  inference-mode:
+    description: 'Inference mode (record or replay)'
+    required: true
+
+runs:
+  using: 'composite'
+  steps:
+    - name: Checkout repository
+      uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+      with:
+        ref: ${{ inputs.inference-mode == 'record' && github.event.pull_request.head.ref || '' }}
+        fetch-depth: ${{ inputs.inference-mode == 'record' && '0' || '1' }}
+
+    - name: Install dependencies
+      uses: ./.github/actions/setup-runner
+      with:
+        python-version: ${{ inputs.python-version }}
+        client-version: ${{ inputs.client-version }}
+
+    - name: Setup ollama
+      if: ${{ inputs.provider == 'ollama' }}
+      uses: ./.github/actions/setup-ollama
+      with:
+        run-vision-tests: ${{ inputs.run-vision-tests }}
+
+    - name: Setup vllm
+      if: ${{ inputs.provider == 'vllm' }}
+      uses: ./.github/actions/setup-vllm
+
+    - name: Build Llama Stack
+      shell: bash
+      run: |
+        uv run llama stack build --template ci-tests --image-type venv
+
+    - name: Configure git for commits
+      shell: bash
+      run: |
+        git config --local user.email "github-actions[bot]@users.noreply.github.com"
+        git config --local user.name "github-actions[bot]" 

.github/actions/setup-vision-ollama/action.yml

@@ -1,11 +0,0 @@
-name: Setup Ollama for Vision Tests
-description: Start Ollama with vision model
-runs:
-  using: "composite"
-  steps:
-    - name: Start Ollama
-      shell: bash
-      run: |
-        docker run -d --name ollama -p 11434:11434 docker.io/llamastack/ollama-with-vision-model
-        echo "Verifying Ollama status..."
-        timeout 30 bash -c 'while ! curl -s -L http://127.0.0.1:11434; do sleep 1 && echo "."; done'

.github/workflows/integration-tests.yml

@@ -13,9 +13,10 @@ on:
       - 'tests/**'
       - 'uv.lock'
       - 'pyproject.toml'
-      - 'requirements.txt'
       - '.github/workflows/integration-tests.yml' # This workflow
       - '.github/actions/setup-ollama/action.yml'
+      - '.github/actions/setup-test-environment/action.yml'
+      - '.github/actions/run-and-record-tests/action.yml'
       - '.github/actions/run-integration-tests/action.yml'
   schedule:
     # If changing the cron schedule, update the provider in the test-matrix job
@@ -92,70 +93,23 @@ jobs:
       pull-requests: write
 
     steps:
-      - name: Checkout repository
+      - name: Setup test environment
-        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        uses: ./.github/actions/setup-test-environment
-
-      - name: Install dependencies
-        uses: ./.github/actions/setup-runner
         with:
           python-version: "3.12"  # Use single Python version for recording
           client-version: "latest"
+          provider: ${{ inputs.test-provider || 'ollama' }}
+          inference-mode: 'record'
 
-      - name: Setup ollama
+      - name: Run and record tests
-        if: ${{ (inputs.test-provider || 'ollama') == 'ollama' }}
+        uses: ./.github/actions/run-and-record-tests
-        uses: ./.github/actions/setup-ollama
-
-      - name: Setup vllm
-        if: ${{ (inputs.test-provider || 'ollama') == 'vllm' }}
-        uses: ./.github/actions/setup-vllm
-
-      - name: Build Llama Stack
-        run: |
-          uv run llama stack build --template ci-tests --image-type venv
-
-      - name: Configure git for commits
-        run: |
-          git config --local user.email "github-actions[bot]@users.noreply.github.com"
-          git config --local user.name "github-actions[bot]"
-
-      - name: Run Integration Tests for All Types (Recording Mode)
-        uses: ./.github/actions/run-integration-tests
         with:
           test-types: ${{ needs.discover-tests.outputs.test-types }}
           stack-config: 'server:ci-tests'  # recording must be done with server since more tests are run
           provider: ${{ inputs.test-provider || 'ollama' }}
           inference-mode: 'record'
 
-      - name: Commit and push recordings
+  run-replay-mode-tests:
-        run: |
-          if [[ -n $(git status --porcelain tests/integration/recordings/) ]]; then
-            echo "New recordings detected, committing and pushing"
-            
-            git checkout -B ${{ github.head_ref }}
-            
-            git add tests/integration/recordings/
-            git commit -m "Update recordings"            
-            git pull --rebase origin ${{ github.head_ref }}
-            git push origin ${{ github.head_ref }}
-          else
-            echo "No recording changes"
-          fi
-
-      - name: Write inference logs to file
-        if: ${{ always() }}
-        run: |
-          sudo docker logs ollama > ollama-recording.log || true
-
-      - name: Upload recording logs
-        if: ${{ always() }}
-        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
-        with:
-          name: recording-logs-${{ github.run_id }}
-          path: |
-            *.log
-          retention-days: 1
-
-  run-tests:
     # Skip this job if we're in recording mode (handled by record-tests job)
     if: ${{ needs.discover-tests.outputs.rerecord-tests != 'true' }}
     needs: discover-tests
@@ -171,44 +125,17 @@ jobs:
         client-version: ${{ (github.event.schedule == '0 0 * * 0' || github.event.inputs.test-all-client-versions == 'true') && fromJSON('["published", "latest"]') || fromJSON('["latest"]') }}
 
     steps:
-      - name: Checkout repository
+      - name: Setup test environment
-        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        uses: ./.github/actions/setup-test-environment
-
-      - name: Install dependencies
-        uses: ./.github/actions/setup-runner
         with:
           python-version: ${{ matrix.python-version }}
           client-version: ${{ matrix.client-version }}
+          provider: ${{ matrix.provider }}
 
-      - name: Build Llama Stack
+      - name: Run and record tests
-        run: |
+        uses: ./.github/actions/run-and-record-tests
-          uv run llama stack build --template ci-tests --image-type venv
-
-      - name: Check Storage and Memory Available Before Tests
-        if: ${{ always() }}
-        run: |
-          free -h
-          df -h
-
-      - name: Run Integration Tests (Replay Mode)
-        uses: ./.github/actions/run-integration-tests
         with:
           test-types: ${{ needs.discover-tests.outputs.test-types }}
           stack-config: ${{ matrix.client-type == 'library' && 'ci-tests' || 'server:ci-tests' }}
           provider: ${{ matrix.provider }}
           inference-mode: 'replay'
-
-      - name: Check Storage and Memory Available After Tests
-        if: ${{ always() }}
-        run: |
-          free -h
-          df -h
-
-      - name: Upload test logs on failure
-        if: ${{ failure() }}
-        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
-        with:
-          name: test-logs-${{ github.run_id }}-${{ github.run_attempt }}-${{ matrix.provider }}-${{ matrix.client-type }}-${{ matrix.python-version }}-${{ matrix.client-version }}
-          path: |
-            *.log
-          retention-days: 1

.github/workflows/integration-vision-tests.yml

@@ -13,9 +13,10 @@ on:
       - 'tests/**'
       - 'uv.lock'
       - 'pyproject.toml'
-      - 'requirements.txt'
       - '.github/workflows/integration-vision-tests.yml' # This workflow
-      - '.github/actions/setup-vision-ollama/action.yml'
+      - '.github/actions/setup-ollama/action.yml'
+      - '.github/actions/setup-test-environment/action.yml'
+      - '.github/actions/run-and-record-tests/action.yml'
       - '.github/actions/run-integration-tests/action.yml'
   workflow_dispatch:
     inputs:
@@ -43,18 +44,12 @@ jobs:
       (github.event.action == 'labeled' && contains(github.event.pull_request.labels.*.name, 're-record-tests'))
     runs-on: ubuntu-latest
     outputs:
-      test-types: ${{ steps.generate-test-types.outputs.test-types }}
       rerecord-tests: ${{ steps.check-rerecord-tests.outputs.rerecord-tests }}
 
     steps:
       - name: Checkout repository
         uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
 
-      - name: Generate test types
-        id: generate-test-types
-        run: |
-          echo "test-types='[\"vision\"]'" >> $GITHUB_OUTPUT
-
       - name: Check if re-record-tests label exists
         id: check-rerecord-tests
         run: |
@@ -75,75 +70,25 @@ jobs:
       pull-requests: write
 
     steps:
-      - name: Checkout repository
+      - name: Setup test environment
-        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        uses: ./.github/actions/setup-test-environment
-        with:
-          ref: ${{ github.event.pull_request.head.ref }}
-          fetch-depth: 0
-
-      - name: Install dependencies
-        uses: ./.github/actions/setup-runner
         with:
           python-version: "3.12"  # Use single Python version for recording
           client-version: "latest"
+          provider: 'ollama'
+          inference-mode: 'record'
+          run-vision-tests: 'true'
 
-      # - name: Setup ollama for vision tests
+      - name: Run and record tests
-      #   uses: ./.github/actions/setup-vision-ollama
+        uses: ./.github/actions/run-and-record-tests
-
-      # - name: Build Llama Stack
-      #   run: |
-      #     uv run llama stack build --template ci-tests --image-type venv
-
-      - name: Configure git for commits
-        run: |
-          git config --local user.email "github-actions[bot]@users.noreply.github.com"
-          git config --local user.name "github-actions[bot]"
-
-      # - name: Run Integration Tests for All Types (Recording Mode)
-      #   uses: ./.github/actions/run-integration-tests
-      #   with:
-      #     test-types: ${{ needs.discover-tests.outputs.test-types }}
-      #     stack-config: 'server:ci-tests'  # re-recording must be done in server mode
-      #     provider: 'ollama'
-      #     inference-mode: 'record'
-      #     run-vision-tests: 'true'
-
-      - name: Commit and push recordings
-        run: |
-          echo "Testing testing" > tests/integration/recordings/test.txt
-          echo "head ref: ${{ github.head_ref }}"
-          echo "ref: ${{ github.ref }}"
-
-          pwd
-          ls -la tests/integration/recordings/
-          git status --porcelain tests/integration/recordings/
-          git status
-
-          if [[ -n $(git status --porcelain tests/integration/recordings/) ]]; then
-            echo "New recordings detected, committing and pushing"
-            git add tests/integration/recordings/
-            git commit -m "Update recordings"
-
-            git push --force-with-lease origin HEAD:${{ github.event.pull_request.head.ref }}
-          else
-            echo "No recording changes"
-          fi
-
-      - name: Write inference logs to file
-        if: ${{ always() }}
-        run: |
-          sudo docker logs ollama > ollama-recording.log || true
-
-      - name: Upload recording logs
-        if: ${{ always() }}
-        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
         with:
-          name: recording-logs-${{ github.run_id }}
+          test-types: '["vision"]'
-          path: |
+          stack-config: 'server:ci-tests'  # re-recording must be done in server mode
-            *.log
+          provider: 'ollama'
-          retention-days: 1
+          inference-mode: 'record'
+          run-vision-tests: 'true'
 
-  run-tests:
+  run-replay-mode-tests:
     # Skip this job if we're in recording mode (handled by record-tests job)
     if: ${{ needs.discover-tests.outputs.rerecord-tests != 'true' }}
     needs: discover-tests
@@ -158,45 +103,19 @@ jobs:
         client-version: ["latest"]
 
     steps:
-      - name: Checkout repository
+      - name: Setup test environment
-        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        uses: ./.github/actions/setup-test-environment
-
-      - name: Install dependencies
-        uses: ./.github/actions/setup-runner
         with:
           python-version: ${{ matrix.python-version }}
           client-version: ${{ matrix.client-version }}
+          provider: ${{ matrix.provider }}
+          run-vision-tests: 'true'
 
-      - name: Build Llama Stack
+      - name: Run and record tests
-        run: |
+        uses: ./.github/actions/run-and-record-tests
-          uv run llama stack build --template ci-tests --image-type venv
-
-      - name: Check Storage and Memory Available Before Tests
-        if: ${{ always() }}
-        run: |
-          free -h
-          df -h
-
-      - name: Run Integration Tests (Replay Mode)
-        uses: ./.github/actions/run-integration-tests
         with:
-          test-types: ${{ needs.discover-tests.outputs.test-types }}
+          test-types: '["vision"]'
           stack-config: ${{ matrix.client-type == 'library' && 'ci-tests' || 'server:ci-tests' }}
           provider: ${{ matrix.provider }}
           inference-mode: 'replay'
           run-vision-tests: 'true'
-
-      - name: Check Storage and Memory Available After Tests
-        if: ${{ always() }}
-        run: |
-          free -h
-          df -h
-
-      - name: Upload test logs on failure
-        if: ${{ failure() }}
-        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
-        with:
-          name: test-logs-${{ github.run_id }}-${{ github.run_attempt }}-${{ matrix.provider }}-${{ matrix.client-type }}-${{ matrix.python-version }}-${{ matrix.client-version }}
-          path: |
-            *.log
-          retention-days: 1