Merge branch 'main' into openai-vector-store/qdrant

.github/workflows/README.md

@@ -13,6 +13,7 @@ Llama Stack uses GitHub Actions for Continuous Integration (CI). Below is a tabl
 | Pre-commit | [pre-commit.yml](pre-commit.yml) | Run pre-commit checks |
 | Test Llama Stack Build | [providers-build.yml](providers-build.yml) | Test llama stack build |
 | Python Package Build Test | [python-build-test.yml](python-build-test.yml) | Test building the llama-stack PyPI project |
+| Integration Tests (Record) | [record-integration-tests.yml](record-integration-tests.yml) | Run the integration test suite from tests/integration |
 | Check semantic PR titles | [semantic-pr.yml](semantic-pr.yml) | Ensure that PR titles follow the conventional commit spec |
 | Close stale issues and PRs | [stale_bot.yml](stale_bot.yml) | Run the Stale Bot action |
 | Test External Providers Installed via Module | [test-external-provider-module.yml](test-external-provider-module.yml) | Test External Provider installation via Python module |

.github/workflows/integration-vector-io-tests.yml

@@ -24,7 +24,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        vector-io-provider: ["inline::faiss", "inline::sqlite-vec", "inline::milvus", "remote::chromadb", "remote::pgvector", "remote::qdrant"]
+        vector-io-provider: ["inline::faiss", "inline::sqlite-vec", "inline::milvus", "remote::chromadb", "remote::pgvector", "remote::weaviate", "remote::qdrant"]
         python-version: ["3.12", "3.13"]
       fail-fast: false # we want to run all tests regardless of failure
 
@@ -48,6 +48,14 @@ jobs:
             -e ANONYMIZED_TELEMETRY=FALSE \
             chromadb/chroma:latest
 
+      - name: Setup Weaviate
+        if: matrix.vector-io-provider == 'remote::weaviate'
+        run: |
+          docker run --rm -d --pull always \
+          --name weaviate \
+          -p 8080:8080 -p 50051:50051 \
+          cr.weaviate.io/semitechnologies/weaviate:1.32.0
+
       - name: Start PGVector DB
         if: matrix.vector-io-provider == 'remote::pgvector'
         run: |
@@ -116,6 +124,21 @@ jobs:
           docker logs chromadb
           exit 1
 
+      - name: Wait for Weaviate to be ready
+        if: matrix.vector-io-provider == 'remote::weaviate'
+        run: |
+          echo "Waiting for Weaviate to be ready..."
+          for i in {1..30}; do
+            if curl -s http://localhost:8080 | grep -q "https://weaviate.io/developers/weaviate/current/"; then
+              echo "Weaviate is ready!"
+              exit 0
+            fi
+            sleep 2
+          done
+          echo "Weaviate failed to start"
+          docker logs weaviate
+          exit 1
+
       - name: Build Llama Stack
         run: |
           uv run llama stack build --template ci-tests --image-type venv
@@ -138,6 +161,8 @@ jobs:
           PGVECTOR_PASSWORD: ${{ matrix.vector-io-provider == 'remote::pgvector' && 'llamastack' || '' }}
           ENABLE_QDRANT: ${{ matrix.vector-io-provider == 'remote::qdrant' && 'true' || '' }}
           QDRANT_URL: ${{ matrix.vector-io-provider == 'remote::qdrant' && 'http://localhost:6333' || '' }}
+          ENABLE_WEAVIATE: ${{ matrix.vector-io-provider == 'remote::weaviate' && 'true' || '' }}
+          WEAVIATE_CLUSTER_URL: ${{ matrix.vector-io-provider == 'remote::weaviate' && 'localhost:8080' || '' }}
         run: |
           uv run pytest -sv --stack-config="inference=inline::sentence-transformers,vector_io=${{ matrix.vector-io-provider }}" \
             tests/integration/vector_io \

.github/workflows/record-integration-tests.yml

@@ -0,0 +1,109 @@
+name: Integration Tests (Record)
+
+run-name: Run the integration test suite from tests/integration
+
+on:
+  pull_request:
+    branches: [ main ]
+    types: [opened, synchronize, labeled]
+    paths:
+      - 'llama_stack/**'
+      - 'tests/**'
+      - 'uv.lock'
+      - 'pyproject.toml'
+      - '.github/workflows/record-integration-tests.yml' # This workflow
+      - '.github/actions/setup-ollama/action.yml'
+      - '.github/actions/setup-test-environment/action.yml'
+      - '.github/actions/run-and-record-tests/action.yml'
+  workflow_dispatch:
+    inputs:
+      test-provider:
+        description: 'Test against a specific provider'
+        type: string
+        default: 'ollama'
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  discover-tests:
+    if: contains(github.event.pull_request.labels.*.name, 're-record-tests') ||
+      contains(github.event.pull_request.labels.*.name, 're-record-vision-tests')
+    runs-on: ubuntu-latest
+    outputs:
+      test-types: ${{ steps.generate-test-types.outputs.test-types }}
+      matrix-modes: ${{ steps.generate-test-types.outputs.matrix-modes }}
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+
+      - name: Generate test types
+        id: generate-test-types
+        run: |
+          # Get test directories dynamically, excluding non-test directories
+          TEST_TYPES=$(find tests/integration -maxdepth 1 -mindepth 1 -type d -printf "%f\n" |
+            grep -Ev "^(__pycache__|fixtures|test_cases|recordings|post_training)$" |
+            sort | jq -R -s -c 'split("\n")[:-1]')
+          echo "test-types=$TEST_TYPES" >> $GITHUB_OUTPUT
+
+          labels=$(gh pr view ${{ github.event.pull_request.number }} --json labels --jq '.labels[].name')
+          echo "labels=$labels"
+
+          modes_array=()
+          if [[ $labels == *"re-record-vision-tests"* ]]; then
+            modes_array+=("vision")
+          fi
+          if [[ $labels == *"re-record-tests"* ]]; then
+            modes_array+=("non-vision")
+          fi
+
+          # Convert to JSON array
+          if [ ${#modes_array[@]} -eq 0 ]; then
+            matrix_modes="[]"
+          else
+            matrix_modes=$(printf '%s\n' "${modes_array[@]}" | jq -R -s -c 'split("\n")[:-1]')
+          fi
+          echo "matrix_modes=$matrix_modes"
+          echo "matrix-modes=$matrix_modes" >> $GITHUB_OUTPUT
+
+        env:
+          GH_TOKEN: ${{ github.token }}
+
+  record-tests:
+    needs: discover-tests
+    runs-on: ubuntu-latest
+
+    permissions:
+      contents: write
+
+    strategy:
+      fail-fast: false
+      matrix:
+        mode: ${{ fromJSON(needs.discover-tests.outputs.matrix-modes) }}
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          ref: ${{ github.event.pull_request.head.ref }}
+          fetch-depth: 0
+
+      - name: Setup test environment
+        uses: ./.github/actions/setup-test-environment
+        with:
+          python-version: "3.12"  # Use single Python version for recording
+          client-version: "latest"
+          provider: ${{ inputs.test-provider || 'ollama' }}
+          run-vision-tests: ${{ matrix.mode == 'vision' && 'true' || 'false' }}
+          inference-mode: 'record'
+
+      - name: Run and record tests
+        uses: ./.github/actions/run-and-record-tests
+        with:
+          test-types: ${{ needs.discover-tests.outputs.test-types }}
+          stack-config: 'server:ci-tests'  # recording must be done with server since more tests are run
+          provider: ${{ inputs.test-provider || 'ollama' }}
+          inference-mode: 'record'
+          run-vision-tests: ${{ matrix.mode == 'vision' && 'true' || 'false' }}