feat(api): ensure StackRunConfig

StackRunConfig is part of our public API, ensure stability of this datatype using a pytest snapshot test.

If the pydantic model changes, it will fail. A snapshot can be re-generated via `@github-actions regenerate snapshots` by a code owner.

The API conformance test will then re-run and pass.

Signed-off-by: Charlie Doern <cdoern@redhat.com>

.github/workflows/README.md

@@ -4,6 +4,7 @@ Llama Stack uses GitHub Actions for Continuous Integration (CI). Below is a tabl
 
 | Name | File | Purpose |
 | ---- | ---- | ------- |
+| PR Bot Commands | [bot-trigger.yml](bot-trigger.yml) | Bot command for PR |
 | Update Changelog | [changelog.yml](changelog.yml) | Creates PR for updating the CHANGELOG.md |
 | API Conformance Tests | [conformance.yml](conformance.yml) | Run the API Conformance test suite on the changes. |
 | Installer CI | [install-script-ci.yml](install-script-ci.yml) | Test the installation script |
@@ -12,10 +13,11 @@ Llama Stack uses GitHub Actions for Continuous Integration (CI). Below is a tabl
 | Integration Tests (Replay) | [integration-tests.yml](integration-tests.yml) | Run the integration test suites from tests/integration in replay mode |
 | Vector IO Integration Tests | [integration-vector-io-tests.yml](integration-vector-io-tests.yml) | Run the integration test suite with various VectorIO providers |
 | Pre-commit | [pre-commit.yml](pre-commit.yml) | Run pre-commit checks |
-| Pre-commit Bot | [precommit-trigger.yml](precommit-trigger.yml) | Pre-commit bot for PR |
+| Run Pre-commit | [precommit-trigger.yml](precommit-trigger.yml) | Run Pre-commit via PR comment |
 | Test Llama Stack Build | [providers-build.yml](providers-build.yml) | Test llama stack build |
 | Python Package Build Test | [python-build-test.yml](python-build-test.yml) | Test building the llama-stack PyPI project |
 | Integration Tests (Record) | [record-integration-tests.yml](record-integration-tests.yml) | Run the integration test suite from tests/integration |
+| Run Snapshot Regeneration | [regenerate-snapshot-trigger.yml](regenerate-snapshot-trigger.yml) | Run Snapshot Regeneration via PR comment |
 | Check semantic PR titles | [semantic-pr.yml](semantic-pr.yml) | Ensure that PR titles follow the conventional commit spec |
 | Close stale issues and PRs | [stale_bot.yml](stale_bot.yml) | Run the Stale Bot action |
 | Test External Providers Installed via Module | [test-external-provider-module.yml](test-external-provider-module.yml) | Test External Provider installation via Python module |

.github/workflows/bot-trigger.yml

@@ -0,0 +1,130 @@
+name: PR Bot Commands
+
+run-name: Bot command for PR #${{ github.event.issue.number }}
+
+on:
+  issue_comment:
+    types: [created]
+
+jobs:
+  # Shared setup job for both pre-commit and snapshot regeneration
+  setup:
+    if: github.event.issue.pull_request && (contains(github.event.comment.body, '@github-actions run precommit') || contains(github.event.comment.body, '@github-actions regenerate snapshots'))
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+      pull-requests: write
+    outputs:
+      authorized: ${{ steps.check_author.outputs.authorized }}
+      pr_number: ${{ steps.check_author.outputs.pr_number }}
+      pr_head_ref: ${{ steps.check_author.outputs.pr_head_ref }}
+      pr_head_sha: ${{ steps.check_author.outputs.pr_head_sha }}
+      pr_head_repo: ${{ steps.check_author.outputs.pr_head_repo }}
+      pr_base_ref: ${{ steps.check_author.outputs.pr_base_ref }}
+      is_fork: ${{ steps.check_author.outputs.is_fork }}
+      command: ${{ steps.detect_command.outputs.command }}
+
+    steps:
+      - name: Detect command
+        id: detect_command
+        run: |
+          COMMENT="${{ github.event.comment.body }}"
+          if [[ "$COMMENT" == *"@github-actions run precommit"* ]]; then
+            echo "command=precommit" >> $GITHUB_OUTPUT
+          elif [[ "$COMMENT" == *"@github-actions regenerate snapshots"* ]]; then
+            echo "command=regenerate-snapshots" >> $GITHUB_OUTPUT
+          fi
+      - name: Check comment author and get PR details
+        id: check_author
+        uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          script: |
+            // Get PR details
+            const pr = await github.rest.pulls.get({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              pull_number: context.issue.number
+            });
+
+            // Check if commenter has write access or is the PR author
+            const commenter = context.payload.comment.user.login;
+            const prAuthor = pr.data.user.login;
+
+            let hasPermission = false;
+
+            // Check if commenter is PR author
+            if (commenter === prAuthor) {
+              hasPermission = true;
+              console.log(`Comment author ${commenter} is the PR author`);
+            } else {
+              // Check if commenter has write/admin access
+              try {
+                const permission = await github.rest.repos.getCollaboratorPermissionLevel({
+                  owner: context.repo.owner,
+                  repo: context.repo.repo,
+                  username: commenter
+                });
+
+                const level = permission.data.permission;
+                hasPermission = ['write', 'admin', 'maintain'].includes(level);
+                console.log(`Comment author ${commenter} has permission: ${level}`);
+              } catch (error) {
+                console.log(`Could not check permissions for ${commenter}: ${error.message}`);
+              }
+            }
+
+            if (!hasPermission) {
+              await github.rest.issues.createComment({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                issue_number: context.issue.number,
+                body: `❌ @${commenter} You don't have permission to trigger bot commands. Only PR authors or repository collaborators can run this command.`
+              });
+              core.setFailed(`User ${commenter} does not have permission`);
+              return;
+            }
+
+            // Save PR info for later steps
+            core.setOutput('pr_number', context.issue.number);
+            core.setOutput('pr_head_ref', pr.data.head.ref);
+            core.setOutput('pr_head_sha', pr.data.head.sha);
+            core.setOutput('pr_head_repo', pr.data.head.repo.full_name);
+            core.setOutput('pr_base_ref', pr.data.base.ref);
+            core.setOutput('is_fork', pr.data.head.repo.full_name !== context.payload.repository.full_name);
+            core.setOutput('authorized', 'true');
+
+      - name: React to comment
+        if: steps.check_author.outputs.authorized == 'true'
+        uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          script: |
+            await github.rest.reactions.createForIssueComment({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              comment_id: context.payload.comment.id,
+              content: 'rocket'
+            });
+
+  pre-commit:
+    needs: setup
+    if: needs.setup.outputs.authorized == 'true' && needs.setup.outputs.command == 'precommit'
+    uses: ./.github/workflows/precommit-trigger.yml
+    with:
+      pr_number: ${{ needs.setup.outputs.pr_number }}
+      pr_head_ref: ${{ needs.setup.outputs.pr_head_ref }}
+      pr_head_sha: ${{ needs.setup.outputs.pr_head_sha }}
+      pr_head_repo: ${{ needs.setup.outputs.pr_head_repo }}
+      is_fork: ${{ needs.setup.outputs.is_fork }}
+
+  regenerate-snapshots:
+    needs: setup
+    if: needs.setup.outputs.authorized == 'true' && needs.setup.outputs.command == 'regenerate-snapshots'
+    uses: ./.github/workflows/regenerate-snapshots-trigger.yml
+    with:
+      pr_number: ${{ needs.setup.outputs.pr_number }}
+      pr_head_ref: ${{ needs.setup.outputs.pr_head_ref }}
+      pr_head_sha: ${{ needs.setup.outputs.pr_head_sha }}
+      pr_head_repo: ${{ needs.setup.outputs.pr_head_repo }}
+      is_fork: ${{ needs.setup.outputs.is_fork }}

.github/workflows/conformance.yml

@@ -40,6 +40,11 @@ jobs:
         with:
           fetch-depth: 0
 
+      - name: Install dependencies
+        uses: ./.github/actions/setup-runner
+        with:
+          python-version: "3.12"
+
       # Check if we should skip conformance testing due to breaking changes
       - name: Check if conformance test should be skipped
         id: skip-check
@@ -137,6 +142,11 @@ jobs:
         run: |
           oasdiff breaking --fail-on ERR $BASE_SPEC $CURRENT_SPEC --match-path '^/v1/'
 
+      # never skip this, instead if a breaking change is properly identified -- we should regenerate our snapshot.
+      - name: Run Pydantic Model Test
+        run: |
+          uv run --no-sync ./scripts/snapshot-test.sh tests/api/test_pydantic_models.py
+
       # Report when test is skipped
       - name: Report skip reason
         if: steps.skip-check.outputs.skip == 'true'

.github/workflows/precommit-trigger.yml

@@ -1,96 +1,40 @@
-name: Pre-commit Bot
+name: Run Pre-commit
 
-run-name: Pre-commit bot for PR #${{ github.event.issue.number }}
+run-name: Run Pre-commit via PR comment
 
 on:
-  issue_comment:
+  workflow_call:
-    types: [created]
+    inputs:
+      pr_number:
+        required: true
+        type: string
+        description: 'PR number'
+      pr_head_ref:
+        required: true
+        type: string
+        description: 'PR head branch ref'
+      pr_head_sha:
+        required: true
+        type: string
+        description: 'PR head SHA'
+      pr_head_repo:
+        required: true
+        type: string
+        description: 'PR head repository full name'
+      is_fork:
+        required: true
+        type: string
+        description: 'Whether PR is from a fork (true/false)'
 
 jobs:
   pre-commit:
-    # Only run on pull request comments
-    if: github.event.issue.pull_request && contains(github.event.comment.body, '@github-actions run precommit')
     runs-on: ubuntu-latest
     permissions:
       contents: write
       pull-requests: write
 
     steps:
-      - name: Check comment author and get PR details
-        id: check_author
-        uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0
-        with:
-          github-token: ${{ secrets.GITHUB_TOKEN }}
-          script: |
-            // Get PR details
-            const pr = await github.rest.pulls.get({
-              owner: context.repo.owner,
-              repo: context.repo.repo,
-              pull_number: context.issue.number
-            });
-
-            // Check if commenter has write access or is the PR author
-            const commenter = context.payload.comment.user.login;
-            const prAuthor = pr.data.user.login;
-
-            let hasPermission = false;
-
-            // Check if commenter is PR author
-            if (commenter === prAuthor) {
-              hasPermission = true;
-              console.log(`Comment author ${commenter} is the PR author`);
-            } else {
-              // Check if commenter has write/admin access
-              try {
-                const permission = await github.rest.repos.getCollaboratorPermissionLevel({
-                  owner: context.repo.owner,
-                  repo: context.repo.repo,
-                  username: commenter
-                });
-
-                const level = permission.data.permission;
-                hasPermission = ['write', 'admin', 'maintain'].includes(level);
-                console.log(`Comment author ${commenter} has permission: ${level}`);
-              } catch (error) {
-                console.log(`Could not check permissions for ${commenter}: ${error.message}`);
-              }
-            }
-
-            if (!hasPermission) {
-              await github.rest.issues.createComment({
-                owner: context.repo.owner,
-                repo: context.repo.repo,
-                issue_number: context.issue.number,
-                body: `❌ @${commenter} You don't have permission to trigger pre-commit. Only PR authors or repository collaborators can run this command.`
-              });
-              core.setFailed(`User ${commenter} does not have permission`);
-              return;
-            }
-
-            // Save PR info for later steps
-            core.setOutput('pr_number', context.issue.number);
-            core.setOutput('pr_head_ref', pr.data.head.ref);
-            core.setOutput('pr_head_sha', pr.data.head.sha);
-            core.setOutput('pr_head_repo', pr.data.head.repo.full_name);
-            core.setOutput('pr_base_ref', pr.data.base.ref);
-            core.setOutput('is_fork', pr.data.head.repo.full_name !== context.payload.repository.full_name);
-            core.setOutput('authorized', 'true');
-
-      - name: React to comment
-        if: steps.check_author.outputs.authorized == 'true'
-        uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0
-        with:
-          github-token: ${{ secrets.GITHUB_TOKEN }}
-          script: |
-            await github.rest.reactions.createForIssueComment({
-              owner: context.repo.owner,
-              repo: context.repo.repo,
-              comment_id: context.payload.comment.id,
-              content: 'rocket'
-            });
-
       - name: Comment starting
-        if: steps.check_author.outputs.authorized == 'true'
         uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0
         with:
           github-token: ${{ secrets.GITHUB_TOKEN }}
@@ -98,39 +42,37 @@ jobs:
             await github.rest.issues.createComment({
               owner: context.repo.owner,
               repo: context.repo.repo,
-              issue_number: ${{ steps.check_author.outputs.pr_number }},
+              issue_number: ${{ inputs.pr_number }},
-              body: `⏳ Running pre-commit hooks on PR #${{ steps.check_author.outputs.pr_number }}...`
+              body: `⏳ Running pre-commit hooks on PR #${{ inputs.pr_number }}...`
             });
 
       - name: Checkout PR branch (same-repo)
-        if: steps.check_author.outputs.authorized == 'true' && steps.check_author.outputs.is_fork == 'false'
+        if: inputs.is_fork == 'false'
         uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
         with:
-          ref: ${{ steps.check_author.outputs.pr_head_ref }}
+          ref: ${{ inputs.pr_head_ref }}
           fetch-depth: 0
           token: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Checkout PR branch (fork)
-        if: steps.check_author.outputs.authorized == 'true' && steps.check_author.outputs.is_fork == 'true'
+        if: inputs.is_fork == 'true'
         uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
         with:
-          repository: ${{ steps.check_author.outputs.pr_head_repo }}
+          repository: ${{ inputs.pr_head_repo }}
-          ref: ${{ steps.check_author.outputs.pr_head_ref }}
+          ref: ${{ inputs.pr_head_ref }}
           fetch-depth: 0
           token: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Verify checkout
-        if: steps.check_author.outputs.authorized == 'true'
         run: |
           echo "Current SHA: $(git rev-parse HEAD)"
-          echo "Expected SHA: ${{ steps.check_author.outputs.pr_head_sha }}"
+          echo "Expected SHA: ${{ inputs.pr_head_sha }}"
-          if [[ "$(git rev-parse HEAD)" != "${{ steps.check_author.outputs.pr_head_sha }}" ]]; then
+          if [[ "$(git rev-parse HEAD)" != "${{ inputs.pr_head_sha }}" ]]; then
             echo "::error::Checked out SHA does not match expected SHA"
             exit 1
           fi
 
       - name: Set up Python
-        if: steps.check_author.outputs.authorized == 'true'
         uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6.0.0
         with:
           python-version: '3.12'
@@ -140,7 +82,6 @@ jobs:
             .pre-commit-config.yaml
 
       - name: Set up Node.js
-        if: steps.check_author.outputs.authorized == 'true'
         uses: actions/setup-node@a0853c24544627f65ddf259abe73b1d18a591444 # v5.0.0
         with:
           node-version: '20'
@@ -148,12 +89,10 @@ jobs:
           cache-dependency-path: 'llama_stack/ui/'
 
       - name: Install npm dependencies
-        if: steps.check_author.outputs.authorized == 'true'
         run: npm ci
         working-directory: llama_stack/ui
 
       - name: Run pre-commit
-        if: steps.check_author.outputs.authorized == 'true'
         id: precommit
         uses: pre-commit/action@2c7b3805fd2a0fd8c1884dcaebf91fc102a13ecd # v3.0.1
         continue-on-error: true
@@ -162,7 +101,6 @@ jobs:
           RUFF_OUTPUT_FORMAT: github
 
       - name: Check for changes
-        if: steps.check_author.outputs.authorized == 'true'
         id: changes
         run: |
           if ! git diff --exit-code || [ -n "$(git ls-files --others --exclude-standard)" ]; then
@@ -174,7 +112,7 @@ jobs:
           fi
 
       - name: Commit and push changes
-        if: steps.check_author.outputs.authorized == 'true' && steps.changes.outputs.has_changes == 'true'
+        if: steps.changes.outputs.has_changes == 'true'
         run: |
           git config --local user.email "github-actions[bot]@users.noreply.github.com"
           git config --local user.name "github-actions[bot]"
@@ -185,10 +123,10 @@ jobs:
           🤖 Applied by @github-actions bot via pre-commit workflow"
 
           # Push changes
-          git push origin HEAD:${{ steps.check_author.outputs.pr_head_ref }}
+          git push origin HEAD:${{ inputs.pr_head_ref }}
 
       - name: Comment success with changes
-        if: steps.check_author.outputs.authorized == 'true' && steps.changes.outputs.has_changes == 'true'
+        if: steps.changes.outputs.has_changes == 'true'
         uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0
         with:
           github-token: ${{ secrets.GITHUB_TOKEN }}
@@ -196,12 +134,12 @@ jobs:
             await github.rest.issues.createComment({
               owner: context.repo.owner,
               repo: context.repo.repo,
-              issue_number: ${{ steps.check_author.outputs.pr_number }},
+              issue_number: ${{ inputs.pr_number }},
               body: `✅ Pre-commit hooks completed successfully!\n\n🔧 Changes have been committed and pushed to the PR branch.`
             });
 
       - name: Comment success without changes
-        if: steps.check_author.outputs.authorized == 'true' && steps.changes.outputs.has_changes == 'false' && steps.precommit.outcome == 'success'
+        if: steps.changes.outputs.has_changes == 'false' && steps.precommit.outcome == 'success'
         uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0
         with:
           github-token: ${{ secrets.GITHUB_TOKEN }}
@@ -209,7 +147,7 @@ jobs:
             await github.rest.issues.createComment({
               owner: context.repo.owner,
               repo: context.repo.repo,
-              issue_number: ${{ steps.check_author.outputs.pr_number }},
+              issue_number: ${{ inputs.pr_number }},
               body: `✅ Pre-commit hooks passed!\n\n✨ No changes needed - your code is already formatted correctly.`
             });
 
@@ -222,6 +160,6 @@ jobs:
             await github.rest.issues.createComment({
               owner: context.repo.owner,
               repo: context.repo.repo,
-              issue_number: ${{ steps.check_author.outputs.pr_number }},
+              issue_number: ${{ inputs.pr_number }},
               body: `❌ Pre-commit workflow failed!\n\nPlease check the [workflow logs](https://github.com/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}) for details.`
             });

.github/workflows/regenerate-snapshot-trigger.yml

@@ -0,0 +1,148 @@
+name: Run Snapshot Regeneration
+
+run-name: Run Snapshot Regeneration via PR comment
+
+
+on:
+  workflow_call:
+    inputs:
+      pr_number:
+        required: true
+        type: string
+        description: 'PR number'
+      pr_head_ref:
+        required: true
+        type: string
+        description: 'PR head branch ref'
+      pr_head_sha:
+        required: true
+        type: string
+        description: 'PR head SHA'
+      pr_head_repo:
+        required: true
+        type: string
+        description: 'PR head repository full name'
+      is_fork:
+        required: true
+        type: string
+        description: 'Whether PR is from a fork (true/false)'
+
+jobs:
+  pre-commit:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+      pull-requests: write
+
+    steps:
+      - name: Comment starting
+        uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          script: |
+            await github.rest.issues.createComment({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              issue_number: ${{ inputs.pr_number }},
+              body: `⏳ Regenerating snapshots for PR #${{ inputs.pr_number }}...`
+            });
+
+      - name: Checkout PR branch (same-repo)
+        if: inputs.is_fork == 'false'
+        uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
+        with:
+          ref: ${{ inputs.pr_head_ref }}
+          fetch-depth: 0
+          token: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Checkout PR branch (fork)
+        if: inputs.is_fork == 'true'
+        uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
+        with:
+          repository: ${{ inputs.pr_head_repo }}
+          ref: ${{ inputs.pr_head_ref }}
+          fetch-depth: 0
+          token: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Verify checkout
+        run: |
+          echo "Current SHA: $(git rev-parse HEAD)"
+          echo "Expected SHA: ${{ inputs.pr_head_sha }}"
+          if [[ "$(git rev-parse HEAD)" != "${{ inputs.pr_head_sha }}" ]]; then
+            echo "::error::Checked out SHA does not match expected SHA"
+            exit 1
+          fi
+
+      - name: Install dependencies
+        uses: ./.github/actions/setup-runner
+        with:
+          python-version: "3.12"
+
+      - name: Run snapshot test with regeneration
+        id: snapshot_test
+        run: |
+          uv run --no-sync ./scripts/snapshot-test.sh tests/api/test_pydantic_models.py true
+
+      - name: Check for changes
+        id: changes
+        run: |
+          if ! git diff --exit-code tests/api/snapshots/ || [ -n "$(git ls-files --others --exclude-standard tests/api/snapshots/)" ]; then
+            echo "has_changes=true" >> $GITHUB_OUTPUT
+            echo "Changes detected in snapshots"
+          else
+            echo "has_changes=false" >> $GITHUB_OUTPUT
+            echo "No snapshot changes"
+          fi
+
+      - name: Commit and push changes
+        if: steps.changes.outputs.has_changes == 'true'
+        run: |
+          git config --local user.email "github-actions[bot]@users.noreply.github.com"
+          git config --local user.name "github-actions[bot]"
+
+          git add tests/api/snapshots/
+          git commit -m "chore: regenerate snapshots for schema changes
+
+          🤖 Applied by @github-actions bot via snapshot-regenerate workflow"
+
+          # Push changes
+          git push origin HEAD:${{ inputs.pr_head_ref }}
+
+      - name: Comment success with changes
+        if: steps.changes.outputs.has_changes == 'true'
+        uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          script: |
+            await github.rest.issues.createComment({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              issue_number: ${{ inputs.pr_number }},
+              body: `✅ Snapshot regeneration completed successfully!\n\n🔧 Updated snapshots have been committed and pushed to the PR branch.`
+            });
+
+      - name: Comment success without changes
+        if: steps.changes.outputs.has_changes == 'false'
+        uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          script: |
+            await github.rest.issues.createComment({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              issue_number: ${{ inputs.pr_number }},
+              body: `✅ Snapshot test passed!\n\n✨ No changes needed - snapshots are already up to date.`
+            });
+
+      - name: Comment failure
+        if: failure()
+        uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          script: |
+            await github.rest.issues.createComment({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              issue_number: ${{ inputs.pr_number }},
+              body: `❌ Snapshot regeneration workflow failed!\n\nPlease check the [workflow logs](https://github.com/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}) for details.`
+            });

.pre-commit-config.yaml

@@ -1,4 +1,4 @@
-exclude: 'build/'
+exclude: 'build/|tests/api/snapshots/'
 
 default_language_version:
     python: python3.12

llama_stack/providers/utils/kvstore/config.py

@@ -54,6 +54,7 @@ class SqliteKVStoreConfig(CommonConfig):
     db_path: str = Field(
         default=(RUNTIME_BASE_DIR / "kvstore.db").as_posix(),
         description="File path for the sqlite database",
+        json_schema_extra={"default": "~/.llama/runtime/kvstore.db"},
     )
 
     @classmethod

llama_stack/providers/utils/sqlstore/sqlstore.py

@@ -39,6 +39,7 @@ class SqliteSqlStoreConfig(SqlAlchemySqlStoreConfig):
     db_path: str = Field(
         default=(RUNTIME_BASE_DIR / "sqlstore.db").as_posix(),
         description="Database path, e.g. ~/.llama/distributions/ollama/sqlstore.db",
+        json_schema_extra={"default": "~/.llama/runtime/sqlstore.db"},
     )
 
     @property

pyproject.toml

@@ -62,6 +62,7 @@ ui = [
 [dependency-groups]
 dev = [
     "pytest>=8.4",
+    "pytest-snapshot>=0.9.0",
     "pytest-timeout",
     "pytest-asyncio>=1.0",
     "pytest-cov",

scripts/snapshot-test.sh

@@ -0,0 +1,65 @@
+#!/bin/bash
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the terms described in the LICENSE file in
+# the root directory of this source tree.
+
+set -eu
+
+# Simple test runner for snapshot tests
+# Runs pytest with snapshot testing to detect schema changes
+
+# Get the script directory
+THIS_DIR=$(dirname "$0")
+ROOT_DIR="$THIS_DIR/.."
+SCRIPT=""
+REGENERATE_SNAPSHOT=""
+
+cd "$ROOT_DIR"
+
+usage() {
+    cat << EOF
+Usage: $0 [OPTIONS] <test-script> [regenerate-snapshot]
+
+Arguments:
+    test-script              Path to the test script to run (required)
+    regenerate-snapshot      Set to "true" to regenerate snapshots (optional)
+
+Options:
+    --help                   Show this help message
+
+Examples:
+    # Run snapshot test
+    $0 tests/api/test_pydantic_models.py
+
+    # Run with snapshot regeneration
+    $0 tests/api/test_pydantic_models.py true
+EOF
+    exit 0
+}
+
+# Parse command line arguments
+if [[ $# -gt 0 ]] && [[ "$1" == "--help" ]]; then
+    usage
+fi
+
+if [[ $# -lt 1 ]]; then
+    echo "Error: Missing required test script argument"
+    usage
+fi
+
+SCRIPT="$1"
+REGENERATE_SNAPSHOT="${2:-false}"
+
+# Run pytest with snapshot testing
+echo "=== Running Snapshot Tests ==="
+  if [[ "$REGENERATE_SNAPSHOT" == "true" ]]; then
+    echo "Regenerating snapshots..."
+    pytest -s -v "$SCRIPT" --snapshot-update
+    exit 0
+  else
+    pytest -s -v "$SCRIPT"  # do not update snapshots.
+  fi
+
+echo "✅ Snapshot Tests Complete"

tests/api/__init__.py

@@ -0,0 +1,5 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the terms described in the LICENSE file in
+# the root directory of this source tree.

tests/api/test_pydantic_models.py

@@ -0,0 +1,17 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the terms described in the LICENSE file in
+# the root directory of this source tree.
+
+import json
+
+from llama_stack.core.datatypes import StackRunConfig
+
+
+def test_run_config_v1_schema_is_unchanged(snapshot):
+    """
+    Ensures the V1 schema never changes.
+    """
+    schema = StackRunConfig.model_json_schema()
+    snapshot.assert_match(json.dumps(schema, indent=2), "stored_run_config_v1_schema.json")

uv.lock

@@ -1812,6 +1812,7 @@ dev = [
     { name = "pytest-cov" },
     { name = "pytest-html" },
     { name = "pytest-json-report" },
+    { name = "pytest-snapshot" },
     { name = "pytest-socket" },
     { name = "pytest-timeout" },
     { name = "ruamel-yaml" },
@@ -1928,6 +1929,7 @@ dev = [
     { name = "pytest-cov" },
     { name = "pytest-html" },
     { name = "pytest-json-report" },
+    { name = "pytest-snapshot", specifier = ">=0.9.0" },
     { name = "pytest-socket" },
     { name = "pytest-timeout" },
     { name = "ruamel-yaml" },
@@ -3692,6 +3694,18 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/3e/43/7e7b2ec865caa92f67b8f0e9231a798d102724ca4c0e1f414316be1c1ef2/pytest_metadata-3.1.1-py3-none-any.whl", hash = "sha256:c8e0844db684ee1c798cfa38908d20d67d0463ecb6137c72e91f418558dd5f4b", size = 11428, upload-time = "2024-02-12T19:38:42.531Z" },
 ]
 
+[[package]]
+name = "pytest-snapshot"
+version = "0.9.0"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "pytest" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/9b/7b/ab8f1fc1e687218aa66acec1c3674d9c443f6a2dc8cb6a50f464548ffa34/pytest-snapshot-0.9.0.tar.gz", hash = "sha256:c7013c3abc3e860f9feff899f8b4debe3708650d8d8242a61bf2625ff64db7f3", size = 19877, upload-time = "2022-04-23T17:35:31.751Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/cb/29/518f32faf6edad9f56d6e0107217f7de6b79f297a47170414a2bd4be7f01/pytest_snapshot-0.9.0-py3-none-any.whl", hash = "sha256:4b9fe1c21c868fe53a545e4e3184d36bc1c88946e3f5c1d9dd676962a9b3d4ab", size = 10715, upload-time = "2022-04-23T17:35:30.288Z" },
+]
+
 [[package]]
 name = "pytest-socket"
 version = "0.7.0"