feat(api): ensure StackRunConfig

StackRunConfig is part of our public API, ensure stability of this datatype using a pytest snapshot test.

If the pydantic model changes, it will fail. A snapshot can be re-generated via `@github-actions regenerate snapshots` by a code owner.

The API conformance test will then re-run and pass.

Signed-off-by: Charlie Doern <cdoern@redhat.com>
This commit is contained in:
Charlie Doern 2025-09-24 16:21:42 -04:00
parent 0dbf79c328
commit af94606828
14 changed files with 2427 additions and 104 deletions

View file

@ -4,6 +4,7 @@ Llama Stack uses GitHub Actions for Continuous Integration (CI). Below is a tabl
| Name | File | Purpose |
| ---- | ---- | ------- |
| PR Bot Commands | [bot-trigger.yml](bot-trigger.yml) | Bot command for PR |
| Update Changelog | [changelog.yml](changelog.yml) | Creates PR for updating the CHANGELOG.md |
| API Conformance Tests | [conformance.yml](conformance.yml) | Run the API Conformance test suite on the changes. |
| Installer CI | [install-script-ci.yml](install-script-ci.yml) | Test the installation script |
@ -12,10 +13,11 @@ Llama Stack uses GitHub Actions for Continuous Integration (CI). Below is a tabl
| Integration Tests (Replay) | [integration-tests.yml](integration-tests.yml) | Run the integration test suites from tests/integration in replay mode |
| Vector IO Integration Tests | [integration-vector-io-tests.yml](integration-vector-io-tests.yml) | Run the integration test suite with various VectorIO providers |
| Pre-commit | [pre-commit.yml](pre-commit.yml) | Run pre-commit checks |
| Pre-commit Bot | [precommit-trigger.yml](precommit-trigger.yml) | Pre-commit bot for PR |
| Run Pre-commit | [precommit-trigger.yml](precommit-trigger.yml) | Run Pre-commit via PR comment |
| Test Llama Stack Build | [providers-build.yml](providers-build.yml) | Test llama stack build |
| Python Package Build Test | [python-build-test.yml](python-build-test.yml) | Test building the llama-stack PyPI project |
| Integration Tests (Record) | [record-integration-tests.yml](record-integration-tests.yml) | Run the integration test suite from tests/integration |
| Run Snapshot Regeneration | [regenerate-snapshot-trigger.yml](regenerate-snapshot-trigger.yml) | Run Snapshot Regeneration via PR comment |
| Check semantic PR titles | [semantic-pr.yml](semantic-pr.yml) | Ensure that PR titles follow the conventional commit spec |
| Close stale issues and PRs | [stale_bot.yml](stale_bot.yml) | Run the Stale Bot action |
| Test External Providers Installed via Module | [test-external-provider-module.yml](test-external-provider-module.yml) | Test External Provider installation via Python module |

130
.github/workflows/bot-trigger.yml vendored Normal file
View file

@ -0,0 +1,130 @@
name: PR Bot Commands
run-name: Bot command for PR #${{ github.event.issue.number }}
on:
issue_comment:
types: [created]
jobs:
# Shared setup job for both pre-commit and snapshot regeneration
setup:
if: github.event.issue.pull_request && (contains(github.event.comment.body, '@github-actions run precommit') || contains(github.event.comment.body, '@github-actions regenerate snapshots'))
runs-on: ubuntu-latest
permissions:
contents: write
pull-requests: write
outputs:
authorized: ${{ steps.check_author.outputs.authorized }}
pr_number: ${{ steps.check_author.outputs.pr_number }}
pr_head_ref: ${{ steps.check_author.outputs.pr_head_ref }}
pr_head_sha: ${{ steps.check_author.outputs.pr_head_sha }}
pr_head_repo: ${{ steps.check_author.outputs.pr_head_repo }}
pr_base_ref: ${{ steps.check_author.outputs.pr_base_ref }}
is_fork: ${{ steps.check_author.outputs.is_fork }}
command: ${{ steps.detect_command.outputs.command }}
steps:
- name: Detect command
id: detect_command
run: |
COMMENT="${{ github.event.comment.body }}"
if [[ "$COMMENT" == *"@github-actions run precommit"* ]]; then
echo "command=precommit" >> $GITHUB_OUTPUT
elif [[ "$COMMENT" == *"@github-actions regenerate snapshots"* ]]; then
echo "command=regenerate-snapshots" >> $GITHUB_OUTPUT
fi
- name: Check comment author and get PR details
id: check_author
uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0
with:
github-token: ${{ secrets.GITHUB_TOKEN }}
script: |
// Get PR details
const pr = await github.rest.pulls.get({
owner: context.repo.owner,
repo: context.repo.repo,
pull_number: context.issue.number
});
// Check if commenter has write access or is the PR author
const commenter = context.payload.comment.user.login;
const prAuthor = pr.data.user.login;
let hasPermission = false;
// Check if commenter is PR author
if (commenter === prAuthor) {
hasPermission = true;
console.log(`Comment author ${commenter} is the PR author`);
} else {
// Check if commenter has write/admin access
try {
const permission = await github.rest.repos.getCollaboratorPermissionLevel({
owner: context.repo.owner,
repo: context.repo.repo,
username: commenter
});
const level = permission.data.permission;
hasPermission = ['write', 'admin', 'maintain'].includes(level);
console.log(`Comment author ${commenter} has permission: ${level}`);
} catch (error) {
console.log(`Could not check permissions for ${commenter}: ${error.message}`);
}
}
if (!hasPermission) {
await github.rest.issues.createComment({
owner: context.repo.owner,
repo: context.repo.repo,
issue_number: context.issue.number,
body: `❌ @${commenter} You don't have permission to trigger bot commands. Only PR authors or repository collaborators can run this command.`
});
core.setFailed(`User ${commenter} does not have permission`);
return;
}
// Save PR info for later steps
core.setOutput('pr_number', context.issue.number);
core.setOutput('pr_head_ref', pr.data.head.ref);
core.setOutput('pr_head_sha', pr.data.head.sha);
core.setOutput('pr_head_repo', pr.data.head.repo.full_name);
core.setOutput('pr_base_ref', pr.data.base.ref);
core.setOutput('is_fork', pr.data.head.repo.full_name !== context.payload.repository.full_name);
core.setOutput('authorized', 'true');
- name: React to comment
if: steps.check_author.outputs.authorized == 'true'
uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0
with:
github-token: ${{ secrets.GITHUB_TOKEN }}
script: |
await github.rest.reactions.createForIssueComment({
owner: context.repo.owner,
repo: context.repo.repo,
comment_id: context.payload.comment.id,
content: 'rocket'
});
pre-commit:
needs: setup
if: needs.setup.outputs.authorized == 'true' && needs.setup.outputs.command == 'precommit'
uses: ./.github/workflows/precommit-trigger.yml
with:
pr_number: ${{ needs.setup.outputs.pr_number }}
pr_head_ref: ${{ needs.setup.outputs.pr_head_ref }}
pr_head_sha: ${{ needs.setup.outputs.pr_head_sha }}
pr_head_repo: ${{ needs.setup.outputs.pr_head_repo }}
is_fork: ${{ needs.setup.outputs.is_fork }}
regenerate-snapshots:
needs: setup
if: needs.setup.outputs.authorized == 'true' && needs.setup.outputs.command == 'regenerate-snapshots'
uses: ./.github/workflows/regenerate-snapshots-trigger.yml
with:
pr_number: ${{ needs.setup.outputs.pr_number }}
pr_head_ref: ${{ needs.setup.outputs.pr_head_ref }}
pr_head_sha: ${{ needs.setup.outputs.pr_head_sha }}
pr_head_repo: ${{ needs.setup.outputs.pr_head_repo }}
is_fork: ${{ needs.setup.outputs.is_fork }}

View file

@ -40,6 +40,11 @@ jobs:
with:
fetch-depth: 0
- name: Install dependencies
uses: ./.github/actions/setup-runner
with:
python-version: "3.12"
# Check if we should skip conformance testing due to breaking changes
- name: Check if conformance test should be skipped
id: skip-check
@ -137,6 +142,11 @@ jobs:
run: |
oasdiff breaking --fail-on ERR $BASE_SPEC $CURRENT_SPEC --match-path '^/v1/'
# never skip this, instead if a breaking change is properly identified -- we should regenerate our snapshot.
- name: Run Pydantic Model Test
run: |
uv run --no-sync ./scripts/snapshot-test.sh tests/api/test_pydantic_models.py
# Report when test is skipped
- name: Report skip reason
if: steps.skip-check.outputs.skip == 'true'

View file

@ -1,96 +1,40 @@
name: Pre-commit Bot
name: Run Pre-commit
run-name: Pre-commit bot for PR #${{ github.event.issue.number }}
run-name: Run Pre-commit via PR comment
on:
issue_comment:
types: [created]
workflow_call:
inputs:
pr_number:
required: true
type: string
description: 'PR number'
pr_head_ref:
required: true
type: string
description: 'PR head branch ref'
pr_head_sha:
required: true
type: string
description: 'PR head SHA'
pr_head_repo:
required: true
type: string
description: 'PR head repository full name'
is_fork:
required: true
type: string
description: 'Whether PR is from a fork (true/false)'
jobs:
pre-commit:
# Only run on pull request comments
if: github.event.issue.pull_request && contains(github.event.comment.body, '@github-actions run precommit')
runs-on: ubuntu-latest
permissions:
contents: write
pull-requests: write
steps:
- name: Check comment author and get PR details
id: check_author
uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0
with:
github-token: ${{ secrets.GITHUB_TOKEN }}
script: |
// Get PR details
const pr = await github.rest.pulls.get({
owner: context.repo.owner,
repo: context.repo.repo,
pull_number: context.issue.number
});
// Check if commenter has write access or is the PR author
const commenter = context.payload.comment.user.login;
const prAuthor = pr.data.user.login;
let hasPermission = false;
// Check if commenter is PR author
if (commenter === prAuthor) {
hasPermission = true;
console.log(`Comment author ${commenter} is the PR author`);
} else {
// Check if commenter has write/admin access
try {
const permission = await github.rest.repos.getCollaboratorPermissionLevel({
owner: context.repo.owner,
repo: context.repo.repo,
username: commenter
});
const level = permission.data.permission;
hasPermission = ['write', 'admin', 'maintain'].includes(level);
console.log(`Comment author ${commenter} has permission: ${level}`);
} catch (error) {
console.log(`Could not check permissions for ${commenter}: ${error.message}`);
}
}
if (!hasPermission) {
await github.rest.issues.createComment({
owner: context.repo.owner,
repo: context.repo.repo,
issue_number: context.issue.number,
body: `❌ @${commenter} You don't have permission to trigger pre-commit. Only PR authors or repository collaborators can run this command.`
});
core.setFailed(`User ${commenter} does not have permission`);
return;
}
// Save PR info for later steps
core.setOutput('pr_number', context.issue.number);
core.setOutput('pr_head_ref', pr.data.head.ref);
core.setOutput('pr_head_sha', pr.data.head.sha);
core.setOutput('pr_head_repo', pr.data.head.repo.full_name);
core.setOutput('pr_base_ref', pr.data.base.ref);
core.setOutput('is_fork', pr.data.head.repo.full_name !== context.payload.repository.full_name);
core.setOutput('authorized', 'true');
- name: React to comment
if: steps.check_author.outputs.authorized == 'true'
uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0
with:
github-token: ${{ secrets.GITHUB_TOKEN }}
script: |
await github.rest.reactions.createForIssueComment({
owner: context.repo.owner,
repo: context.repo.repo,
comment_id: context.payload.comment.id,
content: 'rocket'
});
- name: Comment starting
if: steps.check_author.outputs.authorized == 'true'
uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0
with:
github-token: ${{ secrets.GITHUB_TOKEN }}
@ -98,39 +42,37 @@ jobs:
await github.rest.issues.createComment({
owner: context.repo.owner,
repo: context.repo.repo,
issue_number: ${{ steps.check_author.outputs.pr_number }},
body: `⏳ Running pre-commit hooks on PR #${{ steps.check_author.outputs.pr_number }}...`
issue_number: ${{ inputs.pr_number }},
body: `⏳ Running pre-commit hooks on PR #${{ inputs.pr_number }}...`
});
- name: Checkout PR branch (same-repo)
if: steps.check_author.outputs.authorized == 'true' && steps.check_author.outputs.is_fork == 'false'
if: inputs.is_fork == 'false'
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
with:
ref: ${{ steps.check_author.outputs.pr_head_ref }}
ref: ${{ inputs.pr_head_ref }}
fetch-depth: 0
token: ${{ secrets.GITHUB_TOKEN }}
- name: Checkout PR branch (fork)
if: steps.check_author.outputs.authorized == 'true' && steps.check_author.outputs.is_fork == 'true'
if: inputs.is_fork == 'true'
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
with:
repository: ${{ steps.check_author.outputs.pr_head_repo }}
ref: ${{ steps.check_author.outputs.pr_head_ref }}
repository: ${{ inputs.pr_head_repo }}
ref: ${{ inputs.pr_head_ref }}
fetch-depth: 0
token: ${{ secrets.GITHUB_TOKEN }}
- name: Verify checkout
if: steps.check_author.outputs.authorized == 'true'
run: |
echo "Current SHA: $(git rev-parse HEAD)"
echo "Expected SHA: ${{ steps.check_author.outputs.pr_head_sha }}"
if [[ "$(git rev-parse HEAD)" != "${{ steps.check_author.outputs.pr_head_sha }}" ]]; then
echo "Expected SHA: ${{ inputs.pr_head_sha }}"
if [[ "$(git rev-parse HEAD)" != "${{ inputs.pr_head_sha }}" ]]; then
echo "::error::Checked out SHA does not match expected SHA"
exit 1
fi
- name: Set up Python
if: steps.check_author.outputs.authorized == 'true'
uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6.0.0
with:
python-version: '3.12'
@ -140,7 +82,6 @@ jobs:
.pre-commit-config.yaml
- name: Set up Node.js
if: steps.check_author.outputs.authorized == 'true'
uses: actions/setup-node@a0853c24544627f65ddf259abe73b1d18a591444 # v5.0.0
with:
node-version: '20'
@ -148,12 +89,10 @@ jobs:
cache-dependency-path: 'llama_stack/ui/'
- name: Install npm dependencies
if: steps.check_author.outputs.authorized == 'true'
run: npm ci
working-directory: llama_stack/ui
- name: Run pre-commit
if: steps.check_author.outputs.authorized == 'true'
id: precommit
uses: pre-commit/action@2c7b3805fd2a0fd8c1884dcaebf91fc102a13ecd # v3.0.1
continue-on-error: true
@ -162,7 +101,6 @@ jobs:
RUFF_OUTPUT_FORMAT: github
- name: Check for changes
if: steps.check_author.outputs.authorized == 'true'
id: changes
run: |
if ! git diff --exit-code || [ -n "$(git ls-files --others --exclude-standard)" ]; then
@ -174,7 +112,7 @@ jobs:
fi
- name: Commit and push changes
if: steps.check_author.outputs.authorized == 'true' && steps.changes.outputs.has_changes == 'true'
if: steps.changes.outputs.has_changes == 'true'
run: |
git config --local user.email "github-actions[bot]@users.noreply.github.com"
git config --local user.name "github-actions[bot]"
@ -185,10 +123,10 @@ jobs:
🤖 Applied by @github-actions bot via pre-commit workflow"
# Push changes
git push origin HEAD:${{ steps.check_author.outputs.pr_head_ref }}
git push origin HEAD:${{ inputs.pr_head_ref }}
- name: Comment success with changes
if: steps.check_author.outputs.authorized == 'true' && steps.changes.outputs.has_changes == 'true'
if: steps.changes.outputs.has_changes == 'true'
uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0
with:
github-token: ${{ secrets.GITHUB_TOKEN }}
@ -196,12 +134,12 @@ jobs:
await github.rest.issues.createComment({
owner: context.repo.owner,
repo: context.repo.repo,
issue_number: ${{ steps.check_author.outputs.pr_number }},
issue_number: ${{ inputs.pr_number }},
body: `✅ Pre-commit hooks completed successfully!\n\n🔧 Changes have been committed and pushed to the PR branch.`
});
- name: Comment success without changes
if: steps.check_author.outputs.authorized == 'true' && steps.changes.outputs.has_changes == 'false' && steps.precommit.outcome == 'success'
if: steps.changes.outputs.has_changes == 'false' && steps.precommit.outcome == 'success'
uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0
with:
github-token: ${{ secrets.GITHUB_TOKEN }}
@ -209,7 +147,7 @@ jobs:
await github.rest.issues.createComment({
owner: context.repo.owner,
repo: context.repo.repo,
issue_number: ${{ steps.check_author.outputs.pr_number }},
issue_number: ${{ inputs.pr_number }},
body: `✅ Pre-commit hooks passed!\n\n✨ No changes needed - your code is already formatted correctly.`
});
@ -222,6 +160,6 @@ jobs:
await github.rest.issues.createComment({
owner: context.repo.owner,
repo: context.repo.repo,
issue_number: ${{ steps.check_author.outputs.pr_number }},
issue_number: ${{ inputs.pr_number }},
body: `❌ Pre-commit workflow failed!\n\nPlease check the [workflow logs](https://github.com/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}) for details.`
});

View file

@ -0,0 +1,148 @@
name: Run Snapshot Regeneration
run-name: Run Snapshot Regeneration via PR comment
on:
workflow_call:
inputs:
pr_number:
required: true
type: string
description: 'PR number'
pr_head_ref:
required: true
type: string
description: 'PR head branch ref'
pr_head_sha:
required: true
type: string
description: 'PR head SHA'
pr_head_repo:
required: true
type: string
description: 'PR head repository full name'
is_fork:
required: true
type: string
description: 'Whether PR is from a fork (true/false)'
jobs:
pre-commit:
runs-on: ubuntu-latest
permissions:
contents: write
pull-requests: write
steps:
- name: Comment starting
uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0
with:
github-token: ${{ secrets.GITHUB_TOKEN }}
script: |
await github.rest.issues.createComment({
owner: context.repo.owner,
repo: context.repo.repo,
issue_number: ${{ inputs.pr_number }},
body: `⏳ Regenerating snapshots for PR #${{ inputs.pr_number }}...`
});
- name: Checkout PR branch (same-repo)
if: inputs.is_fork == 'false'
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
with:
ref: ${{ inputs.pr_head_ref }}
fetch-depth: 0
token: ${{ secrets.GITHUB_TOKEN }}
- name: Checkout PR branch (fork)
if: inputs.is_fork == 'true'
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
with:
repository: ${{ inputs.pr_head_repo }}
ref: ${{ inputs.pr_head_ref }}
fetch-depth: 0
token: ${{ secrets.GITHUB_TOKEN }}
- name: Verify checkout
run: |
echo "Current SHA: $(git rev-parse HEAD)"
echo "Expected SHA: ${{ inputs.pr_head_sha }}"
if [[ "$(git rev-parse HEAD)" != "${{ inputs.pr_head_sha }}" ]]; then
echo "::error::Checked out SHA does not match expected SHA"
exit 1
fi
- name: Install dependencies
uses: ./.github/actions/setup-runner
with:
python-version: "3.12"
- name: Run snapshot test with regeneration
id: snapshot_test
run: |
uv run --no-sync ./scripts/snapshot-test.sh tests/api/test_pydantic_models.py true
- name: Check for changes
id: changes
run: |
if ! git diff --exit-code tests/api/snapshots/ || [ -n "$(git ls-files --others --exclude-standard tests/api/snapshots/)" ]; then
echo "has_changes=true" >> $GITHUB_OUTPUT
echo "Changes detected in snapshots"
else
echo "has_changes=false" >> $GITHUB_OUTPUT
echo "No snapshot changes"
fi
- name: Commit and push changes
if: steps.changes.outputs.has_changes == 'true'
run: |
git config --local user.email "github-actions[bot]@users.noreply.github.com"
git config --local user.name "github-actions[bot]"
git add tests/api/snapshots/
git commit -m "chore: regenerate snapshots for schema changes
🤖 Applied by @github-actions bot via snapshot-regenerate workflow"
# Push changes
git push origin HEAD:${{ inputs.pr_head_ref }}
- name: Comment success with changes
if: steps.changes.outputs.has_changes == 'true'
uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0
with:
github-token: ${{ secrets.GITHUB_TOKEN }}
script: |
await github.rest.issues.createComment({
owner: context.repo.owner,
repo: context.repo.repo,
issue_number: ${{ inputs.pr_number }},
body: `✅ Snapshot regeneration completed successfully!\n\n🔧 Updated snapshots have been committed and pushed to the PR branch.`
});
- name: Comment success without changes
if: steps.changes.outputs.has_changes == 'false'
uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0
with:
github-token: ${{ secrets.GITHUB_TOKEN }}
script: |
await github.rest.issues.createComment({
owner: context.repo.owner,
repo: context.repo.repo,
issue_number: ${{ inputs.pr_number }},
body: `✅ Snapshot test passed!\n\n✨ No changes needed - snapshots are already up to date.`
});
- name: Comment failure
if: failure()
uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0
with:
github-token: ${{ secrets.GITHUB_TOKEN }}
script: |
await github.rest.issues.createComment({
owner: context.repo.owner,
repo: context.repo.repo,
issue_number: ${{ inputs.pr_number }},
body: `❌ Snapshot regeneration workflow failed!\n\nPlease check the [workflow logs](https://github.com/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}) for details.`
});