Merge 9f87d67849 into cc88789071

.github/workflows/README.md

@@ -12,6 +12,7 @@ Llama Stack uses GitHub Actions for Continuous Integration (CI). Below is a tabl
 | SqlStore Integration Tests | [integration-sql-store-tests.yml](integration-sql-store-tests.yml) | Run the integration test suite with SqlStore |
 | Integration Tests (Replay) | [integration-tests.yml](integration-tests.yml) | Run the integration test suites from tests/integration in replay mode |
 | Vector IO Integration Tests | [integration-vector-io-tests.yml](integration-vector-io-tests.yml) | Run the integration test suite with various VectorIO providers |
+| Pre-commit Fix | [pre-commit-fix.yml](pre-commit-fix.yml) | Apply a subset of pre-commit fixes |
 | Pre-commit | [pre-commit.yml](pre-commit.yml) | Run pre-commit checks |
 | Test Llama Stack Build | [providers-build.yml](providers-build.yml) | Test llama stack build |
 | Test llama stack list-deps | [providers-list-deps.yml](providers-list-deps.yml) | Test llama stack list-deps |

.github/workflows/pre-commit-fix.yml

@@ -0,0 +1,246 @@
+name: Pre-commit Fix
+run-name: Apply a subset of pre-commit fixes
+
+on:
+  workflow_dispatch:
+    inputs:
+      pr-number:
+        description: Pull request number to update
+        required: true
+
+permissions:
+  contents: write
+  pull-requests: write
+
+jobs:
+  autofix:
+    name: Run pre-commit and push fixes
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Resolve pull request metadata
+        id: pr
+        uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1
+        with:
+          result-encoding: string
+          script: |
+            const prNumber = Number(core.getInput('pr_number', { required: true }));
+            const { data: pr } = await github.rest.pulls.get({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              pull_number: prNumber,
+            });
+
+            if (pr.state !== 'open') {
+              core.setFailed(`Pull request #${prNumber} is not open.`);
+              return;
+            }
+
+            return JSON.stringify({
+              number: prNumber,
+              headRef: pr.head.ref,
+              headRepo: pr.head.repo.full_name,
+              baseRef: pr.base.ref,
+              isFork: pr.head.repo.full_name !== `${context.repo.owner}/${context.repo.repo}`,
+              maintainerCanModify: pr.maintainer_can_modify ? 'true' : 'false',
+              author: pr.user.login,
+            });
+          pr_number: ${{ github.event.inputs.pr-number }}
+
+      - name: Verify push permissions
+        run: |
+          pr_info='${{ steps.pr.outputs.result }}'
+          head_repo=$(echo "$pr_info" | jq -r '.headRepo')
+          maintainer_can_modify=$(echo "$pr_info" | jq -r '.maintainerCanModify')
+          author=$(echo "$pr_info" | jq -r '.author')
+
+          if [ "$head_repo" != "${{ github.repository }}" ] && [ "$maintainer_can_modify" != "true" ] && [ "${{ github.actor }}" != "$author" ]; then
+            echo "::error::This workflow cannot push to $head_repo because 'Allow edits from maintainers' is disabled."
+            echo "Ask the PR author to enable the setting or run the workflow from a fork with sufficient permissions."
+            exit 1
+          fi
+
+      - name: Check out pull request branch
+        uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
+        with:
+          repository: ${{ fromJSON(steps.pr.outputs.result).headRepo }}
+          ref: ${{ fromJSON(steps.pr.outputs.result).headRef }}
+          token: ${{ secrets.GITHUB_TOKEN }}
+          fetch-depth: 0
+          persist-credentials: false
+
+      - name: Retrieve trusted pre-commit config
+        id: trusted-config
+        run: |
+          set -euo pipefail
+          pr_info='${{ steps.pr.outputs.result }}'
+          base_ref=$(echo "$pr_info" | jq -r '.baseRef')
+          git fetch "https://github.com/${{ github.repository }}.git" "$base_ref:refs/remotes/upstream/$base_ref"
+          git show "upstream/$base_ref:.pre-commit-config.yaml" > .pre-commit-config.trusted.yaml
+
+      - name: Set up Python
+        uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6.0.0
+        with:
+          python-version: '3.12'
+          cache: pip
+          cache-dependency-path: |
+            **/requirements*.txt
+            .pre-commit-config.yaml
+            .pre-commit-config.trusted.yaml
+
+      - name: Install pre-commit tooling
+        run: |
+          python -m pip install 'pre-commit>=4.4.0' 'uv>=0.4.27'
+        env:
+          GITHUB_TOKEN: ''
+
+      # Spin up a temporary worktree for the base branch and ask pre-commit to
+      # execute the trusted codegen scripts from there while pointing their
+      # outputs back at the contributor's checkout.
+      - name: Run trusted codegen hooks
+        run: |
+          set -euo pipefail
+
+          pr_info='${{ steps.pr.outputs.result }}'
+          base_ref=$(echo "$pr_info" | jq -r '.baseRef')
+          head_ref=$(echo "$pr_info" | jq -r '.headRef')
+
+          mkdir -p .trusted
+          git worktree add --force --detach .trusted/base "upstream/$base_ref"
+
+          cleanup() {
+            rm -f .trusted-codegen.yaml
+            git worktree remove --force .trusted/base 2>/dev/null || true
+          }
+          trap cleanup EXIT
+
+          cat <<'YAML' > .trusted-codegen.yaml
+          repos:
+            - repo: local
+              hooks:
+                - id: trusted-uv-lock
+                  name: Trusted uv lock
+                  entry: bash
+                  args:
+                    - -c
+                    - '"$TRUSTED_HOOK_ROOT/scripts/uv-run-with-index.sh" lock'
+                  language: system
+                  pass_filenames: false
+                  always_run: true
+                - id: trusted-distro-codegen
+                  name: Trusted distribution codegen
+                  entry: bash
+                  args:
+                    - -c
+                    - |
+                      "$TRUSTED_HOOK_ROOT/scripts/uv-run-with-index.sh" run --group codegen \
+                        "$TRUSTED_HOOK_ROOT/scripts/distro_codegen.py" --repo-root "$TRUSTED_TARGET_ROOT"
+                  language: system
+                  pass_filenames: false
+                  always_run: true
+                - id: trusted-provider-codegen
+                  name: Trusted provider codegen
+                  entry: bash
+                  args:
+                    - -c
+                    - |
+                      "$TRUSTED_HOOK_ROOT/scripts/uv-run-with-index.sh" run --group codegen \
+                        "$TRUSTED_HOOK_ROOT/scripts/provider_codegen.py" --repo-root "$TRUSTED_TARGET_ROOT"
+                  language: system
+                  pass_filenames: false
+                  always_run: true
+                - id: trusted-openapi-generator
+                  name: Trusted OpenAPI generator
+                  entry: bash
+                  args:
+                    - -c
+                    - |
+                      "$TRUSTED_HOOK_ROOT/scripts/uv-run-with-index.sh" run \
+                        "$TRUSTED_HOOK_ROOT/docs/openapi_generator/run_openapi_generator.sh" --target-root "$TRUSTED_TARGET_ROOT"
+                  language: system
+                  pass_filenames: false
+                  always_run: true
+          YAML
+
+          export TRUSTED_HOOK_ROOT="$PWD/.trusted/base"
+          export TRUSTED_TARGET_ROOT="$PWD"
+          export PYTHONPATH="$TRUSTED_TARGET_ROOT:$TRUSTED_TARGET_ROOT/src:${PYTHONPATH:-}"
+          export GITHUB_BASE_REF="$base_ref"
+          export GITHUB_REF="refs/heads/$head_ref"
+
+          pre-commit run --all-files --config .trusted-codegen.yaml
+        env:
+          GITHUB_TOKEN: ''
+
+      - name: Run trusted pre-commit subset
+        id: precommit
+        run: |
+          set -euo pipefail
+          echo "Running trusted pre-commit hooks from base branch"
+          hooks=(
+            trailing-whitespace
+            end-of-file-fixer
+            mixed-line-ending
+            insert-license
+            blacken-docs
+            ruff
+            ruff-format
+          )
+          status=0
+          for hook in "${hooks[@]}"; do
+            echo "::group::Running $hook"
+            if ! pre-commit run "$hook" --show-diff-on-failure --color=always --all-files --config .pre-commit-config.trusted.yaml; then
+              status=$?
+            fi
+            echo "::endgroup::"
+          done
+          exit "$status"
+        env:
+          RUFF_OUTPUT_FORMAT: github
+          GITHUB_TOKEN: ''
+
+      # These hooks come from the repository's base branch configuration, so
+      # contributors cannot smuggle new hook definitions or tweak pinned
+      # versions through the pull request. The selected set intentionally
+      # excludes repo-local entries and tooling that would execute scripts from
+      # the PR itself, trading coverage for a safer runner profile. It now
+      # focuses further on hooks that perform automatic fixes so the run only
+      # attempts changes that can be committed back to the branch.
+      - name: Configure git user
+        run: |
+          git config user.name "github-actions[bot]"
+          git config user.email "github-actions[bot]@users.noreply.github.com"
+
+      - name: Commit and push changes
+        id: push
+        run: |
+          set -e
+          branch='${{ fromJSON(steps.pr.outputs.result).headRef }}'
+          if [ -n "$(git status --porcelain)" ]; then
+            git add -A
+            git commit -m "Apply pre-commit fixes"
+            git push "https://x-access-token:${{ secrets.GITHUB_TOKEN }}@github.com/${{ fromJSON(steps.pr.outputs.result).headRepo }}.git" "HEAD:$branch"
+            echo "pushed=true" >> "$GITHUB_OUTPUT"
+          else
+            echo "No changes to commit"
+            echo "pushed=false" >> "$GITHUB_OUTPUT"
+          fi
+
+      - name: Comment on pull request
+        uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1
+        with:
+          script: |
+            const prNumber = Number(core.getInput('pr_number', { required: true }));
+            const pushed = core.getInput('pushed');
+            const runUrl = `https://github.com/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`;
+            const messages = {
+              true: `✅ Applied trusted pre-commit fixes in [workflow run](${runUrl}).`,
+              false: `ℹ️ Trusted pre-commit workflow completed with no changes. See [workflow run](${runUrl}) for details.`,
+            };
+            await github.rest.issues.createComment({
+              ...context.repo,
+              issue_number: prNumber,
+              body: messages[pushed === 'true' ? 'true' : 'false'],
+            });
+          pr_number: ${{ github.event.inputs.pr-number }}
+          pushed: ${{ steps.push.outputs.pushed }}

docs/openapi_generator/run_openapi_generator.sh

@@ -5,9 +5,34 @@
 #
 # This source code is licensed under the terms described in the LICENSE file in
 # the root directory of this source tree.
+#
+# Pass --target-root to direct generated artifacts into an alternate checkout
+# (used by the trusted autofix workflow running from a base-branch worktree).
 
 PYTHONPATH=${PYTHONPATH:-}
 THIS_DIR="$(cd "$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")" && pwd)"
+SOURCE_STACK_DIR="$(dirname "$(dirname "$THIS_DIR")")"
+TARGET_STACK_DIR="$SOURCE_STACK_DIR"
+
+while [[ $# -gt 0 ]]; do
+    case "$1" in
+        --target-root)
+            shift
+            if [[ $# -eq 0 ]]; then
+                echo "Error: --target-root requires a value" >&2
+                exit 1
+            fi
+            TARGET_STACK_DIR="$1"
+            ;;
+        *)
+            echo "Error: unknown argument '$1'" >&2
+            exit 1
+            ;;
+    esac
+    shift
+done
+TARGET_STATIC_DIR="$TARGET_STACK_DIR/docs/openapi_generator/static"
+TARGET_SPEC_PATH="$TARGET_STACK_DIR/client-sdks/stainless/openapi.yml"
 
 set -euo pipefail
 
@@ -27,8 +52,10 @@ if [ ${#missing_packages[@]} -ne 0 ]; then
     exit 1
 fi
 
-stack_dir=$(dirname $(dirname $THIS_DIR))
-PYTHONPATH=$PYTHONPATH:$stack_dir \
-  python -m docs.openapi_generator.generate $(dirname $THIS_DIR)/static
+mkdir -p "$TARGET_STATIC_DIR"
+mkdir -p "$(dirname "$TARGET_SPEC_PATH")"
 
-cp $stack_dir/docs/static/stainless-llama-stack-spec.yaml $stack_dir/client-sdks/stainless/openapi.yml
+PYTHONPATH="$TARGET_STACK_DIR:$TARGET_STACK_DIR/src:$PYTHONPATH" \
+  python "$SOURCE_STACK_DIR/docs/openapi_generator/generate.py" "$TARGET_STATIC_DIR"
+
+cp "$TARGET_STACK_DIR/docs/static/stainless-llama-stack-spec.yaml" "$TARGET_SPEC_PATH"

scripts/distro_codegen.py

@@ -4,7 +4,12 @@
 #
 # This source code is licensed under the terms described in the LICENSE file in
 # the root directory of this source tree.
+#
+# CI can direct the generated artifacts into an alternate checkout by passing
+# --repo-root, allowing the trusted copy of this script to run from a separate
+# worktree.
 
+import argparse
 import concurrent.futures
 import importlib
 import subprocess
@@ -15,7 +20,28 @@ from pathlib import Path
 
 from rich.progress import Progress, SpinnerColumn, TextColumn
 
-REPO_ROOT = Path(__file__).parent.parent
+_DEFAULT_REPO_ROOT = Path(__file__).parent.parent
+REPO_ROOT = _DEFAULT_REPO_ROOT
+
+
+def set_repo_root(repo_root: Path) -> None:
+    """Update the global repository root used by helper functions."""
+
+    global REPO_ROOT
+    REPO_ROOT = repo_root
+
+
+def parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(
+        description="Generate distribution docs and YAML artifacts."
+    )
+    parser.add_argument(
+        "--repo-root",
+        type=Path,
+        default=_DEFAULT_REPO_ROOT,
+        help="Repository root where generated artifacts should be written.",
+    )
+    return parser.parse_args()
 
 
 class ChangedPathTracker:
@@ -93,6 +119,13 @@ def pre_import_distros(distro_dirs: list[Path]) -> None:
 
 
 def main():
+    args = parse_args()
+    repo_root = args.repo_root
+    if not repo_root.is_absolute():
+        repo_root = (Path.cwd() / repo_root).resolve()
+
+    set_repo_root(repo_root)
+
     distros_dir = REPO_ROOT / "src" / "llama_stack" / "distributions"
     change_tracker = ChangedPathTracker()
 

scripts/provider_codegen.py

@@ -4,7 +4,12 @@
 #
 # This source code is licensed under the terms described in the LICENSE file in
 # the root directory of this source tree.
+#
+# CI can direct the generated artifacts into an alternate checkout by passing
+# --repo-root, allowing the trusted copy of this script to run from a separate
+# worktree.
 
+import argparse
 import subprocess
 import sys
 from pathlib import Path
@@ -15,7 +20,28 @@ from rich.progress import Progress, SpinnerColumn, TextColumn
 
 from llama_stack.core.distribution import get_provider_registry
 
-REPO_ROOT = Path(__file__).parent.parent
+_DEFAULT_REPO_ROOT = Path(__file__).parent.parent
+REPO_ROOT = _DEFAULT_REPO_ROOT
+
+
+def set_repo_root(repo_root: Path) -> None:
+    """Update the global repository root used by helper functions."""
+
+    global REPO_ROOT
+    REPO_ROOT = repo_root
+
+
+def parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(
+        description="Regenerate provider documentation from source definitions."
+    )
+    parser.add_argument(
+        "--repo-root",
+        type=Path,
+        default=_DEFAULT_REPO_ROOT,
+        help="Repository root where generated artifacts should be written.",
+    )
+    return parser.parse_args()
 
 
 def get_api_docstring(api_name: str) -> str | None:
@@ -440,6 +466,13 @@ def check_for_changes(change_tracker: ChangedPathTracker) -> bool:
 
 
 def main():
+    args = parse_args()
+    repo_root = args.repo_root
+    if not repo_root.is_absolute():
+        repo_root = (Path.cwd() / repo_root).resolve()
+
+    set_repo_root(repo_root)
+
     change_tracker = ChangedPathTracker()
 
     with Progress(