From 9f87d678491e3350fe6bcd08e052823e1d229400 Mon Sep 17 00:00:00 2001 From: Ashwin Bharambe Date: Fri, 14 Nov 2025 15:29:31 -0800 Subject: [PATCH] fixes --- .github/workflows/pre-commit-fix.yml | 83 ++++++++++++++++++- .../run_openapi_generator.sh | 35 +++++++- scripts/distro_codegen.py | 35 +++++++- scripts/provider_codegen.py | 35 +++++++- 4 files changed, 180 insertions(+), 8 deletions(-) diff --git a/.github/workflows/pre-commit-fix.yml b/.github/workflows/pre-commit-fix.yml index 400c31edd..17d69360b 100644 --- a/.github/workflows/pre-commit-fix.yml +++ b/.github/workflows/pre-commit-fix.yml @@ -88,8 +88,87 @@ jobs: .pre-commit-config.yaml .pre-commit-config.trusted.yaml - - name: Install pre-commit - run: python -m pip install 'pre-commit>=4.4.0' + - name: Install pre-commit tooling + run: | + python -m pip install 'pre-commit>=4.4.0' 'uv>=0.4.27' + env: + GITHUB_TOKEN: '' + + # Spin up a temporary worktree for the base branch and ask pre-commit to + # execute the trusted codegen scripts from there while pointing their + # outputs back at the contributor's checkout. + - name: Run trusted codegen hooks + run: | + set -euo pipefail + + pr_info='${{ steps.pr.outputs.result }}' + base_ref=$(echo "$pr_info" | jq -r '.baseRef') + head_ref=$(echo "$pr_info" | jq -r '.headRef') + + mkdir -p .trusted + git worktree add --force --detach .trusted/base "upstream/$base_ref" + + cleanup() { + rm -f .trusted-codegen.yaml + git worktree remove --force .trusted/base 2>/dev/null || true + } + trap cleanup EXIT + + cat <<'YAML' > .trusted-codegen.yaml + repos: + - repo: local + hooks: + - id: trusted-uv-lock + name: Trusted uv lock + entry: bash + args: + - -c + - '"$TRUSTED_HOOK_ROOT/scripts/uv-run-with-index.sh" lock' + language: system + pass_filenames: false + always_run: true + - id: trusted-distro-codegen + name: Trusted distribution codegen + entry: bash + args: + - -c + - | + "$TRUSTED_HOOK_ROOT/scripts/uv-run-with-index.sh" run --group codegen \ + "$TRUSTED_HOOK_ROOT/scripts/distro_codegen.py" --repo-root "$TRUSTED_TARGET_ROOT" + language: system + pass_filenames: false + always_run: true + - id: trusted-provider-codegen + name: Trusted provider codegen + entry: bash + args: + - -c + - | + "$TRUSTED_HOOK_ROOT/scripts/uv-run-with-index.sh" run --group codegen \ + "$TRUSTED_HOOK_ROOT/scripts/provider_codegen.py" --repo-root "$TRUSTED_TARGET_ROOT" + language: system + pass_filenames: false + always_run: true + - id: trusted-openapi-generator + name: Trusted OpenAPI generator + entry: bash + args: + - -c + - | + "$TRUSTED_HOOK_ROOT/scripts/uv-run-with-index.sh" run \ + "$TRUSTED_HOOK_ROOT/docs/openapi_generator/run_openapi_generator.sh" --target-root "$TRUSTED_TARGET_ROOT" + language: system + pass_filenames: false + always_run: true + YAML + + export TRUSTED_HOOK_ROOT="$PWD/.trusted/base" + export TRUSTED_TARGET_ROOT="$PWD" + export PYTHONPATH="$TRUSTED_TARGET_ROOT:$TRUSTED_TARGET_ROOT/src:${PYTHONPATH:-}" + export GITHUB_BASE_REF="$base_ref" + export GITHUB_REF="refs/heads/$head_ref" + + pre-commit run --all-files --config .trusted-codegen.yaml env: GITHUB_TOKEN: '' diff --git a/docs/openapi_generator/run_openapi_generator.sh b/docs/openapi_generator/run_openapi_generator.sh index 6cffd42b0..ba277024e 100755 --- a/docs/openapi_generator/run_openapi_generator.sh +++ b/docs/openapi_generator/run_openapi_generator.sh @@ -5,9 +5,34 @@ # # This source code is licensed under the terms described in the LICENSE file in # the root directory of this source tree. +# +# Pass --target-root to direct generated artifacts into an alternate checkout +# (used by the trusted autofix workflow running from a base-branch worktree). PYTHONPATH=${PYTHONPATH:-} THIS_DIR="$(cd "$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")" && pwd)" +SOURCE_STACK_DIR="$(dirname "$(dirname "$THIS_DIR")")" +TARGET_STACK_DIR="$SOURCE_STACK_DIR" + +while [[ $# -gt 0 ]]; do + case "$1" in + --target-root) + shift + if [[ $# -eq 0 ]]; then + echo "Error: --target-root requires a value" >&2 + exit 1 + fi + TARGET_STACK_DIR="$1" + ;; + *) + echo "Error: unknown argument '$1'" >&2 + exit 1 + ;; + esac + shift +done +TARGET_STATIC_DIR="$TARGET_STACK_DIR/docs/openapi_generator/static" +TARGET_SPEC_PATH="$TARGET_STACK_DIR/client-sdks/stainless/openapi.yml" set -euo pipefail @@ -27,8 +52,10 @@ if [ ${#missing_packages[@]} -ne 0 ]; then exit 1 fi -stack_dir=$(dirname $(dirname $THIS_DIR)) -PYTHONPATH=$PYTHONPATH:$stack_dir \ - python -m docs.openapi_generator.generate $(dirname $THIS_DIR)/static +mkdir -p "$TARGET_STATIC_DIR" +mkdir -p "$(dirname "$TARGET_SPEC_PATH")" -cp $stack_dir/docs/static/stainless-llama-stack-spec.yaml $stack_dir/client-sdks/stainless/openapi.yml +PYTHONPATH="$TARGET_STACK_DIR:$TARGET_STACK_DIR/src:$PYTHONPATH" \ + python "$SOURCE_STACK_DIR/docs/openapi_generator/generate.py" "$TARGET_STATIC_DIR" + +cp "$TARGET_STACK_DIR/docs/static/stainless-llama-stack-spec.yaml" "$TARGET_SPEC_PATH" diff --git a/scripts/distro_codegen.py b/scripts/distro_codegen.py index 4dbdda5c4..edade6902 100755 --- a/scripts/distro_codegen.py +++ b/scripts/distro_codegen.py @@ -4,7 +4,12 @@ # # This source code is licensed under the terms described in the LICENSE file in # the root directory of this source tree. +# +# CI can direct the generated artifacts into an alternate checkout by passing +# --repo-root, allowing the trusted copy of this script to run from a separate +# worktree. +import argparse import concurrent.futures import importlib import subprocess @@ -15,7 +20,28 @@ from pathlib import Path from rich.progress import Progress, SpinnerColumn, TextColumn -REPO_ROOT = Path(__file__).parent.parent +_DEFAULT_REPO_ROOT = Path(__file__).parent.parent +REPO_ROOT = _DEFAULT_REPO_ROOT + + +def set_repo_root(repo_root: Path) -> None: + """Update the global repository root used by helper functions.""" + + global REPO_ROOT + REPO_ROOT = repo_root + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="Generate distribution docs and YAML artifacts." + ) + parser.add_argument( + "--repo-root", + type=Path, + default=_DEFAULT_REPO_ROOT, + help="Repository root where generated artifacts should be written.", + ) + return parser.parse_args() class ChangedPathTracker: @@ -93,6 +119,13 @@ def pre_import_distros(distro_dirs: list[Path]) -> None: def main(): + args = parse_args() + repo_root = args.repo_root + if not repo_root.is_absolute(): + repo_root = (Path.cwd() / repo_root).resolve() + + set_repo_root(repo_root) + distros_dir = REPO_ROOT / "src" / "llama_stack" / "distributions" change_tracker = ChangedPathTracker() diff --git a/scripts/provider_codegen.py b/scripts/provider_codegen.py index d62d626ad..8ee430cd4 100755 --- a/scripts/provider_codegen.py +++ b/scripts/provider_codegen.py @@ -4,7 +4,12 @@ # # This source code is licensed under the terms described in the LICENSE file in # the root directory of this source tree. +# +# CI can direct the generated artifacts into an alternate checkout by passing +# --repo-root, allowing the trusted copy of this script to run from a separate +# worktree. +import argparse import subprocess import sys from pathlib import Path @@ -15,7 +20,28 @@ from rich.progress import Progress, SpinnerColumn, TextColumn from llama_stack.core.distribution import get_provider_registry -REPO_ROOT = Path(__file__).parent.parent +_DEFAULT_REPO_ROOT = Path(__file__).parent.parent +REPO_ROOT = _DEFAULT_REPO_ROOT + + +def set_repo_root(repo_root: Path) -> None: + """Update the global repository root used by helper functions.""" + + global REPO_ROOT + REPO_ROOT = repo_root + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="Regenerate provider documentation from source definitions." + ) + parser.add_argument( + "--repo-root", + type=Path, + default=_DEFAULT_REPO_ROOT, + help="Repository root where generated artifacts should be written.", + ) + return parser.parse_args() def get_api_docstring(api_name: str) -> str | None: @@ -440,6 +466,13 @@ def check_for_changes(change_tracker: ChangedPathTracker) -> bool: def main(): + args = parse_args() + repo_root = args.repo_root + if not repo_root.is_absolute(): + repo_root = (Path.cwd() / repo_root).resolve() + + set_repo_root(repo_root) + change_tracker = ChangedPathTracker() with Progress(