From e809d21357f3ff41b66f39a4d231200e5ed25429 Mon Sep 17 00:00:00 2001 From: Ashwin Bharambe Date: Tue, 28 Oct 2025 21:51:56 -0700 Subject: [PATCH] feat: add backward compatibility tests for run.yaml (#3952) This adds automated backward compatibility testing for `run.yaml` files. As we evolve `StackRunConfig`, changes can inadvertently break existing user configurations. This workflow catches those breaks before merge. We test old run.yaml files (from main and the latest release) against the PR's new code. If configs that worked before now fail, the PR is blocked unless explicitly acknowledged as a breaking change. **Two test layers:** - Schema validation: Quick pytest checks that configs parse without errors - Integration tests: Full test suite execution to catch runtime semantic issues (cross-field validations, provider initialization, etc.) **What we test against:** - main branch: Breaking changes here block the PR (this is the gate) - Latest release: Informational only - shows if we've drifted from what users have If tests fail, the PR author must acknowledge the breaking change by adding `!:` to the PR title (e.g., `feat!: change xyz`) or including `BREAKING CHANGE:` in a commit message. Once acknowledged, the check passes with a warning. These jobs are run: 1. `check-main-compatibility` - Schema validation of all distribution run.yaml files from main 2. `test-integration-main` - Full integration test suite using main's ci-tests run.yaml 3. `test-integration-release` - Integration tests with latest release config (informational) 4. `check-schema-release-compatibility` - Schema checks against release (informational) The integration tests catch issues that schema validation alone would miss, like assertion failures in `StackRunConfig.validate_server_stores()` or provider-specific runtime logic. Resolves #3311 Related to #3237 --- .github/workflows/README.md | 1 + .github/workflows/backward-compat.yml | 574 +++++++++++++++++++++++ tests/backward_compat/test_run_config.py | 52 ++ 3 files changed, 627 insertions(+) create mode 100644 .github/workflows/backward-compat.yml create mode 100644 tests/backward_compat/test_run_config.py diff --git a/.github/workflows/README.md b/.github/workflows/README.md index 00a8f54ac..ef6a8bb3c 100644 --- a/.github/workflows/README.md +++ b/.github/workflows/README.md @@ -4,6 +4,7 @@ Llama Stack uses GitHub Actions for Continuous Integration (CI). Below is a tabl | Name | File | Purpose | | ---- | ---- | ------- | +| Backward Compatibility Check | [backward-compat.yml](backward-compat.yml) | Check backward compatibility for run.yaml configs | | Update Changelog | [changelog.yml](changelog.yml) | Creates PR for updating the CHANGELOG.md | | API Conformance Tests | [conformance.yml](conformance.yml) | Run the API Conformance test suite on the changes. | | Installer CI | [install-script-ci.yml](install-script-ci.yml) | Test the installation script | diff --git a/.github/workflows/backward-compat.yml b/.github/workflows/backward-compat.yml new file mode 100644 index 000000000..72d2b0c27 --- /dev/null +++ b/.github/workflows/backward-compat.yml @@ -0,0 +1,574 @@ +name: Backward Compatibility Check + +run-name: Check backward compatibility for run.yaml configs + +on: + pull_request: + branches: [main] + paths: + - 'src/llama_stack/core/datatypes.py' + - 'src/llama_stack/providers/datatypes.py' + - 'src/llama_stack/distributions/**/run.yaml' + - 'tests/backward_compat/**' + - '.github/workflows/backward-compat.yml' + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + check-main-compatibility: + name: Check Compatibility with main + runs-on: ubuntu-latest + + steps: + - name: Checkout PR branch + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + with: + fetch-depth: 0 # Need full history to access main branch + + - name: Set up Python + uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6.0.0 + with: + python-version: '3.12' + + - name: Install uv + uses: astral-sh/setup-uv@6b9c6063abd6010835644d4c2e1bef4cf5cd0fca # v6.0.1 + with: + enable-cache: true + + - name: Install dependencies + run: | + uv sync --group dev + + - name: Extract run.yaml files from main branch + id: extract_configs + run: | + # Get list of run.yaml paths from main + git fetch origin main + CONFIG_PATHS=$(git ls-tree -r --name-only origin/main | grep "src/llama_stack/distributions/.*/run.yaml$" || true) + + if [ -z "$CONFIG_PATHS" ]; then + echo "No run.yaml files found in main branch" + exit 1 + fi + + # Extract all configs to a temp directory + mkdir -p /tmp/main_configs + echo "Extracting configs from main branch:" + + while IFS= read -r config_path; do + if [ -z "$config_path" ]; then + continue + fi + + # Extract filename for storage + filename=$(basename $(dirname "$config_path")) + echo " - $filename (from $config_path)" + + git show origin/main:"$config_path" > "/tmp/main_configs/${filename}.yaml" + done <<< "$CONFIG_PATHS" + + echo "" + echo "Extracted $(ls /tmp/main_configs/*.yaml | wc -l) config files" + + - name: Test all configs from main + id: test_configs + continue-on-error: true + run: | + # Run pytest once with all configs parameterized + if COMPAT_TEST_CONFIGS_DIR=/tmp/main_configs uv run pytest tests/backward_compat/test_run_config.py -v; then + echo "failed=false" >> $GITHUB_OUTPUT + else + echo "failed=true" >> $GITHUB_OUTPUT + exit 1 + fi + + - name: Check for breaking change acknowledgment + id: check_ack + if: steps.test_configs.outputs.failed == 'true' + run: | + echo "Breaking changes detected. Checking for acknowledgment..." + + # Check PR title for '!:' marker (conventional commits) + PR_TITLE="${{ github.event.pull_request.title }}" + if [[ "$PR_TITLE" =~ ^[a-z]+\!: ]]; then + echo "✓ Breaking change acknowledged in PR title" + echo "acknowledged=true" >> $GITHUB_OUTPUT + exit 0 + fi + + # Check commit messages for BREAKING CHANGE: + if git log origin/main..HEAD --format=%B | grep -q "BREAKING CHANGE:"; then + echo "✓ Breaking change acknowledged in commit message" + echo "acknowledged=true" >> $GITHUB_OUTPUT + exit 0 + fi + + echo "✗ Breaking change NOT acknowledged" + echo "acknowledged=false" >> $GITHUB_OUTPUT + env: + GH_TOKEN: ${{ github.token }} + + - name: Evaluate results + if: always() + run: | + FAILED="${{ steps.test_configs.outputs.failed }}" + ACKNOWLEDGED="${{ steps.check_ack.outputs.acknowledged }}" + + if [[ "$FAILED" == "true" ]]; then + if [[ "$ACKNOWLEDGED" == "true" ]]; then + echo "" + echo "⚠️ WARNING: Breaking changes detected but acknowledged" + echo "" + echo "This PR introduces backward-incompatible changes to run.yaml." + echo "The changes have been properly acknowledged." + echo "" + exit 0 # Pass the check + else + echo "" + echo "❌ ERROR: Breaking changes detected without acknowledgment" + echo "" + echo "This PR introduces backward-incompatible changes to run.yaml" + echo "that will break existing user configurations." + echo "" + echo "To acknowledge this breaking change, do ONE of:" + echo " 1. Add '!:' to your PR title (e.g., 'feat!: change xyz')" + echo " 2. Add the 'breaking-change' label to this PR" + echo " 3. Include 'BREAKING CHANGE:' in a commit message" + echo "" + exit 1 # Fail the check + fi + fi + + test-integration-main: + name: Run Integration Tests with main Config + runs-on: ubuntu-latest + + steps: + - name: Checkout PR branch + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + with: + fetch-depth: 0 + + - name: Extract ci-tests run.yaml from main + run: | + git fetch origin main + git show origin/main:src/llama_stack/distributions/ci-tests/run.yaml > /tmp/main-ci-tests-run.yaml + echo "Extracted ci-tests run.yaml from main branch" + + - name: Setup test environment + uses: ./.github/actions/setup-test-environment + with: + python-version: '3.12' + client-version: 'latest' + setup: 'ollama' + suite: 'base' + inference-mode: 'replay' + + - name: Run integration tests with main config + id: test_integration + continue-on-error: true + uses: ./.github/actions/run-and-record-tests + with: + stack-config: /tmp/main-ci-tests-run.yaml + setup: 'ollama' + inference-mode: 'replay' + suite: 'base' + + - name: Check for breaking change acknowledgment + id: check_ack + if: steps.test_integration.outcome == 'failure' + run: | + echo "Integration tests failed. Checking for acknowledgment..." + + # Check PR title for '!:' marker (conventional commits) + PR_TITLE="${{ github.event.pull_request.title }}" + if [[ "$PR_TITLE" =~ ^[a-z]+\!: ]]; then + echo "✓ Breaking change acknowledged in PR title" + echo "acknowledged=true" >> $GITHUB_OUTPUT + exit 0 + fi + + # Check commit messages for BREAKING CHANGE: + if git log origin/main..HEAD --format=%B | grep -q "BREAKING CHANGE:"; then + echo "✓ Breaking change acknowledged in commit message" + echo "acknowledged=true" >> $GITHUB_OUTPUT + exit 0 + fi + + echo "✗ Breaking change NOT acknowledged" + echo "acknowledged=false" >> $GITHUB_OUTPUT + env: + GH_TOKEN: ${{ github.token }} + + - name: Evaluate integration test results + if: always() + run: | + TEST_FAILED="${{ steps.test_integration.outcome == 'failure' }}" + ACKNOWLEDGED="${{ steps.check_ack.outputs.acknowledged }}" + + if [[ "$TEST_FAILED" == "true" ]]; then + if [[ "$ACKNOWLEDGED" == "true" ]]; then + echo "" + echo "⚠️ WARNING: Integration tests failed with main config but acknowledged" + echo "" + exit 0 # Pass the check + else + echo "" + echo "❌ ERROR: Integration tests failed with main config without acknowledgment" + echo "" + echo "To acknowledge this breaking change, do ONE of:" + echo " 1. Add '!:' to your PR title (e.g., 'feat!: change xyz')" + echo " 2. Include 'BREAKING CHANGE:' in a commit message" + echo "" + exit 1 # Fail the check + fi + fi + + test-integration-release: + name: Run Integration Tests with Latest Release (Informational) + runs-on: ubuntu-latest + + steps: + - name: Checkout PR branch + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + with: + fetch-depth: 0 + + - name: Get latest release + id: get_release + run: | + # Get the latest release from GitHub + LATEST_TAG=$(gh release list --limit 1 --json tagName --jq '.[0].tagName' 2>/dev/null || echo "") + + if [ -z "$LATEST_TAG" ]; then + echo "No releases found, skipping release compatibility check" + echo "has_release=false" >> $GITHUB_OUTPUT + exit 0 + fi + + echo "Latest release: $LATEST_TAG" + echo "has_release=true" >> $GITHUB_OUTPUT + echo "tag=$LATEST_TAG" >> $GITHUB_OUTPUT + env: + GH_TOKEN: ${{ github.token }} + + - name: Extract ci-tests run.yaml from release + if: steps.get_release.outputs.has_release == 'true' + id: extract_config + run: | + RELEASE_TAG="${{ steps.get_release.outputs.tag }}" + + # Try with src/ prefix first (newer releases), then without (older releases) + if git show "$RELEASE_TAG:src/llama_stack/distributions/ci-tests/run.yaml" > /tmp/release-ci-tests-run.yaml 2>/dev/null; then + echo "Extracted ci-tests run.yaml from release $RELEASE_TAG (src/ path)" + echo "has_config=true" >> $GITHUB_OUTPUT + elif git show "$RELEASE_TAG:llama_stack/distributions/ci-tests/run.yaml" > /tmp/release-ci-tests-run.yaml 2>/dev/null; then + echo "Extracted ci-tests run.yaml from release $RELEASE_TAG (old path)" + echo "has_config=true" >> $GITHUB_OUTPUT + else + echo "::warning::ci-tests/run.yaml not found in release $RELEASE_TAG" + echo "has_config=false" >> $GITHUB_OUTPUT + fi + + - name: Setup test environment + if: steps.get_release.outputs.has_release == 'true' && steps.extract_config.outputs.has_config == 'true' + uses: ./.github/actions/setup-test-environment + with: + python-version: '3.12' + client-version: 'latest' + setup: 'ollama' + suite: 'base' + inference-mode: 'replay' + + - name: Run integration tests with release config (PR branch) + id: test_release_pr + if: steps.get_release.outputs.has_release == 'true' && steps.extract_config.outputs.has_config == 'true' + continue-on-error: true + uses: ./.github/actions/run-and-record-tests + with: + stack-config: /tmp/release-ci-tests-run.yaml + setup: 'ollama' + inference-mode: 'replay' + suite: 'base' + + - name: Checkout main branch to test baseline + if: steps.get_release.outputs.has_release == 'true' && steps.extract_config.outputs.has_config == 'true' + run: | + git checkout origin/main + + - name: Setup test environment for main + if: steps.get_release.outputs.has_release == 'true' && steps.extract_config.outputs.has_config == 'true' + uses: ./.github/actions/setup-test-environment + with: + python-version: '3.12' + client-version: 'latest' + setup: 'ollama' + suite: 'base' + inference-mode: 'replay' + + - name: Run integration tests with release config (main branch) + id: test_release_main + if: steps.get_release.outputs.has_release == 'true' && steps.extract_config.outputs.has_config == 'true' + continue-on-error: true + uses: ./.github/actions/run-and-record-tests + with: + stack-config: /tmp/release-ci-tests-run.yaml + setup: 'ollama' + inference-mode: 'replay' + suite: 'base' + + - name: Report results and post PR comment + if: always() && steps.get_release.outputs.has_release == 'true' && steps.extract_config.outputs.has_config == 'true' + run: | + RELEASE_TAG="${{ steps.get_release.outputs.tag }}" + PR_OUTCOME="${{ steps.test_release_pr.outcome }}" + MAIN_OUTCOME="${{ steps.test_release_main.outcome }}" + + if [[ "$PR_OUTCOME" == "failure" && "$MAIN_OUTCOME" == "success" ]]; then + # NEW breaking change - PR fails but main passes + echo "::error::🚨 This PR introduces a NEW breaking change!" + + # Check if we already posted a comment (to avoid spam on every push) + EXISTING_COMMENT=$(gh pr view ${{ github.event.pull_request.number }} --json comments --jq '.comments[] | select(.body | contains("🚨 New Breaking Change Detected") and contains("Integration tests")) | .id' | head -1) + + if [[ -z "$EXISTING_COMMENT" ]]; then + gh pr comment ${{ github.event.pull_request.number }} --body "## 🚨 New Breaking Change Detected + + **Integration tests against release \`$RELEASE_TAG\` are now failing** + + ⚠️ This PR introduces a breaking change that affects compatibility with the latest release. + + - Users on release \`$RELEASE_TAG\` may not be able to upgrade + - Existing configurations may break + + The tests pass on \`main\` but fail with this PR's changes. + + > **Note:** This is informational only and does not block merge. + > Consider whether this breaking change is acceptable for users." + else + echo "Comment already exists, skipping to avoid spam" + fi + + cat >> $GITHUB_STEP_SUMMARY < **Note:** This is informational only and does not block merge. + > Consider whether this breaking change is acceptable for users. + EOF + + elif [[ "$PR_OUTCOME" == "failure" ]]; then + # Existing breaking change - both PR and main fail + echo "::warning::Breaking change already exists in main branch" + + cat >> $GITHUB_STEP_SUMMARY < **Note:** This is informational only. + EOF + + else + # Success - tests pass + cat >> $GITHUB_STEP_SUMMARY </dev/null || echo "") + + if [ -z "$LATEST_TAG" ]; then + echo "No releases found, skipping release compatibility check" + echo "has_release=false" >> $GITHUB_OUTPUT + exit 0 + fi + + echo "Latest release: $LATEST_TAG" + echo "has_release=true" >> $GITHUB_OUTPUT + echo "tag=$LATEST_TAG" >> $GITHUB_OUTPUT + env: + GH_TOKEN: ${{ github.token }} + + - name: Extract configs from release + if: steps.get_release.outputs.has_release == 'true' + id: extract_release_configs + run: | + RELEASE_TAG="${{ steps.get_release.outputs.tag }}" + + # Get run.yaml files from the release (try both src/ and old path) + CONFIG_PATHS=$(git ls-tree -r --name-only "$RELEASE_TAG" | grep "llama_stack/distributions/.*/run.yaml$" || true) + + if [ -z "$CONFIG_PATHS" ]; then + echo "::warning::No run.yaml files found in release $RELEASE_TAG" + echo "has_configs=false" >> $GITHUB_OUTPUT + exit 0 + fi + + # Extract all configs to a temp directory + mkdir -p /tmp/release_configs + echo "Extracting configs from release $RELEASE_TAG:" + + while IFS= read -r config_path; do + if [ -z "$config_path" ]; then + continue + fi + + filename=$(basename $(dirname "$config_path")) + echo " - $filename (from $config_path)" + + git show "$RELEASE_TAG:$config_path" > "/tmp/release_configs/${filename}.yaml" 2>/dev/null || true + done <<< "$CONFIG_PATHS" + + echo "" + echo "Extracted $(ls /tmp/release_configs/*.yaml 2>/dev/null | wc -l) config files" + echo "has_configs=true" >> $GITHUB_OUTPUT + + - name: Test against release configs (PR branch) + id: test_schema_pr + if: steps.get_release.outputs.has_release == 'true' && steps.extract_release_configs.outputs.has_configs == 'true' + continue-on-error: true + run: | + RELEASE_TAG="${{ steps.get_release.outputs.tag }}" + COMPAT_TEST_CONFIGS_DIR=/tmp/release_configs uv run pytest tests/backward_compat/test_run_config.py -v --tb=short + + - name: Checkout main branch to test baseline + if: steps.get_release.outputs.has_release == 'true' && steps.extract_release_configs.outputs.has_configs == 'true' + run: | + git checkout origin/main + + - name: Install dependencies for main + if: steps.get_release.outputs.has_release == 'true' && steps.extract_release_configs.outputs.has_configs == 'true' + run: | + uv sync --group dev + + - name: Test against release configs (main branch) + id: test_schema_main + if: steps.get_release.outputs.has_release == 'true' && steps.extract_release_configs.outputs.has_configs == 'true' + continue-on-error: true + run: | + RELEASE_TAG="${{ steps.get_release.outputs.tag }}" + COMPAT_TEST_CONFIGS_DIR=/tmp/release_configs uv run pytest tests/backward_compat/test_run_config.py -v --tb=short + + - name: Report results and post PR comment + if: always() && steps.get_release.outputs.has_release == 'true' && steps.extract_release_configs.outputs.has_configs == 'true' + run: | + RELEASE_TAG="${{ steps.get_release.outputs.tag }}" + PR_OUTCOME="${{ steps.test_schema_pr.outcome }}" + MAIN_OUTCOME="${{ steps.test_schema_main.outcome }}" + + if [[ "$PR_OUTCOME" == "failure" && "$MAIN_OUTCOME" == "success" ]]; then + # NEW breaking change - PR fails but main passes + echo "::error::🚨 This PR introduces a NEW schema breaking change!" + + # Check if we already posted a comment (to avoid spam on every push) + EXISTING_COMMENT=$(gh pr view ${{ github.event.pull_request.number }} --json comments --jq '.comments[] | select(.body | contains("🚨 New Schema Breaking Change Detected")) | .id' | head -1) + + if [[ -z "$EXISTING_COMMENT" ]]; then + gh pr comment ${{ github.event.pull_request.number }} --body "## 🚨 New Schema Breaking Change Detected + + **Schema validation against release \`$RELEASE_TAG\` is now failing** + + ⚠️ This PR introduces a schema breaking change that affects compatibility with the latest release. + + - Users on release \`$RELEASE_TAG\` will not be able to upgrade + - Existing run.yaml configurations will fail validation + + The tests pass on \`main\` but fail with this PR's changes. + + > **Note:** This is informational only and does not block merge. + > Consider whether this breaking change is acceptable for users." + else + echo "Comment already exists, skipping to avoid spam" + fi + + cat >> $GITHUB_STEP_SUMMARY < **Note:** This is informational only and does not block merge. + > Consider whether this breaking change is acceptable for users. + EOF + + elif [[ "$PR_OUTCOME" == "failure" ]]; then + # Existing breaking change - both PR and main fail + echo "::warning::Schema breaking change already exists in main branch" + + cat >> $GITHUB_STEP_SUMMARY < **Note:** This is informational only. + EOF + + else + # Success - tests pass + cat >> $GITHUB_STEP_SUMMARY <