diff --git a/.github/workflows/README.md b/.github/workflows/README.md index 00a8f54ac..ef6a8bb3c 100644 --- a/.github/workflows/README.md +++ b/.github/workflows/README.md @@ -4,6 +4,7 @@ Llama Stack uses GitHub Actions for Continuous Integration (CI). Below is a tabl | Name | File | Purpose | | ---- | ---- | ------- | +| Backward Compatibility Check | [backward-compat.yml](backward-compat.yml) | Check backward compatibility for run.yaml configs | | Update Changelog | [changelog.yml](changelog.yml) | Creates PR for updating the CHANGELOG.md | | API Conformance Tests | [conformance.yml](conformance.yml) | Run the API Conformance test suite on the changes. | | Installer CI | [install-script-ci.yml](install-script-ci.yml) | Test the installation script | diff --git a/.github/workflows/backward-compat.yml b/.github/workflows/backward-compat.yml new file mode 100644 index 000000000..857120af7 --- /dev/null +++ b/.github/workflows/backward-compat.yml @@ -0,0 +1,384 @@ +name: Backward Compatibility Check + +run-name: Check backward compatibility for run.yaml configs + +on: + pull_request: + branches: [main] + paths: + - 'src/llama_stack/core/datatypes.py' + - 'src/llama_stack/providers/datatypes.py' + - 'src/llama_stack/distributions/**/run.yaml' + - 'tests/backward_compat/**' + - '.github/workflows/backward-compat.yml' + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + check-main-compatibility: + name: Check Compatibility with main + runs-on: ubuntu-latest + + steps: + - name: Checkout PR branch + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + with: + fetch-depth: 0 # Need full history to access main branch + + - name: Set up Python + uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6.0.0 + with: + python-version: '3.12' + + - name: Install uv + uses: astral-sh/setup-uv@6b9c6063abd6010835644d4c2e1bef4cf5cd0fca # v6.0.1 + with: + enable-cache: true + + - name: Install dependencies + run: | + uv sync --group dev + + - name: Extract run.yaml files from main branch + id: extract_configs + run: | + # Get list of run.yaml paths from main + git fetch origin main + CONFIG_PATHS=$(git ls-tree -r --name-only origin/main | grep "src/llama_stack/distributions/.*/run.yaml$" || true) + + if [ -z "$CONFIG_PATHS" ]; then + echo "No run.yaml files found in main branch" + exit 1 + fi + + # Extract all configs to a temp directory + mkdir -p /tmp/main_configs + echo "Extracting configs from main branch:" + + while IFS= read -r config_path; do + if [ -z "$config_path" ]; then + continue + fi + + # Extract filename for storage + filename=$(basename $(dirname "$config_path")) + echo " - $filename (from $config_path)" + + git show origin/main:"$config_path" > "/tmp/main_configs/${filename}.yaml" + done <<< "$CONFIG_PATHS" + + echo "" + echo "Extracted $(ls /tmp/main_configs/*.yaml | wc -l) config files" + + - name: Test all configs from main + id: test_configs + continue-on-error: true + run: | + # Run pytest once with all configs parameterized + if COMPAT_TEST_CONFIGS_DIR=/tmp/main_configs uv run pytest tests/backward_compat/test_run_config.py -v; then + echo "failed=false" >> $GITHUB_OUTPUT + else + echo "failed=true" >> $GITHUB_OUTPUT + exit 1 + fi + + - name: Check for breaking change acknowledgment + id: check_ack + if: steps.test_configs.outputs.failed == 'true' + run: | + echo "Breaking changes detected. Checking for acknowledgment..." + + # Check PR title for '!:' marker (conventional commits) + PR_TITLE="${{ github.event.pull_request.title }}" + if [[ "$PR_TITLE" =~ ^[a-z]+\!: ]]; then + echo "✓ Breaking change acknowledged in PR title" + echo "acknowledged=true" >> $GITHUB_OUTPUT + exit 0 + fi + + # Check commit messages for BREAKING CHANGE: + if git log origin/main..HEAD --format=%B | grep -q "BREAKING CHANGE:"; then + echo "✓ Breaking change acknowledged in commit message" + echo "acknowledged=true" >> $GITHUB_OUTPUT + exit 0 + fi + + echo "✗ Breaking change NOT acknowledged" + echo "acknowledged=false" >> $GITHUB_OUTPUT + env: + GH_TOKEN: ${{ github.token }} + + - name: Evaluate results + if: always() + run: | + FAILED="${{ steps.test_configs.outputs.failed }}" + ACKNOWLEDGED="${{ steps.check_ack.outputs.acknowledged }}" + + if [[ "$FAILED" == "true" ]]; then + if [[ "$ACKNOWLEDGED" == "true" ]]; then + echo "" + echo "⚠️ WARNING: Breaking changes detected but acknowledged" + echo "" + echo "This PR introduces backward-incompatible changes to run.yaml." + echo "The changes have been properly acknowledged." + echo "" + exit 0 # Pass the check + else + echo "" + echo "❌ ERROR: Breaking changes detected without acknowledgment" + echo "" + echo "This PR introduces backward-incompatible changes to run.yaml" + echo "that will break existing user configurations." + echo "" + echo "To acknowledge this breaking change, do ONE of:" + echo " 1. Add '!:' to your PR title (e.g., 'feat!: change xyz')" + echo " 2. Add the 'breaking-change' label to this PR" + echo " 3. Include 'BREAKING CHANGE:' in a commit message" + echo "" + exit 1 # Fail the check + fi + fi + + test-integration-main: + name: Run Integration Tests with main Config + runs-on: ubuntu-latest + + steps: + - name: Checkout PR branch + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + with: + fetch-depth: 0 + + - name: Extract ci-tests run.yaml from main + run: | + git fetch origin main + git show origin/main:src/llama_stack/distributions/ci-tests/run.yaml > /tmp/main-ci-tests-run.yaml + echo "Extracted ci-tests run.yaml from main branch" + + - name: Setup test environment + uses: ./.github/actions/setup-test-environment + with: + python-version: '3.12' + client-version: 'latest' + setup: 'ollama' + suite: 'base' + inference-mode: 'replay' + + - name: Run integration tests with main config + id: test_integration + continue-on-error: true + uses: ./.github/actions/run-and-record-tests + with: + stack-config: /tmp/main-ci-tests-run.yaml + setup: 'ollama' + inference-mode: 'replay' + suite: 'base' + + - name: Check for breaking change acknowledgment + id: check_ack + if: steps.test_integration.outcome == 'failure' + run: | + echo "Integration tests failed. Checking for acknowledgment..." + + # Check PR title for '!:' marker (conventional commits) + PR_TITLE="${{ github.event.pull_request.title }}" + if [[ "$PR_TITLE" =~ ^[a-z]+\!: ]]; then + echo "✓ Breaking change acknowledged in PR title" + echo "acknowledged=true" >> $GITHUB_OUTPUT + exit 0 + fi + + # Check commit messages for BREAKING CHANGE: + if git log origin/main..HEAD --format=%B | grep -q "BREAKING CHANGE:"; then + echo "✓ Breaking change acknowledged in commit message" + echo "acknowledged=true" >> $GITHUB_OUTPUT + exit 0 + fi + + echo "✗ Breaking change NOT acknowledged" + echo "acknowledged=false" >> $GITHUB_OUTPUT + env: + GH_TOKEN: ${{ github.token }} + + - name: Evaluate integration test results + if: always() + run: | + TEST_FAILED="${{ steps.test_integration.outcome == 'failure' }}" + ACKNOWLEDGED="${{ steps.check_ack.outputs.acknowledged }}" + + if [[ "$TEST_FAILED" == "true" ]]; then + if [[ "$ACKNOWLEDGED" == "true" ]]; then + echo "" + echo "⚠️ WARNING: Integration tests failed with main config but acknowledged" + echo "" + exit 0 # Pass the check + else + echo "" + echo "❌ ERROR: Integration tests failed with main config without acknowledgment" + echo "" + echo "To acknowledge this breaking change, do ONE of:" + echo " 1. Add '!:' to your PR title (e.g., 'feat!: change xyz')" + echo " 2. Include 'BREAKING CHANGE:' in a commit message" + echo "" + exit 1 # Fail the check + fi + fi + + test-integration-release: + name: Run Integration Tests with Latest Release (Informational) + runs-on: ubuntu-latest + + steps: + - name: Checkout PR branch + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + with: + fetch-depth: 0 + + - name: Get latest release + id: get_release + run: | + # Get the latest release from GitHub + LATEST_TAG=$(gh release list --limit 1 --json tagName --jq '.[0].tagName' 2>/dev/null || echo "") + + if [ -z "$LATEST_TAG" ]; then + echo "No releases found, skipping release compatibility check" + echo "has_release=false" >> $GITHUB_OUTPUT + exit 0 + fi + + echo "Latest release: $LATEST_TAG" + echo "has_release=true" >> $GITHUB_OUTPUT + echo "tag=$LATEST_TAG" >> $GITHUB_OUTPUT + env: + GH_TOKEN: ${{ github.token }} + + - name: Extract ci-tests run.yaml from release + if: steps.get_release.outputs.has_release == 'true' + id: extract_config + run: | + RELEASE_TAG="${{ steps.get_release.outputs.tag }}" + + if git show "$RELEASE_TAG:src/llama_stack/distributions/ci-tests/run.yaml" > /tmp/release-ci-tests-run.yaml 2>/dev/null; then + echo "Extracted ci-tests run.yaml from release $RELEASE_TAG" + echo "has_config=true" >> $GITHUB_OUTPUT + else + echo "::warning::ci-tests/run.yaml not found in release $RELEASE_TAG" + echo "has_config=false" >> $GITHUB_OUTPUT + fi + + - name: Setup test environment + if: steps.get_release.outputs.has_release == 'true' && steps.extract_config.outputs.has_config == 'true' + uses: ./.github/actions/setup-test-environment + with: + python-version: '3.12' + client-version: 'latest' + setup: 'ollama' + suite: 'base' + inference-mode: 'replay' + + - name: Run integration tests with release config + if: steps.get_release.outputs.has_release == 'true' && steps.extract_config.outputs.has_config == 'true' + uses: ./.github/actions/run-and-record-tests + with: + stack-config: /tmp/release-ci-tests-run.yaml + setup: 'ollama' + inference-mode: 'replay' + suite: 'base' + + - name: Report results + if: always() && steps.get_release.outputs.has_release == 'true' && steps.extract_config.outputs.has_config == 'true' + run: | + RELEASE_TAG="${{ steps.get_release.outputs.tag }}" + echo "::warning::Integration tests against release $RELEASE_TAG completed" + echo "::warning::This is informational only - does not block merge" + + check-schema-release-compatibility: + name: Check Schema Compatibility with Latest Release (Informational) + runs-on: ubuntu-latest + + steps: + - name: Checkout PR branch + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + with: + fetch-depth: 0 + + - name: Set up Python + uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6.0.0 + with: + python-version: '3.12' + + - name: Install uv + uses: astral-sh/setup-uv@6b9c6063abd6010835644d4c2e1bef4cf5cd0fca # v6.0.1 + with: + enable-cache: true + + - name: Install dependencies + run: | + uv sync --group dev + + - name: Get latest release + id: get_release + run: | + # Get the latest release from GitHub + LATEST_TAG=$(gh release list --limit 1 --json tagName --jq '.[0].tagName' 2>/dev/null || echo "") + + if [ -z "$LATEST_TAG" ]; then + echo "No releases found, skipping release compatibility check" + echo "has_release=false" >> $GITHUB_OUTPUT + exit 0 + fi + + echo "Latest release: $LATEST_TAG" + echo "has_release=true" >> $GITHUB_OUTPUT + echo "tag=$LATEST_TAG" >> $GITHUB_OUTPUT + env: + GH_TOKEN: ${{ github.token }} + + - name: Extract configs from release + if: steps.get_release.outputs.has_release == 'true' + id: extract_release_configs + run: | + RELEASE_TAG="${{ steps.get_release.outputs.tag }}" + + # Get run.yaml files from the release + CONFIG_PATHS=$(git ls-tree -r --name-only "$RELEASE_TAG" | grep "src/llama_stack/distributions/.*/run.yaml$" || true) + + if [ -z "$CONFIG_PATHS" ]; then + echo "::warning::No run.yaml files found in release $RELEASE_TAG" + echo "has_configs=false" >> $GITHUB_OUTPUT + exit 0 + fi + + # Extract all configs to a temp directory + mkdir -p /tmp/release_configs + echo "Extracting configs from release $RELEASE_TAG:" + + while IFS= read -r config_path; do + if [ -z "$config_path" ]; then + continue + fi + + filename=$(basename $(dirname "$config_path")) + echo " - $filename (from $config_path)" + + git show "$RELEASE_TAG:$config_path" > "/tmp/release_configs/${filename}.yaml" 2>/dev/null || true + done <<< "$CONFIG_PATHS" + + echo "" + echo "Extracted $(ls /tmp/release_configs/*.yaml 2>/dev/null | wc -l) config files" + echo "has_configs=true" >> $GITHUB_OUTPUT + + - name: Test against release configs + if: steps.get_release.outputs.has_release == 'true' && steps.extract_release_configs.outputs.has_configs == 'true' + run: | + RELEASE_TAG="${{ steps.get_release.outputs.tag }}" + + # Run pytest with all release configs + if COMPAT_TEST_CONFIGS_DIR=/tmp/release_configs uv run pytest tests/backward_compat/test_run_config.py -v --tb=short; then + echo "✓ All configs from release $RELEASE_TAG are compatible" + else + echo "::warning::Some configs from release $RELEASE_TAG are incompatible" + echo "::warning::This is informational only - does not block merge" + fi diff --git a/tests/backward_compat/test_run_config.py b/tests/backward_compat/test_run_config.py new file mode 100644 index 000000000..13aac85e4 --- /dev/null +++ b/tests/backward_compat/test_run_config.py @@ -0,0 +1,52 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the terms described in the LICENSE file in +# the root directory of this source tree. + +""" +Backward compatibility test for run.yaml files. + +This test ensures that changes to StackRunConfig don't break +existing run.yaml files from previous versions. +""" + +import os +from pathlib import Path + +import pytest +import yaml + +from llama_stack.core.datatypes import StackRunConfig + + +def get_test_configs(): + configs_dir = os.environ.get("COMPAT_TEST_CONFIGS_DIR") + if configs_dir: + # CI mode: test configs extracted from main/release + config_dir = Path(configs_dir) + if not config_dir.exists(): + pytest.skip(f"Config directory not found: {configs_dir}") + + config_files = sorted(config_dir.glob("*.yaml")) + if not config_files: + pytest.skip(f"No .yaml files found in {configs_dir}") + + return config_files + else: + # Local mode: test current distribution configs + repo_root = Path(__file__).parent.parent.parent + config_files = sorted((repo_root / "src" / "llama_stack" / "distributions").glob("*/run.yaml")) + + if not config_files: + pytest.skip("No run.yaml files found in distributions/") + + return config_files + + +@pytest.mark.parametrize("config_file", get_test_configs(), ids=lambda p: p.stem) +def test_load_run_config(config_file): + with open(config_file) as f: + config_data = yaml.safe_load(f) + + StackRunConfig.model_validate(config_data)