# API Conformance Tests # This workflow ensures that API changes maintain backward compatibility and don't break existing integrations # It runs schema validation and OpenAPI diff checks to catch breaking changes early # # The workflow handles both monolithic and split API specifications: # - If split specs exist (stable/experimental/deprecated), they are stitched together for comparison # - If only monolithic spec exists, it is used directly # This allows for clean API organization while maintaining robust conformance testing name: API Conformance Tests run-name: Run the API Conformance test suite on the changes. on: push: branches: [ main ] pull_request: branches: [ main ] types: [opened, synchronize, reopened, edited] paths: - 'docs/static/llama-stack-spec.yaml' # Legacy monolithic spec - 'docs/static/stable-llama-stack-spec.yaml' # Stable APIs spec - 'docs/static/experimental-llama-stack-spec.yaml' # Experimental APIs spec - 'docs/static/deprecated-llama-stack-spec.yaml' # Deprecated APIs spec - 'docs/static/llama-stack-spec.html' # Legacy HTML spec - '.github/workflows/conformance.yml' # This workflow itself concurrency: group: ${{ github.workflow }}-${{ github.ref == 'refs/heads/main' && github.run_id || github.ref }} # Cancel in-progress runs when new commits are pushed to avoid wasting CI resources cancel-in-progress: true jobs: # Job to check if API schema changes maintain backward compatibility check-schema-compatibility: runs-on: ubuntu-latest steps: - name: Checkout PR Code uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 with: fetch-depth: 0 # Check if we should skip conformance testing due to breaking changes - name: Check if conformance test should be skipped id: skip-check run: | PR_TITLE="${{ github.event.pull_request.title }}" # Skip if title contains "!:" indicating breaking change (like "feat!:") if [[ "$PR_TITLE" == *"!:"* ]]; then echo "skip=true" >> $GITHUB_OUTPUT exit 0 fi # Get all commits in this PR and check for BREAKING CHANGE footer git log --format="%B" ${{ github.event.pull_request.base.sha }}..${{ github.event.pull_request.head.sha }} | \ grep -q "BREAKING CHANGE:" && echo "skip=true" >> $GITHUB_OUTPUT || echo "skip=false" >> $GITHUB_OUTPUT shell: bash # Checkout the base branch to compare against (usually main) # This allows us to diff the current changes against the previous state - name: Checkout Base Branch if: steps.skip-check.outputs.skip != 'true' uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 with: ref: ${{ github.event.pull_request.base.ref }} path: 'base' # Cache oasdiff to avoid checksum failures and speed up builds - name: Cache oasdiff if: steps.skip-check.outputs.skip != 'true' id: cache-oasdiff uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 with: path: ~/oasdiff key: oasdiff-${{ runner.os }} # Install oasdiff: https://github.com/oasdiff/oasdiff, a tool for detecting breaking changes in OpenAPI specs. - name: Install oasdiff if: steps.skip-check.outputs.skip != 'true' && steps.cache-oasdiff.outputs.cache-hit != 'true' run: | curl -fsSL https://raw.githubusercontent.com/oasdiff/oasdiff/main/install.sh | sh cp /usr/local/bin/oasdiff ~/oasdiff # Setup cached oasdiff - name: Setup cached oasdiff if: steps.skip-check.outputs.skip != 'true' && steps.cache-oasdiff.outputs.cache-hit == 'true' run: | sudo cp ~/oasdiff /usr/local/bin/oasdiff sudo chmod +x /usr/local/bin/oasdiff # Install yq for YAML processing - name: Install yq run: | sudo wget -qO /usr/local/bin/yq https://github.com/mikefarah/yq/releases/latest/download/yq_linux_amd64 sudo chmod +x /usr/local/bin/yq # Verify API specs exist for conformance testing - name: Check API Specs if: steps.skip-check.outputs.skip != 'true' run: | echo "Checking for API specification files..." # Check current branch if [ -f "docs/static/stable-llama-stack-spec.yaml" ]; then echo "✓ Found stable API spec in current branch" CURRENT_SPEC="docs/static/stable-llama-stack-spec.yaml" elif [ -f "docs/static/llama-stack-spec.yaml" ]; then echo "✓ Found monolithic API spec in current branch" CURRENT_SPEC="docs/static/llama-stack-spec.yaml" else echo "❌ No API specs found in current branch" exit 1 fi # Check base branch if [ -f "base/docs/static/stable-llama-stack-spec.yaml" ]; then echo "✓ Found stable API spec in base branch" BASE_SPEC="base/docs/static/stable-llama-stack-spec.yaml" elif [ -f "base/docs/static/llama-stack-spec.yaml" ]; then echo "✓ Found monolithic API spec in base branch" BASE_SPEC="base/docs/static/llama-stack-spec.yaml" else echo "❌ No API specs found in base branch" exit 1 fi # Export for next step echo "BASE_SPEC=${BASE_SPEC}" >> $GITHUB_ENV echo "CURRENT_SPEC=${CURRENT_SPEC}" >> $GITHUB_ENV echo "Will compare: ${BASE_SPEC} -> ${CURRENT_SPEC}" # Run oasdiff to detect breaking changes in the API specification # This step will fail if incompatible changes are detected, preventing breaking changes from being merged - name: Run OpenAPI Breaking Change Diff if: steps.skip-check.outputs.skip != 'true' run: | oasdiff breaking --fail-on ERR $BASE_SPEC $CURRENT_SPEC --match-path '^/v1/' # Report when test is skipped - name: Report skip reason if: steps.skip-check.outputs.skip == 'true' run: | echo "Conformance test skipped due to breaking change indicator"