llama-stack-mirror/.github/workflows/conformance.yml

# API Conformance Tests
# This workflow ensures that API changes maintain backward compatibility and don't break existing integrations
# It runs schema validation and OpenAPI diff checks to catch breaking changes early
#
# The workflow handles both monolithic and split API specifications:
# - If split specs exist (stable/experimental/deprecated), they are stitched together for comparison
# - If only monolithic spec exists, it is used directly
# This allows for clean API organization while maintaining robust conformance testing

name: API Conformance Tests

run-name: Run the API Conformance test suite on the changes.

on:
  push:
    branches: [ main ]
  pull_request:
    branches: [ main ]
    types: [opened, synchronize, reopened, edited]
    paths:
      - 'docs/static/llama-stack-spec.yaml'              # Legacy monolithic spec
      - 'docs/static/stable-llama-stack-spec.yaml'       # Stable APIs spec
      - 'docs/static/experimental-llama-stack-spec.yaml' # Experimental APIs spec
      - 'docs/static/deprecated-llama-stack-spec.yaml'   # Deprecated APIs spec
      - 'docs/static/llama-stack-spec.html'              # Legacy HTML spec
      - '.github/workflows/conformance.yml'              # This workflow itself

concurrency:
  group: ${{ github.workflow }}-${{ github.ref == 'refs/heads/main' && github.run_id || github.ref }}
  # Cancel in-progress runs when new commits are pushed to avoid wasting CI resources
  cancel-in-progress: true

jobs:
  # Job to check if API schema changes maintain backward compatibility
  check-schema-compatibility:
    runs-on: ubuntu-latest
    steps:
      - name: Checkout PR Code
        uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
        with:
          fetch-depth: 0

      # Check if we should skip conformance testing due to breaking changes
      - name: Check if conformance test should be skipped
        id: skip-check
        run: |
          PR_TITLE="${{ github.event.pull_request.title }}"

          # Skip if title contains "!:" indicating breaking change (like "feat!:")
          if [[ "$PR_TITLE" == *"!:"* ]]; then
            echo "skip=true" >> $GITHUB_OUTPUT
            exit 0
          fi

          # Get all commits in this PR and check for BREAKING CHANGE footer
          git log --format="%B" ${{ github.event.pull_request.base.sha }}..${{ github.event.pull_request.head.sha }} | \
            grep -q "BREAKING CHANGE:" && echo "skip=true" >> $GITHUB_OUTPUT || echo "skip=false" >> $GITHUB_OUTPUT
        shell: bash
      # Checkout the base branch to compare against (usually main)
      # This allows us to diff the current changes against the previous state
      - name: Checkout Base Branch
        if: steps.skip-check.outputs.skip != 'true'
        uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
        with:
          ref: ${{ github.event.pull_request.base.ref }}
          path: 'base'


      # Cache openai spec
      - name: Cache openai openapi spec
        id: cache-openapi
        uses: actions/cache@0400d5f644dc74513175e3cd8d07132dd4860809
        with:
          path: ~/openai-openapi
          key: openai-openapi.yml

      # Cache oasdiff to avoid checksum failures and speed up builds
      - name: Cache oasdiff
        if: steps.skip-check.outputs.skip != 'true'
        id: cache-oasdiff
        uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830
        with:
          path: ~/oasdiff
          key: oasdiff-${{ runner.os }}

      # Install oasdiff: https://github.com/oasdiff/oasdiff, a tool for detecting breaking changes in OpenAPI specs.
      - name: Install oasdiff
        if: steps.skip-check.outputs.skip != 'true' && steps.cache-oasdiff.outputs.cache-hit != 'true'
        run: |
          curl -fsSL https://raw.githubusercontent.com/oasdiff/oasdiff/main/install.sh | sh
          cp /usr/local/bin/oasdiff ~/oasdiff

      - name: Get openai openapi spec
        if: steps.cache-openapi.outputs.cache-hit != 'true'
        run: |
          mkdir -p ~/openai-openapi
          curl -L https://app.stainless.com/api/spec/documented/openai/openapi.documented.yml -o ~/openai-openapi/openai-openapi.yml

      # Setup cached oasdiff
      - name: Setup cached oasdiff
        if: steps.skip-check.outputs.skip != 'true' && steps.cache-oasdiff.outputs.cache-hit == 'true'
        run: |
          sudo cp ~/oasdiff /usr/local/bin/oasdiff
          sudo chmod +x /usr/local/bin/oasdiff

      # Install yq for YAML processing
      - name: Install yq
        run: |
          sudo wget -qO /usr/local/bin/yq https://github.com/mikefarah/yq/releases/latest/download/yq_linux_amd64
          sudo chmod +x /usr/local/bin/yq

      # Verify API specs exist for conformance testing
      - name: Check API Specs
        if: steps.skip-check.outputs.skip != 'true'
        run: |
          echo "Checking for API specification files..."

          # Check current branch
          if [ -f "docs/static/stable-llama-stack-spec.yaml" ]; then
            echo "✓ Found stable API spec in current branch"
            CURRENT_SPEC="docs/static/stable-llama-stack-spec.yaml"
          elif [ -f "docs/static/llama-stack-spec.yaml" ]; then
            echo "✓ Found monolithic API spec in current branch"
            CURRENT_SPEC="docs/static/llama-stack-spec.yaml"
          else
            echo "❌ No API specs found in current branch"
            exit 1
          fi

          # Check base branch
          if [ -f "base/docs/static/stable-llama-stack-spec.yaml" ]; then
            echo "✓ Found stable API spec in base branch"
            BASE_SPEC="base/docs/static/stable-llama-stack-spec.yaml"
          elif [ -f "base/docs/static/llama-stack-spec.yaml" ]; then
            echo "✓ Found monolithic API spec in base branch"
            BASE_SPEC="base/docs/static/llama-stack-spec.yaml"
          else
            echo "❌ No API specs found in base branch"
            exit 1
          fi

          # Export for next step
          echo "BASE_SPEC=${BASE_SPEC}" >> $GITHUB_ENV
          echo "CURRENT_SPEC=${CURRENT_SPEC}" >> $GITHUB_ENV

          echo "Will compare: ${BASE_SPEC} -> ${CURRENT_SPEC}"

      # Run oasdiff to detect breaking changes in the API specification
      # This step will fail if incompatible changes are detected, preventing breaking changes from being merged
      - name: Run OpenAPI Breaking Change Diff
        if: steps.skip-check.outputs.skip != 'true'
        run: |
          oasdiff breaking --fail-on ERR $BASE_SPEC $CURRENT_SPEC --match-path '^/v1/'

      # Run oasdiff to detect breaking changes in the API specification when compared to the OpenAI openAPI spec
      - name: Run OpenAPI Breaking Change Diff Against OpenAI API
        if: steps.skip-check.outputs.skip != 'true'
        continue-on-error: true
        shell: bash
        run: |
          OPENAI_SPEC=~/openai-openapi/openai-openapi.yml
          LOCAL_SPEC=docs/static/llama-stack-spec.yaml

          # Compare OpenAI spec against Llama Stack spec
          # Strip /v1 prefix from our spec to match OpenAI's paths
          # Filter out api-path-removed errors for OpenAI routes we don't implement
          oasdiff breaking --fail-on ERR \
            "$OPENAI_SPEC" \
            "$LOCAL_SPEC" \
            --strip-prefix-revision "/v1" 2>&1 | \
            grep -v "api-path-removed-without-deprecation" | \
            grep -v "api-removed-without-deprecation" || true

      # Report when test is skipped
      - name: Report skip reason
        if: steps.skip-check.outputs.skip == 'true'
        run: |
          echo "Conformance test skipped due to breaking change indicator"