llama-stack-mirror/.github/workflows/conformance.yml
2025-10-01 09:52:37 -07:00

160 lines
7.3 KiB
YAML

# API Conformance Tests
# This workflow ensures that API changes maintain backward compatibility and don't break existing integrations
# It runs schema validation and OpenAPI diff checks to catch breaking changes early
#
# The workflow handles both monolithic and split API specifications:
# - If split specs exist (stable/experimental/deprecated), they are stitched together for comparison
# - If only monolithic spec exists, it is used directly
# This allows for clean API organization while maintaining robust conformance testing
name: API Conformance Tests
run-name: Run the API Conformance test suite on the changes.
on:
push:
branches: [ main ]
pull_request:
branches: [ main ]
types: [opened, synchronize, reopened, edited]
paths:
- 'docs/static/llama-stack-spec.yaml' # Legacy monolithic spec
- 'docs/static/stable-llama-stack-spec.yaml' # Stable APIs spec
- 'docs/static/experimental-llama-stack-spec.yaml' # Experimental APIs spec
- 'docs/static/deprecated-llama-stack-spec.yaml' # Deprecated APIs spec
- 'docs/static/llama-stack-spec.html' # Legacy HTML spec
- '.github/workflows/conformance.yml' # This workflow itself
concurrency:
group: ${{ github.workflow }}-${{ github.ref == 'refs/heads/main' && github.run_id || github.ref }}
# Cancel in-progress runs when new commits are pushed to avoid wasting CI resources
cancel-in-progress: true
jobs:
# Job to check if API schema changes maintain backward compatibility
check-schema-compatibility:
runs-on: ubuntu-latest
steps:
- name: Checkout PR Code
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
with:
fetch-depth: 0
# Check if we should skip conformance testing due to breaking changes
- name: Check if conformance test should be skipped
id: skip-check
run: |
PR_TITLE="${{ github.event.pull_request.title }}"
# Skip if title contains "!:" indicating breaking change (like "feat!:")
if [[ "$PR_TITLE" == *"!:"* ]]; then
echo "skip=true" >> $GITHUB_OUTPUT
exit 0
fi
# Get all commits in this PR and check for BREAKING CHANGE footer
git log --format="%B" ${{ github.event.pull_request.base.sha }}..${{ github.event.pull_request.head.sha }} | \
grep -q "BREAKING CHANGE:" && echo "skip=true" >> $GITHUB_OUTPUT || echo "skip=false" >> $GITHUB_OUTPUT
shell: bash
# Checkout the base branch to compare against (usually main)
# This allows us to diff the current changes against the previous state
- name: Checkout Base Branch
if: steps.skip-check.outputs.skip != 'true'
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
with:
ref: ${{ github.event.pull_request.base.ref }}
path: 'base'
# Cache oasdiff to avoid checksum failures and speed up builds
- name: Cache oasdiff
if: steps.skip-check.outputs.skip != 'true'
id: cache-oasdiff
uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830
with:
path: ~/oasdiff
key: oasdiff-${{ runner.os }}
# Install oasdiff: https://github.com/oasdiff/oasdiff, a tool for detecting breaking changes in OpenAPI specs.
- name: Install oasdiff
if: steps.skip-check.outputs.skip != 'true' && steps.cache-oasdiff.outputs.cache-hit != 'true'
run: |
curl -fsSL https://raw.githubusercontent.com/oasdiff/oasdiff/main/install.sh | sh
cp /usr/local/bin/oasdiff ~/oasdiff
# Setup cached oasdiff
- name: Setup cached oasdiff
if: steps.skip-check.outputs.skip != 'true' && steps.cache-oasdiff.outputs.cache-hit == 'true'
run: |
sudo cp ~/oasdiff /usr/local/bin/oasdiff
sudo chmod +x /usr/local/bin/oasdiff
# Install yq for YAML processing
- name: Install yq
run: |
sudo wget -qO /usr/local/bin/yq https://github.com/mikefarah/yq/releases/latest/download/yq_linux_amd64
sudo chmod +x /usr/local/bin/yq
# Stitch together complete API specs for conformance testing
# This handles cases where APIs have been split across multiple files
- name: Create Complete API Specs for Comparison
run: |
# Function to create complete spec from split files or use existing monolithic spec
create_complete_spec() {
local source_dir="$1"
local output_file="$2"
# Check if split specs exist
if [ -f "${source_dir}/docs/static/stable-llama-stack-spec.yaml" ] &&
[ -f "${source_dir}/docs/static/experimental-llama-stack-spec.yaml" ] &&
[ -f "${source_dir}/docs/static/deprecated-llama-stack-spec.yaml" ]; then
echo "Found split specs in ${source_dir}, stitching together..."
# Start with stable spec as base
cp "${source_dir}/docs/static/stable-llama-stack-spec.yaml" "${output_file}"
# Merge paths from experimental spec
if [ -s "${source_dir}/docs/static/experimental-llama-stack-spec.yaml" ]; then
yq eval-all 'select(fileIndex == 0) * select(fileIndex == 1)' \
"${output_file}" \
"${source_dir}/docs/static/experimental-llama-stack-spec.yaml" > "${output_file}.tmp"
mv "${output_file}.tmp" "${output_file}"
fi
# Merge paths from deprecated spec
if [ -s "${source_dir}/docs/static/deprecated-llama-stack-spec.yaml" ]; then
yq eval-all 'select(fileIndex == 0) * select(fileIndex == 1)' \
"${output_file}" \
"${source_dir}/docs/static/deprecated-llama-stack-spec.yaml" > "${output_file}.tmp"
mv "${output_file}.tmp" "${output_file}"
fi
elif [ -f "${source_dir}/docs/static/llama-stack-spec.yaml" ]; then
echo "Using monolithic spec from ${source_dir}..."
cp "${source_dir}/docs/static/llama-stack-spec.yaml" "${output_file}"
else
echo "ERROR: No API specs found in ${source_dir}"
ls -la "${source_dir}/docs/static/" || true
exit 1
fi
}
# Create complete specs for both base and current
create_complete_spec "base" "base-complete-spec.yaml"
create_complete_spec "." "current-complete-spec.yaml"
echo "Generated complete specs for comparison:"
echo "Base spec size: $(wc -l < base-complete-spec.yaml) lines"
echo "Current spec size: $(wc -l < current-complete-spec.yaml) lines"
# Run oasdiff to detect breaking changes in the API specification
# This step will fail if incompatible changes are detected, preventing breaking changes from being merged
- name: Run OpenAPI Breaking Change Diff
if: steps.skip-check.outputs.skip != 'true'
run: |
oasdiff breaking --fail-on ERR base/docs/static/llama-stack-spec.yaml docs/static/llama-stack-spec.yaml --match-path '^/v1/'
# Report when test is skipped
- name: Report skip reason
if: steps.skip-check.outputs.skip == 'true'
run: |
echo "Conformance test skipped due to breaking change indicator"