mirror of
https://github.com/meta-llama/llama-stack.git
synced 2025-10-04 04:04:14 +00:00
# What does this PR do? Given the rapidly changing nature of Llama Stack's APIs and the need to have clean, user-friendly API documentation, we want to split the API reference into 3 main buckets; stable, experimental and deprecated. The most straightforward way to do it is to have several automatically generated doctrees, which introduces some complexity in testing APIs for backwards compatibility. This PR updates the API conformance test to handle cases where the API schema is split into several files; it does not change the testing criteria. <!-- Provide a short summary of what this PR does and why. Link to relevant issues if applicable. --> <!-- If resolving an issue, uncomment and update the line below --> <!-- Closes #[issue-number] --> ## Test Plan No developer-facing changes (all existing tests should pass) <!-- Describe the tests you ran to verify your changes with result summaries. *Provide clear instructions so the plan can be easily re-executed.* -->
143 lines
6.1 KiB
YAML
143 lines
6.1 KiB
YAML
# API Conformance Tests
|
|
# This workflow ensures that API changes maintain backward compatibility and don't break existing integrations
|
|
# It runs schema validation and OpenAPI diff checks to catch breaking changes early
|
|
#
|
|
# The workflow handles both monolithic and split API specifications:
|
|
# - If split specs exist (stable/experimental/deprecated), they are stitched together for comparison
|
|
# - If only monolithic spec exists, it is used directly
|
|
# This allows for clean API organization while maintaining robust conformance testing
|
|
|
|
name: API Conformance Tests
|
|
|
|
run-name: Run the API Conformance test suite on the changes.
|
|
|
|
on:
|
|
push:
|
|
branches: [ main ]
|
|
pull_request:
|
|
branches: [ main ]
|
|
types: [opened, synchronize, reopened, edited]
|
|
paths:
|
|
- 'docs/static/llama-stack-spec.yaml' # Legacy monolithic spec
|
|
- 'docs/static/stable-llama-stack-spec.yaml' # Stable APIs spec
|
|
- 'docs/static/experimental-llama-stack-spec.yaml' # Experimental APIs spec
|
|
- 'docs/static/deprecated-llama-stack-spec.yaml' # Deprecated APIs spec
|
|
- 'docs/static/llama-stack-spec.html' # Legacy HTML spec
|
|
- '.github/workflows/conformance.yml' # This workflow itself
|
|
|
|
concurrency:
|
|
group: ${{ github.workflow }}-${{ github.ref == 'refs/heads/main' && github.run_id || github.ref }}
|
|
# Cancel in-progress runs when new commits are pushed to avoid wasting CI resources
|
|
cancel-in-progress: true
|
|
|
|
jobs:
|
|
# Job to check if API schema changes maintain backward compatibility
|
|
check-schema-compatibility:
|
|
runs-on: ubuntu-latest
|
|
steps:
|
|
- name: Checkout PR Code
|
|
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
|
|
with:
|
|
fetch-depth: 0
|
|
|
|
# Check if we should skip conformance testing due to breaking changes
|
|
- name: Check if conformance test should be skipped
|
|
id: skip-check
|
|
run: |
|
|
PR_TITLE="${{ github.event.pull_request.title }}"
|
|
|
|
# Skip if title contains "!:" indicating breaking change (like "feat!:")
|
|
if [[ "$PR_TITLE" == *"!:"* ]]; then
|
|
echo "skip=true" >> $GITHUB_OUTPUT
|
|
exit 0
|
|
fi
|
|
|
|
# Get all commits in this PR and check for BREAKING CHANGE footer
|
|
git log --format="%B" ${{ github.event.pull_request.base.sha }}..${{ github.event.pull_request.head.sha }} | \
|
|
grep -q "BREAKING CHANGE:" && echo "skip=true" >> $GITHUB_OUTPUT || echo "skip=false" >> $GITHUB_OUTPUT
|
|
shell: bash
|
|
# Checkout the base branch to compare against (usually main)
|
|
# This allows us to diff the current changes against the previous state
|
|
- name: Checkout Base Branch
|
|
if: steps.skip-check.outputs.skip != 'true'
|
|
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
|
|
with:
|
|
ref: ${{ github.event.pull_request.base.ref }}
|
|
path: 'base'
|
|
|
|
# Cache oasdiff to avoid checksum failures and speed up builds
|
|
- name: Cache oasdiff
|
|
if: steps.skip-check.outputs.skip != 'true'
|
|
id: cache-oasdiff
|
|
uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830
|
|
with:
|
|
path: ~/oasdiff
|
|
key: oasdiff-${{ runner.os }}
|
|
|
|
# Install oasdiff: https://github.com/oasdiff/oasdiff, a tool for detecting breaking changes in OpenAPI specs.
|
|
- name: Install oasdiff
|
|
if: steps.skip-check.outputs.skip != 'true' && steps.cache-oasdiff.outputs.cache-hit != 'true'
|
|
run: |
|
|
curl -fsSL https://raw.githubusercontent.com/oasdiff/oasdiff/main/install.sh | sh
|
|
cp /usr/local/bin/oasdiff ~/oasdiff
|
|
|
|
# Setup cached oasdiff
|
|
- name: Setup cached oasdiff
|
|
if: steps.skip-check.outputs.skip != 'true' && steps.cache-oasdiff.outputs.cache-hit == 'true'
|
|
run: |
|
|
sudo cp ~/oasdiff /usr/local/bin/oasdiff
|
|
sudo chmod +x /usr/local/bin/oasdiff
|
|
|
|
# Install yq for YAML processing
|
|
- name: Install yq
|
|
run: |
|
|
sudo wget -qO /usr/local/bin/yq https://github.com/mikefarah/yq/releases/latest/download/yq_linux_amd64
|
|
sudo chmod +x /usr/local/bin/yq
|
|
|
|
# Verify API specs exist for conformance testing
|
|
- name: Check API Specs
|
|
run: |
|
|
echo "Checking for API specification files..."
|
|
|
|
# Check current branch
|
|
if [ -f "docs/static/stable-llama-stack-spec.yaml" ]; then
|
|
echo "✓ Found stable API spec in current branch"
|
|
CURRENT_SPEC="docs/static/stable-llama-stack-spec.yaml"
|
|
elif [ -f "docs/static/llama-stack-spec.yaml" ]; then
|
|
echo "✓ Found monolithic API spec in current branch"
|
|
CURRENT_SPEC="docs/static/llama-stack-spec.yaml"
|
|
else
|
|
echo "❌ No API specs found in current branch"
|
|
exit 1
|
|
fi
|
|
|
|
# Check base branch
|
|
if [ -f "base/docs/static/stable-llama-stack-spec.yaml" ]; then
|
|
echo "✓ Found stable API spec in base branch"
|
|
BASE_SPEC="base/docs/static/stable-llama-stack-spec.yaml"
|
|
elif [ -f "base/docs/static/llama-stack-spec.yaml" ]; then
|
|
echo "✓ Found monolithic API spec in base branch"
|
|
BASE_SPEC="base/docs/static/llama-stack-spec.yaml"
|
|
else
|
|
echo "❌ No API specs found in base branch"
|
|
exit 1
|
|
fi
|
|
|
|
# Export for next step
|
|
echo "BASE_SPEC=${BASE_SPEC}" >> $GITHUB_ENV
|
|
echo "CURRENT_SPEC=${CURRENT_SPEC}" >> $GITHUB_ENV
|
|
|
|
echo "Will compare: ${BASE_SPEC} -> ${CURRENT_SPEC}"
|
|
|
|
# Run oasdiff to detect breaking changes in the API specification
|
|
# This step will fail if incompatible changes are detected, preventing breaking changes from being merged
|
|
- name: Run OpenAPI Breaking Change Diff
|
|
if: steps.skip-check.outputs.skip != 'true'
|
|
run: |
|
|
oasdiff breaking --fail-on ERR base/docs/static/llama-stack-spec.yaml docs/static/llama-stack-spec.yaml --match-path '^/v1/'
|
|
|
|
# Report when test is skipped
|
|
- name: Report skip reason
|
|
if: steps.skip-check.outputs.skip == 'true'
|
|
run: |
|
|
oasdiff breaking --fail-on ERR $BASE_SPEC $CURRENT_SPEC --match-path '^/v1/'
|