From 7f1a33f51ca3b779cb38954c536ca23bfc15cecd Mon Sep 17 00:00:00 2001
From: Alexey Rybak <50731695+reluctantfuturist@users.noreply.github.com>
Date: Wed, 1 Oct 2025 10:11:31 -0700
Subject: [PATCH] docs: update API conformance test (#3631)

# What does this PR do?

Given the rapidly changing nature of Llama Stack's APIs and the need to have clean, user-friendly API documentation, we want to split the API reference into 3 main buckets; stable, experimental and deprecated. The most straightforward way to do it is to have several automatically generated doctrees, which introduces some complexity in testing APIs for backwards compatibility.

This PR updates the API conformance test to handle cases where the API schema is split into several files; it does not change the testing criteria.

<!-- Provide a short summary of what this PR does and why. Link to relevant issues if applicable. -->

<!-- If resolving an issue, uncomment and update the line below -->

<!-- Closes #[issue-number] -->

## Test Plan

No developer-facing changes (all existing tests should pass)

<!-- Describe the tests you ran to verify your changes with result summaries. *Provide clear instructions so the plan can be easily re-executed.* -->
---
 .github/workflows/conformance.yml | 57 ++++++++++++++++++++++++++++---
 1 file changed, 53 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/conformance.yml b/.github/workflows/conformance.yml
index dfa6b4167..2dd62a9c4 100644
--- a/.github/workflows/conformance.yml
+++ b/.github/workflows/conformance.yml
@@ -1,6 +1,11 @@
 # API Conformance Tests
 # This workflow ensures that API changes maintain backward compatibility and don't break existing integrations
 # It runs schema validation and OpenAPI diff checks to catch breaking changes early
+#
+# The workflow handles both monolithic and split API specifications:
+# - If split specs exist (stable/experimental/deprecated), they are stitched together for comparison
+# - If only monolithic spec exists, it is used directly
+# This allows for clean API organization while maintaining robust conformance testing
 
 name: API Conformance Tests
 
@@ -13,9 +18,12 @@ on:
     branches: [ main ]
     types: [opened, synchronize, reopened, edited]
     paths:
-      - 'docs/static/llama-stack-spec.yaml'
-      - 'docs/static/llama-stack-spec.html'
-      - '.github/workflows/conformance.yml' # This workflow itself
+      - 'docs/static/llama-stack-spec.yaml'              # Legacy monolithic spec
+      - 'docs/static/stable-llama-stack-spec.yaml'       # Stable APIs spec
+      - 'docs/static/experimental-llama-stack-spec.yaml' # Experimental APIs spec
+      - 'docs/static/deprecated-llama-stack-spec.yaml'   # Deprecated APIs spec
+      - 'docs/static/llama-stack-spec.html'              # Legacy HTML spec
+      - '.github/workflows/conformance.yml'              # This workflow itself
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref == 'refs/heads/main' && github.run_id || github.ref }}
@@ -80,6 +88,47 @@ jobs:
           sudo cp ~/oasdiff /usr/local/bin/oasdiff
           sudo chmod +x /usr/local/bin/oasdiff
 
+      # Install yq for YAML processing
+      - name: Install yq
+        run: |
+          sudo wget -qO /usr/local/bin/yq https://github.com/mikefarah/yq/releases/latest/download/yq_linux_amd64
+          sudo chmod +x /usr/local/bin/yq
+
+      # Verify API specs exist for conformance testing
+      - name: Check API Specs
+        run: |
+          echo "Checking for API specification files..."
+
+          # Check current branch
+          if [ -f "docs/static/stable-llama-stack-spec.yaml" ]; then
+            echo "✓ Found stable API spec in current branch"
+            CURRENT_SPEC="docs/static/stable-llama-stack-spec.yaml"
+          elif [ -f "docs/static/llama-stack-spec.yaml" ]; then
+            echo "✓ Found monolithic API spec in current branch"
+            CURRENT_SPEC="docs/static/llama-stack-spec.yaml"
+          else
+            echo "❌ No API specs found in current branch"
+            exit 1
+          fi
+
+          # Check base branch
+          if [ -f "base/docs/static/stable-llama-stack-spec.yaml" ]; then
+            echo "✓ Found stable API spec in base branch"
+            BASE_SPEC="base/docs/static/stable-llama-stack-spec.yaml"
+          elif [ -f "base/docs/static/llama-stack-spec.yaml" ]; then
+            echo "✓ Found monolithic API spec in base branch"
+            BASE_SPEC="base/docs/static/llama-stack-spec.yaml"
+          else
+            echo "❌ No API specs found in base branch"
+            exit 1
+          fi
+
+          # Export for next step
+          echo "BASE_SPEC=${BASE_SPEC}" >> $GITHUB_ENV
+          echo "CURRENT_SPEC=${CURRENT_SPEC}" >> $GITHUB_ENV
+
+          echo "Will compare: ${BASE_SPEC} -> ${CURRENT_SPEC}"
+
       # Run oasdiff to detect breaking changes in the API specification
       # This step will fail if incompatible changes are detected, preventing breaking changes from being merged
       - name: Run OpenAPI Breaking Change Diff
@@ -91,4 +140,4 @@ jobs:
       - name: Report skip reason
         if: steps.skip-check.outputs.skip == 'true'
         run: |
-          echo "Conformance test skipped due to breaking change indicator"
+          oasdiff breaking --fail-on ERR $BASE_SPEC $CURRENT_SPEC --match-path '^/v1/'