From d9abb2e958e7c3340fea0036f21a9237116fcd5b Mon Sep 17 00:00:00 2001
From: Charlie Doern <cdoern@redhat.com>
Date: Tue, 23 Sep 2025 14:50:08 -0400
Subject: [PATCH] feat(api): oasdiff OpenAI openAPI spec against ours

diff the `/v1/` routes that are OpenAI compatible against the OpenAI openAPI spec. This will of course only trigger on PRs where the spec is changed.

This will catch errors with new handwritten additions to our openAI compat routes.

Signed-off-by: Charlie Doern <cdoern@redhat.com>
---
 .github/workflows/conformance.yml | 34 +++++++++++++++++++++++++++++++
 1 file changed, 34 insertions(+)

diff --git a/.github/workflows/conformance.yml b/.github/workflows/conformance.yml
index 5bbd53e5f..242508710 100644
--- a/.github/workflows/conformance.yml
+++ b/.github/workflows/conformance.yml
@@ -65,6 +65,15 @@ jobs:
           ref: ${{ github.event.pull_request.base.ref }}
           path: 'base'
 
+
+      # Cache openai spec
+      - name: Cache openai openapi spec
+        id: cache-openapi
+        uses: actions/cache@0400d5f644dc74513175e3cd8d07132dd4860809
+        with:
+          path: ~/openai-openapi
+          key: openai-openapi.yml
+
       # Cache oasdiff to avoid checksum failures and speed up builds
       - name: Cache oasdiff
         if: steps.skip-check.outputs.skip != 'true'
@@ -81,6 +90,12 @@ jobs:
           curl -fsSL https://raw.githubusercontent.com/oasdiff/oasdiff/main/install.sh | sh
           cp /usr/local/bin/oasdiff ~/oasdiff
 
+      - name: Get openai openapi spec
+        if: steps.cache-openapi.outputs.cache-hit != 'true'
+        run: |
+          mkdir -p ~/openai-openapi
+          curl -L https://app.stainless.com/api/spec/documented/openai/openapi.documented.yml -o ~/openai-openapi/openai-openapi.yml
+
       # Setup cached oasdiff
       - name: Setup cached oasdiff
         if: steps.skip-check.outputs.skip != 'true' && steps.cache-oasdiff.outputs.cache-hit == 'true'
@@ -137,6 +152,25 @@ jobs:
         run: |
           oasdiff breaking --fail-on ERR $BASE_SPEC $CURRENT_SPEC --match-path '^/v1/'
 
+      # Run oasdiff to detect breaking changes in the API specification when compared to the OpenAI openAPI spec
+      - name: Run OpenAPI Breaking Change Diff Against OpenAI API
+        if: steps.skip-check.outputs.skip != 'true'
+        continue-on-error: true
+        shell: bash
+        run: |
+          OPENAI_SPEC=~/openai-openapi/openai-openapi.yml
+          LOCAL_SPEC=docs/static/llama-stack-spec.yaml
+
+          # Compare OpenAI spec against Llama Stack spec
+          # Strip /v1 prefix from our spec to match OpenAI's paths
+          # Filter out api-path-removed errors for OpenAI routes we don't implement
+          oasdiff breaking --fail-on ERR \
+            "$OPENAI_SPEC" \
+            "$LOCAL_SPEC" \
+            --strip-prefix-revision "/v1" 2>&1 | \
+            grep -v "api-path-removed-without-deprecation" | \
+            grep -v "api-removed-without-deprecation" || true
+
       # Report when test is skipped
       - name: Report skip reason
         if: steps.skip-check.outputs.skip == 'true'