diff --git a/.github/workflows/README.md b/.github/workflows/README.md index 2e0df58b8..059bb873f 100644 --- a/.github/workflows/README.md +++ b/.github/workflows/README.md @@ -5,6 +5,7 @@ Llama Stack uses GitHub Actions for Continuous Integration (CI). Below is a tabl | Name | File | Purpose | | ---- | ---- | ------- | | Update Changelog | [changelog.yml](changelog.yml) | Creates PR for updating the CHANGELOG.md | +| API Conformance Tests | [conformance.yml](conformance.yml) | Run the API Conformance test suite on the changes. | | Installer CI | [install-script-ci.yml](install-script-ci.yml) | Test the installation script | | Integration Auth Tests | [integration-auth-tests.yml](integration-auth-tests.yml) | Run the integration test suite with Kubernetes authentication | | SqlStore Integration Tests | [integration-sql-store-tests.yml](integration-sql-store-tests.yml) | Run the integration test suite with SqlStore | diff --git a/.github/workflows/conformance.yml b/.github/workflows/conformance.yml new file mode 100644 index 000000000..2433b0203 --- /dev/null +++ b/.github/workflows/conformance.yml @@ -0,0 +1,57 @@ +# API Conformance Tests +# This workflow ensures that API changes maintain backward compatibility and don't break existing integrations +# It runs schema validation and OpenAPI diff checks to catch breaking changes early + +name: API Conformance Tests + +run-name: Run the API Conformance test suite on the changes. + +on: + push: + branches: [ main ] + pull_request: + branches: [ main ] + types: [opened, synchronize, reopened] + paths: + - 'llama_stack/**' + - '!llama_stack/ui/**' + - 'tests/**' + - 'uv.lock' + - 'pyproject.toml' + - '.github/workflows/conformance.yml' # This workflow itself + +concurrency: + group: ${{ github.workflow }}-${{ github.ref == 'refs/heads/main' && github.run_id || github.ref }} + # Cancel in-progress runs when new commits are pushed to avoid wasting CI resources + cancel-in-progress: true + +jobs: + # Job to check if API schema changes maintain backward compatibility + check-schema-compatibility: + runs-on: ubuntu-latest + steps: + # Using specific version 4.1.7 because 5.0.0 fails when trying to run this locally using `act` + # This ensures consistent behavior between local testing and CI + - name: Checkout PR Code + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + + # Checkout the base branch to compare against (usually main) + # This allows us to diff the current changes against the previous state + - name: Checkout Base Branch + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + with: + ref: ${{ github.event.pull_request.base.ref }} + path: 'base' + + # Install oasdiff: https://github.com/oasdiff/oasdiff, a tool for detecting breaking changes in OpenAPI specs. + - name: Install oasdiff + run: | + curl -fsSL https://raw.githubusercontent.com/oasdiff/oasdiff/main/install.sh | sh + + # Run oasdiff to detect breaking changes in the API specification + # This step will fail if incompatible changes are detected, preventing breaking changes from being merged + - name: Run OpenAPI Breaking Change Diff + run: | + oasdiff breaking --fail-on ERR base/docs/_static/llama-stack-spec.yaml docs/_static/llama-stack-spec.yaml --match-path '^/v1/openai/v1' \ + --match-path '^/v1/vector-io' \ + --match-path '^/v1/vector-dbs'