feat(api): ensure StackRunConfig

StackRunConfig is part of our public API, ensure stability of this datatype using a pytest snapshot test.

If the pydantic model changes, it will fail.

Signed-off-by: Charlie Doern <cdoern@redhat.com>

.github/workflows/conformance.yml

@@ -40,6 +40,11 @@ jobs:
         with:
           fetch-depth: 0
 
+      - name: Install dependencies
+        uses: ./.github/actions/setup-runner
+        with:
+          python-version: "3.12"
+
       # Check if we should skip conformance testing due to breaking changes
       - name: Check if conformance test should be skipped
         id: skip-check
@@ -137,6 +142,11 @@ jobs:
         run: |
           oasdiff breaking --fail-on ERR $BASE_SPEC $CURRENT_SPEC --match-path '^/v1/'
 
+      - name: Run Pydantic Model Test
+        if: steps.skip-check.outputs.skip != 'true'
+        run: |
+          uv run --no-sync ./scripts/api-conformance.sh tests/api/test_models.py
+
       # Report when test is skipped
       - name: Report skip reason
         if: steps.skip-check.outputs.skip == 'true'

.pre-commit-config.yaml

@@ -1,4 +1,4 @@
-exclude: 'build/'
+exclude: 'build/|tests/api/snapshots/'
 
 default_language_version:
     python: python3.12

llama_stack/providers/utils/kvstore/config.py

@@ -54,6 +54,7 @@ class SqliteKVStoreConfig(CommonConfig):
     db_path: str = Field(
         default=(RUNTIME_BASE_DIR / "kvstore.db").as_posix(),
         description="File path for the sqlite database",
+        json_schema_extra={"default": "~/.llama/runtime/kvstore.db"},
     )
 
     @classmethod

llama_stack/providers/utils/sqlstore/sqlstore.py

@@ -39,6 +39,7 @@ class SqliteSqlStoreConfig(SqlAlchemySqlStoreConfig):
     db_path: str = Field(
         default=(RUNTIME_BASE_DIR / "sqlstore.db").as_posix(),
         description="Database path, e.g. ~/.llama/distributions/ollama/sqlstore.db",
+        json_schema_extra={"default": "~/.llama/runtime/sqlstore.db"},
     )
 
     @property

pyproject.toml

@@ -62,6 +62,7 @@ ui = [
 [dependency-groups]
 dev = [
     "pytest>=8.4",
+    "pytest-snapshot>=0.9.0",
     "pytest-timeout",
     "pytest-asyncio>=1.0",
     "pytest-cov",

scripts/api-conformance.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the terms described in the LICENSE file in
+# the root directory of this source tree.
+
+set -euo pipefail
+
+# Simple test runner for API conformance tests
+# Runs pytest with snapshot testing for Pydantic models
+
+# Get the script directory
+THIS_DIR=$(dirname "$0")
+ROOT_DIR="$THIS_DIR/.."
+
+cd "$ROOT_DIR"
+
+# Run pytest with snapshot testing
+echo "=== Running API Conformance Tests ==="
+pytest -s -v tests/ \
+    --snapshot-update \
+    "$@"
+
+echo "✅ API Conformance Tests Complete"

scripts/pydantic-diff.py

@@ -0,0 +1,14 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the terms described in the LICENSE file in
+# the root directory of this source tree.
+
+from llama_stack.core.datatypes import StackRunConfig
+
+
+def test_build_config_v1_schema_is_unchanged(snapshot):
+    """
+    Ensures the V1 schema never changes.
+    """
+    snapshot.assert_match(StackRunConfig.model_json_schema(), "stored_build_config_v1_schema.json")

tests/api/__init__.py

@@ -0,0 +1,5 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the terms described in the LICENSE file in
+# the root directory of this source tree.

tests/api/test_models.py

@@ -0,0 +1,17 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the terms described in the LICENSE file in
+# the root directory of this source tree.
+
+import json
+
+from llama_stack.core.datatypes import StackRunConfig
+
+
+def test_run_config_v1_schema_is_unchanged(snapshot):
+    """
+    Ensures the V1 schema never changes.
+    """
+    schema = StackRunConfig.model_json_schema()
+    snapshot.assert_match(json.dumps(schema, indent=2), "stored_run_config_v1_schema.json")

uv.lock

@@ -1805,6 +1805,7 @@ dev = [
     { name = "pytest-cov" },
     { name = "pytest-html" },
     { name = "pytest-json-report" },
+    { name = "pytest-snapshot" },
     { name = "pytest-socket" },
     { name = "pytest-timeout" },
     { name = "ruamel-yaml" },
@@ -1923,6 +1924,7 @@ dev = [
     { name = "pytest-cov" },
     { name = "pytest-html" },
     { name = "pytest-json-report" },
+    { name = "pytest-snapshot", specifier = ">=0.9.0" },
     { name = "pytest-socket" },
     { name = "pytest-timeout" },
     { name = "ruamel-yaml" },
@@ -3616,6 +3618,18 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/3e/43/7e7b2ec865caa92f67b8f0e9231a798d102724ca4c0e1f414316be1c1ef2/pytest_metadata-3.1.1-py3-none-any.whl", hash = "sha256:c8e0844db684ee1c798cfa38908d20d67d0463ecb6137c72e91f418558dd5f4b", size = 11428, upload-time = "2024-02-12T19:38:42.531Z" },
 ]
 
+[[package]]
+name = "pytest-snapshot"
+version = "0.9.0"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "pytest" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/9b/7b/ab8f1fc1e687218aa66acec1c3674d9c443f6a2dc8cb6a50f464548ffa34/pytest-snapshot-0.9.0.tar.gz", hash = "sha256:c7013c3abc3e860f9feff899f8b4debe3708650d8d8242a61bf2625ff64db7f3", size = 19877, upload-time = "2022-04-23T17:35:31.751Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/cb/29/518f32faf6edad9f56d6e0107217f7de6b79f297a47170414a2bd4be7f01/pytest_snapshot-0.9.0-py3-none-any.whl", hash = "sha256:4b9fe1c21c868fe53a545e4e3184d36bc1c88946e3f5c1d9dd676962a9b3d4ab", size = 10715, upload-time = "2022-04-23T17:35:30.288Z" },
+]
+
 [[package]]
 name = "pytest-socket"
 version = "0.7.0"