feat(ci): add a ci-tests distro (#2826)

.github/workflows/integration-tests.yml

@@ -71,7 +71,7 @@ jobs:
 
       - name: Build Llama Stack
         run: |
-          uv run llama stack build --template starter --image-type venv
+          uv run llama stack build --template ci-tests --image-type venv
 
       - name: Check Storage and Memory Available Before Tests
         if: ${{ always() }}
@@ -92,9 +92,9 @@ jobs:
         shell: bash
         run: |
           if [ "${{ matrix.client-type }}" == "library" ]; then
-            stack_config="starter"
+            stack_config="ci-tests"
           else
-            stack_config="server:starter"
+            stack_config="server:ci-tests"
           fi
           uv run pytest -s -v tests/integration/${{ matrix.test-type }} --stack-config=${stack_config} \
             -k "not(builtin_tool or safety_with_image or code_interpreter or test_rag)" \

.github/workflows/integration-vector-io-tests.yml

@@ -93,7 +93,7 @@ jobs:
 
       - name: Build Llama Stack
         run: |
-          uv run llama stack build --template starter --image-type venv
+          uv run llama stack build --template ci-tests --image-type venv
 
       - name: Check Storage and Memory Available Before Tests
         if: ${{ always() }}

.github/workflows/providers-build.yml

@@ -97,9 +97,9 @@ jobs:
 
       - name: Build a single provider
         run: |
-          yq -i '.image_type = "container"' llama_stack/templates/starter/build.yaml
-          yq -i '.image_name = "test"' llama_stack/templates/starter/build.yaml
-          USE_COPY_NOT_MOUNT=true LLAMA_STACK_DIR=. uv run llama stack build --config llama_stack/templates/starter/build.yaml
+          yq -i '.image_type = "container"' llama_stack/templates/ci-tests/build.yaml
+          yq -i '.image_name = "test"' llama_stack/templates/ci-tests/build.yaml
+          USE_COPY_NOT_MOUNT=true LLAMA_STACK_DIR=. uv run llama stack build --config llama_stack/templates/ci-tests/build.yaml
 
       - name: Inspect the container image entrypoint
         run: |
@@ -126,14 +126,14 @@ jobs:
             .image_type    = "container" |
             .image_name    = "ubi9-test" |
             .distribution_spec.container_image = "registry.access.redhat.com/ubi9:latest"
-          ' llama_stack/templates/starter/build.yaml
+          ' llama_stack/templates/ci-tests/build.yaml
 
       - name: Build dev container (UBI9)
         env:
           USE_COPY_NOT_MOUNT: "true"
           LLAMA_STACK_DIR: "."
         run: |
-          uv run llama stack build --config llama_stack/templates/starter/build.yaml
+          uv run llama stack build --config llama_stack/templates/ci-tests/build.yaml
 
       - name: Inspect UBI9 image
         run: |

llama_stack/templates/ci-tests/__init__.py

@@ -0,0 +1,7 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the terms described in the LICENSE file in
+# the root directory of this source tree.
+
+from .ci_tests import get_distribution_template  # noqa: F401

llama_stack/templates/ci-tests/build.yaml

@@ -0,0 +1,65 @@
+version: 2
+distribution_spec:
+  description: CI tests for Llama Stack
+  providers:
+    inference:
+    - remote::cerebras
+    - remote::ollama
+    - remote::vllm
+    - remote::tgi
+    - remote::hf::serverless
+    - remote::hf::endpoint
+    - remote::fireworks
+    - remote::together
+    - remote::bedrock
+    - remote::databricks
+    - remote::nvidia
+    - remote::runpod
+    - remote::openai
+    - remote::anthropic
+    - remote::gemini
+    - remote::groq
+    - remote::fireworks-openai-compat
+    - remote::llama-openai-compat
+    - remote::together-openai-compat
+    - remote::groq-openai-compat
+    - remote::sambanova-openai-compat
+    - remote::cerebras-openai-compat
+    - remote::sambanova
+    - remote::passthrough
+    - inline::sentence-transformers
+    vector_io:
+    - inline::faiss
+    - inline::sqlite-vec
+    - inline::milvus
+    - remote::chromadb
+    - remote::pgvector
+    files:
+    - inline::localfs
+    safety:
+    - inline::llama-guard
+    agents:
+    - inline::meta-reference
+    telemetry:
+    - inline::meta-reference
+    post_training:
+    - inline::huggingface
+    eval:
+    - inline::meta-reference
+    datasetio:
+    - remote::huggingface
+    - inline::localfs
+    scoring:
+    - inline::basic
+    - inline::llm-as-judge
+    - inline::braintrust
+    tool_runtime:
+    - remote::brave-search
+    - remote::tavily-search
+    - inline::rag-runtime
+    - remote::model-context-protocol
+image_type: conda
+additional_pip_packages:
+- aiosqlite
+- asyncpg
+- sqlalchemy[asyncio]

llama_stack/templates/ci-tests/ci_tests.py

@@ -0,0 +1,19 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the terms described in the LICENSE file in
+# the root directory of this source tree.
+
+
+from llama_stack.templates.template import DistributionTemplate
+
+from ..starter.starter import get_distribution_template as get_starter_distribution_template
+
+
+def get_distribution_template() -> DistributionTemplate:
+    template = get_starter_distribution_template()
+    name = "ci-tests"
+    template.name = name
+    template.description = "CI tests for Llama Stack"
+
+    return template