autogen build/run

2025-08-02 08:44:44 +00:00 · 2024-12-03 17:04:35 -08:00 · 2024-12-03 17:04:35 -08:00 · 3fc6b10d22
commit 3fc6b10d22
parent 95187891ca
33 changed files with 369 additions and 6 deletions
--- a/distributions/dependencies.json
+++ b/distributions/dependencies.json
@ -2,9 +2,11 @@
  "hf-serverless": [
    "aiohttp",
    "aiosqlite",
+    "autoevals",
    "blobfile",
    "chardet",
    "chromadb-client",
+    "datasets",
    "faiss-cpu",
    "fastapi",
    "fire",
@ -13,6 +15,7 @@
    "matplotlib",
    "nltk",
    "numpy",
+    "openai",
    "pandas",
    "pillow",
    "psycopg2-binary",
@ -29,9 +32,11 @@
  ],
  "together": [
    "aiosqlite",
+    "autoevals",
    "blobfile",
    "chardet",
    "chromadb-client",
+    "datasets",
    "faiss-cpu",
    "fastapi",
    "fire",
@ -39,6 +44,7 @@
    "matplotlib",
    "nltk",
    "numpy",
+    "openai",
    "pandas",
    "pillow",
    "psycopg2-binary",
@ -56,9 +62,11 @@
  ],
  "vllm-gpu": [
    "aiosqlite",
+    "autoevals",
    "blobfile",
    "chardet",
    "chromadb-client",
+    "datasets",
    "faiss-cpu",
    "fastapi",
    "fire",
@ -66,6 +74,7 @@
    "matplotlib",
    "nltk",
    "numpy",
+    "openai",
    "pandas",
    "pillow",
    "psycopg2-binary",
@ -110,9 +119,11 @@
  ],
  "fireworks": [
    "aiosqlite",
+    "autoevals",
    "blobfile",
    "chardet",
    "chromadb-client",
+    "datasets",
    "faiss-cpu",
    "fastapi",
    "fire",
@ -121,6 +132,7 @@
    "matplotlib",
    "nltk",
    "numpy",
+    "openai",
    "pandas",
    "pillow",
    "psycopg2-binary",
@ -138,9 +150,11 @@
  "tgi": [
    "aiohttp",
    "aiosqlite",
+    "autoevals",
    "blobfile",
    "chardet",
    "chromadb-client",
+    "datasets",
    "faiss-cpu",
    "fastapi",
    "fire",
@ -149,6 +163,7 @@
    "matplotlib",
    "nltk",
    "numpy",
+    "openai",
    "pandas",
    "pillow",
    "psycopg2-binary",
@ -165,10 +180,12 @@
  ],
  "bedrock": [
    "aiosqlite",
+    "autoevals",
    "blobfile",
    "boto3",
    "chardet",
    "chromadb-client",
+    "datasets",
    "faiss-cpu",
    "fastapi",
    "fire",
@ -176,6 +193,7 @@
    "matplotlib",
    "nltk",
    "numpy",
+    "openai",
    "pandas",
    "pillow",
    "psycopg2-binary",
@ -193,9 +211,11 @@
  "meta-reference-gpu": [
    "accelerate",
    "aiosqlite",
+    "autoevals",
    "blobfile",
    "chardet",
    "chromadb-client",
+    "datasets",
    "fairscale",
    "faiss-cpu",
    "fastapi",
@ -205,6 +225,7 @@
    "matplotlib",
    "nltk",
    "numpy",
+    "openai",
    "pandas",
    "pillow",
    "psycopg2-binary",
@ -225,9 +246,11 @@
  "meta-reference-quantized-gpu": [
    "accelerate",
    "aiosqlite",
+    "autoevals",
    "blobfile",
    "chardet",
    "chromadb-client",
+    "datasets",
    "fairscale",
    "faiss-cpu",
    "fastapi",
@ -238,6 +261,7 @@
    "matplotlib",
    "nltk",
    "numpy",
+    "openai",
    "pandas",
    "pillow",
    "psycopg2-binary",
@ -259,9 +283,11 @@
  "ollama": [
    "aiohttp",
    "aiosqlite",
+    "autoevals",
    "blobfile",
    "chardet",
    "chromadb-client",
+    "datasets",
    "faiss-cpu",
    "fastapi",
    "fire",
@ -270,6 +296,7 @@
    "nltk",
    "numpy",
    "ollama",
+    "openai",
    "pandas",
    "pillow",
    "psycopg2-binary",
@ -287,9 +314,11 @@
  "hf-endpoint": [
    "aiohttp",
    "aiosqlite",
+    "autoevals",
    "blobfile",
    "chardet",
    "chromadb-client",
+    "datasets",
    "faiss-cpu",
    "fastapi",
    "fire",
@ -298,6 +327,7 @@
    "matplotlib",
    "nltk",
    "numpy",
+    "openai",
    "pandas",
    "pillow",
    "psycopg2-binary",
--- a/docs/source/distributions/self_hosted_distro/bedrock.md
+++ b/docs/source/distributions/self_hosted_distro/bedrock.md
@ -1,6 +1,3 @@
---
-orphan: true
---
 # Bedrock Distribution

 ```{toctree}
@ -15,9 +12,12 @@ The `llamastack/distribution-bedrock` distribution consists of the following pro
 | API | Provider(s) |
 |-----|-------------|
 | agents | `inline::meta-reference` |
+| datasetio | `remote::huggingface`, `inline::localfs` |
+| eval | `inline::meta-reference` |
 | inference | `remote::bedrock` |
 | memory | `inline::faiss`, `remote::chromadb`, `remote::pgvector` |
 | safety | `remote::bedrock` |
+| scoring | `inline::basic`, `inline::llm-as-judge`, `inline::braintrust` |
 | telemetry | `inline::meta-reference` |


--- a/docs/source/distributions/self_hosted_distro/fireworks.md
+++ b/docs/source/distributions/self_hosted_distro/fireworks.md
@ -15,9 +15,12 @@ The `llamastack/distribution-fireworks` distribution consists of the following p
 | API | Provider(s) |
 |-----|-------------|
 | agents | `inline::meta-reference` |
+| datasetio | `remote::huggingface`, `inline::localfs` |
+| eval | `inline::meta-reference` |
 | inference | `remote::fireworks` |
 | memory | `inline::faiss`, `remote::chromadb`, `remote::pgvector` |
 | safety | `inline::llama-guard` |
+| scoring | `inline::basic`, `inline::llm-as-judge`, `inline::braintrust` |
 | telemetry | `inline::meta-reference` |


--- a/docs/source/distributions/self_hosted_distro/meta-reference-gpu.md
+++ b/docs/source/distributions/self_hosted_distro/meta-reference-gpu.md
@ -15,9 +15,12 @@ The `llamastack/distribution-meta-reference-gpu` distribution consists of the fo
 | API | Provider(s) |
 |-----|-------------|
 | agents | `inline::meta-reference` |
+| datasetio | `remote::huggingface`, `inline::localfs` |
+| eval | `inline::meta-reference` |
 | inference | `inline::meta-reference` |
 | memory | `inline::faiss`, `remote::chromadb`, `remote::pgvector` |
 | safety | `inline::llama-guard` |
+| scoring | `inline::basic`, `inline::llm-as-judge`, `inline::braintrust` |
 | telemetry | `inline::meta-reference` |


--- a/docs/source/distributions/self_hosted_distro/meta-reference-quantized-gpu.md
+++ b/docs/source/distributions/self_hosted_distro/meta-reference-quantized-gpu.md
@ -15,9 +15,12 @@ The `llamastack/distribution-meta-reference-quantized-gpu` distribution consists
 | API | Provider(s) |
 |-----|-------------|
 | agents | `inline::meta-reference` |
+| datasetio | `remote::huggingface`, `inline::localfs` |
+| eval | `inline::meta-reference` |
 | inference | `inline::meta-reference-quantized` |
 | memory | `inline::faiss`, `remote::chromadb`, `remote::pgvector` |
 | safety | `inline::llama-guard` |
+| scoring | `inline::basic`, `inline::llm-as-judge`, `inline::braintrust` |
 | telemetry | `inline::meta-reference` |


--- a/docs/source/distributions/self_hosted_distro/ollama.md
+++ b/docs/source/distributions/self_hosted_distro/ollama.md
@ -15,9 +15,12 @@ The `llamastack/distribution-ollama` distribution consists of the following prov
 | API | Provider(s) |
 |-----|-------------|
 | agents | `inline::meta-reference` |
+| datasetio | `remote::huggingface`, `inline::localfs` |
+| eval | `inline::meta-reference` |
 | inference | `remote::ollama` |
 | memory | `inline::faiss`, `remote::chromadb`, `remote::pgvector` |
 | safety | `inline::llama-guard` |
+| scoring | `inline::basic`, `inline::llm-as-judge`, `inline::braintrust` |
 | telemetry | `inline::meta-reference` |


@ -118,9 +121,9 @@ llama stack run ./run-with-safety.yaml \

 ### (Optional) Update Model Serving Configuration

-```{note}
-Please check the [model_aliases](https://github.com/meta-llama/llama-stack/blob/main/llama_stack/providers/remote/inference/ollama/ollama.py#L45) variable for supported Ollama models.
-```
+> [!NOTE]
+> Please check the [OLLAMA_SUPPORTED_MODELS](https://github.com/meta-llama/llama-stack/blob/main/llama_stack/providers.remote/inference/ollama/ollama.py) for the supported Ollama models.
+

 To serve a new model with `ollama`
 ```bash
--- a/docs/source/distributions/self_hosted_distro/tgi.md
+++ b/docs/source/distributions/self_hosted_distro/tgi.md
@ -16,9 +16,12 @@ The `llamastack/distribution-tgi` distribution consists of the following provide
 | API | Provider(s) |
 |-----|-------------|
 | agents | `inline::meta-reference` |
+| datasetio | `remote::huggingface`, `inline::localfs` |
+| eval | `inline::meta-reference` |
 | inference | `remote::tgi` |
 | memory | `inline::faiss`, `remote::chromadb`, `remote::pgvector` |
 | safety | `inline::llama-guard` |
+| scoring | `inline::basic`, `inline::llm-as-judge`, `inline::braintrust` |
 | telemetry | `inline::meta-reference` |


--- a/docs/source/distributions/self_hosted_distro/together.md
+++ b/docs/source/distributions/self_hosted_distro/together.md
@ -15,9 +15,12 @@ The `llamastack/distribution-together` distribution consists of the following pr
 | API | Provider(s) |
 |-----|-------------|
 | agents | `inline::meta-reference` |
+| datasetio | `remote::huggingface`, `inline::localfs` |
+| eval | `inline::meta-reference` |
 | inference | `remote::together` |
 | memory | `inline::faiss`, `remote::chromadb`, `remote::pgvector` |
 | safety | `inline::llama-guard` |
+| scoring | `inline::basic`, `inline::llm-as-judge`, `inline::braintrust` |
 | telemetry | `inline::meta-reference` |


--- a/llama_stack/templates/bedrock/build.yaml
+++ b/llama_stack/templates/bedrock/build.yaml
@ -16,4 +16,13 @@ distribution_spec:
    - inline::meta-reference
    telemetry:
    - inline::meta-reference
+    eval:
+    - inline::meta-reference
+    datasetio:
+    - remote::huggingface
+    - inline::localfs
+    scoring:
+    - inline::basic
+    - inline::llm-as-judge
+    - inline::braintrust
 image_type: conda
--- a/llama_stack/templates/bedrock/run.yaml
+++ b/llama_stack/templates/bedrock/run.yaml
@ -4,9 +4,12 @@ docker_image: null
 conda_env: bedrock
 apis:
 - agents
+- datasetio
+- eval
 - inference
 - memory
 - safety
+- scoring
 - telemetry
 providers:
  inference:
@ -37,6 +40,18 @@ providers:
  - provider_id: meta-reference
    provider_type: inline::meta-reference
    config: {}
+  eval:
+  - provider_id: meta-reference
+    provider_type: inline::meta-reference
+    config: {}
+  datasetio:
+  - provider_id: huggingface
+    provider_type: remote::huggingface
+    config: {}
+  scoring:
+  - provider_id: basic
+    provider_type: inline::basic
+    config: {}
 metadata_store:
  namespace: null
  type: sqlite
--- a/llama_stack/templates/fireworks/build.yaml
+++ b/llama_stack/templates/fireworks/build.yaml
@ -16,4 +16,13 @@ distribution_spec:
    - inline::meta-reference
    telemetry:
    - inline::meta-reference
+    eval:
+    - inline::meta-reference
+    datasetio:
+    - remote::huggingface
+    - inline::localfs
+    scoring:
+    - inline::basic
+    - inline::llm-as-judge
+    - inline::braintrust
 image_type: conda
--- a/llama_stack/templates/fireworks/run.yaml
+++ b/llama_stack/templates/fireworks/run.yaml
@ -4,9 +4,12 @@ docker_image: null
 conda_env: fireworks
 apis:
 - agents
+- datasetio
+- eval
 - inference
 - memory
 - safety
+- scoring
 - telemetry
 providers:
  inference:
@ -39,6 +42,18 @@ providers:
  - provider_id: meta-reference
    provider_type: inline::meta-reference
    config: {}
+  eval:
+  - provider_id: meta-reference
+    provider_type: inline::meta-reference
+    config: {}
+  datasetio:
+  - provider_id: huggingface
+    provider_type: remote::huggingface
+    config: {}
+  scoring:
+  - provider_id: basic
+    provider_type: inline::basic
+    config: {}
 metadata_store:
  namespace: null
  type: sqlite
--- a/llama_stack/templates/hf-endpoint/build.yaml
+++ b/llama_stack/templates/hf-endpoint/build.yaml
@ -16,4 +16,13 @@ distribution_spec:
    - inline::meta-reference
    telemetry:
    - inline::meta-reference
+    eval:
+    - inline::meta-reference
+    datasetio:
+    - remote::huggingface
+    - inline::localfs
+    scoring:
+    - inline::basic
+    - inline::llm-as-judge
+    - inline::braintrust
 image_type: conda
--- a/llama_stack/templates/hf-endpoint/run-with-safety.yaml
+++ b/llama_stack/templates/hf-endpoint/run-with-safety.yaml
@ -4,9 +4,12 @@ docker_image: null
 conda_env: hf-endpoint
 apis:
 - agents
+- datasetio
+- eval
 - inference
 - memory
 - safety
+- scoring
 - telemetry
 providers:
  inference:
@ -44,6 +47,18 @@ providers:
  - provider_id: meta-reference
    provider_type: inline::meta-reference
    config: {}
+  eval:
+  - provider_id: meta-reference
+    provider_type: inline::meta-reference
+    config: {}
+  datasetio:
+  - provider_id: huggingface
+    provider_type: remote::huggingface
+    config: {}
+  scoring:
+  - provider_id: basic
+    provider_type: inline::basic
+    config: {}
 metadata_store:
  namespace: null
  type: sqlite
--- a/llama_stack/templates/hf-endpoint/run.yaml
+++ b/llama_stack/templates/hf-endpoint/run.yaml
@ -4,9 +4,12 @@ docker_image: null
 conda_env: hf-endpoint
 apis:
 - agents
+- datasetio
+- eval
 - inference
 - memory
 - safety
+- scoring
 - telemetry
 providers:
  inference:
@ -39,6 +42,18 @@ providers:
  - provider_id: meta-reference
    provider_type: inline::meta-reference
    config: {}
+  eval:
+  - provider_id: meta-reference
+    provider_type: inline::meta-reference
+    config: {}
+  datasetio:
+  - provider_id: huggingface
+    provider_type: remote::huggingface
+    config: {}
+  scoring:
+  - provider_id: basic
+    provider_type: inline::basic
+    config: {}
 metadata_store:
  namespace: null
  type: sqlite
--- a/llama_stack/templates/hf-serverless/build.yaml
+++ b/llama_stack/templates/hf-serverless/build.yaml
@ -16,4 +16,13 @@ distribution_spec:
    - inline::meta-reference
    telemetry:
    - inline::meta-reference
+    eval:
+    - inline::meta-reference
+    datasetio:
+    - remote::huggingface
+    - inline::localfs
+    scoring:
+    - inline::basic
+    - inline::llm-as-judge
+    - inline::braintrust
 image_type: conda
--- a/llama_stack/templates/hf-serverless/run-with-safety.yaml
+++ b/llama_stack/templates/hf-serverless/run-with-safety.yaml
@ -4,9 +4,12 @@ docker_image: null
 conda_env: hf-serverless
 apis:
 - agents
+- datasetio
+- eval
 - inference
 - memory
 - safety
+- scoring
 - telemetry
 providers:
  inference:
@ -44,6 +47,18 @@ providers:
  - provider_id: meta-reference
    provider_type: inline::meta-reference
    config: {}
+  eval:
+  - provider_id: meta-reference
+    provider_type: inline::meta-reference
+    config: {}
+  datasetio:
+  - provider_id: huggingface
+    provider_type: remote::huggingface
+    config: {}
+  scoring:
+  - provider_id: basic
+    provider_type: inline::basic
+    config: {}
 metadata_store:
  namespace: null
  type: sqlite
--- a/llama_stack/templates/hf-serverless/run.yaml
+++ b/llama_stack/templates/hf-serverless/run.yaml
@ -4,9 +4,12 @@ docker_image: null
 conda_env: hf-serverless
 apis:
 - agents
+- datasetio
+- eval
 - inference
 - memory
 - safety
+- scoring
 - telemetry
 providers:
  inference:
@ -39,6 +42,18 @@ providers:
  - provider_id: meta-reference
    provider_type: inline::meta-reference
    config: {}
+  eval:
+  - provider_id: meta-reference
+    provider_type: inline::meta-reference
+    config: {}
+  datasetio:
+  - provider_id: huggingface
+    provider_type: remote::huggingface
+    config: {}
+  scoring:
+  - provider_id: basic
+    provider_type: inline::basic
+    config: {}
 metadata_store:
  namespace: null
  type: sqlite
--- a/llama_stack/templates/meta-reference-gpu/build.yaml
+++ b/llama_stack/templates/meta-reference-gpu/build.yaml
@ -16,4 +16,13 @@ distribution_spec:
    - inline::meta-reference
    telemetry:
    - inline::meta-reference
+    eval:
+    - inline::meta-reference
+    datasetio:
+    - remote::huggingface
+    - inline::localfs
+    scoring:
+    - inline::basic
+    - inline::llm-as-judge
+    - inline::braintrust
 image_type: conda
--- a/llama_stack/templates/meta-reference-gpu/run-with-safety.yaml
+++ b/llama_stack/templates/meta-reference-gpu/run-with-safety.yaml
@ -4,9 +4,12 @@ docker_image: null
 conda_env: meta-reference-gpu
 apis:
 - agents
+- datasetio
+- eval
 - inference
 - memory
 - safety
+- scoring
 - telemetry
 providers:
  inference:
@ -46,6 +49,18 @@ providers:
  - provider_id: meta-reference
    provider_type: inline::meta-reference
    config: {}
+  eval:
+  - provider_id: meta-reference
+    provider_type: inline::meta-reference
+    config: {}
+  datasetio:
+  - provider_id: huggingface
+    provider_type: remote::huggingface
+    config: {}
+  scoring:
+  - provider_id: basic
+    provider_type: inline::basic
+    config: {}
 metadata_store:
  namespace: null
  type: sqlite
--- a/llama_stack/templates/meta-reference-gpu/run.yaml
+++ b/llama_stack/templates/meta-reference-gpu/run.yaml
@ -4,9 +4,12 @@ docker_image: null
 conda_env: meta-reference-gpu
 apis:
 - agents
+- datasetio
+- eval
 - inference
 - memory
 - safety
+- scoring
 - telemetry
 providers:
  inference:
@ -40,6 +43,18 @@ providers:
  - provider_id: meta-reference
    provider_type: inline::meta-reference
    config: {}
+  eval:
+  - provider_id: meta-reference
+    provider_type: inline::meta-reference
+    config: {}
+  datasetio:
+  - provider_id: huggingface
+    provider_type: remote::huggingface
+    config: {}
+  scoring:
+  - provider_id: basic
+    provider_type: inline::basic
+    config: {}
 metadata_store:
  namespace: null
  type: sqlite
--- a/llama_stack/templates/meta-reference-quantized-gpu/build.yaml
+++ b/llama_stack/templates/meta-reference-quantized-gpu/build.yaml
@ -16,4 +16,13 @@ distribution_spec:
    - inline::meta-reference
    telemetry:
    - inline::meta-reference
+    eval:
+    - inline::meta-reference
+    datasetio:
+    - remote::huggingface
+    - inline::localfs
+    scoring:
+    - inline::basic
+    - inline::llm-as-judge
+    - inline::braintrust
 image_type: conda
--- a/llama_stack/templates/meta-reference-quantized-gpu/run.yaml
+++ b/llama_stack/templates/meta-reference-quantized-gpu/run.yaml
@ -4,9 +4,12 @@ docker_image: null
 conda_env: meta-reference-quantized-gpu
 apis:
 - agents
+- datasetio
+- eval
 - inference
 - memory
 - safety
+- scoring
 - telemetry
 providers:
  inference:
@ -42,6 +45,18 @@ providers:
  - provider_id: meta-reference
    provider_type: inline::meta-reference
    config: {}
+  eval:
+  - provider_id: meta-reference
+    provider_type: inline::meta-reference
+    config: {}
+  datasetio:
+  - provider_id: huggingface
+    provider_type: remote::huggingface
+    config: {}
+  scoring:
+  - provider_id: basic
+    provider_type: inline::basic
+    config: {}
 metadata_store:
  namespace: null
  type: sqlite
--- a/llama_stack/templates/ollama/build.yaml
+++ b/llama_stack/templates/ollama/build.yaml
@ -16,4 +16,13 @@ distribution_spec:
    - inline::meta-reference
    telemetry:
    - inline::meta-reference
+    eval:
+    - inline::meta-reference
+    datasetio:
+    - remote::huggingface
+    - inline::localfs
+    scoring:
+    - inline::basic
+    - inline::llm-as-judge
+    - inline::braintrust
 image_type: conda
--- a/llama_stack/templates/ollama/run-with-safety.yaml
+++ b/llama_stack/templates/ollama/run-with-safety.yaml
@ -4,9 +4,12 @@ docker_image: null
 conda_env: ollama
 apis:
 - agents
+- datasetio
+- eval
 - inference
 - memory
 - safety
+- scoring
 - telemetry
 providers:
  inference:
@ -38,6 +41,18 @@ providers:
  - provider_id: meta-reference
    provider_type: inline::meta-reference
    config: {}
+  eval:
+  - provider_id: meta-reference
+    provider_type: inline::meta-reference
+    config: {}
+  datasetio:
+  - provider_id: huggingface
+    provider_type: remote::huggingface
+    config: {}
+  scoring:
+  - provider_id: basic
+    provider_type: inline::basic
+    config: {}
 metadata_store:
  namespace: null
  type: sqlite
--- a/llama_stack/templates/ollama/run.yaml
+++ b/llama_stack/templates/ollama/run.yaml
@ -4,9 +4,12 @@ docker_image: null
 conda_env: ollama
 apis:
 - agents
+- datasetio
+- eval
 - inference
 - memory
 - safety
+- scoring
 - telemetry
 providers:
  inference:
@ -38,6 +41,18 @@ providers:
  - provider_id: meta-reference
    provider_type: inline::meta-reference
    config: {}
+  eval:
+  - provider_id: meta-reference
+    provider_type: inline::meta-reference
+    config: {}
+  datasetio:
+  - provider_id: huggingface
+    provider_type: remote::huggingface
+    config: {}
+  scoring:
+  - provider_id: basic
+    provider_type: inline::basic
+    config: {}
 metadata_store:
  namespace: null
  type: sqlite
--- a/llama_stack/templates/tgi/build.yaml
+++ b/llama_stack/templates/tgi/build.yaml
@ -16,4 +16,13 @@ distribution_spec:
    - inline::meta-reference
    telemetry:
    - inline::meta-reference
+    eval:
+    - inline::meta-reference
+    datasetio:
+    - remote::huggingface
+    - inline::localfs
+    scoring:
+    - inline::basic
+    - inline::llm-as-judge
+    - inline::braintrust
 image_type: conda
--- a/llama_stack/templates/tgi/run-with-safety.yaml
+++ b/llama_stack/templates/tgi/run-with-safety.yaml
@ -4,9 +4,12 @@ docker_image: null
 conda_env: tgi
 apis:
 - agents
+- datasetio
+- eval
 - inference
 - memory
 - safety
+- scoring
 - telemetry
 providers:
  inference:
@ -42,6 +45,18 @@ providers:
  - provider_id: meta-reference
    provider_type: inline::meta-reference
    config: {}
+  eval:
+  - provider_id: meta-reference
+    provider_type: inline::meta-reference
+    config: {}
+  datasetio:
+  - provider_id: huggingface
+    provider_type: remote::huggingface
+    config: {}
+  scoring:
+  - provider_id: basic
+    provider_type: inline::basic
+    config: {}
 metadata_store:
  namespace: null
  type: sqlite
--- a/llama_stack/templates/tgi/run.yaml
+++ b/llama_stack/templates/tgi/run.yaml
@ -4,9 +4,12 @@ docker_image: null
 conda_env: tgi
 apis:
 - agents
+- datasetio
+- eval
 - inference
 - memory
 - safety
+- scoring
 - telemetry
 providers:
  inference:
@ -38,6 +41,18 @@ providers:
  - provider_id: meta-reference
    provider_type: inline::meta-reference
    config: {}
+  eval:
+  - provider_id: meta-reference
+    provider_type: inline::meta-reference
+    config: {}
+  datasetio:
+  - provider_id: huggingface
+    provider_type: remote::huggingface
+    config: {}
+  scoring:
+  - provider_id: basic
+    provider_type: inline::basic
+    config: {}
 metadata_store:
  namespace: null
  type: sqlite
--- a/llama_stack/templates/together/build.yaml
+++ b/llama_stack/templates/together/build.yaml
@ -16,4 +16,13 @@ distribution_spec:
    - inline::meta-reference
    telemetry:
    - inline::meta-reference
+    eval:
+    - inline::meta-reference
+    datasetio:
+    - remote::huggingface
+    - inline::localfs
+    scoring:
+    - inline::basic
+    - inline::llm-as-judge
+    - inline::braintrust
 image_type: conda
--- a/llama_stack/templates/together/run.yaml
+++ b/llama_stack/templates/together/run.yaml
@ -4,9 +4,12 @@ docker_image: null
 conda_env: together
 apis:
 - agents
+- datasetio
+- eval
 - inference
 - memory
 - safety
+- scoring
 - telemetry
 providers:
  inference:
@ -39,6 +42,18 @@ providers:
  - provider_id: meta-reference
    provider_type: inline::meta-reference
    config: {}
+  eval:
+  - provider_id: meta-reference
+    provider_type: inline::meta-reference
+    config: {}
+  datasetio:
+  - provider_id: huggingface
+    provider_type: remote::huggingface
+    config: {}
+  scoring:
+  - provider_id: basic
+    provider_type: inline::basic
+    config: {}
 metadata_store:
  namespace: null
  type: sqlite
--- a/llama_stack/templates/vllm-gpu/build.yaml
+++ b/llama_stack/templates/vllm-gpu/build.yaml
@ -16,4 +16,13 @@ distribution_spec:
    - inline::meta-reference
    telemetry:
    - inline::meta-reference
+    eval:
+    - inline::meta-reference
+    datasetio:
+    - remote::huggingface
+    - inline::localfs
+    scoring:
+    - inline::basic
+    - inline::llm-as-judge
+    - inline::braintrust
 image_type: conda
--- a/llama_stack/templates/vllm-gpu/run.yaml
+++ b/llama_stack/templates/vllm-gpu/run.yaml
@ -4,9 +4,12 @@ docker_image: null
 conda_env: vllm-gpu
 apis:
 - agents
+- datasetio
+- eval
 - inference
 - memory
 - safety
+- scoring
 - telemetry
 providers:
  inference:
@ -42,6 +45,18 @@ providers:
  - provider_id: meta-reference
    provider_type: inline::meta-reference
    config: {}
+  eval:
+  - provider_id: meta-reference
+    provider_type: inline::meta-reference
+    config: {}
+  datasetio:
+  - provider_id: huggingface
+    provider_type: remote::huggingface
+    config: {}
+  scoring:
+  - provider_id: basic
+    provider_type: inline::basic
+    config: {}
 metadata_store:
  namespace: null
  type: sqlite