autogen build/run

distributions/dependencies.json

@@ -2,9 +2,11 @@
   "hf-serverless": [
     "aiohttp",
     "aiosqlite",
+    "autoevals",
     "blobfile",
     "chardet",
     "chromadb-client",
+    "datasets",
     "faiss-cpu",
     "fastapi",
     "fire",
@@ -13,6 +15,7 @@
     "matplotlib",
     "nltk",
     "numpy",
+    "openai",
     "pandas",
     "pillow",
     "psycopg2-binary",
@@ -29,9 +32,11 @@
   ],
   "together": [
     "aiosqlite",
+    "autoevals",
     "blobfile",
     "chardet",
     "chromadb-client",
+    "datasets",
     "faiss-cpu",
     "fastapi",
     "fire",
@@ -39,6 +44,7 @@
     "matplotlib",
     "nltk",
     "numpy",
+    "openai",
     "pandas",
     "pillow",
     "psycopg2-binary",
@@ -56,9 +62,11 @@
   ],
   "vllm-gpu": [
     "aiosqlite",
+    "autoevals",
     "blobfile",
     "chardet",
     "chromadb-client",
+    "datasets",
     "faiss-cpu",
     "fastapi",
     "fire",
@@ -66,6 +74,7 @@
     "matplotlib",
     "nltk",
     "numpy",
+    "openai",
     "pandas",
     "pillow",
     "psycopg2-binary",
@@ -110,9 +119,11 @@
   ],
   "fireworks": [
     "aiosqlite",
+    "autoevals",
     "blobfile",
     "chardet",
     "chromadb-client",
+    "datasets",
     "faiss-cpu",
     "fastapi",
     "fire",
@@ -121,6 +132,7 @@
     "matplotlib",
     "nltk",
     "numpy",
+    "openai",
     "pandas",
     "pillow",
     "psycopg2-binary",
@@ -138,9 +150,11 @@
   "tgi": [
     "aiohttp",
     "aiosqlite",
+    "autoevals",
     "blobfile",
     "chardet",
     "chromadb-client",
+    "datasets",
     "faiss-cpu",
     "fastapi",
     "fire",
@@ -149,6 +163,7 @@
     "matplotlib",
     "nltk",
     "numpy",
+    "openai",
     "pandas",
     "pillow",
     "psycopg2-binary",
@@ -165,10 +180,12 @@
   ],
   "bedrock": [
     "aiosqlite",
+    "autoevals",
     "blobfile",
     "boto3",
     "chardet",
     "chromadb-client",
+    "datasets",
     "faiss-cpu",
     "fastapi",
     "fire",
@@ -176,6 +193,7 @@
     "matplotlib",
     "nltk",
     "numpy",
+    "openai",
     "pandas",
     "pillow",
     "psycopg2-binary",
@@ -193,9 +211,11 @@
   "meta-reference-gpu": [
     "accelerate",
     "aiosqlite",
+    "autoevals",
     "blobfile",
     "chardet",
     "chromadb-client",
+    "datasets",
     "fairscale",
     "faiss-cpu",
     "fastapi",
@@ -205,6 +225,7 @@
     "matplotlib",
     "nltk",
     "numpy",
+    "openai",
     "pandas",
     "pillow",
     "psycopg2-binary",
@@ -225,9 +246,11 @@
   "meta-reference-quantized-gpu": [
     "accelerate",
     "aiosqlite",
+    "autoevals",
     "blobfile",
     "chardet",
     "chromadb-client",
+    "datasets",
     "fairscale",
     "faiss-cpu",
     "fastapi",
@@ -238,6 +261,7 @@
     "matplotlib",
     "nltk",
     "numpy",
+    "openai",
     "pandas",
     "pillow",
     "psycopg2-binary",
@@ -259,9 +283,11 @@
   "ollama": [
     "aiohttp",
     "aiosqlite",
+    "autoevals",
     "blobfile",
     "chardet",
     "chromadb-client",
+    "datasets",
     "faiss-cpu",
     "fastapi",
     "fire",
@@ -270,6 +296,7 @@
     "nltk",
     "numpy",
     "ollama",
+    "openai",
     "pandas",
     "pillow",
     "psycopg2-binary",
@@ -287,9 +314,11 @@
   "hf-endpoint": [
     "aiohttp",
     "aiosqlite",
+    "autoevals",
     "blobfile",
     "chardet",
     "chromadb-client",
+    "datasets",
     "faiss-cpu",
     "fastapi",
     "fire",
@@ -298,6 +327,7 @@
     "matplotlib",
     "nltk",
     "numpy",
+    "openai",
     "pandas",
     "pillow",
     "psycopg2-binary",

docs/source/distributions/self_hosted_distro/bedrock.md

@@ -1,6 +1,3 @@
----
-orphan: true
----
 # Bedrock Distribution
 
 ```{toctree}
@@ -15,9 +12,12 @@ The `llamastack/distribution-bedrock` distribution consists of the following pro
 | API | Provider(s) |
 |-----|-------------|
 | agents | `inline::meta-reference` |
+| datasetio | `remote::huggingface`, `inline::localfs` |
+| eval | `inline::meta-reference` |
 | inference | `remote::bedrock` |
 | memory | `inline::faiss`, `remote::chromadb`, `remote::pgvector` |
 | safety | `remote::bedrock` |
+| scoring | `inline::basic`, `inline::llm-as-judge`, `inline::braintrust` |
 | telemetry | `inline::meta-reference` |
 
 

docs/source/distributions/self_hosted_distro/fireworks.md

@@ -15,9 +15,12 @@ The `llamastack/distribution-fireworks` distribution consists of the following p
 | API | Provider(s) |
 |-----|-------------|
 | agents | `inline::meta-reference` |
+| datasetio | `remote::huggingface`, `inline::localfs` |
+| eval | `inline::meta-reference` |
 | inference | `remote::fireworks` |
 | memory | `inline::faiss`, `remote::chromadb`, `remote::pgvector` |
 | safety | `inline::llama-guard` |
+| scoring | `inline::basic`, `inline::llm-as-judge`, `inline::braintrust` |
 | telemetry | `inline::meta-reference` |
 
 

docs/source/distributions/self_hosted_distro/meta-reference-gpu.md

@@ -15,9 +15,12 @@ The `llamastack/distribution-meta-reference-gpu` distribution consists of the fo
 | API | Provider(s) |
 |-----|-------------|
 | agents | `inline::meta-reference` |
+| datasetio | `remote::huggingface`, `inline::localfs` |
+| eval | `inline::meta-reference` |
 | inference | `inline::meta-reference` |
 | memory | `inline::faiss`, `remote::chromadb`, `remote::pgvector` |
 | safety | `inline::llama-guard` |
+| scoring | `inline::basic`, `inline::llm-as-judge`, `inline::braintrust` |
 | telemetry | `inline::meta-reference` |
 
 

docs/source/distributions/self_hosted_distro/meta-reference-quantized-gpu.md

@@ -15,9 +15,12 @@ The `llamastack/distribution-meta-reference-quantized-gpu` distribution consists
 | API | Provider(s) |
 |-----|-------------|
 | agents | `inline::meta-reference` |
+| datasetio | `remote::huggingface`, `inline::localfs` |
+| eval | `inline::meta-reference` |
 | inference | `inline::meta-reference-quantized` |
 | memory | `inline::faiss`, `remote::chromadb`, `remote::pgvector` |
 | safety | `inline::llama-guard` |
+| scoring | `inline::basic`, `inline::llm-as-judge`, `inline::braintrust` |
 | telemetry | `inline::meta-reference` |
 
 

docs/source/distributions/self_hosted_distro/ollama.md

@@ -15,9 +15,12 @@ The `llamastack/distribution-ollama` distribution consists of the following prov
 | API | Provider(s) |
 |-----|-------------|
 | agents | `inline::meta-reference` |
+| datasetio | `remote::huggingface`, `inline::localfs` |
+| eval | `inline::meta-reference` |
 | inference | `remote::ollama` |
 | memory | `inline::faiss`, `remote::chromadb`, `remote::pgvector` |
 | safety | `inline::llama-guard` |
+| scoring | `inline::basic`, `inline::llm-as-judge`, `inline::braintrust` |
 | telemetry | `inline::meta-reference` |
 
 
@@ -118,9 +121,9 @@ llama stack run ./run-with-safety.yaml \
 
 ### (Optional) Update Model Serving Configuration
 
-```{note}
+> [!NOTE]
-Please check the [model_aliases](https://github.com/meta-llama/llama-stack/blob/main/llama_stack/providers/remote/inference/ollama/ollama.py#L45) variable for supported Ollama models.
+> Please check the [OLLAMA_SUPPORTED_MODELS](https://github.com/meta-llama/llama-stack/blob/main/llama_stack/providers.remote/inference/ollama/ollama.py) for the supported Ollama models.
-```
+
 
 To serve a new model with `ollama`
 ```bash

docs/source/distributions/self_hosted_distro/tgi.md

@@ -16,9 +16,12 @@ The `llamastack/distribution-tgi` distribution consists of the following provide
 | API | Provider(s) |
 |-----|-------------|
 | agents | `inline::meta-reference` |
+| datasetio | `remote::huggingface`, `inline::localfs` |
+| eval | `inline::meta-reference` |
 | inference | `remote::tgi` |
 | memory | `inline::faiss`, `remote::chromadb`, `remote::pgvector` |
 | safety | `inline::llama-guard` |
+| scoring | `inline::basic`, `inline::llm-as-judge`, `inline::braintrust` |
 | telemetry | `inline::meta-reference` |
 
 

docs/source/distributions/self_hosted_distro/together.md

@@ -15,9 +15,12 @@ The `llamastack/distribution-together` distribution consists of the following pr
 | API | Provider(s) |
 |-----|-------------|
 | agents | `inline::meta-reference` |
+| datasetio | `remote::huggingface`, `inline::localfs` |
+| eval | `inline::meta-reference` |
 | inference | `remote::together` |
 | memory | `inline::faiss`, `remote::chromadb`, `remote::pgvector` |
 | safety | `inline::llama-guard` |
+| scoring | `inline::basic`, `inline::llm-as-judge`, `inline::braintrust` |
 | telemetry | `inline::meta-reference` |
 
 

llama_stack/templates/bedrock/build.yaml

@@ -16,4 +16,13 @@ distribution_spec:
     - inline::meta-reference
     telemetry:
     - inline::meta-reference
+    eval:
+    - inline::meta-reference
+    datasetio:
+    - remote::huggingface
+    - inline::localfs
+    scoring:
+    - inline::basic
+    - inline::llm-as-judge
+    - inline::braintrust
 image_type: conda

llama_stack/templates/bedrock/run.yaml

@@ -4,9 +4,12 @@ docker_image: null
 conda_env: bedrock
 apis:
 - agents
+- datasetio
+- eval
 - inference
 - memory
 - safety
+- scoring
 - telemetry
 providers:
   inference:
@@ -37,6 +40,18 @@ providers:
   - provider_id: meta-reference
     provider_type: inline::meta-reference
     config: {}
+  eval:
+  - provider_id: meta-reference
+    provider_type: inline::meta-reference
+    config: {}
+  datasetio:
+  - provider_id: huggingface
+    provider_type: remote::huggingface
+    config: {}
+  scoring:
+  - provider_id: basic
+    provider_type: inline::basic
+    config: {}
 metadata_store:
   namespace: null
   type: sqlite

llama_stack/templates/fireworks/build.yaml

@@ -16,4 +16,13 @@ distribution_spec:
     - inline::meta-reference
     telemetry:
     - inline::meta-reference
+    eval:
+    - inline::meta-reference
+    datasetio:
+    - remote::huggingface
+    - inline::localfs
+    scoring:
+    - inline::basic
+    - inline::llm-as-judge
+    - inline::braintrust
 image_type: conda

llama_stack/templates/fireworks/run.yaml

@@ -4,9 +4,12 @@ docker_image: null
 conda_env: fireworks
 apis:
 - agents
+- datasetio
+- eval
 - inference
 - memory
 - safety
+- scoring
 - telemetry
 providers:
   inference:
@@ -39,6 +42,18 @@ providers:
   - provider_id: meta-reference
     provider_type: inline::meta-reference
     config: {}
+  eval:
+  - provider_id: meta-reference
+    provider_type: inline::meta-reference
+    config: {}
+  datasetio:
+  - provider_id: huggingface
+    provider_type: remote::huggingface
+    config: {}
+  scoring:
+  - provider_id: basic
+    provider_type: inline::basic
+    config: {}
 metadata_store:
   namespace: null
   type: sqlite

llama_stack/templates/hf-endpoint/build.yaml

@@ -16,4 +16,13 @@ distribution_spec:
     - inline::meta-reference
     telemetry:
     - inline::meta-reference
+    eval:
+    - inline::meta-reference
+    datasetio:
+    - remote::huggingface
+    - inline::localfs
+    scoring:
+    - inline::basic
+    - inline::llm-as-judge
+    - inline::braintrust
 image_type: conda

llama_stack/templates/hf-endpoint/run-with-safety.yaml

@@ -4,9 +4,12 @@ docker_image: null
 conda_env: hf-endpoint
 apis:
 - agents
+- datasetio
+- eval
 - inference
 - memory
 - safety
+- scoring
 - telemetry
 providers:
   inference:
@@ -44,6 +47,18 @@ providers:
   - provider_id: meta-reference
     provider_type: inline::meta-reference
     config: {}
+  eval:
+  - provider_id: meta-reference
+    provider_type: inline::meta-reference
+    config: {}
+  datasetio:
+  - provider_id: huggingface
+    provider_type: remote::huggingface
+    config: {}
+  scoring:
+  - provider_id: basic
+    provider_type: inline::basic
+    config: {}
 metadata_store:
   namespace: null
   type: sqlite

llama_stack/templates/hf-endpoint/run.yaml

@@ -4,9 +4,12 @@ docker_image: null
 conda_env: hf-endpoint
 apis:
 - agents
+- datasetio
+- eval
 - inference
 - memory
 - safety
+- scoring
 - telemetry
 providers:
   inference:
@@ -39,6 +42,18 @@ providers:
   - provider_id: meta-reference
     provider_type: inline::meta-reference
     config: {}
+  eval:
+  - provider_id: meta-reference
+    provider_type: inline::meta-reference
+    config: {}
+  datasetio:
+  - provider_id: huggingface
+    provider_type: remote::huggingface
+    config: {}
+  scoring:
+  - provider_id: basic
+    provider_type: inline::basic
+    config: {}
 metadata_store:
   namespace: null
   type: sqlite

llama_stack/templates/hf-serverless/build.yaml

@@ -16,4 +16,13 @@ distribution_spec:
     - inline::meta-reference
     telemetry:
     - inline::meta-reference
+    eval:
+    - inline::meta-reference
+    datasetio:
+    - remote::huggingface
+    - inline::localfs
+    scoring:
+    - inline::basic
+    - inline::llm-as-judge
+    - inline::braintrust
 image_type: conda

llama_stack/templates/hf-serverless/run-with-safety.yaml

@@ -4,9 +4,12 @@ docker_image: null
 conda_env: hf-serverless
 apis:
 - agents
+- datasetio
+- eval
 - inference
 - memory
 - safety
+- scoring
 - telemetry
 providers:
   inference:
@@ -44,6 +47,18 @@ providers:
   - provider_id: meta-reference
     provider_type: inline::meta-reference
     config: {}
+  eval:
+  - provider_id: meta-reference
+    provider_type: inline::meta-reference
+    config: {}
+  datasetio:
+  - provider_id: huggingface
+    provider_type: remote::huggingface
+    config: {}
+  scoring:
+  - provider_id: basic
+    provider_type: inline::basic
+    config: {}
 metadata_store:
   namespace: null
   type: sqlite

llama_stack/templates/hf-serverless/run.yaml

@@ -4,9 +4,12 @@ docker_image: null
 conda_env: hf-serverless
 apis:
 - agents
+- datasetio
+- eval
 - inference
 - memory
 - safety
+- scoring
 - telemetry
 providers:
   inference:
@@ -39,6 +42,18 @@ providers:
   - provider_id: meta-reference
     provider_type: inline::meta-reference
     config: {}
+  eval:
+  - provider_id: meta-reference
+    provider_type: inline::meta-reference
+    config: {}
+  datasetio:
+  - provider_id: huggingface
+    provider_type: remote::huggingface
+    config: {}
+  scoring:
+  - provider_id: basic
+    provider_type: inline::basic
+    config: {}
 metadata_store:
   namespace: null
   type: sqlite

llama_stack/templates/meta-reference-gpu/build.yaml

@@ -16,4 +16,13 @@ distribution_spec:
     - inline::meta-reference
     telemetry:
     - inline::meta-reference
+    eval:
+    - inline::meta-reference
+    datasetio:
+    - remote::huggingface
+    - inline::localfs
+    scoring:
+    - inline::basic
+    - inline::llm-as-judge
+    - inline::braintrust
 image_type: conda

llama_stack/templates/meta-reference-gpu/run-with-safety.yaml

@@ -4,9 +4,12 @@ docker_image: null
 conda_env: meta-reference-gpu
 apis:
 - agents
+- datasetio
+- eval
 - inference
 - memory
 - safety
+- scoring
 - telemetry
 providers:
   inference:
@@ -46,6 +49,18 @@ providers:
   - provider_id: meta-reference
     provider_type: inline::meta-reference
     config: {}
+  eval:
+  - provider_id: meta-reference
+    provider_type: inline::meta-reference
+    config: {}
+  datasetio:
+  - provider_id: huggingface
+    provider_type: remote::huggingface
+    config: {}
+  scoring:
+  - provider_id: basic
+    provider_type: inline::basic
+    config: {}
 metadata_store:
   namespace: null
   type: sqlite

llama_stack/templates/meta-reference-gpu/run.yaml

@@ -4,9 +4,12 @@ docker_image: null
 conda_env: meta-reference-gpu
 apis:
 - agents
+- datasetio
+- eval
 - inference
 - memory
 - safety
+- scoring
 - telemetry
 providers:
   inference:
@@ -40,6 +43,18 @@ providers:
   - provider_id: meta-reference
     provider_type: inline::meta-reference
     config: {}
+  eval:
+  - provider_id: meta-reference
+    provider_type: inline::meta-reference
+    config: {}
+  datasetio:
+  - provider_id: huggingface
+    provider_type: remote::huggingface
+    config: {}
+  scoring:
+  - provider_id: basic
+    provider_type: inline::basic
+    config: {}
 metadata_store:
   namespace: null
   type: sqlite

llama_stack/templates/meta-reference-quantized-gpu/build.yaml

@@ -16,4 +16,13 @@ distribution_spec:
     - inline::meta-reference
     telemetry:
     - inline::meta-reference
+    eval:
+    - inline::meta-reference
+    datasetio:
+    - remote::huggingface
+    - inline::localfs
+    scoring:
+    - inline::basic
+    - inline::llm-as-judge
+    - inline::braintrust
 image_type: conda

llama_stack/templates/meta-reference-quantized-gpu/run.yaml

@@ -4,9 +4,12 @@ docker_image: null
 conda_env: meta-reference-quantized-gpu
 apis:
 - agents
+- datasetio
+- eval
 - inference
 - memory
 - safety
+- scoring
 - telemetry
 providers:
   inference:
@@ -42,6 +45,18 @@ providers:
   - provider_id: meta-reference
     provider_type: inline::meta-reference
     config: {}
+  eval:
+  - provider_id: meta-reference
+    provider_type: inline::meta-reference
+    config: {}
+  datasetio:
+  - provider_id: huggingface
+    provider_type: remote::huggingface
+    config: {}
+  scoring:
+  - provider_id: basic
+    provider_type: inline::basic
+    config: {}
 metadata_store:
   namespace: null
   type: sqlite

llama_stack/templates/ollama/build.yaml

@@ -16,4 +16,13 @@ distribution_spec:
     - inline::meta-reference
     telemetry:
     - inline::meta-reference
+    eval:
+    - inline::meta-reference
+    datasetio:
+    - remote::huggingface
+    - inline::localfs
+    scoring:
+    - inline::basic
+    - inline::llm-as-judge
+    - inline::braintrust
 image_type: conda

llama_stack/templates/ollama/run-with-safety.yaml

@@ -4,9 +4,12 @@ docker_image: null
 conda_env: ollama
 apis:
 - agents
+- datasetio
+- eval
 - inference
 - memory
 - safety
+- scoring
 - telemetry
 providers:
   inference:
@@ -38,6 +41,18 @@ providers:
   - provider_id: meta-reference
     provider_type: inline::meta-reference
     config: {}
+  eval:
+  - provider_id: meta-reference
+    provider_type: inline::meta-reference
+    config: {}
+  datasetio:
+  - provider_id: huggingface
+    provider_type: remote::huggingface
+    config: {}
+  scoring:
+  - provider_id: basic
+    provider_type: inline::basic
+    config: {}
 metadata_store:
   namespace: null
   type: sqlite

llama_stack/templates/ollama/run.yaml

@@ -4,9 +4,12 @@ docker_image: null
 conda_env: ollama
 apis:
 - agents
+- datasetio
+- eval
 - inference
 - memory
 - safety
+- scoring
 - telemetry
 providers:
   inference:
@@ -38,6 +41,18 @@ providers:
   - provider_id: meta-reference
     provider_type: inline::meta-reference
     config: {}
+  eval:
+  - provider_id: meta-reference
+    provider_type: inline::meta-reference
+    config: {}
+  datasetio:
+  - provider_id: huggingface
+    provider_type: remote::huggingface
+    config: {}
+  scoring:
+  - provider_id: basic
+    provider_type: inline::basic
+    config: {}
 metadata_store:
   namespace: null
   type: sqlite

llama_stack/templates/tgi/build.yaml

@@ -16,4 +16,13 @@ distribution_spec:
     - inline::meta-reference
     telemetry:
     - inline::meta-reference
+    eval:
+    - inline::meta-reference
+    datasetio:
+    - remote::huggingface
+    - inline::localfs
+    scoring:
+    - inline::basic
+    - inline::llm-as-judge
+    - inline::braintrust
 image_type: conda

llama_stack/templates/tgi/run-with-safety.yaml

@@ -4,9 +4,12 @@ docker_image: null
 conda_env: tgi
 apis:
 - agents
+- datasetio
+- eval
 - inference
 - memory
 - safety
+- scoring
 - telemetry
 providers:
   inference:
@@ -42,6 +45,18 @@ providers:
   - provider_id: meta-reference
     provider_type: inline::meta-reference
     config: {}
+  eval:
+  - provider_id: meta-reference
+    provider_type: inline::meta-reference
+    config: {}
+  datasetio:
+  - provider_id: huggingface
+    provider_type: remote::huggingface
+    config: {}
+  scoring:
+  - provider_id: basic
+    provider_type: inline::basic
+    config: {}
 metadata_store:
   namespace: null
   type: sqlite

llama_stack/templates/tgi/run.yaml

@@ -4,9 +4,12 @@ docker_image: null
 conda_env: tgi
 apis:
 - agents
+- datasetio
+- eval
 - inference
 - memory
 - safety
+- scoring
 - telemetry
 providers:
   inference:
@@ -38,6 +41,18 @@ providers:
   - provider_id: meta-reference
     provider_type: inline::meta-reference
     config: {}
+  eval:
+  - provider_id: meta-reference
+    provider_type: inline::meta-reference
+    config: {}
+  datasetio:
+  - provider_id: huggingface
+    provider_type: remote::huggingface
+    config: {}
+  scoring:
+  - provider_id: basic
+    provider_type: inline::basic
+    config: {}
 metadata_store:
   namespace: null
   type: sqlite

llama_stack/templates/together/build.yaml

@@ -16,4 +16,13 @@ distribution_spec:
     - inline::meta-reference
     telemetry:
     - inline::meta-reference
+    eval:
+    - inline::meta-reference
+    datasetio:
+    - remote::huggingface
+    - inline::localfs
+    scoring:
+    - inline::basic
+    - inline::llm-as-judge
+    - inline::braintrust
 image_type: conda

llama_stack/templates/together/run.yaml

@@ -4,9 +4,12 @@ docker_image: null
 conda_env: together
 apis:
 - agents
+- datasetio
+- eval
 - inference
 - memory
 - safety
+- scoring
 - telemetry
 providers:
   inference:
@@ -39,6 +42,18 @@ providers:
   - provider_id: meta-reference
     provider_type: inline::meta-reference
     config: {}
+  eval:
+  - provider_id: meta-reference
+    provider_type: inline::meta-reference
+    config: {}
+  datasetio:
+  - provider_id: huggingface
+    provider_type: remote::huggingface
+    config: {}
+  scoring:
+  - provider_id: basic
+    provider_type: inline::basic
+    config: {}
 metadata_store:
   namespace: null
   type: sqlite

llama_stack/templates/vllm-gpu/build.yaml

@@ -16,4 +16,13 @@ distribution_spec:
     - inline::meta-reference
     telemetry:
     - inline::meta-reference
+    eval:
+    - inline::meta-reference
+    datasetio:
+    - remote::huggingface
+    - inline::localfs
+    scoring:
+    - inline::basic
+    - inline::llm-as-judge
+    - inline::braintrust
 image_type: conda

llama_stack/templates/vllm-gpu/run.yaml

@@ -4,9 +4,12 @@ docker_image: null
 conda_env: vllm-gpu
 apis:
 - agents
+- datasetio
+- eval
 - inference
 - memory
 - safety
+- scoring
 - telemetry
 providers:
   inference:
@@ -42,6 +45,18 @@ providers:
   - provider_id: meta-reference
     provider_type: inline::meta-reference
     config: {}
+  eval:
+  - provider_id: meta-reference
+    provider_type: inline::meta-reference
+    config: {}
+  datasetio:
+  - provider_id: huggingface
+    provider_type: remote::huggingface
+    config: {}
+  scoring:
+  - provider_id: basic
+    provider_type: inline::basic
+    config: {}
 metadata_store:
   namespace: null
   type: sqlite