From 8162f96d00d2e204ebe65892a716469c388ea991 Mon Sep 17 00:00:00 2001 From: Xi Yan Date: Tue, 18 Mar 2025 21:46:41 -0700 Subject: [PATCH] remove evals from top-level --- .../remote_hosted_distro/nvidia.md | 2 - .../self_hosted_distro/bedrock.md | 2 - .../self_hosted_distro/cerebras.md | 2 - .../self_hosted_distro/fireworks.md | 2 - .../distributions/self_hosted_distro/groq.md | 2 - .../self_hosted_distro/meta-reference-gpu.md | 2 - .../meta-reference-quantized-gpu.md | 2 - .../self_hosted_distro/ollama.md | 2 - .../self_hosted_distro/passthrough.md | 2 - .../self_hosted_distro/remote-vllm.md | 2 - .../distributions/self_hosted_distro/tgi.md | 2 - .../self_hosted_distro/together.md | 2 - llama_stack/distribution/distribution.py | 3 +- llama_stack/templates/bedrock/bedrock.py | 8 ++- llama_stack/templates/bedrock/build.yaml | 6 -- llama_stack/templates/bedrock/run.yaml | 22 ------ llama_stack/templates/cerebras/build.yaml | 6 -- llama_stack/templates/cerebras/cerebras.py | 8 ++- llama_stack/templates/cerebras/run.yaml | 22 ------ llama_stack/templates/ci-tests/build.yaml | 6 -- llama_stack/templates/ci-tests/ci_tests.py | 12 ++-- llama_stack/templates/ci-tests/run.yaml | 22 ------ llama_stack/templates/dell/build.yaml | 6 -- llama_stack/templates/dell/dell.py | 2 - .../templates/dell/run-with-safety.yaml | 22 ------ llama_stack/templates/dell/run.yaml | 22 ------ llama_stack/templates/dev/build.yaml | 6 -- llama_stack/templates/dev/dev.py | 45 ++++++++---- llama_stack/templates/dev/run.yaml | 22 ------ llama_stack/templates/fireworks/build.yaml | 6 -- llama_stack/templates/fireworks/fireworks.py | 8 ++- .../templates/fireworks/run-with-safety.yaml | 22 ------ llama_stack/templates/fireworks/run.yaml | 22 ------ llama_stack/templates/groq/build.yaml | 6 -- llama_stack/templates/groq/groq.py | 14 ++-- llama_stack/templates/groq/run.yaml | 22 ------ llama_stack/templates/hf-endpoint/build.yaml | 6 -- .../templates/hf-endpoint/hf_endpoint.py | 2 - .../hf-endpoint/run-with-safety.yaml | 22 ------ llama_stack/templates/hf-endpoint/run.yaml | 22 ------ .../templates/hf-serverless/build.yaml | 6 -- .../templates/hf-serverless/hf_serverless.py | 2 - .../hf-serverless/run-with-safety.yaml | 22 ------ llama_stack/templates/hf-serverless/run.yaml | 22 ------ .../templates/meta-reference-gpu/build.yaml | 6 -- .../meta-reference-gpu/meta_reference.py | 2 - .../meta-reference-gpu/run-with-safety.yaml | 22 ------ .../templates/meta-reference-gpu/run.yaml | 22 ------ .../meta-reference-quantized-gpu/build.yaml | 6 -- .../meta_reference.py | 2 - .../meta-reference-quantized-gpu/run.yaml | 22 ------ llama_stack/templates/nvidia/build.yaml | 4 -- llama_stack/templates/nvidia/nvidia.py | 19 +++-- .../templates/nvidia/run-with-safety.yaml | 15 ---- llama_stack/templates/nvidia/run.yaml | 15 ---- llama_stack/templates/ollama/build.yaml | 6 -- llama_stack/templates/ollama/ollama.py | 2 - .../templates/ollama/run-with-safety.yaml | 22 ------ llama_stack/templates/ollama/run.yaml | 22 ------ .../open-benchmark/open_benchmark.py | 69 ++++++++++--------- llama_stack/templates/passthrough/build.yaml | 6 -- .../templates/passthrough/passthrough.py | 7 +- .../passthrough/run-with-safety.yaml | 22 ------ llama_stack/templates/passthrough/run.yaml | 22 ------ llama_stack/templates/remote-vllm/build.yaml | 6 -- .../remote-vllm/run-with-safety.yaml | 22 ------ llama_stack/templates/remote-vllm/run.yaml | 22 ------ llama_stack/templates/remote-vllm/vllm.py | 2 - llama_stack/templates/sambanova/run.yaml | 1 - llama_stack/templates/tgi/build.yaml | 6 -- .../templates/tgi/run-with-safety.yaml | 22 ------ llama_stack/templates/tgi/run.yaml | 22 ------ llama_stack/templates/tgi/tgi.py | 2 - llama_stack/templates/together/build.yaml | 6 -- .../templates/together/run-with-safety.yaml | 22 ------ llama_stack/templates/together/run.yaml | 22 ------ llama_stack/templates/together/together.py | 8 ++- llama_stack/templates/vllm-gpu/build.yaml | 6 -- llama_stack/templates/vllm-gpu/run.yaml | 22 ------ llama_stack/templates/vllm-gpu/vllm.py | 2 - 80 files changed, 121 insertions(+), 853 deletions(-) diff --git a/docs/source/distributions/remote_hosted_distro/nvidia.md b/docs/source/distributions/remote_hosted_distro/nvidia.md index 774d5ec1b..5d7bb4ef4 100644 --- a/docs/source/distributions/remote_hosted_distro/nvidia.md +++ b/docs/source/distributions/remote_hosted_distro/nvidia.md @@ -7,10 +7,8 @@ The `llamastack/distribution-nvidia` distribution consists of the following prov |-----|-------------| | agents | `inline::meta-reference` | | datasetio | `inline::localfs` | -| eval | `inline::meta-reference` | | inference | `remote::nvidia` | | safety | `remote::nvidia` | -| scoring | `inline::basic` | | telemetry | `inline::meta-reference` | | tool_runtime | `inline::rag-runtime` | | vector_io | `inline::faiss` | diff --git a/docs/source/distributions/self_hosted_distro/bedrock.md b/docs/source/distributions/self_hosted_distro/bedrock.md index 623ab6848..92a934222 100644 --- a/docs/source/distributions/self_hosted_distro/bedrock.md +++ b/docs/source/distributions/self_hosted_distro/bedrock.md @@ -14,10 +14,8 @@ The `llamastack/distribution-bedrock` distribution consists of the following pro |-----|-------------| | agents | `inline::meta-reference` | | datasetio | `remote::huggingface`, `inline::localfs` | -| eval | `inline::meta-reference` | | inference | `remote::bedrock` | | safety | `remote::bedrock` | -| scoring | `inline::basic`, `inline::llm-as-judge`, `inline::braintrust` | | telemetry | `inline::meta-reference` | | tool_runtime | `remote::brave-search`, `remote::tavily-search`, `inline::code-interpreter`, `inline::rag-runtime`, `remote::model-context-protocol` | | vector_io | `inline::faiss`, `remote::chromadb`, `remote::pgvector` | diff --git a/docs/source/distributions/self_hosted_distro/cerebras.md b/docs/source/distributions/self_hosted_distro/cerebras.md index 8f14ae7cc..4b6673830 100644 --- a/docs/source/distributions/self_hosted_distro/cerebras.md +++ b/docs/source/distributions/self_hosted_distro/cerebras.md @@ -7,10 +7,8 @@ The `llamastack/distribution-cerebras` distribution consists of the following pr |-----|-------------| | agents | `inline::meta-reference` | | datasetio | `remote::huggingface`, `inline::localfs` | -| eval | `inline::meta-reference` | | inference | `remote::cerebras`, `inline::sentence-transformers` | | safety | `inline::llama-guard` | -| scoring | `inline::basic`, `inline::llm-as-judge`, `inline::braintrust` | | telemetry | `inline::meta-reference` | | tool_runtime | `remote::brave-search`, `remote::tavily-search`, `inline::code-interpreter`, `inline::rag-runtime` | | vector_io | `inline::faiss`, `remote::chromadb`, `remote::pgvector` | diff --git a/docs/source/distributions/self_hosted_distro/fireworks.md b/docs/source/distributions/self_hosted_distro/fireworks.md index 3c8f5eec9..1ab3db57d 100644 --- a/docs/source/distributions/self_hosted_distro/fireworks.md +++ b/docs/source/distributions/self_hosted_distro/fireworks.md @@ -17,10 +17,8 @@ The `llamastack/distribution-fireworks` distribution consists of the following p |-----|-------------| | agents | `inline::meta-reference` | | datasetio | `remote::huggingface`, `inline::localfs` | -| eval | `inline::meta-reference` | | inference | `remote::fireworks`, `inline::sentence-transformers` | | safety | `inline::llama-guard` | -| scoring | `inline::basic`, `inline::llm-as-judge`, `inline::braintrust` | | telemetry | `inline::meta-reference` | | tool_runtime | `remote::brave-search`, `remote::tavily-search`, `remote::wolfram-alpha`, `inline::code-interpreter`, `inline::rag-runtime`, `remote::model-context-protocol` | | vector_io | `inline::faiss`, `remote::chromadb`, `remote::pgvector` | diff --git a/docs/source/distributions/self_hosted_distro/groq.md b/docs/source/distributions/self_hosted_distro/groq.md index ce3f8aecc..cc25bc955 100644 --- a/docs/source/distributions/self_hosted_distro/groq.md +++ b/docs/source/distributions/self_hosted_distro/groq.md @@ -17,10 +17,8 @@ The `llamastack/distribution-groq` distribution consists of the following provid |-----|-------------| | agents | `inline::meta-reference` | | datasetio | `remote::huggingface`, `inline::localfs` | -| eval | `inline::meta-reference` | | inference | `remote::groq` | | safety | `inline::llama-guard` | -| scoring | `inline::basic`, `inline::llm-as-judge`, `inline::braintrust` | | telemetry | `inline::meta-reference` | | tool_runtime | `remote::brave-search`, `remote::tavily-search`, `inline::code-interpreter`, `inline::rag-runtime` | | vector_io | `inline::faiss` | diff --git a/docs/source/distributions/self_hosted_distro/meta-reference-gpu.md b/docs/source/distributions/self_hosted_distro/meta-reference-gpu.md index b8d1b1714..c5e5fccc2 100644 --- a/docs/source/distributions/self_hosted_distro/meta-reference-gpu.md +++ b/docs/source/distributions/self_hosted_distro/meta-reference-gpu.md @@ -17,10 +17,8 @@ The `llamastack/distribution-meta-reference-gpu` distribution consists of the fo |-----|-------------| | agents | `inline::meta-reference` | | datasetio | `remote::huggingface`, `inline::localfs` | -| eval | `inline::meta-reference` | | inference | `inline::meta-reference` | | safety | `inline::llama-guard` | -| scoring | `inline::basic`, `inline::llm-as-judge`, `inline::braintrust` | | telemetry | `inline::meta-reference` | | tool_runtime | `remote::brave-search`, `remote::tavily-search`, `inline::code-interpreter`, `inline::rag-runtime`, `remote::model-context-protocol` | | vector_io | `inline::faiss`, `remote::chromadb`, `remote::pgvector` | diff --git a/docs/source/distributions/self_hosted_distro/meta-reference-quantized-gpu.md b/docs/source/distributions/self_hosted_distro/meta-reference-quantized-gpu.md index a49175e22..9dcffb536 100644 --- a/docs/source/distributions/self_hosted_distro/meta-reference-quantized-gpu.md +++ b/docs/source/distributions/self_hosted_distro/meta-reference-quantized-gpu.md @@ -17,10 +17,8 @@ The `llamastack/distribution-meta-reference-quantized-gpu` distribution consists |-----|-------------| | agents | `inline::meta-reference` | | datasetio | `remote::huggingface`, `inline::localfs` | -| eval | `inline::meta-reference` | | inference | `inline::meta-reference-quantized` | | safety | `inline::llama-guard` | -| scoring | `inline::basic`, `inline::llm-as-judge`, `inline::braintrust` | | telemetry | `inline::meta-reference` | | tool_runtime | `remote::brave-search`, `remote::tavily-search`, `inline::code-interpreter`, `inline::rag-runtime`, `remote::model-context-protocol` | | vector_io | `inline::faiss`, `remote::chromadb`, `remote::pgvector` | diff --git a/docs/source/distributions/self_hosted_distro/ollama.md b/docs/source/distributions/self_hosted_distro/ollama.md index 9bfa4211c..f525c24aa 100644 --- a/docs/source/distributions/self_hosted_distro/ollama.md +++ b/docs/source/distributions/self_hosted_distro/ollama.md @@ -17,10 +17,8 @@ The `llamastack/distribution-ollama` distribution consists of the following prov |-----|-------------| | agents | `inline::meta-reference` | | datasetio | `remote::huggingface`, `inline::localfs` | -| eval | `inline::meta-reference` | | inference | `remote::ollama` | | safety | `inline::llama-guard` | -| scoring | `inline::basic`, `inline::llm-as-judge`, `inline::braintrust` | | telemetry | `inline::meta-reference` | | tool_runtime | `remote::brave-search`, `remote::tavily-search`, `inline::code-interpreter`, `inline::rag-runtime`, `remote::model-context-protocol`, `remote::wolfram-alpha` | | vector_io | `inline::faiss`, `remote::chromadb`, `remote::pgvector` | diff --git a/docs/source/distributions/self_hosted_distro/passthrough.md b/docs/source/distributions/self_hosted_distro/passthrough.md index 558d7ca08..b13f41cad 100644 --- a/docs/source/distributions/self_hosted_distro/passthrough.md +++ b/docs/source/distributions/self_hosted_distro/passthrough.md @@ -17,10 +17,8 @@ The `llamastack/distribution-passthrough` distribution consists of the following |-----|-------------| | agents | `inline::meta-reference` | | datasetio | `remote::huggingface`, `inline::localfs` | -| eval | `inline::meta-reference` | | inference | `remote::passthrough`, `inline::sentence-transformers` | | safety | `inline::llama-guard` | -| scoring | `inline::basic`, `inline::llm-as-judge`, `inline::braintrust` | | telemetry | `inline::meta-reference` | | tool_runtime | `remote::brave-search`, `remote::tavily-search`, `remote::wolfram-alpha`, `inline::code-interpreter`, `inline::rag-runtime`, `remote::model-context-protocol` | | vector_io | `inline::faiss`, `remote::chromadb`, `remote::pgvector` | diff --git a/docs/source/distributions/self_hosted_distro/remote-vllm.md b/docs/source/distributions/self_hosted_distro/remote-vllm.md index b7e155385..b33105cb2 100644 --- a/docs/source/distributions/self_hosted_distro/remote-vllm.md +++ b/docs/source/distributions/self_hosted_distro/remote-vllm.md @@ -16,10 +16,8 @@ The `llamastack/distribution-remote-vllm` distribution consists of the following |-----|-------------| | agents | `inline::meta-reference` | | datasetio | `remote::huggingface`, `inline::localfs` | -| eval | `inline::meta-reference` | | inference | `remote::vllm`, `inline::sentence-transformers` | | safety | `inline::llama-guard` | -| scoring | `inline::basic`, `inline::llm-as-judge`, `inline::braintrust` | | telemetry | `inline::meta-reference` | | tool_runtime | `remote::brave-search`, `remote::tavily-search`, `inline::code-interpreter`, `inline::rag-runtime`, `remote::model-context-protocol`, `remote::wolfram-alpha` | | vector_io | `inline::faiss`, `remote::chromadb`, `remote::pgvector` | diff --git a/docs/source/distributions/self_hosted_distro/tgi.md b/docs/source/distributions/self_hosted_distro/tgi.md index e126f9a08..6852f843c 100644 --- a/docs/source/distributions/self_hosted_distro/tgi.md +++ b/docs/source/distributions/self_hosted_distro/tgi.md @@ -18,10 +18,8 @@ The `llamastack/distribution-tgi` distribution consists of the following provide |-----|-------------| | agents | `inline::meta-reference` | | datasetio | `remote::huggingface`, `inline::localfs` | -| eval | `inline::meta-reference` | | inference | `remote::tgi`, `inline::sentence-transformers` | | safety | `inline::llama-guard` | -| scoring | `inline::basic`, `inline::llm-as-judge`, `inline::braintrust` | | telemetry | `inline::meta-reference` | | tool_runtime | `remote::brave-search`, `remote::tavily-search`, `inline::code-interpreter`, `inline::rag-runtime`, `remote::model-context-protocol` | | vector_io | `inline::faiss`, `remote::chromadb`, `remote::pgvector` | diff --git a/docs/source/distributions/self_hosted_distro/together.md b/docs/source/distributions/self_hosted_distro/together.md index fa02199b0..ebfda2e26 100644 --- a/docs/source/distributions/self_hosted_distro/together.md +++ b/docs/source/distributions/self_hosted_distro/together.md @@ -17,10 +17,8 @@ The `llamastack/distribution-together` distribution consists of the following pr |-----|-------------| | agents | `inline::meta-reference` | | datasetio | `remote::huggingface`, `inline::localfs` | -| eval | `inline::meta-reference` | | inference | `remote::together`, `inline::sentence-transformers` | | safety | `inline::llama-guard` | -| scoring | `inline::basic`, `inline::llm-as-judge`, `inline::braintrust` | | telemetry | `inline::meta-reference` | | tool_runtime | `remote::brave-search`, `remote::tavily-search`, `inline::code-interpreter`, `inline::rag-runtime`, `remote::model-context-protocol`, `remote::wolfram-alpha` | | vector_io | `inline::faiss`, `remote::chromadb`, `remote::pgvector` | diff --git a/llama_stack/distribution/distribution.py b/llama_stack/distribution/distribution.py index 233712c60..2aa45322e 100644 --- a/llama_stack/distribution/distribution.py +++ b/llama_stack/distribution/distribution.py @@ -53,7 +53,8 @@ def providable_apis() -> List[Api]: return [ api for api in Api - if api not in routing_table_apis and api != Api.inspect and api != Api.providers + if api not in routing_table_apis + and api not in [Api.inspect, Api.providers, Api.benchmarks] ] diff --git a/llama_stack/templates/bedrock/bedrock.py b/llama_stack/templates/bedrock/bedrock.py index 9171ae18a..61999c270 100644 --- a/llama_stack/templates/bedrock/bedrock.py +++ b/llama_stack/templates/bedrock/bedrock.py @@ -9,7 +9,11 @@ from pathlib import Path from llama_stack.distribution.datatypes import Provider, ToolGroupInput from llama_stack.providers.inline.vector_io.faiss.config import FaissVectorIOConfig from llama_stack.providers.remote.inference.bedrock.models import MODEL_ENTRIES -from llama_stack.templates.template import DistributionTemplate, RunConfigSettings, get_model_registry +from llama_stack.templates.template import ( + DistributionTemplate, + get_model_registry, + RunConfigSettings, +) def get_distribution_template() -> DistributionTemplate: @@ -19,9 +23,7 @@ def get_distribution_template() -> DistributionTemplate: "safety": ["remote::bedrock"], "agents": ["inline::meta-reference"], "telemetry": ["inline::meta-reference"], - "eval": ["inline::meta-reference"], "datasetio": ["remote::huggingface", "inline::localfs"], - "scoring": ["inline::basic", "inline::llm-as-judge", "inline::braintrust"], "tool_runtime": [ "remote::brave-search", "remote::tavily-search", diff --git a/llama_stack/templates/bedrock/build.yaml b/llama_stack/templates/bedrock/build.yaml index 6c07b0478..209cd8e34 100644 --- a/llama_stack/templates/bedrock/build.yaml +++ b/llama_stack/templates/bedrock/build.yaml @@ -14,15 +14,9 @@ distribution_spec: - inline::meta-reference telemetry: - inline::meta-reference - eval: - - inline::meta-reference datasetio: - remote::huggingface - inline::localfs - scoring: - - inline::basic - - inline::llm-as-judge - - inline::braintrust tool_runtime: - remote::brave-search - remote::tavily-search diff --git a/llama_stack/templates/bedrock/run.yaml b/llama_stack/templates/bedrock/run.yaml index 39ed8cf48..ae6357345 100644 --- a/llama_stack/templates/bedrock/run.yaml +++ b/llama_stack/templates/bedrock/run.yaml @@ -3,10 +3,8 @@ image_name: bedrock apis: - agents - datasetio -- eval - inference - safety -- scoring - telemetry - tool_runtime - vector_io @@ -42,14 +40,6 @@ providers: service_name: ${env.OTEL_SERVICE_NAME:llama-stack} sinks: ${env.TELEMETRY_SINKS:console,sqlite} sqlite_db_path: ${env.SQLITE_DB_PATH:~/.llama/distributions/bedrock/trace_store.db} - eval: - - provider_id: meta-reference - provider_type: inline::meta-reference - config: - kvstore: - type: sqlite - namespace: null - db_path: ${env.SQLITE_STORE_DIR:~/.llama/distributions/bedrock}/meta_reference_eval.db datasetio: - provider_id: huggingface provider_type: remote::huggingface @@ -65,17 +55,6 @@ providers: type: sqlite namespace: null db_path: ${env.SQLITE_STORE_DIR:~/.llama/distributions/bedrock}/localfs_datasetio.db - scoring: - - provider_id: basic - provider_type: inline::basic - config: {} - - provider_id: llm-as-judge - provider_type: inline::llm-as-judge - config: {} - - provider_id: braintrust - provider_type: inline::braintrust - config: - openai_api_key: ${env.OPENAI_API_KEY:} tool_runtime: - provider_id: brave-search provider_type: remote::brave-search @@ -133,7 +112,6 @@ models: shields: [] vector_dbs: [] datasets: [] -scoring_fns: [] benchmarks: [] tool_groups: - toolgroup_id: builtin::websearch diff --git a/llama_stack/templates/cerebras/build.yaml b/llama_stack/templates/cerebras/build.yaml index ef6c43212..5fe4a6bf0 100644 --- a/llama_stack/templates/cerebras/build.yaml +++ b/llama_stack/templates/cerebras/build.yaml @@ -13,15 +13,9 @@ distribution_spec: - remote::pgvector agents: - inline::meta-reference - eval: - - inline::meta-reference datasetio: - remote::huggingface - inline::localfs - scoring: - - inline::basic - - inline::llm-as-judge - - inline::braintrust telemetry: - inline::meta-reference tool_runtime: diff --git a/llama_stack/templates/cerebras/cerebras.py b/llama_stack/templates/cerebras/cerebras.py index 4a9ad90b4..7d3fe7ca2 100644 --- a/llama_stack/templates/cerebras/cerebras.py +++ b/llama_stack/templates/cerebras/cerebras.py @@ -14,7 +14,11 @@ from llama_stack.providers.inline.inference.sentence_transformers import ( from llama_stack.providers.inline.vector_io.faiss.config import FaissVectorIOConfig from llama_stack.providers.remote.inference.cerebras import CerebrasImplConfig from llama_stack.providers.remote.inference.cerebras.models import MODEL_ENTRIES -from llama_stack.templates.template import DistributionTemplate, RunConfigSettings, get_model_registry +from llama_stack.templates.template import ( + DistributionTemplate, + get_model_registry, + RunConfigSettings, +) def get_distribution_template() -> DistributionTemplate: @@ -23,9 +27,7 @@ def get_distribution_template() -> DistributionTemplate: "safety": ["inline::llama-guard"], "vector_io": ["inline::faiss", "remote::chromadb", "remote::pgvector"], "agents": ["inline::meta-reference"], - "eval": ["inline::meta-reference"], "datasetio": ["remote::huggingface", "inline::localfs"], - "scoring": ["inline::basic", "inline::llm-as-judge", "inline::braintrust"], "telemetry": ["inline::meta-reference"], "tool_runtime": [ "remote::brave-search", diff --git a/llama_stack/templates/cerebras/run.yaml b/llama_stack/templates/cerebras/run.yaml index 8315f75d5..fb3d7ec9b 100644 --- a/llama_stack/templates/cerebras/run.yaml +++ b/llama_stack/templates/cerebras/run.yaml @@ -3,10 +3,8 @@ image_name: cerebras apis: - agents - datasetio -- eval - inference - safety -- scoring - telemetry - tool_runtime - vector_io @@ -41,14 +39,6 @@ providers: type: sqlite namespace: null db_path: ${env.SQLITE_STORE_DIR:~/.llama/distributions/cerebras}/agents_store.db - eval: - - provider_id: meta-reference - provider_type: inline::meta-reference - config: - kvstore: - type: sqlite - namespace: null - db_path: ${env.SQLITE_STORE_DIR:~/.llama/distributions/cerebras}/meta_reference_eval.db datasetio: - provider_id: huggingface provider_type: remote::huggingface @@ -64,17 +54,6 @@ providers: type: sqlite namespace: null db_path: ${env.SQLITE_STORE_DIR:~/.llama/distributions/cerebras}/localfs_datasetio.db - scoring: - - provider_id: basic - provider_type: inline::basic - config: {} - - provider_id: llm-as-judge - provider_type: inline::llm-as-judge - config: {} - - provider_id: braintrust - provider_type: inline::braintrust - config: - openai_api_key: ${env.OPENAI_API_KEY:} telemetry: - provider_id: meta-reference provider_type: inline::meta-reference @@ -131,7 +110,6 @@ models: shields: [] vector_dbs: [] datasets: [] -scoring_fns: [] benchmarks: [] tool_groups: - toolgroup_id: builtin::websearch diff --git a/llama_stack/templates/ci-tests/build.yaml b/llama_stack/templates/ci-tests/build.yaml index a5c615f2f..3c6ff6924 100644 --- a/llama_stack/templates/ci-tests/build.yaml +++ b/llama_stack/templates/ci-tests/build.yaml @@ -15,15 +15,9 @@ distribution_spec: - inline::meta-reference telemetry: - inline::meta-reference - eval: - - inline::meta-reference datasetio: - remote::huggingface - inline::localfs - scoring: - - inline::basic - - inline::llm-as-judge - - inline::braintrust tool_runtime: - remote::brave-search - remote::tavily-search diff --git a/llama_stack/templates/ci-tests/ci_tests.py b/llama_stack/templates/ci-tests/ci_tests.py index b204af5ea..85523ef06 100644 --- a/llama_stack/templates/ci-tests/ci_tests.py +++ b/llama_stack/templates/ci-tests/ci_tests.py @@ -15,10 +15,16 @@ from llama_stack.distribution.datatypes import ( from llama_stack.providers.inline.inference.sentence_transformers import ( SentenceTransformersInferenceConfig, ) -from llama_stack.providers.inline.vector_io.sqlite_vec.config import SQLiteVectorIOConfig +from llama_stack.providers.inline.vector_io.sqlite_vec.config import ( + SQLiteVectorIOConfig, +) from llama_stack.providers.remote.inference.fireworks.config import FireworksImplConfig from llama_stack.providers.remote.inference.fireworks.models import MODEL_ENTRIES -from llama_stack.templates.template import DistributionTemplate, RunConfigSettings, get_model_registry +from llama_stack.templates.template import ( + DistributionTemplate, + get_model_registry, + RunConfigSettings, +) def get_distribution_template() -> DistributionTemplate: @@ -28,9 +34,7 @@ def get_distribution_template() -> DistributionTemplate: "safety": ["inline::llama-guard"], "agents": ["inline::meta-reference"], "telemetry": ["inline::meta-reference"], - "eval": ["inline::meta-reference"], "datasetio": ["remote::huggingface", "inline::localfs"], - "scoring": ["inline::basic", "inline::llm-as-judge", "inline::braintrust"], "tool_runtime": [ "remote::brave-search", "remote::tavily-search", diff --git a/llama_stack/templates/ci-tests/run.yaml b/llama_stack/templates/ci-tests/run.yaml index ae2b3912c..6b351ac9c 100644 --- a/llama_stack/templates/ci-tests/run.yaml +++ b/llama_stack/templates/ci-tests/run.yaml @@ -3,10 +3,8 @@ image_name: ci-tests apis: - agents - datasetio -- eval - inference - safety -- scoring - telemetry - tool_runtime - vector_io @@ -45,14 +43,6 @@ providers: service_name: ${env.OTEL_SERVICE_NAME:llama-stack} sinks: ${env.TELEMETRY_SINKS:console,sqlite} sqlite_db_path: ${env.SQLITE_DB_PATH:~/.llama/distributions/ci-tests/trace_store.db} - eval: - - provider_id: meta-reference - provider_type: inline::meta-reference - config: - kvstore: - type: sqlite - namespace: null - db_path: ${env.SQLITE_STORE_DIR:~/.llama/distributions/ci-tests}/meta_reference_eval.db datasetio: - provider_id: huggingface provider_type: remote::huggingface @@ -68,17 +58,6 @@ providers: type: sqlite namespace: null db_path: ${env.SQLITE_STORE_DIR:~/.llama/distributions/ci-tests}/localfs_datasetio.db - scoring: - - provider_id: basic - provider_type: inline::basic - config: {} - - provider_id: llm-as-judge - provider_type: inline::llm-as-judge - config: {} - - provider_id: braintrust - provider_type: inline::braintrust - config: - openai_api_key: ${env.OPENAI_API_KEY:} tool_runtime: - provider_id: brave-search provider_type: remote::brave-search @@ -209,7 +188,6 @@ shields: - shield_id: meta-llama/Llama-Guard-3-8B vector_dbs: [] datasets: [] -scoring_fns: [] benchmarks: [] tool_groups: - toolgroup_id: builtin::websearch diff --git a/llama_stack/templates/dell/build.yaml b/llama_stack/templates/dell/build.yaml index 05b98d56f..12183da9e 100644 --- a/llama_stack/templates/dell/build.yaml +++ b/llama_stack/templates/dell/build.yaml @@ -16,15 +16,9 @@ distribution_spec: - inline::meta-reference telemetry: - inline::meta-reference - eval: - - inline::meta-reference datasetio: - remote::huggingface - inline::localfs - scoring: - - inline::basic - - inline::llm-as-judge - - inline::braintrust tool_runtime: - remote::brave-search - remote::tavily-search diff --git a/llama_stack/templates/dell/dell.py b/llama_stack/templates/dell/dell.py index 52c5a5476..161a611ae 100644 --- a/llama_stack/templates/dell/dell.py +++ b/llama_stack/templates/dell/dell.py @@ -24,9 +24,7 @@ def get_distribution_template() -> DistributionTemplate: "safety": ["inline::llama-guard"], "agents": ["inline::meta-reference"], "telemetry": ["inline::meta-reference"], - "eval": ["inline::meta-reference"], "datasetio": ["remote::huggingface", "inline::localfs"], - "scoring": ["inline::basic", "inline::llm-as-judge", "inline::braintrust"], "tool_runtime": [ "remote::brave-search", "remote::tavily-search", diff --git a/llama_stack/templates/dell/run-with-safety.yaml b/llama_stack/templates/dell/run-with-safety.yaml index 8a62a5a42..ff074659b 100644 --- a/llama_stack/templates/dell/run-with-safety.yaml +++ b/llama_stack/templates/dell/run-with-safety.yaml @@ -3,10 +3,8 @@ image_name: dell apis: - agents - datasetio -- eval - inference - safety -- scoring - telemetry - tool_runtime - vector_io @@ -48,14 +46,6 @@ providers: service_name: ${env.OTEL_SERVICE_NAME:llama-stack} sinks: ${env.TELEMETRY_SINKS:console,sqlite} sqlite_db_path: ${env.SQLITE_DB_PATH:~/.llama/distributions/dell/trace_store.db} - eval: - - provider_id: meta-reference - provider_type: inline::meta-reference - config: - kvstore: - type: sqlite - namespace: null - db_path: ${env.SQLITE_STORE_DIR:~/.llama/distributions/dell}/meta_reference_eval.db datasetio: - provider_id: huggingface provider_type: remote::huggingface @@ -71,17 +61,6 @@ providers: type: sqlite namespace: null db_path: ${env.SQLITE_STORE_DIR:~/.llama/distributions/dell}/localfs_datasetio.db - scoring: - - provider_id: basic - provider_type: inline::basic - config: {} - - provider_id: llm-as-judge - provider_type: inline::llm-as-judge - config: {} - - provider_id: braintrust - provider_type: inline::braintrust - config: - openai_api_key: ${env.OPENAI_API_KEY:} tool_runtime: - provider_id: brave-search provider_type: remote::brave-search @@ -120,7 +99,6 @@ shields: - shield_id: ${env.SAFETY_MODEL} vector_dbs: [] datasets: [] -scoring_fns: [] benchmarks: [] tool_groups: - toolgroup_id: builtin::websearch diff --git a/llama_stack/templates/dell/run.yaml b/llama_stack/templates/dell/run.yaml index 31c63bd83..762769349 100644 --- a/llama_stack/templates/dell/run.yaml +++ b/llama_stack/templates/dell/run.yaml @@ -3,10 +3,8 @@ image_name: dell apis: - agents - datasetio -- eval - inference - safety -- scoring - telemetry - tool_runtime - vector_io @@ -44,14 +42,6 @@ providers: service_name: ${env.OTEL_SERVICE_NAME:llama-stack} sinks: ${env.TELEMETRY_SINKS:console,sqlite} sqlite_db_path: ${env.SQLITE_DB_PATH:~/.llama/distributions/dell/trace_store.db} - eval: - - provider_id: meta-reference - provider_type: inline::meta-reference - config: - kvstore: - type: sqlite - namespace: null - db_path: ${env.SQLITE_STORE_DIR:~/.llama/distributions/dell}/meta_reference_eval.db datasetio: - provider_id: huggingface provider_type: remote::huggingface @@ -67,17 +57,6 @@ providers: type: sqlite namespace: null db_path: ${env.SQLITE_STORE_DIR:~/.llama/distributions/dell}/localfs_datasetio.db - scoring: - - provider_id: basic - provider_type: inline::basic - config: {} - - provider_id: llm-as-judge - provider_type: inline::llm-as-judge - config: {} - - provider_id: braintrust - provider_type: inline::braintrust - config: - openai_api_key: ${env.OPENAI_API_KEY:} tool_runtime: - provider_id: brave-search provider_type: remote::brave-search @@ -111,7 +90,6 @@ models: shields: [] vector_dbs: [] datasets: [] -scoring_fns: [] benchmarks: [] tool_groups: - toolgroup_id: builtin::websearch diff --git a/llama_stack/templates/dev/build.yaml b/llama_stack/templates/dev/build.yaml index 726ebccca..c98972dac 100644 --- a/llama_stack/templates/dev/build.yaml +++ b/llama_stack/templates/dev/build.yaml @@ -19,15 +19,9 @@ distribution_spec: - inline::meta-reference telemetry: - inline::meta-reference - eval: - - inline::meta-reference datasetio: - remote::huggingface - inline::localfs - scoring: - - inline::basic - - inline::llm-as-judge - - inline::braintrust tool_runtime: - remote::brave-search - remote::tavily-search diff --git a/llama_stack/templates/dev/dev.py b/llama_stack/templates/dev/dev.py index 1aee1bb22..dad8b6a8e 100644 --- a/llama_stack/templates/dev/dev.py +++ b/llama_stack/templates/dev/dev.py @@ -16,20 +16,38 @@ from llama_stack.distribution.datatypes import ( from llama_stack.providers.inline.inference.sentence_transformers import ( SentenceTransformersInferenceConfig, ) -from llama_stack.providers.inline.vector_io.sqlite_vec.config import SQLiteVectorIOConfig +from llama_stack.providers.inline.vector_io.sqlite_vec.config import ( + SQLiteVectorIOConfig, +) from llama_stack.providers.remote.inference.anthropic.config import AnthropicConfig -from llama_stack.providers.remote.inference.anthropic.models import MODEL_ENTRIES as ANTHROPIC_MODEL_ENTRIES +from llama_stack.providers.remote.inference.anthropic.models import ( + MODEL_ENTRIES as ANTHROPIC_MODEL_ENTRIES, +) from llama_stack.providers.remote.inference.fireworks.config import FireworksImplConfig -from llama_stack.providers.remote.inference.fireworks.models import MODEL_ENTRIES as FIREWORKS_MODEL_ENTRIES +from llama_stack.providers.remote.inference.fireworks.models import ( + MODEL_ENTRIES as FIREWORKS_MODEL_ENTRIES, +) from llama_stack.providers.remote.inference.gemini.config import GeminiConfig -from llama_stack.providers.remote.inference.gemini.models import MODEL_ENTRIES as GEMINI_MODEL_ENTRIES +from llama_stack.providers.remote.inference.gemini.models import ( + MODEL_ENTRIES as GEMINI_MODEL_ENTRIES, +) from llama_stack.providers.remote.inference.groq.config import GroqConfig -from llama_stack.providers.remote.inference.groq.models import MODEL_ENTRIES as GROQ_MODEL_ENTRIES +from llama_stack.providers.remote.inference.groq.models import ( + MODEL_ENTRIES as GROQ_MODEL_ENTRIES, +) from llama_stack.providers.remote.inference.openai.config import OpenAIConfig -from llama_stack.providers.remote.inference.openai.models import MODEL_ENTRIES as OPENAI_MODEL_ENTRIES +from llama_stack.providers.remote.inference.openai.models import ( + MODEL_ENTRIES as OPENAI_MODEL_ENTRIES, +) from llama_stack.providers.remote.vector_io.chroma.config import ChromaVectorIOConfig -from llama_stack.providers.remote.vector_io.pgvector.config import PGVectorVectorIOConfig -from llama_stack.templates.template import DistributionTemplate, RunConfigSettings, get_model_registry +from llama_stack.providers.remote.vector_io.pgvector.config import ( + PGVectorVectorIOConfig, +) +from llama_stack.templates.template import ( + DistributionTemplate, + get_model_registry, + RunConfigSettings, +) def get_inference_providers() -> Tuple[List[Provider], List[ModelInput]]: @@ -78,14 +96,15 @@ def get_inference_providers() -> Tuple[List[Provider], List[ModelInput]]: def get_distribution_template() -> DistributionTemplate: inference_providers, available_models = get_inference_providers() providers = { - "inference": ([p.provider_type for p in inference_providers] + ["inline::sentence-transformers"]), + "inference": ( + [p.provider_type for p in inference_providers] + + ["inline::sentence-transformers"] + ), "vector_io": ["inline::sqlite-vec", "remote::chromadb", "remote::pgvector"], "safety": ["inline::llama-guard"], "agents": ["inline::meta-reference"], "telemetry": ["inline::meta-reference"], - "eval": ["inline::meta-reference"], "datasetio": ["remote::huggingface", "inline::localfs"], - "scoring": ["inline::basic", "inline::llm-as-judge", "inline::braintrust"], "tool_runtime": [ "remote::brave-search", "remote::tavily-search", @@ -100,7 +119,9 @@ def get_distribution_template() -> DistributionTemplate: Provider( provider_id="sqlite-vec", provider_type="inline::sqlite-vec", - config=SQLiteVectorIOConfig.sample_run_config(f"~/.llama/distributions/{name}"), + config=SQLiteVectorIOConfig.sample_run_config( + f"~/.llama/distributions/{name}" + ), ), Provider( provider_id="${env.ENABLE_CHROMADB+chromadb}", diff --git a/llama_stack/templates/dev/run.yaml b/llama_stack/templates/dev/run.yaml index dba13b357..dbffbf215 100644 --- a/llama_stack/templates/dev/run.yaml +++ b/llama_stack/templates/dev/run.yaml @@ -3,10 +3,8 @@ image_name: dev apis: - agents - datasetio -- eval - inference - safety -- scoring - telemetry - tool_runtime - vector_io @@ -74,14 +72,6 @@ providers: service_name: ${env.OTEL_SERVICE_NAME:llama-stack} sinks: ${env.TELEMETRY_SINKS:console,sqlite} sqlite_db_path: ${env.SQLITE_DB_PATH:~/.llama/distributions/dev/trace_store.db} - eval: - - provider_id: meta-reference - provider_type: inline::meta-reference - config: - kvstore: - type: sqlite - namespace: null - db_path: ${env.SQLITE_STORE_DIR:~/.llama/distributions/dev}/meta_reference_eval.db datasetio: - provider_id: huggingface provider_type: remote::huggingface @@ -97,17 +87,6 @@ providers: type: sqlite namespace: null db_path: ${env.SQLITE_STORE_DIR:~/.llama/distributions/dev}/localfs_datasetio.db - scoring: - - provider_id: basic - provider_type: inline::basic - config: {} - - provider_id: llm-as-judge - provider_type: inline::llm-as-judge - config: {} - - provider_id: braintrust - provider_type: inline::braintrust - config: - openai_api_key: ${env.OPENAI_API_KEY:} tool_runtime: - provider_id: brave-search provider_type: remote::brave-search @@ -365,7 +344,6 @@ shields: - shield_id: meta-llama/Llama-Guard-3-8B vector_dbs: [] datasets: [] -scoring_fns: [] benchmarks: [] tool_groups: - toolgroup_id: builtin::websearch diff --git a/llama_stack/templates/fireworks/build.yaml b/llama_stack/templates/fireworks/build.yaml index 3907eba78..c5904a7e3 100644 --- a/llama_stack/templates/fireworks/build.yaml +++ b/llama_stack/templates/fireworks/build.yaml @@ -15,15 +15,9 @@ distribution_spec: - inline::meta-reference telemetry: - inline::meta-reference - eval: - - inline::meta-reference datasetio: - remote::huggingface - inline::localfs - scoring: - - inline::basic - - inline::llm-as-judge - - inline::braintrust tool_runtime: - remote::brave-search - remote::tavily-search diff --git a/llama_stack/templates/fireworks/fireworks.py b/llama_stack/templates/fireworks/fireworks.py index 3e6d1ca89..6ea73d3b8 100644 --- a/llama_stack/templates/fireworks/fireworks.py +++ b/llama_stack/templates/fireworks/fireworks.py @@ -19,7 +19,11 @@ from llama_stack.providers.inline.inference.sentence_transformers import ( from llama_stack.providers.inline.vector_io.faiss.config import FaissVectorIOConfig from llama_stack.providers.remote.inference.fireworks.config import FireworksImplConfig from llama_stack.providers.remote.inference.fireworks.models import MODEL_ENTRIES -from llama_stack.templates.template import DistributionTemplate, RunConfigSettings, get_model_registry +from llama_stack.templates.template import ( + DistributionTemplate, + get_model_registry, + RunConfigSettings, +) def get_distribution_template() -> DistributionTemplate: @@ -29,9 +33,7 @@ def get_distribution_template() -> DistributionTemplate: "safety": ["inline::llama-guard"], "agents": ["inline::meta-reference"], "telemetry": ["inline::meta-reference"], - "eval": ["inline::meta-reference"], "datasetio": ["remote::huggingface", "inline::localfs"], - "scoring": ["inline::basic", "inline::llm-as-judge", "inline::braintrust"], "tool_runtime": [ "remote::brave-search", "remote::tavily-search", diff --git a/llama_stack/templates/fireworks/run-with-safety.yaml b/llama_stack/templates/fireworks/run-with-safety.yaml index 2d79a3548..b89323dbc 100644 --- a/llama_stack/templates/fireworks/run-with-safety.yaml +++ b/llama_stack/templates/fireworks/run-with-safety.yaml @@ -3,10 +3,8 @@ image_name: fireworks apis: - agents - datasetio -- eval - inference - safety -- scoring - telemetry - tool_runtime - vector_io @@ -53,14 +51,6 @@ providers: service_name: ${env.OTEL_SERVICE_NAME:llama-stack} sinks: ${env.TELEMETRY_SINKS:console,sqlite} sqlite_db_path: ${env.SQLITE_DB_PATH:~/.llama/distributions/fireworks/trace_store.db} - eval: - - provider_id: meta-reference - provider_type: inline::meta-reference - config: - kvstore: - type: sqlite - namespace: null - db_path: ${env.SQLITE_STORE_DIR:~/.llama/distributions/fireworks}/meta_reference_eval.db datasetio: - provider_id: huggingface provider_type: remote::huggingface @@ -76,17 +66,6 @@ providers: type: sqlite namespace: null db_path: ${env.SQLITE_STORE_DIR:~/.llama/distributions/fireworks}/localfs_datasetio.db - scoring: - - provider_id: basic - provider_type: inline::basic - config: {} - - provider_id: llm-as-judge - provider_type: inline::llm-as-judge - config: {} - - provider_id: braintrust - provider_type: inline::braintrust - config: - openai_api_key: ${env.OPENAI_API_KEY:} tool_runtime: - provider_id: brave-search provider_type: remote::brave-search @@ -226,7 +205,6 @@ shields: provider_id: code-scanner vector_dbs: [] datasets: [] -scoring_fns: [] benchmarks: [] tool_groups: - toolgroup_id: builtin::websearch diff --git a/llama_stack/templates/fireworks/run.yaml b/llama_stack/templates/fireworks/run.yaml index 285495ad9..38f3bb67f 100644 --- a/llama_stack/templates/fireworks/run.yaml +++ b/llama_stack/templates/fireworks/run.yaml @@ -3,10 +3,8 @@ image_name: fireworks apis: - agents - datasetio -- eval - inference - safety -- scoring - telemetry - tool_runtime - vector_io @@ -48,14 +46,6 @@ providers: service_name: ${env.OTEL_SERVICE_NAME:llama-stack} sinks: ${env.TELEMETRY_SINKS:console,sqlite} sqlite_db_path: ${env.SQLITE_DB_PATH:~/.llama/distributions/fireworks/trace_store.db} - eval: - - provider_id: meta-reference - provider_type: inline::meta-reference - config: - kvstore: - type: sqlite - namespace: null - db_path: ${env.SQLITE_STORE_DIR:~/.llama/distributions/fireworks}/meta_reference_eval.db datasetio: - provider_id: huggingface provider_type: remote::huggingface @@ -71,17 +61,6 @@ providers: type: sqlite namespace: null db_path: ${env.SQLITE_STORE_DIR:~/.llama/distributions/fireworks}/localfs_datasetio.db - scoring: - - provider_id: basic - provider_type: inline::basic - config: {} - - provider_id: llm-as-judge - provider_type: inline::llm-as-judge - config: {} - - provider_id: braintrust - provider_type: inline::braintrust - config: - openai_api_key: ${env.OPENAI_API_KEY:} tool_runtime: - provider_id: brave-search provider_type: remote::brave-search @@ -216,7 +195,6 @@ shields: - shield_id: meta-llama/Llama-Guard-3-8B vector_dbs: [] datasets: [] -scoring_fns: [] benchmarks: [] tool_groups: - toolgroup_id: builtin::websearch diff --git a/llama_stack/templates/groq/build.yaml b/llama_stack/templates/groq/build.yaml index 3263ce83b..6a92d0b01 100644 --- a/llama_stack/templates/groq/build.yaml +++ b/llama_stack/templates/groq/build.yaml @@ -12,15 +12,9 @@ distribution_spec: - inline::meta-reference telemetry: - inline::meta-reference - eval: - - inline::meta-reference datasetio: - remote::huggingface - inline::localfs - scoring: - - inline::basic - - inline::llm-as-judge - - inline::braintrust tool_runtime: - remote::brave-search - remote::tavily-search diff --git a/llama_stack/templates/groq/groq.py b/llama_stack/templates/groq/groq.py index 71c504cde..bde870c55 100644 --- a/llama_stack/templates/groq/groq.py +++ b/llama_stack/templates/groq/groq.py @@ -7,17 +7,17 @@ from pathlib import Path from llama_stack.apis.models.models import ModelType -from llama_stack.distribution.datatypes import ( - ModelInput, - Provider, - ToolGroupInput, -) +from llama_stack.distribution.datatypes import ModelInput, Provider, ToolGroupInput from llama_stack.providers.inline.inference.sentence_transformers import ( SentenceTransformersInferenceConfig, ) from llama_stack.providers.remote.inference.groq import GroqConfig from llama_stack.providers.remote.inference.groq.models import MODEL_ENTRIES -from llama_stack.templates.template import DistributionTemplate, RunConfigSettings, get_model_registry +from llama_stack.templates.template import ( + DistributionTemplate, + get_model_registry, + RunConfigSettings, +) def get_distribution_template() -> DistributionTemplate: @@ -27,9 +27,7 @@ def get_distribution_template() -> DistributionTemplate: "safety": ["inline::llama-guard"], "agents": ["inline::meta-reference"], "telemetry": ["inline::meta-reference"], - "eval": ["inline::meta-reference"], "datasetio": ["remote::huggingface", "inline::localfs"], - "scoring": ["inline::basic", "inline::llm-as-judge", "inline::braintrust"], "tool_runtime": [ "remote::brave-search", "remote::tavily-search", diff --git a/llama_stack/templates/groq/run.yaml b/llama_stack/templates/groq/run.yaml index 6afea2355..441e2bda0 100644 --- a/llama_stack/templates/groq/run.yaml +++ b/llama_stack/templates/groq/run.yaml @@ -3,10 +3,8 @@ image_name: groq apis: - agents - datasetio -- eval - inference - safety -- scoring - telemetry - tool_runtime - vector_io @@ -48,14 +46,6 @@ providers: service_name: ${env.OTEL_SERVICE_NAME:llama-stack} sinks: ${env.TELEMETRY_SINKS:console,sqlite} sqlite_db_path: ${env.SQLITE_DB_PATH:~/.llama/distributions/groq/trace_store.db} - eval: - - provider_id: meta-reference - provider_type: inline::meta-reference - config: - kvstore: - type: sqlite - namespace: null - db_path: ${env.SQLITE_STORE_DIR:~/.llama/distributions/groq}/meta_reference_eval.db datasetio: - provider_id: huggingface provider_type: remote::huggingface @@ -71,17 +61,6 @@ providers: type: sqlite namespace: null db_path: ${env.SQLITE_STORE_DIR:~/.llama/distributions/groq}/localfs_datasetio.db - scoring: - - provider_id: basic - provider_type: inline::basic - config: {} - - provider_id: llm-as-judge - provider_type: inline::llm-as-judge - config: {} - - provider_id: braintrust - provider_type: inline::braintrust - config: - openai_api_key: ${env.OPENAI_API_KEY:} tool_runtime: - provider_id: brave-search provider_type: remote::brave-search @@ -156,7 +135,6 @@ models: shields: [] vector_dbs: [] datasets: [] -scoring_fns: [] benchmarks: [] tool_groups: - toolgroup_id: builtin::websearch diff --git a/llama_stack/templates/hf-endpoint/build.yaml b/llama_stack/templates/hf-endpoint/build.yaml index c2eaaa05b..0b6c072aa 100644 --- a/llama_stack/templates/hf-endpoint/build.yaml +++ b/llama_stack/templates/hf-endpoint/build.yaml @@ -14,15 +14,9 @@ distribution_spec: - inline::meta-reference telemetry: - inline::meta-reference - eval: - - inline::meta-reference datasetio: - remote::huggingface - inline::localfs - scoring: - - inline::basic - - inline::llm-as-judge - - inline::braintrust tool_runtime: - remote::brave-search - remote::tavily-search diff --git a/llama_stack/templates/hf-endpoint/hf_endpoint.py b/llama_stack/templates/hf-endpoint/hf_endpoint.py index 0dafe0a01..8fd31cb1d 100644 --- a/llama_stack/templates/hf-endpoint/hf_endpoint.py +++ b/llama_stack/templates/hf-endpoint/hf_endpoint.py @@ -26,9 +26,7 @@ def get_distribution_template() -> DistributionTemplate: "safety": ["inline::llama-guard"], "agents": ["inline::meta-reference"], "telemetry": ["inline::meta-reference"], - "eval": ["inline::meta-reference"], "datasetio": ["remote::huggingface", "inline::localfs"], - "scoring": ["inline::basic", "inline::llm-as-judge", "inline::braintrust"], "tool_runtime": [ "remote::brave-search", "remote::tavily-search", diff --git a/llama_stack/templates/hf-endpoint/run-with-safety.yaml b/llama_stack/templates/hf-endpoint/run-with-safety.yaml index f6f23a987..ce15f76f9 100644 --- a/llama_stack/templates/hf-endpoint/run-with-safety.yaml +++ b/llama_stack/templates/hf-endpoint/run-with-safety.yaml @@ -3,10 +3,8 @@ image_name: hf-endpoint apis: - agents - datasetio -- eval - inference - safety -- scoring - telemetry - tool_runtime - vector_io @@ -53,14 +51,6 @@ providers: service_name: ${env.OTEL_SERVICE_NAME:llama-stack} sinks: ${env.TELEMETRY_SINKS:console,sqlite} sqlite_db_path: ${env.SQLITE_DB_PATH:~/.llama/distributions/hf-endpoint/trace_store.db} - eval: - - provider_id: meta-reference - provider_type: inline::meta-reference - config: - kvstore: - type: sqlite - namespace: null - db_path: ${env.SQLITE_STORE_DIR:~/.llama/distributions/hf-endpoint}/meta_reference_eval.db datasetio: - provider_id: huggingface provider_type: remote::huggingface @@ -76,17 +66,6 @@ providers: type: sqlite namespace: null db_path: ${env.SQLITE_STORE_DIR:~/.llama/distributions/hf-endpoint}/localfs_datasetio.db - scoring: - - provider_id: basic - provider_type: inline::basic - config: {} - - provider_id: llm-as-judge - provider_type: inline::llm-as-judge - config: {} - - provider_id: braintrust - provider_type: inline::braintrust - config: - openai_api_key: ${env.OPENAI_API_KEY:} tool_runtime: - provider_id: brave-search provider_type: remote::brave-search @@ -128,7 +107,6 @@ shields: - shield_id: ${env.SAFETY_MODEL} vector_dbs: [] datasets: [] -scoring_fns: [] benchmarks: [] tool_groups: - toolgroup_id: builtin::websearch diff --git a/llama_stack/templates/hf-endpoint/run.yaml b/llama_stack/templates/hf-endpoint/run.yaml index 461f97128..5cd91e64a 100644 --- a/llama_stack/templates/hf-endpoint/run.yaml +++ b/llama_stack/templates/hf-endpoint/run.yaml @@ -3,10 +3,8 @@ image_name: hf-endpoint apis: - agents - datasetio -- eval - inference - safety -- scoring - telemetry - tool_runtime - vector_io @@ -48,14 +46,6 @@ providers: service_name: ${env.OTEL_SERVICE_NAME:llama-stack} sinks: ${env.TELEMETRY_SINKS:console,sqlite} sqlite_db_path: ${env.SQLITE_DB_PATH:~/.llama/distributions/hf-endpoint/trace_store.db} - eval: - - provider_id: meta-reference - provider_type: inline::meta-reference - config: - kvstore: - type: sqlite - namespace: null - db_path: ${env.SQLITE_STORE_DIR:~/.llama/distributions/hf-endpoint}/meta_reference_eval.db datasetio: - provider_id: huggingface provider_type: remote::huggingface @@ -71,17 +61,6 @@ providers: type: sqlite namespace: null db_path: ${env.SQLITE_STORE_DIR:~/.llama/distributions/hf-endpoint}/localfs_datasetio.db - scoring: - - provider_id: basic - provider_type: inline::basic - config: {} - - provider_id: llm-as-judge - provider_type: inline::llm-as-judge - config: {} - - provider_id: braintrust - provider_type: inline::braintrust - config: - openai_api_key: ${env.OPENAI_API_KEY:} tool_runtime: - provider_id: brave-search provider_type: remote::brave-search @@ -118,7 +97,6 @@ models: shields: [] vector_dbs: [] datasets: [] -scoring_fns: [] benchmarks: [] tool_groups: - toolgroup_id: builtin::websearch diff --git a/llama_stack/templates/hf-serverless/build.yaml b/llama_stack/templates/hf-serverless/build.yaml index c0cc1e2c2..2fff4a7d3 100644 --- a/llama_stack/templates/hf-serverless/build.yaml +++ b/llama_stack/templates/hf-serverless/build.yaml @@ -15,15 +15,9 @@ distribution_spec: - inline::meta-reference telemetry: - inline::meta-reference - eval: - - inline::meta-reference datasetio: - remote::huggingface - inline::localfs - scoring: - - inline::basic - - inline::llm-as-judge - - inline::braintrust tool_runtime: - remote::brave-search - remote::tavily-search diff --git a/llama_stack/templates/hf-serverless/hf_serverless.py b/llama_stack/templates/hf-serverless/hf_serverless.py index 25d4c6b30..e1537a681 100644 --- a/llama_stack/templates/hf-serverless/hf_serverless.py +++ b/llama_stack/templates/hf-serverless/hf_serverless.py @@ -26,9 +26,7 @@ def get_distribution_template() -> DistributionTemplate: "safety": ["inline::llama-guard"], "agents": ["inline::meta-reference"], "telemetry": ["inline::meta-reference"], - "eval": ["inline::meta-reference"], "datasetio": ["remote::huggingface", "inline::localfs"], - "scoring": ["inline::basic", "inline::llm-as-judge", "inline::braintrust"], "tool_runtime": [ "remote::brave-search", "remote::tavily-search", diff --git a/llama_stack/templates/hf-serverless/run-with-safety.yaml b/llama_stack/templates/hf-serverless/run-with-safety.yaml index 7f1724f34..c6bfa291b 100644 --- a/llama_stack/templates/hf-serverless/run-with-safety.yaml +++ b/llama_stack/templates/hf-serverless/run-with-safety.yaml @@ -3,10 +3,8 @@ image_name: hf-serverless apis: - agents - datasetio -- eval - inference - safety -- scoring - telemetry - tool_runtime - vector_io @@ -53,14 +51,6 @@ providers: service_name: ${env.OTEL_SERVICE_NAME:llama-stack} sinks: ${env.TELEMETRY_SINKS:console,sqlite} sqlite_db_path: ${env.SQLITE_DB_PATH:~/.llama/distributions/hf-serverless/trace_store.db} - eval: - - provider_id: meta-reference - provider_type: inline::meta-reference - config: - kvstore: - type: sqlite - namespace: null - db_path: ${env.SQLITE_STORE_DIR:~/.llama/distributions/hf-serverless}/meta_reference_eval.db datasetio: - provider_id: huggingface provider_type: remote::huggingface @@ -76,17 +66,6 @@ providers: type: sqlite namespace: null db_path: ${env.SQLITE_STORE_DIR:~/.llama/distributions/hf-serverless}/localfs_datasetio.db - scoring: - - provider_id: basic - provider_type: inline::basic - config: {} - - provider_id: llm-as-judge - provider_type: inline::llm-as-judge - config: {} - - provider_id: braintrust - provider_type: inline::braintrust - config: - openai_api_key: ${env.OPENAI_API_KEY:} tool_runtime: - provider_id: brave-search provider_type: remote::brave-search @@ -128,7 +107,6 @@ shields: - shield_id: ${env.SAFETY_MODEL} vector_dbs: [] datasets: [] -scoring_fns: [] benchmarks: [] tool_groups: - toolgroup_id: builtin::websearch diff --git a/llama_stack/templates/hf-serverless/run.yaml b/llama_stack/templates/hf-serverless/run.yaml index ac013488b..799c9845b 100644 --- a/llama_stack/templates/hf-serverless/run.yaml +++ b/llama_stack/templates/hf-serverless/run.yaml @@ -3,10 +3,8 @@ image_name: hf-serverless apis: - agents - datasetio -- eval - inference - safety -- scoring - telemetry - tool_runtime - vector_io @@ -48,14 +46,6 @@ providers: service_name: ${env.OTEL_SERVICE_NAME:llama-stack} sinks: ${env.TELEMETRY_SINKS:console,sqlite} sqlite_db_path: ${env.SQLITE_DB_PATH:~/.llama/distributions/hf-serverless/trace_store.db} - eval: - - provider_id: meta-reference - provider_type: inline::meta-reference - config: - kvstore: - type: sqlite - namespace: null - db_path: ${env.SQLITE_STORE_DIR:~/.llama/distributions/hf-serverless}/meta_reference_eval.db datasetio: - provider_id: huggingface provider_type: remote::huggingface @@ -71,17 +61,6 @@ providers: type: sqlite namespace: null db_path: ${env.SQLITE_STORE_DIR:~/.llama/distributions/hf-serverless}/localfs_datasetio.db - scoring: - - provider_id: basic - provider_type: inline::basic - config: {} - - provider_id: llm-as-judge - provider_type: inline::llm-as-judge - config: {} - - provider_id: braintrust - provider_type: inline::braintrust - config: - openai_api_key: ${env.OPENAI_API_KEY:} tool_runtime: - provider_id: brave-search provider_type: remote::brave-search @@ -118,7 +97,6 @@ models: shields: [] vector_dbs: [] datasets: [] -scoring_fns: [] benchmarks: [] tool_groups: - toolgroup_id: builtin::websearch diff --git a/llama_stack/templates/meta-reference-gpu/build.yaml b/llama_stack/templates/meta-reference-gpu/build.yaml index b9130fc7d..0c8da8280 100644 --- a/llama_stack/templates/meta-reference-gpu/build.yaml +++ b/llama_stack/templates/meta-reference-gpu/build.yaml @@ -14,15 +14,9 @@ distribution_spec: - inline::meta-reference telemetry: - inline::meta-reference - eval: - - inline::meta-reference datasetio: - remote::huggingface - inline::localfs - scoring: - - inline::basic - - inline::llm-as-judge - - inline::braintrust tool_runtime: - remote::brave-search - remote::tavily-search diff --git a/llama_stack/templates/meta-reference-gpu/meta_reference.py b/llama_stack/templates/meta-reference-gpu/meta_reference.py index 6bb1fcb0a..12a5013cb 100644 --- a/llama_stack/templates/meta-reference-gpu/meta_reference.py +++ b/llama_stack/templates/meta-reference-gpu/meta_reference.py @@ -30,9 +30,7 @@ def get_distribution_template() -> DistributionTemplate: "safety": ["inline::llama-guard"], "agents": ["inline::meta-reference"], "telemetry": ["inline::meta-reference"], - "eval": ["inline::meta-reference"], "datasetio": ["remote::huggingface", "inline::localfs"], - "scoring": ["inline::basic", "inline::llm-as-judge", "inline::braintrust"], "tool_runtime": [ "remote::brave-search", "remote::tavily-search", diff --git a/llama_stack/templates/meta-reference-gpu/run-with-safety.yaml b/llama_stack/templates/meta-reference-gpu/run-with-safety.yaml index 190c08494..82744f307 100644 --- a/llama_stack/templates/meta-reference-gpu/run-with-safety.yaml +++ b/llama_stack/templates/meta-reference-gpu/run-with-safety.yaml @@ -3,10 +3,8 @@ image_name: meta-reference-gpu apis: - agents - datasetio -- eval - inference - safety -- scoring - telemetry - tool_runtime - vector_io @@ -55,14 +53,6 @@ providers: service_name: ${env.OTEL_SERVICE_NAME:llama-stack} sinks: ${env.TELEMETRY_SINKS:console,sqlite} sqlite_db_path: ${env.SQLITE_DB_PATH:~/.llama/distributions/meta-reference-gpu/trace_store.db} - eval: - - provider_id: meta-reference - provider_type: inline::meta-reference - config: - kvstore: - type: sqlite - namespace: null - db_path: ${env.SQLITE_STORE_DIR:~/.llama/distributions/meta-reference-gpu}/meta_reference_eval.db datasetio: - provider_id: huggingface provider_type: remote::huggingface @@ -78,17 +68,6 @@ providers: type: sqlite namespace: null db_path: ${env.SQLITE_STORE_DIR:~/.llama/distributions/meta-reference-gpu}/localfs_datasetio.db - scoring: - - provider_id: basic - provider_type: inline::basic - config: {} - - provider_id: llm-as-judge - provider_type: inline::llm-as-judge - config: {} - - provider_id: braintrust - provider_type: inline::braintrust - config: - openai_api_key: ${env.OPENAI_API_KEY:} tool_runtime: - provider_id: brave-search provider_type: remote::brave-search @@ -130,7 +109,6 @@ shields: - shield_id: ${env.SAFETY_MODEL} vector_dbs: [] datasets: [] -scoring_fns: [] benchmarks: [] tool_groups: - toolgroup_id: builtin::websearch diff --git a/llama_stack/templates/meta-reference-gpu/run.yaml b/llama_stack/templates/meta-reference-gpu/run.yaml index 07763a4df..1b710c37c 100644 --- a/llama_stack/templates/meta-reference-gpu/run.yaml +++ b/llama_stack/templates/meta-reference-gpu/run.yaml @@ -3,10 +3,8 @@ image_name: meta-reference-gpu apis: - agents - datasetio -- eval - inference - safety -- scoring - telemetry - tool_runtime - vector_io @@ -49,14 +47,6 @@ providers: service_name: ${env.OTEL_SERVICE_NAME:llama-stack} sinks: ${env.TELEMETRY_SINKS:console,sqlite} sqlite_db_path: ${env.SQLITE_DB_PATH:~/.llama/distributions/meta-reference-gpu/trace_store.db} - eval: - - provider_id: meta-reference - provider_type: inline::meta-reference - config: - kvstore: - type: sqlite - namespace: null - db_path: ${env.SQLITE_STORE_DIR:~/.llama/distributions/meta-reference-gpu}/meta_reference_eval.db datasetio: - provider_id: huggingface provider_type: remote::huggingface @@ -72,17 +62,6 @@ providers: type: sqlite namespace: null db_path: ${env.SQLITE_STORE_DIR:~/.llama/distributions/meta-reference-gpu}/localfs_datasetio.db - scoring: - - provider_id: basic - provider_type: inline::basic - config: {} - - provider_id: llm-as-judge - provider_type: inline::llm-as-judge - config: {} - - provider_id: braintrust - provider_type: inline::braintrust - config: - openai_api_key: ${env.OPENAI_API_KEY:} tool_runtime: - provider_id: brave-search provider_type: remote::brave-search @@ -119,7 +98,6 @@ models: shields: [] vector_dbs: [] datasets: [] -scoring_fns: [] benchmarks: [] tool_groups: - toolgroup_id: builtin::websearch diff --git a/llama_stack/templates/meta-reference-quantized-gpu/build.yaml b/llama_stack/templates/meta-reference-quantized-gpu/build.yaml index 7bbcfe5f2..a55d3ddb4 100644 --- a/llama_stack/templates/meta-reference-quantized-gpu/build.yaml +++ b/llama_stack/templates/meta-reference-quantized-gpu/build.yaml @@ -14,15 +14,9 @@ distribution_spec: - inline::meta-reference telemetry: - inline::meta-reference - eval: - - inline::meta-reference datasetio: - remote::huggingface - inline::localfs - scoring: - - inline::basic - - inline::llm-as-judge - - inline::braintrust tool_runtime: - remote::brave-search - remote::tavily-search diff --git a/llama_stack/templates/meta-reference-quantized-gpu/meta_reference.py b/llama_stack/templates/meta-reference-quantized-gpu/meta_reference.py index 5f207bfad..de16930e4 100644 --- a/llama_stack/templates/meta-reference-quantized-gpu/meta_reference.py +++ b/llama_stack/templates/meta-reference-quantized-gpu/meta_reference.py @@ -25,9 +25,7 @@ def get_distribution_template() -> DistributionTemplate: "safety": ["inline::llama-guard"], "agents": ["inline::meta-reference"], "telemetry": ["inline::meta-reference"], - "eval": ["inline::meta-reference"], "datasetio": ["remote::huggingface", "inline::localfs"], - "scoring": ["inline::basic", "inline::llm-as-judge", "inline::braintrust"], "tool_runtime": [ "remote::brave-search", "remote::tavily-search", diff --git a/llama_stack/templates/meta-reference-quantized-gpu/run.yaml b/llama_stack/templates/meta-reference-quantized-gpu/run.yaml index 51b9dc250..13e9177fd 100644 --- a/llama_stack/templates/meta-reference-quantized-gpu/run.yaml +++ b/llama_stack/templates/meta-reference-quantized-gpu/run.yaml @@ -3,10 +3,8 @@ image_name: meta-reference-quantized-gpu apis: - agents - datasetio -- eval - inference - safety -- scoring - telemetry - tool_runtime - vector_io @@ -51,14 +49,6 @@ providers: service_name: ${env.OTEL_SERVICE_NAME:llama-stack} sinks: ${env.TELEMETRY_SINKS:console,sqlite} sqlite_db_path: ${env.SQLITE_DB_PATH:~/.llama/distributions/meta-reference-quantized-gpu/trace_store.db} - eval: - - provider_id: meta-reference - provider_type: inline::meta-reference - config: - kvstore: - type: sqlite - namespace: null - db_path: ${env.SQLITE_STORE_DIR:~/.llama/distributions/meta-reference-quantized-gpu}/meta_reference_eval.db datasetio: - provider_id: huggingface provider_type: remote::huggingface @@ -74,17 +64,6 @@ providers: type: sqlite namespace: null db_path: ${env.SQLITE_STORE_DIR:~/.llama/distributions/meta-reference-quantized-gpu}/localfs_datasetio.db - scoring: - - provider_id: basic - provider_type: inline::basic - config: {} - - provider_id: llm-as-judge - provider_type: inline::llm-as-judge - config: {} - - provider_id: braintrust - provider_type: inline::braintrust - config: - openai_api_key: ${env.OPENAI_API_KEY:} tool_runtime: - provider_id: brave-search provider_type: remote::brave-search @@ -121,7 +100,6 @@ models: shields: [] vector_dbs: [] datasets: [] -scoring_fns: [] benchmarks: [] tool_groups: - toolgroup_id: builtin::websearch diff --git a/llama_stack/templates/nvidia/build.yaml b/llama_stack/templates/nvidia/build.yaml index 0c788ce86..848b6cd45 100644 --- a/llama_stack/templates/nvidia/build.yaml +++ b/llama_stack/templates/nvidia/build.yaml @@ -12,12 +12,8 @@ distribution_spec: - inline::meta-reference telemetry: - inline::meta-reference - eval: - - inline::meta-reference datasetio: - inline::localfs - scoring: - - inline::basic tool_runtime: - inline::rag-runtime image_type: conda diff --git a/llama_stack/templates/nvidia/nvidia.py b/llama_stack/templates/nvidia/nvidia.py index 308c0e2a6..f273c1a17 100644 --- a/llama_stack/templates/nvidia/nvidia.py +++ b/llama_stack/templates/nvidia/nvidia.py @@ -6,11 +6,20 @@ from pathlib import Path -from llama_stack.distribution.datatypes import ModelInput, Provider, ShieldInput, ToolGroupInput +from llama_stack.distribution.datatypes import ( + ModelInput, + Provider, + ShieldInput, + ToolGroupInput, +) from llama_stack.providers.remote.inference.nvidia import NVIDIAConfig from llama_stack.providers.remote.inference.nvidia.models import MODEL_ENTRIES from llama_stack.providers.remote.safety.nvidia import NVIDIASafetyConfig -from llama_stack.templates.template import DistributionTemplate, RunConfigSettings, get_model_registry +from llama_stack.templates.template import ( + DistributionTemplate, + get_model_registry, + RunConfigSettings, +) def get_distribution_template() -> DistributionTemplate: @@ -20,9 +29,7 @@ def get_distribution_template() -> DistributionTemplate: "safety": ["remote::nvidia"], "agents": ["inline::meta-reference"], "telemetry": ["inline::meta-reference"], - "eval": ["inline::meta-reference"], "datasetio": ["inline::localfs"], - "scoring": ["inline::basic"], "tool_runtime": ["inline::rag-runtime"], } @@ -80,7 +87,9 @@ def get_distribution_template() -> DistributionTemplate: ] }, default_models=[inference_model, safety_model], - default_shields=[ShieldInput(shield_id="${env.SAFETY_MODEL}", provider_id="nvidia")], + default_shields=[ + ShieldInput(shield_id="${env.SAFETY_MODEL}", provider_id="nvidia") + ], default_tool_groups=default_tool_groups, ), }, diff --git a/llama_stack/templates/nvidia/run-with-safety.yaml b/llama_stack/templates/nvidia/run-with-safety.yaml index 04da1bcda..a1f6fb5f8 100644 --- a/llama_stack/templates/nvidia/run-with-safety.yaml +++ b/llama_stack/templates/nvidia/run-with-safety.yaml @@ -3,10 +3,8 @@ image_name: nvidia apis: - agents - datasetio -- eval - inference - safety -- scoring - telemetry - tool_runtime - vector_io @@ -51,14 +49,6 @@ providers: service_name: ${env.OTEL_SERVICE_NAME:llama-stack} sinks: ${env.TELEMETRY_SINKS:console,sqlite} sqlite_db_path: ${env.SQLITE_DB_PATH:~/.llama/distributions/nvidia/trace_store.db} - eval: - - provider_id: meta-reference - provider_type: inline::meta-reference - config: - kvstore: - type: sqlite - namespace: null - db_path: ${env.SQLITE_STORE_DIR:~/.llama/distributions/nvidia}/meta_reference_eval.db datasetio: - provider_id: localfs provider_type: inline::localfs @@ -67,10 +57,6 @@ providers: type: sqlite namespace: null db_path: ${env.SQLITE_STORE_DIR:~/.llama/distributions/nvidia}/localfs_datasetio.db - scoring: - - provider_id: basic - provider_type: inline::basic - config: {} tool_runtime: - provider_id: rag-runtime provider_type: inline::rag-runtime @@ -92,7 +78,6 @@ shields: provider_id: nvidia vector_dbs: [] datasets: [] -scoring_fns: [] benchmarks: [] tool_groups: - toolgroup_id: builtin::rag diff --git a/llama_stack/templates/nvidia/run.yaml b/llama_stack/templates/nvidia/run.yaml index 3abdd82a7..4279eda08 100644 --- a/llama_stack/templates/nvidia/run.yaml +++ b/llama_stack/templates/nvidia/run.yaml @@ -3,10 +3,8 @@ image_name: nvidia apis: - agents - datasetio -- eval - inference - safety -- scoring - telemetry - tool_runtime - vector_io @@ -46,14 +44,6 @@ providers: service_name: ${env.OTEL_SERVICE_NAME:llama-stack} sinks: ${env.TELEMETRY_SINKS:console,sqlite} sqlite_db_path: ${env.SQLITE_DB_PATH:~/.llama/distributions/nvidia/trace_store.db} - eval: - - provider_id: meta-reference - provider_type: inline::meta-reference - config: - kvstore: - type: sqlite - namespace: null - db_path: ${env.SQLITE_STORE_DIR:~/.llama/distributions/nvidia}/meta_reference_eval.db datasetio: - provider_id: localfs provider_type: inline::localfs @@ -62,10 +52,6 @@ providers: type: sqlite namespace: null db_path: ${env.SQLITE_STORE_DIR:~/.llama/distributions/nvidia}/localfs_datasetio.db - scoring: - - provider_id: basic - provider_type: inline::basic - config: {} tool_runtime: - provider_id: rag-runtime provider_type: inline::rag-runtime @@ -195,7 +181,6 @@ models: shields: [] vector_dbs: [] datasets: [] -scoring_fns: [] benchmarks: [] tool_groups: - toolgroup_id: builtin::rag diff --git a/llama_stack/templates/ollama/build.yaml b/llama_stack/templates/ollama/build.yaml index 37b72fc1f..d5a195d5f 100644 --- a/llama_stack/templates/ollama/build.yaml +++ b/llama_stack/templates/ollama/build.yaml @@ -14,15 +14,9 @@ distribution_spec: - inline::meta-reference telemetry: - inline::meta-reference - eval: - - inline::meta-reference datasetio: - remote::huggingface - inline::localfs - scoring: - - inline::basic - - inline::llm-as-judge - - inline::braintrust tool_runtime: - remote::brave-search - remote::tavily-search diff --git a/llama_stack/templates/ollama/ollama.py b/llama_stack/templates/ollama/ollama.py index 2d753d3e4..a679607fa 100644 --- a/llama_stack/templates/ollama/ollama.py +++ b/llama_stack/templates/ollama/ollama.py @@ -25,9 +25,7 @@ def get_distribution_template() -> DistributionTemplate: "safety": ["inline::llama-guard"], "agents": ["inline::meta-reference"], "telemetry": ["inline::meta-reference"], - "eval": ["inline::meta-reference"], "datasetio": ["remote::huggingface", "inline::localfs"], - "scoring": ["inline::basic", "inline::llm-as-judge", "inline::braintrust"], "tool_runtime": [ "remote::brave-search", "remote::tavily-search", diff --git a/llama_stack/templates/ollama/run-with-safety.yaml b/llama_stack/templates/ollama/run-with-safety.yaml index 2b8eb44db..0b9e94faa 100644 --- a/llama_stack/templates/ollama/run-with-safety.yaml +++ b/llama_stack/templates/ollama/run-with-safety.yaml @@ -3,10 +3,8 @@ image_name: ollama apis: - agents - datasetio -- eval - inference - safety -- scoring - telemetry - tool_runtime - vector_io @@ -46,14 +44,6 @@ providers: service_name: ${env.OTEL_SERVICE_NAME:llama-stack} sinks: ${env.TELEMETRY_SINKS:console,sqlite} sqlite_db_path: ${env.SQLITE_DB_PATH:~/.llama/distributions/ollama/trace_store.db} - eval: - - provider_id: meta-reference - provider_type: inline::meta-reference - config: - kvstore: - type: sqlite - namespace: null - db_path: ${env.SQLITE_STORE_DIR:~/.llama/distributions/ollama}/meta_reference_eval.db datasetio: - provider_id: huggingface provider_type: remote::huggingface @@ -69,17 +59,6 @@ providers: type: sqlite namespace: null db_path: ${env.SQLITE_STORE_DIR:~/.llama/distributions/ollama}/localfs_datasetio.db - scoring: - - provider_id: basic - provider_type: inline::basic - config: {} - - provider_id: llm-as-judge - provider_type: inline::llm-as-judge - config: {} - - provider_id: braintrust - provider_type: inline::braintrust - config: - openai_api_key: ${env.OPENAI_API_KEY:} tool_runtime: - provider_id: brave-search provider_type: remote::brave-search @@ -129,7 +108,6 @@ shields: provider_id: code-scanner vector_dbs: [] datasets: [] -scoring_fns: [] benchmarks: [] tool_groups: - toolgroup_id: builtin::websearch diff --git a/llama_stack/templates/ollama/run.yaml b/llama_stack/templates/ollama/run.yaml index c9531f417..1451ba96f 100644 --- a/llama_stack/templates/ollama/run.yaml +++ b/llama_stack/templates/ollama/run.yaml @@ -3,10 +3,8 @@ image_name: ollama apis: - agents - datasetio -- eval - inference - safety -- scoring - telemetry - tool_runtime - vector_io @@ -44,14 +42,6 @@ providers: service_name: ${env.OTEL_SERVICE_NAME:llama-stack} sinks: ${env.TELEMETRY_SINKS:console,sqlite} sqlite_db_path: ${env.SQLITE_DB_PATH:~/.llama/distributions/ollama/trace_store.db} - eval: - - provider_id: meta-reference - provider_type: inline::meta-reference - config: - kvstore: - type: sqlite - namespace: null - db_path: ${env.SQLITE_STORE_DIR:~/.llama/distributions/ollama}/meta_reference_eval.db datasetio: - provider_id: huggingface provider_type: remote::huggingface @@ -67,17 +57,6 @@ providers: type: sqlite namespace: null db_path: ${env.SQLITE_STORE_DIR:~/.llama/distributions/ollama}/localfs_datasetio.db - scoring: - - provider_id: basic - provider_type: inline::basic - config: {} - - provider_id: llm-as-judge - provider_type: inline::llm-as-judge - config: {} - - provider_id: braintrust - provider_type: inline::braintrust - config: - openai_api_key: ${env.OPENAI_API_KEY:} tool_runtime: - provider_id: brave-search provider_type: remote::brave-search @@ -119,7 +98,6 @@ models: shields: [] vector_dbs: [] datasets: [] -scoring_fns: [] benchmarks: [] tool_groups: - toolgroup_id: builtin::websearch diff --git a/llama_stack/templates/open-benchmark/open_benchmark.py b/llama_stack/templates/open-benchmark/open_benchmark.py index 03e524dae..185fd867f 100644 --- a/llama_stack/templates/open-benchmark/open_benchmark.py +++ b/llama_stack/templates/open-benchmark/open_benchmark.py @@ -31,12 +31,14 @@ from llama_stack.providers.remote.vector_io.pgvector.config import ( from llama_stack.providers.utils.inference.model_registry import ProviderModelEntry from llama_stack.templates.template import ( DistributionTemplate, - RunConfigSettings, get_model_registry, + RunConfigSettings, ) -def get_inference_providers() -> Tuple[List[Provider], Dict[str, List[ProviderModelEntry]]]: +def get_inference_providers() -> ( + Tuple[List[Provider], Dict[str, List[ProviderModelEntry]]] +): # in this template, we allow each API key to be optional providers = [ ( @@ -102,9 +104,7 @@ def get_distribution_template() -> DistributionTemplate: "safety": ["inline::llama-guard"], "agents": ["inline::meta-reference"], "telemetry": ["inline::meta-reference"], - "eval": ["inline::meta-reference"], "datasetio": ["remote::huggingface", "inline::localfs"], - "scoring": ["inline::basic", "inline::llm-as-judge", "inline::braintrust"], "tool_runtime": [ "remote::brave-search", "remote::tavily-search", @@ -119,7 +119,9 @@ def get_distribution_template() -> DistributionTemplate: Provider( provider_id="sqlite-vec", provider_type="inline::sqlite-vec", - config=SQLiteVectorIOConfig.sample_run_config(f"~/.llama/distributions/{name}"), + config=SQLiteVectorIOConfig.sample_run_config( + f"~/.llama/distributions/{name}" + ), ), Provider( provider_id="${env.ENABLE_CHROMADB+chromadb}", @@ -210,33 +212,36 @@ def get_distribution_template() -> DistributionTemplate: ), ] - default_benchmarks = [ - BenchmarkInput( - benchmark_id="meta-reference-simpleqa", - dataset_id="simpleqa", - grader_ids=["llm-as-judge::405b-simpleqa"], - ), - BenchmarkInput( - benchmark_id="meta-reference-mmlu-cot", - dataset_id="mmlu_cot", - grader_ids=["basic::regex_parser_multiple_choice_answer"], - ), - BenchmarkInput( - benchmark_id="meta-reference-gpqa-cot", - dataset_id="gpqa_cot", - grader_ids=["basic::regex_parser_multiple_choice_answer"], - ), - BenchmarkInput( - benchmark_id="meta-reference-math-500", - dataset_id="math_500", - grader_ids=["basic::regex_parser_math_response"], - ), - BenchmarkInput( - benchmark_id="meta-reference-bfcl", - dataset_id="bfcl", - grader_ids=["basic::bfcl"], - ), - ] + # TODO(xiyan): fix this back as registerable resources + default_benchmarks = [] + # default_benchmarks = [ + # BenchmarkInput( + # benchmark_id="meta-reference-simpleqa", + # dataset_id="simpleqa", + # grader_ids=["llm-as-judge::405b-simpleqa"], + # ), + # BenchmarkInput( + # benchmark_id="meta-reference-mmlu-cot", + # dataset_id="mmlu_cot", + # grader_ids=["basic::regex_parser_multiple_choice_answer"], + # ), + # BenchmarkInput( + # benchmark_id="meta-reference-gpqa-cot", + # dataset_id="gpqa_cot", + # grader_ids=["basic::regex_parser_multiple_choice_answer"], + # ), + # BenchmarkInput( + # benchmark_id="meta-reference-math-500", + # dataset_id="math_500", + # grader_ids=["basic::regex_parser_math_response"], + # ), + # BenchmarkInput( + # benchmark_id="meta-reference-bfcl", + # dataset_id="bfcl", + # grader_ids=["basic::bfcl"], + # ), + # ] + return DistributionTemplate( name=name, distro_type="self_hosted", diff --git a/llama_stack/templates/passthrough/build.yaml b/llama_stack/templates/passthrough/build.yaml index fb1fb1066..6a44293f6 100644 --- a/llama_stack/templates/passthrough/build.yaml +++ b/llama_stack/templates/passthrough/build.yaml @@ -15,15 +15,9 @@ distribution_spec: - inline::meta-reference telemetry: - inline::meta-reference - eval: - - inline::meta-reference datasetio: - remote::huggingface - inline::localfs - scoring: - - inline::basic - - inline::llm-as-judge - - inline::braintrust tool_runtime: - remote::brave-search - remote::tavily-search diff --git a/llama_stack/templates/passthrough/passthrough.py b/llama_stack/templates/passthrough/passthrough.py index cc3f55937..589aca229 100644 --- a/llama_stack/templates/passthrough/passthrough.py +++ b/llama_stack/templates/passthrough/passthrough.py @@ -21,10 +21,7 @@ from llama_stack.providers.remote.inference.passthrough.config import ( PassthroughImplConfig, ) from llama_stack.providers.utils.inference.model_registry import ProviderModelEntry -from llama_stack.templates.template import ( - DistributionTemplate, - RunConfigSettings, -) +from llama_stack.templates.template import DistributionTemplate, RunConfigSettings def get_distribution_template() -> DistributionTemplate: @@ -34,9 +31,7 @@ def get_distribution_template() -> DistributionTemplate: "safety": ["inline::llama-guard"], "agents": ["inline::meta-reference"], "telemetry": ["inline::meta-reference"], - "eval": ["inline::meta-reference"], "datasetio": ["remote::huggingface", "inline::localfs"], - "scoring": ["inline::basic", "inline::llm-as-judge", "inline::braintrust"], "tool_runtime": [ "remote::brave-search", "remote::tavily-search", diff --git a/llama_stack/templates/passthrough/run-with-safety.yaml b/llama_stack/templates/passthrough/run-with-safety.yaml index fbfa4afe7..72aced336 100644 --- a/llama_stack/templates/passthrough/run-with-safety.yaml +++ b/llama_stack/templates/passthrough/run-with-safety.yaml @@ -3,10 +3,8 @@ image_name: passthrough apis: - agents - datasetio -- eval - inference - safety -- scoring - telemetry - tool_runtime - vector_io @@ -53,14 +51,6 @@ providers: service_name: ${env.OTEL_SERVICE_NAME:llama-stack} sinks: ${env.TELEMETRY_SINKS:console,sqlite} sqlite_db_path: ${env.SQLITE_DB_PATH:~/.llama/distributions/passthrough/trace_store.db} - eval: - - provider_id: meta-reference - provider_type: inline::meta-reference - config: - kvstore: - type: sqlite - namespace: null - db_path: ${env.SQLITE_STORE_DIR:~/.llama/distributions/passthrough}/meta_reference_eval.db datasetio: - provider_id: huggingface provider_type: remote::huggingface @@ -76,17 +66,6 @@ providers: type: sqlite namespace: null db_path: ${env.SQLITE_STORE_DIR:~/.llama/distributions/passthrough}/localfs_datasetio.db - scoring: - - provider_id: basic - provider_type: inline::basic - config: {} - - provider_id: llm-as-judge - provider_type: inline::llm-as-judge - config: {} - - provider_id: braintrust - provider_type: inline::braintrust - config: - openai_api_key: ${env.OPENAI_API_KEY:} tool_runtime: - provider_id: brave-search provider_type: remote::brave-search @@ -139,7 +118,6 @@ shields: provider_id: code-scanner vector_dbs: [] datasets: [] -scoring_fns: [] benchmarks: [] tool_groups: - toolgroup_id: builtin::websearch diff --git a/llama_stack/templates/passthrough/run.yaml b/llama_stack/templates/passthrough/run.yaml index 6956bc6e3..a239f2bf7 100644 --- a/llama_stack/templates/passthrough/run.yaml +++ b/llama_stack/templates/passthrough/run.yaml @@ -3,10 +3,8 @@ image_name: passthrough apis: - agents - datasetio -- eval - inference - safety -- scoring - telemetry - tool_runtime - vector_io @@ -48,14 +46,6 @@ providers: service_name: ${env.OTEL_SERVICE_NAME:llama-stack} sinks: ${env.TELEMETRY_SINKS:console,sqlite} sqlite_db_path: ${env.SQLITE_DB_PATH:~/.llama/distributions/passthrough/trace_store.db} - eval: - - provider_id: meta-reference - provider_type: inline::meta-reference - config: - kvstore: - type: sqlite - namespace: null - db_path: ${env.SQLITE_STORE_DIR:~/.llama/distributions/passthrough}/meta_reference_eval.db datasetio: - provider_id: huggingface provider_type: remote::huggingface @@ -71,17 +61,6 @@ providers: type: sqlite namespace: null db_path: ${env.SQLITE_STORE_DIR:~/.llama/distributions/passthrough}/localfs_datasetio.db - scoring: - - provider_id: basic - provider_type: inline::basic - config: {} - - provider_id: llm-as-judge - provider_type: inline::llm-as-judge - config: {} - - provider_id: braintrust - provider_type: inline::braintrust - config: - openai_api_key: ${env.OPENAI_API_KEY:} tool_runtime: - provider_id: brave-search provider_type: remote::brave-search @@ -129,7 +108,6 @@ shields: - shield_id: meta-llama/Llama-Guard-3-8B vector_dbs: [] datasets: [] -scoring_fns: [] benchmarks: [] tool_groups: - toolgroup_id: builtin::websearch diff --git a/llama_stack/templates/remote-vllm/build.yaml b/llama_stack/templates/remote-vllm/build.yaml index b2bbf853a..0437d76d6 100644 --- a/llama_stack/templates/remote-vllm/build.yaml +++ b/llama_stack/templates/remote-vllm/build.yaml @@ -13,15 +13,9 @@ distribution_spec: - inline::llama-guard agents: - inline::meta-reference - eval: - - inline::meta-reference datasetio: - remote::huggingface - inline::localfs - scoring: - - inline::basic - - inline::llm-as-judge - - inline::braintrust telemetry: - inline::meta-reference tool_runtime: diff --git a/llama_stack/templates/remote-vllm/run-with-safety.yaml b/llama_stack/templates/remote-vllm/run-with-safety.yaml index 3830ffcdb..c462ad924 100644 --- a/llama_stack/templates/remote-vllm/run-with-safety.yaml +++ b/llama_stack/templates/remote-vllm/run-with-safety.yaml @@ -3,10 +3,8 @@ image_name: remote-vllm apis: - agents - datasetio -- eval - inference - safety -- scoring - telemetry - tool_runtime - vector_io @@ -50,14 +48,6 @@ providers: type: sqlite namespace: null db_path: ${env.SQLITE_STORE_DIR:~/.llama/distributions/remote-vllm}/agents_store.db - eval: - - provider_id: meta-reference - provider_type: inline::meta-reference - config: - kvstore: - type: sqlite - namespace: null - db_path: ${env.SQLITE_STORE_DIR:~/.llama/distributions/remote-vllm}/meta_reference_eval.db datasetio: - provider_id: huggingface provider_type: remote::huggingface @@ -73,17 +63,6 @@ providers: type: sqlite namespace: null db_path: ${env.SQLITE_STORE_DIR:~/.llama/distributions/remote-vllm}/localfs_datasetio.db - scoring: - - provider_id: basic - provider_type: inline::basic - config: {} - - provider_id: llm-as-judge - provider_type: inline::llm-as-judge - config: {} - - provider_id: braintrust - provider_type: inline::braintrust - config: - openai_api_key: ${env.OPENAI_API_KEY:} telemetry: - provider_id: meta-reference provider_type: inline::meta-reference @@ -136,7 +115,6 @@ shields: - shield_id: ${env.SAFETY_MODEL} vector_dbs: [] datasets: [] -scoring_fns: [] benchmarks: [] tool_groups: - toolgroup_id: builtin::websearch diff --git a/llama_stack/templates/remote-vllm/run.yaml b/llama_stack/templates/remote-vllm/run.yaml index b6bba1252..71f4ea5cc 100644 --- a/llama_stack/templates/remote-vllm/run.yaml +++ b/llama_stack/templates/remote-vllm/run.yaml @@ -3,10 +3,8 @@ image_name: remote-vllm apis: - agents - datasetio -- eval - inference - safety -- scoring - telemetry - tool_runtime - vector_io @@ -43,14 +41,6 @@ providers: type: sqlite namespace: null db_path: ${env.SQLITE_STORE_DIR:~/.llama/distributions/remote-vllm}/agents_store.db - eval: - - provider_id: meta-reference - provider_type: inline::meta-reference - config: - kvstore: - type: sqlite - namespace: null - db_path: ${env.SQLITE_STORE_DIR:~/.llama/distributions/remote-vllm}/meta_reference_eval.db datasetio: - provider_id: huggingface provider_type: remote::huggingface @@ -66,17 +56,6 @@ providers: type: sqlite namespace: null db_path: ${env.SQLITE_STORE_DIR:~/.llama/distributions/remote-vllm}/localfs_datasetio.db - scoring: - - provider_id: basic - provider_type: inline::basic - config: {} - - provider_id: llm-as-judge - provider_type: inline::llm-as-judge - config: {} - - provider_id: braintrust - provider_type: inline::braintrust - config: - openai_api_key: ${env.OPENAI_API_KEY:} telemetry: - provider_id: meta-reference provider_type: inline::meta-reference @@ -124,7 +103,6 @@ models: shields: [] vector_dbs: [] datasets: [] -scoring_fns: [] benchmarks: [] tool_groups: - toolgroup_id: builtin::websearch diff --git a/llama_stack/templates/remote-vllm/vllm.py b/llama_stack/templates/remote-vllm/vllm.py index 9901fc83b..f43873b02 100644 --- a/llama_stack/templates/remote-vllm/vllm.py +++ b/llama_stack/templates/remote-vllm/vllm.py @@ -27,9 +27,7 @@ def get_distribution_template() -> DistributionTemplate: "vector_io": ["inline::faiss", "remote::chromadb", "remote::pgvector"], "safety": ["inline::llama-guard"], "agents": ["inline::meta-reference"], - "eval": ["inline::meta-reference"], "datasetio": ["remote::huggingface", "inline::localfs"], - "scoring": ["inline::basic", "inline::llm-as-judge", "inline::braintrust"], "telemetry": ["inline::meta-reference"], "tool_runtime": [ "remote::brave-search", diff --git a/llama_stack/templates/sambanova/run.yaml b/llama_stack/templates/sambanova/run.yaml index 616d82a61..0a6f71c52 100644 --- a/llama_stack/templates/sambanova/run.yaml +++ b/llama_stack/templates/sambanova/run.yaml @@ -169,7 +169,6 @@ shields: - shield_id: meta-llama/Llama-Guard-3-8B vector_dbs: [] datasets: [] -scoring_fns: [] benchmarks: [] tool_groups: - toolgroup_id: builtin::websearch diff --git a/llama_stack/templates/tgi/build.yaml b/llama_stack/templates/tgi/build.yaml index 9fe79647c..e870c5eb1 100644 --- a/llama_stack/templates/tgi/build.yaml +++ b/llama_stack/templates/tgi/build.yaml @@ -15,15 +15,9 @@ distribution_spec: - inline::meta-reference telemetry: - inline::meta-reference - eval: - - inline::meta-reference datasetio: - remote::huggingface - inline::localfs - scoring: - - inline::basic - - inline::llm-as-judge - - inline::braintrust tool_runtime: - remote::brave-search - remote::tavily-search diff --git a/llama_stack/templates/tgi/run-with-safety.yaml b/llama_stack/templates/tgi/run-with-safety.yaml index db54c0393..2e27ccdbf 100644 --- a/llama_stack/templates/tgi/run-with-safety.yaml +++ b/llama_stack/templates/tgi/run-with-safety.yaml @@ -3,10 +3,8 @@ image_name: tgi apis: - agents - datasetio -- eval - inference - safety -- scoring - telemetry - tool_runtime - vector_io @@ -48,14 +46,6 @@ providers: service_name: ${env.OTEL_SERVICE_NAME:llama-stack} sinks: ${env.TELEMETRY_SINKS:console,sqlite} sqlite_db_path: ${env.SQLITE_DB_PATH:~/.llama/distributions/tgi/trace_store.db} - eval: - - provider_id: meta-reference - provider_type: inline::meta-reference - config: - kvstore: - type: sqlite - namespace: null - db_path: ${env.SQLITE_STORE_DIR:~/.llama/distributions/tgi}/meta_reference_eval.db datasetio: - provider_id: huggingface provider_type: remote::huggingface @@ -71,17 +61,6 @@ providers: type: sqlite namespace: null db_path: ${env.SQLITE_STORE_DIR:~/.llama/distributions/tgi}/localfs_datasetio.db - scoring: - - provider_id: basic - provider_type: inline::basic - config: {} - - provider_id: llm-as-judge - provider_type: inline::llm-as-judge - config: {} - - provider_id: braintrust - provider_type: inline::braintrust - config: - openai_api_key: ${env.OPENAI_API_KEY:} tool_runtime: - provider_id: brave-search provider_type: remote::brave-search @@ -118,7 +97,6 @@ shields: - shield_id: ${env.SAFETY_MODEL} vector_dbs: [] datasets: [] -scoring_fns: [] benchmarks: [] tool_groups: - toolgroup_id: builtin::websearch diff --git a/llama_stack/templates/tgi/run.yaml b/llama_stack/templates/tgi/run.yaml index dafb59aa9..9e0bdeb21 100644 --- a/llama_stack/templates/tgi/run.yaml +++ b/llama_stack/templates/tgi/run.yaml @@ -3,10 +3,8 @@ image_name: tgi apis: - agents - datasetio -- eval - inference - safety -- scoring - telemetry - tool_runtime - vector_io @@ -47,14 +45,6 @@ providers: service_name: ${env.OTEL_SERVICE_NAME:llama-stack} sinks: ${env.TELEMETRY_SINKS:console,sqlite} sqlite_db_path: ${env.SQLITE_DB_PATH:~/.llama/distributions/tgi/trace_store.db} - eval: - - provider_id: meta-reference - provider_type: inline::meta-reference - config: - kvstore: - type: sqlite - namespace: null - db_path: ${env.SQLITE_STORE_DIR:~/.llama/distributions/tgi}/meta_reference_eval.db datasetio: - provider_id: huggingface provider_type: remote::huggingface @@ -70,17 +60,6 @@ providers: type: sqlite namespace: null db_path: ${env.SQLITE_STORE_DIR:~/.llama/distributions/tgi}/localfs_datasetio.db - scoring: - - provider_id: basic - provider_type: inline::basic - config: {} - - provider_id: llm-as-judge - provider_type: inline::llm-as-judge - config: {} - - provider_id: braintrust - provider_type: inline::braintrust - config: - openai_api_key: ${env.OPENAI_API_KEY:} tool_runtime: - provider_id: brave-search provider_type: remote::brave-search @@ -117,7 +96,6 @@ models: shields: [] vector_dbs: [] datasets: [] -scoring_fns: [] benchmarks: [] tool_groups: - toolgroup_id: builtin::websearch diff --git a/llama_stack/templates/tgi/tgi.py b/llama_stack/templates/tgi/tgi.py index 45ea74db6..5fcf336c7 100644 --- a/llama_stack/templates/tgi/tgi.py +++ b/llama_stack/templates/tgi/tgi.py @@ -28,9 +28,7 @@ def get_distribution_template() -> DistributionTemplate: "safety": ["inline::llama-guard"], "agents": ["inline::meta-reference"], "telemetry": ["inline::meta-reference"], - "eval": ["inline::meta-reference"], "datasetio": ["remote::huggingface", "inline::localfs"], - "scoring": ["inline::basic", "inline::llm-as-judge", "inline::braintrust"], "tool_runtime": [ "remote::brave-search", "remote::tavily-search", diff --git a/llama_stack/templates/together/build.yaml b/llama_stack/templates/together/build.yaml index 834a3ecaf..8892475bb 100644 --- a/llama_stack/templates/together/build.yaml +++ b/llama_stack/templates/together/build.yaml @@ -15,15 +15,9 @@ distribution_spec: - inline::meta-reference telemetry: - inline::meta-reference - eval: - - inline::meta-reference datasetio: - remote::huggingface - inline::localfs - scoring: - - inline::basic - - inline::llm-as-judge - - inline::braintrust tool_runtime: - remote::brave-search - remote::tavily-search diff --git a/llama_stack/templates/together/run-with-safety.yaml b/llama_stack/templates/together/run-with-safety.yaml index e0bf46c11..c386aeb83 100644 --- a/llama_stack/templates/together/run-with-safety.yaml +++ b/llama_stack/templates/together/run-with-safety.yaml @@ -3,10 +3,8 @@ image_name: together apis: - agents - datasetio -- eval - inference - safety -- scoring - telemetry - tool_runtime - vector_io @@ -53,14 +51,6 @@ providers: service_name: ${env.OTEL_SERVICE_NAME:llama-stack} sinks: ${env.TELEMETRY_SINKS:console,sqlite} sqlite_db_path: ${env.SQLITE_DB_PATH:~/.llama/distributions/together/trace_store.db} - eval: - - provider_id: meta-reference - provider_type: inline::meta-reference - config: - kvstore: - type: sqlite - namespace: null - db_path: ${env.SQLITE_STORE_DIR:~/.llama/distributions/together}/meta_reference_eval.db datasetio: - provider_id: huggingface provider_type: remote::huggingface @@ -76,17 +66,6 @@ providers: type: sqlite namespace: null db_path: ${env.SQLITE_STORE_DIR:~/.llama/distributions/together}/localfs_datasetio.db - scoring: - - provider_id: basic - provider_type: inline::basic - config: {} - - provider_id: llm-as-judge - provider_type: inline::llm-as-judge - config: {} - - provider_id: braintrust - provider_type: inline::braintrust - config: - openai_api_key: ${env.OPENAI_API_KEY:} tool_runtime: - provider_id: brave-search provider_type: remote::brave-search @@ -233,7 +212,6 @@ shields: provider_id: code-scanner vector_dbs: [] datasets: [] -scoring_fns: [] benchmarks: [] tool_groups: - toolgroup_id: builtin::websearch diff --git a/llama_stack/templates/together/run.yaml b/llama_stack/templates/together/run.yaml index 9d0acaf31..0c808eed6 100644 --- a/llama_stack/templates/together/run.yaml +++ b/llama_stack/templates/together/run.yaml @@ -3,10 +3,8 @@ image_name: together apis: - agents - datasetio -- eval - inference - safety -- scoring - telemetry - tool_runtime - vector_io @@ -48,14 +46,6 @@ providers: service_name: ${env.OTEL_SERVICE_NAME:llama-stack} sinks: ${env.TELEMETRY_SINKS:console,sqlite} sqlite_db_path: ${env.SQLITE_DB_PATH:~/.llama/distributions/together/trace_store.db} - eval: - - provider_id: meta-reference - provider_type: inline::meta-reference - config: - kvstore: - type: sqlite - namespace: null - db_path: ${env.SQLITE_STORE_DIR:~/.llama/distributions/together}/meta_reference_eval.db datasetio: - provider_id: huggingface provider_type: remote::huggingface @@ -71,17 +61,6 @@ providers: type: sqlite namespace: null db_path: ${env.SQLITE_STORE_DIR:~/.llama/distributions/together}/localfs_datasetio.db - scoring: - - provider_id: basic - provider_type: inline::basic - config: {} - - provider_id: llm-as-judge - provider_type: inline::llm-as-judge - config: {} - - provider_id: braintrust - provider_type: inline::braintrust - config: - openai_api_key: ${env.OPENAI_API_KEY:} tool_runtime: - provider_id: brave-search provider_type: remote::brave-search @@ -223,7 +202,6 @@ shields: - shield_id: meta-llama/Llama-Guard-3-8B vector_dbs: [] datasets: [] -scoring_fns: [] benchmarks: [] tool_groups: - toolgroup_id: builtin::websearch diff --git a/llama_stack/templates/together/together.py b/llama_stack/templates/together/together.py index fce03a1b2..fbeeaad09 100644 --- a/llama_stack/templates/together/together.py +++ b/llama_stack/templates/together/together.py @@ -19,7 +19,11 @@ from llama_stack.providers.inline.inference.sentence_transformers import ( from llama_stack.providers.inline.vector_io.faiss.config import FaissVectorIOConfig from llama_stack.providers.remote.inference.together import TogetherImplConfig from llama_stack.providers.remote.inference.together.models import MODEL_ENTRIES -from llama_stack.templates.template import DistributionTemplate, RunConfigSettings, get_model_registry +from llama_stack.templates.template import ( + DistributionTemplate, + get_model_registry, + RunConfigSettings, +) def get_distribution_template() -> DistributionTemplate: @@ -29,9 +33,7 @@ def get_distribution_template() -> DistributionTemplate: "safety": ["inline::llama-guard"], "agents": ["inline::meta-reference"], "telemetry": ["inline::meta-reference"], - "eval": ["inline::meta-reference"], "datasetio": ["remote::huggingface", "inline::localfs"], - "scoring": ["inline::basic", "inline::llm-as-judge", "inline::braintrust"], "tool_runtime": [ "remote::brave-search", "remote::tavily-search", diff --git a/llama_stack/templates/vllm-gpu/build.yaml b/llama_stack/templates/vllm-gpu/build.yaml index 8eb44dc1b..93707544d 100644 --- a/llama_stack/templates/vllm-gpu/build.yaml +++ b/llama_stack/templates/vllm-gpu/build.yaml @@ -15,15 +15,9 @@ distribution_spec: - inline::meta-reference telemetry: - inline::meta-reference - eval: - - inline::meta-reference datasetio: - remote::huggingface - inline::localfs - scoring: - - inline::basic - - inline::llm-as-judge - - inline::braintrust tool_runtime: - remote::brave-search - remote::tavily-search diff --git a/llama_stack/templates/vllm-gpu/run.yaml b/llama_stack/templates/vllm-gpu/run.yaml index bf85de0a2..d3c666438 100644 --- a/llama_stack/templates/vllm-gpu/run.yaml +++ b/llama_stack/templates/vllm-gpu/run.yaml @@ -3,10 +3,8 @@ image_name: vllm-gpu apis: - agents - datasetio -- eval - inference - safety -- scoring - telemetry - tool_runtime - vector_io @@ -52,14 +50,6 @@ providers: service_name: ${env.OTEL_SERVICE_NAME:llama-stack} sinks: ${env.TELEMETRY_SINKS:console,sqlite} sqlite_db_path: ${env.SQLITE_DB_PATH:~/.llama/distributions/vllm-gpu/trace_store.db} - eval: - - provider_id: meta-reference - provider_type: inline::meta-reference - config: - kvstore: - type: sqlite - namespace: null - db_path: ${env.SQLITE_STORE_DIR:~/.llama/distributions/vllm-gpu}/meta_reference_eval.db datasetio: - provider_id: huggingface provider_type: remote::huggingface @@ -75,17 +65,6 @@ providers: type: sqlite namespace: null db_path: ${env.SQLITE_STORE_DIR:~/.llama/distributions/vllm-gpu}/localfs_datasetio.db - scoring: - - provider_id: basic - provider_type: inline::basic - config: {} - - provider_id: llm-as-judge - provider_type: inline::llm-as-judge - config: {} - - provider_id: braintrust - provider_type: inline::braintrust - config: - openai_api_key: ${env.OPENAI_API_KEY:} tool_runtime: - provider_id: brave-search provider_type: remote::brave-search @@ -122,7 +101,6 @@ models: shields: [] vector_dbs: [] datasets: [] -scoring_fns: [] benchmarks: [] tool_groups: - toolgroup_id: builtin::websearch diff --git a/llama_stack/templates/vllm-gpu/vllm.py b/llama_stack/templates/vllm-gpu/vllm.py index 8883f117f..2235c8642 100644 --- a/llama_stack/templates/vllm-gpu/vllm.py +++ b/llama_stack/templates/vllm-gpu/vllm.py @@ -25,9 +25,7 @@ def get_distribution_template() -> DistributionTemplate: "safety": ["inline::llama-guard"], "agents": ["inline::meta-reference"], "telemetry": ["inline::meta-reference"], - "eval": ["inline::meta-reference"], "datasetio": ["remote::huggingface", "inline::localfs"], - "scoring": ["inline::basic", "inline::llm-as-judge", "inline::braintrust"], "tool_runtime": [ "remote::brave-search", "remote::tavily-search",