mirror of
https://github.com/meta-llama/llama-stack.git
synced 2025-07-20 19:56:59 +00:00
d994305f0a
# What does this PR do?
currently when running `llama stack run --template starter...` the
__disabled__ providers, their models, etc are printed alongside the
enabled ones making the output really confusing
in server.py add a utility `remove_disabled_providers` which
post-processes the model_dump output to remove any dict with
`provider_id: __disabled__`
we also have `debug` logs printing the disabled providers, so I think
its safe to say that is the only indicator we need when using starter.
<!-- If resolving an issue, uncomment and update the line below -->
<!-- Closes #[issue-number] -->
## Test Plan
before (output truncated because it was huge):
```
...
model_id: ${env.ENABLE_SAMBANOVA:=__disabled__}/sambanova/Llama-3.2-11B-Vision-Instruct
model_type: llm
provider_id: __disabled__
provider_model_id: sambanova/Llama-3.2-11B-Vision-Instruct
- metadata: {}
model_id: ${env.ENABLE_SAMBANOVA:=__disabled__}/meta-llama/Llama-3.2-11B-Vision-Instruct
model_type: llm
provider_id: __disabled__
provider_model_id: sambanova/Llama-3.2-11B-Vision-Instruct
- metadata: {}
model_id: ${env.ENABLE_SAMBANOVA:=__disabled__}/sambanova/Llama-3.2-90B-Vision-Instruct
model_type: llm
provider_id: __disabled__
provider_model_id: sambanova/Llama-3.2-90B-Vision-Instruct
- metadata: {}
model_id: ${env.ENABLE_SAMBANOVA:=__disabled__}/meta-llama/Llama-3.2-90B-Vision-Instruct
model_type: llm
provider_id: __disabled__
provider_model_id: sambanova/Llama-3.2-90B-Vision-Instruct
- metadata: {}
model_id: ${env.ENABLE_SAMBANOVA:=__disabled__}/sambanova/Llama-4-Scout-17B-16E-Instruct
model_type: llm
provider_id: __disabled__
provider_model_id: sambanova/Llama-4-Scout-17B-16E-Instruct
- metadata: {}
model_id: ${env.ENABLE_SAMBANOVA:=__disabled__}/meta-llama/Llama-4-Scout-17B-16E-Instruct
model_type: llm
provider_id: __disabled__
provider_model_id: sambanova/Llama-4-Scout-17B-16E-Instruct
- metadata: {}
model_id: ${env.ENABLE_SAMBANOVA:=__disabled__}/sambanova/Llama-4-Maverick-17B-128E-Instruct
model_type: llm
provider_id: __disabled__
provider_model_id: sambanova/Llama-4-Maverick-17B-128E-Instruct
- metadata: {}
model_id: ${env.ENABLE_SAMBANOVA:=__disabled__}/meta-llama/Llama-4-Maverick-17B-128E-Instruct
model_type: llm
provider_id: __disabled__
provider_model_id: sambanova/Llama-4-Maverick-17B-128E-Instruct
- metadata: {}
model_id: ${env.ENABLE_SAMBANOVA:=__disabled__}/sambanova/Meta-Llama-Guard-3-8B
model_type: llm
provider_id: __disabled__
provider_model_id: sambanova/Meta-Llama-Guard-3-8B
- metadata: {}
model_id: ${env.ENABLE_SAMBANOVA:=__disabled__}/meta-llama/Llama-Guard-3-8B
model_type: llm
provider_id: __disabled__
provider_model_id: sambanova/Meta-Llama-Guard-3-8B
- metadata:
embedding_dimension: 384
model_id: all-MiniLM-L6-v2
model_type: embedding
provider_id: sentence-transformers
provider_model_id: null
providers:
agents:
- config:
persistence_store:
db_path: /Users/charliedoern/.llama/distributions/starter/agents_store.db
type: sqlite
responses_store:
db_path: /Users/charliedoern/.llama/distributions/starter/responses_store.db
type: sqlite
provider_id: meta-reference
provider_type: inline::meta-reference
datasetio:
- config:
kvstore:
db_path: /Users/charliedoern/.llama/distributions/starter/huggingface_datasetio.db
type: sqlite
provider_id: huggingface
provider_type: remote::huggingface
- config:
kvstore:
db_path: /Users/charliedoern/.llama/distributions/starter/localfs_datasetio.db
type: sqlite
provider_id: localfs
provider_type: inline::localfs
eval:
- config:
kvstore:
db_path: /Users/charliedoern/.llama/distributions/starter/meta_reference_eval.db
type: sqlite
provider_id: meta-reference
provider_type: inline::meta-reference
files:
- config:
metadata_store:
db_path: /Users/charliedoern/.llama/distributions/starter/files_metadata.db
type: sqlite
storage_dir: /Users/charliedoern/.llama/distributions/starter/files
provider_id: meta-reference-files
provider_type: inline::localfs
inference:
- config:
api_key: '********'
base_url: https://api.cerebras.ai
provider_id: __disabled__
provider_type: remote::cerebras
- config:
url: http://localhost:11434
provider_id: ollama
provider_type: remote::ollama
- config:
api_token: '********'
max_tokens: ${env.VLLM_MAX_TOKENS:=4096}
tls_verify: ${env.VLLM_TLS_VERIFY:=true}
url: ${env.VLLM_URL}
provider_id: __disabled__
provider_type: remote::vllm
- config:
url: ${env.TGI_URL}
provider_id: __disabled__
provider_type: remote::tgi
- config:
api_token: '********'
huggingface_repo: ${env.INFERENCE_MODEL}
provider_id: __disabled__
provider_type: remote::hf::serverless
- config:
api_token: '********'
endpoint_name: ${env.INFERENCE_ENDPOINT_NAME}
provider_id: __disabled__
provider_type: remote::hf::endpoint
- config:
api_key: '********'
url: https://api.fireworks.ai/inference/v1
provider_id: __disabled__
provider_type: remote::fireworks
- config:
api_key: '********'
url: https://api.together.xyz/v1
provider_id: __disabled__
provider_type: remote::together
- config: {}
provider_id: __disabled__
provider_type: remote::bedrock
- config:
api_token: '********'
url: ${env.DATABRICKS_URL}
provider_id: __disabled__
provider_type: remote::databricks
- config:
api_key: '********'
append_api_version: ${env.NVIDIA_APPEND_API_VERSION:=True}
url: ${env.NVIDIA_BASE_URL:=https://integrate.api.nvidia.com}
provider_id: __disabled__
provider_type: remote::nvidia
- config:
api_token: '********'
url: ${env.RUNPOD_URL:=}
provider_id: __disabled__
provider_type: remote::runpod
- config:
api_key: '********'
provider_id: __disabled__
provider_type: remote::openai
- config:
api_key: '********'
provider_id: __disabled__
provider_type: remote::anthropic
- config:
api_key: '********'
provider_id: __disabled__
provider_type: remote::gemini
- config:
api_key: '********'
url: https://api.groq.com
provider_id: __disabled__
provider_type: remote::groq
- config:
api_key: '********'
openai_compat_api_base: https://api.fireworks.ai/inference/v1
provider_id: __disabled__
provider_type: remote::fireworks-openai-compat
- config:
api_key: '********'
openai_compat_api_base: https://api.llama.com/compat/v1/
provider_id: __disabled__
provider_type: remote::llama-openai-compat
- config:
api_key: '********'
openai_compat_api_base: https://api.together.xyz/v1
provider_id: __disabled__
provider_type: remote::together-openai-compat
- config:
api_key: '********'
openai_compat_api_base: https://api.groq.com/openai/v1
provider_id: __disabled__
provider_type: remote::groq-openai-compat
- config:
api_key: '********'
openai_compat_api_base: https://api.sambanova.ai/v1
provider_id: __disabled__
provider_type: remote::sambanova-openai-compat
- config:
api_key: '********'
openai_compat_api_base: https://api.cerebras.ai/v1
provider_id: __disabled__
provider_type: remote::cerebras-openai-compat
- config:
api_key: '********'
url: https://api.sambanova.ai/v1
provider_id: __disabled__
provider_type: remote::sambanova
- config:
api_key: '********'
url: ${env.PASSTHROUGH_URL}
provider_id: __disabled__
provider_type: remote::passthrough
- config: {}
provider_id: sentence-transformers
provider_type: inline::sentence-transformers
post_training:
- config:
checkpoint_format: huggingface
device: cpu
distributed_backend: null
provider_id: huggingface
provider_type: inline::huggingface
safety:
- config:
excluded_categories: []
provider_id: llama-guard
provider_type: inline::llama-guard
scoring:
- config: {}
provider_id: basic
provider_type: inline::basic
- config: {}
provider_id: llm-as-judge
provider_type: inline::llm-as-judge
- config:
openai_api_key: '********'
provider_id: braintrust
provider_type: inline::braintrust
telemetry:
- config:
otel_exporter_otlp_endpoint: null
service_name: "\u200B"
sinks: console,sqlite
sqlite_db_path: /Users/charliedoern/.llama/distributions/starter/trace_store.db
provider_id: meta-reference
provider_type: inline::meta-reference
tool_runtime:
- config:
api_key: '********'
max_results: 3
provider_id: brave-search
provider_type: remote::brave-search
- config:
api_key: '********'
max_results: 3
provider_id: tavily-search
provider_type: remote::tavily-search
- config: {}
provider_id: rag-runtime
provider_type: inline::rag-runtime
- config: {}
provider_id: model-context-protocol
provider_type: remote::model-context-protocol
vector_io:
- config:
kvstore:
db_path: /Users/charliedoern/.llama/distributions/starter/faiss_store.db
type: sqlite
provider_id: faiss
provider_type: inline::faiss
- config:
db_path: ${env.SQLITE_STORE_DIR:=~/.llama/distributions/starter}/sqlite_vec.db
kvstore:
db_path: ${env.SQLITE_STORE_DIR:=~/.llama/distributions/starter}/sqlite_vec_registry.db
type: sqlite
provider_id: __disabled__
provider_type: inline::sqlite-vec
- config:
db_path: ${env.MILVUS_DB_PATH:=~/.llama/distributions/starter}/milvus.db
kvstore:
db_path: ${env.SQLITE_STORE_DIR:=~/.llama/distributions/starter}/milvus_registry.db
type: sqlite
provider_id: __disabled__
provider_type: inline::milvus
- config:
url: ${env.CHROMADB_URL:=}
provider_id: __disabled__
provider_type: remote::chromadb
- config:
db: ${env.PGVECTOR_DB:=}
host: ${env.PGVECTOR_HOST:=localhost}
kvstore:
db_path: ${env.SQLITE_STORE_DIR:=~/.llama/distributions/starter}/pgvector_registry.db
type: sqlite
password: '********'
port: ${env.PGVECTOR_PORT:=5432}
user: ${env.PGVECTOR_USER:=}
provider_id: __disabled__
provider_type: remote::pgvector
scoring_fns: []
server:
auth: null
host: null
port: 8321
quota: null
tls_cafile: null
tls_certfile: null
tls_keyfile: null
shields:
- params: null
provider_id: null
provider_shield_id: ollama/__disabled__
shield_id: __disabled__
tool_groups:
- args: null
mcp_endpoint: null
provider_id: tavily-search
toolgroup_id: builtin::websearch
- args: null
mcp_endpoint: null
provider_id: rag-runtime
toolgroup_id: builtin::rag
vector_dbs: []
version: 2
```
after:
```
INFO 2025-07-16 13:00:32,604 __main__:448 server: Run configuration:
INFO 2025-07-16 13:00:32,606 __main__:450 server: apis:
- agents
- datasetio
- eval
- files
- inference
- post_training
- safety
- scoring
- telemetry
- tool_runtime
- vector_io
benchmarks: []
datasets: []
image_name: starter
inference_store:
db_path: /Users/charliedoern/.llama/distributions/starter/inference_store.db
type: sqlite
metadata_store:
db_path: /Users/charliedoern/.llama/distributions/starter/registry.db
type: sqlite
models:
- metadata: {}
model_id: ollama/llama3.2:3b
model_type: llm
provider_id: ollama
provider_model_id: llama3.2:3b
- metadata:
embedding_dimension: 384
model_id: all-MiniLM-L6-v2
model_type: embedding
provider_id: sentence-transformers
providers:
agents:
- config:
persistence_store:
db_path: /Users/charliedoern/.llama/distributions/starter/agents_store.db
type: sqlite
responses_store:
db_path: /Users/charliedoern/.llama/distributions/starter/responses_store.db
type: sqlite
provider_id: meta-reference
provider_type: inline::meta-reference
datasetio:
- config:
kvstore:
db_path: /Users/charliedoern/.llama/distributions/starter/huggingface_datasetio.db
type: sqlite
provider_id: huggingface
provider_type: remote::huggingface
- config:
kvstore:
db_path: /Users/charliedoern/.llama/distributions/starter/localfs_datasetio.db
type: sqlite
provider_id: localfs
provider_type: inline::localfs
eval:
- config:
kvstore:
db_path: /Users/charliedoern/.llama/distributions/starter/meta_reference_eval.db
type: sqlite
provider_id: meta-reference
provider_type: inline::meta-reference
files:
- config:
metadata_store:
db_path: /Users/charliedoern/.llama/distributions/starter/files_metadata.db
type: sqlite
storage_dir: /Users/charliedoern/.llama/distributions/starter/files
provider_id: meta-reference-files
provider_type: inline::localfs
inference:
- config:
url: http://localhost:11434
provider_id: ollama
provider_type: remote::ollama
- config: {}
provider_id: sentence-transformers
provider_type: inline::sentence-transformers
post_training:
- config:
checkpoint_format: huggingface
device: cpu
provider_id: huggingface
provider_type: inline::huggingface
safety:
- config:
excluded_categories: []
provider_id: llama-guard
provider_type: inline::llama-guard
scoring:
- config: {}
provider_id: basic
provider_type: inline::basic
- config: {}
provider_id: llm-as-judge
provider_type: inline::llm-as-judge
- config:
openai_api_key: '********'
provider_id: braintrust
provider_type: inline::braintrust
telemetry:
- config:
service_name: "\u200B"
sinks: console,sqlite
sqlite_db_path: /Users/charliedoern/.llama/distributions/starter/trace_store.db
provider_id: meta-reference
provider_type: inline::meta-reference
tool_runtime:
- config:
api_key: '********'
max_results: 3
provider_id: brave-search
provider_type: remote::brave-search
- config:
api_key: '********'
max_results: 3
provider_id: tavily-search
provider_type: remote::tavily-search
- config: {}
provider_id: rag-runtime
provider_type: inline::rag-runtime
- config: {}
provider_id: model-context-protocol
provider_type: remote::model-context-protocol
vector_io:
- config:
kvstore:
db_path: /Users/charliedoern/.llama/distributions/starter/faiss_store.db
type: sqlite
provider_id: faiss
provider_type: inline::faiss
scoring_fns: []
server:
port: 8321
shields: []
tool_groups:
- provider_id: tavily-search
toolgroup_id: builtin::websearch
- provider_id: rag-runtime
toolgroup_id: builtin::rag
vector_dbs: []
version: 2
```
Signed-off-by: Charlie Doern <cdoern@redhat.com>
|
||
|---|---|---|
| .. | ||
| access_control | ||
| routers | ||
| routing_tables | ||
| server | ||
| store | ||
| ui | ||
| utils | ||
| __init__.py | ||
| build.py | ||
| build_conda_env.sh | ||
| build_container.sh | ||
| build_venv.sh | ||
| client.py | ||
| common.sh | ||
| configure.py | ||
| datatypes.py | ||
| distribution.py | ||
| inspect.py | ||
| library_client.py | ||
| providers.py | ||
| request_headers.py | ||
| resolver.py | ||
| stack.py | ||
| start_stack.sh | ||