Merge branch 'main' into feature/dpo-training

docs/source/advanced_apis/eval/inline_meta-reference.md

@@ -1,3 +1,7 @@
+---
+orphan: true
+---
+
 # inline::meta-reference
 
 ## Description

docs/source/advanced_apis/eval/remote_nvidia.md

@@ -1,3 +1,7 @@
+---
+orphan: true
+---
+
 # remote::nvidia
 
 ## Description

docs/source/advanced_apis/post_training/inline_huggingface.md

@@ -1,3 +1,7 @@
+---
+orphan: true
+---
+
 # inline::huggingface
 
 ## Description

docs/source/advanced_apis/post_training/inline_torchtune.md

@@ -1,3 +1,7 @@
+---
+orphan: true
+---
+
 # inline::torchtune
 
 ## Description

docs/source/advanced_apis/post_training/remote_nvidia.md

@@ -1,3 +1,7 @@
+---
+orphan: true
+---
+
 # remote::nvidia
 
 ## Description

docs/source/advanced_apis/scoring/inline_basic.md

@@ -1,3 +1,7 @@
+---
+orphan: true
+---
+
 # inline::basic
 
 ## Description

docs/source/advanced_apis/scoring/inline_braintrust.md

@@ -1,3 +1,7 @@
+---
+orphan: true
+---
+
 # inline::braintrust
 
 ## Description

docs/source/advanced_apis/scoring/inline_llm-as-judge.md

@@ -1,3 +1,7 @@
+---
+orphan: true
+---
+
 # inline::llm-as-judge
 
 ## Description

docs/source/apis/external.md

@@ -355,7 +355,7 @@ server:
 8. Run the server:
 
 ```bash
-python -m llama_stack.distribution.server.server --yaml-config ~/.llama/run-byoa.yaml
+python -m llama_stack.core.server.server --yaml-config ~/.llama/run-byoa.yaml
 ```
 
 9. Test the API:

docs/source/building_applications/playground/index.md

@@ -103,5 +103,5 @@ llama stack run together
 
 2. Start Streamlit UI
 ```bash
-uv run --with ".[ui]" streamlit run llama_stack/distribution/ui/app.py
+uv run --with ".[ui]" streamlit run llama_stack.core/ui/app.py
 ```

docs/source/contributing/index.md

@@ -11,4 +11,5 @@ See the [Adding a New API Provider](new_api_provider.md) which describes how to
 :hidden:
 
 new_api_provider
+testing
 ```

docs/source/deploying/kubernetes_deployment.md

@@ -174,7 +174,7 @@ spec:
       - name: llama-stack
         image: localhost/llama-stack-run-k8s:latest
         imagePullPolicy: IfNotPresent
-        command: ["python", "-m", "llama_stack.distribution.server.server", "--config", "/app/config.yaml"]
+        command: ["python", "-m", "llama_stack.core.server.server", "--config", "/app/config.yaml"]
         ports:
           - containerPort: 5000
         volumeMounts:

docs/source/distributions/building_distro.md

@@ -59,7 +59,7 @@ Build a Llama stack container
 
 options:
   -h, --help            show this help message and exit
-  --config CONFIG       Path to a config file to use for the build. You can find example configs in llama_stack/distributions/**/build.yaml. If this argument is not provided, you will
+  --config CONFIG       Path to a config file to use for the build. You can find example configs in llama_stack.cores/**/build.yaml. If this argument is not provided, you will
                         be prompted to enter information interactively (default: None)
   --template TEMPLATE   Name of the example template config to use for build. You may use `llama stack build --list-templates` to check out the available templates (default: None)
   --list-templates      Show the available templates for building a Llama Stack distribution (default: False)

docs/source/distributions/importing_as_library.md

@@ -10,7 +10,7 @@ llama stack build --template starter --image-type venv
 ```
 
 ```python
-from llama_stack.distribution.library_client import LlamaStackAsLibraryClient
+from llama_stack.core.library_client import LlamaStackAsLibraryClient
 
 client = LlamaStackAsLibraryClient(
     "starter",

docs/source/distributions/index.md

@@ -9,6 +9,7 @@ This section provides an overview of the distributions available in Llama Stack.
 list_of_distributions
 building_distro
 customizing_run_yaml
+starting_llama_stack_server
 importing_as_library
 configuration
 ```

docs/source/distributions/k8s/stack-configmap.yaml

@@ -34,6 +34,13 @@ data:
         provider_type: remote::chromadb
         config:
           url: ${env.CHROMADB_URL:=}
+          kvstore:
+            type: postgres
+            host: ${env.POSTGRES_HOST:=localhost}
+            port: ${env.POSTGRES_PORT:=5432}
+            db: ${env.POSTGRES_DB:=llamastack}
+            user: ${env.POSTGRES_USER:=llamastack}
+            password: ${env.POSTGRES_PASSWORD:=llamastack}
       safety:
       - provider_id: llama-guard
         provider_type: inline::llama-guard

docs/source/distributions/k8s/stack-k8s.yaml.template

@@ -52,7 +52,7 @@ spec:
           value: "${SAFETY_MODEL}"
         - name: TAVILY_SEARCH_API_KEY
           value: "${TAVILY_SEARCH_API_KEY}"
-        command: ["python", "-m", "llama_stack.distribution.server.server", "--config", "/etc/config/stack_run_config.yaml", "--port", "8321"]
+        command: ["python", "-m", "llama_stack.core.server.server", "--config", "/etc/config/stack_run_config.yaml", "--port", "8321"]
         ports:
           - containerPort: 8321
         volumeMounts:

docs/source/distributions/k8s/stack_run_config.yaml

@@ -31,6 +31,13 @@ providers:
     provider_type: remote::chromadb
     config:
       url: ${env.CHROMADB_URL:=}
+      kvstore:
+        type: postgres
+        host: ${env.POSTGRES_HOST:=localhost}
+        port: ${env.POSTGRES_PORT:=5432}
+        db: ${env.POSTGRES_DB:=llamastack}
+        user: ${env.POSTGRES_USER:=llamastack}
+        password: ${env.POSTGRES_PASSWORD:=llamastack}
   safety:
   - provider_id: llama-guard
     provider_type: inline::llama-guard

docs/source/distributions/self_hosted_distro/starter.md

@@ -100,10 +100,6 @@ The following environment variables can be configured:
 ### Model Configuration
 - `INFERENCE_MODEL`: HuggingFace model for serverless inference
 - `INFERENCE_ENDPOINT_NAME`: HuggingFace endpoint name
-- `OLLAMA_INFERENCE_MODEL`: Ollama model name
-- `OLLAMA_EMBEDDING_MODEL`: Ollama embedding model name
-- `OLLAMA_EMBEDDING_DIMENSION`: Ollama embedding dimension (default: `384`)
-- `VLLM_INFERENCE_MODEL`: vLLM model name
 
 ### Vector Database Configuration
 - `SQLITE_STORE_DIR`: SQLite store directory (default: `~/.llama/distributions/starter`)
@@ -127,43 +123,25 @@ The following environment variables can be configured:
 
 ## Enabling Providers
 
-You can enable specific providers by setting their provider ID to a valid value using environment variables. This is useful when you want to use certain providers or don't have the required API keys.
+You can enable specific providers by setting appropriate environment variables. For example,
 
-### Examples of Enabling Providers
-
-#### Enable FAISS Vector Provider
 ```bash
-export ENABLE_FAISS=faiss
+# self-hosted
+export OLLAMA_URL=http://localhost:11434   # enables the Ollama inference provider
+export VLLM_URL=http://localhost:8000/v1   # enables the vLLM inference provider
+export TGI_URL=http://localhost:8000/v1   # enables the TGI inference provider
+
+# cloud-hosted requiring API key configuration on the server
+export CEREBRAS_API_KEY=your_cerebras_api_key   # enables the Cerebras inference provider
+export NVIDIA_API_KEY=your_nvidia_api_key   # enables the NVIDIA inference provider
+
+# vector providers
+export MILVUS_URL=http://localhost:19530   # enables the Milvus vector provider
+export CHROMADB_URL=http://localhost:8000/v1   # enables the ChromaDB vector provider
+export PGVECTOR_DB=llama_stack_db   # enables the PGVector vector provider
 ```
 
-#### Enable Ollama Models
-```bash
-export ENABLE_OLLAMA=ollama
-```
-
-#### Disable vLLM Models
-```bash
-export VLLM_INFERENCE_MODEL=__disabled__
-```
-
-#### Disable Optional Vector Providers
-```bash
-export ENABLE_SQLITE_VEC=__disabled__
-export ENABLE_CHROMADB=__disabled__
-export ENABLE_PGVECTOR=__disabled__
-```
-
-### Provider ID Patterns
-
-The starter distribution uses several patterns for provider IDs:
-
-1. **Direct provider IDs**: `faiss`, `ollama`, `vllm`
-2. **Environment-based provider IDs**: `${env.ENABLE_SQLITE_VEC:+sqlite-vec}`
-3. **Model-based provider IDs**: `${env.OLLAMA_INFERENCE_MODEL:__disabled__}`
-
-When using the `+` pattern (like `${env.ENABLE_SQLITE_VEC+sqlite-vec}`), the provider is enabled by default and can be disabled by setting the environment variable to `__disabled__`.
-
-When using the `:` pattern (like `${env.OLLAMA_INFERENCE_MODEL:__disabled__}`), the provider is disabled by default and can be enabled by setting the environment variable to a valid value.
+This distribution comes with a default "llama-guard" shield that can be enabled by setting the `SAFETY_MODEL` environment variable to point to an appropriate Llama Guard model id. Use `llama-stack-client models list` to see the list of available models.
 
 ## Running the Distribution
 

docs/source/getting_started/quickstart.md

@@ -16,10 +16,13 @@ as the inference [provider](../providers/inference/index) for a Llama Model.
 ```bash
 ollama run llama3.2:3b --keepalive 60m
 ```
+
 #### Step 2: Run the Llama Stack server
+
 We will use `uv` to run the Llama Stack server.
 ```bash
-uv run --with llama-stack llama stack build --template starter --image-type venv --run
+OLLAMA_URL=http://localhost:11434 \
+  uv run --with llama-stack llama stack build --template starter --image-type venv --run
 ```
 #### Step 3: Run the demo
 Now open up a new terminal and copy the following script into a file named `demo_script.py`.

docs/source/providers/agents/index.md

@@ -1,5 +1,13 @@
-# Agents Providers
+# Agents 
+
+## Overview
 
 This section contains documentation for all available providers for the **agents** API.
 
-- [inline::meta-reference](inline_meta-reference.md)
+## Providers
+
+```{toctree}
+:maxdepth: 1
+
+inline_meta-reference
+```

docs/source/providers/datasetio/index.md

@@ -1,7 +1,15 @@
-# Datasetio Providers
+# Datasetio 
+
+## Overview
 
 This section contains documentation for all available providers for the **datasetio** API.
 
-- [inline::localfs](inline_localfs.md)
-- [remote::huggingface](remote_huggingface.md)
-- [remote::nvidia](remote_nvidia.md)
+## Providers
+
+```{toctree}
+:maxdepth: 1
+
+inline_localfs
+remote_huggingface
+remote_nvidia
+```

docs/source/providers/eval/index.md

@@ -1,6 +1,14 @@
-# Eval Providers
+# Eval 
+
+## Overview
 
 This section contains documentation for all available providers for the **eval** API.
 
-- [inline::meta-reference](inline_meta-reference.md)
-- [remote::nvidia](remote_nvidia.md)
+## Providers
+
+```{toctree}
+:maxdepth: 1
+
+inline_meta-reference
+remote_nvidia
+```

docs/source/providers/files/index.md

@@ -1,5 +1,13 @@
-# Files Providers
+# Files 
+
+## Overview
 
 This section contains documentation for all available providers for the **files** API.
 
-- [inline::localfs](inline_localfs.md)
+## Providers
+
+```{toctree}
+:maxdepth: 1
+
+inline_localfs
+```

docs/source/providers/index.md

@@ -1,4 +1,4 @@
-# API Providers Overview
+# API Providers
 
 The goal of Llama Stack is to build an ecosystem where users can easily swap out different implementations for the same API. Examples for these include:
 - LLM inference providers (e.g., Meta Reference, Ollama, Fireworks, Together, AWS Bedrock, Groq, Cerebras, SambaNova, vLLM, OpenAI, Anthropic, Gemini, WatsonX, etc.),
@@ -12,81 +12,17 @@ Providers come in two flavors:
 
 Importantly, Llama Stack always strives to provide at least one fully inline provider for each API so you can iterate on a fully featured environment locally.
 
-## External Providers
-Llama Stack supports external providers that live outside of the main codebase. This allows you to create and maintain your own providers independently.
-
-```{toctree}
-:maxdepth: 1
-
-external.md
-```
-
-```{include} openai.md
-:start-after: ## OpenAI API Compatibility
-```
-
-## Inference
-Runs inference with an LLM.
-
 ```{toctree}
 :maxdepth: 1
 
+external
+openai
 inference/index
-```
-
-## Agents
-Run multi-step agentic workflows with LLMs with tool usage, memory (RAG), etc.
-
-```{toctree}
-:maxdepth: 1
-
 agents/index
-```
-
-## DatasetIO
-Interfaces with datasets and data loaders.
-
-```{toctree}
-:maxdepth: 1
-
 datasetio/index
-```
-
-## Safety
-Applies safety policies to the output at a Systems (not only model) level.
-
-```{toctree}
-:maxdepth: 1
-
 safety/index
-```
-
-## Telemetry
-Collects telemetry data from the system.
-
-```{toctree}
-:maxdepth: 1
-
 telemetry/index
-```
-
-## Vector IO
-
-Vector IO refers to operations on vector databases, such as adding documents, searching, and deleting documents.
-Vector IO plays a crucial role in [Retreival Augmented Generation (RAG)](../..//building_applications/rag), where the vector
-io and database are used to store and retrieve documents for retrieval.
-
-```{toctree}
-:maxdepth: 1
-
 vector_io/index
-```
-
-## Tool Runtime
-Is associated with the ToolGroup resources.
-
-```{toctree}
-:maxdepth: 1
-
 tool_runtime/index
-```
+files/index
+```

docs/source/providers/inference/index.md

@@ -1,26 +1,34 @@
-# Inference Providers
+# Inference 
+
+## Overview
 
 This section contains documentation for all available providers for the **inference** API.
 
-- [inline::meta-reference](inline_meta-reference.md)
-- [inline::sentence-transformers](inline_sentence-transformers.md)
-- [remote::anthropic](remote_anthropic.md)
-- [remote::bedrock](remote_bedrock.md)
-- [remote::cerebras](remote_cerebras.md)
-- [remote::databricks](remote_databricks.md)
-- [remote::fireworks](remote_fireworks.md)
-- [remote::gemini](remote_gemini.md)
-- [remote::groq](remote_groq.md)
-- [remote::hf::endpoint](remote_hf_endpoint.md)
-- [remote::hf::serverless](remote_hf_serverless.md)
-- [remote::llama-openai-compat](remote_llama-openai-compat.md)
-- [remote::nvidia](remote_nvidia.md)
-- [remote::ollama](remote_ollama.md)
-- [remote::openai](remote_openai.md)
-- [remote::passthrough](remote_passthrough.md)
-- [remote::runpod](remote_runpod.md)
-- [remote::sambanova](remote_sambanova.md)
-- [remote::tgi](remote_tgi.md)
-- [remote::together](remote_together.md)
-- [remote::vllm](remote_vllm.md)
-- [remote::watsonx](remote_watsonx.md)
+## Providers
+
+```{toctree}
+:maxdepth: 1
+
+inline_meta-reference
+inline_sentence-transformers
+remote_anthropic
+remote_bedrock
+remote_cerebras
+remote_databricks
+remote_fireworks
+remote_gemini
+remote_groq
+remote_hf_endpoint
+remote_hf_serverless
+remote_llama-openai-compat
+remote_nvidia
+remote_ollama
+remote_openai
+remote_passthrough
+remote_runpod
+remote_sambanova
+remote_tgi
+remote_together
+remote_vllm
+remote_watsonx
+```

docs/source/providers/inference/remote_cerebras-openai-compat.md

@@ -1,21 +0,0 @@
-# remote::cerebras-openai-compat
-
-## Description
-
-Cerebras OpenAI-compatible provider for using Cerebras models with OpenAI API format.
-
-## Configuration
-
-| Field | Type | Required | Default | Description |
-|-------|------|----------|---------|-------------|
-| `api_key` | `str \| None` | No |  | The Cerebras API key |
-| `openai_compat_api_base` | `<class 'str'>` | No | https://api.cerebras.ai/v1 | The URL for the Cerebras API server |
-
-## Sample Configuration
-
-```yaml
-openai_compat_api_base: https://api.cerebras.ai/v1
-api_key: ${env.CEREBRAS_API_KEY}
-
-```
-

docs/source/providers/inference/remote_fireworks-openai-compat.md

@@ -1,21 +0,0 @@
-# remote::fireworks-openai-compat
-
-## Description
-
-Fireworks AI OpenAI-compatible provider for using Fireworks models with OpenAI API format.
-
-## Configuration
-
-| Field | Type | Required | Default | Description |
-|-------|------|----------|---------|-------------|
-| `api_key` | `str \| None` | No |  | The Fireworks API key |
-| `openai_compat_api_base` | `<class 'str'>` | No | https://api.fireworks.ai/inference/v1 | The URL for the Fireworks API server |
-
-## Sample Configuration
-
-```yaml
-openai_compat_api_base: https://api.fireworks.ai/inference/v1
-api_key: ${env.FIREWORKS_API_KEY}
-
-```
-

docs/source/providers/inference/remote_groq-openai-compat.md

@@ -1,21 +0,0 @@
-# remote::groq-openai-compat
-
-## Description
-
-Groq OpenAI-compatible provider for using Groq models with OpenAI API format.
-
-## Configuration
-
-| Field | Type | Required | Default | Description |
-|-------|------|----------|---------|-------------|
-| `api_key` | `str \| None` | No |  | The Groq API key |
-| `openai_compat_api_base` | `<class 'str'>` | No | https://api.groq.com/openai/v1 | The URL for the Groq API server |
-
-## Sample Configuration
-
-```yaml
-openai_compat_api_base: https://api.groq.com/openai/v1
-api_key: ${env.GROQ_API_KEY}
-
-```
-

docs/source/providers/inference/remote_together-openai-compat.md

@@ -1,21 +0,0 @@
-# remote::together-openai-compat
-
-## Description
-
-Together AI OpenAI-compatible provider for using Together models with OpenAI API format.
-
-## Configuration
-
-| Field | Type | Required | Default | Description |
-|-------|------|----------|---------|-------------|
-| `api_key` | `str \| None` | No |  | The Together API key |
-| `openai_compat_api_base` | `<class 'str'>` | No | https://api.together.xyz/v1 | The URL for the Together API server |
-
-## Sample Configuration
-
-```yaml
-openai_compat_api_base: https://api.together.xyz/v1
-api_key: ${env.TOGETHER_API_KEY}
-
-```
-

docs/source/providers/post_training/index.md

@@ -1,7 +1,15 @@
-# Post_Training Providers
+# Post_Training 
+
+## Overview
 
 This section contains documentation for all available providers for the **post_training** API.
 
-- [inline::huggingface](inline_huggingface.md)
-- [inline::torchtune](inline_torchtune.md)
-- [remote::nvidia](remote_nvidia.md)
+## Providers
+
+```{toctree}
+:maxdepth: 1
+
+inline_huggingface
+inline_torchtune
+remote_nvidia
+```

docs/source/providers/safety/index.md

@@ -1,10 +1,18 @@
-# Safety Providers
+# Safety 
+
+## Overview
 
 This section contains documentation for all available providers for the **safety** API.
 
-- [inline::code-scanner](inline_code-scanner.md)
-- [inline::llama-guard](inline_llama-guard.md)
-- [inline::prompt-guard](inline_prompt-guard.md)
-- [remote::bedrock](remote_bedrock.md)
-- [remote::nvidia](remote_nvidia.md)
-- [remote::sambanova](remote_sambanova.md)
+## Providers
+
+```{toctree}
+:maxdepth: 1
+
+inline_code-scanner
+inline_llama-guard
+inline_prompt-guard
+remote_bedrock
+remote_nvidia
+remote_sambanova
+```

docs/source/providers/scoring/index.md

@@ -1,7 +1,15 @@
-# Scoring Providers
+# Scoring 
+
+## Overview
 
 This section contains documentation for all available providers for the **scoring** API.
 
-- [inline::basic](inline_basic.md)
-- [inline::braintrust](inline_braintrust.md)
-- [inline::llm-as-judge](inline_llm-as-judge.md)
+## Providers
+
+```{toctree}
+:maxdepth: 1
+
+inline_basic
+inline_braintrust
+inline_llm-as-judge
+```

docs/source/providers/telemetry/index.md

@@ -1,5 +1,13 @@
-# Telemetry Providers
+# Telemetry 
+
+## Overview
 
 This section contains documentation for all available providers for the **telemetry** API.
 
-- [inline::meta-reference](inline_meta-reference.md)
+## Providers
+
+```{toctree}
+:maxdepth: 1
+
+inline_meta-reference
+```

docs/source/providers/tool_runtime/index.md

@@ -1,10 +1,18 @@
-# Tool_Runtime Providers
+# Tool_Runtime 
+
+## Overview
 
 This section contains documentation for all available providers for the **tool_runtime** API.
 
-- [inline::rag-runtime](inline_rag-runtime.md)
-- [remote::bing-search](remote_bing-search.md)
-- [remote::brave-search](remote_brave-search.md)
-- [remote::model-context-protocol](remote_model-context-protocol.md)
-- [remote::tavily-search](remote_tavily-search.md)
-- [remote::wolfram-alpha](remote_wolfram-alpha.md)
+## Providers
+
+```{toctree}
+:maxdepth: 1
+
+inline_rag-runtime
+remote_bing-search
+remote_brave-search
+remote_model-context-protocol
+remote_tavily-search
+remote_wolfram-alpha
+```

docs/source/providers/vector_io/index.md

@@ -1,16 +1,24 @@
-# Vector_Io Providers
+# Vector_Io 
+
+## Overview
 
 This section contains documentation for all available providers for the **vector_io** API.
 
-- [inline::chromadb](inline_chromadb.md)
-- [inline::faiss](inline_faiss.md)
-- [inline::meta-reference](inline_meta-reference.md)
-- [inline::milvus](inline_milvus.md)
-- [inline::qdrant](inline_qdrant.md)
-- [inline::sqlite-vec](inline_sqlite-vec.md)
-- [inline::sqlite_vec](inline_sqlite_vec.md)
-- [remote::chromadb](remote_chromadb.md)
-- [remote::milvus](remote_milvus.md)
-- [remote::pgvector](remote_pgvector.md)
-- [remote::qdrant](remote_qdrant.md)
-- [remote::weaviate](remote_weaviate.md)
+## Providers
+
+```{toctree}
+:maxdepth: 1
+
+inline_chromadb
+inline_faiss
+inline_meta-reference
+inline_milvus
+inline_qdrant
+inline_sqlite-vec
+inline_sqlite_vec
+remote_chromadb
+remote_milvus
+remote_pgvector
+remote_qdrant
+remote_weaviate
+```