feat: Enable setting a default embedding model in the stack (#3803)

# What does this PR do?

Enables automatic embedding model detection for vector stores and by
using a `default_configured` boolean that can be defined in the
`run.yaml`.

<!-- If resolving an issue, uncomment and update the line below -->
<!-- Closes #[issue-number] -->

## Test Plan
- Unit tests
- Integration tests
- Simple example below:

Spin up the stack:
```bash
uv run llama stack build --distro starter --image-type venv --run
```
Then test with OpenAI's client:
```python
from openai import OpenAI
client = OpenAI(base_url="http://localhost:8321/v1/", api_key="none")
vs = client.vector_stores.create()
```
Previously you needed:

```python
vs = client.vector_stores.create(
    extra_body={
        "embedding_model": "sentence-transformers/all-MiniLM-L6-v2",
        "embedding_dimension": 384,
    }
)
```

The `extra_body` is now unnecessary.

---------

Signed-off-by: Francisco Javier Arceo <farceo@redhat.com>

tests/unit/core/test_stack_validation.py

@@ -0,0 +1,93 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the terms described in the LICENSE file in
+# the root directory of this source tree.
+
+"""
+Unit tests for Stack validation functions.
+"""
+
+from unittest.mock import AsyncMock
+
+import pytest
+
+from llama_stack.apis.models import Model, ModelType
+from llama_stack.core.stack import validate_default_embedding_model
+from llama_stack.providers.datatypes import Api
+
+
+class TestStackValidation:
+    """Test Stack validation functions."""
+
+    @pytest.mark.parametrize(
+        "models,should_raise",
+        [
+            ([], False),  # No models
+            (
+                [
+                    Model(
+                        identifier="emb1",
+                        model_type=ModelType.embedding,
+                        metadata={"default_configured": True},
+                        provider_id="p",
+                        provider_resource_id="emb1",
+                    )
+                ],
+                False,
+            ),  # Single default
+            (
+                [
+                    Model(
+                        identifier="emb1",
+                        model_type=ModelType.embedding,
+                        metadata={"default_configured": True},
+                        provider_id="p",
+                        provider_resource_id="emb1",
+                    ),
+                    Model(
+                        identifier="emb2",
+                        model_type=ModelType.embedding,
+                        metadata={"default_configured": True},
+                        provider_id="p",
+                        provider_resource_id="emb2",
+                    ),
+                ],
+                True,
+            ),  # Multiple defaults
+            (
+                [
+                    Model(
+                        identifier="emb1",
+                        model_type=ModelType.embedding,
+                        metadata={"default_configured": True},
+                        provider_id="p",
+                        provider_resource_id="emb1",
+                    ),
+                    Model(
+                        identifier="llm1",
+                        model_type=ModelType.llm,
+                        metadata={"default_configured": True},
+                        provider_id="p",
+                        provider_resource_id="llm1",
+                    ),
+                ],
+                False,
+            ),  # Ignores non-embedding
+        ],
+    )
+    async def test_validate_default_embedding_model(self, models, should_raise):
+        """Test validation with various model configurations."""
+        mock_models_impl = AsyncMock()
+        mock_models_impl.list_models.return_value = models
+        impls = {Api.models: mock_models_impl}
+
+        if should_raise:
+            with pytest.raises(ValueError, match="Multiple embedding models marked as default_configured=True"):
+                await validate_default_embedding_model(impls)
+        else:
+            await validate_default_embedding_model(impls)
+
+    async def test_validate_default_embedding_model_no_models_api(self):
+        """Test validation when models API is not available."""
+        await validate_default_embedding_model({})

tests/unit/providers/vector_io/conftest.py

@@ -144,6 +144,7 @@ async def sqlite_vec_adapter(sqlite_vec_db_path, unique_kvstore_config, mock_inf
         config=config,
         inference_api=mock_inference_api,
         files_api=None,
+        models_api=None,
     )
     collection_id = f"sqlite_test_collection_{np.random.randint(1e6)}"
     await adapter.initialize()
@@ -182,6 +183,7 @@ async def faiss_vec_adapter(unique_kvstore_config, mock_inference_api, embedding
         config=config,
         inference_api=mock_inference_api,
         files_api=None,
+        models_api=None,
     )
     await adapter.initialize()
     await adapter.register_vector_db(

tests/unit/providers/vector_io/test_faiss.py

@@ -11,6 +11,7 @@ import numpy as np
 import pytest
 
 from llama_stack.apis.files import Files
+from llama_stack.apis.models import Models
 from llama_stack.apis.vector_dbs import VectorDB
 from llama_stack.apis.vector_io import Chunk, QueryChunksResponse
 from llama_stack.providers.datatypes import HealthStatus
@@ -75,6 +76,12 @@ def mock_files_api():
     return mock_api
 
 
+@pytest.fixture
+def mock_models_api():
+    mock_api = MagicMock(spec=Models)
+    return mock_api
+
+
 @pytest.fixture
 def faiss_config():
     config = MagicMock(spec=FaissVectorIOConfig)
@@ -110,7 +117,7 @@ async def test_faiss_query_vector_returns_infinity_when_query_and_embedding_are_
         assert response.chunks[1] == sample_chunks[1]
 
 
-async def test_health_success():
+async def test_health_success(mock_models_api):
     """Test that the health check returns OK status when faiss is working correctly."""
     # Create a fresh instance of FaissVectorIOAdapter for testing
     config = MagicMock()
@@ -119,7 +126,9 @@ async def test_health_success():
 
     with patch("llama_stack.providers.inline.vector_io.faiss.faiss.faiss.IndexFlatL2") as mock_index_flat:
         mock_index_flat.return_value = MagicMock()
-        adapter = FaissVectorIOAdapter(config=config, inference_api=inference_api, files_api=files_api)
+        adapter = FaissVectorIOAdapter(
+            config=config, inference_api=inference_api, models_api=mock_models_api, files_api=files_api
+        )
 
         # Calling the health method directly
         response = await adapter.health()
@@ -133,7 +142,7 @@ async def test_health_success():
         mock_index_flat.assert_called_once_with(128)  # VECTOR_DIMENSION is 128
 
 
-async def test_health_failure():
+async def test_health_failure(mock_models_api):
     """Test that the health check returns ERROR status when faiss encounters an error."""
     # Create a fresh instance of FaissVectorIOAdapter for testing
     config = MagicMock()
@@ -143,7 +152,9 @@ async def test_health_failure():
     with patch("llama_stack.providers.inline.vector_io.faiss.faiss.faiss.IndexFlatL2") as mock_index_flat:
         mock_index_flat.side_effect = Exception("Test error")
 
-        adapter = FaissVectorIOAdapter(config=config, inference_api=inference_api, files_api=files_api)
+        adapter = FaissVectorIOAdapter(
+            config=config, inference_api=inference_api, models_api=mock_models_api, files_api=files_api
+        )
 
         # Calling the health method directly
         response = await adapter.health()

tests/unit/providers/vector_io/test_vector_io_openai_vector_stores.py

@@ -6,16 +6,18 @@
 
 import json
 import time
-from unittest.mock import AsyncMock, patch
+from unittest.mock import AsyncMock, Mock, patch
 
 import numpy as np
 import pytest
 
 from llama_stack.apis.common.errors import VectorStoreNotFoundError
+from llama_stack.apis.models import Model, ModelType
 from llama_stack.apis.vector_dbs import VectorDB
 from llama_stack.apis.vector_io import (
     Chunk,
     OpenAICreateVectorStoreFileBatchRequestWithExtraBody,
+    OpenAICreateVectorStoreRequestWithExtraBody,
     QueryChunksResponse,
     VectorStoreChunkingStrategyAuto,
     VectorStoreFileObject,
@@ -961,3 +963,93 @@ async def test_max_concurrent_files_per_batch(vector_io_adapter):
     assert batch.status == "in_progress"
     assert batch.file_counts.total == 8
     assert batch.file_counts.in_progress == 8
+
+
+async def test_get_default_embedding_model_success(vector_io_adapter):
+    """Test successful default embedding model detection."""
+    # Mock models API with a default model
+    mock_models_api = Mock()
+    mock_models_api.list_models = AsyncMock(
+        return_value=Mock(
+            data=[
+                Model(
+                    identifier="nomic-embed-text-v1.5",
+                    model_type=ModelType.embedding,
+                    provider_id="test-provider",
+                    metadata={
+                        "embedding_dimension": 768,
+                        "default_configured": True,
+                    },
+                )
+            ]
+        )
+    )
+
+    vector_io_adapter.models_api = mock_models_api
+    result = await vector_io_adapter._get_default_embedding_model_and_dimension()
+
+    assert result is not None
+    model_id, dimension = result
+    assert model_id == "nomic-embed-text-v1.5"
+    assert dimension == 768
+
+
+async def test_get_default_embedding_model_multiple_defaults_error(vector_io_adapter):
+    """Test error when multiple models are marked as default."""
+    mock_models_api = Mock()
+    mock_models_api.list_models = AsyncMock(
+        return_value=Mock(
+            data=[
+                Model(
+                    identifier="model1",
+                    model_type=ModelType.embedding,
+                    provider_id="test-provider",
+                    metadata={"embedding_dimension": 768, "default_configured": True},
+                ),
+                Model(
+                    identifier="model2",
+                    model_type=ModelType.embedding,
+                    provider_id="test-provider",
+                    metadata={"embedding_dimension": 512, "default_configured": True},
+                ),
+            ]
+        )
+    )
+
+    vector_io_adapter.models_api = mock_models_api
+
+    with pytest.raises(ValueError, match="Multiple embedding models marked as default_configured=True"):
+        await vector_io_adapter._get_default_embedding_model_and_dimension()
+
+
+async def test_openai_create_vector_store_uses_default_model(vector_io_adapter):
+    """Test that vector store creation uses default embedding model when none specified."""
+    # Mock models API and dependencies
+    mock_models_api = Mock()
+    mock_models_api.list_models = AsyncMock(
+        return_value=Mock(
+            data=[
+                Model(
+                    identifier="default-model",
+                    model_type=ModelType.embedding,
+                    provider_id="test-provider",
+                    metadata={"embedding_dimension": 512, "default_configured": True},
+                )
+            ]
+        )
+    )
+
+    vector_io_adapter.models_api = mock_models_api
+    vector_io_adapter.register_vector_db = AsyncMock()
+    vector_io_adapter.__provider_id__ = "test-provider"
+
+    # Create vector store without specifying embedding model
+    params = OpenAICreateVectorStoreRequestWithExtraBody(name="test-store")
+    result = await vector_io_adapter.openai_create_vector_store(params)
+
+    # Verify the vector store was created with default model
+    assert result.name == "test-store"
+    vector_io_adapter.register_vector_db.assert_called_once()
+    call_args = vector_io_adapter.register_vector_db.call_args[0][0]
+    assert call_args.embedding_model == "default-model"
+    assert call_args.embedding_dimension == 512