feat: providers use centralized persistence backends

- Add provider_config_resolver to resolve backend references in provider configs
- Providers can now reference persistence.backends instead of duplicating kvstore configs
- Supports kvstore, metadata_store, persistence_store, responses_store references
- Updated ci-tests to use backend references with namespaces
- Eliminates massive config duplication across providers

llama_stack/distributions/ci-tests/run.yaml

@@ -14,7 +14,7 @@ apis:
 - tool_runtime
 - vector_io
 providers:
-    inference:
+  inference:
   - provider_id: ${env.CEREBRAS_API_KEY:+cerebras}
     provider_type: remote::cerebras
     config:
@@ -95,29 +95,29 @@ providers:
     provider_type: inline::faiss
     config:
       kvstore:
-        type: sqlite
-        db_path: ${env.SQLITE_STORE_DIR:=~/.llama/distributions/ci-tests}/faiss_store.db
+        backend: kvstore
+        namespace: faiss
   - provider_id: sqlite-vec
     provider_type: inline::sqlite-vec
     config:
       db_path: ${env.SQLITE_STORE_DIR:=~/.llama/distributions/ci-tests}/sqlite_vec.db
       kvstore:
-        type: sqlite
-        db_path: ${env.SQLITE_STORE_DIR:=~/.llama/distributions/ci-tests}/sqlite_vec_registry.db
+        backend: kvstore
+        namespace: sqlite_vec
   - provider_id: ${env.MILVUS_URL:+milvus}
     provider_type: inline::milvus
     config:
       db_path: ${env.MILVUS_DB_PATH:=~/.llama/distributions/ci-tests}/milvus.db
       kvstore:
-        type: sqlite
-        db_path: ${env.SQLITE_STORE_DIR:=~/.llama/distributions/ci-tests}/milvus_registry.db
+        backend: kvstore
+        namespace: milvus
   - provider_id: ${env.CHROMADB_URL:+chromadb}
     provider_type: remote::chromadb
     config:
       url: ${env.CHROMADB_URL:=}
       kvstore:
-        type: sqlite
-        db_path: ${env.SQLITE_STORE_DIR:=~/.llama/distributions/ci-tests/}/chroma_remote_registry.db
+        backend: kvstore
+        namespace: chromadb
   - provider_id: ${env.PGVECTOR_DB:+pgvector}
     provider_type: remote::pgvector
     config:
@@ -127,16 +127,16 @@ providers:
       user: ${env.PGVECTOR_USER:=}
       password: ${env.PGVECTOR_PASSWORD:=}
       kvstore:
-        type: sqlite
-        db_path: ${env.SQLITE_STORE_DIR:=~/.llama/distributions/ci-tests}/pgvector_registry.db
+        backend: kvstore
+        namespace: pgvector
   files:
   - provider_id: meta-reference-files
     provider_type: inline::localfs
     config:
       storage_dir: ${env.FILES_STORAGE_DIR:=~/.llama/distributions/ci-tests/files}
       metadata_store:
-        type: sqlite
-        db_path: ${env.SQLITE_STORE_DIR:=~/.llama/distributions/ci-tests}/files_metadata.db
+        backend: kvstore
+        namespace: files
   safety:
   - provider_id: llama-guard
     provider_type: inline::llama-guard
@@ -149,11 +149,11 @@ providers:
     provider_type: inline::meta-reference
     config:
       persistence_store:
-        type: sqlite
-        db_path: ${env.SQLITE_STORE_DIR:=~/.llama/distributions/ci-tests}/agents_store.db
+        backend: sqlstore
+        namespace: agents
       responses_store:
-        type: sqlite
-        db_path: ${env.SQLITE_STORE_DIR:=~/.llama/distributions/ci-tests}/responses_store.db
+        backend: sqlstore
+        namespace: responses
   telemetry:
   - provider_id: meta-reference
     provider_type: inline::meta-reference
@@ -172,21 +172,21 @@ providers:
     provider_type: inline::meta-reference
     config:
       kvstore:
-        type: sqlite
-        db_path: ${env.SQLITE_STORE_DIR:=~/.llama/distributions/ci-tests}/meta_reference_eval.db
+        backend: kvstore
+        namespace: eval
   datasetio:
   - provider_id: huggingface
     provider_type: remote::huggingface
     config:
       kvstore:
-        type: sqlite
-        db_path: ${env.SQLITE_STORE_DIR:=~/.llama/distributions/ci-tests}/huggingface_datasetio.db
+        backend: kvstore
+        namespace: huggingface
   - provider_id: localfs
     provider_type: inline::localfs
     config:
       kvstore:
-        type: sqlite
-        db_path: ${env.SQLITE_STORE_DIR:=~/.llama/distributions/ci-tests}/localfs_datasetio.db
+        backend: kvstore
+        namespace: localfs_datasetio
   scoring:
   - provider_id: basic
     provider_type: inline::basic
@@ -216,18 +216,21 @@ providers:
     provider_type: inline::reference
     config:
       kvstore:
-        type: sqlite
-        db_path: ${env.SQLITE_STORE_DIR:=~/.llama/distributions/ci-tests}/batches.db
+        backend: kvstore
+        namespace: batches
 persistence:
   backends:
-    default:
+    kvstore:
       type: sqlite
-      db_path: ${env.SQLITE_STORE_DIR:=~/.llama/distributions/ci-tests}/registry.db
+      db_path: ${env.SQLITE_STORE_DIR:=~/.llama/distributions/ci-tests}/kvstore.db
+    sqlstore:
+      type: sqlite
+      db_path: ${env.SQLITE_STORE_DIR:=~/.llama/distributions/ci-tests}/sqlstore.db
   stores:
     metadata:
-      backend: default
+      backend: kvstore
     inference:
-      backend: default
+      backend: sqlstore
 models: []
 shields:
 - shield_id: llama-guard