feat!: Migrate Vector DB IDs to Vector Store IDs (breaking change) (#3253)

# What does this PR do?
This change migrates the VectorDB id generation to Vector Stores.

This is a breaking change for **_some users_** that may have application
code using the `vector_db_id` parameter in the request of the VectorDB
protocol instead of the `VectorDB.identifier` in the response.

By default we will now create a Vector Store every time we register a
VectorDB. The caveat with this approach is that this maps the
`vector_db_id` → `vector_store.name`. This is a reasonable tradeoff to
transition users towards OpenAI Vector Stores.

As an added benefit, registering VectorDBs will result in them appearing
in the VectorStores admin UI.

### Why?
This PR makes the `POST` API call to `/v1/vector-dbs` swap the
`vector_db_id` parameter in the **request body** into the VectorStore's
name field and sets the `vector_db_id` to the generated vector store id
(e.g., `vs_038247dd-4bbb-4dbb-a6be-d5ecfd46cfdb`).

That means that users would have to do something like follows in their
application code:

```python
res = client.vector_dbs.register(
    vector_db_id='my-vector-db-id', 
    embedding_model='ollama/all-minilm:l6-v2', 
    embedding_dimension=384,
)
vector_db_id = res.identifier
```

And then the rest of their code would behave, including `VectorIO`'s
insert protocol using `vector_db_id` in the request.

An alternative implementation would be to just delete the `vector_db_id`
parameter in `VectorDB` but the end result would still require users
having to write `vector_db_id = res.identifier` since
`VectorStores.create()` generates the ID for you.

So this approach felt the easiest way to migrate users towards
VectorStores (subsequent PRs will be added to trigger `files.create()`
and `vector_stores.files.create()`).

## Test Plan
Unit tests and integration tests have been added.

Signed-off-by: Francisco Javier Arceo <farceo@redhat.com>

tests/integration/vector_io/test_vector_io.py

@@ -47,34 +47,45 @@ def client_with_empty_registry(client_with_models):
 
 
 def test_vector_db_retrieve(client_with_empty_registry, embedding_model_id, embedding_dimension):
-    # Register a memory bank first
-    vector_db_id = "test_vector_db"
-    client_with_empty_registry.vector_dbs.register(
-        vector_db_id=vector_db_id,
+    vector_db_name = "test_vector_db"
+    register_response = client_with_empty_registry.vector_dbs.register(
+        vector_db_id=vector_db_name,
         embedding_model=embedding_model_id,
         embedding_dimension=embedding_dimension,
     )
 
+    actual_vector_db_id = register_response.identifier
+
     # Retrieve the memory bank and validate its properties
-    response = client_with_empty_registry.vector_dbs.retrieve(vector_db_id=vector_db_id)
+    response = client_with_empty_registry.vector_dbs.retrieve(vector_db_id=actual_vector_db_id)
     assert response is not None
-    assert response.identifier == vector_db_id
+    assert response.identifier == actual_vector_db_id
     assert response.embedding_model == embedding_model_id
-    assert response.provider_resource_id == vector_db_id
+    assert response.identifier.startswith("vs_")
 
 
 def test_vector_db_register(client_with_empty_registry, embedding_model_id, embedding_dimension):
-    vector_db_id = "test_vector_db"
-    client_with_empty_registry.vector_dbs.register(
-        vector_db_id=vector_db_id,
+    vector_db_name = "test_vector_db"
+    response = client_with_empty_registry.vector_dbs.register(
+        vector_db_id=vector_db_name,
         embedding_model=embedding_model_id,
         embedding_dimension=embedding_dimension,
     )
 
-    vector_dbs_after_register = [vector_db.identifier for vector_db in client_with_empty_registry.vector_dbs.list()]
-    assert vector_dbs_after_register == [vector_db_id]
+    actual_vector_db_id = response.identifier
+    assert actual_vector_db_id.startswith("vs_")
+    assert actual_vector_db_id != vector_db_name
 
-    client_with_empty_registry.vector_dbs.unregister(vector_db_id=vector_db_id)
+    vector_dbs_after_register = [vector_db.identifier for vector_db in client_with_empty_registry.vector_dbs.list()]
+    assert vector_dbs_after_register == [actual_vector_db_id]
+
+    vector_stores = client_with_empty_registry.vector_stores.list()
+    assert len(vector_stores.data) == 1
+    vector_store = vector_stores.data[0]
+    assert vector_store.id == actual_vector_db_id
+    assert vector_store.name == vector_db_name
+
+    client_with_empty_registry.vector_dbs.unregister(vector_db_id=actual_vector_db_id)
 
     vector_dbs = [vector_db.identifier for vector_db in client_with_empty_registry.vector_dbs.list()]
     assert len(vector_dbs) == 0
@@ -91,20 +102,22 @@ def test_vector_db_register(client_with_empty_registry, embedding_model_id, embe
     ],
 )
 def test_insert_chunks(client_with_empty_registry, embedding_model_id, embedding_dimension, sample_chunks, test_case):
-    vector_db_id = "test_vector_db"
-    client_with_empty_registry.vector_dbs.register(
-        vector_db_id=vector_db_id,
+    vector_db_name = "test_vector_db"
+    register_response = client_with_empty_registry.vector_dbs.register(
+        vector_db_id=vector_db_name,
         embedding_model=embedding_model_id,
         embedding_dimension=embedding_dimension,
     )
 
+    actual_vector_db_id = register_response.identifier
+
     client_with_empty_registry.vector_io.insert(
-        vector_db_id=vector_db_id,
+        vector_db_id=actual_vector_db_id,
         chunks=sample_chunks,
     )
 
     response = client_with_empty_registry.vector_io.query(
-        vector_db_id=vector_db_id,
+        vector_db_id=actual_vector_db_id,
         query="What is the capital of France?",
     )
     assert response is not None
@@ -113,7 +126,7 @@ def test_insert_chunks(client_with_empty_registry, embedding_model_id, embedding
 
     query, expected_doc_id = test_case
     response = client_with_empty_registry.vector_io.query(
-        vector_db_id=vector_db_id,
+        vector_db_id=actual_vector_db_id,
         query=query,
     )
     assert response is not None
@@ -128,13 +141,15 @@ def test_insert_chunks_with_precomputed_embeddings(client_with_empty_registry, e
         "remote::qdrant": {"score_threshold": -1.0},
         "inline::qdrant": {"score_threshold": -1.0},
     }
-    vector_db_id = "test_precomputed_embeddings_db"
-    client_with_empty_registry.vector_dbs.register(
-        vector_db_id=vector_db_id,
+    vector_db_name = "test_precomputed_embeddings_db"
+    register_response = client_with_empty_registry.vector_dbs.register(
+        vector_db_id=vector_db_name,
         embedding_model=embedding_model_id,
         embedding_dimension=embedding_dimension,
     )
 
+    actual_vector_db_id = register_response.identifier
+
     chunks_with_embeddings = [
         Chunk(
             content="This is a test chunk with precomputed embedding.",
@@ -144,13 +159,13 @@ def test_insert_chunks_with_precomputed_embeddings(client_with_empty_registry, e
     ]
 
     client_with_empty_registry.vector_io.insert(
-        vector_db_id=vector_db_id,
+        vector_db_id=actual_vector_db_id,
         chunks=chunks_with_embeddings,
     )
 
     provider = [p.provider_id for p in client_with_empty_registry.providers.list() if p.api == "vector_io"][0]
     response = client_with_empty_registry.vector_io.query(
-        vector_db_id=vector_db_id,
+        vector_db_id=actual_vector_db_id,
         query="precomputed embedding test",
         params=vector_io_provider_params_dict.get(provider, None),
     )
@@ -173,13 +188,15 @@ def test_query_returns_valid_object_when_identical_to_embedding_in_vdb(
         "remote::qdrant": {"score_threshold": 0.0},
         "inline::qdrant": {"score_threshold": 0.0},
     }
-    vector_db_id = "test_precomputed_embeddings_db"
-    client_with_empty_registry.vector_dbs.register(
-        vector_db_id=vector_db_id,
+    vector_db_name = "test_precomputed_embeddings_db"
+    register_response = client_with_empty_registry.vector_dbs.register(
+        vector_db_id=vector_db_name,
         embedding_model=embedding_model_id,
         embedding_dimension=embedding_dimension,
     )
 
+    actual_vector_db_id = register_response.identifier
+
     chunks_with_embeddings = [
         Chunk(
             content="duplicate",
@@ -189,13 +206,13 @@ def test_query_returns_valid_object_when_identical_to_embedding_in_vdb(
     ]
 
     client_with_empty_registry.vector_io.insert(
-        vector_db_id=vector_db_id,
+        vector_db_id=actual_vector_db_id,
         chunks=chunks_with_embeddings,
     )
 
     provider = [p.provider_id for p in client_with_empty_registry.providers.list() if p.api == "vector_io"][0]
     response = client_with_empty_registry.vector_io.query(
-        vector_db_id=vector_db_id,
+        vector_db_id=actual_vector_db_id,
         query="duplicate",
         params=vector_io_provider_params_dict.get(provider, None),
     )