feat: Add missing Vector Store Files API surface (#2468)

# What does this PR do?

This adds the ability to list, retrieve, update, and delete Vector Store
Files. It implements these new APIs for the faiss and sqlite-vec
providers, since those are the two that also have the rest of the vector
store files implementation.

Closes #2445 

## Test Plan

### test_openai_vector_stores Integration Tests

There are a number of new integration tests added, which I ran for each
provider as outlined below.

faiss (from ollama distro):

```
INFERENCE_MODEL="meta-llama/Llama-3.2-3B-Instruct" \
llama stack run llama_stack/templates/ollama/run.yaml

LLAMA_STACK_CONFIG=http://localhost:8321 \
pytest -sv tests/integration/vector_io/test_openai_vector_stores.py \
  --embedding-model=all-MiniLM-L6-v2
```

sqlite-vec (from starter distro):

```
llama stack run llama_stack/templates/starter/run.yaml

LLAMA_STACK_CONFIG=http://localhost:8321 \
pytest -sv tests/integration/vector_io/test_openai_vector_stores.py \
  --embedding-model=all-MiniLM-L6-v2
```

### file_search verification tests

I also ensured the file_search verification tests continue to work, both
for faiss and sqlite-vec.

faiss (ollama distro):

```
INFERENCE_MODEL="meta-llama/Llama-3.2-3B-Instruct" \
llama stack run llama_stack/templates/ollama/run.yaml

pytest -sv tests/verifications/openai_api/test_responses.py \
  -k'file_search' \
  --base-url=http://localhost:8321/v1/openai/v1 \
  --model=meta-llama/Llama-3.2-3B-Instruct
```


sqlite-vec (starter distro):

```
llama stack run llama_stack/templates/starter/run.yaml

pytest -sv tests/verifications/openai_api/test_responses.py \
  -k'file_search' \
  --base-url=http://localhost:8321/v1/openai/v1 \
  --model=together/meta-llama/Llama-3.2-3B-Instruct-Turbo
```

---------

Signed-off-by: Ben Browning <bbrownin@redhat.com>

tests/integration/vector_io/test_openai_vector_stores.py

@@ -6,8 +6,11 @@
 
 import logging
 import time
+from io import BytesIO
 
 import pytest
+from llama_stack_client import BadRequestError, LlamaStackClient
+from openai import BadRequestError as OpenAIBadRequestError
 from openai import OpenAI
 
 from llama_stack.apis.vector_io import Chunk
@@ -73,11 +76,23 @@ def compat_client_with_empty_stores(compat_client):
             logger.warning("Failed to clear vector stores")
             pass
 
+    def clear_files():
+        try:
+            response = compat_client.files.list()
+            for file in response.data:
+                compat_client.files.delete(file_id=file.id)
+        except Exception:
+            # If the API is not available or fails, just continue
+            logger.warning("Failed to clear files")
+            pass
+
     clear_vector_stores()
+    clear_files()
     yield compat_client
 
     # Clean up after the test
     clear_vector_stores()
+    clear_files()
 
 
 def test_openai_create_vector_store(compat_client_with_empty_stores, client_with_models):
@@ -423,3 +438,369 @@ def test_openai_vector_store_search_with_max_num_results(
 
     assert search_response is not None
     assert len(search_response.data) == 2
+
+
+def test_openai_vector_store_attach_file(compat_client_with_empty_stores, client_with_models):
+    """Test OpenAI vector store attach file."""
+    skip_if_provider_doesnt_support_openai_vector_stores(client_with_models)
+
+    if isinstance(compat_client_with_empty_stores, LlamaStackClient):
+        pytest.skip("Vector Store Files attach is not yet supported with LlamaStackClient")
+
+    compat_client = compat_client_with_empty_stores
+
+    # Create a vector store
+    vector_store = compat_client.vector_stores.create(name="test_store")
+
+    # Create a file
+    test_content = b"The secret string is foobazbar."
+    with BytesIO(test_content) as file_buffer:
+        file_buffer.name = "openai_test.txt"
+        file = compat_client.files.create(file=file_buffer, purpose="assistants")
+
+    # Attach the file to the vector store
+    file_attach_response = compat_client.vector_stores.files.create(
+        vector_store_id=vector_store.id,
+        file_id=file.id,
+    )
+
+    assert file_attach_response
+    assert file_attach_response.object == "vector_store.file"
+    assert file_attach_response.id == file.id
+    assert file_attach_response.vector_store_id == vector_store.id
+    assert file_attach_response.status == "completed"
+    assert file_attach_response.chunking_strategy.type == "auto"
+    assert file_attach_response.created_at > 0
+    assert not file_attach_response.last_error
+
+    updated_vector_store = compat_client.vector_stores.retrieve(vector_store_id=vector_store.id)
+    assert updated_vector_store.file_counts.completed == 1
+    assert updated_vector_store.file_counts.total == 1
+    assert updated_vector_store.file_counts.cancelled == 0
+    assert updated_vector_store.file_counts.failed == 0
+    assert updated_vector_store.file_counts.in_progress == 0
+
+    # Search using OpenAI API to confirm our file attached
+    search_response = compat_client.vector_stores.search(
+        vector_store_id=vector_store.id, query="What is the secret string?", max_num_results=1
+    )
+    assert search_response is not None
+    assert len(search_response.data) > 0
+    top_result = search_response.data[0]
+    top_content = top_result.content[0].text
+    assert "foobazbar" in top_content.lower()
+
+
+def test_openai_vector_store_attach_files_on_creation(compat_client_with_empty_stores, client_with_models):
+    """Test OpenAI vector store attach files on creation."""
+    skip_if_provider_doesnt_support_openai_vector_stores(client_with_models)
+
+    if isinstance(compat_client_with_empty_stores, LlamaStackClient):
+        pytest.skip("Vector Store Files attach is not yet supported with LlamaStackClient")
+
+    compat_client = compat_client_with_empty_stores
+
+    # Create some files and attach them to the vector store
+    valid_file_ids = []
+    for i in range(3):
+        with BytesIO(f"This is a test file {i}".encode()) as file_buffer:
+            file_buffer.name = f"openai_test_{i}.txt"
+            file = compat_client.files.create(file=file_buffer, purpose="assistants")
+        valid_file_ids.append(file.id)
+
+    # include an invalid file ID so we can test failed status
+    failed_file_id = "invalid_file_id"
+    file_ids = valid_file_ids + [failed_file_id]
+    num_failed = len(file_ids) - len(valid_file_ids)
+
+    # Create a vector store
+    vector_store = compat_client.vector_stores.create(
+        name="test_store",
+        file_ids=file_ids,
+    )
+
+    assert vector_store.file_counts.completed == len(valid_file_ids)
+    assert vector_store.file_counts.total == len(file_ids)
+    assert vector_store.file_counts.cancelled == 0
+    assert vector_store.file_counts.failed == num_failed
+    assert vector_store.file_counts.in_progress == 0
+
+    files_list = compat_client.vector_stores.files.list(vector_store_id=vector_store.id)
+    assert len(files_list.data) == len(file_ids)
+    assert set(file_ids) == {file.id for file in files_list.data}
+    for file in files_list.data:
+        if file.id in valid_file_ids:
+            assert file.status == "completed"
+        else:
+            assert file.status == "failed"
+
+    failed_list = compat_client.vector_stores.files.list(vector_store_id=vector_store.id, filter="failed")
+    assert len(failed_list.data) == num_failed
+    assert failed_file_id == failed_list.data[0].id
+
+    # Delete the invalid file
+    delete_response = compat_client.vector_stores.files.delete(vector_store_id=vector_store.id, file_id=failed_file_id)
+    assert delete_response.id == failed_file_id
+
+    updated_vector_store = compat_client.vector_stores.retrieve(vector_store_id=vector_store.id)
+    assert updated_vector_store.file_counts.completed == len(valid_file_ids)
+    assert updated_vector_store.file_counts.total == len(valid_file_ids)
+    assert updated_vector_store.file_counts.failed == 0
+
+
+def test_openai_vector_store_list_files(compat_client_with_empty_stores, client_with_models):
+    """Test OpenAI vector store list files."""
+    skip_if_provider_doesnt_support_openai_vector_stores(client_with_models)
+
+    if isinstance(compat_client_with_empty_stores, LlamaStackClient):
+        pytest.skip("Vector Store Files list is not yet supported with LlamaStackClient")
+
+    compat_client = compat_client_with_empty_stores
+
+    # Create a vector store
+    vector_store = compat_client.vector_stores.create(name="test_store")
+
+    # Create some files and attach them to the vector store
+    file_ids = []
+    for i in range(3):
+        with BytesIO(f"This is a test file {i}".encode()) as file_buffer:
+            file_buffer.name = f"openai_test_{i}.txt"
+            file = compat_client.files.create(file=file_buffer, purpose="assistants")
+
+        compat_client.vector_stores.files.create(
+            vector_store_id=vector_store.id,
+            file_id=file.id,
+        )
+        file_ids.append(file.id)
+
+    files_list = compat_client.vector_stores.files.list(vector_store_id=vector_store.id)
+    assert files_list
+    assert files_list.object == "list"
+    assert files_list.data
+    assert not files_list.has_more
+    assert len(files_list.data) == 3
+    assert set(file_ids) == {file.id for file in files_list.data}
+    assert files_list.data[0].object == "vector_store.file"
+    assert files_list.data[0].vector_store_id == vector_store.id
+    assert files_list.data[0].status == "completed"
+    assert files_list.data[0].chunking_strategy.type == "auto"
+    assert files_list.data[0].created_at > 0
+    assert files_list.first_id == files_list.data[0].id
+    assert not files_list.data[0].last_error
+
+    first_page = compat_client.vector_stores.files.list(vector_store_id=vector_store.id, limit=2)
+    assert first_page.has_more
+    assert len(first_page.data) == 2
+    assert first_page.first_id == first_page.data[0].id
+    assert first_page.last_id != first_page.data[-1].id
+
+    next_page = compat_client.vector_stores.files.list(
+        vector_store_id=vector_store.id, limit=2, after=first_page.data[-1].id
+    )
+    assert not next_page.has_more
+    assert len(next_page.data) == 1
+
+    updated_vector_store = compat_client.vector_stores.retrieve(vector_store_id=vector_store.id)
+    assert updated_vector_store.file_counts.completed == 3
+    assert updated_vector_store.file_counts.total == 3
+    assert updated_vector_store.file_counts.cancelled == 0
+    assert updated_vector_store.file_counts.failed == 0
+    assert updated_vector_store.file_counts.in_progress == 0
+
+
+def test_openai_vector_store_list_files_invalid_vector_store(compat_client_with_empty_stores, client_with_models):
+    """Test OpenAI vector store list files with invalid vector store ID."""
+    skip_if_provider_doesnt_support_openai_vector_stores(client_with_models)
+
+    if isinstance(compat_client_with_empty_stores, LlamaStackClient):
+        pytest.skip("Vector Store Files list is not yet supported with LlamaStackClient")
+
+    compat_client = compat_client_with_empty_stores
+
+    with pytest.raises((BadRequestError, OpenAIBadRequestError)):
+        compat_client.vector_stores.files.list(vector_store_id="abc123")
+
+
+def test_openai_vector_store_retrieve_file_contents(compat_client_with_empty_stores, client_with_models):
+    """Test OpenAI vector store retrieve file contents."""
+    skip_if_provider_doesnt_support_openai_vector_stores(client_with_models)
+
+    if isinstance(compat_client_with_empty_stores, LlamaStackClient):
+        pytest.skip("Vector Store Files retrieve contents is not yet supported with LlamaStackClient")
+
+    compat_client = compat_client_with_empty_stores
+
+    # Create a vector store
+    vector_store = compat_client.vector_stores.create(name="test_store")
+
+    # Create a file
+    test_content = b"This is a test file"
+    file_name = "openai_test.txt"
+    attributes = {"foo": "bar"}
+    with BytesIO(test_content) as file_buffer:
+        file_buffer.name = file_name
+        file = compat_client.files.create(file=file_buffer, purpose="assistants")
+
+    # Attach the file to the vector store
+    file_attach_response = compat_client.vector_stores.files.create(
+        vector_store_id=vector_store.id,
+        file_id=file.id,
+        attributes=attributes,
+    )
+
+    assert file_attach_response.status == "completed"
+
+    file_contents = compat_client.vector_stores.files.content(
+        vector_store_id=vector_store.id,
+        file_id=file.id,
+    )
+
+    assert file_contents
+    assert file_contents.content[0]["type"] == "text"
+    assert file_contents.content[0]["text"] == test_content.decode("utf-8")
+    assert file_contents.filename == file_name
+    assert file_contents.attributes == attributes
+
+
+def test_openai_vector_store_delete_file(compat_client_with_empty_stores, client_with_models):
+    """Test OpenAI vector store delete file."""
+    skip_if_provider_doesnt_support_openai_vector_stores(client_with_models)
+
+    if isinstance(compat_client_with_empty_stores, LlamaStackClient):
+        pytest.skip("Vector Store Files list is not yet supported with LlamaStackClient")
+
+    compat_client = compat_client_with_empty_stores
+
+    # Create a vector store
+    vector_store = compat_client.vector_stores.create(name="test_store")
+
+    # Create some files and attach them to the vector store
+    file_ids = []
+    for i in range(3):
+        with BytesIO(f"This is a test file {i}".encode()) as file_buffer:
+            file_buffer.name = f"openai_test_{i}.txt"
+            file = compat_client.files.create(file=file_buffer, purpose="assistants")
+
+        compat_client.vector_stores.files.create(
+            vector_store_id=vector_store.id,
+            file_id=file.id,
+        )
+        file_ids.append(file.id)
+
+    files_list = compat_client.vector_stores.files.list(vector_store_id=vector_store.id)
+    assert len(files_list.data) == 3
+
+    # Delete the first file
+    delete_response = compat_client.vector_stores.files.delete(vector_store_id=vector_store.id, file_id=file_ids[0])
+    assert delete_response
+    assert delete_response.id == file_ids[0]
+    assert delete_response.deleted is True
+    assert delete_response.object == "vector_store.file.deleted"
+
+    updated_vector_store = compat_client.vector_stores.retrieve(vector_store_id=vector_store.id)
+    assert updated_vector_store.file_counts.completed == 2
+    assert updated_vector_store.file_counts.total == 2
+    assert updated_vector_store.file_counts.cancelled == 0
+    assert updated_vector_store.file_counts.failed == 0
+    assert updated_vector_store.file_counts.in_progress == 0
+
+    # Delete the second file
+    delete_response = compat_client.vector_stores.files.delete(vector_store_id=vector_store.id, file_id=file_ids[1])
+    assert delete_response
+    assert delete_response.id == file_ids[1]
+
+    updated_vector_store = compat_client.vector_stores.retrieve(vector_store_id=vector_store.id)
+    assert updated_vector_store.file_counts.completed == 1
+    assert updated_vector_store.file_counts.total == 1
+    assert updated_vector_store.file_counts.cancelled == 0
+    assert updated_vector_store.file_counts.failed == 0
+    assert updated_vector_store.file_counts.in_progress == 0
+
+
+# TODO: Remove this xfail once we have a way to remove embeddings from vector store
+@pytest.mark.xfail(reason="Vector Store Files delete doesn't remove embeddings from vecntor store", strict=True)
+def test_openai_vector_store_delete_file_removes_from_vector_store(compat_client_with_empty_stores, client_with_models):
+    """Test OpenAI vector store delete file removes from vector store."""
+    skip_if_provider_doesnt_support_openai_vector_stores(client_with_models)
+
+    if isinstance(compat_client_with_empty_stores, LlamaStackClient):
+        pytest.skip("Vector Store Files attach is not yet supported with LlamaStackClient")
+
+    compat_client = compat_client_with_empty_stores
+
+    # Create a vector store
+    vector_store = compat_client.vector_stores.create(name="test_store")
+
+    # Create a file
+    test_content = b"The secret string is foobazbar."
+    with BytesIO(test_content) as file_buffer:
+        file_buffer.name = "openai_test.txt"
+        file = compat_client.files.create(file=file_buffer, purpose="assistants")
+
+    # Attach the file to the vector store
+    file_attach_response = compat_client.vector_stores.files.create(
+        vector_store_id=vector_store.id,
+        file_id=file.id,
+    )
+    assert file_attach_response.status == "completed"
+
+    # Search using OpenAI API to confirm our file attached
+    search_response = compat_client.vector_stores.search(
+        vector_store_id=vector_store.id, query="What is the secret string?", max_num_results=1
+    )
+    assert "foobazbar" in search_response.data[0].content[0].text.lower()
+
+    # Delete the file
+    compat_client.vector_stores.files.delete(vector_store_id=vector_store.id, file_id=file.id)
+
+    # Search using OpenAI API to confirm our file deleted
+    search_response = compat_client.vector_stores.search(
+        vector_store_id=vector_store.id, query="What is the secret string?", max_num_results=1
+    )
+    assert not search_response.data
+
+
+def test_openai_vector_store_update_file(compat_client_with_empty_stores, client_with_models):
+    """Test OpenAI vector store update file."""
+    skip_if_provider_doesnt_support_openai_vector_stores(client_with_models)
+
+    if isinstance(compat_client_with_empty_stores, LlamaStackClient):
+        pytest.skip("Vector Store Files update is not yet supported with LlamaStackClient")
+
+    compat_client = compat_client_with_empty_stores
+
+    # Create a vector store
+    vector_store = compat_client.vector_stores.create(name="test_store")
+
+    # Create a file
+    test_content = b"This is a test file"
+    with BytesIO(test_content) as file_buffer:
+        file_buffer.name = "openai_test.txt"
+        file = compat_client.files.create(file=file_buffer, purpose="assistants")
+
+    # Attach the file to the vector store
+    file_attach_response = compat_client.vector_stores.files.create(
+        vector_store_id=vector_store.id,
+        file_id=file.id,
+        attributes={"foo": "bar"},
+    )
+
+    assert file_attach_response.status == "completed"
+    assert file_attach_response.attributes["foo"] == "bar"
+
+    # Update the file's attributes
+    updated_response = compat_client.vector_stores.files.update(
+        vector_store_id=vector_store.id,
+        file_id=file.id,
+        attributes={"foo": "baz"},
+    )
+
+    assert updated_response.status == "completed"
+    assert updated_response.attributes["foo"] == "baz"
+
+    # Ensure we can retrieve the file and see the updated attributes
+    retrieved_file = compat_client.vector_stores.files.retrieve(
+        vector_store_id=vector_store.id,
+        file_id=file.id,
+    )
+    assert retrieved_file.attributes["foo"] == "baz"