diff --git a/docs/source/providers/files/index.md b/docs/source/providers/files/index.md index 692aad3ca..128953223 100644 --- a/docs/source/providers/files/index.md +++ b/docs/source/providers/files/index.md @@ -10,4 +10,5 @@ This section contains documentation for all available providers for the **files* :maxdepth: 1 inline_localfs +remote_s3 ``` diff --git a/docs/source/providers/files/remote_s3.md b/docs/source/providers/files/remote_s3.md new file mode 100644 index 000000000..2e3cebabd --- /dev/null +++ b/docs/source/providers/files/remote_s3.md @@ -0,0 +1,33 @@ +# remote::s3 + +## Description + +AWS S3-based file storage provider for scalable cloud file management with metadata persistence. + +## Configuration + +| Field | Type | Required | Default | Description | +|-------|------|----------|---------|-------------| +| `bucket_name` | `` | No | | S3 bucket name to store files | +| `region` | `` | No | us-east-1 | AWS region where the bucket is located | +| `aws_access_key_id` | `str \| None` | No | | AWS access key ID (optional if using IAM roles) | +| `aws_secret_access_key` | `str \| None` | No | | AWS secret access key (optional if using IAM roles) | +| `endpoint_url` | `str \| None` | No | | Custom S3 endpoint URL (for MinIO, LocalStack, etc.) | +| `auto_create_bucket` | `` | No | False | Automatically create the S3 bucket if it doesn't exist | +| `metadata_store` | `utils.sqlstore.sqlstore.SqliteSqlStoreConfig \| utils.sqlstore.sqlstore.PostgresSqlStoreConfig` | No | sqlite | SQL store configuration for file metadata | + +## Sample Configuration + +```yaml +bucket_name: ${env.S3_BUCKET_NAME} +region: ${env.AWS_REGION:=us-east-1} +aws_access_key_id: ${env.AWS_ACCESS_KEY_ID:=} +aws_secret_access_key: ${env.AWS_SECRET_ACCESS_KEY:=} +endpoint_url: ${env.S3_ENDPOINT_URL:=} +auto_create_bucket: ${env.S3_AUTO_CREATE_BUCKET:=false} +metadata_store: + type: sqlite + db_path: ${env.SQLITE_STORE_DIR:=~/.llama/dummy}/s3_files_metadata.db + +``` + diff --git a/llama_stack/providers/registry/files.py b/llama_stack/providers/registry/files.py index e894debaf..ebe90310c 100644 --- a/llama_stack/providers/registry/files.py +++ b/llama_stack/providers/registry/files.py @@ -5,9 +5,11 @@ # the root directory of this source tree. from llama_stack.providers.datatypes import ( + AdapterSpec, Api, InlineProviderSpec, ProviderSpec, + remote_provider_spec, ) from llama_stack.providers.utils.sqlstore.sqlstore import sql_store_pip_packages @@ -23,4 +25,14 @@ def available_providers() -> list[ProviderSpec]: config_class="llama_stack.providers.inline.files.localfs.config.LocalfsFilesImplConfig", description="Local filesystem-based file storage provider for managing files and documents locally.", ), + remote_provider_spec( + api=Api.files, + adapter=AdapterSpec( + adapter_type="s3", + pip_packages=["boto3"] + sql_store_pip_packages, + module="llama_stack.providers.remote.files.s3", + config_class="llama_stack.providers.remote.files.s3.config.S3FilesImplConfig", + description="AWS S3-based file storage provider for scalable cloud file management with metadata persistence.", + ), + ), ] diff --git a/llama_stack/providers/remote/files/s3/README.md b/llama_stack/providers/remote/files/s3/README.md new file mode 100644 index 000000000..0f33122c7 --- /dev/null +++ b/llama_stack/providers/remote/files/s3/README.md @@ -0,0 +1,237 @@ +# S3 Files Provider + +A remote S3-based implementation of the Llama Stack Files API that provides scalable cloud file storage with metadata persistence. + +## Features + +- **AWS S3 Storage**: Store files in AWS S3 buckets for scalable, durable storage +- **Metadata Management**: Uses SQL database for efficient file metadata queries +- **OpenAI API Compatibility**: Full compatibility with OpenAI Files API endpoints +- **Flexible Authentication**: Support for IAM roles and access keys +- **Custom S3 Endpoints**: Support for MinIO and other S3-compatible services + +## Configuration + +### Basic Configuration + +```yaml +api: files +provider_type: remote::s3 +config: + bucket_name: my-llama-stack-files + region: us-east-1 + metadata_store: + type: sqlite + db_path: ./s3_files_metadata.db +``` + +### Advanced Configuration + +```yaml +api: files +provider_type: remote::s3 +config: + bucket_name: my-llama-stack-files + region: us-east-1 + aws_access_key_id: YOUR_ACCESS_KEY + aws_secret_access_key: YOUR_SECRET_KEY + endpoint_url: https://s3.amazonaws.com # Optional for custom endpoints + metadata_store: + type: sqlite + db_path: ./s3_files_metadata.db +``` + +### Environment Variables + +The configuration supports environment variable substitution: + +```yaml +config: + bucket_name: "${env.S3_BUCKET_NAME}" + region: "${env.AWS_REGION:=us-east-1}" + aws_access_key_id: "${env.AWS_ACCESS_KEY_ID:=}" + aws_secret_access_key: "${env.AWS_SECRET_ACCESS_KEY:=}" + endpoint_url: "${env.S3_ENDPOINT_URL:=}" +``` + +Note: `S3_BUCKET_NAME` has no default value since S3 bucket names must be globally unique. + +## Authentication + +### IAM Roles (Recommended) + +For production deployments, use IAM roles: + +```yaml +config: + bucket_name: my-bucket + region: us-east-1 + # No credentials needed - will use IAM role +``` + +### Access Keys + +For development or specific use cases: + +```yaml +config: + bucket_name: my-bucket + region: us-east-1 + aws_access_key_id: AKIAIOSFODNN7EXAMPLE + aws_secret_access_key: wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY +``` + +## S3 Bucket Setup + +### Required Permissions + +The S3 provider requires the following permissions: + +```json +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "s3:GetObject", + "s3:PutObject", + "s3:DeleteObject", + "s3:ListBucket" + ], + "Resource": [ + "arn:aws:s3:::your-bucket-name", + "arn:aws:s3:::your-bucket-name/*" + ] + } + ] +} +``` + +### Automatic Bucket Creation + +By default, the S3 provider expects the bucket to already exist. If you want the provider to automatically create the bucket when it doesn't exist, set `auto_create_bucket: true` in your configuration: + +```yaml +config: + bucket_name: my-bucket + auto_create_bucket: true # Will create bucket if it doesn't exist + region: us-east-1 +``` + +**Note**: When `auto_create_bucket` is enabled, the provider will need additional permissions: + +```json +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "s3:GetObject", + "s3:PutObject", + "s3:DeleteObject", + "s3:ListBucket", + "s3:CreateBucket" + ], + "Resource": [ + "arn:aws:s3:::your-bucket-name", + "arn:aws:s3:::your-bucket-name/*" + ] + } + ] +} +``` + +### Bucket Policy (Optional) + +For additional security, you can add a bucket policy: + +```json +{ + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "LlamaStackAccess", + "Effect": "Allow", + "Principal": { + "AWS": "arn:aws:iam::YOUR-ACCOUNT:role/LlamaStackRole" + }, + "Action": [ + "s3:GetObject", + "s3:PutObject", + "s3:DeleteObject" + ], + "Resource": "arn:aws:s3:::your-bucket-name/*" + }, + { + "Sid": "LlamaStackBucketAccess", + "Effect": "Allow", + "Principal": { + "AWS": "arn:aws:iam::YOUR-ACCOUNT:role/LlamaStackRole" + }, + "Action": [ + "s3:ListBucket" + ], + "Resource": "arn:aws:s3:::your-bucket-name" + } + ] +} +``` + +## Features + +### Metadata Persistence + +File metadata is stored in a SQL database for fast queries and OpenAI API compatibility. The metadata includes: + +- File ID +- Original filename +- Purpose (assistants, batch, etc.) +- File size in bytes +- Created and expiration timestamps + +### TTL and Cleanup + +Files currently have a fixed long expiration time (100 years). + +## Development and Testing + +### Using MinIO + +For self-hosted S3-compatible storage: + +```yaml +config: + bucket_name: test-bucket + region: us-east-1 + endpoint_url: http://localhost:9000 + aws_access_key_id: minioadmin + aws_secret_access_key: minioadmin +``` + +## Monitoring and Logging + +The provider logs important operations and errors. For production deployments, consider: + +- CloudWatch monitoring for S3 operations +- Custom metrics for file upload/download rates +- Error rate monitoring +- Performance metrics tracking + +## Error Handling + +The provider handles various error scenarios: + +- S3 connectivity issues +- Bucket access permissions +- File not found errors +- Metadata consistency checks + +## Known Limitations + +- Fixed long TTL (100 years) instead of configurable expiration +- No server-side encryption enabled by default +- No support for AWS session tokens +- No S3 key prefix organization support +- No multipart upload support (all files uploaded as single objects) diff --git a/llama_stack/providers/remote/files/s3/__init__.py b/llama_stack/providers/remote/files/s3/__init__.py new file mode 100644 index 000000000..3f5dfc88a --- /dev/null +++ b/llama_stack/providers/remote/files/s3/__init__.py @@ -0,0 +1,20 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the terms described in the LICENSE file in +# the root directory of this source tree. + +from typing import Any + +from llama_stack.core.datatypes import Api + +from .config import S3FilesImplConfig + + +async def get_adapter_impl(config: S3FilesImplConfig, deps: dict[Api, Any]): + from .files import S3FilesImpl + + # TODO: authorization policies and user separation + impl = S3FilesImpl(config) + await impl.initialize() + return impl diff --git a/llama_stack/providers/remote/files/s3/config.py b/llama_stack/providers/remote/files/s3/config.py new file mode 100644 index 000000000..da20d8668 --- /dev/null +++ b/llama_stack/providers/remote/files/s3/config.py @@ -0,0 +1,42 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the terms described in the LICENSE file in +# the root directory of this source tree. + +from typing import Any + +from pydantic import BaseModel, Field + +from llama_stack.providers.utils.sqlstore.sqlstore import SqliteSqlStoreConfig, SqlStoreConfig + + +class S3FilesImplConfig(BaseModel): + """Configuration for S3-based files provider.""" + + bucket_name: str = Field(description="S3 bucket name to store files") + region: str = Field(default="us-east-1", description="AWS region where the bucket is located") + aws_access_key_id: str | None = Field(default=None, description="AWS access key ID (optional if using IAM roles)") + aws_secret_access_key: str | None = Field( + default=None, description="AWS secret access key (optional if using IAM roles)" + ) + endpoint_url: str | None = Field(default=None, description="Custom S3 endpoint URL (for MinIO, LocalStack, etc.)") + auto_create_bucket: bool = Field( + default=False, description="Automatically create the S3 bucket if it doesn't exist" + ) + metadata_store: SqlStoreConfig = Field(description="SQL store configuration for file metadata") + + @classmethod + def sample_run_config(cls, __distro_dir__: str) -> dict[str, Any]: + return { + "bucket_name": "${env.S3_BUCKET_NAME}", # no default, buckets must be globally unique + "region": "${env.AWS_REGION:=us-east-1}", + "aws_access_key_id": "${env.AWS_ACCESS_KEY_ID:=}", + "aws_secret_access_key": "${env.AWS_SECRET_ACCESS_KEY:=}", + "endpoint_url": "${env.S3_ENDPOINT_URL:=}", + "auto_create_bucket": "${env.S3_AUTO_CREATE_BUCKET:=false}", + "metadata_store": SqliteSqlStoreConfig.sample_run_config( + __distro_dir__=__distro_dir__, + db_name="s3_files_metadata.db", + ), + } diff --git a/llama_stack/providers/remote/files/s3/files.py b/llama_stack/providers/remote/files/s3/files.py new file mode 100644 index 000000000..52e0cbbf4 --- /dev/null +++ b/llama_stack/providers/remote/files/s3/files.py @@ -0,0 +1,272 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the terms described in the LICENSE file in +# the root directory of this source tree. + +import time +import uuid +from typing import Annotated + +import boto3 +from botocore.exceptions import BotoCoreError, ClientError, NoCredentialsError +from fastapi import File, Form, Response, UploadFile + +from llama_stack.apis.common.errors import ResourceNotFoundError +from llama_stack.apis.common.responses import Order +from llama_stack.apis.files import ( + Files, + ListOpenAIFileResponse, + OpenAIFileDeleteResponse, + OpenAIFileObject, + OpenAIFilePurpose, +) +from llama_stack.providers.utils.sqlstore.api import ColumnDefinition, ColumnType +from llama_stack.providers.utils.sqlstore.sqlstore import SqlStore, sqlstore_impl + +from .config import S3FilesImplConfig + +# TODO: provider data for S3 credentials + + +def _create_s3_client(config: S3FilesImplConfig) -> boto3.client: + try: + s3_config = { + "region_name": config.region, + } + + # endpoint URL if specified (for MinIO, LocalStack, etc.) + if config.endpoint_url: + s3_config["endpoint_url"] = config.endpoint_url + + if config.aws_access_key_id and config.aws_secret_access_key: + s3_config.update( + { + "aws_access_key_id": config.aws_access_key_id, + "aws_secret_access_key": config.aws_secret_access_key, + } + ) + + return boto3.client("s3", **s3_config) + + except (BotoCoreError, NoCredentialsError) as e: + raise RuntimeError(f"Failed to initialize S3 client: {e}") from e + + +async def _create_bucket_if_not_exists(client: boto3.client, config: S3FilesImplConfig) -> None: + try: + client.head_bucket(Bucket=config.bucket_name) + except ClientError as e: + error_code = e.response["Error"]["Code"] + if error_code == "404": + if not config.auto_create_bucket: + raise RuntimeError( + f"S3 bucket '{config.bucket_name}' does not exist. " + f"Either create the bucket manually or set 'auto_create_bucket: true' in your configuration." + ) from e + try: + # For us-east-1, we can't specify LocationConstraint + if config.region == "us-east-1": + client.create_bucket(Bucket=config.bucket_name) + else: + client.create_bucket( + Bucket=config.bucket_name, + CreateBucketConfiguration={"LocationConstraint": config.region}, + ) + except ClientError as create_error: + raise RuntimeError( + f"Failed to create S3 bucket '{config.bucket_name}': {create_error}" + ) from create_error + elif error_code == "403": + raise RuntimeError(f"Access denied to S3 bucket '{config.bucket_name}'") from e + else: + raise RuntimeError(f"Failed to access S3 bucket '{config.bucket_name}': {e}") from e + + +class S3FilesImpl(Files): + """S3-based implementation of the Files API.""" + + # TODO: implement expiration, for now a silly offset + _SILLY_EXPIRATION_OFFSET = 100 * 365 * 24 * 60 * 60 + + def __init__(self, config: S3FilesImplConfig) -> None: + self._config = config + self._client: boto3.client | None = None + self._sql_store: SqlStore | None = None + + async def initialize(self) -> None: + self._client = _create_s3_client(self._config) + await _create_bucket_if_not_exists(self._client, self._config) + + self._sql_store = sqlstore_impl(self._config.metadata_store) + await self._sql_store.create_table( + "openai_files", + { + "id": ColumnDefinition(type=ColumnType.STRING, primary_key=True), + "filename": ColumnType.STRING, + "purpose": ColumnType.STRING, + "bytes": ColumnType.INTEGER, + "created_at": ColumnType.INTEGER, + "expires_at": ColumnType.INTEGER, + # TODO: add s3_etag field for integrity checking + }, + ) + + async def shutdown(self) -> None: + pass + + @property + def client(self) -> boto3.client: + assert self._client is not None, "Provider not initialized" + return self._client + + @property + def sql_store(self) -> SqlStore: + assert self._sql_store is not None, "Provider not initialized" + return self._sql_store + + async def openai_upload_file( + self, + file: Annotated[UploadFile, File()], + purpose: Annotated[OpenAIFilePurpose, Form()], + ) -> OpenAIFileObject: + file_id = f"file-{uuid.uuid4().hex}" + + filename = getattr(file, "filename", None) or "uploaded_file" + + created_at = int(time.time()) + expires_at = created_at + self._SILLY_EXPIRATION_OFFSET + content = await file.read() + file_size = len(content) + + await self.sql_store.insert( + "openai_files", + { + "id": file_id, + "filename": filename, + "purpose": purpose.value, + "bytes": file_size, + "created_at": created_at, + "expires_at": expires_at, + }, + ) + + try: + self.client.put_object( + Bucket=self._config.bucket_name, + Key=file_id, + Body=content, + # TODO: enable server-side encryption + ) + except ClientError as e: + await self.sql_store.delete("openai_files", where={"id": file_id}) + + raise RuntimeError(f"Failed to upload file to S3: {e}") from e + + return OpenAIFileObject( + id=file_id, + filename=filename, + purpose=purpose, + bytes=file_size, + created_at=created_at, + expires_at=expires_at, + ) + + async def openai_list_files( + self, + after: str | None = None, + limit: int | None = 10000, + order: Order | None = Order.desc, + purpose: OpenAIFilePurpose | None = None, + ) -> ListOpenAIFileResponse: + # this purely defensive. it should not happen because the router also default to Order.desc. + if not order: + order = Order.desc + + where_conditions = {} + if purpose: + where_conditions["purpose"] = purpose.value + + paginated_result = await self.sql_store.fetch_all( + table="openai_files", + where=where_conditions if where_conditions else None, + order_by=[("created_at", order.value)], + cursor=("id", after) if after else None, + limit=limit, + ) + + files = [ + OpenAIFileObject( + id=row["id"], + filename=row["filename"], + purpose=OpenAIFilePurpose(row["purpose"]), + bytes=row["bytes"], + created_at=row["created_at"], + expires_at=row["expires_at"], + ) + for row in paginated_result.data + ] + + return ListOpenAIFileResponse( + data=files, + has_more=paginated_result.has_more, + # empty string or None? spec says str, ref impl returns str | None, we go with spec + first_id=files[0].id if files else "", + last_id=files[-1].id if files else "", + ) + + async def openai_retrieve_file(self, file_id: str) -> OpenAIFileObject: + row = await self.sql_store.fetch_one("openai_files", where={"id": file_id}) + if not row: + raise ResourceNotFoundError(file_id, "File", "files.list()") + + return OpenAIFileObject( + id=row["id"], + filename=row["filename"], + purpose=OpenAIFilePurpose(row["purpose"]), + bytes=row["bytes"], + created_at=row["created_at"], + expires_at=row["expires_at"], + ) + + async def openai_delete_file(self, file_id: str) -> OpenAIFileDeleteResponse: + row = await self.sql_store.fetch_one("openai_files", where={"id": file_id}) + if not row: + raise ResourceNotFoundError(file_id, "File", "files.list()") + + try: + self.client.delete_object( + Bucket=self._config.bucket_name, + Key=row["id"], + ) + except ClientError as e: + if e.response["Error"]["Code"] != "NoSuchKey": + raise RuntimeError(f"Failed to delete file from S3: {e}") from e + + await self.sql_store.delete("openai_files", where={"id": file_id}) + + return OpenAIFileDeleteResponse(id=file_id, deleted=True) + + async def openai_retrieve_file_content(self, file_id: str) -> Response: + row = await self.sql_store.fetch_one("openai_files", where={"id": file_id}) + if not row: + raise ResourceNotFoundError(file_id, "File", "files.list()") + + try: + response = self.client.get_object( + Bucket=self._config.bucket_name, + Key=row["id"], + ) + # TODO: can we stream this instead of loading it into memory + content = response["Body"].read() + except ClientError as e: + if e.response["Error"]["Code"] == "NoSuchKey": + await self.sql_store.delete("openai_files", where={"id": file_id}) + raise ResourceNotFoundError(file_id, "File", "files.list()") from e + raise RuntimeError(f"Failed to download file from S3: {e}") from e + + return Response( + content=content, + media_type="application/octet-stream", + headers={"Content-Disposition": f'attachment; filename="{row["filename"]}"'}, + ) diff --git a/pyproject.toml b/pyproject.toml index 0cdfc6a37..6c76da895 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -98,6 +98,7 @@ unit = [ "together", "coverage", "chromadb>=1.0.15", + "moto[s3]>=5.1.10", ] # These are the core dependencies required for running integration tests. They are shared across all # providers. If a provider requires additional dependencies, please add them to your environment diff --git a/scripts/provider_codegen.py b/scripts/provider_codegen.py index 060acfa72..17efa2138 100755 --- a/scripts/provider_codegen.py +++ b/scripts/provider_codegen.py @@ -157,12 +157,14 @@ def get_config_class_info(config_class_path: str) -> dict[str, Any]: } -def generate_provider_docs(provider_spec: Any, api_name: str) -> str: +def generate_provider_docs(progress, provider_spec: Any, api_name: str) -> str: """Generate markdown documentation for a provider.""" provider_type = provider_spec.provider_type config_class = provider_spec.config_class config_info = get_config_class_info(config_class) + if "error" in config_info: + progress.print(config_info["error"]) md_lines = [] md_lines.append(f"# {provider_type}") @@ -295,7 +297,7 @@ def process_provider_registry(progress, change_tracker: ChangedPathTracker) -> N filename = provider_type.replace("::", "_").replace(":", "_") provider_doc_file = doc_output_dir / f"{filename}.md" - provider_docs = generate_provider_docs(provider, api_name) + provider_docs = generate_provider_docs(progress, provider, api_name) provider_doc_file.write_text(provider_docs) change_tracker.add_paths(provider_doc_file) diff --git a/tests/unit/providers/files/test_s3_files.py b/tests/unit/providers/files/test_s3_files.py new file mode 100644 index 000000000..daa250f10 --- /dev/null +++ b/tests/unit/providers/files/test_s3_files.py @@ -0,0 +1,251 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the terms described in the LICENSE file in +# the root directory of this source tree. + +from unittest.mock import patch + +import boto3 +import pytest +from botocore.exceptions import ClientError +from moto import mock_aws + +from llama_stack.apis.common.errors import ResourceNotFoundError +from llama_stack.apis.files import OpenAIFilePurpose +from llama_stack.providers.remote.files.s3 import ( + S3FilesImplConfig, + get_adapter_impl, +) +from llama_stack.providers.utils.sqlstore.sqlstore import SqliteSqlStoreConfig + + +class MockUploadFile: + def __init__(self, content: bytes, filename: str, content_type: str = "text/plain"): + self.content = content + self.filename = filename + self.content_type = content_type + + async def read(self): + return self.content + + +@pytest.fixture +def s3_config(tmp_path): + db_path = tmp_path / "s3_files_metadata.db" + + return S3FilesImplConfig( + bucket_name="test-bucket", + region="not-a-region", + auto_create_bucket=True, + metadata_store=SqliteSqlStoreConfig(db_path=db_path.as_posix()), + ) + + +@pytest.fixture +def s3_client(): + """Create a mocked S3 client for testing.""" + # we use `with mock_aws()` because @mock_aws decorator does not support being a generator + with mock_aws(): + # must yield or the mock will be reset before it is used + yield boto3.client("s3") + + +@pytest.fixture +async def s3_provider(s3_config, s3_client): + """Create an S3 files provider with mocked S3 for testing.""" + provider = await get_adapter_impl(s3_config, {}) + yield provider + await provider.shutdown() + + +@pytest.fixture +def sample_text_file(): + content = b"Hello, this is a test file for the S3 Files API!" + return MockUploadFile(content, "sample_text_file.txt") + + +class TestS3FilesImpl: + """Test suite for S3 Files implementation.""" + + async def test_upload_file(self, s3_provider, sample_text_file, s3_client, s3_config): + """Test successful file upload.""" + sample_text_file.filename = "test_upload_file" + result = await s3_provider.openai_upload_file( + file=sample_text_file, + purpose=OpenAIFilePurpose.ASSISTANTS, + ) + + assert result.filename == sample_text_file.filename + assert result.purpose == OpenAIFilePurpose.ASSISTANTS + assert result.bytes == len(sample_text_file.content) + assert result.id.startswith("file-") + + # Verify file exists in S3 backend + response = s3_client.head_object(Bucket=s3_config.bucket_name, Key=result.id) + assert response["ResponseMetadata"]["HTTPStatusCode"] == 200 + + async def test_list_files_empty(self, s3_provider): + """Test listing files when no files exist.""" + result = await s3_provider.openai_list_files() + + assert len(result.data) == 0 + assert not result.has_more + assert result.first_id == "" + assert result.last_id == "" + + async def test_retrieve_file(self, s3_provider, sample_text_file): + """Test retrieving file metadata.""" + sample_text_file.filename = "test_retrieve_file" + uploaded = await s3_provider.openai_upload_file( + file=sample_text_file, + purpose=OpenAIFilePurpose.ASSISTANTS, + ) + + retrieved = await s3_provider.openai_retrieve_file(uploaded.id) + + assert retrieved.id == uploaded.id + assert retrieved.filename == uploaded.filename + assert retrieved.purpose == uploaded.purpose + assert retrieved.bytes == uploaded.bytes + + async def test_retrieve_file_content(self, s3_provider, sample_text_file): + """Test retrieving file content.""" + sample_text_file.filename = "test_retrieve_file_content" + uploaded = await s3_provider.openai_upload_file( + file=sample_text_file, + purpose=OpenAIFilePurpose.ASSISTANTS, + ) + + response = await s3_provider.openai_retrieve_file_content(uploaded.id) + + assert response.body == sample_text_file.content + assert response.headers["Content-Disposition"] == f'attachment; filename="{sample_text_file.filename}"' + + async def test_delete_file(self, s3_provider, sample_text_file, s3_config, s3_client): + """Test deleting a file.""" + sample_text_file.filename = "test_delete_file" + uploaded = await s3_provider.openai_upload_file( + file=sample_text_file, + purpose=OpenAIFilePurpose.ASSISTANTS, + ) + + delete_response = await s3_provider.openai_delete_file(uploaded.id) + + assert delete_response.id == uploaded.id + assert delete_response.deleted is True + + with pytest.raises(ResourceNotFoundError, match="not found"): + await s3_provider.openai_retrieve_file(uploaded.id) + + # Verify file is gone from S3 backend + with pytest.raises(ClientError) as exc_info: + s3_client.head_object(Bucket=s3_config.bucket_name, Key=uploaded.id) + assert exc_info.value.response["Error"]["Code"] == "404" + + async def test_list_files(self, s3_provider, sample_text_file): + """Test listing files after uploading some.""" + sample_text_file.filename = "test_list_files_with_content_file1" + file1 = await s3_provider.openai_upload_file( + file=sample_text_file, + purpose=OpenAIFilePurpose.ASSISTANTS, + ) + + file2_content = MockUploadFile(b"Second file content", "test_list_files_with_content_file2") + file2 = await s3_provider.openai_upload_file( + file=file2_content, + purpose=OpenAIFilePurpose.BATCH, + ) + + result = await s3_provider.openai_list_files() + + assert len(result.data) == 2 + file_ids = {f.id for f in result.data} + assert file1.id in file_ids + assert file2.id in file_ids + + async def test_list_files_with_purpose_filter(self, s3_provider, sample_text_file): + """Test listing files with purpose filter.""" + sample_text_file.filename = "test_list_files_with_purpose_filter_file1" + file1 = await s3_provider.openai_upload_file( + file=sample_text_file, + purpose=OpenAIFilePurpose.ASSISTANTS, + ) + + file2_content = MockUploadFile(b"Batch file content", "test_list_files_with_purpose_filter_file2") + await s3_provider.openai_upload_file( + file=file2_content, + purpose=OpenAIFilePurpose.BATCH, + ) + + result = await s3_provider.openai_list_files(purpose=OpenAIFilePurpose.ASSISTANTS) + + assert len(result.data) == 1 + assert result.data[0].id == file1.id + assert result.data[0].purpose == OpenAIFilePurpose.ASSISTANTS + + async def test_nonexistent_file_retrieval(self, s3_provider): + """Test retrieving a non-existent file raises error.""" + with pytest.raises(ResourceNotFoundError, match="not found"): + await s3_provider.openai_retrieve_file("file-nonexistent") + + async def test_nonexistent_file_content_retrieval(self, s3_provider): + """Test retrieving content of a non-existent file raises error.""" + with pytest.raises(ResourceNotFoundError, match="not found"): + await s3_provider.openai_retrieve_file_content("file-nonexistent") + + async def test_nonexistent_file_deletion(self, s3_provider): + """Test deleting a non-existent file raises error.""" + with pytest.raises(ResourceNotFoundError, match="not found"): + await s3_provider.openai_delete_file("file-nonexistent") + + async def test_upload_file_without_filename(self, s3_provider, sample_text_file): + """Test uploading a file without a filename uses the fallback.""" + del sample_text_file.filename + result = await s3_provider.openai_upload_file( + file=sample_text_file, + purpose=OpenAIFilePurpose.ASSISTANTS, + ) + + assert result.purpose == OpenAIFilePurpose.ASSISTANTS + assert result.bytes == len(sample_text_file.content) + + retrieved = await s3_provider.openai_retrieve_file(result.id) + assert retrieved.filename == result.filename + + async def test_file_operations_when_s3_object_deleted(self, s3_provider, sample_text_file, s3_config, s3_client): + """Test file operations when S3 object is deleted but metadata exists (negative test).""" + sample_text_file.filename = "test_orphaned_metadata" + uploaded = await s3_provider.openai_upload_file( + file=sample_text_file, + purpose=OpenAIFilePurpose.ASSISTANTS, + ) + + # Directly delete the S3 object from the backend + s3_client.delete_object(Bucket=s3_config.bucket_name, Key=uploaded.id) + + with pytest.raises(ResourceNotFoundError, match="not found") as exc_info: + await s3_provider.openai_retrieve_file_content(uploaded.id) + assert uploaded.id in str(exc_info).lower() + + listed_files = await s3_provider.openai_list_files() + assert uploaded.id not in [file.id for file in listed_files.data] + + async def test_upload_file_s3_put_object_failure(self, s3_provider, sample_text_file, s3_config, s3_client): + """Test that put_object failure results in exception and no orphaned metadata.""" + sample_text_file.filename = "test_s3_put_object_failure" + + def failing_put_object(*args, **kwargs): + raise ClientError( + error_response={"Error": {"Code": "SolarRadiation", "Message": "Bloop"}}, operation_name="PutObject" + ) + + with patch.object(s3_provider.client, "put_object", side_effect=failing_put_object): + with pytest.raises(RuntimeError, match="Failed to upload file to S3"): + await s3_provider.openai_upload_file( + file=sample_text_file, + purpose=OpenAIFilePurpose.ASSISTANTS, + ) + + files_list = await s3_provider.openai_list_files() + assert len(files_list.data) == 0, "No file metadata should remain after failed upload" diff --git a/uv.lock b/uv.lock index 5d30ad304..385c75bea 100644 --- a/uv.lock +++ b/uv.lock @@ -347,6 +347,34 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ed/4d/1392562369b1139e741b30d624f09fe7091d17dd5579fae5732f044b12bb/blobfile-3.0.0-py3-none-any.whl", hash = "sha256:48ecc3307e622804bd8fe13bf6f40e6463c4439eba7a1f9ad49fd78aa63cc658", size = 75413, upload-time = "2024-08-27T00:02:51.518Z" }, ] +[[package]] +name = "boto3" +version = "1.40.12" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "botocore" }, + { name = "jmespath" }, + { name = "s3transfer" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/41/19/2c4d140a7f99b5903b21b9ccd7253c71f147c346c3c632b2117444cf2d65/boto3-1.40.12.tar.gz", hash = "sha256:c6b32aee193fbd2eb84696d2b5b2410dcda9fb4a385e1926cff908377d222247", size = 111959, upload-time = "2025-08-18T19:30:23.827Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/16/6e/5a9dcf38ad87838fb99742c4a3ab1b7507ad3a02c8c27a9ccda7a0bb5709/boto3-1.40.12-py3-none-any.whl", hash = "sha256:3c3d6731390b5b11f5e489d5d9daa57f0c3e171efb63ac8f47203df9c71812b3", size = 140075, upload-time = "2025-08-18T19:30:22.494Z" }, +] + +[[package]] +name = "botocore" +version = "1.40.12" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "jmespath" }, + { name = "python-dateutil" }, + { name = "urllib3" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/7d/b2/7933590fc5bca1980801b71e09db1a95581afff177cbf3c8a031d922885c/botocore-1.40.12.tar.gz", hash = "sha256:c6560578e799b47b762b7e555bd9c5dd5c29c5d23bd778a8a72e98c979b3c727", size = 14349930, upload-time = "2025-08-18T19:30:13.794Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1e/b6/65fd6e718c9538ba1462c9b71e9262bc723202ff203fe64ff66ff676d823/botocore-1.40.12-py3-none-any.whl", hash = "sha256:84e96004a8b426c5508f6b5600312d6271364269466a3a957dc377ad8effc438", size = 14018004, upload-time = "2025-08-18T19:30:09.054Z" }, +] + [[package]] name = "braintrust-core" version = "0.0.59" @@ -1580,6 +1608,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b3/4a/4175a563579e884192ba6e81725fc0448b042024419be8d83aa8a80a3f44/jiter-0.10.0-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3aa96f2abba33dc77f79b4cf791840230375f9534e5fac927ccceb58c5e604a5", size = 354213, upload-time = "2025-05-18T19:04:41.894Z" }, ] +[[package]] +name = "jmespath" +version = "1.0.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/00/2a/e867e8531cf3e36b41201936b7fa7ba7b5702dbef42922193f05c8976cd6/jmespath-1.0.1.tar.gz", hash = "sha256:90261b206d6defd58fdd5e85f478bf633a2901798906be2ad389150c5c60edbe", size = 25843, upload-time = "2022-06-17T18:00:12.224Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/31/b4/b9b800c45527aadd64d5b442f9b932b00648617eb5d63d2c7a6587b7cafc/jmespath-1.0.1-py3-none-any.whl", hash = "sha256:02e2e4cc71b5bcab88332eebf907519190dd9e6e82107fa7f83b1003a6252980", size = 20256, upload-time = "2022-06-17T18:00:10.251Z" }, +] + [[package]] name = "jsonschema" version = "4.25.0" @@ -1820,6 +1857,7 @@ unit = [ { name = "litellm" }, { name = "mcp" }, { name = "milvus-lite" }, + { name = "moto", extra = ["s3"] }, { name = "ollama" }, { name = "openai" }, { name = "pymilvus" }, @@ -1937,6 +1975,7 @@ unit = [ { name = "litellm" }, { name = "mcp" }, { name = "milvus-lite", specifier = ">=2.5.0" }, + { name = "moto", extras = ["s3"], specifier = ">=5.1.10" }, { name = "ollama" }, { name = "openai" }, { name = "pymilvus", specifier = ">=2.5.12" }, @@ -2224,6 +2263,32 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/16/71/4ad9a42f2772793a03cb698f0fc42499f04e6e8d2560ba2f7da0fb059a8e/mmh3-5.1.0-cp313-cp313-win_arm64.whl", hash = "sha256:b22fe2e54be81f6c07dcb36b96fa250fb72effe08aa52fbb83eade6e1e2d5fd7", size = 38890, upload-time = "2025-01-25T08:39:25.28Z" }, ] +[[package]] +name = "moto" +version = "5.1.10" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "boto3" }, + { name = "botocore" }, + { name = "cryptography" }, + { name = "jinja2" }, + { name = "python-dateutil" }, + { name = "requests" }, + { name = "responses" }, + { name = "werkzeug" }, + { name = "xmltodict" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c4/72/9bc9b4917b816f5a82fc8f0fbd477c2a669d35a7d7941ae15a5411e266d6/moto-5.1.10.tar.gz", hash = "sha256:d6bdc8f82a1e503502927cc0a3da22014f836094d0bf399bb0f695754ae6c7a6", size = 7087004, upload-time = "2025-08-11T20:59:45.542Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c4/37/9b9cb5597eecc2ebfde2f65a8265f3669f6724ebe82bf9b155a3421039f8/moto-5.1.10-py3-none-any.whl", hash = "sha256:9ec1a21a924f97470af225b2bfa854fe46c1ad30fb44655eba458206dedf28b5", size = 5246859, upload-time = "2025-08-11T20:59:43.22Z" }, +] + +[package.optional-dependencies] +s3 = [ + { name = "py-partiql-parser" }, + { name = "pyyaml" }, +] + [[package]] name = "mpmath" version = "1.3.0" @@ -3068,6 +3133,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/8e/37/efad0257dc6e593a18957422533ff0f87ede7c9c6ea010a2177d738fb82f/pure_eval-0.2.3-py3-none-any.whl", hash = "sha256:1db8e35b67b3d218d818ae653e27f06c3aa420901fa7b081ca98cbedc874e0d0", size = 11842, upload-time = "2024-07-21T12:58:20.04Z" }, ] +[[package]] +name = "py-partiql-parser" +version = "0.6.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/58/a1/0a2867e48b232b4f82c4929ef7135f2a5d72c3886b957dccf63c70aa2fcb/py_partiql_parser-0.6.1.tar.gz", hash = "sha256:8583ff2a0e15560ef3bc3df109a7714d17f87d81d33e8c38b7fed4e58a63215d", size = 17120, upload-time = "2024-12-25T22:06:41.327Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/97/84/0e410c20bbe9a504fc56e97908f13261c2b313d16cbb3b738556166f044a/py_partiql_parser-0.6.1-py2.py3-none-any.whl", hash = "sha256:ff6a48067bff23c37e9044021bf1d949c83e195490c17e020715e927fe5b2456", size = 23520, upload-time = "2024-12-25T22:06:39.106Z" }, +] + [[package]] name = "pyaml" version = "25.7.0" @@ -3788,6 +3862,20 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/3b/5d/63d4ae3b9daea098d5d6f5da83984853c1bbacd5dc826764b249fe119d24/requests_oauthlib-2.0.0-py2.py3-none-any.whl", hash = "sha256:7dd8a5c40426b779b0868c404bdef9768deccf22749cde15852df527e6269b36", size = 24179, upload-time = "2024-03-22T20:32:28.055Z" }, ] +[[package]] +name = "responses" +version = "0.25.8" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pyyaml" }, + { name = "requests" }, + { name = "urllib3" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/0e/95/89c054ad70bfef6da605338b009b2e283485835351a9935c7bfbfaca7ffc/responses-0.25.8.tar.gz", hash = "sha256:9374d047a575c8f781b94454db5cab590b6029505f488d12899ddb10a4af1cf4", size = 79320, upload-time = "2025-08-08T19:01:46.709Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1c/4c/cc276ce57e572c102d9542d383b2cfd551276581dc60004cb94fe8774c11/responses-0.25.8-py3-none-any.whl", hash = "sha256:0c710af92def29c8352ceadff0c3fe340ace27cf5af1bbe46fb71275bcd2831c", size = 34769, upload-time = "2025-08-08T19:01:45.018Z" }, +] + [[package]] name = "rich" version = "14.1.0" @@ -3961,6 +4049,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/00/db/c376b0661c24cf770cb8815268190668ec1330eba8374a126ceef8c72d55/ruff-0.12.5-py3-none-win_arm64.whl", hash = "sha256:48cdbfc633de2c5c37d9f090ba3b352d1576b0015bfc3bc98eaf230275b7e805", size = 11951564, upload-time = "2025-07-24T13:26:34.994Z" }, ] +[[package]] +name = "s3transfer" +version = "0.13.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "botocore" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/6d/05/d52bf1e65044b4e5e27d4e63e8d1579dbdec54fce685908ae09bc3720030/s3transfer-0.13.1.tar.gz", hash = "sha256:c3fdba22ba1bd367922f27ec8032d6a1cf5f10c934fb5d68cf60fd5a23d936cf", size = 150589, upload-time = "2025-07-18T19:22:42.31Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/6d/4f/d073e09df851cfa251ef7840007d04db3293a0482ce607d2b993926089be/s3transfer-0.13.1-py3-none-any.whl", hash = "sha256:a981aa7429be23fe6dfc13e80e4020057cbab622b08c0315288758d67cabc724", size = 85308, upload-time = "2025-07-18T19:22:40.947Z" }, +] + [[package]] name = "safetensors" version = "0.5.3" @@ -5107,6 +5207,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/78/58/e860788190eba3bcce367f74d29c4675466ce8dddfba85f7827588416f01/wsproto-1.2.0-py3-none-any.whl", hash = "sha256:b9acddd652b585d75b20477888c56642fdade28bdfd3579aa24a4d2c037dd736", size = 24226, upload-time = "2022-08-23T19:58:19.96Z" }, ] +[[package]] +name = "xmltodict" +version = "0.14.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/50/05/51dcca9a9bf5e1bce52582683ce50980bcadbc4fa5143b9f2b19ab99958f/xmltodict-0.14.2.tar.gz", hash = "sha256:201e7c28bb210e374999d1dde6382923ab0ed1a8a5faeece48ab525b7810a553", size = 51942, upload-time = "2024-10-16T06:10:29.683Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d6/45/fc303eb433e8a2a271739c98e953728422fa61a3c1f36077a49e395c972e/xmltodict-0.14.2-py2.py3-none-any.whl", hash = "sha256:20cc7d723ed729276e808f26fb6b3599f786cbc37e06c65e192ba77c40f20aac", size = 9981, upload-time = "2024-10-16T06:10:27.649Z" }, +] + [[package]] name = "xxhash" version = "3.5.0"