diff --git a/docs/source/providers/files/remote_s3.md b/docs/source/providers/files/remote_s3.md index d90830fa8..2e3cebabd 100644 --- a/docs/source/providers/files/remote_s3.md +++ b/docs/source/providers/files/remote_s3.md @@ -13,6 +13,7 @@ AWS S3-based file storage provider for scalable cloud file management with metad | `aws_access_key_id` | `str \| None` | No | | AWS access key ID (optional if using IAM roles) | | `aws_secret_access_key` | `str \| None` | No | | AWS secret access key (optional if using IAM roles) | | `endpoint_url` | `str \| None` | No | | Custom S3 endpoint URL (for MinIO, LocalStack, etc.) | +| `auto_create_bucket` | `` | No | False | Automatically create the S3 bucket if it doesn't exist | | `metadata_store` | `utils.sqlstore.sqlstore.SqliteSqlStoreConfig \| utils.sqlstore.sqlstore.PostgresSqlStoreConfig` | No | sqlite | SQL store configuration for file metadata | ## Sample Configuration @@ -23,6 +24,7 @@ region: ${env.AWS_REGION:=us-east-1} aws_access_key_id: ${env.AWS_ACCESS_KEY_ID:=} aws_secret_access_key: ${env.AWS_SECRET_ACCESS_KEY:=} endpoint_url: ${env.S3_ENDPOINT_URL:=} +auto_create_bucket: ${env.S3_AUTO_CREATE_BUCKET:=false} metadata_store: type: sqlite db_path: ${env.SQLITE_STORE_DIR:=~/.llama/dummy}/s3_files_metadata.db diff --git a/llama_stack/providers/remote/files/s3/README.md b/llama_stack/providers/remote/files/s3/README.md index 9257a9925..0f33122c7 100644 --- a/llama_stack/providers/remote/files/s3/README.md +++ b/llama_stack/providers/remote/files/s3/README.md @@ -108,6 +108,41 @@ The S3 provider requires the following permissions: } ``` +### Automatic Bucket Creation + +By default, the S3 provider expects the bucket to already exist. If you want the provider to automatically create the bucket when it doesn't exist, set `auto_create_bucket: true` in your configuration: + +```yaml +config: + bucket_name: my-bucket + auto_create_bucket: true # Will create bucket if it doesn't exist + region: us-east-1 +``` + +**Note**: When `auto_create_bucket` is enabled, the provider will need additional permissions: + +```json +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "s3:GetObject", + "s3:PutObject", + "s3:DeleteObject", + "s3:ListBucket", + "s3:CreateBucket" + ], + "Resource": [ + "arn:aws:s3:::your-bucket-name", + "arn:aws:s3:::your-bucket-name/*" + ] + } + ] +} +``` + ### Bucket Policy (Optional) For additional security, you can add a bucket policy: diff --git a/llama_stack/providers/remote/files/s3/config.py b/llama_stack/providers/remote/files/s3/config.py index 6105a4816..da20d8668 100644 --- a/llama_stack/providers/remote/files/s3/config.py +++ b/llama_stack/providers/remote/files/s3/config.py @@ -21,6 +21,9 @@ class S3FilesImplConfig(BaseModel): default=None, description="AWS secret access key (optional if using IAM roles)" ) endpoint_url: str | None = Field(default=None, description="Custom S3 endpoint URL (for MinIO, LocalStack, etc.)") + auto_create_bucket: bool = Field( + default=False, description="Automatically create the S3 bucket if it doesn't exist" + ) metadata_store: SqlStoreConfig = Field(description="SQL store configuration for file metadata") @classmethod @@ -31,6 +34,7 @@ class S3FilesImplConfig(BaseModel): "aws_access_key_id": "${env.AWS_ACCESS_KEY_ID:=}", "aws_secret_access_key": "${env.AWS_SECRET_ACCESS_KEY:=}", "endpoint_url": "${env.S3_ENDPOINT_URL:=}", + "auto_create_bucket": "${env.S3_AUTO_CREATE_BUCKET:=false}", "metadata_store": SqliteSqlStoreConfig.sample_run_config( __distro_dir__=__distro_dir__, db_name="s3_files_metadata.db", diff --git a/llama_stack/providers/remote/files/s3/files.py b/llama_stack/providers/remote/files/s3/files.py index 5f436fe04..52e0cbbf4 100644 --- a/llama_stack/providers/remote/files/s3/files.py +++ b/llama_stack/providers/remote/files/s3/files.py @@ -59,6 +59,11 @@ async def _create_bucket_if_not_exists(client: boto3.client, config: S3FilesImpl except ClientError as e: error_code = e.response["Error"]["Code"] if error_code == "404": + if not config.auto_create_bucket: + raise RuntimeError( + f"S3 bucket '{config.bucket_name}' does not exist. " + f"Either create the bucket manually or set 'auto_create_bucket: true' in your configuration." + ) from e try: # For us-east-1, we can't specify LocationConstraint if config.region == "us-east-1": diff --git a/tests/unit/providers/files/test_s3_files.py b/tests/unit/providers/files/test_s3_files.py index 640411b53..daa250f10 100644 --- a/tests/unit/providers/files/test_s3_files.py +++ b/tests/unit/providers/files/test_s3_files.py @@ -37,6 +37,7 @@ def s3_config(tmp_path): return S3FilesImplConfig( bucket_name="test-bucket", region="not-a-region", + auto_create_bucket=True, metadata_store=SqliteSqlStoreConfig(db_path=db_path.as_posix()), )