diff --git a/client-sdks/stainless/openapi.yml b/client-sdks/stainless/openapi.yml index 359508f30..5e5982d8a 100644 --- a/client-sdks/stainless/openapi.yml +++ b/client-sdks/stainless/openapi.yml @@ -3887,7 +3887,7 @@ paths: schema: $ref: '#/components/schemas/SupervisedFineTuneRequest' required: true - /v1alpha/file-processor/process: + /v1alpha/file-processors/process: post: responses: '200': @@ -3909,13 +3909,13 @@ paths: description: Default Response $ref: '#/components/responses/DefaultError' tags: - - File Processor + - File Processors summary: Process File description: |- Process a file into structured content with optional chunking and embeddings. This method processes raw file data and converts it into text content for applications such as vector store ingestion. - operationId: process_file_v1alpha_file_processor_process_post + operationId: process_file_v1alpha_file_processors_process_post requestBody: content: application/json: @@ -13005,7 +13005,7 @@ components: - benchmarks - tool_groups - files - - file_processor + - file_processors - prompts - conversations - inspect diff --git a/docs/docs/providers/file_processors/index.mdx b/docs/docs/providers/file_processors/index.mdx new file mode 100644 index 000000000..e9119d40e --- /dev/null +++ b/docs/docs/providers/file_processors/index.mdx @@ -0,0 +1,10 @@ +--- +sidebar_label: File Processors +title: File_Processors +--- + +# File_Processors + +## Overview + +This section contains documentation for all available providers for the **file_processors** API. diff --git a/docs/static/deprecated-llama-stack-spec.yaml b/docs/static/deprecated-llama-stack-spec.yaml index 0c3a77bcb..c471cc39a 100644 --- a/docs/static/deprecated-llama-stack-spec.yaml +++ b/docs/static/deprecated-llama-stack-spec.yaml @@ -9813,7 +9813,7 @@ components: - benchmarks - tool_groups - files - - file_processor + - file_processors - prompts - conversations - inspect diff --git a/docs/static/experimental-llama-stack-spec.yaml b/docs/static/experimental-llama-stack-spec.yaml index 0a6493d65..89128f887 100644 --- a/docs/static/experimental-llama-stack-spec.yaml +++ b/docs/static/experimental-llama-stack-spec.yaml @@ -630,7 +630,7 @@ paths: schema: $ref: '#/components/schemas/SupervisedFineTuneRequest' required: true - /v1alpha/file-processor/process: + /v1alpha/file-processors/process: post: responses: '200': @@ -652,13 +652,13 @@ paths: description: Default Response $ref: '#/components/responses/DefaultError' tags: - - File Processor + - File Processors summary: Process File description: |- Process a file into structured content with optional chunking and embeddings. This method processes raw file data and converts it into text content for applications such as vector store ingestion. - operationId: process_file_v1alpha_file_processor_process_post + operationId: process_file_v1alpha_file_processors_process_post requestBody: content: application/json: @@ -8777,7 +8777,7 @@ components: - benchmarks - tool_groups - files - - file_processor + - file_processors - prompts - conversations - inspect diff --git a/docs/static/llama-stack-spec.yaml b/docs/static/llama-stack-spec.yaml index 2c6025f2e..132db6a44 100644 --- a/docs/static/llama-stack-spec.yaml +++ b/docs/static/llama-stack-spec.yaml @@ -11640,7 +11640,7 @@ components: - benchmarks - tool_groups - files - - file_processor + - file_processors - prompts - conversations - inspect diff --git a/docs/static/stainless-llama-stack-spec.yaml b/docs/static/stainless-llama-stack-spec.yaml index 359508f30..5e5982d8a 100644 --- a/docs/static/stainless-llama-stack-spec.yaml +++ b/docs/static/stainless-llama-stack-spec.yaml @@ -3887,7 +3887,7 @@ paths: schema: $ref: '#/components/schemas/SupervisedFineTuneRequest' required: true - /v1alpha/file-processor/process: + /v1alpha/file-processors/process: post: responses: '200': @@ -3909,13 +3909,13 @@ paths: description: Default Response $ref: '#/components/responses/DefaultError' tags: - - File Processor + - File Processors summary: Process File description: |- Process a file into structured content with optional chunking and embeddings. This method processes raw file data and converts it into text content for applications such as vector store ingestion. - operationId: process_file_v1alpha_file_processor_process_post + operationId: process_file_v1alpha_file_processors_process_post requestBody: content: application/json: @@ -13005,7 +13005,7 @@ components: - benchmarks - tool_groups - files - - file_processor + - file_processors - prompts - conversations - inspect diff --git a/src/llama_stack/core/resolver.py b/src/llama_stack/core/resolver.py index df6693338..f8a70f1ee 100644 --- a/src/llama_stack/core/resolver.py +++ b/src/llama_stack/core/resolver.py @@ -34,7 +34,7 @@ from llama_stack_api import ( DatasetsProtocolPrivate, Eval, ExternalApiSpec, - FileProcessor, + FileProcessors, Files, Inference, InferenceProvider, @@ -101,7 +101,7 @@ def api_protocol_map(external_apis: dict[Api, ExternalApiSpec] | None = None) -> Api.files: Files, Api.prompts: Prompts, Api.conversations: Conversations, - Api.file_processor: FileProcessor, + Api.file_processors: FileProcessors, } if external_apis: diff --git a/src/llama_stack/distributions/ci-tests/build.yaml b/src/llama_stack/distributions/ci-tests/build.yaml index ef2e552c0..f29ac7712 100644 --- a/src/llama_stack/distributions/ci-tests/build.yaml +++ b/src/llama_stack/distributions/ci-tests/build.yaml @@ -29,8 +29,6 @@ distribution_spec: - provider_type: remote::weaviate files: - provider_type: inline::localfs - file_processor: - - provider_type: inline::reference safety: - provider_type: inline::llama-guard - provider_type: inline::code-scanner diff --git a/src/llama_stack/distributions/ci-tests/run-with-postgres-store.yaml b/src/llama_stack/distributions/ci-tests/run-with-postgres-store.yaml index 62eb08b3d..7721138c7 100644 --- a/src/llama_stack/distributions/ci-tests/run-with-postgres-store.yaml +++ b/src/llama_stack/distributions/ci-tests/run-with-postgres-store.yaml @@ -5,7 +5,6 @@ apis: - batches - datasetio - eval -- file_processor - files - inference - post_training @@ -154,9 +153,6 @@ providers: metadata_store: table_name: files_metadata backend: sql_default - file_processor: - - provider_id: reference - provider_type: inline::reference safety: - provider_id: llama-guard provider_type: inline::llama-guard diff --git a/src/llama_stack/distributions/ci-tests/run.yaml b/src/llama_stack/distributions/ci-tests/run.yaml index e610c7601..b791e1488 100644 --- a/src/llama_stack/distributions/ci-tests/run.yaml +++ b/src/llama_stack/distributions/ci-tests/run.yaml @@ -5,7 +5,6 @@ apis: - batches - datasetio - eval -- file_processor - files - inference - post_training @@ -154,9 +153,6 @@ providers: metadata_store: table_name: files_metadata backend: sql_default - file_processor: - - provider_id: reference - provider_type: inline::reference safety: - provider_id: llama-guard provider_type: inline::llama-guard diff --git a/src/llama_stack/distributions/starter-gpu/build.yaml b/src/llama_stack/distributions/starter-gpu/build.yaml index e286bc3d8..10cbb1389 100644 --- a/src/llama_stack/distributions/starter-gpu/build.yaml +++ b/src/llama_stack/distributions/starter-gpu/build.yaml @@ -30,8 +30,6 @@ distribution_spec: - provider_type: remote::weaviate files: - provider_type: inline::localfs - file_processor: - - provider_type: inline::reference safety: - provider_type: inline::llama-guard - provider_type: inline::code-scanner diff --git a/src/llama_stack/distributions/starter-gpu/run-with-postgres-store.yaml b/src/llama_stack/distributions/starter-gpu/run-with-postgres-store.yaml index 37199a35a..9c250c05a 100644 --- a/src/llama_stack/distributions/starter-gpu/run-with-postgres-store.yaml +++ b/src/llama_stack/distributions/starter-gpu/run-with-postgres-store.yaml @@ -5,7 +5,6 @@ apis: - batches - datasetio - eval -- file_processor - files - inference - post_training @@ -154,9 +153,6 @@ providers: metadata_store: table_name: files_metadata backend: sql_default - file_processor: - - provider_id: reference - provider_type: inline::reference safety: - provider_id: llama-guard provider_type: inline::llama-guard diff --git a/src/llama_stack/distributions/starter-gpu/run.yaml b/src/llama_stack/distributions/starter-gpu/run.yaml index 14dbeb341..65f9ae326 100644 --- a/src/llama_stack/distributions/starter-gpu/run.yaml +++ b/src/llama_stack/distributions/starter-gpu/run.yaml @@ -5,7 +5,6 @@ apis: - batches - datasetio - eval -- file_processor - files - inference - post_training @@ -154,9 +153,6 @@ providers: metadata_store: table_name: files_metadata backend: sql_default - file_processor: - - provider_id: reference - provider_type: inline::reference safety: - provider_id: llama-guard provider_type: inline::llama-guard diff --git a/src/llama_stack/distributions/starter/build.yaml b/src/llama_stack/distributions/starter/build.yaml index e673cc3e6..acd51f773 100644 --- a/src/llama_stack/distributions/starter/build.yaml +++ b/src/llama_stack/distributions/starter/build.yaml @@ -30,8 +30,6 @@ distribution_spec: - provider_type: remote::weaviate files: - provider_type: inline::localfs - file_processor: - - provider_type: inline::reference safety: - provider_type: inline::llama-guard - provider_type: inline::code-scanner diff --git a/src/llama_stack/distributions/starter/run-with-postgres-store.yaml b/src/llama_stack/distributions/starter/run-with-postgres-store.yaml index a84a5a243..3314bb9e9 100644 --- a/src/llama_stack/distributions/starter/run-with-postgres-store.yaml +++ b/src/llama_stack/distributions/starter/run-with-postgres-store.yaml @@ -5,7 +5,6 @@ apis: - batches - datasetio - eval -- file_processor - files - inference - post_training @@ -154,9 +153,6 @@ providers: metadata_store: table_name: files_metadata backend: sql_default - file_processor: - - provider_id: reference - provider_type: inline::reference safety: - provider_id: llama-guard provider_type: inline::llama-guard diff --git a/src/llama_stack/distributions/starter/run.yaml b/src/llama_stack/distributions/starter/run.yaml index f2ebb4aef..e88539e6a 100644 --- a/src/llama_stack/distributions/starter/run.yaml +++ b/src/llama_stack/distributions/starter/run.yaml @@ -5,7 +5,6 @@ apis: - batches - datasetio - eval -- file_processor - files - inference - post_training @@ -154,9 +153,6 @@ providers: metadata_store: table_name: files_metadata backend: sql_default - file_processor: - - provider_id: reference - provider_type: inline::reference safety: - provider_id: llama-guard provider_type: inline::llama-guard diff --git a/src/llama_stack/distributions/starter/starter.py b/src/llama_stack/distributions/starter/starter.py index 9ce0eb799..32264eebb 100644 --- a/src/llama_stack/distributions/starter/starter.py +++ b/src/llama_stack/distributions/starter/starter.py @@ -123,7 +123,6 @@ def get_distribution_template(name: str = "starter") -> DistributionTemplate: BuildProvider(provider_type="remote::weaviate"), ], "files": [BuildProvider(provider_type="inline::localfs")], - "file_processor": [BuildProvider(provider_type="inline::reference")], "safety": [ BuildProvider(provider_type="inline::llama-guard"), BuildProvider(provider_type="inline::code-scanner"), diff --git a/src/llama_stack/log.py b/src/llama_stack/log.py index 83e6b96b6..92c137073 100644 --- a/src/llama_stack/log.py +++ b/src/llama_stack/log.py @@ -45,7 +45,7 @@ CATEGORIES = [ "providers", "models", "files", - "file_processor", + "file_processors", "vector_io", "tool_runtime", "cli", diff --git a/src/llama_stack/providers/inline/file_processor/reference/__init__.py b/src/llama_stack/providers/inline/file_processor/reference/__init__.py deleted file mode 100644 index 3c8b6a7ec..000000000 --- a/src/llama_stack/providers/inline/file_processor/reference/__init__.py +++ /dev/null @@ -1,15 +0,0 @@ -# Copyright (c) Meta Platforms, Inc. and affiliates. -# All rights reserved. -# -# This source code is licensed under the terms described in the LICENSE file in -# the root directory of this source tree. - -from .config import ReferenceFileProcessorImplConfig - - -async def get_provider_impl(config: ReferenceFileProcessorImplConfig, deps): - from .reference import ReferenceFileProcessorImpl - - impl = ReferenceFileProcessorImpl(config, deps) - await impl.initialize() - return impl diff --git a/src/llama_stack/providers/inline/file_processor/reference/config.py b/src/llama_stack/providers/inline/file_processor/reference/config.py deleted file mode 100644 index 7c6de7483..000000000 --- a/src/llama_stack/providers/inline/file_processor/reference/config.py +++ /dev/null @@ -1,15 +0,0 @@ -# Copyright (c) Meta Platforms, Inc. and affiliates. -# All rights reserved. -# -# This source code is licensed under the terms described in the LICENSE file in -# the root directory of this source tree. - -from pydantic import BaseModel - - -class ReferenceFileProcessorImplConfig(BaseModel): - """Configuration for the reference file processor implementation.""" - - @staticmethod - def sample_run_config(**kwargs): - return {} diff --git a/src/llama_stack/providers/inline/file_processor/reference/reference.py b/src/llama_stack/providers/inline/file_processor/reference/reference.py deleted file mode 100644 index c69675bdf..000000000 --- a/src/llama_stack/providers/inline/file_processor/reference/reference.py +++ /dev/null @@ -1,41 +0,0 @@ -# Copyright (c) Meta Platforms, Inc. and affiliates. -# All rights reserved. -# -# This source code is licensed under the terms described in the LICENSE file in -# the root directory of this source tree. - -from typing import Any - -from llama_stack_api import FileProcessor, ProcessedContent, VectorStoreChunkingStrategy - -from .config import ReferenceFileProcessorImplConfig - - -class ReferenceFileProcessorImpl(FileProcessor): - """Reference implementation of the FileProcessor API.""" - - def __init__(self, config: ReferenceFileProcessorImplConfig, deps: dict[str, Any]): - self.config = config - self.deps = deps - - async def initialize(self) -> None: - pass - - async def process_file( - self, - file_data: bytes, - filename: str, - options: dict[str, Any] | None = None, - chunking_strategy: VectorStoreChunkingStrategy | None = None, - include_embeddings: bool = False, - ) -> ProcessedContent: - """Process a file into structured content.""" - return ProcessedContent( - content="Placeholder content", - chunks=None, - embeddings=None, - metadata={ - "processor": "reference", - "filename": filename, - }, - ) diff --git a/src/llama_stack/providers/registry/file_processor.py b/src/llama_stack/providers/registry/file_processor.py deleted file mode 100644 index b930928a8..000000000 --- a/src/llama_stack/providers/registry/file_processor.py +++ /dev/null @@ -1,20 +0,0 @@ -# Copyright (c) Meta Platforms, Inc. and affiliates. -# All rights reserved. -# -# This source code is licensed under the terms described in the LICENSE file in -# the root directory of this source tree. - -from llama_stack_api import Api, InlineProviderSpec, ProviderSpec - - -def available_providers() -> list[ProviderSpec]: - return [ - InlineProviderSpec( - api=Api.file_processor, - provider_type="inline::reference", - pip_packages=[], - module="llama_stack.providers.inline.file_processor.reference", - config_class="llama_stack.providers.inline.file_processor.reference.config.ReferenceFileProcessorImplConfig", - description="Reference file processor implementation (placeholder for development)", - ), - ] diff --git a/src/llama_stack/providers/registry/file_processors.py b/src/llama_stack/providers/registry/file_processors.py new file mode 100644 index 000000000..fef58de51 --- /dev/null +++ b/src/llama_stack/providers/registry/file_processors.py @@ -0,0 +1,11 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the terms described in the LICENSE file in +# the root directory of this source tree. + +from llama_stack_api import ProviderSpec + + +def available_providers() -> list[ProviderSpec]: + return [] diff --git a/src/llama_stack_api/__init__.py b/src/llama_stack_api/__init__.py index 139627cfa..758059047 100644 --- a/src/llama_stack_api/__init__.py +++ b/src/llama_stack_api/__init__.py @@ -112,7 +112,7 @@ from .datatypes import ( VectorStoresProtocolPrivate, ) from .eval import BenchmarkConfig, Eval, EvalCandidate, EvaluateResponse, ModelCandidate -from .file_processor import FileProcessor, ProcessedContent, ProcessFileRequest +from .file_processors import FileProcessors, ProcessedContent, ProcessFileRequest from .files import ( ExpiresAfter, Files, @@ -519,7 +519,7 @@ __all__ = [ "ExpiresAfter", "ExternalApiSpec", "ExtraBodyField", - "FileProcessor", + "FileProcessors", "Files", "Fp8QuantizationConfig", "clear_dynamic_schema_types", diff --git a/src/llama_stack_api/datatypes.py b/src/llama_stack_api/datatypes.py index 34bb8ddc2..fb8ed7039 100644 --- a/src/llama_stack_api/datatypes.py +++ b/src/llama_stack_api/datatypes.py @@ -110,7 +110,7 @@ class Api(Enum, metaclass=DynamicApiMeta): :cvar benchmarks: Benchmark suite management :cvar tool_groups: Tool group organization :cvar files: File storage and management - :cvar file_processor: File parsing and processing operations + :cvar file_processors: File parsing and processing operations :cvar prompts: Prompt versions and management :cvar inspect: Built-in system inspection and introspection """ @@ -135,7 +135,7 @@ class Api(Enum, metaclass=DynamicApiMeta): benchmarks = "benchmarks" tool_groups = "tool_groups" files = "files" - file_processor = "file_processor" + file_processors = "file_processors" prompts = "prompts" conversations = "conversations" diff --git a/src/llama_stack_api/file_processor.py b/src/llama_stack_api/file_processors.py similarity index 96% rename from src/llama_stack_api/file_processor.py rename to src/llama_stack_api/file_processors.py index eaa8f697a..157b31e15 100644 --- a/src/llama_stack_api/file_processor.py +++ b/src/llama_stack_api/file_processors.py @@ -53,7 +53,7 @@ class ProcessedContent(BaseModel): @telemetry_traceable @runtime_checkable -class FileProcessor(Protocol): +class FileProcessors(Protocol): """ File Processor API for converting files into structured, processable content. @@ -72,7 +72,7 @@ class FileProcessor(Protocol): processing capabilities, and optimization strategies. """ - @webmethod(route="/file-processor/process", method="POST", level=LLAMA_STACK_API_V1ALPHA) + @webmethod(route="/file-processors/process", method="POST", level=LLAMA_STACK_API_V1ALPHA) async def process_file( self, file_data: bytes,