Moving preprocessors.py to a separate directory.

docs/_static/llama-stack-spec.html

@@ -7210,6 +7210,7 @@
                             "preprocessor"
                         ],
                         "title": "ResourceType",
+                        "const": "preprocessor",
                         "default": "preprocessor"
                     },
                     "metadata": {

docs/_static/llama-stack-spec.yaml

@@ -5033,6 +5033,7 @@ components:
             - tool_group
             - preprocessor
           title: ResourceType
+          const: preprocessor
           default: preprocessor
         metadata:
           type: object

llama_stack/apis/preprocessing/preprocessing.py

@@ -9,7 +9,7 @@ from typing import Any, Dict, List, Optional, Protocol, runtime_checkable
 from pydantic import BaseModel
 
 from llama_stack.apis.common.content_types import URL, InterleavedContent
-from llama_stack.apis.preprocessing.preprocessors import Preprocessor
+from llama_stack.apis.preprocessors.preprocessors import Preprocessor
 from llama_stack.apis.vector_io import Chunk
 from llama_stack.schema_utils import json_schema_type, webmethod
 

llama_stack/apis/preprocessors/__init__.py

@@ -0,0 +1,7 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the terms described in the LICENSE file in
+# the root directory of this source tree.
+
+from .preprocessors import *  # noqa: F401 F403

llama_stack/apis/preprocessors/preprocessors.py

@@ -4,7 +4,7 @@
 # This source code is licensed under the terms described in the LICENSE file in
 # the root directory of this source tree.
 
-from typing import Any, Dict, List, Optional, Protocol, runtime_checkable
+from typing import Any, Dict, List, Literal, Optional, Protocol, runtime_checkable
 
 from pydantic import BaseModel
 
@@ -15,7 +15,7 @@ from llama_stack.schema_utils import json_schema_type, webmethod
 
 @json_schema_type
 class Preprocessor(Resource):
-    type: ResourceType = ResourceType.preprocessor
+    type: Literal[ResourceType.preprocessor] = ResourceType.preprocessor
 
     @property
     def preprocessor_id(self) -> str:

llama_stack/distribution/datatypes.py

@@ -15,7 +15,7 @@ from llama_stack.apis.eval import Eval
 from llama_stack.apis.inference import Inference
 from llama_stack.apis.models import Model, ModelInput
 from llama_stack.apis.preprocessing import Preprocessing, Preprocessor
-from llama_stack.apis.preprocessing.preprocessors import PreprocessorInput
+from llama_stack.apis.preprocessors.preprocessors import PreprocessorInput
 from llama_stack.apis.resource import Resource
 from llama_stack.apis.safety import Safety
 from llama_stack.apis.scoring import Scoring

llama_stack/distribution/resolver.py

@@ -18,7 +18,7 @@ from llama_stack.apis.inspect import Inspect
 from llama_stack.apis.models import Models
 from llama_stack.apis.post_training import PostTraining
 from llama_stack.apis.preprocessing import Preprocessing
-from llama_stack.apis.preprocessing.preprocessors import Preprocessors
+from llama_stack.apis.preprocessors.preprocessors import Preprocessors
 from llama_stack.apis.providers import Providers as ProvidersAPI
 from llama_stack.apis.safety import Safety
 from llama_stack.apis.scoring import Scoring

llama_stack/distribution/routers/routing_tables.py

@@ -24,7 +24,7 @@ from llama_stack.apis.datasets import (
     URIDataSource,
 )
 from llama_stack.apis.models import ListModelsResponse, Model, Models, ModelType
-from llama_stack.apis.preprocessing.preprocessors import ListPreprocessorsResponse, Preprocessor, Preprocessors
+from llama_stack.apis.preprocessors.preprocessors import ListPreprocessorsResponse, Preprocessor, Preprocessors
 from llama_stack.apis.resource import ResourceType
 from llama_stack.apis.scoring_functions import (
     ListScoringFunctionsResponse,

llama_stack/distribution/stack.py

@@ -24,7 +24,7 @@ from llama_stack.apis.inspect import Inspect
 from llama_stack.apis.models import Models
 from llama_stack.apis.post_training import PostTraining
 from llama_stack.apis.preprocessing import Preprocessing
-from llama_stack.apis.preprocessing.preprocessors import Preprocessors
+from llama_stack.apis.preprocessors.preprocessors import Preprocessors
 from llama_stack.apis.providers import Providers
 from llama_stack.apis.safety import Safety
 from llama_stack.apis.scoring import Scoring

llama_stack/templates/bedrock/bedrock.py

@@ -6,7 +6,7 @@
 
 from pathlib import Path
 
-from llama_stack.apis.preprocessing.preprocessors import PreprocessorInput
+from llama_stack.apis.preprocessors.preprocessors import PreprocessorInput
 from llama_stack.distribution.datatypes import Provider, ToolGroupInput
 from llama_stack.providers.inline.vector_io.faiss.config import FaissVectorIOConfig
 from llama_stack.providers.remote.inference.bedrock.models import MODEL_ENTRIES

llama_stack/templates/cerebras/cerebras.py

@@ -7,7 +7,7 @@
 from pathlib import Path
 
 from llama_stack.apis.models.models import ModelType
-from llama_stack.apis.preprocessing.preprocessors import PreprocessorInput
+from llama_stack.apis.preprocessors.preprocessors import PreprocessorInput
 from llama_stack.distribution.datatypes import ModelInput, Provider, ToolGroupInput
 from llama_stack.providers.inline.inference.sentence_transformers import (
     SentenceTransformersInferenceConfig,

llama_stack/templates/ci-tests/ci_tests.py

@@ -6,7 +6,7 @@
 
 
 from llama_stack.apis.models.models import ModelType
-from llama_stack.apis.preprocessing.preprocessors import PreprocessorInput
+from llama_stack.apis.preprocessors.preprocessors import PreprocessorInput
 from llama_stack.distribution.datatypes import (
     ModelInput,
     Provider,

llama_stack/templates/dell/dell.py

@@ -5,7 +5,7 @@
 # the root directory of this source tree.
 
 from llama_stack.apis.models.models import ModelType
-from llama_stack.apis.preprocessing.preprocessors import PreprocessorInput
+from llama_stack.apis.preprocessors.preprocessors import PreprocessorInput
 from llama_stack.distribution.datatypes import (
     ModelInput,
     Provider,

llama_stack/templates/dev/dev.py

@@ -7,7 +7,7 @@
 from typing import List, Tuple
 
 from llama_stack.apis.models.models import ModelType
-from llama_stack.apis.preprocessing.preprocessors import PreprocessorInput
+from llama_stack.apis.preprocessors.preprocessors import PreprocessorInput
 from llama_stack.distribution.datatypes import (
     ModelInput,
     Provider,

llama_stack/templates/fireworks/fireworks.py

@@ -7,7 +7,7 @@
 from pathlib import Path
 
 from llama_stack.apis.models.models import ModelType
-from llama_stack.apis.preprocessing.preprocessors import PreprocessorInput
+from llama_stack.apis.preprocessors.preprocessors import PreprocessorInput
 from llama_stack.distribution.datatypes import (
     ModelInput,
     Provider,

llama_stack/templates/groq/groq.py

@@ -7,7 +7,7 @@
 from pathlib import Path
 
 from llama_stack.apis.models.models import ModelType
-from llama_stack.apis.preprocessing.preprocessors import PreprocessorInput
+from llama_stack.apis.preprocessors.preprocessors import PreprocessorInput
 from llama_stack.distribution.datatypes import ModelInput, Provider, ToolGroupInput
 from llama_stack.providers.inline.inference.sentence_transformers import (
     SentenceTransformersInferenceConfig,

llama_stack/templates/hf-endpoint/hf_endpoint.py

@@ -5,7 +5,7 @@
 # the root directory of this source tree.
 
 from llama_stack.apis.models.models import ModelType
-from llama_stack.apis.preprocessing.preprocessors import PreprocessorInput
+from llama_stack.apis.preprocessors.preprocessors import PreprocessorInput
 from llama_stack.distribution.datatypes import (
     ModelInput,
     Provider,

llama_stack/templates/hf-serverless/hf_serverless.py

@@ -5,7 +5,7 @@
 # the root directory of this source tree.
 
 from llama_stack.apis.models.models import ModelType
-from llama_stack.apis.preprocessing.preprocessors import PreprocessorInput
+from llama_stack.apis.preprocessors.preprocessors import PreprocessorInput
 from llama_stack.distribution.datatypes import (
     ModelInput,
     Provider,

llama_stack/templates/meta-reference-gpu/meta_reference.py

@@ -7,7 +7,7 @@
 from pathlib import Path
 
 from llama_stack.apis.models.models import ModelType
-from llama_stack.apis.preprocessing.preprocessors import PreprocessorInput
+from llama_stack.apis.preprocessors.preprocessors import PreprocessorInput
 from llama_stack.distribution.datatypes import (
     ModelInput,
     Provider,

llama_stack/templates/meta-reference-quantized-gpu/meta_reference.py

@@ -7,7 +7,7 @@
 from pathlib import Path
 
 from llama_stack.apis.models.models import ModelType
-from llama_stack.apis.preprocessing.preprocessors import PreprocessorInput
+from llama_stack.apis.preprocessors.preprocessors import PreprocessorInput
 from llama_stack.distribution.datatypes import ModelInput, Provider, ToolGroupInput
 from llama_stack.providers.inline.inference.meta_reference import (
     MetaReferenceQuantizedInferenceConfig,

llama_stack/templates/nvidia/nvidia.py

@@ -6,7 +6,7 @@
 
 from pathlib import Path
 
-from llama_stack.apis.preprocessing.preprocessors import PreprocessorInput
+from llama_stack.apis.preprocessors.preprocessors import PreprocessorInput
 from llama_stack.distribution.datatypes import ModelInput, Provider, ShieldInput, ToolGroupInput
 from llama_stack.providers.remote.inference.nvidia import NVIDIAConfig
 from llama_stack.providers.remote.inference.nvidia.models import MODEL_ENTRIES

llama_stack/templates/ollama/ollama.py

@@ -7,7 +7,7 @@
 from pathlib import Path
 
 from llama_stack.apis.models.models import ModelType
-from llama_stack.apis.preprocessing.preprocessors import PreprocessorInput
+from llama_stack.apis.preprocessors.preprocessors import PreprocessorInput
 from llama_stack.distribution.datatypes import (
     ModelInput,
     Provider,

llama_stack/templates/remote-vllm/vllm.py

@@ -7,7 +7,7 @@
 from pathlib import Path
 
 from llama_stack.apis.models.models import ModelType
-from llama_stack.apis.preprocessing.preprocessors import PreprocessorInput
+from llama_stack.apis.preprocessors.preprocessors import PreprocessorInput
 from llama_stack.distribution.datatypes import (
     ModelInput,
     Provider,

llama_stack/templates/sambanova/sambanova.py

@@ -6,7 +6,7 @@
 
 from pathlib import Path
 
-from llama_stack.apis.preprocessing.preprocessors import PreprocessorInput
+from llama_stack.apis.preprocessors.preprocessors import PreprocessorInput
 from llama_stack.distribution.datatypes import Provider, ShieldInput, ToolGroupInput
 from llama_stack.providers.inline.vector_io.faiss.config import FaissVectorIOConfig
 from llama_stack.providers.remote.inference.sambanova import SambaNovaImplConfig

llama_stack/templates/template.py

@@ -13,7 +13,7 @@ from pydantic import BaseModel, Field
 
 from llama_stack.apis.datasets import DatasetPurpose
 from llama_stack.apis.models.models import ModelType
-from llama_stack.apis.preprocessing.preprocessors import PreprocessorInput
+from llama_stack.apis.preprocessors.preprocessors import PreprocessorInput
 from llama_stack.distribution.datatypes import (
     Api,
     BenchmarkInput,

llama_stack/templates/tgi/tgi.py

@@ -7,7 +7,7 @@
 from pathlib import Path
 
 from llama_stack.apis.models.models import ModelType
-from llama_stack.apis.preprocessing.preprocessors import PreprocessorInput
+from llama_stack.apis.preprocessors.preprocessors import PreprocessorInput
 from llama_stack.distribution.datatypes import (
     ModelInput,
     Provider,

llama_stack/templates/together/together.py

@@ -7,7 +7,7 @@
 from pathlib import Path
 
 from llama_stack.apis.models.models import ModelType
-from llama_stack.apis.preprocessing.preprocessors import PreprocessorInput
+from llama_stack.apis.preprocessors.preprocessors import PreprocessorInput
 from llama_stack.distribution.datatypes import (
     ModelInput,
     Provider,

llama_stack/templates/vllm-gpu/vllm.py

@@ -5,7 +5,7 @@
 # the root directory of this source tree.
 
 from llama_stack.apis.models.models import ModelType
-from llama_stack.apis.preprocessing.preprocessors import PreprocessorInput
+from llama_stack.apis.preprocessors.preprocessors import PreprocessorInput
 from llama_stack.distribution.datatypes import ModelInput, Provider
 from llama_stack.providers.inline.inference.sentence_transformers import (
     SentenceTransformersInferenceConfig,