From 98c55b63b40552ad6e4e9a2fcc38ace252024d39 Mon Sep 17 00:00:00 2001 From: Xi Yan Date: Mon, 16 Sep 2024 12:11:59 -0700 Subject: [PATCH 1/5] delete distribution registry --- .../distribution_registry/local-ollama.yaml | 7 ------ .../local-plus-fireworks-inference.yaml | 7 ------ .../local-plus-tgi-inference.yaml | 6 ----- .../local-plus-together-inference.yaml | 7 ------ .../distribution_registry/local.yaml | 7 ------ llama_toolchain/core/distribution_registry.py | 23 ------------------- 6 files changed, 57 deletions(-) delete mode 100644 llama_toolchain/configs/distributions/distribution_registry/local-ollama.yaml delete mode 100644 llama_toolchain/configs/distributions/distribution_registry/local-plus-fireworks-inference.yaml delete mode 100644 llama_toolchain/configs/distributions/distribution_registry/local-plus-tgi-inference.yaml delete mode 100644 llama_toolchain/configs/distributions/distribution_registry/local-plus-together-inference.yaml delete mode 100644 llama_toolchain/configs/distributions/distribution_registry/local.yaml delete mode 100644 llama_toolchain/core/distribution_registry.py diff --git a/llama_toolchain/configs/distributions/distribution_registry/local-ollama.yaml b/llama_toolchain/configs/distributions/distribution_registry/local-ollama.yaml deleted file mode 100644 index 4518a7eb7..000000000 --- a/llama_toolchain/configs/distributions/distribution_registry/local-ollama.yaml +++ /dev/null @@ -1,7 +0,0 @@ -description: Like local, but use ollama for running LLM inference -providers: - inference: remote::ollama - safety: meta-reference - agentic_system: meta-reference - memory: meta-reference-faiss - telemetry: console diff --git a/llama_toolchain/configs/distributions/distribution_registry/local-plus-fireworks-inference.yaml b/llama_toolchain/configs/distributions/distribution_registry/local-plus-fireworks-inference.yaml deleted file mode 100644 index 520274e1c..000000000 --- a/llama_toolchain/configs/distributions/distribution_registry/local-plus-fireworks-inference.yaml +++ /dev/null @@ -1,7 +0,0 @@ -description: Use Fireworks.ai for running LLM inference -providers: - inference: remote::fireworks - safety: meta-reference - agentic_system: meta-reference - memory: meta-reference-faiss - telemetry: console diff --git a/llama_toolchain/configs/distributions/distribution_registry/local-plus-tgi-inference.yaml b/llama_toolchain/configs/distributions/distribution_registry/local-plus-tgi-inference.yaml deleted file mode 100644 index a0b36b07e..000000000 --- a/llama_toolchain/configs/distributions/distribution_registry/local-plus-tgi-inference.yaml +++ /dev/null @@ -1,6 +0,0 @@ -description: Use TGI (local or with Hugging Face Inference Endpoints for running LLM inference. When using HF Inference Endpoints, you must provide the name of the endpoint). -providers: - inference: remote::tgi - safety: meta-reference - agentic_system: meta-reference - memory: meta-reference-faiss diff --git a/llama_toolchain/configs/distributions/distribution_registry/local-plus-together-inference.yaml b/llama_toolchain/configs/distributions/distribution_registry/local-plus-together-inference.yaml deleted file mode 100644 index 2681b619c..000000000 --- a/llama_toolchain/configs/distributions/distribution_registry/local-plus-together-inference.yaml +++ /dev/null @@ -1,7 +0,0 @@ -description: Use Together.ai for running LLM inference -providers: - inference: remote::together - safety: meta-reference - agentic_system: meta-reference - memory: meta-reference-faiss - telemetry: console diff --git a/llama_toolchain/configs/distributions/distribution_registry/local.yaml b/llama_toolchain/configs/distributions/distribution_registry/local.yaml deleted file mode 100644 index 03dfb1dab..000000000 --- a/llama_toolchain/configs/distributions/distribution_registry/local.yaml +++ /dev/null @@ -1,7 +0,0 @@ -description: Use code from `llama_toolchain` itself to serve all llama stack APIs -providers: - inference: meta-reference - memory: meta-reference-faiss - safety: meta-reference - agentic_system: meta-reference - telemetry: console diff --git a/llama_toolchain/core/distribution_registry.py b/llama_toolchain/core/distribution_registry.py deleted file mode 100644 index 87ac98d7c..000000000 --- a/llama_toolchain/core/distribution_registry.py +++ /dev/null @@ -1,23 +0,0 @@ -# Copyright (c) Meta Platforms, Inc. and affiliates. -# All rights reserved. -# -# This source code is licensed under the terms described in the LICENSE file in -# the root directory of this source tree. - -from functools import lru_cache -from pathlib import Path -from typing import List, Optional -from .datatypes import * # noqa: F403 -import yaml - - -@lru_cache() -def available_distribution_specs() -> List[DistributionSpec]: - distribution_specs = [] - for p in Path("llama_toolchain/configs/distributions/distribution_registry").rglob( - "*.yaml" - ): - with open(p, "r") as f: - distribution_specs.append(DistributionSpec(**yaml.safe_load(f))) - - return distribution_specs From a36699cd11c757c6414b0b8bab423303f1a7eee6 Mon Sep 17 00:00:00 2001 From: Ashwin Bharambe Date: Mon, 16 Sep 2024 12:22:01 -0700 Subject: [PATCH 2/5] Rename the "package" word away --- llama_toolchain/cli/stack/configure.py | 20 ++++++++++---------- llama_toolchain/cli/stack/run.py | 6 +----- llama_toolchain/core/datatypes.py | 4 ++-- 3 files changed, 13 insertions(+), 17 deletions(-) diff --git a/llama_toolchain/cli/stack/configure.py b/llama_toolchain/cli/stack/configure.py index 952f9b50e..4a73f1af4 100644 --- a/llama_toolchain/cli/stack/configure.py +++ b/llama_toolchain/cli/stack/configure.py @@ -102,8 +102,8 @@ class StackConfigure(Subcommand): if output_dir: builds_dir = Path(output_dir) os.makedirs(builds_dir, exist_ok=True) - package_name = build_config.name.replace("::", "-") - package_file = builds_dir / f"{package_name}-run.yaml" + image_name = build_config.name.replace("::", "-") + run_config_file = builds_dir / f"{image_name}-run.yaml" api2providers = build_config.distribution_spec.providers @@ -112,31 +112,31 @@ class StackConfigure(Subcommand): for api_str, provider in api2providers.items() } - if package_file.exists(): + if run_config_file.exists(): cprint( f"Configuration already exists for {build_config.name}. Will overwrite...", "yellow", attrs=["bold"], ) - config = PackageConfig(**yaml.safe_load(package_file.read_text())) + config = StackRunConfig(**yaml.safe_load(run_config_file.read_text())) else: - config = PackageConfig( + config = StackRunConfig( built_at=datetime.now(), - package_name=package_name, + image_name=image_name, providers=stub_config, ) config.providers = configure_api_providers(config.providers) config.docker_image = ( - package_name if build_config.image_type == "docker" else None + image_name if build_config.image_type == "docker" else None ) - config.conda_env = package_name if build_config.image_type == "conda" else None + config.conda_env = image_name if build_config.image_type == "conda" else None - with open(package_file, "w") as f: + with open(run_config_file, "w") as f: to_write = json.loads(json.dumps(config.dict(), cls=EnumEncoder)) f.write(yaml.dump(to_write, sort_keys=False)) cprint( - f"> YAML configuration has been written to {package_file}", + f"> YAML configuration has been written to {run_config_file}", color="blue", ) diff --git a/llama_toolchain/cli/stack/run.py b/llama_toolchain/cli/stack/run.py index 091cc50a9..b5900eaba 100644 --- a/llama_toolchain/cli/stack/run.py +++ b/llama_toolchain/cli/stack/run.py @@ -13,7 +13,6 @@ import yaml from llama_toolchain.cli.subcommand import Subcommand from llama_toolchain.core.datatypes import * # noqa: F403 -from llama_toolchain.common.config_dirs import BUILDS_BASE_DIR class StackRun(Subcommand): @@ -29,8 +28,6 @@ class StackRun(Subcommand): self.parser.set_defaults(func=self._run_stack_run_cmd) def _add_arguments(self): - from llama_toolchain.core.package import ImageType - self.parser.add_argument( "config", type=str, @@ -51,7 +48,6 @@ class StackRun(Subcommand): def _run_stack_run_cmd(self, args: argparse.Namespace) -> None: from llama_toolchain.common.exec import run_with_pty - from llama_toolchain.core.package import ImageType if not args.config: self.parser.error("Must specify a config file to run") @@ -67,7 +63,7 @@ class StackRun(Subcommand): return with open(config_file, "r") as f: - config = PackageConfig(**yaml.safe_load(f)) + config = StackRunConfig(**yaml.safe_load(f)) if config.docker_image: script = pkg_resources.resource_filename( diff --git a/llama_toolchain/core/datatypes.py b/llama_toolchain/core/datatypes.py index 1366eeb0d..f523e0308 100644 --- a/llama_toolchain/core/datatypes.py +++ b/llama_toolchain/core/datatypes.py @@ -163,10 +163,10 @@ class DistributionSpec(BaseModel): @json_schema_type -class PackageConfig(BaseModel): +class StackRunConfig(BaseModel): built_at: datetime - package_name: str = Field( + image_name: str = Field( ..., description=""" Reference to the distribution this package refers to. For unregistered (adhoc) packages, From 82b5c0460e61ddf7f605a807403de7a69ab767e1 Mon Sep 17 00:00:00 2001 From: Xi Yan Date: Thu, 12 Sep 2024 11:29:19 -0700 Subject: [PATCH 3/5] models api --- llama_toolchain/models/api/api.py | 41 +++++++++++++++++++++++++ llama_toolchain/models/api/endpoints.py | 14 --------- 2 files changed, 41 insertions(+), 14 deletions(-) create mode 100644 llama_toolchain/models/api/api.py delete mode 100644 llama_toolchain/models/api/endpoints.py diff --git a/llama_toolchain/models/api/api.py b/llama_toolchain/models/api/api.py new file mode 100644 index 000000000..f18d421f2 --- /dev/null +++ b/llama_toolchain/models/api/api.py @@ -0,0 +1,41 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the terms described in the LICENSE file in +# the root directory of this source tree. + +from typing import Protocol + +from llama_models.schema_utils import webmethod # noqa: F401 + +from pydantic import BaseModel # noqa: F401 + + +@json_schema_type +class ModelSpec(BaseModel): + model_name: str = Field(description="Name of the model") + providers_spec: Dict[Api, Dict[str, str]] = Field( + default_factory=dict, + description="Map of API to the concrete provider specs. E.g. {}".format( + { + "inference": { + "provider_type": "remote::tgi", + "url": "localhost::5555", + "api_token": "hf_xxx", + } + } + ), + ) + + +class Models(Protocol): + @webmethod(route="/models/list", method="GET") + async def list_models(self) -> List[ModelSpec]: ... + + @webmethod(route="/models/get", method="GET") + async def get_model(self, model_name: str) -> ModelSpec: ... + + @webmethod(route="/models/register") + async def register_model( + self, name: str, provider: Api, provider_spec: Dict[str, str] + ) -> ModelSpec: ... diff --git a/llama_toolchain/models/api/endpoints.py b/llama_toolchain/models/api/endpoints.py deleted file mode 100644 index ee1d5f0ba..000000000 --- a/llama_toolchain/models/api/endpoints.py +++ /dev/null @@ -1,14 +0,0 @@ -# Copyright (c) Meta Platforms, Inc. and affiliates. -# All rights reserved. -# -# This source code is licensed under the terms described in the LICENSE file in -# the root directory of this source tree. - -from typing import Protocol - -from llama_models.schema_utils import webmethod # noqa: F401 - -from pydantic import BaseModel # noqa: F401 - - -class Models(Protocol): ... From 5839c610023a0b8932c70b822bad19ca5b74c416 Mon Sep 17 00:00:00 2001 From: Xi Yan Date: Mon, 16 Sep 2024 13:00:39 -0700 Subject: [PATCH 4/5] stage back models api --- llama_toolchain/models/api/api.py | 29 +---------------------------- 1 file changed, 1 insertion(+), 28 deletions(-) diff --git a/llama_toolchain/models/api/api.py b/llama_toolchain/models/api/api.py index f18d421f2..ee1d5f0ba 100644 --- a/llama_toolchain/models/api/api.py +++ b/llama_toolchain/models/api/api.py @@ -11,31 +11,4 @@ from llama_models.schema_utils import webmethod # noqa: F401 from pydantic import BaseModel # noqa: F401 -@json_schema_type -class ModelSpec(BaseModel): - model_name: str = Field(description="Name of the model") - providers_spec: Dict[Api, Dict[str, str]] = Field( - default_factory=dict, - description="Map of API to the concrete provider specs. E.g. {}".format( - { - "inference": { - "provider_type": "remote::tgi", - "url": "localhost::5555", - "api_token": "hf_xxx", - } - } - ), - ) - - -class Models(Protocol): - @webmethod(route="/models/list", method="GET") - async def list_models(self) -> List[ModelSpec]: ... - - @webmethod(route="/models/get", method="GET") - async def get_model(self, model_name: str) -> ModelSpec: ... - - @webmethod(route="/models/register") - async def register_model( - self, name: str, provider: Api, provider_spec: Dict[str, str] - ) -> ModelSpec: ... +class Models(Protocol): ... From f294eac5f52f90ad3367862baa55ec784c543869 Mon Sep 17 00:00:00 2001 From: Xi Yan Date: Mon, 16 Sep 2024 13:10:05 -0700 Subject: [PATCH 5/5] Bump version to 0.0.17 --- requirements.txt | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 45ca7ed06..6dc053c63 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,7 +2,7 @@ blobfile fire httpx huggingface-hub -llama-models>=0.0.16 +llama-models>=0.0.17 pydantic requests termcolor diff --git a/setup.py b/setup.py index 7273bee51..d0cacb22d 100644 --- a/setup.py +++ b/setup.py @@ -16,7 +16,7 @@ def read_requirements(): setup( name="llama_toolchain", - version="0.0.16", + version="0.0.17", author="Meta Llama", author_email="llama-oss@meta.com", description="Llama toolchain",