Introduce Llama stack distributions (#22)

* Add distribution CLI scaffolding

* More progress towards `llama distribution install`

* getting closer to a distro definition, distro install + configure works

* Distribution server now functioning

* read existing configuration, save enums properly

* Remove inference uvicorn server entrypoint and llama inference CLI command

* updated dependency and client model name

* Improved exception handling

* local imports for faster cli

* undo a typo, add a passthrough distribution

* implement full-passthrough in the server

* add safety adapters, configuration handling, server + clients

* cleanup, moving stuff to common, nuke utils

* Add a Path() wrapper at the earliest place

* fixes

* Bring agentic system api to toolchain

Add adapter dependencies and resolve adapters using a topological sort

* refactor to reduce size of `agentic_system`

* move straggler files and fix some important existing bugs

* ApiSurface -> Api

* refactor a method out

* Adapter -> Provider

* Make each inference provider into its own subdirectory

* installation fixes

* Rename Distribution -> DistributionSpec, simplify RemoteProviders

* dict key instead of attr

* update inference config to take model and not model_dir

* Fix passthrough streaming, send headers properly not part of body :facepalm

* update safety to use model sku ids and not model dirs

* Update cli_reference.md

* minor fixes

* add DistributionConfig, fix a bug in model download

* Make install + start scripts do proper configuration automatically

* Update CLI_reference

* Nuke fp8_requirements, fold fbgemm into common requirements

* Update README, add newline between API surface configurations

* Refactor download functionality out of the Command so can be reused

* Add `llama model download` alias for `llama download`

* Show message about checksum file so users can check themselves

* Simpler intro statements

* get ollama working

* Reduce a bunch of dependencies from toolchain

Some improvements to the distribution install script

* Avoid using `conda run` since it buffers everything

* update dependencies and rely on LLAMA_TOOLCHAIN_DIR for dev purposes

* add validation for configuration input

* resort imports

* make optional subclasses default to yes for configuration

* Remove additional_pip_packages; move deps to providers

* for inline make 8b model the default

* Add scripts to MANIFEST

* allow installing from test.pypi.org

* Fix #2 to help with testing packages

* Must install llama-models at that same version first

* fix PIP_ARGS

---------

Co-authored-by: Hardik Shah <hjshah@fb.com>
Co-authored-by: Hardik Shah <hjshah@meta.com>

llama_toolchain/safety/meta_reference/safety.py

@@ -0,0 +1,103 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the terms described in the LICENSE file in
+# the root directory of this source tree.
+
+import asyncio
+from typing import Dict
+
+from llama_models.sku_list import resolve_model
+
+from llama_toolchain.common.model_utils import model_local_dir
+from llama_toolchain.distribution.datatypes import Api, ProviderSpec
+from llama_toolchain.safety.api import *  # noqa
+
+from .config import SafetyConfig
+from .shields import (
+    CodeScannerShield,
+    InjectionShield,
+    JailbreakShield,
+    LlamaGuardShield,
+    PromptGuardShield,
+    ShieldBase,
+    ThirdPartyShield,
+)
+
+
+async def get_provider_impl(config: SafetyConfig, _deps: Dict[Api, ProviderSpec]):
+    assert isinstance(config, SafetyConfig), f"Unexpected config type: {type(config)}"
+
+    impl = MetaReferenceSafetyImpl(config)
+    await impl.initialize()
+    return impl
+
+
+def resolve_and_get_path(model_name: str) -> str:
+    model = resolve_model(model_name)
+    assert model is not None, f"Could not resolve model {model_name}"
+    model_dir = model_local_dir(model)
+    return model_dir
+
+
+class MetaReferenceSafetyImpl(Safety):
+
+    def __init__(self, config: SafetyConfig) -> None:
+        self.config = config
+
+    async def initialize(self) -> None:
+        shield_cfg = self.config.llama_guard_shield
+        if shield_cfg is not None:
+            model_dir = resolve_and_get_path(shield_cfg.model)
+            _ = LlamaGuardShield.instance(
+                model_dir=model_dir,
+                excluded_categories=shield_cfg.excluded_categories,
+                disable_input_check=shield_cfg.disable_input_check,
+                disable_output_check=shield_cfg.disable_output_check,
+            )
+
+        shield_cfg = self.config.prompt_guard_shield
+        if shield_cfg is not None:
+            model_dir = resolve_and_get_path(shield_cfg.model)
+            _ = PromptGuardShield.instance(model_dir)
+
+    async def run_shields(
+        self,
+        request: RunShieldRequest,
+    ) -> RunShieldResponse:
+        shields = [shield_config_to_shield(c, self.config) for c in request.shields]
+
+        responses = await asyncio.gather(
+            *[shield.run(request.messages) for shield in shields]
+        )
+
+        return RunShieldResponse(responses=responses)
+
+
+def shield_config_to_shield(
+    sc: ShieldDefinition, safety_config: SafetyConfig
+) -> ShieldBase:
+    if sc.shield_type == BuiltinShield.llama_guard:
+        assert (
+            safety_config.llama_guard_shield is not None
+        ), "Cannot use LlamaGuardShield since not present in config"
+        model_dir = resolve_and_get_path(safety_config.llama_guard_shield.model)
+        return LlamaGuardShield.instance(model_dir=model_dir)
+    elif sc.shield_type == BuiltinShield.jailbreak_shield:
+        assert (
+            safety_config.prompt_guard_shield is not None
+        ), "Cannot use Jailbreak Shield since Prompt Guard not present in config"
+        model_dir = resolve_and_get_path(safety_config.prompt_guard_shield.model)
+        return JailbreakShield.instance(model_dir)
+    elif sc.shield_type == BuiltinShield.injection_shield:
+        assert (
+            safety_config.prompt_guard_shield is not None
+        ), "Cannot use PromptGuardShield since not present in config"
+        model_dir = resolve_and_get_path(safety_config.prompt_guard_shield.model)
+        return InjectionShield.instance(model_dir)
+    elif sc.shield_type == BuiltinShield.code_scanner_guard:
+        return CodeScannerShield.instance()
+    elif sc.shield_type == BuiltinShield.third_party_shield:
+        return ThirdPartyShield.instance()
+    else:
+        raise ValueError(f"Unknown shield type: {sc.shield_type}")