mirror of
https://github.com/meta-llama/llama-stack.git
synced 2025-06-27 18:50:41 +00:00
0d2eb3bd25
Test Plan: First, start a TGI container with `meta-llama/Llama-Guard-3-8B` model serving on port 5099. See https://github.com/meta-llama/llama-stack/pull/53 and its description for how. Then run llama-stack with the following run config: ``` image_name: safety docker_image: null conda_env: safety apis_to_serve: - models - inference - shields - safety api_providers: inference: providers: - remote::tgi safety: providers: - meta-reference telemetry: provider_id: meta-reference config: {} routing_table: inference: - provider_id: remote::tgi config: url: http://localhost:5099 api_token: null hf_endpoint_name: null routing_key: Llama-Guard-3-8B safety: - provider_id: meta-reference config: llama_guard_shield: model: Llama-Guard-3-8B excluded_categories: [] disable_input_check: false disable_output_check: false prompt_guard_shield: null routing_key: llama_guard ``` Now simply run `python -m llama_stack.apis.safety.client localhost <port>` and check that the llama_guard shield calls run correctly. (The injection_shield calls fail as expected since we have not set up a router for them.)
37 lines
1.2 KiB
Python
37 lines
1.2 KiB
Python
# Copyright (c) Meta Platforms, Inc. and affiliates.
|
|
# All rights reserved.
|
|
#
|
|
# This source code is licensed under the terms described in the LICENSE file in
|
|
# the root directory of this source tree.
|
|
|
|
from typing import List
|
|
|
|
from llama_stack.distribution.datatypes import * # noqa: F403
|
|
|
|
|
|
def available_providers() -> List[ProviderSpec]:
|
|
return [
|
|
InlineProviderSpec(
|
|
api=Api.safety,
|
|
provider_id="meta-reference",
|
|
pip_packages=[
|
|
"codeshield",
|
|
"transformers",
|
|
"torch --index-url https://download.pytorch.org/whl/cpu",
|
|
],
|
|
module="llama_stack.providers.impls.meta_reference.safety",
|
|
config_class="llama_stack.providers.impls.meta_reference.safety.SafetyConfig",
|
|
api_dependencies=[
|
|
Api.inference,
|
|
],
|
|
),
|
|
remote_provider_spec(
|
|
api=Api.safety,
|
|
adapter=AdapterSpec(
|
|
adapter_id="sample",
|
|
pip_packages=[],
|
|
module="llama_stack.providers.adapters.safety.sample",
|
|
config_class="llama_stack.providers.adapters.safety.sample.SampleConfig",
|
|
),
|
|
),
|
|
]
|