phoenix-oss/llama-stack-mirror
0
0
Fork 1
mirror of https://github.com/meta-llama/llama-stack.git synced 2025-12-17 13:42:39 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

all distros

Browse source
This commit is contained in:
Xi Yan 2024-12-03 20:49:30 -08:00
parent a097bfa761
commit 7103892f54
16 changed files with 79 additions and 62 deletions
Download patch file Download diff file Expand all files Collapse all files

10
llama_stack/templates/hf-serverless/hf_serverless.py
View file

@ -5,6 +5,7 @@
# the root directory of this source tree.
from llama_stack.distribution.datatypes import ModelInput, Provider, ShieldInput
from llama_stack.providers.inline.memory.faiss.config import FaissImplConfig
from llama_stack.providers.remote.inference.tgi import InferenceAPIImplConfig
from llama_stack.templates.template import DistributionTemplate, RunConfigSettings
@ -21,11 +22,17 @@ def get_distribution_template() -> DistributionTemplate:
"scoring": ["inline::basic", "inline::llm-as-judge", "inline::braintrust"],
}
name = "hf-serverless"
inference_provider = Provider(
provider_id="hf-serverless",
provider_type="remote::hf::serverless",
config=InferenceAPIImplConfig.sample_run_config(),
)
memory_provider = Provider(
provider_id="faiss",
provider_type="inline::faiss",
config=FaissImplConfig.sample_run_config(f"distributions/{name}"),
)
inference_model = ModelInput(
model_id="${env.INFERENCE_MODEL}",
@ -37,7 +44,7 @@ def get_distribution_template() -> DistributionTemplate:
)
return DistributionTemplate(
name="hf-serverless",
name=name,
distro_type="self_hosted",
description="Use (an external) Hugging Face Inference Endpoint for running LLM inference",
docker_image=None,
@ -48,6 +55,7 @@ def get_distribution_template() -> DistributionTemplate:
"run.yaml": RunConfigSettings(
provider_overrides={
"inference": [inference_provider],
"memory": [memory_provider],
},
default_models=[inference_model],
),