phoenix-oss/llama-stack-mirror
0
0
Fork 1
mirror of https://github.com/meta-llama/llama-stack.git synced 2025-07-29 15:23:51 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

move distribution to yaml files

Browse source
This commit is contained in:
Xi Yan 2024-09-14 15:02:22 -07:00
parent 3802d5acdc
commit 0068d059db
9 changed files with 78 additions and 49 deletions
Download patch file Download diff file Expand all files Collapse all files

2
llama_toolchain/cli/stack/list_distributions.py
View file

@ -40,7 +40,7 @@ class StackListDistributions(Subcommand):
rows = []
for spec in available_distribution_specs():
providers = {k.value: v for k, v in spec.providers.items()}
providers = {k: v for k, v in spec.providers.items()}
rows.append(
[
spec.distribution_type,

2
llama_toolchain/configs/distributions/conda/local-tgi-conda-example-build.yaml
View file

@ -1,7 +1,7 @@
name: local-tgi-conda-example
distribution_spec:
distribution_type: local-plus-tgi-inference
description: Use TGI for running LLM inference
description: Use TGI (local or with Hugging Face Inference Endpoints for running LLM inference. When using HF Inference Endpoints, you must provide the name of the endpoint).
docker_image: null
providers:
inference: remote::tgi

9
llama_toolchain/configs/distributions/distribution_registry/local-ollama.yaml Normal file
View file

@ -0,0 +1,9 @@
distribution_type: local-ollama
description: Like local, but use ollama for running LLM inference
docker_image: null
providers:
inference: remote::ollama
safety: meta-reference
agentic_system: meta-reference
memory: meta-reference-faiss
telemetry: console

9
llama_toolchain/configs/distributions/distribution_registry/local-plus-fireworks-inference.yaml Normal file
View file

@ -0,0 +1,9 @@
distribution_type: local-plus-fireworks-inference
description: Use Fireworks.ai for running LLM inference
docker_image: null
providers:
inference: remote::fireworks
safety: meta-reference
agentic_system: meta-reference
memory: meta-reference-faiss
telemetry: console

8
llama_toolchain/configs/distributions/distribution_registry/local-plus-tgi-inference.yaml Normal file
View file

@ -0,0 +1,8 @@
distribution_type: local-plus-tgi-inference
description: Use TGI (local or with Hugging Face Inference Endpoints for running LLM inference. When using HF Inference Endpoints, you must provide the name of the endpoint).
docker_image: null
providers:
inference: remote::tgi
safety: meta-reference
agentic_system: meta-reference
memory: meta-reference-faiss

9
llama_toolchain/configs/distributions/distribution_registry/local-plus-together-inference.yaml Normal file
View file

@ -0,0 +1,9 @@
distribution_type: local-plus-together-inference
description: Use Together.ai for running LLM inference
docker_image: null
providers:
inference: remote::together
safety: meta-reference
agentic_system: meta-reference
memory: meta-reference-faiss
telemetry: console

9
llama_toolchain/configs/distributions/distribution_registry/local.yaml Normal file
View file

@ -0,0 +1,9 @@
distribution_type: local
description: Use code from `llama_toolchain` itself to serve all llama stack APIs
docker_image: null
providers:
inference: meta-reference
memory: meta-reference-faiss
safety: meta-reference
agentic_system: meta-reference
telemetry: console

21
llama_toolchain/configs/distributions/run.py Normal file
View file

@ -0,0 +1,21 @@
from llama_toolchain.core.distribution_registry import *
import json
import fire
import yaml
from llama_toolchain.common.serialize import EnumEncoder
def main():
for d in available_distribution_specs():
file_path = "./configs/distributions/distribution_registry/{}.yaml".format(
d.distribution_type
)
with open(file_path, "w") as f:
to_write = json.loads(json.dumps(d.dict(), cls=EnumEncoder))
f.write(yaml.dump(to_write, sort_keys=False))
if __name__ == "__main__":
fire.Fire(main)

58
llama_toolchain/core/distribution_registry.py
View file

@ -5,55 +5,19 @@
# the root directory of this source tree.
from functools import lru_cache
from pathlib import Path
from typing import List, Optional
from .datatypes import * # noqa: F403
import yaml
@lru_cache()
# @lru_cache()
def available_distribution_specs() -> List[DistributionSpec]:
return [
DistributionSpec(
distribution_type="local",
description="Use code from `llama_toolchain` itself to serve all llama stack APIs",
providers={
"inference": "meta-reference",
"memory": "meta-reference-faiss",
"safety": "meta-reference",
"agentic_system": "meta-reference",
"telemetry": "console",
},
),
DistributionSpec(
distribution_type="local-ollama",
description="Like local, but use ollama for running LLM inference",
providers={
"inference": remote_provider_type("ollama"),
"safety": "meta-reference",
"agentic_system": "meta-reference",
"memory": "meta-reference-faiss",
"telemetry": "console",
},
),
DistributionSpec(
distribution_type="local-plus-fireworks-inference",
description="Use Fireworks.ai for running LLM inference",
providers={
"inference": remote_provider_type("fireworks"),
"safety": "meta-reference",
"agentic_system": "meta-reference",
"memory": "meta-reference-faiss",
"telemetry": "console",
},
),
DistributionSpec(
distribution_type="local-plus-tgi-inference",
description="Use TGI for running LLM inference",
providers={
"inference": remote_provider_type("tgi"),
"safety": "meta-reference",
"agentic_system": "meta-reference",
"memory": "meta-reference-faiss",
},
),
]
distribution_specs = []
for p in Path("llama_toolchain/configs/distributions/distribution_registry").rglob(
"*.yaml"
):
with open(p, "r") as f:
distribution_specs.append(DistributionSpec(**yaml.safe_load(f)))
return distribution_specs