phoenix-oss/llama-stack-mirror
0
0
Fork 1
mirror of https://github.com/meta-llama/llama-stack.git synced 2025-08-02 08:44:44 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

all distros

Browse source
This commit is contained in:
Xi Yan 2024-12-03 20:51:21 -08:00
parent 7103892f54
commit a0d79c68f9
2 changed files with 9 additions and 8 deletions
Download patch file Download diff file Expand all files Collapse all files

6
llama_stack/templates/vllm-gpu/run.yaml
View file

@ -29,12 +29,6 @@ providers:
type: sqlite type: sqlite
namespace: null namespace: null
db_path: ${env.SQLITE_STORE_DIR:~/.llama/distributions/vllm-gpu}/faiss_store.db db_path: ${env.SQLITE_STORE_DIR:~/.llama/distributions/vllm-gpu}/faiss_store.db
- provider_id: chromadb
provider_type: remote::chromadb
config: {}
- provider_id: pgvector
provider_type: remote::pgvector
config: {}
safety: safety:
- provider_id: llama-guard - provider_id: llama-guard
provider_type: inline::llama-guard provider_type: inline::llama-guard

11
llama_stack/templates/vllm-gpu/vllm.py
View file

@ -6,6 +6,7 @@
from llama_stack.distribution.datatypes import ModelInput, Provider from llama_stack.distribution.datatypes import ModelInput, Provider
from llama_stack.providers.inline.inference.vllm import VLLMConfig from llama_stack.providers.inline.inference.vllm import VLLMConfig
from llama_stack.providers.inline.memory.faiss.config import FaissImplConfig
from llama_stack.templates.template import DistributionTemplate, RunConfigSettings from llama_stack.templates.template import DistributionTemplate, RunConfigSettings
@ -20,12 +21,17 @@ def get_distribution_template() -> DistributionTemplate:
"datasetio": ["remote::huggingface", "inline::localfs"], "datasetio": ["remote::huggingface", "inline::localfs"],
"scoring": ["inline::basic", "inline::llm-as-judge", "inline::braintrust"], "scoring": ["inline::basic", "inline::llm-as-judge", "inline::braintrust"],
} }
name = "vllm-gpu"
inference_provider = Provider( inference_provider = Provider(
provider_id="vllm", provider_id="vllm",
provider_type="inline::vllm", provider_type="inline::vllm",
config=VLLMConfig.sample_run_config(), config=VLLMConfig.sample_run_config(),
) )
memory_provider = Provider(
provider_id="faiss",
provider_type="inline::faiss",
config=FaissImplConfig.sample_run_config(f"distributions/{name}"),
)
inference_model = ModelInput( inference_model = ModelInput(
model_id="${env.INFERENCE_MODEL}", model_id="${env.INFERENCE_MODEL}",
@ -33,7 +39,7 @@ def get_distribution_template() -> DistributionTemplate:
) )
return DistributionTemplate( return DistributionTemplate(
name="vllm-gpu", name=name,
distro_type="self_hosted", distro_type="self_hosted",
description="Use a built-in vLLM engine for running LLM inference", description="Use a built-in vLLM engine for running LLM inference",
docker_image=None, docker_image=None,
@ -44,6 +50,7 @@ def get_distribution_template() -> DistributionTemplate:
"run.yaml": RunConfigSettings( "run.yaml": RunConfigSettings(
provider_overrides={ provider_overrides={
"inference": [inference_provider], "inference": [inference_provider],
"memory": [memory_provider],
}, },
default_models=[inference_model], default_models=[inference_model],
), ),