phoenix-oss/llama-stack-mirror
0
0
Fork 1
mirror of https://github.com/meta-llama/llama-stack.git synced 2025-12-17 17:42:39 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

add embedding model by default

Browse source
This commit is contained in:
Dinesh Yeduguru 2024-12-12 14:46:51 -08:00
parent 2a9b13dd52
commit 2f88006bd0
43 changed files with 446 additions and 85 deletions
Download patch file Download diff file Expand all files Collapse all files

22
llama_stack/templates/tgi/tgi.py
View file

@ -6,7 +6,12 @@
from pathlib import Path
from llama_stack.apis.models.models import ModelType
from llama_stack.distribution.datatypes import ModelInput, Provider, ShieldInput
from llama_stack.providers.inline.inference.sentence_transformers import (
SentenceTransformersInferenceConfig,
)
from llama_stack.providers.inline.memory.faiss.config import FaissImplConfig
from llama_stack.providers.remote.inference.tgi import TGIImplConfig
from llama_stack.templates.template import DistributionTemplate, RunConfigSettings
@ -31,6 +36,11 @@ def get_distribution_template() -> DistributionTemplate:
url="${env.TGI_URL}",
),
)
embedding_provider = Provider(
provider_id="sentence-transformers",
provider_type="inline::sentence-transformers",
config=SentenceTransformersInferenceConfig.sample_run_config(),
)
memory_provider = Provider(
provider_id="faiss",
provider_type="inline::faiss",
@ -41,6 +51,14 @@ def get_distribution_template() -> DistributionTemplate:
model_id="${env.INFERENCE_MODEL}",
provider_id="tgi-inference",
)
embedding_model = ModelInput(
model_id="all-MiniLM-L6-v2",
provider_id="sentence-transformers",
model_type=ModelType.embedding,
metadata={
"embedding_dim": 384,
},
)
safety_model = ModelInput(
model_id="${env.SAFETY_MODEL}",
provider_id="tgi-safety",
@ -57,10 +75,10 @@ def get_distribution_template() -> DistributionTemplate:
run_configs={
"run.yaml": RunConfigSettings(
provider_overrides={
"inference": [inference_provider],
"inference": [inference_provider, embedding_provider],
"memory": [memory_provider],
},
default_models=[inference_model],
default_models=[inference_model, embedding_model],
),
"run-with-safety.yaml": RunConfigSettings(
provider_overrides={