phoenix-oss/llama-stack-mirror
0
0
Fork 1
mirror of https://github.com/meta-llama/llama-stack.git synced 2025-07-17 18:38:11 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

fix: remove sentence-transformers from remote vllm

Browse source 
vLLM itself can perform the embeddings generation so we don't need this
extra provider.

Signed-off-by: Sébastien Han <seb@redhat.com>
This commit is contained in:
Sébastien Han 2025-06-03 18:00:27 +02:00
parent 3c9a10d2fe
commit 7e30b5a466
No known key found for this signature in database
5 changed files with 6 additions and 22 deletions
Download patch file Download diff file Expand all files Collapse all files

15
llama_stack/templates/remote-vllm/vllm.py
View file

@ -13,9 +13,6 @@ from llama_stack.distribution.datatypes import (
ShieldInput,
ToolGroupInput,
)
from llama_stack.providers.inline.inference.sentence_transformers import (
SentenceTransformersInferenceConfig,
)
from llama_stack.providers.inline.vector_io.faiss.config import FaissVectorIOConfig
from llama_stack.providers.remote.inference.vllm import VLLMInferenceAdapterConfig
from llama_stack.templates.template import DistributionTemplate, RunConfigSettings
@ -23,7 +20,7 @@ from llama_stack.templates.template import DistributionTemplate, RunConfigSettin
def get_distribution_template() -> DistributionTemplate:
providers = {
"inference": ["remote::vllm", "inline::sentence-transformers"],
"inference": ["remote::vllm"],
"vector_io": ["inline::faiss", "remote::chromadb", "remote::pgvector"],
"safety": ["inline::llama-guard"],
"agents": ["inline::meta-reference"],
@ -47,11 +44,6 @@ def get_distribution_template() -> DistributionTemplate:
url="${env.VLLM_URL:http://localhost:8000/v1}",
),
)
embedding_provider = Provider(
provider_id="sentence-transformers",
provider_type="inline::sentence-transformers",
config=SentenceTransformersInferenceConfig.sample_run_config(),
)
vector_io_provider = Provider(
provider_id="faiss",
provider_type="inline::faiss",
@ -68,7 +60,7 @@ def get_distribution_template() -> DistributionTemplate:
)
embedding_model = ModelInput(
model_id="all-MiniLM-L6-v2",
provider_id="sentence-transformers",
provider_id="vllm-inference",
model_type=ModelType.embedding,
metadata={
"embedding_dimension": 384,
@ -98,7 +90,7 @@ def get_distribution_template() -> DistributionTemplate:
run_configs={
"run.yaml": RunConfigSettings(
provider_overrides={
"inference": [inference_provider, embedding_provider],
"inference": [inference_provider],
"vector_io": [vector_io_provider],
},
default_models=[inference_model, embedding_model],
@ -115,7 +107,6 @@ def get_distribution_template() -> DistributionTemplate:
url="${env.SAFETY_VLLM_URL}",
),
),
embedding_provider,
],
"vector_io": [vector_io_provider],
},