phoenix-oss/llama-stack
1
0
Fork 0
forked from phoenix-oss/llama-stack-mirror
Code Pull requests Releases Packages 2 Activity Actions

feat(providers): sambanova updated to use LiteLLM openai-compat (#1596)

Browse source 
# What does this PR do?

switch sambanova inference adaptor to LiteLLM usage to simplify
integration and solve issues with current adaptor when streaming and
tool calling, models and templates updated

## Test Plan
pytest -s -v tests/integration/inference/test_text_inference.py
--stack-config=sambanova
--text-model=sambanova/Meta-Llama-3.3-70B-Instruct

pytest -s -v tests/integration/inference/test_vision_inference.py
--stack-config=sambanova
--vision-model=sambanova/Llama-3.2-11B-Vision-Instruct
This commit is contained in:
Jorge Piedrahita Ortiz 2025-05-06 18:50:22 -05:00 • committed by GitHub
parent dd49ef31f1
commit b2b00a216b
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
15 changed files with 529 additions and 404 deletions
Download patch file Download diff file Expand all files Collapse all files

44
llama_stack/templates/sambanova/sambanova.py
View file

@ -6,7 +6,16 @@
from pathlib import Path
from llama_stack.distribution.datatypes import Provider, ShieldInput, ToolGroupInput
from llama_stack.apis.models.models import ModelType
from llama_stack.distribution.datatypes import (
ModelInput,
Provider,
ShieldInput,
ToolGroupInput,
)
from llama_stack.providers.inline.inference.sentence_transformers import (
SentenceTransformersInferenceConfig,
)
from llama_stack.providers.inline.vector_io.faiss.config import FaissVectorIOConfig
from llama_stack.providers.remote.inference.sambanova import SambaNovaImplConfig
from llama_stack.providers.remote.inference.sambanova.models import MODEL_ENTRIES
@ -23,7 +32,7 @@ from llama_stack.templates.template import (
def get_distribution_template() -> DistributionTemplate:
providers = {
"inference": ["remote::sambanova"],
"inference": ["remote::sambanova", "inline::sentence-transformers"],
"vector_io": ["inline::faiss", "remote::chromadb", "remote::pgvector"],
"safety": ["inline::llama-guard"],
"agents": ["inline::meta-reference"],
@ -32,16 +41,29 @@ def get_distribution_template() -> DistributionTemplate:
"remote::brave-search",
"remote::tavily-search",
"inline::rag-runtime",
"remote::model-context-protocol",
"remote::wolfram-alpha",
],
}
name = "sambanova"
inference_provider = Provider(
provider_id=name,
provider_type=f"remote::{name}",
config=SambaNovaImplConfig.sample_run_config(),
)
embedding_provider = Provider(
provider_id="sentence-transformers",
provider_type="inline::sentence-transformers",
config=SentenceTransformersInferenceConfig.sample_run_config(),
)
embedding_model = ModelInput(
model_id="all-MiniLM-L6-v2",
provider_id="sentence-transformers",
model_type=ModelType.embedding,
metadata={
"embedding_dimension": 384,
},
)
vector_io_providers = [
Provider(
provider_id="faiss",
@ -79,23 +101,27 @@ def get_distribution_template() -> DistributionTemplate:
toolgroup_id="builtin::rag",
provider_id="rag-runtime",
),
ToolGroupInput(
toolgroup_id="builtin::wolfram_alpha",
provider_id="wolfram-alpha",
),
]
return DistributionTemplate(
name=name,
distro_type="self_hosted",
description="Use SambaNova.AI for running LLM inference",
docker_image=None,
description="Use SambaNova for running LLM inference",
container_image=None,
template_path=Path(__file__).parent / "doc_template.md",
providers=providers,
available_models_by_provider=available_models,
run_configs={
"run.yaml": RunConfigSettings(
provider_overrides={
"inference": [inference_provider],
"inference": [inference_provider, embedding_provider],
"vector_io": vector_io_providers,
},
default_models=default_models,
default_models=default_models + [embedding_model],
default_shields=[ShieldInput(shield_id="meta-llama/Llama-Guard-3-8B")],
default_tool_groups=default_tool_groups,
),
@ -107,7 +133,7 @@ def get_distribution_template() -> DistributionTemplate:
),
"SAMBANOVA_API_KEY": (
"",
"SambaNova.AI API Key",
"SambaNova API Key",
),
},
)