phoenix-oss/llama-stack-mirror
0
0
Fork 1
mirror of https://github.com/meta-llama/llama-stack.git synced 2025-12-23 12:53:56 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

feat: pre-commit result

Browse source
This commit is contained in:
Young Han 2025-07-14 14:32:58 -07:00
parent f65a260cdd
commit b884bc490b
10 changed files with 286 additions and 288 deletions
Download patch file Download diff file Expand all files Collapse all files

2
docs/source/providers/inference/remote_llamacpp.md
View file

@ -11,7 +11,7 @@
```yaml ```yaml
openai_compat_api_base: ${env.LLAMACPP_URL:http://localhost:8080}/v1 openai_compat_api_base: ${env.LLAMACPP_URL:http://localhost:8080}/v1
api_key: ${env.LLAMACPP_API_KEY:} api_key: ${env.LLAMACPP_API_KEY:=}
``` ```

14
llama_stack/providers/remote/inference/llamacpp/config.py
View file

@ -4,15 +4,15 @@
# This source code is licensed under the terms described in the LICENSE file in # This source code is licensed under the terms described in the LICENSE file in
# the root directory of this source tree. # the root directory of this source tree.
from typing import Any, Union from typing import Any
from llama_stack.schema_utils import json_schema_type
from pydantic import BaseModel, Field from pydantic import BaseModel, Field
from llama_stack.schema_utils import json_schema_type
class LlamaCppProviderDataValidator(BaseModel): class LlamaCppProviderDataValidator(BaseModel):
llamacpp_api_key: Union[str, None] = Field( llamacpp_api_key: str | None = Field(
default=None, default=None,
description="API key for llama.cpp server (optional for local servers)", description="API key for llama.cpp server (optional for local servers)",
) )
@ -20,7 +20,7 @@ class LlamaCppProviderDataValidator(BaseModel):
@json_schema_type @json_schema_type
class LlamaCppImplConfig(BaseModel): class LlamaCppImplConfig(BaseModel):
api_key: Union[str, None] = Field( api_key: str | None = Field(
default=None, default=None,
description="The llama.cpp server API key (optional for local servers)", description="The llama.cpp server API key (optional for local servers)",
) )
@ -31,9 +31,7 @@ class LlamaCppImplConfig(BaseModel):
) )
@classmethod @classmethod
def sample_run_config( def sample_run_config(cls, api_key: str = "${env.LLAMACPP_API_KEY:=}") -> dict[str, Any]:
cls, api_key: str = "${env.LLAMACPP_API_KEY:}"
) -> dict[str, Any]:
return { return {
"openai_compat_api_base": "${env.LLAMACPP_URL:http://localhost:8080}/v1", "openai_compat_api_base": "${env.LLAMACPP_URL:http://localhost:8080}/v1",
"api_key": api_key, "api_key": api_key,

6
llama_stack/templates/llamacpp/build.yaml
View file

@ -1,4 +1,4 @@
version: "2" version: 2
distribution_spec: distribution_spec:
description: Use llama.cpp server for running LLM inference description: Use llama.cpp server for running LLM inference
providers: providers:
@ -34,5 +34,5 @@ distribution_spec:
- remote::model-context-protocol - remote::model-context-protocol
image_type: conda image_type: conda
additional_pip_packages: additional_pip_packages:
- aiosqlite - aiosqlite
- sqlalchemy[asyncio] - sqlalchemy[asyncio]

15
llama_stack/templates/llamacpp/llamacpp.py
View file

@ -21,7 +21,7 @@ from llama_stack.templates.template import DistributionTemplate, RunConfigSettin
def get_distribution_template() -> DistributionTemplate: def get_distribution_template() -> DistributionTemplate:
providers = { providers = {
"inference": ["remote::llamacpp"], "inference": ["remote::llamacpp", "inline::sentence-transformers"],
"vector_io": ["inline::faiss", "remote::chromadb", "remote::pgvector"], "vector_io": ["inline::faiss", "remote::chromadb", "remote::pgvector"],
"safety": ["inline::llama-guard"], "safety": ["inline::llama-guard"],
"agents": ["inline::meta-reference"], "agents": ["inline::meta-reference"],
@ -44,6 +44,11 @@ def get_distribution_template() -> DistributionTemplate:
provider_type="remote::llamacpp", provider_type="remote::llamacpp",
config=LlamaCppImplConfig.sample_run_config(), config=LlamaCppImplConfig.sample_run_config(),
) )
sentence_transformers_provider = Provider(
provider_id="sentence-transformers",
provider_type="inline::sentence-transformers",
config={},
)
vector_io_provider_faiss = Provider( vector_io_provider_faiss = Provider(
provider_id="faiss", provider_id="faiss",
provider_type="inline::faiss", provider_type="inline::faiss",
@ -52,9 +57,7 @@ def get_distribution_template() -> DistributionTemplate:
files_provider = Provider( files_provider = Provider(
provider_id="meta-reference-files", provider_id="meta-reference-files",
provider_type="inline::localfs", provider_type="inline::localfs",
config=LocalfsFilesImplConfig.sample_run_config( config=LocalfsFilesImplConfig.sample_run_config(f"~/.llama/distributions/{name}"),
f"~/.llama/distributions/{name}"
),
) )
inference_model = ModelInput( inference_model = ModelInput(
model_id="${env.INFERENCE_MODEL}", model_id="${env.INFERENCE_MODEL}",
@ -97,7 +100,7 @@ def get_distribution_template() -> DistributionTemplate:
run_configs={ run_configs={
"run.yaml": RunConfigSettings( "run.yaml": RunConfigSettings(
provider_overrides={ provider_overrides={
"inference": [inference_provider], "inference": [inference_provider, sentence_transformers_provider],
"vector_io": [vector_io_provider_faiss], "vector_io": [vector_io_provider_faiss],
"files": [files_provider], "files": [files_provider],
}, },
@ -106,7 +109,7 @@ def get_distribution_template() -> DistributionTemplate:
), ),
"run-with-safety.yaml": RunConfigSettings( "run-with-safety.yaml": RunConfigSettings(
provider_overrides={ provider_overrides={
"inference": [inference_provider], "inference": [inference_provider, sentence_transformers_provider],
"vector_io": [vector_io_provider_faiss], "vector_io": [vector_io_provider_faiss],
"files": [files_provider], "files": [files_provider],
"safety": [ "safety": [

84
llama_stack/templates/llamacpp/run-with-safety.yaml
View file

@ -1,23 +1,23 @@
version: "2" version: 2
image_name: llamacpp image_name: llamacpp
apis: apis:
- agents - agents
- datasetio - datasetio
- eval - eval
- files - files
- inference - inference
- safety - safety
- scoring - scoring
- telemetry - telemetry
- tool_runtime - tool_runtime
- vector_io - vector_io
providers: providers:
inference: inference:
- provider_id: llamacpp - provider_id: llamacpp
provider_type: remote::llamacpp provider_type: remote::llamacpp
config: config:
openai_compat_api_base: ${env.LLAMACPP_URL:http://localhost:8080}/v1 openai_compat_api_base: ${env.LLAMACPP_URL:http://localhost:8080}/v1
api_key: ${env.LLAMACPP_API_KEY:} api_key: ${env.LLAMACPP_API_KEY:=}
- provider_id: sentence-transformers - provider_id: sentence-transformers
provider_type: inline::sentence-transformers provider_type: inline::sentence-transformers
config: {} config: {}
@ -27,8 +27,7 @@ providers:
config: config:
kvstore: kvstore:
type: sqlite type: sqlite
namespace: null db_path: ${env.SQLITE_STORE_DIR:=~/.llama/distributions/llamacpp}/faiss_store.db
db_path: ${env.SQLITE_STORE_DIR:~/.llama/distributions/llamacpp}/faiss_store.db
safety: safety:
- provider_id: llama-guard - provider_id: llama-guard
provider_type: inline::llama-guard provider_type: inline::llama-guard
@ -39,41 +38,38 @@ providers:
config: config:
persistence_store: persistence_store:
type: sqlite type: sqlite
namespace: null db_path: ${env.SQLITE_STORE_DIR:=~/.llama/distributions/llamacpp}/agents_store.db
db_path: ${env.SQLITE_STORE_DIR:~/.llama/distributions/llamacpp}/agents_store.db
responses_store: responses_store:
type: sqlite type: sqlite
db_path: ${env.SQLITE_STORE_DIR:~/.llama/distributions/llamacpp}/responses_store.db db_path: ${env.SQLITE_STORE_DIR:=~/.llama/distributions/llamacpp}/responses_store.db
telemetry: telemetry:
- provider_id: meta-reference - provider_id: meta-reference
provider_type: inline::meta-reference provider_type: inline::meta-reference
config: config:
service_name: "${env.OTEL_SERVICE_NAME:\u200B}" service_name: "${env.OTEL_SERVICE_NAME:=\u200B}"
sinks: ${env.TELEMETRY_SINKS:console,sqlite} sinks: ${env.TELEMETRY_SINKS:=console,sqlite}
sqlite_db_path: ${env.SQLITE_STORE_DIR:~/.llama/distributions/llamacpp}/trace_store.db sqlite_db_path: ${env.SQLITE_STORE_DIR:=~/.llama/distributions/llamacpp}/trace_store.db
otel_exporter_otlp_endpoint: ${env.OTEL_EXPORTER_OTLP_ENDPOINT:=}
eval: eval:
- provider_id: meta-reference - provider_id: meta-reference
provider_type: inline::meta-reference provider_type: inline::meta-reference
config: config:
kvstore: kvstore:
type: sqlite type: sqlite
namespace: null db_path: ${env.SQLITE_STORE_DIR:=~/.llama/distributions/llamacpp}/meta_reference_eval.db
db_path: ${env.SQLITE_STORE_DIR:~/.llama/distributions/llamacpp}/meta_reference_eval.db
datasetio: datasetio:
- provider_id: huggingface - provider_id: huggingface
provider_type: remote::huggingface provider_type: remote::huggingface
config: config:
kvstore: kvstore:
type: sqlite type: sqlite
namespace: null db_path: ${env.SQLITE_STORE_DIR:=~/.llama/distributions/llamacpp}/huggingface_datasetio.db
db_path: ${env.SQLITE_STORE_DIR:~/.llama/distributions/llamacpp}/huggingface_datasetio.db
- provider_id: localfs - provider_id: localfs
provider_type: inline::localfs provider_type: inline::localfs
config: config:
kvstore: kvstore:
type: sqlite type: sqlite
namespace: null db_path: ${env.SQLITE_STORE_DIR:=~/.llama/distributions/llamacpp}/localfs_datasetio.db
db_path: ${env.SQLITE_STORE_DIR:~/.llama/distributions/llamacpp}/localfs_datasetio.db
scoring: scoring:
- provider_id: basic - provider_id: basic
provider_type: inline::basic provider_type: inline::basic
@ -84,69 +80,69 @@ providers:
- provider_id: braintrust - provider_id: braintrust
provider_type: inline::braintrust provider_type: inline::braintrust
config: config:
openai_api_key: ${env.OPENAI_API_KEY:} openai_api_key: ${env.OPENAI_API_KEY:=}
files: files:
- provider_id: meta-reference-files - provider_id: meta-reference-files
provider_type: inline::localfs provider_type: inline::localfs
config: config:
storage_dir: ${env.FILES_STORAGE_DIR:~/.llama/distributions/llamacpp/files} storage_dir: ${env.FILES_STORAGE_DIR:=~/.llama/distributions/llamacpp/files}
metadata_store: metadata_store:
type: sqlite type: sqlite
db_path: ${env.SQLITE_STORE_DIR:~/.llama/distributions/llamacpp}/files_metadata.db db_path: ${env.SQLITE_STORE_DIR:=~/.llama/distributions/llamacpp}/files_metadata.db
tool_runtime: tool_runtime:
- provider_id: brave-search - provider_id: brave-search
provider_type: remote::brave-search provider_type: remote::brave-search
config: config:
api_key: ${env.BRAVE_SEARCH_API_KEY:} api_key: ${env.BRAVE_SEARCH_API_KEY:=}
max_results: 3 max_results: 3
- provider_id: tavily-search - provider_id: tavily-search
provider_type: remote::tavily-search provider_type: remote::tavily-search
config: config:
api_key: ${env.TAVILY_SEARCH_API_KEY:} api_key: ${env.TAVILY_SEARCH_API_KEY:=}
max_results: 3 max_results: 3
- provider_id: wolfram-alpha
provider_type: remote::wolfram-alpha
config:
api_key: ${env.WOLFRAM_ALPHA_API_KEY:=}
- provider_id: rag-runtime - provider_id: rag-runtime
provider_type: inline::rag-runtime provider_type: inline::rag-runtime
config: {} config: {}
- provider_id: model-context-protocol - provider_id: model-context-protocol
provider_type: remote::model-context-protocol provider_type: remote::model-context-protocol
config: {} config: {}
- provider_id: wolfram-alpha
provider_type: remote::wolfram-alpha
config:
api_key: ${env.WOLFRAM_ALPHA_API_KEY:}
metadata_store: metadata_store:
type: sqlite type: sqlite
db_path: ${env.SQLITE_STORE_DIR:~/.llama/distributions/llamacpp}/registry.db db_path: ${env.SQLITE_STORE_DIR:=~/.llama/distributions/llamacpp}/registry.db
inference_store: inference_store:
type: sqlite type: sqlite
db_path: ${env.SQLITE_STORE_DIR:~/.llama/distributions/llamacpp}/inference_store.db db_path: ${env.SQLITE_STORE_DIR:=~/.llama/distributions/llamacpp}/inference_store.db
models: models:
- metadata: {} - metadata: {}
model_id: ${env.INFERENCE_MODEL} model_id: ${env.INFERENCE_MODEL}
provider_id: llamacpp provider_id: llamacpp
model_type: llm model_type: llm
- metadata: {} - metadata: {}
model_id: ${env.SAFETY_MODEL} model_id: ${env.SAFETY_MODEL}
provider_id: llamacpp provider_id: llamacpp
model_type: llm model_type: llm
- metadata: - metadata:
embedding_dimension: 384 embedding_dimension: 384
model_id: all-MiniLM-L6-v2 model_id: all-MiniLM-L6-v2
provider_id: sentence-transformers provider_id: sentence-transformers
model_type: embedding model_type: embedding
shields: shields:
- shield_id: ${env.SAFETY_MODEL} - shield_id: ${env.SAFETY_MODEL}
provider_id: llama-guard provider_id: llama-guard
vector_dbs: [] vector_dbs: []
datasets: [] datasets: []
scoring_fns: [] scoring_fns: []
benchmarks: [] benchmarks: []
tool_groups: tool_groups:
- toolgroup_id: builtin::websearch - toolgroup_id: builtin::websearch
provider_id: tavily-search provider_id: tavily-search
- toolgroup_id: builtin::rag - toolgroup_id: builtin::rag
provider_id: rag-runtime provider_id: rag-runtime
- toolgroup_id: builtin::wolfram_alpha - toolgroup_id: builtin::wolfram_alpha
provider_id: wolfram-alpha provider_id: wolfram-alpha
server: server:
port: 8321 port: 8321

73
llama_stack/templates/llamacpp/run.yaml
View file

@ -1,23 +1,23 @@
version: "2" version: 2
image_name: llamacpp image_name: llamacpp
apis: apis:
- agents - agents
- datasetio - datasetio
- eval - eval
- files - files
- inference - inference
- safety - safety
- scoring - scoring
- telemetry - telemetry
- tool_runtime - tool_runtime
- vector_io - vector_io
providers: providers:
inference: inference:
- provider_id: llamacpp - provider_id: llamacpp
provider_type: remote::llamacpp provider_type: remote::llamacpp
config: config:
openai_compat_api_base: ${env.LLAMACPP_URL:http://localhost:8080}/v1 openai_compat_api_base: ${env.LLAMACPP_URL:http://localhost:8080}/v1
api_key: ${env.LLAMACPP_API_KEY:} api_key: ${env.LLAMACPP_API_KEY:=}
- provider_id: sentence-transformers - provider_id: sentence-transformers
provider_type: inline::sentence-transformers provider_type: inline::sentence-transformers
config: {} config: {}
@ -27,8 +27,7 @@ providers:
config: config:
kvstore: kvstore:
type: sqlite type: sqlite
namespace: null db_path: ${env.SQLITE_STORE_DIR:=~/.llama/distributions/llamacpp}/faiss_store.db
db_path: ${env.SQLITE_STORE_DIR:~/.llama/distributions/llamacpp}/faiss_store.db
safety: safety:
- provider_id: llama-guard - provider_id: llama-guard
provider_type: inline::llama-guard provider_type: inline::llama-guard
@ -40,11 +39,10 @@ providers:
config: config:
persistence_store: persistence_store:
type: sqlite type: sqlite
namespace: null db_path: ${env.SQLITE_STORE_DIR:=~/.llama/distributions/llamacpp}/agents_store.db
db_path: ${env.SQLITE_STORE_DIR:~/.llama/distributions/llamacpp}/agents_store.db
responses_store: responses_store:
type: sqlite type: sqlite
db_path: ${env.SQLITE_STORE_DIR:~/.llama/distributions/llamacpp}/responses_store.db db_path: ${env.SQLITE_STORE_DIR:=~/.llama/distributions/llamacpp}/responses_store.db
telemetry: telemetry:
- provider_id: meta-reference - provider_id: meta-reference
provider_type: inline::meta-reference provider_type: inline::meta-reference
@ -59,23 +57,20 @@ providers:
config: config:
kvstore: kvstore:
type: sqlite type: sqlite
namespace: null db_path: ${env.SQLITE_STORE_DIR:=~/.llama/distributions/llamacpp}/meta_reference_eval.db
db_path: ${env.SQLITE_STORE_DIR:~/.llama/distributions/llamacpp}/meta_reference_eval.db
datasetio: datasetio:
- provider_id: huggingface - provider_id: huggingface
provider_type: remote::huggingface provider_type: remote::huggingface
config: config:
kvstore: kvstore:
type: sqlite type: sqlite
namespace: null db_path: ${env.SQLITE_STORE_DIR:=~/.llama/distributions/llamacpp}/huggingface_datasetio.db
db_path: ${env.SQLITE_STORE_DIR:~/.llama/distributions/llamacpp}/huggingface_datasetio.db
- provider_id: localfs - provider_id: localfs
provider_type: inline::localfs provider_type: inline::localfs
config: config:
kvstore: kvstore:
type: sqlite type: sqlite
namespace: null db_path: ${env.SQLITE_STORE_DIR:=~/.llama/distributions/llamacpp}/localfs_datasetio.db
db_path: ${env.SQLITE_STORE_DIR:~/.llama/distributions/llamacpp}/localfs_datasetio.db
scoring: scoring:
- provider_id: basic - provider_id: basic
provider_type: inline::basic provider_type: inline::basic
@ -86,48 +81,48 @@ providers:
- provider_id: braintrust - provider_id: braintrust
provider_type: inline::braintrust provider_type: inline::braintrust
config: config:
openai_api_key: ${env.OPENAI_API_KEY:} openai_api_key: ${env.OPENAI_API_KEY:=}
files: files:
- provider_id: meta-reference-files - provider_id: meta-reference-files
provider_type: inline::localfs provider_type: inline::localfs
config: config:
storage_dir: ${env.FILES_STORAGE_DIR:~/.llama/distributions/llamacpp/files} storage_dir: ${env.FILES_STORAGE_DIR:=~/.llama/distributions/llamacpp/files}
metadata_store: metadata_store:
type: sqlite type: sqlite
db_path: ${env.SQLITE_STORE_DIR:~/.llama/distributions/llamacpp}/files_metadata.db db_path: ${env.SQLITE_STORE_DIR:=~/.llama/distributions/llamacpp}/files_metadata.db
tool_runtime: tool_runtime:
- provider_id: brave-search - provider_id: brave-search
provider_type: remote::brave-search provider_type: remote::brave-search
config: config:
api_key: ${env.BRAVE_SEARCH_API_KEY:} api_key: ${env.BRAVE_SEARCH_API_KEY:=}
max_results: 3 max_results: 3
- provider_id: tavily-search - provider_id: tavily-search
provider_type: remote::tavily-search provider_type: remote::tavily-search
config: config:
api_key: ${env.TAVILY_SEARCH_API_KEY:} api_key: ${env.TAVILY_SEARCH_API_KEY:=}
max_results: 3 max_results: 3
- provider_id: wolfram-alpha
provider_type: remote::wolfram-alpha
config:
api_key: ${env.WOLFRAM_ALPHA_API_KEY:=}
- provider_id: rag-runtime - provider_id: rag-runtime
provider_type: inline::rag-runtime provider_type: inline::rag-runtime
config: {} config: {}
- provider_id: model-context-protocol - provider_id: model-context-protocol
provider_type: remote::model-context-protocol provider_type: remote::model-context-protocol
config: {} config: {}
- provider_id: wolfram-alpha
provider_type: remote::wolfram-alpha
config:
api_key: ${env.WOLFRAM_ALPHA_API_KEY:}
metadata_store: metadata_store:
type: sqlite type: sqlite
db_path: ${env.SQLITE_STORE_DIR:~/.llama/distributions/llamacpp}/registry.db db_path: ${env.SQLITE_STORE_DIR:=~/.llama/distributions/llamacpp}/registry.db
inference_store: inference_store:
type: sqlite type: sqlite
db_path: ${env.SQLITE_STORE_DIR:~/.llama/distributions/llamacpp}/inference_store.db db_path: ${env.SQLITE_STORE_DIR:=~/.llama/distributions/llamacpp}/inference_store.db
models: models:
- metadata: {} - metadata: {}
model_id: ${env.INFERENCE_MODEL} model_id: ${env.INFERENCE_MODEL}
provider_id: llamacpp provider_id: llamacpp
model_type: llm model_type: llm
- metadata: - metadata:
embedding_dimension: 384 embedding_dimension: 384
model_id: all-MiniLM-L6-v2 model_id: all-MiniLM-L6-v2
provider_id: sentence-transformers provider_id: sentence-transformers
@ -138,11 +133,11 @@ datasets: []
scoring_fns: [] scoring_fns: []
benchmarks: [] benchmarks: []
tool_groups: tool_groups:
- toolgroup_id: builtin::websearch - toolgroup_id: builtin::websearch
provider_id: tavily-search provider_id: tavily-search
- toolgroup_id: builtin::rag - toolgroup_id: builtin::rag
provider_id: rag-runtime provider_id: rag-runtime
- toolgroup_id: builtin::wolfram_alpha - toolgroup_id: builtin::wolfram_alpha
provider_id: wolfram-alpha provider_id: wolfram-alpha
server: server:
port: 8321 port: 8321

1
llama_stack/templates/starter/build.yaml
View file

@ -27,6 +27,7 @@ distribution_spec:
- remote::cerebras-openai-compat - remote::cerebras-openai-compat
- remote::sambanova - remote::sambanova
- remote::passthrough - remote::passthrough
- remote::llamacpp
- inline::sentence-transformers - inline::sentence-transformers
vector_io: vector_io:
- inline::faiss - inline::faiss

5
llama_stack/templates/starter/run.yaml
View file

@ -130,6 +130,11 @@ providers:
config: config:
url: ${env.PASSTHROUGH_URL} url: ${env.PASSTHROUGH_URL}
api_key: ${env.PASSTHROUGH_API_KEY} api_key: ${env.PASSTHROUGH_API_KEY}
- provider_id: ${env.ENABLE_LLAMACPP:=__disabled__}
provider_type: remote::llamacpp
config:
openai_compat_api_base: ${env.LLAMACPP_URL:http://localhost:8080}/v1
api_key: ${env.LLAMACPP_API_KEY:=}
- provider_id: ${env.ENABLE_SENTENCE_TRANSFORMERS:=sentence-transformers} - provider_id: ${env.ENABLE_SENTENCE_TRANSFORMERS:=sentence-transformers}
provider_type: inline::sentence-transformers provider_type: inline::sentence-transformers
config: {} config: {}