phoenix-oss/llama-stack-mirror
0
0
Fork 1
mirror of https://github.com/meta-llama/llama-stack.git synced 2025-12-23 05:02:26 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

feat: pre-commit result

Browse source
This commit is contained in:
Young Han 2025-07-14 14:32:58 -07:00
parent f65a260cdd
commit b884bc490b
10 changed files with 286 additions and 288 deletions
Download patch file Download diff file Expand all files Collapse all files

2
docs/source/providers/inference/remote_llamacpp.md
View file

@ -11,7 +11,7 @@
```yaml
openai_compat_api_base: ${env.LLAMACPP_URL:http://localhost:8080}/v1
api_key: ${env.LLAMACPP_API_KEY:}
api_key: ${env.LLAMACPP_API_KEY:=}
```

14
llama_stack/providers/remote/inference/llamacpp/config.py
View file

@ -4,15 +4,15 @@
# This source code is licensed under the terms described in the LICENSE file in
# the root directory of this source tree.
from typing import Any, Union
from llama_stack.schema_utils import json_schema_type
from typing import Any
from pydantic import BaseModel, Field
from llama_stack.schema_utils import json_schema_type
class LlamaCppProviderDataValidator(BaseModel):
llamacpp_api_key: Union[str, None] = Field(
llamacpp_api_key: str | None = Field(
default=None,
description="API key for llama.cpp server (optional for local servers)",
)
@ -20,7 +20,7 @@ class LlamaCppProviderDataValidator(BaseModel):
@json_schema_type
class LlamaCppImplConfig(BaseModel):
api_key: Union[str, None] = Field(
api_key: str | None = Field(
default=None,
description="The llama.cpp server API key (optional for local servers)",
)
@ -31,9 +31,7 @@ class LlamaCppImplConfig(BaseModel):
)
@classmethod
def sample_run_config(
cls, api_key: str = "${env.LLAMACPP_API_KEY:}"
) -> dict[str, Any]:
def sample_run_config(cls, api_key: str = "${env.LLAMACPP_API_KEY:=}") -> dict[str, Any]:
return {
"openai_compat_api_base": "${env.LLAMACPP_URL:http://localhost:8080}/v1",
"api_key": api_key,

2
llama_stack/templates/llamacpp/build.yaml
View file

@ -1,4 +1,4 @@
version: "2"
version: 2
distribution_spec:
description: Use llama.cpp server for running LLM inference
providers:

15
llama_stack/templates/llamacpp/llamacpp.py
View file

@ -21,7 +21,7 @@ from llama_stack.templates.template import DistributionTemplate, RunConfigSettin
def get_distribution_template() -> DistributionTemplate:
providers = {
"inference": ["remote::llamacpp"],
"inference": ["remote::llamacpp", "inline::sentence-transformers"],
"vector_io": ["inline::faiss", "remote::chromadb", "remote::pgvector"],
"safety": ["inline::llama-guard"],
"agents": ["inline::meta-reference"],
@ -44,6 +44,11 @@ def get_distribution_template() -> DistributionTemplate:
provider_type="remote::llamacpp",
config=LlamaCppImplConfig.sample_run_config(),
)
sentence_transformers_provider = Provider(
provider_id="sentence-transformers",
provider_type="inline::sentence-transformers",
config={},
)
vector_io_provider_faiss = Provider(
provider_id="faiss",
provider_type="inline::faiss",
@ -52,9 +57,7 @@ def get_distribution_template() -> DistributionTemplate:
files_provider = Provider(
provider_id="meta-reference-files",
provider_type="inline::localfs",
config=LocalfsFilesImplConfig.sample_run_config(
f"~/.llama/distributions/{name}"
),
config=LocalfsFilesImplConfig.sample_run_config(f"~/.llama/distributions/{name}"),
)
inference_model = ModelInput(
model_id="${env.INFERENCE_MODEL}",
@ -97,7 +100,7 @@ def get_distribution_template() -> DistributionTemplate:
run_configs={
"run.yaml": RunConfigSettings(
provider_overrides={
"inference": [inference_provider],
"inference": [inference_provider, sentence_transformers_provider],
"vector_io": [vector_io_provider_faiss],
"files": [files_provider],
},
@ -106,7 +109,7 @@ def get_distribution_template() -> DistributionTemplate:
),
"run-with-safety.yaml": RunConfigSettings(
provider_overrides={
"inference": [inference_provider],
"inference": [inference_provider, sentence_transformers_provider],
"vector_io": [vector_io_provider_faiss],
"files": [files_provider],
"safety": [

50
llama_stack/templates/llamacpp/run-with-safety.yaml
View file

@ -1,4 +1,4 @@
version: "2"
version: 2
image_name: llamacpp
apis:
- agents
@ -17,7 +17,7 @@ providers:
provider_type: remote::llamacpp
config:
openai_compat_api_base: ${env.LLAMACPP_URL:http://localhost:8080}/v1
api_key: ${env.LLAMACPP_API_KEY:}
api_key: ${env.LLAMACPP_API_KEY:=}
- provider_id: sentence-transformers
provider_type: inline::sentence-transformers
config: {}
@ -27,8 +27,7 @@ providers:
config:
kvstore:
type: sqlite
namespace: null
db_path: ${env.SQLITE_STORE_DIR:~/.llama/distributions/llamacpp}/faiss_store.db
db_path: ${env.SQLITE_STORE_DIR:=~/.llama/distributions/llamacpp}/faiss_store.db
safety:
- provider_id: llama-guard
provider_type: inline::llama-guard
@ -39,41 +38,38 @@ providers:
config:
persistence_store:
type: sqlite
namespace: null
db_path: ${env.SQLITE_STORE_DIR:~/.llama/distributions/llamacpp}/agents_store.db
db_path: ${env.SQLITE_STORE_DIR:=~/.llama/distributions/llamacpp}/agents_store.db
responses_store:
type: sqlite
db_path: ${env.SQLITE_STORE_DIR:~/.llama/distributions/llamacpp}/responses_store.db
db_path: ${env.SQLITE_STORE_DIR:=~/.llama/distributions/llamacpp}/responses_store.db
telemetry:
- provider_id: meta-reference
provider_type: inline::meta-reference
config:
service_name: "${env.OTEL_SERVICE_NAME:\u200B}"
sinks: ${env.TELEMETRY_SINKS:console,sqlite}
sqlite_db_path: ${env.SQLITE_STORE_DIR:~/.llama/distributions/llamacpp}/trace_store.db
service_name: "${env.OTEL_SERVICE_NAME:=\u200B}"
sinks: ${env.TELEMETRY_SINKS:=console,sqlite}
sqlite_db_path: ${env.SQLITE_STORE_DIR:=~/.llama/distributions/llamacpp}/trace_store.db
otel_exporter_otlp_endpoint: ${env.OTEL_EXPORTER_OTLP_ENDPOINT:=}
eval:
- provider_id: meta-reference
provider_type: inline::meta-reference
config:
kvstore:
type: sqlite
namespace: null
db_path: ${env.SQLITE_STORE_DIR:~/.llama/distributions/llamacpp}/meta_reference_eval.db
db_path: ${env.SQLITE_STORE_DIR:=~/.llama/distributions/llamacpp}/meta_reference_eval.db
datasetio:
- provider_id: huggingface
provider_type: remote::huggingface
config:
kvstore:
type: sqlite
namespace: null
db_path: ${env.SQLITE_STORE_DIR:~/.llama/distributions/llamacpp}/huggingface_datasetio.db
db_path: ${env.SQLITE_STORE_DIR:=~/.llama/distributions/llamacpp}/huggingface_datasetio.db
- provider_id: localfs
provider_type: inline::localfs
config:
kvstore:
type: sqlite
namespace: null
db_path: ${env.SQLITE_STORE_DIR:~/.llama/distributions/llamacpp}/localfs_datasetio.db
db_path: ${env.SQLITE_STORE_DIR:=~/.llama/distributions/llamacpp}/localfs_datasetio.db
scoring:
- provider_id: basic
provider_type: inline::basic
@ -84,42 +80,42 @@ providers:
- provider_id: braintrust
provider_type: inline::braintrust
config:
openai_api_key: ${env.OPENAI_API_KEY:}
openai_api_key: ${env.OPENAI_API_KEY:=}
files:
- provider_id: meta-reference-files
provider_type: inline::localfs
config:
storage_dir: ${env.FILES_STORAGE_DIR:~/.llama/distributions/llamacpp/files}
storage_dir: ${env.FILES_STORAGE_DIR:=~/.llama/distributions/llamacpp/files}
metadata_store:
type: sqlite
db_path: ${env.SQLITE_STORE_DIR:~/.llama/distributions/llamacpp}/files_metadata.db
db_path: ${env.SQLITE_STORE_DIR:=~/.llama/distributions/llamacpp}/files_metadata.db
tool_runtime:
- provider_id: brave-search
provider_type: remote::brave-search
config:
api_key: ${env.BRAVE_SEARCH_API_KEY:}
api_key: ${env.BRAVE_SEARCH_API_KEY:=}
max_results: 3
- provider_id: tavily-search
provider_type: remote::tavily-search
config:
api_key: ${env.TAVILY_SEARCH_API_KEY:}
api_key: ${env.TAVILY_SEARCH_API_KEY:=}
max_results: 3
- provider_id: wolfram-alpha
provider_type: remote::wolfram-alpha
config:
api_key: ${env.WOLFRAM_ALPHA_API_KEY:=}
- provider_id: rag-runtime
provider_type: inline::rag-runtime
config: {}
- provider_id: model-context-protocol
provider_type: remote::model-context-protocol
config: {}
- provider_id: wolfram-alpha
provider_type: remote::wolfram-alpha
config:
api_key: ${env.WOLFRAM_ALPHA_API_KEY:}
metadata_store:
type: sqlite
db_path: ${env.SQLITE_STORE_DIR:~/.llama/distributions/llamacpp}/registry.db
db_path: ${env.SQLITE_STORE_DIR:=~/.llama/distributions/llamacpp}/registry.db
inference_store:
type: sqlite
db_path: ${env.SQLITE_STORE_DIR:~/.llama/distributions/llamacpp}/inference_store.db
db_path: ${env.SQLITE_STORE_DIR:=~/.llama/distributions/llamacpp}/inference_store.db
models:
- metadata: {}
model_id: ${env.INFERENCE_MODEL}

43
llama_stack/templates/llamacpp/run.yaml
View file

@ -1,4 +1,4 @@
version: "2"
version: 2
image_name: llamacpp
apis:
- agents
@ -17,7 +17,7 @@ providers:
provider_type: remote::llamacpp
config:
openai_compat_api_base: ${env.LLAMACPP_URL:http://localhost:8080}/v1
api_key: ${env.LLAMACPP_API_KEY:}
api_key: ${env.LLAMACPP_API_KEY:=}
- provider_id: sentence-transformers
provider_type: inline::sentence-transformers
config: {}
@ -27,8 +27,7 @@ providers:
config:
kvstore:
type: sqlite
namespace: null
db_path: ${env.SQLITE_STORE_DIR:~/.llama/distributions/llamacpp}/faiss_store.db
db_path: ${env.SQLITE_STORE_DIR:=~/.llama/distributions/llamacpp}/faiss_store.db
safety:
- provider_id: llama-guard
provider_type: inline::llama-guard
@ -40,11 +39,10 @@ providers:
config:
persistence_store:
type: sqlite
namespace: null
db_path: ${env.SQLITE_STORE_DIR:~/.llama/distributions/llamacpp}/agents_store.db
db_path: ${env.SQLITE_STORE_DIR:=~/.llama/distributions/llamacpp}/agents_store.db
responses_store:
type: sqlite
db_path: ${env.SQLITE_STORE_DIR:~/.llama/distributions/llamacpp}/responses_store.db
db_path: ${env.SQLITE_STORE_DIR:=~/.llama/distributions/llamacpp}/responses_store.db
telemetry:
- provider_id: meta-reference
provider_type: inline::meta-reference
@ -59,23 +57,20 @@ providers:
config:
kvstore:
type: sqlite
namespace: null
db_path: ${env.SQLITE_STORE_DIR:~/.llama/distributions/llamacpp}/meta_reference_eval.db
db_path: ${env.SQLITE_STORE_DIR:=~/.llama/distributions/llamacpp}/meta_reference_eval.db
datasetio:
- provider_id: huggingface
provider_type: remote::huggingface
config:
kvstore:
type: sqlite
namespace: null
db_path: ${env.SQLITE_STORE_DIR:~/.llama/distributions/llamacpp}/huggingface_datasetio.db
db_path: ${env.SQLITE_STORE_DIR:=~/.llama/distributions/llamacpp}/huggingface_datasetio.db
- provider_id: localfs
provider_type: inline::localfs
config:
kvstore:
type: sqlite
namespace: null
db_path: ${env.SQLITE_STORE_DIR:~/.llama/distributions/llamacpp}/localfs_datasetio.db
db_path: ${env.SQLITE_STORE_DIR:=~/.llama/distributions/llamacpp}/localfs_datasetio.db
scoring:
- provider_id: basic
provider_type: inline::basic
@ -86,42 +81,42 @@ providers:
- provider_id: braintrust
provider_type: inline::braintrust
config:
openai_api_key: ${env.OPENAI_API_KEY:}
openai_api_key: ${env.OPENAI_API_KEY:=}
files:
- provider_id: meta-reference-files
provider_type: inline::localfs
config:
storage_dir: ${env.FILES_STORAGE_DIR:~/.llama/distributions/llamacpp/files}
storage_dir: ${env.FILES_STORAGE_DIR:=~/.llama/distributions/llamacpp/files}
metadata_store:
type: sqlite
db_path: ${env.SQLITE_STORE_DIR:~/.llama/distributions/llamacpp}/files_metadata.db
db_path: ${env.SQLITE_STORE_DIR:=~/.llama/distributions/llamacpp}/files_metadata.db
tool_runtime:
- provider_id: brave-search
provider_type: remote::brave-search
config:
api_key: ${env.BRAVE_SEARCH_API_KEY:}
api_key: ${env.BRAVE_SEARCH_API_KEY:=}
max_results: 3
- provider_id: tavily-search
provider_type: remote::tavily-search
config:
api_key: ${env.TAVILY_SEARCH_API_KEY:}
api_key: ${env.TAVILY_SEARCH_API_KEY:=}
max_results: 3
- provider_id: wolfram-alpha
provider_type: remote::wolfram-alpha
config:
api_key: ${env.WOLFRAM_ALPHA_API_KEY:=}
- provider_id: rag-runtime
provider_type: inline::rag-runtime
config: {}
- provider_id: model-context-protocol
provider_type: remote::model-context-protocol
config: {}
- provider_id: wolfram-alpha
provider_type: remote::wolfram-alpha
config:
api_key: ${env.WOLFRAM_ALPHA_API_KEY:}
metadata_store:
type: sqlite
db_path: ${env.SQLITE_STORE_DIR:~/.llama/distributions/llamacpp}/registry.db
db_path: ${env.SQLITE_STORE_DIR:=~/.llama/distributions/llamacpp}/registry.db
inference_store:
type: sqlite
db_path: ${env.SQLITE_STORE_DIR:~/.llama/distributions/llamacpp}/inference_store.db
db_path: ${env.SQLITE_STORE_DIR:=~/.llama/distributions/llamacpp}/inference_store.db
models:
- metadata: {}
model_id: ${env.INFERENCE_MODEL}

1
llama_stack/templates/starter/build.yaml
View file

@ -27,6 +27,7 @@ distribution_spec:
- remote::cerebras-openai-compat
- remote::sambanova
- remote::passthrough
- remote::llamacpp
- inline::sentence-transformers
vector_io:
- inline::faiss

5
llama_stack/templates/starter/run.yaml
View file

@ -130,6 +130,11 @@ providers:
config:
url: ${env.PASSTHROUGH_URL}
api_key: ${env.PASSTHROUGH_API_KEY}
- provider_id: ${env.ENABLE_LLAMACPP:=__disabled__}
provider_type: remote::llamacpp
config:
openai_compat_api_base: ${env.LLAMACPP_URL:http://localhost:8080}/v1
api_key: ${env.LLAMACPP_API_KEY:=}
- provider_id: ${env.ENABLE_SENTENCE_TRANSFORMERS:=sentence-transformers}
provider_type: inline::sentence-transformers
config: {}