phoenix-oss/llama-stack-mirror
0
0
Fork 1
mirror of https://github.com/meta-llama/llama-stack.git synced 2025-08-02 08:44:44 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

run-with-safety memory

Browse source
This commit is contained in:
Xi Yan 2024-12-03 20:54:59 -08:00
parent bc2452d2e9
commit 26f578cc1d
13 changed files with 18 additions and 47 deletions
Download patch file Download diff file Expand all files Collapse all files

3
llama_stack/templates/hf-endpoint/hf_endpoint.py
View file

@ -69,7 +69,8 @@ def get_distribution_template() -> DistributionTemplate:
endpoint_name="${env.SAFETY_INFERENCE_ENDPOINT_NAME}",
),
),
]
],
"memory": [memory_provider],
},
default_models=[
inference_model,

6
llama_stack/templates/hf-endpoint/run-with-safety.yaml
View file

@ -31,12 +31,6 @@ providers:
type: sqlite
namespace: null
db_path: ${env.SQLITE_STORE_DIR:~/.llama/distributions/hf-endpoint}/faiss_store.db
- provider_id: chromadb
provider_type: remote::chromadb
config: {}
- provider_id: pgvector
provider_type: remote::pgvector
config: {}
safety:
- provider_id: llama-guard
provider_type: inline::llama-guard

3
llama_stack/templates/hf-serverless/hf_serverless.py
View file

@ -70,7 +70,8 @@ def get_distribution_template() -> DistributionTemplate:
repo="${env.SAFETY_MODEL}",
),
),
]
],
"memory": [memory_provider],
},
default_models=[
inference_model,

6
llama_stack/templates/hf-serverless/run-with-safety.yaml
View file

@ -31,12 +31,6 @@ providers:
type: sqlite
namespace: null
db_path: ${env.SQLITE_STORE_DIR:~/.llama/distributions/hf-serverless}/faiss_store.db
- provider_id: chromadb
provider_type: remote::chromadb
config: {}
- provider_id: pgvector
provider_type: remote::pgvector
config: {}
safety:
- provider_id: llama-guard
provider_type: inline::llama-guard

1
llama_stack/templates/meta-reference-gpu/meta_reference.py
View file

@ -77,6 +77,7 @@ def get_distribution_template() -> DistributionTemplate:
),
),
],
"memory": [memory_provider],
},
default_models=[
inference_model,

6
llama_stack/templates/meta-reference-gpu/run-with-safety.yaml
View file

@ -33,12 +33,6 @@ providers:
type: sqlite
namespace: null
db_path: ${env.SQLITE_STORE_DIR:~/.llama/distributions/meta-reference-gpu}/faiss_store.db
- provider_id: chromadb
provider_type: remote::chromadb
config: {}
- provider_id: pgvector
provider_type: remote::pgvector
config: {}
safety:
- provider_id: llama-guard
provider_type: inline::llama-guard

14
llama_stack/templates/ollama/ollama.py
View file

@ -7,6 +7,7 @@
from pathlib import Path
from llama_stack.distribution.datatypes import ModelInput, Provider, ShieldInput
from llama_stack.providers.inline.memory.faiss.config import FaissImplConfig
from llama_stack.providers.remote.inference.ollama import OllamaImplConfig
from llama_stack.templates.template import DistributionTemplate, RunConfigSettings
@ -22,12 +23,17 @@ def get_distribution_template() -> DistributionTemplate:
"datasetio": ["remote::huggingface", "inline::localfs"],
"scoring": ["inline::basic", "inline::llm-as-judge", "inline::braintrust"],
}
name = "ollama"
inference_provider = Provider(
provider_id="ollama",
provider_type="remote::ollama",
config=OllamaImplConfig.sample_run_config(),
)
memory_provider = Provider(
provider_id="faiss",
provider_type="inline::faiss",
config=FaissImplConfig.sample_run_config(f"distributions/{name}"),
)
inference_model = ModelInput(
model_id="${env.INFERENCE_MODEL}",
@ -39,7 +45,7 @@ def get_distribution_template() -> DistributionTemplate:
)
return DistributionTemplate(
name="ollama",
name=name,
distro_type="self_hosted",
description="Use (an external) Ollama server for running LLM inference",
docker_image=None,
@ -50,6 +56,7 @@ def get_distribution_template() -> DistributionTemplate:
"run.yaml": RunConfigSettings(
provider_overrides={
"inference": [inference_provider],
"memory": [memory_provider],
},
default_models=[inference_model],
),
@ -57,7 +64,8 @@ def get_distribution_template() -> DistributionTemplate:
provider_overrides={
"inference": [
inference_provider,
]
],
"memory": [memory_provider],
},
default_models=[
inference_model,

6
llama_stack/templates/ollama/run-with-safety.yaml
View file

@ -25,12 +25,6 @@ providers:
type: sqlite
namespace: null
db_path: ${env.SQLITE_STORE_DIR:~/.llama/distributions/ollama}/faiss_store.db
- provider_id: chromadb
provider_type: remote::chromadb
config: {}
- provider_id: pgvector
provider_type: remote::pgvector
config: {}
safety:
- provider_id: llama-guard
provider_type: inline::llama-guard

6
llama_stack/templates/ollama/run.yaml
View file

@ -25,12 +25,6 @@ providers:
type: sqlite
namespace: null
db_path: ${env.SQLITE_STORE_DIR:~/.llama/distributions/ollama}/faiss_store.db
- provider_id: chromadb
provider_type: remote::chromadb
config: {}
- provider_id: pgvector
provider_type: remote::pgvector
config: {}
safety:
- provider_id: llama-guard
provider_type: inline::llama-guard

6
llama_stack/templates/remote-vllm/run-with-safety.yaml
View file

@ -30,12 +30,6 @@ providers:
type: sqlite
namespace: null
db_path: ${env.SQLITE_STORE_DIR:~/.llama/distributions/remote-vllm}/faiss_store.db
- provider_id: chromadb
provider_type: remote::chromadb
config: {}
- provider_id: pgvector
provider_type: remote::pgvector
config: {}
safety:
- provider_id: llama-guard
provider_type: inline::llama-guard

1
llama_stack/templates/remote-vllm/vllm.py
View file

@ -70,6 +70,7 @@ def get_distribution_template() -> DistributionTemplate:
),
),
],
"memory": [memory_provider],
},
default_models=[
inference_model,

6
llama_stack/templates/tgi/run-with-safety.yaml
View file

@ -29,12 +29,6 @@ providers:
type: sqlite
namespace: null
db_path: ${env.SQLITE_STORE_DIR:~/.llama/distributions/tgi}/faiss_store.db
- provider_id: chromadb
provider_type: remote::chromadb
config: {}
- provider_id: pgvector
provider_type: remote::pgvector
config: {}
safety:
- provider_id: llama-guard
provider_type: inline::llama-guard

1
llama_stack/templates/tgi/tgi.py
View file

@ -74,6 +74,7 @@ def get_distribution_template() -> DistributionTemplate:
),
),
],
"memory": [memory_provider],
},
default_models=[
inference_model,