phoenix-oss/llama-stack-mirror
0
0
Fork 1
mirror of https://github.com/meta-llama/llama-stack.git synced 2025-12-17 12:42:37 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

showcase using env vars

Browse source
This commit is contained in:
Ashwin Bharambe 2024-11-13 10:55:33 -08:00
parent e5ef94ded8
commit 4074fcf83c
3 changed files with 27 additions and 14 deletions
Download patch file Download diff file Expand all files Collapse all files

20
distributions/remote-vllm/run.yaml
View file

@ -16,16 +16,16 @@ providers:
provider_type: remote::vllm
config:
# NOTE: replace with "localhost" if you are running in "host" network mode
url: http://host.docker.internal:5100/v1
max_tokens: 4096
url: ${env.LLAMA_INFERENCE_VLLM_URL:http://host.docker.internal:5100/v1}
max_tokens: ${env.MAX_TOKENS:4096}
api_token: fake
# serves safety llama_guard model
- provider_id: vllm-1
provider_type: remote::vllm
config:
# NOTE: replace with "localhost" if you are running in "host" network mode
url: http://host.docker.internal:5101/v1
max_tokens: 4096
url: ${env.LLAMA_SAFETY_VLLM_URL:http://host.docker.internal:5101/v1}
max_tokens: ${env.MAX_TOKENS:4096}
api_token: fake
memory:
- provider_id: faiss-0
@ -34,7 +34,7 @@ providers:
kvstore:
namespace: null
type: sqlite
db_path: /home/ashwin/.llama/distributions/remote-vllm/faiss_store.db
db_path: "${env.SQLITE_STORE_DIR:/home/ashwin/.llama/distributions/remote-vllm}/faiss_store.db"
safety:
- provider_id: llama-guard
provider_type: inline::llama-guard
@ -50,7 +50,7 @@ providers:
persistence_store:
namespace: null
type: sqlite
db_path: /home/ashwin/.llama/distributions/remote-vllm/agents_store.db
db_path: "${env.SQLITE_STORE_DIR:/home/ashwin/.llama/distributions/remote-vllm}/agents_store.db"
telemetry:
- provider_id: meta0
provider_type: inline::meta-reference
@ -58,11 +58,11 @@ providers:
metadata_store:
namespace: null
type: sqlite
db_path: /home/ashwin/.llama/distributions/remote-vllm/registry.db
db_path: "${env.SQLITE_STORE_DIR:/home/ashwin/.llama/distributions/remote-vllm}/registry.db"
models:
- model_id: Llama3.1-8B-Instruct
- model_id: ${env.LLAMA_INFERENCE_MODEL:Llama3.1-8B-Instruct}
provider_id: vllm-0
- model_id: Llama-Guard-3-1B
- model_id: ${env.LLAMA_SAFETY_MODEL:Llama-Guard-3-1B}
provider_id: vllm-1
shields:
- shield_id: Llama-Guard-3-1B
- shield_id: ${env.LLAMA_SAFETY_MODEL:Llama-Guard-3-1B}