Added distributions for inline and remote

This commit is contained in:
Chantal D Gama Rose 2024-11-20 23:04:48 +00:00
parent a5d413045c
commit 6759744235
6 changed files with 136 additions and 0 deletions

View file

@ -0,0 +1 @@
../../llama_stack/templates/nvidia/build.yaml

View file

@ -0,0 +1,19 @@
services:
llamastack:
image: distribution-nvidia:dev
network_mode: "host"
volumes:
- ~/.llama:/root/.llama
- ./run.yaml:/root/llamastack-run-nvidia.yaml
ports:
- "5000:5000"
environment:
- INFERENCE_MODEL=${INFERENCE_MODEL:-Llama3.1-8B-Instruct}
- NVIDIA_API_KEY=${NVIDIA_API_KEY:-}
entrypoint: bash -c "python -m llama_stack.distribution.server.server --yaml-config /root/llamastack-run-nvidia.yaml"
deploy:
restart_policy:
condition: on-failure
delay: 3s
max_attempts: 5
window: 60s

View file

@ -0,0 +1 @@
../../llama_stack/templates/nvidia/run.yaml