mirror of
https://github.com/meta-llama/llama-stack.git
synced 2025-06-28 02:53:30 +00:00
7fb4bdabea
This Kubernetes cluster has: - vLLM for serving an inference model - vLLM for serving a safety model - Postgres DB (for metadata and other state for the Llama Stack distro) - Chroma DB for Vector IO (memory) Perhaps most importantly, this was me trying to learn Kubernetes for the first time. ## Test Plan Run `sh apply.sh` against an EKS cluster, then after `kubectl port-forward service/llama-stack-service 8321:8321` and after many attempts, we have finally: <img width="1589" alt="image" src="https://github.com/user-attachments/assets/c69f242d-6aaa-4def-9f7c-172113b8bfc1" /> <img width="1978" alt="image" src="https://github.com/user-attachments/assets/cf678404-f551-4fa5-9077-bebe3e8e8ae8" />
30 lines
1 KiB
Bash
Executable file
30 lines
1 KiB
Bash
Executable file
#!/usr/bin/env bash
|
|
|
|
# Copyright (c) Meta Platforms, Inc. and affiliates.
|
|
# All rights reserved.
|
|
#
|
|
# This source code is licensed under the terms described in the LICENSE file in
|
|
# the root directory of this source tree.
|
|
|
|
export POSTGRES_USER=${POSTGRES_USER:-llamastack}
|
|
export POSTGRES_DB=${POSTGRES_DB:-llamastack}
|
|
export POSTGRES_PASSWORD=${POSTGRES_PASSWORD:-llamastack}
|
|
|
|
export INFERENCE_MODEL=${INFERENCE_MODEL:-meta-llama/Llama-3.2-3B-Instruct}
|
|
export SAFETY_MODEL=${SAFETY_MODEL:-meta-llama/Llama-Guard-3-1B}
|
|
|
|
set -euo pipefail
|
|
set -x
|
|
|
|
envsubst < ./vllm-k8s.yaml.template | kubectl apply -f -
|
|
envsubst < ./vllm-safety-k8s.yaml.template | kubectl apply -f -
|
|
envsubst < ./postgres-k8s.yaml.template | kubectl apply -f -
|
|
envsubst < ./chroma-k8s.yaml.template | kubectl apply -f -
|
|
|
|
kubectl create configmap llama-stack-config --from-file=stack_run_config.yaml \
|
|
--dry-run=client -o yaml > stack-configmap.yaml
|
|
|
|
kubectl apply -f stack-configmap.yaml
|
|
|
|
envsubst < ./stack-k8s.yaml.template | kubectl apply -f -
|
|
envsubst < ./ingress-k8s.yaml.template | kubectl apply -f -
|