phoenix-oss/llama-stack-mirror
0
0
Fork 1
mirror of https://github.com/meta-llama/llama-stack.git synced 2025-08-02 08:44:44 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

second try

Browse source
This commit is contained in:
Kai Wu 2025-07-30 14:51:43 -07:00
parent 31a15332c4
commit 1cb9d3bca2
11 changed files with 237 additions and 64 deletions
Download patch file Download diff file Expand all files Collapse all files

41
docs/source/distributions/k8s/apply.sh
View file

@ -6,12 +6,18 @@
# This source code is licensed under the terms described in the LICENSE file in
# the root directory of this source tree.
# Check if NGC_API_KEY is provided as argument
if [ -n "$1" ]; then
export NGC_API_KEY=$1
echo "Using NGC API key provided as argument."
fi
export POSTGRES_USER=llamastack
export POSTGRES_DB=llamastack
export POSTGRES_PASSWORD=llamastack
export INFERENCE_MODEL=meta-llama/Llama-3.2-3B-Instruct
export SAFETY_MODEL=meta-llama/Llama-Guard-3-1B
export INFERENCE_MODEL=meta-llama/Llama-3.1-8B-Instruct
export CODE_MODEL=bigcode/starcoder2-7b
# Set USE_EBS to false if you don't have permission to use EKS EBS
export USE_EBS=${USE_EBS:-false}
@ -24,13 +30,16 @@ else
exit 1
fi
if [ -z "${GITHUB_CLIENT_ID:-}" ]; then
echo "ERROR: GITHUB_CLIENT_ID not set. You need it for Github login to work. Refer to https://llama-stack.readthedocs.io/en/latest/deploying/index.html#kubernetes-deployment-guide"
exit 1
fi
if [ -z "${GITHUB_CLIENT_SECRET:-}" ]; then
echo "ERROR: GITHUB_CLIENT_SECRET not set. You need it for Github login to work. Refer to https://llama-stack.readthedocs.io/en/latest/deploying/index.html#kubernetes-deployment-guide"
# NGC_API_KEY should be set by the user or provided as argument; base64 encode it for the secret
if [ -n "${NGC_API_KEY:-}" ]; then
export NGC_API_KEY_BASE64=$(echo -n "$NGC_API_KEY" | base64)
# Create Docker config JSON for NGC image pull
NGC_DOCKER_CONFIG="{\"auths\":{\"nvcr.io\":{\"username\":\"\$oauthtoken\",\"password\":\"$NGC_API_KEY\"}}}"
export NGC_DOCKER_CONFIG_JSON=$(echo -n "$NGC_DOCKER_CONFIG" | base64)
else
echo "ERROR: NGC_API_KEY not set. You need it for NIM to download models from NVIDIA."
echo "Usage: $0 [your-ngc-api-key]"
echo "You can either provide your NGC API key as an argument or set it as an environment variable."
exit 1
fi
@ -41,20 +50,18 @@ fi
# Apply the HF token secret if HF_TOKEN is provided
if [ -n "${HF_TOKEN:-}" ]; then
envsubst < ./set-secret.yaml.template | kubectl apply -f -
fi
set -euo pipefail
set -x
# Apply the HF token secret if HF_TOKEN is provided
if [ -n "${HF_TOKEN:-}" ]; then
envsubst < ./hf-token-secret.yaml.template | kubectl apply -f -
fi
# Apply templates with appropriate storage configuration based on USE_EBS setting
if [ "$USE_EBS" = "true" ]; then
echo "Using EBS storage for persistent volumes"
envsubst < ./vllm-k8s.yaml.template | kubectl apply -f -
envsubst < ./vllm-safety-k8s.yaml.template | kubectl apply -f -
envsubst < ./llama-nim.yaml.template | kubectl apply -f -
envsubst < ./postgres-k8s.yaml.template | kubectl apply -f -
envsubst < ./chroma-k8s.yaml.template | kubectl apply -f -
@ -70,7 +77,7 @@ else
echo "Using emptyDir for storage (data will not persist across pod restarts)"
# Process templates to replace EBS storage with emptyDir
envsubst < ./vllm-k8s.yaml.template | sed 's/persistentVolumeClaim:/emptyDir: {}/g' | sed '/claimName:/d' | kubectl apply -f -
envsubst < ./vllm-safety-k8s.yaml.template | sed 's/persistentVolumeClaim:/emptyDir: {}/g' | sed '/claimName:/d' | kubectl apply -f -
envsubst < ./llama-nim.yaml.template | kubectl apply -f -
envsubst < ./postgres-k8s.yaml.template | sed 's/persistentVolumeClaim:/emptyDir: {}/g' | sed '/claimName:/d' | kubectl apply -f -
envsubst < ./chroma-k8s.yaml.template | sed 's/persistentVolumeClaim:/emptyDir: {}/g' | sed '/claimName:/d' | kubectl apply -f -