mirror of
https://github.com/meta-llama/llama-stack.git
synced 2025-10-22 16:23:08 +00:00
115 lines
4.9 KiB
Bash
Executable file
115 lines
4.9 KiB
Bash
Executable file
#!/usr/bin/env bash
|
|
|
|
# Copyright (c) Meta Platforms, Inc. and affiliates.
|
|
# All rights reserved.
|
|
#
|
|
# This source code is licensed under the terms described in the LICENSE file in
|
|
# the root directory of this source tree.
|
|
|
|
# This script is used to apply the Kubernetes resources for the Llama Stack.
|
|
|
|
export POSTGRES_USER=llamastack
|
|
export POSTGRES_DB=llamastack
|
|
export POSTGRES_PASSWORD=llamastack
|
|
|
|
export INFERENCE_MODEL=meta-llama/Llama-3.1-8B-Instruct
|
|
export CODE_MODEL=bigcode/starcoder2-7b
|
|
export OLLAMA_MODEL=llama-guard3:1b
|
|
# Set USE_EBS to false if you don't have permission to use EKS EBS
|
|
export USE_EBS=${USE_EBS:-false}
|
|
set -euo pipefail
|
|
set -x
|
|
# HF_TOKEN should be set by the user; base64 encode it for the secret
|
|
if [ -n "${HF_TOKEN:-}" ]; then
|
|
export HF_TOKEN_BASE64=$(echo -n "$HF_TOKEN" | base64)
|
|
else
|
|
echo "ERROR: HF_TOKEN not set. You need it for vLLM to download models from Hugging Face."
|
|
exit 1
|
|
fi
|
|
|
|
# NGC_API_KEY should be set by the user or provided as argument; base64 encode it for the secret
|
|
if [ -n "${NGC_API_KEY:-}" ]; then
|
|
export NGC_API_KEY_BASE64=$(echo -n "$NGC_API_KEY" | base64)
|
|
# Create Docker config JSON for NGC image pull
|
|
NGC_DOCKER_CONFIG="{\"auths\":{\"nvcr.io\":{\"username\":\"\$oauthtoken\",\"password\":\"$NGC_API_KEY\"}}}"
|
|
export NGC_DOCKER_CONFIG_JSON=$(echo -n "$NGC_DOCKER_CONFIG" | base64)
|
|
else
|
|
echo "ERROR: NGC_API_KEY not set. You need it for NIM to download models from NVIDIA."
|
|
echo "Usage: $0 [your-ngc-api-key]"
|
|
echo "You can either provide your NGC API key as an argument or set it as an environment variable."
|
|
exit 1
|
|
fi
|
|
|
|
|
|
|
|
|
|
# Create secrets and capture any errors
|
|
echo "Creating Kubernetes secrets..."
|
|
SECRET_OUTPUT=$(envsubst < ./set-secret.yaml.template | kubectl apply -f - 2>&1) || {
|
|
echo "ERROR: Failed to create secrets. Error output:"
|
|
echo "$SECRET_OUTPUT"
|
|
exit 1
|
|
}
|
|
|
|
# Wait a moment for Kubernetes to process the secrets
|
|
echo "Waiting for secrets to be processed..."
|
|
sleep 2
|
|
|
|
# Verify that the secrets were created successfully
|
|
echo "Verifying that secrets were created successfully..."
|
|
|
|
# Check each secret with better error handling
|
|
for SECRET_NAME in "hf-token-secret" "ngc-docker-registry" "ngc-api"; do
|
|
echo "Checking for secret: $SECRET_NAME"
|
|
if ! kubectl get secret "$SECRET_NAME" &> /dev/null; then
|
|
echo "ERROR: Secret '$SECRET_NAME' not found in the cluster."
|
|
echo "Secret creation output was:"
|
|
echo "$SECRET_OUTPUT"
|
|
echo "Current secrets in the namespace:"
|
|
kubectl get secrets
|
|
exit 1
|
|
fi
|
|
done
|
|
|
|
echo "Secret verification successful. All required secrets are present."
|
|
# Apply templates with appropriate storage configuration based on USE_EBS setting
|
|
if [ "$USE_EBS" = "true" ]; then
|
|
echo "Using EBS storage for persistent volumes"
|
|
envsubst < ./vllm-k8s.yaml.template | kubectl apply -f -
|
|
envsubst < ./ollama-safety-k8s.yaml.template | kubectl apply -f -
|
|
envsubst < ./llama-nim.yaml.template | kubectl apply -f -
|
|
envsubst < ./postgres-k8s.yaml.template | kubectl apply -f -
|
|
envsubst < ./chroma-k8s.yaml.template | kubectl apply -f -
|
|
envsubst < ./jaeger-k8s.yaml.template | kubectl apply -f -
|
|
|
|
|
|
kubectl create configmap llama-stack-config --from-file=stack_run_config.yaml \
|
|
--dry-run=client -o yaml > stack-configmap.yaml
|
|
|
|
kubectl apply -f stack-configmap.yaml
|
|
|
|
envsubst < ./stack-k8s.yaml.template | kubectl apply -f -
|
|
envsubst < ./ingress-k8s.yaml.template | kubectl apply -f -
|
|
envsubst < ./ui-k8s.yaml.template | kubectl apply -f -
|
|
envsubst < ./ui-service-k8s.yaml.template | kubectl apply -f -
|
|
else
|
|
echo "Using emptyDir for storage (data will not persist across pod restarts)"
|
|
# Process templates to replace EBS storage with emptyDir
|
|
envsubst < ./vllm-k8s.yaml.template | sed 's/persistentVolumeClaim:/emptyDir: {}/g' | sed '/claimName:/d' | kubectl apply -f -
|
|
envsubst < ./ollama-safety-k8s.yaml.template | sed 's/persistentVolumeClaim:/emptyDir: {}/g' | sed '/claimName:/d' | kubectl apply -f -
|
|
envsubst < ./llama-nim.yaml.template | kubectl apply -f -
|
|
envsubst < ./postgres-k8s.yaml.template | sed 's/persistentVolumeClaim:/emptyDir: {}/g' | sed '/claimName:/d' | kubectl apply -f -
|
|
envsubst < ./chroma-k8s.yaml.template | sed 's/persistentVolumeClaim:/emptyDir: {}/g' | sed '/claimName:/d' | kubectl apply -f -
|
|
envsubst < ./jaeger-k8s.yaml.template | kubectl apply -f -
|
|
|
|
kubectl create configmap llama-stack-config --from-file=stack_run_config.yaml \
|
|
--dry-run=client -o yaml > stack-configmap.yaml
|
|
|
|
kubectl apply -f stack-configmap.yaml
|
|
|
|
# Apply the same emptyDir transformation to the remaining templates
|
|
envsubst < ./stack-k8s.yaml.template | sed 's/persistentVolumeClaim:/emptyDir: {}/g' | sed '/claimName:/d' | kubectl apply -f -
|
|
envsubst < ./ingress-k8s.yaml.template | sed 's/persistentVolumeClaim:/emptyDir: {}/g' | sed '/claimName:/d' | kubectl apply -f -
|
|
envsubst < ./ui-k8s.yaml.template | sed 's/persistentVolumeClaim:/emptyDir: {}/g' | sed '/claimName:/d' | kubectl apply -f -
|
|
envsubst < ./ui-service-k8s.yaml.template | kubectl apply -f -
|
|
fi
|