#!/usr/bin/env bash # Copyright (c) Meta Platforms, Inc. and affiliates. # All rights reserved. # # This source code is licensed under the terms described in the LICENSE file in # the root directory of this source tree. # Check if NGC_API_KEY is provided as argument if [ -n "$1" ]; then export NGC_API_KEY=$1 echo "Using NGC API key provided as argument." fi export POSTGRES_USER=llamastack export POSTGRES_DB=llamastack export POSTGRES_PASSWORD=llamastack export INFERENCE_MODEL=meta-llama/Llama-3.2-3B-Instruct export CODE_MODEL=bigcode/starcoder2-7b export OLLAMA_MODEL=llama-guard3:1b # Set USE_EBS to false if you don't have permission to use EKS EBS export USE_EBS=${USE_EBS:-false} set -euo pipefail set -x # HF_TOKEN should be set by the user; base64 encode it for the secret if [ -n "${HF_TOKEN:-}" ]; then export HF_TOKEN_BASE64=$(echo -n "$HF_TOKEN" | base64) else echo "ERROR: HF_TOKEN not set. You need it for vLLM to download models from Hugging Face." exit 1 fi # NGC_API_KEY should be set by the user or provided as argument; base64 encode it for the secret if [ -n "${NGC_API_KEY:-}" ]; then export NGC_API_KEY_BASE64=$(echo -n "$NGC_API_KEY" | base64) # Create Docker config JSON for NGC image pull NGC_DOCKER_CONFIG="{\"auths\":{\"nvcr.io\":{\"username\":\"\$oauthtoken\",\"password\":\"$NGC_API_KEY\"}}}" export NGC_DOCKER_CONFIG_JSON=$(echo -n "$NGC_DOCKER_CONFIG" | base64) else echo "ERROR: NGC_API_KEY not set. You need it for NIM to download models from NVIDIA." echo "Usage: $0 [your-ngc-api-key]" echo "You can either provide your NGC API key as an argument or set it as an environment variable." exit 1 fi # Create secrets and capture any errors echo "Creating Kubernetes secrets..." SECRET_OUTPUT=$(envsubst < ./set-secret.yaml.template | kubectl apply -f - 2>&1) || { echo "ERROR: Failed to create secrets. Error output:" echo "$SECRET_OUTPUT" exit 1 } # Wait a moment for Kubernetes to process the secrets echo "Waiting for secrets to be processed..." sleep 2 # Verify that the secrets were created successfully echo "Verifying that secrets were created successfully..." # Check each secret with better error handling for SECRET_NAME in "hf-token-secret" "ngc-docker-registry" "ngc-api"; do echo "Checking for secret: $SECRET_NAME" if ! kubectl get secret "$SECRET_NAME" &> /dev/null; then echo "ERROR: Secret '$SECRET_NAME' not found in the cluster." echo "Secret creation output was:" echo "$SECRET_OUTPUT" echo "Current secrets in the namespace:" kubectl get secrets exit 1 fi done echo "Secret verification successful. All required secrets are present." # Apply templates with appropriate storage configuration based on USE_EBS setting if [ "$USE_EBS" = "true" ]; then echo "Using EBS storage for persistent volumes" envsubst < ./vllm-k8s.yaml.template | kubectl apply -f - envsubst < ./ollama-safety-k8s.yaml.template | kubectl apply -f - envsubst < ./llama-nim.yaml.template | kubectl apply -f - envsubst < ./postgres-k8s.yaml.template | kubectl apply -f - envsubst < ./chroma-k8s.yaml.template | kubectl apply -f - # Create monitoring namespace kubectl create namespace monitoring --dry-run=client -o yaml | kubectl apply -f - # Install Prometheus Operator CRDs ./install-prometheus-operator.sh # Apply RBAC for Prometheus kubectl apply -f ./prometheus-rbac.yaml # Apply monitoring resources after CRDs are installed envsubst < ./monitoring-k8s.yaml.template | kubectl apply -f - kubectl create configmap llama-stack-config --from-file=stack_run_config.yaml \ --dry-run=client -o yaml > stack-configmap.yaml kubectl apply -f stack-configmap.yaml envsubst < ./stack-k8s.yaml.template | kubectl apply -f - envsubst < ./ingress-k8s.yaml.template | kubectl apply -f - envsubst < ./ui-k8s.yaml.template | kubectl apply -f - envsubst < ./ui-service-k8s.yaml.template | kubectl apply -f - else echo "Using emptyDir for storage (data will not persist across pod restarts)" # Process templates to replace EBS storage with emptyDir envsubst < ./vllm-k8s.yaml.template | sed 's/persistentVolumeClaim:/emptyDir: {}/g' | sed '/claimName:/d' | kubectl apply -f - envsubst < ./ollama-safety-k8s.yaml.template | sed 's/persistentVolumeClaim:/emptyDir: {}/g' | sed '/claimName:/d' | kubectl apply -f - envsubst < ./llama-nim.yaml.template | kubectl apply -f - envsubst < ./postgres-k8s.yaml.template | sed 's/persistentVolumeClaim:/emptyDir: {}/g' | sed '/claimName:/d' | kubectl apply -f - envsubst < ./chroma-k8s.yaml.template | sed 's/persistentVolumeClaim:/emptyDir: {}/g' | sed '/claimName:/d' | kubectl apply -f - # Create monitoring namespace kubectl create namespace monitoring --dry-run=client -o yaml | kubectl apply -f - # Apply RBAC for Prometheus kubectl apply -f ./prometheus-rbac.yaml # Apply monitoring resources after CRDs are installed envsubst < ./monitoring-k8s.yaml.template | kubectl apply -f - kubectl create configmap llama-stack-config --from-file=stack_run_config.yaml \ --dry-run=client -o yaml > stack-configmap.yaml kubectl apply -f stack-configmap.yaml # Apply the same emptyDir transformation to the remaining templates envsubst < ./stack-k8s.yaml.template | sed 's/persistentVolumeClaim:/emptyDir: {}/g' | sed '/claimName:/d' | kubectl apply -f - envsubst < ./ingress-k8s.yaml.template | sed 's/persistentVolumeClaim:/emptyDir: {}/g' | sed '/claimName:/d' | kubectl apply -f - envsubst < ./ui-k8s.yaml.template | sed 's/persistentVolumeClaim:/emptyDir: {}/g' | sed '/claimName:/d' | kubectl apply -f - envsubst < ./ui-service-k8s.yaml.template | kubectl apply -f - fi