llama-stack-mirror/benchmarking/k8s-benchmark/scripts/run-all-benchmarks.sh

#!/usr/bin/env bash

# Copyright (c) Meta Platforms, Inc. and affiliates.
# All rights reserved.
#
# This source code is licensed under the terms described in the LICENSE file in
# the root directory of this source tree.

# Define benchmark configurations: (target, stack_replicas, vllm_replicas, stack_workers)
configs=(
    "stack 1 1 1"
    "stack 1 1 2"
    "stack 1 1 4"
    "vllm 1 1 -"
)

set -euo pipefail

# Get the directory where this script is located
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"

echo "Running comprehensive GuideLL benchmark suite..."
echo "Start time: $(date)"

# Default deployment names
STACK_DEPLOYMENT="llama-stack-benchmark-server"
VLLM_DEPLOYMENT="vllm-server"

# Scaling function
scale_deployments() {
    local stack_replicas=$1
    local vllm_replicas=$2
    local workers=$3

    echo "Scaling deployments..."

    if [[ "$vllm_replicas" != "-" ]]; then
        echo "Scaling $VLLM_DEPLOYMENT to $vllm_replicas replicas..."
        kubectl scale deployment $VLLM_DEPLOYMENT --replicas=$vllm_replicas
        kubectl rollout status deployment $VLLM_DEPLOYMENT --timeout=600s
    fi

    if [[ "$target" == "stack" ]]; then
        if [[ "$stack_replicas" != "-" ]]; then
            echo "Scaling $STACK_DEPLOYMENT to $stack_replicas replicas..."
            kubectl scale deployment $STACK_DEPLOYMENT --replicas=$stack_replicas
            kubectl rollout status deployment $STACK_DEPLOYMENT --timeout=600s
        fi

        if [[ "$workers" != "-" ]]; then
            echo "Updating $STACK_DEPLOYMENT to use $workers workers..."
            kubectl set env deployment/$STACK_DEPLOYMENT LLAMA_STACK_WORKERS=$workers
            kubectl rollout status deployment $STACK_DEPLOYMENT --timeout=600s
        fi
    fi

    echo "All scaling operations completed. Waiting additional 30s for services to stabilize..."
    sleep 30
}


for config in "${configs[@]}"; do
    read -r target stack_replicas vllm_replicas workers <<< "$config"

    echo ""
    echo "=========================================="
    if [[ "$workers" != "-" ]]; then
        echo "Running benchmark: $target (stack=$stack_replicas, vllm=$vllm_replicas, workers=$workers)"
    else
        echo "Running benchmark: $target (stack=$stack_replicas, vllm=$vllm_replicas)"
    fi
    echo "Start: $(date)"
    echo "=========================================="

    # Scale deployments before running benchmark
    scale_deployments "$stack_replicas" "$vllm_replicas" "$workers"

    # Generate output filename with setup info
    TIMESTAMP=$(date +%Y%m%d-%H%M%S)
    if [[ "$target" == "stack" ]]; then
        OUTPUT_FILE="results/guidellm-benchmark-${target}-s${stack_replicas}-sw${workers}-v${vllm_replicas}-${TIMESTAMP}.txt"
    else
        OUTPUT_FILE="results/guidellm-benchmark-${target}-v${vllm_replicas}-${TIMESTAMP}.txt"
    fi

    # Run the benchmark with the cluster as configured
    "$SCRIPT_DIR/run-guidellm-benchmark.sh" \
        --target "$target" \
        --output-file "$OUTPUT_FILE"

    echo "Completed: $(date)"
    echo "Waiting 30 seconds before next benchmark..."
    sleep 30
done

echo ""
echo "=========================================="
echo "All benchmarks completed!"
echo "End time: $(date)"
echo "=========================================="
echo ""
echo "Results files generated:"
ls -la results/guidellm-*.txt results/guidellm-*.json 2>/dev/null || echo "No result files found"