From 1fc0fd593529542f87f89f17a867bda92a7429ba Mon Sep 17 00:00:00 2001 From: Ashwin Bharambe Date: Thu, 16 Oct 2025 14:31:10 -0700 Subject: [PATCH] feat(ci): add support for docker:distro in tests Also a critical bug fix so test recordings can be found inside docker --- llama_stack/core/build_container.sh | 2 +- llama_stack/testing/api_recorder.py | 12 +- scripts/docker.sh | 358 ++++++++++++++++++++++++++++ scripts/integration-tests.sh | 105 +++++++- tests/integration/conftest.py | 8 +- 5 files changed, 477 insertions(+), 8 deletions(-) create mode 100755 scripts/docker.sh diff --git a/llama_stack/core/build_container.sh b/llama_stack/core/build_container.sh index 09878f535..03ed846d9 100755 --- a/llama_stack/core/build_container.sh +++ b/llama_stack/core/build_container.sh @@ -338,7 +338,7 @@ fi # Add other require item commands genearic to all containers add_to_container << EOF -RUN mkdir -p /.llama /.cache && chmod -R g+rw /app /.llama /.cache +RUN mkdir -p /.llama /.cache && chmod -R g+rw /.llama /.cache && (chmod -R g+rw /app 2>/dev/null || true) EOF printf "Containerfile created successfully in %s/Containerfile\n\n" "$TEMP_DIR" diff --git a/llama_stack/testing/api_recorder.py b/llama_stack/testing/api_recorder.py index b0d68fd8a..4a4644dad 100644 --- a/llama_stack/testing/api_recorder.py +++ b/llama_stack/testing/api_recorder.py @@ -355,9 +355,15 @@ class ResponseStorage: test_file = test_id.split("::")[0] # Remove test function part test_dir = Path(test_file).parent # Get parent directory - # Put recordings in a "recordings" subdirectory of the test's parent dir - # e.g., "tests/integration/inference" -> "tests/integration/inference/recordings" - return test_dir / "recordings" + # Make it absolute by resolving against base_dir + # If base_dir is absolute, use it as the root, otherwise resolve relative to cwd + if self.base_dir.is_absolute(): + # base_dir is something like /app/llama-stack-source/tests/integration/common + # We need to go up to the repo root and then back down to the test dir + repo_root = self.base_dir.parent.parent.parent # go up from common -> integration -> tests -> repo + return repo_root / test_dir / "recordings" + else: + return test_dir / "recordings" else: # Fallback for non-test contexts return self.base_dir / "recordings" diff --git a/scripts/docker.sh b/scripts/docker.sh new file mode 100755 index 000000000..1ba1d9adf --- /dev/null +++ b/scripts/docker.sh @@ -0,0 +1,358 @@ +#!/bin/bash +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the terms described in the LICENSE file in +# the root directory of this source tree. + +set -euo pipefail + +# Docker container management script for Llama Stack +# Allows starting/stopping/restarting a Llama Stack docker container for testing + +# Default values +DISTRO="" +PORT=8321 +INFERENCE_MODE="replay" +COMMAND="" +USE_COPY_NOT_MOUNT=false +NO_REBUILD=false + +# Function to display usage +usage() { + cat </dev/null || true + echo "Removing container: $CONTAINER_NAME" + docker rm "$CONTAINER_NAME" 2>/dev/null || true + echo "✅ Container stopped and removed" + else + echo "⚠️ Container $CONTAINER_NAME does not exist" + fi +} + +# Function to build docker image +build_image() { + echo "=== Building Docker Image for distribution: $DISTRO ===" + # Get the repo root (parent of scripts directory) + SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) + REPO_ROOT=$(cd "$SCRIPT_DIR/.." && pwd) + + # Determine whether to copy or mount source + # Copy in CI or if explicitly requested, otherwise mount for live development + BUILD_ENV="LLAMA_STACK_DIR=$REPO_ROOT" + if [[ "$USE_COPY_NOT_MOUNT" == "true" ]] || [[ "${CI:-false}" == "true" ]] || [[ "${GITHUB_ACTIONS:-false}" == "true" ]]; then + echo "Copying source into image (USE_COPY_NOT_MOUNT=true, CI=${CI:-false}, GITHUB_ACTIONS=${GITHUB_ACTIONS:-false})" + BUILD_ENV="USE_COPY_NOT_MOUNT=true $BUILD_ENV" + else + echo "Will mount source for live development" + fi + + if ! eval "$BUILD_ENV llama stack build --distro '$DISTRO' --image-type container"; then + echo "❌ Failed to build Docker image" + exit 1 + fi + echo "✅ Docker image built successfully" +} + +# Function to start container +start_container() { + # Check if already running + if is_container_running; then + echo "⚠️ Container $CONTAINER_NAME is already running" + echo "URL: http://localhost:$PORT" + exit 0 + fi + + # Stop and remove if exists but not running + if container_exists; then + echo "Removing existing stopped container..." + docker rm "$CONTAINER_NAME" 2>/dev/null || true + fi + + # Build the image unless --no-rebuild was specified + if [[ "$NO_REBUILD" == "true" ]]; then + echo "Skipping build (--no-rebuild specified)" + # Check if image exists (with or without localhost/ prefix) + if ! docker images --format "{{.Repository}}:{{.Tag}}" | grep -q "distribution-$DISTRO:dev$"; then + echo "❌ Error: Image distribution-$DISTRO:dev does not exist" + echo "Either build it first without --no-rebuild, or run: llama stack build --distro $DISTRO --image-type container" + exit 1 + fi + echo "✅ Found existing image for distribution-$DISTRO:dev" + else + build_image + fi + + echo "" + echo "=== Starting Docker Container ===" + + # Get the repo root for volume mount + SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) + REPO_ROOT=$(cd "$SCRIPT_DIR/.." && pwd) + + # Determine the actual image name (may have localhost/ prefix) + IMAGE_NAME=$(docker images --format "{{.Repository}}:{{.Tag}}" | grep "distribution-$DISTRO:dev$" | head -1) + if [[ -z "$IMAGE_NAME" ]]; then + echo "❌ Error: Could not find image for distribution-$DISTRO:dev" + exit 1 + fi + echo "Using image: $IMAGE_NAME" + + # Build environment variables for docker run + DOCKER_ENV_VARS="" + DOCKER_ENV_VARS="$DOCKER_ENV_VARS -e LLAMA_STACK_TEST_INFERENCE_MODE=$INFERENCE_MODE" + DOCKER_ENV_VARS="$DOCKER_ENV_VARS -e LLAMA_STACK_TEST_STACK_CONFIG_TYPE=server" + + # Set default OLLAMA_URL if not provided + # On macOS/Windows, use host.docker.internal to reach host from container + # On Linux with --network host, use localhost + if [[ "$(uname)" == "Darwin" ]] || [[ "$(uname)" == *"MINGW"* ]]; then + OLLAMA_URL="${OLLAMA_URL:-http://host.docker.internal:11434}" + else + OLLAMA_URL="${OLLAMA_URL:-http://localhost:11434}" + fi + DOCKER_ENV_VARS="$DOCKER_ENV_VARS -e OLLAMA_URL=$OLLAMA_URL" + + # Pass through API keys if they exist + [ -n "${TOGETHER_API_KEY:-}" ] && DOCKER_ENV_VARS="$DOCKER_ENV_VARS -e TOGETHER_API_KEY=$TOGETHER_API_KEY" + [ -n "${FIREWORKS_API_KEY:-}" ] && DOCKER_ENV_VARS="$DOCKER_ENV_VARS -e FIREWORKS_API_KEY=$FIREWORKS_API_KEY" + [ -n "${TAVILY_SEARCH_API_KEY:-}" ] && DOCKER_ENV_VARS="$DOCKER_ENV_VARS -e TAVILY_SEARCH_API_KEY=$TAVILY_SEARCH_API_KEY" + [ -n "${OPENAI_API_KEY:-}" ] && DOCKER_ENV_VARS="$DOCKER_ENV_VARS -e OPENAI_API_KEY=$OPENAI_API_KEY" + [ -n "${ANTHROPIC_API_KEY:-}" ] && DOCKER_ENV_VARS="$DOCKER_ENV_VARS -e ANTHROPIC_API_KEY=$ANTHROPIC_API_KEY" + [ -n "${GROQ_API_KEY:-}" ] && DOCKER_ENV_VARS="$DOCKER_ENV_VARS -e GROQ_API_KEY=$GROQ_API_KEY" + [ -n "${GEMINI_API_KEY:-}" ] && DOCKER_ENV_VARS="$DOCKER_ENV_VARS -e GEMINI_API_KEY=$GEMINI_API_KEY" + [ -n "${SAFETY_MODEL:-}" ] && DOCKER_ENV_VARS="$DOCKER_ENV_VARS -e SAFETY_MODEL=$SAFETY_MODEL" + [ -n "${SQLITE_STORE_DIR:-}" ] && DOCKER_ENV_VARS="$DOCKER_ENV_VARS -e SQLITE_STORE_DIR=$SQLITE_STORE_DIR" + + # Use --network host on Linux only (macOS doesn't support it properly) + NETWORK_MODE="" + if [[ "$(uname)" != "Darwin" ]] && [[ "$(uname)" != *"MINGW"* ]]; then + NETWORK_MODE="--network host" + fi + + docker run -d $NETWORK_MODE --name "$CONTAINER_NAME" \ + -p $PORT:$PORT \ + $DOCKER_ENV_VARS \ + -v "$REPO_ROOT":/app/llama-stack-source \ + "$IMAGE_NAME" \ + --port $PORT + + echo "Waiting for container to start..." + for i in {1..30}; do + if curl -s http://localhost:$PORT/v1/health 2>/dev/null | grep -q "OK"; then + echo "✅ Container started successfully" + echo "" + echo "=== Container Information ===" + echo "Container name: $CONTAINER_NAME" + echo "URL: http://localhost:$PORT" + echo "Health check: http://localhost:$PORT/v1/health" + echo "" + echo "To view logs: $0 logs --distro $DISTRO" + echo "To stop: $0 stop --distro $DISTRO" + return 0 + fi + if [[ $i -eq 30 ]]; then + echo "❌ Container failed to start within timeout" + echo "Showing container logs:" + docker logs "$CONTAINER_NAME" + exit 1 + fi + sleep 1 + done +} + +# Execute command +case "$COMMAND" in +start) + start_container + ;; +stop) + stop_container + ;; +restart) + echo "Restarting container: $CONTAINER_NAME" + stop_container + echo "" + start_container + ;; +status) + if is_container_running; then + echo "✅ Container $CONTAINER_NAME is running" + echo "URL: http://localhost:$PORT" + # Try to get the actual port from the container + ACTUAL_PORT=$(docker port "$CONTAINER_NAME" 2>/dev/null | grep "8321/tcp" | cut -d':' -f2 | head -1) + if [[ -n "$ACTUAL_PORT" ]]; then + echo "Port: $ACTUAL_PORT" + fi + elif container_exists; then + echo "⚠️ Container $CONTAINER_NAME exists but is not running" + echo "Start it with: $0 start --distro $DISTRO" + else + echo "❌ Container $CONTAINER_NAME does not exist" + echo "Start it with: $0 start --distro $DISTRO" + fi + ;; +logs) + if container_exists; then + echo "=== Logs for $CONTAINER_NAME ===" + # Check if -f flag was passed after 'logs' command + if [[ "${1:-}" == "-f" || "${1:-}" == "--follow" ]]; then + docker logs --tail 100 --follow "$CONTAINER_NAME" + else + docker logs --tail 100 "$CONTAINER_NAME" + fi + else + echo "❌ Container $CONTAINER_NAME does not exist" + exit 1 + fi + ;; +esac diff --git a/scripts/integration-tests.sh b/scripts/integration-tests.sh index 138f1d144..e19a5cc55 100755 --- a/scripts/integration-tests.sh +++ b/scripts/integration-tests.sh @@ -42,9 +42,12 @@ Setups are defined in tests/integration/setups.py and provide global configurati You can also specify subdirectories (of tests/integration) to select tests from, which will override the suite. Examples: - # Basic inference tests with ollama + # Basic inference tests with ollama (server mode) $0 --stack-config server:ci-tests --suite base --setup ollama + # Basic inference tests with docker + $0 --stack-config docker:ci-tests --suite base --setup ollama + # Multiple test directories with vllm $0 --stack-config server:ci-tests --subdirs 'inference,agents' --setup vllm @@ -153,7 +156,7 @@ echo "Setting SQLITE_STORE_DIR: $SQLITE_STORE_DIR" # Determine stack config type for api_recorder test isolation if [[ "$COLLECT_ONLY" == false ]]; then - if [[ "$STACK_CONFIG" == server:* ]]; then + if [[ "$STACK_CONFIG" == server:* ]] || [[ "$STACK_CONFIG" == docker:* ]]; then export LLAMA_STACK_TEST_STACK_CONFIG_TYPE="server" echo "Setting stack config type: server" else @@ -229,6 +232,104 @@ if [[ "$STACK_CONFIG" == *"server:"* && "$COLLECT_ONLY" == false ]]; then trap stop_server EXIT ERR INT TERM fi +# Start Docker Container if needed +if [[ "$STACK_CONFIG" == *"docker:"* && "$COLLECT_ONLY" == false ]]; then + stop_container() { + echo "Stopping Docker container..." + container_name="llama-stack-test-$DISTRO" + if docker ps -a --format '{{.Names}}' | grep -q "^${container_name}$"; then + echo "Stopping and removing container: $container_name" + docker stop "$container_name" 2>/dev/null || true + docker rm "$container_name" 2>/dev/null || true + else + echo "No container named $container_name found" + fi + echo "Docker container stopped" + } + + # Extract distribution name from docker:distro format + DISTRO=$(echo "$STACK_CONFIG" | sed 's/^docker://') + export LLAMA_STACK_PORT=8321 + + echo "=== Building Docker Image for distribution: $DISTRO ===" + # Set LLAMA_STACK_DIR to repo root + # USE_COPY_NOT_MOUNT copies files into image (for CI), otherwise mounts for live development + BUILD_ENV="LLAMA_STACK_DIR=$ROOT_DIR" + if [[ "${CI:-false}" == "true" ]] || [[ "${GITHUB_ACTIONS:-false}" == "true" ]]; then + echo "CI detected (CI=$CI, GITHUB_ACTIONS=$GITHUB_ACTIONS): copying source into image" + BUILD_ENV="USE_COPY_NOT_MOUNT=true $BUILD_ENV" + else + echo "Local mode: will mount source for live development" + fi + + eval "$BUILD_ENV llama stack build --distro '$DISTRO' --image-type container" + + if [ $? -ne 0 ]; then + echo "❌ Failed to build Docker image" + exit 1 + fi + + echo "" + echo "=== Starting Docker Container ===" + container_name="llama-stack-test-$DISTRO" + + # Stop and remove existing container if it exists + docker stop "$container_name" 2>/dev/null || true + docker rm "$container_name" 2>/dev/null || true + + # Build environment variables for docker run + DOCKER_ENV_VARS="" + DOCKER_ENV_VARS="$DOCKER_ENV_VARS -e LLAMA_STACK_TEST_INFERENCE_MODE=$INFERENCE_MODE" + DOCKER_ENV_VARS="$DOCKER_ENV_VARS -e LLAMA_STACK_TEST_STACK_CONFIG_TYPE=server" + + # Pass through API keys if they exist + [ -n "${TOGETHER_API_KEY:-}" ] && DOCKER_ENV_VARS="$DOCKER_ENV_VARS -e TOGETHER_API_KEY=$TOGETHER_API_KEY" + [ -n "${FIREWORKS_API_KEY:-}" ] && DOCKER_ENV_VARS="$DOCKER_ENV_VARS -e FIREWORKS_API_KEY=$FIREWORKS_API_KEY" + [ -n "${TAVILY_SEARCH_API_KEY:-}" ] && DOCKER_ENV_VARS="$DOCKER_ENV_VARS -e TAVILY_SEARCH_API_KEY=$TAVILY_SEARCH_API_KEY" + [ -n "${OPENAI_API_KEY:-}" ] && DOCKER_ENV_VARS="$DOCKER_ENV_VARS -e OPENAI_API_KEY=$OPENAI_API_KEY" + [ -n "${ANTHROPIC_API_KEY:-}" ] && DOCKER_ENV_VARS="$DOCKER_ENV_VARS -e ANTHROPIC_API_KEY=$ANTHROPIC_API_KEY" + [ -n "${GROQ_API_KEY:-}" ] && DOCKER_ENV_VARS="$DOCKER_ENV_VARS -e GROQ_API_KEY=$GROQ_API_KEY" + [ -n "${GEMINI_API_KEY:-}" ] && DOCKER_ENV_VARS="$DOCKER_ENV_VARS -e GEMINI_API_KEY=$GEMINI_API_KEY" + [ -n "${OLLAMA_URL:-}" ] && DOCKER_ENV_VARS="$DOCKER_ENV_VARS -e OLLAMA_URL=$OLLAMA_URL" + [ -n "${SAFETY_MODEL:-}" ] && DOCKER_ENV_VARS="$DOCKER_ENV_VARS -e SAFETY_MODEL=$SAFETY_MODEL" + + # Determine the actual image name (may have localhost/ prefix) + IMAGE_NAME=$(docker images --format "{{.Repository}}:{{.Tag}}" | grep "distribution-$DISTRO:dev$" | head -1) + if [[ -z "$IMAGE_NAME" ]]; then + echo "❌ Error: Could not find image for distribution-$DISTRO:dev" + exit 1 + fi + echo "Using image: $IMAGE_NAME" + + docker run -d --network host --name "$container_name" \ + -p $LLAMA_STACK_PORT:$LLAMA_STACK_PORT \ + $DOCKER_ENV_VARS \ + -v $ROOT_DIR:/app/llama-stack-source \ + "$IMAGE_NAME" \ + --port $LLAMA_STACK_PORT + + echo "Waiting for Docker container to start..." + for i in {1..30}; do + if curl -s http://localhost:$LLAMA_STACK_PORT/v1/health 2>/dev/null | grep -q "OK"; then + echo "✅ Docker container started successfully" + break + fi + if [[ $i -eq 30 ]]; then + echo "❌ Docker container failed to start" + echo "Container logs:" + docker logs "$container_name" + exit 1 + fi + sleep 1 + done + echo "" + + # Update STACK_CONFIG to point to the running container + STACK_CONFIG="http://localhost:$LLAMA_STACK_PORT" + + trap stop_container EXIT ERR INT TERM +fi + # Run tests echo "=== Running Integration Tests ===" EXCLUDE_TESTS="builtin_tool or safety_with_image or code_interpreter or test_rag" diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 7734b8654..3137de0de 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -42,7 +42,9 @@ def pytest_sessionstart(session): # Set test stack config type for api_recorder test isolation stack_config = session.config.getoption("--stack-config", default=None) - if stack_config and (stack_config.startswith("server:") or stack_config.startswith("http")): + if stack_config and ( + stack_config.startswith("server:") or stack_config.startswith("docker:") or stack_config.startswith("http") + ): os.environ["LLAMA_STACK_TEST_STACK_CONFIG_TYPE"] = "server" logger.info(f"Test stack config type: server (stack_config={stack_config})") else: @@ -139,7 +141,9 @@ def pytest_addoption(parser): a 'pointer' to the stack. this can be either be: (a) a template name like `starter`, or (b) a path to a run.yaml file, or - (c) an adhoc config spec, e.g. `inference=fireworks,safety=llama-guard,agents=meta-reference` + (c) an adhoc config spec, e.g. `inference=fireworks,safety=llama-guard,agents=meta-reference`, or + (d) a server config like `server:ci-tests`, or + (e) a docker config like `docker:ci-tests` (builds and runs container) """ ), )