llama distribution -> llama stack + containers (WIP)

llama_toolchain/distribution/build_conda_env.sh

@@ -47,25 +47,25 @@ ensure_conda_env_python310() {
 
   # Check if conda command is available
   if ! command -v conda &>/dev/null; then
-    echo -e "${RED}Error: conda command not found. Is Conda installed and in your PATH?${NC}" >&2
+    printf "${RED}Error: conda command not found. Is Conda installed and in your PATH?${NC}" >&2
     exit 1
   fi
 
   # Check if the environment exists
   if conda env list | grep -q "^${env_name} "; then
-    echo "Conda environment '${env_name}' exists. Checking Python version..."
+    printf "Conda environment '${env_name}' exists. Checking Python version..."
 
     # Check Python version in the environment
     current_version=$(conda run -n "${env_name}" python --version 2>&1 | cut -d' ' -f2 | cut -d'.' -f1,2)
 
     if [ "$current_version" = "$python_version" ]; then
-      echo "Environment '${env_name}' already has Python ${python_version}. No action needed."
+      printf "Environment '${env_name}' already has Python ${python_version}. No action needed."
     else
-      echo "Updating environment '${env_name}' to Python ${python_version}..."
+      printf "Updating environment '${env_name}' to Python ${python_version}..."
       conda install -n "${env_name}" python="${python_version}" -y
     fi
   else
-    echo "Conda environment '${env_name}' does not exist. Creating with Python ${python_version}..."
+    printf "Conda environment '${env_name}' does not exist. Creating with Python ${python_version}..."
     conda create -n "${env_name}" python="${python_version}" -y
 
     ENVNAME="${env_name}"
@@ -83,11 +83,11 @@ ensure_conda_env_python310() {
     # Re-installing llama-toolchain in the new conda environment
     if [ -n "$LLAMA_TOOLCHAIN_DIR" ]; then
       if [ ! -d "$LLAMA_TOOLCHAIN_DIR" ]; then
-        echo -e "${RED}Warning: LLAMA_TOOLCHAIN_DIR is set but directory does not exist: $LLAMA_TOOLCHAIN_DIR${NC}" >&2
+        printf "${RED}Warning: LLAMA_TOOLCHAIN_DIR is set but directory does not exist: $LLAMA_TOOLCHAIN_DIR${NC}" >&2
         exit 1
       fi
 
-      echo "Installing from LLAMA_TOOLCHAIN_DIR: $LLAMA_TOOLCHAIN_DIR"
+      printf "Installing from LLAMA_TOOLCHAIN_DIR: $LLAMA_TOOLCHAIN_DIR"
       pip install --no-cache-dir -e "$LLAMA_TOOLCHAIN_DIR"
     else
       pip install --no-cache-dir llama-toolchain
@@ -95,18 +95,18 @@ ensure_conda_env_python310() {
 
     if [ -n "$LLAMA_MODELS_DIR" ]; then
       if [ ! -d "$LLAMA_MODELS_DIR" ]; then
-        echo -e "${RED}Warning: LLAMA_MODELS_DIR is set but directory does not exist: $LLAMA_MODELS_DIR${NC}" >&2
+        printf "${RED}Warning: LLAMA_MODELS_DIR is set but directory does not exist: $LLAMA_MODELS_DIR${NC}" >&2
         exit 1
       fi
 
-      echo "Installing from LLAMA_MODELS_DIR: $LLAMA_MODELS_DIR"
+      printf "Installing from LLAMA_MODELS_DIR: $LLAMA_MODELS_DIR"
       pip uninstall -y llama-models
       pip install --no-cache-dir -e "$LLAMA_MODELS_DIR"
     fi
 
     # Install pip dependencies
     if [ -n "$pip_dependencies" ]; then
-      echo "Installing pip dependencies: $pip_dependencies"
+      printf "Installing pip dependencies: $pip_dependencies"
       pip install $pip_dependencies
     fi
   fi
@@ -114,6 +114,6 @@ ensure_conda_env_python310() {
 
 ensure_conda_env_python310 "$env_name" "$pip_dependencies"
 
-echo -e "${GREEN}Successfully setup conda environment. Configuring build...${NC}"
+printf "${GREEN}Successfully setup conda environment. Configuring build...${NC}"
 
 $CONDA_PREFIX/bin/python3 -m llama_toolchain.cli.llama api configure "$api_or_stack" --name "$env_name"

llama_toolchain/distribution/build_container.sh

@@ -1,44 +1,113 @@
 #!/bin/bash
 
-if [ "$#" -ne 3 ]; then
-  echo "Usage: $0 <image_name> <base_image> <pip_dependencies>
-  echo "Example: $0 my-fastapi-app python:3.9-slim 'fastapi uvicorn'
+LLAMA_MODELS_DIR=${LLAMA_MODELS_DIR:-}
+LLAMA_TOOLCHAIN_DIR=${LLAMA_TOOLCHAIN_DIR:-}
+TEST_PYPI_VERSION=${TEST_PYPI_VERSION:-}
+
+
+if [ "$#" -ne 4 ]; then
+  echo "Usage: $0 [api|stack] <image_name> <docker_base> <pip_dependencies>
+  echo "Example: $0 agentic_system my-fastapi-app python:3.9-slim 'fastapi uvicorn'
   exit 1
 fi
 
-IMAGE_NAME=$1
-BASE_IMAGE=$2
-PIP_DEPENDENCIES=$3
+api_or_stack=$1
+image_name=$2
+docker_base=$3
+pip_dependencies=$4
+
+# Define color codes
+RED='\033[0;31m'
+GREEN='\033[0;32m'
+NC='\033[0m' # No Color
 
 set -euo pipefail
 
-PORT=8001
-
 SCRIPT_DIR=$(dirname "$(readlink -f "$0")")
-SOURCE_DIR=$(dirname $(dirname $(dirname "$SCRIPT_DIR")))
-echo $SOURCE_DIR
+REPO_DIR=$(dirname $(dirname "$SCRIPT_DIR"))
 
 TEMP_DIR=$(mktemp -d)
-echo "Created temporary directory: $TEMP_DIR"
 
-cat <<EOF >"$TEMP_DIR/Dockerfile"
-FROM $BASE_IMAGE
+add_to_docker() {
+  local input
+  output_file="$TEMP_DIR/Dockerfile"
+  if [ -t 0 ]; then
+    printf '%s\n' "$1" >> "$output_file"
+  else
+    # If stdin is not a terminal, read from it (heredoc)
+    cat >> "$output_file"
+  fi
+}
+
+
+add_to_docker <<EOF
+FROM $docker_base
 WORKDIR /app
-COPY llama-stack/llama_toolchain /app/llama_toolchain
-COPY llama-models/models /app/llama_models
 
-RUN pip install $PIP_DEPENDENCIES
-
-EXPOSE $PORT
-ENTRYPOINT ["python3", "-m", "llama_toolchain.distribution.server", "--port", "$PORT"]
+RUN apt-get update && apt-get install -y \
+       iputils-ping net-tools iproute2 dnsutils telnet \
+       curl wget telnet \
+       procps psmisc lsof \
+       traceroute \
+       && rm -rf /var/lib/apt/lists/*
 
 EOF
 
-echo "Dockerfile created successfully in $TEMP_DIR/Dockerfile"
+toolchain_mount="/app/llama-toolchain-source"
+models_mount="/app/llama-models-source"
 
-podman build -t $IMAGE_NAME -f "$TEMP_DIR/Dockerfile" "$SOURCE_DIR"
+if [ -n "$LLAMA_TOOLCHAIN_DIR" ]; then
+  if [ ! -d "$LLAMA_TOOLCHAIN_DIR" ]; then
+    echo "${RED}Warning: LLAMA_TOOLCHAIN_DIR is set but directory does not exist: $LLAMA_TOOLCHAIN_DIR${NC}" >&2
+    exit 1
+  fi
+  add_to_docker "RUN pip install $toolchain_mount"
+else
+  add_to_docker "RUN pip install llama-toolchain"
+fi
 
-echo "Podman image '$IMAGE_NAME' built successfully."
-echo "You can run it with: podman run -p 8000:8000 $IMAGE_NAME"
+if [ -n "$LLAMA_MODELS_DIR" ]; then
+ if [ ! -d "$LLAMA_MODELS_DIR" ]; then
+    echo "${RED}Warning: LLAMA_MODELS_DIR is set but directory does not exist: $LLAMA_MODELS_DIR${NC}" >&2
+    exit 1
+  fi
 
-rm -rf "$TEMP_DIR"
+  add_to_docker <<EOF
+RUN pip uninstall -y llama-models
+RUN pip install $models_mount
+
+EOF
+fi
+
+if [ -n "$pip_dependencies" ]; then
+  add_to_docker "RUN pip install $pip_dependencies"
+fi
+
+add_to_docker <<EOF
+
+# This would be good in production but for debugging flexibility lets not add it right now
+# We need a more solid production ready entrypoint.sh anyway
+#
+# ENTRYPOINT ["python", "-m", "llama_toolchain.distribution.server"]
+
+EOF
+
+printf "Dockerfile created successfully in $TEMP_DIR/Dockerfile"
+cat $TEMP_DIR/Dockerfile
+printf "\n"
+
+mounts=""
+if [ -n "$LLAMA_TOOLCHAIN_DIR" ]; then
+  mounts="$mounts -v $(readlink -f $LLAMA_TOOLCHAIN_DIR):$toolchain_mount"
+fi
+if [ -n "$LLAMA_MODELS_DIR" ]; then
+  mounts="$mounts -v $(readlink -f $LLAMA_MODELS_DIR):$models_mount"
+fi
+set -x
+podman build -t $image_name -f "$TEMP_DIR/Dockerfile" "$REPO_DIR" $mounts
+set +x
+
+printf "${GREEN}Succesfully setup Podman image. Configuring build...${NC}"
+echo "You can run it with: podman run -p 8000:8000 $image_name"
+
+$CONDA_PREFIX/bin/python3 -m llama_toolchain.cli.llama api configure "$api_or_stack" --name "$image_name"

llama_toolchain/distribution/datatypes.py

@@ -115,6 +115,10 @@ as being "Llama Stack compatible"
 """,
     )
 
+    @property
+    def docker_image(self) -> Optional[str]:
+        return None
+
     @property
     def module(self) -> str:
         if self.adapter:

llama_toolchain/distribution/start_conda_env.sh

@@ -18,15 +18,24 @@ error_handler() {
 
 trap 'error_handler ${LINENO}' ERR
 
-if [ $# -lt 2 ]; then
-  echo "Usage: $0 <environment_name> <script_args...>"
+if [ $# -lt 3 ]; then
+  echo "Usage: $0 <env_name> <yaml_config> <port> <script_args...>"
   exit 1
 fi
 
 env_name="$1"
 shift
 
+yaml_config="$1"
+shift
+
+port="$1"
+shift
+
 eval "$(conda shell.bash hook)"
 conda deactivate && conda activate "$env_name"
 
-$CONDA_PREFIX/bin/python -m llama_toolchain.distribution.server "$@"
+$CONDA_PREFIX/bin/python \
+  -m llama_toolchain.distribution.server \
+  --yaml_config "$yaml_config" \
+  --port "$port" "$@"

llama_toolchain/distribution/start_container.sh

@@ -8,4 +8,35 @@
 
 set -euo pipefail
 
-podman run -it -p 8001:8001 -v ~/.llama/test.yaml:/app/test.yaml test-image --yaml_config /app/test.yaml --disable-ipv6
+RED='\033[0;31m'
+NC='\033[0m' # No Color
+
+error_handler() {
+  echo "Error occurred in script at line: ${1}" >&2
+  exit 1
+}
+
+trap 'error_handler ${LINENO}' ERR
+
+if [ $# -lt 3 ]; then
+  echo "Usage: $0 <docker_image> <yaml_config> <port> <other_args...>"
+  exit 1
+fi
+
+docker_image="$1"
+shift
+
+yaml_config="$1"
+shift
+
+port="$1"
+shift
+
+set -x
+podman run -it \
+  -p $port:$port \
+  -v "$yaml_config:/app/config.yaml" \
+  $docker_image \
+  python -m llama_toolchain.distribution.server \
+  --yaml_config /app/config.yaml \
+  --port $port "$@"