From a8091d0c6ae60cee8afc2d8b653efaa8a6079910 Mon Sep 17 00:00:00 2001 From: Francisco Arceo Date: Mon, 18 Aug 2025 06:04:21 -0600 Subject: [PATCH 1/7] chore: Update benchmarking location in contributing docs (#3180) # What does this PR do? Small docs change as requested in https://github.com/llamastack/llama-stack/pull/3160#pullrequestreview-3125038932 ## Test Plan --- docs/source/contributing/index.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/source/contributing/index.md b/docs/source/contributing/index.md index 24bf3f66c..1846f4d97 100644 --- a/docs/source/contributing/index.md +++ b/docs/source/contributing/index.md @@ -23,12 +23,7 @@ new_vector_database ```{include} ../../../tests/README.md ``` -## Benchmarking - -```{include} ../../../docs/source/distributions/k8s-benchmark/README.md -``` - -### Advanced Topics +## Advanced Topics For developers who need deeper understanding of the testing system internals: @@ -37,3 +32,8 @@ For developers who need deeper understanding of the testing system internals: testing/record-replay ``` + +### Benchmarking + +```{include} ../../../docs/source/distributions/k8s-benchmark/README.md +``` From f4cecaade94456135f33057ec678542593a839bf Mon Sep 17 00:00:00 2001 From: Ashwin Bharambe Date: Mon, 18 Aug 2025 10:11:55 -0700 Subject: [PATCH 2/7] chore(ci): dont run llama stack server always (#3188) Sometimes the server has already been started (e.g., via docker). Just a convenience here so we can reuse this script more. --- scripts/integration-tests.sh | 43 ++++++++++++++++++++++-------------- 1 file changed, 26 insertions(+), 17 deletions(-) diff --git a/scripts/integration-tests.sh b/scripts/integration-tests.sh index bf7671348..e9a5283e1 100755 --- a/scripts/integration-tests.sh +++ b/scripts/integration-tests.sh @@ -133,6 +133,10 @@ else EXTRA_PARAMS="" fi +THIS_DIR=$(dirname "$0") +ROOT_DIR="$THIS_DIR/.." +cd $ROOT_DIR + # Set recording directory if [[ "$RUN_VISION_TESTS" == "true" ]]; then export LLAMA_STACK_TEST_RECORDING_DIR="tests/integration/recordings/vision" @@ -142,24 +146,29 @@ fi # Start Llama Stack Server if needed if [[ "$STACK_CONFIG" == *"server:"* ]]; then - echo "=== Starting Llama Stack Server ===" - nohup uv run llama stack run ci-tests --image-type venv > server.log 2>&1 & + # check if server is already running + if curl -s http://localhost:8321/v1/health 2>/dev/null | grep -q "OK"; then + echo "Llama Stack Server is already running, skipping start" + else + echo "=== Starting Llama Stack Server ===" + nohup uv run llama stack run ci-tests --image-type venv > server.log 2>&1 & - echo "Waiting for Llama Stack Server to start..." - for i in {1..30}; do - if curl -s http://localhost:8321/v1/health 2>/dev/null | grep -q "OK"; then - echo "✅ Llama Stack Server started successfully" - break - fi - if [[ $i -eq 30 ]]; then - echo "❌ Llama Stack Server failed to start" - echo "Server logs:" - cat server.log - exit 1 - fi - sleep 1 - done - echo "" + echo "Waiting for Llama Stack Server to start..." + for i in {1..30}; do + if curl -s http://localhost:8321/v1/health 2>/dev/null | grep -q "OK"; then + echo "✅ Llama Stack Server started successfully" + break + fi + if [[ $i -eq 30 ]]; then + echo "❌ Llama Stack Server failed to start" + echo "Server logs:" + cat server.log + exit 1 + fi + sleep 1 + done + echo "" + fi fi # Run tests From 4ae39b94ffcd3940f2dcc3aa9f50165b96ab39a8 Mon Sep 17 00:00:00 2001 From: Charlie Doern Date: Mon, 18 Aug 2025 13:23:23 -0400 Subject: [PATCH 3/7] fix: remove category prints (#3189) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit # What does this PR do? commands where the output is important like `llama stack build --print-deps-only` (soon to be `llama stack show`) print some log.py `cprint`'s on _every_ execution of the CLI for example: Screenshot 2025-08-18 at 1 16 30 PM the yellow text is likely unnecessary. Signed-off-by: Charlie Doern --- llama_stack/log.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/llama_stack/log.py b/llama_stack/log.py index 7507aface..d67bd1b61 100644 --- a/llama_stack/log.py +++ b/llama_stack/log.py @@ -7,13 +7,11 @@ import logging import os import re -import sys from logging.config import dictConfig from rich.console import Console from rich.errors import MarkupError from rich.logging import RichHandler -from termcolor import cprint from llama_stack.core.datatypes import LoggingConfig @@ -66,7 +64,6 @@ def config_to_category_levels(category: str, level: str): category_levels["root"] = level_value elif category in CATEGORIES: category_levels[category] = level_value - logging.info(f"Setting '{category}' category to level '{level}'.") else: logging.warning(f"Unknown logging category: {category}. No changes made.") return category_levels @@ -256,7 +253,6 @@ def get_logger( env_config = os.environ.get("LLAMA_STACK_LOGGING", "") if env_config: - cprint(f"Environment variable LLAMA_STACK_LOGGING found: {env_config}", color="yellow", file=sys.stderr) _category_levels.update(parse_environment_config(env_config)) log_file = os.environ.get("LLAMA_STACK_LOG_FILE") From fa431e15e02f550f0278ac7920d55bb432641759 Mon Sep 17 00:00:00 2001 From: Francisco Arceo Date: Mon, 18 Aug 2025 11:23:51 -0600 Subject: [PATCH 4/7] chore: Update TRIAGERS.md (#3186) # What does this PR do? Update triagers to current state ## Test Plan --- .github/TRIAGERS.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/TRIAGERS.md b/.github/TRIAGERS.md index ed4f4a6c6..f5bd11531 100644 --- a/.github/TRIAGERS.md +++ b/.github/TRIAGERS.md @@ -1,2 +1,2 @@ # This file documents Triage members in the Llama Stack community - @bbrowning @franciscojavierarceo @leseb + @franciscojavierarceo From 739b18edf8755390cd75b819852090de3be940c3 Mon Sep 17 00:00:00 2001 From: Maor Friedman Date: Mon, 18 Aug 2025 20:24:24 +0300 Subject: [PATCH 5/7] feat: add support for postgres ssl mode and root cert (#3182) this PR adds support for configuring `sslmode` and `sslrootcert` when initiating the psycopg2 connection. closes #3181 --- llama_stack/providers/utils/kvstore/config.py | 2 ++ llama_stack/providers/utils/kvstore/postgres/postgres.py | 2 ++ 2 files changed, 4 insertions(+) diff --git a/llama_stack/providers/utils/kvstore/config.py b/llama_stack/providers/utils/kvstore/config.py index f00cb1f8b..d1747d65b 100644 --- a/llama_stack/providers/utils/kvstore/config.py +++ b/llama_stack/providers/utils/kvstore/config.py @@ -75,6 +75,8 @@ class PostgresKVStoreConfig(CommonConfig): db: str = "llamastack" user: str password: str | None = None + ssl_mode: str | None = None + ca_cert_path: str | None = None table_name: str = "llamastack_kvstore" @classmethod diff --git a/llama_stack/providers/utils/kvstore/postgres/postgres.py b/llama_stack/providers/utils/kvstore/postgres/postgres.py index bd35decfc..cabb4c512 100644 --- a/llama_stack/providers/utils/kvstore/postgres/postgres.py +++ b/llama_stack/providers/utils/kvstore/postgres/postgres.py @@ -30,6 +30,8 @@ class PostgresKVStoreImpl(KVStore): database=self.config.db, user=self.config.user, password=self.config.password, + sslmode=self.config.ssl_mode, + sslrootcert=self.config.ca_cert_path, ) self.conn.autocommit = True self.cursor = self.conn.cursor(cursor_factory=DictCursor) From f8398d25ff312ce23afb21616a385de3e0a3d6da Mon Sep 17 00:00:00 2001 From: IAN MILLER <75687988+r3v5@users.noreply.github.com> Date: Mon, 18 Aug 2025 20:17:44 +0100 Subject: [PATCH 6/7] fix: kill build_conda_env.sh (#3190) # What does this PR do? I noticed somehow [build_conda_env.sh](https://github.com/llamastack/llama-stack/blob/main/llama_stack/core/build_conda_env.sh) exists in main branch. We need to kill it to be consistent with [#2969](https://github.com/llamastack/llama-stack/pull/2969) ## Test Plan --- llama_stack/core/build_conda_env.sh | 207 ---------------------------- 1 file changed, 207 deletions(-) delete mode 100755 llama_stack/core/build_conda_env.sh diff --git a/llama_stack/core/build_conda_env.sh b/llama_stack/core/build_conda_env.sh deleted file mode 100755 index 48ac3a1ab..000000000 --- a/llama_stack/core/build_conda_env.sh +++ /dev/null @@ -1,207 +0,0 @@ -#!/bin/bash - -# Copyright (c) Meta Platforms, Inc. and affiliates. -# All rights reserved. -# -# This source code is licensed under the terms described in the LICENSE file in -# the root directory of this source tree. - -LLAMA_STACK_DIR=${LLAMA_STACK_DIR:-} -LLAMA_STACK_CLIENT_DIR=${LLAMA_STACK_CLIENT_DIR:-} -TEST_PYPI_VERSION=${TEST_PYPI_VERSION:-} -PYPI_VERSION=${PYPI_VERSION:-} -# This timeout (in seconds) is necessary when installing PyTorch via uv since it's likely to time out -# Reference: https://github.com/astral-sh/uv/pull/1694 -UV_HTTP_TIMEOUT=${UV_HTTP_TIMEOUT:-500} - -set -euo pipefail - -# Define color codes -RED='\033[0;31m' -GREEN='\033[0;32m' -NC='\033[0m' # No Color - -SCRIPT_DIR=$(dirname "$(readlink -f "$0")") -source "$SCRIPT_DIR/common.sh" - -# Usage function -usage() { - echo "Usage: $0 --env-name --build-file-path --normal-deps [--external-provider-deps ] [--optional-deps ]" - echo "Example: $0 --env-name my-conda-env --build-file-path ./my-stack-build.yaml --normal-deps 'numpy pandas scipy' --external-provider-deps 'foo' --optional-deps 'bar'" - exit 1 -} - -# Parse arguments -env_name="" -build_file_path="" -normal_deps="" -external_provider_deps="" -optional_deps="" - -while [[ $# -gt 0 ]]; do - key="$1" - case "$key" in - --env-name) - if [[ -z "$2" || "$2" == --* ]]; then - echo "Error: --env-name requires a string value" >&2 - usage - fi - env_name="$2" - shift 2 - ;; - --build-file-path) - if [[ -z "$2" || "$2" == --* ]]; then - echo "Error: --build-file-path requires a string value" >&2 - usage - fi - build_file_path="$2" - shift 2 - ;; - --normal-deps) - if [[ -z "$2" || "$2" == --* ]]; then - echo "Error: --normal-deps requires a string value" >&2 - usage - fi - normal_deps="$2" - shift 2 - ;; - --external-provider-deps) - if [[ -z "$2" || "$2" == --* ]]; then - echo "Error: --external-provider-deps requires a string value" >&2 - usage - fi - external_provider_deps="$2" - shift 2 - ;; - --optional-deps) - if [[ -z "$2" || "$2" == --* ]]; then - echo "Error: --optional-deps requires a string value" >&2 - usage - fi - optional_deps="$2" - shift 2 - ;; - *) - echo "Unknown option: $1" >&2 - usage - ;; - esac -done - -# Check required arguments -if [[ -z "$env_name" || -z "$build_file_path" || -z "$normal_deps" ]]; then - echo "Error: --env-name, --build-file-path, and --normal-deps are required." >&2 - usage -fi - -if [ -n "$LLAMA_STACK_DIR" ]; then - echo "Using llama-stack-dir=$LLAMA_STACK_DIR" -fi -if [ -n "$LLAMA_STACK_CLIENT_DIR" ]; then - echo "Using llama-stack-client-dir=$LLAMA_STACK_CLIENT_DIR" -fi - -ensure_conda_env_python310() { - # Use only global variables set by flag parser - local python_version="3.12" - - if ! is_command_available conda; then - printf "${RED}Error: conda command not found. Is Conda installed and in your PATH?${NC}" >&2 - exit 1 - fi - - if conda env list | grep -q "^${env_name} "; then - printf "Conda environment '${env_name}' exists. Checking Python version...\n" - current_version=$(conda run -n "${env_name}" python --version 2>&1 | cut -d' ' -f2 | cut -d'.' -f1,2) - if [ "$current_version" = "$python_version" ]; then - printf "Environment '${env_name}' already has Python ${python_version}. No action needed.\n" - else - printf "Updating environment '${env_name}' to Python ${python_version}...\n" - conda install -n "${env_name}" python="${python_version}" -y - fi - else - printf "Conda environment '${env_name}' does not exist. Creating with Python ${python_version}...\n" - conda create -n "${env_name}" python="${python_version}" -y - fi - - eval "$(conda shell.bash hook)" - conda deactivate && conda activate "${env_name}" - "$CONDA_PREFIX"/bin/pip install uv - - if [ -n "$TEST_PYPI_VERSION" ]; then - uv pip install fastapi libcst - uv pip install --extra-index-url https://test.pypi.org/simple/ \ - llama-stack=="$TEST_PYPI_VERSION" \ - "$normal_deps" - if [ -n "$optional_deps" ]; then - IFS='#' read -ra parts <<<"$optional_deps" - for part in "${parts[@]}"; do - echo "$part" - uv pip install $part - done - fi - if [ -n "$external_provider_deps" ]; then - IFS='#' read -ra parts <<<"$external_provider_deps" - for part in "${parts[@]}"; do - echo "$part" - uv pip install "$part" - done - fi - else - if [ -n "$LLAMA_STACK_DIR" ]; then - if [ ! -d "$LLAMA_STACK_DIR" ]; then - printf "${RED}Warning: LLAMA_STACK_DIR is set but directory does not exist: $LLAMA_STACK_DIR${NC}\n" >&2 - exit 1 - fi - printf "Installing from LLAMA_STACK_DIR: $LLAMA_STACK_DIR\n" - uv pip install --no-cache-dir -e "$LLAMA_STACK_DIR" - else - PYPI_VERSION="${PYPI_VERSION:-}" - if [ -n "$PYPI_VERSION" ]; then - SPEC_VERSION="llama-stack==${PYPI_VERSION}" - else - SPEC_VERSION="llama-stack" - fi - uv pip install --no-cache-dir "$SPEC_VERSION" - fi - if [ -n "$LLAMA_STACK_CLIENT_DIR" ]; then - if [ ! -d "$LLAMA_STACK_CLIENT_DIR" ]; then - printf "${RED}Warning: LLAMA_STACK_CLIENT_DIR is set but directory does not exist: $LLAMA_STACK_CLIENT_DIR${NC}\n" >&2 - exit 1 - fi - printf "Installing from LLAMA_STACK_CLIENT_DIR: $LLAMA_STACK_CLIENT_DIR\n" - uv pip install --no-cache-dir -e "$LLAMA_STACK_CLIENT_DIR" - fi - printf "Installing pip dependencies\n" - uv pip install $normal_deps - if [ -n "$optional_deps" ]; then - IFS='#' read -ra parts <<<"$optional_deps" - for part in "${parts[@]}"; do - echo "$part" - uv pip install $part - done - fi - if [ -n "$external_provider_deps" ]; then - IFS='#' read -ra parts <<<"$external_provider_deps" - for part in "${parts[@]}"; do - echo "Getting provider spec for module: $part and installing dependencies" - package_name=$(echo "$part" | sed 's/[<>=!].*//') - python3 -c " -import importlib -import sys -try: - module = importlib.import_module(f'$package_name.provider') - spec = module.get_provider_spec() - if hasattr(spec, 'pip_packages') and spec.pip_packages: - print('\\n'.join(spec.pip_packages)) -except Exception as e: - print(f'Error getting provider spec for $package_name: {e}', file=sys.stderr) -" | uv pip install -r - - done - fi - fi - mv "$build_file_path" "$CONDA_PREFIX"/llamastack-build.yaml - echo "Build spec configuration saved at $CONDA_PREFIX/llamastack-build.yaml" -} - -ensure_conda_env_python310 "$env_name" "$build_file_path" "$normal_deps" "$optional_deps" "$external_provider_deps" From 27d6becfd0f2b3071fc650eee0ae8e15a7ae8115 Mon Sep 17 00:00:00 2001 From: Ashwin Bharambe Date: Mon, 18 Aug 2025 12:20:50 -0700 Subject: [PATCH 7/7] fix(misc): pin openai dependency to < 1.100.0 (#3192) This OpenAI client release https://github.com/openai/openai-python/commit/0843a1116498bc3312db9904adf71a4fb0a0a77e ends up breaking litellm https://github.com/BerriAI/litellm/blob/169a17400f1f5e36815c7d89128754975cd0584d/litellm/types/llms/openai.py#L40 Update the dependency pin. Also make the imports a bit more defensive anyhow if something else during `llama stack build` ends up moving openai to a previous version. ## Test Plan Run pre-release script integration tests. --- .../utils/inference/openai_compat.py | 12 +++++++++--- pyproject.toml | 2 +- scripts/integration-tests.sh | 19 ++++++++++++++++--- uv.lock | 2 +- 4 files changed, 27 insertions(+), 8 deletions(-) diff --git a/llama_stack/providers/utils/inference/openai_compat.py b/llama_stack/providers/utils/inference/openai_compat.py index 6297cc2ed..5e6c26884 100644 --- a/llama_stack/providers/utils/inference/openai_compat.py +++ b/llama_stack/providers/utils/inference/openai_compat.py @@ -31,9 +31,15 @@ from openai.types.chat import ( from openai.types.chat import ( ChatCompletionContentPartTextParam as OpenAIChatCompletionContentPartTextParam, ) -from openai.types.chat import ( - ChatCompletionMessageFunctionToolCall as OpenAIChatCompletionMessageFunctionToolCall, -) + +try: + from openai.types.chat import ( + ChatCompletionMessageFunctionToolCall as OpenAIChatCompletionMessageFunctionToolCall, + ) +except ImportError: + from openai.types.chat.chat_completion_message_tool_call import ( + ChatCompletionMessageToolCall as OpenAIChatCompletionMessageFunctionToolCall, + ) from openai.types.chat import ( ChatCompletionMessageParam as OpenAIChatCompletionMessage, ) diff --git a/pyproject.toml b/pyproject.toml index db0ad1f00..f02c02c41 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -33,7 +33,7 @@ dependencies = [ "jsonschema", "llama-stack-client>=0.2.17", "llama-api-client>=0.1.2", - "openai>=1.99.6", + "openai>=1.99.6,<1.100.0", "prompt-toolkit", "python-dotenv", "python-jose[cryptography]", diff --git a/scripts/integration-tests.sh b/scripts/integration-tests.sh index e9a5283e1..66e6d8e57 100755 --- a/scripts/integration-tests.sh +++ b/scripts/integration-tests.sh @@ -144,6 +144,19 @@ else export LLAMA_STACK_TEST_RECORDING_DIR="tests/integration/recordings" fi +# check if "llama" and "pytest" are available. this script does not use `uv run` given +# it can be used in a pre-release environment where we have not been able to tell +# uv about pre-release dependencies properly (yet). +if ! command -v llama &> /dev/null; then + echo "llama could not be found, ensure llama-stack is installed" + exit 1 +fi + +if ! command -v pytest &> /dev/null; then + echo "pytest could not be found, ensure pytest is installed" + exit 1 +fi + # Start Llama Stack Server if needed if [[ "$STACK_CONFIG" == *"server:"* ]]; then # check if server is already running @@ -151,7 +164,7 @@ if [[ "$STACK_CONFIG" == *"server:"* ]]; then echo "Llama Stack Server is already running, skipping start" else echo "=== Starting Llama Stack Server ===" - nohup uv run llama stack run ci-tests --image-type venv > server.log 2>&1 & + nohup llama stack run ci-tests --image-type venv > server.log 2>&1 & echo "Waiting for Llama Stack Server to start..." for i in {1..30}; do @@ -189,7 +202,7 @@ fi if [[ "$RUN_VISION_TESTS" == "true" ]]; then echo "Running vision tests..." set +e - uv run pytest -s -v tests/integration/inference/test_vision_inference.py \ + pytest -s -v tests/integration/inference/test_vision_inference.py \ --stack-config="$STACK_CONFIG" \ -k "$PYTEST_PATTERN" \ --vision-model=ollama/llama3.2-vision:11b \ @@ -257,7 +270,7 @@ echo "=== Running all collected tests in a single pytest command ===" echo "Total test files: $(echo $TEST_FILES | wc -w)" set +e -uv run pytest -s -v $TEST_FILES \ +pytest -s -v $TEST_FILES \ --stack-config="$STACK_CONFIG" \ -k "$PYTEST_PATTERN" \ --text-model="$TEXT_MODEL" \ diff --git a/uv.lock b/uv.lock index a09406770..3e3bf7e24 100644 --- a/uv.lock +++ b/uv.lock @@ -1856,7 +1856,7 @@ requires-dist = [ { name = "llama-api-client", specifier = ">=0.1.2" }, { name = "llama-stack-client", specifier = ">=0.2.17" }, { name = "llama-stack-client", marker = "extra == 'ui'", specifier = ">=0.2.17" }, - { name = "openai", specifier = ">=1.99.6" }, + { name = "openai", specifier = ">=1.99.6,<1.100.0" }, { name = "opentelemetry-exporter-otlp-proto-http", specifier = ">=1.30.0" }, { name = "opentelemetry-sdk", specifier = ">=1.30.0" }, { name = "pandas", marker = "extra == 'ui'" },