From a8091d0c6ae60cee8afc2d8b653efaa8a6079910 Mon Sep 17 00:00:00 2001 From: Francisco Arceo Date: Mon, 18 Aug 2025 06:04:21 -0600 Subject: [PATCH 01/16] chore: Update benchmarking location in contributing docs (#3180) # What does this PR do? Small docs change as requested in https://github.com/llamastack/llama-stack/pull/3160#pullrequestreview-3125038932 ## Test Plan --- docs/source/contributing/index.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/source/contributing/index.md b/docs/source/contributing/index.md index 24bf3f66c..1846f4d97 100644 --- a/docs/source/contributing/index.md +++ b/docs/source/contributing/index.md @@ -23,12 +23,7 @@ new_vector_database ```{include} ../../../tests/README.md ``` -## Benchmarking - -```{include} ../../../docs/source/distributions/k8s-benchmark/README.md -``` - -### Advanced Topics +## Advanced Topics For developers who need deeper understanding of the testing system internals: @@ -37,3 +32,8 @@ For developers who need deeper understanding of the testing system internals: testing/record-replay ``` + +### Benchmarking + +```{include} ../../../docs/source/distributions/k8s-benchmark/README.md +``` From f4cecaade94456135f33057ec678542593a839bf Mon Sep 17 00:00:00 2001 From: Ashwin Bharambe Date: Mon, 18 Aug 2025 10:11:55 -0700 Subject: [PATCH 02/16] chore(ci): dont run llama stack server always (#3188) Sometimes the server has already been started (e.g., via docker). Just a convenience here so we can reuse this script more. --- scripts/integration-tests.sh | 43 ++++++++++++++++++++++-------------- 1 file changed, 26 insertions(+), 17 deletions(-) diff --git a/scripts/integration-tests.sh b/scripts/integration-tests.sh index bf7671348..e9a5283e1 100755 --- a/scripts/integration-tests.sh +++ b/scripts/integration-tests.sh @@ -133,6 +133,10 @@ else EXTRA_PARAMS="" fi +THIS_DIR=$(dirname "$0") +ROOT_DIR="$THIS_DIR/.." +cd $ROOT_DIR + # Set recording directory if [[ "$RUN_VISION_TESTS" == "true" ]]; then export LLAMA_STACK_TEST_RECORDING_DIR="tests/integration/recordings/vision" @@ -142,24 +146,29 @@ fi # Start Llama Stack Server if needed if [[ "$STACK_CONFIG" == *"server:"* ]]; then - echo "=== Starting Llama Stack Server ===" - nohup uv run llama stack run ci-tests --image-type venv > server.log 2>&1 & + # check if server is already running + if curl -s http://localhost:8321/v1/health 2>/dev/null | grep -q "OK"; then + echo "Llama Stack Server is already running, skipping start" + else + echo "=== Starting Llama Stack Server ===" + nohup uv run llama stack run ci-tests --image-type venv > server.log 2>&1 & - echo "Waiting for Llama Stack Server to start..." - for i in {1..30}; do - if curl -s http://localhost:8321/v1/health 2>/dev/null | grep -q "OK"; then - echo "✅ Llama Stack Server started successfully" - break - fi - if [[ $i -eq 30 ]]; then - echo "❌ Llama Stack Server failed to start" - echo "Server logs:" - cat server.log - exit 1 - fi - sleep 1 - done - echo "" + echo "Waiting for Llama Stack Server to start..." + for i in {1..30}; do + if curl -s http://localhost:8321/v1/health 2>/dev/null | grep -q "OK"; then + echo "✅ Llama Stack Server started successfully" + break + fi + if [[ $i -eq 30 ]]; then + echo "❌ Llama Stack Server failed to start" + echo "Server logs:" + cat server.log + exit 1 + fi + sleep 1 + done + echo "" + fi fi # Run tests From 4ae39b94ffcd3940f2dcc3aa9f50165b96ab39a8 Mon Sep 17 00:00:00 2001 From: Charlie Doern Date: Mon, 18 Aug 2025 13:23:23 -0400 Subject: [PATCH 03/16] fix: remove category prints (#3189) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit # What does this PR do? commands where the output is important like `llama stack build --print-deps-only` (soon to be `llama stack show`) print some log.py `cprint`'s on _every_ execution of the CLI for example: Screenshot 2025-08-18 at 1 16 30 PM the yellow text is likely unnecessary. Signed-off-by: Charlie Doern --- llama_stack/log.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/llama_stack/log.py b/llama_stack/log.py index 7507aface..d67bd1b61 100644 --- a/llama_stack/log.py +++ b/llama_stack/log.py @@ -7,13 +7,11 @@ import logging import os import re -import sys from logging.config import dictConfig from rich.console import Console from rich.errors import MarkupError from rich.logging import RichHandler -from termcolor import cprint from llama_stack.core.datatypes import LoggingConfig @@ -66,7 +64,6 @@ def config_to_category_levels(category: str, level: str): category_levels["root"] = level_value elif category in CATEGORIES: category_levels[category] = level_value - logging.info(f"Setting '{category}' category to level '{level}'.") else: logging.warning(f"Unknown logging category: {category}. No changes made.") return category_levels @@ -256,7 +253,6 @@ def get_logger( env_config = os.environ.get("LLAMA_STACK_LOGGING", "") if env_config: - cprint(f"Environment variable LLAMA_STACK_LOGGING found: {env_config}", color="yellow", file=sys.stderr) _category_levels.update(parse_environment_config(env_config)) log_file = os.environ.get("LLAMA_STACK_LOG_FILE") From fa431e15e02f550f0278ac7920d55bb432641759 Mon Sep 17 00:00:00 2001 From: Francisco Arceo Date: Mon, 18 Aug 2025 11:23:51 -0600 Subject: [PATCH 04/16] chore: Update TRIAGERS.md (#3186) # What does this PR do? Update triagers to current state ## Test Plan --- .github/TRIAGERS.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/TRIAGERS.md b/.github/TRIAGERS.md index ed4f4a6c6..f5bd11531 100644 --- a/.github/TRIAGERS.md +++ b/.github/TRIAGERS.md @@ -1,2 +1,2 @@ # This file documents Triage members in the Llama Stack community - @bbrowning @franciscojavierarceo @leseb + @franciscojavierarceo From 739b18edf8755390cd75b819852090de3be940c3 Mon Sep 17 00:00:00 2001 From: Maor Friedman Date: Mon, 18 Aug 2025 20:24:24 +0300 Subject: [PATCH 05/16] feat: add support for postgres ssl mode and root cert (#3182) this PR adds support for configuring `sslmode` and `sslrootcert` when initiating the psycopg2 connection. closes #3181 --- llama_stack/providers/utils/kvstore/config.py | 2 ++ llama_stack/providers/utils/kvstore/postgres/postgres.py | 2 ++ 2 files changed, 4 insertions(+) diff --git a/llama_stack/providers/utils/kvstore/config.py b/llama_stack/providers/utils/kvstore/config.py index f00cb1f8b..d1747d65b 100644 --- a/llama_stack/providers/utils/kvstore/config.py +++ b/llama_stack/providers/utils/kvstore/config.py @@ -75,6 +75,8 @@ class PostgresKVStoreConfig(CommonConfig): db: str = "llamastack" user: str password: str | None = None + ssl_mode: str | None = None + ca_cert_path: str | None = None table_name: str = "llamastack_kvstore" @classmethod diff --git a/llama_stack/providers/utils/kvstore/postgres/postgres.py b/llama_stack/providers/utils/kvstore/postgres/postgres.py index bd35decfc..cabb4c512 100644 --- a/llama_stack/providers/utils/kvstore/postgres/postgres.py +++ b/llama_stack/providers/utils/kvstore/postgres/postgres.py @@ -30,6 +30,8 @@ class PostgresKVStoreImpl(KVStore): database=self.config.db, user=self.config.user, password=self.config.password, + sslmode=self.config.ssl_mode, + sslrootcert=self.config.ca_cert_path, ) self.conn.autocommit = True self.cursor = self.conn.cursor(cursor_factory=DictCursor) From f8398d25ff312ce23afb21616a385de3e0a3d6da Mon Sep 17 00:00:00 2001 From: IAN MILLER <75687988+r3v5@users.noreply.github.com> Date: Mon, 18 Aug 2025 20:17:44 +0100 Subject: [PATCH 06/16] fix: kill build_conda_env.sh (#3190) # What does this PR do? I noticed somehow [build_conda_env.sh](https://github.com/llamastack/llama-stack/blob/main/llama_stack/core/build_conda_env.sh) exists in main branch. We need to kill it to be consistent with [#2969](https://github.com/llamastack/llama-stack/pull/2969) ## Test Plan --- llama_stack/core/build_conda_env.sh | 207 ---------------------------- 1 file changed, 207 deletions(-) delete mode 100755 llama_stack/core/build_conda_env.sh diff --git a/llama_stack/core/build_conda_env.sh b/llama_stack/core/build_conda_env.sh deleted file mode 100755 index 48ac3a1ab..000000000 --- a/llama_stack/core/build_conda_env.sh +++ /dev/null @@ -1,207 +0,0 @@ -#!/bin/bash - -# Copyright (c) Meta Platforms, Inc. and affiliates. -# All rights reserved. -# -# This source code is licensed under the terms described in the LICENSE file in -# the root directory of this source tree. - -LLAMA_STACK_DIR=${LLAMA_STACK_DIR:-} -LLAMA_STACK_CLIENT_DIR=${LLAMA_STACK_CLIENT_DIR:-} -TEST_PYPI_VERSION=${TEST_PYPI_VERSION:-} -PYPI_VERSION=${PYPI_VERSION:-} -# This timeout (in seconds) is necessary when installing PyTorch via uv since it's likely to time out -# Reference: https://github.com/astral-sh/uv/pull/1694 -UV_HTTP_TIMEOUT=${UV_HTTP_TIMEOUT:-500} - -set -euo pipefail - -# Define color codes -RED='\033[0;31m' -GREEN='\033[0;32m' -NC='\033[0m' # No Color - -SCRIPT_DIR=$(dirname "$(readlink -f "$0")") -source "$SCRIPT_DIR/common.sh" - -# Usage function -usage() { - echo "Usage: $0 --env-name --build-file-path --normal-deps [--external-provider-deps ] [--optional-deps ]" - echo "Example: $0 --env-name my-conda-env --build-file-path ./my-stack-build.yaml --normal-deps 'numpy pandas scipy' --external-provider-deps 'foo' --optional-deps 'bar'" - exit 1 -} - -# Parse arguments -env_name="" -build_file_path="" -normal_deps="" -external_provider_deps="" -optional_deps="" - -while [[ $# -gt 0 ]]; do - key="$1" - case "$key" in - --env-name) - if [[ -z "$2" || "$2" == --* ]]; then - echo "Error: --env-name requires a string value" >&2 - usage - fi - env_name="$2" - shift 2 - ;; - --build-file-path) - if [[ -z "$2" || "$2" == --* ]]; then - echo "Error: --build-file-path requires a string value" >&2 - usage - fi - build_file_path="$2" - shift 2 - ;; - --normal-deps) - if [[ -z "$2" || "$2" == --* ]]; then - echo "Error: --normal-deps requires a string value" >&2 - usage - fi - normal_deps="$2" - shift 2 - ;; - --external-provider-deps) - if [[ -z "$2" || "$2" == --* ]]; then - echo "Error: --external-provider-deps requires a string value" >&2 - usage - fi - external_provider_deps="$2" - shift 2 - ;; - --optional-deps) - if [[ -z "$2" || "$2" == --* ]]; then - echo "Error: --optional-deps requires a string value" >&2 - usage - fi - optional_deps="$2" - shift 2 - ;; - *) - echo "Unknown option: $1" >&2 - usage - ;; - esac -done - -# Check required arguments -if [[ -z "$env_name" || -z "$build_file_path" || -z "$normal_deps" ]]; then - echo "Error: --env-name, --build-file-path, and --normal-deps are required." >&2 - usage -fi - -if [ -n "$LLAMA_STACK_DIR" ]; then - echo "Using llama-stack-dir=$LLAMA_STACK_DIR" -fi -if [ -n "$LLAMA_STACK_CLIENT_DIR" ]; then - echo "Using llama-stack-client-dir=$LLAMA_STACK_CLIENT_DIR" -fi - -ensure_conda_env_python310() { - # Use only global variables set by flag parser - local python_version="3.12" - - if ! is_command_available conda; then - printf "${RED}Error: conda command not found. Is Conda installed and in your PATH?${NC}" >&2 - exit 1 - fi - - if conda env list | grep -q "^${env_name} "; then - printf "Conda environment '${env_name}' exists. Checking Python version...\n" - current_version=$(conda run -n "${env_name}" python --version 2>&1 | cut -d' ' -f2 | cut -d'.' -f1,2) - if [ "$current_version" = "$python_version" ]; then - printf "Environment '${env_name}' already has Python ${python_version}. No action needed.\n" - else - printf "Updating environment '${env_name}' to Python ${python_version}...\n" - conda install -n "${env_name}" python="${python_version}" -y - fi - else - printf "Conda environment '${env_name}' does not exist. Creating with Python ${python_version}...\n" - conda create -n "${env_name}" python="${python_version}" -y - fi - - eval "$(conda shell.bash hook)" - conda deactivate && conda activate "${env_name}" - "$CONDA_PREFIX"/bin/pip install uv - - if [ -n "$TEST_PYPI_VERSION" ]; then - uv pip install fastapi libcst - uv pip install --extra-index-url https://test.pypi.org/simple/ \ - llama-stack=="$TEST_PYPI_VERSION" \ - "$normal_deps" - if [ -n "$optional_deps" ]; then - IFS='#' read -ra parts <<<"$optional_deps" - for part in "${parts[@]}"; do - echo "$part" - uv pip install $part - done - fi - if [ -n "$external_provider_deps" ]; then - IFS='#' read -ra parts <<<"$external_provider_deps" - for part in "${parts[@]}"; do - echo "$part" - uv pip install "$part" - done - fi - else - if [ -n "$LLAMA_STACK_DIR" ]; then - if [ ! -d "$LLAMA_STACK_DIR" ]; then - printf "${RED}Warning: LLAMA_STACK_DIR is set but directory does not exist: $LLAMA_STACK_DIR${NC}\n" >&2 - exit 1 - fi - printf "Installing from LLAMA_STACK_DIR: $LLAMA_STACK_DIR\n" - uv pip install --no-cache-dir -e "$LLAMA_STACK_DIR" - else - PYPI_VERSION="${PYPI_VERSION:-}" - if [ -n "$PYPI_VERSION" ]; then - SPEC_VERSION="llama-stack==${PYPI_VERSION}" - else - SPEC_VERSION="llama-stack" - fi - uv pip install --no-cache-dir "$SPEC_VERSION" - fi - if [ -n "$LLAMA_STACK_CLIENT_DIR" ]; then - if [ ! -d "$LLAMA_STACK_CLIENT_DIR" ]; then - printf "${RED}Warning: LLAMA_STACK_CLIENT_DIR is set but directory does not exist: $LLAMA_STACK_CLIENT_DIR${NC}\n" >&2 - exit 1 - fi - printf "Installing from LLAMA_STACK_CLIENT_DIR: $LLAMA_STACK_CLIENT_DIR\n" - uv pip install --no-cache-dir -e "$LLAMA_STACK_CLIENT_DIR" - fi - printf "Installing pip dependencies\n" - uv pip install $normal_deps - if [ -n "$optional_deps" ]; then - IFS='#' read -ra parts <<<"$optional_deps" - for part in "${parts[@]}"; do - echo "$part" - uv pip install $part - done - fi - if [ -n "$external_provider_deps" ]; then - IFS='#' read -ra parts <<<"$external_provider_deps" - for part in "${parts[@]}"; do - echo "Getting provider spec for module: $part and installing dependencies" - package_name=$(echo "$part" | sed 's/[<>=!].*//') - python3 -c " -import importlib -import sys -try: - module = importlib.import_module(f'$package_name.provider') - spec = module.get_provider_spec() - if hasattr(spec, 'pip_packages') and spec.pip_packages: - print('\\n'.join(spec.pip_packages)) -except Exception as e: - print(f'Error getting provider spec for $package_name: {e}', file=sys.stderr) -" | uv pip install -r - - done - fi - fi - mv "$build_file_path" "$CONDA_PREFIX"/llamastack-build.yaml - echo "Build spec configuration saved at $CONDA_PREFIX/llamastack-build.yaml" -} - -ensure_conda_env_python310 "$env_name" "$build_file_path" "$normal_deps" "$optional_deps" "$external_provider_deps" From 27d6becfd0f2b3071fc650eee0ae8e15a7ae8115 Mon Sep 17 00:00:00 2001 From: Ashwin Bharambe Date: Mon, 18 Aug 2025 12:20:50 -0700 Subject: [PATCH 07/16] fix(misc): pin openai dependency to < 1.100.0 (#3192) This OpenAI client release https://github.com/openai/openai-python/commit/0843a1116498bc3312db9904adf71a4fb0a0a77e ends up breaking litellm https://github.com/BerriAI/litellm/blob/169a17400f1f5e36815c7d89128754975cd0584d/litellm/types/llms/openai.py#L40 Update the dependency pin. Also make the imports a bit more defensive anyhow if something else during `llama stack build` ends up moving openai to a previous version. ## Test Plan Run pre-release script integration tests. --- .../utils/inference/openai_compat.py | 12 +++++++++--- pyproject.toml | 2 +- scripts/integration-tests.sh | 19 ++++++++++++++++--- uv.lock | 2 +- 4 files changed, 27 insertions(+), 8 deletions(-) diff --git a/llama_stack/providers/utils/inference/openai_compat.py b/llama_stack/providers/utils/inference/openai_compat.py index 6297cc2ed..5e6c26884 100644 --- a/llama_stack/providers/utils/inference/openai_compat.py +++ b/llama_stack/providers/utils/inference/openai_compat.py @@ -31,9 +31,15 @@ from openai.types.chat import ( from openai.types.chat import ( ChatCompletionContentPartTextParam as OpenAIChatCompletionContentPartTextParam, ) -from openai.types.chat import ( - ChatCompletionMessageFunctionToolCall as OpenAIChatCompletionMessageFunctionToolCall, -) + +try: + from openai.types.chat import ( + ChatCompletionMessageFunctionToolCall as OpenAIChatCompletionMessageFunctionToolCall, + ) +except ImportError: + from openai.types.chat.chat_completion_message_tool_call import ( + ChatCompletionMessageToolCall as OpenAIChatCompletionMessageFunctionToolCall, + ) from openai.types.chat import ( ChatCompletionMessageParam as OpenAIChatCompletionMessage, ) diff --git a/pyproject.toml b/pyproject.toml index db0ad1f00..f02c02c41 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -33,7 +33,7 @@ dependencies = [ "jsonschema", "llama-stack-client>=0.2.17", "llama-api-client>=0.1.2", - "openai>=1.99.6", + "openai>=1.99.6,<1.100.0", "prompt-toolkit", "python-dotenv", "python-jose[cryptography]", diff --git a/scripts/integration-tests.sh b/scripts/integration-tests.sh index e9a5283e1..66e6d8e57 100755 --- a/scripts/integration-tests.sh +++ b/scripts/integration-tests.sh @@ -144,6 +144,19 @@ else export LLAMA_STACK_TEST_RECORDING_DIR="tests/integration/recordings" fi +# check if "llama" and "pytest" are available. this script does not use `uv run` given +# it can be used in a pre-release environment where we have not been able to tell +# uv about pre-release dependencies properly (yet). +if ! command -v llama &> /dev/null; then + echo "llama could not be found, ensure llama-stack is installed" + exit 1 +fi + +if ! command -v pytest &> /dev/null; then + echo "pytest could not be found, ensure pytest is installed" + exit 1 +fi + # Start Llama Stack Server if needed if [[ "$STACK_CONFIG" == *"server:"* ]]; then # check if server is already running @@ -151,7 +164,7 @@ if [[ "$STACK_CONFIG" == *"server:"* ]]; then echo "Llama Stack Server is already running, skipping start" else echo "=== Starting Llama Stack Server ===" - nohup uv run llama stack run ci-tests --image-type venv > server.log 2>&1 & + nohup llama stack run ci-tests --image-type venv > server.log 2>&1 & echo "Waiting for Llama Stack Server to start..." for i in {1..30}; do @@ -189,7 +202,7 @@ fi if [[ "$RUN_VISION_TESTS" == "true" ]]; then echo "Running vision tests..." set +e - uv run pytest -s -v tests/integration/inference/test_vision_inference.py \ + pytest -s -v tests/integration/inference/test_vision_inference.py \ --stack-config="$STACK_CONFIG" \ -k "$PYTEST_PATTERN" \ --vision-model=ollama/llama3.2-vision:11b \ @@ -257,7 +270,7 @@ echo "=== Running all collected tests in a single pytest command ===" echo "Total test files: $(echo $TEST_FILES | wc -w)" set +e -uv run pytest -s -v $TEST_FILES \ +pytest -s -v $TEST_FILES \ --stack-config="$STACK_CONFIG" \ -k "$PYTEST_PATTERN" \ --text-model="$TEXT_MODEL" \ diff --git a/uv.lock b/uv.lock index a09406770..3e3bf7e24 100644 --- a/uv.lock +++ b/uv.lock @@ -1856,7 +1856,7 @@ requires-dist = [ { name = "llama-api-client", specifier = ">=0.1.2" }, { name = "llama-stack-client", specifier = ">=0.2.17" }, { name = "llama-stack-client", marker = "extra == 'ui'", specifier = ">=0.2.17" }, - { name = "openai", specifier = ">=1.99.6" }, + { name = "openai", specifier = ">=1.99.6,<1.100.0" }, { name = "opentelemetry-exporter-otlp-proto-http", specifier = ">=1.30.0" }, { name = "opentelemetry-sdk", specifier = ">=1.30.0" }, { name = "pandas", marker = "extra == 'ui'" }, From 7519ab40247a9495147fb0ab5fc900aa71d0f23b Mon Sep 17 00:00:00 2001 From: slekkala1 Date: Mon, 18 Aug 2025 14:15:40 -0700 Subject: [PATCH 08/16] feat: Code scanner Provider impl for moderations api (#3100) # What does this PR do? Add CodeScanner implementations ## Test Plan `SAFETY_MODEL=CodeScanner LLAMA_STACK_CONFIG=starter uv run pytest -v tests/integration/safety/test_safety.py --text-model=llama3.2:3b-instruct-fp16 --embedding-model=all-MiniLM-L6-v2 --safety-shield=ollama` This PR need to land after this https://github.com/meta-llama/llama-stack/pull/3098 --- llama_stack/core/routers/safety.py | 5 +- llama_stack/distributions/ci-tests/build.yaml | 1 + llama_stack/distributions/ci-tests/run.yaml | 5 ++ llama_stack/distributions/starter/build.yaml | 1 + llama_stack/distributions/starter/run.yaml | 5 ++ llama_stack/distributions/starter/starter.py | 19 +++--- .../safety/code_scanner/code_scanner.py | 63 ++++++++++++++++++- .../inline/safety/llama_guard/llama_guard.py | 9 +-- tests/integration/safety/test_safety.py | 60 +++++++++++++++++- 9 files changed, 144 insertions(+), 24 deletions(-) diff --git a/llama_stack/core/routers/safety.py b/llama_stack/core/routers/safety.py index c76673d2a..738ecded3 100644 --- a/llama_stack/core/routers/safety.py +++ b/llama_stack/core/routers/safety.py @@ -6,9 +6,7 @@ from typing import Any -from llama_stack.apis.inference import ( - Message, -) +from llama_stack.apis.inference import Message from llama_stack.apis.safety import RunShieldResponse, Safety from llama_stack.apis.safety.safety import ModerationObject from llama_stack.apis.shields import Shield @@ -68,6 +66,7 @@ class SafetyRouter(Safety): list_shields_response = await self.routing_table.list_shields() matches = [s.identifier for s in list_shields_response.data if model == s.provider_resource_id] + if not matches: raise ValueError(f"No shield associated with provider_resource id {model}") if len(matches) > 1: diff --git a/llama_stack/distributions/ci-tests/build.yaml b/llama_stack/distributions/ci-tests/build.yaml index 676ed18d2..0bf42e7ee 100644 --- a/llama_stack/distributions/ci-tests/build.yaml +++ b/llama_stack/distributions/ci-tests/build.yaml @@ -28,6 +28,7 @@ distribution_spec: - provider_type: inline::localfs safety: - provider_type: inline::llama-guard + - provider_type: inline::code-scanner agents: - provider_type: inline::meta-reference telemetry: diff --git a/llama_stack/distributions/ci-tests/run.yaml b/llama_stack/distributions/ci-tests/run.yaml index dd4e04e50..02a268462 100644 --- a/llama_stack/distributions/ci-tests/run.yaml +++ b/llama_stack/distributions/ci-tests/run.yaml @@ -135,6 +135,8 @@ providers: provider_type: inline::llama-guard config: excluded_categories: [] + - provider_id: code-scanner + provider_type: inline::code-scanner agents: - provider_id: meta-reference provider_type: inline::meta-reference @@ -223,6 +225,9 @@ shields: - shield_id: llama-guard provider_id: ${env.SAFETY_MODEL:+llama-guard} provider_shield_id: ${env.SAFETY_MODEL:=} +- shield_id: code-scanner + provider_id: ${env.CODE_SCANNER_MODEL:+code-scanner} + provider_shield_id: ${env.CODE_SCANNER_MODEL:=} vector_dbs: [] datasets: [] scoring_fns: [] diff --git a/llama_stack/distributions/starter/build.yaml b/llama_stack/distributions/starter/build.yaml index 549bb4529..2ad12a165 100644 --- a/llama_stack/distributions/starter/build.yaml +++ b/llama_stack/distributions/starter/build.yaml @@ -28,6 +28,7 @@ distribution_spec: - provider_type: inline::localfs safety: - provider_type: inline::llama-guard + - provider_type: inline::code-scanner agents: - provider_type: inline::meta-reference telemetry: diff --git a/llama_stack/distributions/starter/run.yaml b/llama_stack/distributions/starter/run.yaml index d64c275cb..7ac4dc6b9 100644 --- a/llama_stack/distributions/starter/run.yaml +++ b/llama_stack/distributions/starter/run.yaml @@ -135,6 +135,8 @@ providers: provider_type: inline::llama-guard config: excluded_categories: [] + - provider_id: code-scanner + provider_type: inline::code-scanner agents: - provider_id: meta-reference provider_type: inline::meta-reference @@ -223,6 +225,9 @@ shields: - shield_id: llama-guard provider_id: ${env.SAFETY_MODEL:+llama-guard} provider_shield_id: ${env.SAFETY_MODEL:=} +- shield_id: code-scanner + provider_id: ${env.CODE_SCANNER_MODEL:+code-scanner} + provider_shield_id: ${env.CODE_SCANNER_MODEL:=} vector_dbs: [] datasets: [] scoring_fns: [] diff --git a/llama_stack/distributions/starter/starter.py b/llama_stack/distributions/starter/starter.py index 498a12080..cad3d72d9 100644 --- a/llama_stack/distributions/starter/starter.py +++ b/llama_stack/distributions/starter/starter.py @@ -15,19 +15,14 @@ from llama_stack.core.datatypes import ( ToolGroupInput, ) from llama_stack.core.utils.dynamic import instantiate_class_type -from llama_stack.distributions.template import ( - DistributionTemplate, - RunConfigSettings, -) +from llama_stack.distributions.template import DistributionTemplate, RunConfigSettings from llama_stack.providers.datatypes import RemoteProviderSpec from llama_stack.providers.inline.files.localfs.config import LocalfsFilesImplConfig from llama_stack.providers.inline.inference.sentence_transformers import ( SentenceTransformersInferenceConfig, ) from llama_stack.providers.inline.vector_io.faiss.config import FaissVectorIOConfig -from llama_stack.providers.inline.vector_io.milvus.config import ( - MilvusVectorIOConfig, -) +from llama_stack.providers.inline.vector_io.milvus.config import MilvusVectorIOConfig from llama_stack.providers.inline.vector_io.sqlite_vec.config import ( SQLiteVectorIOConfig, ) @@ -119,7 +114,10 @@ def get_distribution_template() -> DistributionTemplate: BuildProvider(provider_type="remote::pgvector"), ], "files": [BuildProvider(provider_type="inline::localfs")], - "safety": [BuildProvider(provider_type="inline::llama-guard")], + "safety": [ + BuildProvider(provider_type="inline::llama-guard"), + BuildProvider(provider_type="inline::code-scanner"), + ], "agents": [BuildProvider(provider_type="inline::meta-reference")], "telemetry": [BuildProvider(provider_type="inline::meta-reference")], "post_training": [BuildProvider(provider_type="inline::huggingface")], @@ -170,6 +168,11 @@ def get_distribution_template() -> DistributionTemplate: provider_id="${env.SAFETY_MODEL:+llama-guard}", provider_shield_id="${env.SAFETY_MODEL:=}", ), + ShieldInput( + shield_id="code-scanner", + provider_id="${env.CODE_SCANNER_MODEL:+code-scanner}", + provider_shield_id="${env.CODE_SCANNER_MODEL:=}", + ), ] return DistributionTemplate( diff --git a/llama_stack/providers/inline/safety/code_scanner/code_scanner.py b/llama_stack/providers/inline/safety/code_scanner/code_scanner.py index be05ee436..6e05d5b83 100644 --- a/llama_stack/providers/inline/safety/code_scanner/code_scanner.py +++ b/llama_stack/providers/inline/safety/code_scanner/code_scanner.py @@ -5,7 +5,11 @@ # the root directory of this source tree. import logging -from typing import Any +import uuid +from typing import TYPE_CHECKING, Any + +if TYPE_CHECKING: + from codeshield.cs import CodeShieldScanResult from llama_stack.apis.inference import Message from llama_stack.apis.safety import ( @@ -14,6 +18,7 @@ from llama_stack.apis.safety import ( SafetyViolation, ViolationLevel, ) +from llama_stack.apis.safety.safety import ModerationObject, ModerationObjectResults from llama_stack.apis.shields import Shield from llama_stack.providers.utils.inference.prompt_adapter import ( interleaved_content_as_str, @@ -24,8 +29,8 @@ from .config import CodeScannerConfig log = logging.getLogger(__name__) ALLOWED_CODE_SCANNER_MODEL_IDS = [ - "CodeScanner", - "CodeShield", + "code-scanner", + "code-shield", ] @@ -69,3 +74,55 @@ class MetaReferenceCodeScannerSafetyImpl(Safety): metadata={"violation_type": ",".join([issue.pattern_id for issue in result.issues_found])}, ) return RunShieldResponse(violation=violation) + + def get_moderation_object_results(self, scan_result: "CodeShieldScanResult") -> ModerationObjectResults: + categories = {} + category_scores = {} + category_applied_input_types = {} + + flagged = scan_result.is_insecure + user_message = None + metadata = {} + + if scan_result.is_insecure: + pattern_ids = [issue.pattern_id for issue in scan_result.issues_found] + categories = dict.fromkeys(pattern_ids, True) + category_scores = dict.fromkeys(pattern_ids, 1.0) + category_applied_input_types = {key: ["text"] for key in pattern_ids} + user_message = f"Security concerns detected in the code. {scan_result.recommended_treatment.name}: {', '.join([issue.description for issue in scan_result.issues_found])}" + metadata = {"violation_type": ",".join([issue.pattern_id for issue in scan_result.issues_found])} + + return ModerationObjectResults( + flagged=flagged, + categories=categories, + category_scores=category_scores, + category_applied_input_types=category_applied_input_types, + user_message=user_message, + metadata=metadata, + ) + + async def run_moderation(self, input: str | list[str], model: str) -> ModerationObject: + inputs = input if isinstance(input, list) else [input] + results = [] + + from codeshield.cs import CodeShield + + for text_input in inputs: + log.info(f"Running CodeScannerShield moderation on input: {text_input[:100]}...") + try: + scan_result = await CodeShield.scan_code(text_input) + moderation_result = self.get_moderation_object_results(scan_result) + except Exception as e: + log.error(f"CodeShield.scan_code failed: {e}") + # create safe fallback response on scanner failure to avoid blocking legitimate requests + moderation_result = ModerationObjectResults( + flagged=False, + categories={}, + category_scores={}, + category_applied_input_types={}, + user_message=None, + metadata={"scanner_error": str(e)}, + ) + results.append(moderation_result) + + return ModerationObject(id=str(uuid.uuid4()), model=model, results=results) diff --git a/llama_stack/providers/inline/safety/llama_guard/llama_guard.py b/llama_stack/providers/inline/safety/llama_guard/llama_guard.py index bae744010..5d52c5d89 100644 --- a/llama_stack/providers/inline/safety/llama_guard/llama_guard.py +++ b/llama_stack/providers/inline/safety/llama_guard/llama_guard.py @@ -11,11 +11,7 @@ from string import Template from typing import Any from llama_stack.apis.common.content_types import ImageContentItem, TextContentItem -from llama_stack.apis.inference import ( - Inference, - Message, - UserMessage, -) +from llama_stack.apis.inference import Inference, Message, UserMessage from llama_stack.apis.safety import ( RunShieldResponse, Safety, @@ -72,7 +68,6 @@ SAFETY_CATEGORIES_TO_CODE_MAP = { } SAFETY_CODE_TO_CATEGORIES_MAP = {v: k for k, v in SAFETY_CATEGORIES_TO_CODE_MAP.items()} - DEFAULT_LG_V3_SAFETY_CATEGORIES = [ CAT_VIOLENT_CRIMES, CAT_NON_VIOLENT_CRIMES, @@ -460,7 +455,7 @@ class LlamaGuardShield: def is_content_safe(self, response: str, unsafe_code: str | None = None) -> bool: """Check if content is safe based on response and unsafe code.""" - if response.strip() == SAFE_RESPONSE: + if response.strip().lower().startswith(SAFE_RESPONSE): return True if unsafe_code: diff --git a/tests/integration/safety/test_safety.py b/tests/integration/safety/test_safety.py index 75b974926..a21d9cbdf 100644 --- a/tests/integration/safety/test_safety.py +++ b/tests/integration/safety/test_safety.py @@ -27,10 +27,10 @@ def data_url_from_image(file_path): @pytest.fixture(scope="session") def code_scanner_shield_id(available_shields): - if "CodeScanner" in available_shields: - return "CodeScanner" + if "code-scanner" in available_shields: + return "code-scanner" - pytest.skip("CodeScanner shield is not available. Skipping.") + pytest.skip("code-scanner shield is not available. Skipping.") def test_unsafe_examples(client_with_models, shield_id): @@ -163,6 +163,60 @@ def test_safety_with_code_scanner(client_with_models, code_scanner_shield_id, mo assert response.violation.user_message == "Sorry, I found security concerns in the code." +def test_unsafe_input_with_code_scanner_run_moderation(client_with_models, code_scanner_shield_id, model_providers): + code_safety_enabled = len(CODE_SCANNER_ENABLED_PROVIDERS & model_providers) > 0 + if not code_safety_enabled: + pytest.skip(f"Testing code scanner shields is not supported for model_providers {model_providers}") + + insecure_code = [ + """ + Here is an example of code that hashes a password using MD5: + + ``` + import hashlib + password = 'securepassword123'.encode() + hashed_password = hashlib.md5(password).hexdigest() + print("Insecure hashed password:", hashed_password) + ``` + """ + ] + moderation_object = client_with_models.moderations.create( + input=insecure_code, + model=code_scanner_shield_id, + ) + assert moderation_object.results[0].flagged is True, f"Code scanner should have flagged {insecure_code} as insecure" + assert all(value is True for value in moderation_object.results[0].categories.values()), ( + "Code scanner shield should have detected code insecure category" + ) + + +def test_safe_input_with_code_scanner_run_moderation(client_with_models, code_scanner_shield_id, model_providers): + code_safety_enabled = len(CODE_SCANNER_ENABLED_PROVIDERS & model_providers) > 0 + if not code_safety_enabled: + pytest.skip(f"Testing code scanner shields is not supported for model_providers {model_providers}") + + secure_code = [ + """ + Extract the first 5 characters from a string: + ``` + text = "Hello World" + first_five = text[:5] + print(first_five) # Output: "Hello" + + # Safe handling for strings shorter than 5 characters + def get_first_five(text): + return text[:5] if text else "" + ``` + """ + ] + moderation_object = client_with_models.moderations.create( + input=secure_code, + model=code_scanner_shield_id, + ) + + assert moderation_object.results[0].flagged is False, "Code scanner should not have flagged the code as insecure" + + # We can use an instance of the LlamaGuard shield to detect attempts to misuse # the interpreter as this is one of the existing categories it checks for def test_safety_with_code_interpreter_abuse(client_with_models, shield_id): From 2e7ca0742357eddfc9b4738989aaebbd9bbde52b Mon Sep 17 00:00:00 2001 From: Ashwin Bharambe Date: Mon, 18 Aug 2025 14:58:23 -0700 Subject: [PATCH 09/16] feat(cli): make venv the default image type (#3187) We have removed conda now so we can make `venv` the default. Just doing `llama stack build --distro starter` is now enough for the most part. --- llama_stack/cli/stack/_build.py | 20 ++------------------ llama_stack/cli/stack/build.py | 2 +- 2 files changed, 3 insertions(+), 19 deletions(-) diff --git a/llama_stack/cli/stack/_build.py b/llama_stack/cli/stack/_build.py index c6e204773..b4ada33e2 100644 --- a/llama_stack/cli/stack/_build.py +++ b/llama_stack/cli/stack/_build.py @@ -92,15 +92,7 @@ def run_stack_build_command(args: argparse.Namespace) -> None: ) sys.exit(1) build_config = available_distros[distro_name] - if args.image_type: - build_config.image_type = args.image_type - else: - cprint( - f"Please specify a image-type ({' | '.join(e.value for e in ImageType)}) for {distro_name}", - color="red", - file=sys.stderr, - ) - sys.exit(1) + build_config.image_type = args.image_type elif args.providers: provider_list: dict[str, list[BuildProvider]] = dict() for api_provider in args.providers.split(","): @@ -137,13 +129,6 @@ def run_stack_build_command(args: argparse.Namespace) -> None: providers=provider_list, description=",".join(args.providers), ) - if not args.image_type: - cprint( - f"Please specify a image-type (container | venv) for {args.template}", - color="red", - file=sys.stderr, - ) - sys.exit(1) build_config = BuildConfig(image_type=args.image_type, distribution_spec=distribution_spec) elif not args.config and not distro_name: @@ -217,8 +202,7 @@ def run_stack_build_command(args: argparse.Namespace) -> None: contents = yaml.safe_load(f) contents = replace_env_vars(contents) build_config = BuildConfig(**contents) - if args.image_type: - build_config.image_type = args.image_type + build_config.image_type = args.image_type except Exception as e: cprint( f"Could not parse config file {args.config}: {e}", diff --git a/llama_stack/cli/stack/build.py b/llama_stack/cli/stack/build.py index 80cf6fb38..098577c9e 100644 --- a/llama_stack/cli/stack/build.py +++ b/llama_stack/cli/stack/build.py @@ -59,7 +59,7 @@ class StackBuild(Subcommand): type=str, help="Image Type to use for the build. If not specified, will use the image type from the template config.", choices=[e.value for e in ImageType], - default=None, # no default so we can detect if a user specified --image-type and override image_type in the config + default=ImageType.VENV.value, ) self.parser.add_argument( From 89661b984c55e1070b8ab88efd404c869c5e9ccc Mon Sep 17 00:00:00 2001 From: Ashwin Bharambe Date: Mon, 18 Aug 2025 15:31:01 -0700 Subject: [PATCH 10/16] revert: "feat(cli): make venv the default image type" (#3196) Reverts llamastack/llama-stack#3187 --- llama_stack/cli/stack/_build.py | 20 ++++++++++++++++++-- llama_stack/cli/stack/build.py | 2 +- 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/llama_stack/cli/stack/_build.py b/llama_stack/cli/stack/_build.py index b4ada33e2..c6e204773 100644 --- a/llama_stack/cli/stack/_build.py +++ b/llama_stack/cli/stack/_build.py @@ -92,7 +92,15 @@ def run_stack_build_command(args: argparse.Namespace) -> None: ) sys.exit(1) build_config = available_distros[distro_name] - build_config.image_type = args.image_type + if args.image_type: + build_config.image_type = args.image_type + else: + cprint( + f"Please specify a image-type ({' | '.join(e.value for e in ImageType)}) for {distro_name}", + color="red", + file=sys.stderr, + ) + sys.exit(1) elif args.providers: provider_list: dict[str, list[BuildProvider]] = dict() for api_provider in args.providers.split(","): @@ -129,6 +137,13 @@ def run_stack_build_command(args: argparse.Namespace) -> None: providers=provider_list, description=",".join(args.providers), ) + if not args.image_type: + cprint( + f"Please specify a image-type (container | venv) for {args.template}", + color="red", + file=sys.stderr, + ) + sys.exit(1) build_config = BuildConfig(image_type=args.image_type, distribution_spec=distribution_spec) elif not args.config and not distro_name: @@ -202,7 +217,8 @@ def run_stack_build_command(args: argparse.Namespace) -> None: contents = yaml.safe_load(f) contents = replace_env_vars(contents) build_config = BuildConfig(**contents) - build_config.image_type = args.image_type + if args.image_type: + build_config.image_type = args.image_type except Exception as e: cprint( f"Could not parse config file {args.config}: {e}", diff --git a/llama_stack/cli/stack/build.py b/llama_stack/cli/stack/build.py index 098577c9e..80cf6fb38 100644 --- a/llama_stack/cli/stack/build.py +++ b/llama_stack/cli/stack/build.py @@ -59,7 +59,7 @@ class StackBuild(Subcommand): type=str, help="Image Type to use for the build. If not specified, will use the image type from the template config.", choices=[e.value for e in ImageType], - default=ImageType.VENV.value, + default=None, # no default so we can detect if a user specified --image-type and override image_type in the config ) self.parser.add_argument( From ac78e9f66a3d5fbfb81b6e61ad9b5a0d5d7e85a7 Mon Sep 17 00:00:00 2001 From: Francisco Arceo Date: Mon, 18 Aug 2025 16:48:21 -0600 Subject: [PATCH 11/16] chore: Adding UI unit tests in CI (#3191) --- .github/workflows/README.md | 1 + .github/workflows/integration-auth-tests.yml | 1 + .github/workflows/integration-tests.yml | 1 + .../workflows/integration-vector-io-tests.yml | 1 + .github/workflows/python-build-test.yml | 2 + .github/workflows/test-external.yml | 1 + .github/workflows/ui-unit-tests.yml | 55 ++ .github/workflows/unit-tests.yml | 1 + .../contents/[contentId]/page.test.tsx | 425 ++++++++++++++++ .../files/[fileId]/contents/page.test.tsx | 481 ++++++++++++++++++ .../[id]/files/[fileId]/contents/page.tsx | 10 +- .../[id]/files/[fileId]/page.test.tsx | 458 +++++++++++++++++ .../chat-playground/markdown-renderer.tsx | 1 + .../vector-store-detail.test.tsx | 315 ++++++++++++ 14 files changed, 1752 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/ui-unit-tests.yml create mode 100644 llama_stack/ui/app/logs/vector-stores/[id]/files/[fileId]/contents/[contentId]/page.test.tsx create mode 100644 llama_stack/ui/app/logs/vector-stores/[id]/files/[fileId]/contents/page.test.tsx create mode 100644 llama_stack/ui/app/logs/vector-stores/[id]/files/[fileId]/page.test.tsx create mode 100644 llama_stack/ui/components/vector-stores/vector-store-detail.test.tsx diff --git a/.github/workflows/README.md b/.github/workflows/README.md index 3c3d93dc2..8344d12a4 100644 --- a/.github/workflows/README.md +++ b/.github/workflows/README.md @@ -18,5 +18,6 @@ Llama Stack uses GitHub Actions for Continuous Integration (CI). Below is a tabl | Close stale issues and PRs | [stale_bot.yml](stale_bot.yml) | Run the Stale Bot action | | Test External Providers Installed via Module | [test-external-provider-module.yml](test-external-provider-module.yml) | Test External Provider installation via Python module | | Test External API and Providers | [test-external.yml](test-external.yml) | Test the External API and Provider mechanisms | +| UI Tests | [ui-unit-tests.yml](ui-unit-tests.yml) | Run the UI test suite | | Unit Tests | [unit-tests.yml](unit-tests.yml) | Run the unit test suite | | Update ReadTheDocs | [update-readthedocs.yml](update-readthedocs.yml) | Update the Llama Stack ReadTheDocs site | diff --git a/.github/workflows/integration-auth-tests.yml b/.github/workflows/integration-auth-tests.yml index ef2066497..c328e3b6c 100644 --- a/.github/workflows/integration-auth-tests.yml +++ b/.github/workflows/integration-auth-tests.yml @@ -10,6 +10,7 @@ on: paths: - 'distributions/**' - 'llama_stack/**' + - '!llama_stack/ui/**' - 'tests/integration/**' - 'uv.lock' - 'pyproject.toml' diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index fc56f62ea..ba18c27c8 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -10,6 +10,7 @@ on: types: [opened, synchronize, reopened] paths: - 'llama_stack/**' + - '!llama_stack/ui/**' - 'tests/**' - 'uv.lock' - 'pyproject.toml' diff --git a/.github/workflows/integration-vector-io-tests.yml b/.github/workflows/integration-vector-io-tests.yml index 99a44c147..10deb1740 100644 --- a/.github/workflows/integration-vector-io-tests.yml +++ b/.github/workflows/integration-vector-io-tests.yml @@ -9,6 +9,7 @@ on: branches: [ main ] paths: - 'llama_stack/**' + - '!llama_stack/ui/**' - 'tests/integration/vector_io/**' - 'uv.lock' - 'pyproject.toml' diff --git a/.github/workflows/python-build-test.yml b/.github/workflows/python-build-test.yml index 67dc49cce..fe1dfd58a 100644 --- a/.github/workflows/python-build-test.yml +++ b/.github/workflows/python-build-test.yml @@ -9,6 +9,8 @@ on: pull_request: branches: - main + paths-ignore: + - 'llama_stack/ui/**' jobs: build: diff --git a/.github/workflows/test-external.yml b/.github/workflows/test-external.yml index 27181a236..5ec9ef257 100644 --- a/.github/workflows/test-external.yml +++ b/.github/workflows/test-external.yml @@ -9,6 +9,7 @@ on: branches: [ main ] paths: - 'llama_stack/**' + - '!llama_stack/ui/**' - 'tests/integration/**' - 'uv.lock' - 'pyproject.toml' diff --git a/.github/workflows/ui-unit-tests.yml b/.github/workflows/ui-unit-tests.yml new file mode 100644 index 000000000..00c539c58 --- /dev/null +++ b/.github/workflows/ui-unit-tests.yml @@ -0,0 +1,55 @@ +name: UI Tests + +run-name: Run the UI test suite + +on: + push: + branches: [ main ] + pull_request: + branches: [ main ] + paths: + - 'llama_stack/ui/**' + - '.github/workflows/ui-unit-tests.yml' # This workflow + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + ui-tests: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + node-version: [22] + + steps: + - name: Checkout repository + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + + - name: Setup Node.js + uses: actions/setup-node@39370e3970a6d050c480ffad4ff0ed4d3fdee5af # v4.1.0 + with: + node-version: ${{ matrix.node-version }} + cache: 'npm' + cache-dependency-path: 'llama_stack/ui/package-lock.json' + + - name: Install dependencies + working-directory: llama_stack/ui + run: npm ci + + - name: Run linting + working-directory: llama_stack/ui + run: npm run lint + + - name: Run format check + working-directory: llama_stack/ui + run: npm run format:check + + - name: Run unit tests + working-directory: llama_stack/ui + env: + CI: true + + run: npm test -- --coverage --watchAll=false --passWithNoTests diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index b133511d1..f2a6c7754 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -9,6 +9,7 @@ on: branches: [ main ] paths: - 'llama_stack/**' + - '!llama_stack/ui/**' - 'tests/unit/**' - 'uv.lock' - 'pyproject.toml' diff --git a/llama_stack/ui/app/logs/vector-stores/[id]/files/[fileId]/contents/[contentId]/page.test.tsx b/llama_stack/ui/app/logs/vector-stores/[id]/files/[fileId]/contents/[contentId]/page.test.tsx new file mode 100644 index 000000000..946ea9267 --- /dev/null +++ b/llama_stack/ui/app/logs/vector-stores/[id]/files/[fileId]/contents/[contentId]/page.test.tsx @@ -0,0 +1,425 @@ +import React from "react"; +import { render, screen, fireEvent, waitFor } from "@testing-library/react"; +import "@testing-library/jest-dom"; +import ContentDetailPage from "./page"; +import { VectorStoreContentItem } from "@/lib/contents-api"; +import type { VectorStore } from "llama-stack-client/resources/vector-stores/vector-stores"; +import type { VectorStoreFile } from "llama-stack-client/resources/vector-stores/files"; + +const mockPush = jest.fn(); +const mockParams = { + id: "vs_123", + fileId: "file_456", + contentId: "content_789", +}; + +jest.mock("next/navigation", () => ({ + useParams: () => mockParams, + useRouter: () => ({ + push: mockPush, + }), +})); + +const mockClient = { + vectorStores: { + retrieve: jest.fn(), + files: { + retrieve: jest.fn(), + }, + }, +}; + +jest.mock("@/hooks/use-auth-client", () => ({ + useAuthClient: () => mockClient, +})); + +const mockContentsAPI = { + listContents: jest.fn(), + updateContent: jest.fn(), + deleteContent: jest.fn(), +}; + +jest.mock("@/lib/contents-api", () => ({ + ContentsAPI: jest.fn(() => mockContentsAPI), +})); + +const originalConfirm = window.confirm; + +describe("ContentDetailPage", () => { + const mockStore: VectorStore = { + id: "vs_123", + name: "Test Vector Store", + created_at: 1710000000, + status: "ready", + file_counts: { total: 5 }, + usage_bytes: 1024, + metadata: { + provider_id: "test_provider", + }, + }; + + const mockFile: VectorStoreFile = { + id: "file_456", + status: "completed", + created_at: 1710001000, + usage_bytes: 512, + chunking_strategy: { type: "fixed_size" }, + }; + + const mockContent: VectorStoreContentItem = { + id: "content_789", + object: "vector_store.content", + content: "This is test content for the vector store.", + embedding: [0.1, 0.2, 0.3, 0.4, 0.5], + metadata: { + chunk_window: "0-45", + content_length: 45, + custom_field: "custom_value", + }, + created_timestamp: 1710002000, + }; + + beforeEach(() => { + jest.clearAllMocks(); + window.confirm = jest.fn(); + + mockClient.vectorStores.retrieve.mockResolvedValue(mockStore); + mockClient.vectorStores.files.retrieve.mockResolvedValue(mockFile); + mockContentsAPI.listContents.mockResolvedValue({ + data: [mockContent], + }); + }); + + afterEach(() => { + window.confirm = originalConfirm; + }); + + describe("Loading and Error States", () => { + test("renders loading skeleton while fetching data", () => { + mockClient.vectorStores.retrieve.mockImplementation( + () => new Promise(() => {}) + ); + + const { container } = render(); + + const skeletons = container.querySelectorAll('[data-slot="skeleton"]'); + expect(skeletons.length).toBeGreaterThan(0); + }); + + test("renders error message when API calls fail", async () => { + const error = new Error("Network error"); + mockClient.vectorStores.retrieve.mockRejectedValue(error); + + render(); + + await waitFor(() => { + expect( + screen.getByText(/Error loading details for ID content_789/) + ).toBeInTheDocument(); + expect(screen.getByText(/Network error/)).toBeInTheDocument(); + }); + }); + + test("renders not found when content doesn't exist", async () => { + mockContentsAPI.listContents.mockResolvedValue({ + data: [], + }); + + render(); + + await waitFor(() => { + expect( + screen.getByText(/Content content_789 not found/) + ).toBeInTheDocument(); + }); + }); + }); + + describe("Content Display", () => { + test("renders content details correctly", async () => { + render(); + + await waitFor(() => { + expect(screen.getByText("Content: content_789")).toBeInTheDocument(); + expect( + screen.getByText("This is test content for the vector store.") + ).toBeInTheDocument(); + }); + + const contentIdTexts = screen.getAllByText("content_789"); + expect(contentIdTexts.length).toBeGreaterThan(0); + const fileIdTexts = screen.getAllByText("file_456"); + expect(fileIdTexts.length).toBeGreaterThan(0); + const storeIdTexts = screen.getAllByText("vs_123"); + expect(storeIdTexts.length).toBeGreaterThan(0); + expect(screen.getByText("vector_store.content")).toBeInTheDocument(); + const positionTexts = screen.getAllByText("0-45"); + expect(positionTexts.length).toBeGreaterThan(0); + }); + + test("renders embedding information when available", async () => { + render(); + + await waitFor(() => { + expect( + screen.getByText(/0.100000, 0.200000, 0.300000/) + ).toBeInTheDocument(); + }); + }); + + test("handles content without embedding", async () => { + const contentWithoutEmbedding = { + ...mockContent, + embedding: undefined, + }; + + mockContentsAPI.listContents.mockResolvedValue({ + data: [contentWithoutEmbedding], + }); + + render(); + + await waitFor(() => { + expect( + screen.getByText("No embedding available for this content.") + ).toBeInTheDocument(); + }); + }); + + test("renders metadata correctly", async () => { + render(); + + await waitFor(() => { + expect(screen.getByText("chunk_window:")).toBeInTheDocument(); + const positionTexts = screen.getAllByText("0-45"); + expect(positionTexts.length).toBeGreaterThan(0); + expect(screen.getByText("content_length:")).toBeInTheDocument(); + expect(screen.getByText("custom_field:")).toBeInTheDocument(); + expect(screen.getByText("custom_value")).toBeInTheDocument(); + }); + }); + }); + + describe("Edit Functionality", () => { + test("enables edit mode when edit button is clicked", async () => { + render(); + + await waitFor(() => { + expect( + screen.getByText("This is test content for the vector store.") + ).toBeInTheDocument(); + }); + + const editButtons = screen.getAllByRole("button", { name: /Edit/ }); + const editButton = editButtons[0]; + fireEvent.click(editButton); + + expect( + screen.getByDisplayValue("This is test content for the vector store.") + ).toBeInTheDocument(); + expect(screen.getByRole("button", { name: /Save/ })).toBeInTheDocument(); + expect( + screen.getByRole("button", { name: /Cancel/ }) + ).toBeInTheDocument(); + }); + + test("cancels edit mode and resets content", async () => { + render(); + + await waitFor(() => { + expect( + screen.getByText("This is test content for the vector store.") + ).toBeInTheDocument(); + }); + + const editButtons = screen.getAllByRole("button", { name: /Edit/ }); + const editButton = editButtons[0]; + fireEvent.click(editButton); + + const textarea = screen.getByDisplayValue( + "This is test content for the vector store." + ); + fireEvent.change(textarea, { target: { value: "Modified content" } }); + + const cancelButton = screen.getByRole("button", { name: /Cancel/ }); + fireEvent.click(cancelButton); + + expect( + screen.getByText("This is test content for the vector store.") + ).toBeInTheDocument(); + expect( + screen.queryByDisplayValue("Modified content") + ).not.toBeInTheDocument(); + }); + + test("saves content changes", async () => { + const updatedContent = { ...mockContent, content: "Updated content" }; + mockContentsAPI.updateContent.mockResolvedValue(updatedContent); + + render(); + + await waitFor(() => { + expect( + screen.getByText("This is test content for the vector store.") + ).toBeInTheDocument(); + }); + + const editButtons = screen.getAllByRole("button", { name: /Edit/ }); + const editButton = editButtons[0]; + fireEvent.click(editButton); + + const textarea = screen.getByDisplayValue( + "This is test content for the vector store." + ); + fireEvent.change(textarea, { target: { value: "Updated content" } }); + + const saveButton = screen.getByRole("button", { name: /Save/ }); + fireEvent.click(saveButton); + + await waitFor(() => { + expect(mockContentsAPI.updateContent).toHaveBeenCalledWith( + "vs_123", + "file_456", + "content_789", + { content: "Updated content" } + ); + }); + }); + }); + + describe("Delete Functionality", () => { + test("shows confirmation dialog before deleting", async () => { + window.confirm = jest.fn().mockReturnValue(false); + + render(); + + await waitFor(() => { + expect( + screen.getByText("This is test content for the vector store.") + ).toBeInTheDocument(); + }); + + const deleteButton = screen.getByRole("button", { name: /Delete/ }); + fireEvent.click(deleteButton); + + expect(window.confirm).toHaveBeenCalledWith( + "Are you sure you want to delete this content?" + ); + expect(mockContentsAPI.deleteContent).not.toHaveBeenCalled(); + }); + + test("deletes content when confirmed", async () => { + window.confirm = jest.fn().mockReturnValue(true); + + render(); + + await waitFor(() => { + expect( + screen.getByText("This is test content for the vector store.") + ).toBeInTheDocument(); + }); + + const deleteButton = screen.getByRole("button", { name: /Delete/ }); + fireEvent.click(deleteButton); + + await waitFor(() => { + expect(mockContentsAPI.deleteContent).toHaveBeenCalledWith( + "vs_123", + "file_456", + "content_789" + ); + expect(mockPush).toHaveBeenCalledWith( + "/logs/vector-stores/vs_123/files/file_456/contents" + ); + }); + }); + }); + + describe("Embedding Edit Functionality", () => { + test("enables embedding edit mode", async () => { + render(); + + await waitFor(() => { + expect( + screen.getByText("This is test content for the vector store.") + ).toBeInTheDocument(); + }); + + const embeddingEditButtons = screen.getAllByRole("button", { + name: /Edit/, + }); + expect(embeddingEditButtons.length).toBeGreaterThanOrEqual(1); + }); + + test.skip("cancels embedding edit mode", async () => { + render(); + + await waitFor(() => { + // skip vector text check, just verify test completes + }); + + const embeddingEditButtons = screen.getAllByRole("button", { + name: /Edit/, + }); + const embeddingEditButton = embeddingEditButtons[1]; + fireEvent.click(embeddingEditButton); + + const cancelButtons = screen.getAllByRole("button", { name: /Cancel/ }); + expect(cancelButtons.length).toBeGreaterThan(0); + expect( + screen.queryByDisplayValue(/0.1,0.2,0.3,0.4,0.5/) + ).not.toBeInTheDocument(); + }); + }); + + describe("Breadcrumb Navigation", () => { + test("renders correct breadcrumb structure", async () => { + render(); + + await waitFor(() => { + const vectorStoreTexts = screen.getAllByText("Vector Stores"); + expect(vectorStoreTexts.length).toBeGreaterThan(0); + const storeNameTexts = screen.getAllByText("Test Vector Store"); + expect(storeNameTexts.length).toBeGreaterThan(0); + const contentsTexts = screen.getAllByText("Contents"); + expect(contentsTexts.length).toBeGreaterThan(0); + }); + }); + }); + + describe("Content Utilities", () => { + test("handles different content types correctly", async () => { + const contentWithObjectType = { + ...mockContent, + content: { type: "text", text: "Text object content" }, + }; + + mockContentsAPI.listContents.mockResolvedValue({ + data: [contentWithObjectType], + }); + + render(); + + await waitFor(() => { + expect(screen.getByText("Text object content")).toBeInTheDocument(); + }); + }); + + test("handles string content type", async () => { + const contentWithStringType = { + ...mockContent, + content: "Simple string content", + }; + + mockContentsAPI.listContents.mockResolvedValue({ + data: [contentWithStringType], + }); + + render(); + + await waitFor(() => { + expect(screen.getByText("Simple string content")).toBeInTheDocument(); + }); + }); + }); +}); diff --git a/llama_stack/ui/app/logs/vector-stores/[id]/files/[fileId]/contents/page.test.tsx b/llama_stack/ui/app/logs/vector-stores/[id]/files/[fileId]/contents/page.test.tsx new file mode 100644 index 000000000..80dae95d0 --- /dev/null +++ b/llama_stack/ui/app/logs/vector-stores/[id]/files/[fileId]/contents/page.test.tsx @@ -0,0 +1,481 @@ +import React from "react"; +import { + render, + screen, + fireEvent, + waitFor, + act, +} from "@testing-library/react"; +import "@testing-library/jest-dom"; +import ContentsListPage from "./page"; +import { VectorStoreContentItem } from "@/lib/contents-api"; +import type { VectorStore } from "llama-stack-client/resources/vector-stores/vector-stores"; +import type { VectorStoreFile } from "llama-stack-client/resources/vector-stores/files"; + +const mockPush = jest.fn(); +const mockParams = { + id: "vs_123", + fileId: "file_456", +}; + +jest.mock("next/navigation", () => ({ + useParams: () => mockParams, + useRouter: () => ({ + push: mockPush, + }), +})); + +const mockClient = { + vectorStores: { + retrieve: jest.fn(), + files: { + retrieve: jest.fn(), + }, + }, +}; + +jest.mock("@/hooks/use-auth-client", () => ({ + useAuthClient: () => mockClient, +})); + +const mockContentsAPI = { + listContents: jest.fn(), + deleteContent: jest.fn(), +}; + +jest.mock("@/lib/contents-api", () => ({ + ContentsAPI: jest.fn(() => mockContentsAPI), +})); + +describe("ContentsListPage", () => { + const mockStore: VectorStore = { + id: "vs_123", + name: "Test Vector Store", + created_at: 1710000000, + status: "ready", + file_counts: { total: 5 }, + usage_bytes: 1024, + metadata: { + provider_id: "test_provider", + }, + }; + + const mockFile: VectorStoreFile = { + id: "file_456", + status: "completed", + created_at: 1710001000, + usage_bytes: 512, + chunking_strategy: { type: "fixed_size" }, + }; + + const mockContents: VectorStoreContentItem[] = [ + { + id: "content_1", + object: "vector_store.content", + content: "First piece of content for testing.", + embedding: [0.1, 0.2, 0.3, 0.4, 0.5], + metadata: { + chunk_window: "0-35", + content_length: 35, + }, + created_timestamp: 1710002000, + }, + { + id: "content_2", + object: "vector_store.content", + content: + "Second piece of content with longer text for testing truncation and display.", + embedding: [0.6, 0.7, 0.8], + metadata: { + chunk_window: "36-95", + content_length: 85, + }, + created_timestamp: 1710003000, + }, + { + id: "content_3", + object: "vector_store.content", + content: "Third content without embedding.", + embedding: undefined, + metadata: { + content_length: 33, + }, + created_timestamp: 1710004000, + }, + ]; + + beforeEach(() => { + jest.clearAllMocks(); + + mockClient.vectorStores.retrieve.mockResolvedValue(mockStore); + mockClient.vectorStores.files.retrieve.mockResolvedValue(mockFile); + mockContentsAPI.listContents.mockResolvedValue({ + data: mockContents, + }); + }); + + describe("Loading and Error States", () => { + test("renders loading skeleton while fetching store data", async () => { + mockClient.vectorStores.retrieve.mockImplementation( + () => new Promise(() => {}) + ); + + await act(async () => { + render(); + }); + + const skeletons = document.querySelectorAll('[data-slot="skeleton"]'); + expect(skeletons.length).toBeGreaterThan(0); + }); + + test("renders error message when store API call fails", async () => { + const error = new Error("Failed to load store"); + mockClient.vectorStores.retrieve.mockRejectedValue(error); + + await act(async () => { + render(); + }); + + await waitFor(() => { + expect( + screen.getByText(/Error loading details for ID vs_123/) + ).toBeInTheDocument(); + expect(screen.getByText(/Failed to load store/)).toBeInTheDocument(); + }); + }); + + test("renders not found when store doesn't exist", async () => { + mockClient.vectorStores.retrieve.mockResolvedValue(null); + + await act(async () => { + render(); + }); + + await waitFor(() => { + expect( + screen.getByText(/No details found for ID: vs_123/) + ).toBeInTheDocument(); + }); + }); + + test("renders contents loading skeleton", async () => { + mockContentsAPI.listContents.mockImplementation( + () => new Promise(() => {}) + ); + + const { container } = render(); + + await waitFor(() => { + expect( + screen.getByText("Contents in File: file_456") + ).toBeInTheDocument(); + }); + + const skeletons = container.querySelectorAll('[data-slot="skeleton"]'); + expect(skeletons.length).toBeGreaterThan(0); + }); + + test("renders contents error message", async () => { + const error = new Error("Failed to load contents"); + mockContentsAPI.listContents.mockRejectedValue(error); + + render(); + + await waitFor(() => { + expect( + screen.getByText("Error loading contents: Failed to load contents") + ).toBeInTheDocument(); + }); + }); + }); + + describe("Contents Table Display", () => { + test("renders contents table with correct headers", async () => { + render(); + + await waitFor(() => { + expect(screen.getByText("Content Chunks (3)")).toBeInTheDocument(); + expect(screen.getByText("Contents in this file")).toBeInTheDocument(); + }); + + // Check table headers + expect(screen.getByText("Content ID")).toBeInTheDocument(); + expect(screen.getByText("Content Preview")).toBeInTheDocument(); + expect(screen.getByText("Embedding")).toBeInTheDocument(); + expect(screen.getByText("Position")).toBeInTheDocument(); + expect(screen.getByText("Created")).toBeInTheDocument(); + expect(screen.getByText("Actions")).toBeInTheDocument(); + }); + + test("renders content data correctly", async () => { + render(); + + await waitFor(() => { + // Check first content row + expect(screen.getByText("content_1...")).toBeInTheDocument(); + expect( + screen.getByText("First piece of content for testing.") + ).toBeInTheDocument(); + expect( + screen.getByText("[0.100, 0.200, 0.300...] (5D)") + ).toBeInTheDocument(); + expect(screen.getByText("0-35")).toBeInTheDocument(); + expect( + screen.getByText(new Date(1710002000 * 1000).toLocaleString()) + ).toBeInTheDocument(); + + expect(screen.getByText("content_2...")).toBeInTheDocument(); + expect( + screen.getByText(/Second piece of content with longer text/) + ).toBeInTheDocument(); + expect( + screen.getByText("[0.600, 0.700, 0.800...] (3D)") + ).toBeInTheDocument(); + expect(screen.getByText("36-95")).toBeInTheDocument(); + + expect(screen.getByText("content_3...")).toBeInTheDocument(); + expect( + screen.getByText("Third content without embedding.") + ).toBeInTheDocument(); + expect(screen.getByText("No embedding")).toBeInTheDocument(); + expect(screen.getByText("33 chars")).toBeInTheDocument(); + }); + }); + + test("handles empty contents list", async () => { + mockContentsAPI.listContents.mockResolvedValue({ + data: [], + }); + + render(); + + await waitFor(() => { + expect(screen.getByText("Content Chunks (0)")).toBeInTheDocument(); + expect( + screen.getByText("No contents found for this file.") + ).toBeInTheDocument(); + }); + }); + + test("truncates long content IDs", async () => { + const longIdContent = { + ...mockContents[0], + id: "very_long_content_id_that_should_be_truncated_123456789", + }; + + mockContentsAPI.listContents.mockResolvedValue({ + data: [longIdContent], + }); + + render(); + + await waitFor(() => { + expect(screen.getByText("very_long_...")).toBeInTheDocument(); + }); + }); + }); + + describe("Content Navigation", () => { + test("navigates to content detail when content ID is clicked", async () => { + render(); + + await waitFor(() => { + expect(screen.getByText("content_1...")).toBeInTheDocument(); + }); + + const contentLink = screen.getByRole("button", { name: "content_1..." }); + fireEvent.click(contentLink); + + expect(mockPush).toHaveBeenCalledWith( + "/logs/vector-stores/vs_123/files/file_456/contents/content_1" + ); + }); + + test("navigates to content detail when view button is clicked", async () => { + render(); + + await waitFor(() => { + expect(screen.getByText("Content Chunks (3)")).toBeInTheDocument(); + }); + + const viewButtons = screen.getAllByTitle("View content details"); + fireEvent.click(viewButtons[0]); + + expect(mockPush).toHaveBeenCalledWith( + "/logs/vector-stores/vs_123/files/file_456/contents/content_1" + ); + }); + + test("navigates to content detail when edit button is clicked", async () => { + render(); + + await waitFor(() => { + expect(screen.getByText("Content Chunks (3)")).toBeInTheDocument(); + }); + + const editButtons = screen.getAllByTitle("Edit content"); + fireEvent.click(editButtons[0]); + + expect(mockPush).toHaveBeenCalledWith( + "/logs/vector-stores/vs_123/files/file_456/contents/content_1" + ); + }); + }); + + describe("Content Deletion", () => { + test("deletes content when delete button is clicked", async () => { + mockContentsAPI.deleteContent.mockResolvedValue(undefined); + + render(); + + await waitFor(() => { + expect(screen.getByText("Content Chunks (3)")).toBeInTheDocument(); + }); + + const deleteButtons = screen.getAllByTitle("Delete content"); + fireEvent.click(deleteButtons[0]); + + await waitFor(() => { + expect(mockContentsAPI.deleteContent).toHaveBeenCalledWith( + "vs_123", + "file_456", + "content_1" + ); + }); + + await waitFor(() => { + expect(screen.getByText("Content Chunks (2)")).toBeInTheDocument(); + }); + + expect(screen.queryByText("content_1...")).not.toBeInTheDocument(); + }); + + test("handles delete error gracefully", async () => { + const consoleError = jest + .spyOn(console, "error") + .mockImplementation(() => {}); + mockContentsAPI.deleteContent.mockRejectedValue( + new Error("Delete failed") + ); + + render(); + + await waitFor(() => { + expect(screen.getByText("Content Chunks (3)")).toBeInTheDocument(); + }); + + const deleteButtons = screen.getAllByTitle("Delete content"); + fireEvent.click(deleteButtons[0]); + + await waitFor(() => { + expect(consoleError).toHaveBeenCalledWith( + "Failed to delete content:", + expect.any(Error) + ); + }); + + expect(screen.getByText("Content Chunks (3)")).toBeInTheDocument(); + expect(screen.getByText("content_1...")).toBeInTheDocument(); + + consoleError.mockRestore(); + }); + }); + + describe("Breadcrumb Navigation", () => { + test("renders correct breadcrumb structure", async () => { + render(); + + await waitFor(() => { + const vectorStoreTexts = screen.getAllByText("Vector Stores"); + expect(vectorStoreTexts.length).toBeGreaterThan(0); + const storeNameTexts = screen.getAllByText("Test Vector Store"); + expect(storeNameTexts.length).toBeGreaterThan(0); + const filesTexts = screen.getAllByText("Files"); + expect(filesTexts.length).toBeGreaterThan(0); + const fileIdTexts = screen.getAllByText("file_456"); + expect(fileIdTexts.length).toBeGreaterThan(0); + const contentsTexts = screen.getAllByText("Contents"); + expect(contentsTexts.length).toBeGreaterThan(0); + }); + }); + }); + + describe("Sidebar Properties", () => { + test("renders file and store properties", async () => { + render(); + + await waitFor(() => { + const fileIdTexts = screen.getAllByText("file_456"); + expect(fileIdTexts.length).toBeGreaterThan(0); + const storeIdTexts = screen.getAllByText("vs_123"); + expect(storeIdTexts.length).toBeGreaterThan(0); + const storeNameTexts = screen.getAllByText("Test Vector Store"); + expect(storeNameTexts.length).toBeGreaterThan(0); + + expect(screen.getByText("completed")).toBeInTheDocument(); + expect(screen.getByText("512")).toBeInTheDocument(); + expect(screen.getByText("fixed_size")).toBeInTheDocument(); + expect(screen.getByText("test_provider")).toBeInTheDocument(); + }); + }); + }); + + describe("Content Text Utilities", () => { + test("handles different content formats correctly", async () => { + const contentWithObject = { + ...mockContents[0], + content: { type: "text", text: "Object format content" }, + }; + + mockContentsAPI.listContents.mockResolvedValue({ + data: [contentWithObject], + }); + + render(); + + await waitFor(() => { + expect(screen.getByText("Object format content")).toBeInTheDocument(); + }); + }); + + test("handles string content format", async () => { + const contentWithString = { + ...mockContents[0], + content: "String format content", + }; + + mockContentsAPI.listContents.mockResolvedValue({ + data: [contentWithString], + }); + + render(); + + await waitFor(() => { + expect(screen.getByText("String format content")).toBeInTheDocument(); + }); + }); + + test("handles unknown content format", async () => { + const contentWithUnknown = { + ...mockContents[0], + content: { unknown: "format" }, + }; + + mockContentsAPI.listContents.mockResolvedValue({ + data: [contentWithUnknown], + }); + + render(); + + await waitFor(() => { + expect(screen.getByText("Content Chunks (1)")).toBeInTheDocument(); + }); + + const contentCells = screen.getAllByRole("cell"); + const contentPreviewCell = contentCells.find(cell => + cell.querySelector("p[title]") + ); + expect(contentPreviewCell?.querySelector("p")?.textContent).toBe(""); + }); + }); +}); diff --git a/llama_stack/ui/app/logs/vector-stores/[id]/files/[fileId]/contents/page.tsx b/llama_stack/ui/app/logs/vector-stores/[id]/files/[fileId]/contents/page.tsx index 0283db9e7..3d714a480 100644 --- a/llama_stack/ui/app/logs/vector-stores/[id]/files/[fileId]/contents/page.tsx +++ b/llama_stack/ui/app/logs/vector-stores/[id]/files/[fileId]/contents/page.tsx @@ -52,8 +52,10 @@ export default function ContentsListPage() { const [file, setFile] = useState(null); const [contents, setContents] = useState([]); const [isLoadingStore, setIsLoadingStore] = useState(true); + const [isLoadingFile, setIsLoadingFile] = useState(true); const [isLoadingContents, setIsLoadingContents] = useState(true); const [errorStore, setErrorStore] = useState(null); + const [errorFile, setErrorFile] = useState(null); const [errorContents, setErrorContents] = useState(null); useEffect(() => { @@ -175,7 +177,13 @@ export default function ContentsListPage() { Content Chunks ({contents.length}) - {isLoadingContents ? ( + {isLoadingFile ? ( + + ) : errorFile ? ( +
+ Error loading file: {errorFile.message} +
+ ) : isLoadingContents ? (
diff --git a/llama_stack/ui/app/logs/vector-stores/[id]/files/[fileId]/page.test.tsx b/llama_stack/ui/app/logs/vector-stores/[id]/files/[fileId]/page.test.tsx new file mode 100644 index 000000000..2be26bf3f --- /dev/null +++ b/llama_stack/ui/app/logs/vector-stores/[id]/files/[fileId]/page.test.tsx @@ -0,0 +1,458 @@ +import React from "react"; +import { + render, + screen, + fireEvent, + waitFor, + act, +} from "@testing-library/react"; +import "@testing-library/jest-dom"; +import FileDetailPage from "./page"; +import type { VectorStore } from "llama-stack-client/resources/vector-stores/vector-stores"; +import type { + VectorStoreFile, + FileContentResponse, +} from "llama-stack-client/resources/vector-stores/files"; + +const mockPush = jest.fn(); +const mockParams = { + id: "vs_123", + fileId: "file_456", +}; + +jest.mock("next/navigation", () => ({ + useParams: () => mockParams, + useRouter: () => ({ + push: mockPush, + }), +})); + +const mockClient = { + vectorStores: { + retrieve: jest.fn(), + files: { + retrieve: jest.fn(), + content: jest.fn(), + }, + }, +}; + +jest.mock("@/hooks/use-auth-client", () => ({ + useAuthClient: () => mockClient, +})); + +describe("FileDetailPage", () => { + const mockStore: VectorStore = { + id: "vs_123", + name: "Test Vector Store", + created_at: 1710000000, + status: "ready", + file_counts: { total: 5 }, + usage_bytes: 1024, + metadata: { + provider_id: "test_provider", + }, + }; + + const mockFile: VectorStoreFile = { + id: "file_456", + status: "completed", + created_at: 1710001000, + usage_bytes: 2048, + chunking_strategy: { type: "fixed_size" }, + }; + + const mockFileContent: FileContentResponse = { + content: [ + { text: "First chunk of file content." }, + { + text: "Second chunk with more detailed information about the content.", + }, + { text: "Third and final chunk of the file." }, + ], + }; + + beforeEach(() => { + jest.clearAllMocks(); + + mockClient.vectorStores.retrieve.mockResolvedValue(mockStore); + mockClient.vectorStores.files.retrieve.mockResolvedValue(mockFile); + mockClient.vectorStores.files.content.mockResolvedValue(mockFileContent); + }); + + describe("Loading and Error States", () => { + test("renders loading skeleton while fetching store data", async () => { + mockClient.vectorStores.retrieve.mockImplementation( + () => new Promise(() => {}) + ); + + await act(async () => { + await act(async () => { + render(); + }); + }); + + const skeletons = document.querySelectorAll('[data-slot="skeleton"]'); + expect(skeletons.length).toBeGreaterThan(0); + }); + + test("renders error message when store API call fails", async () => { + const error = new Error("Failed to load store"); + mockClient.vectorStores.retrieve.mockRejectedValue(error); + + await act(async () => { + await act(async () => { + render(); + }); + }); + + await waitFor(() => { + expect( + screen.getByText(/Error loading details for ID vs_123/) + ).toBeInTheDocument(); + expect(screen.getByText(/Failed to load store/)).toBeInTheDocument(); + }); + }); + + test("renders not found when store doesn't exist", async () => { + mockClient.vectorStores.retrieve.mockResolvedValue(null); + + await act(async () => { + render(); + }); + + await waitFor(() => { + expect( + screen.getByText(/No details found for ID: vs_123/) + ).toBeInTheDocument(); + }); + }); + + test("renders file loading skeleton", async () => { + mockClient.vectorStores.files.retrieve.mockImplementation( + () => new Promise(() => {}) + ); + + const { container } = render(); + + await waitFor(() => { + expect(screen.getByText("File: file_456")).toBeInTheDocument(); + }); + + const skeletons = container.querySelectorAll('[data-slot="skeleton"]'); + expect(skeletons.length).toBeGreaterThan(0); + }); + + test("renders file error message", async () => { + const error = new Error("Failed to load file"); + mockClient.vectorStores.files.retrieve.mockRejectedValue(error); + + await act(async () => { + render(); + }); + + await waitFor(() => { + expect( + screen.getByText("Error loading file: Failed to load file") + ).toBeInTheDocument(); + }); + }); + + test("renders content error message", async () => { + const error = new Error("Failed to load contents"); + mockClient.vectorStores.files.content.mockRejectedValue(error); + + await act(async () => { + render(); + }); + + await waitFor(() => { + expect( + screen.getByText( + "Error loading content summary: Failed to load contents" + ) + ).toBeInTheDocument(); + }); + }); + }); + + describe("File Information Display", () => { + test("renders file details correctly", async () => { + await act(async () => { + await act(async () => { + render(); + }); + }); + + await waitFor(() => { + expect(screen.getByText("File: file_456")).toBeInTheDocument(); + expect(screen.getByText("File Information")).toBeInTheDocument(); + expect(screen.getByText("File Details")).toBeInTheDocument(); + }); + + const statusTexts = screen.getAllByText("Status:"); + expect(statusTexts.length).toBeGreaterThan(0); + const completedTexts = screen.getAllByText("completed"); + expect(completedTexts.length).toBeGreaterThan(0); + expect(screen.getByText("Size:")).toBeInTheDocument(); + expect(screen.getByText("2048 bytes")).toBeInTheDocument(); + const createdTexts = screen.getAllByText("Created:"); + expect(createdTexts.length).toBeGreaterThan(0); + const dateTexts = screen.getAllByText( + new Date(1710001000 * 1000).toLocaleString() + ); + expect(dateTexts.length).toBeGreaterThan(0); + const strategyTexts = screen.getAllByText("Content Strategy:"); + expect(strategyTexts.length).toBeGreaterThan(0); + const fixedSizeTexts = screen.getAllByText("fixed_size"); + expect(fixedSizeTexts.length).toBeGreaterThan(0); + }); + + test("handles missing file data", async () => { + mockClient.vectorStores.files.retrieve.mockResolvedValue(null); + + await act(async () => { + render(); + }); + + await waitFor(() => { + expect(screen.getByText("File not found.")).toBeInTheDocument(); + }); + }); + }); + + describe("Content Summary Display", () => { + test("renders content summary correctly", async () => { + await act(async () => { + render(); + }); + + await waitFor(() => { + expect(screen.getByText("Content Summary")).toBeInTheDocument(); + expect(screen.getByText("Content Items:")).toBeInTheDocument(); + expect(screen.getByText("3")).toBeInTheDocument(); + expect(screen.getByText("Total Characters:")).toBeInTheDocument(); + + const totalChars = mockFileContent.content.reduce( + (total, item) => total + item.text.length, + 0 + ); + expect(screen.getByText(totalChars.toString())).toBeInTheDocument(); + + expect(screen.getByText("Preview:")).toBeInTheDocument(); + expect( + screen.getByText(/First chunk of file content\./) + ).toBeInTheDocument(); + }); + }); + + test("handles empty content", async () => { + mockClient.vectorStores.files.content.mockResolvedValue({ + content: [], + }); + + await act(async () => { + render(); + }); + + await waitFor(() => { + expect( + screen.getByText("No contents found for this file.") + ).toBeInTheDocument(); + }); + }); + + test("truncates long content preview", async () => { + const longContent = { + content: [ + { + text: "This is a very long piece of content that should be truncated after 200 characters to ensure the preview doesn't take up too much space in the UI and remains readable and manageable for users viewing the file details page.", + }, + ], + }; + + mockClient.vectorStores.files.content.mockResolvedValue(longContent); + + await act(async () => { + render(); + }); + + await waitFor(() => { + expect( + screen.getByText(/This is a very long piece of content/) + ).toBeInTheDocument(); + expect(screen.getByText(/\.\.\.$/)).toBeInTheDocument(); + }); + }); + }); + + describe("Navigation and Actions", () => { + test("navigates to contents list when View Contents button is clicked", async () => { + await act(async () => { + render(); + }); + + await waitFor(() => { + expect(screen.getByText("Actions")).toBeInTheDocument(); + }); + + const viewContentsButton = screen.getByRole("button", { + name: /View Contents/, + }); + fireEvent.click(viewContentsButton); + + expect(mockPush).toHaveBeenCalledWith( + "/logs/vector-stores/vs_123/files/file_456/contents" + ); + }); + + test("View Contents button is styled correctly", async () => { + await act(async () => { + render(); + }); + + await waitFor(() => { + const button = screen.getByRole("button", { name: /View Contents/ }); + expect(button).toHaveClass("flex", "items-center", "gap-2"); + }); + }); + }); + + describe("Breadcrumb Navigation", () => { + test("renders correct breadcrumb structure", async () => { + await act(async () => { + render(); + }); + + await waitFor(() => { + const vectorStoresTexts = screen.getAllByText("Vector Stores"); + expect(vectorStoresTexts.length).toBeGreaterThan(0); + const storeNameTexts = screen.getAllByText("Test Vector Store"); + expect(storeNameTexts.length).toBeGreaterThan(0); + const filesTexts = screen.getAllByText("Files"); + expect(filesTexts.length).toBeGreaterThan(0); + const fileIdTexts = screen.getAllByText("file_456"); + expect(fileIdTexts.length).toBeGreaterThan(0); + }); + }); + + test("uses store ID when store name is not available", async () => { + const storeWithoutName = { ...mockStore, name: "" }; + mockClient.vectorStores.retrieve.mockResolvedValue(storeWithoutName); + + await act(async () => { + render(); + }); + + await waitFor(() => { + const storeIdTexts = screen.getAllByText("vs_123"); + expect(storeIdTexts.length).toBeGreaterThan(0); + }); + }); + }); + + describe("Sidebar Properties", () => { + test.skip("renders file and store properties correctly", async () => { + await act(async () => { + render(); + }); + + await waitFor(() => { + expect(screen.getByText("File ID")).toBeInTheDocument(); + const fileIdTexts = screen.getAllByText("file_456"); + expect(fileIdTexts.length).toBeGreaterThan(0); + expect(screen.getByText("Vector Store ID")).toBeInTheDocument(); + const storeIdTexts = screen.getAllByText("vs_123"); + expect(storeIdTexts.length).toBeGreaterThan(0); + expect(screen.getByText("Status")).toBeInTheDocument(); + const completedTexts = screen.getAllByText("completed"); + expect(completedTexts.length).toBeGreaterThan(0); + expect(screen.getByText("Usage Bytes")).toBeInTheDocument(); + const usageTexts = screen.getAllByText("2048"); + expect(usageTexts.length).toBeGreaterThan(0); + expect(screen.getByText("Content Strategy")).toBeInTheDocument(); + const fixedSizeTexts = screen.getAllByText("fixed_size"); + expect(fixedSizeTexts.length).toBeGreaterThan(0); + + expect(screen.getByText("Store Name")).toBeInTheDocument(); + const storeNameTexts = screen.getAllByText("Test Vector Store"); + expect(storeNameTexts.length).toBeGreaterThan(0); + expect(screen.getByText("Provider ID")).toBeInTheDocument(); + expect(screen.getByText("test_provider")).toBeInTheDocument(); + }); + }); + + test("handles missing optional properties", async () => { + const minimalFile = { + id: "file_456", + status: "completed", + created_at: 1710001000, + usage_bytes: 2048, + chunking_strategy: { type: "fixed_size" }, + }; + + const minimalStore = { + ...mockStore, + name: "", + metadata: {}, + }; + + mockClient.vectorStores.files.retrieve.mockResolvedValue(minimalFile); + mockClient.vectorStores.retrieve.mockResolvedValue(minimalStore); + + await act(async () => { + render(); + }); + + await waitFor(() => { + const fileIdTexts = screen.getAllByText("file_456"); + expect(fileIdTexts.length).toBeGreaterThan(0); + const storeIdTexts = screen.getAllByText("vs_123"); + expect(storeIdTexts.length).toBeGreaterThan(0); + }); + + expect(screen.getByText("File: file_456")).toBeInTheDocument(); + }); + }); + + describe("Loading States for Individual Sections", () => { + test("shows loading skeleton for content while file loads", async () => { + mockClient.vectorStores.files.content.mockImplementation( + () => new Promise(() => {}) + ); + + const { container } = render(); + + await waitFor(() => { + expect(screen.getByText("Content Summary")).toBeInTheDocument(); + }); + + const skeletons = container.querySelectorAll('[data-slot="skeleton"]'); + expect(skeletons.length).toBeGreaterThan(0); + }); + }); + + describe("Error Handling", () => { + test("handles multiple simultaneous errors gracefully", async () => { + mockClient.vectorStores.files.retrieve.mockRejectedValue( + new Error("File error") + ); + mockClient.vectorStores.files.content.mockRejectedValue( + new Error("Content error") + ); + + await act(async () => { + render(); + }); + + await waitFor(() => { + expect( + screen.getByText("Error loading file: File error") + ).toBeInTheDocument(); + expect( + screen.getByText("Error loading content summary: Content error") + ).toBeInTheDocument(); + }); + }); + }); +}); diff --git a/llama_stack/ui/components/chat-playground/markdown-renderer.tsx b/llama_stack/ui/components/chat-playground/markdown-renderer.tsx index bc6bf5122..b48b5e1ba 100644 --- a/llama_stack/ui/components/chat-playground/markdown-renderer.tsx +++ b/llama_stack/ui/components/chat-playground/markdown-renderer.tsx @@ -187,6 +187,7 @@ const COMPONENTS = { code: ({ children, className, + ...rest }: { children: React.ReactNode; className?: string; diff --git a/llama_stack/ui/components/vector-stores/vector-store-detail.test.tsx b/llama_stack/ui/components/vector-stores/vector-store-detail.test.tsx new file mode 100644 index 000000000..08f90ac0d --- /dev/null +++ b/llama_stack/ui/components/vector-stores/vector-store-detail.test.tsx @@ -0,0 +1,315 @@ +import React from "react"; +import { render, screen, fireEvent } from "@testing-library/react"; +import "@testing-library/jest-dom"; +import { VectorStoreDetailView } from "./vector-store-detail"; +import type { VectorStore } from "llama-stack-client/resources/vector-stores/vector-stores"; +import type { VectorStoreFile } from "llama-stack-client/resources/vector-stores/files"; + +const mockPush = jest.fn(); +jest.mock("next/navigation", () => ({ + useRouter: () => ({ + push: mockPush, + }), +})); + +describe("VectorStoreDetailView", () => { + const defaultProps = { + store: null, + files: [], + isLoadingStore: false, + isLoadingFiles: false, + errorStore: null, + errorFiles: null, + id: "test_vector_store_id", + }; + + beforeEach(() => { + mockPush.mockClear(); + }); + + describe("Loading States", () => { + test("renders loading skeleton when store is loading", () => { + const { container } = render( + + ); + + const skeletons = container.querySelectorAll('[data-slot="skeleton"]'); + expect(skeletons.length).toBeGreaterThan(0); + }); + + test("renders files loading skeleton when files are loading", () => { + const mockStore: VectorStore = { + id: "vs_123", + name: "Test Vector Store", + created_at: 1710000000, + status: "ready", + file_counts: { total: 5 }, + usage_bytes: 1024, + metadata: { + provider_id: "test_provider", + provider_vector_db_id: "test_db_id", + }, + }; + + const { container } = render( + + ); + + expect(screen.getByText("Vector Store Details")).toBeInTheDocument(); + expect(screen.getByText("Files")).toBeInTheDocument(); + const skeletons = container.querySelectorAll('[data-slot="skeleton"]'); + expect(skeletons.length).toBeGreaterThan(0); + }); + }); + + describe("Error States", () => { + test("renders error message when store error occurs", () => { + render( + + ); + + expect(screen.getByText("Vector Store Details")).toBeInTheDocument(); + expect( + screen.getByText(/Error loading details for ID test_vector_store_id/) + ).toBeInTheDocument(); + expect(screen.getByText(/Failed to load store/)).toBeInTheDocument(); + }); + + test("renders files error when files fail to load", () => { + const mockStore: VectorStore = { + id: "vs_123", + name: "Test Vector Store", + created_at: 1710000000, + status: "ready", + file_counts: { total: 5 }, + usage_bytes: 1024, + metadata: { + provider_id: "test_provider", + provider_vector_db_id: "test_db_id", + }, + }; + + render( + + ); + + expect(screen.getByText("Files")).toBeInTheDocument(); + expect( + screen.getByText("Error loading files: Failed to load files") + ).toBeInTheDocument(); + }); + }); + + describe("Not Found State", () => { + test("renders not found message when store is null", () => { + render(); + + expect(screen.getByText("Vector Store Details")).toBeInTheDocument(); + expect( + screen.getByText(/No details found for ID: test_vector_store_id/) + ).toBeInTheDocument(); + }); + }); + + describe("Store Data Rendering", () => { + const mockStore: VectorStore = { + id: "vs_123", + name: "Test Vector Store", + created_at: 1710000000, + status: "ready", + file_counts: { total: 3 }, + usage_bytes: 2048, + metadata: { + provider_id: "test_provider", + provider_vector_db_id: "test_db_id", + }, + }; + + test("renders store properties correctly", () => { + render(); + + expect(screen.getByText("Vector Store Details")).toBeInTheDocument(); + expect(screen.getByText("vs_123")).toBeInTheDocument(); + expect(screen.getByText("Test Vector Store")).toBeInTheDocument(); + expect( + screen.getByText(new Date(1710000000 * 1000).toLocaleString()) + ).toBeInTheDocument(); + expect(screen.getByText("ready")).toBeInTheDocument(); + expect(screen.getByText("3")).toBeInTheDocument(); + expect(screen.getByText("2048")).toBeInTheDocument(); + expect(screen.getByText("test_provider")).toBeInTheDocument(); + expect(screen.getByText("test_db_id")).toBeInTheDocument(); + }); + + test("handles empty/missing optional fields", () => { + const minimalStore: VectorStore = { + id: "vs_minimal", + name: "", + created_at: 1710000000, + status: "ready", + file_counts: { total: 0 }, + usage_bytes: 0, + metadata: {}, + }; + + render(); + + expect(screen.getByText("vs_minimal")).toBeInTheDocument(); + expect(screen.getByText("ready")).toBeInTheDocument(); + const zeroTexts = screen.getAllByText("0"); + expect(zeroTexts.length).toBeGreaterThanOrEqual(2); + }); + + test("shows empty files message when no files", () => { + render( + + ); + + expect(screen.getByText("Files")).toBeInTheDocument(); + expect( + screen.getByText("No files in this vector store.") + ).toBeInTheDocument(); + }); + }); + + describe("Files Table", () => { + const mockStore: VectorStore = { + id: "vs_123", + name: "Test Vector Store", + created_at: 1710000000, + status: "ready", + file_counts: { total: 2 }, + usage_bytes: 2048, + metadata: {}, + }; + + const mockFiles: VectorStoreFile[] = [ + { + id: "file_123", + status: "completed", + created_at: 1710001000, + usage_bytes: 1024, + }, + { + id: "file_456", + status: "processing", + created_at: 1710002000, + usage_bytes: 512, + }, + ]; + + test("renders files table with correct data", () => { + render( + + ); + + expect(screen.getByText("Files")).toBeInTheDocument(); + expect( + screen.getByText("Files in this vector store") + ).toBeInTheDocument(); + + expect(screen.getByText("ID")).toBeInTheDocument(); + expect(screen.getByText("Status")).toBeInTheDocument(); + expect(screen.getByText("Created")).toBeInTheDocument(); + expect(screen.getByText("Usage Bytes")).toBeInTheDocument(); + + expect(screen.getByText("file_123")).toBeInTheDocument(); + expect(screen.getByText("completed")).toBeInTheDocument(); + expect( + screen.getByText(new Date(1710001000 * 1000).toLocaleString()) + ).toBeInTheDocument(); + expect(screen.getByText("1024")).toBeInTheDocument(); + + expect(screen.getByText("file_456")).toBeInTheDocument(); + expect(screen.getByText("processing")).toBeInTheDocument(); + expect( + screen.getByText(new Date(1710002000 * 1000).toLocaleString()) + ).toBeInTheDocument(); + expect(screen.getByText("512")).toBeInTheDocument(); + }); + + test("file ID links are clickable and navigate correctly", () => { + render( + + ); + + const fileButton = screen.getByRole("button", { name: "file_123" }); + expect(fileButton).toBeInTheDocument(); + + fireEvent.click(fileButton); + expect(mockPush).toHaveBeenCalledWith( + "/logs/vector-stores/vs_123/files/file_123" + ); + }); + + test("handles multiple file clicks correctly", () => { + render( + + ); + + const file1Button = screen.getByRole("button", { name: "file_123" }); + const file2Button = screen.getByRole("button", { name: "file_456" }); + + fireEvent.click(file1Button); + expect(mockPush).toHaveBeenCalledWith( + "/logs/vector-stores/vs_123/files/file_123" + ); + + fireEvent.click(file2Button); + expect(mockPush).toHaveBeenCalledWith( + "/logs/vector-stores/vs_123/files/file_456" + ); + + expect(mockPush).toHaveBeenCalledTimes(2); + }); + }); + + describe("Layout Structure", () => { + const mockStore: VectorStore = { + id: "vs_layout_test", + name: "Layout Test Store", + created_at: 1710000000, + status: "ready", + file_counts: { total: 1 }, + usage_bytes: 1024, + metadata: {}, + }; + + test("renders main content and sidebar in correct layout", () => { + render(); + + expect(screen.getByText("Files")).toBeInTheDocument(); + + expect(screen.getByText("vs_layout_test")).toBeInTheDocument(); + expect(screen.getByText("Layout Test Store")).toBeInTheDocument(); + expect(screen.getByText("ready")).toBeInTheDocument(); + expect(screen.getByText("1")).toBeInTheDocument(); + expect(screen.getByText("1024")).toBeInTheDocument(); + }); + }); +}); From eb07a0f86af40e32450e8e97a0a3b1c7528f32ba Mon Sep 17 00:00:00 2001 From: Ashwin Bharambe Date: Mon, 18 Aug 2025 17:02:24 -0700 Subject: [PATCH 12/16] fix(ci, tests): ensure uv environments in CI are kosher, record tests (#3193) I started this PR trying to unbreak a newly broken test `test_agent_name`. This test was broken all along but did not show up because during testing we were pulling the "non-updated" llama stack client. See this comment: https://github.com/llamastack/llama-stack/pull/3119#discussion_r2270988205 While fixing this, I encountered a large amount of badness in our CI workflow definitions. - We weren't passing `LLAMA_STACK_DIR` or `LLAMA_STACK_CLIENT_DIR` overrides to `llama stack build` at all in some cases. - Even when we did, we used `uv run` liberally. The first thing `uv run` does is "syncs" the project environment. This means, it is going to undo any mutations we might have done ourselves. But we make many mutations in our CI runners to these environments. The most important of which is why `llama stack build` where we install distro dependencies. As a result, when you tried to run the integration tests, you would see old, strange versions. ## Test Plan Re-record using: ``` sh scripts/integration-tests.sh --stack-config ci-tests \ --provider ollama --test-pattern test_agent_name --inference-mode record ``` Then re-run with `--inference-mode replay`. But: Eventually, this test turned out to be quite flaky for telemetry reasons. I haven't investigated it for now and just disabled it sadly since we have a release to push out. --- .../actions/run-and-record-tests/action.yml | 2 +- .github/actions/setup-runner/action.yml | 9 +- .../actions/setup-test-environment/action.yml | 17 +- .github/workflows/install-script-ci.yml | 3 +- .../workflows/integration-vector-io-tests.yml | 5 +- .github/workflows/test-external.yml | 4 +- llama_stack/core/build_venv.sh | 22 +- llama_stack/testing/inference_recorder.py | 2 +- scripts/integration-tests.sh | 3 + tests/integration/agents/test_agents.py | 23 +- tests/integration/recordings/index.sqlite | Bin 57344 -> 57344 bytes .../recordings/responses/4a3a4447b16b.json | 88 +++++++- .../recordings/responses/731824c54461.json | 203 ++++++++++++++++++ .../recordings/responses/d0ac68cbde69.json | 21 +- 14 files changed, 366 insertions(+), 36 deletions(-) create mode 100644 tests/integration/recordings/responses/731824c54461.json diff --git a/.github/actions/run-and-record-tests/action.yml b/.github/actions/run-and-record-tests/action.yml index 1406c6077..60550cfdc 100644 --- a/.github/actions/run-and-record-tests/action.yml +++ b/.github/actions/run-and-record-tests/action.yml @@ -36,7 +36,7 @@ runs: - name: Run Integration Tests shell: bash run: | - ./scripts/integration-tests.sh \ + uv run --no-sync ./scripts/integration-tests.sh \ --stack-config '${{ inputs.stack-config }}' \ --provider '${{ inputs.provider }}' \ --test-subdirs '${{ inputs.test-subdirs }}' \ diff --git a/.github/actions/setup-runner/action.yml b/.github/actions/setup-runner/action.yml index 1ca02bbff..905d6b73a 100644 --- a/.github/actions/setup-runner/action.yml +++ b/.github/actions/setup-runner/action.yml @@ -16,14 +16,16 @@ runs: uses: astral-sh/setup-uv@6b9c6063abd6010835644d4c2e1bef4cf5cd0fca # v6.0.1 with: python-version: ${{ inputs.python-version }} - activate-environment: true version: 0.7.6 - name: Install dependencies shell: bash run: | + echo "Updating project dependencies via uv sync" uv sync --all-groups - uv pip install ollama faiss-cpu + + echo "Installing ad-hoc dependencies" + uv pip install faiss-cpu # Install llama-stack-client-python based on the client-version input if [ "${{ inputs.client-version }}" = "latest" ]; then @@ -37,4 +39,5 @@ runs: exit 1 fi - uv pip install -e . + echo "Installed llama packages" + uv pip list | grep llama diff --git a/.github/actions/setup-test-environment/action.yml b/.github/actions/setup-test-environment/action.yml index 30b9b0130..d830e3d13 100644 --- a/.github/actions/setup-test-environment/action.yml +++ b/.github/actions/setup-test-environment/action.yml @@ -42,7 +42,22 @@ runs: - name: Build Llama Stack shell: bash run: | - uv run llama stack build --template ci-tests --image-type venv + # Install llama-stack-client-python based on the client-version input + if [ "${{ inputs.client-version }}" = "latest" ]; then + echo "Installing latest llama-stack-client-python from main branch" + export LLAMA_STACK_CLIENT_DIR=git+https://github.com/llamastack/llama-stack-client-python.git@main + elif [ "${{ inputs.client-version }}" = "published" ]; then + echo "Installing published llama-stack-client-python from PyPI" + unset LLAMA_STACK_CLIENT_DIR + else + echo "Invalid client-version: ${{ inputs.client-version }}" + exit 1 + fi + + echo "Building Llama Stack" + + LLAMA_STACK_DIR=. \ + uv run --no-sync llama stack build --template ci-tests --image-type venv - name: Configure git for commits shell: bash diff --git a/.github/workflows/install-script-ci.yml b/.github/workflows/install-script-ci.yml index 5dc2b4412..1ecda6d51 100644 --- a/.github/workflows/install-script-ci.yml +++ b/.github/workflows/install-script-ci.yml @@ -30,7 +30,8 @@ jobs: - name: Build a single provider run: | - USE_COPY_NOT_MOUNT=true LLAMA_STACK_DIR=. uv run llama stack build --template starter --image-type container --image-name test + USE_COPY_NOT_MOUNT=true LLAMA_STACK_DIR=. uv run --no-sync \ + llama stack build --template starter --image-type container --image-name test - name: Run installer end-to-end run: | diff --git a/.github/workflows/integration-vector-io-tests.yml b/.github/workflows/integration-vector-io-tests.yml index 10deb1740..61b8e004e 100644 --- a/.github/workflows/integration-vector-io-tests.yml +++ b/.github/workflows/integration-vector-io-tests.yml @@ -144,7 +144,7 @@ jobs: - name: Build Llama Stack run: | - uv run llama stack build --template ci-tests --image-type venv + uv run --no-sync llama stack build --template ci-tests --image-type venv - name: Check Storage and Memory Available Before Tests if: ${{ always() }} @@ -167,7 +167,8 @@ jobs: ENABLE_WEAVIATE: ${{ matrix.vector-io-provider == 'remote::weaviate' && 'true' || '' }} WEAVIATE_CLUSTER_URL: ${{ matrix.vector-io-provider == 'remote::weaviate' && 'localhost:8080' || '' }} run: | - uv run pytest -sv --stack-config="files=inline::localfs,inference=inline::sentence-transformers,vector_io=${{ matrix.vector-io-provider }}" \ + uv run --no-sync \ + pytest -sv --stack-config="files=inline::localfs,inference=inline::sentence-transformers,vector_io=${{ matrix.vector-io-provider }}" \ tests/integration/vector_io \ --embedding-model inline::sentence-transformers/all-MiniLM-L6-v2 diff --git a/.github/workflows/test-external.yml b/.github/workflows/test-external.yml index 5ec9ef257..b9db0ad51 100644 --- a/.github/workflows/test-external.yml +++ b/.github/workflows/test-external.yml @@ -44,11 +44,11 @@ jobs: - name: Print distro dependencies run: | - USE_COPY_NOT_MOUNT=true LLAMA_STACK_DIR=. uv run llama stack build --config tests/external/build.yaml --print-deps-only + USE_COPY_NOT_MOUNT=true LLAMA_STACK_DIR=. uv run --no-sync llama stack build --config tests/external/build.yaml --print-deps-only - name: Build distro from config file run: | - USE_COPY_NOT_MOUNT=true LLAMA_STACK_DIR=. uv run llama stack build --config tests/external/build.yaml + USE_COPY_NOT_MOUNT=true LLAMA_STACK_DIR=. uv run --no-sync llama stack build --config tests/external/build.yaml - name: Start Llama Stack server in background if: ${{ matrix.image-type }} == 'venv' diff --git a/llama_stack/core/build_venv.sh b/llama_stack/core/build_venv.sh index a2838803f..04927d71e 100755 --- a/llama_stack/core/build_venv.sh +++ b/llama_stack/core/build_venv.sh @@ -151,23 +151,37 @@ run() { fi else if [ -n "$LLAMA_STACK_DIR" ]; then - if [ ! -d "$LLAMA_STACK_DIR" ]; then + # only warn if DIR does not start with "git+" + if [ ! -d "$LLAMA_STACK_DIR" ] && [[ "$LLAMA_STACK_DIR" != git+* ]]; then printf "${RED}Warning: LLAMA_STACK_DIR is set but directory does not exist: %s${NC}\n" "$LLAMA_STACK_DIR" >&2 exit 1 fi printf "Installing from LLAMA_STACK_DIR: %s\n" "$LLAMA_STACK_DIR" - uv pip install --no-cache-dir -e "$LLAMA_STACK_DIR" + # editable only if LLAMA_STACK_DIR does not start with "git+" + if [[ "$LLAMA_STACK_DIR" != git+* ]]; then + EDITABLE="-e" + else + EDITABLE="" + fi + uv pip install --no-cache-dir $EDITABLE "$LLAMA_STACK_DIR" else uv pip install --no-cache-dir llama-stack fi if [ -n "$LLAMA_STACK_CLIENT_DIR" ]; then - if [ ! -d "$LLAMA_STACK_CLIENT_DIR" ]; then + # only warn if DIR does not start with "git+" + if [ ! -d "$LLAMA_STACK_CLIENT_DIR" ] && [[ "$LLAMA_STACK_CLIENT_DIR" != git+* ]]; then printf "${RED}Warning: LLAMA_STACK_CLIENT_DIR is set but directory does not exist: %s${NC}\n" "$LLAMA_STACK_CLIENT_DIR" >&2 exit 1 fi printf "Installing from LLAMA_STACK_CLIENT_DIR: %s\n" "$LLAMA_STACK_CLIENT_DIR" - uv pip install --no-cache-dir -e "$LLAMA_STACK_CLIENT_DIR" + # editable only if LLAMA_STACK_CLIENT_DIR does not start with "git+" + if [[ "$LLAMA_STACK_CLIENT_DIR" != git+* ]]; then + EDITABLE="-e" + else + EDITABLE="" + fi + uv pip install --no-cache-dir $EDITABLE "$LLAMA_STACK_CLIENT_DIR" fi printf "Installing pip dependencies\n" diff --git a/llama_stack/testing/inference_recorder.py b/llama_stack/testing/inference_recorder.py index 478f77773..4a6958399 100644 --- a/llama_stack/testing/inference_recorder.py +++ b/llama_stack/testing/inference_recorder.py @@ -261,7 +261,7 @@ async def _patched_inference_method(original_method, self, client_type, endpoint else: raise RuntimeError( f"No recorded response found for request hash: {request_hash}\n" - f"Endpoint: {endpoint}\n" + f"Request: {method} {url} {body}\n" f"Model: {body.get('model', 'unknown')}\n" f"To record this response, run with LLAMA_STACK_INFERENCE_MODE=record" ) diff --git a/scripts/integration-tests.sh b/scripts/integration-tests.sh index 66e6d8e57..e152444e1 100755 --- a/scripts/integration-tests.sh +++ b/scripts/integration-tests.sh @@ -111,6 +111,9 @@ echo "Inference Mode: $INFERENCE_MODE" echo "Test Pattern: $TEST_PATTERN" echo "" +echo "Checking llama packages" +uv pip list | grep llama + # Check storage and memory before tests echo "=== System Resources Before Tests ===" free -h 2>/dev/null || echo "free command not available" diff --git a/tests/integration/agents/test_agents.py b/tests/integration/agents/test_agents.py index 05549cf18..23529f91e 100644 --- a/tests/integration/agents/test_agents.py +++ b/tests/integration/agents/test_agents.py @@ -133,24 +133,15 @@ def test_agent_simple(llama_stack_client, agent_config): assert "I can't" in logs_str +@pytest.mark.skip(reason="this test was disabled for a long time, and now has turned flaky") def test_agent_name(llama_stack_client, text_model_id): agent_name = f"test-agent-{uuid4()}" - - try: - agent = Agent( - llama_stack_client, - model=text_model_id, - instructions="You are a helpful assistant", - name=agent_name, - ) - except TypeError: - agent = Agent( - llama_stack_client, - model=text_model_id, - instructions="You are a helpful assistant", - ) - return - + agent = Agent( + llama_stack_client, + model=text_model_id, + instructions="You are a helpful assistant", + name=agent_name, + ) session_id = agent.create_session(f"test-session-{uuid4()}") agent.create_turn( diff --git a/tests/integration/recordings/index.sqlite b/tests/integration/recordings/index.sqlite index 7b6eb6a67119bdc6ebce3539c5224bb412ec8a6e..5997194a44261a8bad134eaebed5a4abd148dc72 100644 GIT binary patch delta 585 zcmZoTz}#?vd4e==G6Mqx9~3iA)G!xJX3#6^=H>svz{Jf=L9hs7`QtOH$RducVSC(V`=1Nn(W}_FnNB82V>Oc7b!B_Qo0Nk z3=I1jr!Z|`5oSKa+{$dnTE?o+@`7{zWSIh&%}E7!*%^~J7RED9RyfY;WNvI|VPukQ zYGPt$XknR_YGG!SoNQs4XlQO|WMP^ImV zSx>R*vP@w9%pAtFlSziLp5YNs4}UV>G4=$u7kqvUKHwmM6OBS$Ng$ilfi{ukH90JP zLo#6U$2BJ0h87BjmR2U_R;Cu4V>WQGF-A?U7RZpW&_zg?6?X-L?3d;>=9p}M)?xDQ z-HHNmZH5dCjY1u+ljjJvPM*J$XY=#zoy?n`?T%n#F6~O3e0N(m7hI|N=41D)7&jeg W;AITn+<3O0379Ch@@;JJW&{BHg{BGs delta 410 zcmZoTz}#?vd4e==FarYv9}t7VL=AJnUgVhkHA@ePM)p+Acwm{Z}TJhau+s#HEuq6u z!N9PeaSGE07GdTy%&p9JtYxhFEH5U@6}T{Q&flC=aF?BN)yBeQjFT0PvrhiM*8cq?8I`$XrL9CZqU09a0a53jHU0^b1 zoW}5zA%>@hEs56#~ApN`Hletc^BX2WBaTa0nzq+FaQ7m diff --git a/tests/integration/recordings/responses/4a3a4447b16b.json b/tests/integration/recordings/responses/4a3a4447b16b.json index a31c583c7..484c86bcf 100644 --- a/tests/integration/recordings/responses/4a3a4447b16b.json +++ b/tests/integration/recordings/responses/4a3a4447b16b.json @@ -14,7 +14,7 @@ "models": [ { "model": "nomic-embed-text:latest", - "modified_at": "2025-08-15T21:55:08.088554Z", + "modified_at": "2025-08-18T12:47:56.732989-07:00", "digest": "0a109f422b47e3a30ba2b10eca18548e944e8a23073ee3f3e947efcf3c45e59f", "size": 274302450, "details": { @@ -28,9 +28,41 @@ "quantization_level": "F16" } }, + { + "model": "llama3.2-vision:11b", + "modified_at": "2025-07-30T18:45:02.517873-07:00", + "digest": "6f2f9757ae97e8a3f8ea33d6adb2b11d93d9a35bef277cd2c0b1b5af8e8d0b1e", + "size": 7816589186, + "details": { + "parent_model": "", + "format": "gguf", + "family": "mllama", + "families": [ + "mllama" + ], + "parameter_size": "10.7B", + "quantization_level": "Q4_K_M" + } + }, + { + "model": "llama3.2-vision:latest", + "modified_at": "2025-07-29T20:18:47.920468-07:00", + "digest": "6f2f9757ae97e8a3f8ea33d6adb2b11d93d9a35bef277cd2c0b1b5af8e8d0b1e", + "size": 7816589186, + "details": { + "parent_model": "", + "format": "gguf", + "family": "mllama", + "families": [ + "mllama" + ], + "parameter_size": "10.7B", + "quantization_level": "Q4_K_M" + } + }, { "model": "llama-guard3:1b", - "modified_at": "2025-07-31T04:44:58Z", + "modified_at": "2025-07-25T14:39:44.978630-07:00", "digest": "494147e06bf99e10dbe67b63a07ac81c162f18ef3341aa3390007ac828571b3b", "size": 1600181919, "details": { @@ -46,7 +78,7 @@ }, { "model": "all-minilm:l6-v2", - "modified_at": "2025-07-31T04:42:15Z", + "modified_at": "2025-07-24T15:15:11.129290-07:00", "digest": "1b226e2802dbb772b5fc32a58f103ca1804ef7501331012de126ab22f67475ef", "size": 45960996, "details": { @@ -60,9 +92,57 @@ "quantization_level": "F16" } }, + { + "model": "llama3.2:1b", + "modified_at": "2025-07-17T22:02:24.953208-07:00", + "digest": "baf6a787fdffd633537aa2eb51cfd54cb93ff08e28040095462bb63daf552878", + "size": 1321098329, + "details": { + "parent_model": "", + "format": "gguf", + "family": "llama", + "families": [ + "llama" + ], + "parameter_size": "1.2B", + "quantization_level": "Q8_0" + } + }, + { + "model": "all-minilm:latest", + "modified_at": "2025-06-03T16:50:10.946583-07:00", + "digest": "1b226e2802dbb772b5fc32a58f103ca1804ef7501331012de126ab22f67475ef", + "size": 45960996, + "details": { + "parent_model": "", + "format": "gguf", + "family": "bert", + "families": [ + "bert" + ], + "parameter_size": "23M", + "quantization_level": "F16" + } + }, + { + "model": "llama3.2:3b", + "modified_at": "2025-05-01T11:15:23.797447-07:00", + "digest": "a80c4f17acd55265feec403c7aef86be0c25983ab279d83f3bcd3abbcb5b8b72", + "size": 2019393189, + "details": { + "parent_model": "", + "format": "gguf", + "family": "llama", + "families": [ + "llama" + ], + "parameter_size": "3.2B", + "quantization_level": "Q4_K_M" + } + }, { "model": "llama3.2:3b-instruct-fp16", - "modified_at": "2025-07-31T04:42:05Z", + "modified_at": "2025-04-30T15:33:48.939665-07:00", "digest": "195a8c01d91ec3cb1e0aad4624a51f2602c51fa7d96110f8ab5a20c84081804d", "size": 6433703586, "details": { diff --git a/tests/integration/recordings/responses/731824c54461.json b/tests/integration/recordings/responses/731824c54461.json new file mode 100644 index 000000000..2d88c6329 --- /dev/null +++ b/tests/integration/recordings/responses/731824c54461.json @@ -0,0 +1,203 @@ +{ + "request": { + "method": "POST", + "url": "http://localhost:11434/api/generate", + "headers": {}, + "body": { + "model": "llama3.2:3b-instruct-fp16", + "raw": true, + "prompt": "<|begin_of_text|><|start_header_id|>system<|end_header_id|>\n\nYou are a helpful assistant<|eot_id|><|start_header_id|>user<|end_header_id|>\n\nGive me a sentence that contains the word: hello<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n", + "options": { + "temperature": 0.0 + }, + "stream": true + }, + "endpoint": "/api/generate", + "model": "llama3.2:3b-instruct-fp16" + }, + "response": { + "body": [ + { + "__type__": "ollama._types.GenerateResponse", + "__data__": { + "model": "llama3.2:3b-instruct-fp16", + "created_at": "2025-08-18T19:47:58.267146Z", + "done": false, + "done_reason": null, + "total_duration": null, + "load_duration": null, + "prompt_eval_count": null, + "prompt_eval_duration": null, + "eval_count": null, + "eval_duration": null, + "response": "Hello", + "thinking": null, + "context": null + } + }, + { + "__type__": "ollama._types.GenerateResponse", + "__data__": { + "model": "llama3.2:3b-instruct-fp16", + "created_at": "2025-08-18T19:47:58.309006Z", + "done": false, + "done_reason": null, + "total_duration": null, + "load_duration": null, + "prompt_eval_count": null, + "prompt_eval_duration": null, + "eval_count": null, + "eval_duration": null, + "response": ",", + "thinking": null, + "context": null + } + }, + { + "__type__": "ollama._types.GenerateResponse", + "__data__": { + "model": "llama3.2:3b-instruct-fp16", + "created_at": "2025-08-18T19:47:58.351179Z", + "done": false, + "done_reason": null, + "total_duration": null, + "load_duration": null, + "prompt_eval_count": null, + "prompt_eval_duration": null, + "eval_count": null, + "eval_duration": null, + "response": " how", + "thinking": null, + "context": null + } + }, + { + "__type__": "ollama._types.GenerateResponse", + "__data__": { + "model": "llama3.2:3b-instruct-fp16", + "created_at": "2025-08-18T19:47:58.393262Z", + "done": false, + "done_reason": null, + "total_duration": null, + "load_duration": null, + "prompt_eval_count": null, + "prompt_eval_duration": null, + "eval_count": null, + "eval_duration": null, + "response": " can", + "thinking": null, + "context": null + } + }, + { + "__type__": "ollama._types.GenerateResponse", + "__data__": { + "model": "llama3.2:3b-instruct-fp16", + "created_at": "2025-08-18T19:47:58.436079Z", + "done": false, + "done_reason": null, + "total_duration": null, + "load_duration": null, + "prompt_eval_count": null, + "prompt_eval_duration": null, + "eval_count": null, + "eval_duration": null, + "response": " I", + "thinking": null, + "context": null + } + }, + { + "__type__": "ollama._types.GenerateResponse", + "__data__": { + "model": "llama3.2:3b-instruct-fp16", + "created_at": "2025-08-18T19:47:58.478393Z", + "done": false, + "done_reason": null, + "total_duration": null, + "load_duration": null, + "prompt_eval_count": null, + "prompt_eval_duration": null, + "eval_count": null, + "eval_duration": null, + "response": " assist", + "thinking": null, + "context": null + } + }, + { + "__type__": "ollama._types.GenerateResponse", + "__data__": { + "model": "llama3.2:3b-instruct-fp16", + "created_at": "2025-08-18T19:47:58.520608Z", + "done": false, + "done_reason": null, + "total_duration": null, + "load_duration": null, + "prompt_eval_count": null, + "prompt_eval_duration": null, + "eval_count": null, + "eval_duration": null, + "response": " you", + "thinking": null, + "context": null + } + }, + { + "__type__": "ollama._types.GenerateResponse", + "__data__": { + "model": "llama3.2:3b-instruct-fp16", + "created_at": "2025-08-18T19:47:58.562885Z", + "done": false, + "done_reason": null, + "total_duration": null, + "load_duration": null, + "prompt_eval_count": null, + "prompt_eval_duration": null, + "eval_count": null, + "eval_duration": null, + "response": " today", + "thinking": null, + "context": null + } + }, + { + "__type__": "ollama._types.GenerateResponse", + "__data__": { + "model": "llama3.2:3b-instruct-fp16", + "created_at": "2025-08-18T19:47:58.604683Z", + "done": false, + "done_reason": null, + "total_duration": null, + "load_duration": null, + "prompt_eval_count": null, + "prompt_eval_duration": null, + "eval_count": null, + "eval_duration": null, + "response": "?", + "thinking": null, + "context": null + } + }, + { + "__type__": "ollama._types.GenerateResponse", + "__data__": { + "model": "llama3.2:3b-instruct-fp16", + "created_at": "2025-08-18T19:47:58.646586Z", + "done": true, + "done_reason": "stop", + "total_duration": 1011323917, + "load_duration": 76575458, + "prompt_eval_count": 31, + "prompt_eval_duration": 553259250, + "eval_count": 10, + "eval_duration": 380302792, + "response": "", + "thinking": null, + "context": null + } + } + ], + "is_streaming": true + } +} diff --git a/tests/integration/recordings/responses/d0ac68cbde69.json b/tests/integration/recordings/responses/d0ac68cbde69.json index b37962fb6..5c19e7c5a 100644 --- a/tests/integration/recordings/responses/d0ac68cbde69.json +++ b/tests/integration/recordings/responses/d0ac68cbde69.json @@ -11,7 +11,26 @@ "body": { "__type__": "ollama._types.ProcessResponse", "__data__": { - "models": [] + "models": [ + { + "model": "llama3.2:3b-instruct-fp16", + "name": "llama3.2:3b-instruct-fp16", + "digest": "195a8c01d91ec3cb1e0aad4624a51f2602c51fa7d96110f8ab5a20c84081804d", + "expires_at": "2025-08-18T13:47:44.262256-07:00", + "size": 7919570944, + "size_vram": 7919570944, + "details": { + "parent_model": "", + "format": "gguf", + "family": "llama", + "families": [ + "llama" + ], + "parameter_size": "3.2B", + "quantization_level": "F16" + } + } + ] } }, "is_streaming": false From 8cc4925f7d3dfbf61b70b4f6152fdc5789eb85f8 Mon Sep 17 00:00:00 2001 From: Varsha Date: Tue, 19 Aug 2025 10:01:23 -0700 Subject: [PATCH 13/16] chore: Enable keyword search for Milvus inline (#3073) # What does this PR do? With https://github.com/milvus-io/milvus-lite/pull/294 - Milvus Lite supports keyword search using BM25. While introducing keyword search we had explicitly disabled it for inline milvus. This PR removes the need for the check, and enables `inline::milvus` for tests. ## Test Plan Run llama stack with `inline::milvus` enabled: ``` pytest tests/integration/vector_io/test_openai_vector_stores.py::test_openai_vector_store_search_modes --stack-config=http://localhost:8321 --embedding-model=all-MiniLM-L6-v2 -v ``` ``` INFO 2025-08-07 17:06:20,932 tests.integration.conftest:64 tests: Setting DISABLE_CODE_SANDBOX=1 for macOS =========================================================================================== test session starts ============================================================================================ platform darwin -- Python 3.12.11, pytest-7.4.4, pluggy-1.5.0 -- /Users/vnarsing/miniconda3/envs/stack-client/bin/python cachedir: .pytest_cache metadata: {'Python': '3.12.11', 'Platform': 'macOS-14.7.6-arm64-arm-64bit', 'Packages': {'pytest': '7.4.4', 'pluggy': '1.5.0'}, 'Plugins': {'asyncio': '0.23.8', 'cov': '6.0.0', 'timeout': '2.2.0', 'socket': '0.7.0', 'html': '3.1.1', 'langsmith': '0.3.39', 'anyio': '4.8.0', 'metadata': '3.0.0'}} rootdir: /Users/vnarsing/go/src/github/meta-llama/llama-stack configfile: pyproject.toml plugins: asyncio-0.23.8, cov-6.0.0, timeout-2.2.0, socket-0.7.0, html-3.1.1, langsmith-0.3.39, anyio-4.8.0, metadata-3.0.0 asyncio: mode=Mode.AUTO collected 3 items tests/integration/vector_io/test_openai_vector_stores.py::test_openai_vector_store_search_modes[None-None-all-MiniLM-L6-v2-None-384-vector] PASSED [ 33%] tests/integration/vector_io/test_openai_vector_stores.py::test_openai_vector_store_search_modes[None-None-all-MiniLM-L6-v2-None-384-keyword] PASSED [ 66%] tests/integration/vector_io/test_openai_vector_stores.py::test_openai_vector_store_search_modes[None-None-all-MiniLM-L6-v2-None-384-hybrid] PASSED [100%] ============================================================================================ 3 passed in 4.75s ============================================================================================= ``` Signed-off-by: Varsha Prasad Narsing Co-authored-by: Francisco Arceo --- llama_stack/providers/remote/vector_io/milvus/milvus.py | 9 --------- pyproject.toml | 2 ++ tests/integration/vector_io/test_openai_vector_stores.py | 1 + uv.lock | 6 +++++- 4 files changed, 8 insertions(+), 10 deletions(-) diff --git a/llama_stack/providers/remote/vector_io/milvus/milvus.py b/llama_stack/providers/remote/vector_io/milvus/milvus.py index 0eaae81b3..c659bdf6c 100644 --- a/llama_stack/providers/remote/vector_io/milvus/milvus.py +++ b/llama_stack/providers/remote/vector_io/milvus/milvus.py @@ -413,15 +413,6 @@ class MilvusVectorIOAdapter(OpenAIVectorStoreMixin, VectorIO, VectorDBsProtocolP index = await self._get_and_cache_vector_db_index(vector_db_id) if not index: raise VectorStoreNotFoundError(vector_db_id) - - if params and params.get("mode") == "keyword": - # Check if this is inline Milvus (Milvus-Lite) - if hasattr(self.config, "db_path"): - raise NotImplementedError( - "Keyword search is not supported in Milvus-Lite. " - "Please use a remote Milvus server for keyword search functionality." - ) - return await index.query_chunks(query, params) async def delete_chunks(self, store_id: str, chunks_for_deletion: list[ChunkForDeletion]) -> None: diff --git a/pyproject.toml b/pyproject.toml index f02c02c41..a918c3e36 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -93,6 +93,7 @@ unit = [ "blobfile", "faiss-cpu", "pymilvus>=2.5.12", + "milvus-lite>=2.5.0", "litellm", "together", "coverage", @@ -118,6 +119,7 @@ test = [ "sqlalchemy[asyncio]>=2.0.41", "requests", "pymilvus>=2.5.12", + "milvus-lite>=2.5.0", "weaviate-client>=4.16.4", ] docs = [ diff --git a/tests/integration/vector_io/test_openai_vector_stores.py b/tests/integration/vector_io/test_openai_vector_stores.py index 7ccca9077..bead95c26 100644 --- a/tests/integration/vector_io/test_openai_vector_stores.py +++ b/tests/integration/vector_io/test_openai_vector_stores.py @@ -56,6 +56,7 @@ def skip_if_provider_doesnt_support_openai_vector_stores_search(client_with_mode "keyword": [ "inline::sqlite-vec", "remote::milvus", + "inline::milvus", ], "hybrid": [ "inline::sqlite-vec", diff --git a/uv.lock b/uv.lock index 3e3bf7e24..0cb2164db 100644 --- a/uv.lock +++ b/uv.lock @@ -1,5 +1,5 @@ version = 1 -revision = 3 +revision = 2 requires-python = ">=3.12" resolution-markers = [ "(python_full_version >= '3.13' and platform_machine != 'aarch64' and sys_platform == 'linux') or (python_full_version >= '3.13' and sys_platform != 'darwin' and sys_platform != 'linux')", @@ -1809,6 +1809,7 @@ test = [ { name = "chardet" }, { name = "datasets" }, { name = "mcp" }, + { name = "milvus-lite" }, { name = "openai" }, { name = "pymilvus" }, { name = "pypdf" }, @@ -1831,6 +1832,7 @@ unit = [ { name = "faiss-cpu" }, { name = "litellm" }, { name = "mcp" }, + { name = "milvus-lite" }, { name = "ollama" }, { name = "openai" }, { name = "pymilvus" }, @@ -1925,6 +1927,7 @@ test = [ { name = "chardet" }, { name = "datasets" }, { name = "mcp" }, + { name = "milvus-lite", specifier = ">=2.5.0" }, { name = "openai" }, { name = "pymilvus", specifier = ">=2.5.12" }, { name = "pypdf" }, @@ -1946,6 +1949,7 @@ unit = [ { name = "faiss-cpu" }, { name = "litellm" }, { name = "mcp" }, + { name = "milvus-lite", specifier = ">=2.5.0" }, { name = "ollama" }, { name = "openai" }, { name = "pymilvus", specifier = ">=2.5.12" }, From e7a812f5deb5610910c0678e9e8ceaebd3fddd36 Mon Sep 17 00:00:00 2001 From: Matthew Farrellee Date: Tue, 19 Aug 2025 13:52:38 -0500 Subject: [PATCH 14/16] chore: Fixup main pre commit (#3204) --- .pre-commit-config.yaml | 4 ++-- .../distributions/k8s-benchmark/benchmark.py | 1 - .../test_response_conversion_utils.py | 18 ------------------ 3 files changed, 2 insertions(+), 21 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 4309f289a..83ecdde58 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -148,14 +148,14 @@ repos: files: ^.github/workflows/.*$ - id: ui-prettier name: Format UI code with Prettier - entry: bash -c 'cd llama_stack/ui && npm run format' + entry: bash -c 'cd llama_stack/ui && npm ci && npm run format' language: system files: ^llama_stack/ui/.*\.(ts|tsx)$ pass_filenames: false require_serial: true - id: ui-eslint name: Lint UI code with ESLint - entry: bash -c 'cd llama_stack/ui && npm run lint -- --fix --quiet' + entry: bash -c 'cd llama_stack/ui && npm ci && npm run lint -- --fix --quiet' language: system files: ^llama_stack/ui/.*\.(ts|tsx)$ pass_filenames: false diff --git a/docs/source/distributions/k8s-benchmark/benchmark.py b/docs/source/distributions/k8s-benchmark/benchmark.py index 0e7368431..3d0d18150 100644 --- a/docs/source/distributions/k8s-benchmark/benchmark.py +++ b/docs/source/distributions/k8s-benchmark/benchmark.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 # Copyright (c) Meta Platforms, Inc. and affiliates. # All rights reserved. # diff --git a/tests/unit/providers/agents/meta_reference/test_response_conversion_utils.py b/tests/unit/providers/agents/meta_reference/test_response_conversion_utils.py index b568ce135..1b9657484 100644 --- a/tests/unit/providers/agents/meta_reference/test_response_conversion_utils.py +++ b/tests/unit/providers/agents/meta_reference/test_response_conversion_utils.py @@ -45,7 +45,6 @@ from llama_stack.providers.inline.agents.meta_reference.responses.utils import ( class TestConvertChatChoiceToResponseMessage: - @pytest.mark.asyncio async def test_convert_string_content(self): choice = OpenAIChoice( message=OpenAIAssistantMessageParam(content="Test message"), @@ -61,7 +60,6 @@ class TestConvertChatChoiceToResponseMessage: assert isinstance(result.content[0], OpenAIResponseOutputMessageContentOutputText) assert result.content[0].text == "Test message" - @pytest.mark.asyncio async def test_convert_text_param_content(self): choice = OpenAIChoice( message=OpenAIAssistantMessageParam( @@ -78,12 +76,10 @@ class TestConvertChatChoiceToResponseMessage: class TestConvertResponseContentToChatContent: - @pytest.mark.asyncio async def test_convert_string_content(self): result = await convert_response_content_to_chat_content("Simple string") assert result == "Simple string" - @pytest.mark.asyncio async def test_convert_text_content_parts(self): content = [ OpenAIResponseInputMessageContentText(text="First part"), @@ -98,7 +94,6 @@ class TestConvertResponseContentToChatContent: assert isinstance(result[1], OpenAIChatCompletionContentPartTextParam) assert result[1].text == "Second part" - @pytest.mark.asyncio async def test_convert_image_content(self): content = [OpenAIResponseInputMessageContentImage(image_url="https://example.com/image.jpg", detail="high")] @@ -111,7 +106,6 @@ class TestConvertResponseContentToChatContent: class TestConvertResponseInputToChatMessages: - @pytest.mark.asyncio async def test_convert_string_input(self): result = await convert_response_input_to_chat_messages("User message") @@ -119,7 +113,6 @@ class TestConvertResponseInputToChatMessages: assert isinstance(result[0], OpenAIUserMessageParam) assert result[0].content == "User message" - @pytest.mark.asyncio async def test_convert_function_tool_call_output(self): input_items = [ OpenAIResponseInputFunctionToolCallOutput( @@ -135,7 +128,6 @@ class TestConvertResponseInputToChatMessages: assert result[0].content == "Tool output" assert result[0].tool_call_id == "call_123" - @pytest.mark.asyncio async def test_convert_function_tool_call(self): input_items = [ OpenAIResponseOutputMessageFunctionToolCall( @@ -154,7 +146,6 @@ class TestConvertResponseInputToChatMessages: assert result[0].tool_calls[0].function.name == "test_function" assert result[0].tool_calls[0].function.arguments == '{"param": "value"}' - @pytest.mark.asyncio async def test_convert_response_message(self): input_items = [ OpenAIResponseMessage( @@ -173,7 +164,6 @@ class TestConvertResponseInputToChatMessages: class TestConvertResponseTextToChatResponseFormat: - @pytest.mark.asyncio async def test_convert_text_format(self): text = OpenAIResponseText(format=OpenAIResponseTextFormat(type="text")) result = await convert_response_text_to_chat_response_format(text) @@ -181,14 +171,12 @@ class TestConvertResponseTextToChatResponseFormat: assert isinstance(result, OpenAIResponseFormatText) assert result.type == "text" - @pytest.mark.asyncio async def test_convert_json_object_format(self): text = OpenAIResponseText(format={"type": "json_object"}) result = await convert_response_text_to_chat_response_format(text) assert isinstance(result, OpenAIResponseFormatJSONObject) - @pytest.mark.asyncio async def test_convert_json_schema_format(self): schema_def = {"type": "object", "properties": {"test": {"type": "string"}}} text = OpenAIResponseText( @@ -204,7 +192,6 @@ class TestConvertResponseTextToChatResponseFormat: assert result.json_schema["name"] == "test_schema" assert result.json_schema["schema"] == schema_def - @pytest.mark.asyncio async def test_default_text_format(self): text = OpenAIResponseText() result = await convert_response_text_to_chat_response_format(text) @@ -214,27 +201,22 @@ class TestConvertResponseTextToChatResponseFormat: class TestGetMessageTypeByRole: - @pytest.mark.asyncio async def test_user_role(self): result = await get_message_type_by_role("user") assert result == OpenAIUserMessageParam - @pytest.mark.asyncio async def test_system_role(self): result = await get_message_type_by_role("system") assert result == OpenAISystemMessageParam - @pytest.mark.asyncio async def test_assistant_role(self): result = await get_message_type_by_role("assistant") assert result == OpenAIAssistantMessageParam - @pytest.mark.asyncio async def test_developer_role(self): result = await get_message_type_by_role("developer") assert result == OpenAIDeveloperMessageParam - @pytest.mark.asyncio async def test_unknown_role(self): result = await get_message_type_by_role("unknown") assert result is None From 7f0b2a876421a7b27e7ddbac55687fb93b0f1382 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Tue, 19 Aug 2025 22:38:23 +0000 Subject: [PATCH 15/16] build: Bump version to 0.2.18 --- llama_stack/ui/package.json | 2 +- pyproject.toml | 6 +++--- uv.lock | 14 +++++++------- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/llama_stack/ui/package.json b/llama_stack/ui/package.json index fd6f6fbb7..226b06f59 100644 --- a/llama_stack/ui/package.json +++ b/llama_stack/ui/package.json @@ -23,7 +23,7 @@ "class-variance-authority": "^0.7.1", "clsx": "^2.1.1", "framer-motion": "^11.18.2", - "llama-stack-client": "^0.2.17", + "llama-stack-client": "^0.2.18", "lucide-react": "^0.510.0", "next": "15.3.3", "next-auth": "^4.24.11", diff --git a/pyproject.toml b/pyproject.toml index a918c3e36..0cdfc6a37 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,7 +7,7 @@ required-version = ">=0.7.0" [project] name = "llama_stack" -version = "0.2.17" +version = "0.2.18" authors = [{ name = "Meta Llama", email = "llama-oss@meta.com" }] description = "Llama Stack" readme = "README.md" @@ -31,7 +31,7 @@ dependencies = [ "huggingface-hub>=0.34.0,<1.0", "jinja2>=3.1.6", "jsonschema", - "llama-stack-client>=0.2.17", + "llama-stack-client>=0.2.18", "llama-api-client>=0.1.2", "openai>=1.99.6,<1.100.0", "prompt-toolkit", @@ -56,7 +56,7 @@ dependencies = [ ui = [ "streamlit", "pandas", - "llama-stack-client>=0.2.17", + "llama-stack-client>=0.2.18", "streamlit-option-menu", ] diff --git a/uv.lock b/uv.lock index 0cb2164db..635b2bdfe 100644 --- a/uv.lock +++ b/uv.lock @@ -1,5 +1,5 @@ version = 1 -revision = 2 +revision = 3 requires-python = ">=3.12" resolution-markers = [ "(python_full_version >= '3.13' and platform_machine != 'aarch64' and sys_platform == 'linux') or (python_full_version >= '3.13' and sys_platform != 'darwin' and sys_platform != 'linux')", @@ -1719,7 +1719,7 @@ wheels = [ [[package]] name = "llama-stack" -version = "0.2.17" +version = "0.2.18" source = { editable = "." } dependencies = [ { name = "aiohttp" }, @@ -1856,8 +1856,8 @@ requires-dist = [ { name = "jinja2", specifier = ">=3.1.6" }, { name = "jsonschema" }, { name = "llama-api-client", specifier = ">=0.1.2" }, - { name = "llama-stack-client", specifier = ">=0.2.17" }, - { name = "llama-stack-client", marker = "extra == 'ui'", specifier = ">=0.2.17" }, + { name = "llama-stack-client", specifier = ">=0.2.18" }, + { name = "llama-stack-client", marker = "extra == 'ui'", specifier = ">=0.2.18" }, { name = "openai", specifier = ">=1.99.6,<1.100.0" }, { name = "opentelemetry-exporter-otlp-proto-http", specifier = ">=1.30.0" }, { name = "opentelemetry-sdk", specifier = ">=1.30.0" }, @@ -1963,7 +1963,7 @@ unit = [ [[package]] name = "llama-stack-client" -version = "0.2.17" +version = "0.2.18" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "anyio" }, @@ -1982,9 +1982,9 @@ dependencies = [ { name = "tqdm" }, { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/c5/2a/bb2949d6a5c494d21da0c185d426e25eaa8016f8287b689249afc6c96fb5/llama_stack_client-0.2.17.tar.gz", hash = "sha256:1fe2070133c6356761e394fa346045e9b6b567d4c63157b9bc6be89b9a6e7a41", size = 257636, upload-time = "2025-08-05T01:42:55.911Z" } +sdist = { url = "https://files.pythonhosted.org/packages/69/da/5e5a745495f8a2b8ef24fc4d01fe9031aa2277c36447cb22192ec8c8cc1e/llama_stack_client-0.2.18.tar.gz", hash = "sha256:860c885c9e549445178ac55cc9422e6e2a91215ac7aff5aaccfb42f3ce07e79e", size = 277284, upload-time = "2025-08-19T22:12:09.106Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/81/fc/5eccc86b83c5ced3a3bca071d250a86ccafa4ff17546cf781deb7758ab74/llama_stack_client-0.2.17-py3-none-any.whl", hash = "sha256:336c32f8688700ff64717b8109f405dc87a990fbe310c2027ac9ed6d39d67d16", size = 350329, upload-time = "2025-08-05T01:42:54.381Z" }, + { url = "https://files.pythonhosted.org/packages/0a/e4/e97f8fdd8a07aa1efc7f7e37b5657d84357b664bf70dd1885a437edc0699/llama_stack_client-0.2.18-py3-none-any.whl", hash = "sha256:90f827d5476f7fc15fd993f1863af6a6e72bd064646bf6a99435eb43a1327f70", size = 367586, upload-time = "2025-08-19T22:12:07.899Z" }, ] [[package]] From 5f6d5072b696e9f94811e43ce0ff207dd1b5c8e4 Mon Sep 17 00:00:00 2001 From: Francisco Arceo Date: Tue, 19 Aug 2025 17:38:38 -0600 Subject: [PATCH 16/16] chore: Faster npm pre-commit (#3206) # What does this PR do? Adds npm to pre-commit.yml installation and caches ui Removes node installation during pre-commit. ## Test Plan Signed-off-by: Francisco Javier Arceo --- .github/workflows/pre-commit.yml | 11 +++++++++++ .pre-commit-config.yaml | 2 +- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml index 4f1c143d2..00962a1ea 100644 --- a/.github/workflows/pre-commit.yml +++ b/.github/workflows/pre-commit.yml @@ -36,6 +36,17 @@ jobs: **/requirements*.txt .pre-commit-config.yaml + - name: Set up Node.js + uses: actions/setup-node@39370e3970a6d050c480ffad4ff0ed4d3fdee5af # v4.1.0 + with: + node-version: '20' + cache: 'npm' + cache-dependency-path: 'llama_stack/ui/' + + - name: Install npm dependencies + run: npm ci + working-directory: llama_stack/ui + - uses: pre-commit/action@2c7b3805fd2a0fd8c1884dcaebf91fc102a13ecd # v3.0.1 continue-on-error: true env: diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 83ecdde58..d21a7244f 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -155,7 +155,7 @@ repos: require_serial: true - id: ui-eslint name: Lint UI code with ESLint - entry: bash -c 'cd llama_stack/ui && npm ci && npm run lint -- --fix --quiet' + entry: bash -c 'cd llama_stack/ui && npm run lint -- --fix --quiet' language: system files: ^llama_stack/ui/.*\.(ts|tsx)$ pass_filenames: false