diff --git a/.github/TRIAGERS.md b/.github/TRIAGERS.md index ed4f4a6c6..f5bd11531 100644 --- a/.github/TRIAGERS.md +++ b/.github/TRIAGERS.md @@ -1,2 +1,2 @@ # This file documents Triage members in the Llama Stack community - @bbrowning @franciscojavierarceo @leseb + @franciscojavierarceo diff --git a/.github/actions/run-and-record-tests/action.yml b/.github/actions/run-and-record-tests/action.yml index 1406c6077..60550cfdc 100644 --- a/.github/actions/run-and-record-tests/action.yml +++ b/.github/actions/run-and-record-tests/action.yml @@ -36,7 +36,7 @@ runs: - name: Run Integration Tests shell: bash run: | - ./scripts/integration-tests.sh \ + uv run --no-sync ./scripts/integration-tests.sh \ --stack-config '${{ inputs.stack-config }}' \ --provider '${{ inputs.provider }}' \ --test-subdirs '${{ inputs.test-subdirs }}' \ diff --git a/.github/actions/setup-runner/action.yml b/.github/actions/setup-runner/action.yml index 1ca02bbff..905d6b73a 100644 --- a/.github/actions/setup-runner/action.yml +++ b/.github/actions/setup-runner/action.yml @@ -16,14 +16,16 @@ runs: uses: astral-sh/setup-uv@6b9c6063abd6010835644d4c2e1bef4cf5cd0fca # v6.0.1 with: python-version: ${{ inputs.python-version }} - activate-environment: true version: 0.7.6 - name: Install dependencies shell: bash run: | + echo "Updating project dependencies via uv sync" uv sync --all-groups - uv pip install ollama faiss-cpu + + echo "Installing ad-hoc dependencies" + uv pip install faiss-cpu # Install llama-stack-client-python based on the client-version input if [ "${{ inputs.client-version }}" = "latest" ]; then @@ -37,4 +39,5 @@ runs: exit 1 fi - uv pip install -e . + echo "Installed llama packages" + uv pip list | grep llama diff --git a/.github/actions/setup-test-environment/action.yml b/.github/actions/setup-test-environment/action.yml index 30b9b0130..d830e3d13 100644 --- a/.github/actions/setup-test-environment/action.yml +++ b/.github/actions/setup-test-environment/action.yml @@ -42,7 +42,22 @@ runs: - name: Build Llama Stack shell: bash run: | - uv run llama stack build --template ci-tests --image-type venv + # Install llama-stack-client-python based on the client-version input + if [ "${{ inputs.client-version }}" = "latest" ]; then + echo "Installing latest llama-stack-client-python from main branch" + export LLAMA_STACK_CLIENT_DIR=git+https://github.com/llamastack/llama-stack-client-python.git@main + elif [ "${{ inputs.client-version }}" = "published" ]; then + echo "Installing published llama-stack-client-python from PyPI" + unset LLAMA_STACK_CLIENT_DIR + else + echo "Invalid client-version: ${{ inputs.client-version }}" + exit 1 + fi + + echo "Building Llama Stack" + + LLAMA_STACK_DIR=. \ + uv run --no-sync llama stack build --template ci-tests --image-type venv - name: Configure git for commits shell: bash diff --git a/.github/workflows/README.md b/.github/workflows/README.md index 3c3d93dc2..8344d12a4 100644 --- a/.github/workflows/README.md +++ b/.github/workflows/README.md @@ -18,5 +18,6 @@ Llama Stack uses GitHub Actions for Continuous Integration (CI). Below is a tabl | Close stale issues and PRs | [stale_bot.yml](stale_bot.yml) | Run the Stale Bot action | | Test External Providers Installed via Module | [test-external-provider-module.yml](test-external-provider-module.yml) | Test External Provider installation via Python module | | Test External API and Providers | [test-external.yml](test-external.yml) | Test the External API and Provider mechanisms | +| UI Tests | [ui-unit-tests.yml](ui-unit-tests.yml) | Run the UI test suite | | Unit Tests | [unit-tests.yml](unit-tests.yml) | Run the unit test suite | | Update ReadTheDocs | [update-readthedocs.yml](update-readthedocs.yml) | Update the Llama Stack ReadTheDocs site | diff --git a/.github/workflows/install-script-ci.yml b/.github/workflows/install-script-ci.yml index 5dc2b4412..1ecda6d51 100644 --- a/.github/workflows/install-script-ci.yml +++ b/.github/workflows/install-script-ci.yml @@ -30,7 +30,8 @@ jobs: - name: Build a single provider run: | - USE_COPY_NOT_MOUNT=true LLAMA_STACK_DIR=. uv run llama stack build --template starter --image-type container --image-name test + USE_COPY_NOT_MOUNT=true LLAMA_STACK_DIR=. uv run --no-sync \ + llama stack build --template starter --image-type container --image-name test - name: Run installer end-to-end run: | diff --git a/.github/workflows/integration-auth-tests.yml b/.github/workflows/integration-auth-tests.yml index ef2066497..c328e3b6c 100644 --- a/.github/workflows/integration-auth-tests.yml +++ b/.github/workflows/integration-auth-tests.yml @@ -10,6 +10,7 @@ on: paths: - 'distributions/**' - 'llama_stack/**' + - '!llama_stack/ui/**' - 'tests/integration/**' - 'uv.lock' - 'pyproject.toml' diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index fc56f62ea..ba18c27c8 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -10,6 +10,7 @@ on: types: [opened, synchronize, reopened] paths: - 'llama_stack/**' + - '!llama_stack/ui/**' - 'tests/**' - 'uv.lock' - 'pyproject.toml' diff --git a/.github/workflows/integration-vector-io-tests.yml b/.github/workflows/integration-vector-io-tests.yml index 99a44c147..61b8e004e 100644 --- a/.github/workflows/integration-vector-io-tests.yml +++ b/.github/workflows/integration-vector-io-tests.yml @@ -9,6 +9,7 @@ on: branches: [ main ] paths: - 'llama_stack/**' + - '!llama_stack/ui/**' - 'tests/integration/vector_io/**' - 'uv.lock' - 'pyproject.toml' @@ -143,7 +144,7 @@ jobs: - name: Build Llama Stack run: | - uv run llama stack build --template ci-tests --image-type venv + uv run --no-sync llama stack build --template ci-tests --image-type venv - name: Check Storage and Memory Available Before Tests if: ${{ always() }} @@ -166,7 +167,8 @@ jobs: ENABLE_WEAVIATE: ${{ matrix.vector-io-provider == 'remote::weaviate' && 'true' || '' }} WEAVIATE_CLUSTER_URL: ${{ matrix.vector-io-provider == 'remote::weaviate' && 'localhost:8080' || '' }} run: | - uv run pytest -sv --stack-config="files=inline::localfs,inference=inline::sentence-transformers,vector_io=${{ matrix.vector-io-provider }}" \ + uv run --no-sync \ + pytest -sv --stack-config="files=inline::localfs,inference=inline::sentence-transformers,vector_io=${{ matrix.vector-io-provider }}" \ tests/integration/vector_io \ --embedding-model inline::sentence-transformers/all-MiniLM-L6-v2 diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml index 4f1c143d2..00962a1ea 100644 --- a/.github/workflows/pre-commit.yml +++ b/.github/workflows/pre-commit.yml @@ -36,6 +36,17 @@ jobs: **/requirements*.txt .pre-commit-config.yaml + - name: Set up Node.js + uses: actions/setup-node@39370e3970a6d050c480ffad4ff0ed4d3fdee5af # v4.1.0 + with: + node-version: '20' + cache: 'npm' + cache-dependency-path: 'llama_stack/ui/' + + - name: Install npm dependencies + run: npm ci + working-directory: llama_stack/ui + - uses: pre-commit/action@2c7b3805fd2a0fd8c1884dcaebf91fc102a13ecd # v3.0.1 continue-on-error: true env: diff --git a/.github/workflows/python-build-test.yml b/.github/workflows/python-build-test.yml index 67dc49cce..fe1dfd58a 100644 --- a/.github/workflows/python-build-test.yml +++ b/.github/workflows/python-build-test.yml @@ -9,6 +9,8 @@ on: pull_request: branches: - main + paths-ignore: + - 'llama_stack/ui/**' jobs: build: diff --git a/.github/workflows/test-external.yml b/.github/workflows/test-external.yml index 27181a236..b9db0ad51 100644 --- a/.github/workflows/test-external.yml +++ b/.github/workflows/test-external.yml @@ -9,6 +9,7 @@ on: branches: [ main ] paths: - 'llama_stack/**' + - '!llama_stack/ui/**' - 'tests/integration/**' - 'uv.lock' - 'pyproject.toml' @@ -43,11 +44,11 @@ jobs: - name: Print distro dependencies run: | - USE_COPY_NOT_MOUNT=true LLAMA_STACK_DIR=. uv run llama stack build --config tests/external/build.yaml --print-deps-only + USE_COPY_NOT_MOUNT=true LLAMA_STACK_DIR=. uv run --no-sync llama stack build --config tests/external/build.yaml --print-deps-only - name: Build distro from config file run: | - USE_COPY_NOT_MOUNT=true LLAMA_STACK_DIR=. uv run llama stack build --config tests/external/build.yaml + USE_COPY_NOT_MOUNT=true LLAMA_STACK_DIR=. uv run --no-sync llama stack build --config tests/external/build.yaml - name: Start Llama Stack server in background if: ${{ matrix.image-type }} == 'venv' diff --git a/.github/workflows/ui-unit-tests.yml b/.github/workflows/ui-unit-tests.yml new file mode 100644 index 000000000..00c539c58 --- /dev/null +++ b/.github/workflows/ui-unit-tests.yml @@ -0,0 +1,55 @@ +name: UI Tests + +run-name: Run the UI test suite + +on: + push: + branches: [ main ] + pull_request: + branches: [ main ] + paths: + - 'llama_stack/ui/**' + - '.github/workflows/ui-unit-tests.yml' # This workflow + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + ui-tests: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + node-version: [22] + + steps: + - name: Checkout repository + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + + - name: Setup Node.js + uses: actions/setup-node@39370e3970a6d050c480ffad4ff0ed4d3fdee5af # v4.1.0 + with: + node-version: ${{ matrix.node-version }} + cache: 'npm' + cache-dependency-path: 'llama_stack/ui/package-lock.json' + + - name: Install dependencies + working-directory: llama_stack/ui + run: npm ci + + - name: Run linting + working-directory: llama_stack/ui + run: npm run lint + + - name: Run format check + working-directory: llama_stack/ui + run: npm run format:check + + - name: Run unit tests + working-directory: llama_stack/ui + env: + CI: true + + run: npm test -- --coverage --watchAll=false --passWithNoTests diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index b133511d1..f2a6c7754 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -9,6 +9,7 @@ on: branches: [ main ] paths: - 'llama_stack/**' + - '!llama_stack/ui/**' - 'tests/unit/**' - 'uv.lock' - 'pyproject.toml' diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 4309f289a..d21a7244f 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -148,7 +148,7 @@ repos: files: ^.github/workflows/.*$ - id: ui-prettier name: Format UI code with Prettier - entry: bash -c 'cd llama_stack/ui && npm run format' + entry: bash -c 'cd llama_stack/ui && npm ci && npm run format' language: system files: ^llama_stack/ui/.*\.(ts|tsx)$ pass_filenames: false diff --git a/docs/source/contributing/index.md b/docs/source/contributing/index.md index 24bf3f66c..1846f4d97 100644 --- a/docs/source/contributing/index.md +++ b/docs/source/contributing/index.md @@ -23,12 +23,7 @@ new_vector_database ```{include} ../../../tests/README.md ``` -## Benchmarking - -```{include} ../../../docs/source/distributions/k8s-benchmark/README.md -``` - -### Advanced Topics +## Advanced Topics For developers who need deeper understanding of the testing system internals: @@ -37,3 +32,8 @@ For developers who need deeper understanding of the testing system internals: testing/record-replay ``` + +### Benchmarking + +```{include} ../../../docs/source/distributions/k8s-benchmark/README.md +``` diff --git a/docs/source/distributions/k8s-benchmark/benchmark.py b/docs/source/distributions/k8s-benchmark/benchmark.py index 0e7368431..3d0d18150 100644 --- a/docs/source/distributions/k8s-benchmark/benchmark.py +++ b/docs/source/distributions/k8s-benchmark/benchmark.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 # Copyright (c) Meta Platforms, Inc. and affiliates. # All rights reserved. # diff --git a/llama_stack/core/build_conda_env.sh b/llama_stack/core/build_conda_env.sh deleted file mode 100755 index 48ac3a1ab..000000000 --- a/llama_stack/core/build_conda_env.sh +++ /dev/null @@ -1,207 +0,0 @@ -#!/bin/bash - -# Copyright (c) Meta Platforms, Inc. and affiliates. -# All rights reserved. -# -# This source code is licensed under the terms described in the LICENSE file in -# the root directory of this source tree. - -LLAMA_STACK_DIR=${LLAMA_STACK_DIR:-} -LLAMA_STACK_CLIENT_DIR=${LLAMA_STACK_CLIENT_DIR:-} -TEST_PYPI_VERSION=${TEST_PYPI_VERSION:-} -PYPI_VERSION=${PYPI_VERSION:-} -# This timeout (in seconds) is necessary when installing PyTorch via uv since it's likely to time out -# Reference: https://github.com/astral-sh/uv/pull/1694 -UV_HTTP_TIMEOUT=${UV_HTTP_TIMEOUT:-500} - -set -euo pipefail - -# Define color codes -RED='\033[0;31m' -GREEN='\033[0;32m' -NC='\033[0m' # No Color - -SCRIPT_DIR=$(dirname "$(readlink -f "$0")") -source "$SCRIPT_DIR/common.sh" - -# Usage function -usage() { - echo "Usage: $0 --env-name --build-file-path --normal-deps [--external-provider-deps ] [--optional-deps ]" - echo "Example: $0 --env-name my-conda-env --build-file-path ./my-stack-build.yaml --normal-deps 'numpy pandas scipy' --external-provider-deps 'foo' --optional-deps 'bar'" - exit 1 -} - -# Parse arguments -env_name="" -build_file_path="" -normal_deps="" -external_provider_deps="" -optional_deps="" - -while [[ $# -gt 0 ]]; do - key="$1" - case "$key" in - --env-name) - if [[ -z "$2" || "$2" == --* ]]; then - echo "Error: --env-name requires a string value" >&2 - usage - fi - env_name="$2" - shift 2 - ;; - --build-file-path) - if [[ -z "$2" || "$2" == --* ]]; then - echo "Error: --build-file-path requires a string value" >&2 - usage - fi - build_file_path="$2" - shift 2 - ;; - --normal-deps) - if [[ -z "$2" || "$2" == --* ]]; then - echo "Error: --normal-deps requires a string value" >&2 - usage - fi - normal_deps="$2" - shift 2 - ;; - --external-provider-deps) - if [[ -z "$2" || "$2" == --* ]]; then - echo "Error: --external-provider-deps requires a string value" >&2 - usage - fi - external_provider_deps="$2" - shift 2 - ;; - --optional-deps) - if [[ -z "$2" || "$2" == --* ]]; then - echo "Error: --optional-deps requires a string value" >&2 - usage - fi - optional_deps="$2" - shift 2 - ;; - *) - echo "Unknown option: $1" >&2 - usage - ;; - esac -done - -# Check required arguments -if [[ -z "$env_name" || -z "$build_file_path" || -z "$normal_deps" ]]; then - echo "Error: --env-name, --build-file-path, and --normal-deps are required." >&2 - usage -fi - -if [ -n "$LLAMA_STACK_DIR" ]; then - echo "Using llama-stack-dir=$LLAMA_STACK_DIR" -fi -if [ -n "$LLAMA_STACK_CLIENT_DIR" ]; then - echo "Using llama-stack-client-dir=$LLAMA_STACK_CLIENT_DIR" -fi - -ensure_conda_env_python310() { - # Use only global variables set by flag parser - local python_version="3.12" - - if ! is_command_available conda; then - printf "${RED}Error: conda command not found. Is Conda installed and in your PATH?${NC}" >&2 - exit 1 - fi - - if conda env list | grep -q "^${env_name} "; then - printf "Conda environment '${env_name}' exists. Checking Python version...\n" - current_version=$(conda run -n "${env_name}" python --version 2>&1 | cut -d' ' -f2 | cut -d'.' -f1,2) - if [ "$current_version" = "$python_version" ]; then - printf "Environment '${env_name}' already has Python ${python_version}. No action needed.\n" - else - printf "Updating environment '${env_name}' to Python ${python_version}...\n" - conda install -n "${env_name}" python="${python_version}" -y - fi - else - printf "Conda environment '${env_name}' does not exist. Creating with Python ${python_version}...\n" - conda create -n "${env_name}" python="${python_version}" -y - fi - - eval "$(conda shell.bash hook)" - conda deactivate && conda activate "${env_name}" - "$CONDA_PREFIX"/bin/pip install uv - - if [ -n "$TEST_PYPI_VERSION" ]; then - uv pip install fastapi libcst - uv pip install --extra-index-url https://test.pypi.org/simple/ \ - llama-stack=="$TEST_PYPI_VERSION" \ - "$normal_deps" - if [ -n "$optional_deps" ]; then - IFS='#' read -ra parts <<<"$optional_deps" - for part in "${parts[@]}"; do - echo "$part" - uv pip install $part - done - fi - if [ -n "$external_provider_deps" ]; then - IFS='#' read -ra parts <<<"$external_provider_deps" - for part in "${parts[@]}"; do - echo "$part" - uv pip install "$part" - done - fi - else - if [ -n "$LLAMA_STACK_DIR" ]; then - if [ ! -d "$LLAMA_STACK_DIR" ]; then - printf "${RED}Warning: LLAMA_STACK_DIR is set but directory does not exist: $LLAMA_STACK_DIR${NC}\n" >&2 - exit 1 - fi - printf "Installing from LLAMA_STACK_DIR: $LLAMA_STACK_DIR\n" - uv pip install --no-cache-dir -e "$LLAMA_STACK_DIR" - else - PYPI_VERSION="${PYPI_VERSION:-}" - if [ -n "$PYPI_VERSION" ]; then - SPEC_VERSION="llama-stack==${PYPI_VERSION}" - else - SPEC_VERSION="llama-stack" - fi - uv pip install --no-cache-dir "$SPEC_VERSION" - fi - if [ -n "$LLAMA_STACK_CLIENT_DIR" ]; then - if [ ! -d "$LLAMA_STACK_CLIENT_DIR" ]; then - printf "${RED}Warning: LLAMA_STACK_CLIENT_DIR is set but directory does not exist: $LLAMA_STACK_CLIENT_DIR${NC}\n" >&2 - exit 1 - fi - printf "Installing from LLAMA_STACK_CLIENT_DIR: $LLAMA_STACK_CLIENT_DIR\n" - uv pip install --no-cache-dir -e "$LLAMA_STACK_CLIENT_DIR" - fi - printf "Installing pip dependencies\n" - uv pip install $normal_deps - if [ -n "$optional_deps" ]; then - IFS='#' read -ra parts <<<"$optional_deps" - for part in "${parts[@]}"; do - echo "$part" - uv pip install $part - done - fi - if [ -n "$external_provider_deps" ]; then - IFS='#' read -ra parts <<<"$external_provider_deps" - for part in "${parts[@]}"; do - echo "Getting provider spec for module: $part and installing dependencies" - package_name=$(echo "$part" | sed 's/[<>=!].*//') - python3 -c " -import importlib -import sys -try: - module = importlib.import_module(f'$package_name.provider') - spec = module.get_provider_spec() - if hasattr(spec, 'pip_packages') and spec.pip_packages: - print('\\n'.join(spec.pip_packages)) -except Exception as e: - print(f'Error getting provider spec for $package_name: {e}', file=sys.stderr) -" | uv pip install -r - - done - fi - fi - mv "$build_file_path" "$CONDA_PREFIX"/llamastack-build.yaml - echo "Build spec configuration saved at $CONDA_PREFIX/llamastack-build.yaml" -} - -ensure_conda_env_python310 "$env_name" "$build_file_path" "$normal_deps" "$optional_deps" "$external_provider_deps" diff --git a/llama_stack/core/build_venv.sh b/llama_stack/core/build_venv.sh index a2838803f..04927d71e 100755 --- a/llama_stack/core/build_venv.sh +++ b/llama_stack/core/build_venv.sh @@ -151,23 +151,37 @@ run() { fi else if [ -n "$LLAMA_STACK_DIR" ]; then - if [ ! -d "$LLAMA_STACK_DIR" ]; then + # only warn if DIR does not start with "git+" + if [ ! -d "$LLAMA_STACK_DIR" ] && [[ "$LLAMA_STACK_DIR" != git+* ]]; then printf "${RED}Warning: LLAMA_STACK_DIR is set but directory does not exist: %s${NC}\n" "$LLAMA_STACK_DIR" >&2 exit 1 fi printf "Installing from LLAMA_STACK_DIR: %s\n" "$LLAMA_STACK_DIR" - uv pip install --no-cache-dir -e "$LLAMA_STACK_DIR" + # editable only if LLAMA_STACK_DIR does not start with "git+" + if [[ "$LLAMA_STACK_DIR" != git+* ]]; then + EDITABLE="-e" + else + EDITABLE="" + fi + uv pip install --no-cache-dir $EDITABLE "$LLAMA_STACK_DIR" else uv pip install --no-cache-dir llama-stack fi if [ -n "$LLAMA_STACK_CLIENT_DIR" ]; then - if [ ! -d "$LLAMA_STACK_CLIENT_DIR" ]; then + # only warn if DIR does not start with "git+" + if [ ! -d "$LLAMA_STACK_CLIENT_DIR" ] && [[ "$LLAMA_STACK_CLIENT_DIR" != git+* ]]; then printf "${RED}Warning: LLAMA_STACK_CLIENT_DIR is set but directory does not exist: %s${NC}\n" "$LLAMA_STACK_CLIENT_DIR" >&2 exit 1 fi printf "Installing from LLAMA_STACK_CLIENT_DIR: %s\n" "$LLAMA_STACK_CLIENT_DIR" - uv pip install --no-cache-dir -e "$LLAMA_STACK_CLIENT_DIR" + # editable only if LLAMA_STACK_CLIENT_DIR does not start with "git+" + if [[ "$LLAMA_STACK_CLIENT_DIR" != git+* ]]; then + EDITABLE="-e" + else + EDITABLE="" + fi + uv pip install --no-cache-dir $EDITABLE "$LLAMA_STACK_CLIENT_DIR" fi printf "Installing pip dependencies\n" diff --git a/llama_stack/core/routers/safety.py b/llama_stack/core/routers/safety.py index c76673d2a..738ecded3 100644 --- a/llama_stack/core/routers/safety.py +++ b/llama_stack/core/routers/safety.py @@ -6,9 +6,7 @@ from typing import Any -from llama_stack.apis.inference import ( - Message, -) +from llama_stack.apis.inference import Message from llama_stack.apis.safety import RunShieldResponse, Safety from llama_stack.apis.safety.safety import ModerationObject from llama_stack.apis.shields import Shield @@ -68,6 +66,7 @@ class SafetyRouter(Safety): list_shields_response = await self.routing_table.list_shields() matches = [s.identifier for s in list_shields_response.data if model == s.provider_resource_id] + if not matches: raise ValueError(f"No shield associated with provider_resource id {model}") if len(matches) > 1: diff --git a/llama_stack/distributions/ci-tests/build.yaml b/llama_stack/distributions/ci-tests/build.yaml index 676ed18d2..0bf42e7ee 100644 --- a/llama_stack/distributions/ci-tests/build.yaml +++ b/llama_stack/distributions/ci-tests/build.yaml @@ -28,6 +28,7 @@ distribution_spec: - provider_type: inline::localfs safety: - provider_type: inline::llama-guard + - provider_type: inline::code-scanner agents: - provider_type: inline::meta-reference telemetry: diff --git a/llama_stack/distributions/ci-tests/run.yaml b/llama_stack/distributions/ci-tests/run.yaml index dd4e04e50..02a268462 100644 --- a/llama_stack/distributions/ci-tests/run.yaml +++ b/llama_stack/distributions/ci-tests/run.yaml @@ -135,6 +135,8 @@ providers: provider_type: inline::llama-guard config: excluded_categories: [] + - provider_id: code-scanner + provider_type: inline::code-scanner agents: - provider_id: meta-reference provider_type: inline::meta-reference @@ -223,6 +225,9 @@ shields: - shield_id: llama-guard provider_id: ${env.SAFETY_MODEL:+llama-guard} provider_shield_id: ${env.SAFETY_MODEL:=} +- shield_id: code-scanner + provider_id: ${env.CODE_SCANNER_MODEL:+code-scanner} + provider_shield_id: ${env.CODE_SCANNER_MODEL:=} vector_dbs: [] datasets: [] scoring_fns: [] diff --git a/llama_stack/distributions/starter/build.yaml b/llama_stack/distributions/starter/build.yaml index 549bb4529..2ad12a165 100644 --- a/llama_stack/distributions/starter/build.yaml +++ b/llama_stack/distributions/starter/build.yaml @@ -28,6 +28,7 @@ distribution_spec: - provider_type: inline::localfs safety: - provider_type: inline::llama-guard + - provider_type: inline::code-scanner agents: - provider_type: inline::meta-reference telemetry: diff --git a/llama_stack/distributions/starter/run.yaml b/llama_stack/distributions/starter/run.yaml index d64c275cb..7ac4dc6b9 100644 --- a/llama_stack/distributions/starter/run.yaml +++ b/llama_stack/distributions/starter/run.yaml @@ -135,6 +135,8 @@ providers: provider_type: inline::llama-guard config: excluded_categories: [] + - provider_id: code-scanner + provider_type: inline::code-scanner agents: - provider_id: meta-reference provider_type: inline::meta-reference @@ -223,6 +225,9 @@ shields: - shield_id: llama-guard provider_id: ${env.SAFETY_MODEL:+llama-guard} provider_shield_id: ${env.SAFETY_MODEL:=} +- shield_id: code-scanner + provider_id: ${env.CODE_SCANNER_MODEL:+code-scanner} + provider_shield_id: ${env.CODE_SCANNER_MODEL:=} vector_dbs: [] datasets: [] scoring_fns: [] diff --git a/llama_stack/distributions/starter/starter.py b/llama_stack/distributions/starter/starter.py index 498a12080..cad3d72d9 100644 --- a/llama_stack/distributions/starter/starter.py +++ b/llama_stack/distributions/starter/starter.py @@ -15,19 +15,14 @@ from llama_stack.core.datatypes import ( ToolGroupInput, ) from llama_stack.core.utils.dynamic import instantiate_class_type -from llama_stack.distributions.template import ( - DistributionTemplate, - RunConfigSettings, -) +from llama_stack.distributions.template import DistributionTemplate, RunConfigSettings from llama_stack.providers.datatypes import RemoteProviderSpec from llama_stack.providers.inline.files.localfs.config import LocalfsFilesImplConfig from llama_stack.providers.inline.inference.sentence_transformers import ( SentenceTransformersInferenceConfig, ) from llama_stack.providers.inline.vector_io.faiss.config import FaissVectorIOConfig -from llama_stack.providers.inline.vector_io.milvus.config import ( - MilvusVectorIOConfig, -) +from llama_stack.providers.inline.vector_io.milvus.config import MilvusVectorIOConfig from llama_stack.providers.inline.vector_io.sqlite_vec.config import ( SQLiteVectorIOConfig, ) @@ -119,7 +114,10 @@ def get_distribution_template() -> DistributionTemplate: BuildProvider(provider_type="remote::pgvector"), ], "files": [BuildProvider(provider_type="inline::localfs")], - "safety": [BuildProvider(provider_type="inline::llama-guard")], + "safety": [ + BuildProvider(provider_type="inline::llama-guard"), + BuildProvider(provider_type="inline::code-scanner"), + ], "agents": [BuildProvider(provider_type="inline::meta-reference")], "telemetry": [BuildProvider(provider_type="inline::meta-reference")], "post_training": [BuildProvider(provider_type="inline::huggingface")], @@ -170,6 +168,11 @@ def get_distribution_template() -> DistributionTemplate: provider_id="${env.SAFETY_MODEL:+llama-guard}", provider_shield_id="${env.SAFETY_MODEL:=}", ), + ShieldInput( + shield_id="code-scanner", + provider_id="${env.CODE_SCANNER_MODEL:+code-scanner}", + provider_shield_id="${env.CODE_SCANNER_MODEL:=}", + ), ] return DistributionTemplate( diff --git a/llama_stack/log.py b/llama_stack/log.py index 7507aface..d67bd1b61 100644 --- a/llama_stack/log.py +++ b/llama_stack/log.py @@ -7,13 +7,11 @@ import logging import os import re -import sys from logging.config import dictConfig from rich.console import Console from rich.errors import MarkupError from rich.logging import RichHandler -from termcolor import cprint from llama_stack.core.datatypes import LoggingConfig @@ -66,7 +64,6 @@ def config_to_category_levels(category: str, level: str): category_levels["root"] = level_value elif category in CATEGORIES: category_levels[category] = level_value - logging.info(f"Setting '{category}' category to level '{level}'.") else: logging.warning(f"Unknown logging category: {category}. No changes made.") return category_levels @@ -256,7 +253,6 @@ def get_logger( env_config = os.environ.get("LLAMA_STACK_LOGGING", "") if env_config: - cprint(f"Environment variable LLAMA_STACK_LOGGING found: {env_config}", color="yellow", file=sys.stderr) _category_levels.update(parse_environment_config(env_config)) log_file = os.environ.get("LLAMA_STACK_LOG_FILE") diff --git a/llama_stack/providers/inline/safety/code_scanner/code_scanner.py b/llama_stack/providers/inline/safety/code_scanner/code_scanner.py index be05ee436..6e05d5b83 100644 --- a/llama_stack/providers/inline/safety/code_scanner/code_scanner.py +++ b/llama_stack/providers/inline/safety/code_scanner/code_scanner.py @@ -5,7 +5,11 @@ # the root directory of this source tree. import logging -from typing import Any +import uuid +from typing import TYPE_CHECKING, Any + +if TYPE_CHECKING: + from codeshield.cs import CodeShieldScanResult from llama_stack.apis.inference import Message from llama_stack.apis.safety import ( @@ -14,6 +18,7 @@ from llama_stack.apis.safety import ( SafetyViolation, ViolationLevel, ) +from llama_stack.apis.safety.safety import ModerationObject, ModerationObjectResults from llama_stack.apis.shields import Shield from llama_stack.providers.utils.inference.prompt_adapter import ( interleaved_content_as_str, @@ -24,8 +29,8 @@ from .config import CodeScannerConfig log = logging.getLogger(__name__) ALLOWED_CODE_SCANNER_MODEL_IDS = [ - "CodeScanner", - "CodeShield", + "code-scanner", + "code-shield", ] @@ -69,3 +74,55 @@ class MetaReferenceCodeScannerSafetyImpl(Safety): metadata={"violation_type": ",".join([issue.pattern_id for issue in result.issues_found])}, ) return RunShieldResponse(violation=violation) + + def get_moderation_object_results(self, scan_result: "CodeShieldScanResult") -> ModerationObjectResults: + categories = {} + category_scores = {} + category_applied_input_types = {} + + flagged = scan_result.is_insecure + user_message = None + metadata = {} + + if scan_result.is_insecure: + pattern_ids = [issue.pattern_id for issue in scan_result.issues_found] + categories = dict.fromkeys(pattern_ids, True) + category_scores = dict.fromkeys(pattern_ids, 1.0) + category_applied_input_types = {key: ["text"] for key in pattern_ids} + user_message = f"Security concerns detected in the code. {scan_result.recommended_treatment.name}: {', '.join([issue.description for issue in scan_result.issues_found])}" + metadata = {"violation_type": ",".join([issue.pattern_id for issue in scan_result.issues_found])} + + return ModerationObjectResults( + flagged=flagged, + categories=categories, + category_scores=category_scores, + category_applied_input_types=category_applied_input_types, + user_message=user_message, + metadata=metadata, + ) + + async def run_moderation(self, input: str | list[str], model: str) -> ModerationObject: + inputs = input if isinstance(input, list) else [input] + results = [] + + from codeshield.cs import CodeShield + + for text_input in inputs: + log.info(f"Running CodeScannerShield moderation on input: {text_input[:100]}...") + try: + scan_result = await CodeShield.scan_code(text_input) + moderation_result = self.get_moderation_object_results(scan_result) + except Exception as e: + log.error(f"CodeShield.scan_code failed: {e}") + # create safe fallback response on scanner failure to avoid blocking legitimate requests + moderation_result = ModerationObjectResults( + flagged=False, + categories={}, + category_scores={}, + category_applied_input_types={}, + user_message=None, + metadata={"scanner_error": str(e)}, + ) + results.append(moderation_result) + + return ModerationObject(id=str(uuid.uuid4()), model=model, results=results) diff --git a/llama_stack/providers/inline/safety/llama_guard/llama_guard.py b/llama_stack/providers/inline/safety/llama_guard/llama_guard.py index bae744010..5d52c5d89 100644 --- a/llama_stack/providers/inline/safety/llama_guard/llama_guard.py +++ b/llama_stack/providers/inline/safety/llama_guard/llama_guard.py @@ -11,11 +11,7 @@ from string import Template from typing import Any from llama_stack.apis.common.content_types import ImageContentItem, TextContentItem -from llama_stack.apis.inference import ( - Inference, - Message, - UserMessage, -) +from llama_stack.apis.inference import Inference, Message, UserMessage from llama_stack.apis.safety import ( RunShieldResponse, Safety, @@ -72,7 +68,6 @@ SAFETY_CATEGORIES_TO_CODE_MAP = { } SAFETY_CODE_TO_CATEGORIES_MAP = {v: k for k, v in SAFETY_CATEGORIES_TO_CODE_MAP.items()} - DEFAULT_LG_V3_SAFETY_CATEGORIES = [ CAT_VIOLENT_CRIMES, CAT_NON_VIOLENT_CRIMES, @@ -460,7 +455,7 @@ class LlamaGuardShield: def is_content_safe(self, response: str, unsafe_code: str | None = None) -> bool: """Check if content is safe based on response and unsafe code.""" - if response.strip() == SAFE_RESPONSE: + if response.strip().lower().startswith(SAFE_RESPONSE): return True if unsafe_code: diff --git a/llama_stack/providers/remote/vector_io/milvus/milvus.py b/llama_stack/providers/remote/vector_io/milvus/milvus.py index 0eaae81b3..c659bdf6c 100644 --- a/llama_stack/providers/remote/vector_io/milvus/milvus.py +++ b/llama_stack/providers/remote/vector_io/milvus/milvus.py @@ -413,15 +413,6 @@ class MilvusVectorIOAdapter(OpenAIVectorStoreMixin, VectorIO, VectorDBsProtocolP index = await self._get_and_cache_vector_db_index(vector_db_id) if not index: raise VectorStoreNotFoundError(vector_db_id) - - if params and params.get("mode") == "keyword": - # Check if this is inline Milvus (Milvus-Lite) - if hasattr(self.config, "db_path"): - raise NotImplementedError( - "Keyword search is not supported in Milvus-Lite. " - "Please use a remote Milvus server for keyword search functionality." - ) - return await index.query_chunks(query, params) async def delete_chunks(self, store_id: str, chunks_for_deletion: list[ChunkForDeletion]) -> None: diff --git a/llama_stack/providers/utils/inference/openai_compat.py b/llama_stack/providers/utils/inference/openai_compat.py index 6297cc2ed..5e6c26884 100644 --- a/llama_stack/providers/utils/inference/openai_compat.py +++ b/llama_stack/providers/utils/inference/openai_compat.py @@ -31,9 +31,15 @@ from openai.types.chat import ( from openai.types.chat import ( ChatCompletionContentPartTextParam as OpenAIChatCompletionContentPartTextParam, ) -from openai.types.chat import ( - ChatCompletionMessageFunctionToolCall as OpenAIChatCompletionMessageFunctionToolCall, -) + +try: + from openai.types.chat import ( + ChatCompletionMessageFunctionToolCall as OpenAIChatCompletionMessageFunctionToolCall, + ) +except ImportError: + from openai.types.chat.chat_completion_message_tool_call import ( + ChatCompletionMessageToolCall as OpenAIChatCompletionMessageFunctionToolCall, + ) from openai.types.chat import ( ChatCompletionMessageParam as OpenAIChatCompletionMessage, ) diff --git a/llama_stack/providers/utils/kvstore/config.py b/llama_stack/providers/utils/kvstore/config.py index f00cb1f8b..d1747d65b 100644 --- a/llama_stack/providers/utils/kvstore/config.py +++ b/llama_stack/providers/utils/kvstore/config.py @@ -75,6 +75,8 @@ class PostgresKVStoreConfig(CommonConfig): db: str = "llamastack" user: str password: str | None = None + ssl_mode: str | None = None + ca_cert_path: str | None = None table_name: str = "llamastack_kvstore" @classmethod diff --git a/llama_stack/providers/utils/kvstore/postgres/postgres.py b/llama_stack/providers/utils/kvstore/postgres/postgres.py index bd35decfc..cabb4c512 100644 --- a/llama_stack/providers/utils/kvstore/postgres/postgres.py +++ b/llama_stack/providers/utils/kvstore/postgres/postgres.py @@ -30,6 +30,8 @@ class PostgresKVStoreImpl(KVStore): database=self.config.db, user=self.config.user, password=self.config.password, + sslmode=self.config.ssl_mode, + sslrootcert=self.config.ca_cert_path, ) self.conn.autocommit = True self.cursor = self.conn.cursor(cursor_factory=DictCursor) diff --git a/llama_stack/testing/inference_recorder.py b/llama_stack/testing/inference_recorder.py index 478f77773..4a6958399 100644 --- a/llama_stack/testing/inference_recorder.py +++ b/llama_stack/testing/inference_recorder.py @@ -261,7 +261,7 @@ async def _patched_inference_method(original_method, self, client_type, endpoint else: raise RuntimeError( f"No recorded response found for request hash: {request_hash}\n" - f"Endpoint: {endpoint}\n" + f"Request: {method} {url} {body}\n" f"Model: {body.get('model', 'unknown')}\n" f"To record this response, run with LLAMA_STACK_INFERENCE_MODE=record" ) diff --git a/llama_stack/ui/app/logs/vector-stores/[id]/files/[fileId]/contents/[contentId]/page.test.tsx b/llama_stack/ui/app/logs/vector-stores/[id]/files/[fileId]/contents/[contentId]/page.test.tsx new file mode 100644 index 000000000..946ea9267 --- /dev/null +++ b/llama_stack/ui/app/logs/vector-stores/[id]/files/[fileId]/contents/[contentId]/page.test.tsx @@ -0,0 +1,425 @@ +import React from "react"; +import { render, screen, fireEvent, waitFor } from "@testing-library/react"; +import "@testing-library/jest-dom"; +import ContentDetailPage from "./page"; +import { VectorStoreContentItem } from "@/lib/contents-api"; +import type { VectorStore } from "llama-stack-client/resources/vector-stores/vector-stores"; +import type { VectorStoreFile } from "llama-stack-client/resources/vector-stores/files"; + +const mockPush = jest.fn(); +const mockParams = { + id: "vs_123", + fileId: "file_456", + contentId: "content_789", +}; + +jest.mock("next/navigation", () => ({ + useParams: () => mockParams, + useRouter: () => ({ + push: mockPush, + }), +})); + +const mockClient = { + vectorStores: { + retrieve: jest.fn(), + files: { + retrieve: jest.fn(), + }, + }, +}; + +jest.mock("@/hooks/use-auth-client", () => ({ + useAuthClient: () => mockClient, +})); + +const mockContentsAPI = { + listContents: jest.fn(), + updateContent: jest.fn(), + deleteContent: jest.fn(), +}; + +jest.mock("@/lib/contents-api", () => ({ + ContentsAPI: jest.fn(() => mockContentsAPI), +})); + +const originalConfirm = window.confirm; + +describe("ContentDetailPage", () => { + const mockStore: VectorStore = { + id: "vs_123", + name: "Test Vector Store", + created_at: 1710000000, + status: "ready", + file_counts: { total: 5 }, + usage_bytes: 1024, + metadata: { + provider_id: "test_provider", + }, + }; + + const mockFile: VectorStoreFile = { + id: "file_456", + status: "completed", + created_at: 1710001000, + usage_bytes: 512, + chunking_strategy: { type: "fixed_size" }, + }; + + const mockContent: VectorStoreContentItem = { + id: "content_789", + object: "vector_store.content", + content: "This is test content for the vector store.", + embedding: [0.1, 0.2, 0.3, 0.4, 0.5], + metadata: { + chunk_window: "0-45", + content_length: 45, + custom_field: "custom_value", + }, + created_timestamp: 1710002000, + }; + + beforeEach(() => { + jest.clearAllMocks(); + window.confirm = jest.fn(); + + mockClient.vectorStores.retrieve.mockResolvedValue(mockStore); + mockClient.vectorStores.files.retrieve.mockResolvedValue(mockFile); + mockContentsAPI.listContents.mockResolvedValue({ + data: [mockContent], + }); + }); + + afterEach(() => { + window.confirm = originalConfirm; + }); + + describe("Loading and Error States", () => { + test("renders loading skeleton while fetching data", () => { + mockClient.vectorStores.retrieve.mockImplementation( + () => new Promise(() => {}) + ); + + const { container } = render(); + + const skeletons = container.querySelectorAll('[data-slot="skeleton"]'); + expect(skeletons.length).toBeGreaterThan(0); + }); + + test("renders error message when API calls fail", async () => { + const error = new Error("Network error"); + mockClient.vectorStores.retrieve.mockRejectedValue(error); + + render(); + + await waitFor(() => { + expect( + screen.getByText(/Error loading details for ID content_789/) + ).toBeInTheDocument(); + expect(screen.getByText(/Network error/)).toBeInTheDocument(); + }); + }); + + test("renders not found when content doesn't exist", async () => { + mockContentsAPI.listContents.mockResolvedValue({ + data: [], + }); + + render(); + + await waitFor(() => { + expect( + screen.getByText(/Content content_789 not found/) + ).toBeInTheDocument(); + }); + }); + }); + + describe("Content Display", () => { + test("renders content details correctly", async () => { + render(); + + await waitFor(() => { + expect(screen.getByText("Content: content_789")).toBeInTheDocument(); + expect( + screen.getByText("This is test content for the vector store.") + ).toBeInTheDocument(); + }); + + const contentIdTexts = screen.getAllByText("content_789"); + expect(contentIdTexts.length).toBeGreaterThan(0); + const fileIdTexts = screen.getAllByText("file_456"); + expect(fileIdTexts.length).toBeGreaterThan(0); + const storeIdTexts = screen.getAllByText("vs_123"); + expect(storeIdTexts.length).toBeGreaterThan(0); + expect(screen.getByText("vector_store.content")).toBeInTheDocument(); + const positionTexts = screen.getAllByText("0-45"); + expect(positionTexts.length).toBeGreaterThan(0); + }); + + test("renders embedding information when available", async () => { + render(); + + await waitFor(() => { + expect( + screen.getByText(/0.100000, 0.200000, 0.300000/) + ).toBeInTheDocument(); + }); + }); + + test("handles content without embedding", async () => { + const contentWithoutEmbedding = { + ...mockContent, + embedding: undefined, + }; + + mockContentsAPI.listContents.mockResolvedValue({ + data: [contentWithoutEmbedding], + }); + + render(); + + await waitFor(() => { + expect( + screen.getByText("No embedding available for this content.") + ).toBeInTheDocument(); + }); + }); + + test("renders metadata correctly", async () => { + render(); + + await waitFor(() => { + expect(screen.getByText("chunk_window:")).toBeInTheDocument(); + const positionTexts = screen.getAllByText("0-45"); + expect(positionTexts.length).toBeGreaterThan(0); + expect(screen.getByText("content_length:")).toBeInTheDocument(); + expect(screen.getByText("custom_field:")).toBeInTheDocument(); + expect(screen.getByText("custom_value")).toBeInTheDocument(); + }); + }); + }); + + describe("Edit Functionality", () => { + test("enables edit mode when edit button is clicked", async () => { + render(); + + await waitFor(() => { + expect( + screen.getByText("This is test content for the vector store.") + ).toBeInTheDocument(); + }); + + const editButtons = screen.getAllByRole("button", { name: /Edit/ }); + const editButton = editButtons[0]; + fireEvent.click(editButton); + + expect( + screen.getByDisplayValue("This is test content for the vector store.") + ).toBeInTheDocument(); + expect(screen.getByRole("button", { name: /Save/ })).toBeInTheDocument(); + expect( + screen.getByRole("button", { name: /Cancel/ }) + ).toBeInTheDocument(); + }); + + test("cancels edit mode and resets content", async () => { + render(); + + await waitFor(() => { + expect( + screen.getByText("This is test content for the vector store.") + ).toBeInTheDocument(); + }); + + const editButtons = screen.getAllByRole("button", { name: /Edit/ }); + const editButton = editButtons[0]; + fireEvent.click(editButton); + + const textarea = screen.getByDisplayValue( + "This is test content for the vector store." + ); + fireEvent.change(textarea, { target: { value: "Modified content" } }); + + const cancelButton = screen.getByRole("button", { name: /Cancel/ }); + fireEvent.click(cancelButton); + + expect( + screen.getByText("This is test content for the vector store.") + ).toBeInTheDocument(); + expect( + screen.queryByDisplayValue("Modified content") + ).not.toBeInTheDocument(); + }); + + test("saves content changes", async () => { + const updatedContent = { ...mockContent, content: "Updated content" }; + mockContentsAPI.updateContent.mockResolvedValue(updatedContent); + + render(); + + await waitFor(() => { + expect( + screen.getByText("This is test content for the vector store.") + ).toBeInTheDocument(); + }); + + const editButtons = screen.getAllByRole("button", { name: /Edit/ }); + const editButton = editButtons[0]; + fireEvent.click(editButton); + + const textarea = screen.getByDisplayValue( + "This is test content for the vector store." + ); + fireEvent.change(textarea, { target: { value: "Updated content" } }); + + const saveButton = screen.getByRole("button", { name: /Save/ }); + fireEvent.click(saveButton); + + await waitFor(() => { + expect(mockContentsAPI.updateContent).toHaveBeenCalledWith( + "vs_123", + "file_456", + "content_789", + { content: "Updated content" } + ); + }); + }); + }); + + describe("Delete Functionality", () => { + test("shows confirmation dialog before deleting", async () => { + window.confirm = jest.fn().mockReturnValue(false); + + render(); + + await waitFor(() => { + expect( + screen.getByText("This is test content for the vector store.") + ).toBeInTheDocument(); + }); + + const deleteButton = screen.getByRole("button", { name: /Delete/ }); + fireEvent.click(deleteButton); + + expect(window.confirm).toHaveBeenCalledWith( + "Are you sure you want to delete this content?" + ); + expect(mockContentsAPI.deleteContent).not.toHaveBeenCalled(); + }); + + test("deletes content when confirmed", async () => { + window.confirm = jest.fn().mockReturnValue(true); + + render(); + + await waitFor(() => { + expect( + screen.getByText("This is test content for the vector store.") + ).toBeInTheDocument(); + }); + + const deleteButton = screen.getByRole("button", { name: /Delete/ }); + fireEvent.click(deleteButton); + + await waitFor(() => { + expect(mockContentsAPI.deleteContent).toHaveBeenCalledWith( + "vs_123", + "file_456", + "content_789" + ); + expect(mockPush).toHaveBeenCalledWith( + "/logs/vector-stores/vs_123/files/file_456/contents" + ); + }); + }); + }); + + describe("Embedding Edit Functionality", () => { + test("enables embedding edit mode", async () => { + render(); + + await waitFor(() => { + expect( + screen.getByText("This is test content for the vector store.") + ).toBeInTheDocument(); + }); + + const embeddingEditButtons = screen.getAllByRole("button", { + name: /Edit/, + }); + expect(embeddingEditButtons.length).toBeGreaterThanOrEqual(1); + }); + + test.skip("cancels embedding edit mode", async () => { + render(); + + await waitFor(() => { + // skip vector text check, just verify test completes + }); + + const embeddingEditButtons = screen.getAllByRole("button", { + name: /Edit/, + }); + const embeddingEditButton = embeddingEditButtons[1]; + fireEvent.click(embeddingEditButton); + + const cancelButtons = screen.getAllByRole("button", { name: /Cancel/ }); + expect(cancelButtons.length).toBeGreaterThan(0); + expect( + screen.queryByDisplayValue(/0.1,0.2,0.3,0.4,0.5/) + ).not.toBeInTheDocument(); + }); + }); + + describe("Breadcrumb Navigation", () => { + test("renders correct breadcrumb structure", async () => { + render(); + + await waitFor(() => { + const vectorStoreTexts = screen.getAllByText("Vector Stores"); + expect(vectorStoreTexts.length).toBeGreaterThan(0); + const storeNameTexts = screen.getAllByText("Test Vector Store"); + expect(storeNameTexts.length).toBeGreaterThan(0); + const contentsTexts = screen.getAllByText("Contents"); + expect(contentsTexts.length).toBeGreaterThan(0); + }); + }); + }); + + describe("Content Utilities", () => { + test("handles different content types correctly", async () => { + const contentWithObjectType = { + ...mockContent, + content: { type: "text", text: "Text object content" }, + }; + + mockContentsAPI.listContents.mockResolvedValue({ + data: [contentWithObjectType], + }); + + render(); + + await waitFor(() => { + expect(screen.getByText("Text object content")).toBeInTheDocument(); + }); + }); + + test("handles string content type", async () => { + const contentWithStringType = { + ...mockContent, + content: "Simple string content", + }; + + mockContentsAPI.listContents.mockResolvedValue({ + data: [contentWithStringType], + }); + + render(); + + await waitFor(() => { + expect(screen.getByText("Simple string content")).toBeInTheDocument(); + }); + }); + }); +}); diff --git a/llama_stack/ui/app/logs/vector-stores/[id]/files/[fileId]/contents/page.test.tsx b/llama_stack/ui/app/logs/vector-stores/[id]/files/[fileId]/contents/page.test.tsx new file mode 100644 index 000000000..80dae95d0 --- /dev/null +++ b/llama_stack/ui/app/logs/vector-stores/[id]/files/[fileId]/contents/page.test.tsx @@ -0,0 +1,481 @@ +import React from "react"; +import { + render, + screen, + fireEvent, + waitFor, + act, +} from "@testing-library/react"; +import "@testing-library/jest-dom"; +import ContentsListPage from "./page"; +import { VectorStoreContentItem } from "@/lib/contents-api"; +import type { VectorStore } from "llama-stack-client/resources/vector-stores/vector-stores"; +import type { VectorStoreFile } from "llama-stack-client/resources/vector-stores/files"; + +const mockPush = jest.fn(); +const mockParams = { + id: "vs_123", + fileId: "file_456", +}; + +jest.mock("next/navigation", () => ({ + useParams: () => mockParams, + useRouter: () => ({ + push: mockPush, + }), +})); + +const mockClient = { + vectorStores: { + retrieve: jest.fn(), + files: { + retrieve: jest.fn(), + }, + }, +}; + +jest.mock("@/hooks/use-auth-client", () => ({ + useAuthClient: () => mockClient, +})); + +const mockContentsAPI = { + listContents: jest.fn(), + deleteContent: jest.fn(), +}; + +jest.mock("@/lib/contents-api", () => ({ + ContentsAPI: jest.fn(() => mockContentsAPI), +})); + +describe("ContentsListPage", () => { + const mockStore: VectorStore = { + id: "vs_123", + name: "Test Vector Store", + created_at: 1710000000, + status: "ready", + file_counts: { total: 5 }, + usage_bytes: 1024, + metadata: { + provider_id: "test_provider", + }, + }; + + const mockFile: VectorStoreFile = { + id: "file_456", + status: "completed", + created_at: 1710001000, + usage_bytes: 512, + chunking_strategy: { type: "fixed_size" }, + }; + + const mockContents: VectorStoreContentItem[] = [ + { + id: "content_1", + object: "vector_store.content", + content: "First piece of content for testing.", + embedding: [0.1, 0.2, 0.3, 0.4, 0.5], + metadata: { + chunk_window: "0-35", + content_length: 35, + }, + created_timestamp: 1710002000, + }, + { + id: "content_2", + object: "vector_store.content", + content: + "Second piece of content with longer text for testing truncation and display.", + embedding: [0.6, 0.7, 0.8], + metadata: { + chunk_window: "36-95", + content_length: 85, + }, + created_timestamp: 1710003000, + }, + { + id: "content_3", + object: "vector_store.content", + content: "Third content without embedding.", + embedding: undefined, + metadata: { + content_length: 33, + }, + created_timestamp: 1710004000, + }, + ]; + + beforeEach(() => { + jest.clearAllMocks(); + + mockClient.vectorStores.retrieve.mockResolvedValue(mockStore); + mockClient.vectorStores.files.retrieve.mockResolvedValue(mockFile); + mockContentsAPI.listContents.mockResolvedValue({ + data: mockContents, + }); + }); + + describe("Loading and Error States", () => { + test("renders loading skeleton while fetching store data", async () => { + mockClient.vectorStores.retrieve.mockImplementation( + () => new Promise(() => {}) + ); + + await act(async () => { + render(); + }); + + const skeletons = document.querySelectorAll('[data-slot="skeleton"]'); + expect(skeletons.length).toBeGreaterThan(0); + }); + + test("renders error message when store API call fails", async () => { + const error = new Error("Failed to load store"); + mockClient.vectorStores.retrieve.mockRejectedValue(error); + + await act(async () => { + render(); + }); + + await waitFor(() => { + expect( + screen.getByText(/Error loading details for ID vs_123/) + ).toBeInTheDocument(); + expect(screen.getByText(/Failed to load store/)).toBeInTheDocument(); + }); + }); + + test("renders not found when store doesn't exist", async () => { + mockClient.vectorStores.retrieve.mockResolvedValue(null); + + await act(async () => { + render(); + }); + + await waitFor(() => { + expect( + screen.getByText(/No details found for ID: vs_123/) + ).toBeInTheDocument(); + }); + }); + + test("renders contents loading skeleton", async () => { + mockContentsAPI.listContents.mockImplementation( + () => new Promise(() => {}) + ); + + const { container } = render(); + + await waitFor(() => { + expect( + screen.getByText("Contents in File: file_456") + ).toBeInTheDocument(); + }); + + const skeletons = container.querySelectorAll('[data-slot="skeleton"]'); + expect(skeletons.length).toBeGreaterThan(0); + }); + + test("renders contents error message", async () => { + const error = new Error("Failed to load contents"); + mockContentsAPI.listContents.mockRejectedValue(error); + + render(); + + await waitFor(() => { + expect( + screen.getByText("Error loading contents: Failed to load contents") + ).toBeInTheDocument(); + }); + }); + }); + + describe("Contents Table Display", () => { + test("renders contents table with correct headers", async () => { + render(); + + await waitFor(() => { + expect(screen.getByText("Content Chunks (3)")).toBeInTheDocument(); + expect(screen.getByText("Contents in this file")).toBeInTheDocument(); + }); + + // Check table headers + expect(screen.getByText("Content ID")).toBeInTheDocument(); + expect(screen.getByText("Content Preview")).toBeInTheDocument(); + expect(screen.getByText("Embedding")).toBeInTheDocument(); + expect(screen.getByText("Position")).toBeInTheDocument(); + expect(screen.getByText("Created")).toBeInTheDocument(); + expect(screen.getByText("Actions")).toBeInTheDocument(); + }); + + test("renders content data correctly", async () => { + render(); + + await waitFor(() => { + // Check first content row + expect(screen.getByText("content_1...")).toBeInTheDocument(); + expect( + screen.getByText("First piece of content for testing.") + ).toBeInTheDocument(); + expect( + screen.getByText("[0.100, 0.200, 0.300...] (5D)") + ).toBeInTheDocument(); + expect(screen.getByText("0-35")).toBeInTheDocument(); + expect( + screen.getByText(new Date(1710002000 * 1000).toLocaleString()) + ).toBeInTheDocument(); + + expect(screen.getByText("content_2...")).toBeInTheDocument(); + expect( + screen.getByText(/Second piece of content with longer text/) + ).toBeInTheDocument(); + expect( + screen.getByText("[0.600, 0.700, 0.800...] (3D)") + ).toBeInTheDocument(); + expect(screen.getByText("36-95")).toBeInTheDocument(); + + expect(screen.getByText("content_3...")).toBeInTheDocument(); + expect( + screen.getByText("Third content without embedding.") + ).toBeInTheDocument(); + expect(screen.getByText("No embedding")).toBeInTheDocument(); + expect(screen.getByText("33 chars")).toBeInTheDocument(); + }); + }); + + test("handles empty contents list", async () => { + mockContentsAPI.listContents.mockResolvedValue({ + data: [], + }); + + render(); + + await waitFor(() => { + expect(screen.getByText("Content Chunks (0)")).toBeInTheDocument(); + expect( + screen.getByText("No contents found for this file.") + ).toBeInTheDocument(); + }); + }); + + test("truncates long content IDs", async () => { + const longIdContent = { + ...mockContents[0], + id: "very_long_content_id_that_should_be_truncated_123456789", + }; + + mockContentsAPI.listContents.mockResolvedValue({ + data: [longIdContent], + }); + + render(); + + await waitFor(() => { + expect(screen.getByText("very_long_...")).toBeInTheDocument(); + }); + }); + }); + + describe("Content Navigation", () => { + test("navigates to content detail when content ID is clicked", async () => { + render(); + + await waitFor(() => { + expect(screen.getByText("content_1...")).toBeInTheDocument(); + }); + + const contentLink = screen.getByRole("button", { name: "content_1..." }); + fireEvent.click(contentLink); + + expect(mockPush).toHaveBeenCalledWith( + "/logs/vector-stores/vs_123/files/file_456/contents/content_1" + ); + }); + + test("navigates to content detail when view button is clicked", async () => { + render(); + + await waitFor(() => { + expect(screen.getByText("Content Chunks (3)")).toBeInTheDocument(); + }); + + const viewButtons = screen.getAllByTitle("View content details"); + fireEvent.click(viewButtons[0]); + + expect(mockPush).toHaveBeenCalledWith( + "/logs/vector-stores/vs_123/files/file_456/contents/content_1" + ); + }); + + test("navigates to content detail when edit button is clicked", async () => { + render(); + + await waitFor(() => { + expect(screen.getByText("Content Chunks (3)")).toBeInTheDocument(); + }); + + const editButtons = screen.getAllByTitle("Edit content"); + fireEvent.click(editButtons[0]); + + expect(mockPush).toHaveBeenCalledWith( + "/logs/vector-stores/vs_123/files/file_456/contents/content_1" + ); + }); + }); + + describe("Content Deletion", () => { + test("deletes content when delete button is clicked", async () => { + mockContentsAPI.deleteContent.mockResolvedValue(undefined); + + render(); + + await waitFor(() => { + expect(screen.getByText("Content Chunks (3)")).toBeInTheDocument(); + }); + + const deleteButtons = screen.getAllByTitle("Delete content"); + fireEvent.click(deleteButtons[0]); + + await waitFor(() => { + expect(mockContentsAPI.deleteContent).toHaveBeenCalledWith( + "vs_123", + "file_456", + "content_1" + ); + }); + + await waitFor(() => { + expect(screen.getByText("Content Chunks (2)")).toBeInTheDocument(); + }); + + expect(screen.queryByText("content_1...")).not.toBeInTheDocument(); + }); + + test("handles delete error gracefully", async () => { + const consoleError = jest + .spyOn(console, "error") + .mockImplementation(() => {}); + mockContentsAPI.deleteContent.mockRejectedValue( + new Error("Delete failed") + ); + + render(); + + await waitFor(() => { + expect(screen.getByText("Content Chunks (3)")).toBeInTheDocument(); + }); + + const deleteButtons = screen.getAllByTitle("Delete content"); + fireEvent.click(deleteButtons[0]); + + await waitFor(() => { + expect(consoleError).toHaveBeenCalledWith( + "Failed to delete content:", + expect.any(Error) + ); + }); + + expect(screen.getByText("Content Chunks (3)")).toBeInTheDocument(); + expect(screen.getByText("content_1...")).toBeInTheDocument(); + + consoleError.mockRestore(); + }); + }); + + describe("Breadcrumb Navigation", () => { + test("renders correct breadcrumb structure", async () => { + render(); + + await waitFor(() => { + const vectorStoreTexts = screen.getAllByText("Vector Stores"); + expect(vectorStoreTexts.length).toBeGreaterThan(0); + const storeNameTexts = screen.getAllByText("Test Vector Store"); + expect(storeNameTexts.length).toBeGreaterThan(0); + const filesTexts = screen.getAllByText("Files"); + expect(filesTexts.length).toBeGreaterThan(0); + const fileIdTexts = screen.getAllByText("file_456"); + expect(fileIdTexts.length).toBeGreaterThan(0); + const contentsTexts = screen.getAllByText("Contents"); + expect(contentsTexts.length).toBeGreaterThan(0); + }); + }); + }); + + describe("Sidebar Properties", () => { + test("renders file and store properties", async () => { + render(); + + await waitFor(() => { + const fileIdTexts = screen.getAllByText("file_456"); + expect(fileIdTexts.length).toBeGreaterThan(0); + const storeIdTexts = screen.getAllByText("vs_123"); + expect(storeIdTexts.length).toBeGreaterThan(0); + const storeNameTexts = screen.getAllByText("Test Vector Store"); + expect(storeNameTexts.length).toBeGreaterThan(0); + + expect(screen.getByText("completed")).toBeInTheDocument(); + expect(screen.getByText("512")).toBeInTheDocument(); + expect(screen.getByText("fixed_size")).toBeInTheDocument(); + expect(screen.getByText("test_provider")).toBeInTheDocument(); + }); + }); + }); + + describe("Content Text Utilities", () => { + test("handles different content formats correctly", async () => { + const contentWithObject = { + ...mockContents[0], + content: { type: "text", text: "Object format content" }, + }; + + mockContentsAPI.listContents.mockResolvedValue({ + data: [contentWithObject], + }); + + render(); + + await waitFor(() => { + expect(screen.getByText("Object format content")).toBeInTheDocument(); + }); + }); + + test("handles string content format", async () => { + const contentWithString = { + ...mockContents[0], + content: "String format content", + }; + + mockContentsAPI.listContents.mockResolvedValue({ + data: [contentWithString], + }); + + render(); + + await waitFor(() => { + expect(screen.getByText("String format content")).toBeInTheDocument(); + }); + }); + + test("handles unknown content format", async () => { + const contentWithUnknown = { + ...mockContents[0], + content: { unknown: "format" }, + }; + + mockContentsAPI.listContents.mockResolvedValue({ + data: [contentWithUnknown], + }); + + render(); + + await waitFor(() => { + expect(screen.getByText("Content Chunks (1)")).toBeInTheDocument(); + }); + + const contentCells = screen.getAllByRole("cell"); + const contentPreviewCell = contentCells.find(cell => + cell.querySelector("p[title]") + ); + expect(contentPreviewCell?.querySelector("p")?.textContent).toBe(""); + }); + }); +}); diff --git a/llama_stack/ui/app/logs/vector-stores/[id]/files/[fileId]/contents/page.tsx b/llama_stack/ui/app/logs/vector-stores/[id]/files/[fileId]/contents/page.tsx index 0283db9e7..3d714a480 100644 --- a/llama_stack/ui/app/logs/vector-stores/[id]/files/[fileId]/contents/page.tsx +++ b/llama_stack/ui/app/logs/vector-stores/[id]/files/[fileId]/contents/page.tsx @@ -52,8 +52,10 @@ export default function ContentsListPage() { const [file, setFile] = useState(null); const [contents, setContents] = useState([]); const [isLoadingStore, setIsLoadingStore] = useState(true); + const [isLoadingFile, setIsLoadingFile] = useState(true); const [isLoadingContents, setIsLoadingContents] = useState(true); const [errorStore, setErrorStore] = useState(null); + const [errorFile, setErrorFile] = useState(null); const [errorContents, setErrorContents] = useState(null); useEffect(() => { @@ -175,7 +177,13 @@ export default function ContentsListPage() { Content Chunks ({contents.length}) - {isLoadingContents ? ( + {isLoadingFile ? ( + + ) : errorFile ? ( +
+ Error loading file: {errorFile.message} +
+ ) : isLoadingContents ? (
diff --git a/llama_stack/ui/app/logs/vector-stores/[id]/files/[fileId]/page.test.tsx b/llama_stack/ui/app/logs/vector-stores/[id]/files/[fileId]/page.test.tsx new file mode 100644 index 000000000..2be26bf3f --- /dev/null +++ b/llama_stack/ui/app/logs/vector-stores/[id]/files/[fileId]/page.test.tsx @@ -0,0 +1,458 @@ +import React from "react"; +import { + render, + screen, + fireEvent, + waitFor, + act, +} from "@testing-library/react"; +import "@testing-library/jest-dom"; +import FileDetailPage from "./page"; +import type { VectorStore } from "llama-stack-client/resources/vector-stores/vector-stores"; +import type { + VectorStoreFile, + FileContentResponse, +} from "llama-stack-client/resources/vector-stores/files"; + +const mockPush = jest.fn(); +const mockParams = { + id: "vs_123", + fileId: "file_456", +}; + +jest.mock("next/navigation", () => ({ + useParams: () => mockParams, + useRouter: () => ({ + push: mockPush, + }), +})); + +const mockClient = { + vectorStores: { + retrieve: jest.fn(), + files: { + retrieve: jest.fn(), + content: jest.fn(), + }, + }, +}; + +jest.mock("@/hooks/use-auth-client", () => ({ + useAuthClient: () => mockClient, +})); + +describe("FileDetailPage", () => { + const mockStore: VectorStore = { + id: "vs_123", + name: "Test Vector Store", + created_at: 1710000000, + status: "ready", + file_counts: { total: 5 }, + usage_bytes: 1024, + metadata: { + provider_id: "test_provider", + }, + }; + + const mockFile: VectorStoreFile = { + id: "file_456", + status: "completed", + created_at: 1710001000, + usage_bytes: 2048, + chunking_strategy: { type: "fixed_size" }, + }; + + const mockFileContent: FileContentResponse = { + content: [ + { text: "First chunk of file content." }, + { + text: "Second chunk with more detailed information about the content.", + }, + { text: "Third and final chunk of the file." }, + ], + }; + + beforeEach(() => { + jest.clearAllMocks(); + + mockClient.vectorStores.retrieve.mockResolvedValue(mockStore); + mockClient.vectorStores.files.retrieve.mockResolvedValue(mockFile); + mockClient.vectorStores.files.content.mockResolvedValue(mockFileContent); + }); + + describe("Loading and Error States", () => { + test("renders loading skeleton while fetching store data", async () => { + mockClient.vectorStores.retrieve.mockImplementation( + () => new Promise(() => {}) + ); + + await act(async () => { + await act(async () => { + render(); + }); + }); + + const skeletons = document.querySelectorAll('[data-slot="skeleton"]'); + expect(skeletons.length).toBeGreaterThan(0); + }); + + test("renders error message when store API call fails", async () => { + const error = new Error("Failed to load store"); + mockClient.vectorStores.retrieve.mockRejectedValue(error); + + await act(async () => { + await act(async () => { + render(); + }); + }); + + await waitFor(() => { + expect( + screen.getByText(/Error loading details for ID vs_123/) + ).toBeInTheDocument(); + expect(screen.getByText(/Failed to load store/)).toBeInTheDocument(); + }); + }); + + test("renders not found when store doesn't exist", async () => { + mockClient.vectorStores.retrieve.mockResolvedValue(null); + + await act(async () => { + render(); + }); + + await waitFor(() => { + expect( + screen.getByText(/No details found for ID: vs_123/) + ).toBeInTheDocument(); + }); + }); + + test("renders file loading skeleton", async () => { + mockClient.vectorStores.files.retrieve.mockImplementation( + () => new Promise(() => {}) + ); + + const { container } = render(); + + await waitFor(() => { + expect(screen.getByText("File: file_456")).toBeInTheDocument(); + }); + + const skeletons = container.querySelectorAll('[data-slot="skeleton"]'); + expect(skeletons.length).toBeGreaterThan(0); + }); + + test("renders file error message", async () => { + const error = new Error("Failed to load file"); + mockClient.vectorStores.files.retrieve.mockRejectedValue(error); + + await act(async () => { + render(); + }); + + await waitFor(() => { + expect( + screen.getByText("Error loading file: Failed to load file") + ).toBeInTheDocument(); + }); + }); + + test("renders content error message", async () => { + const error = new Error("Failed to load contents"); + mockClient.vectorStores.files.content.mockRejectedValue(error); + + await act(async () => { + render(); + }); + + await waitFor(() => { + expect( + screen.getByText( + "Error loading content summary: Failed to load contents" + ) + ).toBeInTheDocument(); + }); + }); + }); + + describe("File Information Display", () => { + test("renders file details correctly", async () => { + await act(async () => { + await act(async () => { + render(); + }); + }); + + await waitFor(() => { + expect(screen.getByText("File: file_456")).toBeInTheDocument(); + expect(screen.getByText("File Information")).toBeInTheDocument(); + expect(screen.getByText("File Details")).toBeInTheDocument(); + }); + + const statusTexts = screen.getAllByText("Status:"); + expect(statusTexts.length).toBeGreaterThan(0); + const completedTexts = screen.getAllByText("completed"); + expect(completedTexts.length).toBeGreaterThan(0); + expect(screen.getByText("Size:")).toBeInTheDocument(); + expect(screen.getByText("2048 bytes")).toBeInTheDocument(); + const createdTexts = screen.getAllByText("Created:"); + expect(createdTexts.length).toBeGreaterThan(0); + const dateTexts = screen.getAllByText( + new Date(1710001000 * 1000).toLocaleString() + ); + expect(dateTexts.length).toBeGreaterThan(0); + const strategyTexts = screen.getAllByText("Content Strategy:"); + expect(strategyTexts.length).toBeGreaterThan(0); + const fixedSizeTexts = screen.getAllByText("fixed_size"); + expect(fixedSizeTexts.length).toBeGreaterThan(0); + }); + + test("handles missing file data", async () => { + mockClient.vectorStores.files.retrieve.mockResolvedValue(null); + + await act(async () => { + render(); + }); + + await waitFor(() => { + expect(screen.getByText("File not found.")).toBeInTheDocument(); + }); + }); + }); + + describe("Content Summary Display", () => { + test("renders content summary correctly", async () => { + await act(async () => { + render(); + }); + + await waitFor(() => { + expect(screen.getByText("Content Summary")).toBeInTheDocument(); + expect(screen.getByText("Content Items:")).toBeInTheDocument(); + expect(screen.getByText("3")).toBeInTheDocument(); + expect(screen.getByText("Total Characters:")).toBeInTheDocument(); + + const totalChars = mockFileContent.content.reduce( + (total, item) => total + item.text.length, + 0 + ); + expect(screen.getByText(totalChars.toString())).toBeInTheDocument(); + + expect(screen.getByText("Preview:")).toBeInTheDocument(); + expect( + screen.getByText(/First chunk of file content\./) + ).toBeInTheDocument(); + }); + }); + + test("handles empty content", async () => { + mockClient.vectorStores.files.content.mockResolvedValue({ + content: [], + }); + + await act(async () => { + render(); + }); + + await waitFor(() => { + expect( + screen.getByText("No contents found for this file.") + ).toBeInTheDocument(); + }); + }); + + test("truncates long content preview", async () => { + const longContent = { + content: [ + { + text: "This is a very long piece of content that should be truncated after 200 characters to ensure the preview doesn't take up too much space in the UI and remains readable and manageable for users viewing the file details page.", + }, + ], + }; + + mockClient.vectorStores.files.content.mockResolvedValue(longContent); + + await act(async () => { + render(); + }); + + await waitFor(() => { + expect( + screen.getByText(/This is a very long piece of content/) + ).toBeInTheDocument(); + expect(screen.getByText(/\.\.\.$/)).toBeInTheDocument(); + }); + }); + }); + + describe("Navigation and Actions", () => { + test("navigates to contents list when View Contents button is clicked", async () => { + await act(async () => { + render(); + }); + + await waitFor(() => { + expect(screen.getByText("Actions")).toBeInTheDocument(); + }); + + const viewContentsButton = screen.getByRole("button", { + name: /View Contents/, + }); + fireEvent.click(viewContentsButton); + + expect(mockPush).toHaveBeenCalledWith( + "/logs/vector-stores/vs_123/files/file_456/contents" + ); + }); + + test("View Contents button is styled correctly", async () => { + await act(async () => { + render(); + }); + + await waitFor(() => { + const button = screen.getByRole("button", { name: /View Contents/ }); + expect(button).toHaveClass("flex", "items-center", "gap-2"); + }); + }); + }); + + describe("Breadcrumb Navigation", () => { + test("renders correct breadcrumb structure", async () => { + await act(async () => { + render(); + }); + + await waitFor(() => { + const vectorStoresTexts = screen.getAllByText("Vector Stores"); + expect(vectorStoresTexts.length).toBeGreaterThan(0); + const storeNameTexts = screen.getAllByText("Test Vector Store"); + expect(storeNameTexts.length).toBeGreaterThan(0); + const filesTexts = screen.getAllByText("Files"); + expect(filesTexts.length).toBeGreaterThan(0); + const fileIdTexts = screen.getAllByText("file_456"); + expect(fileIdTexts.length).toBeGreaterThan(0); + }); + }); + + test("uses store ID when store name is not available", async () => { + const storeWithoutName = { ...mockStore, name: "" }; + mockClient.vectorStores.retrieve.mockResolvedValue(storeWithoutName); + + await act(async () => { + render(); + }); + + await waitFor(() => { + const storeIdTexts = screen.getAllByText("vs_123"); + expect(storeIdTexts.length).toBeGreaterThan(0); + }); + }); + }); + + describe("Sidebar Properties", () => { + test.skip("renders file and store properties correctly", async () => { + await act(async () => { + render(); + }); + + await waitFor(() => { + expect(screen.getByText("File ID")).toBeInTheDocument(); + const fileIdTexts = screen.getAllByText("file_456"); + expect(fileIdTexts.length).toBeGreaterThan(0); + expect(screen.getByText("Vector Store ID")).toBeInTheDocument(); + const storeIdTexts = screen.getAllByText("vs_123"); + expect(storeIdTexts.length).toBeGreaterThan(0); + expect(screen.getByText("Status")).toBeInTheDocument(); + const completedTexts = screen.getAllByText("completed"); + expect(completedTexts.length).toBeGreaterThan(0); + expect(screen.getByText("Usage Bytes")).toBeInTheDocument(); + const usageTexts = screen.getAllByText("2048"); + expect(usageTexts.length).toBeGreaterThan(0); + expect(screen.getByText("Content Strategy")).toBeInTheDocument(); + const fixedSizeTexts = screen.getAllByText("fixed_size"); + expect(fixedSizeTexts.length).toBeGreaterThan(0); + + expect(screen.getByText("Store Name")).toBeInTheDocument(); + const storeNameTexts = screen.getAllByText("Test Vector Store"); + expect(storeNameTexts.length).toBeGreaterThan(0); + expect(screen.getByText("Provider ID")).toBeInTheDocument(); + expect(screen.getByText("test_provider")).toBeInTheDocument(); + }); + }); + + test("handles missing optional properties", async () => { + const minimalFile = { + id: "file_456", + status: "completed", + created_at: 1710001000, + usage_bytes: 2048, + chunking_strategy: { type: "fixed_size" }, + }; + + const minimalStore = { + ...mockStore, + name: "", + metadata: {}, + }; + + mockClient.vectorStores.files.retrieve.mockResolvedValue(minimalFile); + mockClient.vectorStores.retrieve.mockResolvedValue(minimalStore); + + await act(async () => { + render(); + }); + + await waitFor(() => { + const fileIdTexts = screen.getAllByText("file_456"); + expect(fileIdTexts.length).toBeGreaterThan(0); + const storeIdTexts = screen.getAllByText("vs_123"); + expect(storeIdTexts.length).toBeGreaterThan(0); + }); + + expect(screen.getByText("File: file_456")).toBeInTheDocument(); + }); + }); + + describe("Loading States for Individual Sections", () => { + test("shows loading skeleton for content while file loads", async () => { + mockClient.vectorStores.files.content.mockImplementation( + () => new Promise(() => {}) + ); + + const { container } = render(); + + await waitFor(() => { + expect(screen.getByText("Content Summary")).toBeInTheDocument(); + }); + + const skeletons = container.querySelectorAll('[data-slot="skeleton"]'); + expect(skeletons.length).toBeGreaterThan(0); + }); + }); + + describe("Error Handling", () => { + test("handles multiple simultaneous errors gracefully", async () => { + mockClient.vectorStores.files.retrieve.mockRejectedValue( + new Error("File error") + ); + mockClient.vectorStores.files.content.mockRejectedValue( + new Error("Content error") + ); + + await act(async () => { + render(); + }); + + await waitFor(() => { + expect( + screen.getByText("Error loading file: File error") + ).toBeInTheDocument(); + expect( + screen.getByText("Error loading content summary: Content error") + ).toBeInTheDocument(); + }); + }); + }); +}); diff --git a/llama_stack/ui/components/chat-playground/markdown-renderer.tsx b/llama_stack/ui/components/chat-playground/markdown-renderer.tsx index bc6bf5122..b48b5e1ba 100644 --- a/llama_stack/ui/components/chat-playground/markdown-renderer.tsx +++ b/llama_stack/ui/components/chat-playground/markdown-renderer.tsx @@ -187,6 +187,7 @@ const COMPONENTS = { code: ({ children, className, + ...rest }: { children: React.ReactNode; className?: string; diff --git a/llama_stack/ui/components/vector-stores/vector-store-detail.test.tsx b/llama_stack/ui/components/vector-stores/vector-store-detail.test.tsx new file mode 100644 index 000000000..08f90ac0d --- /dev/null +++ b/llama_stack/ui/components/vector-stores/vector-store-detail.test.tsx @@ -0,0 +1,315 @@ +import React from "react"; +import { render, screen, fireEvent } from "@testing-library/react"; +import "@testing-library/jest-dom"; +import { VectorStoreDetailView } from "./vector-store-detail"; +import type { VectorStore } from "llama-stack-client/resources/vector-stores/vector-stores"; +import type { VectorStoreFile } from "llama-stack-client/resources/vector-stores/files"; + +const mockPush = jest.fn(); +jest.mock("next/navigation", () => ({ + useRouter: () => ({ + push: mockPush, + }), +})); + +describe("VectorStoreDetailView", () => { + const defaultProps = { + store: null, + files: [], + isLoadingStore: false, + isLoadingFiles: false, + errorStore: null, + errorFiles: null, + id: "test_vector_store_id", + }; + + beforeEach(() => { + mockPush.mockClear(); + }); + + describe("Loading States", () => { + test("renders loading skeleton when store is loading", () => { + const { container } = render( + + ); + + const skeletons = container.querySelectorAll('[data-slot="skeleton"]'); + expect(skeletons.length).toBeGreaterThan(0); + }); + + test("renders files loading skeleton when files are loading", () => { + const mockStore: VectorStore = { + id: "vs_123", + name: "Test Vector Store", + created_at: 1710000000, + status: "ready", + file_counts: { total: 5 }, + usage_bytes: 1024, + metadata: { + provider_id: "test_provider", + provider_vector_db_id: "test_db_id", + }, + }; + + const { container } = render( + + ); + + expect(screen.getByText("Vector Store Details")).toBeInTheDocument(); + expect(screen.getByText("Files")).toBeInTheDocument(); + const skeletons = container.querySelectorAll('[data-slot="skeleton"]'); + expect(skeletons.length).toBeGreaterThan(0); + }); + }); + + describe("Error States", () => { + test("renders error message when store error occurs", () => { + render( + + ); + + expect(screen.getByText("Vector Store Details")).toBeInTheDocument(); + expect( + screen.getByText(/Error loading details for ID test_vector_store_id/) + ).toBeInTheDocument(); + expect(screen.getByText(/Failed to load store/)).toBeInTheDocument(); + }); + + test("renders files error when files fail to load", () => { + const mockStore: VectorStore = { + id: "vs_123", + name: "Test Vector Store", + created_at: 1710000000, + status: "ready", + file_counts: { total: 5 }, + usage_bytes: 1024, + metadata: { + provider_id: "test_provider", + provider_vector_db_id: "test_db_id", + }, + }; + + render( + + ); + + expect(screen.getByText("Files")).toBeInTheDocument(); + expect( + screen.getByText("Error loading files: Failed to load files") + ).toBeInTheDocument(); + }); + }); + + describe("Not Found State", () => { + test("renders not found message when store is null", () => { + render(); + + expect(screen.getByText("Vector Store Details")).toBeInTheDocument(); + expect( + screen.getByText(/No details found for ID: test_vector_store_id/) + ).toBeInTheDocument(); + }); + }); + + describe("Store Data Rendering", () => { + const mockStore: VectorStore = { + id: "vs_123", + name: "Test Vector Store", + created_at: 1710000000, + status: "ready", + file_counts: { total: 3 }, + usage_bytes: 2048, + metadata: { + provider_id: "test_provider", + provider_vector_db_id: "test_db_id", + }, + }; + + test("renders store properties correctly", () => { + render(); + + expect(screen.getByText("Vector Store Details")).toBeInTheDocument(); + expect(screen.getByText("vs_123")).toBeInTheDocument(); + expect(screen.getByText("Test Vector Store")).toBeInTheDocument(); + expect( + screen.getByText(new Date(1710000000 * 1000).toLocaleString()) + ).toBeInTheDocument(); + expect(screen.getByText("ready")).toBeInTheDocument(); + expect(screen.getByText("3")).toBeInTheDocument(); + expect(screen.getByText("2048")).toBeInTheDocument(); + expect(screen.getByText("test_provider")).toBeInTheDocument(); + expect(screen.getByText("test_db_id")).toBeInTheDocument(); + }); + + test("handles empty/missing optional fields", () => { + const minimalStore: VectorStore = { + id: "vs_minimal", + name: "", + created_at: 1710000000, + status: "ready", + file_counts: { total: 0 }, + usage_bytes: 0, + metadata: {}, + }; + + render(); + + expect(screen.getByText("vs_minimal")).toBeInTheDocument(); + expect(screen.getByText("ready")).toBeInTheDocument(); + const zeroTexts = screen.getAllByText("0"); + expect(zeroTexts.length).toBeGreaterThanOrEqual(2); + }); + + test("shows empty files message when no files", () => { + render( + + ); + + expect(screen.getByText("Files")).toBeInTheDocument(); + expect( + screen.getByText("No files in this vector store.") + ).toBeInTheDocument(); + }); + }); + + describe("Files Table", () => { + const mockStore: VectorStore = { + id: "vs_123", + name: "Test Vector Store", + created_at: 1710000000, + status: "ready", + file_counts: { total: 2 }, + usage_bytes: 2048, + metadata: {}, + }; + + const mockFiles: VectorStoreFile[] = [ + { + id: "file_123", + status: "completed", + created_at: 1710001000, + usage_bytes: 1024, + }, + { + id: "file_456", + status: "processing", + created_at: 1710002000, + usage_bytes: 512, + }, + ]; + + test("renders files table with correct data", () => { + render( + + ); + + expect(screen.getByText("Files")).toBeInTheDocument(); + expect( + screen.getByText("Files in this vector store") + ).toBeInTheDocument(); + + expect(screen.getByText("ID")).toBeInTheDocument(); + expect(screen.getByText("Status")).toBeInTheDocument(); + expect(screen.getByText("Created")).toBeInTheDocument(); + expect(screen.getByText("Usage Bytes")).toBeInTheDocument(); + + expect(screen.getByText("file_123")).toBeInTheDocument(); + expect(screen.getByText("completed")).toBeInTheDocument(); + expect( + screen.getByText(new Date(1710001000 * 1000).toLocaleString()) + ).toBeInTheDocument(); + expect(screen.getByText("1024")).toBeInTheDocument(); + + expect(screen.getByText("file_456")).toBeInTheDocument(); + expect(screen.getByText("processing")).toBeInTheDocument(); + expect( + screen.getByText(new Date(1710002000 * 1000).toLocaleString()) + ).toBeInTheDocument(); + expect(screen.getByText("512")).toBeInTheDocument(); + }); + + test("file ID links are clickable and navigate correctly", () => { + render( + + ); + + const fileButton = screen.getByRole("button", { name: "file_123" }); + expect(fileButton).toBeInTheDocument(); + + fireEvent.click(fileButton); + expect(mockPush).toHaveBeenCalledWith( + "/logs/vector-stores/vs_123/files/file_123" + ); + }); + + test("handles multiple file clicks correctly", () => { + render( + + ); + + const file1Button = screen.getByRole("button", { name: "file_123" }); + const file2Button = screen.getByRole("button", { name: "file_456" }); + + fireEvent.click(file1Button); + expect(mockPush).toHaveBeenCalledWith( + "/logs/vector-stores/vs_123/files/file_123" + ); + + fireEvent.click(file2Button); + expect(mockPush).toHaveBeenCalledWith( + "/logs/vector-stores/vs_123/files/file_456" + ); + + expect(mockPush).toHaveBeenCalledTimes(2); + }); + }); + + describe("Layout Structure", () => { + const mockStore: VectorStore = { + id: "vs_layout_test", + name: "Layout Test Store", + created_at: 1710000000, + status: "ready", + file_counts: { total: 1 }, + usage_bytes: 1024, + metadata: {}, + }; + + test("renders main content and sidebar in correct layout", () => { + render(); + + expect(screen.getByText("Files")).toBeInTheDocument(); + + expect(screen.getByText("vs_layout_test")).toBeInTheDocument(); + expect(screen.getByText("Layout Test Store")).toBeInTheDocument(); + expect(screen.getByText("ready")).toBeInTheDocument(); + expect(screen.getByText("1")).toBeInTheDocument(); + expect(screen.getByText("1024")).toBeInTheDocument(); + }); + }); +}); diff --git a/llama_stack/ui/package.json b/llama_stack/ui/package.json index fd6f6fbb7..226b06f59 100644 --- a/llama_stack/ui/package.json +++ b/llama_stack/ui/package.json @@ -23,7 +23,7 @@ "class-variance-authority": "^0.7.1", "clsx": "^2.1.1", "framer-motion": "^11.18.2", - "llama-stack-client": "^0.2.17", + "llama-stack-client": "^0.2.18", "lucide-react": "^0.510.0", "next": "15.3.3", "next-auth": "^4.24.11", diff --git a/pyproject.toml b/pyproject.toml index db0ad1f00..0cdfc6a37 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,7 +7,7 @@ required-version = ">=0.7.0" [project] name = "llama_stack" -version = "0.2.17" +version = "0.2.18" authors = [{ name = "Meta Llama", email = "llama-oss@meta.com" }] description = "Llama Stack" readme = "README.md" @@ -31,9 +31,9 @@ dependencies = [ "huggingface-hub>=0.34.0,<1.0", "jinja2>=3.1.6", "jsonschema", - "llama-stack-client>=0.2.17", + "llama-stack-client>=0.2.18", "llama-api-client>=0.1.2", - "openai>=1.99.6", + "openai>=1.99.6,<1.100.0", "prompt-toolkit", "python-dotenv", "python-jose[cryptography]", @@ -56,7 +56,7 @@ dependencies = [ ui = [ "streamlit", "pandas", - "llama-stack-client>=0.2.17", + "llama-stack-client>=0.2.18", "streamlit-option-menu", ] @@ -93,6 +93,7 @@ unit = [ "blobfile", "faiss-cpu", "pymilvus>=2.5.12", + "milvus-lite>=2.5.0", "litellm", "together", "coverage", @@ -118,6 +119,7 @@ test = [ "sqlalchemy[asyncio]>=2.0.41", "requests", "pymilvus>=2.5.12", + "milvus-lite>=2.5.0", "weaviate-client>=4.16.4", ] docs = [ diff --git a/scripts/integration-tests.sh b/scripts/integration-tests.sh index bf7671348..e152444e1 100755 --- a/scripts/integration-tests.sh +++ b/scripts/integration-tests.sh @@ -111,6 +111,9 @@ echo "Inference Mode: $INFERENCE_MODE" echo "Test Pattern: $TEST_PATTERN" echo "" +echo "Checking llama packages" +uv pip list | grep llama + # Check storage and memory before tests echo "=== System Resources Before Tests ===" free -h 2>/dev/null || echo "free command not available" @@ -133,6 +136,10 @@ else EXTRA_PARAMS="" fi +THIS_DIR=$(dirname "$0") +ROOT_DIR="$THIS_DIR/.." +cd $ROOT_DIR + # Set recording directory if [[ "$RUN_VISION_TESTS" == "true" ]]; then export LLAMA_STACK_TEST_RECORDING_DIR="tests/integration/recordings/vision" @@ -140,26 +147,44 @@ else export LLAMA_STACK_TEST_RECORDING_DIR="tests/integration/recordings" fi +# check if "llama" and "pytest" are available. this script does not use `uv run` given +# it can be used in a pre-release environment where we have not been able to tell +# uv about pre-release dependencies properly (yet). +if ! command -v llama &> /dev/null; then + echo "llama could not be found, ensure llama-stack is installed" + exit 1 +fi + +if ! command -v pytest &> /dev/null; then + echo "pytest could not be found, ensure pytest is installed" + exit 1 +fi + # Start Llama Stack Server if needed if [[ "$STACK_CONFIG" == *"server:"* ]]; then - echo "=== Starting Llama Stack Server ===" - nohup uv run llama stack run ci-tests --image-type venv > server.log 2>&1 & + # check if server is already running + if curl -s http://localhost:8321/v1/health 2>/dev/null | grep -q "OK"; then + echo "Llama Stack Server is already running, skipping start" + else + echo "=== Starting Llama Stack Server ===" + nohup llama stack run ci-tests --image-type venv > server.log 2>&1 & - echo "Waiting for Llama Stack Server to start..." - for i in {1..30}; do - if curl -s http://localhost:8321/v1/health 2>/dev/null | grep -q "OK"; then - echo "✅ Llama Stack Server started successfully" - break - fi - if [[ $i -eq 30 ]]; then - echo "❌ Llama Stack Server failed to start" - echo "Server logs:" - cat server.log - exit 1 - fi - sleep 1 - done - echo "" + echo "Waiting for Llama Stack Server to start..." + for i in {1..30}; do + if curl -s http://localhost:8321/v1/health 2>/dev/null | grep -q "OK"; then + echo "✅ Llama Stack Server started successfully" + break + fi + if [[ $i -eq 30 ]]; then + echo "❌ Llama Stack Server failed to start" + echo "Server logs:" + cat server.log + exit 1 + fi + sleep 1 + done + echo "" + fi fi # Run tests @@ -180,7 +205,7 @@ fi if [[ "$RUN_VISION_TESTS" == "true" ]]; then echo "Running vision tests..." set +e - uv run pytest -s -v tests/integration/inference/test_vision_inference.py \ + pytest -s -v tests/integration/inference/test_vision_inference.py \ --stack-config="$STACK_CONFIG" \ -k "$PYTEST_PATTERN" \ --vision-model=ollama/llama3.2-vision:11b \ @@ -248,7 +273,7 @@ echo "=== Running all collected tests in a single pytest command ===" echo "Total test files: $(echo $TEST_FILES | wc -w)" set +e -uv run pytest -s -v $TEST_FILES \ +pytest -s -v $TEST_FILES \ --stack-config="$STACK_CONFIG" \ -k "$PYTEST_PATTERN" \ --text-model="$TEXT_MODEL" \ diff --git a/tests/integration/agents/test_agents.py b/tests/integration/agents/test_agents.py index 05549cf18..23529f91e 100644 --- a/tests/integration/agents/test_agents.py +++ b/tests/integration/agents/test_agents.py @@ -133,24 +133,15 @@ def test_agent_simple(llama_stack_client, agent_config): assert "I can't" in logs_str +@pytest.mark.skip(reason="this test was disabled for a long time, and now has turned flaky") def test_agent_name(llama_stack_client, text_model_id): agent_name = f"test-agent-{uuid4()}" - - try: - agent = Agent( - llama_stack_client, - model=text_model_id, - instructions="You are a helpful assistant", - name=agent_name, - ) - except TypeError: - agent = Agent( - llama_stack_client, - model=text_model_id, - instructions="You are a helpful assistant", - ) - return - + agent = Agent( + llama_stack_client, + model=text_model_id, + instructions="You are a helpful assistant", + name=agent_name, + ) session_id = agent.create_session(f"test-session-{uuid4()}") agent.create_turn( diff --git a/tests/integration/recordings/index.sqlite b/tests/integration/recordings/index.sqlite index 7b6eb6a67..5997194a4 100644 Binary files a/tests/integration/recordings/index.sqlite and b/tests/integration/recordings/index.sqlite differ diff --git a/tests/integration/recordings/responses/4a3a4447b16b.json b/tests/integration/recordings/responses/4a3a4447b16b.json index a31c583c7..484c86bcf 100644 --- a/tests/integration/recordings/responses/4a3a4447b16b.json +++ b/tests/integration/recordings/responses/4a3a4447b16b.json @@ -14,7 +14,7 @@ "models": [ { "model": "nomic-embed-text:latest", - "modified_at": "2025-08-15T21:55:08.088554Z", + "modified_at": "2025-08-18T12:47:56.732989-07:00", "digest": "0a109f422b47e3a30ba2b10eca18548e944e8a23073ee3f3e947efcf3c45e59f", "size": 274302450, "details": { @@ -28,9 +28,41 @@ "quantization_level": "F16" } }, + { + "model": "llama3.2-vision:11b", + "modified_at": "2025-07-30T18:45:02.517873-07:00", + "digest": "6f2f9757ae97e8a3f8ea33d6adb2b11d93d9a35bef277cd2c0b1b5af8e8d0b1e", + "size": 7816589186, + "details": { + "parent_model": "", + "format": "gguf", + "family": "mllama", + "families": [ + "mllama" + ], + "parameter_size": "10.7B", + "quantization_level": "Q4_K_M" + } + }, + { + "model": "llama3.2-vision:latest", + "modified_at": "2025-07-29T20:18:47.920468-07:00", + "digest": "6f2f9757ae97e8a3f8ea33d6adb2b11d93d9a35bef277cd2c0b1b5af8e8d0b1e", + "size": 7816589186, + "details": { + "parent_model": "", + "format": "gguf", + "family": "mllama", + "families": [ + "mllama" + ], + "parameter_size": "10.7B", + "quantization_level": "Q4_K_M" + } + }, { "model": "llama-guard3:1b", - "modified_at": "2025-07-31T04:44:58Z", + "modified_at": "2025-07-25T14:39:44.978630-07:00", "digest": "494147e06bf99e10dbe67b63a07ac81c162f18ef3341aa3390007ac828571b3b", "size": 1600181919, "details": { @@ -46,7 +78,7 @@ }, { "model": "all-minilm:l6-v2", - "modified_at": "2025-07-31T04:42:15Z", + "modified_at": "2025-07-24T15:15:11.129290-07:00", "digest": "1b226e2802dbb772b5fc32a58f103ca1804ef7501331012de126ab22f67475ef", "size": 45960996, "details": { @@ -60,9 +92,57 @@ "quantization_level": "F16" } }, + { + "model": "llama3.2:1b", + "modified_at": "2025-07-17T22:02:24.953208-07:00", + "digest": "baf6a787fdffd633537aa2eb51cfd54cb93ff08e28040095462bb63daf552878", + "size": 1321098329, + "details": { + "parent_model": "", + "format": "gguf", + "family": "llama", + "families": [ + "llama" + ], + "parameter_size": "1.2B", + "quantization_level": "Q8_0" + } + }, + { + "model": "all-minilm:latest", + "modified_at": "2025-06-03T16:50:10.946583-07:00", + "digest": "1b226e2802dbb772b5fc32a58f103ca1804ef7501331012de126ab22f67475ef", + "size": 45960996, + "details": { + "parent_model": "", + "format": "gguf", + "family": "bert", + "families": [ + "bert" + ], + "parameter_size": "23M", + "quantization_level": "F16" + } + }, + { + "model": "llama3.2:3b", + "modified_at": "2025-05-01T11:15:23.797447-07:00", + "digest": "a80c4f17acd55265feec403c7aef86be0c25983ab279d83f3bcd3abbcb5b8b72", + "size": 2019393189, + "details": { + "parent_model": "", + "format": "gguf", + "family": "llama", + "families": [ + "llama" + ], + "parameter_size": "3.2B", + "quantization_level": "Q4_K_M" + } + }, { "model": "llama3.2:3b-instruct-fp16", - "modified_at": "2025-07-31T04:42:05Z", + "modified_at": "2025-04-30T15:33:48.939665-07:00", "digest": "195a8c01d91ec3cb1e0aad4624a51f2602c51fa7d96110f8ab5a20c84081804d", "size": 6433703586, "details": { diff --git a/tests/integration/recordings/responses/731824c54461.json b/tests/integration/recordings/responses/731824c54461.json new file mode 100644 index 000000000..2d88c6329 --- /dev/null +++ b/tests/integration/recordings/responses/731824c54461.json @@ -0,0 +1,203 @@ +{ + "request": { + "method": "POST", + "url": "http://localhost:11434/api/generate", + "headers": {}, + "body": { + "model": "llama3.2:3b-instruct-fp16", + "raw": true, + "prompt": "<|begin_of_text|><|start_header_id|>system<|end_header_id|>\n\nYou are a helpful assistant<|eot_id|><|start_header_id|>user<|end_header_id|>\n\nGive me a sentence that contains the word: hello<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n", + "options": { + "temperature": 0.0 + }, + "stream": true + }, + "endpoint": "/api/generate", + "model": "llama3.2:3b-instruct-fp16" + }, + "response": { + "body": [ + { + "__type__": "ollama._types.GenerateResponse", + "__data__": { + "model": "llama3.2:3b-instruct-fp16", + "created_at": "2025-08-18T19:47:58.267146Z", + "done": false, + "done_reason": null, + "total_duration": null, + "load_duration": null, + "prompt_eval_count": null, + "prompt_eval_duration": null, + "eval_count": null, + "eval_duration": null, + "response": "Hello", + "thinking": null, + "context": null + } + }, + { + "__type__": "ollama._types.GenerateResponse", + "__data__": { + "model": "llama3.2:3b-instruct-fp16", + "created_at": "2025-08-18T19:47:58.309006Z", + "done": false, + "done_reason": null, + "total_duration": null, + "load_duration": null, + "prompt_eval_count": null, + "prompt_eval_duration": null, + "eval_count": null, + "eval_duration": null, + "response": ",", + "thinking": null, + "context": null + } + }, + { + "__type__": "ollama._types.GenerateResponse", + "__data__": { + "model": "llama3.2:3b-instruct-fp16", + "created_at": "2025-08-18T19:47:58.351179Z", + "done": false, + "done_reason": null, + "total_duration": null, + "load_duration": null, + "prompt_eval_count": null, + "prompt_eval_duration": null, + "eval_count": null, + "eval_duration": null, + "response": " how", + "thinking": null, + "context": null + } + }, + { + "__type__": "ollama._types.GenerateResponse", + "__data__": { + "model": "llama3.2:3b-instruct-fp16", + "created_at": "2025-08-18T19:47:58.393262Z", + "done": false, + "done_reason": null, + "total_duration": null, + "load_duration": null, + "prompt_eval_count": null, + "prompt_eval_duration": null, + "eval_count": null, + "eval_duration": null, + "response": " can", + "thinking": null, + "context": null + } + }, + { + "__type__": "ollama._types.GenerateResponse", + "__data__": { + "model": "llama3.2:3b-instruct-fp16", + "created_at": "2025-08-18T19:47:58.436079Z", + "done": false, + "done_reason": null, + "total_duration": null, + "load_duration": null, + "prompt_eval_count": null, + "prompt_eval_duration": null, + "eval_count": null, + "eval_duration": null, + "response": " I", + "thinking": null, + "context": null + } + }, + { + "__type__": "ollama._types.GenerateResponse", + "__data__": { + "model": "llama3.2:3b-instruct-fp16", + "created_at": "2025-08-18T19:47:58.478393Z", + "done": false, + "done_reason": null, + "total_duration": null, + "load_duration": null, + "prompt_eval_count": null, + "prompt_eval_duration": null, + "eval_count": null, + "eval_duration": null, + "response": " assist", + "thinking": null, + "context": null + } + }, + { + "__type__": "ollama._types.GenerateResponse", + "__data__": { + "model": "llama3.2:3b-instruct-fp16", + "created_at": "2025-08-18T19:47:58.520608Z", + "done": false, + "done_reason": null, + "total_duration": null, + "load_duration": null, + "prompt_eval_count": null, + "prompt_eval_duration": null, + "eval_count": null, + "eval_duration": null, + "response": " you", + "thinking": null, + "context": null + } + }, + { + "__type__": "ollama._types.GenerateResponse", + "__data__": { + "model": "llama3.2:3b-instruct-fp16", + "created_at": "2025-08-18T19:47:58.562885Z", + "done": false, + "done_reason": null, + "total_duration": null, + "load_duration": null, + "prompt_eval_count": null, + "prompt_eval_duration": null, + "eval_count": null, + "eval_duration": null, + "response": " today", + "thinking": null, + "context": null + } + }, + { + "__type__": "ollama._types.GenerateResponse", + "__data__": { + "model": "llama3.2:3b-instruct-fp16", + "created_at": "2025-08-18T19:47:58.604683Z", + "done": false, + "done_reason": null, + "total_duration": null, + "load_duration": null, + "prompt_eval_count": null, + "prompt_eval_duration": null, + "eval_count": null, + "eval_duration": null, + "response": "?", + "thinking": null, + "context": null + } + }, + { + "__type__": "ollama._types.GenerateResponse", + "__data__": { + "model": "llama3.2:3b-instruct-fp16", + "created_at": "2025-08-18T19:47:58.646586Z", + "done": true, + "done_reason": "stop", + "total_duration": 1011323917, + "load_duration": 76575458, + "prompt_eval_count": 31, + "prompt_eval_duration": 553259250, + "eval_count": 10, + "eval_duration": 380302792, + "response": "", + "thinking": null, + "context": null + } + } + ], + "is_streaming": true + } +} diff --git a/tests/integration/recordings/responses/d0ac68cbde69.json b/tests/integration/recordings/responses/d0ac68cbde69.json index b37962fb6..5c19e7c5a 100644 --- a/tests/integration/recordings/responses/d0ac68cbde69.json +++ b/tests/integration/recordings/responses/d0ac68cbde69.json @@ -11,7 +11,26 @@ "body": { "__type__": "ollama._types.ProcessResponse", "__data__": { - "models": [] + "models": [ + { + "model": "llama3.2:3b-instruct-fp16", + "name": "llama3.2:3b-instruct-fp16", + "digest": "195a8c01d91ec3cb1e0aad4624a51f2602c51fa7d96110f8ab5a20c84081804d", + "expires_at": "2025-08-18T13:47:44.262256-07:00", + "size": 7919570944, + "size_vram": 7919570944, + "details": { + "parent_model": "", + "format": "gguf", + "family": "llama", + "families": [ + "llama" + ], + "parameter_size": "3.2B", + "quantization_level": "F16" + } + } + ] } }, "is_streaming": false diff --git a/tests/integration/safety/test_safety.py b/tests/integration/safety/test_safety.py index 75b974926..a21d9cbdf 100644 --- a/tests/integration/safety/test_safety.py +++ b/tests/integration/safety/test_safety.py @@ -27,10 +27,10 @@ def data_url_from_image(file_path): @pytest.fixture(scope="session") def code_scanner_shield_id(available_shields): - if "CodeScanner" in available_shields: - return "CodeScanner" + if "code-scanner" in available_shields: + return "code-scanner" - pytest.skip("CodeScanner shield is not available. Skipping.") + pytest.skip("code-scanner shield is not available. Skipping.") def test_unsafe_examples(client_with_models, shield_id): @@ -163,6 +163,60 @@ def test_safety_with_code_scanner(client_with_models, code_scanner_shield_id, mo assert response.violation.user_message == "Sorry, I found security concerns in the code." +def test_unsafe_input_with_code_scanner_run_moderation(client_with_models, code_scanner_shield_id, model_providers): + code_safety_enabled = len(CODE_SCANNER_ENABLED_PROVIDERS & model_providers) > 0 + if not code_safety_enabled: + pytest.skip(f"Testing code scanner shields is not supported for model_providers {model_providers}") + + insecure_code = [ + """ + Here is an example of code that hashes a password using MD5: + + ``` + import hashlib + password = 'securepassword123'.encode() + hashed_password = hashlib.md5(password).hexdigest() + print("Insecure hashed password:", hashed_password) + ``` + """ + ] + moderation_object = client_with_models.moderations.create( + input=insecure_code, + model=code_scanner_shield_id, + ) + assert moderation_object.results[0].flagged is True, f"Code scanner should have flagged {insecure_code} as insecure" + assert all(value is True for value in moderation_object.results[0].categories.values()), ( + "Code scanner shield should have detected code insecure category" + ) + + +def test_safe_input_with_code_scanner_run_moderation(client_with_models, code_scanner_shield_id, model_providers): + code_safety_enabled = len(CODE_SCANNER_ENABLED_PROVIDERS & model_providers) > 0 + if not code_safety_enabled: + pytest.skip(f"Testing code scanner shields is not supported for model_providers {model_providers}") + + secure_code = [ + """ + Extract the first 5 characters from a string: + ``` + text = "Hello World" + first_five = text[:5] + print(first_five) # Output: "Hello" + + # Safe handling for strings shorter than 5 characters + def get_first_five(text): + return text[:5] if text else "" + ``` + """ + ] + moderation_object = client_with_models.moderations.create( + input=secure_code, + model=code_scanner_shield_id, + ) + + assert moderation_object.results[0].flagged is False, "Code scanner should not have flagged the code as insecure" + + # We can use an instance of the LlamaGuard shield to detect attempts to misuse # the interpreter as this is one of the existing categories it checks for def test_safety_with_code_interpreter_abuse(client_with_models, shield_id): diff --git a/tests/integration/vector_io/test_openai_vector_stores.py b/tests/integration/vector_io/test_openai_vector_stores.py index 7ccca9077..bead95c26 100644 --- a/tests/integration/vector_io/test_openai_vector_stores.py +++ b/tests/integration/vector_io/test_openai_vector_stores.py @@ -56,6 +56,7 @@ def skip_if_provider_doesnt_support_openai_vector_stores_search(client_with_mode "keyword": [ "inline::sqlite-vec", "remote::milvus", + "inline::milvus", ], "hybrid": [ "inline::sqlite-vec", diff --git a/tests/unit/providers/agents/meta_reference/test_response_conversion_utils.py b/tests/unit/providers/agents/meta_reference/test_response_conversion_utils.py index b568ce135..1b9657484 100644 --- a/tests/unit/providers/agents/meta_reference/test_response_conversion_utils.py +++ b/tests/unit/providers/agents/meta_reference/test_response_conversion_utils.py @@ -45,7 +45,6 @@ from llama_stack.providers.inline.agents.meta_reference.responses.utils import ( class TestConvertChatChoiceToResponseMessage: - @pytest.mark.asyncio async def test_convert_string_content(self): choice = OpenAIChoice( message=OpenAIAssistantMessageParam(content="Test message"), @@ -61,7 +60,6 @@ class TestConvertChatChoiceToResponseMessage: assert isinstance(result.content[0], OpenAIResponseOutputMessageContentOutputText) assert result.content[0].text == "Test message" - @pytest.mark.asyncio async def test_convert_text_param_content(self): choice = OpenAIChoice( message=OpenAIAssistantMessageParam( @@ -78,12 +76,10 @@ class TestConvertChatChoiceToResponseMessage: class TestConvertResponseContentToChatContent: - @pytest.mark.asyncio async def test_convert_string_content(self): result = await convert_response_content_to_chat_content("Simple string") assert result == "Simple string" - @pytest.mark.asyncio async def test_convert_text_content_parts(self): content = [ OpenAIResponseInputMessageContentText(text="First part"), @@ -98,7 +94,6 @@ class TestConvertResponseContentToChatContent: assert isinstance(result[1], OpenAIChatCompletionContentPartTextParam) assert result[1].text == "Second part" - @pytest.mark.asyncio async def test_convert_image_content(self): content = [OpenAIResponseInputMessageContentImage(image_url="https://example.com/image.jpg", detail="high")] @@ -111,7 +106,6 @@ class TestConvertResponseContentToChatContent: class TestConvertResponseInputToChatMessages: - @pytest.mark.asyncio async def test_convert_string_input(self): result = await convert_response_input_to_chat_messages("User message") @@ -119,7 +113,6 @@ class TestConvertResponseInputToChatMessages: assert isinstance(result[0], OpenAIUserMessageParam) assert result[0].content == "User message" - @pytest.mark.asyncio async def test_convert_function_tool_call_output(self): input_items = [ OpenAIResponseInputFunctionToolCallOutput( @@ -135,7 +128,6 @@ class TestConvertResponseInputToChatMessages: assert result[0].content == "Tool output" assert result[0].tool_call_id == "call_123" - @pytest.mark.asyncio async def test_convert_function_tool_call(self): input_items = [ OpenAIResponseOutputMessageFunctionToolCall( @@ -154,7 +146,6 @@ class TestConvertResponseInputToChatMessages: assert result[0].tool_calls[0].function.name == "test_function" assert result[0].tool_calls[0].function.arguments == '{"param": "value"}' - @pytest.mark.asyncio async def test_convert_response_message(self): input_items = [ OpenAIResponseMessage( @@ -173,7 +164,6 @@ class TestConvertResponseInputToChatMessages: class TestConvertResponseTextToChatResponseFormat: - @pytest.mark.asyncio async def test_convert_text_format(self): text = OpenAIResponseText(format=OpenAIResponseTextFormat(type="text")) result = await convert_response_text_to_chat_response_format(text) @@ -181,14 +171,12 @@ class TestConvertResponseTextToChatResponseFormat: assert isinstance(result, OpenAIResponseFormatText) assert result.type == "text" - @pytest.mark.asyncio async def test_convert_json_object_format(self): text = OpenAIResponseText(format={"type": "json_object"}) result = await convert_response_text_to_chat_response_format(text) assert isinstance(result, OpenAIResponseFormatJSONObject) - @pytest.mark.asyncio async def test_convert_json_schema_format(self): schema_def = {"type": "object", "properties": {"test": {"type": "string"}}} text = OpenAIResponseText( @@ -204,7 +192,6 @@ class TestConvertResponseTextToChatResponseFormat: assert result.json_schema["name"] == "test_schema" assert result.json_schema["schema"] == schema_def - @pytest.mark.asyncio async def test_default_text_format(self): text = OpenAIResponseText() result = await convert_response_text_to_chat_response_format(text) @@ -214,27 +201,22 @@ class TestConvertResponseTextToChatResponseFormat: class TestGetMessageTypeByRole: - @pytest.mark.asyncio async def test_user_role(self): result = await get_message_type_by_role("user") assert result == OpenAIUserMessageParam - @pytest.mark.asyncio async def test_system_role(self): result = await get_message_type_by_role("system") assert result == OpenAISystemMessageParam - @pytest.mark.asyncio async def test_assistant_role(self): result = await get_message_type_by_role("assistant") assert result == OpenAIAssistantMessageParam - @pytest.mark.asyncio async def test_developer_role(self): result = await get_message_type_by_role("developer") assert result == OpenAIDeveloperMessageParam - @pytest.mark.asyncio async def test_unknown_role(self): result = await get_message_type_by_role("unknown") assert result is None diff --git a/uv.lock b/uv.lock index a09406770..635b2bdfe 100644 --- a/uv.lock +++ b/uv.lock @@ -1719,7 +1719,7 @@ wheels = [ [[package]] name = "llama-stack" -version = "0.2.17" +version = "0.2.18" source = { editable = "." } dependencies = [ { name = "aiohttp" }, @@ -1809,6 +1809,7 @@ test = [ { name = "chardet" }, { name = "datasets" }, { name = "mcp" }, + { name = "milvus-lite" }, { name = "openai" }, { name = "pymilvus" }, { name = "pypdf" }, @@ -1831,6 +1832,7 @@ unit = [ { name = "faiss-cpu" }, { name = "litellm" }, { name = "mcp" }, + { name = "milvus-lite" }, { name = "ollama" }, { name = "openai" }, { name = "pymilvus" }, @@ -1854,9 +1856,9 @@ requires-dist = [ { name = "jinja2", specifier = ">=3.1.6" }, { name = "jsonschema" }, { name = "llama-api-client", specifier = ">=0.1.2" }, - { name = "llama-stack-client", specifier = ">=0.2.17" }, - { name = "llama-stack-client", marker = "extra == 'ui'", specifier = ">=0.2.17" }, - { name = "openai", specifier = ">=1.99.6" }, + { name = "llama-stack-client", specifier = ">=0.2.18" }, + { name = "llama-stack-client", marker = "extra == 'ui'", specifier = ">=0.2.18" }, + { name = "openai", specifier = ">=1.99.6,<1.100.0" }, { name = "opentelemetry-exporter-otlp-proto-http", specifier = ">=1.30.0" }, { name = "opentelemetry-sdk", specifier = ">=1.30.0" }, { name = "pandas", marker = "extra == 'ui'" }, @@ -1925,6 +1927,7 @@ test = [ { name = "chardet" }, { name = "datasets" }, { name = "mcp" }, + { name = "milvus-lite", specifier = ">=2.5.0" }, { name = "openai" }, { name = "pymilvus", specifier = ">=2.5.12" }, { name = "pypdf" }, @@ -1946,6 +1949,7 @@ unit = [ { name = "faiss-cpu" }, { name = "litellm" }, { name = "mcp" }, + { name = "milvus-lite", specifier = ">=2.5.0" }, { name = "ollama" }, { name = "openai" }, { name = "pymilvus", specifier = ">=2.5.12" }, @@ -1959,7 +1963,7 @@ unit = [ [[package]] name = "llama-stack-client" -version = "0.2.17" +version = "0.2.18" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "anyio" }, @@ -1978,9 +1982,9 @@ dependencies = [ { name = "tqdm" }, { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/c5/2a/bb2949d6a5c494d21da0c185d426e25eaa8016f8287b689249afc6c96fb5/llama_stack_client-0.2.17.tar.gz", hash = "sha256:1fe2070133c6356761e394fa346045e9b6b567d4c63157b9bc6be89b9a6e7a41", size = 257636, upload-time = "2025-08-05T01:42:55.911Z" } +sdist = { url = "https://files.pythonhosted.org/packages/69/da/5e5a745495f8a2b8ef24fc4d01fe9031aa2277c36447cb22192ec8c8cc1e/llama_stack_client-0.2.18.tar.gz", hash = "sha256:860c885c9e549445178ac55cc9422e6e2a91215ac7aff5aaccfb42f3ce07e79e", size = 277284, upload-time = "2025-08-19T22:12:09.106Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/81/fc/5eccc86b83c5ced3a3bca071d250a86ccafa4ff17546cf781deb7758ab74/llama_stack_client-0.2.17-py3-none-any.whl", hash = "sha256:336c32f8688700ff64717b8109f405dc87a990fbe310c2027ac9ed6d39d67d16", size = 350329, upload-time = "2025-08-05T01:42:54.381Z" }, + { url = "https://files.pythonhosted.org/packages/0a/e4/e97f8fdd8a07aa1efc7f7e37b5657d84357b664bf70dd1885a437edc0699/llama_stack_client-0.2.18-py3-none-any.whl", hash = "sha256:90f827d5476f7fc15fd993f1863af6a6e72bd064646bf6a99435eb43a1327f70", size = 367586, upload-time = "2025-08-19T22:12:07.899Z" }, ] [[package]]