diff --git a/.coveragerc b/.coveragerc index e16c2e461..d4925275f 100644 --- a/.coveragerc +++ b/.coveragerc @@ -4,3 +4,9 @@ omit = */llama_stack/providers/* */llama_stack/templates/* .venv/* + */llama_stack/cli/scripts/* + */llama_stack/ui/* + */llama_stack/distribution/ui/* + */llama_stack/strong_typing/* + */llama_stack/env.py + */__init__.py diff --git a/.github/workflows/coverage-badge.yml b/.github/workflows/coverage-badge.yml new file mode 100644 index 000000000..6b2f133dd --- /dev/null +++ b/.github/workflows/coverage-badge.yml @@ -0,0 +1,57 @@ +name: Coverage Badge + +on: + push: + branches: [ main ] + paths: + - 'llama_stack/**' + - 'tests/unit/**' + - 'uv.lock' + - 'pyproject.toml' + - 'requirements.txt' + - '.github/workflows/unit-tests.yml' + - '.github/workflows/coverage-badge.yml' # This workflow + workflow_dispatch: + +jobs: + unit-tests: + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + + - name: Install dependencies + uses: ./.github/actions/setup-runner + + - name: Run unit tests + run: | + ./scripts/unit-tests.sh + + - name: Coverage Badge + uses: tj-actions/coverage-badge-py@1788babcb24544eb5bbb6e0d374df5d1e54e670f # v2.0.4 + + - name: Verify Changed files + uses: tj-actions/verify-changed-files@a1c6acee9df209257a246f2cc6ae8cb6581c1edf # v20.0.4 + id: verify-changed-files + with: + files: coverage.svg + + - name: Commit files + if: steps.verify-changed-files.outputs.files_changed == 'true' + run: | + git config --local user.email "github-actions[bot]@users.noreply.github.com" + git config --local user.name "github-actions[bot]" + git add coverage.svg + git commit -m "Updated coverage.svg" + + - name: Create Pull Request + if: steps.verify-changed-files.outputs.files_changed == 'true' + uses: peter-evans/create-pull-request@271a8d0340265f705b14b6d32b9829c1cb33d45e # v7.0.8 + with: + token: ${{ secrets.GITHUB_TOKEN }} + title: "ci: [Automatic] Coverage Badge Update" + body: | + This PR updates the coverage badge based on the latest coverage report. + + Automatically generated by the [workflow coverage-badge.yaml](.github/workflows/coverage-badge.yaml) + delete-branch: true diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index e29045e52..41034b45f 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -36,7 +36,7 @@ jobs: - name: Run unit tests run: | - PYTHON_VERSION=${{ matrix.python }} ./scripts/unit-tests.sh --cov=llama_stack --junitxml=pytest-report-${{ matrix.python }}.xml --cov-report=html:htmlcov-${{ matrix.python }} + PYTHON_VERSION=${{ matrix.python }} ./scripts/unit-tests.sh --junitxml=pytest-report-${{ matrix.python }}.xml - name: Upload test results if: always() diff --git a/README.md b/README.md index 9148ce05d..7f0fed345 100644 --- a/README.md +++ b/README.md @@ -6,6 +6,7 @@ [![Discord](https://img.shields.io/discord/1257833999603335178?color=6A7EC2&logo=discord&logoColor=ffffff)](https://discord.gg/llama-stack) [![Unit Tests](https://github.com/meta-llama/llama-stack/actions/workflows/unit-tests.yml/badge.svg?branch=main)](https://github.com/meta-llama/llama-stack/actions/workflows/unit-tests.yml?query=branch%3Amain) [![Integration Tests](https://github.com/meta-llama/llama-stack/actions/workflows/integration-tests.yml/badge.svg?branch=main)](https://github.com/meta-llama/llama-stack/actions/workflows/integration-tests.yml?query=branch%3Amain) +![coverage badge](./coverage.svg) [**Quick Start**](https://llama-stack.readthedocs.io/en/latest/getting_started/index.html) | [**Documentation**](https://llama-stack.readthedocs.io/en/latest/index.html) | [**Colab Notebook**](./docs/getting_started.ipynb) | [**Discord**](https://discord.gg/llama-stack) diff --git a/coverage.svg b/coverage.svg new file mode 100644 index 000000000..636889bb0 --- /dev/null +++ b/coverage.svg @@ -0,0 +1,21 @@ + + + + + + + + + + + + + + + + coverage + coverage + 44% + 44% + + diff --git a/pyproject.toml b/pyproject.toml index 4d54bece0..15e2e10b4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -91,6 +91,7 @@ unit = [ "pymilvus>=2.5.12", "litellm", "together", + "coverage", ] # These are the core dependencies required for running integration tests. They are shared across all # providers. If a provider requires additional dependencies, please add them to your environment diff --git a/scripts/unit-tests.sh b/scripts/unit-tests.sh index 68d6458fc..458cd383d 100755 --- a/scripts/unit-tests.sh +++ b/scripts/unit-tests.sh @@ -16,4 +16,9 @@ if [ $FOUND_PYTHON -ne 0 ]; then uv python install "$PYTHON_VERSION" fi -uv run --python "$PYTHON_VERSION" --with-editable . --group unit pytest -s -v tests/unit/ $@ +# Run unit tests with coverage +uv run --python "$PYTHON_VERSION" --with-editable . --group unit \ + coverage run --source=llama_stack -m pytest -s -v tests/unit/ "$@" + +# Generate HTML coverage report +uv run --python "$PYTHON_VERSION" coverage html -d htmlcov-$PYTHON_VERSION diff --git a/tests/unit/README.md b/tests/unit/README.md index c95c3a0e7..06e22fb8c 100644 --- a/tests/unit/README.md +++ b/tests/unit/README.md @@ -1,9 +1,17 @@ # Llama Stack Unit Tests +## Unit Tests + +Unit tests verify individual components and functions in isolation. They are fast, reliable, and don't require external services. + +### Prerequisites + +1. **Python Environment**: Ensure you have Python 3.12+ installed +2. **uv Package Manager**: Install `uv` if not already installed + You can run the unit tests by running: ```bash -source .venv/bin/activate ./scripts/unit-tests.sh [PYTEST_ARGS] ``` @@ -19,3 +27,21 @@ If you'd like to run for a non-default version of Python (currently 3.12), pass source .venv/bin/activate PYTHON_VERSION=3.13 ./scripts/unit-tests.sh ``` + +### Test Configuration + +- **Test Discovery**: Tests are automatically discovered in the `tests/unit/` directory +- **Async Support**: Tests use `--asyncio-mode=auto` for automatic async test handling +- **Coverage**: Tests generate coverage reports in `htmlcov/` directory +- **Python Version**: Defaults to Python 3.12, but can be overridden with `PYTHON_VERSION` environment variable + +### Coverage Reports + +After running tests, you can view coverage reports: + +```bash +# Open HTML coverage report in browser +open htmlcov/index.html # macOS +xdg-open htmlcov/index.html # Linux +start htmlcov/index.html # Windows +``` diff --git a/uv.lock b/uv.lock index 7a9c5cab0..2c5197988 100644 --- a/uv.lock +++ b/uv.lock @@ -1390,6 +1390,7 @@ unit = [ { name = "aiosqlite" }, { name = "blobfile" }, { name = "chardet" }, + { name = "coverage" }, { name = "faiss-cpu" }, { name = "litellm" }, { name = "mcp" }, @@ -1499,6 +1500,7 @@ unit = [ { name = "aiosqlite" }, { name = "blobfile" }, { name = "chardet" }, + { name = "coverage" }, { name = "faiss-cpu" }, { name = "litellm" }, { name = "mcp" },