mirror of
https://github.com/meta-llama/llama-stack.git
synced 2025-10-04 20:14:13 +00:00
Some checks failed
Test External Providers Installed via Module / test-external-providers-from-module (venv) (push) Has been skipped
Python Package Build Test / build (3.13) (push) Failing after 3s
Vector IO Integration Tests / test-matrix (push) Failing after 6s
Integration Tests (Replay) / Integration Tests (, , , client=, ) (push) Failing after 5s
SqlStore Integration Tests / test-postgres (3.13) (push) Failing after 8s
SqlStore Integration Tests / test-postgres (3.12) (push) Failing after 13s
Unit Tests / unit-tests (3.13) (push) Failing after 4s
Test External API and Providers / test-external (venv) (push) Failing after 7s
Unit Tests / unit-tests (3.12) (push) Failing after 6s
Python Package Build Test / build (3.12) (push) Failing after 10s
Integration Auth Tests / test-matrix (oauth2_token) (push) Failing after 18s
API Conformance Tests / check-schema-compatibility (push) Successful in 22s
UI Tests / ui-tests (22) (push) Successful in 29s
Pre-commit / pre-commit (push) Successful in 1m25s
# What does this PR do? As shown in #3421, we can scale stack to handle more RPS with k8s replicas. This PR enables multi process stack with uvicorn --workers so that we can achieve the same scaling without being in k8s. To achieve that we refactor main to split out the app construction logic. This method needs to be non-async. We created a new `Stack` class to house impls and have a `start()` method to be called in lifespan to start background tasks instead of starting them in the old `construct_stack`. This way we avoid having to manage an event loop manually. ## Test Plan CI > uv run --with llama-stack python -m llama_stack.core.server.server benchmarking/k8s-benchmark/stack_run_config.yaml works. > LLAMA_STACK_CONFIG=benchmarking/k8s-benchmark/stack_run_config.yaml uv run uvicorn llama_stack.core.server.server:create_app --port 8321 --workers 4 works.
145 lines
5.1 KiB
Python
145 lines
5.1 KiB
Python
# Copyright (c) Meta Platforms, Inc. and affiliates.
|
|
# All rights reserved.
|
|
#
|
|
# This source code is licensed under the terms described in the LICENSE file in
|
|
# the root directory of this source tree.
|
|
|
|
"""
|
|
Unit tests for LlamaStackAsLibraryClient automatic initialization.
|
|
|
|
These tests ensure that the library client is automatically initialized
|
|
and ready to use immediately after construction.
|
|
"""
|
|
|
|
from llama_stack.core.library_client import (
|
|
AsyncLlamaStackAsLibraryClient,
|
|
LlamaStackAsLibraryClient,
|
|
)
|
|
from llama_stack.core.server.routes import RouteImpls
|
|
|
|
|
|
class TestLlamaStackAsLibraryClientAutoInitialization:
|
|
"""Test automatic initialization of library clients."""
|
|
|
|
def test_sync_client_auto_initialization(self, monkeypatch):
|
|
"""Test that sync client is automatically initialized after construction."""
|
|
# Mock the stack construction to avoid dependency issues
|
|
mock_impls = {}
|
|
mock_route_impls = RouteImpls({})
|
|
|
|
class MockStack:
|
|
def __init__(self, config, custom_provider_registry=None):
|
|
self.impls = mock_impls
|
|
|
|
async def initialize(self):
|
|
pass
|
|
|
|
def mock_initialize_route_impls(impls):
|
|
return mock_route_impls
|
|
|
|
monkeypatch.setattr("llama_stack.core.library_client.Stack", MockStack)
|
|
monkeypatch.setattr("llama_stack.core.library_client.initialize_route_impls", mock_initialize_route_impls)
|
|
|
|
client = LlamaStackAsLibraryClient("ci-tests")
|
|
|
|
assert client.async_client.route_impls is not None
|
|
|
|
async def test_async_client_auto_initialization(self, monkeypatch):
|
|
"""Test that async client can be initialized and works properly."""
|
|
# Mock the stack construction to avoid dependency issues
|
|
mock_impls = {}
|
|
mock_route_impls = RouteImpls({})
|
|
|
|
class MockStack:
|
|
def __init__(self, config, custom_provider_registry=None):
|
|
self.impls = mock_impls
|
|
|
|
async def initialize(self):
|
|
pass
|
|
|
|
def mock_initialize_route_impls(impls):
|
|
return mock_route_impls
|
|
|
|
monkeypatch.setattr("llama_stack.core.library_client.Stack", MockStack)
|
|
monkeypatch.setattr("llama_stack.core.library_client.initialize_route_impls", mock_initialize_route_impls)
|
|
|
|
client = AsyncLlamaStackAsLibraryClient("ci-tests")
|
|
|
|
# Initialize the client
|
|
result = await client.initialize()
|
|
assert result is True
|
|
assert client.route_impls is not None
|
|
|
|
def test_initialize_method_backward_compatibility(self, monkeypatch):
|
|
"""Test that initialize() method still works for backward compatibility."""
|
|
# Mock the stack construction to avoid dependency issues
|
|
mock_impls = {}
|
|
mock_route_impls = RouteImpls({})
|
|
|
|
class MockStack:
|
|
def __init__(self, config, custom_provider_registry=None):
|
|
self.impls = mock_impls
|
|
|
|
async def initialize(self):
|
|
pass
|
|
|
|
def mock_initialize_route_impls(impls):
|
|
return mock_route_impls
|
|
|
|
monkeypatch.setattr("llama_stack.core.library_client.Stack", MockStack)
|
|
monkeypatch.setattr("llama_stack.core.library_client.initialize_route_impls", mock_initialize_route_impls)
|
|
|
|
client = LlamaStackAsLibraryClient("ci-tests")
|
|
|
|
result = client.initialize()
|
|
assert result is None
|
|
|
|
result2 = client.initialize()
|
|
assert result2 is None
|
|
|
|
async def test_async_initialize_method_idempotent(self, monkeypatch):
|
|
"""Test that async initialize() method can be called multiple times safely."""
|
|
mock_impls = {}
|
|
mock_route_impls = RouteImpls({})
|
|
|
|
class MockStack:
|
|
def __init__(self, config, custom_provider_registry=None):
|
|
self.impls = mock_impls
|
|
|
|
async def initialize(self):
|
|
pass
|
|
|
|
def mock_initialize_route_impls(impls):
|
|
return mock_route_impls
|
|
|
|
monkeypatch.setattr("llama_stack.core.library_client.Stack", MockStack)
|
|
monkeypatch.setattr("llama_stack.core.library_client.initialize_route_impls", mock_initialize_route_impls)
|
|
|
|
client = AsyncLlamaStackAsLibraryClient("ci-tests")
|
|
|
|
result1 = await client.initialize()
|
|
assert result1 is True
|
|
|
|
result2 = await client.initialize()
|
|
assert result2 is True
|
|
|
|
def test_route_impls_automatically_set(self, monkeypatch):
|
|
"""Test that route_impls is automatically set during construction."""
|
|
mock_impls = {}
|
|
mock_route_impls = RouteImpls({})
|
|
|
|
class MockStack:
|
|
def __init__(self, config, custom_provider_registry=None):
|
|
self.impls = mock_impls
|
|
|
|
async def initialize(self):
|
|
pass
|
|
|
|
def mock_initialize_route_impls(impls):
|
|
return mock_route_impls
|
|
|
|
monkeypatch.setattr("llama_stack.core.library_client.Stack", MockStack)
|
|
monkeypatch.setattr("llama_stack.core.library_client.initialize_route_impls", mock_initialize_route_impls)
|
|
|
|
sync_client = LlamaStackAsLibraryClient("ci-tests")
|
|
assert sync_client.async_client.route_impls is not None
|