This commit is contained in:
Swapna Lekkala 2025-11-04 11:14:20 -08:00
parent d0796388e7
commit 97ff9b5ef8
4 changed files with 16 additions and 40 deletions

View file

@ -2,11 +2,15 @@
description: | description: |
Evaluations Evaluations
<<<<<<< HEAD
<<<<<<< HEAD <<<<<<< HEAD
Llama Stack Evaluation API for running evaluations on model and agent candidates. Llama Stack Evaluation API for running evaluations on model and agent candidates.
======= =======
Llama Stack Evaluation API for running evaluations on model and agent candidates." Llama Stack Evaluation API for running evaluations on model and agent candidates."
>>>>>>> eb10a349 (clean) >>>>>>> eb10a349 (clean)
=======
Llama Stack Evaluation API for running evaluations on model and agent candidates."
>>>>>>> 94479abf (fix uts)
sidebar_label: Eval sidebar_label: Eval
title: Eval title: Eval
--- ---

View file

@ -32,7 +32,7 @@
│ B. Vector IO (depends on inference, job_scheduler) │ │ B. Vector IO (depends on inference, job_scheduler) │
│ ├─ deps = { │ │ ├─ deps = { │
│ │ Api.inference: <impl>, │ │ │ Api.inference: <impl>, │
│ │ Api.job_scheduler: <scheduler_impl> │ │ Api.job_scheduler: <scheduler>
│ │ } │ │ │ } │
│ ├─ get_provider_impl(config, deps) │ │ ├─ get_provider_impl(config, deps) │
│ │ ├─ adapter = FaissVectorIOAdapter( │ │ │ ├─ adapter = FaissVectorIOAdapter( │

View file

@ -7,41 +7,10 @@
from .api import JobStatus, Scheduler from .api import JobStatus, Scheduler
from .config import CelerySchedulerConfig, InlineSchedulerConfig, SchedulerConfig from .config import CelerySchedulerConfig, InlineSchedulerConfig, SchedulerConfig
async def scheduler_impl(config: SchedulerConfig) -> Scheduler:
"""
Factory function to instantiate scheduler implementations.
Args:
config: Scheduler configuration (InlineSchedulerConfig or CelerySchedulerConfig)
Returns:
Scheduler: An initialized scheduler instance
Raises:
ValueError: If the config type is unknown
"""
impl: Scheduler
if isinstance(config, InlineSchedulerConfig):
from .inline import InlineSchedulerImpl
impl = InlineSchedulerImpl(config)
elif isinstance(config, CelerySchedulerConfig):
from .celery import CelerySchedulerImpl
impl = CelerySchedulerImpl(config)
else:
raise ValueError(f"Unknown scheduler config type: {type(config)}")
await impl.initialize()
return impl
__all__ = [ __all__ = [
"JobStatus", "JobStatus",
"Scheduler", "Scheduler",
"SchedulerConfig", "SchedulerConfig",
"InlineSchedulerConfig", "InlineSchedulerConfig",
"CelerySchedulerConfig", "CelerySchedulerConfig",
"scheduler_impl",
] ]

View file

@ -12,10 +12,7 @@ from unittest.mock import AsyncMock, MagicMock
import pytest import pytest
from llama_stack.core.storage.datatypes import KVStoreReference, SqliteKVStoreConfig from llama_stack.core.storage.datatypes import KVStoreReference, SqliteKVStoreConfig
from llama_stack.providers.utils.job_scheduler import ( from llama_stack.providers.utils.job_scheduler import InlineSchedulerConfig
InlineSchedulerConfig,
scheduler_impl,
)
from llama_stack.providers.utils.kvstore import register_kvstore_backends from llama_stack.providers.utils.kvstore import register_kvstore_backends
@ -45,7 +42,9 @@ def scheduler_config():
async def test_scheduler_api_exists(scheduler_config): async def test_scheduler_api_exists(scheduler_config):
"""Test that scheduler API is properly defined.""" """Test that scheduler API is properly defined."""
scheduler = await scheduler_impl(scheduler_config) from llama_stack.providers.utils.job_scheduler.inline import InlineSchedulerImpl
scheduler = InlineSchedulerImpl(scheduler_config)
# Verify all required methods exist # Verify all required methods exist
assert hasattr(scheduler, "initialize") assert hasattr(scheduler, "initialize")
@ -61,7 +60,9 @@ async def test_scheduler_api_exists(scheduler_config):
async def test_scheduler_not_implemented(scheduler_config): async def test_scheduler_not_implemented(scheduler_config):
"""Test that scheduler methods raise NotImplementedError.""" """Test that scheduler methods raise NotImplementedError."""
scheduler = await scheduler_impl(scheduler_config) from llama_stack.providers.utils.job_scheduler.inline import InlineSchedulerImpl
scheduler = InlineSchedulerImpl(scheduler_config)
# Test that all methods raise NotImplementedError # Test that all methods raise NotImplementedError
with pytest.raises(NotImplementedError, match="not yet available"): with pytest.raises(NotImplementedError, match="not yet available"):
@ -94,7 +95,9 @@ async def test_scheduler_not_implemented(scheduler_config):
async def test_two_phase_initialization_pattern(scheduler_config): async def test_two_phase_initialization_pattern(scheduler_config):
"""Test that the two-phase initialization pattern is supported.""" """Test that the two-phase initialization pattern is supported."""
scheduler = await scheduler_impl(scheduler_config) from llama_stack.providers.utils.job_scheduler.inline import InlineSchedulerImpl
scheduler = InlineSchedulerImpl(scheduler_config)
# Mock the methods to test the pattern # Mock the methods to test the pattern
scheduler.initialize = AsyncMock() scheduler.initialize = AsyncMock()