This commit is contained in:
Swapna Lekkala 2025-11-04 11:14:20 -08:00
parent d0796388e7
commit 97ff9b5ef8
4 changed files with 16 additions and 40 deletions

View file

@ -2,11 +2,15 @@
description: |
Evaluations
<<<<<<< HEAD
<<<<<<< HEAD
Llama Stack Evaluation API for running evaluations on model and agent candidates.
=======
Llama Stack Evaluation API for running evaluations on model and agent candidates."
>>>>>>> eb10a349 (clean)
=======
Llama Stack Evaluation API for running evaluations on model and agent candidates."
>>>>>>> 94479abf (fix uts)
sidebar_label: Eval
title: Eval
---

View file

@ -32,7 +32,7 @@
│ B. Vector IO (depends on inference, job_scheduler) │
│ ├─ deps = { │
│ │ Api.inference: <impl>, │
│ │ Api.job_scheduler: <scheduler_impl>
│ │ Api.job_scheduler: <scheduler>
│ │ } │
│ ├─ get_provider_impl(config, deps) │
│ │ ├─ adapter = FaissVectorIOAdapter( │

View file

@ -7,41 +7,10 @@
from .api import JobStatus, Scheduler
from .config import CelerySchedulerConfig, InlineSchedulerConfig, SchedulerConfig
async def scheduler_impl(config: SchedulerConfig) -> Scheduler:
"""
Factory function to instantiate scheduler implementations.
Args:
config: Scheduler configuration (InlineSchedulerConfig or CelerySchedulerConfig)
Returns:
Scheduler: An initialized scheduler instance
Raises:
ValueError: If the config type is unknown
"""
impl: Scheduler
if isinstance(config, InlineSchedulerConfig):
from .inline import InlineSchedulerImpl
impl = InlineSchedulerImpl(config)
elif isinstance(config, CelerySchedulerConfig):
from .celery import CelerySchedulerImpl
impl = CelerySchedulerImpl(config)
else:
raise ValueError(f"Unknown scheduler config type: {type(config)}")
await impl.initialize()
return impl
__all__ = [
"JobStatus",
"Scheduler",
"SchedulerConfig",
"InlineSchedulerConfig",
"CelerySchedulerConfig",
"scheduler_impl",
]

View file

@ -12,10 +12,7 @@ from unittest.mock import AsyncMock, MagicMock
import pytest
from llama_stack.core.storage.datatypes import KVStoreReference, SqliteKVStoreConfig
from llama_stack.providers.utils.job_scheduler import (
InlineSchedulerConfig,
scheduler_impl,
)
from llama_stack.providers.utils.job_scheduler import InlineSchedulerConfig
from llama_stack.providers.utils.kvstore import register_kvstore_backends
@ -45,7 +42,9 @@ def scheduler_config():
async def test_scheduler_api_exists(scheduler_config):
"""Test that scheduler API is properly defined."""
scheduler = await scheduler_impl(scheduler_config)
from llama_stack.providers.utils.job_scheduler.inline import InlineSchedulerImpl
scheduler = InlineSchedulerImpl(scheduler_config)
# Verify all required methods exist
assert hasattr(scheduler, "initialize")
@ -61,7 +60,9 @@ async def test_scheduler_api_exists(scheduler_config):
async def test_scheduler_not_implemented(scheduler_config):
"""Test that scheduler methods raise NotImplementedError."""
scheduler = await scheduler_impl(scheduler_config)
from llama_stack.providers.utils.job_scheduler.inline import InlineSchedulerImpl
scheduler = InlineSchedulerImpl(scheduler_config)
# Test that all methods raise NotImplementedError
with pytest.raises(NotImplementedError, match="not yet available"):
@ -94,7 +95,9 @@ async def test_scheduler_not_implemented(scheduler_config):
async def test_two_phase_initialization_pattern(scheduler_config):
"""Test that the two-phase initialization pattern is supported."""
scheduler = await scheduler_impl(scheduler_config)
from llama_stack.providers.utils.job_scheduler.inline import InlineSchedulerImpl
scheduler = InlineSchedulerImpl(scheduler_config)
# Mock the methods to test the pattern
scheduler.initialize = AsyncMock()