This commit is contained in:
Swapna Lekkala 2025-11-04 11:14:20 -08:00
parent d0796388e7
commit 97ff9b5ef8
4 changed files with 16 additions and 40 deletions

View file

@ -2,11 +2,15 @@
description: |
Evaluations
<<<<<<< HEAD
<<<<<<< HEAD
Llama Stack Evaluation API for running evaluations on model and agent candidates.
=======
Llama Stack Evaluation API for running evaluations on model and agent candidates."
>>>>>>> eb10a349 (clean)
=======
Llama Stack Evaluation API for running evaluations on model and agent candidates."
>>>>>>> 94479abf (fix uts)
sidebar_label: Eval
title: Eval
---
@ -17,6 +21,6 @@ title: Eval
Evaluations
Llama Stack Evaluation API for running evaluations on model and agent candidates.
Llama Stack Evaluation API for running evaluations on model and agent candidates.
This section contains documentation for all available providers for the **eval** API.

View file

@ -32,7 +32,7 @@
│ B. Vector IO (depends on inference, job_scheduler) │
│ ├─ deps = { │
│ │ Api.inference: <impl>, │
│ │ Api.job_scheduler: <scheduler_impl>
│ │ Api.job_scheduler: <scheduler>
│ │ } │
│ ├─ get_provider_impl(config, deps) │
│ │ ├─ adapter = FaissVectorIOAdapter( │

View file

@ -7,41 +7,10 @@
from .api import JobStatus, Scheduler
from .config import CelerySchedulerConfig, InlineSchedulerConfig, SchedulerConfig
async def scheduler_impl(config: SchedulerConfig) -> Scheduler:
"""
Factory function to instantiate scheduler implementations.
Args:
config: Scheduler configuration (InlineSchedulerConfig or CelerySchedulerConfig)
Returns:
Scheduler: An initialized scheduler instance
Raises:
ValueError: If the config type is unknown
"""
impl: Scheduler
if isinstance(config, InlineSchedulerConfig):
from .inline import InlineSchedulerImpl
impl = InlineSchedulerImpl(config)
elif isinstance(config, CelerySchedulerConfig):
from .celery import CelerySchedulerImpl
impl = CelerySchedulerImpl(config)
else:
raise ValueError(f"Unknown scheduler config type: {type(config)}")
await impl.initialize()
return impl
__all__ = [
"JobStatus",
"Scheduler",
"SchedulerConfig",
"InlineSchedulerConfig",
"CelerySchedulerConfig",
"scheduler_impl",
]

View file

@ -12,10 +12,7 @@ from unittest.mock import AsyncMock, MagicMock
import pytest
from llama_stack.core.storage.datatypes import KVStoreReference, SqliteKVStoreConfig
from llama_stack.providers.utils.job_scheduler import (
InlineSchedulerConfig,
scheduler_impl,
)
from llama_stack.providers.utils.job_scheduler import InlineSchedulerConfig
from llama_stack.providers.utils.kvstore import register_kvstore_backends
@ -45,7 +42,9 @@ def scheduler_config():
async def test_scheduler_api_exists(scheduler_config):
"""Test that scheduler API is properly defined."""
scheduler = await scheduler_impl(scheduler_config)
from llama_stack.providers.utils.job_scheduler.inline import InlineSchedulerImpl
scheduler = InlineSchedulerImpl(scheduler_config)
# Verify all required methods exist
assert hasattr(scheduler, "initialize")
@ -61,7 +60,9 @@ async def test_scheduler_api_exists(scheduler_config):
async def test_scheduler_not_implemented(scheduler_config):
"""Test that scheduler methods raise NotImplementedError."""
scheduler = await scheduler_impl(scheduler_config)
from llama_stack.providers.utils.job_scheduler.inline import InlineSchedulerImpl
scheduler = InlineSchedulerImpl(scheduler_config)
# Test that all methods raise NotImplementedError
with pytest.raises(NotImplementedError, match="not yet available"):
@ -94,7 +95,9 @@ async def test_scheduler_not_implemented(scheduler_config):
async def test_two_phase_initialization_pattern(scheduler_config):
"""Test that the two-phase initialization pattern is supported."""
scheduler = await scheduler_impl(scheduler_config)
from llama_stack.providers.utils.job_scheduler.inline import InlineSchedulerImpl
scheduler = InlineSchedulerImpl(scheduler_config)
# Mock the methods to test the pattern
scheduler.initialize = AsyncMock()