From fd581c3d883e18f20246014da0d677fa29c1179d Mon Sep 17 00:00:00 2001 From: Xi Yan Date: Thu, 7 Nov 2024 16:17:49 -0800 Subject: [PATCH] only keep 1 run_eval --- llama_stack/apis/eval/eval.py | 22 +++------ llama_stack/distribution/routers/routers.py | 48 ++++++++----------- .../inline/meta_reference/eval/eval.py | 36 +++++--------- llama_stack/providers/tests/eval/test_eval.py | 15 +++--- 4 files changed, 45 insertions(+), 76 deletions(-) diff --git a/llama_stack/apis/eval/eval.py b/llama_stack/apis/eval/eval.py index 6aa4cae34..5ae779ca7 100644 --- a/llama_stack/apis/eval/eval.py +++ b/llama_stack/apis/eval/eval.py @@ -66,36 +66,28 @@ class EvaluateResponse(BaseModel): class Eval(Protocol): - @webmethod(route="/eval/run_benchmark", method="POST") - async def run_benchmark( - self, - benchmark_id: str, - benchmark_config: BenchmarkEvalTaskConfig, - ) -> Job: ... - @webmethod(route="/eval/run_eval", method="POST") async def run_eval( self, - task: EvalTaskDef, - task_config: AppEvalTaskConfig, + task_id: str, + task_def: EvalTaskDef, + task_config: EvalTaskConfig, ) -> Job: ... @webmethod(route="/eval/evaluate_rows", method="POST") async def evaluate_rows( self, + task_id: str, input_rows: List[Dict[str, Any]], scoring_functions: List[str], task_config: EvalTaskConfig, - eval_task_id: Optional[str] = None, ) -> EvaluateResponse: ... @webmethod(route="/eval/job/status", method="GET") - async def job_status( - self, job_id: str, eval_task_id: str - ) -> Optional[JobStatus]: ... + async def job_status(self, task_id: str, job_id: str) -> Optional[JobStatus]: ... @webmethod(route="/eval/job/cancel", method="POST") - async def job_cancel(self, job_id: str, eval_task_id: str) -> None: ... + async def job_cancel(self, task_id: str, job_id: str) -> None: ... @webmethod(route="/eval/job/result", method="GET") - async def job_result(self, job_id: str, eval_task_id: str) -> EvaluateResponse: ... + async def job_result(self, task_id: str, job_id: str) -> EvaluateResponse: ... diff --git a/llama_stack/distribution/routers/routers.py b/llama_stack/distribution/routers/routers.py index 06d50bd65..4b28a20d7 100644 --- a/llama_stack/distribution/routers/routers.py +++ b/llama_stack/distribution/routers/routers.py @@ -16,10 +16,6 @@ from llama_stack.apis.datasetio import * # noqa: F403 from llama_stack.apis.scoring import * # noqa: F403 from llama_stack.apis.eval import * # noqa: F403 -from llama_stack.providers.inline.meta_reference.eval.eval import ( - DEFAULT_EVAL_TASK_IDENTIFIER, -) - class MemoryRouter(Memory): """Routes to an provider based on the memory bank identifier""" @@ -268,36 +264,28 @@ class EvalRouter(Eval): async def shutdown(self) -> None: pass - async def run_benchmark( - self, - benchmark_id: str, - benchmark_config: BenchmarkEvalTaskConfig, - ) -> Job: - pass - async def run_eval( self, - task: EvalTaskDef, + task_id: str, + task_def: EvalTaskDef, task_config: AppEvalTaskConfig, ) -> Job: - return await self.routing_table.get_provider_impl(task.identifier).run_eval( - task=task, + return await self.routing_table.get_provider_impl(task_id).run_eval( + task_id=task_id, + task_def=task_def, task_config=task_config, ) @webmethod(route="/eval/evaluate_rows", method="POST") async def evaluate_rows( self, + task_id: str, input_rows: List[Dict[str, Any]], scoring_functions: List[str], task_config: EvalTaskConfig, - eval_task_id: Optional[str] = None, ) -> EvaluateResponse: - # NOTE: This is to deal with the case where we do not pre-register an eval benchmark_task - # We use default DEFAULT_EVAL_TASK_IDENTIFIER as identifier - if eval_task_id is None: - eval_task_id = DEFAULT_EVAL_TASK_IDENTIFIER - return await self.routing_table.get_provider_impl(eval_task_id).evaluate_rows( + return await self.routing_table.get_provider_impl(task_id).evaluate_rows( + task_id=task_id, input_rows=input_rows, scoring_functions=scoring_functions, task_config=task_config, @@ -305,27 +293,29 @@ class EvalRouter(Eval): async def job_status( self, + task_id: str, job_id: str, - eval_task_id: str, ) -> Optional[JobStatus]: - return await self.routing_table.get_provider_impl(eval_task_id).job_status( - job_id, eval_task_id + return await self.routing_table.get_provider_impl(task_id).job_status( + task_id, job_id ) async def job_cancel( self, + task_id: str, job_id: str, - eval_task_id: str, ) -> None: - await self.routing_table.get_provider_impl(eval_task_id).job_cancel( - job_id, eval_task_id + await self.routing_table.get_provider_impl(task_id).job_cancel( + task_id, + job_id, ) async def job_result( self, + task_id: str, job_id: str, - eval_task_id: str, ) -> EvaluateResponse: - return await self.routing_table.get_provider_impl(eval_task_id).job_result( - job_id, eval_task_id + return await self.routing_table.get_provider_impl(task_id).job_result( + task_id, + job_id, ) diff --git a/llama_stack/providers/inline/meta_reference/eval/eval.py b/llama_stack/providers/inline/meta_reference/eval/eval.py index a9a1978e9..d20cf30d2 100644 --- a/llama_stack/providers/inline/meta_reference/eval/eval.py +++ b/llama_stack/providers/inline/meta_reference/eval/eval.py @@ -7,14 +7,7 @@ from enum import Enum from llama_models.llama3.api.datatypes import * # noqa: F403 from .....apis.common.job_types import Job -from .....apis.eval.eval import ( - AppEvalTaskConfig, - BenchmarkEvalTaskConfig, - Eval, - EvalTaskConfig, - EvaluateResponse, - JobStatus, -) +from .....apis.eval.eval import Eval, EvalTaskConfig, EvaluateResponse, JobStatus from llama_stack.apis.common.type_system import * # noqa: F403 from llama_stack.apis.datasetio import DatasetIO from llama_stack.apis.datasets import Datasets @@ -98,21 +91,15 @@ class MetaReferenceEvalImpl(Eval, EvalTasksProtocolPrivate): f"Dataset {dataset_id} does not have a correct input schema in {expected_schemas}" ) - async def run_benchmark( - self, - benchmark_id: str, - benchmark_config: BenchmarkEvalTaskConfig, - ) -> Job: - raise NotImplementedError("Benchmark eval is not implemented yet") - async def run_eval( self, - task: EvalTaskDef, - task_config: AppEvalTaskConfig, + task_id: str, + task_def: EvalTaskDef, + task_config: EvalTaskConfig, ) -> Job: - dataset_id = task.dataset_id + dataset_id = task_def.dataset_id candidate = task_config.eval_candidate - scoring_functions = task.scoring_functions + scoring_functions = task_def.scoring_functions await self.validate_eval_input_dataset_schema(dataset_id=dataset_id) all_rows = await self.datasetio_api.get_rows_paginated( @@ -120,6 +107,7 @@ class MetaReferenceEvalImpl(Eval, EvalTasksProtocolPrivate): rows_in_page=-1, ) res = await self.evaluate_rows( + task_id=task_id, input_rows=all_rows.rows, scoring_functions=scoring_functions, task_config=task_config, @@ -133,10 +121,10 @@ class MetaReferenceEvalImpl(Eval, EvalTasksProtocolPrivate): async def evaluate_rows( self, + task_id: str, input_rows: List[Dict[str, Any]], scoring_functions: List[str], task_config: EvalTaskConfig, - eval_task_id: Optional[str] = None, ) -> EvaluateResponse: candidate = task_config.eval_candidate if candidate.type == "agent": @@ -206,17 +194,17 @@ class MetaReferenceEvalImpl(Eval, EvalTasksProtocolPrivate): return EvaluateResponse(generations=generations, scores=score_response.results) - async def job_status(self, job_id: str, eval_task_id: str) -> Optional[JobStatus]: + async def job_status(self, task_id: str, job_id: str) -> Optional[JobStatus]: if job_id in self.jobs: return JobStatus.completed return None - async def job_cancel(self, job_id: str, eval_task_id: str) -> None: + async def job_cancel(self, task_id: str, job_id: str) -> None: raise NotImplementedError("Job cancel is not implemented yet") - async def job_result(self, job_id: str, eval_task_id: str) -> EvaluateResponse: - status = await self.job_status(job_id, eval_task_id) + async def job_result(self, task_id: str, job_id: str) -> EvaluateResponse: + status = await self.job_status(task_id, job_id) if not status or status != JobStatus.completed: raise ValueError(f"Job is not completed, Status: {status.value}") diff --git a/llama_stack/providers/tests/eval/test_eval.py b/llama_stack/providers/tests/eval/test_eval.py index d97f74ec4..794026e63 100644 --- a/llama_stack/providers/tests/eval/test_eval.py +++ b/llama_stack/providers/tests/eval/test_eval.py @@ -50,6 +50,7 @@ class Testeval: ] response = await eval_impl.evaluate_rows( + task_id="meta-reference::app_eval", input_rows=rows.rows, scoring_functions=scoring_functions, task_config=AppEvalTaskConfig( @@ -75,10 +76,12 @@ class Testeval: "meta-reference::subset_of", ] + task_id = "meta-reference::app_eval" response = await eval_impl.run_eval( - task=EvalTaskDef( + task_id=task_id, + task_def=EvalTaskDef( # NOTE: this is needed to make the router work for all app evals - identifier="meta-reference::app_eval", + identifier=task_id, dataset_id="test_dataset_for_eval", scoring_functions=scoring_functions, ), @@ -90,13 +93,9 @@ class Testeval: ), ) assert response.job_id == "0" - job_status = await eval_impl.job_status( - response.job_id, "meta-reference::app_eval" - ) + job_status = await eval_impl.job_status(task_id, response.job_id) assert job_status and job_status.value == "completed" - eval_response = await eval_impl.job_result( - response.job_id, "meta-reference::app_eval" - ) + eval_response = await eval_impl.job_result(task_id, response.job_id) assert eval_response is not None assert len(eval_response.generations) == 5