only keep 1 run_eval

llama_stack/apis/eval/eval.py

@@ -66,36 +66,28 @@ class EvaluateResponse(BaseModel):
 
 
 class Eval(Protocol):
-    @webmethod(route="/eval/run_benchmark", method="POST")
-    async def run_benchmark(
-        self,
-        benchmark_id: str,
-        benchmark_config: BenchmarkEvalTaskConfig,
-    ) -> Job: ...
-
     @webmethod(route="/eval/run_eval", method="POST")
     async def run_eval(
         self,
-        task: EvalTaskDef,
-        task_config: AppEvalTaskConfig,
+        task_id: str,
+        task_def: EvalTaskDef,
+        task_config: EvalTaskConfig,
     ) -> Job: ...
 
     @webmethod(route="/eval/evaluate_rows", method="POST")
     async def evaluate_rows(
         self,
+        task_id: str,
         input_rows: List[Dict[str, Any]],
         scoring_functions: List[str],
         task_config: EvalTaskConfig,
-        eval_task_id: Optional[str] = None,
     ) -> EvaluateResponse: ...
 
     @webmethod(route="/eval/job/status", method="GET")
-    async def job_status(
-        self, job_id: str, eval_task_id: str
-    ) -> Optional[JobStatus]: ...
+    async def job_status(self, task_id: str, job_id: str) -> Optional[JobStatus]: ...
 
     @webmethod(route="/eval/job/cancel", method="POST")
-    async def job_cancel(self, job_id: str, eval_task_id: str) -> None: ...
+    async def job_cancel(self, task_id: str, job_id: str) -> None: ...
 
     @webmethod(route="/eval/job/result", method="GET")
-    async def job_result(self, job_id: str, eval_task_id: str) -> EvaluateResponse: ...
+    async def job_result(self, task_id: str, job_id: str) -> EvaluateResponse: ...

llama_stack/distribution/routers/routers.py

@@ -16,10 +16,6 @@ from llama_stack.apis.datasetio import *  # noqa: F403
 from llama_stack.apis.scoring import *  # noqa: F403
 from llama_stack.apis.eval import *  # noqa: F403
 
-from llama_stack.providers.inline.meta_reference.eval.eval import (
-    DEFAULT_EVAL_TASK_IDENTIFIER,
-)
-
 
 class MemoryRouter(Memory):
     """Routes to an provider based on the memory bank identifier"""
@@ -268,36 +264,28 @@ class EvalRouter(Eval):
     async def shutdown(self) -> None:
         pass
 
-    async def run_benchmark(
-        self,
-        benchmark_id: str,
-        benchmark_config: BenchmarkEvalTaskConfig,
-    ) -> Job:
-        pass
-
     async def run_eval(
         self,
-        task: EvalTaskDef,
+        task_id: str,
+        task_def: EvalTaskDef,
         task_config: AppEvalTaskConfig,
     ) -> Job:
-        return await self.routing_table.get_provider_impl(task.identifier).run_eval(
-            task=task,
+        return await self.routing_table.get_provider_impl(task_id).run_eval(
+            task_id=task_id,
+            task_def=task_def,
             task_config=task_config,
         )
 
     @webmethod(route="/eval/evaluate_rows", method="POST")
     async def evaluate_rows(
         self,
+        task_id: str,
         input_rows: List[Dict[str, Any]],
         scoring_functions: List[str],
         task_config: EvalTaskConfig,
-        eval_task_id: Optional[str] = None,
     ) -> EvaluateResponse:
-        # NOTE: This is to deal with the case where we do not pre-register an eval benchmark_task
-        # We use default DEFAULT_EVAL_TASK_IDENTIFIER as identifier
-        if eval_task_id is None:
-            eval_task_id = DEFAULT_EVAL_TASK_IDENTIFIER
-        return await self.routing_table.get_provider_impl(eval_task_id).evaluate_rows(
+        return await self.routing_table.get_provider_impl(task_id).evaluate_rows(
+            task_id=task_id,
             input_rows=input_rows,
             scoring_functions=scoring_functions,
             task_config=task_config,
@@ -305,27 +293,29 @@ class EvalRouter(Eval):
 
     async def job_status(
         self,
+        task_id: str,
         job_id: str,
-        eval_task_id: str,
     ) -> Optional[JobStatus]:
-        return await self.routing_table.get_provider_impl(eval_task_id).job_status(
-            job_id, eval_task_id
+        return await self.routing_table.get_provider_impl(task_id).job_status(
+            task_id, job_id
         )
 
     async def job_cancel(
         self,
+        task_id: str,
         job_id: str,
-        eval_task_id: str,
     ) -> None:
-        await self.routing_table.get_provider_impl(eval_task_id).job_cancel(
-            job_id, eval_task_id
+        await self.routing_table.get_provider_impl(task_id).job_cancel(
+            task_id,
+            job_id,
         )
 
     async def job_result(
         self,
+        task_id: str,
         job_id: str,
-        eval_task_id: str,
     ) -> EvaluateResponse:
-        return await self.routing_table.get_provider_impl(eval_task_id).job_result(
-            job_id, eval_task_id
+        return await self.routing_table.get_provider_impl(task_id).job_result(
+            task_id,
+            job_id,
         )

llama_stack/providers/inline/meta_reference/eval/eval.py

@@ -7,14 +7,7 @@ from enum import Enum
 from llama_models.llama3.api.datatypes import *  # noqa: F403
 
 from .....apis.common.job_types import Job
-from .....apis.eval.eval import (
-    AppEvalTaskConfig,
-    BenchmarkEvalTaskConfig,
-    Eval,
-    EvalTaskConfig,
-    EvaluateResponse,
-    JobStatus,
-)
+from .....apis.eval.eval import Eval, EvalTaskConfig, EvaluateResponse, JobStatus
 from llama_stack.apis.common.type_system import *  # noqa: F403
 from llama_stack.apis.datasetio import DatasetIO
 from llama_stack.apis.datasets import Datasets
@@ -98,21 +91,15 @@ class MetaReferenceEvalImpl(Eval, EvalTasksProtocolPrivate):
                 f"Dataset {dataset_id} does not have a correct input schema in {expected_schemas}"
             )
 
-    async def run_benchmark(
-        self,
-        benchmark_id: str,
-        benchmark_config: BenchmarkEvalTaskConfig,
-    ) -> Job:
-        raise NotImplementedError("Benchmark eval is not implemented yet")
-
     async def run_eval(
         self,
-        task: EvalTaskDef,
-        task_config: AppEvalTaskConfig,
+        task_id: str,
+        task_def: EvalTaskDef,
+        task_config: EvalTaskConfig,
     ) -> Job:
-        dataset_id = task.dataset_id
+        dataset_id = task_def.dataset_id
         candidate = task_config.eval_candidate
-        scoring_functions = task.scoring_functions
+        scoring_functions = task_def.scoring_functions
 
         await self.validate_eval_input_dataset_schema(dataset_id=dataset_id)
         all_rows = await self.datasetio_api.get_rows_paginated(
@@ -120,6 +107,7 @@ class MetaReferenceEvalImpl(Eval, EvalTasksProtocolPrivate):
             rows_in_page=-1,
         )
         res = await self.evaluate_rows(
+            task_id=task_id,
             input_rows=all_rows.rows,
             scoring_functions=scoring_functions,
             task_config=task_config,
@@ -133,10 +121,10 @@ class MetaReferenceEvalImpl(Eval, EvalTasksProtocolPrivate):
 
     async def evaluate_rows(
         self,
+        task_id: str,
         input_rows: List[Dict[str, Any]],
         scoring_functions: List[str],
         task_config: EvalTaskConfig,
-        eval_task_id: Optional[str] = None,
     ) -> EvaluateResponse:
         candidate = task_config.eval_candidate
         if candidate.type == "agent":
@@ -206,17 +194,17 @@ class MetaReferenceEvalImpl(Eval, EvalTasksProtocolPrivate):
 
         return EvaluateResponse(generations=generations, scores=score_response.results)
 
-    async def job_status(self, job_id: str, eval_task_id: str) -> Optional[JobStatus]:
+    async def job_status(self, task_id: str, job_id: str) -> Optional[JobStatus]:
         if job_id in self.jobs:
             return JobStatus.completed
 
         return None
 
-    async def job_cancel(self, job_id: str, eval_task_id: str) -> None:
+    async def job_cancel(self, task_id: str, job_id: str) -> None:
         raise NotImplementedError("Job cancel is not implemented yet")
 
-    async def job_result(self, job_id: str, eval_task_id: str) -> EvaluateResponse:
-        status = await self.job_status(job_id, eval_task_id)
+    async def job_result(self, task_id: str, job_id: str) -> EvaluateResponse:
+        status = await self.job_status(task_id, job_id)
         if not status or status != JobStatus.completed:
             raise ValueError(f"Job is not completed, Status: {status.value}")
 

llama_stack/providers/tests/eval/test_eval.py

@@ -50,6 +50,7 @@ class Testeval:
         ]
 
         response = await eval_impl.evaluate_rows(
+            task_id="meta-reference::app_eval",
             input_rows=rows.rows,
             scoring_functions=scoring_functions,
             task_config=AppEvalTaskConfig(
@@ -75,10 +76,12 @@ class Testeval:
             "meta-reference::subset_of",
         ]
 
+        task_id = "meta-reference::app_eval"
         response = await eval_impl.run_eval(
-            task=EvalTaskDef(
+            task_id=task_id,
+            task_def=EvalTaskDef(
                 # NOTE: this is needed to make the router work for all app evals
-                identifier="meta-reference::app_eval",
+                identifier=task_id,
                 dataset_id="test_dataset_for_eval",
                 scoring_functions=scoring_functions,
             ),
@@ -90,13 +93,9 @@ class Testeval:
             ),
         )
         assert response.job_id == "0"
-        job_status = await eval_impl.job_status(
-            response.job_id, "meta-reference::app_eval"
-        )
+        job_status = await eval_impl.job_status(task_id, response.job_id)
         assert job_status and job_status.value == "completed"
-        eval_response = await eval_impl.job_result(
-            response.job_id, "meta-reference::app_eval"
-        )
+        eval_response = await eval_impl.job_result(task_id, response.job_id)
 
         assert eval_response is not None
         assert len(eval_response.generations) == 5