fix eval

llama_stack/distribution/distribution.py

@@ -43,6 +43,10 @@ def builtin_automatically_routed_apis() -> List[AutoRoutedApiInfo]:
             routing_table_api=Api.scoring_functions,
             router_api=Api.scoring,
         ),
+        AutoRoutedApiInfo(
+            routing_table_api=Api.eval_tasks,
+            router_api=Api.eval,
+        ),
     ]
 
 

llama_stack/distribution/resolver.py

@@ -15,6 +15,7 @@ from llama_stack.apis.agents import Agents
 from llama_stack.apis.datasetio import DatasetIO
 from llama_stack.apis.datasets import Datasets
 from llama_stack.apis.eval import Eval
+from llama_stack.apis.eval_tasks import EvalTasks
 from llama_stack.apis.inference import Inference
 from llama_stack.apis.inspect import Inspect
 from llama_stack.apis.memory import Memory
@@ -46,6 +47,7 @@ def api_protocol_map() -> Dict[Api, Any]:
         Api.scoring: Scoring,
         Api.scoring_functions: ScoringFunctions,
         Api.eval: Eval,
+        Api.eval_tasks: EvalTasks,
     }
 
 
@@ -56,6 +58,7 @@ def additional_protocols_map() -> Dict[Api, Any]:
         Api.safety: (ShieldsProtocolPrivate, Shields),
         Api.datasetio: (DatasetsProtocolPrivate, Datasets),
         Api.scoring: (ScoringFunctionsProtocolPrivate, ScoringFunctions),
+        Api.eval_tasks: (EvalTasksProtocolPrivate, EvalTasks),
     }
 
 

llama_stack/distribution/routers/__init__.py

@@ -12,6 +12,7 @@ from llama_stack.distribution.store import DistributionRegistry
 
 from .routing_tables import (
     DatasetsRoutingTable,
+    EvalTasksRoutingTable,
     MemoryBanksRoutingTable,
     ModelsRoutingTable,
     ScoringFunctionsRoutingTable,
@@ -31,6 +32,7 @@ async def get_routing_table_impl(
         "shields": ShieldsRoutingTable,
         "datasets": DatasetsRoutingTable,
         "scoring_functions": ScoringFunctionsRoutingTable,
+        "eval_tasks": EvalTasksRoutingTable,
     }
 
     if api.value not in api_to_tables:
@@ -44,6 +46,7 @@ async def get_routing_table_impl(
 async def get_auto_router_impl(api: Api, routing_table: RoutingTable, _deps) -> Any:
     from .routers import (
         DatasetIORouter,
+        EvalRouter,
         InferenceRouter,
         MemoryRouter,
         SafetyRouter,
@@ -56,6 +59,7 @@ async def get_auto_router_impl(api: Api, routing_table: RoutingTable, _deps) ->
         "safety": SafetyRouter,
         "datasetio": DatasetIORouter,
         "scoring": ScoringRouter,
+        "eval": EvalRouter,
     }
     if api.value not in api_to_routers:
         raise ValueError(f"API {api.value} not found in router map")

llama_stack/distribution/routers/routers.py

@@ -14,6 +14,7 @@ from llama_stack.apis.inference import *  # noqa: F403
 from llama_stack.apis.safety import *  # noqa: F403
 from llama_stack.apis.datasetio import *  # noqa: F403
 from llama_stack.apis.scoring import *  # noqa: F403
+from llama_stack.apis.eval import *  # noqa: F403
 
 
 class MemoryRouter(Memory):
@@ -252,3 +253,49 @@ class ScoringRouter(Scoring):
             res.update(score_response.results)
 
         return ScoreResponse(results=res)
+
+
+class EvalRouter(Eval):
+    def __init__(
+        self,
+        routing_table: RoutingTable,
+    ) -> None:
+        self.routing_table = routing_table
+
+    async def initialize(self) -> None:
+        pass
+
+    async def shutdown(self) -> None:
+        pass
+
+    async def run_benchmark_eval(
+        self,
+        benchmark_id: str,
+        eval_task_config: BenchmarkEvalTaskConfig,
+    ) -> Job:
+        pass
+
+    async def run_eval(
+        self,
+        eval_task_def: EvalTaskDef,
+        eval_task_config: EvalTaskConfig,
+    ) -> Job:
+        pass
+
+    @webmethod(route="/eval/evaluate_rows", method="POST")
+    async def evaluate_rows(
+        self,
+        input_rows: List[Dict[str, Any]],
+        scoring_functions: List[str],
+        eval_task_config: EvalTaskConfig,  # type: ignore
+    ) -> EvaluateResponse:
+        pass
+
+    async def job_status(self, job_id: str) -> Optional[JobStatus]:
+        pass
+
+    async def job_cancel(self, job_id: str) -> None:
+        pass
+
+    async def job_result(self, job_id: str) -> EvaluateResponse:
+        pass

llama_stack/distribution/routers/routing_tables.py

@@ -12,6 +12,8 @@ from llama_stack.apis.models import *  # noqa: F403
 from llama_stack.apis.shields import *  # noqa: F403
 from llama_stack.apis.memory_banks import *  # noqa: F403
 from llama_stack.apis.datasets import *  # noqa: F403
+from llama_stack.apis.eval_tasks import *  # noqa: F403
+
 
 from llama_stack.distribution.store import DistributionRegistry
 from llama_stack.distribution.datatypes import *  # noqa: F403
@@ -40,6 +42,8 @@ async def register_object_with_provider(obj: RoutableObject, p: Any) -> None:
         await p.register_dataset(obj)
     elif api == Api.scoring:
         await p.register_scoring_function(obj)
+    elif api == Api.eval:
+        await p.register_eval_task(obj)
     else:
         raise ValueError(f"Unknown API {api} for registering object with provider")
 
@@ -103,6 +107,11 @@ class CommonRoutingTableImpl(RoutingTable):
                 scoring_functions = await p.list_scoring_functions()
                 await add_objects(scoring_functions, pid, ScoringFnDefWithProvider)
 
+            elif api == Api.eval:
+                p.eval_task_store = self
+                eval_tasks = await p.list_eval_tasks()
+                await add_objects(eval_tasks, pid, EvalTaskDefWithProvider)
+
     async def shutdown(self) -> None:
         for p in self.impls_by_provider_id.values():
             await p.shutdown()
@@ -121,6 +130,8 @@ class CommonRoutingTableImpl(RoutingTable):
                 return ("DatasetIO", "dataset")
             elif isinstance(self, ScoringFunctionsRoutingTable):
                 return ("Scoring", "scoring_function")
+            elif isinstance(self, EvalTasksRoutingTable):
+                return ("Eval", "eval_task")
             else:
                 raise ValueError("Unknown routing table type")
 
@@ -246,7 +257,7 @@ class DatasetsRoutingTable(CommonRoutingTableImpl, Datasets):
         await self.register_object(dataset_def)
 
 
-class ScoringFunctionsRoutingTable(CommonRoutingTableImpl, Scoring):
+class ScoringFunctionsRoutingTable(CommonRoutingTableImpl, ScoringFunctions):
     async def list_scoring_functions(self) -> List[ScoringFnDefWithProvider]:
         return await self.get_all_with_type("scoring_fn")
 
@@ -259,3 +270,14 @@ class ScoringFunctionsRoutingTable(CommonRoutingTableImpl, Scoring):
         self, function_def: ScoringFnDefWithProvider
     ) -> None:
         await self.register_object(function_def)
+
+
+class EvalTasksRoutingTable(CommonRoutingTableImpl, EvalTasks):
+    async def list_eval_tasks(self) -> List[ScoringFnDefWithProvider]:
+        return await self.get_all_with_type("eval_task")
+
+    async def get_eval_task(self, name: str) -> Optional[EvalTaskDefWithProvider]:
+        return await self.get_object_by_identifier(name)
+
+    async def register_eval_task(self, eval_task_def: EvalTaskDefWithProvider) -> None:
+        await self.register_object(eval_task_def)