move benchmark task def to file

llama_stack/providers/inline/meta_reference/eval/eval.py

@@ -19,6 +19,7 @@ from llama_stack.apis.scoring import Scoring
 from llama_stack.providers.datatypes import EvalTasksProtocolPrivate
 
 from .config import MetaReferenceEvalConfig
+from .eval_task_defs.meta_reference_mmlu import meta_reference_mmlu
 
 
 class ColumnName(Enum):
@@ -51,15 +52,7 @@ class MetaReferenceEvalImpl(Eval, EvalTasksProtocolPrivate):
 
     async def initialize(self) -> None:
         # pre-register eval tasks
-        benchmark_tasks = [
+        benchmark_tasks = [meta_reference_mmlu]
-            EvalTaskDef(
-                identifier="meta-reference-mmlu",
-                dataset_id="llamastack_mmlu_loose",
-                scoring_functions=[
-                    "meta-reference::regex_parser_multiple_choice_answer"
-                ],
-            )
-        ]
         self.eval_tasks = {x.identifier: x for x in benchmark_tasks}
 
     async def shutdown(self) -> None: ...

llama_stack/providers/inline/meta_reference/eval/eval_task_defs/__init__.py

@@ -0,0 +1,5 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the terms described in the LICENSE file in
+# the root directory of this source tree.

llama_stack/providers/inline/meta_reference/eval/eval_task_defs/meta_reference_mmlu.py

@@ -0,0 +1,13 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the terms described in the LICENSE file in
+# the root directory of this source tree.
+
+from llama_stack.apis.eval import EvalTaskDef
+
+meta_reference_mmlu = EvalTaskDef(
+    identifier="meta-reference-mmlu",
+    dataset_id="llamastack_mmlu_loose",
+    scoring_functions=["meta-reference::regex_parser_multiple_choice_answer"],
+)