fix

llama_stack/providers/inline/huggingface/datasetio/dataset_defs/llamastack_mmlu.py

@@ -5,7 +5,7 @@
 # the root directory of this source tree.
 
 from llama_models.llama3.api.datatypes import URL
-from llama_stack.apis.common.type_system import StringType
+from llama_stack.apis.common.type_system import CompletionInputType, StringType
 from llama_stack.apis.datasetio import DatasetDef
 
 
@@ -15,7 +15,7 @@ llamastack_mmlu = DatasetDef(
     dataset_schema={
         "expected_answer": StringType(),
         "input_query": StringType(),
-        "generated_answer": StringType(),
+        "chat_completion_input": CompletionInputType(),
     },
     metadata={"path": "yanxi0830/ls-mmlu", "split": "train"},
 )

llama_stack/providers/inline/huggingface/datasetio/huggingface.py

@@ -10,6 +10,7 @@ from llama_stack.apis.datasetio import *  # noqa: F403
 
 from datasets import Dataset, load_dataset
 from llama_stack.providers.datatypes import DatasetsProtocolPrivate
+from llama_stack.providers.utils.datasetio.url_utils import get_dataframe_from_url
 
 from .config import HuggingfaceDatasetIOConfig
 from .dataset_defs.llamastack_mmlu import llamastack_mmlu

llama_stack/providers/inline/meta_reference/eval/eval.py

@@ -49,7 +49,18 @@ class MetaReferenceEvalImpl(Eval, EvalTasksProtocolPrivate):
 
         self.eval_tasks = {}
 
-    async def initialize(self) -> None: ...
+    async def initialize(self) -> None:
+        # pre-register eval tasks
+        benchmark_tasks = [
+            EvalTaskDef(
+                identifier="meta-reference-mmlu",
+                dataset_id="llamastack_mmlu",
+                scoring_functions=[
+                    "meta-reference::regex_parser_multiple_choice_answer"
+                ],
+            )
+        ]
+        self.eval_tasks = {x.identifier: x for x in benchmark_tasks}
 
     async def shutdown(self) -> None: ...