From 58c6138df12aa726d6a2ad0bd1cd014a5ace20c1 Mon Sep 17 00:00:00 2001
From: Xi Yan <xiyan@meta.com>
Date: Fri, 8 Nov 2024 11:42:16 -0800
Subject: [PATCH] move dataset to hf llamastack repo

---
 ...llamastack_mmlu_loose.py => llamastack_mmlu.py} | 14 ++++++++++----
 .../inline/huggingface/datasetio/huggingface.py    |  4 ++--
 .../eval/eval_task_defs/meta_reference_mmlu.py     |  2 +-
 3 files changed, 13 insertions(+), 7 deletions(-)
 rename llama_stack/providers/inline/huggingface/datasetio/dataset_defs/{llamastack_mmlu_loose.py => llamastack_mmlu.py} (64%)

diff --git a/llama_stack/providers/inline/huggingface/datasetio/dataset_defs/llamastack_mmlu_loose.py b/llama_stack/providers/inline/huggingface/datasetio/dataset_defs/llamastack_mmlu.py
similarity index 64%
rename from llama_stack/providers/inline/huggingface/datasetio/dataset_defs/llamastack_mmlu_loose.py
rename to llama_stack/providers/inline/huggingface/datasetio/dataset_defs/llamastack_mmlu.py
index 884b6da67..d59f7d8d0 100644
--- a/llama_stack/providers/inline/huggingface/datasetio/dataset_defs/llamastack_mmlu_loose.py
+++ b/llama_stack/providers/inline/huggingface/datasetio/dataset_defs/llamastack_mmlu.py
@@ -9,13 +9,19 @@ from llama_stack.apis.common.type_system import ChatCompletionInputType, StringT
 from llama_stack.apis.datasetio import DatasetDef
 
 # TODO: migrate these to meta-llama/ HF repo with finalized schema
-llamastack_mmlu_loose = DatasetDef(
-    identifier="llamastack_mmlu_loose",
-    url=URL(uri="https://huggingface.co/datasets/yanxi0830/ls-mmlu"),
+llamastack_mmlu = DatasetDef(
+    identifier="llamastack_mmlu",
+    url=URL(
+        uri="https://huggingface.co/datasets/llamastack/Llama-3.2-1B-Instruct-evals"
+    ),
     dataset_schema={
         "input_query": StringType(),
         "expected_answer": StringType(),
         "chat_completion_input": ChatCompletionInputType(),
     },
-    metadata={"path": "yanxi0830/ls-mmlu", "split": "train"},
+    metadata={
+        "path": "llamastack/Llama-3.2-1B-Instruct-evals",
+        "name": "Llama-3.2-1B-Instruct-evals__mmlu__details",
+        "split": "train",
+    },
 )
diff --git a/llama_stack/providers/inline/huggingface/datasetio/huggingface.py b/llama_stack/providers/inline/huggingface/datasetio/huggingface.py
index 0185d22dc..8958f6161 100644
--- a/llama_stack/providers/inline/huggingface/datasetio/huggingface.py
+++ b/llama_stack/providers/inline/huggingface/datasetio/huggingface.py
@@ -13,7 +13,7 @@ from llama_stack.providers.datatypes import DatasetsProtocolPrivate
 from llama_stack.providers.utils.datasetio.url_utils import get_dataframe_from_url
 
 from .config import HuggingfaceDatasetIOConfig
-from .dataset_defs.llamastack_mmlu_loose import llamastack_mmlu_loose
+from .dataset_defs.llamastack_mmlu import llamastack_mmlu
 
 
 def load_hf_dataset(dataset_def: DatasetDef):
@@ -37,7 +37,7 @@ class HuggingfaceDatasetIOImpl(DatasetIO, DatasetsProtocolPrivate):
 
     async def initialize(self) -> None:
         # pre-registered benchmark datasets
-        self.pre_registered_datasets = [llamastack_mmlu_loose]
+        self.pre_registered_datasets = [llamastack_mmlu]
         self.dataset_infos = {x.identifier: x for x in self.pre_registered_datasets}
 
     async def shutdown(self) -> None: ...
diff --git a/llama_stack/providers/inline/meta_reference/eval/eval_task_defs/meta_reference_mmlu.py b/llama_stack/providers/inline/meta_reference/eval/eval_task_defs/meta_reference_mmlu.py
index c14ce439c..938055316 100644
--- a/llama_stack/providers/inline/meta_reference/eval/eval_task_defs/meta_reference_mmlu.py
+++ b/llama_stack/providers/inline/meta_reference/eval/eval_task_defs/meta_reference_mmlu.py
@@ -8,6 +8,6 @@ from llama_stack.apis.eval import EvalTaskDef
 
 meta_reference_mmlu = EvalTaskDef(
     identifier="meta-reference-mmlu",
-    dataset_id="llamastack_mmlu_loose",
+    dataset_id="llamastack_mmlu",
     scoring_functions=["meta-reference::regex_parser_multiple_choice_answer"],
 )