rename

llama_stack/providers/adapters/datasetio/huggingface/benchmarks/mmlu.py

@@ -0,0 +1,24 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the terms described in the LICENSE file in
+# the root directory of this source tree.
+
+from llama_models.llama3.api.datatypes import URL
+from llama_stack.apis.common.type_system import ChatCompletionInputType, StringType
+from llama_stack.apis.datasetio import DatasetDef
+
+mmlu = DatasetDef(
+    identifier="mmlu",
+    url=URL(uri="https://huggingface.co/datasets/llamastack/evals"),
+    dataset_schema={
+        "input_query": StringType(),
+        "expected_answer": StringType(),
+        "chat_completion_input": ChatCompletionInputType(),
+    },
+    metadata={
+        "path": "llamastack/evals",
+        "name": "evals__mmlu__details",
+        "split": "train",
+    },
+)