This commit is contained in:
Xi Yan 2024-11-11 10:48:47 -05:00
parent 1031f1404b
commit 75ccc05296
4 changed files with 9 additions and 11 deletions

View file

@ -3,4 +3,4 @@
# #
# This source code is licensed under the terms described in the LICENSE file in # This source code is licensed under the terms described in the LICENSE file in
# the root directory of this source tree. # the root directory of this source tree.
from .llamastack_mmlu import llamastack_mmlu # noqa: F401 from .mmlu import mmlu # noqa: F401

View file

@ -8,19 +8,17 @@ from llama_models.llama3.api.datatypes import URL
from llama_stack.apis.common.type_system import ChatCompletionInputType, StringType from llama_stack.apis.common.type_system import ChatCompletionInputType, StringType
from llama_stack.apis.datasetio import DatasetDef from llama_stack.apis.datasetio import DatasetDef
llamastack_mmlu = DatasetDef( mmlu = DatasetDef(
identifier="llamastack_mmlu", identifier="mmlu",
url=URL( url=URL(uri="https://huggingface.co/datasets/llamastack/evals"),
uri="https://huggingface.co/datasets/llamastack/Llama-3.2-1B-Instruct-evals"
),
dataset_schema={ dataset_schema={
"input_query": StringType(), "input_query": StringType(),
"expected_answer": StringType(), "expected_answer": StringType(),
"chat_completion_input": ChatCompletionInputType(), "chat_completion_input": ChatCompletionInputType(),
}, },
metadata={ metadata={
"path": "llamastack/Llama-3.2-1B-Instruct-evals", "path": "llamastack/evals",
"name": "Llama-3.2-1B-Instruct-evals__mmlu__details", "name": "evals__mmlu__details",
"split": "train", "split": "train",
}, },
) )

View file

@ -12,7 +12,7 @@ import datasets as hf_datasets
from llama_stack.providers.datatypes import DatasetsProtocolPrivate from llama_stack.providers.datatypes import DatasetsProtocolPrivate
from llama_stack.providers.utils.datasetio.url_utils import get_dataframe_from_url from llama_stack.providers.utils.datasetio.url_utils import get_dataframe_from_url
from .benchmarks import llamastack_mmlu from .benchmarks import mmlu
from .config import HuggingfaceDatasetIOConfig from .config import HuggingfaceDatasetIOConfig
@ -38,7 +38,7 @@ class HuggingfaceDatasetIOImpl(DatasetIO, DatasetsProtocolPrivate):
async def initialize(self) -> None: async def initialize(self) -> None:
# pre-registered benchmark datasets # pre-registered benchmark datasets
pre_registered_datasets = [llamastack_mmlu] pre_registered_datasets = [mmlu]
self.dataset_infos = {x.identifier: x for x in pre_registered_datasets} self.dataset_infos = {x.identifier: x for x in pre_registered_datasets}
async def shutdown(self) -> None: ... async def shutdown(self) -> None: ...

View file

@ -8,6 +8,6 @@ from llama_stack.apis.eval import EvalTaskDef
meta_reference_mmlu = EvalTaskDef( meta_reference_mmlu = EvalTaskDef(
identifier="meta-reference-mmlu", identifier="meta-reference-mmlu",
dataset_id="llamastack_mmlu", dataset_id="mmlu",
scoring_functions=["meta-reference::regex_parser_multiple_choice_answer"], scoring_functions=["meta-reference::regex_parser_multiple_choice_answer"],
) )