phoenix-oss/llama-stack-mirror
0
0
Fork 1
mirror of https://github.com/meta-llama/llama-stack.git synced 2025-07-31 16:01:46 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

move dataset to hf llamastack repo

Browse source
This commit is contained in:
Xi Yan 2024-11-08 11:42:16 -08:00
parent 72b2c885ee
commit 58c6138df1
3 changed files with 13 additions and 7 deletions
Download patch file Download diff file Expand all files Collapse all files

14
llama_stack/providers/inline/huggingface/datasetio/dataset_defs/llamastack_mmlu_loose.py → llama_stack/providers/inline/huggingface/datasetio/dataset_defs/llamastack_mmlu.py
View file

@ -9,13 +9,19 @@ from llama_stack.apis.common.type_system import ChatCompletionInputType, StringT
from llama_stack.apis.datasetio import DatasetDef from llama_stack.apis.datasetio import DatasetDef
# TODO: migrate these to meta-llama/ HF repo with finalized schema # TODO: migrate these to meta-llama/ HF repo with finalized schema
llamastack_mmlu_loose = DatasetDef( llamastack_mmlu = DatasetDef(
identifier="llamastack_mmlu_loose", identifier="llamastack_mmlu",
url=URL(uri="https://huggingface.co/datasets/yanxi0830/ls-mmlu"), url=URL(
uri="https://huggingface.co/datasets/llamastack/Llama-3.2-1B-Instruct-evals"
),
dataset_schema={ dataset_schema={
"input_query": StringType(), "input_query": StringType(),
"expected_answer": StringType(), "expected_answer": StringType(),
"chat_completion_input": ChatCompletionInputType(), "chat_completion_input": ChatCompletionInputType(),
}, },
metadata={"path": "yanxi0830/ls-mmlu", "split": "train"}, metadata={
"path": "llamastack/Llama-3.2-1B-Instruct-evals",
"name": "Llama-3.2-1B-Instruct-evals__mmlu__details",
"split": "train",
},
) )

4
llama_stack/providers/inline/huggingface/datasetio/huggingface.py
View file

@ -13,7 +13,7 @@ from llama_stack.providers.datatypes import DatasetsProtocolPrivate
from llama_stack.providers.utils.datasetio.url_utils import get_dataframe_from_url from llama_stack.providers.utils.datasetio.url_utils import get_dataframe_from_url
from .config import HuggingfaceDatasetIOConfig from .config import HuggingfaceDatasetIOConfig
from .dataset_defs.llamastack_mmlu_loose import llamastack_mmlu_loose from .dataset_defs.llamastack_mmlu import llamastack_mmlu
def load_hf_dataset(dataset_def: DatasetDef): def load_hf_dataset(dataset_def: DatasetDef):
@ -37,7 +37,7 @@ class HuggingfaceDatasetIOImpl(DatasetIO, DatasetsProtocolPrivate):
async def initialize(self) -> None: async def initialize(self) -> None:
# pre-registered benchmark datasets # pre-registered benchmark datasets
self.pre_registered_datasets = [llamastack_mmlu_loose] self.pre_registered_datasets = [llamastack_mmlu]
self.dataset_infos = {x.identifier: x for x in self.pre_registered_datasets} self.dataset_infos = {x.identifier: x for x in self.pre_registered_datasets}
async def shutdown(self) -> None: ... async def shutdown(self) -> None: ...

2
llama_stack/providers/inline/meta_reference/eval/eval_task_defs/meta_reference_mmlu.py
View file

@ -8,6 +8,6 @@ from llama_stack.apis.eval import EvalTaskDef
meta_reference_mmlu = EvalTaskDef( meta_reference_mmlu = EvalTaskDef(
identifier="meta-reference-mmlu", identifier="meta-reference-mmlu",
dataset_id="llamastack_mmlu_loose", dataset_id="llamastack_mmlu",
scoring_functions=["meta-reference::regex_parser_multiple_choice_answer"], scoring_functions=["meta-reference::regex_parser_multiple_choice_answer"],
) )