phoenix-oss/llama-stack-mirror
0
0
Fork 1
mirror of https://github.com/meta-llama/llama-stack.git synced 2025-10-17 07:07:19 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

mmlu loose

Browse source
This commit is contained in:
Xi Yan 2024-11-07 18:36:41 -08:00
parent 6ee02ca23b
commit edeb6dcf04
4 changed files with 46 additions and 59 deletions
Download patch file Download diff file Expand all files Collapse all files

10
llama_stack/providers/inline/huggingface/datasetio/dataset_defs/llamastack_mmlu.py → llama_stack/providers/inline/huggingface/datasetio/dataset_defs/llamastack_mmlu_loose.py
View file

@ -5,17 +5,17 @@
# the root directory of this source tree.
from llama_models.llama3.api.datatypes import URL
from llama_stack.apis.common.type_system import CompletionInputType, StringType
from llama_stack.apis.common.type_system import ChatCompletionInputType, StringType
from llama_stack.apis.datasetio import DatasetDef
llamastack_mmlu = DatasetDef(
identifier="llamastack_mmlu",
llamastack_mmlu_loose = DatasetDef(
identifier="llamastack_mmlu_loose",
url=URL(uri="https://huggingface.co/datasets/yanxi0830/ls-mmlu"),
dataset_schema={
"expected_answer": StringType(),
"input_query": StringType(),
"chat_completion_input": CompletionInputType(),
"expected_answer": StringType(),
"chat_completion_input": ChatCompletionInputType(),
},
metadata={"path": "yanxi0830/ls-mmlu", "split": "train"},
)

8
llama_stack/providers/inline/huggingface/datasetio/huggingface.py
View file

@ -13,11 +13,11 @@ from llama_stack.providers.datatypes import DatasetsProtocolPrivate
from llama_stack.providers.utils.datasetio.url_utils import get_dataframe_from_url
from .config import HuggingfaceDatasetIOConfig
from .dataset_defs.llamastack_mmlu import llamastack_mmlu
from .dataset_defs.llamastack_mmlu_loose import llamastack_mmlu_loose
def load_hf_dataset(dataset_def: DatasetDef):
if dataset_def.metadata.get("dataset_path", None):
if dataset_def.metadata.get("path", None):
return load_dataset(**dataset_def.metadata)
df = get_dataframe_from_url(dataset_def.url)
@ -37,7 +37,7 @@ class HuggingfaceDatasetIOImpl(DatasetIO, DatasetsProtocolPrivate):
async def initialize(self) -> None:
# pre-registered benchmark datasets
self.pre_registered_datasets = [llamastack_mmlu]
self.pre_registered_datasets = [llamastack_mmlu_loose]
self.dataset_infos = {x.identifier: x for x in self.pre_registered_datasets}
async def shutdown(self) -> None: ...
@ -46,8 +46,6 @@ class HuggingfaceDatasetIOImpl(DatasetIO, DatasetsProtocolPrivate):
self,
dataset_def: DatasetDef,
) -> None:
print("registering dataset", dataset_def)
self.dataset_infos[dataset_def.identifier] = dataset_def
async def list_datasets(self) -> List[DatasetDef]:

2
llama_stack/providers/inline/meta_reference/eval/eval.py
View file

@ -54,7 +54,7 @@ class MetaReferenceEvalImpl(Eval, EvalTasksProtocolPrivate):
benchmark_tasks = [
EvalTaskDef(
identifier="meta-reference-mmlu",
dataset_id="llamastack_mmlu",
dataset_id="llamastack_mmlu_loose",
scoring_functions=[
"meta-reference::regex_parser_multiple_choice_answer"
],