mirror of
https://github.com/meta-llama/llama-stack.git
synced 2025-07-31 16:01:46 +00:00
rename
This commit is contained in:
parent
1031f1404b
commit
75ccc05296
4 changed files with 9 additions and 11 deletions
|
@ -3,4 +3,4 @@
|
||||||
#
|
#
|
||||||
# This source code is licensed under the terms described in the LICENSE file in
|
# This source code is licensed under the terms described in the LICENSE file in
|
||||||
# the root directory of this source tree.
|
# the root directory of this source tree.
|
||||||
from .llamastack_mmlu import llamastack_mmlu # noqa: F401
|
from .mmlu import mmlu # noqa: F401
|
||||||
|
|
|
@ -8,19 +8,17 @@ from llama_models.llama3.api.datatypes import URL
|
||||||
from llama_stack.apis.common.type_system import ChatCompletionInputType, StringType
|
from llama_stack.apis.common.type_system import ChatCompletionInputType, StringType
|
||||||
from llama_stack.apis.datasetio import DatasetDef
|
from llama_stack.apis.datasetio import DatasetDef
|
||||||
|
|
||||||
llamastack_mmlu = DatasetDef(
|
mmlu = DatasetDef(
|
||||||
identifier="llamastack_mmlu",
|
identifier="mmlu",
|
||||||
url=URL(
|
url=URL(uri="https://huggingface.co/datasets/llamastack/evals"),
|
||||||
uri="https://huggingface.co/datasets/llamastack/Llama-3.2-1B-Instruct-evals"
|
|
||||||
),
|
|
||||||
dataset_schema={
|
dataset_schema={
|
||||||
"input_query": StringType(),
|
"input_query": StringType(),
|
||||||
"expected_answer": StringType(),
|
"expected_answer": StringType(),
|
||||||
"chat_completion_input": ChatCompletionInputType(),
|
"chat_completion_input": ChatCompletionInputType(),
|
||||||
},
|
},
|
||||||
metadata={
|
metadata={
|
||||||
"path": "llamastack/Llama-3.2-1B-Instruct-evals",
|
"path": "llamastack/evals",
|
||||||
"name": "Llama-3.2-1B-Instruct-evals__mmlu__details",
|
"name": "evals__mmlu__details",
|
||||||
"split": "train",
|
"split": "train",
|
||||||
},
|
},
|
||||||
)
|
)
|
|
@ -12,7 +12,7 @@ import datasets as hf_datasets
|
||||||
from llama_stack.providers.datatypes import DatasetsProtocolPrivate
|
from llama_stack.providers.datatypes import DatasetsProtocolPrivate
|
||||||
from llama_stack.providers.utils.datasetio.url_utils import get_dataframe_from_url
|
from llama_stack.providers.utils.datasetio.url_utils import get_dataframe_from_url
|
||||||
|
|
||||||
from .benchmarks import llamastack_mmlu
|
from .benchmarks import mmlu
|
||||||
|
|
||||||
from .config import HuggingfaceDatasetIOConfig
|
from .config import HuggingfaceDatasetIOConfig
|
||||||
|
|
||||||
|
@ -38,7 +38,7 @@ class HuggingfaceDatasetIOImpl(DatasetIO, DatasetsProtocolPrivate):
|
||||||
|
|
||||||
async def initialize(self) -> None:
|
async def initialize(self) -> None:
|
||||||
# pre-registered benchmark datasets
|
# pre-registered benchmark datasets
|
||||||
pre_registered_datasets = [llamastack_mmlu]
|
pre_registered_datasets = [mmlu]
|
||||||
self.dataset_infos = {x.identifier: x for x in pre_registered_datasets}
|
self.dataset_infos = {x.identifier: x for x in pre_registered_datasets}
|
||||||
|
|
||||||
async def shutdown(self) -> None: ...
|
async def shutdown(self) -> None: ...
|
||||||
|
|
|
@ -8,6 +8,6 @@ from llama_stack.apis.eval import EvalTaskDef
|
||||||
|
|
||||||
meta_reference_mmlu = EvalTaskDef(
|
meta_reference_mmlu = EvalTaskDef(
|
||||||
identifier="meta-reference-mmlu",
|
identifier="meta-reference-mmlu",
|
||||||
dataset_id="llamastack_mmlu",
|
dataset_id="mmlu",
|
||||||
scoring_functions=["meta-reference::regex_parser_multiple_choice_answer"],
|
scoring_functions=["meta-reference::regex_parser_multiple_choice_answer"],
|
||||||
)
|
)
|
||||||
|
|
Loading…
Add table
Add a link
Reference in a new issue