delete preregistered dataset/eval task

This commit is contained in:
Xi Yan 2024-11-11 11:05:47 -05:00
parent 8bebe3fd1f
commit 9ff903e63b
5 changed files with 4 additions and 64 deletions

View file

@ -1,12 +0,0 @@
# Copyright (c) Meta Platforms, Inc. and affiliates.
# All rights reserved.
#
# This source code is licensed under the terms described in the LICENSE file in
# the root directory of this source tree.
# # Copyright (c) Meta Platforms, Inc. and affiliates.
# # All rights reserved.
# #
# # This source code is licensed under the terms described in the LICENSE file in
# # the root directory of this source tree.
# from .mmlu import mmlu # noqa: F401

View file

@ -1,30 +0,0 @@
# Copyright (c) Meta Platforms, Inc. and affiliates.
# All rights reserved.
#
# This source code is licensed under the terms described in the LICENSE file in
# the root directory of this source tree.
# # Copyright (c) Meta Platforms, Inc. and affiliates.
# # All rights reserved.
# #
# # This source code is licensed under the terms described in the LICENSE file in
# # the root directory of this source tree.
# from llama_models.llama3.api.datatypes import URL
# from llama_stack.apis.common.type_system import ChatCompletionInputType, StringType
# from llama_stack.apis.datasetio import DatasetDef
# mmlu = DatasetDef(
# identifier="mmlu",
# url=URL(uri="https://huggingface.co/datasets/llamastack/evals"),
# dataset_schema={
# "input_query": StringType(),
# "expected_answer": StringType(),
# "chat_completion_input": ChatCompletionInputType(),
# },
# metadata={
# "path": "llamastack/evals",
# "name": "evals__mmlu__details",
# "split": "train",
# },
# )

View file

@ -19,7 +19,6 @@ from llama_stack.apis.scoring import Scoring
from llama_stack.providers.datatypes import EvalTasksProtocolPrivate from llama_stack.providers.datatypes import EvalTasksProtocolPrivate
from .config import MetaReferenceEvalConfig from .config import MetaReferenceEvalConfig
from .eval_task_defs.meta_reference_mmlu import meta_reference_mmlu
class ColumnName(Enum): class ColumnName(Enum):
@ -51,9 +50,10 @@ class MetaReferenceEvalImpl(Eval, EvalTasksProtocolPrivate):
self.eval_tasks = {} self.eval_tasks = {}
async def initialize(self) -> None: async def initialize(self) -> None:
# pre-register eval tasks pass
benchmark_tasks = [meta_reference_mmlu] # # pre-register eval tasks
self.eval_tasks = {x.identifier: x for x in benchmark_tasks} # benchmark_tasks = [meta_reference_mmlu]
# self.eval_tasks = {x.identifier: x for x in benchmark_tasks}
async def shutdown(self) -> None: ... async def shutdown(self) -> None: ...

View file

@ -1,5 +0,0 @@
# Copyright (c) Meta Platforms, Inc. and affiliates.
# All rights reserved.
#
# This source code is licensed under the terms described in the LICENSE file in
# the root directory of this source tree.

View file

@ -1,13 +0,0 @@
# Copyright (c) Meta Platforms, Inc. and affiliates.
# All rights reserved.
#
# This source code is licensed under the terms described in the LICENSE file in
# the root directory of this source tree.
from llama_stack.apis.eval import EvalTaskDef
meta_reference_mmlu = EvalTaskDef(
identifier="meta-reference-mmlu",
dataset_id="mmlu",
scoring_functions=["meta-reference::regex_parser_multiple_choice_answer"],
)