phoenix-oss/llama-stack-mirror
0
0
Fork 1
mirror of https://github.com/meta-llama/llama-stack.git synced 2025-07-31 16:01:46 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

move benchmark task def to file

Browse source
This commit is contained in:
Xi Yan 2024-11-07 21:35:02 -08:00
parent f429e75b3e
commit 989f070bc0
3 changed files with 20 additions and 9 deletions
Download patch file Download diff file Expand all files Collapse all files

11
llama_stack/providers/inline/meta_reference/eval/eval.py
View file

@ -19,6 +19,7 @@ from llama_stack.apis.scoring import Scoring
from llama_stack.providers.datatypes import EvalTasksProtocolPrivate
from .config import MetaReferenceEvalConfig
from .eval_task_defs.meta_reference_mmlu import meta_reference_mmlu
class ColumnName(Enum):
@ -51,15 +52,7 @@ class MetaReferenceEvalImpl(Eval, EvalTasksProtocolPrivate):
async def initialize(self) -> None:
# pre-register eval tasks
benchmark_tasks = [
EvalTaskDef(
identifier="meta-reference-mmlu",
dataset_id="llamastack_mmlu_loose",
scoring_functions=[
"meta-reference::regex_parser_multiple_choice_answer"
],
)
]
benchmark_tasks = [meta_reference_mmlu]
self.eval_tasks = {x.identifier: x for x in benchmark_tasks}
async def shutdown(self) -> None: ...

5
llama_stack/providers/inline/meta_reference/eval/eval_task_defs/__init__.py Normal file
View file

@ -0,0 +1,5 @@
# Copyright (c) Meta Platforms, Inc. and affiliates.
# All rights reserved.
#
# This source code is licensed under the terms described in the LICENSE file in
# the root directory of this source tree.

13
llama_stack/providers/inline/meta_reference/eval/eval_task_defs/meta_reference_mmlu.py Normal file
View file

@ -0,0 +1,13 @@
# Copyright (c) Meta Platforms, Inc. and affiliates.
# All rights reserved.
#
# This source code is licensed under the terms described in the LICENSE file in
# the root directory of this source tree.
from llama_stack.apis.eval import EvalTaskDef
meta_reference_mmlu = EvalTaskDef(
identifier="meta-reference-mmlu",
dataset_id="llamastack_mmlu_loose",
scoring_functions=["meta-reference::regex_parser_multiple_choice_answer"],
)