phoenix-oss/llama-stack-mirror
0
0
Fork 1
mirror of https://github.com/meta-llama/llama-stack.git synced 2025-10-16 14:57:20 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

fix

Browse source
This commit is contained in:
Xi Yan 2024-11-07 18:25:39 -08:00
parent 33b6d9b7b7
commit 6ee02ca23b
6 changed files with 100 additions and 87 deletions
Download patch file Download diff file Expand all files Collapse all files

13
llama_stack/providers/inline/meta_reference/eval/eval.py
View file

@ -49,7 +49,18 @@ class MetaReferenceEvalImpl(Eval, EvalTasksProtocolPrivate):
self.eval_tasks = {}
async def initialize(self) -> None: ...
async def initialize(self) -> None:
# pre-register eval tasks
benchmark_tasks = [
EvalTaskDef(
identifier="meta-reference-mmlu",
dataset_id="llamastack_mmlu",
scoring_functions=[
"meta-reference::regex_parser_multiple_choice_answer"
],
)
]
self.eval_tasks = {x.identifier: x for x in benchmark_tasks}
async def shutdown(self) -> None: ...