phoenix-oss/llama-stack-mirror
0
0
Fork 1
mirror of https://github.com/meta-llama/llama-stack.git synced 2025-12-31 05:59:59 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

pre-commit scripts

Browse source
This commit is contained in:
Yang Yang 2025-03-16 23:55:49 -07:00
parent 9c26992390
commit d4df61e47a
2 changed files with 98 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

52
llama_stack/templates/open-benchmark/open_benchmark.py
View file

@ -189,6 +189,20 @@ def get_distribution_template() -> DistributionTemplate:
uri="huggingface://datasets/llamastack/gpqa_0shot_cot?split=test&name=gpqa_main",
),
),
DatasetInput(
dataset_id="gpqa_cot_diamond",
purpose=DatasetPurpose.eval_messages_answer,
source=URIDataSource(
uri="huggingface://datasets/llamastack/gpqa_0shot_cot?split=test&name=gpqa_diamond",
),
),
DatasetInput(
dataset_id="gpqa",
purpose=DatasetPurpose.eval_messages_answer,
source=URIDataSource(
uri="huggingface://datasets/llamastack/gpqa_0shot?split=test&name=gpqa_main",
),
),
DatasetInput(
dataset_id="math_500",
purpose=DatasetPurpose.eval_messages_answer,
@ -210,6 +224,34 @@ def get_distribution_template() -> DistributionTemplate:
uri="huggingface://datasets/llamastack/docvqa?split=val",
),
),
DatasetInput(
dataset_id="MMMU",
purpose=DatasetPurpose.eval_messages_answer,
source=URIDataSource(
uri="huggingface://datasets/llamastack/mmmu_v3?split=validation",
),
),
DatasetInput(
dataset_id="MMMU_Pro_standard",
purpose=DatasetPurpose.eval_messages_answer,
source=URIDataSource(
uri="huggingface://datasets/llamastack/MMMU_Pro?name=standard%20(10%20options)&split=test",
),
),
DatasetInput(
dataset_id="MMMU_Pro_vision",
purpose=DatasetPurpose.eval_messages_answer,
source=URIDataSource(
uri="huggingface://datasets/llamastack/MMMU_Pro?name=vision&split=test",
),
),
DatasetInput(
dataset_id="ai2d",
purpose=DatasetPurpose.eval_messages_answer,
source=URIDataSource(
uri="huggingface://datasets/llamastack/ai2d?split=test",
),
),
]
default_benchmarks = [
@ -243,6 +285,16 @@ def get_distribution_template() -> DistributionTemplate:
dataset_id="docvqa",
scoring_functions=["basic::docvqa"],
),
BenchmarkInput(
benchmark_id="meta-reference-MMMU_Pro_standard",
dataset_id="MMMU_Pro_standard",
scoring_functions=["basic::regex_parser_multiple_choice_answer"],
),
BenchmarkInput(
benchmark_id="meta-reference-MMMU_Pro_vision",
dataset_id="MMMU_Pro_vision",
scoring_functions=["basic::regex_parser_multiple_choice_answer"],
),
]
return DistributionTemplate(
name=name,