phoenix-oss/llama-stack-mirror
0
0
Fork 1
mirror of https://github.com/meta-llama/llama-stack.git synced 2025-12-31 07:33:51 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

add DocVQA scoring

Browse source
This commit is contained in:
Yang Yang 2025-03-14 11:01:11 -07:00
parent ab777ef5cd
commit 74ccc7e26a
6 changed files with 305 additions and 1 deletions
Download patch file Download diff file Expand all files Collapse all files

18
llama_stack/templates/open-benchmark/open_benchmark.py
View file

@ -203,6 +203,19 @@ def get_distribution_template() -> DistributionTemplate:
uri="huggingface://datasets/llamastack/bfcl_v3?split=train",
),
),
DatasetInput(
dataset_id="docvqa",
provider_id="huggingface",
url=URL(uri="https://huggingface.co/datasets/llamastack/docvqa"),
metadata={
"path": "llamastack/docvqa",
"split": "train",
},
dataset_schema={
"chat_completion_input": {"type": "string"},
"expected_answer": {"type": "string"},
},
),
]
default_benchmarks = [
@ -231,6 +244,11 @@ def get_distribution_template() -> DistributionTemplate:
dataset_id="bfcl",
scoring_functions=["basic::bfcl"],
),
BenchmarkInput(
benchmark_id="meta-reference-docvqa",
dataset_id="docvqa",
scoring_functions=["basic::docvqa"],
),
]
return DistributionTemplate(
name=name,