add DocVQA scoring

llama_stack/templates/open-benchmark/open_benchmark.py

@@ -203,6 +203,19 @@ def get_distribution_template() -> DistributionTemplate:
                 uri="huggingface://datasets/llamastack/bfcl_v3?split=train",
             ),
         ),
+        DatasetInput(
+            dataset_id="docvqa",
+            provider_id="huggingface",
+            url=URL(uri="https://huggingface.co/datasets/llamastack/docvqa"),
+            metadata={
+                "path": "llamastack/docvqa",
+                "split": "train",
+            },
+            dataset_schema={
+                "chat_completion_input": {"type": "string"},
+                "expected_answer": {"type": "string"},
+            },
+        ),
     ]
 
     default_benchmarks = [
@@ -231,6 +244,11 @@ def get_distribution_template() -> DistributionTemplate:
             dataset_id="bfcl",
             scoring_functions=["basic::bfcl"],
         ),
+        BenchmarkInput(
+            benchmark_id="meta-reference-docvqa",
+            dataset_id="docvqa",
+            scoring_functions=["basic::docvqa"],
+        ),
     ]
     return DistributionTemplate(
         name=name,

llama_stack/templates/open-benchmark/run.yaml

@@ -188,6 +188,22 @@ datasets:
     uri: huggingface://datasets/llamastack/bfcl_v3?split=train
   metadata: {}
   dataset_id: bfcl
+<<<<<<< HEAD
+=======
+  provider_id: huggingface
+- dataset_schema:
+    chat_completion_input:
+      type: string
+    expected_answer:
+      type: string
+  url:
+    uri: https://huggingface.co/datasets/llamastack/docvqa
+  metadata:
+    path: llamastack/docvqa
+    split: train
+  dataset_id: docvqa
+  provider_id: huggingface
+>>>>>>> 81df5a4e (add DocVQA scoring)
 scoring_fns: []
 benchmarks:
 - dataset_id: simpleqa
@@ -215,6 +231,11 @@ benchmarks:
   - basic::bfcl
   metadata: {}
   benchmark_id: meta-reference-bfcl
+- dataset_id: docvqa
+  scoring_functions:
+  - basic::docvqa
+  metadata: {}
+  benchmark_id: meta-reference-docvqa
 tool_groups:
 - toolgroup_id: builtin::websearch
   provider_id: tavily-search