Merge branch 'main' into pr1573

This commit is contained in:
Xi Yan 2025-03-14 15:58:27 -07:00
commit cba4842a87
22 changed files with 1658 additions and 56 deletions

View file

@ -200,6 +200,22 @@ def get_distribution_template() -> DistributionTemplate:
uri="huggingface://llamastack/math_500?split=test",
),
),
DatasetInput(
dataset_id="bfcl",
provider_id="huggingface",
url=URL(uri="https://huggingface.co/datasets/llamastack/bfcl_v3"),
metadata={
"path": "llamastack/bfcl_v3",
"split": "train",
},
dataset_schema={
"function": {"type": "string"},
"language": {"type": "string"},
"ground_truth": {"type": "string"},
"id": {"type": "string"},
"chat_completion_input": {"type": "string"},
},
),
]
default_benchmarks = [
@ -223,6 +239,11 @@ def get_distribution_template() -> DistributionTemplate:
dataset_id="math_500",
scoring_functions=["basic::regex_parser_math_response"],
),
BenchmarkInput(
benchmark_id="meta-reference-bfcl",
dataset_id="bfcl",
scoring_functions=["basic::bfcl"],
),
]
return DistributionTemplate(
name=name,

View file

@ -186,6 +186,24 @@ datasets:
metadata: {}
dataset_id: math_500
provider_id: huggingface
- dataset_schema:
function:
type: string
language:
type: string
ground_truth:
type: string
id:
type: string
chat_completion_input:
type: string
url:
uri: https://huggingface.co/datasets/llamastack/bfcl_v3
metadata:
path: llamastack/bfcl_v3
split: train
dataset_id: bfcl
provider_id: huggingface
scoring_fns: []
benchmarks:
- dataset_id: simpleqa
@ -208,6 +226,11 @@ benchmarks:
- basic::regex_parser_math_response
metadata: {}
benchmark_id: meta-reference-math-500
- dataset_id: bfcl
scoring_functions:
- basic::bfcl
metadata: {}
benchmark_id: meta-reference-bfcl
tool_groups:
- toolgroup_id: builtin::websearch
provider_id: tavily-search