phoenix-oss/llama-stack-mirror
0
0
Fork 1
mirror of https://github.com/meta-llama/llama-stack.git synced 2025-08-12 13:00:39 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

update notebook

Browse source
This commit is contained in:
Xi Yan 2025-02-20 13:06:42 -08:00
parent c26d1a9a28
commit 547a49eb7a
1 changed files with 10 additions and 10 deletions
Download patch file Download diff file Expand all files Collapse all files

20
docs/notebooks/Llama_Stack_Benchmark_Evals.ipynb
View file

@ -1017,14 +1017,14 @@
" \"content\": SYSTEM_PROMPT_TEMPLATE.format(subject=subset),\n",
"}\n",
"\n",
"client.eval_tasks.register(\n",
" eval_task_id=\"meta-reference::mmmu\",\n",
"client.benchmarks.register(\n",
" benchmark_id=\"meta-reference::mmmu\",\n",
" dataset_id=f\"mmmu-{subset}-{split}\",\n",
" scoring_functions=[\"basic::regex_parser_multiple_choice_answer\"],\n",
")\n",
"\n",
"response = client.eval.evaluate_rows(\n",
" task_id=\"meta-reference::mmmu\",\n",
"response = client.eval.evaluate_rows_alpha(\n",
" benchmark_id=\"meta-reference::mmmu\",\n",
" input_rows=eval_rows,\n",
" scoring_functions=[\"basic::regex_parser_multiple_choice_answer\"],\n",
" task_config={\n",
@ -1196,14 +1196,14 @@
" provider_id=\"together\",\n",
")\n",
"\n",
"client.eval_tasks.register(\n",
" eval_task_id=\"meta-reference::simpleqa\",\n",
"client.benchmarks.register(\n",
" benchmark_id=\"meta-reference::simpleqa\",\n",
" dataset_id=simpleqa_dataset_id,\n",
" scoring_functions=[\"llm-as-judge::405b-simpleqa\"],\n",
")\n",
"\n",
"response = client.eval.evaluate_rows(\n",
" task_id=\"meta-reference::simpleqa\",\n",
"response = client.eval.evaluate_rows_alpha(\n",
" benchmark_id=\"meta-reference::simpleqa\",\n",
" input_rows=eval_rows.rows,\n",
" scoring_functions=[\"llm-as-judge::405b-simpleqa\"],\n",
" task_config={\n",
@ -1351,8 +1351,8 @@
" \"enable_session_persistence\": False,\n",
"}\n",
"\n",
"response = client.eval.evaluate_rows(\n",
" task_id=\"meta-reference::simpleqa\",\n",
"response = client.eval.evaluate_rows_alpha(\n",
" benchmark_id=\"meta-reference::simpleqa\",\n",
" input_rows=eval_rows.rows,\n",
" scoring_functions=[\"llm-as-judge::405b-simpleqa\"],\n",
" task_config={\n",