phoenix-oss/llama-stack-mirror
0
0
Fork 1
mirror of https://github.com/meta-llama/llama-stack.git synced 2025-08-12 13:00:39 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

fix

Browse source
This commit is contained in:
Xi Yan 2025-02-12 21:16:47 -08:00
parent 2d0f6865ac
commit 1395de57a6
3 changed files with 11 additions and 11 deletions
Download patch file Download diff file Expand all files Collapse all files

4
docs/_static/llama-stack-spec.html vendored
View file

@ -81,7 +81,7 @@
"deprecated": true "deprecated": true
} }
}, },
"/v1/eval-tasks/{benchmark_id}": { "/v1/eval-tasks/{task_id}": {
"get": { "get": {
"responses": { "responses": {
"200": { "200": {
@ -109,7 +109,7 @@
"parameters": [ "parameters": [
{ {
"name": "task_id", "name": "task_id",
"in": "query", "in": "path",
"required": true, "required": true,
"schema": { "schema": {
"type": "string" "type": "string"

4
docs/_static/llama-stack-spec.yaml vendored
View file

@ -35,7 +35,7 @@ paths:
$ref: '#/components/schemas/DeprecatedEvaluateRowsRequest' $ref: '#/components/schemas/DeprecatedEvaluateRowsRequest'
required: true required: true
deprecated: true deprecated: true
/v1/eval-tasks/{benchmark_id}: /v1/eval-tasks/{task_id}:
get: get:
responses: responses:
'200': '200':
@ -51,7 +51,7 @@ paths:
description: '' description: ''
parameters: parameters:
- name: task_id - name: task_id
in: query in: path
required: true required: true
schema: schema:
type: string type: string

14
docs/notebooks/Llama_Stack_Benchmark_Evals.ipynb
View file

@ -1017,14 +1017,14 @@
" \"content\": SYSTEM_PROMPT_TEMPLATE.format(subject=subset),\n", " \"content\": SYSTEM_PROMPT_TEMPLATE.format(subject=subset),\n",
"}\n", "}\n",
"\n", "\n",
"client.benchmarks.register(\n", "client.eval_tasks.register(\n",
" benchmark_id=\"meta-reference::mmmu\",\n", " eval_task_id=\"meta-reference::mmmu\",\n",
" dataset_id=f\"mmmu-{subset}-{split}\",\n", " dataset_id=f\"mmmu-{subset}-{split}\",\n",
" scoring_functions=[\"basic::regex_parser_multiple_choice_answer\"],\n", " scoring_functions=[\"basic::regex_parser_multiple_choice_answer\"],\n",
")\n", ")\n",
"\n", "\n",
"response = client.eval.evaluate_rows(\n", "response = client.eval.evaluate_rows(\n",
" benchmark_id=\"meta-reference::mmmu\",\n", " task_id=\"meta-reference::mmmu\",\n",
" input_rows=eval_rows,\n", " input_rows=eval_rows,\n",
" scoring_functions=[\"basic::regex_parser_multiple_choice_answer\"],\n", " scoring_functions=[\"basic::regex_parser_multiple_choice_answer\"],\n",
" task_config={\n", " task_config={\n",
@ -1196,8 +1196,8 @@
" provider_id=\"together\",\n", " provider_id=\"together\",\n",
")\n", ")\n",
"\n", "\n",
"client.benchmarks.register(\n", "client.eval_tasks.register(\n",
" benchmark_id=\"meta-reference::simpleqa\",\n", " eval_task_id=\"meta-reference::simpleqa\",\n",
" dataset_id=simpleqa_dataset_id,\n", " dataset_id=simpleqa_dataset_id,\n",
" scoring_functions=[\"llm-as-judge::405b-simpleqa\"],\n", " scoring_functions=[\"llm-as-judge::405b-simpleqa\"],\n",
")\n", ")\n",
@ -1214,7 +1214,7 @@
" \"sampling_params\": {\n", " \"sampling_params\": {\n",
" \"strategy\": {\n", " \"strategy\": {\n",
" \"type\": \"greedy\",\n", " \"type\": \"greedy\",\n",
" },b\n", " },\n",
" \"max_tokens\": 4096,\n", " \"max_tokens\": 4096,\n",
" \"repeat_penalty\": 1.0,\n", " \"repeat_penalty\": 1.0,\n",
" },\n", " },\n",
@ -1352,7 +1352,7 @@
"}\n", "}\n",
"\n", "\n",
"response = client.eval.evaluate_rows(\n", "response = client.eval.evaluate_rows(\n",
" benchmark_id=\"meta-reference::simpleqa\",\n", " task_id=\"meta-reference::simpleqa\",\n",
" input_rows=eval_rows.rows,\n", " input_rows=eval_rows.rows,\n",
" scoring_functions=[\"llm-as-judge::405b-simpleqa\"],\n", " scoring_functions=[\"llm-as-judge::405b-simpleqa\"],\n",
" task_config={\n", " task_config={\n",