phoenix-oss/llama-stack-mirror
0
0
Fork 1
mirror of https://github.com/meta-llama/llama-stack.git synced 2025-08-10 04:08:31 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

eval

Browse source
This commit is contained in:
Xi Yan 2025-03-15 17:16:38 -07:00
parent b561cfd902
commit 659f5e86ee
5 changed files with 1094 additions and 1108 deletions
Download patch file Download diff file Expand all files Collapse all files

2154
docs/notebooks/Alpha_Llama_Stack_Post_Training.ipynb
View file

File diff suppressed because one or more lines are too long

10
docs/notebooks/Llama_Stack_Benchmark_Evals.ipynb
View file

@ -847,10 +847,10 @@
}, },
"outputs": [], "outputs": [],
"source": [ "source": [
"eval_rows = client.datasetio.get_rows_paginated(\n", "eval_rows = client.datasets.iterrows(\n",
" dataset_id=simpleqa_dataset_id,\n", " dataset_id=simpleqa_dataset_id,\n",
" rows_in_page=5,\n", " limit=5,\n",
")\n" ")"
] ]
}, },
{ {
@ -957,7 +957,7 @@
"\n", "\n",
"response = client.eval.evaluate_rows_alpha(\n", "response = client.eval.evaluate_rows_alpha(\n",
" benchmark_id=\"meta-reference::simpleqa\",\n", " benchmark_id=\"meta-reference::simpleqa\",\n",
" input_rows=eval_rows.rows,\n", " input_rows=eval_rows.data,\n",
" scoring_functions=[\"llm-as-judge::405b-simpleqa\"],\n", " scoring_functions=[\"llm-as-judge::405b-simpleqa\"],\n",
" benchmark_config={\n", " benchmark_config={\n",
" \"type\": \"benchmark\",\n", " \"type\": \"benchmark\",\n",
@ -1106,7 +1106,7 @@
"\n", "\n",
"response = client.eval.evaluate_rows_alpha(\n", "response = client.eval.evaluate_rows_alpha(\n",
" benchmark_id=\"meta-reference::simpleqa\",\n", " benchmark_id=\"meta-reference::simpleqa\",\n",
" input_rows=eval_rows.rows,\n", " input_rows=eval_rows.data,\n",
" scoring_functions=[\"llm-as-judge::405b-simpleqa\"],\n", " scoring_functions=[\"llm-as-judge::405b-simpleqa\"],\n",
" benchmark_config={\n", " benchmark_config={\n",
" \"type\": \"benchmark\",\n", " \"type\": \"benchmark\",\n",

8
docs/source/references/evals_reference/index.md
View file

@ -128,9 +128,9 @@ _ = client.datasets.register(
}, },
) )
eval_rows = client.datasetio.get_rows_paginated( eval_rows = client.datasets.iterrows(
dataset_id=simpleqa_dataset_id, dataset_id=simpleqa_dataset_id,
rows_in_page=5, limit=5,
) )
``` ```
@ -143,7 +143,7 @@ client.benchmarks.register(
response = client.eval.evaluate_rows( response = client.eval.evaluate_rows(
benchmark_id="meta-reference::simpleqa", benchmark_id="meta-reference::simpleqa",
input_rows=eval_rows.rows, input_rows=eval_rows.data,
scoring_functions=["llm-as-judge::405b-simpleqa"], scoring_functions=["llm-as-judge::405b-simpleqa"],
benchmark_config={ benchmark_config={
"eval_candidate": { "eval_candidate": {
@ -191,7 +191,7 @@ agent_config = {
response = client.eval.evaluate_rows( response = client.eval.evaluate_rows(
benchmark_id="meta-reference::simpleqa", benchmark_id="meta-reference::simpleqa",
input_rows=eval_rows.rows, input_rows=eval_rows.data,
scoring_functions=["llm-as-judge::405b-simpleqa"], scoring_functions=["llm-as-judge::405b-simpleqa"],
benchmark_config={ benchmark_config={
"eval_candidate": { "eval_candidate": {

6
llama_stack/distribution/ui/page/evaluations/native_eval.py
View file

@ -194,7 +194,7 @@ def run_evaluation_3():
if st.button("Run Evaluation"): if st.button("Run Evaluation"):
progress_text = "Running evaluation..." progress_text = "Running evaluation..."
progress_bar = st.progress(0, text=progress_text) progress_bar = st.progress(0, text=progress_text)
rows = rows.rows rows = rows.data
if num_rows < total_rows: if num_rows < total_rows:
rows = rows[:num_rows] rows = rows[:num_rows]
@ -229,7 +229,9 @@ def run_evaluation_3():
output_res[scoring_fn] = [] output_res[scoring_fn] = []
output_res[scoring_fn].append(eval_res.scores[scoring_fn].score_rows[0]) output_res[scoring_fn].append(eval_res.scores[scoring_fn].score_rows[0])
progress_text_container.write(f"Expand to see current processed result ({i + 1} / {len(rows)})") progress_text_container.write(
f"Expand to see current processed result ({i + 1} / {len(rows)})"
)
results_container.json(eval_res, expanded=2) results_container.json(eval_res, expanded=2)
progress_bar.progress(1.0, text="Evaluation complete!") progress_bar.progress(1.0, text="Evaluation complete!")

24
tests/integration/eval/test_eval.py
View file

@ -16,15 +16,17 @@ from ..datasetio.test_datasetio import register_dataset
@pytest.mark.parametrize("scoring_fn_id", ["basic::equality"]) @pytest.mark.parametrize("scoring_fn_id", ["basic::equality"])
def test_evaluate_rows(llama_stack_client, text_model_id, scoring_fn_id): def test_evaluate_rows(llama_stack_client, text_model_id, scoring_fn_id):
register_dataset(llama_stack_client, for_generation=True, dataset_id="test_dataset_for_eval") register_dataset(
llama_stack_client, for_generation=True, dataset_id="test_dataset_for_eval"
)
response = llama_stack_client.datasets.list() response = llama_stack_client.datasets.list()
assert any(x.identifier == "test_dataset_for_eval" for x in response) assert any(x.identifier == "test_dataset_for_eval" for x in response)
rows = llama_stack_client.datasetio.get_rows_paginated( rows = llama_stack_client.datasets.iterrows(
dataset_id="test_dataset_for_eval", dataset_id="test_dataset_for_eval",
rows_in_page=3, limit=3,
) )
assert len(rows.rows) == 3 assert len(rows.data) == 3
scoring_functions = [ scoring_functions = [
scoring_fn_id, scoring_fn_id,
@ -40,7 +42,7 @@ def test_evaluate_rows(llama_stack_client, text_model_id, scoring_fn_id):
response = llama_stack_client.eval.evaluate_rows( response = llama_stack_client.eval.evaluate_rows(
benchmark_id=benchmark_id, benchmark_id=benchmark_id,
input_rows=rows.rows, input_rows=rows.data,
scoring_functions=scoring_functions, scoring_functions=scoring_functions,
benchmark_config={ benchmark_config={
"eval_candidate": { "eval_candidate": {
@ -59,7 +61,9 @@ def test_evaluate_rows(llama_stack_client, text_model_id, scoring_fn_id):
@pytest.mark.parametrize("scoring_fn_id", ["basic::subset_of"]) @pytest.mark.parametrize("scoring_fn_id", ["basic::subset_of"])
def test_evaluate_benchmark(llama_stack_client, text_model_id, scoring_fn_id): def test_evaluate_benchmark(llama_stack_client, text_model_id, scoring_fn_id):
register_dataset(llama_stack_client, for_generation=True, dataset_id="test_dataset_for_eval_2") register_dataset(
llama_stack_client, for_generation=True, dataset_id="test_dataset_for_eval_2"
)
benchmark_id = str(uuid.uuid4()) benchmark_id = str(uuid.uuid4())
llama_stack_client.benchmarks.register( llama_stack_client.benchmarks.register(
benchmark_id=benchmark_id, benchmark_id=benchmark_id,
@ -80,10 +84,14 @@ def test_evaluate_benchmark(llama_stack_client, text_model_id, scoring_fn_id):
}, },
) )
assert response.job_id == "0" assert response.job_id == "0"
job_status = llama_stack_client.eval.jobs.status(job_id=response.job_id, benchmark_id=benchmark_id) job_status = llama_stack_client.eval.jobs.status(
job_id=response.job_id, benchmark_id=benchmark_id
)
assert job_status and job_status == "completed" assert job_status and job_status == "completed"
eval_response = llama_stack_client.eval.jobs.retrieve(job_id=response.job_id, benchmark_id=benchmark_id) eval_response = llama_stack_client.eval.jobs.retrieve(
job_id=response.job_id, benchmark_id=benchmark_id
)
assert eval_response is not None assert eval_response is not None
assert len(eval_response.generations) == 5 assert len(eval_response.generations) == 5
assert scoring_fn_id in eval_response.scores assert scoring_fn_id in eval_response.scores