phoenix-oss/llama-stack-mirror
0
0
Fork 1
mirror of https://github.com/meta-llama/llama-stack.git synced 2025-10-15 22:47:59 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

test wip

Browse source
This commit is contained in:
Xi Yan 2024-11-07 11:11:27 -08:00
parent 3322aa9ee4
commit 93995ecc4c
4 changed files with 54 additions and 2 deletions
Download patch file Download diff file Expand all files Collapse all files

29
llama_stack/providers/tests/eval/test_eval.py
View file

@ -98,3 +98,32 @@ class Testeval:
assert len(eval_response.generations) == 5
assert "meta-reference::subset_of" in eval_response.scores
assert "meta-reference::llm_as_judge_8b_correctness" in eval_response.scores
@pytest.mark.asyncio
async def test_eval_run_benchmark_eval(self, eval_stack):
eval_impl, eval_tasks_impl, _, _, datasetio_impl, datasets_impl = eval_stack
response = await datasets_impl.list_datasets()
assert len(response) == 1
rows = await datasetio_impl.get_rows_paginated(
dataset_id="llamastack_mmlu",
rows_in_page=3,
)
assert len(rows.rows) == 3
scoring_functions = [
"meta-reference::regex_parser_multiple_choice_answer",
]
response = await eval_impl.evaluate_rows(
input_rows=rows.rows,
scoring_functions=scoring_functions,
eval_task_config=AppEvalTaskConfig(
eval_candidate=ModelCandidate(
model="Llama3.2-3B-Instruct",
sampling_params=SamplingParams(),
),
),
)
print(response)