phoenix-oss/llama-stack-mirror
0
0
Fork 1
mirror of https://github.com/meta-llama/llama-stack.git synced 2025-10-16 06:53:47 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

mmlu loose

Browse source
This commit is contained in:
Xi Yan 2024-11-07 18:36:41 -08:00
parent 6ee02ca23b
commit edeb6dcf04
4 changed files with 46 additions and 59 deletions
Download patch file Download diff file Expand all files Collapse all files

85
llama_stack/providers/tests/eval/test_eval.py
View file

@ -33,7 +33,6 @@ class Testeval:
_, eval_tasks_impl, _, _, _, _ = eval_stack
response = await eval_tasks_impl.list_eval_tasks()
assert isinstance(response, list)
assert len(response) == 0
@pytest.mark.asyncio
async def test_eval_evaluate_rows(self, eval_stack):
@ -41,11 +40,6 @@ class Testeval:
await register_dataset(
datasets_impl, for_generation=True, dataset_id="test_dataset_for_eval"
)
provider = datasetio_impl.routing_table.get_provider_impl(
"test_dataset_for_eval"
)
# if provider.__provider_spec__.provider_type != "meta-reference":
# pytest.skip("Only meta-reference provider supports registering datasets")
response = await datasets_impl.list_datasets()
assert len(response) >= 1
@ -83,49 +77,44 @@ class Testeval:
assert "meta-reference::llm_as_judge_8b_correctness" in response.scores
assert "meta-reference::equality" in response.scores
# @pytest.mark.asyncio
# async def test_eval_run_eval(self, eval_stack):
# eval_impl, eval_tasks_impl, _, _, datasetio_impl, datasets_impl = eval_stack
# await register_dataset(
# datasets_impl, for_generation=True, dataset_id="test_dataset_for_eval"
# )
# provider = datasetio_impl.routing_table.get_provider_impl(
# "test_dataset_for_eval"
# )
# if provider.__provider_spec__.provider_type != "meta-reference":
# pytest.skip("Only meta-reference provider supports registering datasets")
@pytest.mark.asyncio
async def test_eval_run_eval(self, eval_stack):
eval_impl, eval_tasks_impl, _, _, datasetio_impl, datasets_impl = eval_stack
await register_dataset(
datasets_impl, for_generation=True, dataset_id="test_dataset_for_eval"
)
# scoring_functions = [
# "meta-reference::llm_as_judge_8b_correctness",
# "meta-reference::subset_of",
# ]
scoring_functions = [
"meta-reference::llm_as_judge_8b_correctness",
"meta-reference::subset_of",
]
# task_id = "meta-reference::app_eval-2"
# task_def = EvalTaskDefWithProvider(
# identifier=task_id,
# dataset_id="test_dataset_for_eval",
# scoring_functions=scoring_functions,
# provider_id="meta-reference",
# )
# await eval_tasks_impl.register_eval_task(task_def)
# response = await eval_impl.run_eval(
# task_id=task_id,
# task_config=AppEvalTaskConfig(
# eval_candidate=ModelCandidate(
# model="Llama3.2-3B-Instruct",
# sampling_params=SamplingParams(),
# ),
# ),
# )
# assert response.job_id == "0"
# job_status = await eval_impl.job_status(task_id, response.job_id)
# assert job_status and job_status.value == "completed"
# eval_response = await eval_impl.job_result(task_id, response.job_id)
task_id = "meta-reference::app_eval-2"
task_def = EvalTaskDefWithProvider(
identifier=task_id,
dataset_id="test_dataset_for_eval",
scoring_functions=scoring_functions,
provider_id="meta-reference",
)
await eval_tasks_impl.register_eval_task(task_def)
response = await eval_impl.run_eval(
task_id=task_id,
task_config=AppEvalTaskConfig(
eval_candidate=ModelCandidate(
model="Llama3.2-3B-Instruct",
sampling_params=SamplingParams(),
),
),
)
assert response.job_id == "0"
job_status = await eval_impl.job_status(task_id, response.job_id)
assert job_status and job_status.value == "completed"
eval_response = await eval_impl.job_result(task_id, response.job_id)
# assert eval_response is not None
# assert len(eval_response.generations) == 5
# assert "meta-reference::subset_of" in eval_response.scores
# assert "meta-reference::llm_as_judge_8b_correctness" in eval_response.scores
assert eval_response is not None
assert len(eval_response.generations) == 5
assert "meta-reference::subset_of" in eval_response.scores
assert "meta-reference::llm_as_judge_8b_correctness" in eval_response.scores
@pytest.mark.asyncio
async def test_eval_run_benchmark_eval(self, eval_stack):
@ -152,8 +141,8 @@ class Testeval:
num_examples=3,
),
)
job_status = await eval_impl.job_status(response.job_id, benchmark_id)
job_status = await eval_impl.job_status(benchmark_id, response.job_id)
assert job_status and job_status.value == "completed"
eval_response = await eval_impl.job_result(response.job_id, benchmark_id)
eval_response = await eval_impl.job_result(benchmark_id, response.job_id)
assert eval_response is not None
assert len(eval_response.generations) == 3