phoenix-oss/llama-stack-mirror
0
0
Fork 1
mirror of https://github.com/meta-llama/llama-stack.git synced 2025-10-15 22:47:59 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

fix

Browse source
This commit is contained in:
Xi Yan 2024-11-07 18:25:39 -08:00
parent 33b6d9b7b7
commit 6ee02ca23b
6 changed files with 100 additions and 87 deletions
Download patch file Download diff file Expand all files Collapse all files

87
llama_stack/providers/tests/eval/test_eval.py
View file

@ -11,6 +11,7 @@ from llama_models.llama3.api import SamplingParams
from llama_stack.apis.eval.eval import (
AppEvalTaskConfig,
BenchmarkEvalTaskConfig,
EvalTaskDefWithProvider,
ModelCandidate,
)
@ -82,49 +83,49 @@ class Testeval:
assert "meta-reference::llm_as_judge_8b_correctness" in response.scores
assert "meta-reference::equality" in response.scores
@pytest.mark.asyncio
async def test_eval_run_eval(self, eval_stack):
eval_impl, eval_tasks_impl, _, _, datasetio_impl, datasets_impl = eval_stack
await register_dataset(
datasets_impl, for_generation=True, dataset_id="test_dataset_for_eval"
)
provider = datasetio_impl.routing_table.get_provider_impl(
"test_dataset_for_eval"
)
if provider.__provider_spec__.provider_type != "meta-reference":
pytest.skip("Only meta-reference provider supports registering datasets")
# @pytest.mark.asyncio
# async def test_eval_run_eval(self, eval_stack):
# eval_impl, eval_tasks_impl, _, _, datasetio_impl, datasets_impl = eval_stack
# await register_dataset(
# datasets_impl, for_generation=True, dataset_id="test_dataset_for_eval"
# )
# provider = datasetio_impl.routing_table.get_provider_impl(
# "test_dataset_for_eval"
# )
# if provider.__provider_spec__.provider_type != "meta-reference":
# pytest.skip("Only meta-reference provider supports registering datasets")
scoring_functions = [
"meta-reference::llm_as_judge_8b_correctness",
"meta-reference::subset_of",
]
# scoring_functions = [
# "meta-reference::llm_as_judge_8b_correctness",
# "meta-reference::subset_of",
# ]
task_id = "meta-reference::app_eval-2"
task_def = EvalTaskDefWithProvider(
identifier=task_id,
dataset_id="test_dataset_for_eval",
scoring_functions=scoring_functions,
provider_id="meta-reference",
)
await eval_tasks_impl.register_eval_task(task_def)
response = await eval_impl.run_eval(
task_id=task_id,
task_config=AppEvalTaskConfig(
eval_candidate=ModelCandidate(
model="Llama3.2-3B-Instruct",
sampling_params=SamplingParams(),
),
),
)
assert response.job_id == "0"
job_status = await eval_impl.job_status(task_id, response.job_id)
assert job_status and job_status.value == "completed"
eval_response = await eval_impl.job_result(task_id, response.job_id)
# task_id = "meta-reference::app_eval-2"
# task_def = EvalTaskDefWithProvider(
# identifier=task_id,
# dataset_id="test_dataset_for_eval",
# scoring_functions=scoring_functions,
# provider_id="meta-reference",
# )
# await eval_tasks_impl.register_eval_task(task_def)
# response = await eval_impl.run_eval(
# task_id=task_id,
# task_config=AppEvalTaskConfig(
# eval_candidate=ModelCandidate(
# model="Llama3.2-3B-Instruct",
# sampling_params=SamplingParams(),
# ),
# ),
# )
# assert response.job_id == "0"
# job_status = await eval_impl.job_status(task_id, response.job_id)
# assert job_status and job_status.value == "completed"
# eval_response = await eval_impl.job_result(task_id, response.job_id)
assert eval_response is not None
assert len(eval_response.generations) == 5
assert "meta-reference::subset_of" in eval_response.scores
assert "meta-reference::llm_as_judge_8b_correctness" in eval_response.scores
# assert eval_response is not None
# assert len(eval_response.generations) == 5
# assert "meta-reference::subset_of" in eval_response.scores
# assert "meta-reference::llm_as_judge_8b_correctness" in eval_response.scores
@pytest.mark.asyncio
async def test_eval_run_benchmark_eval(self, eval_stack):
@ -141,9 +142,9 @@ class Testeval:
assert len(response) > 0
benchmark_id = "meta-reference-mmlu"
response = await eval_impl.run_benchmark(
benchmark_id=benchmark_id,
benchmark_config=BenchmarkEvalTaskConfig(
response = await eval_impl.run_eval(
task_id=benchmark_id,
task_config=BenchmarkEvalTaskConfig(
eval_candidate=ModelCandidate(
model="Llama3.2-3B-Instruct",
sampling_params=SamplingParams(),