phoenix-oss/llama-stack-mirror
0
0
Fork 1
mirror of https://github.com/meta-llama/llama-stack.git synced 2025-10-15 22:47:59 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

test full generation + eval

Browse source
This commit is contained in:
Xi Yan 2024-10-25 16:52:59 -07:00
parent 16620a8185
commit 247a53d393
5 changed files with 15 additions and 3 deletions
Download patch file Download diff file Expand all files Collapse all files

9
llama_stack/providers/tests/eval/test_eval.py
View file

@ -65,7 +65,10 @@ async def test_eval(eval_settings):
model="Llama3.2-1B-Instruct",
sampling_params=SamplingParams(),
),
scoring_functions=["subset_of"],
scoring_functions=[
"meta-reference::subset_of",
"meta-reference::llm_as_judge_8b_correctness",
],
)
assert response.job_id == "0"
job_status = await eval_impl.job_status(response.job_id)
@ -74,6 +77,8 @@ async def test_eval(eval_settings):
eval_response = await eval_impl.job_result(response.job_id)
print(eval_response)
assert eval_response is not None
assert len(eval_response.generations) == 5
assert "subset_of" in eval_response.scores
assert "meta-reference::subset_of" in eval_response.scores
assert "meta-reference::llm_as_judge_8b_correctness" in eval_response.scores