phoenix-oss/llama-stack-mirror
0
0
Fork 1
mirror of https://github.com/meta-llama/llama-stack.git synced 2025-10-16 14:57:20 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 2

register task required

Browse source
This commit is contained in:
Xi Yan 2024-11-07 16:41:23 -08:00
parent 7ca479f400
commit 94a56cc3f3
4 changed files with 30 additions and 22 deletions
Download patch file Download diff file Expand all files Collapse all files

33
llama_stack/providers/tests/eval/test_eval.py
View file

@ -9,7 +9,11 @@ import pytest
from llama_models.llama3.api import SamplingParams
from llama_stack.apis.eval.eval import AppEvalTaskConfig, EvalTaskDef, ModelCandidate
from llama_stack.apis.eval.eval import (
AppEvalTaskConfig,
EvalTaskDefWithProvider,
ModelCandidate,
)
from llama_stack.providers.tests.datasetio.test_datasetio import register_dataset
@ -28,7 +32,7 @@ class Testeval:
_, eval_tasks_impl, _, _, _, _ = eval_stack
response = await eval_tasks_impl.list_eval_tasks()
assert isinstance(response, list)
assert len(response) >= 1
assert len(response) == 0
@pytest.mark.asyncio
async def test_eval_evaluate_rows(self, eval_stack):
@ -48,9 +52,17 @@ class Testeval:
"meta-reference::llm_as_judge_8b_correctness",
"meta-reference::equality",
]
task_id = "meta-reference::app_eval"
task_def = EvalTaskDefWithProvider(
identifier=task_id,
dataset_id="test_dataset_for_eval",
scoring_functions=scoring_functions,
provider_id="meta-reference",
)
await eval_tasks_impl.register_eval_task(task_def)
response = await eval_impl.evaluate_rows(
task_id="meta-reference::app_eval",
task_id=task_id,
input_rows=rows.rows,
scoring_functions=scoring_functions,
task_config=AppEvalTaskConfig(
@ -76,15 +88,16 @@ class Testeval:
"meta-reference::subset_of",
]
task_id = "meta-reference::app_eval"
task_id = "meta-reference::app_eval-2"
task_def = EvalTaskDefWithProvider(
identifier=task_id,
dataset_id="test_dataset_for_eval",
scoring_functions=scoring_functions,
provider_id="meta-reference",
)
await eval_tasks_impl.register_eval_task(task_def)
response = await eval_impl.run_eval(
task_id=task_id,
task_def=EvalTaskDef(
# NOTE: this is needed to make the router work for all app evals
identifier=task_id,
dataset_id="test_dataset_for_eval",
scoring_functions=scoring_functions,
),
task_config=AppEvalTaskConfig(
eval_candidate=ModelCandidate(
model="Llama3.2-3B-Instruct",