phoenix-oss/llama-stack-mirror
0
0
Fork 1
mirror of https://github.com/meta-llama/llama-stack.git synced 2025-12-08 19:10:56 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

evals api mvp

Browse source
This commit is contained in:
Xi Yan 2024-10-04 00:50:03 -07:00
parent 3cbe3a72e8
commit 2441e66d14
3 changed files with 15 additions and 28 deletions
Download patch file Download diff file Expand all files Collapse all files

14
llama_stack/providers/impls/meta_reference/evals/evals.py
View file

@ -35,8 +35,7 @@ class MetaReferenceEvalsImpl(Evals):
task: str,
) -> EvaluateResponse:
cprint(f"model={model}, dataset={dataset}, task={task}", "red")
dataset = get_dataset("mmlu-simple-eval-en")
dataset = get_dataset(dataset)
task_impl = get_task(task, dataset)
x1 = task_impl.preprocess()
@ -52,11 +51,6 @@ class MetaReferenceEvalsImpl(Evals):
generation_outputs.append(x.completion_message.content)
x2 = task_impl.postprocess(generation_outputs)
scores = task_impl.score(x2)
print(scores)
return EvaluateResponse(
metrics={
"accuracy": 0.5,
}
)
eval_results = task_impl.score(x2)
eval_response = task_impl.aggregate_results(eval_results)
return eval_response