phoenix-oss/llama-stack-mirror
0
0
Fork 1
mirror of https://github.com/meta-llama/llama-stack.git synced 2025-08-12 04:50:39 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

tmp eval

Browse source
This commit is contained in:
Xi Yan 2025-03-05 16:41:37 -08:00
parent 54abeeebce
commit fd68b0dc9a
3 changed files with 153 additions and 158 deletions
Download patch file Download diff file Expand all files Collapse all files

5
llama_stack/providers/utils/scoring/base_scoring_fn.py
View file

@ -73,6 +73,11 @@ class RegisteredBaseScoringFn(BaseScoringFn):
raise ValueError(f"Scoring function def with identifier {scoring_fn.identifier} already exists.") raise ValueError(f"Scoring function def with identifier {scoring_fn.identifier} already exists.")
self.supported_fn_defs_registry[scoring_fn.identifier] = scoring_fn self.supported_fn_defs_registry[scoring_fn.identifier] = scoring_fn
def unregister_scoring_fn_def(self, scoring_fn_id: str) -> None:
if scoring_fn_id not in self.supported_fn_defs_registry:
raise ValueError(f"Scoring function def with identifier {scoring_fn_id} does not exist.")
del self.supported_fn_defs_registry[scoring_fn_id]
@abstractmethod @abstractmethod
async def score_row( async def score_row(
self, self,

304
tests/integration/eval/test_eval.py
View file

@ -5,179 +5,169 @@
# the root directory of this source tree. # the root directory of this source tree.
import pytest
from llama_stack.apis.common.content_types import URL
from llama_stack.apis.common.type_system import ChatCompletionInputType, StringType
from llama_stack.apis.eval.eval import (
ModelCandidate,
)
from llama_stack.apis.inference import SamplingParams
from llama_stack.apis.scoring_functions import LLMAsJudgeScoringFnParams
from llama_stack.distribution.datatypes import Api
from ..datasetio.test_datasetio import register_dataset
from .constants import JUDGE_PROMPT
# How to run this test: # How to run this test:
# #
# pytest llama_stack/providers/tests/eval/test_eval.py # LLAMA_STACK_CONFIG="template-name" pytest -v tests/integration/eval
# -m "meta_reference_eval_together_inference_huggingface_datasetio"
# -v -s --tb=short --disable-warnings
@pytest.mark.skip(reason="FIXME FIXME @yanxi0830 this needs to be migrated to use the API") def test_benchmarks_list(llama_stack_client):
class Testeval: response = llama_stack_client.benchmarks.list()
@pytest.mark.asyncio assert isinstance(response, list)
async def test_benchmarks_list(self, eval_stack): assert len(response) == 0
# NOTE: this needs you to ensure that you are starting from a clean state
# but so far we don't have an unregister API unfortunately, so be careful
benchmarks_impl = eval_stack[Api.benchmarks]
response = await benchmarks_impl.list_benchmarks()
assert isinstance(response, list)
@pytest.mark.asyncio
async def test_eval_evaluate_rows(self, eval_stack, inference_model, judge_model):
eval_impl, benchmarks_impl, datasetio_impl, datasets_impl = (
eval_stack[Api.eval],
eval_stack[Api.benchmarks],
eval_stack[Api.datasetio],
eval_stack[Api.datasets],
)
await register_dataset(datasets_impl, for_generation=True, dataset_id="test_dataset_for_eval") # @pytest.mark.skip(reason="FIXME FIXME @yanxi0830 this needs to be migrated to use the API")
response = await datasets_impl.list_datasets() # class Testeval:
# @pytest.mark.asyncio
# async def test_benchmarks_list(self, eval_stack):
# # NOTE: this needs you to ensure that you are starting from a clean state
# # but so far we don't have an unregister API unfortunately, so be careful
# benchmarks_impl = eval_stack[Api.benchmarks]
# response = await benchmarks_impl.list_benchmarks()
# assert isinstance(response, list)
rows = await datasetio_impl.get_rows_paginated( # @pytest.mark.asyncio
dataset_id="test_dataset_for_eval", # async def test_eval_evaluate_rows(self, eval_stack, inference_model, judge_model):
rows_in_page=3, # eval_impl, benchmarks_impl, datasetio_impl, datasets_impl = (
) # eval_stack[Api.eval],
assert len(rows.rows) == 3 # eval_stack[Api.benchmarks],
# eval_stack[Api.datasetio],
# eval_stack[Api.datasets],
# )
scoring_functions = [ # await register_dataset(datasets_impl, for_generation=True, dataset_id="test_dataset_for_eval")
"basic::equality", # response = await datasets_impl.list_datasets()
]
benchmark_id = "meta-reference::app_eval"
await benchmarks_impl.register_benchmark(
benchmark_id=benchmark_id,
dataset_id="test_dataset_for_eval",
scoring_functions=scoring_functions,
)
response = await eval_impl.evaluate_rows(
benchmark_id=benchmark_id,
input_rows=rows.rows,
scoring_functions=scoring_functions,
benchmark_config=dict(
eval_candidate=ModelCandidate(
model=inference_model,
sampling_params=SamplingParams(),
),
scoring_params={
"meta-reference::llm_as_judge_base": LLMAsJudgeScoringFnParams(
judge_model=judge_model,
prompt_template=JUDGE_PROMPT,
judge_score_regexes=[
r"Total rating: (\d+)",
r"rating: (\d+)",
r"Rating: (\d+)",
],
)
},
),
)
assert len(response.generations) == 3
assert "basic::equality" in response.scores
@pytest.mark.asyncio # rows = await datasetio_impl.get_rows_paginated(
async def test_eval_run_eval(self, eval_stack, inference_model, judge_model): # dataset_id="test_dataset_for_eval",
eval_impl, benchmarks_impl, datasets_impl = ( # rows_in_page=3,
eval_stack[Api.eval], # )
eval_stack[Api.benchmarks], # assert len(rows.rows) == 3
eval_stack[Api.datasets],
)
await register_dataset(datasets_impl, for_generation=True, dataset_id="test_dataset_for_eval") # scoring_functions = [
# "basic::equality",
# ]
# benchmark_id = "meta-reference::app_eval"
# await benchmarks_impl.register_benchmark(
# benchmark_id=benchmark_id,
# dataset_id="test_dataset_for_eval",
# scoring_functions=scoring_functions,
# )
# response = await eval_impl.evaluate_rows(
# benchmark_id=benchmark_id,
# input_rows=rows.rows,
# scoring_functions=scoring_functions,
# benchmark_config=dict(
# eval_candidate=ModelCandidate(
# model=inference_model,
# sampling_params=SamplingParams(),
# ),
# scoring_params={
# "meta-reference::llm_as_judge_base": LLMAsJudgeScoringFnParams(
# judge_model=judge_model,
# prompt_template=JUDGE_PROMPT,
# judge_score_regexes=[
# r"Total rating: (\d+)",
# r"rating: (\d+)",
# r"Rating: (\d+)",
# ],
# )
# },
# ),
# )
# assert len(response.generations) == 3
# assert "basic::equality" in response.scores
scoring_functions = [ # @pytest.mark.asyncio
"basic::subset_of", # async def test_eval_run_eval(self, eval_stack, inference_model, judge_model):
] # eval_impl, benchmarks_impl, datasets_impl = (
# eval_stack[Api.eval],
# eval_stack[Api.benchmarks],
# eval_stack[Api.datasets],
# )
benchmark_id = "meta-reference::app_eval-2" # await register_dataset(datasets_impl, for_generation=True, dataset_id="test_dataset_for_eval")
await benchmarks_impl.register_benchmark(
benchmark_id=benchmark_id,
dataset_id="test_dataset_for_eval",
scoring_functions=scoring_functions,
)
response = await eval_impl.run_eval(
benchmark_id=benchmark_id,
benchmark_config=dict(
eval_candidate=ModelCandidate(
model=inference_model,
sampling_params=SamplingParams(),
),
),
)
assert response.job_id == "0"
job_status = await eval_impl.job_status(benchmark_id, response.job_id)
assert job_status and job_status.value == "completed"
eval_response = await eval_impl.job_result(benchmark_id, response.job_id)
assert eval_response is not None # scoring_functions = [
assert len(eval_response.generations) == 5 # "basic::subset_of",
assert "basic::subset_of" in eval_response.scores # ]
@pytest.mark.asyncio # benchmark_id = "meta-reference::app_eval-2"
async def test_eval_run_benchmark_eval(self, eval_stack, inference_model): # await benchmarks_impl.register_benchmark(
eval_impl, benchmarks_impl, datasets_impl = ( # benchmark_id=benchmark_id,
eval_stack[Api.eval], # dataset_id="test_dataset_for_eval",
eval_stack[Api.benchmarks], # scoring_functions=scoring_functions,
eval_stack[Api.datasets], # )
) # response = await eval_impl.run_eval(
# benchmark_id=benchmark_id,
# benchmark_config=dict(
# eval_candidate=ModelCandidate(
# model=inference_model,
# sampling_params=SamplingParams(),
# ),
# ),
# )
# assert response.job_id == "0"
# job_status = await eval_impl.job_status(benchmark_id, response.job_id)
# assert job_status and job_status.value == "completed"
# eval_response = await eval_impl.job_result(benchmark_id, response.job_id)
response = await datasets_impl.list_datasets() # assert eval_response is not None
assert len(response) > 0 # assert len(eval_response.generations) == 5
if response[0].provider_id != "huggingface": # assert "basic::subset_of" in eval_response.scores
pytest.skip("Only huggingface provider supports pre-registered remote datasets")
await datasets_impl.register_dataset( # @pytest.mark.asyncio
dataset_id="mmlu", # async def test_eval_run_benchmark_eval(self, eval_stack, inference_model):
dataset_schema={ # eval_impl, benchmarks_impl, datasets_impl = (
"input_query": StringType(), # eval_stack[Api.eval],
"expected_answer": StringType(), # eval_stack[Api.benchmarks],
"chat_completion_input": ChatCompletionInputType(), # eval_stack[Api.datasets],
}, # )
url=URL(uri="https://huggingface.co/datasets/llamastack/evals"),
metadata={
"path": "llamastack/evals",
"name": "evals__mmlu__details",
"split": "train",
},
)
# register eval task # response = await datasets_impl.list_datasets()
await benchmarks_impl.register_benchmark( # assert len(response) > 0
benchmark_id="meta-reference-mmlu", # if response[0].provider_id != "huggingface":
dataset_id="mmlu", # pytest.skip("Only huggingface provider supports pre-registered remote datasets")
scoring_functions=["basic::regex_parser_multiple_choice_answer"],
)
# list benchmarks # await datasets_impl.register_dataset(
response = await benchmarks_impl.list_benchmarks() # dataset_id="mmlu",
assert len(response) > 0 # dataset_schema={
# "input_query": StringType(),
# "expected_answer": StringType(),
# "chat_completion_input": ChatCompletionInputType(),
# },
# url=URL(uri="https://huggingface.co/datasets/llamastack/evals"),
# metadata={
# "path": "llamastack/evals",
# "name": "evals__mmlu__details",
# "split": "train",
# },
# )
benchmark_id = "meta-reference-mmlu" # # register eval task
response = await eval_impl.run_eval( # await benchmarks_impl.register_benchmark(
benchmark_id=benchmark_id, # benchmark_id="meta-reference-mmlu",
benchmark_config=dict( # dataset_id="mmlu",
eval_candidate=ModelCandidate( # scoring_functions=["basic::regex_parser_multiple_choice_answer"],
model=inference_model, # )
sampling_params=SamplingParams(),
), # # list benchmarks
num_examples=3, # response = await benchmarks_impl.list_benchmarks()
), # assert len(response) > 0
)
job_status = await eval_impl.job_status(benchmark_id, response.job_id) # benchmark_id = "meta-reference-mmlu"
assert job_status and job_status.value == "completed" # response = await eval_impl.run_eval(
eval_response = await eval_impl.job_result(benchmark_id, response.job_id) # benchmark_id=benchmark_id,
assert eval_response is not None # benchmark_config=dict(
assert len(eval_response.generations) == 3 # eval_candidate=ModelCandidate(
# model=inference_model,
# sampling_params=SamplingParams(),
# ),
# num_examples=3,
# ),
# )
# job_status = await eval_impl.job_status(benchmark_id, response.job_id)
# assert job_status and job_status.value == "completed"
# eval_response = await eval_impl.job_result(benchmark_id, response.job_id)
# assert eval_response is not None
# assert len(eval_response.generations) == 3

2
tests/integration/scoring/test_scoring.py
View file

@ -76,7 +76,7 @@ def test_scoring_functions_register(
assert len(list_response) > 0 assert len(list_response) > 0
assert any(x.identifier == sample_scoring_fn_id for x in list_response) assert any(x.identifier == sample_scoring_fn_id for x in list_response)
# TODO: add unregister to make clean state # TODO: add unregister api for scoring functions
def test_scoring_score(llama_stack_client): def test_scoring_score(llama_stack_client):