mirror of
https://github.com/meta-llama/llama-stack.git
synced 2025-08-02 00:34:44 +00:00
mmlu loose
This commit is contained in:
Xi Yan
parent
6ee02ca23b
commit
edeb6dcf04
4 changed files with 46 additions and 59 deletions
|
|
@ -5,17 +5,17 @@
|
|||
# the root directory of this source tree.
|
||||
|
||||
from llama_models.llama3.api.datatypes import URL
|
||||
from llama_stack.apis.common.type_system import CompletionInputType, StringType
|
||||
from llama_stack.apis.common.type_system import ChatCompletionInputType, StringType
|
||||
from llama_stack.apis.datasetio import DatasetDef
|
||||
|
||||
|
||||
llamastack_mmlu = DatasetDef(
|
||||
identifier="llamastack_mmlu",
|
||||
llamastack_mmlu_loose = DatasetDef(
|
||||
identifier="llamastack_mmlu_loose",
|
||||
url=URL(uri="https://huggingface.co/datasets/yanxi0830/ls-mmlu"),
|
||||
dataset_schema={
|
||||
"expected_answer": StringType(),
|
||||
"input_query": StringType(),
|
||||
"chat_completion_input": CompletionInputType(),
|
||||
"expected_answer": StringType(),
|
||||
"chat_completion_input": ChatCompletionInputType(),
|
||||
},
|
||||
metadata={"path": "yanxi0830/ls-mmlu", "split": "train"},
|
||||
)
|
||||
|
|
@ -13,11 +13,11 @@ from llama_stack.providers.datatypes import DatasetsProtocolPrivate
|
|||
from llama_stack.providers.utils.datasetio.url_utils import get_dataframe_from_url
|
||||
|
||||
from .config import HuggingfaceDatasetIOConfig
|
||||
from .dataset_defs.llamastack_mmlu import llamastack_mmlu
|
||||
from .dataset_defs.llamastack_mmlu_loose import llamastack_mmlu_loose
|
||||
|
||||
|
||||
def load_hf_dataset(dataset_def: DatasetDef):
|
||||
if dataset_def.metadata.get("dataset_path", None):
|
||||
if dataset_def.metadata.get("path", None):
|
||||
return load_dataset(**dataset_def.metadata)
|
||||
|
||||
df = get_dataframe_from_url(dataset_def.url)
|
||||
|
|
@ -37,7 +37,7 @@ class HuggingfaceDatasetIOImpl(DatasetIO, DatasetsProtocolPrivate):
|
|||
|
||||
async def initialize(self) -> None:
|
||||
# pre-registered benchmark datasets
|
||||
self.pre_registered_datasets = [llamastack_mmlu]
|
||||
self.pre_registered_datasets = [llamastack_mmlu_loose]
|
||||
self.dataset_infos = {x.identifier: x for x in self.pre_registered_datasets}
|
||||
|
||||
async def shutdown(self) -> None: ...
|
||||
|
|
@ -46,8 +46,6 @@ class HuggingfaceDatasetIOImpl(DatasetIO, DatasetsProtocolPrivate):
|
|||
self,
|
||||
dataset_def: DatasetDef,
|
||||
) -> None:
|
||||
|
||||
print("registering dataset", dataset_def)
|
||||
self.dataset_infos[dataset_def.identifier] = dataset_def
|
||||
|
||||
async def list_datasets(self) -> List[DatasetDef]:
|
||||
|
|
|
|||
|
|
@ -54,7 +54,7 @@ class MetaReferenceEvalImpl(Eval, EvalTasksProtocolPrivate):
|
|||
benchmark_tasks = [
|
||||
EvalTaskDef(
|
||||
identifier="meta-reference-mmlu",
|
||||
dataset_id="llamastack_mmlu",
|
||||
dataset_id="llamastack_mmlu_loose",
|
||||
scoring_functions=[
|
||||
"meta-reference::regex_parser_multiple_choice_answer"
|
||||
],
|
||||
|
|
|
|||
|
|
@ -33,7 +33,6 @@ class Testeval:
|
|||
_, eval_tasks_impl, _, _, _, _ = eval_stack
|
||||
response = await eval_tasks_impl.list_eval_tasks()
|
||||
assert isinstance(response, list)
|
||||
assert len(response) == 0
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_eval_evaluate_rows(self, eval_stack):
|
||||
|
|
@ -41,11 +40,6 @@ class Testeval:
|
|||
await register_dataset(
|
||||
datasets_impl, for_generation=True, dataset_id="test_dataset_for_eval"
|
||||
)
|
||||
provider = datasetio_impl.routing_table.get_provider_impl(
|
||||
"test_dataset_for_eval"
|
||||
)
|
||||
# if provider.__provider_spec__.provider_type != "meta-reference":
|
||||
# pytest.skip("Only meta-reference provider supports registering datasets")
|
||||
|
||||
response = await datasets_impl.list_datasets()
|
||||
assert len(response) >= 1
|
||||
|
|
@ -83,49 +77,44 @@ class Testeval:
|
|||
assert "meta-reference::llm_as_judge_8b_correctness" in response.scores
|
||||
assert "meta-reference::equality" in response.scores
|
||||
|
||||
# @pytest.mark.asyncio
|
||||
# async def test_eval_run_eval(self, eval_stack):
|
||||
# eval_impl, eval_tasks_impl, _, _, datasetio_impl, datasets_impl = eval_stack
|
||||
# await register_dataset(
|
||||
# datasets_impl, for_generation=True, dataset_id="test_dataset_for_eval"
|
||||
# )
|
||||
# provider = datasetio_impl.routing_table.get_provider_impl(
|
||||
# "test_dataset_for_eval"
|
||||
# )
|
||||
# if provider.__provider_spec__.provider_type != "meta-reference":
|
||||
# pytest.skip("Only meta-reference provider supports registering datasets")
|
||||
@pytest.mark.asyncio
|
||||
async def test_eval_run_eval(self, eval_stack):
|
||||
eval_impl, eval_tasks_impl, _, _, datasetio_impl, datasets_impl = eval_stack
|
||||
await register_dataset(
|
||||
datasets_impl, for_generation=True, dataset_id="test_dataset_for_eval"
|
||||
)
|
||||
|
||||
# scoring_functions = [
|
||||
# "meta-reference::llm_as_judge_8b_correctness",
|
||||
# "meta-reference::subset_of",
|
||||
# ]
|
||||
scoring_functions = [
|
||||
"meta-reference::llm_as_judge_8b_correctness",
|
||||
"meta-reference::subset_of",
|
||||
]
|
||||
|
||||
# task_id = "meta-reference::app_eval-2"
|
||||
# task_def = EvalTaskDefWithProvider(
|
||||
# identifier=task_id,
|
||||
# dataset_id="test_dataset_for_eval",
|
||||
# scoring_functions=scoring_functions,
|
||||
# provider_id="meta-reference",
|
||||
# )
|
||||
# await eval_tasks_impl.register_eval_task(task_def)
|
||||
# response = await eval_impl.run_eval(
|
||||
# task_id=task_id,
|
||||
# task_config=AppEvalTaskConfig(
|
||||
# eval_candidate=ModelCandidate(
|
||||
# model="Llama3.2-3B-Instruct",
|
||||
# sampling_params=SamplingParams(),
|
||||
# ),
|
||||
# ),
|
||||
# )
|
||||
# assert response.job_id == "0"
|
||||
# job_status = await eval_impl.job_status(task_id, response.job_id)
|
||||
# assert job_status and job_status.value == "completed"
|
||||
# eval_response = await eval_impl.job_result(task_id, response.job_id)
|
||||
task_id = "meta-reference::app_eval-2"
|
||||
task_def = EvalTaskDefWithProvider(
|
||||
identifier=task_id,
|
||||
dataset_id="test_dataset_for_eval",
|
||||
scoring_functions=scoring_functions,
|
||||
provider_id="meta-reference",
|
||||
)
|
||||
await eval_tasks_impl.register_eval_task(task_def)
|
||||
response = await eval_impl.run_eval(
|
||||
task_id=task_id,
|
||||
task_config=AppEvalTaskConfig(
|
||||
eval_candidate=ModelCandidate(
|
||||
model="Llama3.2-3B-Instruct",
|
||||
sampling_params=SamplingParams(),
|
||||
),
|
||||
),
|
||||
)
|
||||
assert response.job_id == "0"
|
||||
job_status = await eval_impl.job_status(task_id, response.job_id)
|
||||
assert job_status and job_status.value == "completed"
|
||||
eval_response = await eval_impl.job_result(task_id, response.job_id)
|
||||
|
||||
# assert eval_response is not None
|
||||
# assert len(eval_response.generations) == 5
|
||||
# assert "meta-reference::subset_of" in eval_response.scores
|
||||
# assert "meta-reference::llm_as_judge_8b_correctness" in eval_response.scores
|
||||
assert eval_response is not None
|
||||
assert len(eval_response.generations) == 5
|
||||
assert "meta-reference::subset_of" in eval_response.scores
|
||||
assert "meta-reference::llm_as_judge_8b_correctness" in eval_response.scores
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_eval_run_benchmark_eval(self, eval_stack):
|
||||
|
|
@ -152,8 +141,8 @@ class Testeval:
|
|||
num_examples=3,
|
||||
),
|
||||
)
|
||||
job_status = await eval_impl.job_status(response.job_id, benchmark_id)
|
||||
job_status = await eval_impl.job_status(benchmark_id, response.job_id)
|
||||
assert job_status and job_status.value == "completed"
|
||||
eval_response = await eval_impl.job_result(response.job_id, benchmark_id)
|
||||
eval_response = await eval_impl.job_result(benchmark_id, response.job_id)
|
||||
assert eval_response is not None
|
||||
assert len(eval_response.generations) == 3
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue