diff --git a/llama_stack/providers/inline/huggingface/datasetio/dataset_defs/llamastack_mmlu.py b/llama_stack/providers/inline/huggingface/datasetio/dataset_defs/llamastack_mmlu_loose.py similarity index 71% rename from llama_stack/providers/inline/huggingface/datasetio/dataset_defs/llamastack_mmlu.py rename to llama_stack/providers/inline/huggingface/datasetio/dataset_defs/llamastack_mmlu_loose.py index aa1850b3f..ae7d4c7c2 100644 --- a/llama_stack/providers/inline/huggingface/datasetio/dataset_defs/llamastack_mmlu.py +++ b/llama_stack/providers/inline/huggingface/datasetio/dataset_defs/llamastack_mmlu_loose.py @@ -5,17 +5,17 @@ # the root directory of this source tree. from llama_models.llama3.api.datatypes import URL -from llama_stack.apis.common.type_system import CompletionInputType, StringType +from llama_stack.apis.common.type_system import ChatCompletionInputType, StringType from llama_stack.apis.datasetio import DatasetDef -llamastack_mmlu = DatasetDef( - identifier="llamastack_mmlu", +llamastack_mmlu_loose = DatasetDef( + identifier="llamastack_mmlu_loose", url=URL(uri="https://huggingface.co/datasets/yanxi0830/ls-mmlu"), dataset_schema={ - "expected_answer": StringType(), "input_query": StringType(), - "chat_completion_input": CompletionInputType(), + "expected_answer": StringType(), + "chat_completion_input": ChatCompletionInputType(), }, metadata={"path": "yanxi0830/ls-mmlu", "split": "train"}, ) diff --git a/llama_stack/providers/inline/huggingface/datasetio/huggingface.py b/llama_stack/providers/inline/huggingface/datasetio/huggingface.py index 849e4e202..0185d22dc 100644 --- a/llama_stack/providers/inline/huggingface/datasetio/huggingface.py +++ b/llama_stack/providers/inline/huggingface/datasetio/huggingface.py @@ -13,11 +13,11 @@ from llama_stack.providers.datatypes import DatasetsProtocolPrivate from llama_stack.providers.utils.datasetio.url_utils import get_dataframe_from_url from .config import HuggingfaceDatasetIOConfig -from .dataset_defs.llamastack_mmlu import llamastack_mmlu +from .dataset_defs.llamastack_mmlu_loose import llamastack_mmlu_loose def load_hf_dataset(dataset_def: DatasetDef): - if dataset_def.metadata.get("dataset_path", None): + if dataset_def.metadata.get("path", None): return load_dataset(**dataset_def.metadata) df = get_dataframe_from_url(dataset_def.url) @@ -37,7 +37,7 @@ class HuggingfaceDatasetIOImpl(DatasetIO, DatasetsProtocolPrivate): async def initialize(self) -> None: # pre-registered benchmark datasets - self.pre_registered_datasets = [llamastack_mmlu] + self.pre_registered_datasets = [llamastack_mmlu_loose] self.dataset_infos = {x.identifier: x for x in self.pre_registered_datasets} async def shutdown(self) -> None: ... @@ -46,8 +46,6 @@ class HuggingfaceDatasetIOImpl(DatasetIO, DatasetsProtocolPrivate): self, dataset_def: DatasetDef, ) -> None: - - print("registering dataset", dataset_def) self.dataset_infos[dataset_def.identifier] = dataset_def async def list_datasets(self) -> List[DatasetDef]: diff --git a/llama_stack/providers/inline/meta_reference/eval/eval.py b/llama_stack/providers/inline/meta_reference/eval/eval.py index 57bedc1b1..05f34f2fa 100644 --- a/llama_stack/providers/inline/meta_reference/eval/eval.py +++ b/llama_stack/providers/inline/meta_reference/eval/eval.py @@ -54,7 +54,7 @@ class MetaReferenceEvalImpl(Eval, EvalTasksProtocolPrivate): benchmark_tasks = [ EvalTaskDef( identifier="meta-reference-mmlu", - dataset_id="llamastack_mmlu", + dataset_id="llamastack_mmlu_loose", scoring_functions=[ "meta-reference::regex_parser_multiple_choice_answer" ], diff --git a/llama_stack/providers/tests/eval/test_eval.py b/llama_stack/providers/tests/eval/test_eval.py index 91db2e7bb..721421d37 100644 --- a/llama_stack/providers/tests/eval/test_eval.py +++ b/llama_stack/providers/tests/eval/test_eval.py @@ -33,7 +33,6 @@ class Testeval: _, eval_tasks_impl, _, _, _, _ = eval_stack response = await eval_tasks_impl.list_eval_tasks() assert isinstance(response, list) - assert len(response) == 0 @pytest.mark.asyncio async def test_eval_evaluate_rows(self, eval_stack): @@ -41,11 +40,6 @@ class Testeval: await register_dataset( datasets_impl, for_generation=True, dataset_id="test_dataset_for_eval" ) - provider = datasetio_impl.routing_table.get_provider_impl( - "test_dataset_for_eval" - ) - # if provider.__provider_spec__.provider_type != "meta-reference": - # pytest.skip("Only meta-reference provider supports registering datasets") response = await datasets_impl.list_datasets() assert len(response) >= 1 @@ -83,49 +77,44 @@ class Testeval: assert "meta-reference::llm_as_judge_8b_correctness" in response.scores assert "meta-reference::equality" in response.scores - # @pytest.mark.asyncio - # async def test_eval_run_eval(self, eval_stack): - # eval_impl, eval_tasks_impl, _, _, datasetio_impl, datasets_impl = eval_stack - # await register_dataset( - # datasets_impl, for_generation=True, dataset_id="test_dataset_for_eval" - # ) - # provider = datasetio_impl.routing_table.get_provider_impl( - # "test_dataset_for_eval" - # ) - # if provider.__provider_spec__.provider_type != "meta-reference": - # pytest.skip("Only meta-reference provider supports registering datasets") + @pytest.mark.asyncio + async def test_eval_run_eval(self, eval_stack): + eval_impl, eval_tasks_impl, _, _, datasetio_impl, datasets_impl = eval_stack + await register_dataset( + datasets_impl, for_generation=True, dataset_id="test_dataset_for_eval" + ) - # scoring_functions = [ - # "meta-reference::llm_as_judge_8b_correctness", - # "meta-reference::subset_of", - # ] + scoring_functions = [ + "meta-reference::llm_as_judge_8b_correctness", + "meta-reference::subset_of", + ] - # task_id = "meta-reference::app_eval-2" - # task_def = EvalTaskDefWithProvider( - # identifier=task_id, - # dataset_id="test_dataset_for_eval", - # scoring_functions=scoring_functions, - # provider_id="meta-reference", - # ) - # await eval_tasks_impl.register_eval_task(task_def) - # response = await eval_impl.run_eval( - # task_id=task_id, - # task_config=AppEvalTaskConfig( - # eval_candidate=ModelCandidate( - # model="Llama3.2-3B-Instruct", - # sampling_params=SamplingParams(), - # ), - # ), - # ) - # assert response.job_id == "0" - # job_status = await eval_impl.job_status(task_id, response.job_id) - # assert job_status and job_status.value == "completed" - # eval_response = await eval_impl.job_result(task_id, response.job_id) + task_id = "meta-reference::app_eval-2" + task_def = EvalTaskDefWithProvider( + identifier=task_id, + dataset_id="test_dataset_for_eval", + scoring_functions=scoring_functions, + provider_id="meta-reference", + ) + await eval_tasks_impl.register_eval_task(task_def) + response = await eval_impl.run_eval( + task_id=task_id, + task_config=AppEvalTaskConfig( + eval_candidate=ModelCandidate( + model="Llama3.2-3B-Instruct", + sampling_params=SamplingParams(), + ), + ), + ) + assert response.job_id == "0" + job_status = await eval_impl.job_status(task_id, response.job_id) + assert job_status and job_status.value == "completed" + eval_response = await eval_impl.job_result(task_id, response.job_id) - # assert eval_response is not None - # assert len(eval_response.generations) == 5 - # assert "meta-reference::subset_of" in eval_response.scores - # assert "meta-reference::llm_as_judge_8b_correctness" in eval_response.scores + assert eval_response is not None + assert len(eval_response.generations) == 5 + assert "meta-reference::subset_of" in eval_response.scores + assert "meta-reference::llm_as_judge_8b_correctness" in eval_response.scores @pytest.mark.asyncio async def test_eval_run_benchmark_eval(self, eval_stack): @@ -152,8 +141,8 @@ class Testeval: num_examples=3, ), ) - job_status = await eval_impl.job_status(response.job_id, benchmark_id) + job_status = await eval_impl.job_status(benchmark_id, response.job_id) assert job_status and job_status.value == "completed" - eval_response = await eval_impl.job_result(response.job_id, benchmark_id) + eval_response = await eval_impl.job_result(benchmark_id, response.job_id) assert eval_response is not None assert len(eval_response.generations) == 3