phoenix-oss/llama-stack-mirror
0
0
Fork 1
mirror of https://github.com/meta-llama/llama-stack.git synced 2025-08-02 08:44:44 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

mmlu loose

Browse source
This commit is contained in:
Xi Yan 2024-11-07 18:36:41 -08:00
parent 6ee02ca23b
commit edeb6dcf04
4 changed files with 46 additions and 59 deletions
Download patch file Download diff file Expand all files Collapse all files

10
llama_stack/providers/inline/huggingface/datasetio/dataset_defs/llamastack_mmlu.py → llama_stack/providers/inline/huggingface/datasetio/dataset_defs/llamastack_mmlu_loose.py
View file

@ -5,17 +5,17 @@
# the root directory of this source tree. # the root directory of this source tree.
from llama_models.llama3.api.datatypes import URL from llama_models.llama3.api.datatypes import URL
from llama_stack.apis.common.type_system import CompletionInputType, StringType from llama_stack.apis.common.type_system import ChatCompletionInputType, StringType
from llama_stack.apis.datasetio import DatasetDef from llama_stack.apis.datasetio import DatasetDef
llamastack_mmlu = DatasetDef( llamastack_mmlu_loose = DatasetDef(
identifier="llamastack_mmlu", identifier="llamastack_mmlu_loose",
url=URL(uri="https://huggingface.co/datasets/yanxi0830/ls-mmlu"), url=URL(uri="https://huggingface.co/datasets/yanxi0830/ls-mmlu"),
dataset_schema={ dataset_schema={
"expected_answer": StringType(),
"input_query": StringType(), "input_query": StringType(),
"chat_completion_input": CompletionInputType(), "expected_answer": StringType(),
"chat_completion_input": ChatCompletionInputType(),
}, },
metadata={"path": "yanxi0830/ls-mmlu", "split": "train"}, metadata={"path": "yanxi0830/ls-mmlu", "split": "train"},
) )

8
llama_stack/providers/inline/huggingface/datasetio/huggingface.py
View file

@ -13,11 +13,11 @@ from llama_stack.providers.datatypes import DatasetsProtocolPrivate
from llama_stack.providers.utils.datasetio.url_utils import get_dataframe_from_url from llama_stack.providers.utils.datasetio.url_utils import get_dataframe_from_url
from .config import HuggingfaceDatasetIOConfig from .config import HuggingfaceDatasetIOConfig
from .dataset_defs.llamastack_mmlu import llamastack_mmlu from .dataset_defs.llamastack_mmlu_loose import llamastack_mmlu_loose
def load_hf_dataset(dataset_def: DatasetDef): def load_hf_dataset(dataset_def: DatasetDef):
if dataset_def.metadata.get("dataset_path", None): if dataset_def.metadata.get("path", None):
return load_dataset(**dataset_def.metadata) return load_dataset(**dataset_def.metadata)
df = get_dataframe_from_url(dataset_def.url) df = get_dataframe_from_url(dataset_def.url)
@ -37,7 +37,7 @@ class HuggingfaceDatasetIOImpl(DatasetIO, DatasetsProtocolPrivate):
async def initialize(self) -> None: async def initialize(self) -> None:
# pre-registered benchmark datasets # pre-registered benchmark datasets
self.pre_registered_datasets = [llamastack_mmlu] self.pre_registered_datasets = [llamastack_mmlu_loose]
self.dataset_infos = {x.identifier: x for x in self.pre_registered_datasets} self.dataset_infos = {x.identifier: x for x in self.pre_registered_datasets}
async def shutdown(self) -> None: ... async def shutdown(self) -> None: ...
@ -46,8 +46,6 @@ class HuggingfaceDatasetIOImpl(DatasetIO, DatasetsProtocolPrivate):
self, self,
dataset_def: DatasetDef, dataset_def: DatasetDef,
) -> None: ) -> None:
print("registering dataset", dataset_def)
self.dataset_infos[dataset_def.identifier] = dataset_def self.dataset_infos[dataset_def.identifier] = dataset_def
async def list_datasets(self) -> List[DatasetDef]: async def list_datasets(self) -> List[DatasetDef]:

2
llama_stack/providers/inline/meta_reference/eval/eval.py
View file

@ -54,7 +54,7 @@ class MetaReferenceEvalImpl(Eval, EvalTasksProtocolPrivate):
benchmark_tasks = [ benchmark_tasks = [
EvalTaskDef( EvalTaskDef(
identifier="meta-reference-mmlu", identifier="meta-reference-mmlu",
dataset_id="llamastack_mmlu", dataset_id="llamastack_mmlu_loose",
scoring_functions=[ scoring_functions=[
"meta-reference::regex_parser_multiple_choice_answer" "meta-reference::regex_parser_multiple_choice_answer"
], ],

85
llama_stack/providers/tests/eval/test_eval.py
View file

@ -33,7 +33,6 @@ class Testeval:
_, eval_tasks_impl, _, _, _, _ = eval_stack _, eval_tasks_impl, _, _, _, _ = eval_stack
response = await eval_tasks_impl.list_eval_tasks() response = await eval_tasks_impl.list_eval_tasks()
assert isinstance(response, list) assert isinstance(response, list)
assert len(response) == 0
@pytest.mark.asyncio @pytest.mark.asyncio
async def test_eval_evaluate_rows(self, eval_stack): async def test_eval_evaluate_rows(self, eval_stack):
@ -41,11 +40,6 @@ class Testeval:
await register_dataset( await register_dataset(
datasets_impl, for_generation=True, dataset_id="test_dataset_for_eval" datasets_impl, for_generation=True, dataset_id="test_dataset_for_eval"
) )
provider = datasetio_impl.routing_table.get_provider_impl(
"test_dataset_for_eval"
)
# if provider.__provider_spec__.provider_type != "meta-reference":
# pytest.skip("Only meta-reference provider supports registering datasets")
response = await datasets_impl.list_datasets() response = await datasets_impl.list_datasets()
assert len(response) >= 1 assert len(response) >= 1
@ -83,49 +77,44 @@ class Testeval:
assert "meta-reference::llm_as_judge_8b_correctness" in response.scores assert "meta-reference::llm_as_judge_8b_correctness" in response.scores
assert "meta-reference::equality" in response.scores assert "meta-reference::equality" in response.scores
# @pytest.mark.asyncio @pytest.mark.asyncio
# async def test_eval_run_eval(self, eval_stack): async def test_eval_run_eval(self, eval_stack):
# eval_impl, eval_tasks_impl, _, _, datasetio_impl, datasets_impl = eval_stack eval_impl, eval_tasks_impl, _, _, datasetio_impl, datasets_impl = eval_stack
# await register_dataset( await register_dataset(
# datasets_impl, for_generation=True, dataset_id="test_dataset_for_eval" datasets_impl, for_generation=True, dataset_id="test_dataset_for_eval"
# ) )
# provider = datasetio_impl.routing_table.get_provider_impl(
# "test_dataset_for_eval"
# )
# if provider.__provider_spec__.provider_type != "meta-reference":
# pytest.skip("Only meta-reference provider supports registering datasets")
# scoring_functions = [ scoring_functions = [
# "meta-reference::llm_as_judge_8b_correctness", "meta-reference::llm_as_judge_8b_correctness",
# "meta-reference::subset_of", "meta-reference::subset_of",
# ] ]
# task_id = "meta-reference::app_eval-2" task_id = "meta-reference::app_eval-2"
# task_def = EvalTaskDefWithProvider( task_def = EvalTaskDefWithProvider(
# identifier=task_id, identifier=task_id,
# dataset_id="test_dataset_for_eval", dataset_id="test_dataset_for_eval",
# scoring_functions=scoring_functions, scoring_functions=scoring_functions,
# provider_id="meta-reference", provider_id="meta-reference",
# ) )
# await eval_tasks_impl.register_eval_task(task_def) await eval_tasks_impl.register_eval_task(task_def)
# response = await eval_impl.run_eval( response = await eval_impl.run_eval(
# task_id=task_id, task_id=task_id,
# task_config=AppEvalTaskConfig( task_config=AppEvalTaskConfig(
# eval_candidate=ModelCandidate( eval_candidate=ModelCandidate(
# model="Llama3.2-3B-Instruct", model="Llama3.2-3B-Instruct",
# sampling_params=SamplingParams(), sampling_params=SamplingParams(),
# ), ),
# ), ),
# ) )
# assert response.job_id == "0" assert response.job_id == "0"
# job_status = await eval_impl.job_status(task_id, response.job_id) job_status = await eval_impl.job_status(task_id, response.job_id)
# assert job_status and job_status.value == "completed" assert job_status and job_status.value == "completed"
# eval_response = await eval_impl.job_result(task_id, response.job_id) eval_response = await eval_impl.job_result(task_id, response.job_id)
# assert eval_response is not None assert eval_response is not None
# assert len(eval_response.generations) == 5 assert len(eval_response.generations) == 5
# assert "meta-reference::subset_of" in eval_response.scores assert "meta-reference::subset_of" in eval_response.scores
# assert "meta-reference::llm_as_judge_8b_correctness" in eval_response.scores assert "meta-reference::llm_as_judge_8b_correctness" in eval_response.scores
@pytest.mark.asyncio @pytest.mark.asyncio
async def test_eval_run_benchmark_eval(self, eval_stack): async def test_eval_run_benchmark_eval(self, eval_stack):
@ -152,8 +141,8 @@ class Testeval:
num_examples=3, num_examples=3,
), ),
) )
job_status = await eval_impl.job_status(response.job_id, benchmark_id) job_status = await eval_impl.job_status(benchmark_id, response.job_id)
assert job_status and job_status.value == "completed" assert job_status and job_status.value == "completed"
eval_response = await eval_impl.job_result(response.job_id, benchmark_id) eval_response = await eval_impl.job_result(benchmark_id, response.job_id)
assert eval_response is not None assert eval_response is not None
assert len(eval_response.generations) == 3 assert len(eval_response.generations) == 3