phoenix-oss/llama-stack-mirror
0
0
Fork 1
mirror of https://github.com/meta-llama/llama-stack.git synced 2025-07-31 16:01:46 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

register to client

Browse source
This commit is contained in:
Xi Yan 2024-11-11 11:03:01 -05:00
parent 75ccc05296
commit 8bebe3fd1f
4 changed files with 67 additions and 24 deletions
Download patch file Download diff file Expand all files Collapse all files

8
llama_stack/providers/adapters/datasetio/huggingface/benchmarks/__init__.py
View file

@ -3,4 +3,10 @@
# #
# This source code is licensed under the terms described in the LICENSE file in # This source code is licensed under the terms described in the LICENSE file in
# the root directory of this source tree. # the root directory of this source tree.
from .mmlu import mmlu # noqa: F401
# # Copyright (c) Meta Platforms, Inc. and affiliates.
# # All rights reserved.
# #
# # This source code is licensed under the terms described in the LICENSE file in
# # the root directory of this source tree.
# from .mmlu import mmlu # noqa: F401

40
llama_stack/providers/adapters/datasetio/huggingface/benchmarks/mmlu.py
View file

@ -4,21 +4,27 @@
# This source code is licensed under the terms described in the LICENSE file in # This source code is licensed under the terms described in the LICENSE file in
# the root directory of this source tree. # the root directory of this source tree.
from llama_models.llama3.api.datatypes import URL # # Copyright (c) Meta Platforms, Inc. and affiliates.
from llama_stack.apis.common.type_system import ChatCompletionInputType, StringType # # All rights reserved.
from llama_stack.apis.datasetio import DatasetDef # #
# # This source code is licensed under the terms described in the LICENSE file in
# # the root directory of this source tree.
mmlu = DatasetDef( # from llama_models.llama3.api.datatypes import URL
identifier="mmlu", # from llama_stack.apis.common.type_system import ChatCompletionInputType, StringType
url=URL(uri="https://huggingface.co/datasets/llamastack/evals"), # from llama_stack.apis.datasetio import DatasetDef
dataset_schema={
"input_query": StringType(), # mmlu = DatasetDef(
"expected_answer": StringType(), # identifier="mmlu",
"chat_completion_input": ChatCompletionInputType(), # url=URL(uri="https://huggingface.co/datasets/llamastack/evals"),
}, # dataset_schema={
metadata={ # "input_query": StringType(),
"path": "llamastack/evals", # "expected_answer": StringType(),
"name": "evals__mmlu__details", # "chat_completion_input": ChatCompletionInputType(),
"split": "train", # },
}, # metadata={
) # "path": "llamastack/evals",
# "name": "evals__mmlu__details",
# "split": "train",
# },
# )

7
llama_stack/providers/adapters/datasetio/huggingface/huggingface.py
View file

@ -12,8 +12,6 @@ import datasets as hf_datasets
from llama_stack.providers.datatypes import DatasetsProtocolPrivate from llama_stack.providers.datatypes import DatasetsProtocolPrivate
from llama_stack.providers.utils.datasetio.url_utils import get_dataframe_from_url from llama_stack.providers.utils.datasetio.url_utils import get_dataframe_from_url
from .benchmarks import mmlu
from .config import HuggingfaceDatasetIOConfig from .config import HuggingfaceDatasetIOConfig
@ -37,9 +35,10 @@ class HuggingfaceDatasetIOImpl(DatasetIO, DatasetsProtocolPrivate):
self.dataset_infos = {} self.dataset_infos = {}
async def initialize(self) -> None: async def initialize(self) -> None:
pass
# pre-registered benchmark datasets # pre-registered benchmark datasets
pre_registered_datasets = [mmlu] # pre_registered_datasets = [mmlu]
self.dataset_infos = {x.identifier: x for x in pre_registered_datasets} # self.dataset_infos = {x.identifier: x for x in pre_registered_datasets}
async def shutdown(self) -> None: ... async def shutdown(self) -> None: ...

36
llama_stack/providers/tests/eval/test_eval.py
View file

@ -7,7 +7,11 @@
import pytest import pytest
from llama_models.llama3.api import SamplingParams from llama_models.llama3.api import SamplingParams, URL
from llama_stack.apis.common.type_system import ChatCompletionInputType, StringType
from llama_stack.apis.datasetio.datasetio import DatasetDefWithProvider
from llama_stack.apis.eval.eval import ( from llama_stack.apis.eval.eval import (
AppEvalTaskConfig, AppEvalTaskConfig,
@ -153,8 +157,36 @@ class Testeval:
assert len(response) > 0 assert len(response) > 0
if response[0].provider_id != "huggingface": if response[0].provider_id != "huggingface":
pytest.skip( pytest.skip(
"Only huggingface provider supports pre-registered benchmarks datasets" "Only huggingface provider supports pre-registered remote datasets"
) )
# register dataset
mmlu = DatasetDefWithProvider(
identifier="mmlu",
url=URL(uri="https://huggingface.co/datasets/llamastack/evals"),
dataset_schema={
"input_query": StringType(),
"expected_answer": StringType(),
"chat_completion_input": ChatCompletionInputType(),
},
metadata={
"path": "llamastack/evals",
"name": "evals__mmlu__details",
"split": "train",
},
provider_id="",
)
await datasets_impl.register_dataset(mmlu)
# register eval task
meta_reference_mmlu = EvalTaskDefWithProvider(
identifier="meta-reference-mmlu",
dataset_id="mmlu",
scoring_functions=["meta-reference::regex_parser_multiple_choice_answer"],
provider_id="",
)
await eval_tasks_impl.register_eval_task(meta_reference_mmlu)
# list benchmarks # list benchmarks
response = await eval_tasks_impl.list_eval_tasks() response = await eval_tasks_impl.list_eval_tasks()