phoenix-oss/llama-stack-mirror
0
0
Fork 1
mirror of https://github.com/meta-llama/llama-stack.git synced 2025-08-02 00:34:44 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

fix

Browse source
This commit is contained in:
Xi Yan 2024-11-07 18:25:39 -08:00
parent 33b6d9b7b7
commit 6ee02ca23b
6 changed files with 100 additions and 87 deletions
Download patch file Download diff file Expand all files Collapse all files

4
llama_stack/providers/inline/huggingface/datasetio/dataset_defs/llamastack_mmlu.py
View file

@ -5,7 +5,7 @@
# the root directory of this source tree.
from llama_models.llama3.api.datatypes import URL
from llama_stack.apis.common.type_system import StringType
from llama_stack.apis.common.type_system import CompletionInputType, StringType
from llama_stack.apis.datasetio import DatasetDef
@ -15,7 +15,7 @@ llamastack_mmlu = DatasetDef(
dataset_schema={
"expected_answer": StringType(),
"input_query": StringType(),
"generated_answer": StringType(),
"chat_completion_input": CompletionInputType(),
},
metadata={"path": "yanxi0830/ls-mmlu", "split": "train"},
)

1
llama_stack/providers/inline/huggingface/datasetio/huggingface.py
View file

@ -10,6 +10,7 @@ from llama_stack.apis.datasetio import * # noqa: F403
from datasets import Dataset, load_dataset
from llama_stack.providers.datatypes import DatasetsProtocolPrivate
from llama_stack.providers.utils.datasetio.url_utils import get_dataframe_from_url
from .config import HuggingfaceDatasetIOConfig
from .dataset_defs.llamastack_mmlu import llamastack_mmlu

13
llama_stack/providers/inline/meta_reference/eval/eval.py
View file

@ -49,7 +49,18 @@ class MetaReferenceEvalImpl(Eval, EvalTasksProtocolPrivate):
self.eval_tasks = {}
async def initialize(self) -> None: ...
async def initialize(self) -> None:
# pre-register eval tasks
benchmark_tasks = [
EvalTaskDef(
identifier="meta-reference-mmlu",
dataset_id="llamastack_mmlu",
scoring_functions=[
"meta-reference::regex_parser_multiple_choice_answer"
],
)
]
self.eval_tasks = {x.identifier: x for x in benchmark_tasks}
async def shutdown(self) -> None: ...

87
llama_stack/providers/tests/eval/test_eval.py
View file

@ -11,6 +11,7 @@ from llama_models.llama3.api import SamplingParams
from llama_stack.apis.eval.eval import (
AppEvalTaskConfig,
BenchmarkEvalTaskConfig,
EvalTaskDefWithProvider,
ModelCandidate,
)
@ -82,49 +83,49 @@ class Testeval:
assert "meta-reference::llm_as_judge_8b_correctness" in response.scores
assert "meta-reference::equality" in response.scores
@pytest.mark.asyncio
async def test_eval_run_eval(self, eval_stack):
eval_impl, eval_tasks_impl, _, _, datasetio_impl, datasets_impl = eval_stack
await register_dataset(
datasets_impl, for_generation=True, dataset_id="test_dataset_for_eval"
)
provider = datasetio_impl.routing_table.get_provider_impl(
"test_dataset_for_eval"
)
if provider.__provider_spec__.provider_type != "meta-reference":
pytest.skip("Only meta-reference provider supports registering datasets")
# @pytest.mark.asyncio
# async def test_eval_run_eval(self, eval_stack):
# eval_impl, eval_tasks_impl, _, _, datasetio_impl, datasets_impl = eval_stack
# await register_dataset(
# datasets_impl, for_generation=True, dataset_id="test_dataset_for_eval"
# )
# provider = datasetio_impl.routing_table.get_provider_impl(
# "test_dataset_for_eval"
# )
# if provider.__provider_spec__.provider_type != "meta-reference":
# pytest.skip("Only meta-reference provider supports registering datasets")
scoring_functions = [
"meta-reference::llm_as_judge_8b_correctness",
"meta-reference::subset_of",
]
# scoring_functions = [
# "meta-reference::llm_as_judge_8b_correctness",
# "meta-reference::subset_of",
# ]
task_id = "meta-reference::app_eval-2"
task_def = EvalTaskDefWithProvider(
identifier=task_id,
dataset_id="test_dataset_for_eval",
scoring_functions=scoring_functions,
provider_id="meta-reference",
)
await eval_tasks_impl.register_eval_task(task_def)
response = await eval_impl.run_eval(
task_id=task_id,
task_config=AppEvalTaskConfig(
eval_candidate=ModelCandidate(
model="Llama3.2-3B-Instruct",
sampling_params=SamplingParams(),
),
),
)
assert response.job_id == "0"
job_status = await eval_impl.job_status(task_id, response.job_id)
assert job_status and job_status.value == "completed"
eval_response = await eval_impl.job_result(task_id, response.job_id)
# task_id = "meta-reference::app_eval-2"
# task_def = EvalTaskDefWithProvider(
# identifier=task_id,
# dataset_id="test_dataset_for_eval",
# scoring_functions=scoring_functions,
# provider_id="meta-reference",
# )
# await eval_tasks_impl.register_eval_task(task_def)
# response = await eval_impl.run_eval(
# task_id=task_id,
# task_config=AppEvalTaskConfig(
# eval_candidate=ModelCandidate(
# model="Llama3.2-3B-Instruct",
# sampling_params=SamplingParams(),
# ),
# ),
# )
# assert response.job_id == "0"
# job_status = await eval_impl.job_status(task_id, response.job_id)
# assert job_status and job_status.value == "completed"
# eval_response = await eval_impl.job_result(task_id, response.job_id)
assert eval_response is not None
assert len(eval_response.generations) == 5
assert "meta-reference::subset_of" in eval_response.scores
assert "meta-reference::llm_as_judge_8b_correctness" in eval_response.scores
# assert eval_response is not None
# assert len(eval_response.generations) == 5
# assert "meta-reference::subset_of" in eval_response.scores
# assert "meta-reference::llm_as_judge_8b_correctness" in eval_response.scores
@pytest.mark.asyncio
async def test_eval_run_benchmark_eval(self, eval_stack):
@ -141,9 +142,9 @@ class Testeval:
assert len(response) > 0
benchmark_id = "meta-reference-mmlu"
response = await eval_impl.run_benchmark(
benchmark_id=benchmark_id,
benchmark_config=BenchmarkEvalTaskConfig(
response = await eval_impl.run_eval(
task_id=benchmark_id,
task_config=BenchmarkEvalTaskConfig(
eval_candidate=ModelCandidate(
model="Llama3.2-3B-Instruct",
sampling_params=SamplingParams(),

41
llama_stack/providers/utils/datasetio/url_utils.py
View file

@ -5,22 +5,41 @@
# the root directory of this source tree.
import base64
import mimetypes
import os
import io
from urllib.parse import unquote
import pandas
from llama_models.llama3.api.datatypes import URL
from llama_stack.providers.utils.memory.vector_store import parse_data_url
def data_url_from_file(file_path: str) -> URL:
if not os.path.exists(file_path):
raise FileNotFoundError(f"File not found: {file_path}")
with open(file_path, "rb") as file:
file_content = file.read()
def get_dataframe_from_url(url: URL):
df = None
if url.uri.endswith(".csv"):
df = pandas.read_csv(url.uri)
elif url.uri.endswith(".xlsx"):
df = pandas.read_excel(url.uri)
elif url.uri.startswith("data:"):
parts = parse_data_url(url.uri)
data = parts["data"]
if parts["is_base64"]:
data = base64.b64decode(data)
else:
data = unquote(data)
encoding = parts["encoding"] or "utf-8"
data = data.encode(encoding)
base64_content = base64.b64encode(file_content).decode("utf-8")
mime_type, _ = mimetypes.guess_type(file_path)
mime_type = parts["mimetype"]
mime_category = mime_type.split("/")[0]
data_bytes = io.BytesIO(data)
data_url = f"data:{mime_type};base64,{base64_content}"
if mime_category == "text":
df = pandas.read_csv(data_bytes)
else:
df = pandas.read_excel(data_bytes)
else:
raise ValueError(f"Unsupported file type: {url}")
return URL(uri=data_url)
return df

41
llama_stack/providers/utils/memory/file_utils.py
View file

@ -5,41 +5,22 @@
# the root directory of this source tree.
import base64
import io
from urllib.parse import unquote
import pandas
import mimetypes
import os
from llama_models.llama3.api.datatypes import URL
from llama_stack.providers.utils.memory.vector_store import parse_data_url
def data_url_from_file(file_path: str) -> URL:
if not os.path.exists(file_path):
raise FileNotFoundError(f"File not found: {file_path}")
def get_dataframe_from_url(url: URL):
df = None
if url.uri.endswith(".csv"):
df = pandas.read_csv(url.uri)
elif url.uri.endswith(".xlsx"):
df = pandas.read_excel(url.uri)
elif url.uri.startswith("data:"):
parts = parse_data_url(url.uri)
data = parts["data"]
if parts["is_base64"]:
data = base64.b64decode(data)
else:
data = unquote(data)
encoding = parts["encoding"] or "utf-8"
data = data.encode(encoding)
with open(file_path, "rb") as file:
file_content = file.read()
mime_type = parts["mimetype"]
mime_category = mime_type.split("/")[0]
data_bytes = io.BytesIO(data)
base64_content = base64.b64encode(file_content).decode("utf-8")
mime_type, _ = mimetypes.guess_type(file_path)
if mime_category == "text":
df = pandas.read_csv(data_bytes)
else:
df = pandas.read_excel(data_bytes)
else:
raise ValueError(f"Unsupported file type: {url}")
data_url = f"data:{mime_type};base64,{base64_content}"
return df
return URL(uri=data_url)