evals new rebase

llama_stack/distribution/registry/__init__.py

@@ -0,0 +1,5 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the terms described in the LICENSE file in
+# the root directory of this source tree.

llama_stack/distribution/registry/datasets/__init__.py

@@ -0,0 +1,23 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the terms described in the LICENSE file in
+# the root directory of this source tree.
+
+# TODO: make these import config based
+from .dataset import CustomDataset, HFDataset
+from .dataset_registry import DatasetRegistry
+
+DATASETS_REGISTRY = {
+    "mmlu-simple-eval-en": CustomDataset(
+        name="mmlu_eval",
+        url="https://openaipublic.blob.core.windows.net/simple-evals/mmlu.csv",
+    ),
+    "hellaswag": HFDataset(
+        name="hellaswag",
+        url="hf://hellaswag?split=validation&trust_remote_code=True",
+    ),
+}
+
+for k, v in DATASETS_REGISTRY.items():
+    DatasetRegistry.register(k, v)

llama_stack/distribution/registry/datasets/dataset.py

@@ -0,0 +1,62 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the terms described in the LICENSE file in
+# the root directory of this source tree.
+
+from abc import ABC, abstractmethod
+from urllib.parse import parse_qs, urlparse
+
+import pandas
+from datasets import Dataset, load_dataset
+
+
+class BaseDataset(ABC):
+    def __init__(self, name: str):
+        self.dataset = None
+        self.dataset_id = name
+        self.type = self.__class__.__name__
+
+    def __iter__(self):
+        return iter(self.dataset)
+
+    @abstractmethod
+    def load(self):
+        pass
+
+
+class CustomDataset(BaseDataset):
+    def __init__(self, name, url):
+        super().__init__(name)
+        self.url = url
+
+    def load(self):
+        if self.dataset:
+            return
+        # TODO: better support w/ data url
+        if self.url.endswith(".csv"):
+            df = pandas.read_csv(self.url)
+        elif self.url.endswith(".xlsx"):
+            df = pandas.read_excel(self.url)
+
+        self.dataset = Dataset.from_pandas(df)
+
+
+class HFDataset(BaseDataset):
+    def __init__(self, name, url):
+        super().__init__(name)
+        self.url = url
+
+    def load(self):
+        if self.dataset:
+            return
+
+        parsed = urlparse(self.url)
+
+        if parsed.scheme != "hf":
+            raise ValueError(f"Unknown HF dataset: {self.url}")
+
+        query = parse_qs(parsed.query)
+        query = {k: v[0] for k, v in query.items()}
+        path = parsed.netloc
+        self.dataset = load_dataset(path, **query)

llama_stack/distribution/registry/datasets/dataset_registry.py

@@ -0,0 +1,32 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the terms described in the LICENSE file in
+# the root directory of this source tree.
+from typing import AbstractSet, Dict
+
+from .dataset import BaseDataset
+
+
+class DatasetRegistry:
+    _REGISTRY: Dict[str, BaseDataset] = {}
+
+    @staticmethod
+    def names() -> AbstractSet[str]:
+        return DatasetRegistry._REGISTRY.keys()
+
+    @staticmethod
+    def register(name: str, task: BaseDataset) -> None:
+        if name in DatasetRegistry._REGISTRY:
+            raise ValueError(f"Dataset {name} already exists.")
+        DatasetRegistry._REGISTRY[name] = task
+
+    @staticmethod
+    def get_dataset(name: str) -> BaseDataset:
+        if name not in DatasetRegistry._REGISTRY:
+            raise ValueError(f"Dataset {name} not found.")
+        return DatasetRegistry._REGISTRY[name]
+
+    @staticmethod
+    def reset() -> None:
+        DatasetRegistry._REGISTRY = {}

llama_stack/distribution/registry/tasks/__init__.py

@@ -0,0 +1,13 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the terms described in the LICENSE file in
+# the root directory of this source tree.
+# TODO: make these import config based
+from llama_stack.providers.impls.meta_reference.evals.tasks.mmlu_task import MMLUTask
+from .task_registry import TaskRegistry
+
+TaskRegistry.register(
+    "mmlu",
+    MMLUTask,
+)

llama_stack/distribution/registry/tasks/task.py

@@ -0,0 +1,49 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the terms described in the LICENSE file in
+# the root directory of this source tree.
+from abc import ABC, abstractmethod
+
+
+class BaseTask(ABC):
+    """
+    A task represents a single evaluation benchmark, including it's dataset, preprocessing, postprocessing and scoring methods.
+    Base class for all evaluation tasks. Each task needs to implement the following methods:
+    - F1: preprocess_sample(self)
+    - F2: postprocess_sample(self)
+    - F3: score_sample(self)
+    """
+
+    def __init__(self, dataset, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self._name = self.__class__.__name__
+        self.dataset = dataset
+
+    @abstractmethod
+    def preprocess_sample(self, sample):
+        raise NotImplementedError()
+
+    @abstractmethod
+    def postprocess_sample(self, sample):
+        raise NotImplementedError()
+
+    @abstractmethod
+    def score_sample(self, sample, ground_truth):
+        raise NotImplementedError()
+
+    @abstractmethod
+    def aggregate_results(self, eval_results):
+        raise NotImplementedError()
+
+    def preprocess(self):
+        return [self.preprocess_sample(sample) for sample in self.dataset]
+
+    def postprocess(self, generation):
+        return [self.postprocess_sample(sample) for sample in generation]
+
+    def score(self, postprocessed):
+        return [
+            self.score_sample(sample, ground_truth)
+            for sample, ground_truth in zip(postprocessed, self.dataset)
+        ]

llama_stack/distribution/registry/tasks/task_registry.py

@@ -0,0 +1,32 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the terms described in the LICENSE file in
+# the root directory of this source tree.
+from typing import AbstractSet, Dict
+
+from .task import BaseTask
+
+
+class TaskRegistry:
+    _REGISTRY: Dict[str, BaseTask] = {}
+
+    @staticmethod
+    def names() -> AbstractSet[str]:
+        return TaskRegistry._REGISTRY.keys()
+
+    @staticmethod
+    def register(name: str, task: BaseTask) -> None:
+        if name in TaskRegistry._REGISTRY:
+            raise ValueError(f"Task {name} already exists.")
+        TaskRegistry._REGISTRY[name] = task
+
+    @staticmethod
+    def get_task(name: str) -> BaseTask:
+        if name not in TaskRegistry._REGISTRY:
+            raise ValueError(f"Task {name} not found.")
+        return TaskRegistry._REGISTRY[name]
+
+    @staticmethod
+    def reset() -> None:
+        TaskRegistry._REGISTRY = {}

llama_stack/distribution/resolver.py

@@ -12,6 +12,7 @@ from llama_stack.providers.datatypes import *  # noqa: F403
 from llama_stack.distribution.datatypes import *  # noqa: F403
 
 from llama_stack.apis.agents import Agents
+from llama_stack.apis.evals import Evals
 from llama_stack.apis.inference import Inference
 from llama_stack.apis.inspect import Inspect
 from llama_stack.apis.memory import Memory
@@ -38,6 +39,7 @@ def api_protocol_map() -> Dict[Api, Any]:
         Api.safety: Safety,
         Api.shields: Shields,
         Api.telemetry: Telemetry,
+        Api.evals: Evals,
     }