mvp

llama_stack/apis/evals/client.py

@@ -33,6 +33,7 @@ class EvaluationClient(Evals):
                     "task": task,
                 },
                 headers={"Content-Type": "application/json"},
+                timeout=3600,
             )
             response.raise_for_status()
             return EvaluateResponse(**response.json())
@@ -43,7 +44,7 @@ async def run_main(host: str, port: int):
 
     response = await client.run_evals(
         "Llama3.1-8B-Instruct",
-        "mmlu.csv",
+        "mmlu-simple-eval-en",
         "mmlu",
     )
     cprint(f"evaluate response={response}", "green")

llama_stack/apis/inference/client.py

@@ -109,7 +109,7 @@ async def run_main(host: str, port: int, stream: bool):
     cprint(f"User>{message.content}", "green")
     iterator = client.chat_completion(
         model="Llama3.1-8B-Instruct",
-        messages=[message],
+        messages=[message, UserMessage(content="write me 3 sentence about the sun.")],
         stream=stream,
     )
     async for log in EventLogger().log(iterator):

llama_stack/providers/impls/meta_reference/evals/datas/dataset_registry.py

@@ -3,17 +3,22 @@
 #
 # This source code is licensed under the terms described in the LICENSE file in
 # the root directory of this source tree.
-from .datasets import CustomDataset
+from .datasets import CustomDataset, HFDataset
 
 # TODO: make this into a config based registry
 DATASETS_REGISTRY = {
-    "mmlu_eval": CustomDataset(
+    "mmlu-simple-eval-en": CustomDataset(
         name="mmlu_eval",
         url="https://openaipublic.blob.core.windows.net/simple-evals/mmlu.csv",
     ),
+    "mmmu-accounting": HFDataset(
+        name="mmlu_eval",
+        url="hf://hellaswag?split=validation&trust_remote_code=True",
+    ),
 }
 
 
 def get_dataset(dataset_id: str):
-    # get dataset concrete dataset implementation
-    return DATASETS_REGISTRY[dataset_id]
+    dataset = DATASETS_REGISTRY[dataset_id]
+    dataset.load()
+    return dataset

llama_stack/providers/impls/meta_reference/evals/datas/datasets.py

@@ -4,23 +4,35 @@
 # This source code is licensed under the terms described in the LICENSE file in
 # the root directory of this source tree.
 
+from abc import ABC, abstractmethod
 from urllib.parse import parse_qs, urlparse
 
 import pandas
 from datasets import Dataset, load_dataset
 
 
-class BaseDataset:
+class BaseDataset(ABC):
     def __init__(self, name: str):
         self.dataset = None
         self.dataset_id = name
         self.type = self.__class__.__name__
 
+    def __iter__(self):
+        return iter(self.dataset)
+
+    @abstractmethod
+    def load(self):
+        pass
+
 
 class CustomDataset(BaseDataset):
     def __init__(self, name, url):
         super().__init__(name)
         self.url = url
+
+    def load(self):
+        if self.dataset:
+            return
         df = pandas.read_csv(self.url)
         self.dataset = Dataset.from_pandas(df)
 
@@ -28,13 +40,18 @@ class CustomDataset(BaseDataset):
 class HFDataset(BaseDataset):
     def __init__(self, name, url):
         super().__init__(name)
-        # URL following OpenAI's evals - hf://hendrycks_test?name=business_ethics&split=validation
         self.url = url
-        parsed = urlparse(url)
-        query = parse_qs(parsed.query)
-        query = {k: v[0] for k, v in query.items()}
+
+    def load(self):
+        if self.dataset:
+            return
+
+        parsed = urlparse(self.url)
 
         if parsed.scheme != "hf":
-            raise ValueError(f"Unknown HF dataset: {url}")
+            raise ValueError(f"Unknown HF dataset: {self.url}")
 
+        query = parse_qs(parsed.query)
+        query = {k: v[0] for k, v in query.items()}
+        path = parsed.netloc
         self.dataset = load_dataset(path, **query)

llama_stack/providers/impls/meta_reference/evals/evals.py

@@ -8,10 +8,10 @@ from llama_stack.apis.inference import *  # noqa: F403
 from llama_stack.apis.evals import *  # noqa: F403
 from termcolor import cprint
 
-from llama_stack.providers.impls.meta_reference.evals.datas.utils import (  # noqa: F403
+from llama_stack.providers.impls.meta_reference.evals.datas.dataset_registry import (
     get_dataset,
 )
-from llama_stack.providers.impls.meta_reference.evals.tasks.utils import (  # noqa: F403
+from llama_stack.providers.impls.meta_reference.evals.tasks.task_registry import (
     get_task,
 )
 
@@ -35,27 +35,25 @@ class MetaReferenceEvalsImpl(Evals):
         task: str,
     ) -> EvaluateResponse:
         cprint(f"model={model}, dataset={dataset}, task={task}", "red")
+        dataset = get_dataset("mmlu-simple-eval-en")
 
-        # resolve dataset
-        # - either a custom URL dataset or HF URL dataset
-        dataset = get_dataset("mmlu_eval")
-        print(dataset.dataset)
-
-        # # resolve task and execute task
         task_impl = get_task(task, dataset)
-        print(task_impl)
+        x1 = task_impl.preprocess()
 
-        # # F1: this will generate a preprocessed list of input messages for model
-        # x1 = task_impl.preprocess(dataset)
+        # TODO: replace w/ batch inference & async return eval job
+        generation_outputs = []
+        for msg in x1[:5]:
+            response = self.inference_api.chat_completion(
+                model=model,
+                messages=[msg],
+                stream=False,
+            )
+            async for x in response:
+                generation_outputs.append(x.completion_message.content)
 
-        # # call inference API w/ model
-        # generation_outputs = ["response1", "response2", "response3"]
-
-        # # F2: post process
-        # x2 = task_impl.postprocess(generation_outputs)
-
-        # # F3: score generation outputs
-        # scores = task_impl.score(x2)
+        x2 = task_impl.postprocess(generation_outputs)
+        scores = task_impl.score(x2)
+        print(scores)
 
         return EvaluateResponse(
             metrics={

llama_stack/providers/impls/meta_reference/evals/tasks/__init__.py

@@ -0,0 +1,5 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the terms described in the LICENSE file in
+# the root directory of this source tree.

llama_stack/providers/impls/meta_reference/evals/tasks/mmlu_task.py

@@ -0,0 +1,152 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the terms described in the LICENSE file in
+# the root directory of this source tree.
+import re
+
+from .task import BaseTask
+
+QUERY_TEMPLATE_MULTICHOICE = """
+Answer the following multiple choice question and make the answer very simple. The last line of your response should be of the following format: 'Answer: $LETTER' (without quotes) where LETTER is one of ABCD.
+
+{Question}
+
+A) {A}
+B) {B}
+C) {C}
+D) {D}
+""".strip()
+
+MULTILINGUAL_ANSWER_REGEXES = [
+    r"Answer\s*:",
+    r"Answer\s*:​​​​​​",  # Korean invisible character
+    r"উত্তর\s*:",
+    r"उत्तर\s*:",
+    r"উত্তরঃ",
+    r"উত্তর\s*:",
+    r"Antwort\s*:",
+    r"답변\s*:",
+    r"정답\s*:",
+    r"답\s*:",
+    r"答案\s*：",
+    r"答案\s*:",
+    r"答\s*：",
+    r"答\s*:",
+    r"答复\s*：",
+    r"答曰\s*：",
+    r"الإجابة:",
+    r"الجواب:",
+    r"إجابة:",
+    r"الإجابة النهائية:",
+    r"الإجابة الصحيحة:",
+    r"الإجابة الصحيحة هي:",
+    r"الإجابة هي:",
+    r"Respuesta\s*:",
+    r"Risposta\s*:",
+    r"答え\s*:",
+    r"答え\s*：",
+    r"回答\s*:",
+    r"回答\s*：",
+    r"解答\s*:",
+    r"Jawaban\s*:",
+    r"Réponse\s*:",
+    r"Resposta\s*:",
+    r"Jibu\s*:",
+    r"Idahun\s*:",
+    r"Ìdáhùn\s*:",
+    r"Idáhùn\s*:",
+    r"Àmọ̀nà\s*:",
+    r"Àdáhùn\s*:",
+    r"Ànúgọ\s*:",
+    r"Àṣàyàn\s*:",
+]
+
+MULTILINGUAL_ANSWER_PATTERN_TEMPLATE = (
+    r"(?i){}\s*([A-D]|[أ-د]|[অ]|[ব]|[ড]|[ঢ]|[Ａ]|[Ｂ]|[Ｃ]|[Ｄ])"
+)
+
+
+def normalize_response(response: str) -> str:
+    """
+    Normalize the response by removing markdown and LaTeX formatting that may prevent a match.
+    """
+
+    return (
+        response.replace("**", "")
+        .replace("$\\boxed{", "")
+        .replace("}$", "")
+        .replace("\\$", "")
+        .replace("$\\text{", "")
+        .replace("$", "")
+        .replace("\\mathrm{", "")
+        .replace("\\{", "")
+        .replace("\\text", "")
+        .replace("\\(", "")
+        .replace("\\mathbf{", "")
+        .replace("{", "")
+        .replace("\\boxed", "")
+    )
+
+
+def normalize_extracted_answer(extracted_answer: str) -> str:
+    return (
+        # In arabic these are the letters used for A-D in multiple choice questions
+        extracted_answer.replace("أ", " A")
+        .replace("ب", " B")
+        .replace("ج", " C")
+        .replace("د", " D")
+        # In Bengali these are the letters used for A-D in multiple choice questions
+        .replace("অ", " A")
+        .replace("ব", " B")
+        .replace("ড", " C")
+        .replace("ঢ", " D")
+        # In Japanese these are the letters sometimes used for A-D in multiple choice questions
+        .replace("Ａ", " A")
+        .replace("Ｂ", " B")
+        .replace("Ｃ", " C")
+        .replace("Ｄ", " D")
+        .strip()
+    )
+
+
+class MMLUTask(BaseTask):
+    """
+    MMLU Task.
+    """
+
+    def __init__(self, dataset, *args, **kwargs):
+        super().__init__(dataset, *args, **kwargs)
+
+    def preprocess_sample(self, sample):
+        """
+        F1: preprocess sample
+        """
+        content = QUERY_TEMPLATE_MULTICHOICE.format(**sample)
+        return {
+            "role": "user",
+            "content": content,
+        }
+
+    def postprocess_sample(self, sample):
+        """
+        F2: postprocess sample
+        """
+        normalized = normalize_response(sample)
+        return normalized
+
+    def score_sample(self, sample, expected):
+        """
+        F3: score sample
+        """
+        extracted_answer = None
+        for answer_regex in MULTILINGUAL_ANSWER_REGEXES:
+            regex = MULTILINGUAL_ANSWER_PATTERN_TEMPLATE.format(answer_regex)
+            match = re.search(regex, sample)
+            if match:
+                extracted_answer = normalize_extracted_answer(match.group(1))
+                break
+        score = (
+            1.0 if extracted_answer and extracted_answer == expected["Answer"] else 0.0
+        )
+        return score

llama_stack/providers/impls/meta_reference/evals/tasks/task.py

@@ -8,8 +8,7 @@ from abc import ABC, abstractmethod
 
 class BaseTask(ABC):
     """
-    Base class for all evaluation tasks.
-    Each task needs to implement the following methods:
+    Base class for all evaluation tasks. Each task needs to implement the following methods:
     - F1: preprocess_sample(self)
     - F2: postprocess_sample(self)
     - F3: score_sample(self)
@@ -42,40 +41,13 @@ class BaseTask(ABC):
         raise NotImplementedError()
 
     def preprocess(self):
-        pass
+        return [self.preprocess_sample(sample) for sample in self.dataset]
 
-    def postprocess(self):
-        pass
+    def postprocess(self, generation):
+        return [self.postprocess_sample(sample) for sample in generation]
 
-    def score(self, generation):
-        pass
-
-
-class MMLUTask(BaseTask):
-    """
-    MMLU Task. Each task needs to implement the following methods:
-    - F1: preprocess_sample(self)
-    - F2: postprocess_sample(self)
-    - F3: score_sample(self)
-    """
-
-    def __init__(self, dataset, *args, **kwargs):
-        super().__init__(dataset, *args, **kwargs)
-
-    def preprocess_sample(self, sample):
-        """
-        F1: preprocess sample
-        """
-        pass
-
-    def postprocess_sample(self, sample):
-        """
-        F2: postprocess sample
-        """
-        pass
-
-    def score_sample(self, sample):
-        """
-        F3: score sample
-        """
-        pass
+    def score(self, postprocessed):
+        return [
+            self.score_sample(sample, ground_truth)
+            for sample, ground_truth in zip(postprocessed, self.dataset)
+        ]

llama_stack/providers/impls/meta_reference/evals/tasks/task_registry.py

@@ -3,7 +3,7 @@
 #
 # This source code is licensed under the terms described in the LICENSE file in
 # the root directory of this source tree.
-from .tasks import *  # noqa: F403
+from .mmlu_task import MMLUTask
 
 # TODO: make this into a config based registry
 TASKS_REGISTRY = {

tests/examples/local-run.yaml

@@ -36,14 +36,18 @@ api_providers:
     config: {}
 routing_table:
   inference:
-  - provider_type: meta-reference
+  # - provider_type: meta-reference
+  #   config:
+  #     model: Llama3.2-1B-Instruct
+  #     quantization: null
+  #     torch_seed: null
+  #     max_seq_len: 4096
+  #     max_batch_size: 1
+  #   routing_key: Llama3.2-1B-Instruct
+  - provider_type: remote::tgi
     config:
-      model: Llama3.2-1B
-      quantization: null
-      torch_seed: null
-      max_seq_len: 4096
-      max_batch_size: 1
-    routing_key: Llama3.2-1B
+      url: http://127.0.0.1:5009
+    routing_key: Llama3.1-8B-Instruct
   safety:
   - provider_type: meta-reference
     config: