scorer only api

llama_stack/providers/impls/meta_reference/evals/evals.py

@@ -13,6 +13,7 @@ from llama_stack.apis.datasets import *  # noqa: F403
 
 from .config import MetaReferenceEvalsImplConfig
 from .tasks.run_eval_task import RunEvalTask
+from .tasks.run_scoring_task import RunScoringTask
 
 
 class MetaReferenceEvalsImpl(Evals):
@@ -44,7 +45,7 @@ class MetaReferenceEvalsImpl(Evals):
             # construct eval task config from inputs
             eval_task_config = EvaluateTaskConfig(
                 dataset_config=EvaluateDatasetConfig(
-                    dataset_name=dataset,
+                    dataset_identifier=dataset,
                     row_limit=3,
                 ),
                 processor_config=EvaluateProcessorConfig(
@@ -76,8 +77,10 @@ class MetaReferenceEvalsImpl(Evals):
     ) -> EvaluateResponse:
         cprint("run_scorer")
 
-        # main logic, we need to convert the datset into List[ScorerInputSample]
+        run_task = RunScoringTask()
+        eval_result = await run_task.run(dataset_config, eval_scoring_config)
 
         return EvaluateResponse(
-            eval_result={},
+            eval_result=eval_result,
+            formatted_report=json.dumps(eval_result.json(), indent=4),
         )

llama_stack/providers/impls/meta_reference/evals/scorer/basic_scorers.py

@@ -31,9 +31,14 @@ class AccuracyScorer(BaseScorer[ScorerInputSample]):
         extracted_answer = scorer_input_sample.generated_answer
         expected_answer = scorer_input_sample.expected_answer
 
-        accuracy = (
-            1.0 if extracted_answer and extracted_answer == expected_answer else 0.0
-        )
+        if isinstance(expected_answer, list):
+            accuracy = (
+                1.0 if extracted_answer and extracted_answer in expected_answer else 0.0
+            )
+        else:
+            accuracy = (
+                1.0 if extracted_answer and extracted_answer == expected_answer else 0.0
+            )
 
         return SingleEvalResult(score_data={"accuracy": accuracy})
 

llama_stack/providers/impls/meta_reference/evals/tasks/run_eval_task.py

@@ -43,7 +43,9 @@ class RunEvalTask(BaseTask):
         print(f"Running eval task w/ {eval_task_config}")
 
         print(DatasetRegistry.names())
-        dataset = DatasetRegistry.get(eval_task_config.dataset_config.dataset_name)
+        dataset = DatasetRegistry.get(
+            eval_task_config.dataset_config.dataset_identifier
+        )
         dataset.load(n_samples=eval_task_config.dataset_config.row_limit)
         print(f"Running on {len(dataset)} samples")
 

llama_stack/providers/impls/meta_reference/evals/tasks/run_scoring_task.py

@@ -0,0 +1,80 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the terms described in the LICENSE file in
+# the root directory of this source tree.
+from llama_stack.distribution.registry.datasets import DatasetRegistry
+from llama_stack.distribution.registry.scorers import ScorerRegistry
+
+from llama_stack.providers.impls.meta_reference.evals.scorer.aggregate_scorer import *  # noqa: F403
+from llama_stack.providers.impls.meta_reference.evals.scorer.basic_scorers import *  # noqa: F403
+
+from llama_stack.apis.evals import *  # noqa: F403
+from llama_stack.apis.inference import *  # noqa: F403
+from termcolor import cprint
+
+
+class RunScoringTask(BaseTask):
+    """
+    RunScoringTask - only run scoring (F3) based on dataset and scoring config
+    """
+
+    def __init__(
+        self,
+        *args,
+        **kwargs,
+    ) -> None:
+        super().__init__(*args, **kwargs)
+
+    def transform_score_input_sample(
+        self, dataset: BaseDataset
+    ) -> List[ScorerInputSample]:
+        scorer_inputs = []
+        for x in dataset:
+            expected_answer = x.data["expected_answer"]
+            generated_answer = x.data["generated_answer"]
+
+            scorer_inputs.append(
+                ScorerInputSample(
+                    expected_answer=expected_answer,
+                    generated_answer=generated_answer,
+                )
+            )
+
+        return scorer_inputs
+
+    async def run(
+        self,
+        dataset_config: EvaluateDatasetConfig,
+        eval_scoring_config: EvaluateScoringConfig,
+        *args,
+        **kwargs,
+    ) -> EvalResult:
+        print(
+            f"Running scoring task w/ dataset={dataset_config} scoring={eval_scoring_config}"
+        )
+
+        dataset = DatasetRegistry.get(dataset_config.dataset_identifier)
+        dataset.load(n_samples=dataset_config.row_limit)
+        print(f"Running on {len(dataset)} samples")
+
+        # transform dataset into
+        postprocessed = self.transform_score_input_sample(dataset)
+        cprint(postprocessed, "blue")
+
+        # F3 - scorer
+        scorer_config_list = eval_scoring_config.scorer_config_list
+        scorer_list = []
+        for s_conf in scorer_config_list:
+            scorer = ScorerRegistry.get(s_conf.scorer_name)
+            scorer_list.append(scorer())
+
+        scorer = AggregateScorer(
+            scorers=scorer_list,
+        )
+
+        scorer_results = scorer.score(postprocessed)
+        cprint(scorer_results, "magenta")
+        eval_result = scorer.aggregate_results(scorer_results)
+
+        return eval_result