eleuther eval fix

llama_stack/apis/evals/client.py

@@ -54,27 +54,29 @@ class EvaluationClient(Evals):
 async def run_main(host: str, port: int):
     client = EvaluationClient(f"http://{host}:{port}")
 
-    # CustomDataset
+    # Custom Eval Task
+    # response = await client.run_evals(
+    #     model="Llama3.1-8B-Instruct",
+    #     dataset="mmlu-simple-eval-en",
+    #     task="mmlu",
+    #     eval_task_config=EvaluateTaskConfig(
+    #         n_samples=2,
+    #     ),
+    # )
+
+    # Eleuther Eval Task
     response = await client.run_evals(
         model="Llama3.1-8B-Instruct",
-        dataset="mmlu-simple-eval-en",
+        # task="meta_mmlu_pro_instruct",
-        task="mmlu",
+        task="meta_ifeval",
         eval_task_config=EvaluateTaskConfig(
             n_samples=2,
         ),
     )
-    cprint(f"evaluate response={response}", "green")
+    if response.formatted_report:
-
+        cprint(response.formatted_report, "green")
-    # Eleuther Eval Task
+    else:
-    # response = await client.run_evals(
+        cprint(f"evaluate response={response}", "green")
-    #     model="Llama3.1-8B-Instruct",
-    #     task="meta_mmlu_pro_instruct",
-    #     # task="meta_ifeval",
-    #     eval_task_config=EvaluateTaskConfig(
-    #         n_samples=2,
-    #     )
-    # )
-    # cprint(response.metrics["metrics_table"], "red")
 
 
 def main(host: str, port: int):

llama_stack/providers/impls/third_party/evals/eleuther/eleuther.py

@@ -157,12 +157,14 @@ class EleutherEvalsAdapter(Evals):
             limit=eval_task_config.n_samples,
         )
 
+        eval_result = EvalResult(
+            metrics={},
+        )
         formatted_output = lm_eval.utils.make_table(output)
 
         cprint(formatted_output, "green")
 
         return EvaluateResponse(
-            metrics={
+            eval_result=eval_result,
-                "metrics_table": formatted_output,
+            formatted_report=formatted_output,
-            },
         )

tests/examples/local-run.yaml

@@ -14,8 +14,8 @@ apis:
 - evals
 providers:
   evals:
-  - provider_id: meta-reference
+  - provider_id: eleuther
-    provider_type: meta-reference
+    provider_type: eleuther
     config: {}
   inference:
   - provider_id: remote::tgi