From 4fc92e52d79be1ec10d50ec54ccaf1a62e57a69c Mon Sep 17 00:00:00 2001
From: Xi Yan <xiyan@meta.com>
Date: Tue, 5 Nov 2024 11:23:24 -0800
Subject: [PATCH] wip

---
 llama_stack/apis/eval/eval.py | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/llama_stack/apis/eval/eval.py b/llama_stack/apis/eval/eval.py
index 51f49da15..53fe49a8d 100644
--- a/llama_stack/apis/eval/eval.py
+++ b/llama_stack/apis/eval/eval.py
@@ -34,6 +34,12 @@ EvalCandidate = Annotated[
     Union[ModelCandidate, AgentCandidate], Field(discriminator="type")
 ]
 
+# @json_schema_type
+# class EvalTaskDef(BaseModel):
+#     dataset_id: str
+#     candidate: EvalCandidate
+#     scoring_functions: List[str]
+
 
 @json_schema_type
 class EvaluateResponse(BaseModel):
@@ -48,7 +54,7 @@ class Eval(Protocol):
     async def evaluate_batch(
         self,
         dataset_id: str,
-        candidate: EvalCandidate,
+        candidate: EvalCandidate,  # type: ignore
         scoring_functions: List[str],
     ) -> Job: ...
 
@@ -56,7 +62,7 @@ class Eval(Protocol):
     async def evaluate(
         self,
         input_rows: List[Dict[str, Any]],
-        candidate: EvalCandidate,
+        candidate: EvalCandidate,  # type: ignore
         scoring_functions: List[str],
     ) -> EvaluateResponse: ...