wip

llama_stack/apis/dataset/dataset.py

@@ -4,7 +4,7 @@
 # This source code is licensed under the terms described in the LICENSE file in
 # the root directory of this source tree.
 
-from enum import Enum
+# from enum import Enum
 from typing import Any, Dict, Optional, Protocol
 
 from llama_models.llama3.api.datatypes import URL
@@ -14,22 +14,12 @@ from llama_models.schema_utils import json_schema_type, webmethod
 from pydantic import BaseModel
 
 
-@json_schema_type
-class TrainEvalDatasetColumnType(Enum):
-    dialog = "dialog"
-    text = "text"
-    media = "media"
-    number = "number"
-    json = "json"
-
-
 @json_schema_type
 class TrainEvalDataset(BaseModel):
     """Dataset to be used for training or evaluating language models."""
 
-    # TODO(ashwin): figure out if we need to add an enum for a "dataset type"
-
-    columns: Dict[str, TrainEvalDatasetColumnType]
+    # unique identifier associated with the dataset
+    dataset_id: str
     content_url: URL
     metadata: Optional[Dict[str, Any]] = None
 

llama_stack/apis/evals/evals.py

@@ -71,6 +71,7 @@ class EvaluateSummarizationRequest(EvaluateTaskRequestCommon):
     metrics: List[SummarizationMetric]
 
 
+@json_schema_type
 class EvaluationJobStatusResponse(BaseModel):
     job_uuid: str
 
@@ -82,24 +83,34 @@ class EvaluationJobArtifactsResponse(BaseModel):
     job_uuid: str
 
 
+@json_schema_type
+class EvaluationJobCreateResponse(BaseModel):
+    """Response to create a evaluation job."""
+
+    job_uuid: str
+
+
 class Evaluations(Protocol):
     @webmethod(route="/evaluate/text_generation/")
-    def evaluate_text_generation(
-        self,
-        metrics: List[TextGenerationMetric],
-    ) -> EvaluationJob: ...
+    def create_evaluation_job(self, model: str, dataset: str) -> EvaluationJob: ...
 
-    @webmethod(route="/evaluate/question_answering/")
-    def evaluate_question_answering(
-        self,
-        metrics: List[QuestionAnsweringMetric],
-    ) -> EvaluationJob: ...
+    # @webmethod(route="/evaluate/text_generation/")
+    # def evaluate_text_generation(
+    #     self,
+    #     metrics: List[TextGenerationMetric],
+    # ) -> EvaluationJob: ...
 
-    @webmethod(route="/evaluate/summarization/")
-    def evaluate_summarization(
-        self,
-        metrics: List[SummarizationMetric],
-    ) -> EvaluationJob: ...
+    # @webmethod(route="/evaluate/question_answering/")
+    # def evaluate_question_answering(
+    #     self,
+    #     metrics: List[QuestionAnsweringMetric],
+    # ) -> EvaluationJob: ...
+
+    # @webmethod(route="/evaluate/summarization/")
+    # def evaluate_summarization(
+    #     self,
+    #     metrics: List[SummarizationMetric],
+    # ) -> EvaluationJob: ...
 
     @webmethod(route="/evaluate/jobs")
     def get_evaluation_jobs(self) -> List[EvaluationJob]: ...