phoenix-oss/llama-stack-mirror
0
0
Fork 1
mirror of https://github.com/meta-llama/llama-stack.git synced 2025-07-29 15:23:51 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

wip

Browse source
This commit is contained in:
Xi Yan 2024-10-03 11:18:23 -07:00
parent b9b1e8b08b
commit 5e9301de90
2 changed files with 28 additions and 27 deletions
Download patch file Download diff file Expand all files Collapse all files

16
llama_stack/apis/dataset/dataset.py
View file

@ -4,7 +4,7 @@
# This source code is licensed under the terms described in the LICENSE file in # This source code is licensed under the terms described in the LICENSE file in
# the root directory of this source tree. # the root directory of this source tree.
from enum import Enum # from enum import Enum
from typing import Any, Dict, Optional, Protocol from typing import Any, Dict, Optional, Protocol
from llama_models.llama3.api.datatypes import URL from llama_models.llama3.api.datatypes import URL
@ -14,22 +14,12 @@ from llama_models.schema_utils import json_schema_type, webmethod
from pydantic import BaseModel from pydantic import BaseModel
@json_schema_type
class TrainEvalDatasetColumnType(Enum):
dialog = "dialog"
text = "text"
media = "media"
number = "number"
json = "json"
@json_schema_type @json_schema_type
class TrainEvalDataset(BaseModel): class TrainEvalDataset(BaseModel):
"""Dataset to be used for training or evaluating language models.""" """Dataset to be used for training or evaluating language models."""
# TODO(ashwin): figure out if we need to add an enum for a "dataset type" # unique identifier associated with the dataset
dataset_id: str
columns: Dict[str, TrainEvalDatasetColumnType]
content_url: URL content_url: URL
metadata: Optional[Dict[str, Any]] = None metadata: Optional[Dict[str, Any]] = None

39
llama_stack/apis/evals/evals.py
View file

@ -71,6 +71,7 @@ class EvaluateSummarizationRequest(EvaluateTaskRequestCommon):
metrics: List[SummarizationMetric] metrics: List[SummarizationMetric]
@json_schema_type
class EvaluationJobStatusResponse(BaseModel): class EvaluationJobStatusResponse(BaseModel):
job_uuid: str job_uuid: str
@ -82,24 +83,34 @@ class EvaluationJobArtifactsResponse(BaseModel):
job_uuid: str job_uuid: str
@json_schema_type
class EvaluationJobCreateResponse(BaseModel):
"""Response to create a evaluation job."""
job_uuid: str
class Evaluations(Protocol): class Evaluations(Protocol):
@webmethod(route="/evaluate/text_generation/") @webmethod(route="/evaluate/text_generation/")
def evaluate_text_generation( def create_evaluation_job(self, model: str, dataset: str) -> EvaluationJob: ...
self,
metrics: List[TextGenerationMetric],
) -> EvaluationJob: ...
@webmethod(route="/evaluate/question_answering/") # @webmethod(route="/evaluate/text_generation/")
def evaluate_question_answering( # def evaluate_text_generation(
self, # self,
metrics: List[QuestionAnsweringMetric], # metrics: List[TextGenerationMetric],
) -> EvaluationJob: ... # ) -> EvaluationJob: ...
@webmethod(route="/evaluate/summarization/") # @webmethod(route="/evaluate/question_answering/")
def evaluate_summarization( # def evaluate_question_answering(
self, # self,
metrics: List[SummarizationMetric], # metrics: List[QuestionAnsweringMetric],
) -> EvaluationJob: ... # ) -> EvaluationJob: ...
# @webmethod(route="/evaluate/summarization/")
# def evaluate_summarization(
# self,
# metrics: List[SummarizationMetric],
# ) -> EvaluationJob: ...
@webmethod(route="/evaluate/jobs") @webmethod(route="/evaluate/jobs")
def get_evaluation_jobs(self) -> List[EvaluationJob]: ... def get_evaluation_jobs(self) -> List[EvaluationJob]: ...