phoenix-oss/llama-stack-mirror
0
0
Fork 1
mirror of https://github.com/meta-llama/llama-stack.git synced 2025-12-09 03:19:20 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

wip

Browse source
This commit is contained in:
Xi Yan 2024-10-03 11:18:23 -07:00
parent b9b1e8b08b
commit 5e9301de90
2 changed files with 28 additions and 27 deletions
Download patch file Download diff file Expand all files Collapse all files

39
llama_stack/apis/evals/evals.py
View file

@ -71,6 +71,7 @@ class EvaluateSummarizationRequest(EvaluateTaskRequestCommon):
metrics: List[SummarizationMetric]
@json_schema_type
class EvaluationJobStatusResponse(BaseModel):
job_uuid: str
@ -82,24 +83,34 @@ class EvaluationJobArtifactsResponse(BaseModel):
job_uuid: str
@json_schema_type
class EvaluationJobCreateResponse(BaseModel):
"""Response to create a evaluation job."""
job_uuid: str
class Evaluations(Protocol):
@webmethod(route="/evaluate/text_generation/")
def evaluate_text_generation(
self,
metrics: List[TextGenerationMetric],
) -> EvaluationJob: ...
def create_evaluation_job(self, model: str, dataset: str) -> EvaluationJob: ...
@webmethod(route="/evaluate/question_answering/")
def evaluate_question_answering(
self,
metrics: List[QuestionAnsweringMetric],
) -> EvaluationJob: ...
# @webmethod(route="/evaluate/text_generation/")
# def evaluate_text_generation(
# self,
# metrics: List[TextGenerationMetric],
# ) -> EvaluationJob: ...
@webmethod(route="/evaluate/summarization/")
def evaluate_summarization(
self,
metrics: List[SummarizationMetric],
) -> EvaluationJob: ...
# @webmethod(route="/evaluate/question_answering/")
# def evaluate_question_answering(
# self,
# metrics: List[QuestionAnsweringMetric],
# ) -> EvaluationJob: ...
# @webmethod(route="/evaluate/summarization/")
# def evaluate_summarization(
# self,
# metrics: List[SummarizationMetric],
# ) -> EvaluationJob: ...
@webmethod(route="/evaluate/jobs")
def get_evaluation_jobs(self) -> List[EvaluationJob]: ...