phoenix-oss/llama-stack-mirror
0
0
Fork 1
mirror of https://github.com/meta-llama/llama-stack.git synced 2025-10-16 06:53:47 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

eval api

Browse source
This commit is contained in:
Xi Yan 2024-11-05 14:55:59 -08:00
parent fe91608321
commit bca96b5b35
2 changed files with 85 additions and 5 deletions
Download patch file Download diff file Expand all files Collapse all files

35
llama_stack/apis/eval/eval.py
View file

@ -14,6 +14,8 @@ from llama_stack.apis.scoring_functions import * # noqa: F403
from llama_stack.apis.agents import AgentConfig
from llama_stack.apis.common.job_types import Job, JobStatus
from llama_stack.apis.scoring import * # noqa: F403
from llama_stack.apis.scoring_functions import * # noqa: F403
from llama_stack.apis.eval_tasks import * # noqa: F403
@json_schema_type
@ -34,11 +36,27 @@ EvalCandidate = Annotated[
Union[ModelCandidate, AgentCandidate], Field(discriminator="type")
]
# @json_schema_type
# class EvalTaskDef(BaseModel):
# dataset_id: str
# candidate: EvalCandidate
# scoring_functions: List[str]
@json_schema_type
class BenchmarkEvalTaskConfig(BaseModel):
type: Literal["benchmark"] = "benchmark"
eval_candidate: EvalCandidate # type: ignore
@json_schema_type
class AppEvalTaskConfig(BaseModel):
type: Literal["app"] = "app"
eval_candidate: EvalCandidate # type: ignore
scoring_functions_config: Dict[str, ScoringFnConfig] = Field(
description="Map between scoring function id and parameters",
default_factory=dict,
)
# we could optinally add any GenEval specific dataset config here
EvalTaskConfig = Annotated[
Union[BenchmarkEvalTaskConfig, AppEvalTaskConfig], Field(discriminator="type")
]
@json_schema_type
@ -50,6 +68,13 @@ class EvaluateResponse(BaseModel):
class Eval(Protocol):
@webmethod(route="/eval/evaluate_batch", method="POST")
async def evaluate_task(
self,
eval_task_def: Union[str, EvalTaskDef], # type: ignore
eval_task_config: EvalTaskConfig, # type: ignore
) -> Job: ...
@webmethod(route="/eval/evaluate_batch", method="POST")
async def evaluate_batch(
self,