phoenix-oss/llama-stack-mirror
0
0
Fork 1
mirror of https://github.com/meta-llama/llama-stack.git synced 2025-07-30 23:51:00 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

wip

Browse source
This commit is contained in:
Xi Yan 2024-11-05 11:23:24 -08:00
parent 8de845a96d
commit 4fc92e52d7
1 changed files with 8 additions and 2 deletions
Download patch file Download diff file Expand all files Collapse all files

10
llama_stack/apis/eval/eval.py
View file

@ -34,6 +34,12 @@ EvalCandidate = Annotated[
Union[ModelCandidate, AgentCandidate], Field(discriminator="type") Union[ModelCandidate, AgentCandidate], Field(discriminator="type")
] ]
# @json_schema_type
# class EvalTaskDef(BaseModel):
# dataset_id: str
# candidate: EvalCandidate
# scoring_functions: List[str]
@json_schema_type @json_schema_type
class EvaluateResponse(BaseModel): class EvaluateResponse(BaseModel):
@ -48,7 +54,7 @@ class Eval(Protocol):
async def evaluate_batch( async def evaluate_batch(
self, self,
dataset_id: str, dataset_id: str,
candidate: EvalCandidate, candidate: EvalCandidate, # type: ignore
scoring_functions: List[str], scoring_functions: List[str],
) -> Job: ... ) -> Job: ...
@ -56,7 +62,7 @@ class Eval(Protocol):
async def evaluate( async def evaluate(
self, self,
input_rows: List[Dict[str, Any]], input_rows: List[Dict[str, Any]],
candidate: EvalCandidate, candidate: EvalCandidate, # type: ignore
scoring_functions: List[str], scoring_functions: List[str],
) -> EvaluateResponse: ... ) -> EvaluateResponse: ...