phoenix-oss/llama-stack-mirror
0
0
Fork 1
mirror of https://github.com/meta-llama/llama-stack.git synced 2025-08-02 08:44:44 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

only keep 1 run_eval

Browse source
This commit is contained in:
Xi Yan 2024-11-07 16:17:49 -08:00
parent 6b889651d6
commit fd581c3d88
4 changed files with 45 additions and 76 deletions
Download patch file Download diff file Expand all files Collapse all files

22
llama_stack/apis/eval/eval.py
View file

@ -66,36 +66,28 @@ class EvaluateResponse(BaseModel):
class Eval(Protocol): class Eval(Protocol):
@webmethod(route="/eval/run_benchmark", method="POST")
async def run_benchmark(
self,
benchmark_id: str,
benchmark_config: BenchmarkEvalTaskConfig,
) -> Job: ...
@webmethod(route="/eval/run_eval", method="POST") @webmethod(route="/eval/run_eval", method="POST")
async def run_eval( async def run_eval(
self, self,
task: EvalTaskDef, task_id: str,
task_config: AppEvalTaskConfig, task_def: EvalTaskDef,
task_config: EvalTaskConfig,
) -> Job: ... ) -> Job: ...
@webmethod(route="/eval/evaluate_rows", method="POST") @webmethod(route="/eval/evaluate_rows", method="POST")
async def evaluate_rows( async def evaluate_rows(
self, self,
task_id: str,
input_rows: List[Dict[str, Any]], input_rows: List[Dict[str, Any]],
scoring_functions: List[str], scoring_functions: List[str],
task_config: EvalTaskConfig, task_config: EvalTaskConfig,
eval_task_id: Optional[str] = None,
) -> EvaluateResponse: ... ) -> EvaluateResponse: ...
@webmethod(route="/eval/job/status", method="GET") @webmethod(route="/eval/job/status", method="GET")
async def job_status( async def job_status(self, task_id: str, job_id: str) -> Optional[JobStatus]: ...
self, job_id: str, eval_task_id: str
) -> Optional[JobStatus]: ...
@webmethod(route="/eval/job/cancel", method="POST") @webmethod(route="/eval/job/cancel", method="POST")
async def job_cancel(self, job_id: str, eval_task_id: str) -> None: ... async def job_cancel(self, task_id: str, job_id: str) -> None: ...
@webmethod(route="/eval/job/result", method="GET") @webmethod(route="/eval/job/result", method="GET")
async def job_result(self, job_id: str, eval_task_id: str) -> EvaluateResponse: ... async def job_result(self, task_id: str, job_id: str) -> EvaluateResponse: ...

48
llama_stack/distribution/routers/routers.py
View file

@ -16,10 +16,6 @@ from llama_stack.apis.datasetio import * # noqa: F403
from llama_stack.apis.scoring import * # noqa: F403 from llama_stack.apis.scoring import * # noqa: F403
from llama_stack.apis.eval import * # noqa: F403 from llama_stack.apis.eval import * # noqa: F403
from llama_stack.providers.inline.meta_reference.eval.eval import (
DEFAULT_EVAL_TASK_IDENTIFIER,
)
class MemoryRouter(Memory): class MemoryRouter(Memory):
"""Routes to an provider based on the memory bank identifier""" """Routes to an provider based on the memory bank identifier"""
@ -268,36 +264,28 @@ class EvalRouter(Eval):
async def shutdown(self) -> None: async def shutdown(self) -> None:
pass pass
async def run_benchmark(
self,
benchmark_id: str,
benchmark_config: BenchmarkEvalTaskConfig,
) -> Job:
pass
async def run_eval( async def run_eval(
self, self,
task: EvalTaskDef, task_id: str,
task_def: EvalTaskDef,
task_config: AppEvalTaskConfig, task_config: AppEvalTaskConfig,
) -> Job: ) -> Job:
return await self.routing_table.get_provider_impl(task.identifier).run_eval( return await self.routing_table.get_provider_impl(task_id).run_eval(
task=task, task_id=task_id,
task_def=task_def,
task_config=task_config, task_config=task_config,
) )
@webmethod(route="/eval/evaluate_rows", method="POST") @webmethod(route="/eval/evaluate_rows", method="POST")
async def evaluate_rows( async def evaluate_rows(
self, self,
task_id: str,
input_rows: List[Dict[str, Any]], input_rows: List[Dict[str, Any]],
scoring_functions: List[str], scoring_functions: List[str],
task_config: EvalTaskConfig, task_config: EvalTaskConfig,
eval_task_id: Optional[str] = None,
) -> EvaluateResponse: ) -> EvaluateResponse:
# NOTE: This is to deal with the case where we do not pre-register an eval benchmark_task return await self.routing_table.get_provider_impl(task_id).evaluate_rows(
# We use default DEFAULT_EVAL_TASK_IDENTIFIER as identifier task_id=task_id,
if eval_task_id is None:
eval_task_id = DEFAULT_EVAL_TASK_IDENTIFIER
return await self.routing_table.get_provider_impl(eval_task_id).evaluate_rows(
input_rows=input_rows, input_rows=input_rows,
scoring_functions=scoring_functions, scoring_functions=scoring_functions,
task_config=task_config, task_config=task_config,
@ -305,27 +293,29 @@ class EvalRouter(Eval):
async def job_status( async def job_status(
self, self,
task_id: str,
job_id: str, job_id: str,
eval_task_id: str,
) -> Optional[JobStatus]: ) -> Optional[JobStatus]:
return await self.routing_table.get_provider_impl(eval_task_id).job_status( return await self.routing_table.get_provider_impl(task_id).job_status(
job_id, eval_task_id task_id, job_id
) )
async def job_cancel( async def job_cancel(
self, self,
task_id: str,
job_id: str, job_id: str,
eval_task_id: str,
) -> None: ) -> None:
await self.routing_table.get_provider_impl(eval_task_id).job_cancel( await self.routing_table.get_provider_impl(task_id).job_cancel(
job_id, eval_task_id task_id,
job_id,
) )
async def job_result( async def job_result(
self, self,
task_id: str,
job_id: str, job_id: str,
eval_task_id: str,
) -> EvaluateResponse: ) -> EvaluateResponse:
return await self.routing_table.get_provider_impl(eval_task_id).job_result( return await self.routing_table.get_provider_impl(task_id).job_result(
job_id, eval_task_id task_id,
job_id,
) )

36
llama_stack/providers/inline/meta_reference/eval/eval.py
View file

@ -7,14 +7,7 @@ from enum import Enum
from llama_models.llama3.api.datatypes import * # noqa: F403 from llama_models.llama3.api.datatypes import * # noqa: F403
from .....apis.common.job_types import Job from .....apis.common.job_types import Job
from .....apis.eval.eval import ( from .....apis.eval.eval import Eval, EvalTaskConfig, EvaluateResponse, JobStatus
AppEvalTaskConfig,
BenchmarkEvalTaskConfig,
Eval,
EvalTaskConfig,
EvaluateResponse,
JobStatus,
)
from llama_stack.apis.common.type_system import * # noqa: F403 from llama_stack.apis.common.type_system import * # noqa: F403
from llama_stack.apis.datasetio import DatasetIO from llama_stack.apis.datasetio import DatasetIO
from llama_stack.apis.datasets import Datasets from llama_stack.apis.datasets import Datasets
@ -98,21 +91,15 @@ class MetaReferenceEvalImpl(Eval, EvalTasksProtocolPrivate):
f"Dataset {dataset_id} does not have a correct input schema in {expected_schemas}" f"Dataset {dataset_id} does not have a correct input schema in {expected_schemas}"
) )
async def run_benchmark(
self,
benchmark_id: str,
benchmark_config: BenchmarkEvalTaskConfig,
) -> Job:
raise NotImplementedError("Benchmark eval is not implemented yet")
async def run_eval( async def run_eval(
self, self,
task: EvalTaskDef, task_id: str,
task_config: AppEvalTaskConfig, task_def: EvalTaskDef,
task_config: EvalTaskConfig,
) -> Job: ) -> Job:
dataset_id = task.dataset_id dataset_id = task_def.dataset_id
candidate = task_config.eval_candidate candidate = task_config.eval_candidate
scoring_functions = task.scoring_functions scoring_functions = task_def.scoring_functions
await self.validate_eval_input_dataset_schema(dataset_id=dataset_id) await self.validate_eval_input_dataset_schema(dataset_id=dataset_id)
all_rows = await self.datasetio_api.get_rows_paginated( all_rows = await self.datasetio_api.get_rows_paginated(
@ -120,6 +107,7 @@ class MetaReferenceEvalImpl(Eval, EvalTasksProtocolPrivate):
rows_in_page=-1, rows_in_page=-1,
) )
res = await self.evaluate_rows( res = await self.evaluate_rows(
task_id=task_id,
input_rows=all_rows.rows, input_rows=all_rows.rows,
scoring_functions=scoring_functions, scoring_functions=scoring_functions,
task_config=task_config, task_config=task_config,
@ -133,10 +121,10 @@ class MetaReferenceEvalImpl(Eval, EvalTasksProtocolPrivate):
async def evaluate_rows( async def evaluate_rows(
self, self,
task_id: str,
input_rows: List[Dict[str, Any]], input_rows: List[Dict[str, Any]],
scoring_functions: List[str], scoring_functions: List[str],
task_config: EvalTaskConfig, task_config: EvalTaskConfig,
eval_task_id: Optional[str] = None,
) -> EvaluateResponse: ) -> EvaluateResponse:
candidate = task_config.eval_candidate candidate = task_config.eval_candidate
if candidate.type == "agent": if candidate.type == "agent":
@ -206,17 +194,17 @@ class MetaReferenceEvalImpl(Eval, EvalTasksProtocolPrivate):
return EvaluateResponse(generations=generations, scores=score_response.results) return EvaluateResponse(generations=generations, scores=score_response.results)
async def job_status(self, job_id: str, eval_task_id: str) -> Optional[JobStatus]: async def job_status(self, task_id: str, job_id: str) -> Optional[JobStatus]:
if job_id in self.jobs: if job_id in self.jobs:
return JobStatus.completed return JobStatus.completed
return None return None
async def job_cancel(self, job_id: str, eval_task_id: str) -> None: async def job_cancel(self, task_id: str, job_id: str) -> None:
raise NotImplementedError("Job cancel is not implemented yet") raise NotImplementedError("Job cancel is not implemented yet")
async def job_result(self, job_id: str, eval_task_id: str) -> EvaluateResponse: async def job_result(self, task_id: str, job_id: str) -> EvaluateResponse:
status = await self.job_status(job_id, eval_task_id) status = await self.job_status(task_id, job_id)
if not status or status != JobStatus.completed: if not status or status != JobStatus.completed:
raise ValueError(f"Job is not completed, Status: {status.value}") raise ValueError(f"Job is not completed, Status: {status.value}")

15
llama_stack/providers/tests/eval/test_eval.py
View file

@ -50,6 +50,7 @@ class Testeval:
] ]
response = await eval_impl.evaluate_rows( response = await eval_impl.evaluate_rows(
task_id="meta-reference::app_eval",
input_rows=rows.rows, input_rows=rows.rows,
scoring_functions=scoring_functions, scoring_functions=scoring_functions,
task_config=AppEvalTaskConfig( task_config=AppEvalTaskConfig(
@ -75,10 +76,12 @@ class Testeval:
"meta-reference::subset_of", "meta-reference::subset_of",
] ]
task_id = "meta-reference::app_eval"
response = await eval_impl.run_eval( response = await eval_impl.run_eval(
task=EvalTaskDef( task_id=task_id,
task_def=EvalTaskDef(
# NOTE: this is needed to make the router work for all app evals # NOTE: this is needed to make the router work for all app evals
identifier="meta-reference::app_eval", identifier=task_id,
dataset_id="test_dataset_for_eval", dataset_id="test_dataset_for_eval",
scoring_functions=scoring_functions, scoring_functions=scoring_functions,
), ),
@ -90,13 +93,9 @@ class Testeval:
), ),
) )
assert response.job_id == "0" assert response.job_id == "0"
job_status = await eval_impl.job_status( job_status = await eval_impl.job_status(task_id, response.job_id)
response.job_id, "meta-reference::app_eval"
)
assert job_status and job_status.value == "completed" assert job_status and job_status.value == "completed"
eval_response = await eval_impl.job_result( eval_response = await eval_impl.job_result(task_id, response.job_id)
response.job_id, "meta-reference::app_eval"
)
assert eval_response is not None assert eval_response is not None
assert len(eval_response.generations) == 5 assert len(eval_response.generations) == 5