forked from phoenix-oss/llama-stack-mirror
feat(eval api): (2.2/n) delete eval / scoring / scoring_fn apis (#1700)
# What does this PR do? - To make it easier, delete existing `eval/scoring/scoring_function` apis. There will be a bunch of broken impls here. The sequence is: 1. migrate benchmark graders 2. clean up existing scoring functions - Add a skeleton evaluation impl to make tests pass. ## Test Plan tested in following PRs [//]: # (## Documentation)
This commit is contained in:
parent
0048274ec0
commit
c1d18283d2
113 changed files with 408 additions and 3900 deletions
|
@ -0,0 +1,71 @@
|
|||
# Copyright (c) Meta Platforms, Inc. and affiliates.
|
||||
# All rights reserved.
|
||||
#
|
||||
# This source code is licensed under the terms described in the LICENSE file in
|
||||
# the root directory of this source tree.
|
||||
|
||||
from llama_stack.apis.agents import Agents
|
||||
from llama_stack.apis.datasetio import DatasetIO
|
||||
from llama_stack.apis.datasets import Datasets
|
||||
from llama_stack.apis.inference import Inference
|
||||
from llama_stack.providers.datatypes import BenchmarksProtocolPrivate
|
||||
|
||||
from .....apis.benchmarks import Benchmark
|
||||
from .....apis.evaluation.evaluation import (
|
||||
Evaluation,
|
||||
EvaluationCandidate,
|
||||
EvaluationJob,
|
||||
EvaluationResponse,
|
||||
EvaluationTask,
|
||||
)
|
||||
from .config import MetaReferenceEvaluationConfig
|
||||
|
||||
EVAL_TASKS_PREFIX = "benchmarks:"
|
||||
|
||||
|
||||
class MetaReferenceEvaluationImpl(
|
||||
Evaluation,
|
||||
BenchmarksProtocolPrivate,
|
||||
):
|
||||
def __init__(
|
||||
self,
|
||||
config: MetaReferenceEvaluationConfig,
|
||||
datasetio_api: DatasetIO,
|
||||
datasets_api: Datasets,
|
||||
inference_api: Inference,
|
||||
agents_api: Agents,
|
||||
) -> None:
|
||||
self.config = config
|
||||
self.datasetio_api = datasetio_api
|
||||
self.datasets_api = datasets_api
|
||||
self.inference_api = inference_api
|
||||
self.agents_api = agents_api
|
||||
|
||||
async def initialize(self) -> None:
|
||||
pass
|
||||
|
||||
async def shutdown(self) -> None:
|
||||
pass
|
||||
|
||||
async def register_benchmark(self, benchmark: Benchmark) -> None:
|
||||
pass
|
||||
|
||||
async def run(
|
||||
self,
|
||||
task: EvaluationTask,
|
||||
candidate: EvaluationCandidate,
|
||||
) -> EvaluationJob:
|
||||
raise NotImplementedError("Run is not implemented yet")
|
||||
|
||||
async def run_sync(
|
||||
self,
|
||||
task: EvaluationTask,
|
||||
candidate: EvaluationCandidate,
|
||||
) -> EvaluationResponse:
|
||||
raise NotImplementedError("Run sync is not implemented yet")
|
||||
|
||||
async def grade(self, task: EvaluationTask) -> EvaluationJob:
|
||||
raise NotImplementedError("Grade is not implemented yet")
|
||||
|
||||
async def grade_sync(self, task: EvaluationTask) -> EvaluationResponse:
|
||||
raise NotImplementedError("Grade sync is not implemented yet")
|
Loading…
Add table
Add a link
Reference in a new issue