mirror of
https://github.com/meta-llama/llama-stack.git
synced 2025-07-29 15:23:51 +00:00
scorer wip
This commit is contained in:
parent
70c08e694d
commit
35981a1a3b
5 changed files with 72 additions and 2 deletions
|
@ -14,6 +14,7 @@ from llama_stack.apis.scoring_functions import * # noqa: F403
|
|||
|
||||
|
||||
ScoringResult = Dict[str, Any]
|
||||
SingleScoringResult = Dict[str, Any]
|
||||
|
||||
|
||||
@json_schema_type
|
||||
|
|
|
@ -0,0 +1,5 @@
|
|||
# Copyright (c) Meta Platforms, Inc. and affiliates.
|
||||
# All rights reserved.
|
||||
#
|
||||
# This source code is licensed under the terms described in the LICENSE file in
|
||||
# the root directory of this source tree.
|
|
@ -0,0 +1,35 @@
|
|||
# Copyright (c) Meta Platforms, Inc. and affiliates.
|
||||
# All rights reserved.
|
||||
#
|
||||
# This source code is licensed under the terms described in the LICENSE file in
|
||||
# the root directory of this source tree.
|
||||
from abc import ABC, abstractmethod
|
||||
from typing import Any, Dict, List
|
||||
from llama_stack.apis.scoring_functions import * # noqa: F401, F403
|
||||
from llama_stack.apis.scoring import * # noqa: F401, F403
|
||||
|
||||
|
||||
class BaseScorer(ABC):
|
||||
"""
|
||||
Base interface class for all meta-reference scorers.
|
||||
Each scorer needs to implement the following methods:
|
||||
- score_row(self, row)
|
||||
- aggregate(self, scorer_results)
|
||||
"""
|
||||
|
||||
def __init__(self, *args, **kwargs) -> None:
|
||||
super().__init__(*args, **kwargs)
|
||||
|
||||
def __str__(self) -> str:
|
||||
return self.__class__.__name__
|
||||
|
||||
@abstractmethod
|
||||
def score_row(self, input_row: Dict[str, Any]) -> ScoringResult:
|
||||
raise NotImplementedError()
|
||||
|
||||
@abstractmethod
|
||||
def aggregate(self, scoring_results: List[ScoringResult]) -> ScoringResult:
|
||||
raise NotImplementedError()
|
||||
|
||||
def score(self, input_rows: List[Dict[str, Any]]) -> List[ScoringResult]:
|
||||
return [self.score_row(input_row) for input_row in input_rows]
|
|
@ -0,0 +1,24 @@
|
|||
# Copyright (c) Meta Platforms, Inc. and affiliates.
|
||||
# All rights reserved.
|
||||
#
|
||||
# This source code is licensed under the terms described in the LICENSE file in
|
||||
# the root directory of this source tree.
|
||||
|
||||
from llama_stack.providers.impls.meta_reference.scoring.scorer.base_scorer import (
|
||||
BaseScorer,
|
||||
)
|
||||
|
||||
|
||||
class EqualityScorer(BaseScorer):
|
||||
"""
|
||||
A scorer that assigns a score of 1.0 if the input string matches the target string, and 0.0 otherwise.
|
||||
"""
|
||||
|
||||
def __init__(self, target: str) -> None:
|
||||
"""
|
||||
Initialize the EqualityScorer with a target string.
|
||||
|
||||
Args:
|
||||
target (str): The target string to match against.
|
||||
"""
|
||||
self.target = target
|
|
@ -41,7 +41,9 @@ class MetaReferenceScoringImpl(Scoring, ScoringFunctionsProtocolPrivate):
|
|||
]
|
||||
|
||||
async def register_scoring_function(self, function_def: ScoringFunctionDef) -> None:
|
||||
pass
|
||||
raise NotImplementedError(
|
||||
"Dynamically registering scoring functions is not supported"
|
||||
)
|
||||
|
||||
async def score_batch(
|
||||
self, dataset_id: str, scoring_functions: List[str]
|
||||
|
@ -51,4 +53,7 @@ class MetaReferenceScoringImpl(Scoring, ScoringFunctionsProtocolPrivate):
|
|||
async def score(
|
||||
self, input_rows: List[Dict[str, Any]], scoring_functions: List[str]
|
||||
) -> ScoreResponse:
|
||||
print("!!!!score")
|
||||
print(
|
||||
f"scoring input_rows {input_rows} on scoring_functions {scoring_functions}"
|
||||
)
|
||||
return ScoreResponse()
|
||||
|
|
Loading…
Add table
Add a link
Reference in a new issue