phoenix-oss/llama-stack-mirror
0
0
Fork 1
mirror of https://github.com/meta-llama/llama-stack.git synced 2025-07-29 15:23:51 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

json -> class

Browse source
This commit is contained in:
Xi Yan 2024-10-28 11:30:02 -07:00
parent 0dad0d0d67
commit e2f6c04e1c
6 changed files with 47 additions and 23 deletions
Download patch file Download diff file Expand all files Collapse all files

13
llama_stack/providers/impls/braintrust/scoring/scoring_fn/braintrust_scoring_fn.py
View file

@ -20,6 +20,12 @@ from llama_stack.apis.scoring_functions import * # noqa: F403
from llama_stack.apis.common.type_system import * # noqa: F403
from autoevals.llm import Factuality
from autoevals.ragas import AnswerCorrectness
from llama_stack.providers.impls.braintrust.scoring.scoring_fn.fn_defs.answer_correctness import (
answer_correctness_fn_def,
)
from llama_stack.providers.impls.braintrust.scoring.scoring_fn.fn_defs.factuality import (
factuality_fn_def,
)
BRAINTRUST_FN_DEFS_PATH = Path(__file__).parent / "fn_defs"
@ -36,9 +42,10 @@ class BraintrustScoringFn(BaseScoringFn):
"braintrust::factuality": Factuality(),
"braintrust::answer-correctness": AnswerCorrectness(),
}
self.defs_paths = [
str(x) for x in sorted(BRAINTRUST_FN_DEFS_PATH.glob("*.json"))
]
self.supported_fn_defs_registry = {
factuality_fn_def.identifier: factuality_fn_def,
answer_correctness_fn_def.identifier: answer_correctness_fn_def,
}
async def score_row(
self,

5
llama_stack/providers/impls/braintrust/scoring/scoring_fn/fn_defs/__init__.py Normal file
View file

@ -0,0 +1,5 @@
# Copyright (c) Meta Platforms, Inc. and affiliates.
# All rights reserved.
#
# This source code is licensed under the terms described in the LICENSE file in
# the root directory of this source tree.

10
llama_stack/providers/impls/braintrust/scoring/scoring_fn/fn_defs/answer-correctness.json
View file

@ -1,10 +0,0 @@
{
"identifier": "braintrust::answer-correctness",
"description": "Test whether an output is factual, compared to an original (`expected`) value. One of Braintrust LLM basd scorer https://github.com/braintrustdata/autoevals/blob/main/py/autoevals/llm.py",
"metadata": {},
"parameters": [],
"return_type": {
"type": "number"
},
"context": null
}

16
llama_stack/providers/impls/braintrust/scoring/scoring_fn/fn_defs/answer_correctness.py Normal file
View file

@ -0,0 +1,16 @@
# Copyright (c) Meta Platforms, Inc. and affiliates.
# All rights reserved.
#
# This source code is licensed under the terms described in the LICENSE file in
# the root directory of this source tree.
from llama_stack.apis.common.type_system import NumberType
from llama_stack.apis.scoring_functions import ScoringFnDef
answer_correctness_fn_def = ScoringFnDef(
identifier="braintrust::answer-correctness",
description="Test whether an output is factual, compared to an original (`expected`) value. One of Braintrust LLM basd scorer https://github.com/braintrustdata/autoevals/blob/main/py/autoevals/llm.py",
parameters=[],
return_type=NumberType(),
)

10
llama_stack/providers/impls/braintrust/scoring/scoring_fn/fn_defs/factuality.json
View file

@ -1,10 +0,0 @@
{
"identifier": "braintrust::factuality",
"description": "Test whether an output is factual, compared to an original (`expected`) value. One of Braintrust LLM basd scorer https://github.com/braintrustdata/autoevals/blob/main/py/autoevals/llm.py",
"metadata": {},
"parameters": [],
"return_type": {
"type": "number"
},
"context": null
}

16
llama_stack/providers/impls/braintrust/scoring/scoring_fn/fn_defs/factuality.py Normal file
View file

@ -0,0 +1,16 @@
# Copyright (c) Meta Platforms, Inc. and affiliates.
# All rights reserved.
#
# This source code is licensed under the terms described in the LICENSE file in
# the root directory of this source tree.
from llama_stack.apis.common.type_system import NumberType
from llama_stack.apis.scoring_functions import ScoringFnDef
factuality_fn_def = ScoringFnDef(
identifier="braintrust::factuality",
description="Test whether an output is factual, compared to an original (`expected`) value. One of Braintrust LLM basd scorer https://github.com/braintrustdata/autoevals/blob/main/py/autoevals/llm.py",
parameters=[],
return_type=NumberType(),
)