json -> class

llama_stack/providers/impls/braintrust/scoring/scoring_fn/braintrust_scoring_fn.py

@@ -20,6 +20,12 @@ from llama_stack.apis.scoring_functions import *  # noqa: F403
 from llama_stack.apis.common.type_system import *  # noqa: F403
 from autoevals.llm import Factuality
 from autoevals.ragas import AnswerCorrectness
+from llama_stack.providers.impls.braintrust.scoring.scoring_fn.fn_defs.answer_correctness import (
+    answer_correctness_fn_def,
+)
+from llama_stack.providers.impls.braintrust.scoring.scoring_fn.fn_defs.factuality import (
+    factuality_fn_def,
+)
 
 
 BRAINTRUST_FN_DEFS_PATH = Path(__file__).parent / "fn_defs"
@@ -36,9 +42,10 @@ class BraintrustScoringFn(BaseScoringFn):
             "braintrust::factuality": Factuality(),
             "braintrust::answer-correctness": AnswerCorrectness(),
         }
-        self.defs_paths = [
+        self.supported_fn_defs_registry = {
-            str(x) for x in sorted(BRAINTRUST_FN_DEFS_PATH.glob("*.json"))
+            factuality_fn_def.identifier: factuality_fn_def,
-        ]
+            answer_correctness_fn_def.identifier: answer_correctness_fn_def,
+        }
 
     async def score_row(
         self,

llama_stack/providers/impls/braintrust/scoring/scoring_fn/fn_defs/__init__.py

@@ -0,0 +1,5 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the terms described in the LICENSE file in
+# the root directory of this source tree.

llama_stack/providers/impls/braintrust/scoring/scoring_fn/fn_defs/answer-correctness.json

@@ -1,10 +0,0 @@
-{
-    "identifier": "braintrust::answer-correctness",
-    "description": "Test whether an output is factual, compared to an original (`expected`) value. One of Braintrust LLM basd scorer https://github.com/braintrustdata/autoevals/blob/main/py/autoevals/llm.py",
-    "metadata": {},
-    "parameters": [],
-    "return_type": {
-        "type": "number"
-    },
-    "context": null
-}

llama_stack/providers/impls/braintrust/scoring/scoring_fn/fn_defs/answer_correctness.py

@@ -0,0 +1,16 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the terms described in the LICENSE file in
+# the root directory of this source tree.
+
+from llama_stack.apis.common.type_system import NumberType
+from llama_stack.apis.scoring_functions import ScoringFnDef
+
+
+answer_correctness_fn_def = ScoringFnDef(
+    identifier="braintrust::answer-correctness",
+    description="Test whether an output is factual, compared to an original (`expected`) value. One of Braintrust LLM basd scorer https://github.com/braintrustdata/autoevals/blob/main/py/autoevals/llm.py",
+    parameters=[],
+    return_type=NumberType(),
+)

llama_stack/providers/impls/braintrust/scoring/scoring_fn/fn_defs/factuality.json

@@ -1,10 +0,0 @@
-{
-    "identifier": "braintrust::factuality",
-    "description": "Test whether an output is factual, compared to an original (`expected`) value. One of Braintrust LLM basd scorer https://github.com/braintrustdata/autoevals/blob/main/py/autoevals/llm.py",
-    "metadata": {},
-    "parameters": [],
-    "return_type": {
-        "type": "number"
-    },
-    "context": null
-}

llama_stack/providers/impls/braintrust/scoring/scoring_fn/fn_defs/factuality.py

@@ -0,0 +1,16 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the terms described in the LICENSE file in
+# the root directory of this source tree.
+
+from llama_stack.apis.common.type_system import NumberType
+from llama_stack.apis.scoring_functions import ScoringFnDef
+
+
+factuality_fn_def = ScoringFnDef(
+    identifier="braintrust::factuality",
+    description="Test whether an output is factual, compared to an original (`expected`) value. One of Braintrust LLM basd scorer https://github.com/braintrustdata/autoevals/blob/main/py/autoevals/llm.py",
+    parameters=[],
+    return_type=NumberType(),
+)