From 3a4039aea9dbe588e9dbcd3a9cf6c48aaf9dfe86 Mon Sep 17 00:00:00 2001 From: Xi Yan Date: Mon, 28 Oct 2024 11:42:07 -0700 Subject: [PATCH] check identifier prefix --- .../providers/impls/meta_reference/scoring/scoring.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/llama_stack/providers/impls/meta_reference/scoring/scoring.py b/llama_stack/providers/impls/meta_reference/scoring/scoring.py index 8797e898d..41b24a512 100644 --- a/llama_stack/providers/impls/meta_reference/scoring/scoring.py +++ b/llama_stack/providers/impls/meta_reference/scoring/scoring.py @@ -60,12 +60,19 @@ class MetaReferenceScoringImpl(Scoring, ScoringFunctionsProtocolPrivate): async def shutdown(self) -> None: ... async def list_scoring_functions(self) -> List[ScoringFnDef]: - return [ + scoring_fn_defs_list = [ fn_def for impl in self.scoring_fn_id_impls.values() for fn_def in impl.get_supported_scoring_fn_defs() ] + for f in scoring_fn_defs_list: + assert f.identifier.startswith( + "meta-reference" + ), "All meta-reference scoring fn must have identifier prefixed with 'meta-reference'! " + + return scoring_fn_defs_list + async def register_scoring_function(self, function_def: ScoringFnDef) -> None: self.llm_as_judge_fn.register_scoring_fn_def(function_def) self.scoring_fn_id_impls[function_def.identifier] = self.llm_as_judge_fn