phoenix-oss/llama-stack-mirror
0
0
Fork 1
mirror of https://github.com/meta-llama/llama-stack.git synced 2025-07-30 23:51:00 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

naming fix

Browse source
This commit is contained in:
Xi Yan 2024-11-05 16:20:16 -08:00
parent 9759911e47
commit 979cd4cd44
2 changed files with 12 additions and 12 deletions
Download patch file Download diff file Expand all files Collapse all files

2
llama_stack/apis/eval/eval.py
View file

@ -46,7 +46,7 @@ class BenchmarkEvalTaskConfig(BaseModel):
class AppEvalTaskConfig(BaseModel): class AppEvalTaskConfig(BaseModel):
type: Literal["app"] = "app" type: Literal["app"] = "app"
eval_candidate: EvalCandidate # type: ignore eval_candidate: EvalCandidate # type: ignore
scoring_functions_config: Dict[str, ScoringFnConfig] = Field( scoring_functions_params: Dict[str, ScoringFnParams] = Field( # type: ignore
description="Map between scoring function id and parameters", description="Map between scoring function id and parameters",
default_factory=dict, default_factory=dict,
) )

22
llama_stack/apis/scoring_functions/scoring_functions.py
View file

@ -28,11 +28,11 @@ from llama_stack.apis.common.type_system import ParamType
@json_schema_type @json_schema_type
class ScoringConfigType(Enum): class ScoringConfigType(Enum):
llm_as_judge = "llm_as_judge" llm_as_judge = "llm_as_judge"
answer_parsing = "answer_parsing" regex_parser = "regex_parser"
@json_schema_type @json_schema_type
class LLMAsJudgeScoringFnConfig(BaseModel): class LLMAsJudgeScoringFnParams(BaseModel):
type: Literal[ScoringConfigType.llm_as_judge.value] = ( # type: ignore type: Literal[ScoringConfigType.llm_as_judge.value] = ( # type: ignore
ScoringConfigType.llm_as_judge.value ScoringConfigType.llm_as_judge.value
) )
@ -42,20 +42,20 @@ class LLMAsJudgeScoringFnConfig(BaseModel):
@json_schema_type @json_schema_type
class AnswerParsingScoringFnConfig(BaseModel): class RegexParserScoringFnParams(BaseModel):
type: Literal[ScoringConfigType.answer_parsing.value] = ( # type: ignore type: Literal[ScoringConfigType.regex_parser.value] = ( # type: ignore
ScoringConfigType.answer_parsing.value ScoringConfigType.regex_parser.value
) )
parsing_regex: Optional[List[str]] = Field( parsing_regexes: Optional[List[str]] = Field(
description="Regex to extract the answer from generated response", description="Regex to extract the answer from generated response",
default_factory=list, default_factory=list,
) )
ScoringFnConfig = Annotated[ ScoringFnParams = Annotated[
Union[ Union[
LLMAsJudgeScoringFnConfig, LLMAsJudgeScoringFnParams,
AnswerParsingScoringFnConfig, RegexParserScoringFnParams,
], ],
Field(discriminator="type"), Field(discriminator="type"),
] ]
@ -72,8 +72,8 @@ class ScoringFnDef(BaseModel):
return_type: ParamType = Field( return_type: ParamType = Field(
description="The return type of the deterministic function", description="The return type of the deterministic function",
) )
config: Optional[ScoringFnConfig] = Field( # type: ignore params: Optional[ScoringFnParams] = Field( # type: ignore
description="The configuration for the scoring function for benchmark eval, we could override this for app eval", description="The parameters for the scoring function for benchmark eval, we could override this for app eval",
default=None, default=None,
) )
# We can optionally add information here to support packaging of code, etc. # We can optionally add information here to support packaging of code, etc.