phoenix-oss/llama-stack-mirror
0
0
Fork 1
mirror of https://github.com/meta-llama/llama-stack.git synced 2025-07-30 23:51:00 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

scoring fn api

Browse source
This commit is contained in:
Xi Yan 2024-11-05 14:34:56 -08:00
parent 4fc92e52d7
commit fe91608321
1 changed files with 46 additions and 16 deletions
Download patch file Download diff file Expand all files Collapse all files

62
llama_stack/apis/scoring_functions/scoring_functions.py
View file

@ -4,32 +4,66 @@
# This source code is licensed under the terms described in the LICENSE file in # This source code is licensed under the terms described in the LICENSE file in
# the root directory of this source tree. # the root directory of this source tree.
from typing import Any, Dict, List, Literal, Optional, Protocol, runtime_checkable from enum import Enum
from typing import (
Any,
Dict,
List,
Literal,
Optional,
Protocol,
runtime_checkable,
Union,
)
from llama_models.schema_utils import json_schema_type, webmethod from llama_models.schema_utils import json_schema_type, webmethod
from pydantic import BaseModel, Field from pydantic import BaseModel, Field
from typing_extensions import Annotated
from llama_stack.apis.common.type_system import ParamType from llama_stack.apis.common.type_system import ParamType
@json_schema_type
class Parameter(BaseModel):
name: str
type: ParamType
description: Optional[str] = None
# Perhaps more structure can be imposed on these functions. Maybe they could be associated # Perhaps more structure can be imposed on these functions. Maybe they could be associated
# with standard metrics so they can be rolled up? # with standard metrics so they can be rolled up?
@json_schema_type
class ScoringContextType(Enum):
llm_as_judge = "llm_as_judge"
answer_parsing = "answer_parsing"
@json_schema_type
class LLMAsJudgeContext(BaseModel): class LLMAsJudgeContext(BaseModel):
type: Literal[ScoringContextType.llm_as_judge.value] = ( # type: ignore
ScoringContextType.llm_as_judge.value
)
judge_model: str judge_model: str
prompt_template: Optional[str] = None prompt_template: Optional[str] = None
judge_score_regex: Optional[List[str]] = Field( judge_score_regex: Optional[List[str]] = Field()
description="Regex to extract the score from the judge response",
default=None,
@json_schema_type
class AnswerParsingContext(BaseModel):
type: Literal[ScoringContextType.answer_parsing.value] = ( # type: ignore
ScoringContextType.answer_parsing.value
) )
parsing_regex: Optional[List[str]] = Field(
description="Regex to extract the answer from generated response",
default_factory=list,
)
ScoringContext = Annotated[
Union[
LLMAsJudgeContext,
AnswerParsingContext,
],
Field(discriminator="type"),
]
@json_schema_type
class ScoringFnConfig(BaseModel):
scoring_context: ScoringContext # type: ignore
@json_schema_type @json_schema_type
@ -40,14 +74,10 @@ class ScoringFnDef(BaseModel):
default_factory=dict, default_factory=dict,
description="Any additional metadata for this definition", description="Any additional metadata for this definition",
) )
parameters: List[Parameter] = Field(
description="List of parameters for the deterministic function",
default_factory=list,
)
return_type: ParamType = Field( return_type: ParamType = Field(
description="The return type of the deterministic function", description="The return type of the deterministic function",
) )
context: Optional[LLMAsJudgeContext] = None context: Optional[ScoringContext] = None # type: ignore
# We can optionally add information here to support packaging of code, etc. # We can optionally add information here to support packaging of code, etc.