diff --git a/docs/_static/llama-stack-spec.html b/docs/_static/llama-stack-spec.html
index ab73dc345..d0b2f8aec 100644
--- a/docs/_static/llama-stack-spec.html
+++ b/docs/_static/llama-stack-spec.html
@@ -6505,36 +6505,11 @@
"type": "string",
"const": "equality",
"default": "equality"
- },
- "equality": {
- "type": "object",
- "properties": {
- "aggregation_functions": {
- "type": "array",
- "items": {
- "type": "string",
- "enum": [
- "average",
- "median",
- "categorical_count",
- "accuracy"
- ],
- "title": "AggregationFunctionType",
- "description": "A type of aggregation function."
- }
- }
- },
- "additionalProperties": false,
- "required": [
- "aggregation_functions"
- ],
- "title": "BasicGraderParams"
}
},
"additionalProperties": false,
"required": [
- "type",
- "equality"
+ "type"
],
"title": "EqualityGrader"
},
@@ -6545,36 +6520,11 @@
"type": "string",
"const": "factuality",
"default": "factuality"
- },
- "factuality": {
- "type": "object",
- "properties": {
- "aggregation_functions": {
- "type": "array",
- "items": {
- "type": "string",
- "enum": [
- "average",
- "median",
- "categorical_count",
- "accuracy"
- ],
- "title": "AggregationFunctionType",
- "description": "A type of aggregation function."
- }
- }
- },
- "additionalProperties": false,
- "required": [
- "aggregation_functions"
- ],
- "title": "BasicGraderParams"
}
},
"additionalProperties": false,
"required": [
- "type",
- "factuality"
+ "type"
],
"title": "FactualityGrader"
},
@@ -6585,36 +6535,11 @@
"type": "string",
"const": "faithfulness",
"default": "faithfulness"
- },
- "faithfulness": {
- "type": "object",
- "properties": {
- "aggregation_functions": {
- "type": "array",
- "items": {
- "type": "string",
- "enum": [
- "average",
- "median",
- "categorical_count",
- "accuracy"
- ],
- "title": "AggregationFunctionType",
- "description": "A type of aggregation function."
- }
- }
- },
- "additionalProperties": false,
- "required": [
- "aggregation_functions"
- ],
- "title": "BasicGraderParams"
}
},
"additionalProperties": false,
"required": [
- "type",
- "faithfulness"
+ "type"
],
"title": "FaithfulnessGrader"
},
@@ -6733,28 +6658,13 @@
"items": {
"type": "string"
}
- },
- "aggregation_functions": {
- "type": "array",
- "items": {
- "type": "string",
- "enum": [
- "average",
- "median",
- "categorical_count",
- "accuracy"
- ],
- "title": "AggregationFunctionType",
- "description": "A type of aggregation function."
- }
}
},
"additionalProperties": false,
"required": [
"model",
"prompt",
- "score_regexes",
- "aggregation_functions"
+ "score_regexes"
],
"title": "LlmGraderParams"
}
@@ -6782,26 +6692,11 @@
"items": {
"type": "string"
}
- },
- "aggregation_functions": {
- "type": "array",
- "items": {
- "type": "string",
- "enum": [
- "average",
- "median",
- "categorical_count",
- "accuracy"
- ],
- "title": "AggregationFunctionType",
- "description": "A type of aggregation function."
- }
}
},
"additionalProperties": false,
"required": [
- "parsing_regexes",
- "aggregation_functions"
+ "parsing_regexes"
],
"title": "RegexParserGraderParams"
}
@@ -6820,36 +6715,11 @@
"type": "string",
"const": "subset_of",
"default": "subset_of"
- },
- "subset_of": {
- "type": "object",
- "properties": {
- "aggregation_functions": {
- "type": "array",
- "items": {
- "type": "string",
- "enum": [
- "average",
- "median",
- "categorical_count",
- "accuracy"
- ],
- "title": "AggregationFunctionType",
- "description": "A type of aggregation function."
- }
- }
- },
- "additionalProperties": false,
- "required": [
- "aggregation_functions"
- ],
- "title": "BasicGraderParams"
}
},
"additionalProperties": false,
"required": [
- "type",
- "subset_of"
+ "type"
],
"title": "SubsetOfGrader"
},
diff --git a/docs/_static/llama-stack-spec.yaml b/docs/_static/llama-stack-spec.yaml
index 66044bb65..86e9c1c36 100644
--- a/docs/_static/llama-stack-spec.yaml
+++ b/docs/_static/llama-stack-spec.yaml
@@ -4557,28 +4557,9 @@ components:
type: string
const: equality
default: equality
- equality:
- type: object
- properties:
- aggregation_functions:
- type: array
- items:
- type: string
- enum:
- - average
- - median
- - categorical_count
- - accuracy
- title: AggregationFunctionType
- description: A type of aggregation function.
- additionalProperties: false
- required:
- - aggregation_functions
- title: BasicGraderParams
additionalProperties: false
required:
- type
- - equality
title: EqualityGrader
FactualityGrader:
type: object
@@ -4587,28 +4568,9 @@ components:
type: string
const: factuality
default: factuality
- factuality:
- type: object
- properties:
- aggregation_functions:
- type: array
- items:
- type: string
- enum:
- - average
- - median
- - categorical_count
- - accuracy
- title: AggregationFunctionType
- description: A type of aggregation function.
- additionalProperties: false
- required:
- - aggregation_functions
- title: BasicGraderParams
additionalProperties: false
required:
- type
- - factuality
title: FactualityGrader
FaithfulnessGrader:
type: object
@@ -4617,28 +4579,9 @@ components:
type: string
const: faithfulness
default: faithfulness
- faithfulness:
- type: object
- properties:
- aggregation_functions:
- type: array
- items:
- type: string
- enum:
- - average
- - median
- - categorical_count
- - accuracy
- title: AggregationFunctionType
- description: A type of aggregation function.
- additionalProperties: false
- required:
- - aggregation_functions
- title: BasicGraderParams
additionalProperties: false
required:
- type
- - faithfulness
title: FaithfulnessGrader
Grader:
type: object
@@ -4711,23 +4654,11 @@ components:
type: array
items:
type: string
- aggregation_functions:
- type: array
- items:
- type: string
- enum:
- - average
- - median
- - categorical_count
- - accuracy
- title: AggregationFunctionType
- description: A type of aggregation function.
additionalProperties: false
required:
- model
- prompt
- score_regexes
- - aggregation_functions
title: LlmGraderParams
additionalProperties: false
required:
@@ -4748,21 +4679,9 @@ components:
type: array
items:
type: string
- aggregation_functions:
- type: array
- items:
- type: string
- enum:
- - average
- - median
- - categorical_count
- - accuracy
- title: AggregationFunctionType
- description: A type of aggregation function.
additionalProperties: false
required:
- parsing_regexes
- - aggregation_functions
title: RegexParserGraderParams
additionalProperties: false
required:
@@ -4776,28 +4695,9 @@ components:
type: string
const: subset_of
default: subset_of
- subset_of:
- type: object
- properties:
- aggregation_functions:
- type: array
- items:
- type: string
- enum:
- - average
- - median
- - categorical_count
- - accuracy
- title: AggregationFunctionType
- description: A type of aggregation function.
- additionalProperties: false
- required:
- - aggregation_functions
- title: BasicGraderParams
additionalProperties: false
required:
- type
- - subset_of
title: SubsetOfGrader
Model:
type: object
diff --git a/llama_stack/apis/graders/graders.py b/llama_stack/apis/graders/graders.py
index 31e03b6d1..76abf1998 100644
--- a/llama_stack/apis/graders/graders.py
+++ b/llama_stack/apis/graders/graders.py
@@ -13,8 +13,8 @@ from typing import (
Literal,
Optional,
Protocol,
- Union,
runtime_checkable,
+ Union,
)
from pydantic import BaseModel, Field
@@ -63,35 +63,14 @@ class GraderTypeInfo(BaseModel):
)
-class AggregationFunctionType(Enum):
- """
- A type of aggregation function.
- :cvar average: Average the scores of each row.
- :cvar median: Median the scores of each row.
- :cvar categorical_count: Count the number of rows that match each category.
- :cvar accuracy: Number of correct results over total results.
- """
-
- average = "average"
- median = "median"
- categorical_count = "categorical_count"
- accuracy = "accuracy"
-
-
-class BasicGraderParams(BaseModel):
- aggregation_functions: List[AggregationFunctionType]
-
-
class LlmGraderParams(BaseModel):
model: str
prompt: str
score_regexes: List[str]
- aggregation_functions: List[AggregationFunctionType]
class RegexParserGraderParams(BaseModel):
parsing_regexes: List[str]
- aggregation_functions: List[AggregationFunctionType]
@json_schema_type
@@ -109,25 +88,21 @@ class RegexParserGrader(BaseModel):
@json_schema_type
class EqualityGrader(BaseModel):
type: Literal["equality"] = "equality"
- equality: BasicGraderParams
@json_schema_type
class SubsetOfGrader(BaseModel):
type: Literal["subset_of"] = "subset_of"
- subset_of: BasicGraderParams
@json_schema_type
class FactualityGrader(BaseModel):
type: Literal["factuality"] = "factuality"
- factuality: BasicGraderParams
@json_schema_type
class FaithfulnessGrader(BaseModel):
type: Literal["faithfulness"] = "faithfulness"
- faithfulness: BasicGraderParams
GraderDefinition = register_schema(