phoenix-oss/llama-stack
1
0
Fork 0
forked from phoenix-oss/llama-stack-mirror
Code Pull requests Releases Packages 2 Activity Actions

remove aggregation functions

Browse source
This commit is contained in:
Xi Yan 2025-03-23 16:17:09 -07:00
parent 64388de068
commit 2723b05164
3 changed files with 7 additions and 262 deletions
Download patch file Download diff file Expand all files Collapse all files

142
docs/_static/llama-stack-spec.html vendored
View file

@ -6505,36 +6505,11 @@
"type": "string",
"const": "equality",
"default": "equality"
},
"equality": {
"type": "object",
"properties": {
"aggregation_functions": {
"type": "array",
"items": {
"type": "string",
"enum": [
"average",
"median",
"categorical_count",
"accuracy"
],
"title": "AggregationFunctionType",
"description": "A type of aggregation function."
}
}
},
"additionalProperties": false,
"required": [
"aggregation_functions"
],
"title": "BasicGraderParams"
}
},
"additionalProperties": false,
"required": [
"type",
"equality"
"type"
],
"title": "EqualityGrader"
},
@ -6545,36 +6520,11 @@
"type": "string",
"const": "factuality",
"default": "factuality"
},
"factuality": {
"type": "object",
"properties": {
"aggregation_functions": {
"type": "array",
"items": {
"type": "string",
"enum": [
"average",
"median",
"categorical_count",
"accuracy"
],
"title": "AggregationFunctionType",
"description": "A type of aggregation function."
}
}
},
"additionalProperties": false,
"required": [
"aggregation_functions"
],
"title": "BasicGraderParams"
}
},
"additionalProperties": false,
"required": [
"type",
"factuality"
"type"
],
"title": "FactualityGrader"
},
@ -6585,36 +6535,11 @@
"type": "string",
"const": "faithfulness",
"default": "faithfulness"
},
"faithfulness": {
"type": "object",
"properties": {
"aggregation_functions": {
"type": "array",
"items": {
"type": "string",
"enum": [
"average",
"median",
"categorical_count",
"accuracy"
],
"title": "AggregationFunctionType",
"description": "A type of aggregation function."
}
}
},
"additionalProperties": false,
"required": [
"aggregation_functions"
],
"title": "BasicGraderParams"
}
},
"additionalProperties": false,
"required": [
"type",
"faithfulness"
"type"
],
"title": "FaithfulnessGrader"
},
@ -6733,28 +6658,13 @@
"items": {
"type": "string"
}
},
"aggregation_functions": {
"type": "array",
"items": {
"type": "string",
"enum": [
"average",
"median",
"categorical_count",
"accuracy"
],
"title": "AggregationFunctionType",
"description": "A type of aggregation function."
}
}
},
"additionalProperties": false,
"required": [
"model",
"prompt",
"score_regexes",
"aggregation_functions"
"score_regexes"
],
"title": "LlmGraderParams"
}
@ -6782,26 +6692,11 @@
"items": {
"type": "string"
}
},
"aggregation_functions": {
"type": "array",
"items": {
"type": "string",
"enum": [
"average",
"median",
"categorical_count",
"accuracy"
],
"title": "AggregationFunctionType",
"description": "A type of aggregation function."
}
}
},
"additionalProperties": false,
"required": [
"parsing_regexes",
"aggregation_functions"
"parsing_regexes"
],
"title": "RegexParserGraderParams"
}
@ -6820,36 +6715,11 @@
"type": "string",
"const": "subset_of",
"default": "subset_of"
},
"subset_of": {
"type": "object",
"properties": {
"aggregation_functions": {
"type": "array",
"items": {
"type": "string",
"enum": [
"average",
"median",
"categorical_count",
"accuracy"
],
"title": "AggregationFunctionType",
"description": "A type of aggregation function."
}
}
},
"additionalProperties": false,
"required": [
"aggregation_functions"
],
"title": "BasicGraderParams"
}
},
"additionalProperties": false,
"required": [
"type",
"subset_of"
"type"
],
"title": "SubsetOfGrader"
},

100
docs/_static/llama-stack-spec.yaml vendored
View file

@ -4557,28 +4557,9 @@ components:
type: string
const: equality
default: equality
equality:
type: object
properties:
aggregation_functions:
type: array
items:
type: string
enum:
- average
- median
- categorical_count
- accuracy
title: AggregationFunctionType
description: A type of aggregation function.
additionalProperties: false
required:
- aggregation_functions
title: BasicGraderParams
additionalProperties: false
required:
- type
- equality
title: EqualityGrader
FactualityGrader:
type: object
@ -4587,28 +4568,9 @@ components:
type: string
const: factuality
default: factuality
factuality:
type: object
properties:
aggregation_functions:
type: array
items:
type: string
enum:
- average
- median
- categorical_count
- accuracy
title: AggregationFunctionType
description: A type of aggregation function.
additionalProperties: false
required:
- aggregation_functions
title: BasicGraderParams
additionalProperties: false
required:
- type
- factuality
title: FactualityGrader
FaithfulnessGrader:
type: object
@ -4617,28 +4579,9 @@ components:
type: string
const: faithfulness
default: faithfulness
faithfulness:
type: object
properties:
aggregation_functions:
type: array
items:
type: string
enum:
- average
- median
- categorical_count
- accuracy
title: AggregationFunctionType
description: A type of aggregation function.
additionalProperties: false
required:
- aggregation_functions
title: BasicGraderParams
additionalProperties: false
required:
- type
- faithfulness
title: FaithfulnessGrader
Grader:
type: object
@ -4711,23 +4654,11 @@ components:
type: array
items:
type: string
aggregation_functions:
type: array
items:
type: string
enum:
- average
- median
- categorical_count
- accuracy
title: AggregationFunctionType
description: A type of aggregation function.
additionalProperties: false
required:
- model
- prompt
- score_regexes
- aggregation_functions
title: LlmGraderParams
additionalProperties: false
required:
@ -4748,21 +4679,9 @@ components:
type: array
items:
type: string
aggregation_functions:
type: array
items:
type: string
enum:
- average
- median
- categorical_count
- accuracy
title: AggregationFunctionType
description: A type of aggregation function.
additionalProperties: false
required:
- parsing_regexes
- aggregation_functions
title: RegexParserGraderParams
additionalProperties: false
required:
@ -4776,28 +4695,9 @@ components:
type: string
const: subset_of
default: subset_of
subset_of:
type: object
properties:
aggregation_functions:
type: array
items:
type: string
enum:
- average
- median
- categorical_count
- accuracy
title: AggregationFunctionType
description: A type of aggregation function.
additionalProperties: false
required:
- aggregation_functions
title: BasicGraderParams
additionalProperties: false
required:
- type
- subset_of
title: SubsetOfGrader
Model:
type: object

27
llama_stack/apis/graders/graders.py
View file

@ -13,8 +13,8 @@ from typing import (
Literal,
Optional,
Protocol,
Union,
runtime_checkable,
Union,
)
from pydantic import BaseModel, Field
@ -63,35 +63,14 @@ class GraderTypeInfo(BaseModel):
)
class AggregationFunctionType(Enum):
"""
A type of aggregation function.
:cvar average: Average the scores of each row.
:cvar median: Median the scores of each row.
:cvar categorical_count: Count the number of rows that match each category.
:cvar accuracy: Number of correct results over total results.
"""
average = "average"
median = "median"
categorical_count = "categorical_count"
accuracy = "accuracy"
class BasicGraderParams(BaseModel):
aggregation_functions: List[AggregationFunctionType]
class LlmGraderParams(BaseModel):
model: str
prompt: str
score_regexes: List[str]
aggregation_functions: List[AggregationFunctionType]
class RegexParserGraderParams(BaseModel):
parsing_regexes: List[str]
aggregation_functions: List[AggregationFunctionType]
@json_schema_type
@ -109,25 +88,21 @@ class RegexParserGrader(BaseModel):
@json_schema_type
class EqualityGrader(BaseModel):
type: Literal["equality"] = "equality"
equality: BasicGraderParams
@json_schema_type
class SubsetOfGrader(BaseModel):
type: Literal["subset_of"] = "subset_of"
subset_of: BasicGraderParams
@json_schema_type
class FactualityGrader(BaseModel):
type: Literal["factuality"] = "factuality"
factuality: BasicGraderParams
@json_schema_type
class FaithfulnessGrader(BaseModel):
type: Literal["faithfulness"] = "faithfulness"
faithfulness: BasicGraderParams
GraderDefinition = register_schema(