From 2723b051640902b691b5091fa55391b8c2241958 Mon Sep 17 00:00:00 2001 From: Xi Yan Date: Sun, 23 Mar 2025 16:17:09 -0700 Subject: [PATCH] remove aggregation functions --- docs/_static/llama-stack-spec.html | 142 ++-------------------------- docs/_static/llama-stack-spec.yaml | 100 -------------------- llama_stack/apis/graders/graders.py | 27 +----- 3 files changed, 7 insertions(+), 262 deletions(-) diff --git a/docs/_static/llama-stack-spec.html b/docs/_static/llama-stack-spec.html index ab73dc345..d0b2f8aec 100644 --- a/docs/_static/llama-stack-spec.html +++ b/docs/_static/llama-stack-spec.html @@ -6505,36 +6505,11 @@ "type": "string", "const": "equality", "default": "equality" - }, - "equality": { - "type": "object", - "properties": { - "aggregation_functions": { - "type": "array", - "items": { - "type": "string", - "enum": [ - "average", - "median", - "categorical_count", - "accuracy" - ], - "title": "AggregationFunctionType", - "description": "A type of aggregation function." - } - } - }, - "additionalProperties": false, - "required": [ - "aggregation_functions" - ], - "title": "BasicGraderParams" } }, "additionalProperties": false, "required": [ - "type", - "equality" + "type" ], "title": "EqualityGrader" }, @@ -6545,36 +6520,11 @@ "type": "string", "const": "factuality", "default": "factuality" - }, - "factuality": { - "type": "object", - "properties": { - "aggregation_functions": { - "type": "array", - "items": { - "type": "string", - "enum": [ - "average", - "median", - "categorical_count", - "accuracy" - ], - "title": "AggregationFunctionType", - "description": "A type of aggregation function." - } - } - }, - "additionalProperties": false, - "required": [ - "aggregation_functions" - ], - "title": "BasicGraderParams" } }, "additionalProperties": false, "required": [ - "type", - "factuality" + "type" ], "title": "FactualityGrader" }, @@ -6585,36 +6535,11 @@ "type": "string", "const": "faithfulness", "default": "faithfulness" - }, - "faithfulness": { - "type": "object", - "properties": { - "aggregation_functions": { - "type": "array", - "items": { - "type": "string", - "enum": [ - "average", - "median", - "categorical_count", - "accuracy" - ], - "title": "AggregationFunctionType", - "description": "A type of aggregation function." - } - } - }, - "additionalProperties": false, - "required": [ - "aggregation_functions" - ], - "title": "BasicGraderParams" } }, "additionalProperties": false, "required": [ - "type", - "faithfulness" + "type" ], "title": "FaithfulnessGrader" }, @@ -6733,28 +6658,13 @@ "items": { "type": "string" } - }, - "aggregation_functions": { - "type": "array", - "items": { - "type": "string", - "enum": [ - "average", - "median", - "categorical_count", - "accuracy" - ], - "title": "AggregationFunctionType", - "description": "A type of aggregation function." - } } }, "additionalProperties": false, "required": [ "model", "prompt", - "score_regexes", - "aggregation_functions" + "score_regexes" ], "title": "LlmGraderParams" } @@ -6782,26 +6692,11 @@ "items": { "type": "string" } - }, - "aggregation_functions": { - "type": "array", - "items": { - "type": "string", - "enum": [ - "average", - "median", - "categorical_count", - "accuracy" - ], - "title": "AggregationFunctionType", - "description": "A type of aggregation function." - } } }, "additionalProperties": false, "required": [ - "parsing_regexes", - "aggregation_functions" + "parsing_regexes" ], "title": "RegexParserGraderParams" } @@ -6820,36 +6715,11 @@ "type": "string", "const": "subset_of", "default": "subset_of" - }, - "subset_of": { - "type": "object", - "properties": { - "aggregation_functions": { - "type": "array", - "items": { - "type": "string", - "enum": [ - "average", - "median", - "categorical_count", - "accuracy" - ], - "title": "AggregationFunctionType", - "description": "A type of aggregation function." - } - } - }, - "additionalProperties": false, - "required": [ - "aggregation_functions" - ], - "title": "BasicGraderParams" } }, "additionalProperties": false, "required": [ - "type", - "subset_of" + "type" ], "title": "SubsetOfGrader" }, diff --git a/docs/_static/llama-stack-spec.yaml b/docs/_static/llama-stack-spec.yaml index 66044bb65..86e9c1c36 100644 --- a/docs/_static/llama-stack-spec.yaml +++ b/docs/_static/llama-stack-spec.yaml @@ -4557,28 +4557,9 @@ components: type: string const: equality default: equality - equality: - type: object - properties: - aggregation_functions: - type: array - items: - type: string - enum: - - average - - median - - categorical_count - - accuracy - title: AggregationFunctionType - description: A type of aggregation function. - additionalProperties: false - required: - - aggregation_functions - title: BasicGraderParams additionalProperties: false required: - type - - equality title: EqualityGrader FactualityGrader: type: object @@ -4587,28 +4568,9 @@ components: type: string const: factuality default: factuality - factuality: - type: object - properties: - aggregation_functions: - type: array - items: - type: string - enum: - - average - - median - - categorical_count - - accuracy - title: AggregationFunctionType - description: A type of aggregation function. - additionalProperties: false - required: - - aggregation_functions - title: BasicGraderParams additionalProperties: false required: - type - - factuality title: FactualityGrader FaithfulnessGrader: type: object @@ -4617,28 +4579,9 @@ components: type: string const: faithfulness default: faithfulness - faithfulness: - type: object - properties: - aggregation_functions: - type: array - items: - type: string - enum: - - average - - median - - categorical_count - - accuracy - title: AggregationFunctionType - description: A type of aggregation function. - additionalProperties: false - required: - - aggregation_functions - title: BasicGraderParams additionalProperties: false required: - type - - faithfulness title: FaithfulnessGrader Grader: type: object @@ -4711,23 +4654,11 @@ components: type: array items: type: string - aggregation_functions: - type: array - items: - type: string - enum: - - average - - median - - categorical_count - - accuracy - title: AggregationFunctionType - description: A type of aggregation function. additionalProperties: false required: - model - prompt - score_regexes - - aggregation_functions title: LlmGraderParams additionalProperties: false required: @@ -4748,21 +4679,9 @@ components: type: array items: type: string - aggregation_functions: - type: array - items: - type: string - enum: - - average - - median - - categorical_count - - accuracy - title: AggregationFunctionType - description: A type of aggregation function. additionalProperties: false required: - parsing_regexes - - aggregation_functions title: RegexParserGraderParams additionalProperties: false required: @@ -4776,28 +4695,9 @@ components: type: string const: subset_of default: subset_of - subset_of: - type: object - properties: - aggregation_functions: - type: array - items: - type: string - enum: - - average - - median - - categorical_count - - accuracy - title: AggregationFunctionType - description: A type of aggregation function. - additionalProperties: false - required: - - aggregation_functions - title: BasicGraderParams additionalProperties: false required: - type - - subset_of title: SubsetOfGrader Model: type: object diff --git a/llama_stack/apis/graders/graders.py b/llama_stack/apis/graders/graders.py index 31e03b6d1..76abf1998 100644 --- a/llama_stack/apis/graders/graders.py +++ b/llama_stack/apis/graders/graders.py @@ -13,8 +13,8 @@ from typing import ( Literal, Optional, Protocol, - Union, runtime_checkable, + Union, ) from pydantic import BaseModel, Field @@ -63,35 +63,14 @@ class GraderTypeInfo(BaseModel): ) -class AggregationFunctionType(Enum): - """ - A type of aggregation function. - :cvar average: Average the scores of each row. - :cvar median: Median the scores of each row. - :cvar categorical_count: Count the number of rows that match each category. - :cvar accuracy: Number of correct results over total results. - """ - - average = "average" - median = "median" - categorical_count = "categorical_count" - accuracy = "accuracy" - - -class BasicGraderParams(BaseModel): - aggregation_functions: List[AggregationFunctionType] - - class LlmGraderParams(BaseModel): model: str prompt: str score_regexes: List[str] - aggregation_functions: List[AggregationFunctionType] class RegexParserGraderParams(BaseModel): parsing_regexes: List[str] - aggregation_functions: List[AggregationFunctionType] @json_schema_type @@ -109,25 +88,21 @@ class RegexParserGrader(BaseModel): @json_schema_type class EqualityGrader(BaseModel): type: Literal["equality"] = "equality" - equality: BasicGraderParams @json_schema_type class SubsetOfGrader(BaseModel): type: Literal["subset_of"] = "subset_of" - subset_of: BasicGraderParams @json_schema_type class FactualityGrader(BaseModel): type: Literal["factuality"] = "factuality" - factuality: BasicGraderParams @json_schema_type class FaithfulnessGrader(BaseModel): type: Literal["faithfulness"] = "faithfulness" - faithfulness: BasicGraderParams GraderDefinition = register_schema(