diff --git a/docs/resources/llama-stack-spec.html b/docs/resources/llama-stack-spec.html
index 14e311cfc..9a9a29439 100644
--- a/docs/resources/llama-stack-spec.html
+++ b/docs/resources/llama-stack-spec.html
@@ -4926,6 +4926,15 @@
"config"
]
},
+ "AggregationFunctionType": {
+ "type": "string",
+ "enum": [
+ "average",
+ "median",
+ "categorical_count",
+ "accuracy"
+ ]
+ },
"AppEvalTaskConfig": {
"type": "object",
"properties": {
@@ -4953,6 +4962,9 @@
},
{
"$ref": "#/components/schemas/RegexParserScoringFnParams"
+ },
+ {
+ "$ref": "#/components/schemas/BasicScoringFnParams"
}
]
}
@@ -4968,6 +4980,26 @@
"scoring_params"
]
},
+ "BasicScoringFnParams": {
+ "type": "object",
+ "properties": {
+ "type": {
+ "type": "string",
+ "const": "basic",
+ "default": "basic"
+ },
+ "aggregation_functions": {
+ "type": "array",
+ "items": {
+ "$ref": "#/components/schemas/AggregationFunctionType"
+ }
+ }
+ },
+ "additionalProperties": false,
+ "required": [
+ "type"
+ ]
+ },
"BenchmarkEvalTaskConfig": {
"type": "object",
"properties": {
@@ -5015,6 +5047,12 @@
"items": {
"type": "string"
}
+ },
+ "aggregation_functions": {
+ "type": "array",
+ "items": {
+ "$ref": "#/components/schemas/AggregationFunctionType"
+ }
}
},
"additionalProperties": false,
@@ -5061,6 +5099,12 @@
"items": {
"type": "string"
}
+ },
+ "aggregation_functions": {
+ "type": "array",
+ "items": {
+ "$ref": "#/components/schemas/AggregationFunctionType"
+ }
}
},
"additionalProperties": false,
@@ -6014,6 +6058,9 @@
},
{
"$ref": "#/components/schemas/RegexParserScoringFnParams"
+ },
+ {
+ "$ref": "#/components/schemas/BasicScoringFnParams"
}
]
}
@@ -7771,6 +7818,9 @@
},
{
"$ref": "#/components/schemas/RegexParserScoringFnParams"
+ },
+ {
+ "$ref": "#/components/schemas/BasicScoringFnParams"
}
]
}
@@ -7998,6 +8048,9 @@
},
{
"$ref": "#/components/schemas/RegexParserScoringFnParams"
+ },
+ {
+ "$ref": "#/components/schemas/BasicScoringFnParams"
}
]
},
@@ -8046,6 +8099,9 @@
},
{
"$ref": "#/components/schemas/RegexParserScoringFnParams"
+ },
+ {
+ "$ref": "#/components/schemas/BasicScoringFnParams"
}
]
},
@@ -8491,6 +8547,10 @@
{
"name": "Agents"
},
+ {
+ "name": "AggregationFunctionType",
+ "description": ""
+ },
{
"name": "AppEvalTaskConfig",
"description": ""
@@ -8503,6 +8563,10 @@
"name": "Attachment",
"description": ""
},
+ {
+ "name": "BasicScoringFnParams",
+ "description": ""
+ },
{
"name": "BatchChatCompletionRequest",
"description": ""
@@ -9146,9 +9210,11 @@
"AgentTurnResponseStreamChunk",
"AgentTurnResponseTurnCompletePayload",
"AgentTurnResponseTurnStartPayload",
+ "AggregationFunctionType",
"AppEvalTaskConfig",
"AppendRowsRequest",
"Attachment",
+ "BasicScoringFnParams",
"BatchChatCompletionRequest",
"BatchChatCompletionResponse",
"BatchCompletionRequest",
diff --git a/docs/resources/llama-stack-spec.yaml b/docs/resources/llama-stack-spec.yaml
index 86fcae23d..a1cd08387 100644
--- a/docs/resources/llama-stack-spec.yaml
+++ b/docs/resources/llama-stack-spec.yaml
@@ -216,6 +216,13 @@ components:
- event_type
- turn_id
type: object
+ AggregationFunctionType:
+ enum:
+ - average
+ - median
+ - categorical_count
+ - accuracy
+ type: string
AppEvalTaskConfig:
additionalProperties: false
properties:
@@ -230,6 +237,7 @@ components:
oneOf:
- $ref: '#/components/schemas/LLMAsJudgeScoringFnParams'
- $ref: '#/components/schemas/RegexParserScoringFnParams'
+ - $ref: '#/components/schemas/BasicScoringFnParams'
type: object
type:
const: app
@@ -280,6 +288,20 @@ components:
- content
- mime_type
type: object
+ BasicScoringFnParams:
+ additionalProperties: false
+ properties:
+ aggregation_functions:
+ items:
+ $ref: '#/components/schemas/AggregationFunctionType'
+ type: array
+ type:
+ const: basic
+ default: basic
+ type: string
+ required:
+ - type
+ type: object
BatchChatCompletionRequest:
additionalProperties: false
properties:
@@ -1280,6 +1302,10 @@ components:
LLMAsJudgeScoringFnParams:
additionalProperties: false
properties:
+ aggregation_functions:
+ items:
+ $ref: '#/components/schemas/AggregationFunctionType'
+ type: array
judge_model:
type: string
judge_score_regexes:
@@ -1984,6 +2010,10 @@ components:
RegexParserScoringFnParams:
additionalProperties: false
properties:
+ aggregation_functions:
+ items:
+ $ref: '#/components/schemas/AggregationFunctionType'
+ type: array
parsing_regexes:
items:
type: string
@@ -2195,6 +2225,7 @@ components:
oneOf:
- $ref: '#/components/schemas/LLMAsJudgeScoringFnParams'
- $ref: '#/components/schemas/RegexParserScoringFnParams'
+ - $ref: '#/components/schemas/BasicScoringFnParams'
provider_id:
type: string
provider_scoring_fn_id:
@@ -2515,6 +2546,7 @@ components:
- oneOf:
- $ref: '#/components/schemas/LLMAsJudgeScoringFnParams'
- $ref: '#/components/schemas/RegexParserScoringFnParams'
+ - $ref: '#/components/schemas/BasicScoringFnParams'
- type: 'null'
type: object
required:
@@ -2555,6 +2587,7 @@ components:
- oneOf:
- $ref: '#/components/schemas/LLMAsJudgeScoringFnParams'
- $ref: '#/components/schemas/RegexParserScoringFnParams'
+ - $ref: '#/components/schemas/BasicScoringFnParams'
- type: 'null'
type: object
required:
@@ -2592,6 +2625,7 @@ components:
oneOf:
- $ref: '#/components/schemas/LLMAsJudgeScoringFnParams'
- $ref: '#/components/schemas/RegexParserScoringFnParams'
+ - $ref: '#/components/schemas/BasicScoringFnParams'
provider_id:
type: string
provider_resource_id:
@@ -5161,6 +5195,9 @@ tags:
/>
name: AgentTurnResponseTurnStartPayload
- name: Agents
+- description:
+ name: AggregationFunctionType
- description:
name: AppEvalTaskConfig
@@ -5169,6 +5206,9 @@ tags:
name: AppendRowsRequest
- description:
name: Attachment
+- description:
+ name: BasicScoringFnParams
- description:
name: BatchChatCompletionRequest
@@ -5636,9 +5676,11 @@ x-tagGroups:
- AgentTurnResponseStreamChunk
- AgentTurnResponseTurnCompletePayload
- AgentTurnResponseTurnStartPayload
+ - AggregationFunctionType
- AppEvalTaskConfig
- AppendRowsRequest
- Attachment
+ - BasicScoringFnParams
- BatchChatCompletionRequest
- BatchChatCompletionResponse
- BatchCompletionRequest