phoenix-oss/llama-stack-mirror
0
0
Fork 1
mirror of https://github.com/meta-llama/llama-stack.git synced 2026-01-01 17:44:32 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge branch 'eval_api_final' into delete_eval_scoring_scoring_fn

Browse source
This commit is contained in:
Xi Yan 2025-03-19 09:50:40 -07:00
commit e23531c9d0
4 changed files with 50 additions and 38 deletions
Download patch file Download diff file Expand all files Collapse all files

37
docs/_static/llama-stack-spec.html vendored
View file

@ -7617,7 +7617,7 @@
"EvaluationResponse": {
"type": "object",
"properties": {
"generations": {
"result_rows": {
"type": "array",
"items": {
"type": "object",
@ -7644,20 +7644,39 @@
]
}
},
"description": "The generations in rows for the evaluation."
"description": "The result data containing inputs, generations and grades in each row."
},
"scores": {
"grades": {
"type": "object",
"additionalProperties": {
"$ref": "#/components/schemas/ScoringResult"
"oneOf": [
{
"type": "null"
},
{
"type": "boolean"
},
{
"type": "number"
},
{
"type": "string"
},
{
"type": "array"
},
{
"type": "object"
}
]
},
"description": "The scores for the evaluation. Map of grader id to ScoringResult."
"description": "Map of grader id to aggregated value."
}
},
"additionalProperties": false,
"required": [
"generations",
"scores"
"result_rows",
"grades"
],
"title": "EvaluationResponse",
"description": "A response to an inline evaluation."
@ -9313,14 +9332,14 @@
"properties": {
"dataset_id": {
"type": "string",
"description": "The ID of the dataset to be used to run the benchmark."
"description": "The ID of the dataset to be used to run the benchmark. ID obtained through `datasets.register()`"
},
"grader_ids": {
"type": "array",
"items": {
"type": "string"
},
"description": "List of grader ids to use for this benchmark."
"description": "List of grader ids to use for this benchmark. ID obtained through `graders.register()`"
},
"benchmark_id": {
"type": "string",

26
docs/_static/llama-stack-spec.yaml vendored
View file

@ -5328,7 +5328,7 @@ components:
EvaluationResponse:
type: object
properties:
generations:
result_rows:
type: array
items:
type: object
@ -5341,17 +5341,22 @@ components:
- type: array
- type: object
description: >-
The generations in rows for the evaluation.
scores:
The result data containing inputs, generations and grades in each row.
grades:
type: object
additionalProperties:
$ref: '#/components/schemas/ScoringResult'
description: >-
The scores for the evaluation. Map of grader id to ScoringResult.
oneOf:
- type: 'null'
- type: boolean
- type: number
- type: string
- type: array
- type: object
description: Map of grader id to aggregated value.
additionalProperties: false
required:
- generations
- scores
- result_rows
- grades
title: EvaluationResponse
description: A response to an inline evaluation.
ScoringResult:
@ -6404,13 +6409,14 @@ components:
dataset_id:
type: string
description: >-
The ID of the dataset to be used to run the benchmark.
The ID of the dataset to be used to run the benchmark. ID obtained through
`datasets.register()`
grader_ids:
type: array
items:
type: string
description: >-
List of grader ids to use for this benchmark.
List of grader ids to use for this benchmark. ID obtained through `graders.register()`
benchmark_id:
type: string
description: >-