phoenix-oss/llama-stack-mirror
0
0
Fork 1
mirror of https://github.com/meta-llama/llama-stack.git synced 2025-08-07 19:12:09 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

scoring api update

Browse source
This commit is contained in:
Xi Yan 2025-03-11 21:53:47 -07:00
parent bbb1947fb4
commit 2bb6ca818a
3 changed files with 141 additions and 383 deletions
Download patch file Download diff file Expand all files Collapse all files

301
docs/_static/llama-stack-spec.html vendored
View file

@ -2800,7 +2800,14 @@
"post": { "post": {
"responses": { "responses": {
"200": { "200": {
"description": "OK" "description": "OK",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/ScoringFn"
}
}
}
}, },
"400": { "400": {
"$ref": "#/components/responses/BadRequest400" "$ref": "#/components/responses/BadRequest400"
@ -2818,7 +2825,7 @@
"tags": [ "tags": [
"ScoringFunctions" "ScoringFunctions"
], ],
"description": "", "description": "Register a new scoring function with given parameters. Only valid scoring function type that can be parameterized can be registered.",
"parameters": [], "parameters": [],
"requestBody": { "requestBody": {
"content": { "content": {
@ -7148,175 +7155,6 @@
"title": "PaginatedRowsResult", "title": "PaginatedRowsResult",
"description": "A paginated list of rows from a dataset." "description": "A paginated list of rows from a dataset."
}, },
"AgentTurnInputType": {
"type": "object",
"properties": {
"type": {
"type": "string",
"const": "agent_turn_input",
"default": "agent_turn_input"
}
},
"additionalProperties": false,
"required": [
"type"
],
"title": "AgentTurnInputType"
},
"ArrayType": {
"type": "object",
"properties": {
"type": {
"type": "string",
"const": "array",
"default": "array"
}
},
"additionalProperties": false,
"required": [
"type"
],
"title": "ArrayType"
},
"BooleanType": {
"type": "object",
"properties": {
"type": {
"type": "string",
"const": "boolean",
"default": "boolean"
}
},
"additionalProperties": false,
"required": [
"type"
],
"title": "BooleanType"
},
"ChatCompletionInputType": {
"type": "object",
"properties": {
"type": {
"type": "string",
"const": "chat_completion_input",
"default": "chat_completion_input"
}
},
"additionalProperties": false,
"required": [
"type"
],
"title": "ChatCompletionInputType"
},
"CompletionInputType": {
"type": "object",
"properties": {
"type": {
"type": "string",
"const": "completion_input",
"default": "completion_input"
}
},
"additionalProperties": false,
"required": [
"type"
],
"title": "CompletionInputType"
},
"JsonType": {
"type": "object",
"properties": {
"type": {
"type": "string",
"const": "json",
"default": "json"
}
},
"additionalProperties": false,
"required": [
"type"
],
"title": "JsonType"
},
"NumberType": {
"type": "object",
"properties": {
"type": {
"type": "string",
"const": "number",
"default": "number"
}
},
"additionalProperties": false,
"required": [
"type"
],
"title": "NumberType"
},
"ObjectType": {
"type": "object",
"properties": {
"type": {
"type": "string",
"const": "object",
"default": "object"
}
},
"additionalProperties": false,
"required": [
"type"
],
"title": "ObjectType"
},
"ParamType": {
"oneOf": [
{
"$ref": "#/components/schemas/StringType"
},
{
"$ref": "#/components/schemas/NumberType"
},
{
"$ref": "#/components/schemas/BooleanType"
},
{
"$ref": "#/components/schemas/ArrayType"
},
{
"$ref": "#/components/schemas/ObjectType"
},
{
"$ref": "#/components/schemas/JsonType"
},
{
"$ref": "#/components/schemas/UnionType"
},
{
"$ref": "#/components/schemas/ChatCompletionInputType"
},
{
"$ref": "#/components/schemas/CompletionInputType"
},
{
"$ref": "#/components/schemas/AgentTurnInputType"
}
],
"discriminator": {
"propertyName": "type",
"mapping": {
"string": "#/components/schemas/StringType",
"number": "#/components/schemas/NumberType",
"boolean": "#/components/schemas/BooleanType",
"array": "#/components/schemas/ArrayType",
"object": "#/components/schemas/ObjectType",
"json": "#/components/schemas/JsonType",
"union": "#/components/schemas/UnionType",
"chat_completion_input": "#/components/schemas/ChatCompletionInputType",
"completion_input": "#/components/schemas/CompletionInputType",
"agent_turn_input": "#/components/schemas/AgentTurnInputType"
}
}
},
"ScoringFn": { "ScoringFn": {
"type": "object", "type": "object",
"properties": { "properties": {
@ -7334,9 +7172,15 @@
"const": "scoring_function", "const": "scoring_function",
"default": "scoring_function" "default": "scoring_function"
}, },
"scoring_fn_type": {
"$ref": "#/components/schemas/ScoringFunctionType"
},
"description": { "description": {
"type": "string" "type": "string"
}, },
"params": {
"$ref": "#/components/schemas/ScoringFnParams"
},
"metadata": { "metadata": {
"type": "object", "type": "object",
"additionalProperties": { "additionalProperties": {
@ -7361,12 +7205,6 @@
} }
] ]
} }
},
"return_type": {
"$ref": "#/components/schemas/ParamType"
},
"params": {
"$ref": "#/components/schemas/ScoringFnParams"
} }
}, },
"additionalProperties": false, "additionalProperties": false,
@ -7375,40 +7213,31 @@
"provider_resource_id", "provider_resource_id",
"provider_id", "provider_id",
"type", "type",
"metadata", "scoring_fn_type",
"return_type" "metadata"
], ],
"title": "ScoringFn" "title": "ScoringFn"
}, },
"StringType": { "ScoringFunctionType": {
"type": "object",
"properties": {
"type": {
"type": "string", "type": "string",
"const": "string", "enum": [
"default": "string" "custom_llm_as_judge",
} "regex_parser",
}, "regex_parser_math_response",
"additionalProperties": false, "equality",
"required": [ "subset_of",
"type" "factuality",
"faithfulness",
"answer_correctness",
"answer_relevancy",
"answer_similarity",
"context_entity_recall",
"context_precision",
"context_recall",
"context_relevancy"
], ],
"title": "StringType" "title": "ScoringFunctionType",
}, "description": "A type of scoring function. Each type is a criteria for evaluating answers."
"UnionType": {
"type": "object",
"properties": {
"type": {
"type": "string",
"const": "union",
"default": "union"
}
},
"additionalProperties": false,
"required": [
"type"
],
"title": "UnionType"
}, },
"Shield": { "Shield": {
"type": "object", "type": "object",
@ -9389,7 +9218,7 @@
}, },
"data_reference": { "data_reference": {
"$ref": "#/components/schemas/DataReference", "$ref": "#/components/schemas/DataReference",
"description": "The data reference of the dataset. Examples: - { \"type\": \"uri\", \"uri\": \"https://mywebsite.com/mydata.jsonl\" } - { \"type\": \"uri\", \"uri\": \"lsfs://mydata.jsonl\" } - { \"type\": \"huggingface\", \"dataset_path\": \"tatsu-lab/alpaca\", \"params\": { \"split\": \"train\" } } - { \"type\": \"rows\", \"rows\": [{\"message\": \"Hello, world!\"}] }" "description": "The data reference of the dataset. Examples: - { \"type\": \"uri\", \"uri\": \"https://mywebsite.com/mydata.jsonl\" } - { \"type\": \"uri\", \"uri\": \"lsfs://mydata.jsonl\" } - { \"type\": \"huggingface\", \"dataset_path\": \"tatsu-lab/alpaca\", \"params\": { \"split\": \"train\" } } - { \"type\": \"rows\", \"rows\": [ { \"messages\": [ {\"role\": \"user\", \"content\": \"Hello, world!\"}, {\"role\": \"assistant\", \"content\": \"Hello, world!\"}, ] } ] }"
}, },
"metadata": { "metadata": {
"type": "object", "type": "object",
@ -9479,30 +9308,52 @@
"RegisterScoringFunctionRequest": { "RegisterScoringFunctionRequest": {
"type": "object", "type": "object",
"properties": { "properties": {
"scoring_fn_id": { "scoring_fn_type": {
"type": "string" "$ref": "#/components/schemas/ScoringFunctionType",
}, "description": "The type of scoring function to register. A function type can only be registered if it is a valid type."
"description": {
"type": "string"
},
"return_type": {
"$ref": "#/components/schemas/ParamType"
},
"provider_scoring_fn_id": {
"type": "string"
},
"provider_id": {
"type": "string"
}, },
"params": { "params": {
"$ref": "#/components/schemas/ScoringFnParams" "$ref": "#/components/schemas/ScoringFnParams",
"description": "The parameters for the scoring function."
},
"scoring_fn_id": {
"type": "string",
"description": "(Optional) The ID of the scoring function to register. If not provided, a random ID will be generated."
},
"description": {
"type": "string",
"description": "(Optional) The description of the scoring function."
},
"metadata": {
"type": "object",
"additionalProperties": {
"oneOf": [
{
"type": "null"
},
{
"type": "boolean"
},
{
"type": "number"
},
{
"type": "string"
},
{
"type": "array"
},
{
"type": "object"
}
]
},
"description": "(Optional) Any additional metadata to be associated with the scoring function."
} }
}, },
"additionalProperties": false, "additionalProperties": false,
"required": [ "required": [
"scoring_fn_id", "scoring_fn_type"
"description",
"return_type"
], ],
"title": "RegisterScoringFunctionRequest" "title": "RegisterScoringFunctionRequest"
}, },

217
docs/_static/llama-stack-spec.yaml vendored
View file

@ -1892,6 +1892,10 @@ paths:
responses: responses:
'200': '200':
description: OK description: OK
content:
application/json:
schema:
$ref: '#/components/schemas/ScoringFn'
'400': '400':
$ref: '#/components/responses/BadRequest400' $ref: '#/components/responses/BadRequest400'
'429': '429':
@ -1904,7 +1908,9 @@ paths:
$ref: '#/components/responses/DefaultError' $ref: '#/components/responses/DefaultError'
tags: tags:
- ScoringFunctions - ScoringFunctions
description: '' description: >-
Register a new scoring function with given parameters. Only valid scoring
function type that can be parameterized can be registered.
parameters: [] parameters: []
requestBody: requestBody:
content: content:
@ -4916,119 +4922,6 @@ components:
- total_count - total_count
title: PaginatedRowsResult title: PaginatedRowsResult
description: A paginated list of rows from a dataset. description: A paginated list of rows from a dataset.
AgentTurnInputType:
type: object
properties:
type:
type: string
const: agent_turn_input
default: agent_turn_input
additionalProperties: false
required:
- type
title: AgentTurnInputType
ArrayType:
type: object
properties:
type:
type: string
const: array
default: array
additionalProperties: false
required:
- type
title: ArrayType
BooleanType:
type: object
properties:
type:
type: string
const: boolean
default: boolean
additionalProperties: false
required:
- type
title: BooleanType
ChatCompletionInputType:
type: object
properties:
type:
type: string
const: chat_completion_input
default: chat_completion_input
additionalProperties: false
required:
- type
title: ChatCompletionInputType
CompletionInputType:
type: object
properties:
type:
type: string
const: completion_input
default: completion_input
additionalProperties: false
required:
- type
title: CompletionInputType
JsonType:
type: object
properties:
type:
type: string
const: json
default: json
additionalProperties: false
required:
- type
title: JsonType
NumberType:
type: object
properties:
type:
type: string
const: number
default: number
additionalProperties: false
required:
- type
title: NumberType
ObjectType:
type: object
properties:
type:
type: string
const: object
default: object
additionalProperties: false
required:
- type
title: ObjectType
ParamType:
oneOf:
- $ref: '#/components/schemas/StringType'
- $ref: '#/components/schemas/NumberType'
- $ref: '#/components/schemas/BooleanType'
- $ref: '#/components/schemas/ArrayType'
- $ref: '#/components/schemas/ObjectType'
- $ref: '#/components/schemas/JsonType'
- $ref: '#/components/schemas/UnionType'
- $ref: '#/components/schemas/ChatCompletionInputType'
- $ref: '#/components/schemas/CompletionInputType'
- $ref: '#/components/schemas/AgentTurnInputType'
discriminator:
propertyName: type
mapping:
string: '#/components/schemas/StringType'
number: '#/components/schemas/NumberType'
boolean: '#/components/schemas/BooleanType'
array: '#/components/schemas/ArrayType'
object: '#/components/schemas/ObjectType'
json: '#/components/schemas/JsonType'
union: '#/components/schemas/UnionType'
chat_completion_input: '#/components/schemas/ChatCompletionInputType'
completion_input: '#/components/schemas/CompletionInputType'
agent_turn_input: '#/components/schemas/AgentTurnInputType'
ScoringFn: ScoringFn:
type: object type: object
properties: properties:
@ -5042,8 +4935,12 @@ components:
type: string type: string
const: scoring_function const: scoring_function
default: scoring_function default: scoring_function
scoring_fn_type:
$ref: '#/components/schemas/ScoringFunctionType'
description: description:
type: string type: string
params:
$ref: '#/components/schemas/ScoringFnParams'
metadata: metadata:
type: object type: object
additionalProperties: additionalProperties:
@ -5054,41 +4951,35 @@ components:
- type: string - type: string
- type: array - type: array
- type: object - type: object
return_type:
$ref: '#/components/schemas/ParamType'
params:
$ref: '#/components/schemas/ScoringFnParams'
additionalProperties: false additionalProperties: false
required: required:
- identifier - identifier
- provider_resource_id - provider_resource_id
- provider_id - provider_id
- type - type
- scoring_fn_type
- metadata - metadata
- return_type
title: ScoringFn title: ScoringFn
StringType: ScoringFunctionType:
type: object
properties:
type:
type: string type: string
const: string enum:
default: string - custom_llm_as_judge
additionalProperties: false - regex_parser
required: - regex_parser_math_response
- type - equality
title: StringType - subset_of
UnionType: - factuality
type: object - faithfulness
properties: - answer_correctness
type: - answer_relevancy
type: string - answer_similarity
const: union - context_entity_recall
default: union - context_precision
additionalProperties: false - context_recall
required: - context_relevancy
- type title: ScoringFunctionType
title: UnionType description: >-
A type of scoring function. Each type is a criteria for evaluating answers.
Shield: Shield:
type: object type: object
properties: properties:
@ -6356,8 +6247,9 @@ components:
The data reference of the dataset. Examples: - { "type": "uri", "uri": The data reference of the dataset. Examples: - { "type": "uri", "uri":
"https://mywebsite.com/mydata.jsonl" } - { "type": "uri", "uri": "lsfs://mydata.jsonl" "https://mywebsite.com/mydata.jsonl" } - { "type": "uri", "uri": "lsfs://mydata.jsonl"
} - { "type": "huggingface", "dataset_path": "tatsu-lab/alpaca", "params": } - { "type": "huggingface", "dataset_path": "tatsu-lab/alpaca", "params":
{ "split": "train" } } - { "type": "rows", "rows": [{"message": "Hello, { "split": "train" } } - { "type": "rows", "rows": [ { "messages": [ {"role":
world!"}] } "user", "content": "Hello, world!"}, {"role": "assistant", "content":
"Hello, world!"}, ] } ] }
metadata: metadata:
type: object type: object
additionalProperties: additionalProperties:
@ -6407,23 +6299,38 @@ components:
RegisterScoringFunctionRequest: RegisterScoringFunctionRequest:
type: object type: object
properties: properties:
scoring_fn_id: scoring_fn_type:
type: string $ref: '#/components/schemas/ScoringFunctionType'
description: description: >-
type: string The type of scoring function to register. A function type can only be
return_type: registered if it is a valid type.
$ref: '#/components/schemas/ParamType'
provider_scoring_fn_id:
type: string
provider_id:
type: string
params: params:
$ref: '#/components/schemas/ScoringFnParams' $ref: '#/components/schemas/ScoringFnParams'
description: The parameters for the scoring function.
scoring_fn_id:
type: string
description: >-
(Optional) The ID of the scoring function to register. If not provided,
a random ID will be generated.
description:
type: string
description: >-
(Optional) The description of the scoring function.
metadata:
type: object
additionalProperties:
oneOf:
- type: 'null'
- type: boolean
- type: number
- type: string
- type: array
- type: object
description: >-
(Optional) Any additional metadata to be associated with the scoring function.
additionalProperties: false additionalProperties: false
required: required:
- scoring_fn_id - scoring_fn_type
- description
- return_type
title: RegisterScoringFunctionRequest title: RegisterScoringFunctionRequest
RegisterShieldRequest: RegisterShieldRequest:
type: object type: object

2
llama_stack/apis/scoring_functions/scoring_functions.py
View file

@ -175,7 +175,7 @@ class ScoringFunctions(Protocol):
scoring_fn_id: Optional[str] = None, scoring_fn_id: Optional[str] = None,
description: Optional[str] = None, description: Optional[str] = None,
metadata: Optional[Dict[str, Any]] = None, metadata: Optional[Dict[str, Any]] = None,
): ) -> ScoringFn:
""" """
Register a new scoring function with given parameters. Register a new scoring function with given parameters.
Only valid scoring function type that can be parameterized can be registered. Only valid scoring function type that can be parameterized can be registered.