forked from phoenix-oss/llama-stack-mirror
precommit
This commit is contained in:
Xi Yan
parent
66cd83fb58
commit
452b2b1284
5 changed files with 515 additions and 658 deletions
603
docs/_static/llama-stack-spec.html
vendored
603
docs/_static/llama-stack-spec.html
vendored
|
|
@ -2293,67 +2293,6 @@
|
|||
]
|
||||
}
|
||||
},
|
||||
"/v1/eval/benchmarks/{benchmark_id}/jobs/{job_id}": {
|
||||
"get": {
|
||||
"responses": {
|
||||
"200": {
|
||||
"description": "OK",
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"$ref": "#/components/schemas/IterrowsResponse"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"400": {
|
||||
"$ref": "#/components/responses/BadRequest400"
|
||||
},
|
||||
"429": {
|
||||
"$ref": "#/components/responses/TooManyRequests429"
|
||||
},
|
||||
"500": {
|
||||
"$ref": "#/components/responses/InternalServerError500"
|
||||
},
|
||||
"default": {
|
||||
"$ref": "#/components/responses/DefaultError"
|
||||
}
|
||||
},
|
||||
"tags": [
|
||||
"DatasetIO"
|
||||
],
|
||||
"description": "Get a paginated list of rows from a dataset. Uses cursor-based pagination.",
|
||||
"parameters": [
|
||||
{
|
||||
"name": "dataset_id",
|
||||
"in": "path",
|
||||
"description": "The ID of the dataset to get the rows from.",
|
||||
"required": true,
|
||||
"schema": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "start_index",
|
||||
"in": "query",
|
||||
"description": "Index into dataset for the first row to get. Get all rows if None.",
|
||||
"required": false,
|
||||
"schema": {
|
||||
"type": "integer"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "limit",
|
||||
"in": "query",
|
||||
"description": "The number of rows to get per page.",
|
||||
"required": false,
|
||||
"schema": {
|
||||
"type": "integer"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"/v1/agents/{agent_id}/sessions": {
|
||||
"get": {
|
||||
"responses": {
|
||||
|
|
@ -6613,69 +6552,77 @@
|
|||
"const": "factuality",
|
||||
"default": "factuality"
|
||||
},
|
||||
"dataset_id": {
|
||||
"type": "string"
|
||||
},
|
||||
"scoring_functions": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"metadata": {
|
||||
"factuality": {
|
||||
"type": "object",
|
||||
"additionalProperties": {
|
||||
"oneOf": [
|
||||
{
|
||||
"type": "null"
|
||||
},
|
||||
{
|
||||
"type": "boolean"
|
||||
},
|
||||
{
|
||||
"type": "number"
|
||||
},
|
||||
{
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"type": "array"
|
||||
},
|
||||
{
|
||||
"type": "object"
|
||||
"properties": {
|
||||
"aggregation_functions": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"average",
|
||||
"median",
|
||||
"categorical_count",
|
||||
"accuracy"
|
||||
],
|
||||
"title": "AggregationFunctionType",
|
||||
"description": "A type of aggregation function."
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
"additionalProperties": false,
|
||||
"required": [
|
||||
"aggregation_functions"
|
||||
],
|
||||
"title": "BasicGraderParams"
|
||||
}
|
||||
},
|
||||
"additionalProperties": false,
|
||||
"required": [
|
||||
"identifier",
|
||||
"provider_resource_id",
|
||||
"provider_id",
|
||||
"type",
|
||||
"dataset_id",
|
||||
"scoring_functions",
|
||||
"metadata"
|
||||
"factuality"
|
||||
],
|
||||
"title": "Benchmark"
|
||||
"title": "FactualityGrader"
|
||||
},
|
||||
"DataSource": {
|
||||
"oneOf": [
|
||||
{
|
||||
"$ref": "#/components/schemas/URIDataSource"
|
||||
"FaithfulnessGrader": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"type": {
|
||||
"type": "string",
|
||||
"const": "faithfulness",
|
||||
"default": "faithfulness"
|
||||
},
|
||||
{
|
||||
"$ref": "#/components/schemas/RowsDataSource"
|
||||
"faithfulness": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"aggregation_functions": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"average",
|
||||
"median",
|
||||
"categorical_count",
|
||||
"accuracy"
|
||||
],
|
||||
"title": "AggregationFunctionType",
|
||||
"description": "A type of aggregation function."
|
||||
}
|
||||
}
|
||||
},
|
||||
"additionalProperties": false,
|
||||
"required": [
|
||||
"aggregation_functions"
|
||||
],
|
||||
"title": "BasicGraderParams"
|
||||
}
|
||||
},
|
||||
"additionalProperties": false,
|
||||
"required": [
|
||||
"type",
|
||||
"faithfulness"
|
||||
],
|
||||
"discriminator": {
|
||||
"propertyName": "type",
|
||||
"mapping": {
|
||||
"uri": "#/components/schemas/URIDataSource",
|
||||
"rows": "#/components/schemas/RowsDataSource"
|
||||
}
|
||||
}
|
||||
"title": "FaithfulnessGrader"
|
||||
},
|
||||
"Grader": {
|
||||
"type": "object",
|
||||
|
|
@ -6694,18 +6641,11 @@
|
|||
"const": "grader",
|
||||
"default": "grader"
|
||||
},
|
||||
"purpose": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"post-training/messages",
|
||||
"eval/question-answer",
|
||||
"eval/messages-answer"
|
||||
],
|
||||
"title": "DatasetPurpose",
|
||||
"description": "Purpose of the dataset. Each purpose has a required input data schema."
|
||||
"grader": {
|
||||
"$ref": "#/components/schemas/GraderDefinition"
|
||||
},
|
||||
"source": {
|
||||
"$ref": "#/components/schemas/DataSource"
|
||||
"description": {
|
||||
"type": "string"
|
||||
},
|
||||
"metadata": {
|
||||
"type": "object",
|
||||
|
|
@ -6739,78 +6679,98 @@
|
|||
"provider_resource_id",
|
||||
"provider_id",
|
||||
"type",
|
||||
"purpose",
|
||||
"source",
|
||||
"grader",
|
||||
"metadata"
|
||||
],
|
||||
"title": "Dataset"
|
||||
"title": "Grader"
|
||||
},
|
||||
"RowsDataSource": {
|
||||
"GraderDefinition": {
|
||||
"oneOf": [
|
||||
{
|
||||
"$ref": "#/components/schemas/LlmGrader"
|
||||
},
|
||||
{
|
||||
"$ref": "#/components/schemas/RegexParserGrader"
|
||||
},
|
||||
{
|
||||
"$ref": "#/components/schemas/EqualityGrader"
|
||||
},
|
||||
{
|
||||
"$ref": "#/components/schemas/SubsetOfGrader"
|
||||
},
|
||||
{
|
||||
"$ref": "#/components/schemas/FactualityGrader"
|
||||
},
|
||||
{
|
||||
"$ref": "#/components/schemas/FaithfulnessGrader"
|
||||
}
|
||||
],
|
||||
"discriminator": {
|
||||
"propertyName": "type",
|
||||
"mapping": {
|
||||
"llm": "#/components/schemas/LlmGrader",
|
||||
"regex_parser": "#/components/schemas/RegexParserGrader",
|
||||
"equality": "#/components/schemas/EqualityGrader",
|
||||
"subset_of": "#/components/schemas/SubsetOfGrader",
|
||||
"factuality": "#/components/schemas/FactualityGrader",
|
||||
"faithfulness": "#/components/schemas/FaithfulnessGrader"
|
||||
}
|
||||
}
|
||||
},
|
||||
"LlmGrader": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"type": {
|
||||
"type": "string",
|
||||
"const": "rows",
|
||||
"default": "rows"
|
||||
"const": "llm",
|
||||
"default": "llm"
|
||||
},
|
||||
"rows": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "object",
|
||||
"additionalProperties": {
|
||||
"oneOf": [
|
||||
{
|
||||
"type": "null"
|
||||
},
|
||||
{
|
||||
"type": "boolean"
|
||||
},
|
||||
{
|
||||
"type": "number"
|
||||
},
|
||||
{
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"type": "array"
|
||||
},
|
||||
{
|
||||
"type": "object"
|
||||
}
|
||||
]
|
||||
"llm": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"model": {
|
||||
"type": "string"
|
||||
},
|
||||
"prompt": {
|
||||
"type": "string"
|
||||
},
|
||||
"score_regexes": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"aggregation_functions": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"average",
|
||||
"median",
|
||||
"categorical_count",
|
||||
"accuracy"
|
||||
],
|
||||
"title": "AggregationFunctionType",
|
||||
"description": "A type of aggregation function."
|
||||
}
|
||||
}
|
||||
},
|
||||
"description": "The dataset is stored in rows. E.g. - [ {\"messages\": [{\"role\": \"user\", \"content\": \"Hello, world!\"}, {\"role\": \"assistant\", \"content\": \"Hello, world!\"}]} ]"
|
||||
"additionalProperties": false,
|
||||
"required": [
|
||||
"model",
|
||||
"prompt",
|
||||
"score_regexes",
|
||||
"aggregation_functions"
|
||||
],
|
||||
"title": "LlmGraderParams"
|
||||
}
|
||||
},
|
||||
"additionalProperties": false,
|
||||
"required": [
|
||||
"type",
|
||||
"rows"
|
||||
"llm"
|
||||
],
|
||||
"title": "RowsDataSource",
|
||||
"description": "A dataset stored in rows."
|
||||
},
|
||||
"URIDataSource": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"type": {
|
||||
"type": "string",
|
||||
"const": "uri",
|
||||
"default": "uri"
|
||||
},
|
||||
"uri": {
|
||||
"type": "string",
|
||||
"description": "The dataset can be obtained from a URI. E.g. - \"https://mywebsite.com/mydata.jsonl\" - \"lsfs://mydata.jsonl\" - \"data:csv;base64,{base64_content}\""
|
||||
}
|
||||
},
|
||||
"additionalProperties": false,
|
||||
"required": [
|
||||
"type",
|
||||
"uri"
|
||||
],
|
||||
"title": "URIDataSource",
|
||||
"description": "A dataset that can be obtained from a URI."
|
||||
"title": "LlmGrader"
|
||||
},
|
||||
"RegexParserGrader": {
|
||||
"type": "object",
|
||||
|
|
@ -6859,182 +6819,45 @@
|
|||
],
|
||||
"title": "RegexParserGrader"
|
||||
},
|
||||
"ModelType": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"llm",
|
||||
"embedding"
|
||||
],
|
||||
"title": "ModelType"
|
||||
},
|
||||
"AgentTurnInputType": {
|
||||
"SubsetOfGrader": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"type": {
|
||||
"type": "string",
|
||||
"const": "agent_turn_input",
|
||||
"default": "agent_turn_input"
|
||||
"const": "subset_of",
|
||||
"default": "subset_of"
|
||||
},
|
||||
"subset_of": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"aggregation_functions": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"average",
|
||||
"median",
|
||||
"categorical_count",
|
||||
"accuracy"
|
||||
],
|
||||
"title": "AggregationFunctionType",
|
||||
"description": "A type of aggregation function."
|
||||
}
|
||||
}
|
||||
},
|
||||
"additionalProperties": false,
|
||||
"required": [
|
||||
"aggregation_functions"
|
||||
],
|
||||
"title": "BasicGraderParams"
|
||||
}
|
||||
},
|
||||
"additionalProperties": false,
|
||||
"required": [
|
||||
"type"
|
||||
"type",
|
||||
"subset_of"
|
||||
],
|
||||
"title": "AgentTurnInputType"
|
||||
},
|
||||
"ArrayType": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"type": {
|
||||
"type": "string",
|
||||
"const": "array",
|
||||
"default": "array"
|
||||
}
|
||||
},
|
||||
"additionalProperties": false,
|
||||
"required": [
|
||||
"type"
|
||||
],
|
||||
"title": "ArrayType"
|
||||
},
|
||||
"BooleanType": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"type": {
|
||||
"type": "string",
|
||||
"const": "boolean",
|
||||
"default": "boolean"
|
||||
}
|
||||
},
|
||||
"additionalProperties": false,
|
||||
"required": [
|
||||
"type"
|
||||
],
|
||||
"title": "BooleanType"
|
||||
},
|
||||
"ChatCompletionInputType": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"type": {
|
||||
"type": "string",
|
||||
"const": "chat_completion_input",
|
||||
"default": "chat_completion_input"
|
||||
}
|
||||
},
|
||||
"additionalProperties": false,
|
||||
"required": [
|
||||
"type"
|
||||
],
|
||||
"title": "ChatCompletionInputType"
|
||||
},
|
||||
"CompletionInputType": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"type": {
|
||||
"type": "string",
|
||||
"const": "completion_input",
|
||||
"default": "completion_input"
|
||||
}
|
||||
},
|
||||
"additionalProperties": false,
|
||||
"required": [
|
||||
"type"
|
||||
],
|
||||
"title": "CompletionInputType"
|
||||
},
|
||||
"JsonType": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"type": {
|
||||
"type": "string",
|
||||
"const": "json",
|
||||
"default": "json"
|
||||
}
|
||||
},
|
||||
"additionalProperties": false,
|
||||
"required": [
|
||||
"type"
|
||||
],
|
||||
"title": "JsonType"
|
||||
},
|
||||
"NumberType": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"type": {
|
||||
"type": "string",
|
||||
"const": "number",
|
||||
"default": "number"
|
||||
}
|
||||
},
|
||||
"additionalProperties": false,
|
||||
"required": [
|
||||
"type"
|
||||
],
|
||||
"title": "NumberType"
|
||||
},
|
||||
"ObjectType": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"type": {
|
||||
"type": "string",
|
||||
"const": "object",
|
||||
"default": "object"
|
||||
}
|
||||
},
|
||||
"additionalProperties": false,
|
||||
"required": [
|
||||
"type"
|
||||
],
|
||||
"title": "ObjectType"
|
||||
},
|
||||
"ParamType": {
|
||||
"oneOf": [
|
||||
{
|
||||
"$ref": "#/components/schemas/StringType"
|
||||
},
|
||||
{
|
||||
"$ref": "#/components/schemas/NumberType"
|
||||
},
|
||||
{
|
||||
"$ref": "#/components/schemas/BooleanType"
|
||||
},
|
||||
{
|
||||
"$ref": "#/components/schemas/ArrayType"
|
||||
},
|
||||
{
|
||||
"$ref": "#/components/schemas/ObjectType"
|
||||
},
|
||||
{
|
||||
"$ref": "#/components/schemas/JsonType"
|
||||
},
|
||||
{
|
||||
"$ref": "#/components/schemas/UnionType"
|
||||
},
|
||||
{
|
||||
"$ref": "#/components/schemas/ChatCompletionInputType"
|
||||
},
|
||||
{
|
||||
"$ref": "#/components/schemas/CompletionInputType"
|
||||
},
|
||||
{
|
||||
"$ref": "#/components/schemas/AgentTurnInputType"
|
||||
}
|
||||
],
|
||||
"discriminator": {
|
||||
"propertyName": "type",
|
||||
"mapping": {
|
||||
"string": "#/components/schemas/StringType",
|
||||
"number": "#/components/schemas/NumberType",
|
||||
"boolean": "#/components/schemas/BooleanType",
|
||||
"array": "#/components/schemas/ArrayType",
|
||||
"object": "#/components/schemas/ObjectType",
|
||||
"json": "#/components/schemas/JsonType",
|
||||
"union": "#/components/schemas/UnionType",
|
||||
"chat_completion_input": "#/components/schemas/ChatCompletionInputType",
|
||||
"completion_input": "#/components/schemas/CompletionInputType",
|
||||
"agent_turn_input": "#/components/schemas/AgentTurnInputType"
|
||||
}
|
||||
}
|
||||
"title": "SubsetOfGrader"
|
||||
},
|
||||
"Model": {
|
||||
"type": "object",
|
||||
|
|
@ -7090,39 +6913,17 @@
|
|||
"provider_id",
|
||||
"type",
|
||||
"metadata",
|
||||
"return_type"
|
||||
"model_type"
|
||||
],
|
||||
"title": "ScoringFn"
|
||||
"title": "Model"
|
||||
},
|
||||
"StringType": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"type": {
|
||||
"type": "string",
|
||||
"const": "string",
|
||||
"default": "string"
|
||||
}
|
||||
},
|
||||
"additionalProperties": false,
|
||||
"required": [
|
||||
"type"
|
||||
"ModelType": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"llm",
|
||||
"embedding"
|
||||
],
|
||||
"title": "StringType"
|
||||
},
|
||||
"UnionType": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"type": {
|
||||
"type": "string",
|
||||
"const": "union",
|
||||
"default": "union"
|
||||
}
|
||||
},
|
||||
"additionalProperties": false,
|
||||
"required": [
|
||||
"type"
|
||||
],
|
||||
"title": "UnionType"
|
||||
"title": "ModelType"
|
||||
},
|
||||
"Shield": {
|
||||
"type": "object",
|
||||
|
|
@ -9677,6 +9478,50 @@
|
|||
"purpose",
|
||||
"source"
|
||||
],
|
||||
"title": "RegisterDatasetRequest"
|
||||
},
|
||||
"RegisterGraderRequest": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"grader": {
|
||||
"$ref": "#/components/schemas/GraderDefinition",
|
||||
"description": "The grader definition, E.g. - { \"type\": \"llm\", \"llm\": { \"model\": \"llama-405b\", \"prompt\": \"You are a judge. Score the answer based on the question. {question} {answer}\", } }"
|
||||
},
|
||||
"grader_id": {
|
||||
"type": "string",
|
||||
"description": "(Optional) The ID of the grader. If not provided, a random ID will be generated."
|
||||
},
|
||||
"metadata": {
|
||||
"type": "object",
|
||||
"additionalProperties": {
|
||||
"oneOf": [
|
||||
{
|
||||
"type": "null"
|
||||
},
|
||||
{
|
||||
"type": "boolean"
|
||||
},
|
||||
{
|
||||
"type": "number"
|
||||
},
|
||||
{
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"type": "array"
|
||||
},
|
||||
{
|
||||
"type": "object"
|
||||
}
|
||||
]
|
||||
},
|
||||
"description": "(Optional) Any additional metadata for this grader. - E.g. { \"description\": \"A grader that scores the answer based on the question.\", }"
|
||||
}
|
||||
},
|
||||
"additionalProperties": false,
|
||||
"required": [
|
||||
"grader"
|
||||
],
|
||||
"title": "RegisterGraderRequest"
|
||||
},
|
||||
"RegisterModelRequest": {
|
||||
|
|
@ -10354,6 +10199,9 @@
|
|||
{
|
||||
"name": "Files"
|
||||
},
|
||||
{
|
||||
"name": "Graders"
|
||||
},
|
||||
{
|
||||
"name": "Inference",
|
||||
"description": "This API provides the raw interface to the underlying models. Two kinds of models are supported:\n- LLM models: these models generate \"raw\" and \"chat\" (conversational) completions.\n- Embedding models: these models generate embeddings to be used for semantic search.",
|
||||
|
|
@ -10406,8 +10254,9 @@
|
|||
"Benchmarks",
|
||||
"DatasetIO",
|
||||
"Datasets",
|
||||
"Eval",
|
||||
"Evaluation",
|
||||
"Files",
|
||||
"Graders",
|
||||
"Inference",
|
||||
"Inspect",
|
||||
"Models",
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue