diff --git a/docs/openapi_generator/generate.py b/docs/openapi_generator/generate.py index 871c01a80..f9f56119b 100644 --- a/docs/openapi_generator/generate.py +++ b/docs/openapi_generator/generate.py @@ -33,14 +33,16 @@ schema_utils.json_schema_type = json_schema_type from llama_models.llama3.api.datatypes import * # noqa: F403 from llama_stack.apis.agents import * # noqa: F403 -from llama_stack.apis.dataset import * # noqa: F403 -from llama_stack.apis.evals import * # noqa: F403 +from llama_stack.apis.datasets import * # noqa: F403 +from llama_stack.apis.datasetio import * # noqa: F403 +from llama_stack.apis.scoring import * # noqa: F403 +from llama_stack.apis.scoring_functions import * # noqa: F403 +from llama_stack.apis.eval import * # noqa: F403 from llama_stack.apis.inference import * # noqa: F403 from llama_stack.apis.batch_inference import * # noqa: F403 from llama_stack.apis.memory import * # noqa: F403 from llama_stack.apis.telemetry import * # noqa: F403 from llama_stack.apis.post_training import * # noqa: F403 -from llama_stack.apis.reward_scoring import * # noqa: F403 from llama_stack.apis.synthetic_data_generation import * # noqa: F403 from llama_stack.apis.safety import * # noqa: F403 from llama_stack.apis.models import * # noqa: F403 @@ -54,14 +56,16 @@ class LlamaStack( Inference, BatchInference, Agents, - RewardScoring, Safety, SyntheticDataGeneration, Datasets, Telemetry, PostTraining, Memory, - Evaluations, + Eval, + Scoring, + ScoringFunctions, + DatasetIO, Models, Shields, Inspect, diff --git a/docs/resources/llama-stack-spec.html b/docs/resources/llama-stack-spec.html index 8e6683931..886634fba 100644 --- a/docs/resources/llama-stack-spec.html +++ b/docs/resources/llama-stack-spec.html @@ -21,7 +21,7 @@ "info": { "title": "[DRAFT] Llama Stack Specification", "version": "0.0.1", - "description": "This is the specification of the llama stack that provides\n a set of endpoints and their corresponding interfaces that are tailored to\n best leverage Llama Models. The specification is still in draft and subject to change.\n Generated at 2024-10-18 20:48:17.730988" + "description": "This is the specification of the llama stack that provides\n a set of endpoints and their corresponding interfaces that are tailored to\n best leverage Llama Models. The specification is still in draft and subject to change.\n Generated at 2024-10-24 17:40:59.576117" }, "servers": [ { @@ -109,39 +109,6 @@ } } }, - "/evaluate/job/cancel": { - "post": { - "responses": { - "200": { - "description": "OK" - } - }, - "tags": [ - "Evaluations" - ], - "parameters": [ - { - "name": "X-LlamaStack-ProviderData", - "in": "header", - "description": "JSON-encoded provider data which will be made available to the adapter servicing the API", - "required": false, - "schema": { - "type": "string" - } - } - ], - "requestBody": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/CancelEvaluationJobRequest" - } - } - }, - "required": true - } - } - }, "/post_training/job/cancel": { "post": { "responses": { @@ -389,39 +356,6 @@ } } }, - "/datasets/create": { - "post": { - "responses": { - "200": { - "description": "OK" - } - }, - "tags": [ - "Datasets" - ], - "parameters": [ - { - "name": "X-LlamaStack-ProviderData", - "in": "header", - "description": "JSON-encoded provider data which will be made available to the adapter servicing the API", - "required": false, - "schema": { - "type": "string" - } - } - ], - "requestBody": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/CreateDatasetRequest" - } - } - }, - "required": true - } - } - }, "/agents/delete": { "post": { "responses": { @@ -488,39 +422,6 @@ } } }, - "/datasets/delete": { - "post": { - "responses": { - "200": { - "description": "OK" - } - }, - "tags": [ - "Datasets" - ], - "parameters": [ - { - "name": "X-LlamaStack-ProviderData", - "in": "header", - "description": "JSON-encoded provider data which will be made available to the adapter servicing the API", - "required": false, - "schema": { - "type": "string" - } - } - ], - "requestBody": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/DeleteDatasetRequest" - } - } - }, - "required": true - } - } - }, "/inference/embeddings": { "post": { "responses": { @@ -561,7 +462,7 @@ } } }, - "/evaluate/question_answering/": { + "/eval/evaluate": { "post": { "responses": { "200": { @@ -569,14 +470,14 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/EvaluationJob" + "$ref": "#/components/schemas/EvaluateResponse" } } } } }, "tags": [ - "Evaluations" + "Eval" ], "parameters": [ { @@ -593,7 +494,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/EvaluateQuestionAnsweringRequest" + "$ref": "#/components/schemas/EvaluateRequest" } } }, @@ -601,7 +502,7 @@ } } }, - "/evaluate/summarization/": { + "/eval/evaluate_batch": { "post": { "responses": { "200": { @@ -609,14 +510,14 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/EvaluationJob" + "$ref": "#/components/schemas/Job" } } } } }, "tags": [ - "Evaluations" + "Eval" ], "parameters": [ { @@ -633,47 +534,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/EvaluateSummarizationRequest" - } - } - }, - "required": true - } - } - }, - "/evaluate/text_generation/": { - "post": { - "responses": { - "200": { - "description": "OK", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/EvaluationJob" - } - } - } - } - }, - "tags": [ - "Evaluations" - ], - "parameters": [ - { - "name": "X-LlamaStack-ProviderData", - "in": "header", - "description": "JSON-encoded provider data which will be made available to the adapter servicing the API", - "required": false, - "schema": { - "type": "string" - } - } - ], - "requestBody": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/EvaluateTextGenerationRequest" + "$ref": "#/components/schemas/EvaluateBatchRequest" } } }, @@ -763,6 +624,14 @@ "type": "string" } }, + { + "name": "session_id", + "in": "query", + "required": true, + "schema": { + "type": "string" + } + }, { "name": "turn_id", "in": "query", @@ -817,6 +686,14 @@ "type": "string" } }, + { + "name": "session_id", + "in": "query", + "required": true, + "schema": { + "type": "string" + } + }, { "name": "turn_id", "in": "query", @@ -845,7 +722,14 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/TrainEvalDataset" + "oneOf": [ + { + "$ref": "#/components/schemas/DatasetDefWithProvider" + }, + { + "type": "null" + } + ] } } } @@ -856,7 +740,7 @@ ], "parameters": [ { - "name": "dataset_uuid", + "name": "dataset_identifier", "in": "query", "required": true, "schema": { @@ -875,150 +759,6 @@ ] } }, - "/evaluate/job/artifacts": { - "get": { - "responses": { - "200": { - "description": "OK", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/EvaluationJobArtifactsResponse" - } - } - } - } - }, - "tags": [ - "Evaluations" - ], - "parameters": [ - { - "name": "job_uuid", - "in": "query", - "required": true, - "schema": { - "type": "string" - } - }, - { - "name": "X-LlamaStack-ProviderData", - "in": "header", - "description": "JSON-encoded provider data which will be made available to the adapter servicing the API", - "required": false, - "schema": { - "type": "string" - } - } - ] - } - }, - "/evaluate/job/logs": { - "get": { - "responses": { - "200": { - "description": "OK", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/EvaluationJobLogStream" - } - } - } - } - }, - "tags": [ - "Evaluations" - ], - "parameters": [ - { - "name": "job_uuid", - "in": "query", - "required": true, - "schema": { - "type": "string" - } - }, - { - "name": "X-LlamaStack-ProviderData", - "in": "header", - "description": "JSON-encoded provider data which will be made available to the adapter servicing the API", - "required": false, - "schema": { - "type": "string" - } - } - ] - } - }, - "/evaluate/job/status": { - "get": { - "responses": { - "200": { - "description": "OK", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/EvaluationJobStatusResponse" - } - } - } - } - }, - "tags": [ - "Evaluations" - ], - "parameters": [ - { - "name": "job_uuid", - "in": "query", - "required": true, - "schema": { - "type": "string" - } - }, - { - "name": "X-LlamaStack-ProviderData", - "in": "header", - "description": "JSON-encoded provider data which will be made available to the adapter servicing the API", - "required": false, - "schema": { - "type": "string" - } - } - ] - } - }, - "/evaluate/jobs": { - "get": { - "responses": { - "200": { - "description": "OK", - "content": { - "application/jsonl": { - "schema": { - "$ref": "#/components/schemas/EvaluationJob" - } - } - } - } - }, - "tags": [ - "Evaluations" - ], - "parameters": [ - { - "name": "X-LlamaStack-ProviderData", - "in": "header", - "description": "JSON-encoded provider data which will be made available to the adapter servicing the API", - "required": false, - "schema": { - "type": "string" - } - } - ] - } - }, "/memory_banks/get": { "get": { "responses": { @@ -1122,6 +862,113 @@ ] } }, + "/datasetio/get_rows_paginated": { + "get": { + "responses": { + "200": { + "description": "OK", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/PaginatedRowsResult" + } + } + } + } + }, + "tags": [ + "DatasetIO" + ], + "parameters": [ + { + "name": "dataset_id", + "in": "query", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "rows_in_page", + "in": "query", + "required": true, + "schema": { + "type": "integer" + } + }, + { + "name": "page_token", + "in": "query", + "required": false, + "schema": { + "type": "string" + } + }, + { + "name": "filter_condition", + "in": "query", + "required": false, + "schema": { + "type": "string" + } + }, + { + "name": "X-LlamaStack-ProviderData", + "in": "header", + "description": "JSON-encoded provider data which will be made available to the adapter servicing the API", + "required": false, + "schema": { + "type": "string" + } + } + ] + } + }, + "/scoring_functions/get": { + "get": { + "responses": { + "200": { + "description": "OK", + "content": { + "application/json": { + "schema": { + "oneOf": [ + { + "$ref": "#/components/schemas/ScoringFunctionDefWithProvider" + }, + { + "type": "null" + } + ] + } + } + } + } + }, + "tags": [ + "ScoringFunctions" + ], + "parameters": [ + { + "name": "name", + "in": "query", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "X-LlamaStack-ProviderData", + "in": "header", + "description": "JSON-encoded provider data which will be made available to the adapter servicing the API", + "required": false, + "schema": { + "type": "string" + } + } + ] + } + }, "/shields/get": { "get": { "responses": { @@ -1412,6 +1259,152 @@ } } }, + "/eval/job/cancel": { + "post": { + "responses": { + "200": { + "description": "OK" + } + }, + "tags": [ + "Eval" + ], + "parameters": [ + { + "name": "X-LlamaStack-ProviderData", + "in": "header", + "description": "JSON-encoded provider data which will be made available to the adapter servicing the API", + "required": false, + "schema": { + "type": "string" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/JobCancelRequest" + } + } + }, + "required": true + } + } + }, + "/eval/job/result": { + "get": { + "responses": { + "200": { + "description": "OK", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/EvaluateResponse" + } + } + } + } + }, + "tags": [ + "Eval" + ], + "parameters": [ + { + "name": "job_id", + "in": "query", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "X-LlamaStack-ProviderData", + "in": "header", + "description": "JSON-encoded provider data which will be made available to the adapter servicing the API", + "required": false, + "schema": { + "type": "string" + } + } + ] + } + }, + "/eval/job/status": { + "get": { + "responses": { + "200": { + "description": "OK", + "content": { + "application/json": { + "schema": { + "oneOf": [ + { + "$ref": "#/components/schemas/JobStatus" + }, + { + "type": "null" + } + ] + } + } + } + } + }, + "tags": [ + "Eval" + ], + "parameters": [ + { + "name": "job_id", + "in": "query", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "X-LlamaStack-ProviderData", + "in": "header", + "description": "JSON-encoded provider data which will be made available to the adapter servicing the API", + "required": false, + "schema": { + "type": "string" + } + } + ] + } + }, + "/datasets/list": { + "get": { + "responses": { + "200": { + "description": "OK", + "content": { + "application/jsonl": { + "schema": { + "$ref": "#/components/schemas/DatasetDefWithProvider" + } + } + } + } + }, + "tags": [ + "Datasets" + ], + "parameters": [ + { + "name": "X-LlamaStack-ProviderData", + "in": "header", + "description": "JSON-encoded provider data which will be made available to the adapter servicing the API", + "required": false, + "schema": { + "type": "string" + } + } + ] + } + }, "/memory_banks/list": { "get": { "responses": { @@ -1554,6 +1547,36 @@ ] } }, + "/scoring_functions/list": { + "get": { + "responses": { + "200": { + "description": "OK", + "content": { + "application/jsonl": { + "schema": { + "$ref": "#/components/schemas/ScoringFunctionDefWithProvider" + } + } + } + } + }, + "tags": [ + "ScoringFunctions" + ], + "parameters": [ + { + "name": "X-LlamaStack-ProviderData", + "in": "header", + "description": "JSON-encoded provider data which will be made available to the adapter servicing the API", + "required": false, + "schema": { + "type": "string" + } + } + ] + } + }, "/shields/list": { "get": { "responses": { @@ -1697,6 +1720,39 @@ } } }, + "/datasets/register": { + "post": { + "responses": { + "200": { + "description": "OK" + } + }, + "tags": [ + "Datasets" + ], + "parameters": [ + { + "name": "X-LlamaStack-ProviderData", + "in": "header", + "description": "JSON-encoded provider data which will be made available to the adapter servicing the API", + "required": false, + "schema": { + "type": "string" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/RegisterDatasetRequest" + } + } + }, + "required": true + } + } + }, "/memory_banks/register": { "post": { "responses": { @@ -1763,6 +1819,39 @@ } } }, + "/scoring_functions/register": { + "post": { + "responses": { + "200": { + "description": "OK" + } + }, + "tags": [ + "ScoringFunctions" + ], + "parameters": [ + { + "name": "X-LlamaStack-ProviderData", + "in": "header", + "description": "JSON-encoded provider data which will be made available to the adapter servicing the API", + "required": false, + "schema": { + "type": "string" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/RegisterScoringFunctionRequest" + } + } + }, + "required": true + } + } + }, "/shields/register": { "post": { "responses": { @@ -1796,46 +1885,6 @@ } } }, - "/reward_scoring/score": { - "post": { - "responses": { - "200": { - "description": "OK", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/RewardScoringResponse" - } - } - } - } - }, - "tags": [ - "RewardScoring" - ], - "parameters": [ - { - "name": "X-LlamaStack-ProviderData", - "in": "header", - "description": "JSON-encoded provider data which will be made available to the adapter servicing the API", - "required": false, - "schema": { - "type": "string" - } - } - ], - "requestBody": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/RewardScoreRequest" - } - } - }, - "required": true - } - } - }, "/safety/run_shield": { "post": { "responses": { @@ -1876,6 +1925,86 @@ } } }, + "/scoring/score": { + "post": { + "responses": { + "200": { + "description": "OK", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ScoreResponse" + } + } + } + } + }, + "tags": [ + "Scoring" + ], + "parameters": [ + { + "name": "X-LlamaStack-ProviderData", + "in": "header", + "description": "JSON-encoded provider data which will be made available to the adapter servicing the API", + "required": false, + "schema": { + "type": "string" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ScoreRequest" + } + } + }, + "required": true + } + } + }, + "/scoring/score_batch": { + "post": { + "responses": { + "200": { + "description": "OK", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ScoreBatchResponse" + } + } + } + } + }, + "tags": [ + "Scoring" + ], + "parameters": [ + { + "name": "X-LlamaStack-ProviderData", + "in": "header", + "description": "JSON-encoded provider data which will be made available to the adapter servicing the API", + "required": false, + "schema": { + "type": "string" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ScoreBatchRequest" + } + } + }, + "required": true + } + } + }, "/post_training/supervised_fine_tune": { "post": { "responses": { @@ -2571,18 +2700,6 @@ "completion_message_batch" ] }, - "CancelEvaluationJobRequest": { - "type": "object", - "properties": { - "job_uuid": { - "type": "string" - } - }, - "additionalProperties": false, - "required": [ - "job_uuid" - ] - }, "CancelTrainingJobRequest": { "type": "object", "properties": { @@ -2635,6 +2752,90 @@ "tool_prompt_format": { "$ref": "#/components/schemas/ToolPromptFormat" }, + "response_format": { + "oneOf": [ + { + "type": "object", + "properties": { + "type": { + "type": "string", + "const": "json_schema", + "default": "json_schema" + }, + "schema": { + "type": "object", + "additionalProperties": { + "oneOf": [ + { + "type": "null" + }, + { + "type": "boolean" + }, + { + "type": "number" + }, + { + "type": "string" + }, + { + "type": "array" + }, + { + "type": "object" + } + ] + } + } + }, + "additionalProperties": false, + "required": [ + "type", + "schema" + ] + }, + { + "type": "object", + "properties": { + "type": { + "type": "string", + "const": "grammar", + "default": "grammar" + }, + "bnf": { + "type": "object", + "additionalProperties": { + "oneOf": [ + { + "type": "null" + }, + { + "type": "boolean" + }, + { + "type": "number" + }, + { + "type": "string" + }, + { + "type": "array" + }, + { + "type": "object" + } + ] + } + } + }, + "additionalProperties": false, + "required": [ + "type", + "bnf" + ] + } + ] + }, "stream": { "type": "boolean" }, @@ -2807,6 +3008,90 @@ "sampling_params": { "$ref": "#/components/schemas/SamplingParams" }, + "response_format": { + "oneOf": [ + { + "type": "object", + "properties": { + "type": { + "type": "string", + "const": "json_schema", + "default": "json_schema" + }, + "schema": { + "type": "object", + "additionalProperties": { + "oneOf": [ + { + "type": "null" + }, + { + "type": "boolean" + }, + { + "type": "number" + }, + { + "type": "string" + }, + { + "type": "array" + }, + { + "type": "object" + } + ] + } + } + }, + "additionalProperties": false, + "required": [ + "type", + "schema" + ] + }, + { + "type": "object", + "properties": { + "type": { + "type": "string", + "const": "grammar", + "default": "grammar" + }, + "bnf": { + "type": "object", + "additionalProperties": { + "oneOf": [ + { + "type": "null" + }, + { + "type": "boolean" + }, + { + "type": "number" + }, + { + "type": "string" + }, + { + "type": "array" + }, + { + "type": "object" + } + ] + } + } + }, + "additionalProperties": false, + "required": [ + "type", + "bnf" + ] + } + ] + }, "stream": { "type": "boolean" }, @@ -4094,77 +4379,6 @@ "error" ] }, - "TrainEvalDataset": { - "type": "object", - "properties": { - "columns": { - "type": "object", - "additionalProperties": { - "$ref": "#/components/schemas/TrainEvalDatasetColumnType" - } - }, - "content_url": { - "$ref": "#/components/schemas/URL" - }, - "metadata": { - "type": "object", - "additionalProperties": { - "oneOf": [ - { - "type": "null" - }, - { - "type": "boolean" - }, - { - "type": "number" - }, - { - "type": "string" - }, - { - "type": "array" - }, - { - "type": "object" - } - ] - } - } - }, - "additionalProperties": false, - "required": [ - "columns", - "content_url" - ], - "title": "Dataset to be used for training or evaluating language models." - }, - "TrainEvalDatasetColumnType": { - "type": "string", - "enum": [ - "dialog", - "text", - "media", - "number", - "json" - ] - }, - "CreateDatasetRequest": { - "type": "object", - "properties": { - "uuid": { - "type": "string" - }, - "dataset": { - "$ref": "#/components/schemas/TrainEvalDataset" - } - }, - "additionalProperties": false, - "required": [ - "uuid", - "dataset" - ] - }, "DeleteAgentsRequest": { "type": "object", "properties": { @@ -4193,18 +4407,6 @@ "session_id" ] }, - "DeleteDatasetRequest": { - "type": "object", - "properties": { - "dataset_uuid": { - "type": "string" - } - }, - "additionalProperties": false, - "required": [ - "dataset_uuid" - ] - }, "EmbeddingsRequest": { "type": "object", "properties": { @@ -4262,74 +4464,251 @@ "embeddings" ] }, - "EvaluateQuestionAnsweringRequest": { + "AgentCandidate": { "type": "object", "properties": { - "metrics": { + "type": { + "type": "string", + "const": "agent", + "default": "agent" + }, + "config": { + "$ref": "#/components/schemas/AgentConfig" + } + }, + "additionalProperties": false, + "required": [ + "type", + "config" + ] + }, + "ModelCandidate": { + "type": "object", + "properties": { + "type": { + "type": "string", + "const": "model", + "default": "model" + }, + "model": { + "type": "string" + }, + "sampling_params": { + "$ref": "#/components/schemas/SamplingParams" + }, + "system_message": { + "$ref": "#/components/schemas/SystemMessage" + } + }, + "additionalProperties": false, + "required": [ + "type", + "model", + "sampling_params" + ] + }, + "EvaluateRequest": { + "type": "object", + "properties": { + "input_rows": { "type": "array", "items": { - "type": "string", - "enum": [ - "em", - "f1" + "type": "object", + "additionalProperties": { + "oneOf": [ + { + "type": "null" + }, + { + "type": "boolean" + }, + { + "type": "number" + }, + { + "type": "string" + }, + { + "type": "array" + }, + { + "type": "object" + } + ] + } + } + }, + "candidate": { + "oneOf": [ + { + "$ref": "#/components/schemas/ModelCandidate" + }, + { + "$ref": "#/components/schemas/AgentCandidate" + } + ] + }, + "scoring_functions": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "additionalProperties": false, + "required": [ + "input_rows", + "candidate", + "scoring_functions" + ] + }, + "EvaluateResponse": { + "type": "object", + "properties": { + "generations": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": { + "oneOf": [ + { + "type": "null" + }, + { + "type": "boolean" + }, + { + "type": "number" + }, + { + "type": "string" + }, + { + "type": "array" + }, + { + "type": "object" + } + ] + } + } + }, + "scores": { + "type": "object", + "additionalProperties": { + "$ref": "#/components/schemas/ScoringResult" + } + } + }, + "additionalProperties": false, + "required": [ + "generations", + "scores" + ] + }, + "ScoringResult": { + "type": "object", + "properties": { + "score_rows": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": { + "oneOf": [ + { + "type": "null" + }, + { + "type": "boolean" + }, + { + "type": "number" + }, + { + "type": "string" + }, + { + "type": "array" + }, + { + "type": "object" + } + ] + } + } + }, + "aggregated_results": { + "type": "object", + "additionalProperties": { + "oneOf": [ + { + "type": "null" + }, + { + "type": "boolean" + }, + { + "type": "number" + }, + { + "type": "string" + }, + { + "type": "array" + }, + { + "type": "object" + } ] } } }, "additionalProperties": false, "required": [ - "metrics" + "score_rows", + "aggregated_results" ] }, - "EvaluationJob": { + "EvaluateBatchRequest": { "type": "object", "properties": { - "job_uuid": { + "dataset_id": { + "type": "string" + }, + "candidate": { + "oneOf": [ + { + "$ref": "#/components/schemas/ModelCandidate" + }, + { + "$ref": "#/components/schemas/AgentCandidate" + } + ] + }, + "scoring_functions": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "additionalProperties": false, + "required": [ + "dataset_id", + "candidate", + "scoring_functions" + ] + }, + "Job": { + "type": "object", + "properties": { + "job_id": { "type": "string" } }, "additionalProperties": false, "required": [ - "job_uuid" - ] - }, - "EvaluateSummarizationRequest": { - "type": "object", - "properties": { - "metrics": { - "type": "array", - "items": { - "type": "string", - "enum": [ - "rouge", - "bleu" - ] - } - } - }, - "additionalProperties": false, - "required": [ - "metrics" - ] - }, - "EvaluateTextGenerationRequest": { - "type": "object", - "properties": { - "metrics": { - "type": "array", - "items": { - "type": "string", - "enum": [ - "perplexity", - "rouge", - "bleu" - ] - } - } - }, - "additionalProperties": false, - "required": [ - "metrics" + "job_id" ] }, "GetAgentsSessionRequest": { @@ -4517,41 +4896,216 @@ "step" ] }, - "EvaluationJobArtifactsResponse": { + "DatasetDefWithProvider": { "type": "object", "properties": { - "job_uuid": { + "identifier": { + "type": "string" + }, + "dataset_schema": { + "type": "object", + "additionalProperties": { + "oneOf": [ + { + "type": "object", + "properties": { + "type": { + "type": "string", + "const": "string", + "default": "string" + } + }, + "additionalProperties": false, + "required": [ + "type" + ] + }, + { + "type": "object", + "properties": { + "type": { + "type": "string", + "const": "number", + "default": "number" + } + }, + "additionalProperties": false, + "required": [ + "type" + ] + }, + { + "type": "object", + "properties": { + "type": { + "type": "string", + "const": "boolean", + "default": "boolean" + } + }, + "additionalProperties": false, + "required": [ + "type" + ] + }, + { + "type": "object", + "properties": { + "type": { + "type": "string", + "const": "array", + "default": "array" + } + }, + "additionalProperties": false, + "required": [ + "type" + ] + }, + { + "type": "object", + "properties": { + "type": { + "type": "string", + "const": "object", + "default": "object" + } + }, + "additionalProperties": false, + "required": [ + "type" + ] + }, + { + "type": "object", + "properties": { + "type": { + "type": "string", + "const": "json", + "default": "json" + } + }, + "additionalProperties": false, + "required": [ + "type" + ] + }, + { + "type": "object", + "properties": { + "type": { + "type": "string", + "const": "union", + "default": "union" + } + }, + "additionalProperties": false, + "required": [ + "type" + ] + }, + { + "type": "object", + "properties": { + "type": { + "type": "string", + "const": "custom", + "default": "custom" + }, + "validator_class": { + "type": "string" + } + }, + "additionalProperties": false, + "required": [ + "type", + "validator_class" + ] + }, + { + "type": "object", + "properties": { + "type": { + "type": "string", + "const": "chat_completion_input", + "default": "chat_completion_input" + } + }, + "additionalProperties": false, + "required": [ + "type" + ] + }, + { + "type": "object", + "properties": { + "type": { + "type": "string", + "const": "completion_input", + "default": "completion_input" + } + }, + "additionalProperties": false, + "required": [ + "type" + ] + }, + { + "type": "object", + "properties": { + "type": { + "type": "string", + "const": "agent_turn_input", + "default": "agent_turn_input" + } + }, + "additionalProperties": false, + "required": [ + "type" + ] + } + ] + } + }, + "url": { + "$ref": "#/components/schemas/URL" + }, + "metadata": { + "type": "object", + "additionalProperties": { + "oneOf": [ + { + "type": "null" + }, + { + "type": "boolean" + }, + { + "type": "number" + }, + { + "type": "string" + }, + { + "type": "array" + }, + { + "type": "object" + } + ] + } + }, + "provider_id": { "type": "string" } }, "additionalProperties": false, "required": [ - "job_uuid" - ], - "title": "Artifacts of a evaluation job." - }, - "EvaluationJobLogStream": { - "type": "object", - "properties": { - "job_uuid": { - "type": "string" - } - }, - "additionalProperties": false, - "required": [ - "job_uuid" - ] - }, - "EvaluationJobStatusResponse": { - "type": "object", - "properties": { - "job_uuid": { - "type": "string" - } - }, - "additionalProperties": false, - "required": [ - "job_uuid" + "identifier", + "dataset_schema", + "url", + "metadata", + "provider_id" ] }, "ModelDefWithProvider": { @@ -4600,6 +5154,458 @@ "provider_id" ] }, + "PaginatedRowsResult": { + "type": "object", + "properties": { + "rows": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": { + "oneOf": [ + { + "type": "null" + }, + { + "type": "boolean" + }, + { + "type": "number" + }, + { + "type": "string" + }, + { + "type": "array" + }, + { + "type": "object" + } + ] + } + } + }, + "total_count": { + "type": "integer" + }, + "next_page_token": { + "type": "string" + } + }, + "additionalProperties": false, + "required": [ + "rows", + "total_count" + ] + }, + "Parameter": { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "type": { + "oneOf": [ + { + "type": "object", + "properties": { + "type": { + "type": "string", + "const": "string", + "default": "string" + } + }, + "additionalProperties": false, + "required": [ + "type" + ] + }, + { + "type": "object", + "properties": { + "type": { + "type": "string", + "const": "number", + "default": "number" + } + }, + "additionalProperties": false, + "required": [ + "type" + ] + }, + { + "type": "object", + "properties": { + "type": { + "type": "string", + "const": "boolean", + "default": "boolean" + } + }, + "additionalProperties": false, + "required": [ + "type" + ] + }, + { + "type": "object", + "properties": { + "type": { + "type": "string", + "const": "array", + "default": "array" + } + }, + "additionalProperties": false, + "required": [ + "type" + ] + }, + { + "type": "object", + "properties": { + "type": { + "type": "string", + "const": "object", + "default": "object" + } + }, + "additionalProperties": false, + "required": [ + "type" + ] + }, + { + "type": "object", + "properties": { + "type": { + "type": "string", + "const": "json", + "default": "json" + } + }, + "additionalProperties": false, + "required": [ + "type" + ] + }, + { + "type": "object", + "properties": { + "type": { + "type": "string", + "const": "union", + "default": "union" + } + }, + "additionalProperties": false, + "required": [ + "type" + ] + }, + { + "type": "object", + "properties": { + "type": { + "type": "string", + "const": "custom", + "default": "custom" + }, + "validator_class": { + "type": "string" + } + }, + "additionalProperties": false, + "required": [ + "type", + "validator_class" + ] + }, + { + "type": "object", + "properties": { + "type": { + "type": "string", + "const": "chat_completion_input", + "default": "chat_completion_input" + } + }, + "additionalProperties": false, + "required": [ + "type" + ] + }, + { + "type": "object", + "properties": { + "type": { + "type": "string", + "const": "completion_input", + "default": "completion_input" + } + }, + "additionalProperties": false, + "required": [ + "type" + ] + }, + { + "type": "object", + "properties": { + "type": { + "type": "string", + "const": "agent_turn_input", + "default": "agent_turn_input" + } + }, + "additionalProperties": false, + "required": [ + "type" + ] + } + ] + }, + "description": { + "type": "string" + } + }, + "additionalProperties": false, + "required": [ + "name", + "type" + ] + }, + "ScoringFunctionDefWithProvider": { + "type": "object", + "properties": { + "identifier": { + "type": "string" + }, + "description": { + "type": "string" + }, + "metadata": { + "type": "object", + "additionalProperties": { + "oneOf": [ + { + "type": "null" + }, + { + "type": "boolean" + }, + { + "type": "number" + }, + { + "type": "string" + }, + { + "type": "array" + }, + { + "type": "object" + } + ] + } + }, + "parameters": { + "type": "array", + "items": { + "$ref": "#/components/schemas/Parameter" + } + }, + "return_type": { + "oneOf": [ + { + "type": "object", + "properties": { + "type": { + "type": "string", + "const": "string", + "default": "string" + } + }, + "additionalProperties": false, + "required": [ + "type" + ] + }, + { + "type": "object", + "properties": { + "type": { + "type": "string", + "const": "number", + "default": "number" + } + }, + "additionalProperties": false, + "required": [ + "type" + ] + }, + { + "type": "object", + "properties": { + "type": { + "type": "string", + "const": "boolean", + "default": "boolean" + } + }, + "additionalProperties": false, + "required": [ + "type" + ] + }, + { + "type": "object", + "properties": { + "type": { + "type": "string", + "const": "array", + "default": "array" + } + }, + "additionalProperties": false, + "required": [ + "type" + ] + }, + { + "type": "object", + "properties": { + "type": { + "type": "string", + "const": "object", + "default": "object" + } + }, + "additionalProperties": false, + "required": [ + "type" + ] + }, + { + "type": "object", + "properties": { + "type": { + "type": "string", + "const": "json", + "default": "json" + } + }, + "additionalProperties": false, + "required": [ + "type" + ] + }, + { + "type": "object", + "properties": { + "type": { + "type": "string", + "const": "union", + "default": "union" + } + }, + "additionalProperties": false, + "required": [ + "type" + ] + }, + { + "type": "object", + "properties": { + "type": { + "type": "string", + "const": "custom", + "default": "custom" + }, + "validator_class": { + "type": "string" + } + }, + "additionalProperties": false, + "required": [ + "type", + "validator_class" + ] + }, + { + "type": "object", + "properties": { + "type": { + "type": "string", + "const": "chat_completion_input", + "default": "chat_completion_input" + } + }, + "additionalProperties": false, + "required": [ + "type" + ] + }, + { + "type": "object", + "properties": { + "type": { + "type": "string", + "const": "completion_input", + "default": "completion_input" + } + }, + "additionalProperties": false, + "required": [ + "type" + ] + }, + { + "type": "object", + "properties": { + "type": { + "type": "string", + "const": "agent_turn_input", + "default": "agent_turn_input" + } + }, + "additionalProperties": false, + "required": [ + "type" + ] + } + ] + }, + "context": { + "type": "object", + "properties": { + "judge_model": { + "type": "string" + }, + "prompt_template": { + "type": "string" + } + }, + "additionalProperties": false, + "required": [ + "judge_model" + ] + }, + "provider_id": { + "type": "string" + } + }, + "additionalProperties": false, + "required": [ + "identifier", + "metadata", + "parameters", + "return_type", + "provider_id" + ] + }, "ShieldDefWithProvider": { "type": "object", "properties": { @@ -4898,6 +5904,25 @@ "documents" ] }, + "JobCancelRequest": { + "type": "object", + "properties": { + "job_id": { + "type": "string" + } + }, + "additionalProperties": false, + "required": [ + "job_id" + ] + }, + "JobStatus": { + "type": "string", + "enum": [ + "completed", + "in_progress" + ] + }, "ProviderInfo": { "type": "object", "properties": { @@ -5315,10 +6340,10 @@ "$ref": "#/components/schemas/URL" }, "dataset": { - "$ref": "#/components/schemas/TrainEvalDataset" + "type": "string" }, "validation_dataset": { - "$ref": "#/components/schemas/TrainEvalDataset" + "type": "string" }, "algorithm": { "$ref": "#/components/schemas/RLHFAlgorithm" @@ -5517,6 +6542,18 @@ "scores" ] }, + "RegisterDatasetRequest": { + "type": "object", + "properties": { + "dataset_def": { + "$ref": "#/components/schemas/DatasetDefWithProvider" + } + }, + "additionalProperties": false, + "required": [ + "dataset_def" + ] + }, "RegisterMemoryBankRequest": { "type": "object", "properties": { @@ -5554,6 +6591,18 @@ "model" ] }, + "RegisterScoringFunctionRequest": { + "type": "object", + "properties": { + "function_def": { + "$ref": "#/components/schemas/ScoringFunctionDefWithProvider" + } + }, + "additionalProperties": false, + "required": [ + "function_def" + ] + }, "RegisterShieldRequest": { "type": "object", "properties": { @@ -5566,153 +6615,6 @@ "shield" ] }, - "DialogGenerations": { - "type": "object", - "properties": { - "dialog": { - "type": "array", - "items": { - "oneOf": [ - { - "$ref": "#/components/schemas/UserMessage" - }, - { - "$ref": "#/components/schemas/SystemMessage" - }, - { - "$ref": "#/components/schemas/ToolResponseMessage" - }, - { - "$ref": "#/components/schemas/CompletionMessage" - } - ] - } - }, - "sampled_generations": { - "type": "array", - "items": { - "oneOf": [ - { - "$ref": "#/components/schemas/UserMessage" - }, - { - "$ref": "#/components/schemas/SystemMessage" - }, - { - "$ref": "#/components/schemas/ToolResponseMessage" - }, - { - "$ref": "#/components/schemas/CompletionMessage" - } - ] - } - } - }, - "additionalProperties": false, - "required": [ - "dialog", - "sampled_generations" - ] - }, - "RewardScoreRequest": { - "type": "object", - "properties": { - "dialog_generations": { - "type": "array", - "items": { - "$ref": "#/components/schemas/DialogGenerations" - } - }, - "model": { - "type": "string" - } - }, - "additionalProperties": false, - "required": [ - "dialog_generations", - "model" - ] - }, - "RewardScoringResponse": { - "type": "object", - "properties": { - "scored_generations": { - "type": "array", - "items": { - "$ref": "#/components/schemas/ScoredDialogGenerations" - } - } - }, - "additionalProperties": false, - "required": [ - "scored_generations" - ], - "title": "Response from the reward scoring. Batch of (prompt, response, score) tuples that pass the threshold." - }, - "ScoredDialogGenerations": { - "type": "object", - "properties": { - "dialog": { - "type": "array", - "items": { - "oneOf": [ - { - "$ref": "#/components/schemas/UserMessage" - }, - { - "$ref": "#/components/schemas/SystemMessage" - }, - { - "$ref": "#/components/schemas/ToolResponseMessage" - }, - { - "$ref": "#/components/schemas/CompletionMessage" - } - ] - } - }, - "scored_generations": { - "type": "array", - "items": { - "$ref": "#/components/schemas/ScoredMessage" - } - } - }, - "additionalProperties": false, - "required": [ - "dialog", - "scored_generations" - ] - }, - "ScoredMessage": { - "type": "object", - "properties": { - "message": { - "oneOf": [ - { - "$ref": "#/components/schemas/UserMessage" - }, - { - "$ref": "#/components/schemas/SystemMessage" - }, - { - "$ref": "#/components/schemas/ToolResponseMessage" - }, - { - "$ref": "#/components/schemas/CompletionMessage" - } - ] - }, - "score": { - "type": "number" - } - }, - "additionalProperties": false, - "required": [ - "message", - "score" - ] - }, "RunShieldRequest": { "type": "object", "properties": { @@ -5780,6 +6682,106 @@ }, "additionalProperties": false }, + "ScoreRequest": { + "type": "object", + "properties": { + "input_rows": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": { + "oneOf": [ + { + "type": "null" + }, + { + "type": "boolean" + }, + { + "type": "number" + }, + { + "type": "string" + }, + { + "type": "array" + }, + { + "type": "object" + } + ] + } + } + }, + "scoring_functions": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "additionalProperties": false, + "required": [ + "input_rows", + "scoring_functions" + ] + }, + "ScoreResponse": { + "type": "object", + "properties": { + "results": { + "type": "object", + "additionalProperties": { + "$ref": "#/components/schemas/ScoringResult" + } + } + }, + "additionalProperties": false, + "required": [ + "results" + ] + }, + "ScoreBatchRequest": { + "type": "object", + "properties": { + "dataset_id": { + "type": "string" + }, + "scoring_functions": { + "type": "array", + "items": { + "type": "string" + } + }, + "save_results_dataset": { + "type": "boolean" + } + }, + "additionalProperties": false, + "required": [ + "dataset_id", + "scoring_functions", + "save_results_dataset" + ] + }, + "ScoreBatchResponse": { + "type": "object", + "properties": { + "dataset_id": { + "type": "string" + }, + "results": { + "type": "object", + "additionalProperties": { + "$ref": "#/components/schemas/ScoringResult" + } + } + }, + "additionalProperties": false, + "required": [ + "results" + ] + }, "DoraFinetuningConfig": { "type": "object", "properties": { @@ -5892,10 +6894,10 @@ "type": "string" }, "dataset": { - "$ref": "#/components/schemas/TrainEvalDataset" + "type": "string" }, "validation_dataset": { - "$ref": "#/components/schemas/TrainEvalDataset" + "type": "string" }, "algorithm": { "$ref": "#/components/schemas/FinetuningAlgorithm" @@ -6034,7 +7036,29 @@ "synthetic_data": { "type": "array", "items": { - "$ref": "#/components/schemas/ScoredDialogGenerations" + "type": "object", + "additionalProperties": { + "oneOf": [ + { + "type": "null" + }, + { + "type": "boolean" + }, + { + "type": "number" + }, + { + "type": "string" + }, + { + "type": "array" + }, + { + "type": "object" + } + ] + } } }, "statistics": { @@ -6079,49 +7103,55 @@ ], "tags": [ { - "name": "Models" + "name": "Eval" }, { - "name": "RewardScoring" - }, - { - "name": "MemoryBanks" - }, - { - "name": "Shields" + "name": "ScoringFunctions" }, { "name": "SyntheticDataGeneration" }, - { - "name": "Inference" - }, { "name": "Inspect" }, - { - "name": "BatchInference" - }, - { - "name": "Memory" - }, - { - "name": "Datasets" - }, - { - "name": "Agents" - }, { "name": "PostTraining" }, { - "name": "Telemetry" + "name": "Models" }, { "name": "Safety" }, { - "name": "Evaluations" + "name": "MemoryBanks" + }, + { + "name": "DatasetIO" + }, + { + "name": "Memory" + }, + { + "name": "Scoring" + }, + { + "name": "Shields" + }, + { + "name": "Datasets" + }, + { + "name": "Inference" + }, + { + "name": "Telemetry" + }, + { + "name": "BatchInference" + }, + { + "name": "Agents" }, { "name": "BuiltinTool", @@ -6199,10 +7229,6 @@ "name": "BatchCompletionResponse", "description": "" }, - { - "name": "CancelEvaluationJobRequest", - "description": "" - }, { "name": "CancelTrainingJobRequest", "description": "" @@ -6371,18 +7397,6 @@ "name": "ViolationLevel", "description": "" }, - { - "name": "TrainEvalDataset", - "description": "Dataset to be used for training or evaluating language models.\n\n" - }, - { - "name": "TrainEvalDatasetColumnType", - "description": "" - }, - { - "name": "CreateDatasetRequest", - "description": "" - }, { "name": "DeleteAgentsRequest", "description": "" @@ -6391,10 +7405,6 @@ "name": "DeleteAgentsSessionRequest", "description": "" }, - { - "name": "DeleteDatasetRequest", - "description": "" - }, { "name": "EmbeddingsRequest", "description": "" @@ -6404,20 +7414,32 @@ "description": "" }, { - "name": "EvaluateQuestionAnsweringRequest", - "description": "" + "name": "AgentCandidate", + "description": "" }, { - "name": "EvaluationJob", - "description": "" + "name": "ModelCandidate", + "description": "" }, { - "name": "EvaluateSummarizationRequest", - "description": "" + "name": "EvaluateRequest", + "description": "" }, { - "name": "EvaluateTextGenerationRequest", - "description": "" + "name": "EvaluateResponse", + "description": "" + }, + { + "name": "ScoringResult", + "description": "" + }, + { + "name": "EvaluateBatchRequest", + "description": "" + }, + { + "name": "Job", + "description": "" }, { "name": "GetAgentsSessionRequest", @@ -6448,21 +7470,25 @@ "description": "" }, { - "name": "EvaluationJobArtifactsResponse", - "description": "Artifacts of a evaluation job.\n\n" - }, - { - "name": "EvaluationJobLogStream", - "description": "" - }, - { - "name": "EvaluationJobStatusResponse", - "description": "" + "name": "DatasetDefWithProvider", + "description": "" }, { "name": "ModelDefWithProvider", "description": "" }, + { + "name": "PaginatedRowsResult", + "description": "" + }, + { + "name": "Parameter", + "description": "" + }, + { + "name": "ScoringFunctionDefWithProvider", + "description": "" + }, { "name": "ShieldDefWithProvider", "description": "" @@ -6507,6 +7533,14 @@ "name": "InsertDocumentsRequest", "description": "" }, + { + "name": "JobCancelRequest", + "description": "" + }, + { + "name": "JobStatus", + "description": "" + }, { "name": "ProviderInfo", "description": "" @@ -6575,6 +7609,10 @@ "name": "QueryDocumentsResponse", "description": "" }, + { + "name": "RegisterDatasetRequest", + "description": "" + }, { "name": "RegisterMemoryBankRequest", "description": "" @@ -6583,30 +7621,14 @@ "name": "RegisterModelRequest", "description": "" }, + { + "name": "RegisterScoringFunctionRequest", + "description": "" + }, { "name": "RegisterShieldRequest", "description": "" }, - { - "name": "DialogGenerations", - "description": "" - }, - { - "name": "RewardScoreRequest", - "description": "" - }, - { - "name": "RewardScoringResponse", - "description": "Response from the reward scoring. Batch of (prompt, response, score) tuples that pass the threshold.\n\n" - }, - { - "name": "ScoredDialogGenerations", - "description": "" - }, - { - "name": "ScoredMessage", - "description": "" - }, { "name": "RunShieldRequest", "description": "" @@ -6615,6 +7637,22 @@ "name": "RunShieldResponse", "description": "" }, + { + "name": "ScoreRequest", + "description": "" + }, + { + "name": "ScoreResponse", + "description": "" + }, + { + "name": "ScoreBatchRequest", + "description": "" + }, + { + "name": "ScoreBatchResponse", + "description": "" + }, { "name": "DoraFinetuningConfig", "description": "" @@ -6650,16 +7688,18 @@ "tags": [ "Agents", "BatchInference", + "DatasetIO", "Datasets", - "Evaluations", + "Eval", "Inference", "Inspect", "Memory", "MemoryBanks", "Models", "PostTraining", - "RewardScoring", "Safety", + "Scoring", + "ScoringFunctions", "Shields", "SyntheticDataGeneration", "Telemetry" @@ -6668,6 +7708,7 @@ { "name": "Types", "tags": [ + "AgentCandidate", "AgentConfig", "AgentCreateResponse", "AgentSessionCreateResponse", @@ -6685,7 +7726,6 @@ "BatchCompletionRequest", "BatchCompletionResponse", "BuiltinTool", - "CancelEvaluationJobRequest", "CancelTrainingJobRequest", "ChatCompletionRequest", "ChatCompletionResponse", @@ -6701,22 +7741,16 @@ "CreateAgentRequest", "CreateAgentSessionRequest", "CreateAgentTurnRequest", - "CreateDatasetRequest", "DPOAlignmentConfig", + "DatasetDefWithProvider", "DeleteAgentsRequest", "DeleteAgentsSessionRequest", - "DeleteDatasetRequest", - "DialogGenerations", "DoraFinetuningConfig", "EmbeddingsRequest", "EmbeddingsResponse", - "EvaluateQuestionAnsweringRequest", - "EvaluateSummarizationRequest", - "EvaluateTextGenerationRequest", - "EvaluationJob", - "EvaluationJobArtifactsResponse", - "EvaluationJobLogStream", - "EvaluationJobStatusResponse", + "EvaluateBatchRequest", + "EvaluateRequest", + "EvaluateResponse", "FinetuningAlgorithm", "FunctionCallToolDefinition", "GetAgentsSessionRequest", @@ -6725,6 +7759,9 @@ "ImageMedia", "InferenceStep", "InsertDocumentsRequest", + "Job", + "JobCancelRequest", + "JobStatus", "KeyValueMemoryBankDef", "KeywordMemoryBankDef", "LogEventRequest", @@ -6734,8 +7771,11 @@ "MemoryRetrievalStep", "MemoryToolDefinition", "MetricEvent", + "ModelCandidate", "ModelDefWithProvider", "OptimizerConfig", + "PaginatedRowsResult", + "Parameter", "PhotogenToolDefinition", "PostTrainingJob", "PostTrainingJobArtifactsResponse", @@ -6748,21 +7788,25 @@ "QueryDocumentsRequest", "QueryDocumentsResponse", "RLHFAlgorithm", + "RegisterDatasetRequest", "RegisterMemoryBankRequest", "RegisterModelRequest", + "RegisterScoringFunctionRequest", "RegisterShieldRequest", "RestAPIExecutionConfig", "RestAPIMethod", - "RewardScoreRequest", - "RewardScoringResponse", "RouteInfo", "RunShieldRequest", "RunShieldResponse", "SafetyViolation", "SamplingParams", "SamplingStrategy", - "ScoredDialogGenerations", - "ScoredMessage", + "ScoreBatchRequest", + "ScoreBatchResponse", + "ScoreRequest", + "ScoreResponse", + "ScoringFunctionDefWithProvider", + "ScoringResult", "SearchToolDefinition", "Session", "ShieldCallStep", @@ -6788,8 +7832,6 @@ "ToolResponse", "ToolResponseMessage", "Trace", - "TrainEvalDataset", - "TrainEvalDatasetColumnType", "TrainingConfig", "Turn", "URL", diff --git a/docs/resources/llama-stack-spec.yaml b/docs/resources/llama-stack-spec.yaml index 906d3934a..9dcdbb028 100644 --- a/docs/resources/llama-stack-spec.yaml +++ b/docs/resources/llama-stack-spec.yaml @@ -1,6 +1,19 @@ components: responses: {} schemas: + AgentCandidate: + additionalProperties: false + properties: + config: + $ref: '#/components/schemas/AgentConfig' + type: + const: agent + default: agent + type: string + required: + - type + - config + type: object AgentConfig: additionalProperties: false properties: @@ -315,14 +328,6 @@ components: - photogen - code_interpreter type: string - CancelEvaluationJobRequest: - additionalProperties: false - properties: - job_uuid: - type: string - required: - - job_uuid - type: object CancelTrainingJobRequest: additionalProperties: false properties: @@ -351,6 +356,48 @@ components: type: array model: type: string + response_format: + oneOf: + - additionalProperties: false + properties: + schema: + additionalProperties: + oneOf: + - type: 'null' + - type: boolean + - type: number + - type: string + - type: array + - type: object + type: object + type: + const: json_schema + default: json_schema + type: string + required: + - type + - schema + type: object + - additionalProperties: false + properties: + bnf: + additionalProperties: + oneOf: + - type: 'null' + - type: boolean + - type: number + - type: string + - type: array + - type: object + type: object + type: + const: grammar + default: grammar + type: string + required: + - type + - bnf + type: object sampling_params: $ref: '#/components/schemas/SamplingParams' stream: @@ -490,6 +537,48 @@ components: type: object model: type: string + response_format: + oneOf: + - additionalProperties: false + properties: + schema: + additionalProperties: + oneOf: + - type: 'null' + - type: boolean + - type: number + - type: string + - type: array + - type: object + type: object + type: + const: json_schema + default: json_schema + type: string + required: + - type + - schema + type: object + - additionalProperties: false + properties: + bnf: + additionalProperties: + oneOf: + - type: 'null' + - type: boolean + - type: number + - type: string + - type: array + - type: object + type: object + type: + const: grammar + default: grammar + type: string + required: + - type + - bnf + type: object sampling_params: $ref: '#/components/schemas/SamplingParams' stream: @@ -572,17 +661,6 @@ components: - session_id - messages type: object - CreateDatasetRequest: - additionalProperties: false - properties: - dataset: - $ref: '#/components/schemas/TrainEvalDataset' - uuid: - type: string - required: - - uuid - - dataset - type: object DPOAlignmentConfig: additionalProperties: false properties: @@ -600,6 +678,138 @@ components: - epsilon - gamma type: object + DatasetDefWithProvider: + additionalProperties: false + properties: + dataset_schema: + additionalProperties: + oneOf: + - additionalProperties: false + properties: + type: + const: string + default: string + type: string + required: + - type + type: object + - additionalProperties: false + properties: + type: + const: number + default: number + type: string + required: + - type + type: object + - additionalProperties: false + properties: + type: + const: boolean + default: boolean + type: string + required: + - type + type: object + - additionalProperties: false + properties: + type: + const: array + default: array + type: string + required: + - type + type: object + - additionalProperties: false + properties: + type: + const: object + default: object + type: string + required: + - type + type: object + - additionalProperties: false + properties: + type: + const: json + default: json + type: string + required: + - type + type: object + - additionalProperties: false + properties: + type: + const: union + default: union + type: string + required: + - type + type: object + - additionalProperties: false + properties: + type: + const: custom + default: custom + type: string + validator_class: + type: string + required: + - type + - validator_class + type: object + - additionalProperties: false + properties: + type: + const: chat_completion_input + default: chat_completion_input + type: string + required: + - type + type: object + - additionalProperties: false + properties: + type: + const: completion_input + default: completion_input + type: string + required: + - type + type: object + - additionalProperties: false + properties: + type: + const: agent_turn_input + default: agent_turn_input + type: string + required: + - type + type: object + type: object + identifier: + type: string + metadata: + additionalProperties: + oneOf: + - type: 'null' + - type: boolean + - type: number + - type: string + - type: array + - type: object + type: object + provider_id: + type: string + url: + $ref: '#/components/schemas/URL' + required: + - identifier + - dataset_schema + - url + - metadata + - provider_id + type: object DeleteAgentsRequest: additionalProperties: false properties: @@ -619,37 +829,6 @@ components: - agent_id - session_id type: object - DeleteDatasetRequest: - additionalProperties: false - properties: - dataset_uuid: - type: string - required: - - dataset_uuid - type: object - DialogGenerations: - additionalProperties: false - properties: - dialog: - items: - oneOf: - - $ref: '#/components/schemas/UserMessage' - - $ref: '#/components/schemas/SystemMessage' - - $ref: '#/components/schemas/ToolResponseMessage' - - $ref: '#/components/schemas/CompletionMessage' - type: array - sampled_generations: - items: - oneOf: - - $ref: '#/components/schemas/UserMessage' - - $ref: '#/components/schemas/SystemMessage' - - $ref: '#/components/schemas/ToolResponseMessage' - - $ref: '#/components/schemas/CompletionMessage' - type: array - required: - - dialog - - sampled_generations - type: object DoraFinetuningConfig: additionalProperties: false properties: @@ -704,78 +883,74 @@ components: required: - embeddings type: object - EvaluateQuestionAnsweringRequest: + EvaluateBatchRequest: additionalProperties: false properties: - metrics: + candidate: + oneOf: + - $ref: '#/components/schemas/ModelCandidate' + - $ref: '#/components/schemas/AgentCandidate' + dataset_id: + type: string + scoring_functions: items: - enum: - - em - - f1 type: string type: array required: - - metrics + - dataset_id + - candidate + - scoring_functions type: object - EvaluateSummarizationRequest: + EvaluateRequest: additionalProperties: false properties: - metrics: + candidate: + oneOf: + - $ref: '#/components/schemas/ModelCandidate' + - $ref: '#/components/schemas/AgentCandidate' + input_rows: + items: + additionalProperties: + oneOf: + - type: 'null' + - type: boolean + - type: number + - type: string + - type: array + - type: object + type: object + type: array + scoring_functions: items: - enum: - - rouge - - bleu type: string type: array required: - - metrics + - input_rows + - candidate + - scoring_functions type: object - EvaluateTextGenerationRequest: + EvaluateResponse: additionalProperties: false properties: - metrics: + generations: items: - enum: - - perplexity - - rouge - - bleu - type: string + additionalProperties: + oneOf: + - type: 'null' + - type: boolean + - type: number + - type: string + - type: array + - type: object + type: object type: array + scores: + additionalProperties: + $ref: '#/components/schemas/ScoringResult' + type: object required: - - metrics - type: object - EvaluationJob: - additionalProperties: false - properties: - job_uuid: - type: string - required: - - job_uuid - type: object - EvaluationJobArtifactsResponse: - additionalProperties: false - properties: - job_uuid: - type: string - required: - - job_uuid - title: Artifacts of a evaluation job. - type: object - EvaluationJobLogStream: - additionalProperties: false - properties: - job_uuid: - type: string - required: - - job_uuid - type: object - EvaluationJobStatusResponse: - additionalProperties: false - properties: - job_uuid: - type: string - required: - - job_uuid + - generations + - scores type: object FinetuningAlgorithm: enum: @@ -905,6 +1080,27 @@ components: - bank_id - documents type: object + Job: + additionalProperties: false + properties: + job_id: + type: string + required: + - job_id + type: object + JobCancelRequest: + additionalProperties: false + properties: + job_id: + type: string + required: + - job_id + type: object + JobStatus: + enum: + - completed + - in_progress + type: string KeyValueMemoryBankDef: additionalProperties: false properties: @@ -1220,6 +1416,24 @@ components: - value - unit type: object + ModelCandidate: + additionalProperties: false + properties: + model: + type: string + sampling_params: + $ref: '#/components/schemas/SamplingParams' + system_message: + $ref: '#/components/schemas/SystemMessage' + type: + const: model + default: model + type: string + required: + - type + - model + - sampling_params + type: object ModelDefWithProvider: additionalProperties: false properties: @@ -1266,6 +1480,144 @@ components: - lr_min - weight_decay type: object + PaginatedRowsResult: + additionalProperties: false + properties: + next_page_token: + type: string + rows: + items: + additionalProperties: + oneOf: + - type: 'null' + - type: boolean + - type: number + - type: string + - type: array + - type: object + type: object + type: array + total_count: + type: integer + required: + - rows + - total_count + type: object + Parameter: + additionalProperties: false + properties: + description: + type: string + name: + type: string + type: + oneOf: + - additionalProperties: false + properties: + type: + const: string + default: string + type: string + required: + - type + type: object + - additionalProperties: false + properties: + type: + const: number + default: number + type: string + required: + - type + type: object + - additionalProperties: false + properties: + type: + const: boolean + default: boolean + type: string + required: + - type + type: object + - additionalProperties: false + properties: + type: + const: array + default: array + type: string + required: + - type + type: object + - additionalProperties: false + properties: + type: + const: object + default: object + type: string + required: + - type + type: object + - additionalProperties: false + properties: + type: + const: json + default: json + type: string + required: + - type + type: object + - additionalProperties: false + properties: + type: + const: union + default: union + type: string + required: + - type + type: object + - additionalProperties: false + properties: + type: + const: custom + default: custom + type: string + validator_class: + type: string + required: + - type + - validator_class + type: object + - additionalProperties: false + properties: + type: + const: chat_completion_input + default: chat_completion_input + type: string + required: + - type + type: object + - additionalProperties: false + properties: + type: + const: completion_input + default: completion_input + type: string + required: + - type + type: object + - additionalProperties: false + properties: + type: + const: agent_turn_input + default: agent_turn_input + type: string + required: + - type + type: object + required: + - name + - type + type: object PhotogenToolDefinition: additionalProperties: false properties: @@ -1373,7 +1725,7 @@ components: algorithm_config: $ref: '#/components/schemas/DPOAlignmentConfig' dataset: - $ref: '#/components/schemas/TrainEvalDataset' + type: string finetuned_model: $ref: '#/components/schemas/URL' hyperparam_search_config: @@ -1403,7 +1755,7 @@ components: training_config: $ref: '#/components/schemas/TrainingConfig' validation_dataset: - $ref: '#/components/schemas/TrainEvalDataset' + type: string required: - job_uuid - finetuned_model @@ -1515,6 +1867,14 @@ components: enum: - dpo type: string + RegisterDatasetRequest: + additionalProperties: false + properties: + dataset_def: + $ref: '#/components/schemas/DatasetDefWithProvider' + required: + - dataset_def + type: object RegisterMemoryBankRequest: additionalProperties: false properties: @@ -1535,6 +1895,14 @@ components: required: - model type: object + RegisterScoringFunctionRequest: + additionalProperties: false + properties: + function_def: + $ref: '#/components/schemas/ScoringFunctionDefWithProvider' + required: + - function_def + type: object RegisterShieldRequest: additionalProperties: false properties: @@ -1591,31 +1959,6 @@ components: - PUT - DELETE type: string - RewardScoreRequest: - additionalProperties: false - properties: - dialog_generations: - items: - $ref: '#/components/schemas/DialogGenerations' - type: array - model: - type: string - required: - - dialog_generations - - model - type: object - RewardScoringResponse: - additionalProperties: false - properties: - scored_generations: - items: - $ref: '#/components/schemas/ScoredDialogGenerations' - type: array - required: - - scored_generations - title: Response from the reward scoring. Batch of (prompt, response, score) - tuples that pass the threshold. - type: object RouteInfo: additionalProperties: false properties: @@ -1717,39 +2060,239 @@ components: - top_p - top_k type: string - ScoredDialogGenerations: + ScoreBatchRequest: additionalProperties: false properties: - dialog: + dataset_id: + type: string + save_results_dataset: + type: boolean + scoring_functions: items: - oneOf: - - $ref: '#/components/schemas/UserMessage' - - $ref: '#/components/schemas/SystemMessage' - - $ref: '#/components/schemas/ToolResponseMessage' - - $ref: '#/components/schemas/CompletionMessage' - type: array - scored_generations: - items: - $ref: '#/components/schemas/ScoredMessage' + type: string type: array required: - - dialog - - scored_generations + - dataset_id + - scoring_functions + - save_results_dataset type: object - ScoredMessage: + ScoreBatchResponse: additionalProperties: false properties: - message: - oneOf: - - $ref: '#/components/schemas/UserMessage' - - $ref: '#/components/schemas/SystemMessage' - - $ref: '#/components/schemas/ToolResponseMessage' - - $ref: '#/components/schemas/CompletionMessage' - score: - type: number + dataset_id: + type: string + results: + additionalProperties: + $ref: '#/components/schemas/ScoringResult' + type: object required: - - message - - score + - results + type: object + ScoreRequest: + additionalProperties: false + properties: + input_rows: + items: + additionalProperties: + oneOf: + - type: 'null' + - type: boolean + - type: number + - type: string + - type: array + - type: object + type: object + type: array + scoring_functions: + items: + type: string + type: array + required: + - input_rows + - scoring_functions + type: object + ScoreResponse: + additionalProperties: false + properties: + results: + additionalProperties: + $ref: '#/components/schemas/ScoringResult' + type: object + required: + - results + type: object + ScoringFunctionDefWithProvider: + additionalProperties: false + properties: + context: + additionalProperties: false + properties: + judge_model: + type: string + prompt_template: + type: string + required: + - judge_model + type: object + description: + type: string + identifier: + type: string + metadata: + additionalProperties: + oneOf: + - type: 'null' + - type: boolean + - type: number + - type: string + - type: array + - type: object + type: object + parameters: + items: + $ref: '#/components/schemas/Parameter' + type: array + provider_id: + type: string + return_type: + oneOf: + - additionalProperties: false + properties: + type: + const: string + default: string + type: string + required: + - type + type: object + - additionalProperties: false + properties: + type: + const: number + default: number + type: string + required: + - type + type: object + - additionalProperties: false + properties: + type: + const: boolean + default: boolean + type: string + required: + - type + type: object + - additionalProperties: false + properties: + type: + const: array + default: array + type: string + required: + - type + type: object + - additionalProperties: false + properties: + type: + const: object + default: object + type: string + required: + - type + type: object + - additionalProperties: false + properties: + type: + const: json + default: json + type: string + required: + - type + type: object + - additionalProperties: false + properties: + type: + const: union + default: union + type: string + required: + - type + type: object + - additionalProperties: false + properties: + type: + const: custom + default: custom + type: string + validator_class: + type: string + required: + - type + - validator_class + type: object + - additionalProperties: false + properties: + type: + const: chat_completion_input + default: chat_completion_input + type: string + required: + - type + type: object + - additionalProperties: false + properties: + type: + const: completion_input + default: completion_input + type: string + required: + - type + type: object + - additionalProperties: false + properties: + type: + const: agent_turn_input + default: agent_turn_input + type: string + required: + - type + type: object + required: + - identifier + - metadata + - parameters + - return_type + - provider_id + type: object + ScoringResult: + additionalProperties: false + properties: + aggregated_results: + additionalProperties: + oneOf: + - type: 'null' + - type: boolean + - type: number + - type: string + - type: array + - type: object + type: object + score_rows: + items: + additionalProperties: + oneOf: + - type: 'null' + - type: boolean + - type: number + - type: string + - type: array + - type: object + type: object + type: array + required: + - score_rows + - aggregated_results type: object SearchToolDefinition: additionalProperties: false @@ -1942,7 +2485,7 @@ components: - $ref: '#/components/schemas/QLoraFinetuningConfig' - $ref: '#/components/schemas/DoraFinetuningConfig' dataset: - $ref: '#/components/schemas/TrainEvalDataset' + type: string hyperparam_search_config: additionalProperties: oneOf: @@ -1972,7 +2515,7 @@ components: training_config: $ref: '#/components/schemas/TrainingConfig' validation_dataset: - $ref: '#/components/schemas/TrainEvalDataset' + type: string required: - job_uuid - model @@ -2027,7 +2570,15 @@ components: type: object synthetic_data: items: - $ref: '#/components/schemas/ScoredDialogGenerations' + additionalProperties: + oneOf: + - type: 'null' + - type: boolean + - type: number + - type: string + - type: array + - type: object + type: object type: array required: - synthetic_data @@ -2282,38 +2833,6 @@ components: - root_span_id - start_time type: object - TrainEvalDataset: - additionalProperties: false - properties: - columns: - additionalProperties: - $ref: '#/components/schemas/TrainEvalDatasetColumnType' - type: object - content_url: - $ref: '#/components/schemas/URL' - metadata: - additionalProperties: - oneOf: - - type: 'null' - - type: boolean - - type: number - - type: string - - type: array - - type: object - type: object - required: - - columns - - content_url - title: Dataset to be used for training or evaluating language models. - type: object - TrainEvalDatasetColumnType: - enum: - - dialog - - text - - media - - number - - json - type: string TrainingConfig: additionalProperties: false properties: @@ -2510,7 +3029,7 @@ info: description: "This is the specification of the llama stack that provides\n \ \ a set of endpoints and their corresponding interfaces that are tailored\ \ to\n best leverage Llama Models. The specification is still in\ - \ draft and subject to change.\n Generated at 2024-10-18 20:48:17.730988" + \ draft and subject to change.\n Generated at 2024-10-24 17:40:59.576117" title: '[DRAFT] Llama Stack Specification' version: 0.0.1 jsonSchemaDialect: https://json-schema.org/draft/2020-12/schema @@ -2651,6 +3170,11 @@ paths: required: true schema: type: string + - in: query + name: session_id + required: true + schema: + type: string - in: query name: turn_id required: true @@ -2710,6 +3234,11 @@ paths: required: true schema: type: string + - in: query + name: session_id + required: true + schema: + type: string - in: query name: turn_id required: true @@ -2781,9 +3310,29 @@ paths: description: OK tags: - BatchInference - /datasets/create: - post: + /datasetio/get_rows_paginated: + get: parameters: + - in: query + name: dataset_id + required: true + schema: + type: string + - in: query + name: rows_in_page + required: true + schema: + type: integer + - in: query + name: page_token + required: false + schema: + type: string + - in: query + name: filter_condition + required: false + schema: + type: string - description: JSON-encoded provider data which will be made available to the adapter servicing the API in: header @@ -2791,43 +3340,20 @@ paths: required: false schema: type: string - requestBody: - content: - application/json: - schema: - $ref: '#/components/schemas/CreateDatasetRequest' - required: true responses: '200': + content: + application/json: + schema: + $ref: '#/components/schemas/PaginatedRowsResult' description: OK tags: - - Datasets - /datasets/delete: - post: - parameters: - - description: JSON-encoded provider data which will be made available to the - adapter servicing the API - in: header - name: X-LlamaStack-ProviderData - required: false - schema: - type: string - requestBody: - content: - application/json: - schema: - $ref: '#/components/schemas/DeleteDatasetRequest' - required: true - responses: - '200': - description: OK - tags: - - Datasets + - DatasetIO /datasets/get: get: parameters: - in: query - name: dataset_uuid + name: dataset_identifier required: true schema: type: string @@ -2843,104 +3369,13 @@ paths: content: application/json: schema: - $ref: '#/components/schemas/TrainEvalDataset' + oneOf: + - $ref: '#/components/schemas/DatasetDefWithProvider' + - type: 'null' description: OK tags: - Datasets - /evaluate/job/artifacts: - get: - parameters: - - in: query - name: job_uuid - required: true - schema: - type: string - - description: JSON-encoded provider data which will be made available to the - adapter servicing the API - in: header - name: X-LlamaStack-ProviderData - required: false - schema: - type: string - responses: - '200': - content: - application/json: - schema: - $ref: '#/components/schemas/EvaluationJobArtifactsResponse' - description: OK - tags: - - Evaluations - /evaluate/job/cancel: - post: - parameters: - - description: JSON-encoded provider data which will be made available to the - adapter servicing the API - in: header - name: X-LlamaStack-ProviderData - required: false - schema: - type: string - requestBody: - content: - application/json: - schema: - $ref: '#/components/schemas/CancelEvaluationJobRequest' - required: true - responses: - '200': - description: OK - tags: - - Evaluations - /evaluate/job/logs: - get: - parameters: - - in: query - name: job_uuid - required: true - schema: - type: string - - description: JSON-encoded provider data which will be made available to the - adapter servicing the API - in: header - name: X-LlamaStack-ProviderData - required: false - schema: - type: string - responses: - '200': - content: - application/json: - schema: - $ref: '#/components/schemas/EvaluationJobLogStream' - description: OK - tags: - - Evaluations - /evaluate/job/status: - get: - parameters: - - in: query - name: job_uuid - required: true - schema: - type: string - - description: JSON-encoded provider data which will be made available to the - adapter servicing the API - in: header - name: X-LlamaStack-ProviderData - required: false - schema: - type: string - responses: - '200': - content: - application/json: - schema: - $ref: '#/components/schemas/EvaluationJobStatusResponse' - description: OK - tags: - - Evaluations - /evaluate/jobs: + /datasets/list: get: parameters: - description: JSON-encoded provider data which will be made available to the @@ -2955,11 +3390,11 @@ paths: content: application/jsonl: schema: - $ref: '#/components/schemas/EvaluationJob' + $ref: '#/components/schemas/DatasetDefWithProvider' description: OK tags: - - Evaluations - /evaluate/question_answering/: + - Datasets + /datasets/register: post: parameters: - description: JSON-encoded provider data which will be made available to the @@ -2973,18 +3408,14 @@ paths: content: application/json: schema: - $ref: '#/components/schemas/EvaluateQuestionAnsweringRequest' + $ref: '#/components/schemas/RegisterDatasetRequest' required: true responses: '200': - content: - application/json: - schema: - $ref: '#/components/schemas/EvaluationJob' description: OK tags: - - Evaluations - /evaluate/summarization/: + - Datasets + /eval/evaluate: post: parameters: - description: JSON-encoded provider data which will be made available to the @@ -2998,18 +3429,18 @@ paths: content: application/json: schema: - $ref: '#/components/schemas/EvaluateSummarizationRequest' + $ref: '#/components/schemas/EvaluateRequest' required: true responses: '200': content: application/json: schema: - $ref: '#/components/schemas/EvaluationJob' + $ref: '#/components/schemas/EvaluateResponse' description: OK tags: - - Evaluations - /evaluate/text_generation/: + - Eval + /eval/evaluate_batch: post: parameters: - description: JSON-encoded provider data which will be made available to the @@ -3023,17 +3454,88 @@ paths: content: application/json: schema: - $ref: '#/components/schemas/EvaluateTextGenerationRequest' + $ref: '#/components/schemas/EvaluateBatchRequest' required: true responses: '200': content: application/json: schema: - $ref: '#/components/schemas/EvaluationJob' + $ref: '#/components/schemas/Job' description: OK tags: - - Evaluations + - Eval + /eval/job/cancel: + post: + parameters: + - description: JSON-encoded provider data which will be made available to the + adapter servicing the API + in: header + name: X-LlamaStack-ProviderData + required: false + schema: + type: string + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/JobCancelRequest' + required: true + responses: + '200': + description: OK + tags: + - Eval + /eval/job/result: + get: + parameters: + - in: query + name: job_id + required: true + schema: + type: string + - description: JSON-encoded provider data which will be made available to the + adapter servicing the API + in: header + name: X-LlamaStack-ProviderData + required: false + schema: + type: string + responses: + '200': + content: + application/json: + schema: + $ref: '#/components/schemas/EvaluateResponse' + description: OK + tags: + - Eval + /eval/job/status: + get: + parameters: + - in: query + name: job_id + required: true + schema: + type: string + - description: JSON-encoded provider data which will be made available to the + adapter servicing the API + in: header + name: X-LlamaStack-ProviderData + required: false + schema: + type: string + responses: + '200': + content: + application/json: + schema: + oneOf: + - $ref: '#/components/schemas/JobStatus' + - type: 'null' + description: OK + tags: + - Eval /health: get: parameters: @@ -3501,31 +4003,6 @@ paths: description: OK tags: - Inspect - /reward_scoring/score: - post: - parameters: - - description: JSON-encoded provider data which will be made available to the - adapter servicing the API - in: header - name: X-LlamaStack-ProviderData - required: false - schema: - type: string - requestBody: - content: - application/json: - schema: - $ref: '#/components/schemas/RewardScoreRequest' - required: true - responses: - '200': - content: - application/json: - schema: - $ref: '#/components/schemas/RewardScoringResponse' - description: OK - tags: - - RewardScoring /routes/list: get: parameters: @@ -3574,6 +4051,122 @@ paths: description: OK tags: - Safety + /scoring/score: + post: + parameters: + - description: JSON-encoded provider data which will be made available to the + adapter servicing the API + in: header + name: X-LlamaStack-ProviderData + required: false + schema: + type: string + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/ScoreRequest' + required: true + responses: + '200': + content: + application/json: + schema: + $ref: '#/components/schemas/ScoreResponse' + description: OK + tags: + - Scoring + /scoring/score_batch: + post: + parameters: + - description: JSON-encoded provider data which will be made available to the + adapter servicing the API + in: header + name: X-LlamaStack-ProviderData + required: false + schema: + type: string + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/ScoreBatchRequest' + required: true + responses: + '200': + content: + application/json: + schema: + $ref: '#/components/schemas/ScoreBatchResponse' + description: OK + tags: + - Scoring + /scoring_functions/get: + get: + parameters: + - in: query + name: name + required: true + schema: + type: string + - description: JSON-encoded provider data which will be made available to the + adapter servicing the API + in: header + name: X-LlamaStack-ProviderData + required: false + schema: + type: string + responses: + '200': + content: + application/json: + schema: + oneOf: + - $ref: '#/components/schemas/ScoringFunctionDefWithProvider' + - type: 'null' + description: OK + tags: + - ScoringFunctions + /scoring_functions/list: + get: + parameters: + - description: JSON-encoded provider data which will be made available to the + adapter servicing the API + in: header + name: X-LlamaStack-ProviderData + required: false + schema: + type: string + responses: + '200': + content: + application/jsonl: + schema: + $ref: '#/components/schemas/ScoringFunctionDefWithProvider' + description: OK + tags: + - ScoringFunctions + /scoring_functions/register: + post: + parameters: + - description: JSON-encoded provider data which will be made available to the + adapter servicing the API + in: header + name: X-LlamaStack-ProviderData + required: false + schema: + type: string + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/RegisterScoringFunctionRequest' + required: true + responses: + '200': + description: OK + tags: + - ScoringFunctions /shields/get: get: parameters: @@ -3715,21 +4308,23 @@ security: servers: - url: http://any-hosted-llama-stack.com tags: -- name: Models -- name: RewardScoring -- name: MemoryBanks -- name: Shields +- name: Eval +- name: ScoringFunctions - name: SyntheticDataGeneration -- name: Inference - name: Inspect -- name: BatchInference -- name: Memory -- name: Datasets -- name: Agents - name: PostTraining -- name: Telemetry +- name: Models - name: Safety -- name: Evaluations +- name: MemoryBanks +- name: DatasetIO +- name: Memory +- name: Scoring +- name: Shields +- name: Datasets +- name: Inference +- name: Telemetry +- name: BatchInference +- name: Agents - description: name: BuiltinTool - description: name: BatchCompletionResponse -- description: - name: CancelEvaluationJobRequest - description: name: CancelTrainingJobRequest @@ -3922,43 +4514,35 @@ tags: name: Turn - description: name: ViolationLevel -- description: 'Dataset to be used for training or evaluating language models. - - - ' - name: TrainEvalDataset -- description: - name: TrainEvalDatasetColumnType -- description: - name: CreateDatasetRequest - description: name: DeleteAgentsRequest - description: name: DeleteAgentsSessionRequest -- description: - name: DeleteDatasetRequest - description: name: EmbeddingsRequest - description: name: EmbeddingsResponse -- description: + name: AgentCandidate +- description: + name: ModelCandidate +- description: - name: EvaluateQuestionAnsweringRequest -- description: - name: EvaluationJob -- description: - name: EvaluateSummarizationRequest -- description: + name: ScoringResult +- description: - name: EvaluateTextGenerationRequest + name: EvaluateBatchRequest +- description: + name: Job - description: name: GetAgentsSessionRequest @@ -3982,21 +4566,20 @@ tags: - description: name: AgentStepResponse -- description: 'Artifacts of a evaluation job. - - - ' - name: EvaluationJobArtifactsResponse -- description: - name: EvaluationJobLogStream -- description: - name: EvaluationJobStatusResponse + name: DatasetDefWithProvider - description: name: ModelDefWithProvider +- description: + name: PaginatedRowsResult +- description: + name: Parameter +- description: + name: ScoringFunctionDefWithProvider - description: name: ShieldDefWithProvider @@ -4038,6 +4621,11 @@ tags: - description: name: InsertDocumentsRequest +- description: + name: JobCancelRequest +- description: + name: JobStatus - description: name: ProviderInfo - description: @@ -4081,38 +4669,37 @@ tags: - description: name: QueryDocumentsResponse +- description: + name: RegisterDatasetRequest - description: name: RegisterMemoryBankRequest - description: name: RegisterModelRequest +- description: + name: RegisterScoringFunctionRequest - description: name: RegisterShieldRequest -- description: - name: DialogGenerations -- description: - name: RewardScoreRequest -- description: 'Response from the reward scoring. Batch of (prompt, response, score) - tuples that pass the threshold. - - - ' - name: RewardScoringResponse -- description: - name: ScoredDialogGenerations -- description: - name: ScoredMessage - description: name: RunShieldRequest - description: name: RunShieldResponse +- description: + name: ScoreRequest +- description: + name: ScoreResponse +- description: + name: ScoreBatchRequest +- description: + name: ScoreBatchResponse - description: name: DoraFinetuningConfig @@ -4143,21 +4730,24 @@ x-tagGroups: tags: - Agents - BatchInference + - DatasetIO - Datasets - - Evaluations + - Eval - Inference - Inspect - Memory - MemoryBanks - Models - PostTraining - - RewardScoring - Safety + - Scoring + - ScoringFunctions - Shields - SyntheticDataGeneration - Telemetry - name: Types tags: + - AgentCandidate - AgentConfig - AgentCreateResponse - AgentSessionCreateResponse @@ -4175,7 +4765,6 @@ x-tagGroups: - BatchCompletionRequest - BatchCompletionResponse - BuiltinTool - - CancelEvaluationJobRequest - CancelTrainingJobRequest - ChatCompletionRequest - ChatCompletionResponse @@ -4191,22 +4780,16 @@ x-tagGroups: - CreateAgentRequest - CreateAgentSessionRequest - CreateAgentTurnRequest - - CreateDatasetRequest - DPOAlignmentConfig + - DatasetDefWithProvider - DeleteAgentsRequest - DeleteAgentsSessionRequest - - DeleteDatasetRequest - - DialogGenerations - DoraFinetuningConfig - EmbeddingsRequest - EmbeddingsResponse - - EvaluateQuestionAnsweringRequest - - EvaluateSummarizationRequest - - EvaluateTextGenerationRequest - - EvaluationJob - - EvaluationJobArtifactsResponse - - EvaluationJobLogStream - - EvaluationJobStatusResponse + - EvaluateBatchRequest + - EvaluateRequest + - EvaluateResponse - FinetuningAlgorithm - FunctionCallToolDefinition - GetAgentsSessionRequest @@ -4215,6 +4798,9 @@ x-tagGroups: - ImageMedia - InferenceStep - InsertDocumentsRequest + - Job + - JobCancelRequest + - JobStatus - KeyValueMemoryBankDef - KeywordMemoryBankDef - LogEventRequest @@ -4224,8 +4810,11 @@ x-tagGroups: - MemoryRetrievalStep - MemoryToolDefinition - MetricEvent + - ModelCandidate - ModelDefWithProvider - OptimizerConfig + - PaginatedRowsResult + - Parameter - PhotogenToolDefinition - PostTrainingJob - PostTrainingJobArtifactsResponse @@ -4238,21 +4827,25 @@ x-tagGroups: - QueryDocumentsRequest - QueryDocumentsResponse - RLHFAlgorithm + - RegisterDatasetRequest - RegisterMemoryBankRequest - RegisterModelRequest + - RegisterScoringFunctionRequest - RegisterShieldRequest - RestAPIExecutionConfig - RestAPIMethod - - RewardScoreRequest - - RewardScoringResponse - RouteInfo - RunShieldRequest - RunShieldResponse - SafetyViolation - SamplingParams - SamplingStrategy - - ScoredDialogGenerations - - ScoredMessage + - ScoreBatchRequest + - ScoreBatchResponse + - ScoreRequest + - ScoreResponse + - ScoringFunctionDefWithProvider + - ScoringResult - SearchToolDefinition - Session - ShieldCallStep @@ -4278,8 +4871,6 @@ x-tagGroups: - ToolResponse - ToolResponseMessage - Trace - - TrainEvalDataset - - TrainEvalDatasetColumnType - TrainingConfig - Turn - URL diff --git a/llama_stack/apis/common/type_system.py b/llama_stack/apis/common/type_system.py index 5f94e85f3..cffc4e936 100644 --- a/llama_stack/apis/common/type_system.py +++ b/llama_stack/apis/common/type_system.py @@ -4,7 +4,7 @@ # This source code is licensed under the terms described in the LICENSE file in # the root directory of this source tree. -from typing import Dict, List, Literal, Union +from typing import Literal, Union from pydantic import BaseModel, Field from typing_extensions import Annotated @@ -24,12 +24,10 @@ class BooleanType(BaseModel): class ArrayType(BaseModel): type: Literal["array"] = "array" - items: "ParamType" class ObjectType(BaseModel): type: Literal["object"] = "object" - properties: Dict[str, "ParamType"] = Field(default_factory=dict) class JsonType(BaseModel): @@ -38,7 +36,6 @@ class JsonType(BaseModel): class UnionType(BaseModel): type: Literal["union"] = "union" - options: List["ParamType"] = Field(default_factory=list) class CustomType(BaseModel): @@ -77,7 +74,3 @@ ParamType = Annotated[ ], Field(discriminator="type"), ] - -ArrayType.model_rebuild() -ObjectType.model_rebuild() -UnionType.model_rebuild() diff --git a/llama_stack/apis/post_training/post_training.py b/llama_stack/apis/post_training/post_training.py index d943f48b2..abf21f6c2 100644 --- a/llama_stack/apis/post_training/post_training.py +++ b/llama_stack/apis/post_training/post_training.py @@ -14,7 +14,7 @@ from llama_models.schema_utils import json_schema_type, webmethod from pydantic import BaseModel, Field from llama_models.llama3.api.datatypes import * # noqa: F403 -from llama_stack.apis.dataset import * # noqa: F403 +from llama_stack.apis.datasets import * # noqa: F403 from llama_stack.apis.common.training_types import * # noqa: F403 @@ -107,8 +107,8 @@ class PostTrainingSFTRequest(BaseModel): job_uuid: str model: str - dataset: TrainEvalDataset - validation_dataset: TrainEvalDataset + dataset: str + validation_dataset: str algorithm: FinetuningAlgorithm algorithm_config: Union[ @@ -131,8 +131,8 @@ class PostTrainingRLHFRequest(BaseModel): finetuned_model: URL - dataset: TrainEvalDataset - validation_dataset: TrainEvalDataset + dataset: str + validation_dataset: str algorithm: RLHFAlgorithm algorithm_config: Union[DPOAlignmentConfig] @@ -181,8 +181,8 @@ class PostTraining(Protocol): self, job_uuid: str, model: str, - dataset: TrainEvalDataset, - validation_dataset: TrainEvalDataset, + dataset: str, + validation_dataset: str, algorithm: FinetuningAlgorithm, algorithm_config: Union[ LoraFinetuningConfig, QLoraFinetuningConfig, DoraFinetuningConfig @@ -198,8 +198,8 @@ class PostTraining(Protocol): self, job_uuid: str, finetuned_model: URL, - dataset: TrainEvalDataset, - validation_dataset: TrainEvalDataset, + dataset: str, + validation_dataset: str, algorithm: RLHFAlgorithm, algorithm_config: Union[DPOAlignmentConfig], optimizer_config: OptimizerConfig, diff --git a/llama_stack/apis/synthetic_data_generation/synthetic_data_generation.py b/llama_stack/apis/synthetic_data_generation/synthetic_data_generation.py index 60c756128..05b49036d 100644 --- a/llama_stack/apis/synthetic_data_generation/synthetic_data_generation.py +++ b/llama_stack/apis/synthetic_data_generation/synthetic_data_generation.py @@ -13,7 +13,6 @@ from llama_models.schema_utils import json_schema_type, webmethod from pydantic import BaseModel from llama_models.llama3.api.datatypes import * # noqa: F403 -from llama_stack.apis.reward_scoring import * # noqa: F403 class FilteringFunction(Enum): @@ -40,7 +39,7 @@ class SyntheticDataGenerationRequest(BaseModel): class SyntheticDataGenerationResponse(BaseModel): """Response from the synthetic data generation. Batch of (prompt, response, score) tuples that pass the threshold.""" - synthetic_data: List[ScoredDialogGenerations] + synthetic_data: List[Dict[str, Any]] statistics: Optional[Dict[str, Any]] = None