llama-stack-mirror/docs/static/experimental-llama-stack-spec.yaml

openapi: 3.1.0
info:
  title: >-
    Llama Stack Specification - Experimental APIs
  version: v1
  description: >-
    This is the specification of the Llama Stack that provides
                    a set of endpoints and their corresponding interfaces that are
    tailored to
                    best leverage Llama Models.

    **🧪 EXPERIMENTAL**: Pre-release APIs (v1alpha, v1beta) that may change before
    becoming stable.
servers:
  - url: http://any-hosted-llama-stack.com
paths:
  /v1beta/datasetio/append-rows/{dataset_id}:
    post:
      responses:
        '200':
          description: OK
        '400':
          $ref: '#/components/responses/BadRequest400'
        '429':
          $ref: >-
            #/components/responses/TooManyRequests429
        '500':
          $ref: >-
            #/components/responses/InternalServerError500
        default:
          $ref: '#/components/responses/DefaultError'
      tags:
        - DatasetIO
      summary: Append rows to a dataset.
      description: Append rows to a dataset.
      parameters:
        - name: dataset_id
          in: path
          description: >-
            The ID of the dataset to append the rows to.
          required: true
          schema:
            type: string
      requestBody:
        content:
          application/json:
            schema:
              $ref: '#/components/schemas/AppendRowsRequest'
        required: true
      deprecated: false
  /v1beta/datasetio/iterrows/{dataset_id}:
    get:
      responses:
        '200':
          description: A PaginatedResponse.
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/PaginatedResponse'
        '400':
          $ref: '#/components/responses/BadRequest400'
        '429':
          $ref: >-
            #/components/responses/TooManyRequests429
        '500':
          $ref: >-
            #/components/responses/InternalServerError500
        default:
          $ref: '#/components/responses/DefaultError'
      tags:
        - DatasetIO
      summary: >-
        Get a paginated list of rows from a dataset.
      description: >-
        Get a paginated list of rows from a dataset.

        Uses offset-based pagination where:

        - start_index: The starting index (0-based). If None, starts from beginning.

        - limit: Number of items to return. If None or -1, returns all items.


        The response includes:

        - data: List of items for the current page.

        - has_more: Whether there are more items available after this set.
      parameters:
        - name: dataset_id
          in: path
          description: >-
            The ID of the dataset to get the rows from.
          required: true
          schema:
            type: string
        - name: start_index
          in: query
          description: >-
            Index into dataset for the first row to get. Get all rows if None.
          required: false
          schema:
            type: integer
        - name: limit
          in: query
          description: The number of rows to get.
          required: false
          schema:
            type: integer
      deprecated: false
  /v1beta/datasets:
    get:
      responses:
        '200':
          description: A ListDatasetsResponse.
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/ListDatasetsResponse'
        '400':
          $ref: '#/components/responses/BadRequest400'
        '429':
          $ref: >-
            #/components/responses/TooManyRequests429
        '500':
          $ref: >-
            #/components/responses/InternalServerError500
        default:
          $ref: '#/components/responses/DefaultError'
      tags:
        - Datasets
      summary: List all datasets.
      description: List all datasets.
      parameters: []
      deprecated: false
    post:
      responses:
        '200':
          description: A Dataset.
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/Dataset'
        '400':
          $ref: '#/components/responses/BadRequest400'
        '429':
          $ref: >-
            #/components/responses/TooManyRequests429
        '500':
          $ref: >-
            #/components/responses/InternalServerError500
        default:
          $ref: '#/components/responses/DefaultError'
      tags:
        - Datasets
      summary: Register a new dataset.
      description: Register a new dataset.
      parameters: []
      requestBody:
        content:
          application/json:
            schema:
              $ref: '#/components/schemas/RegisterDatasetRequest'
        required: true
      deprecated: true
  /v1beta/datasets/{dataset_id}:
    get:
      responses:
        '200':
          description: A Dataset.
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/Dataset'
        '400':
          $ref: '#/components/responses/BadRequest400'
        '429':
          $ref: >-
            #/components/responses/TooManyRequests429
        '500':
          $ref: >-
            #/components/responses/InternalServerError500
        default:
          $ref: '#/components/responses/DefaultError'
      tags:
        - Datasets
      summary: Get a dataset by its ID.
      description: Get a dataset by its ID.
      parameters:
        - name: dataset_id
          in: path
          description: The ID of the dataset to get.
          required: true
          schema:
            type: string
      deprecated: false
    delete:
      responses:
        '200':
          description: OK
        '400':
          $ref: '#/components/responses/BadRequest400'
        '429':
          $ref: >-
            #/components/responses/TooManyRequests429
        '500':
          $ref: >-
            #/components/responses/InternalServerError500
        default:
          $ref: '#/components/responses/DefaultError'
      tags:
        - Datasets
      summary: Unregister a dataset by its ID.
      description: Unregister a dataset by its ID.
      parameters:
        - name: dataset_id
          in: path
          description: The ID of the dataset to unregister.
          required: true
          schema:
            type: string
      deprecated: true
  /v1alpha/eval/benchmarks:
    get:
      responses:
        '200':
          description: A ListBenchmarksResponse.
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/ListBenchmarksResponse'
        '400':
          $ref: '#/components/responses/BadRequest400'
        '429':
          $ref: >-
            #/components/responses/TooManyRequests429
        '500':
          $ref: >-
            #/components/responses/InternalServerError500
        default:
          $ref: '#/components/responses/DefaultError'
      tags:
        - Benchmarks
      summary: List all benchmarks.
      description: List all benchmarks.
      parameters: []
      deprecated: false
    post:
      responses:
        '200':
          description: OK
        '400':
          $ref: '#/components/responses/BadRequest400'
        '429':
          $ref: >-
            #/components/responses/TooManyRequests429
        '500':
          $ref: >-
            #/components/responses/InternalServerError500
        default:
          $ref: '#/components/responses/DefaultError'
      tags:
        - Benchmarks
      summary: Register a benchmark.
      description: Register a benchmark.
      parameters: []
      requestBody:
        content:
          application/json:
            schema:
              $ref: '#/components/schemas/RegisterBenchmarkRequest'
        required: true
      deprecated: true
  /v1alpha/eval/benchmarks/{benchmark_id}:
    get:
      responses:
        '200':
          description: A Benchmark.
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/Benchmark'
        '400':
          $ref: '#/components/responses/BadRequest400'
        '429':
          $ref: >-
            #/components/responses/TooManyRequests429
        '500':
          $ref: >-
            #/components/responses/InternalServerError500
        default:
          $ref: '#/components/responses/DefaultError'
      tags:
        - Benchmarks
      summary: Get a benchmark by its ID.
      description: Get a benchmark by its ID.
      parameters:
        - name: benchmark_id
          in: path
          description: The ID of the benchmark to get.
          required: true
          schema:
            type: string
      deprecated: false
    delete:
      responses:
        '200':
          description: OK
        '400':
          $ref: '#/components/responses/BadRequest400'
        '429':
          $ref: >-
            #/components/responses/TooManyRequests429
        '500':
          $ref: >-
            #/components/responses/InternalServerError500
        default:
          $ref: '#/components/responses/DefaultError'
      tags:
        - Benchmarks
      summary: Unregister a benchmark.
      description: Unregister a benchmark.
      parameters:
        - name: benchmark_id
          in: path
          description: The ID of the benchmark to unregister.
          required: true
          schema:
            type: string
      deprecated: true
  /v1alpha/eval/benchmarks/{benchmark_id}/evaluations:
    post:
      responses:
        '200':
          description: >-
            EvaluateResponse object containing generations and scores.
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/EvaluateResponse'
        '400':
          $ref: '#/components/responses/BadRequest400'
        '429':
          $ref: >-
            #/components/responses/TooManyRequests429
        '500':
          $ref: >-
            #/components/responses/InternalServerError500
        default:
          $ref: '#/components/responses/DefaultError'
      tags:
        - Eval
      summary: Evaluate a list of rows on a benchmark.
      description: Evaluate a list of rows on a benchmark.
      parameters:
        - name: benchmark_id
          in: path
          description: >-
            The ID of the benchmark to run the evaluation on.
          required: true
          schema:
            type: string
      requestBody:
        content:
          application/json:
            schema:
              $ref: '#/components/schemas/EvaluateRowsRequest'
        required: true
      deprecated: false
  /v1alpha/eval/benchmarks/{benchmark_id}/jobs:
    post:
      responses:
        '200':
          description: >-
            The job that was created to run the evaluation.
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/Job'
        '400':
          $ref: '#/components/responses/BadRequest400'
        '429':
          $ref: >-
            #/components/responses/TooManyRequests429
        '500':
          $ref: >-
            #/components/responses/InternalServerError500
        default:
          $ref: '#/components/responses/DefaultError'
      tags:
        - Eval
      summary: Run an evaluation on a benchmark.
      description: Run an evaluation on a benchmark.
      parameters:
        - name: benchmark_id
          in: path
          description: >-
            The ID of the benchmark to run the evaluation on.
          required: true
          schema:
            type: string
      requestBody:
        content:
          application/json:
            schema:
              $ref: '#/components/schemas/RunEvalRequest'
        required: true
      deprecated: false
  /v1alpha/eval/benchmarks/{benchmark_id}/jobs/{job_id}:
    get:
      responses:
        '200':
          description: The status of the evaluation job.
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/Job'
        '400':
          $ref: '#/components/responses/BadRequest400'
        '429':
          $ref: >-
            #/components/responses/TooManyRequests429
        '500':
          $ref: >-
            #/components/responses/InternalServerError500
        default:
          $ref: '#/components/responses/DefaultError'
      tags:
        - Eval
      summary: Get the status of a job.
      description: Get the status of a job.
      parameters:
        - name: benchmark_id
          in: path
          description: >-
            The ID of the benchmark to run the evaluation on.
          required: true
          schema:
            type: string
        - name: job_id
          in: path
          description: The ID of the job to get the status of.
          required: true
          schema:
            type: string
      deprecated: false
    delete:
      responses:
        '200':
          description: OK
        '400':
          $ref: '#/components/responses/BadRequest400'
        '429':
          $ref: >-
            #/components/responses/TooManyRequests429
        '500':
          $ref: >-
            #/components/responses/InternalServerError500
        default:
          $ref: '#/components/responses/DefaultError'
      tags:
        - Eval
      summary: Cancel a job.
      description: Cancel a job.
      parameters:
        - name: benchmark_id
          in: path
          description: >-
            The ID of the benchmark to run the evaluation on.
          required: true
          schema:
            type: string
        - name: job_id
          in: path
          description: The ID of the job to cancel.
          required: true
          schema:
            type: string
      deprecated: false
  /v1alpha/eval/benchmarks/{benchmark_id}/jobs/{job_id}/result:
    get:
      responses:
        '200':
          description: The result of the job.
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/EvaluateResponse'
        '400':
          $ref: '#/components/responses/BadRequest400'
        '429':
          $ref: >-
            #/components/responses/TooManyRequests429
        '500':
          $ref: >-
            #/components/responses/InternalServerError500
        default:
          $ref: '#/components/responses/DefaultError'
      tags:
        - Eval
      summary: Get the result of a job.
      description: Get the result of a job.
      parameters:
        - name: benchmark_id
          in: path
          description: >-
            The ID of the benchmark to run the evaluation on.
          required: true
          schema:
            type: string
        - name: job_id
          in: path
          description: The ID of the job to get the result of.
          required: true
          schema:
            type: string
      deprecated: false
  /v1alpha/inference/rerank:
    post:
      responses:
        '200':
          description: >-
            RerankResponse with indices sorted by relevance score (descending).
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/RerankResponse'
        '400':
          $ref: '#/components/responses/BadRequest400'
        '429':
          $ref: >-
            #/components/responses/TooManyRequests429
        '500':
          $ref: >-
            #/components/responses/InternalServerError500
        default:
          $ref: '#/components/responses/DefaultError'
      tags:
        - Inference
      summary: >-
        Rerank a list of documents based on their relevance to a query.
      description: >-
        Rerank a list of documents based on their relevance to a query.
      parameters: []
      requestBody:
        content:
          application/json:
            schema:
              $ref: '#/components/schemas/RerankRequest'
        required: true
      deprecated: false
  /v1alpha/post-training/job/artifacts:
    get:
      responses:
        '200':
          description: A PostTrainingJobArtifactsResponse.
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/PostTrainingJobArtifactsResponse'
        '400':
          $ref: '#/components/responses/BadRequest400'
        '429':
          $ref: >-
            #/components/responses/TooManyRequests429
        '500':
          $ref: >-
            #/components/responses/InternalServerError500
        default:
          $ref: '#/components/responses/DefaultError'
      tags:
        - PostTraining (Coming Soon)
      summary: Get the artifacts of a training job.
      description: Get the artifacts of a training job.
      parameters:
        - name: job_uuid
          in: query
          description: >-
            The UUID of the job to get the artifacts of.
          required: true
          schema:
            type: string
      deprecated: false
  /v1alpha/post-training/job/cancel:
    post:
      responses:
        '200':
          description: OK
        '400':
          $ref: '#/components/responses/BadRequest400'
        '429':
          $ref: >-
            #/components/responses/TooManyRequests429
        '500':
          $ref: >-
            #/components/responses/InternalServerError500
        default:
          $ref: '#/components/responses/DefaultError'
      tags:
        - PostTraining (Coming Soon)
      summary: Cancel a training job.
      description: Cancel a training job.
      parameters: []
      requestBody:
        content:
          application/json:
            schema:
              $ref: '#/components/schemas/CancelTrainingJobRequest'
        required: true
      deprecated: false
  /v1alpha/post-training/job/status:
    get:
      responses:
        '200':
          description: A PostTrainingJobStatusResponse.
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/PostTrainingJobStatusResponse'
        '400':
          $ref: '#/components/responses/BadRequest400'
        '429':
          $ref: >-
            #/components/responses/TooManyRequests429
        '500':
          $ref: >-
            #/components/responses/InternalServerError500
        default:
          $ref: '#/components/responses/DefaultError'
      tags:
        - PostTraining (Coming Soon)
      summary: Get the status of a training job.
      description: Get the status of a training job.
      parameters:
        - name: job_uuid
          in: query
          description: >-
            The UUID of the job to get the status of.
          required: true
          schema:
            type: string
      deprecated: false
  /v1alpha/post-training/jobs:
    get:
      responses:
        '200':
          description: A ListPostTrainingJobsResponse.
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/ListPostTrainingJobsResponse'
        '400':
          $ref: '#/components/responses/BadRequest400'
        '429':
          $ref: >-
            #/components/responses/TooManyRequests429
        '500':
          $ref: >-
            #/components/responses/InternalServerError500
        default:
          $ref: '#/components/responses/DefaultError'
      tags:
        - PostTraining (Coming Soon)
      summary: Get all training jobs.
      description: Get all training jobs.
      parameters: []
      deprecated: false
  /v1alpha/post-training/preference-optimize:
    post:
      responses:
        '200':
          description: A PostTrainingJob.
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/PostTrainingJob'
        '400':
          $ref: '#/components/responses/BadRequest400'
        '429':
          $ref: >-
            #/components/responses/TooManyRequests429
        '500':
          $ref: >-
            #/components/responses/InternalServerError500
        default:
          $ref: '#/components/responses/DefaultError'
      tags:
        - PostTraining (Coming Soon)
      summary: Run preference optimization of a model.
      description: Run preference optimization of a model.
      parameters: []
      requestBody:
        content:
          application/json:
            schema:
              $ref: '#/components/schemas/PreferenceOptimizeRequest'
        required: true
      deprecated: false
  /v1alpha/post-training/supervised-fine-tune:
    post:
      responses:
        '200':
          description: A PostTrainingJob.
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/PostTrainingJob'
        '400':
          $ref: '#/components/responses/BadRequest400'
        '429':
          $ref: >-
            #/components/responses/TooManyRequests429
        '500':
          $ref: >-
            #/components/responses/InternalServerError500
        default:
          $ref: '#/components/responses/DefaultError'
      tags:
        - PostTraining (Coming Soon)
      summary: Run supervised fine-tuning of a model.
      description: Run supervised fine-tuning of a model.
      parameters: []
      requestBody:
        content:
          application/json:
            schema:
              $ref: '#/components/schemas/SupervisedFineTuneRequest'
        required: true
      deprecated: false
jsonSchemaDialect: >-
  https://json-schema.org/draft/2020-12/schema
components:
  schemas:
    Error:
      type: object
      properties:
        status:
          type: integer
          description: HTTP status code
        title:
          type: string
          description: >-
            Error title, a short summary of the error which is invariant for an error
            type
        detail:
          type: string
          description: >-
            Error detail, a longer human-readable description of the error
        instance:
          type: string
          description: >-
            (Optional) A URL which can be used to retrieve more information about
            the specific occurrence of the error
      additionalProperties: false
      required:
        - status
        - title
        - detail
      title: Error
      description: >-
        Error response from the API. Roughly follows RFC 7807.
    AppendRowsRequest:
      type: object
      properties:
        rows:
          type: array
          items:
            type: object
            additionalProperties:
              oneOf:
                - type: 'null'
                - type: boolean
                - type: number
                - type: string
                - type: array
                - type: object
          description: The rows to append to the dataset.
      additionalProperties: false
      required:
        - rows
      title: AppendRowsRequest
    PaginatedResponse:
      type: object
      properties:
        data:
          type: array
          items:
            type: object
            additionalProperties:
              oneOf:
                - type: 'null'
                - type: boolean
                - type: number
                - type: string
                - type: array
                - type: object
          description: The list of items for the current page
        has_more:
          type: boolean
          description: >-
            Whether there are more items available after this set
        url:
          type: string
          description: The URL for accessing this list
      additionalProperties: false
      required:
        - data
        - has_more
      title: PaginatedResponse
      description: >-
        A generic paginated response that follows a simple format.
    Dataset:
      type: object
      properties:
        identifier:
          type: string
        provider_resource_id:
          type: string
        provider_id:
          type: string
        type:
          type: string
          enum:
            - model
            - shield
            - vector_store
            - dataset
            - scoring_function
            - benchmark
            - tool
            - tool_group
            - prompt
          const: dataset
          default: dataset
          description: >-
            Type of resource, always 'dataset' for datasets
        purpose:
          type: string
          enum:
            - post-training/messages
            - eval/question-answer
            - eval/messages-answer
          description: >-
            Purpose of the dataset indicating its intended use
        source:
          oneOf:
            - $ref: '#/components/schemas/URIDataSource'
            - $ref: '#/components/schemas/RowsDataSource'
          discriminator:
            propertyName: type
            mapping:
              uri: '#/components/schemas/URIDataSource'
              rows: '#/components/schemas/RowsDataSource'
          description: >-
            Data source configuration for the dataset
        metadata:
          type: object
          additionalProperties:
            oneOf:
              - type: 'null'
              - type: boolean
              - type: number
              - type: string
              - type: array
              - type: object
          description: Additional metadata for the dataset
      additionalProperties: false
      required:
        - identifier
        - provider_id
        - type
        - purpose
        - source
        - metadata
      title: Dataset
      description: >-
        Dataset resource for storing and accessing training or evaluation data.
    RowsDataSource:
      type: object
      properties:
        type:
          type: string
          const: rows
          default: rows
        rows:
          type: array
          items:
            type: object
            additionalProperties:
              oneOf:
                - type: 'null'
                - type: boolean
                - type: number
                - type: string
                - type: array
                - type: object
          description: >-
            The dataset is stored in rows. E.g. - [ {"messages": [{"role": "user",
            "content": "Hello, world!"}, {"role": "assistant", "content": "Hello,
            world!"}]} ]
      additionalProperties: false
      required:
        - type
        - rows
      title: RowsDataSource
      description: A dataset stored in rows.
    URIDataSource:
      type: object
      properties:
        type:
          type: string
          const: uri
          default: uri
        uri:
          type: string
          description: >-
            The dataset can be obtained from a URI. E.g. - "https://mywebsite.com/mydata.jsonl"
            - "lsfs://mydata.jsonl" - "data:csv;base64,{base64_content}"
      additionalProperties: false
      required:
        - type
        - uri
      title: URIDataSource
      description: >-
        A dataset that can be obtained from a URI.
    ListDatasetsResponse:
      type: object
      properties:
        data:
          type: array
          items:
            $ref: '#/components/schemas/Dataset'
          description: List of datasets
      additionalProperties: false
      required:
        - data
      title: ListDatasetsResponse
      description: Response from listing datasets.
    Benchmark:
      type: object
      properties:
        identifier:
          type: string
        provider_resource_id:
          type: string
        provider_id:
          type: string
        type:
          type: string
          enum:
            - model
            - shield
            - vector_store
            - dataset
            - scoring_function
            - benchmark
            - tool
            - tool_group
            - prompt
          const: benchmark
          default: benchmark
          description: The resource type, always benchmark
        dataset_id:
          type: string
          description: >-
            Identifier of the dataset to use for the benchmark evaluation
        scoring_functions:
          type: array
          items:
            type: string
          description: >-
            List of scoring function identifiers to apply during evaluation
        metadata:
          type: object
          additionalProperties:
            oneOf:
              - type: 'null'
              - type: boolean
              - type: number
              - type: string
              - type: array
              - type: object
          description: Metadata for this evaluation task
      additionalProperties: false
      required:
        - identifier
        - provider_id
        - type
        - dataset_id
        - scoring_functions
        - metadata
      title: Benchmark
      description: >-
        A benchmark resource for evaluating model performance.
    ListBenchmarksResponse:
      type: object
      properties:
        data:
          type: array
          items:
            $ref: '#/components/schemas/Benchmark'
      additionalProperties: false
      required:
        - data
      title: ListBenchmarksResponse
    AggregationFunctionType:
      type: string
      enum:
        - average
        - weighted_average
        - median
        - categorical_count
        - accuracy
      title: AggregationFunctionType
      description: >-
        Types of aggregation functions for scoring results.
    BasicScoringFnParams:
      type: object
      properties:
        type:
          $ref: '#/components/schemas/ScoringFnParamsType'
          const: basic
          default: basic
          description: >-
            The type of scoring function parameters, always basic
        aggregation_functions:
          type: array
          items:
            $ref: '#/components/schemas/AggregationFunctionType'
          description: >-
            Aggregation functions to apply to the scores of each row
      additionalProperties: false
      required:
        - type
        - aggregation_functions
      title: BasicScoringFnParams
      description: >-
        Parameters for basic scoring function configuration.
    BenchmarkConfig:
      type: object
      properties:
        eval_candidate:
          $ref: '#/components/schemas/ModelCandidate'
          description: The candidate to evaluate.
        scoring_params:
          type: object
          additionalProperties:
            $ref: '#/components/schemas/ScoringFnParams'
          description: >-
            Map between scoring function id and parameters for each scoring function
            you want to run
        num_examples:
          type: integer
          description: >-
            (Optional) The number of examples to evaluate. If not provided, all examples
            in the dataset will be evaluated
      additionalProperties: false
      required:
        - eval_candidate
        - scoring_params
      title: BenchmarkConfig
      description: >-
        A benchmark configuration for evaluation.
    GreedySamplingStrategy:
      type: object
      properties:
        type:
          type: string
          const: greedy
          default: greedy
          description: >-
            Must be "greedy" to identify this sampling strategy
      additionalProperties: false
      required:
        - type
      title: GreedySamplingStrategy
      description: >-
        Greedy sampling strategy that selects the highest probability token at each
        step.
    ImageContentItem:
      type: object
      properties:
        type:
          type: string
          const: image
          default: image
          description: >-
            Discriminator type of the content item. Always "image"
        image:
          type: object
          properties:
            url:
              $ref: '#/components/schemas/URL'
              description: >-
                A URL of the image or data URL in the format of data:image/{type};base64,{data}.
                Note that URL could have length limits.
            data:
              type: string
              contentEncoding: base64
              description: base64 encoded image data as string
          additionalProperties: false
          description: >-
            Image as a base64 encoded string or an URL
      additionalProperties: false
      required:
        - type
        - image
      title: ImageContentItem
      description: A image content item
    InterleavedContent:
      oneOf:
        - type: string
        - $ref: '#/components/schemas/InterleavedContentItem'
        - type: array
          items:
            $ref: '#/components/schemas/InterleavedContentItem'
    InterleavedContentItem:
      oneOf:
        - $ref: '#/components/schemas/ImageContentItem'
        - $ref: '#/components/schemas/TextContentItem'
      discriminator:
        propertyName: type
        mapping:
          image: '#/components/schemas/ImageContentItem'
          text: '#/components/schemas/TextContentItem'
    LLMAsJudgeScoringFnParams:
      type: object
      properties:
        type:
          $ref: '#/components/schemas/ScoringFnParamsType'
          const: llm_as_judge
          default: llm_as_judge
          description: >-
            The type of scoring function parameters, always llm_as_judge
        judge_model:
          type: string
          description: >-
            Identifier of the LLM model to use as a judge for scoring
        prompt_template:
          type: string
          description: >-
            (Optional) Custom prompt template for the judge model
        judge_score_regexes:
          type: array
          items:
            type: string
          description: >-
            Regexes to extract the answer from generated response
        aggregation_functions:
          type: array
          items:
            $ref: '#/components/schemas/AggregationFunctionType'
          description: >-
            Aggregation functions to apply to the scores of each row
      additionalProperties: false
      required:
        - type
        - judge_model
        - judge_score_regexes
        - aggregation_functions
      title: LLMAsJudgeScoringFnParams
      description: >-
        Parameters for LLM-as-judge scoring function configuration.
    ModelCandidate:
      type: object
      properties:
        type:
          type: string
          const: model
          default: model
        model:
          type: string
          description: The model ID to evaluate.
        sampling_params:
          $ref: '#/components/schemas/SamplingParams'
          description: The sampling parameters for the model.
        system_message:
          $ref: '#/components/schemas/SystemMessage'
          description: >-
            (Optional) The system message providing instructions or context to the
            model.
      additionalProperties: false
      required:
        - type
        - model
        - sampling_params
      title: ModelCandidate
      description: A model candidate for evaluation.
    RegexParserScoringFnParams:
      type: object
      properties:
        type:
          $ref: '#/components/schemas/ScoringFnParamsType'
          const: regex_parser
          default: regex_parser
          description: >-
            The type of scoring function parameters, always regex_parser
        parsing_regexes:
          type: array
          items:
            type: string
          description: >-
            Regex to extract the answer from generated response
        aggregation_functions:
          type: array
          items:
            $ref: '#/components/schemas/AggregationFunctionType'
          description: >-
            Aggregation functions to apply to the scores of each row
      additionalProperties: false
      required:
        - type
        - parsing_regexes
        - aggregation_functions
      title: RegexParserScoringFnParams
      description: >-
        Parameters for regex parser scoring function configuration.
    SamplingParams:
      type: object
      properties:
        strategy:
          oneOf:
            - $ref: '#/components/schemas/GreedySamplingStrategy'
            - $ref: '#/components/schemas/TopPSamplingStrategy'
            - $ref: '#/components/schemas/TopKSamplingStrategy'
          discriminator:
            propertyName: type
            mapping:
              greedy: '#/components/schemas/GreedySamplingStrategy'
              top_p: '#/components/schemas/TopPSamplingStrategy'
              top_k: '#/components/schemas/TopKSamplingStrategy'
          description: The sampling strategy.
        max_tokens:
          type: integer
          description: >-
            The maximum number of tokens that can be generated in the completion.
            The token count of your prompt plus max_tokens cannot exceed the model's
            context length.
        repetition_penalty:
          type: number
          default: 1.0
          description: >-
            Number between -2.0 and 2.0. Positive values penalize new tokens based
            on whether they appear in the text so far, increasing the model's likelihood
            to talk about new topics.
        stop:
          type: array
          items:
            type: string
          description: >-
            Up to 4 sequences where the API will stop generating further tokens. The
            returned text will not contain the stop sequence.
      additionalProperties: false
      required:
        - strategy
      title: SamplingParams
      description: Sampling parameters.
    ScoringFnParams:
      oneOf:
        - $ref: '#/components/schemas/LLMAsJudgeScoringFnParams'
        - $ref: '#/components/schemas/RegexParserScoringFnParams'
        - $ref: '#/components/schemas/BasicScoringFnParams'
      discriminator:
        propertyName: type
        mapping:
          llm_as_judge: '#/components/schemas/LLMAsJudgeScoringFnParams'
          regex_parser: '#/components/schemas/RegexParserScoringFnParams'
          basic: '#/components/schemas/BasicScoringFnParams'
    ScoringFnParamsType:
      type: string
      enum:
        - llm_as_judge
        - regex_parser
        - basic
      title: ScoringFnParamsType
      description: >-
        Types of scoring function parameter configurations.
    SystemMessage:
      type: object
      properties:
        role:
          type: string
          const: system
          default: system
          description: >-
            Must be "system" to identify this as a system message
        content:
          $ref: '#/components/schemas/InterleavedContent'
          description: >-
            The content of the "system prompt". If multiple system messages are provided,
            they are concatenated. The underlying Llama Stack code may also add other
            system messages (for example, for formatting tool definitions).
      additionalProperties: false
      required:
        - role
        - content
      title: SystemMessage
      description: >-
        A system message providing instructions or context to the model.
    TextContentItem:
      type: object
      properties:
        type:
          type: string
          const: text
          default: text
          description: >-
            Discriminator type of the content item. Always "text"
        text:
          type: string
          description: Text content
      additionalProperties: false
      required:
        - type
        - text
      title: TextContentItem
      description: A text content item
    TopKSamplingStrategy:
      type: object
      properties:
        type:
          type: string
          const: top_k
          default: top_k
          description: >-
            Must be "top_k" to identify this sampling strategy
        top_k:
          type: integer
          description: >-
            Number of top tokens to consider for sampling. Must be at least 1
      additionalProperties: false
      required:
        - type
        - top_k
      title: TopKSamplingStrategy
      description: >-
        Top-k sampling strategy that restricts sampling to the k most likely tokens.
    TopPSamplingStrategy:
      type: object
      properties:
        type:
          type: string
          const: top_p
          default: top_p
          description: >-
            Must be "top_p" to identify this sampling strategy
        temperature:
          type: number
          description: >-
            Controls randomness in sampling. Higher values increase randomness
        top_p:
          type: number
          default: 0.95
          description: >-
            Cumulative probability threshold for nucleus sampling. Defaults to 0.95
      additionalProperties: false
      required:
        - type
      title: TopPSamplingStrategy
      description: >-
        Top-p (nucleus) sampling strategy that samples from the smallest set of tokens
        with cumulative probability >= p.
    URL:
      type: object
      properties:
        uri:
          type: string
          description: The URL string pointing to the resource
      additionalProperties: false
      required:
        - uri
      title: URL
      description: A URL reference to external content.
    EvaluateRowsRequest:
      type: object
      properties:
        input_rows:
          type: array
          items:
            type: object
            additionalProperties:
              oneOf:
                - type: 'null'
                - type: boolean
                - type: number
                - type: string
                - type: array
                - type: object
          description: The rows to evaluate.
        scoring_functions:
          type: array
          items:
            type: string
          description: >-
            The scoring functions to use for the evaluation.
        benchmark_config:
          $ref: '#/components/schemas/BenchmarkConfig'
          description: The configuration for the benchmark.
      additionalProperties: false
      required:
        - input_rows
        - scoring_functions
        - benchmark_config
      title: EvaluateRowsRequest
    EvaluateResponse:
      type: object
      properties:
        generations:
          type: array
          items:
            type: object
            additionalProperties:
              oneOf:
                - type: 'null'
                - type: boolean
                - type: number
                - type: string
                - type: array
                - type: object
          description: The generations from the evaluation.
        scores:
          type: object
          additionalProperties:
            $ref: '#/components/schemas/ScoringResult'
          description: The scores from the evaluation.
      additionalProperties: false
      required:
        - generations
        - scores
      title: EvaluateResponse
      description: The response from an evaluation.
    ScoringResult:
      type: object
      properties:
        score_rows:
          type: array
          items:
            type: object
            additionalProperties:
              oneOf:
                - type: 'null'
                - type: boolean
                - type: number
                - type: string
                - type: array
                - type: object
          description: >-
            The scoring result for each row. Each row is a map of column name to value.
        aggregated_results:
          type: object
          additionalProperties:
            oneOf:
              - type: 'null'
              - type: boolean
              - type: number
              - type: string
              - type: array
              - type: object
          description: Map of metric name to aggregated value
      additionalProperties: false
      required:
        - score_rows
        - aggregated_results
      title: ScoringResult
      description: A scoring result for a single row.
    RunEvalRequest:
      type: object
      properties:
        benchmark_config:
          $ref: '#/components/schemas/BenchmarkConfig'
          description: The configuration for the benchmark.
      additionalProperties: false
      required:
        - benchmark_config
      title: RunEvalRequest
    Job:
      type: object
      properties:
        job_id:
          type: string
          description: Unique identifier for the job
        status:
          type: string
          enum:
            - completed
            - in_progress
            - failed
            - scheduled
            - cancelled
          description: Current execution status of the job
      additionalProperties: false
      required:
        - job_id
        - status
      title: Job
      description: >-
        A job execution instance with status tracking.
    "OpenAIChatCompletionContentPartImageParam":
      type: object
      properties:
        type:
          type: string
          const: image_url
          default: image_url
          description: >-
            Must be "image_url" to identify this as image content
        image_url:
          $ref: '#/components/schemas/OpenAIImageURL'
          description: >-
            Image URL specification and processing details
      additionalProperties: false
      required:
        - type
        - image_url
      title: >-
        OpenAIChatCompletionContentPartImageParam
      description: >-
        Image content part for OpenAI-compatible chat completion messages.
    OpenAIChatCompletionContentPartTextParam:
      type: object
      properties:
        type:
          type: string
          const: text
          default: text
          description: >-
            Must be "text" to identify this as text content
        text:
          type: string
          description: The text content of the message
      additionalProperties: false
      required:
        - type
        - text
      title: OpenAIChatCompletionContentPartTextParam
      description: >-
        Text content part for OpenAI-compatible chat completion messages.
    OpenAIImageURL:
      type: object
      properties:
        url:
          type: string
          description: >-
            URL of the image to include in the message
        detail:
          type: string
          description: >-
            (Optional) Level of detail for image processing. Can be "low", "high",
            or "auto"
      additionalProperties: false
      required:
        - url
      title: OpenAIImageURL
      description: >-
        Image URL specification for OpenAI-compatible chat completion messages.
    RerankRequest:
      type: object
      properties:
        model:
          type: string
          description: >-
            The identifier of the reranking model to use.
        query:
          oneOf:
            - type: string
            - $ref: '#/components/schemas/OpenAIChatCompletionContentPartTextParam'
            - $ref: '#/components/schemas/OpenAIChatCompletionContentPartImageParam'
          description: >-
            The search query to rank items against. Can be a string, text content
            part, or image content part. The input must not exceed the model's max
            input token length.
        items:
          type: array
          items:
            oneOf:
              - type: string
              - $ref: '#/components/schemas/OpenAIChatCompletionContentPartTextParam'
              - $ref: '#/components/schemas/OpenAIChatCompletionContentPartImageParam'
          description: >-
            List of items to rerank. Each item can be a string, text content part,
            or image content part. Each input must not exceed the model's max input
            token length.
        max_num_results:
          type: integer
          description: >-
            (Optional) Maximum number of results to return. Default: returns all.
      additionalProperties: false
      required:
        - model
        - query
        - items
      title: RerankRequest
    RerankData:
      type: object
      properties:
        index:
          type: integer
          description: >-
            The original index of the document in the input list
        relevance_score:
          type: number
          description: >-
            The relevance score from the model output. Values are inverted when applicable
            so that higher scores indicate greater relevance.
      additionalProperties: false
      required:
        - index
        - relevance_score
      title: RerankData
      description: >-
        A single rerank result from a reranking response.
    RerankResponse:
      type: object
      properties:
        data:
          type: array
          items:
            $ref: '#/components/schemas/RerankData'
          description: >-
            List of rerank result objects, sorted by relevance score (descending)
      additionalProperties: false
      required:
        - data
      title: RerankResponse
      description: Response from a reranking request.
    Checkpoint:
      type: object
      properties:
        identifier:
          type: string
          description: Unique identifier for the checkpoint
        created_at:
          type: string
          format: date-time
          description: >-
            Timestamp when the checkpoint was created
        epoch:
          type: integer
          description: >-
            Training epoch when the checkpoint was saved
        post_training_job_id:
          type: string
          description: >-
            Identifier of the training job that created this checkpoint
        path:
          type: string
          description: >-
            File system path where the checkpoint is stored
        training_metrics:
          $ref: '#/components/schemas/PostTrainingMetric'
          description: >-
            (Optional) Training metrics associated with this checkpoint
      additionalProperties: false
      required:
        - identifier
        - created_at
        - epoch
        - post_training_job_id
        - path
      title: Checkpoint
      description: Checkpoint created during training runs.
    PostTrainingJobArtifactsResponse:
      type: object
      properties:
        job_uuid:
          type: string
          description: Unique identifier for the training job
        checkpoints:
          type: array
          items:
            $ref: '#/components/schemas/Checkpoint'
          description: >-
            List of model checkpoints created during training
      additionalProperties: false
      required:
        - job_uuid
        - checkpoints
      title: PostTrainingJobArtifactsResponse
      description: Artifacts of a finetuning job.
    PostTrainingMetric:
      type: object
      properties:
        epoch:
          type: integer
          description: Training epoch number
        train_loss:
          type: number
          description: Loss value on the training dataset
        validation_loss:
          type: number
          description: Loss value on the validation dataset
        perplexity:
          type: number
          description: >-
            Perplexity metric indicating model confidence
      additionalProperties: false
      required:
        - epoch
        - train_loss
        - validation_loss
        - perplexity
      title: PostTrainingMetric
      description: >-
        Training metrics captured during post-training jobs.
    CancelTrainingJobRequest:
      type: object
      properties:
        job_uuid:
          type: string
          description: The UUID of the job to cancel.
      additionalProperties: false
      required:
        - job_uuid
      title: CancelTrainingJobRequest
    PostTrainingJobStatusResponse:
      type: object
      properties:
        job_uuid:
          type: string
          description: Unique identifier for the training job
        status:
          type: string
          enum:
            - completed
            - in_progress
            - failed
            - scheduled
            - cancelled
          description: Current status of the training job
        scheduled_at:
          type: string
          format: date-time
          description: >-
            (Optional) Timestamp when the job was scheduled
        started_at:
          type: string
          format: date-time
          description: >-
            (Optional) Timestamp when the job execution began
        completed_at:
          type: string
          format: date-time
          description: >-
            (Optional) Timestamp when the job finished, if completed
        resources_allocated:
          type: object
          additionalProperties:
            oneOf:
              - type: 'null'
              - type: boolean
              - type: number
              - type: string
              - type: array
              - type: object
          description: >-
            (Optional) Information about computational resources allocated to the
            job
        checkpoints:
          type: array
          items:
            $ref: '#/components/schemas/Checkpoint'
          description: >-
            List of model checkpoints created during training
      additionalProperties: false
      required:
        - job_uuid
        - status
        - checkpoints
      title: PostTrainingJobStatusResponse
      description: Status of a finetuning job.
    ListPostTrainingJobsResponse:
      type: object
      properties:
        data:
          type: array
          items:
            type: object
            properties:
              job_uuid:
                type: string
            additionalProperties: false
            required:
              - job_uuid
            title: PostTrainingJob
      additionalProperties: false
      required:
        - data
      title: ListPostTrainingJobsResponse
    DPOAlignmentConfig:
      type: object
      properties:
        beta:
          type: number
          description: Temperature parameter for the DPO loss
        loss_type:
          $ref: '#/components/schemas/DPOLossType'
          default: sigmoid
          description: The type of loss function to use for DPO
      additionalProperties: false
      required:
        - beta
        - loss_type
      title: DPOAlignmentConfig
      description: >-
        Configuration for Direct Preference Optimization (DPO) alignment.
    DPOLossType:
      type: string
      enum:
        - sigmoid
        - hinge
        - ipo
        - kto_pair
      title: DPOLossType
    DataConfig:
      type: object
      properties:
        dataset_id:
          type: string
          description: >-
            Unique identifier for the training dataset
        batch_size:
          type: integer
          description: Number of samples per training batch
        shuffle:
          type: boolean
          description: >-
            Whether to shuffle the dataset during training
        data_format:
          $ref: '#/components/schemas/DatasetFormat'
          description: >-
            Format of the dataset (instruct or dialog)
        validation_dataset_id:
          type: string
          description: >-
            (Optional) Unique identifier for the validation dataset
        packed:
          type: boolean
          default: false
          description: >-
            (Optional) Whether to pack multiple samples into a single sequence for
            efficiency
        train_on_input:
          type: boolean
          default: false
          description: >-
            (Optional) Whether to compute loss on input tokens as well as output tokens
      additionalProperties: false
      required:
        - dataset_id
        - batch_size
        - shuffle
        - data_format
      title: DataConfig
      description: >-
        Configuration for training data and data loading.
    DatasetFormat:
      type: string
      enum:
        - instruct
        - dialog
      title: DatasetFormat
      description: Format of the training dataset.
    EfficiencyConfig:
      type: object
      properties:
        enable_activation_checkpointing:
          type: boolean
          default: false
          description: >-
            (Optional) Whether to use activation checkpointing to reduce memory usage
        enable_activation_offloading:
          type: boolean
          default: false
          description: >-
            (Optional) Whether to offload activations to CPU to save GPU memory
        memory_efficient_fsdp_wrap:
          type: boolean
          default: false
          description: >-
            (Optional) Whether to use memory-efficient FSDP wrapping
        fsdp_cpu_offload:
          type: boolean
          default: false
          description: >-
            (Optional) Whether to offload FSDP parameters to CPU
      additionalProperties: false
      title: EfficiencyConfig
      description: >-
        Configuration for memory and compute efficiency optimizations.
    OptimizerConfig:
      type: object
      properties:
        optimizer_type:
          $ref: '#/components/schemas/OptimizerType'
          description: >-
            Type of optimizer to use (adam, adamw, or sgd)
        lr:
          type: number
          description: Learning rate for the optimizer
        weight_decay:
          type: number
          description: >-
            Weight decay coefficient for regularization
        num_warmup_steps:
          type: integer
          description: Number of steps for learning rate warmup
      additionalProperties: false
      required:
        - optimizer_type
        - lr
        - weight_decay
        - num_warmup_steps
      title: OptimizerConfig
      description: >-
        Configuration parameters for the optimization algorithm.
    OptimizerType:
      type: string
      enum:
        - adam
        - adamw
        - sgd
      title: OptimizerType
      description: >-
        Available optimizer algorithms for training.
    TrainingConfig:
      type: object
      properties:
        n_epochs:
          type: integer
          description: Number of training epochs to run
        max_steps_per_epoch:
          type: integer
          default: 1
          description: Maximum number of steps to run per epoch
        gradient_accumulation_steps:
          type: integer
          default: 1
          description: >-
            Number of steps to accumulate gradients before updating
        max_validation_steps:
          type: integer
          default: 1
          description: >-
            (Optional) Maximum number of validation steps per epoch
        data_config:
          $ref: '#/components/schemas/DataConfig'
          description: >-
            (Optional) Configuration for data loading and formatting
        optimizer_config:
          $ref: '#/components/schemas/OptimizerConfig'
          description: >-
            (Optional) Configuration for the optimization algorithm
        efficiency_config:
          $ref: '#/components/schemas/EfficiencyConfig'
          description: >-
            (Optional) Configuration for memory and compute optimizations
        dtype:
          type: string
          default: bf16
          description: >-
            (Optional) Data type for model parameters (bf16, fp16, fp32)
      additionalProperties: false
      required:
        - n_epochs
        - max_steps_per_epoch
        - gradient_accumulation_steps
      title: TrainingConfig
      description: >-
        Comprehensive configuration for the training process.
    PreferenceOptimizeRequest:
      type: object
      properties:
        job_uuid:
          type: string
          description: The UUID of the job to create.
        finetuned_model:
          type: string
          description: The model to fine-tune.
        algorithm_config:
          $ref: '#/components/schemas/DPOAlignmentConfig'
          description: The algorithm configuration.
        training_config:
          $ref: '#/components/schemas/TrainingConfig'
          description: The training configuration.
        hyperparam_search_config:
          type: object
          additionalProperties:
            oneOf:
              - type: 'null'
              - type: boolean
              - type: number
              - type: string
              - type: array
              - type: object
          description: The hyperparam search configuration.
        logger_config:
          type: object
          additionalProperties:
            oneOf:
              - type: 'null'
              - type: boolean
              - type: number
              - type: string
              - type: array
              - type: object
          description: The logger configuration.
      additionalProperties: false
      required:
        - job_uuid
        - finetuned_model
        - algorithm_config
        - training_config
        - hyperparam_search_config
        - logger_config
      title: PreferenceOptimizeRequest
    PostTrainingJob:
      type: object
      properties:
        job_uuid:
          type: string
      additionalProperties: false
      required:
        - job_uuid
      title: PostTrainingJob
    AlgorithmConfig:
      oneOf:
        - $ref: '#/components/schemas/LoraFinetuningConfig'
        - $ref: '#/components/schemas/QATFinetuningConfig'
      discriminator:
        propertyName: type
        mapping:
          LoRA: '#/components/schemas/LoraFinetuningConfig'
          QAT: '#/components/schemas/QATFinetuningConfig'
    LoraFinetuningConfig:
      type: object
      properties:
        type:
          type: string
          const: LoRA
          default: LoRA
          description: Algorithm type identifier, always "LoRA"
        lora_attn_modules:
          type: array
          items:
            type: string
          description: >-
            List of attention module names to apply LoRA to
        apply_lora_to_mlp:
          type: boolean
          description: Whether to apply LoRA to MLP layers
        apply_lora_to_output:
          type: boolean
          description: >-
            Whether to apply LoRA to output projection layers
        rank:
          type: integer
          description: >-
            Rank of the LoRA adaptation (lower rank = fewer parameters)
        alpha:
          type: integer
          description: >-
            LoRA scaling parameter that controls adaptation strength
        use_dora:
          type: boolean
          default: false
          description: >-
            (Optional) Whether to use DoRA (Weight-Decomposed Low-Rank Adaptation)
        quantize_base:
          type: boolean
          default: false
          description: >-
            (Optional) Whether to quantize the base model weights
      additionalProperties: false
      required:
        - type
        - lora_attn_modules
        - apply_lora_to_mlp
        - apply_lora_to_output
        - rank
        - alpha
      title: LoraFinetuningConfig
      description: >-
        Configuration for Low-Rank Adaptation (LoRA) fine-tuning.
    QATFinetuningConfig:
      type: object
      properties:
        type:
          type: string
          const: QAT
          default: QAT
          description: Algorithm type identifier, always "QAT"
        quantizer_name:
          type: string
          description: >-
            Name of the quantization algorithm to use
        group_size:
          type: integer
          description: Size of groups for grouped quantization
      additionalProperties: false
      required:
        - type
        - quantizer_name
        - group_size
      title: QATFinetuningConfig
      description: >-
        Configuration for Quantization-Aware Training (QAT) fine-tuning.
    SupervisedFineTuneRequest:
      type: object
      properties:
        job_uuid:
          type: string
          description: The UUID of the job to create.
        training_config:
          $ref: '#/components/schemas/TrainingConfig'
          description: The training configuration.
        hyperparam_search_config:
          type: object
          additionalProperties:
            oneOf:
              - type: 'null'
              - type: boolean
              - type: number
              - type: string
              - type: array
              - type: object
          description: The hyperparam search configuration.
        logger_config:
          type: object
          additionalProperties:
            oneOf:
              - type: 'null'
              - type: boolean
              - type: number
              - type: string
              - type: array
              - type: object
          description: The logger configuration.
        model:
          type: string
          description: The model to fine-tune.
        checkpoint_dir:
          type: string
          description: The directory to save checkpoint(s) to.
        algorithm_config:
          $ref: '#/components/schemas/AlgorithmConfig'
          description: The algorithm configuration.
      additionalProperties: false
      required:
        - job_uuid
        - training_config
        - hyperparam_search_config
        - logger_config
      title: SupervisedFineTuneRequest
    DataSource:
      oneOf:
        - $ref: '#/components/schemas/URIDataSource'
        - $ref: '#/components/schemas/RowsDataSource'
      discriminator:
        propertyName: type
        mapping:
          uri: '#/components/schemas/URIDataSource'
          rows: '#/components/schemas/RowsDataSource'
    RegisterDatasetRequest:
      type: object
      properties:
        purpose:
          type: string
          enum:
            - post-training/messages
            - eval/question-answer
            - eval/messages-answer
          description: >-
            The purpose of the dataset. One of: - "post-training/messages": The dataset
            contains a messages column with list of messages for post-training. {
            "messages": [ {"role": "user", "content": "Hello, world!"}, {"role": "assistant",
            "content": "Hello, world!"}, ] } - "eval/question-answer": The dataset
            contains a question column and an answer column for evaluation. { "question":
            "What is the capital of France?", "answer": "Paris" } - "eval/messages-answer":
            The dataset contains a messages column with list of messages and an answer
            column for evaluation. { "messages": [ {"role": "user", "content": "Hello,
            my name is John Doe."}, {"role": "assistant", "content": "Hello, John
            Doe. How can I help you today?"}, {"role": "user", "content": "What's
            my name?"}, ], "answer": "John Doe" }
        source:
          $ref: '#/components/schemas/DataSource'
          description: >-
            The data source of the dataset. Ensure that the data source schema is
            compatible with the purpose of the dataset. Examples: - { "type": "uri",
            "uri": "https://mywebsite.com/mydata.jsonl" } - { "type": "uri", "uri":
            "lsfs://mydata.jsonl" } - { "type": "uri", "uri": "data:csv;base64,{base64_content}"
            } - { "type": "uri", "uri": "huggingface://llamastack/simpleqa?split=train"
            } - { "type": "rows", "rows": [ { "messages": [ {"role": "user", "content":
            "Hello, world!"}, {"role": "assistant", "content": "Hello, world!"}, ]
            } ] }
        metadata:
          type: object
          additionalProperties:
            oneOf:
              - type: 'null'
              - type: boolean
              - type: number
              - type: string
              - type: array
              - type: object
          description: >-
            The metadata for the dataset. - E.g. {"description": "My dataset"}.
        dataset_id:
          type: string
          description: >-
            The ID of the dataset. If not provided, an ID will be generated.
      additionalProperties: false
      required:
        - purpose
        - source
      title: RegisterDatasetRequest
    RegisterBenchmarkRequest:
      type: object
      properties:
        benchmark_id:
          type: string
          description: The ID of the benchmark to register.
        dataset_id:
          type: string
          description: >-
            The ID of the dataset to use for the benchmark.
        scoring_functions:
          type: array
          items:
            type: string
          description: >-
            The scoring functions to use for the benchmark.
        provider_benchmark_id:
          type: string
          description: >-
            The ID of the provider benchmark to use for the benchmark.
        provider_id:
          type: string
          description: >-
            The ID of the provider to use for the benchmark.
        metadata:
          type: object
          additionalProperties:
            oneOf:
              - type: 'null'
              - type: boolean
              - type: number
              - type: string
              - type: array
              - type: object
          description: The metadata to use for the benchmark.
      additionalProperties: false
      required:
        - benchmark_id
        - dataset_id
        - scoring_functions
      title: RegisterBenchmarkRequest
  responses:
    BadRequest400:
      description: The request was invalid or malformed
      content:
        application/json:
          schema:
            $ref: '#/components/schemas/Error'
          example:
            status: 400
            title: Bad Request
            detail: The request was invalid or malformed
    TooManyRequests429:
      description: >-
        The client has sent too many requests in a given amount of time
      content:
        application/json:
          schema:
            $ref: '#/components/schemas/Error'
          example:
            status: 429
            title: Too Many Requests
            detail: >-
              You have exceeded the rate limit. Please try again later.
    InternalServerError500:
      description: >-
        The server encountered an unexpected error
      content:
        application/json:
          schema:
            $ref: '#/components/schemas/Error'
          example:
            status: 500
            title: Internal Server Error
            detail: >-
              An unexpected error occurred. Our team has been notified.
    DefaultError:
      description: An unexpected error occurred
      content:
        application/json:
          schema:
            $ref: '#/components/schemas/Error'
          example:
            status: 0
            title: Error
            detail: An unexpected error occurred
security:
  - Default: []
tags:
  - name: Benchmarks
    description: ''
  - name: DatasetIO
    description: ''
  - name: Datasets
    description: ''
  - name: Eval
    description: >-
      Llama Stack Evaluation API for running evaluations on model and agent candidates.
    x-displayName: Evaluations
  - name: PostTraining (Coming Soon)
    description: ''
x-tagGroups:
  - name: Operations
    tags:
      - Benchmarks
      - DatasetIO
      - Datasets
      - Eval
      - PostTraining (Coming Soon)