mirror of
https://github.com/meta-llama/llama-stack.git
synced 2025-10-04 12:07:34 +00:00
71 lines
No EOL
61 KiB
Text
71 lines
No EOL
61 KiB
Text
---
|
|
id: evaluate-a-list-of-rows-on-a-benchmark
|
|
title: "Evaluate a list of rows on a benchmark."
|
|
description: "Evaluate a list of rows on a benchmark."
|
|
sidebar_label: "Evaluate a list of rows on a benchmark."
|
|
hide_title: true
|
|
hide_table_of_contents: true
|
|
api: eJztHdluJLfxV4h58RoYjbSGncSbA9BesZw9BGmNIPAKY6qbmqHVl/uQNBHm31MH2c0+5pJkQ9rQD/aom10s1sWqYrF8O8pVkaVJoYrRi9vRNwcH+J9QFUGus1KnyejF6M2VjCpZqhMzUqTnv6qgFEGalFInOpmJmUpULnF8IWQSiiJIAe5kNB7hIJWUCFVmWaQDGrX/a4Ggb0dFMFexxF/lIlMwGcOGD7M8zVReakbMmcAZLPNcLmCsLlVcDAGRYajxIxkdt8Clifp4MXrxc/1FUkXRaDmu/z5P00jJxH2UVPG5yt0nRZnD4t0njJDzwGCyPFsuxx2yfpqrFt0u8jQWJTxUTG94OAFQI6blLqtbT0qCN83T6y+KkrgqlEQgVRWV4iLNhZLBXMA6J+KN+SU0iKeIZSbSCxDfqIoTkchYiTIVSHRFBJezWa5mIPDhlKE9doJ06fGeFxgrABLUC2yWxWsdISGHV3Eho0KNwTT8VulchbAaV2oGKXQ2HpW6jBCxU2bFCb0ZdZE7HGKVFAU8iRSxa7SSwWqVluyyEteU1NrloN81d70VIDbWajI+Mumggwh9O2RL+dvfKlWU4loWQifwnQ4F0CCWEZAiBiQfzGoWpSwrV3o1gJ2BpHWX9MOnT8eCR4NehApZYOjRFcnup2/yHJCnwWPk4zzNS1FUcSzzBUoh8YrGXM81qKHmRedaJob3iXlP8xDvYVuJtp2ZR+PUUZrA2sS8imWylysZynOQKOebFjo4k05gzUmwxSqffcxYuL4Wh+Knk3dmMQEgf65EVYBSgYrlqHHqSokYhAqWifwkzgl5nlYlTV5kKtAXoJdpEFR5rmD6Dl47qSVz2HKrJp4rzwR1mHhtMUYcDo+PJuIkrWbzaAHsiSLQeHHy9pX4818O/kyqqW5knLFkWPkCUW/meylDccIi3uCzpeQbzfnm+2HNCSINWiHm8HmBP8o0hY+ThQWMUEEQZvpKwX/jtIIxSFsdK69UXqmemFJ9830z3yeQ9Pco6UazCle1/pNWoBNAIXUTKBUi2VDZYA8TkY51ORHH4HEAQiVIj5yBzy4ieJnbneq7VTtVofIrYD6QE1VJAXlQsKpE3QC90ZFQhgxetbxqPSHV+s7dr45QsgFzccrS3sxv1OuwL/KWGOFEfKxAEpSMaVc6V7DxJGkJBIN3Rr1CdSHRE+6p2DrAXqm8Uj0tpToYmm0LFSItQVGTswJXg/EXoj6chEIW64LcOgxGBZIVqJ8Ec5C1S8w4ZTKHiBd0uqAoHMNfjLrtkKlG3dIIM5PlfDC6O3ptuVF/Rpytkk7oKTDcc5lR5pUa91XThvPLMx4MO/jLNFzgiAfScp1kVfnFZXWIxUB5Q3FVJ8QA7vSiSoJtM4I1KmtSRzU8nLFCfUB71E/INaIEvLvQsxWxCr2rONNQg2pJ6nqO4qxTsC2hDmHlKyaxr9tEGvcZuGISfts3fIA9hG8vRjHsA9HI2cPMEyAD/9hoNBFNGopa1WMlWhHkJSntNsnLEt3a2WKFv2rACTvsIUkxy8G3XrRo4Txq5d8qMFCwI3w2Az6PcOE6BDXXFwuQA9j+eqjuZvIJSce+/5MmOjVATy3MLmY8rD854CRhw1YREKYgQZ3r2RxDdaDRuTzXkS5hUHqJcXXJmdWiVNlkQJd3pWuZZtOsRdbmyTBV6f2WRAUSqTjDhF+Vu2gYa9Wd5BVY4zyNCoiekjCNE1VQTsECnogfkC45Z1DxVZBTZNUMpylpAYOTmTUeTL7/rjd3FVcQl2lwHVpUn8MGPU+jkIxIUgWRqgoHo9cMk6wWgr2fIH1Ks+ONYgSD9jLxzODy9Up5Ih/BydkWsYwilKpCcWIGBaoQ17oEL2p49f/4u3gwKbvsSdnlBim73F7KCNpmJ/sDiQIvPrMEgCkQT5gmJyY3zLXoADUxhC/F8zuw12LXZvO/tmLz5SrmglDi8ULp0AOWgVy+BHsPmEb6UkXGaBTAQnTpNIKPdSJL8A+BWIaHiw/spBlsY3CEkHt47scWduO+4I333Yx3z1R5I+6N+O9mxHtW0hvzJ2rMl01uaNSbGoMDeTPlwSvIaFVoMFyQNzquYpE45GXSIq4m3WKOcVWIql1SwIWCSlGaQChs8AJ7CLRIqxyFMs7ALkagfA2KCDJJS5M+J2AUsXxVUI2LukF+JbNyTjKcqwxmwXmmmQKmlYu1RuL5CqE5V+U1Zgv3vpkcUM0M/HcijtNCk/oY20Qz6P8qkahrS4VziZkmCCqv56pEOwb/WgjYM5XMLTEI6SIVFzIfWwNHXHXWRjzV8zSltFUpo0uToeK5Mh0UHKTB73uE2D+hZohvwVz8VmGiq0C8c2UTTWA4okjgHHU1DMXiOS+NBU7w6SGY/wQZhIujz5BrpgyJbROCsRPtWBdQG4Czvmgfc3jaXZp9LZqcE1NsAftrPI1hJ5EztdnewWa11t4xvJbBcx4NWzweMGTyJBbB8GthUVy20s2D2Qx8adNiFrxRqM+jiTi6EGD7Sw062AEO0+W0H1wBHuHYiGtOQAMgeYJazCyuErCa0QJJ+i6SsRSnpQwuKRsN+gpfRSDTwFGRknR053lGZT+cmhyT8eXka8kGLY0EEFAnJA/F14PJASdn9VCJAx0jhV3mNU9cOr92vWNKhFtyW+qjyk3EYXQtFwXwgMB8HlESO95K0Ko8GjiGoFy2mYoA4aF8KEtJL4xqMSVxGL55QeP2b3G65V/RIv3p2/EtvllOxIcUE1Joq/F7sMDg9tD5JFtRPpQsNmfAqlxvl1/Cafi1yFLYZZz9C3Y2MPzBjtVOOLNjBgA+2TVY3gpGI3veJCCnFLeMmCA9XPkxnafiKS3TmgiNFQ02R7wBzzbII4JBKt2BbgjChyu0gjv4GixXDilovle84KMhC3RoluXK7EPE72Dz2x6ffXAvFUIorEEEb7O04dZjLeWdnDecxvXd4O/15CydKQ017xRKb2kivDXz1sxbsw3WrGstvFG7v1FzQ0qqyYAY+UqFLpzeSaIThXhvzds3b9+8t/b4DJv31rw189bMe2tflFHbxltbnq0bsxvulCJtkpRujpaSgO9NInPgulkrSWjykCidWPiZV6bkC3hlc/1GeylNvmMO2dpxUzXVLW9ysH6PQ17V1V19tLlmqinwotRmqxDtvvIM1EjaAs1POBe8Y2Ebfdqgu9nA+tKvp1894KsGfNWAL/3y1QK+9OsJGm9f+uWNuC/98sbcl3750i9f+vVQpV98D6+YaxWFu1xMg0+B/nf+FoubZnlaZWu/W1fttN4s8y3KAYbLfPbobxbuIgq0ULMsRxoOMb3xCVD7JxL537DhHeKI5dAQYgi3KpkiX3a6kbkN2XtmBTs8mSRjSQh0RmwC4Fxs/mHtxelrtMJ2GhGmitwl98btH7fSetYWCtNVDkRnr3HysvW34pmazCZjk71G40mb1tc703NLIjYTW0o2ktjTB2df4UvG7Sn/XW8I4Nk0gOEPC5NTdcg3nVylfM2YV1a3J/jjlXONJBq/mLfDOs/oLO2C9gBbV7qbxTdq3hGbNjbOFy2PK42Oa9Hr4n/csLSuNDUdzpD0A73NnBW/MxfbG31idhLTZBCorCQpgXeycx70GO3umpUe1lgKuxinO0Jpjehu7OwwCeRnwBNu1QBzzwZwIHLwFHWsrM9XTOpNdRrMUz3cHEIBfXByWZWpKyzjUQLU7qLziuGs1FvEDC+duxqLcgPAsX1EgI4Qts3A07a5TtrHFKYTBZZIo06cK9O1Aeuqj0pYMTiTITVLwE+OmoMP8ZbGoSMTyIyDNiCyNY3mDASRznKFxdqmx4GlDnvVUz6gXUck6mcwHtk79tNS4ttsUc7hD+zo0NMxgvyWAffUjJ15cy6M+hVAZEqrgEALotV98V+Vp9iIhLqERcW6RTSnHRti07Y8rPKm/p8kY8A2nLVOlJgCKw3B0CLrhY3r9Y5xwUi7iXgFMSBdA+AeKgtjWes2lwhENg1WyOq2ciwNySc4k0lOPLgsr1yypXHhMGmeYthl5ZnWgwIN0c/LhTDUHLeuRFAMJEtMjpXNsi0A9DA0tiOCDQWsa1Zya5qGcWJP/NLVsgmu65cXJoK2KNBNDXONAvt/oZT9ePrxg+kCPAzJJc0wRIjSotRYXyn+Zj/4e/0l8vMfEJrOhmdwqD08AUdTiO8xDRXFAoLFG7G357SQsbMZavfv7kzP1VxeaU5Zr1TzDJWIFD2LZKDWsv8VGRuyWYbr6ZXKc3BhiD2G2aJ1v4ZI0DrnfmnQmvDcQILDjDWZnCTjE3XPvo0I7DqHWRZMcsK/itVgOBm4BgkO6DuIYWJHg0RUhgqmfuSr29taHpybO8vlV5TNS0LslqOcpELNT2c0bgJYBXSOzZgKsObYmatlo5h9Gx2Ols3mLWMg/dw9GzeGbWJTYzq5AHdTd6KlgfzY84PlHZqsmBSnJpva6mJTH+67dQ/bgTbcahVM9MBuF8FZRWB73QIyrt2xUkUKuxcvKDtml9SELCrBuGpagPRwLi4vQKFVu4nXQOA0WMVUY3QRyZkVKpQ+m2Mz03CxFHZ5A8KCMNlZQw7cuIeWs3usmKXptsW2ulVO8QA1C+4mNTXdo8ajGUQUscz7lqnJkzsfDGXL7dbCcZJNx7dnaXTKfQ7kcf98WkELij9td4y9retjFhrLgiwE6gDNwKfC7czsOKev/zVm4mn0m0o8hBO/ZItQAl2DX+5TYeQS1HEnf4THp/TUdq9e4UD3DZWzyL1ZRYa7XmfTMvs+5/F3lkwzZHuptDDbZ778DIh9nlw8QSl8+eGtMIuo3Vtm33qZvJN8IYlax9Q0785CZfBdL1B3qg7YYFG8rfS28jHYylqY/ngZ9Vbzi7aa7jF8Z4adULe1yq2ooHusZcKdu9DEJNd6EFeXPifdYuJ+7fOdtoxuLOVbYvrSOl8X7UvqnmpJnS+le+qldL4u+hEab18X7Y24r4v2xtzXRfu6aF8X7Vti+paYviWmb0vi25JYX8M3WXqM/UjuFEr7Jkvemnlr5pssPVaj5ltievvm7dvjsG/esHlvzVszb82+DGvmvTXfEvMLbIlpOljeW659a0xfReBLwHz1wGOvHvBVA0+9asCXgD1C4+1LwLwR9yVg3pj7EjBfAuZLwHxrzLVm2bfG9K0xfWvMbenpW2P61pi+NeaalfrWmD29fcwNEBFp3xrTt8b0rTF9a0zfGrOv5r41pm+NuaLNjW+NuaJFpG+N6VtjPoGgxbd7860xH4cUfrFN3u5UHeBbYz5OKfW20rfGfEzy+MVazf/r1pju4t/AiwYaRvBBmq8vLN5aZjfoqtWcKIqnspj+WoWdSze9F13xLM05lMG5CWCbM4CxkHxloANslaHoDMshrrxBWhQUXp7LQgetc26e+W3CB914MEa7EH4+3TL8PGqCTpMefPfuvQlIbW4K80YElCtjeFY6uuO0G2ayIuTgLqeVrzjba1IPFkQdVvJ8dedSXhNOraZEFrXNgeWq4oYThkC9TW/KXAZ8eCGT4hrLf7BmqilUsTrM5fszmJ1EeWr5vQMiTt7pCqATk6+Vns1hnmnzKFahpugXUyQzoHYgMdNdUcm/DAIwOMGirZY1Wm8NVp/6h2J8dEppYGcdtdwWLndx1Zh1HTj1Ohz8FkgJHh4VG3EaCFlFc1HlYp5e380BcGR5WApWMcUhD0j0YfEjftvRmNVngEwM+HBPFnssiz09b4X/u0RX1vZ09LuxPb0X97E9HWC/r+3BD9CAP5COeg199Bra5fgW+kisPSbR2lEhaRLBUvk7KCRLuauJzZP7qKCF8vvqnhf9P1j0Nwr6S+TcjiJO3N4s23fKvLTkbOuw1rvK3lX2G/Gjt0ZPylVuK+zOpsh7zt5zfvIK+5g9ZyO2Oyum96C9CjwRD7p1Ga4zQ48q72VW3/3qQdYhFbtkbTT4xm537CKtxLXkSs+8opIYkJmp6YC8qq5ohWOIWtRct7MwWv/LJmzy7Nbao0ZFzVBzAwvrY/BiKJUmnqv68x1L8/GzaeAcD3Ty+S5jVRLMY5lf3uV8gm9qgdAVzhyN/IDCW+jT/hnGG7O2E/j8BC+dFSULQzODqTAGZs5T+HOUpQWZPlnO4a/9q+f7uNL9epZi/7aZUYfL/ebEgzBU+RXVjP1sWmqN5mWZvdjfl8lib55iIdJehAWiewUWiE6ClFuQFQpsgcabivDha2tRfz7Dd3imaLB/mYaLN8xQFB+HOPAdwelRiC/Ooc06GyAWAOlwspFK61+amKkxfQNNYdy2L+Zzc31/2b5rejB8P/O5vcXYYNur8kTY3Iu96a9et0V3r9n1jpZ6mnewpAq7i5TwtQ6xU7h76h5CwjzIVNbJq+cDOxwfiJO1bJ1e2uDRgUxVv0ZFi8+J6Pwj7a1tlYTUsawggwNQNPZAytmNMx2SwABdmEJPACpz9RlvWkZpToXEfeBUaRwp3p4MVtTqiHcHlP1YJs1Fr1qBnGJclDYsRJeiFqZJlyK3Dl+2B2HkBq927EPYq8limr50rI8/M/VRYl3NR8V74WqlGWPVEgQfVQ+/v73Fy7k/5dFyiY9BqXLUOfh5JXON9YqkgaEu8Hc4XIzoLu/ZiTElbKKPXlue1/gY808PG6wEbkwr1mxdkAQdELpYBX/Bz0u1QC/LXSidp8+VDDG0AsR5iOnhtWdcBwsCt3UjmftUHY9pU/7ikO4trR175ljJ44+nn5ADKbf+QAuBEZm8RocM/k3oplntVNGz21Ekk1lFmszV+Uv8539FUjEd
|
|
sidebar_class_name: "post api-method"
|
|
info_path: docs/api/llama-stack-specification
|
|
custom_edit_url: null
|
|
---
|
|
|
|
import MethodEndpoint from "@theme/ApiExplorer/MethodEndpoint";
|
|
import ParamsDetails from "@theme/ParamsDetails";
|
|
import RequestSchema from "@theme/RequestSchema";
|
|
import StatusCodes from "@theme/StatusCodes";
|
|
import OperationTabs from "@theme/OperationTabs";
|
|
import TabItem from "@theme/TabItem";
|
|
import Heading from "@theme/Heading";
|
|
|
|
<Heading
|
|
as={"h1"}
|
|
className={"openapi__heading"}
|
|
children={"Evaluate a list of rows on a benchmark."}
|
|
>
|
|
</Heading>
|
|
|
|
<MethodEndpoint
|
|
method={"post"}
|
|
path={"/v1/eval/benchmarks/{benchmark_id}/evaluations"}
|
|
context={"endpoint"}
|
|
>
|
|
|
|
</MethodEndpoint>
|
|
|
|
|
|
|
|
Evaluate a list of rows on a benchmark.
|
|
|
|
<Heading
|
|
id={"request"}
|
|
as={"h2"}
|
|
className={"openapi-tabs__heading"}
|
|
children={"Request"}
|
|
>
|
|
</Heading>
|
|
|
|
<ParamsDetails
|
|
parameters={[{"name":"benchmark_id","in":"path","description":"The ID of the benchmark to run the evaluation on.","required":true,"schema":{"type":"string"}}]}
|
|
>
|
|
|
|
</ParamsDetails>
|
|
|
|
<RequestSchema
|
|
title={"Body"}
|
|
body={{"content":{"application/json":{"schema":{"type":"object","properties":{"input_rows":{"type":"array","items":{"type":"object","additionalProperties":{"oneOf":[{"type":"null"},{"type":"boolean"},{"type":"number"},{"type":"string"},{"type":"array"},{"type":"object"}]}},"description":"The rows to evaluate."},"scoring_functions":{"type":"array","items":{"type":"string"},"description":"The scoring functions to use for the evaluation."},"benchmark_config":{"description":"The configuration for the benchmark.","type":"object","properties":{"eval_candidate":{"description":"The candidate to evaluate.","oneOf":[{"type":"object","properties":{"type":{"type":"string","const":"model","default":"model"},"model":{"type":"string","description":"The model ID to evaluate."},"sampling_params":{"type":"object","properties":{"strategy":{"description":"The sampling strategy.","oneOf":[{"type":"object","properties":{"type":{"type":"string","const":"greedy","default":"greedy","description":"Must be \"greedy\" to identify this sampling strategy"}},"additionalProperties":false,"required":["type"],"title":"GreedySamplingStrategy","description":"Greedy sampling strategy that selects the highest probability token at each step."},{"type":"object","properties":{"type":{"type":"string","const":"top_p","default":"top_p","description":"Must be \"top_p\" to identify this sampling strategy"},"temperature":{"type":"number","description":"Controls randomness in sampling. Higher values increase randomness"},"top_p":{"type":"number","default":0.95,"description":"Cumulative probability threshold for nucleus sampling. Defaults to 0.95"}},"additionalProperties":false,"required":["type"],"title":"TopPSamplingStrategy","description":"Top-p (nucleus) sampling strategy that samples from the smallest set of tokens with cumulative probability >= p."},{"type":"object","properties":{"type":{"type":"string","const":"top_k","default":"top_k","description":"Must be \"top_k\" to identify this sampling strategy"},"top_k":{"type":"integer","description":"Number of top tokens to consider for sampling. Must be at least 1"}},"additionalProperties":false,"required":["type","top_k"],"title":"TopKSamplingStrategy","description":"Top-k sampling strategy that restricts sampling to the k most likely tokens."}],"discriminator":{"propertyName":"type","mapping":{"greedy":{"type":"object","properties":{"type":{"type":"string","const":"greedy","default":"greedy","description":"Must be \"greedy\" to identify this sampling strategy"}},"additionalProperties":false,"required":["type"],"title":"GreedySamplingStrategy","description":"Greedy sampling strategy that selects the highest probability token at each step."},"top_p":{"type":"object","properties":{"type":{"type":"string","const":"top_p","default":"top_p","description":"Must be \"top_p\" to identify this sampling strategy"},"temperature":{"type":"number","description":"Controls randomness in sampling. Higher values increase randomness"},"top_p":{"type":"number","default":0.95,"description":"Cumulative probability threshold for nucleus sampling. Defaults to 0.95"}},"additionalProperties":false,"required":["type"],"title":"TopPSamplingStrategy","description":"Top-p (nucleus) sampling strategy that samples from the smallest set of tokens with cumulative probability >= p."},"top_k":{"type":"object","properties":{"type":{"type":"string","const":"top_k","default":"top_k","description":"Must be \"top_k\" to identify this sampling strategy"},"top_k":{"type":"integer","description":"Number of top tokens to consider for sampling. Must be at least 1"}},"additionalProperties":false,"required":["type","top_k"],"title":"TopKSamplingStrategy","description":"Top-k sampling strategy that restricts sampling to the k most likely tokens."}}},"title":"SamplingStrategy"},"max_tokens":{"type":"integer","default":0,"description":"The maximum number of tokens that can be generated in the completion. The token count of your prompt plus max_tokens cannot exceed the model's context length."},"repetition_penalty":{"type":"number","default":1,"description":"Number between -2.0 and 2.0. Positive values penalize new tokens based on whether they appear in the text so far, increasing the model's likelihood to talk about new topics."},"stop":{"type":"array","items":{"type":"string"},"description":"Up to 4 sequences where the API will stop generating further tokens. The returned text will not contain the stop sequence."}},"additionalProperties":false,"required":["strategy"],"title":"SamplingParams","description":"Sampling parameters."},"system_message":{"type":"object","properties":{"role":{"type":"string","const":"system","default":"system","description":"Must be \"system\" to identify this as a system message"},"content":{"description":"The content of the \"system prompt\". If multiple system messages are provided, they are concatenated. The underlying Llama Stack code may also add other system messages (for example, for formatting tool definitions).","oneOf":[{"type":"string"},{"oneOf":[{"type":"object","properties":{"type":{"type":"string","const":"image","default":"image","description":"Discriminator type of the content item. Always \"image\""},"image":{"type":"object","properties":{"url":{"description":"A URL of the image or data URL in the format of data:image/{type};base64,{data}. Note that URL could have length limits.","type":"object","properties":{"uri":{"type":"string","description":"The URL string pointing to the resource"}},"additionalProperties":false,"required":["uri"],"title":"URL"},"data":{"type":"string","contentEncoding":"base64","description":"base64 encoded image data as string"}},"additionalProperties":false,"description":"Image as a base64 encoded string or an URL"}},"additionalProperties":false,"required":["type","image"],"title":"ImageContentItem","description":"A image content item"},{"type":"object","properties":{"type":{"type":"string","const":"text","default":"text","description":"Discriminator type of the content item. Always \"text\""},"text":{"type":"string","description":"Text content"}},"additionalProperties":false,"required":["type","text"],"title":"TextContentItem","description":"A text content item"}],"discriminator":{"propertyName":"type","mapping":{"image":{"type":"object","properties":{"type":{"type":"string","const":"image","default":"image","description":"Discriminator type of the content item. Always \"image\""},"image":{"type":"object","properties":{"url":{"description":"A URL of the image or data URL in the format of data:image/{type};base64,{data}. Note that URL could have length limits.","type":"object","properties":{"uri":{"type":"string","description":"The URL string pointing to the resource"}},"additionalProperties":false,"required":["uri"],"title":"URL"},"data":{"type":"string","contentEncoding":"base64","description":"base64 encoded image data as string"}},"additionalProperties":false,"description":"Image as a base64 encoded string or an URL"}},"additionalProperties":false,"required":["type","image"],"title":"ImageContentItem","description":"A image content item"},"text":{"type":"object","properties":{"type":{"type":"string","const":"text","default":"text","description":"Discriminator type of the content item. Always \"text\""},"text":{"type":"string","description":"Text content"}},"additionalProperties":false,"required":["type","text"],"title":"TextContentItem","description":"A text content item"}}},"title":"InterleavedContentItem"},{"type":"array","items":{"oneOf":[{"type":"object","properties":{"type":{"type":"string","const":"image","default":"image","description":"Discriminator type of the content item. Always \"image\""},"image":{"type":"object","properties":{"url":{"description":"A URL of the image or data URL in the format of data:image/{type};base64,{data}. Note that URL could have length limits.","type":"object","properties":{"uri":{"type":"string","description":"The URL string pointing to the resource"}},"additionalProperties":false,"required":["uri"],"title":"URL"},"data":{"type":"string","contentEncoding":"base64","description":"base64 encoded image data as string"}},"additionalProperties":false,"description":"Image as a base64 encoded string or an URL"}},"additionalProperties":false,"required":["type","image"],"title":"ImageContentItem","description":"A image content item"},{"type":"object","properties":{"type":{"type":"string","const":"text","default":"text","description":"Discriminator type of the content item. Always \"text\""},"text":{"type":"string","description":"Text content"}},"additionalProperties":false,"required":["type","text"],"title":"TextContentItem","description":"A text content item"}],"discriminator":{"propertyName":"type","mapping":{"image":{"type":"object","properties":{"type":{"type":"string","const":"image","default":"image","description":"Discriminator type of the content item. Always \"image\""},"image":{"type":"object","properties":{"url":{"description":"A URL of the image or data URL in the format of data:image/{type};base64,{data}. Note that URL could have length limits.","type":"object","properties":{"uri":{"type":"string","description":"The URL string pointing to the resource"}},"additionalProperties":false,"required":["uri"],"title":"URL"},"data":{"type":"string","contentEncoding":"base64","description":"base64 encoded image data as string"}},"additionalProperties":false,"description":"Image as a base64 encoded string or an URL"}},"additionalProperties":false,"required":["type","image"],"title":"ImageContentItem","description":"A image content item"},"text":{"type":"object","properties":{"type":{"type":"string","const":"text","default":"text","description":"Discriminator type of the content item. Always \"text\""},"text":{"type":"string","description":"Text content"}},"additionalProperties":false,"required":["type","text"],"title":"TextContentItem","description":"A text content item"}}},"title":"InterleavedContentItem"}}],"title":"InterleavedContent"}},"additionalProperties":false,"required":["role","content"],"title":"SystemMessage","description":"A system message providing instructions or context to the model."}},"additionalProperties":false,"required":["type","model","sampling_params"],"title":"ModelCandidate","description":"A model candidate for evaluation."},{"type":"object","properties":{"type":{"type":"string","const":"agent","default":"agent"},"config":{"description":"The configuration for the agent candidate.","type":"object","properties":{"sampling_params":{"type":"object","properties":{"strategy":{"description":"The sampling strategy.","oneOf":[{"type":"object","properties":{"type":{"type":"string","const":"greedy","default":"greedy","description":"Must be \"greedy\" to identify this sampling strategy"}},"additionalProperties":false,"required":["type"],"title":"GreedySamplingStrategy","description":"Greedy sampling strategy that selects the highest probability token at each step."},{"type":"object","properties":{"type":{"type":"string","const":"top_p","default":"top_p","description":"Must be \"top_p\" to identify this sampling strategy"},"temperature":{"type":"number","description":"Controls randomness in sampling. Higher values increase randomness"},"top_p":{"type":"number","default":0.95,"description":"Cumulative probability threshold for nucleus sampling. Defaults to 0.95"}},"additionalProperties":false,"required":["type"],"title":"TopPSamplingStrategy","description":"Top-p (nucleus) sampling strategy that samples from the smallest set of tokens with cumulative probability >= p."},{"type":"object","properties":{"type":{"type":"string","const":"top_k","default":"top_k","description":"Must be \"top_k\" to identify this sampling strategy"},"top_k":{"type":"integer","description":"Number of top tokens to consider for sampling. Must be at least 1"}},"additionalProperties":false,"required":["type","top_k"],"title":"TopKSamplingStrategy","description":"Top-k sampling strategy that restricts sampling to the k most likely tokens."}],"discriminator":{"propertyName":"type","mapping":{"greedy":{"type":"object","properties":{"type":{"type":"string","const":"greedy","default":"greedy","description":"Must be \"greedy\" to identify this sampling strategy"}},"additionalProperties":false,"required":["type"],"title":"GreedySamplingStrategy","description":"Greedy sampling strategy that selects the highest probability token at each step."},"top_p":{"type":"object","properties":{"type":{"type":"string","const":"top_p","default":"top_p","description":"Must be \"top_p\" to identify this sampling strategy"},"temperature":{"type":"number","description":"Controls randomness in sampling. Higher values increase randomness"},"top_p":{"type":"number","default":0.95,"description":"Cumulative probability threshold for nucleus sampling. Defaults to 0.95"}},"additionalProperties":false,"required":["type"],"title":"TopPSamplingStrategy","description":"Top-p (nucleus) sampling strategy that samples from the smallest set of tokens with cumulative probability >= p."},"top_k":{"type":"object","properties":{"type":{"type":"string","const":"top_k","default":"top_k","description":"Must be \"top_k\" to identify this sampling strategy"},"top_k":{"type":"integer","description":"Number of top tokens to consider for sampling. Must be at least 1"}},"additionalProperties":false,"required":["type","top_k"],"title":"TopKSamplingStrategy","description":"Top-k sampling strategy that restricts sampling to the k most likely tokens."}}},"title":"SamplingStrategy"},"max_tokens":{"type":"integer","default":0,"description":"The maximum number of tokens that can be generated in the completion. The token count of your prompt plus max_tokens cannot exceed the model's context length."},"repetition_penalty":{"type":"number","default":1,"description":"Number between -2.0 and 2.0. Positive values penalize new tokens based on whether they appear in the text so far, increasing the model's likelihood to talk about new topics."},"stop":{"type":"array","items":{"type":"string"},"description":"Up to 4 sequences where the API will stop generating further tokens. The returned text will not contain the stop sequence."}},"additionalProperties":false,"required":["strategy"],"title":"SamplingParams","description":"Sampling parameters."},"input_shields":{"type":"array","items":{"type":"string"}},"output_shields":{"type":"array","items":{"type":"string"}},"toolgroups":{"type":"array","items":{"oneOf":[{"type":"string"},{"type":"object","properties":{"name":{"type":"string"},"args":{"type":"object","additionalProperties":{"oneOf":[{"type":"null"},{"type":"boolean"},{"type":"number"},{"type":"string"},{"type":"array"},{"type":"object"}]}}},"additionalProperties":false,"required":["name","args"],"title":"AgentToolGroupWithArgs"}],"title":"AgentTool"}},"client_tools":{"type":"array","items":{"type":"object","properties":{"name":{"type":"string","description":"Name of the tool"},"description":{"type":"string","description":"(Optional) Human-readable description of what the tool does"},"parameters":{"type":"array","items":{"type":"object","properties":{"name":{"type":"string","description":"Name of the parameter"},"parameter_type":{"type":"string","description":"Type of the parameter (e.g., string, integer)"},"description":{"type":"string","description":"Human-readable description of what the parameter does"},"required":{"type":"boolean","default":true,"description":"Whether this parameter is required for tool invocation"},"default":{"oneOf":[{"type":"null"},{"type":"boolean"},{"type":"number"},{"type":"string"},{"type":"array"},{"type":"object"}],"description":"(Optional) Default value for the parameter if not provided"}},"additionalProperties":false,"required":["name","parameter_type","description","required"],"title":"ToolParameter","description":"Parameter definition for a tool."},"description":"(Optional) List of parameters this tool accepts"},"metadata":{"type":"object","additionalProperties":{"oneOf":[{"type":"null"},{"type":"boolean"},{"type":"number"},{"type":"string"},{"type":"array"},{"type":"object"}]},"description":"(Optional) Additional metadata about the tool"}},"additionalProperties":false,"required":["name"],"title":"ToolDef","description":"Tool definition used in runtime contexts."}},"tool_choice":{"type":"string","enum":["auto","required","none"],"title":"ToolChoice","description":"Whether tool use is required or automatic. This is a hint to the model which may not be followed. It depends on the Instruction Following capabilities of the model.","deprecated":true},"tool_prompt_format":{"type":"string","enum":["json","function_tag","python_list"],"title":"ToolPromptFormat","description":"Prompt format for calling custom / zero shot tools.","deprecated":true},"tool_config":{"type":"object","properties":{"tool_choice":{"oneOf":[{"type":"string","enum":["auto","required","none"],"title":"ToolChoice","description":"Whether tool use is required or automatic. This is a hint to the model which may not be followed. It depends on the Instruction Following capabilities of the model."},{"type":"string"}],"default":"auto","description":"(Optional) Whether tool use is automatic, required, or none. Can also specify a tool name to use a specific tool. Defaults to ToolChoice.auto."},"tool_prompt_format":{"type":"string","enum":["json","function_tag","python_list"],"description":"(Optional) Instructs the model how to format tool calls. By default, Llama Stack will attempt to use a format that is best adapted to the model. - `ToolPromptFormat.json`: The tool calls are formatted as a JSON object. - `ToolPromptFormat.function_tag`: The tool calls are enclosed in a <function=function_name> tag. - `ToolPromptFormat.python_list`: The tool calls are output as Python syntax -- a list of function calls."},"system_message_behavior":{"type":"string","enum":["append","replace"],"description":"(Optional) Config for how to override the default system prompt. - `SystemMessageBehavior.append`: Appends the provided system message to the default system prompt. - `SystemMessageBehavior.replace`: Replaces the default system prompt with the provided system message. The system message can include the string '{{function_definitions}}' to indicate where the function definitions should be inserted.","default":"append"}},"additionalProperties":false,"title":"ToolConfig","description":"Configuration for tool use."},"max_infer_iters":{"type":"integer","default":10},"model":{"type":"string","description":"The model identifier to use for the agent"},"instructions":{"type":"string","description":"The system instructions for the agent"},"name":{"type":"string","description":"Optional name for the agent, used in telemetry and identification"},"enable_session_persistence":{"type":"boolean","default":false,"description":"Optional flag indicating whether session data has to be persisted"},"response_format":{"description":"Optional response format configuration","oneOf":[{"type":"object","properties":{"type":{"type":"string","enum":["json_schema","grammar"],"description":"Must be \"json_schema\" to identify this format type","const":"json_schema","default":"json_schema"},"json_schema":{"type":"object","additionalProperties":{"oneOf":[{"type":"null"},{"type":"boolean"},{"type":"number"},{"type":"string"},{"type":"array"},{"type":"object"}]},"description":"The JSON schema the response should conform to. In a Python SDK, this is often a `pydantic` model."}},"additionalProperties":false,"required":["type","json_schema"],"title":"JsonSchemaResponseFormat","description":"Configuration for JSON schema-guided response generation."},{"type":"object","properties":{"type":{"type":"string","enum":["json_schema","grammar"],"description":"Must be \"grammar\" to identify this format type","const":"grammar","default":"grammar"},"bnf":{"type":"object","additionalProperties":{"oneOf":[{"type":"null"},{"type":"boolean"},{"type":"number"},{"type":"string"},{"type":"array"},{"type":"object"}]},"description":"The BNF grammar specification the response should conform to"}},"additionalProperties":false,"required":["type","bnf"],"title":"GrammarResponseFormat","description":"Configuration for grammar-guided response generation."}],"discriminator":{"propertyName":"type","mapping":{"json_schema":{"type":"object","properties":{"type":{"type":"string","enum":["json_schema","grammar"],"description":"Must be \"json_schema\" to identify this format type","const":"json_schema","default":"json_schema"},"json_schema":{"type":"object","additionalProperties":{"oneOf":[{"type":"null"},{"type":"boolean"},{"type":"number"},{"type":"string"},{"type":"array"},{"type":"object"}]},"description":"The JSON schema the response should conform to. In a Python SDK, this is often a `pydantic` model."}},"additionalProperties":false,"required":["type","json_schema"],"title":"JsonSchemaResponseFormat","description":"Configuration for JSON schema-guided response generation."},"grammar":{"type":"object","properties":{"type":{"type":"string","enum":["json_schema","grammar"],"description":"Must be \"grammar\" to identify this format type","const":"grammar","default":"grammar"},"bnf":{"type":"object","additionalProperties":{"oneOf":[{"type":"null"},{"type":"boolean"},{"type":"number"},{"type":"string"},{"type":"array"},{"type":"object"}]},"description":"The BNF grammar specification the response should conform to"}},"additionalProperties":false,"required":["type","bnf"],"title":"GrammarResponseFormat","description":"Configuration for grammar-guided response generation."}}},"title":"ResponseFormat"}},"additionalProperties":false,"required":["model","instructions"],"title":"AgentConfig"}},"additionalProperties":false,"required":["type","config"],"title":"AgentCandidate","description":"An agent candidate for evaluation."}],"discriminator":{"propertyName":"type","mapping":{"model":{"type":"object","properties":{"type":{"type":"string","const":"model","default":"model"},"model":{"type":"string","description":"The model ID to evaluate."},"sampling_params":{"type":"object","properties":{"strategy":{"description":"The sampling strategy.","oneOf":[{"type":"object","properties":{"type":{"type":"string","const":"greedy","default":"greedy","description":"Must be \"greedy\" to identify this sampling strategy"}},"additionalProperties":false,"required":["type"],"title":"GreedySamplingStrategy","description":"Greedy sampling strategy that selects the highest probability token at each step."},{"type":"object","properties":{"type":{"type":"string","const":"top_p","default":"top_p","description":"Must be \"top_p\" to identify this sampling strategy"},"temperature":{"type":"number","description":"Controls randomness in sampling. Higher values increase randomness"},"top_p":{"type":"number","default":0.95,"description":"Cumulative probability threshold for nucleus sampling. Defaults to 0.95"}},"additionalProperties":false,"required":["type"],"title":"TopPSamplingStrategy","description":"Top-p (nucleus) sampling strategy that samples from the smallest set of tokens with cumulative probability >= p."},{"type":"object","properties":{"type":{"type":"string","const":"top_k","default":"top_k","description":"Must be \"top_k\" to identify this sampling strategy"},"top_k":{"type":"integer","description":"Number of top tokens to consider for sampling. Must be at least 1"}},"additionalProperties":false,"required":["type","top_k"],"title":"TopKSamplingStrategy","description":"Top-k sampling strategy that restricts sampling to the k most likely tokens."}],"discriminator":{"propertyName":"type","mapping":{"greedy":{"type":"object","properties":{"type":{"type":"string","const":"greedy","default":"greedy","description":"Must be \"greedy\" to identify this sampling strategy"}},"additionalProperties":false,"required":["type"],"title":"GreedySamplingStrategy","description":"Greedy sampling strategy that selects the highest probability token at each step."},"top_p":{"type":"object","properties":{"type":{"type":"string","const":"top_p","default":"top_p","description":"Must be \"top_p\" to identify this sampling strategy"},"temperature":{"type":"number","description":"Controls randomness in sampling. Higher values increase randomness"},"top_p":{"type":"number","default":0.95,"description":"Cumulative probability threshold for nucleus sampling. Defaults to 0.95"}},"additionalProperties":false,"required":["type"],"title":"TopPSamplingStrategy","description":"Top-p (nucleus) sampling strategy that samples from the smallest set of tokens with cumulative probability >= p."},"top_k":{"type":"object","properties":{"type":{"type":"string","const":"top_k","default":"top_k","description":"Must be \"top_k\" to identify this sampling strategy"},"top_k":{"type":"integer","description":"Number of top tokens to consider for sampling. Must be at least 1"}},"additionalProperties":false,"required":["type","top_k"],"title":"TopKSamplingStrategy","description":"Top-k sampling strategy that restricts sampling to the k most likely tokens."}}},"title":"SamplingStrategy"},"max_tokens":{"type":"integer","default":0,"description":"The maximum number of tokens that can be generated in the completion. The token count of your prompt plus max_tokens cannot exceed the model's context length."},"repetition_penalty":{"type":"number","default":1,"description":"Number between -2.0 and 2.0. Positive values penalize new tokens based on whether they appear in the text so far, increasing the model's likelihood to talk about new topics."},"stop":{"type":"array","items":{"type":"string"},"description":"Up to 4 sequences where the API will stop generating further tokens. The returned text will not contain the stop sequence."}},"additionalProperties":false,"required":["strategy"],"title":"SamplingParams","description":"Sampling parameters."},"system_message":{"type":"object","properties":{"role":{"type":"string","const":"system","default":"system","description":"Must be \"system\" to identify this as a system message"},"content":{"description":"The content of the \"system prompt\". If multiple system messages are provided, they are concatenated. The underlying Llama Stack code may also add other system messages (for example, for formatting tool definitions).","oneOf":[{"type":"string"},{"oneOf":[{"type":"object","properties":{"type":{"type":"string","const":"image","default":"image","description":"Discriminator type of the content item. Always \"image\""},"image":{"type":"object","properties":{"url":{"description":"A URL of the image or data URL in the format of data:image/{type};base64,{data}. Note that URL could have length limits.","type":"object","properties":{"uri":{"type":"string","description":"The URL string pointing to the resource"}},"additionalProperties":false,"required":["uri"],"title":"URL"},"data":{"type":"string","contentEncoding":"base64","description":"base64 encoded image data as string"}},"additionalProperties":false,"description":"Image as a base64 encoded string or an URL"}},"additionalProperties":false,"required":["type","image"],"title":"ImageContentItem","description":"A image content item"},{"type":"object","properties":{"type":{"type":"string","const":"text","default":"text","description":"Discriminator type of the content item. Always \"text\""},"text":{"type":"string","description":"Text content"}},"additionalProperties":false,"required":["type","text"],"title":"TextContentItem","description":"A text content item"}],"discriminator":{"propertyName":"type","mapping":{"image":{"type":"object","properties":{"type":{"type":"string","const":"image","default":"image","description":"Discriminator type of the content item. Always \"image\""},"image":{"type":"object","properties":{"url":{"description":"A URL of the image or data URL in the format of data:image/{type};base64,{data}. Note that URL could have length limits.","type":"object","properties":{"uri":{"type":"string","description":"The URL string pointing to the resource"}},"additionalProperties":false,"required":["uri"],"title":"URL"},"data":{"type":"string","contentEncoding":"base64","description":"base64 encoded image data as string"}},"additionalProperties":false,"description":"Image as a base64 encoded string or an URL"}},"additionalProperties":false,"required":["type","image"],"title":"ImageContentItem","description":"A image content item"},"text":{"type":"object","properties":{"type":{"type":"string","const":"text","default":"text","description":"Discriminator type of the content item. Always \"text\""},"text":{"type":"string","description":"Text content"}},"additionalProperties":false,"required":["type","text"],"title":"TextContentItem","description":"A text content item"}}},"title":"InterleavedContentItem"},{"type":"array","items":{"oneOf":[{"type":"object","properties":{"type":{"type":"string","const":"image","default":"image","description":"Discriminator type of the content item. Always \"image\""},"image":{"type":"object","properties":{"url":{"description":"A URL of the image or data URL in the format of data:image/{type};base64,{data}. Note that URL could have length limits.","type":"object","properties":{"uri":{"type":"string","description":"The URL string pointing to the resource"}},"additionalProperties":false,"required":["uri"],"title":"URL"},"data":{"type":"string","contentEncoding":"base64","description":"base64 encoded image data as string"}},"additionalProperties":false,"description":"Image as a base64 encoded string or an URL"}},"additionalProperties":false,"required":["type","image"],"title":"ImageContentItem","description":"A image content item"},{"type":"object","properties":{"type":{"type":"string","const":"text","default":"text","description":"Discriminator type of the content item. Always \"text\""},"text":{"type":"string","description":"Text content"}},"additionalProperties":false,"required":["type","text"],"title":"TextContentItem","description":"A text content item"}],"discriminator":{"propertyName":"type","mapping":{"image":{"type":"object","properties":{"type":{"type":"string","const":"image","default":"image","description":"Discriminator type of the content item. Always \"image\""},"image":{"type":"object","properties":{"url":{"description":"A URL of the image or data URL in the format of data:image/{type};base64,{data}. Note that URL could have length limits.","type":"object","properties":{"uri":{"type":"string","description":"The URL string pointing to the resource"}},"additionalProperties":false,"required":["uri"],"title":"URL"},"data":{"type":"string","contentEncoding":"base64","description":"base64 encoded image data as string"}},"additionalProperties":false,"description":"Image as a base64 encoded string or an URL"}},"additionalProperties":false,"required":["type","image"],"title":"ImageContentItem","description":"A image content item"},"text":{"type":"object","properties":{"type":{"type":"string","const":"text","default":"text","description":"Discriminator type of the content item. Always \"text\""},"text":{"type":"string","description":"Text content"}},"additionalProperties":false,"required":["type","text"],"title":"TextContentItem","description":"A text content item"}}},"title":"InterleavedContentItem"}}],"title":"InterleavedContent"}},"additionalProperties":false,"required":["role","content"],"title":"SystemMessage","description":"A system message providing instructions or context to the model."}},"additionalProperties":false,"required":["type","model","sampling_params"],"title":"ModelCandidate","description":"A model candidate for evaluation."},"agent":{"type":"object","properties":{"type":{"type":"string","const":"agent","default":"agent"},"config":{"description":"The configuration for the agent candidate.","type":"object","properties":{"sampling_params":{"type":"object","properties":{"strategy":{"description":"The sampling strategy.","oneOf":[{"type":"object","properties":{"type":{"type":"string","const":"greedy","default":"greedy","description":"Must be \"greedy\" to identify this sampling strategy"}},"additionalProperties":false,"required":["type"],"title":"GreedySamplingStrategy","description":"Greedy sampling strategy that selects the highest probability token at each step."},{"type":"object","properties":{"type":{"type":"string","const":"top_p","default":"top_p","description":"Must be \"top_p\" to identify this sampling strategy"},"temperature":{"type":"number","description":"Controls randomness in sampling. Higher values increase randomness"},"top_p":{"type":"number","default":0.95,"description":"Cumulative probability threshold for nucleus sampling. Defaults to 0.95"}},"additionalProperties":false,"required":["type"],"title":"TopPSamplingStrategy","description":"Top-p (nucleus) sampling strategy that samples from the smallest set of tokens with cumulative probability >= p."},{"type":"object","properties":{"type":{"type":"string","const":"top_k","default":"top_k","description":"Must be \"top_k\" to identify this sampling strategy"},"top_k":{"type":"integer","description":"Number of top tokens to consider for sampling. Must be at least 1"}},"additionalProperties":false,"required":["type","top_k"],"title":"TopKSamplingStrategy","description":"Top-k sampling strategy that restricts sampling to the k most likely tokens."}],"discriminator":{"propertyName":"type","mapping":{"greedy":{"type":"object","properties":{"type":{"type":"string","const":"greedy","default":"greedy","description":"Must be \"greedy\" to identify this sampling strategy"}},"additionalProperties":false,"required":["type"],"title":"GreedySamplingStrategy","description":"Greedy sampling strategy that selects the highest probability token at each step."},"top_p":{"type":"object","properties":{"type":{"type":"string","const":"top_p","default":"top_p","description":"Must be \"top_p\" to identify this sampling strategy"},"temperature":{"type":"number","description":"Controls randomness in sampling. Higher values increase randomness"},"top_p":{"type":"number","default":0.95,"description":"Cumulative probability threshold for nucleus sampling. Defaults to 0.95"}},"additionalProperties":false,"required":["type"],"title":"TopPSamplingStrategy","description":"Top-p (nucleus) sampling strategy that samples from the smallest set of tokens with cumulative probability >= p."},"top_k":{"type":"object","properties":{"type":{"type":"string","const":"top_k","default":"top_k","description":"Must be \"top_k\" to identify this sampling strategy"},"top_k":{"type":"integer","description":"Number of top tokens to consider for sampling. Must be at least 1"}},"additionalProperties":false,"required":["type","top_k"],"title":"TopKSamplingStrategy","description":"Top-k sampling strategy that restricts sampling to the k most likely tokens."}}},"title":"SamplingStrategy"},"max_tokens":{"type":"integer","default":0,"description":"The maximum number of tokens that can be generated in the completion. The token count of your prompt plus max_tokens cannot exceed the model's context length."},"repetition_penalty":{"type":"number","default":1,"description":"Number between -2.0 and 2.0. Positive values penalize new tokens based on whether they appear in the text so far, increasing the model's likelihood to talk about new topics."},"stop":{"type":"array","items":{"type":"string"},"description":"Up to 4 sequences where the API will stop generating further tokens. The returned text will not contain the stop sequence."}},"additionalProperties":false,"required":["strategy"],"title":"SamplingParams","description":"Sampling parameters."},"input_shields":{"type":"array","items":{"type":"string"}},"output_shields":{"type":"array","items":{"type":"string"}},"toolgroups":{"type":"array","items":{"oneOf":[{"type":"string"},{"type":"object","properties":{"name":{"type":"string"},"args":{"type":"object","additionalProperties":{"oneOf":[{"type":"null"},{"type":"boolean"},{"type":"number"},{"type":"string"},{"type":"array"},{"type":"object"}]}}},"additionalProperties":false,"required":["name","args"],"title":"AgentToolGroupWithArgs"}],"title":"AgentTool"}},"client_tools":{"type":"array","items":{"type":"object","properties":{"name":{"type":"string","description":"Name of the tool"},"description":{"type":"string","description":"(Optional) Human-readable description of what the tool does"},"parameters":{"type":"array","items":{"type":"object","properties":{"name":{"type":"string","description":"Name of the parameter"},"parameter_type":{"type":"string","description":"Type of the parameter (e.g., string, integer)"},"description":{"type":"string","description":"Human-readable description of what the parameter does"},"required":{"type":"boolean","default":true,"description":"Whether this parameter is required for tool invocation"},"default":{"oneOf":[{"type":"null"},{"type":"boolean"},{"type":"number"},{"type":"string"},{"type":"array"},{"type":"object"}],"description":"(Optional) Default value for the parameter if not provided"}},"additionalProperties":false,"required":["name","parameter_type","description","required"],"title":"ToolParameter","description":"Parameter definition for a tool."},"description":"(Optional) List of parameters this tool accepts"},"metadata":{"type":"object","additionalProperties":{"oneOf":[{"type":"null"},{"type":"boolean"},{"type":"number"},{"type":"string"},{"type":"array"},{"type":"object"}]},"description":"(Optional) Additional metadata about the tool"}},"additionalProperties":false,"required":["name"],"title":"ToolDef","description":"Tool definition used in runtime contexts."}},"tool_choice":{"type":"string","enum":["auto","required","none"],"title":"ToolChoice","description":"Whether tool use is required or automatic. This is a hint to the model which may not be followed. It depends on the Instruction Following capabilities of the model.","deprecated":true},"tool_prompt_format":{"type":"string","enum":["json","function_tag","python_list"],"title":"ToolPromptFormat","description":"Prompt format for calling custom / zero shot tools.","deprecated":true},"tool_config":{"type":"object","properties":{"tool_choice":{"oneOf":[{"type":"string","enum":["auto","required","none"],"title":"ToolChoice","description":"Whether tool use is required or automatic. This is a hint to the model which may not be followed. It depends on the Instruction Following capabilities of the model."},{"type":"string"}],"default":"auto","description":"(Optional) Whether tool use is automatic, required, or none. Can also specify a tool name to use a specific tool. Defaults to ToolChoice.auto."},"tool_prompt_format":{"type":"string","enum":["json","function_tag","python_list"],"description":"(Optional) Instructs the model how to format tool calls. By default, Llama Stack will attempt to use a format that is best adapted to the model. - `ToolPromptFormat.json`: The tool calls are formatted as a JSON object. - `ToolPromptFormat.function_tag`: The tool calls are enclosed in a <function=function_name> tag. - `ToolPromptFormat.python_list`: The tool calls are output as Python syntax -- a list of function calls."},"system_message_behavior":{"type":"string","enum":["append","replace"],"description":"(Optional) Config for how to override the default system prompt. - `SystemMessageBehavior.append`: Appends the provided system message to the default system prompt. - `SystemMessageBehavior.replace`: Replaces the default system prompt with the provided system message. The system message can include the string '{{function_definitions}}' to indicate where the function definitions should be inserted.","default":"append"}},"additionalProperties":false,"title":"ToolConfig","description":"Configuration for tool use."},"max_infer_iters":{"type":"integer","default":10},"model":{"type":"string","description":"The model identifier to use for the agent"},"instructions":{"type":"string","description":"The system instructions for the agent"},"name":{"type":"string","description":"Optional name for the agent, used in telemetry and identification"},"enable_session_persistence":{"type":"boolean","default":false,"description":"Optional flag indicating whether session data has to be persisted"},"response_format":{"description":"Optional response format configuration","oneOf":[{"type":"object","properties":{"type":{"type":"string","enum":["json_schema","grammar"],"description":"Must be \"json_schema\" to identify this format type","const":"json_schema","default":"json_schema"},"json_schema":{"type":"object","additionalProperties":{"oneOf":[{"type":"null"},{"type":"boolean"},{"type":"number"},{"type":"string"},{"type":"array"},{"type":"object"}]},"description":"The JSON schema the response should conform to. In a Python SDK, this is often a `pydantic` model."}},"additionalProperties":false,"required":["type","json_schema"],"title":"JsonSchemaResponseFormat","description":"Configuration for JSON schema-guided response generation."},{"type":"object","properties":{"type":{"type":"string","enum":["json_schema","grammar"],"description":"Must be \"grammar\" to identify this format type","const":"grammar","default":"grammar"},"bnf":{"type":"object","additionalProperties":{"oneOf":[{"type":"null"},{"type":"boolean"},{"type":"number"},{"type":"string"},{"type":"array"},{"type":"object"}]},"description":"The BNF grammar specification the response should conform to"}},"additionalProperties":false,"required":["type","bnf"],"title":"GrammarResponseFormat","description":"Configuration for grammar-guided response generation."}],"discriminator":{"propertyName":"type","mapping":{"json_schema":{"type":"object","properties":{"type":{"type":"string","enum":["json_schema","grammar"],"description":"Must be \"json_schema\" to identify this format type","const":"json_schema","default":"json_schema"},"json_schema":{"type":"object","additionalProperties":{"oneOf":[{"type":"null"},{"type":"boolean"},{"type":"number"},{"type":"string"},{"type":"array"},{"type":"object"}]},"description":"The JSON schema the response should conform to. In a Python SDK, this is often a `pydantic` model."}},"additionalProperties":false,"required":["type","json_schema"],"title":"JsonSchemaResponseFormat","description":"Configuration for JSON schema-guided response generation."},"grammar":{"type":"object","properties":{"type":{"type":"string","enum":["json_schema","grammar"],"description":"Must be \"grammar\" to identify this format type","const":"grammar","default":"grammar"},"bnf":{"type":"object","additionalProperties":{"oneOf":[{"type":"null"},{"type":"boolean"},{"type":"number"},{"type":"string"},{"type":"array"},{"type":"object"}]},"description":"The BNF grammar specification the response should conform to"}},"additionalProperties":false,"required":["type","bnf"],"title":"GrammarResponseFormat","description":"Configuration for grammar-guided response generation."}}},"title":"ResponseFormat"}},"additionalProperties":false,"required":["model","instructions"],"title":"AgentConfig"}},"additionalProperties":false,"required":["type","config"],"title":"AgentCandidate","description":"An agent candidate for evaluation."}}},"title":"EvalCandidate"},"scoring_params":{"type":"object","additionalProperties":{"oneOf":[{"type":"object","properties":{"type":{"const":"llm_as_judge","default":"llm_as_judge","description":"The type of scoring function parameters, always llm_as_judge","type":"string","enum":["llm_as_judge","regex_parser","basic"],"title":"ScoringFnParamsType"},"judge_model":{"type":"string","description":"Identifier of the LLM model to use as a judge for scoring"},"prompt_template":{"type":"string","description":"(Optional) Custom prompt template for the judge model"},"judge_score_regexes":{"type":"array","items":{"type":"string"},"description":"Regexes to extract the answer from generated response"},"aggregation_functions":{"type":"array","items":{"type":"string","enum":["average","weighted_average","median","categorical_count","accuracy"],"title":"AggregationFunctionType","description":"Types of aggregation functions for scoring results."},"description":"Aggregation functions to apply to the scores of each row"}},"additionalProperties":false,"required":["type","judge_model","judge_score_regexes","aggregation_functions"],"title":"LLMAsJudgeScoringFnParams","description":"Parameters for LLM-as-judge scoring function configuration."},{"type":"object","properties":{"type":{"const":"regex_parser","default":"regex_parser","description":"The type of scoring function parameters, always regex_parser","type":"string","enum":["llm_as_judge","regex_parser","basic"],"title":"ScoringFnParamsType"},"parsing_regexes":{"type":"array","items":{"type":"string"},"description":"Regex to extract the answer from generated response"},"aggregation_functions":{"type":"array","items":{"type":"string","enum":["average","weighted_average","median","categorical_count","accuracy"],"title":"AggregationFunctionType","description":"Types of aggregation functions for scoring results."},"description":"Aggregation functions to apply to the scores of each row"}},"additionalProperties":false,"required":["type","parsing_regexes","aggregation_functions"],"title":"RegexParserScoringFnParams","description":"Parameters for regex parser scoring function configuration."},{"type":"object","properties":{"type":{"const":"basic","default":"basic","description":"The type of scoring function parameters, always basic","type":"string","enum":["llm_as_judge","regex_parser","basic"],"title":"ScoringFnParamsType"},"aggregation_functions":{"type":"array","items":{"type":"string","enum":["average","weighted_average","median","categorical_count","accuracy"],"title":"AggregationFunctionType","description":"Types of aggregation functions for scoring results."},"description":"Aggregation functions to apply to the scores of each row"}},"additionalProperties":false,"required":["type","aggregation_functions"],"title":"BasicScoringFnParams","description":"Parameters for basic scoring function configuration."}],"discriminator":{"propertyName":"type","mapping":{"llm_as_judge":{"type":"object","properties":{"type":{"const":"llm_as_judge","default":"llm_as_judge","description":"The type of scoring function parameters, always llm_as_judge","type":"string","enum":["llm_as_judge","regex_parser","basic"],"title":"ScoringFnParamsType"},"judge_model":{"type":"string","description":"Identifier of the LLM model to use as a judge for scoring"},"prompt_template":{"type":"string","description":"(Optional) Custom prompt template for the judge model"},"judge_score_regexes":{"type":"array","items":{"type":"string"},"description":"Regexes to extract the answer from generated response"},"aggregation_functions":{"type":"array","items":{"type":"string","enum":["average","weighted_average","median","categorical_count","accuracy"],"title":"AggregationFunctionType","description":"Types of aggregation functions for scoring results."},"description":"Aggregation functions to apply to the scores of each row"}},"additionalProperties":false,"required":["type","judge_model","judge_score_regexes","aggregation_functions"],"title":"LLMAsJudgeScoringFnParams","description":"Parameters for LLM-as-judge scoring function configuration."},"regex_parser":{"type":"object","properties":{"type":{"const":"regex_parser","default":"regex_parser","description":"The type of scoring function parameters, always regex_parser","type":"string","enum":["llm_as_judge","regex_parser","basic"],"title":"ScoringFnParamsType"},"parsing_regexes":{"type":"array","items":{"type":"string"},"description":"Regex to extract the answer from generated response"},"aggregation_functions":{"type":"array","items":{"type":"string","enum":["average","weighted_average","median","categorical_count","accuracy"],"title":"AggregationFunctionType","description":"Types of aggregation functions for scoring results."},"description":"Aggregation functions to apply to the scores of each row"}},"additionalProperties":false,"required":["type","parsing_regexes","aggregation_functions"],"title":"RegexParserScoringFnParams","description":"Parameters for regex parser scoring function configuration."},"basic":{"type":"object","properties":{"type":{"const":"basic","default":"basic","description":"The type of scoring function parameters, always basic","type":"string","enum":["llm_as_judge","regex_parser","basic"],"title":"ScoringFnParamsType"},"aggregation_functions":{"type":"array","items":{"type":"string","enum":["average","weighted_average","median","categorical_count","accuracy"],"title":"AggregationFunctionType","description":"Types of aggregation functions for scoring results."},"description":"Aggregation functions to apply to the scores of each row"}},"additionalProperties":false,"required":["type","aggregation_functions"],"title":"BasicScoringFnParams","description":"Parameters for basic scoring function configuration."}}},"title":"ScoringFnParams"},"description":"Map between scoring function id and parameters for each scoring function you want to run"},"num_examples":{"type":"integer","description":"(Optional) The number of examples to evaluate. If not provided, all examples in the dataset will be evaluated"}},"additionalProperties":false,"required":["eval_candidate","scoring_params"],"title":"BenchmarkConfig"}},"additionalProperties":false,"required":["input_rows","scoring_functions","benchmark_config"],"title":"EvaluateRowsRequest"}}},"required":true}}
|
|
>
|
|
|
|
</RequestSchema>
|
|
|
|
<StatusCodes
|
|
id={undefined}
|
|
label={undefined}
|
|
responses={{"200":{"description":"EvaluateResponse object containing generations and scores.","content":{"application/json":{"schema":{"type":"object","properties":{"generations":{"type":"array","items":{"type":"object","additionalProperties":{"oneOf":[{"type":"null"},{"type":"boolean"},{"type":"number"},{"type":"string"},{"type":"array"},{"type":"object"}]}},"description":"The generations from the evaluation."},"scores":{"type":"object","additionalProperties":{"type":"object","properties":{"score_rows":{"type":"array","items":{"type":"object","additionalProperties":{"oneOf":[{"type":"null"},{"type":"boolean"},{"type":"number"},{"type":"string"},{"type":"array"},{"type":"object"}]}},"description":"The scoring result for each row. Each row is a map of column name to value."},"aggregated_results":{"type":"object","additionalProperties":{"oneOf":[{"type":"null"},{"type":"boolean"},{"type":"number"},{"type":"string"},{"type":"array"},{"type":"object"}]},"description":"Map of metric name to aggregated value"}},"additionalProperties":false,"required":["score_rows","aggregated_results"],"title":"ScoringResult","description":"A scoring result for a single row."},"description":"The scores from the evaluation."}},"additionalProperties":false,"required":["generations","scores"],"title":"EvaluateResponse","description":"The response from an evaluation."}}}},"400":{"description":"The request was invalid or malformed","content":{"application/json":{"schema":{"type":"object","properties":{"status":{"type":"integer","description":"HTTP status code"},"title":{"type":"string","description":"Error title, a short summary of the error which is invariant for an error type"},"detail":{"type":"string","description":"Error detail, a longer human-readable description of the error"},"instance":{"type":"string","description":"(Optional) A URL which can be used to retrieve more information about the specific occurrence of the error"}},"additionalProperties":false,"required":["status","title","detail"],"title":"Error","description":"Error response from the API. Roughly follows RFC 7807."},"example":{"status":400,"title":"Bad Request","detail":"The request was invalid or malformed"}}}},"429":{"description":"The client has sent too many requests in a given amount of time","content":{"application/json":{"schema":{"type":"object","properties":{"status":{"type":"integer","description":"HTTP status code"},"title":{"type":"string","description":"Error title, a short summary of the error which is invariant for an error type"},"detail":{"type":"string","description":"Error detail, a longer human-readable description of the error"},"instance":{"type":"string","description":"(Optional) A URL which can be used to retrieve more information about the specific occurrence of the error"}},"additionalProperties":false,"required":["status","title","detail"],"title":"Error","description":"Error response from the API. Roughly follows RFC 7807."},"example":{"status":429,"title":"Too Many Requests","detail":"You have exceeded the rate limit. Please try again later."}}}},"500":{"description":"The server encountered an unexpected error","content":{"application/json":{"schema":{"type":"object","properties":{"status":{"type":"integer","description":"HTTP status code"},"title":{"type":"string","description":"Error title, a short summary of the error which is invariant for an error type"},"detail":{"type":"string","description":"Error detail, a longer human-readable description of the error"},"instance":{"type":"string","description":"(Optional) A URL which can be used to retrieve more information about the specific occurrence of the error"}},"additionalProperties":false,"required":["status","title","detail"],"title":"Error","description":"Error response from the API. Roughly follows RFC 7807."},"example":{"status":500,"title":"Internal Server Error","detail":"An unexpected error occurred. Our team has been notified."}}}},"default":{"description":"An unexpected error occurred","content":{"application/json":{"schema":{"type":"object","properties":{"status":{"type":"integer","description":"HTTP status code"},"title":{"type":"string","description":"Error title, a short summary of the error which is invariant for an error type"},"detail":{"type":"string","description":"Error detail, a longer human-readable description of the error"},"instance":{"type":"string","description":"(Optional) A URL which can be used to retrieve more information about the specific occurrence of the error"}},"additionalProperties":false,"required":["status","title","detail"],"title":"Error","description":"Error response from the API. Roughly follows RFC 7807."},"example":{"status":0,"title":"Error","detail":"An unexpected error occurred"}}}}}}
|
|
>
|
|
|
|
</StatusCodes>
|
|
|
|
|
|
|