phoenix-oss/llama-stack-mirror
0
0
Fork 1
mirror of https://github.com/meta-llama/llama-stack.git synced 2025-07-29 15:23:51 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Add a special header per-client call to parser provider data

Browse source
This commit is contained in:
Ashwin Bharambe 2024-09-18 09:17:59 -07:00 • committed by Xi Yan
parent a6be32bc3d
commit 32beecb20d
11 changed files with 955 additions and 104 deletions
Download patch file Download diff file Expand all files Collapse all files

527
docs/llama-stack-spec.html
View file

@ -21,7 +21,7 @@
"info": { "info": {
"title": "[DRAFT] Llama Stack Specification", "title": "[DRAFT] Llama Stack Specification",
"version": "0.0.1", "version": "0.0.1",
"description": "This is the specification of the llama stack that provides\n a set of endpoints and their corresponding interfaces that are tailored to\n best leverage Llama Models. The specification is still in draft and subject to change.\n Generated at 2024-09-18 19:27:39.955190" "description": "This is the specification of the llama stack that provides\n a set of endpoints and their corresponding interfaces that are tailored to\n best leverage Llama Models. The specification is still in draft and subject to change.\n Generated at 2024-09-20 13:32:23.841908"
}, },
"servers": [ "servers": [
{ {
@ -46,7 +46,17 @@
"tags": [ "tags": [
"BatchInference" "BatchInference"
], ],
"parameters": [], "parameters": [
{
"name": "X-LlamaStack-ProviderData",
"in": "header",
"description": "JSON-encoded provider data which will be made available to the adapter servicing the API",
"required": false,
"schema": {
"type": "string"
}
}
],
"requestBody": { "requestBody": {
"content": { "content": {
"application/json": { "application/json": {
@ -76,7 +86,17 @@
"tags": [ "tags": [
"BatchInference" "BatchInference"
], ],
"parameters": [], "parameters": [
{
"name": "X-LlamaStack-ProviderData",
"in": "header",
"description": "JSON-encoded provider data which will be made available to the adapter servicing the API",
"required": false,
"schema": {
"type": "string"
}
}
],
"requestBody": { "requestBody": {
"content": { "content": {
"application/json": { "application/json": {
@ -99,7 +119,17 @@
"tags": [ "tags": [
"Evaluations" "Evaluations"
], ],
"parameters": [], "parameters": [
{
"name": "X-LlamaStack-ProviderData",
"in": "header",
"description": "JSON-encoded provider data which will be made available to the adapter servicing the API",
"required": false,
"schema": {
"type": "string"
}
}
],
"requestBody": { "requestBody": {
"content": { "content": {
"application/json": { "application/json": {
@ -122,7 +152,17 @@
"tags": [ "tags": [
"PostTraining" "PostTraining"
], ],
"parameters": [], "parameters": [
{
"name": "X-LlamaStack-ProviderData",
"in": "header",
"description": "JSON-encoded provider data which will be made available to the adapter servicing the API",
"required": false,
"schema": {
"type": "string"
}
}
],
"requestBody": { "requestBody": {
"content": { "content": {
"application/json": { "application/json": {
@ -159,7 +199,17 @@
"tags": [ "tags": [
"Inference" "Inference"
], ],
"parameters": [], "parameters": [
{
"name": "X-LlamaStack-ProviderData",
"in": "header",
"description": "JSON-encoded provider data which will be made available to the adapter servicing the API",
"required": false,
"schema": {
"type": "string"
}
}
],
"requestBody": { "requestBody": {
"content": { "content": {
"application/json": { "application/json": {
@ -196,7 +246,17 @@
"tags": [ "tags": [
"Inference" "Inference"
], ],
"parameters": [], "parameters": [
{
"name": "X-LlamaStack-ProviderData",
"in": "header",
"description": "JSON-encoded provider data which will be made available to the adapter servicing the API",
"required": false,
"schema": {
"type": "string"
}
}
],
"requestBody": { "requestBody": {
"content": { "content": {
"application/json": { "application/json": {
@ -226,7 +286,17 @@
"tags": [ "tags": [
"Agents" "Agents"
], ],
"parameters": [], "parameters": [
{
"name": "X-LlamaStack-ProviderData",
"in": "header",
"description": "JSON-encoded provider data which will be made available to the adapter servicing the API",
"required": false,
"schema": {
"type": "string"
}
}
],
"requestBody": { "requestBody": {
"content": { "content": {
"application/json": { "application/json": {
@ -256,7 +326,17 @@
"tags": [ "tags": [
"Agents" "Agents"
], ],
"parameters": [], "parameters": [
{
"name": "X-LlamaStack-ProviderData",
"in": "header",
"description": "JSON-encoded provider data which will be made available to the adapter servicing the API",
"required": false,
"schema": {
"type": "string"
}
}
],
"requestBody": { "requestBody": {
"content": { "content": {
"application/json": { "application/json": {
@ -286,7 +366,17 @@
"tags": [ "tags": [
"Agents" "Agents"
], ],
"parameters": [], "parameters": [
{
"name": "X-LlamaStack-ProviderData",
"in": "header",
"description": "JSON-encoded provider data which will be made available to the adapter servicing the API",
"required": false,
"schema": {
"type": "string"
}
}
],
"requestBody": { "requestBody": {
"content": { "content": {
"application/json": { "application/json": {
@ -309,7 +399,17 @@
"tags": [ "tags": [
"Datasets" "Datasets"
], ],
"parameters": [], "parameters": [
{
"name": "X-LlamaStack-ProviderData",
"in": "header",
"description": "JSON-encoded provider data which will be made available to the adapter servicing the API",
"required": false,
"schema": {
"type": "string"
}
}
],
"requestBody": { "requestBody": {
"content": { "content": {
"application/json": { "application/json": {
@ -339,7 +439,17 @@
"tags": [ "tags": [
"Memory" "Memory"
], ],
"parameters": [], "parameters": [
{
"name": "X-LlamaStack-ProviderData",
"in": "header",
"description": "JSON-encoded provider data which will be made available to the adapter servicing the API",
"required": false,
"schema": {
"type": "string"
}
}
],
"requestBody": { "requestBody": {
"content": { "content": {
"application/json": { "application/json": {
@ -362,7 +472,17 @@
"tags": [ "tags": [
"Agents" "Agents"
], ],
"parameters": [], "parameters": [
{
"name": "X-LlamaStack-ProviderData",
"in": "header",
"description": "JSON-encoded provider data which will be made available to the adapter servicing the API",
"required": false,
"schema": {
"type": "string"
}
}
],
"requestBody": { "requestBody": {
"content": { "content": {
"application/json": { "application/json": {
@ -385,7 +505,17 @@
"tags": [ "tags": [
"Agents" "Agents"
], ],
"parameters": [], "parameters": [
{
"name": "X-LlamaStack-ProviderData",
"in": "header",
"description": "JSON-encoded provider data which will be made available to the adapter servicing the API",
"required": false,
"schema": {
"type": "string"
}
}
],
"requestBody": { "requestBody": {
"content": { "content": {
"application/json": { "application/json": {
@ -408,7 +538,17 @@
"tags": [ "tags": [
"Datasets" "Datasets"
], ],
"parameters": [], "parameters": [
{
"name": "X-LlamaStack-ProviderData",
"in": "header",
"description": "JSON-encoded provider data which will be made available to the adapter servicing the API",
"required": false,
"schema": {
"type": "string"
}
}
],
"requestBody": { "requestBody": {
"content": { "content": {
"application/json": { "application/json": {
@ -431,7 +571,17 @@
"tags": [ "tags": [
"Memory" "Memory"
], ],
"parameters": [], "parameters": [
{
"name": "X-LlamaStack-ProviderData",
"in": "header",
"description": "JSON-encoded provider data which will be made available to the adapter servicing the API",
"required": false,
"schema": {
"type": "string"
}
}
],
"requestBody": { "requestBody": {
"content": { "content": {
"application/json": { "application/json": {
@ -461,7 +611,17 @@
"tags": [ "tags": [
"Memory" "Memory"
], ],
"parameters": [], "parameters": [
{
"name": "X-LlamaStack-ProviderData",
"in": "header",
"description": "JSON-encoded provider data which will be made available to the adapter servicing the API",
"required": false,
"schema": {
"type": "string"
}
}
],
"requestBody": { "requestBody": {
"content": { "content": {
"application/json": { "application/json": {
@ -491,7 +651,17 @@
"tags": [ "tags": [
"Inference" "Inference"
], ],
"parameters": [], "parameters": [
{
"name": "X-LlamaStack-ProviderData",
"in": "header",
"description": "JSON-encoded provider data which will be made available to the adapter servicing the API",
"required": false,
"schema": {
"type": "string"
}
}
],
"requestBody": { "requestBody": {
"content": { "content": {
"application/json": { "application/json": {
@ -521,7 +691,17 @@
"tags": [ "tags": [
"Evaluations" "Evaluations"
], ],
"parameters": [], "parameters": [
{
"name": "X-LlamaStack-ProviderData",
"in": "header",
"description": "JSON-encoded provider data which will be made available to the adapter servicing the API",
"required": false,
"schema": {
"type": "string"
}
}
],
"requestBody": { "requestBody": {
"content": { "content": {
"application/json": { "application/json": {
@ -551,7 +731,17 @@
"tags": [ "tags": [
"Evaluations" "Evaluations"
], ],
"parameters": [], "parameters": [
{
"name": "X-LlamaStack-ProviderData",
"in": "header",
"description": "JSON-encoded provider data which will be made available to the adapter servicing the API",
"required": false,
"schema": {
"type": "string"
}
}
],
"requestBody": { "requestBody": {
"content": { "content": {
"application/json": { "application/json": {
@ -581,7 +771,17 @@
"tags": [ "tags": [
"Evaluations" "Evaluations"
], ],
"parameters": [], "parameters": [
{
"name": "X-LlamaStack-ProviderData",
"in": "header",
"description": "JSON-encoded provider data which will be made available to the adapter servicing the API",
"required": false,
"schema": {
"type": "string"
}
}
],
"requestBody": { "requestBody": {
"content": { "content": {
"application/json": { "application/json": {
@ -627,6 +827,15 @@
"schema": { "schema": {
"type": "string" "type": "string"
} }
},
{
"name": "X-LlamaStack-ProviderData",
"in": "header",
"description": "JSON-encoded provider data which will be made available to the adapter servicing the API",
"required": false,
"schema": {
"type": "string"
}
} }
], ],
"requestBody": { "requestBody": {
@ -682,6 +891,15 @@
"schema": { "schema": {
"type": "string" "type": "string"
} }
},
{
"name": "X-LlamaStack-ProviderData",
"in": "header",
"description": "JSON-encoded provider data which will be made available to the adapter servicing the API",
"required": false,
"schema": {
"type": "string"
}
} }
] ]
} }
@ -719,6 +937,15 @@
"schema": { "schema": {
"type": "string" "type": "string"
} }
},
{
"name": "X-LlamaStack-ProviderData",
"in": "header",
"description": "JSON-encoded provider data which will be made available to the adapter servicing the API",
"required": false,
"schema": {
"type": "string"
}
} }
] ]
} }
@ -748,6 +975,15 @@
"schema": { "schema": {
"type": "string" "type": "string"
} }
},
{
"name": "X-LlamaStack-ProviderData",
"in": "header",
"description": "JSON-encoded provider data which will be made available to the adapter servicing the API",
"required": false,
"schema": {
"type": "string"
}
} }
] ]
} }
@ -777,6 +1013,15 @@
"schema": { "schema": {
"type": "string" "type": "string"
} }
},
{
"name": "X-LlamaStack-ProviderData",
"in": "header",
"description": "JSON-encoded provider data which will be made available to the adapter servicing the API",
"required": false,
"schema": {
"type": "string"
}
} }
], ],
"requestBody": { "requestBody": {
@ -816,6 +1061,15 @@
"schema": { "schema": {
"type": "string" "type": "string"
} }
},
{
"name": "X-LlamaStack-ProviderData",
"in": "header",
"description": "JSON-encoded provider data which will be made available to the adapter servicing the API",
"required": false,
"schema": {
"type": "string"
}
} }
] ]
} }
@ -845,6 +1099,15 @@
"schema": { "schema": {
"type": "string" "type": "string"
} }
},
{
"name": "X-LlamaStack-ProviderData",
"in": "header",
"description": "JSON-encoded provider data which will be made available to the adapter servicing the API",
"required": false,
"schema": {
"type": "string"
}
} }
] ]
} }
@ -874,6 +1137,15 @@
"schema": { "schema": {
"type": "string" "type": "string"
} }
},
{
"name": "X-LlamaStack-ProviderData",
"in": "header",
"description": "JSON-encoded provider data which will be made available to the adapter servicing the API",
"required": false,
"schema": {
"type": "string"
}
} }
] ]
} }
@ -895,7 +1167,17 @@
"tags": [ "tags": [
"Evaluations" "Evaluations"
], ],
"parameters": [] "parameters": [
{
"name": "X-LlamaStack-ProviderData",
"in": "header",
"description": "JSON-encoded provider data which will be made available to the adapter servicing the API",
"required": false,
"schema": {
"type": "string"
}
}
]
} }
}, },
"/memory_banks/get": { "/memory_banks/get": {
@ -930,6 +1212,15 @@
"schema": { "schema": {
"type": "string" "type": "string"
} }
},
{
"name": "X-LlamaStack-ProviderData",
"in": "header",
"description": "JSON-encoded provider data which will be made available to the adapter servicing the API",
"required": false,
"schema": {
"type": "string"
}
} }
] ]
} }
@ -959,6 +1250,15 @@
"schema": { "schema": {
"type": "string" "type": "string"
} }
},
{
"name": "X-LlamaStack-ProviderData",
"in": "header",
"description": "JSON-encoded provider data which will be made available to the adapter servicing the API",
"required": false,
"schema": {
"type": "string"
}
} }
] ]
} }
@ -988,6 +1288,15 @@
"schema": { "schema": {
"type": "string" "type": "string"
} }
},
{
"name": "X-LlamaStack-ProviderData",
"in": "header",
"description": "JSON-encoded provider data which will be made available to the adapter servicing the API",
"required": false,
"schema": {
"type": "string"
}
} }
] ]
} }
@ -1017,6 +1326,15 @@
"schema": { "schema": {
"type": "string" "type": "string"
} }
},
{
"name": "X-LlamaStack-ProviderData",
"in": "header",
"description": "JSON-encoded provider data which will be made available to the adapter servicing the API",
"required": false,
"schema": {
"type": "string"
}
} }
] ]
} }
@ -1046,6 +1364,15 @@
"schema": { "schema": {
"type": "string" "type": "string"
} }
},
{
"name": "X-LlamaStack-ProviderData",
"in": "header",
"description": "JSON-encoded provider data which will be made available to the adapter servicing the API",
"required": false,
"schema": {
"type": "string"
}
} }
] ]
} }
@ -1067,7 +1394,17 @@
"tags": [ "tags": [
"PostTraining" "PostTraining"
], ],
"parameters": [] "parameters": [
{
"name": "X-LlamaStack-ProviderData",
"in": "header",
"description": "JSON-encoded provider data which will be made available to the adapter servicing the API",
"required": false,
"schema": {
"type": "string"
}
}
]
} }
}, },
"/memory_bank/insert": { "/memory_bank/insert": {
@ -1080,7 +1417,17 @@
"tags": [ "tags": [
"Memory" "Memory"
], ],
"parameters": [], "parameters": [
{
"name": "X-LlamaStack-ProviderData",
"in": "header",
"description": "JSON-encoded provider data which will be made available to the adapter servicing the API",
"required": false,
"schema": {
"type": "string"
}
}
],
"requestBody": { "requestBody": {
"content": { "content": {
"application/json": { "application/json": {
@ -1110,7 +1457,17 @@
"tags": [ "tags": [
"Memory" "Memory"
], ],
"parameters": [] "parameters": [
{
"name": "X-LlamaStack-ProviderData",
"in": "header",
"description": "JSON-encoded provider data which will be made available to the adapter servicing the API",
"required": false,
"schema": {
"type": "string"
}
}
]
} }
}, },
"/telemetry/log_event": { "/telemetry/log_event": {
@ -1123,7 +1480,17 @@
"tags": [ "tags": [
"Telemetry" "Telemetry"
], ],
"parameters": [], "parameters": [
{
"name": "X-LlamaStack-ProviderData",
"in": "header",
"description": "JSON-encoded provider data which will be made available to the adapter servicing the API",
"required": false,
"schema": {
"type": "string"
}
}
],
"requestBody": { "requestBody": {
"content": { "content": {
"application/json": { "application/json": {
@ -1153,7 +1520,17 @@
"tags": [ "tags": [
"PostTraining" "PostTraining"
], ],
"parameters": [], "parameters": [
{
"name": "X-LlamaStack-ProviderData",
"in": "header",
"description": "JSON-encoded provider data which will be made available to the adapter servicing the API",
"required": false,
"schema": {
"type": "string"
}
}
],
"requestBody": { "requestBody": {
"content": { "content": {
"application/json": { "application/json": {
@ -1183,7 +1560,17 @@
"tags": [ "tags": [
"Memory" "Memory"
], ],
"parameters": [], "parameters": [
{
"name": "X-LlamaStack-ProviderData",
"in": "header",
"description": "JSON-encoded provider data which will be made available to the adapter servicing the API",
"required": false,
"schema": {
"type": "string"
}
}
],
"requestBody": { "requestBody": {
"content": { "content": {
"application/json": { "application/json": {
@ -1213,7 +1600,17 @@
"tags": [ "tags": [
"RewardScoring" "RewardScoring"
], ],
"parameters": [], "parameters": [
{
"name": "X-LlamaStack-ProviderData",
"in": "header",
"description": "JSON-encoded provider data which will be made available to the adapter servicing the API",
"required": false,
"schema": {
"type": "string"
}
}
],
"requestBody": { "requestBody": {
"content": { "content": {
"application/json": { "application/json": {
@ -1243,7 +1640,17 @@
"tags": [ "tags": [
"Safety" "Safety"
], ],
"parameters": [], "parameters": [
{
"name": "X-LlamaStack-ProviderData",
"in": "header",
"description": "JSON-encoded provider data which will be made available to the adapter servicing the API",
"required": false,
"schema": {
"type": "string"
}
}
],
"requestBody": { "requestBody": {
"content": { "content": {
"application/json": { "application/json": {
@ -1273,7 +1680,17 @@
"tags": [ "tags": [
"PostTraining" "PostTraining"
], ],
"parameters": [], "parameters": [
{
"name": "X-LlamaStack-ProviderData",
"in": "header",
"description": "JSON-encoded provider data which will be made available to the adapter servicing the API",
"required": false,
"schema": {
"type": "string"
}
}
],
"requestBody": { "requestBody": {
"content": { "content": {
"application/json": { "application/json": {
@ -1303,7 +1720,17 @@
"tags": [ "tags": [
"SyntheticDataGeneration" "SyntheticDataGeneration"
], ],
"parameters": [], "parameters": [
{
"name": "X-LlamaStack-ProviderData",
"in": "header",
"description": "JSON-encoded provider data which will be made available to the adapter servicing the API",
"required": false,
"schema": {
"type": "string"
}
}
],
"requestBody": { "requestBody": {
"content": { "content": {
"application/json": { "application/json": {
@ -1326,7 +1753,17 @@
"tags": [ "tags": [
"Memory" "Memory"
], ],
"parameters": [], "parameters": [
{
"name": "X-LlamaStack-ProviderData",
"in": "header",
"description": "JSON-encoded provider data which will be made available to the adapter servicing the API",
"required": false,
"schema": {
"type": "string"
}
}
],
"requestBody": { "requestBody": {
"content": { "content": {
"application/json": { "application/json": {
@ -5155,31 +5592,31 @@
"name": "PostTraining" "name": "PostTraining"
}, },
{ {
"name": "Inference" "name": "Memory"
},
{
"name": "Safety"
},
{
"name": "RewardScoring"
},
{
"name": "Telemetry"
}, },
{ {
"name": "Evaluations" "name": "Evaluations"
}, },
{
"name": "Datasets"
},
{ {
"name": "SyntheticDataGeneration" "name": "SyntheticDataGeneration"
}, },
{ {
"name": "Memory" "name": "RewardScoring"
}, },
{ {
"name": "Agents" "name": "Agents"
}, },
{ {
"name": "Datasets" "name": "Safety"
},
{
"name": "Inference"
},
{
"name": "Telemetry"
}, },
{ {
"name": "BuiltinTool", "name": "BuiltinTool",

395
docs/llama-stack-spec.yaml
View file

@ -2362,7 +2362,7 @@ info:
description: "This is the specification of the llama stack that provides\n \ description: "This is the specification of the llama stack that provides\n \
\ a set of endpoints and their corresponding interfaces that are tailored\ \ a set of endpoints and their corresponding interfaces that are tailored\
\ to\n best leverage Llama Models. The specification is still in\ \ to\n best leverage Llama Models. The specification is still in\
\ draft and subject to change.\n Generated at 2024-09-18 19:27:39.955190" \ draft and subject to change.\n Generated at 2024-09-20 13:32:23.841908"
title: '[DRAFT] Llama Stack Specification' title: '[DRAFT] Llama Stack Specification'
version: 0.0.1 version: 0.0.1
jsonSchemaDialect: https://json-schema.org/draft/2020-12/schema jsonSchemaDialect: https://json-schema.org/draft/2020-12/schema
@ -2370,7 +2370,14 @@ openapi: 3.1.0
paths: paths:
/agents/create: /agents/create:
post: post:
parameters: [] parameters:
- description: JSON-encoded provider data which will be made available to the
adapter servicing the API
in: header
name: X-LlamaStack-ProviderData
required: false
schema:
type: string
requestBody: requestBody:
content: content:
application/json: application/json:
@ -2388,7 +2395,14 @@ paths:
- Agents - Agents
/agents/delete: /agents/delete:
post: post:
parameters: [] parameters:
- description: JSON-encoded provider data which will be made available to the
adapter servicing the API
in: header
name: X-LlamaStack-ProviderData
required: false
schema:
type: string
requestBody: requestBody:
content: content:
application/json: application/json:
@ -2402,7 +2416,14 @@ paths:
- Agents - Agents
/agents/session/create: /agents/session/create:
post: post:
parameters: [] parameters:
- description: JSON-encoded provider data which will be made available to the
adapter servicing the API
in: header
name: X-LlamaStack-ProviderData
required: false
schema:
type: string
requestBody: requestBody:
content: content:
application/json: application/json:
@ -2420,7 +2441,14 @@ paths:
- Agents - Agents
/agents/session/delete: /agents/session/delete:
post: post:
parameters: [] parameters:
- description: JSON-encoded provider data which will be made available to the
adapter servicing the API
in: header
name: X-LlamaStack-ProviderData
required: false
schema:
type: string
requestBody: requestBody:
content: content:
application/json: application/json:
@ -2445,6 +2473,13 @@ paths:
required: true required: true
schema: schema:
type: string type: string
- description: JSON-encoded provider data which will be made available to the
adapter servicing the API
in: header
name: X-LlamaStack-ProviderData
required: false
schema:
type: string
requestBody: requestBody:
content: content:
application/json: application/json:
@ -2478,6 +2513,13 @@ paths:
required: true required: true
schema: schema:
type: string type: string
- description: JSON-encoded provider data which will be made available to the
adapter servicing the API
in: header
name: X-LlamaStack-ProviderData
required: false
schema:
type: string
responses: responses:
'200': '200':
content: content:
@ -2489,7 +2531,14 @@ paths:
- Agents - Agents
/agents/turn/create: /agents/turn/create:
post: post:
parameters: [] parameters:
- description: JSON-encoded provider data which will be made available to the
adapter servicing the API
in: header
name: X-LlamaStack-ProviderData
required: false
schema:
type: string
requestBody: requestBody:
content: content:
application/json: application/json:
@ -2518,6 +2567,13 @@ paths:
required: true required: true
schema: schema:
type: string type: string
- description: JSON-encoded provider data which will be made available to the
adapter servicing the API
in: header
name: X-LlamaStack-ProviderData
required: false
schema:
type: string
responses: responses:
'200': '200':
content: content:
@ -2529,7 +2585,14 @@ paths:
- Agents - Agents
/batch_inference/chat_completion: /batch_inference/chat_completion:
post: post:
parameters: [] parameters:
- description: JSON-encoded provider data which will be made available to the
adapter servicing the API
in: header
name: X-LlamaStack-ProviderData
required: false
schema:
type: string
requestBody: requestBody:
content: content:
application/json: application/json:
@ -2547,7 +2610,14 @@ paths:
- BatchInference - BatchInference
/batch_inference/completion: /batch_inference/completion:
post: post:
parameters: [] parameters:
- description: JSON-encoded provider data which will be made available to the
adapter servicing the API
in: header
name: X-LlamaStack-ProviderData
required: false
schema:
type: string
requestBody: requestBody:
content: content:
application/json: application/json:
@ -2565,7 +2635,14 @@ paths:
- BatchInference - BatchInference
/datasets/create: /datasets/create:
post: post:
parameters: [] parameters:
- description: JSON-encoded provider data which will be made available to the
adapter servicing the API
in: header
name: X-LlamaStack-ProviderData
required: false
schema:
type: string
requestBody: requestBody:
content: content:
application/json: application/json:
@ -2579,7 +2656,14 @@ paths:
- Datasets - Datasets
/datasets/delete: /datasets/delete:
post: post:
parameters: [] parameters:
- description: JSON-encoded provider data which will be made available to the
adapter servicing the API
in: header
name: X-LlamaStack-ProviderData
required: false
schema:
type: string
requestBody: requestBody:
content: content:
application/json: application/json:
@ -2599,6 +2683,13 @@ paths:
required: true required: true
schema: schema:
type: string type: string
- description: JSON-encoded provider data which will be made available to the
adapter servicing the API
in: header
name: X-LlamaStack-ProviderData
required: false
schema:
type: string
responses: responses:
'200': '200':
content: content:
@ -2616,6 +2707,13 @@ paths:
required: true required: true
schema: schema:
type: string type: string
- description: JSON-encoded provider data which will be made available to the
adapter servicing the API
in: header
name: X-LlamaStack-ProviderData
required: false
schema:
type: string
responses: responses:
'200': '200':
content: content:
@ -2627,7 +2725,14 @@ paths:
- Evaluations - Evaluations
/evaluate/job/cancel: /evaluate/job/cancel:
post: post:
parameters: [] parameters:
- description: JSON-encoded provider data which will be made available to the
adapter servicing the API
in: header
name: X-LlamaStack-ProviderData
required: false
schema:
type: string
requestBody: requestBody:
content: content:
application/json: application/json:
@ -2647,6 +2752,13 @@ paths:
required: true required: true
schema: schema:
type: string type: string
- description: JSON-encoded provider data which will be made available to the
adapter servicing the API
in: header
name: X-LlamaStack-ProviderData
required: false
schema:
type: string
responses: responses:
'200': '200':
content: content:
@ -2664,6 +2776,13 @@ paths:
required: true required: true
schema: schema:
type: string type: string
- description: JSON-encoded provider data which will be made available to the
adapter servicing the API
in: header
name: X-LlamaStack-ProviderData
required: false
schema:
type: string
responses: responses:
'200': '200':
content: content:
@ -2675,7 +2794,14 @@ paths:
- Evaluations - Evaluations
/evaluate/jobs: /evaluate/jobs:
get: get:
parameters: [] parameters:
- description: JSON-encoded provider data which will be made available to the
adapter servicing the API
in: header
name: X-LlamaStack-ProviderData
required: false
schema:
type: string
responses: responses:
'200': '200':
content: content:
@ -2687,7 +2813,14 @@ paths:
- Evaluations - Evaluations
/evaluate/question_answering/: /evaluate/question_answering/:
post: post:
parameters: [] parameters:
- description: JSON-encoded provider data which will be made available to the
adapter servicing the API
in: header
name: X-LlamaStack-ProviderData
required: false
schema:
type: string
requestBody: requestBody:
content: content:
application/json: application/json:
@ -2705,7 +2838,14 @@ paths:
- Evaluations - Evaluations
/evaluate/summarization/: /evaluate/summarization/:
post: post:
parameters: [] parameters:
- description: JSON-encoded provider data which will be made available to the
adapter servicing the API
in: header
name: X-LlamaStack-ProviderData
required: false
schema:
type: string
requestBody: requestBody:
content: content:
application/json: application/json:
@ -2723,7 +2863,14 @@ paths:
- Evaluations - Evaluations
/evaluate/text_generation/: /evaluate/text_generation/:
post: post:
parameters: [] parameters:
- description: JSON-encoded provider data which will be made available to the
adapter servicing the API
in: header
name: X-LlamaStack-ProviderData
required: false
schema:
type: string
requestBody: requestBody:
content: content:
application/json: application/json:
@ -2741,7 +2888,14 @@ paths:
- Evaluations - Evaluations
/inference/chat_completion: /inference/chat_completion:
post: post:
parameters: [] parameters:
- description: JSON-encoded provider data which will be made available to the
adapter servicing the API
in: header
name: X-LlamaStack-ProviderData
required: false
schema:
type: string
requestBody: requestBody:
content: content:
application/json: application/json:
@ -2761,7 +2915,14 @@ paths:
- Inference - Inference
/inference/completion: /inference/completion:
post: post:
parameters: [] parameters:
- description: JSON-encoded provider data which will be made available to the
adapter servicing the API
in: header
name: X-LlamaStack-ProviderData
required: false
schema:
type: string
requestBody: requestBody:
content: content:
application/json: application/json:
@ -2781,7 +2942,14 @@ paths:
- Inference - Inference
/inference/embeddings: /inference/embeddings:
post: post:
parameters: [] parameters:
- description: JSON-encoded provider data which will be made available to the
adapter servicing the API
in: header
name: X-LlamaStack-ProviderData
required: false
schema:
type: string
requestBody: requestBody:
content: content:
application/json: application/json:
@ -2799,7 +2967,14 @@ paths:
- Inference - Inference
/memory_bank/documents/delete: /memory_bank/documents/delete:
post: post:
parameters: [] parameters:
- description: JSON-encoded provider data which will be made available to the
adapter servicing the API
in: header
name: X-LlamaStack-ProviderData
required: false
schema:
type: string
requestBody: requestBody:
content: content:
application/json: application/json:
@ -2819,6 +2994,13 @@ paths:
required: true required: true
schema: schema:
type: string type: string
- description: JSON-encoded provider data which will be made available to the
adapter servicing the API
in: header
name: X-LlamaStack-ProviderData
required: false
schema:
type: string
requestBody: requestBody:
content: content:
application/json: application/json:
@ -2836,7 +3018,14 @@ paths:
- Memory - Memory
/memory_bank/insert: /memory_bank/insert:
post: post:
parameters: [] parameters:
- description: JSON-encoded provider data which will be made available to the
adapter servicing the API
in: header
name: X-LlamaStack-ProviderData
required: false
schema:
type: string
requestBody: requestBody:
content: content:
application/json: application/json:
@ -2850,7 +3039,14 @@ paths:
- Memory - Memory
/memory_bank/query: /memory_bank/query:
post: post:
parameters: [] parameters:
- description: JSON-encoded provider data which will be made available to the
adapter servicing the API
in: header
name: X-LlamaStack-ProviderData
required: false
schema:
type: string
requestBody: requestBody:
content: content:
application/json: application/json:
@ -2868,7 +3064,14 @@ paths:
- Memory - Memory
/memory_bank/update: /memory_bank/update:
post: post:
parameters: [] parameters:
- description: JSON-encoded provider data which will be made available to the
adapter servicing the API
in: header
name: X-LlamaStack-ProviderData
required: false
schema:
type: string
requestBody: requestBody:
content: content:
application/json: application/json:
@ -2882,7 +3085,14 @@ paths:
- Memory - Memory
/memory_banks/create: /memory_banks/create:
post: post:
parameters: [] parameters:
- description: JSON-encoded provider data which will be made available to the
adapter servicing the API
in: header
name: X-LlamaStack-ProviderData
required: false
schema:
type: string
requestBody: requestBody:
content: content:
application/json: application/json:
@ -2900,7 +3110,14 @@ paths:
- Memory - Memory
/memory_banks/drop: /memory_banks/drop:
post: post:
parameters: [] parameters:
- description: JSON-encoded provider data which will be made available to the
adapter servicing the API
in: header
name: X-LlamaStack-ProviderData
required: false
schema:
type: string
requestBody: requestBody:
content: content:
application/json: application/json:
@ -2924,6 +3141,13 @@ paths:
required: true required: true
schema: schema:
type: string type: string
- description: JSON-encoded provider data which will be made available to the
adapter servicing the API
in: header
name: X-LlamaStack-ProviderData
required: false
schema:
type: string
responses: responses:
'200': '200':
content: content:
@ -2937,7 +3161,14 @@ paths:
- Memory - Memory
/memory_banks/list: /memory_banks/list:
get: get:
parameters: [] parameters:
- description: JSON-encoded provider data which will be made available to the
adapter servicing the API
in: header
name: X-LlamaStack-ProviderData
required: false
schema:
type: string
responses: responses:
'200': '200':
content: content:
@ -2955,6 +3186,13 @@ paths:
required: true required: true
schema: schema:
type: string type: string
- description: JSON-encoded provider data which will be made available to the
adapter servicing the API
in: header
name: X-LlamaStack-ProviderData
required: false
schema:
type: string
responses: responses:
'200': '200':
content: content:
@ -2966,7 +3204,14 @@ paths:
- PostTraining - PostTraining
/post_training/job/cancel: /post_training/job/cancel:
post: post:
parameters: [] parameters:
- description: JSON-encoded provider data which will be made available to the
adapter servicing the API
in: header
name: X-LlamaStack-ProviderData
required: false
schema:
type: string
requestBody: requestBody:
content: content:
application/json: application/json:
@ -2986,6 +3231,13 @@ paths:
required: true required: true
schema: schema:
type: string type: string
- description: JSON-encoded provider data which will be made available to the
adapter servicing the API
in: header
name: X-LlamaStack-ProviderData
required: false
schema:
type: string
responses: responses:
'200': '200':
content: content:
@ -3003,6 +3255,13 @@ paths:
required: true required: true
schema: schema:
type: string type: string
- description: JSON-encoded provider data which will be made available to the
adapter servicing the API
in: header
name: X-LlamaStack-ProviderData
required: false
schema:
type: string
responses: responses:
'200': '200':
content: content:
@ -3014,7 +3273,14 @@ paths:
- PostTraining - PostTraining
/post_training/jobs: /post_training/jobs:
get: get:
parameters: [] parameters:
- description: JSON-encoded provider data which will be made available to the
adapter servicing the API
in: header
name: X-LlamaStack-ProviderData
required: false
schema:
type: string
responses: responses:
'200': '200':
content: content:
@ -3026,7 +3292,14 @@ paths:
- PostTraining - PostTraining
/post_training/preference_optimize: /post_training/preference_optimize:
post: post:
parameters: [] parameters:
- description: JSON-encoded provider data which will be made available to the
adapter servicing the API
in: header
name: X-LlamaStack-ProviderData
required: false
schema:
type: string
requestBody: requestBody:
content: content:
application/json: application/json:
@ -3044,7 +3317,14 @@ paths:
- PostTraining - PostTraining
/post_training/supervised_fine_tune: /post_training/supervised_fine_tune:
post: post:
parameters: [] parameters:
- description: JSON-encoded provider data which will be made available to the
adapter servicing the API
in: header
name: X-LlamaStack-ProviderData
required: false
schema:
type: string
requestBody: requestBody:
content: content:
application/json: application/json:
@ -3062,7 +3342,14 @@ paths:
- PostTraining - PostTraining
/reward_scoring/score: /reward_scoring/score:
post: post:
parameters: [] parameters:
- description: JSON-encoded provider data which will be made available to the
adapter servicing the API
in: header
name: X-LlamaStack-ProviderData
required: false
schema:
type: string
requestBody: requestBody:
content: content:
application/json: application/json:
@ -3080,7 +3367,14 @@ paths:
- RewardScoring - RewardScoring
/safety/run_shields: /safety/run_shields:
post: post:
parameters: [] parameters:
- description: JSON-encoded provider data which will be made available to the
adapter servicing the API
in: header
name: X-LlamaStack-ProviderData
required: false
schema:
type: string
requestBody: requestBody:
content: content:
application/json: application/json:
@ -3098,7 +3392,14 @@ paths:
- Safety - Safety
/synthetic_data_generation/generate: /synthetic_data_generation/generate:
post: post:
parameters: [] parameters:
- description: JSON-encoded provider data which will be made available to the
adapter servicing the API
in: header
name: X-LlamaStack-ProviderData
required: false
schema:
type: string
requestBody: requestBody:
content: content:
application/json: application/json:
@ -3122,6 +3423,13 @@ paths:
required: true required: true
schema: schema:
type: string type: string
- description: JSON-encoded provider data which will be made available to the
adapter servicing the API
in: header
name: X-LlamaStack-ProviderData
required: false
schema:
type: string
responses: responses:
'200': '200':
content: content:
@ -3133,7 +3441,14 @@ paths:
- Telemetry - Telemetry
/telemetry/log_event: /telemetry/log_event:
post: post:
parameters: [] parameters:
- description: JSON-encoded provider data which will be made available to the
adapter servicing the API
in: header
name: X-LlamaStack-ProviderData
required: false
schema:
type: string
requestBody: requestBody:
content: content:
application/json: application/json:
@ -3152,15 +3467,15 @@ servers:
tags: tags:
- name: BatchInference - name: BatchInference
- name: PostTraining - name: PostTraining
- name: Inference
- name: Safety
- name: RewardScoring
- name: Telemetry
- name: Evaluations
- name: SyntheticDataGeneration
- name: Memory - name: Memory
- name: Agents - name: Evaluations
- name: Datasets - name: Datasets
- name: SyntheticDataGeneration
- name: RewardScoring
- name: Agents
- name: Safety
- name: Inference
- name: Telemetry
- description: <SchemaDefinition schemaRef="#/components/schemas/BuiltinTool" /> - description: <SchemaDefinition schemaRef="#/components/schemas/BuiltinTool" />
name: BuiltinTool name: BuiltinTool
- description: <SchemaDefinition schemaRef="#/components/schemas/CompletionMessage" - description: <SchemaDefinition schemaRef="#/components/schemas/CompletionMessage"

9
docs/openapi_generator/pyopenapi/generator.py
View file

@ -462,6 +462,15 @@ class Generator:
# parameters passed anywhere # parameters passed anywhere
parameters = path_parameters + query_parameters parameters = path_parameters + query_parameters
parameters += [
Parameter(
name="X-LlamaStack-ProviderData",
in_=ParameterLocation.Header,
description="JSON-encoded provider data which will be made available to the adapter servicing the API",
required=False,
schema=self.schema_builder.classdef_to_ref(str),
)
]
# data passed in payload # data passed in payload
if op.request_params: if op.request_params:

5
docs/openapi_generator/run_openapi_generator.sh
View file

@ -7,6 +7,7 @@
# the root directory of this source tree. # the root directory of this source tree.
PYTHONPATH=${PYTHONPATH:-} PYTHONPATH=${PYTHONPATH:-}
THIS_DIR="$(cd "$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")" && pwd)"
set -euo pipefail set -euo pipefail
@ -28,4 +29,6 @@ if [ ${#missing_packages[@]} -ne 0 ]; then
exit 1 exit 1
fi fi
PYTHONPATH=$PYTHONPATH:../.. python -m docs.openapi_generator.generate $* stack_dir=$(dirname $(dirname $THIS_DIR))
models_dir=$(dirname $stack_dir)/llama-models
PYTHONPATH=$PYTHONPATH:$stack_dir:$models_dir python -m docs.openapi_generator.generate $(dirname $THIS_DIR)

12
llama_stack/distribution/datatypes.py
View file

@ -92,6 +92,9 @@ Fully-qualified name of the module to import. The module is expected to have:
default=None, default=None,
description="Fully-qualified classname of the config for this provider", description="Fully-qualified classname of the config for this provider",
) )
provider_data_validator: Optional[str] = Field(
default=None,
)
@json_schema_type @json_schema_type
@ -115,6 +118,9 @@ Fully-qualified name of the module to import. The module is expected to have:
- `get_provider_impl(config, deps)`: returns the local implementation - `get_provider_impl(config, deps)`: returns the local implementation
""", """,
) )
provider_data_validator: Optional[str] = Field(
default=None,
)
class RemoteProviderConfig(BaseModel): class RemoteProviderConfig(BaseModel):
@ -159,6 +165,12 @@ as being "Llama Stack compatible"
return self.adapter.pip_packages return self.adapter.pip_packages
return [] return []
@property
def provider_data_validator(self) -> Optional[str]:
if self.adapter:
return self.adapter.provider_data_validator
return None
# Can avoid this by using Pydantic computed_field # Can avoid this by using Pydantic computed_field
def remote_provider_spec( def remote_provider_spec(

49
llama_stack/distribution/request_headers.py Normal file
View file

@ -0,0 +1,49 @@
# Copyright (c) Meta Platforms, Inc. and affiliates.
# All rights reserved.
#
# This source code is licensed under the terms described in the LICENSE file in
# the root directory of this source tree.
import json
import threading
from typing import Any, Dict, Optional
from .utils.dynamic import instantiate_class_type
_THREAD_LOCAL = threading.local()
def get_request_provider_data() -> Any:
return getattr(_THREAD_LOCAL, "provider_data", None)
def set_request_provider_data(headers: Dict[str, str], validator_class: Optional[str]):
if not validator_class:
return
keys = [
"X-LlamaStack-ProviderData",
"x-llamastack-providerdata",
]
for key in keys:
val = headers.get(key, None)
if val:
break
if not val:
return
try:
val = json.loads(val)
except json.JSONDecodeError:
print("Provider data not encoded as a JSON object!", val)
return
validator = instantiate_class_type(validator_class)
try:
provider_data = validator(**val)
except Exception as e:
print("Error parsing provider data", e)
return
_THREAD_LOCAL.provider_data = provider_data

45
llama_stack/distribution/server/server.py
View file

@ -49,6 +49,7 @@ from typing_extensions import Annotated
from llama_stack.distribution.datatypes import * # noqa: F403 from llama_stack.distribution.datatypes import * # noqa: F403
from llama_stack.distribution.distribution import api_endpoints, api_providers from llama_stack.distribution.distribution import api_endpoints, api_providers
from llama_stack.distribution.request_headers import set_request_provider_data
from llama_stack.distribution.utils.dynamic import instantiate_provider from llama_stack.distribution.utils.dynamic import instantiate_provider
@ -177,9 +178,9 @@ def create_dynamic_passthrough(
return endpoint return endpoint
def create_dynamic_typed_route(func: Any, method: str): def create_dynamic_typed_route(
cprint(f"> create_dynamic_typed_route func={func}", "red") func: Any, method: str, provider_data_validator: Optional[str]
cprint(f"> create_dynamic_typed_route method={method}", "red") ):
hints = get_type_hints(func) hints = get_type_hints(func)
response_model = hints.get("return") response_model = hints.get("return")
@ -191,9 +192,11 @@ def create_dynamic_typed_route(func: Any, method: str):
if is_streaming: if is_streaming:
async def endpoint(**kwargs): async def endpoint(request: Request, **kwargs):
await start_trace(func.__name__) await start_trace(func.__name__)
set_request_provider_data(request.headers, provider_data_validator)
async def sse_generator(event_gen): async def sse_generator(event_gen):
try: try:
async for item in event_gen: async for item in event_gen:
@ -220,8 +223,11 @@ def create_dynamic_typed_route(func: Any, method: str):
else: else:
async def endpoint(**kwargs): async def endpoint(request: Request, **kwargs):
await start_trace(func.__name__) await start_trace(func.__name__)
set_request_provider_data(request.headers, provider_data_validator)
try: try:
return ( return (
await func(**kwargs) await func(**kwargs)
@ -235,20 +241,23 @@ def create_dynamic_typed_route(func: Any, method: str):
await end_trace() await end_trace()
sig = inspect.signature(func) sig = inspect.signature(func)
new_params = [
inspect.Parameter(
"request", inspect.Parameter.POSITIONAL_OR_KEYWORD, annotation=Request
)
]
new_params.extend(sig.parameters.values())
if method == "post": if method == "post":
# make sure every parameter is annotated with Body() so FASTAPI doesn't # make sure every parameter is annotated with Body() so FASTAPI doesn't
# do anything too intelligent and ask for some parameters in the query # do anything too intelligent and ask for some parameters in the query
# and some in the body # and some in the body
endpoint.__signature__ = sig.replace( new_params = [new_params[0]] + [
parameters=[ param.replace(annotation=Annotated[param.annotation, Body(..., embed=True)])
param.replace( for param in new_params[1:]
annotation=Annotated[param.annotation, Body(..., embed=True)] ]
)
for param in sig.parameters.values() endpoint.__signature__ = sig.replace(parameters=new_params)
]
)
else:
endpoint.__signature__ = sig
return endpoint return endpoint
@ -420,7 +429,11 @@ def run_main_DEPRECATED(
impl_method = getattr(impl, endpoint.name) impl_method = getattr(impl, endpoint.name)
getattr(app, endpoint.method)(endpoint.route, response_model=None)( getattr(app, endpoint.method)(endpoint.route, response_model=None)(
create_dynamic_typed_route(impl_method, endpoint.method) create_dynamic_typed_route(
impl_method,
endpoint.method,
provider_spec.provider_data_validator,
)
) )
for route in app.routes: for route in app.routes:

2
llama_stack/providers/adapters/inference/together/__init__.py
View file

@ -4,7 +4,7 @@
# This source code is licensed under the terms described in the LICENSE file in # This source code is licensed under the terms described in the LICENSE file in
# the root directory of this source tree. # the root directory of this source tree.
from .config import TogetherImplConfig from .config import TogetherImplConfig, TogetherHeaderExtractor
async def get_adapter_impl(config: TogetherImplConfig, _deps): async def get_adapter_impl(config: TogetherImplConfig, _deps):

11
llama_stack/providers/adapters/inference/together/config.py
View file

@ -4,9 +4,18 @@
# This source code is licensed under the terms described in the LICENSE file in # This source code is licensed under the terms described in the LICENSE file in
# the root directory of this source tree. # the root directory of this source tree.
from llama_models.schema_utils import json_schema_type
from pydantic import BaseModel, Field from pydantic import BaseModel, Field
from llama_models.schema_utils import json_schema_type
from llama_stack.distribution.request_headers import annotate_header
class TogetherHeaderExtractor(BaseModel):
api_key: annotate_header(
"X-LlamaStack-Together-ApiKey", str, "The API Key for the request"
)
@json_schema_type @json_schema_type
class TogetherImplConfig(BaseModel): class TogetherImplConfig(BaseModel):

1
llama_stack/providers/registry/inference.py
View file

@ -63,6 +63,7 @@ def available_providers() -> List[ProviderSpec]:
], ],
module="llama_stack.providers.adapters.inference.together", module="llama_stack.providers.adapters.inference.together",
config_class="llama_stack.providers.adapters.inference.together.TogetherImplConfig", config_class="llama_stack.providers.adapters.inference.together.TogetherImplConfig",
header_extractor_class="llama_stack.providers.adapters.inference.together.TogetherHeaderExtractor",
), ),
), ),
] ]

3
llama_stack/providers/utils/memory/vector_store.py
View file

@ -16,6 +16,7 @@ import httpx
import numpy as np import numpy as np
from numpy.typing import NDArray from numpy.typing import NDArray
from pypdf import PdfReader from pypdf import PdfReader
from termcolor import cprint
from llama_models.llama3.api.datatypes import * # noqa: F403 from llama_models.llama3.api.datatypes import * # noqa: F403
from llama_models.llama3.api.tokenizer import Tokenizer from llama_models.llama3.api.tokenizer import Tokenizer
@ -160,6 +161,8 @@ class BankWithIndex:
self.bank.config.overlap_size_in_tokens self.bank.config.overlap_size_in_tokens
or (self.bank.config.chunk_size_in_tokens // 4), or (self.bank.config.chunk_size_in_tokens // 4),
) )
if not chunks:
continue
embeddings = model.encode([x.content for x in chunks]).astype(np.float32) embeddings = model.encode([x.content for x in chunks]).astype(np.float32)
await self.index.add_chunks(chunks, embeddings) await self.index.add_chunks(chunks, embeddings)