phoenix-oss/llama-stack-mirror
0
0
Fork 1
mirror of https://github.com/meta-llama/llama-stack.git synced 2025-12-12 04:00:42 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge 4306ecdbe7 into 98a5047f9d

Browse source
This commit is contained in:
raghotham 2025-10-27 11:36:30 -07:00 • committed by GitHub
commit fb49732f2f
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
13 changed files with 4819 additions and 41 deletions
Download patch file Download diff file Expand all files Collapse all files

489
client-sdks/stainless/openapi.yml
View file

@ -15,6 +15,178 @@ info:
servers:
- url: http://any-hosted-llama-stack.com
paths:
/v1/admin/providers/{api}:
post:
responses:
'200':
description: >-
RegisterProviderResponse with the registered provider info.
content:
application/json:
schema:
$ref: '#/components/schemas/RegisterProviderResponse'
'400':
$ref: '#/components/responses/BadRequest400'
'429':
$ref: >-
#/components/responses/TooManyRequests429
'500':
$ref: >-
#/components/responses/InternalServerError500
default:
$ref: '#/components/responses/DefaultError'
tags:
- Providers
summary: Register a new dynamic provider.
description: >-
Register a new dynamic provider.
Register a new provider instance at runtime. The provider will be validated,
instantiated, and persisted to the kvstore. Requires appropriate ABAC permissions.
parameters:
- name: api
in: path
description: >-
API namespace this provider implements (e.g., 'inference', 'vector_io').
required: true
schema:
type: string
requestBody:
content:
application/json:
schema:
$ref: '#/components/schemas/RegisterProviderRequest'
required: true
deprecated: false
/v1/admin/providers/{api}/{provider_id}:
post:
responses:
'200':
description: >-
UpdateProviderResponse with updated provider info
content:
application/json:
schema:
$ref: '#/components/schemas/UpdateProviderResponse'
'400':
$ref: '#/components/responses/BadRequest400'
'429':
$ref: >-
#/components/responses/TooManyRequests429
'500':
$ref: >-
#/components/responses/InternalServerError500
default:
$ref: '#/components/responses/DefaultError'
tags:
- Providers
summary: >-
Update an existing provider's configuration.
description: >-
Update an existing provider's configuration.
Update the configuration and/or attributes of a dynamic provider. The provider
will be re-instantiated with the new configuration (hot-reload).
parameters:
- name: api
in: path
description: API namespace the provider implements
required: true
schema:
type: string
- name: provider_id
in: path
description: ID of the provider to update
required: true
schema:
type: string
requestBody:
content:
application/json:
schema:
$ref: '#/components/schemas/UpdateProviderRequest'
required: true
deprecated: false
delete:
responses:
'200':
description: OK
'400':
$ref: '#/components/responses/BadRequest400'
'429':
$ref: >-
#/components/responses/TooManyRequests429
'500':
$ref: >-
#/components/responses/InternalServerError500
default:
$ref: '#/components/responses/DefaultError'
tags:
- Providers
summary: Unregister a dynamic provider.
description: >-
Unregister a dynamic provider.
Remove a dynamic provider, shutting down its instance and removing it from
the kvstore.
parameters:
- name: api
in: path
description: API namespace the provider implements
required: true
schema:
type: string
- name: provider_id
in: path
description: ID of the provider to unregister.
required: true
schema:
type: string
deprecated: false
/v1/admin/providers/{api}/{provider_id}/test:
post:
responses:
'200':
description: >-
TestProviderConnectionResponse with health status.
content:
application/json:
schema:
$ref: '#/components/schemas/TestProviderConnectionResponse'
'400':
$ref: '#/components/responses/BadRequest400'
'429':
$ref: >-
#/components/responses/TooManyRequests429
'500':
$ref: >-
#/components/responses/InternalServerError500
default:
$ref: '#/components/responses/DefaultError'
tags:
- Providers
summary: Test a provider connection.
description: >-
Test a provider connection.
Execute a health check on a provider to verify it is reachable and functioning.
parameters:
- name: api
in: path
description: API namespace the provider implements.
required: true
schema:
type: string
- name: provider_id
in: path
description: ID of the provider to test.
required: true
schema:
type: string
deprecated: false
/v1/chat/completions:
get:
responses:
@ -1254,7 +1426,43 @@ paths:
List all available providers.
parameters: []
deprecated: false
/v1/providers/{provider_id}:
/v1/providers/{api}:
get:
responses:
'200':
description: >-
A ListProvidersResponse containing providers for the specified API.
content:
application/json:
schema:
$ref: '#/components/schemas/ListProvidersResponse'
'400':
$ref: '#/components/responses/BadRequest400'
'429':
$ref: >-
#/components/responses/TooManyRequests429
'500':
$ref: >-
#/components/responses/InternalServerError500
default:
$ref: '#/components/responses/DefaultError'
tags:
- Providers
summary: List providers for a specific API.
description: >-
List providers for a specific API.
List all providers that implement a specific API.
parameters:
- name: api
in: path
description: >-
The API namespace to filter by (e.g., 'inference', 'vector_io')
required: true
schema:
type: string
deprecated: false
/v1/providers/{api}/{provider_id}:
get:
responses:
'200':
@ -1276,12 +1484,18 @@ paths:
$ref: '#/components/responses/DefaultError'
tags:
- Providers
summary: Get provider.
summary: Get provider for specific API.
description: >-
Get provider.
Get provider for specific API.
Get detailed information about a specific provider.
Get detailed information about a specific provider for a specific API.
parameters:
- name: api
in: path
description: The API namespace.
required: true
schema:
type: string
- name: provider_id
in: path
description: The ID of the provider to inspect.
@ -4212,6 +4426,273 @@ components:
title: Error
description: >-
Error response from the API. Roughly follows RFC 7807.
RegisterProviderRequest:
type: object
properties:
provider_id:
type: string
description: >-
Unique identifier for this provider instance.
provider_type:
type: string
description: Provider type (e.g., 'remote::openai').
config:
type: object
additionalProperties:
oneOf:
- type: 'null'
- type: boolean
- type: number
- type: string
- type: array
- type: object
description: >-
Provider configuration (API keys, endpoints, etc.).
attributes:
type: object
additionalProperties:
type: array
items:
type: string
description: >-
Optional attributes for ABAC access control.
additionalProperties: false
required:
- provider_id
- provider_type
- config
title: RegisterProviderRequest
ProviderConnectionInfo:
type: object
properties:
provider_id:
type: string
description: >-
Unique identifier for this provider instance
api:
type: string
description: >-
API namespace (e.g., "inference", "vector_io", "safety")
provider_type:
type: string
description: >-
Provider type identifier (e.g., "remote::openai", "inline::faiss")
config:
type: object
additionalProperties:
oneOf:
- type: 'null'
- type: boolean
- type: number
- type: string
- type: array
- type: object
description: >-
Provider-specific configuration (API keys, endpoints, etc.)
status:
$ref: '#/components/schemas/ProviderConnectionStatus'
description: Current connection status
health:
$ref: '#/components/schemas/ProviderHealth'
description: Most recent health check result
created_at:
type: string
format: date-time
description: Timestamp when provider was registered
updated_at:
type: string
format: date-time
description: Timestamp of last update
last_health_check:
type: string
format: date-time
description: Timestamp of last health check
error_message:
type: string
description: Error message if status is failed
metadata:
type: object
additionalProperties:
oneOf:
- type: 'null'
- type: boolean
- type: number
- type: string
- type: array
- type: object
description: >-
User-defined metadata (deprecated, use attributes)
owner:
type: object
properties:
principal:
type: string
attributes:
type: object
additionalProperties:
type: array
items:
type: string
additionalProperties: false
required:
- principal
description: >-
User who created this provider connection
attributes:
type: object
additionalProperties:
type: array
items:
type: string
description: >-
Key-value attributes for ABAC access control
additionalProperties: false
required:
- provider_id
- api
- provider_type
- config
- status
- created_at
- updated_at
- metadata
title: ProviderConnectionInfo
description: >-
Information about a dynamically managed provider connection.
This model represents a provider that has been registered at runtime
via the /providers API, as opposed to static providers configured in run.yaml.
Dynamic providers support full lifecycle management including registration,
configuration updates, health monitoring, and removal.
ProviderConnectionStatus:
type: string
enum:
- pending
- initializing
- connected
- failed
- disconnected
- testing
title: ProviderConnectionStatus
description: Status of a dynamic provider connection.
ProviderHealth:
type: object
properties:
status:
type: string
enum:
- OK
- Error
- Not Implemented
description: >-
Health status (OK, ERROR, NOT_IMPLEMENTED)
message:
type: string
description: Optional error or status message
metrics:
type: object
additionalProperties:
oneOf:
- type: 'null'
- type: boolean
- type: number
- type: string
- type: array
- type: object
description: Provider-specific health metrics
last_checked:
type: string
format: date-time
description: Timestamp of last health check
additionalProperties: false
required:
- status
- metrics
- last_checked
title: ProviderHealth
description: >-
Structured wrapper around provider health status.
This wraps the existing dict-based HealthResponse for API responses
while maintaining backward compatibility with existing provider implementations.
RegisterProviderResponse:
type: object
properties:
provider:
$ref: '#/components/schemas/ProviderConnectionInfo'
description: >-
Information about the registered provider
additionalProperties: false
required:
- provider
title: RegisterProviderResponse
description: Response after registering a provider.
UpdateProviderRequest:
type: object
properties:
config:
type: object
additionalProperties:
oneOf:
- type: 'null'
- type: boolean
- type: number
- type: string
- type: array
- type: object
description: >-
New configuration parameters (merged with existing)
attributes:
type: object
additionalProperties:
type: array
items:
type: string
description: New attributes for access control
additionalProperties: false
title: UpdateProviderRequest
UpdateProviderResponse:
type: object
properties:
provider:
$ref: '#/components/schemas/ProviderConnectionInfo'
description: Updated provider information
additionalProperties: false
required:
- provider
title: UpdateProviderResponse
description: Response after updating a provider.
TestProviderConnectionResponse:
type: object
properties:
success:
type: boolean
description: Whether the connection test succeeded
health:
type: object
additionalProperties:
oneOf:
- type: 'null'
- type: boolean
- type: number
- type: string
- type: array
- type: object
description: Health status from the provider
error_message:
type: string
description: Error message if test failed
additionalProperties: false
required:
- success
title: TestProviderConnectionResponse
description: >-
Response from testing a provider connection.
Order:
type: string
enum:

148
docs/static/deprecated-llama-stack-spec.html vendored
View file

@ -3526,6 +3526,51 @@
},
"deprecated": true
}
},
"/v1/providers/{provider_id}": {
"get": {
"responses": {
"200": {
"description": "A ListProvidersResponse containing all providers with matching provider_id.",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/ListProvidersResponse"
}
}
}
},
"400": {
"$ref": "#/components/responses/BadRequest400"
},
"429": {
"$ref": "#/components/responses/TooManyRequests429"
},
"500": {
"$ref": "#/components/responses/InternalServerError500"
},
"default": {
"$ref": "#/components/responses/DefaultError"
}
},
"tags": [
"Providers"
],
"summary": "Get providers by ID (deprecated - use /providers/{api}/{provider_id} instead).",
"description": "Get providers by ID (deprecated - use /providers/{api}/{provider_id} instead).\nDEPRECATED: Returns all providers with the given provider_id across all APIs.\nThis can return multiple providers if the same ID is used for different APIs.\nUse /providers/{api}/{provider_id} for unambiguous access.",
"parameters": [
{
"name": "provider_id",
"in": "path",
"description": "The ID of the provider(s) to inspect.",
"required": true,
"schema": {
"type": "string"
}
}
],
"deprecated": true
}
}
},
"jsonSchemaDialect": "https://json-schema.org/draft/2020-12/schema",
@ -13350,6 +13395,103 @@
"logger_config"
],
"title": "SupervisedFineTuneRequest"
},
"ProviderInfo": {
"type": "object",
"properties": {
"api": {
"type": "string",
"description": "The API name this provider implements"
},
"provider_id": {
"type": "string",
"description": "Unique identifier for the provider"
},
"provider_type": {
"type": "string",
"description": "The type of provider implementation"
},
"config": {
"type": "object",
"additionalProperties": {
"oneOf": [
{
"type": "null"
},
{
"type": "boolean"
},
{
"type": "number"
},
{
"type": "string"
},
{
"type": "array"
},
{
"type": "object"
}
]
},
"description": "Configuration parameters for the provider"
},
"health": {
"type": "object",
"additionalProperties": {
"oneOf": [
{
"type": "null"
},
{
"type": "boolean"
},
{
"type": "number"
},
{
"type": "string"
},
{
"type": "array"
},
{
"type": "object"
}
]
},
"description": "Current health status of the provider"
}
},
"additionalProperties": false,
"required": [
"api",
"provider_id",
"provider_type",
"config",
"health"
],
"title": "ProviderInfo",
"description": "Information about a registered provider including its configuration and health status."
},
"ListProvidersResponse": {
"type": "object",
"properties": {
"data": {
"type": "array",
"items": {
"$ref": "#/components/schemas/ProviderInfo"
},
"description": "List of provider information objects"
}
},
"additionalProperties": false,
"required": [
"data"
],
"title": "ListProvidersResponse",
"description": "Response containing a list of all available providers."
}
},
"responses": {
@ -13461,6 +13603,11 @@
"name": "PostTraining (Coming Soon)",
"description": ""
},
{
"name": "Providers",
"description": "Providers API for inspecting, listing, and modifying providers and their configurations.",
"x-displayName": "Providers"
},
{
"name": "Safety",
"description": "OpenAI-compatible Moderations API.",
@ -13484,6 +13631,7 @@
"Inference",
"Models",
"PostTraining (Coming Soon)",
"Providers",
"Safety",
"VectorIO"
]

105
docs/static/deprecated-llama-stack-spec.yaml vendored
View file

@ -2600,6 +2600,46 @@ paths:
$ref: '#/components/schemas/SupervisedFineTuneRequest'
required: true
deprecated: true
/v1/providers/{provider_id}:
get:
responses:
'200':
description: >-
A ListProvidersResponse containing all providers with matching provider_id.
content:
application/json:
schema:
$ref: '#/components/schemas/ListProvidersResponse'
'400':
$ref: '#/components/responses/BadRequest400'
'429':
$ref: >-
#/components/responses/TooManyRequests429
'500':
$ref: >-
#/components/responses/InternalServerError500
default:
$ref: '#/components/responses/DefaultError'
tags:
- Providers
summary: >-
Get providers by ID (deprecated - use /providers/{api}/{provider_id} instead).
description: >-
Get providers by ID (deprecated - use /providers/{api}/{provider_id} instead).
DEPRECATED: Returns all providers with the given provider_id across all APIs.
This can return multiple providers if the same ID is used for different APIs.
Use /providers/{api}/{provider_id} for unambiguous access.
parameters:
- name: provider_id
in: path
description: The ID of the provider(s) to inspect.
required: true
schema:
type: string
deprecated: true
jsonSchemaDialect: >-
https://json-schema.org/draft/2020-12/schema
components:
@ -10121,6 +10161,66 @@ components:
- hyperparam_search_config
- logger_config
title: SupervisedFineTuneRequest
ProviderInfo:
type: object
properties:
api:
type: string
description: The API name this provider implements
provider_id:
type: string
description: Unique identifier for the provider
provider_type:
type: string
description: The type of provider implementation
config:
type: object
additionalProperties:
oneOf:
- type: 'null'
- type: boolean
- type: number
- type: string
- type: array
- type: object
description: >-
Configuration parameters for the provider
health:
type: object
additionalProperties:
oneOf:
- type: 'null'
- type: boolean
- type: number
- type: string
- type: array
- type: object
description: Current health status of the provider
additionalProperties: false
required:
- api
- provider_id
- provider_type
- config
- health
title: ProviderInfo
description: >-
Information about a registered provider including its configuration and health
status.
ListProvidersResponse:
type: object
properties:
data:
type: array
items:
$ref: '#/components/schemas/ProviderInfo'
description: List of provider information objects
additionalProperties: false
required:
- data
title: ListProvidersResponse
description: >-
Response containing a list of all available providers.
responses:
BadRequest400:
description: The request was invalid or malformed
@ -10226,6 +10326,10 @@ tags:
description: ''
- name: PostTraining (Coming Soon)
description: ''
- name: Providers
description: >-
Providers API for inspecting, listing, and modifying providers and their configurations.
x-displayName: Providers
- name: Safety
description: OpenAI-compatible Moderations API.
x-displayName: Safety
@ -10243,5 +10347,6 @@ x-tagGroups:
- Inference
- Models
- PostTraining (Coming Soon)
- Providers
- Safety
- VectorIO

663
docs/static/llama-stack-spec.html vendored
View file

@ -40,6 +40,224 @@
}
],
"paths": {
"/v1/admin/providers/{api}": {
"post": {
"responses": {
"200": {
"description": "RegisterProviderResponse with the registered provider info.",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/RegisterProviderResponse"
}
}
}
},
"400": {
"$ref": "#/components/responses/BadRequest400"
},
"429": {
"$ref": "#/components/responses/TooManyRequests429"
},
"500": {
"$ref": "#/components/responses/InternalServerError500"
},
"default": {
"$ref": "#/components/responses/DefaultError"
}
},
"tags": [
"Providers"
],
"summary": "Register a new dynamic provider.",
"description": "Register a new dynamic provider.\nRegister a new provider instance at runtime. The provider will be validated,\ninstantiated, and persisted to the kvstore. Requires appropriate ABAC permissions.",
"parameters": [
{
"name": "api",
"in": "path",
"description": "API namespace this provider implements (e.g., 'inference', 'vector_io').",
"required": true,
"schema": {
"type": "string"
}
}
],
"requestBody": {
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/RegisterProviderRequest"
}
}
},
"required": true
},
"deprecated": false
}
},
"/v1/admin/providers/{api}/{provider_id}": {
"post": {
"responses": {
"200": {
"description": "UpdateProviderResponse with updated provider info",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/UpdateProviderResponse"
}
}
}
},
"400": {
"$ref": "#/components/responses/BadRequest400"
},
"429": {
"$ref": "#/components/responses/TooManyRequests429"
},
"500": {
"$ref": "#/components/responses/InternalServerError500"
},
"default": {
"$ref": "#/components/responses/DefaultError"
}
},
"tags": [
"Providers"
],
"summary": "Update an existing provider's configuration.",
"description": "Update an existing provider's configuration.\nUpdate the configuration and/or attributes of a dynamic provider. The provider\nwill be re-instantiated with the new configuration (hot-reload).",
"parameters": [
{
"name": "api",
"in": "path",
"description": "API namespace the provider implements",
"required": true,
"schema": {
"type": "string"
}
},
{
"name": "provider_id",
"in": "path",
"description": "ID of the provider to update",
"required": true,
"schema": {
"type": "string"
}
}
],
"requestBody": {
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/UpdateProviderRequest"
}
}
},
"required": true
},
"deprecated": false
},
"delete": {
"responses": {
"200": {
"description": "OK"
},
"400": {
"$ref": "#/components/responses/BadRequest400"
},
"429": {
"$ref": "#/components/responses/TooManyRequests429"
},
"500": {
"$ref": "#/components/responses/InternalServerError500"
},
"default": {
"$ref": "#/components/responses/DefaultError"
}
},
"tags": [
"Providers"
],
"summary": "Unregister a dynamic provider.",
"description": "Unregister a dynamic provider.\nRemove a dynamic provider, shutting down its instance and removing it from\nthe kvstore.",
"parameters": [
{
"name": "api",
"in": "path",
"description": "API namespace the provider implements",
"required": true,
"schema": {
"type": "string"
}
},
{
"name": "provider_id",
"in": "path",
"description": "ID of the provider to unregister.",
"required": true,
"schema": {
"type": "string"
}
}
],
"deprecated": false
}
},
"/v1/admin/providers/{api}/{provider_id}/test": {
"post": {
"responses": {
"200": {
"description": "TestProviderConnectionResponse with health status.",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/TestProviderConnectionResponse"
}
}
}
},
"400": {
"$ref": "#/components/responses/BadRequest400"
},
"429": {
"$ref": "#/components/responses/TooManyRequests429"
},
"500": {
"$ref": "#/components/responses/InternalServerError500"
},
"default": {
"$ref": "#/components/responses/DefaultError"
}
},
"tags": [
"Providers"
],
"summary": "Test a provider connection.",
"description": "Test a provider connection.\nExecute a health check on a provider to verify it is reachable and functioning.",
"parameters": [
{
"name": "api",
"in": "path",
"description": "API namespace the provider implements.",
"required": true,
"schema": {
"type": "string"
}
},
{
"name": "provider_id",
"in": "path",
"description": "ID of the provider to test.",
"required": true,
"schema": {
"type": "string"
}
}
],
"deprecated": false
}
},
"/v1/chat/completions": {
"get": {
"responses": {
@ -1635,7 +1853,52 @@
"deprecated": false
}
},
"/v1/providers/{provider_id}": {
"/v1/providers/{api}": {
"get": {
"responses": {
"200": {
"description": "A ListProvidersResponse containing providers for the specified API.",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/ListProvidersResponse"
}
}
}
},
"400": {
"$ref": "#/components/responses/BadRequest400"
},
"429": {
"$ref": "#/components/responses/TooManyRequests429"
},
"500": {
"$ref": "#/components/responses/InternalServerError500"
},
"default": {
"$ref": "#/components/responses/DefaultError"
}
},
"tags": [
"Providers"
],
"summary": "List providers for a specific API.",
"description": "List providers for a specific API.\nList all providers that implement a specific API.",
"parameters": [
{
"name": "api",
"in": "path",
"description": "The API namespace to filter by (e.g., 'inference', 'vector_io')",
"required": true,
"schema": {
"type": "string"
}
}
],
"deprecated": false
}
},
"/v1/providers/{api}/{provider_id}": {
"get": {
"responses": {
"200": {
@ -1664,9 +1927,18 @@
"tags": [
"Providers"
],
"summary": "Get provider.",
"description": "Get provider.\nGet detailed information about a specific provider.",
"summary": "Get provider for specific API.",
"description": "Get provider for specific API.\nGet detailed information about a specific provider for a specific API.",
"parameters": [
{
"name": "api",
"in": "path",
"description": "The API namespace.",
"required": true,
"schema": {
"type": "string"
}
},
{
"name": "provider_id",
"in": "path",
@ -4005,6 +4277,391 @@
"title": "Error",
"description": "Error response from the API. Roughly follows RFC 7807."
},
"RegisterProviderRequest": {
"type": "object",
"properties": {
"provider_id": {
"type": "string",
"description": "Unique identifier for this provider instance."
},
"provider_type": {
"type": "string",
"description": "Provider type (e.g., 'remote::openai')."
},
"config": {
"type": "object",
"additionalProperties": {
"oneOf": [
{
"type": "null"
},
{
"type": "boolean"
},
{
"type": "number"
},
{
"type": "string"
},
{
"type": "array"
},
{
"type": "object"
}
]
},
"description": "Provider configuration (API keys, endpoints, etc.)."
},
"attributes": {
"type": "object",
"additionalProperties": {
"type": "array",
"items": {
"type": "string"
}
},
"description": "Optional attributes for ABAC access control."
}
},
"additionalProperties": false,
"required": [
"provider_id",
"provider_type",
"config"
],
"title": "RegisterProviderRequest"
},
"ProviderConnectionInfo": {
"type": "object",
"properties": {
"provider_id": {
"type": "string",
"description": "Unique identifier for this provider instance"
},
"api": {
"type": "string",
"description": "API namespace (e.g., \"inference\", \"vector_io\", \"safety\")"
},
"provider_type": {
"type": "string",
"description": "Provider type identifier (e.g., \"remote::openai\", \"inline::faiss\")"
},
"config": {
"type": "object",
"additionalProperties": {
"oneOf": [
{
"type": "null"
},
{
"type": "boolean"
},
{
"type": "number"
},
{
"type": "string"
},
{
"type": "array"
},
{
"type": "object"
}
]
},
"description": "Provider-specific configuration (API keys, endpoints, etc.)"
},
"status": {
"$ref": "#/components/schemas/ProviderConnectionStatus",
"description": "Current connection status"
},
"health": {
"$ref": "#/components/schemas/ProviderHealth",
"description": "Most recent health check result"
},
"created_at": {
"type": "string",
"format": "date-time",
"description": "Timestamp when provider was registered"
},
"updated_at": {
"type": "string",
"format": "date-time",
"description": "Timestamp of last update"
},
"last_health_check": {
"type": "string",
"format": "date-time",
"description": "Timestamp of last health check"
},
"error_message": {
"type": "string",
"description": "Error message if status is failed"
},
"metadata": {
"type": "object",
"additionalProperties": {
"oneOf": [
{
"type": "null"
},
{
"type": "boolean"
},
{
"type": "number"
},
{
"type": "string"
},
{
"type": "array"
},
{
"type": "object"
}
]
},
"description": "User-defined metadata (deprecated, use attributes)"
},
"owner": {
"type": "object",
"properties": {
"principal": {
"type": "string"
},
"attributes": {
"type": "object",
"additionalProperties": {
"type": "array",
"items": {
"type": "string"
}
}
}
},
"additionalProperties": false,
"required": [
"principal"
],
"description": "User who created this provider connection"
},
"attributes": {
"type": "object",
"additionalProperties": {
"type": "array",
"items": {
"type": "string"
}
},
"description": "Key-value attributes for ABAC access control"
}
},
"additionalProperties": false,
"required": [
"provider_id",
"api",
"provider_type",
"config",
"status",
"created_at",
"updated_at",
"metadata"
],
"title": "ProviderConnectionInfo",
"description": "Information about a dynamically managed provider connection.\nThis model represents a provider that has been registered at runtime\nvia the /providers API, as opposed to static providers configured in run.yaml.\n\nDynamic providers support full lifecycle management including registration,\nconfiguration updates, health monitoring, and removal."
},
"ProviderConnectionStatus": {
"type": "string",
"enum": [
"pending",
"initializing",
"connected",
"failed",
"disconnected",
"testing"
],
"title": "ProviderConnectionStatus",
"description": "Status of a dynamic provider connection."
},
"ProviderHealth": {
"type": "object",
"properties": {
"status": {
"type": "string",
"enum": [
"OK",
"Error",
"Not Implemented"
],
"description": "Health status (OK, ERROR, NOT_IMPLEMENTED)"
},
"message": {
"type": "string",
"description": "Optional error or status message"
},
"metrics": {
"type": "object",
"additionalProperties": {
"oneOf": [
{
"type": "null"
},
{
"type": "boolean"
},
{
"type": "number"
},
{
"type": "string"
},
{
"type": "array"
},
{
"type": "object"
}
]
},
"description": "Provider-specific health metrics"
},
"last_checked": {
"type": "string",
"format": "date-time",
"description": "Timestamp of last health check"
}
},
"additionalProperties": false,
"required": [
"status",
"metrics",
"last_checked"
],
"title": "ProviderHealth",
"description": "Structured wrapper around provider health status.\nThis wraps the existing dict-based HealthResponse for API responses\nwhile maintaining backward compatibility with existing provider implementations."
},
"RegisterProviderResponse": {
"type": "object",
"properties": {
"provider": {
"$ref": "#/components/schemas/ProviderConnectionInfo",
"description": "Information about the registered provider"
}
},
"additionalProperties": false,
"required": [
"provider"
],
"title": "RegisterProviderResponse",
"description": "Response after registering a provider."
},
"UpdateProviderRequest": {
"type": "object",
"properties": {
"config": {
"type": "object",
"additionalProperties": {
"oneOf": [
{
"type": "null"
},
{
"type": "boolean"
},
{
"type": "number"
},
{
"type": "string"
},
{
"type": "array"
},
{
"type": "object"
}
]
},
"description": "New configuration parameters (merged with existing)"
},
"attributes": {
"type": "object",
"additionalProperties": {
"type": "array",
"items": {
"type": "string"
}
},
"description": "New attributes for access control"
}
},
"additionalProperties": false,
"title": "UpdateProviderRequest"
},
"UpdateProviderResponse": {
"type": "object",
"properties": {
"provider": {
"$ref": "#/components/schemas/ProviderConnectionInfo",
"description": "Updated provider information"
}
},
"additionalProperties": false,
"required": [
"provider"
],
"title": "UpdateProviderResponse",
"description": "Response after updating a provider."
},
"TestProviderConnectionResponse": {
"type": "object",
"properties": {
"success": {
"type": "boolean",
"description": "Whether the connection test succeeded"
},
"health": {
"type": "object",
"additionalProperties": {
"oneOf": [
{
"type": "null"
},
{
"type": "boolean"
},
{
"type": "number"
},
{
"type": "string"
},
{
"type": "array"
},
{
"type": "object"
}
]
},
"description": "Health status from the provider"
},
"error_message": {
"type": "string",
"description": "Error message if test failed"
}
},
"additionalProperties": false,
"required": [
"success"
],
"title": "TestProviderConnectionResponse",
"description": "Response from testing a provider connection."
},
"Order": {
"type": "string",
"enum": [

489
docs/static/llama-stack-spec.yaml vendored
View file

@ -12,6 +12,178 @@ info:
servers:
- url: http://any-hosted-llama-stack.com
paths:
/v1/admin/providers/{api}:
post:
responses:
'200':
description: >-
RegisterProviderResponse with the registered provider info.
content:
application/json:
schema:
$ref: '#/components/schemas/RegisterProviderResponse'
'400':
$ref: '#/components/responses/BadRequest400'
'429':
$ref: >-
#/components/responses/TooManyRequests429
'500':
$ref: >-
#/components/responses/InternalServerError500
default:
$ref: '#/components/responses/DefaultError'
tags:
- Providers
summary: Register a new dynamic provider.
description: >-
Register a new dynamic provider.
Register a new provider instance at runtime. The provider will be validated,
instantiated, and persisted to the kvstore. Requires appropriate ABAC permissions.
parameters:
- name: api
in: path
description: >-
API namespace this provider implements (e.g., 'inference', 'vector_io').
required: true
schema:
type: string
requestBody:
content:
application/json:
schema:
$ref: '#/components/schemas/RegisterProviderRequest'
required: true
deprecated: false
/v1/admin/providers/{api}/{provider_id}:
post:
responses:
'200':
description: >-
UpdateProviderResponse with updated provider info
content:
application/json:
schema:
$ref: '#/components/schemas/UpdateProviderResponse'
'400':
$ref: '#/components/responses/BadRequest400'
'429':
$ref: >-
#/components/responses/TooManyRequests429
'500':
$ref: >-
#/components/responses/InternalServerError500
default:
$ref: '#/components/responses/DefaultError'
tags:
- Providers
summary: >-
Update an existing provider's configuration.
description: >-
Update an existing provider's configuration.
Update the configuration and/or attributes of a dynamic provider. The provider
will be re-instantiated with the new configuration (hot-reload).
parameters:
- name: api
in: path
description: API namespace the provider implements
required: true
schema:
type: string
- name: provider_id
in: path
description: ID of the provider to update
required: true
schema:
type: string
requestBody:
content:
application/json:
schema:
$ref: '#/components/schemas/UpdateProviderRequest'
required: true
deprecated: false
delete:
responses:
'200':
description: OK
'400':
$ref: '#/components/responses/BadRequest400'
'429':
$ref: >-
#/components/responses/TooManyRequests429
'500':
$ref: >-
#/components/responses/InternalServerError500
default:
$ref: '#/components/responses/DefaultError'
tags:
- Providers
summary: Unregister a dynamic provider.
description: >-
Unregister a dynamic provider.
Remove a dynamic provider, shutting down its instance and removing it from
the kvstore.
parameters:
- name: api
in: path
description: API namespace the provider implements
required: true
schema:
type: string
- name: provider_id
in: path
description: ID of the provider to unregister.
required: true
schema:
type: string
deprecated: false
/v1/admin/providers/{api}/{provider_id}/test:
post:
responses:
'200':
description: >-
TestProviderConnectionResponse with health status.
content:
application/json:
schema:
$ref: '#/components/schemas/TestProviderConnectionResponse'
'400':
$ref: '#/components/responses/BadRequest400'
'429':
$ref: >-
#/components/responses/TooManyRequests429
'500':
$ref: >-
#/components/responses/InternalServerError500
default:
$ref: '#/components/responses/DefaultError'
tags:
- Providers
summary: Test a provider connection.
description: >-
Test a provider connection.
Execute a health check on a provider to verify it is reachable and functioning.
parameters:
- name: api
in: path
description: API namespace the provider implements.
required: true
schema:
type: string
- name: provider_id
in: path
description: ID of the provider to test.
required: true
schema:
type: string
deprecated: false
/v1/chat/completions:
get:
responses:
@ -1251,7 +1423,43 @@ paths:
List all available providers.
parameters: []
deprecated: false
/v1/providers/{provider_id}:
/v1/providers/{api}:
get:
responses:
'200':
description: >-
A ListProvidersResponse containing providers for the specified API.
content:
application/json:
schema:
$ref: '#/components/schemas/ListProvidersResponse'
'400':
$ref: '#/components/responses/BadRequest400'
'429':
$ref: >-
#/components/responses/TooManyRequests429
'500':
$ref: >-
#/components/responses/InternalServerError500
default:
$ref: '#/components/responses/DefaultError'
tags:
- Providers
summary: List providers for a specific API.
description: >-
List providers for a specific API.
List all providers that implement a specific API.
parameters:
- name: api
in: path
description: >-
The API namespace to filter by (e.g., 'inference', 'vector_io')
required: true
schema:
type: string
deprecated: false
/v1/providers/{api}/{provider_id}:
get:
responses:
'200':
@ -1273,12 +1481,18 @@ paths:
$ref: '#/components/responses/DefaultError'
tags:
- Providers
summary: Get provider.
summary: Get provider for specific API.
description: >-
Get provider.
Get provider for specific API.
Get detailed information about a specific provider.
Get detailed information about a specific provider for a specific API.
parameters:
- name: api
in: path
description: The API namespace.
required: true
schema:
type: string
- name: provider_id
in: path
description: The ID of the provider to inspect.
@ -2999,6 +3213,273 @@ components:
title: Error
description: >-
Error response from the API. Roughly follows RFC 7807.
RegisterProviderRequest:
type: object
properties:
provider_id:
type: string
description: >-
Unique identifier for this provider instance.
provider_type:
type: string
description: Provider type (e.g., 'remote::openai').
config:
type: object
additionalProperties:
oneOf:
- type: 'null'
- type: boolean
- type: number
- type: string
- type: array
- type: object
description: >-
Provider configuration (API keys, endpoints, etc.).
attributes:
type: object
additionalProperties:
type: array
items:
type: string
description: >-
Optional attributes for ABAC access control.
additionalProperties: false
required:
- provider_id
- provider_type
- config
title: RegisterProviderRequest
ProviderConnectionInfo:
type: object
properties:
provider_id:
type: string
description: >-
Unique identifier for this provider instance
api:
type: string
description: >-
API namespace (e.g., "inference", "vector_io", "safety")
provider_type:
type: string
description: >-
Provider type identifier (e.g., "remote::openai", "inline::faiss")
config:
type: object
additionalProperties:
oneOf:
- type: 'null'
- type: boolean
- type: number
- type: string
- type: array
- type: object
description: >-
Provider-specific configuration (API keys, endpoints, etc.)
status:
$ref: '#/components/schemas/ProviderConnectionStatus'
description: Current connection status
health:
$ref: '#/components/schemas/ProviderHealth'
description: Most recent health check result
created_at:
type: string
format: date-time
description: Timestamp when provider was registered
updated_at:
type: string
format: date-time
description: Timestamp of last update
last_health_check:
type: string
format: date-time
description: Timestamp of last health check
error_message:
type: string
description: Error message if status is failed
metadata:
type: object
additionalProperties:
oneOf:
- type: 'null'
- type: boolean
- type: number
- type: string
- type: array
- type: object
description: >-
User-defined metadata (deprecated, use attributes)
owner:
type: object
properties:
principal:
type: string
attributes:
type: object
additionalProperties:
type: array
items:
type: string
additionalProperties: false
required:
- principal
description: >-
User who created this provider connection
attributes:
type: object
additionalProperties:
type: array
items:
type: string
description: >-
Key-value attributes for ABAC access control
additionalProperties: false
required:
- provider_id
- api
- provider_type
- config
- status
- created_at
- updated_at
- metadata
title: ProviderConnectionInfo
description: >-
Information about a dynamically managed provider connection.
This model represents a provider that has been registered at runtime
via the /providers API, as opposed to static providers configured in run.yaml.
Dynamic providers support full lifecycle management including registration,
configuration updates, health monitoring, and removal.
ProviderConnectionStatus:
type: string
enum:
- pending
- initializing
- connected
- failed
- disconnected
- testing
title: ProviderConnectionStatus
description: Status of a dynamic provider connection.
ProviderHealth:
type: object
properties:
status:
type: string
enum:
- OK
- Error
- Not Implemented
description: >-
Health status (OK, ERROR, NOT_IMPLEMENTED)
message:
type: string
description: Optional error or status message
metrics:
type: object
additionalProperties:
oneOf:
- type: 'null'
- type: boolean
- type: number
- type: string
- type: array
- type: object
description: Provider-specific health metrics
last_checked:
type: string
format: date-time
description: Timestamp of last health check
additionalProperties: false
required:
- status
- metrics
- last_checked
title: ProviderHealth
description: >-
Structured wrapper around provider health status.
This wraps the existing dict-based HealthResponse for API responses
while maintaining backward compatibility with existing provider implementations.
RegisterProviderResponse:
type: object
properties:
provider:
$ref: '#/components/schemas/ProviderConnectionInfo'
description: >-
Information about the registered provider
additionalProperties: false
required:
- provider
title: RegisterProviderResponse
description: Response after registering a provider.
UpdateProviderRequest:
type: object
properties:
config:
type: object
additionalProperties:
oneOf:
- type: 'null'
- type: boolean
- type: number
- type: string
- type: array
- type: object
description: >-
New configuration parameters (merged with existing)
attributes:
type: object
additionalProperties:
type: array
items:
type: string
description: New attributes for access control
additionalProperties: false
title: UpdateProviderRequest
UpdateProviderResponse:
type: object
properties:
provider:
$ref: '#/components/schemas/ProviderConnectionInfo'
description: Updated provider information
additionalProperties: false
required:
- provider
title: UpdateProviderResponse
description: Response after updating a provider.
TestProviderConnectionResponse:
type: object
properties:
success:
type: boolean
description: Whether the connection test succeeded
health:
type: object
additionalProperties:
oneOf:
- type: 'null'
- type: boolean
- type: number
- type: string
- type: array
- type: object
description: Health status from the provider
error_message:
type: string
description: Error message if test failed
additionalProperties: false
required:
- success
title: TestProviderConnectionResponse
description: >-
Response from testing a provider connection.
Order:
type: string
enum:

663
docs/static/stainless-llama-stack-spec.html vendored
View file

@ -40,6 +40,224 @@
}
],
"paths": {
"/v1/admin/providers/{api}": {
"post": {
"responses": {
"200": {
"description": "RegisterProviderResponse with the registered provider info.",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/RegisterProviderResponse"
}
}
}
},
"400": {
"$ref": "#/components/responses/BadRequest400"
},
"429": {
"$ref": "#/components/responses/TooManyRequests429"
},
"500": {
"$ref": "#/components/responses/InternalServerError500"
},
"default": {
"$ref": "#/components/responses/DefaultError"
}
},
"tags": [
"Providers"
],
"summary": "Register a new dynamic provider.",
"description": "Register a new dynamic provider.\nRegister a new provider instance at runtime. The provider will be validated,\ninstantiated, and persisted to the kvstore. Requires appropriate ABAC permissions.",
"parameters": [
{
"name": "api",
"in": "path",
"description": "API namespace this provider implements (e.g., 'inference', 'vector_io').",
"required": true,
"schema": {
"type": "string"
}
}
],
"requestBody": {
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/RegisterProviderRequest"
}
}
},
"required": true
},
"deprecated": false
}
},
"/v1/admin/providers/{api}/{provider_id}": {
"post": {
"responses": {
"200": {
"description": "UpdateProviderResponse with updated provider info",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/UpdateProviderResponse"
}
}
}
},
"400": {
"$ref": "#/components/responses/BadRequest400"
},
"429": {
"$ref": "#/components/responses/TooManyRequests429"
},
"500": {
"$ref": "#/components/responses/InternalServerError500"
},
"default": {
"$ref": "#/components/responses/DefaultError"
}
},
"tags": [
"Providers"
],
"summary": "Update an existing provider's configuration.",
"description": "Update an existing provider's configuration.\nUpdate the configuration and/or attributes of a dynamic provider. The provider\nwill be re-instantiated with the new configuration (hot-reload).",
"parameters": [
{
"name": "api",
"in": "path",
"description": "API namespace the provider implements",
"required": true,
"schema": {
"type": "string"
}
},
{
"name": "provider_id",
"in": "path",
"description": "ID of the provider to update",
"required": true,
"schema": {
"type": "string"
}
}
],
"requestBody": {
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/UpdateProviderRequest"
}
}
},
"required": true
},
"deprecated": false
},
"delete": {
"responses": {
"200": {
"description": "OK"
},
"400": {
"$ref": "#/components/responses/BadRequest400"
},
"429": {
"$ref": "#/components/responses/TooManyRequests429"
},
"500": {
"$ref": "#/components/responses/InternalServerError500"
},
"default": {
"$ref": "#/components/responses/DefaultError"
}
},
"tags": [
"Providers"
],
"summary": "Unregister a dynamic provider.",
"description": "Unregister a dynamic provider.\nRemove a dynamic provider, shutting down its instance and removing it from\nthe kvstore.",
"parameters": [
{
"name": "api",
"in": "path",
"description": "API namespace the provider implements",
"required": true,
"schema": {
"type": "string"
}
},
{
"name": "provider_id",
"in": "path",
"description": "ID of the provider to unregister.",
"required": true,
"schema": {
"type": "string"
}
}
],
"deprecated": false
}
},
"/v1/admin/providers/{api}/{provider_id}/test": {
"post": {
"responses": {
"200": {
"description": "TestProviderConnectionResponse with health status.",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/TestProviderConnectionResponse"
}
}
}
},
"400": {
"$ref": "#/components/responses/BadRequest400"
},
"429": {
"$ref": "#/components/responses/TooManyRequests429"
},
"500": {
"$ref": "#/components/responses/InternalServerError500"
},
"default": {
"$ref": "#/components/responses/DefaultError"
}
},
"tags": [
"Providers"
],
"summary": "Test a provider connection.",
"description": "Test a provider connection.\nExecute a health check on a provider to verify it is reachable and functioning.",
"parameters": [
{
"name": "api",
"in": "path",
"description": "API namespace the provider implements.",
"required": true,
"schema": {
"type": "string"
}
},
{
"name": "provider_id",
"in": "path",
"description": "ID of the provider to test.",
"required": true,
"schema": {
"type": "string"
}
}
],
"deprecated": false
}
},
"/v1/chat/completions": {
"get": {
"responses": {
@ -1635,7 +1853,52 @@
"deprecated": false
}
},
"/v1/providers/{provider_id}": {
"/v1/providers/{api}": {
"get": {
"responses": {
"200": {
"description": "A ListProvidersResponse containing providers for the specified API.",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/ListProvidersResponse"
}
}
}
},
"400": {
"$ref": "#/components/responses/BadRequest400"
},
"429": {
"$ref": "#/components/responses/TooManyRequests429"
},
"500": {
"$ref": "#/components/responses/InternalServerError500"
},
"default": {
"$ref": "#/components/responses/DefaultError"
}
},
"tags": [
"Providers"
],
"summary": "List providers for a specific API.",
"description": "List providers for a specific API.\nList all providers that implement a specific API.",
"parameters": [
{
"name": "api",
"in": "path",
"description": "The API namespace to filter by (e.g., 'inference', 'vector_io')",
"required": true,
"schema": {
"type": "string"
}
}
],
"deprecated": false
}
},
"/v1/providers/{api}/{provider_id}": {
"get": {
"responses": {
"200": {
@ -1664,9 +1927,18 @@
"tags": [
"Providers"
],
"summary": "Get provider.",
"description": "Get provider.\nGet detailed information about a specific provider.",
"summary": "Get provider for specific API.",
"description": "Get provider for specific API.\nGet detailed information about a specific provider for a specific API.",
"parameters": [
{
"name": "api",
"in": "path",
"description": "The API namespace.",
"required": true,
"schema": {
"type": "string"
}
},
{
"name": "provider_id",
"in": "path",
@ -5677,6 +5949,391 @@
"title": "Error",
"description": "Error response from the API. Roughly follows RFC 7807."
},
"RegisterProviderRequest": {
"type": "object",
"properties": {
"provider_id": {
"type": "string",
"description": "Unique identifier for this provider instance."
},
"provider_type": {
"type": "string",
"description": "Provider type (e.g., 'remote::openai')."
},
"config": {
"type": "object",
"additionalProperties": {
"oneOf": [
{
"type": "null"
},
{
"type": "boolean"
},
{
"type": "number"
},
{
"type": "string"
},
{
"type": "array"
},
{
"type": "object"
}
]
},
"description": "Provider configuration (API keys, endpoints, etc.)."
},
"attributes": {
"type": "object",
"additionalProperties": {
"type": "array",
"items": {
"type": "string"
}
},
"description": "Optional attributes for ABAC access control."
}
},
"additionalProperties": false,
"required": [
"provider_id",
"provider_type",
"config"
],
"title": "RegisterProviderRequest"
},
"ProviderConnectionInfo": {
"type": "object",
"properties": {
"provider_id": {
"type": "string",
"description": "Unique identifier for this provider instance"
},
"api": {
"type": "string",
"description": "API namespace (e.g., \"inference\", \"vector_io\", \"safety\")"
},
"provider_type": {
"type": "string",
"description": "Provider type identifier (e.g., \"remote::openai\", \"inline::faiss\")"
},
"config": {
"type": "object",
"additionalProperties": {
"oneOf": [
{
"type": "null"
},
{
"type": "boolean"
},
{
"type": "number"
},
{
"type": "string"
},
{
"type": "array"
},
{
"type": "object"
}
]
},
"description": "Provider-specific configuration (API keys, endpoints, etc.)"
},
"status": {
"$ref": "#/components/schemas/ProviderConnectionStatus",
"description": "Current connection status"
},
"health": {
"$ref": "#/components/schemas/ProviderHealth",
"description": "Most recent health check result"
},
"created_at": {
"type": "string",
"format": "date-time",
"description": "Timestamp when provider was registered"
},
"updated_at": {
"type": "string",
"format": "date-time",
"description": "Timestamp of last update"
},
"last_health_check": {
"type": "string",
"format": "date-time",
"description": "Timestamp of last health check"
},
"error_message": {
"type": "string",
"description": "Error message if status is failed"
},
"metadata": {
"type": "object",
"additionalProperties": {
"oneOf": [
{
"type": "null"
},
{
"type": "boolean"
},
{
"type": "number"
},
{
"type": "string"
},
{
"type": "array"
},
{
"type": "object"
}
]
},
"description": "User-defined metadata (deprecated, use attributes)"
},
"owner": {
"type": "object",
"properties": {
"principal": {
"type": "string"
},
"attributes": {
"type": "object",
"additionalProperties": {
"type": "array",
"items": {
"type": "string"
}
}
}
},
"additionalProperties": false,
"required": [
"principal"
],
"description": "User who created this provider connection"
},
"attributes": {
"type": "object",
"additionalProperties": {
"type": "array",
"items": {
"type": "string"
}
},
"description": "Key-value attributes for ABAC access control"
}
},
"additionalProperties": false,
"required": [
"provider_id",
"api",
"provider_type",
"config",
"status",
"created_at",
"updated_at",
"metadata"
],
"title": "ProviderConnectionInfo",
"description": "Information about a dynamically managed provider connection.\nThis model represents a provider that has been registered at runtime\nvia the /providers API, as opposed to static providers configured in run.yaml.\n\nDynamic providers support full lifecycle management including registration,\nconfiguration updates, health monitoring, and removal."
},
"ProviderConnectionStatus": {
"type": "string",
"enum": [
"pending",
"initializing",
"connected",
"failed",
"disconnected",
"testing"
],
"title": "ProviderConnectionStatus",
"description": "Status of a dynamic provider connection."
},
"ProviderHealth": {
"type": "object",
"properties": {
"status": {
"type": "string",
"enum": [
"OK",
"Error",
"Not Implemented"
],
"description": "Health status (OK, ERROR, NOT_IMPLEMENTED)"
},
"message": {
"type": "string",
"description": "Optional error or status message"
},
"metrics": {
"type": "object",
"additionalProperties": {
"oneOf": [
{
"type": "null"
},
{
"type": "boolean"
},
{
"type": "number"
},
{
"type": "string"
},
{
"type": "array"
},
{
"type": "object"
}
]
},
"description": "Provider-specific health metrics"
},
"last_checked": {
"type": "string",
"format": "date-time",
"description": "Timestamp of last health check"
}
},
"additionalProperties": false,
"required": [
"status",
"metrics",
"last_checked"
],
"title": "ProviderHealth",
"description": "Structured wrapper around provider health status.\nThis wraps the existing dict-based HealthResponse for API responses\nwhile maintaining backward compatibility with existing provider implementations."
},
"RegisterProviderResponse": {
"type": "object",
"properties": {
"provider": {
"$ref": "#/components/schemas/ProviderConnectionInfo",
"description": "Information about the registered provider"
}
},
"additionalProperties": false,
"required": [
"provider"
],
"title": "RegisterProviderResponse",
"description": "Response after registering a provider."
},
"UpdateProviderRequest": {
"type": "object",
"properties": {
"config": {
"type": "object",
"additionalProperties": {
"oneOf": [
{
"type": "null"
},
{
"type": "boolean"
},
{
"type": "number"
},
{
"type": "string"
},
{
"type": "array"
},
{
"type": "object"
}
]
},
"description": "New configuration parameters (merged with existing)"
},
"attributes": {
"type": "object",
"additionalProperties": {
"type": "array",
"items": {
"type": "string"
}
},
"description": "New attributes for access control"
}
},
"additionalProperties": false,
"title": "UpdateProviderRequest"
},
"UpdateProviderResponse": {
"type": "object",
"properties": {
"provider": {
"$ref": "#/components/schemas/ProviderConnectionInfo",
"description": "Updated provider information"
}
},
"additionalProperties": false,
"required": [
"provider"
],
"title": "UpdateProviderResponse",
"description": "Response after updating a provider."
},
"TestProviderConnectionResponse": {
"type": "object",
"properties": {
"success": {
"type": "boolean",
"description": "Whether the connection test succeeded"
},
"health": {
"type": "object",
"additionalProperties": {
"oneOf": [
{
"type": "null"
},
{
"type": "boolean"
},
{
"type": "number"
},
{
"type": "string"
},
{
"type": "array"
},
{
"type": "object"
}
]
},
"description": "Health status from the provider"
},
"error_message": {
"type": "string",
"description": "Error message if test failed"
}
},
"additionalProperties": false,
"required": [
"success"
],
"title": "TestProviderConnectionResponse",
"description": "Response from testing a provider connection."
},
"Order": {
"type": "string",
"enum": [

489
docs/static/stainless-llama-stack-spec.yaml vendored
View file

@ -15,6 +15,178 @@ info:
servers:
- url: http://any-hosted-llama-stack.com
paths:
/v1/admin/providers/{api}:
post:
responses:
'200':
description: >-
RegisterProviderResponse with the registered provider info.
content:
application/json:
schema:
$ref: '#/components/schemas/RegisterProviderResponse'
'400':
$ref: '#/components/responses/BadRequest400'
'429':
$ref: >-
#/components/responses/TooManyRequests429
'500':
$ref: >-
#/components/responses/InternalServerError500
default:
$ref: '#/components/responses/DefaultError'
tags:
- Providers
summary: Register a new dynamic provider.
description: >-
Register a new dynamic provider.
Register a new provider instance at runtime. The provider will be validated,
instantiated, and persisted to the kvstore. Requires appropriate ABAC permissions.
parameters:
- name: api
in: path
description: >-
API namespace this provider implements (e.g., 'inference', 'vector_io').
required: true
schema:
type: string
requestBody:
content:
application/json:
schema:
$ref: '#/components/schemas/RegisterProviderRequest'
required: true
deprecated: false
/v1/admin/providers/{api}/{provider_id}:
post:
responses:
'200':
description: >-
UpdateProviderResponse with updated provider info
content:
application/json:
schema:
$ref: '#/components/schemas/UpdateProviderResponse'
'400':
$ref: '#/components/responses/BadRequest400'
'429':
$ref: >-
#/components/responses/TooManyRequests429
'500':
$ref: >-
#/components/responses/InternalServerError500
default:
$ref: '#/components/responses/DefaultError'
tags:
- Providers
summary: >-
Update an existing provider's configuration.
description: >-
Update an existing provider's configuration.
Update the configuration and/or attributes of a dynamic provider. The provider
will be re-instantiated with the new configuration (hot-reload).
parameters:
- name: api
in: path
description: API namespace the provider implements
required: true
schema:
type: string
- name: provider_id
in: path
description: ID of the provider to update
required: true
schema:
type: string
requestBody:
content:
application/json:
schema:
$ref: '#/components/schemas/UpdateProviderRequest'
required: true
deprecated: false
delete:
responses:
'200':
description: OK
'400':
$ref: '#/components/responses/BadRequest400'
'429':
$ref: >-
#/components/responses/TooManyRequests429
'500':
$ref: >-
#/components/responses/InternalServerError500
default:
$ref: '#/components/responses/DefaultError'
tags:
- Providers
summary: Unregister a dynamic provider.
description: >-
Unregister a dynamic provider.
Remove a dynamic provider, shutting down its instance and removing it from
the kvstore.
parameters:
- name: api
in: path
description: API namespace the provider implements
required: true
schema:
type: string
- name: provider_id
in: path
description: ID of the provider to unregister.
required: true
schema:
type: string
deprecated: false
/v1/admin/providers/{api}/{provider_id}/test:
post:
responses:
'200':
description: >-
TestProviderConnectionResponse with health status.
content:
application/json:
schema:
$ref: '#/components/schemas/TestProviderConnectionResponse'
'400':
$ref: '#/components/responses/BadRequest400'
'429':
$ref: >-
#/components/responses/TooManyRequests429
'500':
$ref: >-
#/components/responses/InternalServerError500
default:
$ref: '#/components/responses/DefaultError'
tags:
- Providers
summary: Test a provider connection.
description: >-
Test a provider connection.
Execute a health check on a provider to verify it is reachable and functioning.
parameters:
- name: api
in: path
description: API namespace the provider implements.
required: true
schema:
type: string
- name: provider_id
in: path
description: ID of the provider to test.
required: true
schema:
type: string
deprecated: false
/v1/chat/completions:
get:
responses:
@ -1254,7 +1426,43 @@ paths:
List all available providers.
parameters: []
deprecated: false
/v1/providers/{provider_id}:
/v1/providers/{api}:
get:
responses:
'200':
description: >-
A ListProvidersResponse containing providers for the specified API.
content:
application/json:
schema:
$ref: '#/components/schemas/ListProvidersResponse'
'400':
$ref: '#/components/responses/BadRequest400'
'429':
$ref: >-
#/components/responses/TooManyRequests429
'500':
$ref: >-
#/components/responses/InternalServerError500
default:
$ref: '#/components/responses/DefaultError'
tags:
- Providers
summary: List providers for a specific API.
description: >-
List providers for a specific API.
List all providers that implement a specific API.
parameters:
- name: api
in: path
description: >-
The API namespace to filter by (e.g., 'inference', 'vector_io')
required: true
schema:
type: string
deprecated: false
/v1/providers/{api}/{provider_id}:
get:
responses:
'200':
@ -1276,12 +1484,18 @@ paths:
$ref: '#/components/responses/DefaultError'
tags:
- Providers
summary: Get provider.
summary: Get provider for specific API.
description: >-
Get provider.
Get provider for specific API.
Get detailed information about a specific provider.
Get detailed information about a specific provider for a specific API.
parameters:
- name: api
in: path
description: The API namespace.
required: true
schema:
type: string
- name: provider_id
in: path
description: The ID of the provider to inspect.
@ -4212,6 +4426,273 @@ components:
title: Error
description: >-
Error response from the API. Roughly follows RFC 7807.
RegisterProviderRequest:
type: object
properties:
provider_id:
type: string
description: >-
Unique identifier for this provider instance.
provider_type:
type: string
description: Provider type (e.g., 'remote::openai').
config:
type: object
additionalProperties:
oneOf:
- type: 'null'
- type: boolean
- type: number
- type: string
- type: array
- type: object
description: >-
Provider configuration (API keys, endpoints, etc.).
attributes:
type: object
additionalProperties:
type: array
items:
type: string
description: >-
Optional attributes for ABAC access control.
additionalProperties: false
required:
- provider_id
- provider_type
- config
title: RegisterProviderRequest
ProviderConnectionInfo:
type: object
properties:
provider_id:
type: string
description: >-
Unique identifier for this provider instance
api:
type: string
description: >-
API namespace (e.g., "inference", "vector_io", "safety")
provider_type:
type: string
description: >-
Provider type identifier (e.g., "remote::openai", "inline::faiss")
config:
type: object
additionalProperties:
oneOf:
- type: 'null'
- type: boolean
- type: number
- type: string
- type: array
- type: object
description: >-
Provider-specific configuration (API keys, endpoints, etc.)
status:
$ref: '#/components/schemas/ProviderConnectionStatus'
description: Current connection status
health:
$ref: '#/components/schemas/ProviderHealth'
description: Most recent health check result
created_at:
type: string
format: date-time
description: Timestamp when provider was registered
updated_at:
type: string
format: date-time
description: Timestamp of last update
last_health_check:
type: string
format: date-time
description: Timestamp of last health check
error_message:
type: string
description: Error message if status is failed
metadata:
type: object
additionalProperties:
oneOf:
- type: 'null'
- type: boolean
- type: number
- type: string
- type: array
- type: object
description: >-
User-defined metadata (deprecated, use attributes)
owner:
type: object
properties:
principal:
type: string
attributes:
type: object
additionalProperties:
type: array
items:
type: string
additionalProperties: false
required:
- principal
description: >-
User who created this provider connection
attributes:
type: object
additionalProperties:
type: array
items:
type: string
description: >-
Key-value attributes for ABAC access control
additionalProperties: false
required:
- provider_id
- api
- provider_type
- config
- status
- created_at
- updated_at
- metadata
title: ProviderConnectionInfo
description: >-
Information about a dynamically managed provider connection.
This model represents a provider that has been registered at runtime
via the /providers API, as opposed to static providers configured in run.yaml.
Dynamic providers support full lifecycle management including registration,
configuration updates, health monitoring, and removal.
ProviderConnectionStatus:
type: string
enum:
- pending
- initializing
- connected
- failed
- disconnected
- testing
title: ProviderConnectionStatus
description: Status of a dynamic provider connection.
ProviderHealth:
type: object
properties:
status:
type: string
enum:
- OK
- Error
- Not Implemented
description: >-
Health status (OK, ERROR, NOT_IMPLEMENTED)
message:
type: string
description: Optional error or status message
metrics:
type: object
additionalProperties:
oneOf:
- type: 'null'
- type: boolean
- type: number
- type: string
- type: array
- type: object
description: Provider-specific health metrics
last_checked:
type: string
format: date-time
description: Timestamp of last health check
additionalProperties: false
required:
- status
- metrics
- last_checked
title: ProviderHealth
description: >-
Structured wrapper around provider health status.
This wraps the existing dict-based HealthResponse for API responses
while maintaining backward compatibility with existing provider implementations.
RegisterProviderResponse:
type: object
properties:
provider:
$ref: '#/components/schemas/ProviderConnectionInfo'
description: >-
Information about the registered provider
additionalProperties: false
required:
- provider
title: RegisterProviderResponse
description: Response after registering a provider.
UpdateProviderRequest:
type: object
properties:
config:
type: object
additionalProperties:
oneOf:
- type: 'null'
- type: boolean
- type: number
- type: string
- type: array
- type: object
description: >-
New configuration parameters (merged with existing)
attributes:
type: object
additionalProperties:
type: array
items:
type: string
description: New attributes for access control
additionalProperties: false
title: UpdateProviderRequest
UpdateProviderResponse:
type: object
properties:
provider:
$ref: '#/components/schemas/ProviderConnectionInfo'
description: Updated provider information
additionalProperties: false
required:
- provider
title: UpdateProviderResponse
description: Response after updating a provider.
TestProviderConnectionResponse:
type: object
properties:
success:
type: boolean
description: Whether the connection test succeeded
health:
type: object
additionalProperties:
oneOf:
- type: 'null'
- type: boolean
- type: number
- type: string
- type: array
- type: object
description: Health status from the provider
error_message:
type: string
description: Error message if test failed
additionalProperties: false
required:
- success
title: TestProviderConnectionResponse
description: >-
Response from testing a provider connection.
Order:
type: string
enum:

117
llama_stack/apis/providers/connection.py Normal file
View file

@ -0,0 +1,117 @@
# Copyright (c) Meta Platforms, Inc. and affiliates.
# All rights reserved.
#
# This source code is licensed under the terms described in the LICENSE file in
# the root directory of this source tree.
from datetime import UTC, datetime
from enum import StrEnum
from typing import Any
from pydantic import BaseModel, Field
from llama_stack.core.datatypes import User
from llama_stack.providers.datatypes import HealthStatus
from llama_stack.schema_utils import json_schema_type
@json_schema_type
class ProviderConnectionStatus(StrEnum):
"""Status of a dynamic provider connection.
:cvar pending: Configuration stored, not yet initialized
:cvar initializing: In the process of connecting
:cvar connected: Successfully connected and healthy
:cvar failed: Connection attempt failed
:cvar disconnected: Previously connected, now disconnected
:cvar testing: Health check in progress
"""
pending = "pending"
initializing = "initializing"
connected = "connected"
failed = "failed"
disconnected = "disconnected"
testing = "testing"
@json_schema_type
class ProviderHealth(BaseModel):
"""Structured wrapper around provider health status.
This wraps the existing dict-based HealthResponse for API responses
while maintaining backward compatibility with existing provider implementations.
:param status: Health status (OK, ERROR, NOT_IMPLEMENTED)
:param message: Optional error or status message
:param metrics: Provider-specific health metrics
:param last_checked: Timestamp of last health check
"""
status: HealthStatus
message: str | None = None
metrics: dict[str, Any] = Field(default_factory=dict)
last_checked: datetime
@classmethod
def from_health_response(cls, response: dict[str, Any]) -> "ProviderHealth":
"""Convert dict-based HealthResponse to ProviderHealth.
This allows us to maintain the existing dict[str, Any] return type
for provider.health() methods while providing a structured model
for API responses.
:param response: Dict with 'status' and optional 'message', 'metrics'
:returns: ProviderHealth instance
"""
return cls(
status=HealthStatus(response.get("status", HealthStatus.NOT_IMPLEMENTED)),
message=response.get("message"),
metrics=response.get("metrics", {}),
last_checked=datetime.now(UTC),
)
@json_schema_type
class ProviderConnectionInfo(BaseModel):
"""Information about a dynamically managed provider connection.
This model represents a provider that has been registered at runtime
via the /providers API, as opposed to static providers configured in run.yaml.
Dynamic providers support full lifecycle management including registration,
configuration updates, health monitoring, and removal.
:param provider_id: Unique identifier for this provider instance
:param api: API namespace (e.g., "inference", "vector_io", "safety")
:param provider_type: Provider type identifier (e.g., "remote::openai", "inline::faiss")
:param config: Provider-specific configuration (API keys, endpoints, etc.)
:param status: Current connection status
:param health: Most recent health check result
:param created_at: Timestamp when provider was registered
:param updated_at: Timestamp of last update
:param last_health_check: Timestamp of last health check
:param error_message: Error message if status is failed
:param metadata: User-defined metadata (deprecated, use attributes)
:param owner: User who created this provider connection
:param attributes: Key-value attributes for ABAC access control
"""
provider_id: str
api: str
provider_type: str
config: dict[str, Any]
status: ProviderConnectionStatus
health: ProviderHealth | None = None
created_at: datetime
updated_at: datetime
last_health_check: datetime | None = None
error_message: str | None = None
metadata: dict[str, Any] = Field(
default_factory=dict,
description="Deprecated: use attributes for access control",
)
# ABAC fields (same as ResourceWithOwner)
owner: User | None = None
attributes: dict[str, list[str]] | None = None

183
llama_stack/apis/providers/providers.py
View file

@ -8,6 +8,7 @@ from typing import Any, Protocol, runtime_checkable
from pydantic import BaseModel
from llama_stack.apis.providers.connection import ProviderConnectionInfo
from llama_stack.apis.version import LLAMA_STACK_API_V1
from llama_stack.providers.datatypes import HealthResponse
from llama_stack.schema_utils import json_schema_type, webmethod
@ -40,6 +41,85 @@ class ListProvidersResponse(BaseModel):
data: list[ProviderInfo]
# ===== Dynamic Provider Management API Models =====
@json_schema_type
class RegisterProviderRequest(BaseModel):
"""Request to register a new dynamic provider.
:param provider_id: Unique identifier for the provider instance
:param api: API namespace (e.g., 'inference', 'vector_io', 'safety')
:param provider_type: Provider type identifier (e.g., 'remote::openai', 'inline::faiss')
:param config: Provider-specific configuration (API keys, endpoints, etc.)
:param attributes: Optional key-value attributes for ABAC access control
"""
provider_id: str
api: str
provider_type: str
config: dict[str, Any]
attributes: dict[str, list[str]] | None = None
@json_schema_type
class RegisterProviderResponse(BaseModel):
"""Response after registering a provider.
:param provider: Information about the registered provider
"""
provider: ProviderConnectionInfo
@json_schema_type
class UpdateProviderRequest(BaseModel):
"""Request to update an existing provider's configuration.
:param config: New configuration parameters (will be merged with existing)
:param attributes: Optional updated attributes for access control
"""
config: dict[str, Any] | None = None
attributes: dict[str, list[str]] | None = None
@json_schema_type
class UpdateProviderResponse(BaseModel):
"""Response after updating a provider.
:param provider: Updated provider information
"""
provider: ProviderConnectionInfo
@json_schema_type
class UnregisterProviderResponse(BaseModel):
"""Response after unregistering a provider.
:param success: Whether the operation succeeded
:param message: Optional status message
"""
success: bool
message: str | None = None
@json_schema_type
class TestProviderConnectionResponse(BaseModel):
"""Response from testing a provider connection.
:param success: Whether the connection test succeeded
:param health: Health status from the provider
:param error_message: Error message if test failed
"""
success: bool
health: HealthResponse | None = None
error_message: str | None = None
@runtime_checkable
class Providers(Protocol):
"""Providers
@ -57,12 +137,107 @@ class Providers(Protocol):
"""
...
@webmethod(route="/providers/{provider_id}", method="GET", level=LLAMA_STACK_API_V1)
async def inspect_provider(self, provider_id: str) -> ProviderInfo:
"""Get provider.
@webmethod(route="/providers/{provider_id}", method="GET", level=LLAMA_STACK_API_V1, deprecated=True)
async def inspect_provider(self, provider_id: str) -> ListProvidersResponse:
"""Get providers by ID (deprecated - use /providers/{api}/{provider_id} instead).
Get detailed information about a specific provider.
DEPRECATED: Returns all providers with the given provider_id across all APIs.
This can return multiple providers if the same ID is used for different APIs.
Use /providers/{api}/{provider_id} for unambiguous access.
:param provider_id: The ID of the provider(s) to inspect.
:returns: A ListProvidersResponse containing all providers with matching provider_id.
"""
...
# ===== Dynamic Provider Management Methods =====
@webmethod(route="/admin/providers/{api}", method="POST", level=LLAMA_STACK_API_V1)
async def register_provider(
self,
api: str,
provider_id: str,
provider_type: str,
config: dict[str, Any],
attributes: dict[str, list[str]] | None = None,
) -> RegisterProviderResponse:
"""Register a new dynamic provider.
Register a new provider instance at runtime. The provider will be validated,
instantiated, and persisted to the kvstore. Requires appropriate ABAC permissions.
:param api: API namespace this provider implements (e.g., 'inference', 'vector_io').
:param provider_id: Unique identifier for this provider instance.
:param provider_type: Provider type (e.g., 'remote::openai').
:param config: Provider configuration (API keys, endpoints, etc.).
:param attributes: Optional attributes for ABAC access control.
:returns: RegisterProviderResponse with the registered provider info.
"""
...
@webmethod(route="/admin/providers/{api}/{provider_id}", method="PUT", level=LLAMA_STACK_API_V1)
async def update_provider(
self,
api: str,
provider_id: str,
config: dict[str, Any] | None = None,
attributes: dict[str, list[str]] | None = None,
) -> UpdateProviderResponse:
"""Update an existing provider's configuration.
Update the configuration and/or attributes of a dynamic provider. The provider
will be re-instantiated with the new configuration (hot-reload).
:param api: API namespace the provider implements
:param provider_id: ID of the provider to update
:param config: New configuration parameters (merged with existing)
:param attributes: New attributes for access control
:returns: UpdateProviderResponse with updated provider info
"""
...
@webmethod(route="/admin/providers/{api}/{provider_id}", method="DELETE", level=LLAMA_STACK_API_V1)
async def unregister_provider(self, api: str, provider_id: str) -> None:
"""Unregister a dynamic provider.
Remove a dynamic provider, shutting down its instance and removing it from
the kvstore.
:param api: API namespace the provider implements
:param provider_id: ID of the provider to unregister.
"""
...
@webmethod(route="/admin/providers/{api}/{provider_id}/test", method="POST", level=LLAMA_STACK_API_V1)
async def test_provider_connection(self, api: str, provider_id: str) -> TestProviderConnectionResponse:
"""Test a provider connection.
Execute a health check on a provider to verify it is reachable and functioning.
:param api: API namespace the provider implements.
:param provider_id: ID of the provider to test.
:returns: TestProviderConnectionResponse with health status.
"""
...
@webmethod(route="/providers/{api}", method="GET", level=LLAMA_STACK_API_V1)
async def list_providers_for_api(self, api: str) -> ListProvidersResponse:
"""List providers for a specific API.
List all providers that implement a specific API.
:param api: The API namespace to filter by (e.g., 'inference', 'vector_io')
:returns: A ListProvidersResponse containing providers for the specified API.
"""
...
@webmethod(route="/providers/{api}/{provider_id}", method="GET", level=LLAMA_STACK_API_V1)
async def inspect_provider_for_api(self, api: str, provider_id: str) -> ProviderInfo:
"""Get provider for specific API.
Get detailed information about a specific provider for a specific API.
:param api: The API namespace.
:param provider_id: The ID of the provider to inspect.
:returns: A ProviderInfo object containing the provider's details.
"""

525
llama_stack/core/providers.py
View file

@ -5,22 +5,45 @@
# the root directory of this source tree.
import asyncio
from datetime import UTC, datetime
from typing import Any
from pydantic import BaseModel
from llama_stack.apis.providers import ListProvidersResponse, ProviderInfo, Providers
from llama_stack.apis.providers import (
ListProvidersResponse,
ProviderInfo,
Providers,
RegisterProviderResponse,
TestProviderConnectionResponse,
UpdateProviderResponse,
)
from llama_stack.apis.providers.connection import (
ProviderConnectionInfo,
ProviderConnectionStatus,
ProviderHealth,
)
from llama_stack.core.request_headers import get_authenticated_user
from llama_stack.core.resolver import ProviderWithSpec, instantiate_provider
from llama_stack.log import get_logger
from llama_stack.providers.datatypes import HealthResponse, HealthStatus
from llama_stack.providers.datatypes import Api, HealthResponse, HealthStatus
from .datatypes import StackRunConfig
from .utils.config import redact_sensitive_fields
logger = get_logger(name=__name__, category="core")
# Storage constants for dynamic provider connections
# Use composite key format: provider_connections:v1::{api}::{provider_id}
# This allows the same provider_id to be used for different APIs
PROVIDER_CONNECTIONS_PREFIX = "provider_connections:v1::"
class ProviderImplConfig(BaseModel):
run_config: StackRunConfig
provider_registry: Any | None = None # ProviderRegistry from resolver
dist_registry: Any | None = None # DistributionRegistry
policy: list[Any] | None = None # list[AccessRule]
async def get_provider_impl(config, deps):
@ -33,19 +56,71 @@ class ProviderImpl(Providers):
def __init__(self, config, deps):
self.config = config
self.deps = deps
self.kvstore = None # KVStore for dynamic provider persistence
# Runtime cache uses composite key: "{api}::{provider_id}"
# This allows the same provider_id to be used for different APIs
self.dynamic_providers: dict[str, ProviderConnectionInfo] = {} # Runtime cache
self.dynamic_provider_impls: dict[str, Any] = {} # Initialized provider instances
# Store registry references for provider instantiation
self.provider_registry = config.provider_registry
self.dist_registry = config.dist_registry
self.policy = config.policy or []
async def initialize(self) -> None:
pass
# Initialize kvstore for dynamic providers
# Use the metadata store from the new storage config structure
if not (self.config.run_config.storage and self.config.run_config.storage.stores.metadata):
raise RuntimeError(
"No metadata store configured in storage.stores.metadata. "
"Provider management requires a configured metadata store (kv_memory, kv_sqlite, etc)."
)
from llama_stack.providers.utils.kvstore import kvstore_impl
self.kvstore = await kvstore_impl(self.config.run_config.storage.stores.metadata)
logger.info("Initialized kvstore for dynamic provider management")
# Load existing dynamic providers from kvstore
await self._load_dynamic_providers()
logger.info(f"Loaded {len(self.dynamic_providers)} existing dynamic providers from kvstore")
for provider_id, conn_info in self.dynamic_providers.items():
if conn_info.status == ProviderConnectionStatus.connected:
try:
impl = await self._instantiate_provider(conn_info)
self.dynamic_provider_impls[provider_id] = impl
except Exception as e:
logger.error(f"Failed to instantiate provider {provider_id}: {e}")
# Update status to failed
conn_info.status = ProviderConnectionStatus.failed
conn_info.error_message = str(e)
conn_info.updated_at = datetime.now(UTC)
await self._store_connection(conn_info)
async def shutdown(self) -> None:
logger.debug("ProviderImpl.shutdown")
pass
# Shutdown all dynamic provider instances
for provider_id, impl in self.dynamic_provider_impls.items():
try:
if hasattr(impl, "shutdown"):
await impl.shutdown()
logger.debug(f"Shutdown dynamic provider {provider_id}")
except Exception as e:
logger.warning(f"Error shutting down dynamic provider {provider_id}: {e}")
# Shutdown kvstore
if self.kvstore and hasattr(self.kvstore, "shutdown"):
await self.kvstore.shutdown()
async def list_providers(self) -> ListProvidersResponse:
run_config = self.config.run_config
safe_config = StackRunConfig(**redact_sensitive_fields(run_config.model_dump()))
providers_health = await self.get_providers_health()
ret = []
# Add static providers (from run.yaml)
for api, providers in safe_config.providers.items():
for p in providers:
# Skip providers that are not enabled
@ -66,15 +141,62 @@ class ProviderImpl(Providers):
)
)
# Add dynamic providers (from kvstore)
for _provider_id, conn_info in self.dynamic_providers.items():
# Redact sensitive config for API response
redacted_config = self._redact_sensitive_config(conn_info.config)
# Convert ProviderHealth to HealthResponse dict for API compatibility
health_dict: HealthResponse | None = None
if conn_info.health:
health_dict = HealthResponse(
status=conn_info.health.status,
message=conn_info.health.message,
)
if conn_info.health.metrics:
health_dict["metrics"] = conn_info.health.metrics
ret.append(
ProviderInfo(
api=conn_info.api,
provider_id=conn_info.provider_id,
provider_type=conn_info.provider_type,
config=redacted_config,
health=health_dict
or HealthResponse(status=HealthStatus.NOT_IMPLEMENTED, message="No health check available"),
)
)
return ListProvidersResponse(data=ret)
async def inspect_provider(self, provider_id: str) -> ProviderInfo:
async def inspect_provider(self, provider_id: str) -> ListProvidersResponse:
"""Get all providers with the given provider_id (deprecated).
Returns all providers across all APIs that have this provider_id.
This is deprecated - use inspect_provider_for_api() for unambiguous access.
"""
all_providers = await self.list_providers()
matching = [p for p in all_providers.data if p.provider_id == provider_id]
if not matching:
raise ValueError(f"Provider {provider_id} not found")
return ListProvidersResponse(data=matching)
async def list_providers_for_api(self, api: str) -> ListProvidersResponse:
"""List providers for a specific API."""
all_providers = await self.list_providers()
filtered = [p for p in all_providers.data if p.api == api]
return ListProvidersResponse(data=filtered)
async def inspect_provider_for_api(self, api: str, provider_id: str) -> ProviderInfo:
"""Get a specific provider for a specific API."""
all_providers = await self.list_providers()
for p in all_providers.data:
if p.provider_id == provider_id:
if p.api == api and p.provider_id == provider_id:
return p
raise ValueError(f"Provider {provider_id} not found")
raise ValueError(f"Provider {provider_id} not found for API {api}")
async def get_providers_health(self) -> dict[str, dict[str, HealthResponse]]:
"""Get health status for all providers.
@ -135,3 +257,392 @@ class ProviderImpl(Providers):
providers_health[api_name] = health_response
return providers_health
# Storage helper methods for dynamic providers
async def _store_connection(self, info: ProviderConnectionInfo) -> None:
"""Store provider connection info in kvstore.
:param info: ProviderConnectionInfo to store
"""
if not self.kvstore:
raise RuntimeError("KVStore not initialized")
# Use composite key: provider_connections:v1::{api}::{provider_id}
key = f"{PROVIDER_CONNECTIONS_PREFIX}{info.api}::{info.provider_id}"
await self.kvstore.set(key, info.model_dump_json())
logger.debug(f"Stored provider connection: {info.api}::{info.provider_id}")
async def _load_connection(self, provider_id: str) -> ProviderConnectionInfo | None:
"""Load provider connection info from kvstore.
:param provider_id: Provider ID to load
:returns: ProviderConnectionInfo if found, None otherwise
"""
if not self.kvstore:
return None
key = f"{PROVIDER_CONNECTIONS_PREFIX}{provider_id}"
value = await self.kvstore.get(key)
if value:
return ProviderConnectionInfo.model_validate_json(value)
return None
async def _delete_connection(self, api: str, provider_id: str) -> None:
"""Delete provider connection from kvstore.
:param api: API namespace
:param provider_id: Provider ID to delete
"""
if not self.kvstore:
raise RuntimeError("KVStore not initialized")
# Use composite key: provider_connections:v1::{api}::{provider_id}
key = f"{PROVIDER_CONNECTIONS_PREFIX}{api}::{provider_id}"
await self.kvstore.delete(key)
logger.debug(f"Deleted provider connection: {api}::{provider_id}")
async def _list_connections(self) -> list[ProviderConnectionInfo]:
"""List all dynamic provider connections from kvstore.
:returns: List of ProviderConnectionInfo
"""
if not self.kvstore:
return []
start_key = PROVIDER_CONNECTIONS_PREFIX
end_key = f"{PROVIDER_CONNECTIONS_PREFIX}\xff"
values = await self.kvstore.values_in_range(start_key, end_key)
return [ProviderConnectionInfo.model_validate_json(v) for v in values]
async def _load_dynamic_providers(self) -> None:
"""Load dynamic providers from kvstore into runtime cache."""
connections = await self._list_connections()
for conn in connections:
# Use composite key for runtime cache
cache_key = f"{conn.api}::{conn.provider_id}"
self.dynamic_providers[cache_key] = conn
logger.debug(f"Loaded dynamic provider: {cache_key} (status: {conn.status})")
def _find_provider_cache_key(self, provider_id: str) -> str | None:
"""Find the cache key for a provider by its provider_id.
Since we use composite keys ({api}::{provider_id}), this searches for the matching key.
Returns None if not found.
"""
for key in self.dynamic_providers.keys():
if key.endswith(f"::{provider_id}"):
return key
return None
# Helper methods for dynamic provider management
def _redact_sensitive_config(self, config: dict[str, Any]) -> dict[str, Any]:
"""Redact sensitive fields in provider config for API responses.
:param config: Provider configuration dict
:returns: Config with sensitive fields redacted
"""
return redact_sensitive_fields(config)
async def _instantiate_provider(self, conn_info: ProviderConnectionInfo) -> Any:
"""Instantiate a provider from connection info.
Uses the resolver's instantiate_provider() to create a provider instance
with all necessary dependencies.
:param conn_info: Provider connection information
:returns: Instantiated provider implementation
:raises RuntimeError: If provider cannot be instantiated
"""
if not self.provider_registry:
raise RuntimeError("Provider registry not available for provider instantiation")
if not self.dist_registry:
raise RuntimeError("Distribution registry not available for provider instantiation")
# Get provider spec from registry
api = Api(conn_info.api)
if api not in self.provider_registry:
raise ValueError(f"API {conn_info.api} not found in provider registry")
if conn_info.provider_type not in self.provider_registry[api]:
raise ValueError(f"Provider type {conn_info.provider_type} not found for API {conn_info.api}")
provider_spec = self.provider_registry[api][conn_info.provider_type]
# Create ProviderWithSpec for instantiation
provider_with_spec = ProviderWithSpec(
provider_id=conn_info.provider_id,
provider_type=conn_info.provider_type,
config=conn_info.config,
spec=provider_spec,
)
# Resolve dependencies
deps = {}
for dep_api in provider_spec.api_dependencies:
if dep_api not in self.deps:
raise RuntimeError(
f"Required dependency {dep_api.value} not available for provider {conn_info.provider_id}"
)
deps[dep_api] = self.deps[dep_api]
# Add optional dependencies if available
for dep_api in provider_spec.optional_api_dependencies:
if dep_api in self.deps:
deps[dep_api] = self.deps[dep_api]
# Instantiate provider using resolver
impl = await instantiate_provider(
provider_with_spec,
deps,
{}, # inner_impls (empty for dynamic providers)
self.dist_registry,
self.config.run_config,
self.policy,
)
logger.debug(f"Instantiated provider {conn_info.provider_id} (type={conn_info.provider_type})")
return impl
# Dynamic Provider Management Methods
async def register_provider(
self,
api: str,
provider_id: str,
provider_type: str,
config: dict[str, Any],
attributes: dict[str, list[str]] | None = None,
) -> RegisterProviderResponse:
"""Register a new provider.
This is used both for:
- Providers from run.yaml (registered at startup)
- Providers registered via API (registered at runtime)
All providers are stored in kvstore and treated equally.
"""
if not self.kvstore:
raise RuntimeError("Dynamic provider management is not enabled (no kvstore configured)")
# Use composite key to allow same provider_id for different APIs
cache_key = f"{api}::{provider_id}"
# Check if provider already exists for this API
if cache_key in self.dynamic_providers:
raise ValueError(f"Provider {provider_id} already exists for API {api}")
# Get authenticated user as owner
user = get_authenticated_user()
# Create ProviderConnectionInfo
now = datetime.now(UTC)
conn_info = ProviderConnectionInfo(
provider_id=provider_id,
api=api,
provider_type=provider_type,
config=config,
status=ProviderConnectionStatus.initializing,
created_at=now,
updated_at=now,
owner=user,
attributes=attributes,
)
try:
# Store in kvstore
await self._store_connection(conn_info)
impl = await self._instantiate_provider(conn_info)
# Use composite key for impl cache too
self.dynamic_provider_impls[cache_key] = impl
# Update status to connected after successful instantiation
conn_info.status = ProviderConnectionStatus.connected
conn_info.updated_at = datetime.now(UTC)
logger.info(f"Registered and instantiated dynamic provider {provider_id} (api={api}, type={provider_type})")
# Store updated status
await self._store_connection(conn_info)
# Add to runtime cache using composite key
self.dynamic_providers[cache_key] = conn_info
return RegisterProviderResponse(provider=conn_info)
except Exception as e:
# Mark as failed and store
conn_info.status = ProviderConnectionStatus.failed
conn_info.error_message = str(e)
conn_info.updated_at = datetime.now(UTC)
await self._store_connection(conn_info)
self.dynamic_providers[cache_key] = conn_info
logger.error(f"Failed to register provider {provider_id}: {e}")
raise RuntimeError(f"Failed to register provider: {e}") from e
async def update_provider(
self,
api: str,
provider_id: str,
config: dict[str, Any] | None = None,
attributes: dict[str, list[str]] | None = None,
) -> UpdateProviderResponse:
"""Update an existing provider's configuration.
Updates persist to kvstore and survive server restarts.
This works for all providers (whether originally from run.yaml or API).
"""
if not self.kvstore:
raise RuntimeError("Dynamic provider management is not enabled (no kvstore configured)")
# Use composite key
cache_key = f"{api}::{provider_id}"
if cache_key not in self.dynamic_providers:
raise ValueError(f"Provider {provider_id} not found for API {api}")
conn_info = self.dynamic_providers[cache_key]
# Update config if provided
if config is not None:
conn_info.config.update(config)
# Update attributes if provided
if attributes is not None:
conn_info.attributes = attributes
conn_info.updated_at = datetime.now(UTC)
conn_info.status = ProviderConnectionStatus.initializing
try:
# Store updated config
await self._store_connection(conn_info)
# Hot-reload: Shutdown old instance and reinstantiate with new config
# Shutdown old instance if it exists
if cache_key in self.dynamic_provider_impls:
old_impl = self.dynamic_provider_impls[cache_key]
if hasattr(old_impl, "shutdown"):
try:
await old_impl.shutdown()
logger.debug(f"Shutdown old instance of provider {provider_id}")
except Exception as e:
logger.warning(f"Error shutting down old instance of {provider_id}: {e}")
# Reinstantiate with new config
impl = await self._instantiate_provider(conn_info)
self.dynamic_provider_impls[cache_key] = impl
# Update status to connected after successful reinstantiation
conn_info.status = ProviderConnectionStatus.connected
conn_info.updated_at = datetime.now(UTC)
await self._store_connection(conn_info)
logger.info(f"Hot-reloaded dynamic provider {provider_id}")
return UpdateProviderResponse(provider=conn_info)
except Exception as e:
conn_info.status = ProviderConnectionStatus.failed
conn_info.error_message = str(e)
conn_info.updated_at = datetime.now(UTC)
await self._store_connection(conn_info)
logger.error(f"Failed to update provider {provider_id}: {e}")
raise RuntimeError(f"Failed to update provider: {e}") from e
async def unregister_provider(self, api: str, provider_id: str) -> None:
"""Unregister a provider.
Removes the provider from kvstore and shuts down its instance.
This works for all providers (whether originally from run.yaml or API).
"""
if not self.kvstore:
raise RuntimeError("Dynamic provider management is not enabled (no kvstore configured)")
# Use composite key
cache_key = f"{api}::{provider_id}"
if cache_key not in self.dynamic_providers:
raise ValueError(f"Provider {provider_id} not found for API {api}")
conn_info = self.dynamic_providers[cache_key]
try:
# Shutdown provider instance if it exists
if cache_key in self.dynamic_provider_impls:
impl = self.dynamic_provider_impls[cache_key]
if hasattr(impl, "shutdown"):
await impl.shutdown()
del self.dynamic_provider_impls[cache_key]
# Remove from kvstore (using the api and provider_id from conn_info)
await self._delete_connection(conn_info.api, provider_id)
# Remove from runtime cache
del self.dynamic_providers[cache_key]
logger.info(f"Unregistered dynamic provider {provider_id}")
except Exception as e:
logger.error(f"Failed to unregister provider {provider_id}: {e}")
raise RuntimeError(f"Failed to unregister provider: {e}") from e
async def test_provider_connection(self, api: str, provider_id: str) -> TestProviderConnectionResponse:
"""Test a provider connection."""
# Check if provider exists (static or dynamic)
provider_impl = None
cache_key = f"{api}::{provider_id}"
# Check dynamic providers first (using composite keys)
if cache_key in self.dynamic_provider_impls:
provider_impl = self.dynamic_provider_impls[cache_key]
# Check static providers
if not provider_impl and provider_id in self.deps:
provider_impl = self.deps[provider_id]
if not provider_impl:
return TestProviderConnectionResponse(
success=False, error_message=f"Provider {provider_id} not found for API {api}"
)
# Check if provider has health method
if not hasattr(provider_impl, "health"):
return TestProviderConnectionResponse(
success=False,
health=HealthResponse(
status=HealthStatus.NOT_IMPLEMENTED, message="Provider does not implement health check"
),
)
# Call health check
try:
health_result = await asyncio.wait_for(provider_impl.health(), timeout=5.0)
# Update health in dynamic provider cache if applicable
if cache_key and cache_key in self.dynamic_providers:
conn_info = self.dynamic_providers[cache_key]
conn_info.health = ProviderHealth.from_health_response(health_result)
conn_info.last_health_check = datetime.now(UTC)
await self._store_connection(conn_info)
logger.debug(f"Tested provider connection {provider_id}: status={health_result.get('status', 'UNKNOWN')}")
return TestProviderConnectionResponse(
success=health_result.get("status") == HealthStatus.OK,
health=health_result,
)
except TimeoutError:
health = HealthResponse(status=HealthStatus.ERROR, message="Health check timed out after 5 seconds")
return TestProviderConnectionResponse(success=False, health=health)
except Exception as e:
health = HealthResponse(status=HealthStatus.ERROR, message=f"Health check failed: {str(e)}")
return TestProviderConnectionResponse(success=False, health=health, error_message=str(e))

224
llama_stack/core/stack.py
View file

@ -34,13 +34,20 @@ from llama_stack.apis.synthetic_data_generation import SyntheticDataGeneration
from llama_stack.apis.telemetry import Telemetry
from llama_stack.apis.tools import RAGToolRuntime, ToolGroups, ToolRuntime
from llama_stack.apis.vector_io import VectorIO
from llama_stack.core.access_control.datatypes import AccessRule
from llama_stack.core.conversations.conversations import ConversationServiceConfig, ConversationServiceImpl
from llama_stack.core.datatypes import Provider, SafetyConfig, StackRunConfig, VectorStoresConfig
from llama_stack.core.distribution import get_provider_registry
from llama_stack.core.distribution import builtin_automatically_routed_apis, get_provider_registry
from llama_stack.core.inspect import DistributionInspectConfig, DistributionInspectImpl
from llama_stack.core.prompts.prompts import PromptServiceConfig, PromptServiceImpl
from llama_stack.core.providers import ProviderImpl, ProviderImplConfig
from llama_stack.core.resolver import ProviderRegistry, resolve_impls
from llama_stack.core.resolver import (
ProviderRegistry,
instantiate_provider,
sort_providers_by_deps,
specs_for_autorouted_apis,
validate_and_prepare_providers,
)
from llama_stack.core.routing_tables.common import CommonRoutingTableImpl
from llama_stack.core.storage.datatypes import (
InferenceStoreReference,
@ -52,10 +59,12 @@ from llama_stack.core.storage.datatypes import (
StorageBackendConfig,
StorageConfig,
)
from llama_stack.core.store.registry import create_dist_registry
from llama_stack.core.store.registry import DistributionRegistry, create_dist_registry
from llama_stack.core.utils.dynamic import instantiate_class_type
from llama_stack.log import get_logger
from llama_stack.providers.datatypes import Api
from llama_stack.providers.utils.kvstore.kvstore import register_kvstore_backends
from llama_stack.providers.utils.sqlstore.sqlstore import register_sqlstore_backends
logger = get_logger(name=__name__, category="core")
@ -341,12 +350,21 @@ def cast_image_name_to_string(config_dict: dict[str, Any]) -> dict[str, Any]:
return config_dict
def add_internal_implementations(impls: dict[Api, Any], run_config: StackRunConfig) -> None:
def add_internal_implementations(
impls: dict[Api, Any],
run_config: StackRunConfig,
provider_registry=None,
dist_registry=None,
policy=None,
) -> None:
"""Add internal implementations (inspect and providers) to the implementations dictionary.
Args:
impls: Dictionary of API implementations
run_config: Stack run configuration
provider_registry: Provider registry for dynamic provider instantiation
dist_registry: Distribution registry
policy: Access control policy
"""
inspect_impl = DistributionInspectImpl(
DistributionInspectConfig(run_config=run_config),
@ -355,7 +373,12 @@ def add_internal_implementations(impls: dict[Api, Any], run_config: StackRunConf
impls[Api.inspect] = inspect_impl
providers_impl = ProviderImpl(
ProviderImplConfig(run_config=run_config),
ProviderImplConfig(
run_config=run_config,
provider_registry=provider_registry,
dist_registry=dist_registry,
policy=policy,
),
deps=impls,
)
impls[Api.providers] = providers_impl
@ -385,13 +408,179 @@ def _initialize_storage(run_config: StackRunConfig):
else:
raise ValueError(f"Unknown storage backend type: {type}")
from llama_stack.providers.utils.kvstore.kvstore import register_kvstore_backends
from llama_stack.providers.utils.sqlstore.sqlstore import register_sqlstore_backends
register_kvstore_backends(kv_backends)
register_sqlstore_backends(sql_backends)
async def resolve_impls_via_provider_registration(
run_config: StackRunConfig,
provider_registry: ProviderRegistry,
dist_registry: DistributionRegistry,
policy: list[AccessRule],
internal_impls: dict[Api, Any],
) -> dict[Api, Any]:
"""
Resolves provider implementations by registering them through ProviderImpl.
This ensures all providers (startup and runtime) go through the same registration code path.
Args:
run_config: Stack run configuration with providers from run.yaml
provider_registry: Registry of available provider types
dist_registry: Distribution registry
policy: Access control policy
internal_impls: Internal implementations (inspect, providers) already initialized
Returns:
Dictionary mapping API to implementation instances
"""
routing_table_apis = {x.routing_table_api for x in builtin_automatically_routed_apis()}
router_apis = {x.router_api for x in builtin_automatically_routed_apis()}
# Validate and prepare providers from run.yaml
providers_with_specs = validate_and_prepare_providers(
run_config, provider_registry, routing_table_apis, router_apis
)
apis_to_serve = run_config.apis or set(
list(providers_with_specs.keys()) + [x.value for x in routing_table_apis] + [x.value for x in router_apis]
)
providers_with_specs.update(specs_for_autorouted_apis(apis_to_serve))
# Sort providers in dependency order
sorted_providers = sort_providers_by_deps(providers_with_specs, run_config)
# Get the ProviderImpl instance
providers_impl = internal_impls[Api.providers]
# Register each provider through ProviderImpl
impls = internal_impls.copy()
logger.info(f"Provider registration for {len(sorted_providers)} providers from run.yaml")
for api_str, provider in sorted_providers:
# Skip providers that are not enabled
if provider.provider_id is None:
continue
# Skip internal APIs (already initialized)
if api_str in ["providers", "inspect"]:
continue
# Handle different provider types
try:
# Check if this is a router (system infrastructure)
is_router = not api_str.startswith("inner-") and (
Api(api_str) in router_apis or provider.spec.provider_type == "router"
)
if api_str.startswith("inner-") or provider.spec.provider_type == "routing_table":
# Inner providers or routing tables cannot be registered through the API
# They need to be instantiated directly
logger.info(f"Instantiating {provider.provider_id} for {api_str}")
deps = {a: impls[a] for a in provider.spec.api_dependencies if a in impls}
for a in provider.spec.optional_api_dependencies:
if a in impls:
deps[a] = impls[a]
# Get inner impls if available
inner_impls = {}
# For routing tables of autorouted APIs, get inner impls from the router API
# E.g., tool_groups routing table needs inner-tool_runtime providers
if provider.spec.provider_type == "routing_table":
autorouted_map = {
info.routing_table_api: info.router_api for info in builtin_automatically_routed_apis()
}
if Api(api_str) in autorouted_map:
router_api_str = autorouted_map[Api(api_str)].value
inner_key = f"inner-{router_api_str}"
if inner_key in impls:
inner_impls = impls[inner_key]
else:
# For regular inner providers, use their own inner key
inner_key = f"inner-{api_str}"
if inner_key in impls:
inner_impls = impls[inner_key]
impl = await instantiate_provider(provider, deps, inner_impls, dist_registry, run_config, policy)
# Store appropriately
if api_str.startswith("inner-"):
if api_str not in impls:
impls[api_str] = {}
impls[api_str][provider.provider_id] = impl
else:
api = Api(api_str)
impls[api] = impl
# Update providers_impl.deps so subsequent providers can depend on this
providers_impl.deps[api] = impl
elif is_router:
# Router providers also need special handling
logger.info(f"Instantiating router {provider.provider_id} for {api_str}")
deps = {a: impls[a] for a in provider.spec.api_dependencies if a in impls}
for a in provider.spec.optional_api_dependencies:
if a in impls:
deps[a] = impls[a]
# Get inner impls if this is a router
inner_impls = {}
inner_key = f"inner-{api_str}"
if inner_key in impls:
inner_impls = impls[inner_key]
impl = await instantiate_provider(provider, deps, inner_impls, dist_registry, run_config, policy)
api = Api(api_str)
impls[api] = impl
# Update providers_impl.deps so subsequent providers can depend on this
providers_impl.deps[api] = impl
else:
# Regular providers - register through ProviderImpl
api = Api(api_str)
cache_key = f"{api.value}::{provider.provider_id}"
# Check if provider already exists (loaded from kvstore during initialization)
if cache_key in providers_impl.dynamic_providers:
logger.info(
f"Provider {provider.provider_id} for {api.value} already exists, using existing instance"
)
impl = providers_impl.dynamic_provider_impls.get(cache_key)
if impl is None:
# Provider exists but not instantiated, instantiate it
conn_info = providers_impl.dynamic_providers[cache_key]
impl = await providers_impl._instantiate_provider(conn_info)
providers_impl.dynamic_provider_impls[cache_key] = impl
else:
logger.info(f"Registering {provider.provider_id} for {api.value}")
await providers_impl.register_provider(
api=api.value,
provider_id=provider.provider_id,
provider_type=provider.spec.provider_type,
config=provider.config,
attributes=getattr(provider, "attributes", None),
)
# Get the instantiated impl from dynamic_provider_impls using composite key
impl = providers_impl.dynamic_provider_impls[cache_key]
logger.info(f"Successfully registered startup provider: {provider.provider_id}")
impls[api] = impl
# IMPORTANT: Update providers_impl.deps so subsequent providers can depend on this one
providers_impl.deps[api] = impl
except Exception as e:
logger.error(f"Failed to handle provider {provider.provider_id}: {e}")
raise
return impls
class Stack:
def __init__(self, run_config: StackRunConfig, provider_registry: ProviderRegistry | None = None):
self.run_config = run_config
@ -416,13 +605,24 @@ class Stack:
raise ValueError("storage.stores.metadata must be configured with a kv_* backend")
dist_registry, _ = await create_dist_registry(stores.metadata, self.run_config.image_name)
policy = self.run_config.server.auth.access_policy if self.run_config.server.auth else []
provider_registry = self.provider_registry or get_provider_registry(self.run_config)
internal_impls = {}
add_internal_implementations(internal_impls, self.run_config)
impls = await resolve_impls(
add_internal_implementations(
internal_impls,
self.run_config,
self.provider_registry or get_provider_registry(self.run_config),
provider_registry=provider_registry,
dist_registry=dist_registry,
policy=policy,
)
# Initialize the ProviderImpl so it has access to kvstore
await internal_impls[Api.providers].initialize()
# Register all providers from run.yaml through ProviderImpl
impls = await resolve_impls_via_provider_registration(
self.run_config,
provider_registry,
dist_registry,
policy,
internal_impls,

354
tests/integration/providers/test_dynamic_providers.py Normal file
View file

@ -0,0 +1,354 @@
# Copyright (c) Meta Platforms, Inc. and affiliates.
# All rights reserved.
#
# This source code is licensed under the terms described in the LICENSE file in
# the root directory of this source tree.
import pytest
from llama_stack_client import LlamaStackClient
from llama_stack import LlamaStackAsLibraryClient
pytestmark = pytest.mark.skip(reason="Requires client SDK update for new provider management APIs")
class TestDynamicProviderManagement:
"""Integration tests for dynamic provider registration, update, and unregistration."""
def test_register_and_unregister_inference_provider(
self, llama_stack_client: LlamaStackAsLibraryClient | LlamaStackClient
):
"""Test registering and unregistering an inference provider."""
provider_id = "test-dynamic-inference"
# Clean up if exists from previous test
try:
llama_stack_client.providers.unregister(provider_id)
except Exception:
pass
# Register a new inference provider (using Ollama since it's available in test setup)
response = llama_stack_client.providers.register(
provider_id=provider_id,
api="inference",
provider_type="remote::ollama",
config={
"url": "http://localhost:11434",
"api_token": "",
},
)
# Verify registration
assert response.provider.provider_id == provider_id
assert response.provider.api == "inference"
assert response.provider.provider_type == "remote::ollama"
assert response.provider.status in ["connected", "initializing"]
# Verify provider appears in list
providers = llama_stack_client.providers.list()
provider_ids = [p.provider_id for p in providers]
assert provider_id in provider_ids
# Verify we can retrieve it
provider = llama_stack_client.providers.retrieve(provider_id)
assert provider.provider_id == provider_id
# Unregister the provider
llama_stack_client.providers.unregister(provider_id)
# Verify it's no longer in the list
providers = llama_stack_client.providers.list()
provider_ids = [p.provider_id for p in providers]
assert provider_id not in provider_ids
def test_register_and_unregister_vector_store_provider(
self, llama_stack_client: LlamaStackAsLibraryClient | LlamaStackClient
):
"""Test registering and unregistering a vector store provider."""
provider_id = "test-dynamic-vector-store"
# Clean up if exists
try:
llama_stack_client.providers.unregister(provider_id)
except Exception:
pass
# Register a new vector_io provider (using Faiss inline)
response = llama_stack_client.providers.register(
provider_id=provider_id,
api="vector_io",
provider_type="inline::faiss",
config={
"embedding_dimension": 768,
"kvstore": {
"type": "sqlite",
"namespace": f"test_vector_store_{provider_id}",
},
},
)
# Verify registration
assert response.provider.provider_id == provider_id
assert response.provider.api == "vector_io"
assert response.provider.provider_type == "inline::faiss"
# Verify provider appears in list
providers = llama_stack_client.providers.list()
provider_ids = [p.provider_id for p in providers]
assert provider_id in provider_ids
# Unregister
llama_stack_client.providers.unregister(provider_id)
# Verify removal
providers = llama_stack_client.providers.list()
provider_ids = [p.provider_id for p in providers]
assert provider_id not in provider_ids
def test_update_provider_config(self, llama_stack_client: LlamaStackAsLibraryClient | LlamaStackClient):
"""Test updating a provider's configuration."""
provider_id = "test-update-config"
# Clean up if exists
try:
llama_stack_client.providers.unregister(provider_id)
except Exception:
pass
# Register provider
llama_stack_client.providers.register(
provider_id=provider_id,
api="inference",
provider_type="remote::ollama",
config={
"url": "http://localhost:11434",
"api_token": "old-token",
},
)
# Update the configuration
response = llama_stack_client.providers.update(
provider_id=provider_id,
config={
"url": "http://localhost:11434",
"api_token": "new-token",
},
)
# Verify update
assert response.provider.provider_id == provider_id
assert response.provider.config["api_token"] == "new-token"
# Clean up
llama_stack_client.providers.unregister(provider_id)
def test_update_provider_attributes(self, llama_stack_client: LlamaStackAsLibraryClient | LlamaStackClient):
"""Test updating a provider's ABAC attributes."""
provider_id = "test-update-attributes"
# Clean up if exists
try:
llama_stack_client.providers.unregister(provider_id)
except Exception:
pass
# Register provider with initial attributes
llama_stack_client.providers.register(
provider_id=provider_id,
api="inference",
provider_type="remote::ollama",
config={
"url": "http://localhost:11434",
},
attributes={"team": ["team-a"]},
)
# Update attributes
response = llama_stack_client.providers.update(
provider_id=provider_id,
attributes={"team": ["team-a", "team-b"], "environment": ["test"]},
)
# Verify attributes were updated
assert response.provider.attributes["team"] == ["team-a", "team-b"]
assert response.provider.attributes["environment"] == ["test"]
# Clean up
llama_stack_client.providers.unregister(provider_id)
def test_test_provider_connection(self, llama_stack_client: LlamaStackAsLibraryClient | LlamaStackClient):
"""Test the connection testing functionality."""
provider_id = "test-connection-check"
# Clean up if exists
try:
llama_stack_client.providers.unregister(provider_id)
except Exception:
pass
# Register provider
llama_stack_client.providers.register(
provider_id=provider_id,
api="inference",
provider_type="remote::ollama",
config={
"url": "http://localhost:11434",
},
)
# Test the connection
response = llama_stack_client.providers.test_connection(provider_id)
# Verify response structure
assert hasattr(response, "success")
assert hasattr(response, "health")
# Note: success may be True or False depending on whether Ollama is actually running
# but the test should at least verify the API works
# Clean up
llama_stack_client.providers.unregister(provider_id)
def test_register_duplicate_provider_fails(self, llama_stack_client: LlamaStackAsLibraryClient | LlamaStackClient):
"""Test that registering a duplicate provider ID fails."""
provider_id = "test-duplicate"
# Clean up if exists
try:
llama_stack_client.providers.unregister(provider_id)
except Exception:
pass
# Register first provider
llama_stack_client.providers.register(
provider_id=provider_id,
api="inference",
provider_type="remote::ollama",
config={"url": "http://localhost:11434"},
)
# Try to register with same ID - should fail
with pytest.raises(Exception) as exc_info:
llama_stack_client.providers.register(
provider_id=provider_id,
api="inference",
provider_type="remote::ollama",
config={"url": "http://localhost:11435"},
)
# Verify error message mentions the provider already exists
assert "already exists" in str(exc_info.value).lower() or "duplicate" in str(exc_info.value).lower()
# Clean up
llama_stack_client.providers.unregister(provider_id)
def test_unregister_nonexistent_provider_fails(
self, llama_stack_client: LlamaStackAsLibraryClient | LlamaStackClient
):
"""Test that unregistering a non-existent provider fails."""
with pytest.raises(Exception) as exc_info:
llama_stack_client.providers.unregister("nonexistent-provider-12345")
# Verify error message mentions provider not found
assert "not found" in str(exc_info.value).lower() or "does not exist" in str(exc_info.value).lower()
def test_update_nonexistent_provider_fails(self, llama_stack_client: LlamaStackAsLibraryClient | LlamaStackClient):
"""Test that updating a non-existent provider fails."""
with pytest.raises(Exception) as exc_info:
llama_stack_client.providers.update(
provider_id="nonexistent-provider-12345",
config={"url": "http://localhost:11434"},
)
# Verify error message mentions provider not found
assert "not found" in str(exc_info.value).lower() or "does not exist" in str(exc_info.value).lower()
def test_provider_lifecycle_with_inference(self, llama_stack_client: LlamaStackAsLibraryClient | LlamaStackClient):
"""Test full lifecycle: register, use for inference (if Ollama available), update, unregister."""
provider_id = "test-lifecycle-inference"
# Clean up if exists
try:
llama_stack_client.providers.unregister(provider_id)
except Exception:
pass
# Register provider
response = llama_stack_client.providers.register(
provider_id=provider_id,
api="inference",
provider_type="remote::ollama",
config={
"url": "http://localhost:11434",
},
)
assert response.provider.status in ["connected", "initializing"]
# Test connection
conn_test = llama_stack_client.providers.test_connection(provider_id)
assert hasattr(conn_test, "success")
# Update configuration
update_response = llama_stack_client.providers.update(
provider_id=provider_id,
config={
"url": "http://localhost:11434",
"api_token": "updated-token",
},
)
assert update_response.provider.config["api_token"] == "updated-token"
# Unregister
llama_stack_client.providers.unregister(provider_id)
# Verify it's gone
providers = llama_stack_client.providers.list()
provider_ids = [p.provider_id for p in providers]
assert provider_id not in provider_ids
def test_multiple_providers_same_type(self, llama_stack_client: LlamaStackAsLibraryClient | LlamaStackClient):
"""Test registering multiple providers of the same type with different IDs."""
provider_id_1 = "test-multi-ollama-1"
provider_id_2 = "test-multi-ollama-2"
# Clean up if exists
for pid in [provider_id_1, provider_id_2]:
try:
llama_stack_client.providers.unregister(pid)
except Exception:
pass
# Register first provider
response1 = llama_stack_client.providers.register(
provider_id=provider_id_1,
api="inference",
provider_type="remote::ollama",
config={"url": "http://localhost:11434"},
)
assert response1.provider.provider_id == provider_id_1
# Register second provider with same type but different ID
response2 = llama_stack_client.providers.register(
provider_id=provider_id_2,
api="inference",
provider_type="remote::ollama",
config={"url": "http://localhost:11434"},
)
assert response2.provider.provider_id == provider_id_2
# Verify both are in the list
providers = llama_stack_client.providers.list()
provider_ids = [p.provider_id for p in providers]
assert provider_id_1 in provider_ids
assert provider_id_2 in provider_ids
# Clean up both
llama_stack_client.providers.unregister(provider_id_1)
llama_stack_client.providers.unregister(provider_id_2)
# Verify both are gone
providers = llama_stack_client.providers.list()
provider_ids = [p.provider_id for p in providers]
assert provider_id_1 not in provider_ids
assert provider_id_2 not in provider_ids

411
tests/unit/core/test_dynamic_providers.py Normal file
View file

@ -0,0 +1,411 @@
# Copyright (c) Meta Platforms, Inc. and affiliates.
# All rights reserved.
#
# This source code is licensed under the terms described in the LICENSE file in
# the root directory of this source tree.
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
from llama_stack.apis.providers.connection import ProviderConnectionStatus, ProviderHealth
from llama_stack.core.datatypes import StackRunConfig
from llama_stack.core.providers import ProviderImpl, ProviderImplConfig
from llama_stack.core.storage.datatypes import KVStoreReference, ServerStoresConfig, SqliteKVStoreConfig, StorageConfig
from llama_stack.providers.datatypes import Api, HealthStatus
from llama_stack.providers.utils.kvstore.sqlite import SqliteKVStoreImpl
@pytest.fixture
async def kvstore(tmp_path):
"""Create a temporary kvstore for testing."""
db_path = tmp_path / "test_providers.db"
kvstore_config = SqliteKVStoreConfig(db_path=db_path.as_posix())
kvstore = SqliteKVStoreImpl(kvstore_config)
await kvstore.initialize()
yield kvstore
@pytest.fixture
async def provider_impl(kvstore, tmp_path):
"""Create a ProviderImpl instance with mocked dependencies."""
db_path = (tmp_path / "test_providers.db").as_posix()
# Create storage config with required structure
storage_config = StorageConfig(
backends={
"default": SqliteKVStoreConfig(db_path=db_path),
},
stores=ServerStoresConfig(
metadata=KVStoreReference(backend="default", namespace="test_metadata"),
),
)
# Create minimal run config with storage
run_config = StackRunConfig(
image_name="test",
apis=[],
providers={},
storage=storage_config,
)
# Mock provider registry
mock_provider_registry = MagicMock()
config = ProviderImplConfig(
run_config=run_config,
provider_registry=mock_provider_registry,
dist_registry=None,
policy=None,
)
impl = ProviderImpl(config, deps={})
# Manually set the kvstore instead of going through initialize
# This avoids the complex backend registration logic
impl.kvstore = kvstore
impl.provider_registry = mock_provider_registry
impl.dist_registry = None
impl.policy = []
yield impl
class TestDynamicProviderManagement:
"""Unit tests for dynamic provider registration, update, and unregistration."""
async def test_register_inference_provider(self, provider_impl):
"""Test registering a new inference provider."""
# Mock the provider instantiation
mock_provider_instance = AsyncMock()
mock_provider_instance.health = AsyncMock(return_value={"status": HealthStatus.OK})
with patch.object(provider_impl, "_instantiate_provider", return_value=mock_provider_instance):
# Register a mock inference provider
response = await provider_impl.register_provider(
api=Api.inference.value,
provider_id="test-inference-1",
provider_type="remote::openai",
config={"api_key": "test-key", "url": "https://api.openai.com/v1"},
attributes={"team": ["test-team"]},
)
# Verify response
assert response.provider.provider_id == "test-inference-1"
assert response.provider.api == Api.inference.value
assert response.provider.provider_type == "remote::openai"
assert response.provider.status == ProviderConnectionStatus.connected
assert response.provider.config["api_key"] == "test-key"
assert response.provider.attributes == {"team": ["test-team"]}
# Verify provider is stored (using composite key)
assert "inference::test-inference-1" in provider_impl.dynamic_providers
assert "inference::test-inference-1" in provider_impl.dynamic_provider_impls
async def test_register_vector_store_provider(self, provider_impl):
"""Test registering a new vector store provider."""
# Mock the provider instantiation
mock_provider_instance = AsyncMock()
mock_provider_instance.health = AsyncMock(return_value={"status": HealthStatus.OK})
with patch.object(provider_impl, "_instantiate_provider", return_value=mock_provider_instance):
# Register a mock vector_io provider
response = await provider_impl.register_provider(
api=Api.vector_io.value,
provider_id="test-vector-store-1",
provider_type="inline::faiss",
config={"dimension": 768, "index_path": "/tmp/faiss_index"},
)
# Verify response
assert response.provider.provider_id == "test-vector-store-1"
assert response.provider.api == Api.vector_io.value
assert response.provider.provider_type == "inline::faiss"
assert response.provider.status == ProviderConnectionStatus.connected
assert response.provider.config["dimension"] == 768
async def test_register_duplicate_provider_fails(self, provider_impl):
"""Test that registering a duplicate provider_id fails."""
mock_provider_instance = AsyncMock()
mock_provider_instance.health = AsyncMock(return_value={"status": HealthStatus.OK})
with patch.object(provider_impl, "_instantiate_provider", return_value=mock_provider_instance):
# Register first provider
await provider_impl.register_provider(
api=Api.inference.value,
provider_id="test-duplicate",
provider_type="remote::openai",
config={"api_key": "key1"},
)
# Try to register with same ID
with pytest.raises(ValueError, match="already exists"):
await provider_impl.register_provider(
api=Api.inference.value,
provider_id="test-duplicate",
provider_type="remote::openai",
config={"api_key": "key2"},
)
async def test_update_provider_config(self, provider_impl):
"""Test updating a provider's configuration."""
mock_provider_instance = AsyncMock()
mock_provider_instance.health = AsyncMock(return_value={"status": HealthStatus.OK})
with patch.object(provider_impl, "_instantiate_provider", return_value=mock_provider_instance):
# Register provider
await provider_impl.register_provider(
api=Api.inference.value,
provider_id="test-update",
provider_type="remote::openai",
config={"api_key": "old-key", "timeout": 30},
)
# Update configuration
response = await provider_impl.update_provider(
api=Api.inference.value,
provider_id="test-update",
config={"api_key": "new-key", "timeout": 60},
)
# Verify updated config
assert response.provider.provider_id == "test-update"
assert response.provider.config["api_key"] == "new-key"
assert response.provider.config["timeout"] == 60
assert response.provider.status == ProviderConnectionStatus.connected
async def test_update_provider_attributes(self, provider_impl):
"""Test updating a provider's attributes."""
mock_provider_instance = AsyncMock()
mock_provider_instance.health = AsyncMock(return_value={"status": HealthStatus.OK})
with patch.object(provider_impl, "_instantiate_provider", return_value=mock_provider_instance):
# Register provider with initial attributes
await provider_impl.register_provider(
api=Api.inference.value,
provider_id="test-attributes",
provider_type="remote::openai",
config={"api_key": "test-key"},
attributes={"team": ["team-a"]},
)
# Update attributes
response = await provider_impl.update_provider(
api=Api.inference.value,
provider_id="test-attributes",
attributes={"team": ["team-a", "team-b"], "environment": ["prod"]},
)
# Verify updated attributes
assert response.provider.attributes == {"team": ["team-a", "team-b"], "environment": ["prod"]}
async def test_update_nonexistent_provider_fails(self, provider_impl):
"""Test that updating a non-existent provider fails."""
with pytest.raises(ValueError, match="not found"):
await provider_impl.update_provider(
api=Api.inference.value,
provider_id="nonexistent",
config={"api_key": "new-key"},
)
async def test_unregister_provider(self, provider_impl):
"""Test unregistering a provider."""
mock_provider_instance = AsyncMock()
mock_provider_instance.health = AsyncMock(return_value={"status": HealthStatus.OK})
mock_provider_instance.shutdown = AsyncMock()
with patch.object(provider_impl, "_instantiate_provider", return_value=mock_provider_instance):
# Register provider
await provider_impl.register_provider(
api=Api.inference.value,
provider_id="test-unregister",
provider_type="remote::openai",
config={"api_key": "test-key"},
)
# Verify it exists
cache_key = f"{Api.inference.value}::test-unregister"
assert cache_key in provider_impl.dynamic_providers
# Unregister provider
await provider_impl.unregister_provider(api=Api.inference.value, provider_id="test-unregister")
# Verify it's removed
assert cache_key not in provider_impl.dynamic_providers
assert cache_key not in provider_impl.dynamic_provider_impls
# Verify shutdown was called
mock_provider_instance.shutdown.assert_called_once()
async def test_unregister_nonexistent_provider_fails(self, provider_impl):
"""Test that unregistering a non-existent provider fails."""
with pytest.raises(ValueError, match="not found"):
await provider_impl.unregister_provider(api=Api.inference.value, provider_id="nonexistent")
async def test_test_provider_connection_healthy(self, provider_impl):
"""Test testing a healthy provider connection."""
mock_provider_instance = AsyncMock()
mock_provider_instance.health = AsyncMock(return_value={"status": HealthStatus.OK, "message": "All good"})
with patch.object(provider_impl, "_instantiate_provider", return_value=mock_provider_instance):
# Register provider
await provider_impl.register_provider(
api=Api.inference.value,
provider_id="test-health",
provider_type="remote::openai",
config={"api_key": "test-key"},
)
# Test connection
response = await provider_impl.test_provider_connection(api=Api.inference.value, provider_id="test-health")
# Verify response
assert response.success is True
assert response.health["status"] == HealthStatus.OK
assert response.health["message"] == "All good"
assert response.error_message is None
async def test_test_provider_connection_unhealthy(self, provider_impl):
"""Test testing an unhealthy provider connection."""
mock_provider_instance = AsyncMock()
mock_provider_instance.health = AsyncMock(
return_value={"status": HealthStatus.ERROR, "message": "Connection failed"}
)
with patch.object(provider_impl, "_instantiate_provider", return_value=mock_provider_instance):
# Register provider
await provider_impl.register_provider(
api=Api.inference.value,
provider_id="test-unhealthy",
provider_type="remote::openai",
config={"api_key": "invalid-key"},
)
# Test connection
response = await provider_impl.test_provider_connection(
api=Api.inference.value, provider_id="test-unhealthy"
)
# Verify response shows unhealthy status
assert response.success is False
assert response.health["status"] == HealthStatus.ERROR
async def test_list_providers_includes_dynamic(self, provider_impl):
"""Test that list_providers includes dynamically registered providers."""
mock_provider_instance = AsyncMock()
mock_provider_instance.health = AsyncMock(return_value={"status": HealthStatus.OK})
with patch.object(provider_impl, "_instantiate_provider", return_value=mock_provider_instance):
# Register multiple providers
await provider_impl.register_provider(
api=Api.inference.value,
provider_id="dynamic-1",
provider_type="remote::openai",
config={"api_key": "key1"},
)
await provider_impl.register_provider(
api=Api.vector_io.value,
provider_id="dynamic-2",
provider_type="inline::faiss",
config={"dimension": 768},
)
# List all providers
response = await provider_impl.list_providers()
# Verify both dynamic providers are in the list
provider_ids = [p.provider_id for p in response.data]
assert "dynamic-1" in provider_ids
assert "dynamic-2" in provider_ids
async def test_inspect_provider(self, provider_impl):
"""Test inspecting a specific provider."""
mock_provider_instance = AsyncMock()
mock_provider_instance.health = AsyncMock(return_value={"status": HealthStatus.OK})
with patch.object(provider_impl, "_instantiate_provider", return_value=mock_provider_instance):
# Register provider
await provider_impl.register_provider(
api=Api.inference.value,
provider_id="test-inspect",
provider_type="remote::openai",
config={"api_key": "test-key", "model": "gpt-4"},
)
# Update the stored health info to reflect OK status
# (In reality, the health check happens during registration,
# but our mock may not have been properly called)
cache_key = f"{Api.inference.value}::test-inspect"
conn_info = provider_impl.dynamic_providers[cache_key]
conn_info.health = ProviderHealth.from_health_response({"status": HealthStatus.OK})
# Inspect provider
response = await provider_impl.inspect_provider(provider_id="test-inspect")
# Verify response
assert len(response.data) == 1
provider_info = response.data[0]
assert provider_info.provider_id == "test-inspect"
assert provider_info.api == Api.inference.value
assert provider_info.provider_type == "remote::openai"
assert provider_info.config["model"] == "gpt-4"
assert provider_info.health["status"] == HealthStatus.OK
async def test_provider_persistence(self, provider_impl, kvstore, tmp_path):
"""Test that providers persist across restarts."""
mock_provider_instance = AsyncMock()
mock_provider_instance.health = AsyncMock(return_value={"status": HealthStatus.OK})
with patch.object(provider_impl, "_instantiate_provider", return_value=mock_provider_instance):
# Register provider
await provider_impl.register_provider(
api=Api.inference.value,
provider_id="test-persist",
provider_type="remote::openai",
config={"api_key": "persist-key"},
)
# Create a new provider impl (simulating restart) - reuse the same kvstore
db_path = (tmp_path / "test_providers.db").as_posix()
storage_config = StorageConfig(
backends={
"default": SqliteKVStoreConfig(db_path=db_path),
},
stores=ServerStoresConfig(
metadata=KVStoreReference(backend="default", namespace="test_metadata"),
),
)
run_config = StackRunConfig(
image_name="test",
apis=[],
providers={},
storage=storage_config,
)
config = ProviderImplConfig(
run_config=run_config,
provider_registry=MagicMock(),
dist_registry=None,
policy=None,
)
new_impl = ProviderImpl(config, deps={})
# Manually set the kvstore (reusing the same one)
new_impl.kvstore = kvstore
new_impl.provider_registry = MagicMock()
new_impl.dist_registry = None
new_impl.policy = []
# Load providers from kvstore
with patch.object(new_impl, "_instantiate_provider", return_value=mock_provider_instance):
await new_impl._load_dynamic_providers()
# Verify the provider was loaded from kvstore
cache_key = f"{Api.inference.value}::test-persist"
assert cache_key in new_impl.dynamic_providers
assert new_impl.dynamic_providers[cache_key].config["api_key"] == "persist-key"