mirror of
https://github.com/meta-llama/llama-stack.git
synced 2025-12-27 12:48:04 +00:00
feat: implement provider updating
add `v1/providers/` which uses PUT to allow users to change their provider configuration this is a follow up to #1429 and related to #1359 a user can call something like: `llama_stack_client.providers.update(api="inference", provider_id="ollama", provider_type="remote::ollama", config={'url': 'http:/localhost:12345'})` or `llama-stack-client providers update inference ollama remote::ollama "{'url': 'http://localhost:12345'}"` this API works by adding a `RequestMiddleware` to the server which checks requests, and if the user is using PUT /v1/providers, the routes are re-registered with the re-initialized provider configurations/methods for the client, `self.impls` is updated to hold the proper methods+configurations this depends on a client PR, the CI will fail until then but succeeded locally Signed-off-by: Charlie Doern <cdoern@redhat.com>
This commit is contained in:
parent
d165000bbc
commit
436f8ade9e
8 changed files with 449 additions and 56 deletions
|
|
@ -25,6 +25,7 @@ from llama_stack_client import (
|
|||
AsyncStream,
|
||||
LlamaStackClient,
|
||||
)
|
||||
from llama_stack_client.types import provider_info
|
||||
from pydantic import BaseModel, TypeAdapter
|
||||
from rich.console import Console
|
||||
from termcolor import cprint
|
||||
|
|
@ -293,6 +294,22 @@ class AsyncLlamaStackAsLibraryClient(AsyncLlamaStackClient):
|
|||
cast_to=cast_to,
|
||||
options=options,
|
||||
)
|
||||
# Check if response is of a certain type
|
||||
# this indicates we have done a provider update
|
||||
if (
|
||||
isinstance(response, provider_info.ProviderInfo)
|
||||
and hasattr(response, "config")
|
||||
and options.method.lower() == "put"
|
||||
):
|
||||
# patch in the new provider config
|
||||
for api, providers in self.config.providers.items():
|
||||
if api != response.api:
|
||||
continue
|
||||
for prov in providers:
|
||||
if prov.provider_id == response.provider_id:
|
||||
prov.config = response.config
|
||||
break
|
||||
await self.initialize()
|
||||
return response
|
||||
|
||||
async def _call_non_streaming(
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue