From a062723d031fa715dd2f1e98ef299e1845ad0a41 Mon Sep 17 00:00:00 2001 From: Charlie Doern Date: Thu, 13 Mar 2025 18:07:21 -0400 Subject: [PATCH] feat: add provider API for listing and inspecting provider info (#1429) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit # What does this PR do? currently the `inspect` API for providers is really a `list` API. Create a new `providers` API which has a GET `providers/{provider_id}` inspect API which returns "user friendly" configuration to the end user. Also add a GET `/providers` endpoint which returns the list of providers as `inspect/providers` does today. This API follows CRUD and is more intuitive/RESTful. This work is part of the RFC at https://github.com/meta-llama/llama-stack/pull/1359 sensitive fields are redacted using `redact_sensetive_fields` on the server side before returning a response: Screenshot 2025-03-13 at 4 40 21 PM ## Test Plan using https://github.com/meta-llama/llama-stack-client-python/pull/181 a user is able to to run the following: `llama stack build --template ollama --image-type venv` `llama stack run --image-type venv ~/.llama/distributions/ollama/ollama-run.yaml` `llama-stack-client providers inspect ollama` Screenshot 2025-03-13 at 4 39 35 PM also, was able to run the new test_list integration test locally with ollama: Screenshot 2025-03-13 at 11 03 40 AM Signed-off-by: Charlie Doern --- docs/_static/llama-stack-spec.html | 76 ++++++++++++++++++- docs/_static/llama-stack-spec.yaml | 51 +++++++++++++ llama_stack/apis/datatypes.py | 1 + llama_stack/apis/inspect/inspect.py | 18 ++--- llama_stack/apis/providers/__init__.py | 7 ++ llama_stack/apis/providers/providers.py | 40 ++++++++++ llama_stack/distribution/configure.py | 2 +- llama_stack/distribution/distribution.py | 2 +- llama_stack/distribution/providers.py | 59 ++++++++++++++ llama_stack/distribution/resolver.py | 21 +++++ llama_stack/distribution/server/server.py | 1 + llama_stack/distribution/stack.py | 2 + pyproject.toml | 1 + tests/integration/providers/__init__.py | 5 ++ tests/integration/providers/test_providers.py | 17 +++++ 15 files changed, 291 insertions(+), 12 deletions(-) create mode 100644 llama_stack/apis/providers/__init__.py create mode 100644 llama_stack/apis/providers/providers.py create mode 100644 llama_stack/distribution/providers.py create mode 100644 tests/integration/providers/__init__.py create mode 100644 tests/integration/providers/test_providers.py diff --git a/docs/_static/llama-stack-spec.html b/docs/_static/llama-stack-spec.html index 22fa781ac..e62f66bd6 100644 --- a/docs/_static/llama-stack-spec.html +++ b/docs/_static/llama-stack-spec.html @@ -2642,7 +2642,81 @@ } } }, - "/v1/inspect/providers": { + "/v1/providers": { + "get": { + "responses": { + "200": { + "description": "OK", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ListProvidersResponse" + } + } + } + }, + "400": { + "$ref": "#/components/responses/BadRequest400" + }, + "429": { + "$ref": "#/components/responses/TooManyRequests429" + }, + "500": { + "$ref": "#/components/responses/InternalServerError500" + }, + "default": { + "$ref": "#/components/responses/DefaultError" + } + }, + "tags": [ + "Providers" + ], + "description": "", + "parameters": [] + } + }, + "/v1/providers/{provider_id}": { + "get": { + "responses": { + "200": { + "description": "OK", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/GetProviderResponse" + } + } + } + }, + "400": { + "$ref": "#/components/responses/BadRequest400" + }, + "429": { + "$ref": "#/components/responses/TooManyRequests429" + }, + "500": { + "$ref": "#/components/responses/InternalServerError500" + }, + "default": { + "$ref": "#/components/responses/DefaultError" + } + }, + "tags": [ + "Providers" + ], + "description": "", + "parameters": [ + { + "name": "provider_id", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + } + ] + }, + "/v1/inspect/providers": { "get": { "responses": { "200": { diff --git a/docs/_static/llama-stack-spec.yaml b/docs/_static/llama-stack-spec.yaml index 1f01351e9..cb31848ee 100644 --- a/docs/_static/llama-stack-spec.yaml +++ b/docs/_static/llama-stack-spec.yaml @@ -1782,6 +1782,57 @@ paths: schema: $ref: '#/components/schemas/RegisterModelRequest' required: true + /v1/providers: + get: + responses: + '200': + description: OK + content: + application/json: + schema: + $ref: '#/components/schemas/ListProvidersResponse' + '400': + $ref: '#/components/responses/BadRequest400' + '429': + $ref: >- + #/components/responses/TooManyRequests429 + '500': + $ref: >- + #/components/responses/InternalServerError500 + default: + $ref: '#/components/responses/DefaultError' + tags: + - Providers + description: '' + parameters: [] + /v1/providers/{provider_id}: + get: + responses: + '200': + description: OK + content: + application/json: + schema: + $ref: '#/components/schemas/GetProviderResponse' + '400': + $ref: '#/components/responses/BadRequest400' + '429': + $ref: >- + #/components/responses/TooManyRequests429 + '500': + $ref: >- + #/components/responses/InternalServerError500 + default: + $ref: '#/components/responses/DefaultError' + tags: + - Providers + description: '' + parameters: + - name: provider_id + in: path + required: true + schema: + type: string /v1/inspect/providers: get: responses: diff --git a/llama_stack/apis/datatypes.py b/llama_stack/apis/datatypes.py index 842a2b63d..f644e5137 100644 --- a/llama_stack/apis/datatypes.py +++ b/llama_stack/apis/datatypes.py @@ -14,6 +14,7 @@ from llama_stack.schema_utils import json_schema_type @json_schema_type class Api(Enum): + providers = "providers" inference = "inference" safety = "safety" agents = "agents" diff --git a/llama_stack/apis/inspect/inspect.py b/llama_stack/apis/inspect/inspect.py index 4a647a2d9..25937bb61 100644 --- a/llama_stack/apis/inspect/inspect.py +++ b/llama_stack/apis/inspect/inspect.py @@ -11,13 +11,6 @@ from pydantic import BaseModel from llama_stack.schema_utils import json_schema_type, webmethod -@json_schema_type -class ProviderInfo(BaseModel): - api: str - provider_id: str - provider_type: str - - @json_schema_type class RouteInfo(BaseModel): route: str @@ -32,14 +25,21 @@ class HealthInfo(BaseModel): @json_schema_type -class VersionInfo(BaseModel): - version: str +class ProviderInfo(BaseModel): + api: str + provider_id: str + provider_type: str class ListProvidersResponse(BaseModel): data: List[ProviderInfo] +@json_schema_type +class VersionInfo(BaseModel): + version: str + + class ListRoutesResponse(BaseModel): data: List[RouteInfo] diff --git a/llama_stack/apis/providers/__init__.py b/llama_stack/apis/providers/__init__.py new file mode 100644 index 000000000..b554a5d23 --- /dev/null +++ b/llama_stack/apis/providers/__init__.py @@ -0,0 +1,7 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the terms described in the LICENSE file in +# the root directory of this source tree. + +from .providers import * # noqa: F401 F403 diff --git a/llama_stack/apis/providers/providers.py b/llama_stack/apis/providers/providers.py new file mode 100644 index 000000000..fd37bd500 --- /dev/null +++ b/llama_stack/apis/providers/providers.py @@ -0,0 +1,40 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the terms described in the LICENSE file in +# the root directory of this source tree. + +from typing import List, Protocol, runtime_checkable + +from pydantic import BaseModel + +from llama_stack.distribution.datatypes import Provider +from llama_stack.schema_utils import json_schema_type, webmethod + + +@json_schema_type +class ProviderInfo(BaseModel): + api: str + provider_id: str + provider_type: str + + +class GetProviderResponse(BaseModel): + data: Provider | None + + +class ListProvidersResponse(BaseModel): + data: List[ProviderInfo] + + +@runtime_checkable +class Providers(Protocol): + """ + Providers API for inspecting, listing, and modifying providers and their configurations. + """ + + @webmethod(route="/providers", method="GET") + async def list_providers(self) -> ListProvidersResponse: ... + + @webmethod(route="/providers/{provider_id}", method="GET") + async def inspect_provider(self, provider_id: str) -> GetProviderResponse: ... diff --git a/llama_stack/distribution/configure.py b/llama_stack/distribution/configure.py index 715bb5db4..2a3bf7053 100644 --- a/llama_stack/distribution/configure.py +++ b/llama_stack/distribution/configure.py @@ -62,7 +62,7 @@ def configure_api_providers(config: StackRunConfig, build_spec: DistributionSpec if config.apis: apis_to_serve = config.apis else: - apis_to_serve = [a.value for a in Api if a not in (Api.telemetry, Api.inspect)] + apis_to_serve = [a.value for a in Api if a not in (Api.telemetry, Api.inspect, Api.providers)] for api_str in apis_to_serve: api = Api(api_str) diff --git a/llama_stack/distribution/distribution.py b/llama_stack/distribution/distribution.py index 308081415..ddb727663 100644 --- a/llama_stack/distribution/distribution.py +++ b/llama_stack/distribution/distribution.py @@ -56,7 +56,7 @@ def builtin_automatically_routed_apis() -> List[AutoRoutedApiInfo]: def providable_apis() -> List[Api]: routing_table_apis = {x.routing_table_api for x in builtin_automatically_routed_apis()} - return [api for api in Api if api not in routing_table_apis and api != Api.inspect] + return [api for api in Api if api not in routing_table_apis and api != Api.inspect and api != Api.providers] def get_provider_registry() -> Dict[Api, Dict[str, ProviderSpec]]: diff --git a/llama_stack/distribution/providers.py b/llama_stack/distribution/providers.py new file mode 100644 index 000000000..219384900 --- /dev/null +++ b/llama_stack/distribution/providers.py @@ -0,0 +1,59 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the terms described in the LICENSE file in +# the root directory of this source tree. + +from pydantic import BaseModel + +from llama_stack.apis.providers import GetProviderResponse, ListProvidersResponse, ProviderInfo, Providers + +from .datatypes import StackRunConfig +from .stack import redact_sensitive_fields + + +class ProviderImplConfig(BaseModel): + run_config: StackRunConfig + + +async def get_provider_impl(config, deps): + impl = ProviderImpl(config, deps) + await impl.initialize() + return impl + + +class ProviderImpl(Providers): + def __init__(self, config, deps): + self.config = config + self.deps = deps + + async def initialize(self) -> None: + pass + + async def list_providers(self) -> ListProvidersResponse: + run_config = self.config.run_config + ret = [] + for api, providers in run_config.providers.items(): + ret.extend( + [ + ProviderInfo( + api=api, + provider_id=p.provider_id, + provider_type=p.provider_type, + ) + for p in providers + ] + ) + + return ListProvidersResponse(data=ret) + + async def inspect_provider(self, provider_id: str) -> GetProviderResponse: + run_config = self.config.run_config + safe_config = StackRunConfig(**redact_sensitive_fields(run_config.model_dump())) + ret = None + for _, providers in safe_config.providers.items(): + for p in providers: + if p.provider_id == provider_id: + ret = p + + return GetProviderResponse(data=ret) diff --git a/llama_stack/distribution/resolver.py b/llama_stack/distribution/resolver.py index ab075f399..e9e406699 100644 --- a/llama_stack/distribution/resolver.py +++ b/llama_stack/distribution/resolver.py @@ -16,6 +16,7 @@ from llama_stack.apis.inference import Inference from llama_stack.apis.inspect import Inspect from llama_stack.apis.models import Models from llama_stack.apis.post_training import PostTraining +from llama_stack.apis.providers import Providers as ProvidersAPI from llama_stack.apis.safety import Safety from llama_stack.apis.scoring import Scoring from llama_stack.apis.scoring_functions import ScoringFunctions @@ -59,6 +60,7 @@ class InvalidProviderError(Exception): def api_protocol_map() -> Dict[Api, Any]: return { + Api.providers: ProvidersAPI, Api.agents: Agents, Api.inference: Inference, Api.inspect: Inspect, @@ -247,6 +249,25 @@ def sort_providers_by_deps( ) ) + sorted_providers.append( + ( + "providers", + ProviderWithSpec( + provider_id="__builtin__", + provider_type="__builtin__", + config={"run_config": run_config.model_dump()}, + spec=InlineProviderSpec( + api=Api.providers, + provider_type="__builtin__", + config_class="llama_stack.distribution.providers.ProviderImplConfig", + module="llama_stack.distribution.providers", + api_dependencies=apis, + deps__=[x.value for x in apis], + ), + ), + ) + ) + logger.debug(f"Resolved {len(sorted_providers)} providers") for api_str, provider in sorted_providers: logger.debug(f" {api_str} => {provider.provider_id}") diff --git a/llama_stack/distribution/server/server.py b/llama_stack/distribution/server/server.py index 7ca009b13..8f9500ae9 100644 --- a/llama_stack/distribution/server/server.py +++ b/llama_stack/distribution/server/server.py @@ -368,6 +368,7 @@ def main(): apis_to_serve.add(inf.routing_table_api.value) apis_to_serve.add("inspect") + apis_to_serve.add("providers") for api_str in apis_to_serve: api = Api(api_str) diff --git a/llama_stack/distribution/stack.py b/llama_stack/distribution/stack.py index 2b974739a..9c9289a77 100644 --- a/llama_stack/distribution/stack.py +++ b/llama_stack/distribution/stack.py @@ -23,6 +23,7 @@ from llama_stack.apis.inference import Inference from llama_stack.apis.inspect import Inspect from llama_stack.apis.models import Models from llama_stack.apis.post_training import PostTraining +from llama_stack.apis.providers import Providers from llama_stack.apis.safety import Safety from llama_stack.apis.scoring import Scoring from llama_stack.apis.scoring_functions import ScoringFunctions @@ -44,6 +45,7 @@ logger = get_logger(name=__name__, category="core") class LlamaStack( + Providers, VectorDBs, Inference, BatchInference, diff --git a/pyproject.toml b/pyproject.toml index 055fa7a55..aaea4f7c4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -170,6 +170,7 @@ exclude = [ "^llama_stack/apis/inspect/inspect\\.py$", "^llama_stack/apis/models/models\\.py$", "^llama_stack/apis/post_training/post_training\\.py$", + "^llama_stack/apis/providers/providers\\.py$", "^llama_stack/apis/resource\\.py$", "^llama_stack/apis/safety/safety\\.py$", "^llama_stack/apis/scoring/scoring\\.py$", diff --git a/tests/integration/providers/__init__.py b/tests/integration/providers/__init__.py new file mode 100644 index 000000000..756f351d8 --- /dev/null +++ b/tests/integration/providers/__init__.py @@ -0,0 +1,5 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the terms described in the LICENSE file in +# the root directory of this source tree. diff --git a/tests/integration/providers/test_providers.py b/tests/integration/providers/test_providers.py new file mode 100644 index 000000000..174d01b5c --- /dev/null +++ b/tests/integration/providers/test_providers.py @@ -0,0 +1,17 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the terms described in the LICENSE file in +# the root directory of this source tree. + +import pytest +from llama_stack_client import LlamaStackClient + +from llama_stack import LlamaStackAsLibraryClient + + +class TestProviders: + @pytest.mark.asyncio + def test_list(self, llama_stack_client: LlamaStackAsLibraryClient | LlamaStackClient): + provider_list = llama_stack_client.providers.list() + assert provider_list is not None