From 0cec86453bdfdee10760946f63f54426f6306271 Mon Sep 17 00:00:00 2001 From: Xi Yan Date: Wed, 23 Oct 2024 15:27:22 -0700 Subject: [PATCH] Fix issue w/ routing_table api getting added when router api is not specified (#298) * fix issue w/ enforcing api * cleanup * inference only yaml --- llama_stack/distribution/server/server.py | 3 +++ tests/examples/inference-run.yaml | 14 ++++++++++++++ 2 files changed, 17 insertions(+) create mode 100644 tests/examples/inference-run.yaml diff --git a/llama_stack/distribution/server/server.py b/llama_stack/distribution/server/server.py index 84c369364..185c89e7e 100644 --- a/llama_stack/distribution/server/server.py +++ b/llama_stack/distribution/server/server.py @@ -288,6 +288,9 @@ def main( apis_to_serve = set(impls.keys()) for inf in builtin_automatically_routed_apis(): + # if we do not serve the corresponding router API, we should not serve the routing table API + if inf.router_api.value not in apis_to_serve: + continue apis_to_serve.add(inf.routing_table_api.value) apis_to_serve.add("inspect") diff --git a/tests/examples/inference-run.yaml b/tests/examples/inference-run.yaml new file mode 100644 index 000000000..87ab5146b --- /dev/null +++ b/tests/examples/inference-run.yaml @@ -0,0 +1,14 @@ +version: '2' +built_at: '2024-10-08T17:40:45.325529' +image_name: local +docker_image: null +conda_env: local +apis: +- models +- inference +providers: + inference: + - provider_id: tgi0 + provider_type: remote::tgi + config: + url: http://127.0.0.1:5009