phoenix-oss/llama-stack-mirror
0
0
Fork 1
mirror of https://github.com/meta-llama/llama-stack.git synced 2025-10-05 20:27:35 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

add safety adapters, configuration handling, server + clients

Browse source
This commit is contained in:
Ashwin Bharambe 2024-08-03 19:46:59 -07:00
parent 9dafa6ad94
commit fe582a739d
13 changed files with 286 additions and 67 deletions
Download patch file Download diff file Expand all files Collapse all files

9
llama_toolchain/distribution/registry.py
View file

@ -8,6 +8,7 @@ from functools import lru_cache
from typing import List, Optional
from llama_toolchain.inference.adapters import available_inference_adapters
from llama_toolchain.safety.adapters import available_safety_adapters
from .datatypes import ApiSurface, Distribution, PassthroughApiAdapter
@ -45,6 +46,7 @@ COMMON_DEPENDENCIES = [
@lru_cache()
def available_distributions() -> List[Distribution]:
inference_adapters_by_id = {a.adapter_id: a for a in available_inference_adapters()}
safety_adapters_by_id = {a.adapter_id: a for a in available_safety_adapters()}
return [
Distribution(
@ -53,6 +55,7 @@ def available_distributions() -> List[Distribution]:
additional_pip_packages=COMMON_DEPENDENCIES,
adapters={
ApiSurface.inference: inference_adapters_by_id["meta-reference"],
ApiSurface.safety: safety_adapters_by_id["meta-reference"],
},
),
Distribution(
@ -78,6 +81,11 @@ def available_distributions() -> List[Distribution]:
adapter_id="inference-passthrough",
base_url="http://localhost:5001",
),
ApiSurface.safety: PassthroughApiAdapter(
api_surface=ApiSurface.safety,
adapter_id="safety-passthrough",
base_url="http://localhost:5001",
),
},
),
Distribution(
@ -86,6 +94,7 @@ def available_distributions() -> List[Distribution]:
additional_pip_packages=COMMON_DEPENDENCIES,
adapters={
ApiSurface.inference: inference_adapters_by_id["meta-ollama"],
ApiSurface.safety: safety_adapters_by_id["meta-reference"],
},
),
]

12
llama_toolchain/distribution/server.py
View file

@ -136,7 +136,7 @@ async def passthrough(
def handle_sigint(*args, **kwargs):
print("SIGINT or CTRL-C detected. Exiting gracefully", args)
print("SIGINT or CTRL-C detected. Exiting gracefully...")
loop = asyncio.get_event_loop()
for task in asyncio.all_tasks(loop):
task.cancel()
@ -198,8 +198,16 @@ def create_dynamic_typed_route(func: Any):
async def endpoint(request: request_model):
try:
return func(request)
return (
await func(request)
if asyncio.iscoroutinefunction(func)
else func(request)
)
except Exception as e:
print(e)
import traceback
traceback.print_exc()
raise translate_exception(e) from e
return endpoint