phoenix-oss/llama-stack-mirror
0
0
Fork 1
mirror of https://github.com/meta-llama/llama-stack.git synced 2025-10-04 12:07:34 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

split batch_inference from inference

Browse source
This commit is contained in:
Ashwin Bharambe 2024-08-26 13:17:59 -07:00
parent 986a865e62
commit dc433f6c90
5 changed files with 75 additions and 12 deletions
Download patch file Download diff file Expand all files Collapse all files

12
llama_toolchain/inference/api/api.py
View file

@ -185,15 +185,3 @@ class Inference(Protocol):
model: str,
contents: List[InterleavedTextMedia],
) -> EmbeddingsResponse: ...
@webmethod(route="/inference/batch_completion")
async def batch_completion(
self,
request: BatchCompletionRequest,
) -> BatchCompletionResponse: ...
@webmethod(route="/inference/batch_chat_completion")
async def batch_chat_completion(
self,
request: BatchChatCompletionRequest,
) -> BatchChatCompletionResponse: ...