phoenix-oss/llama-stack-mirror
0
0
Fork 1
mirror of https://github.com/meta-llama/llama-stack.git synced 2025-07-29 15:23:51 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

migrate inference/completion

Browse source
This commit is contained in:
Xi Yan 2024-09-11 12:29:22 -07:00
parent 0c7c6b7e02
commit a7be58e4e1
2 changed files with 6 additions and 1 deletions
Download patch file Download diff file Expand all files Collapse all files

6
llama_toolchain/inference/api/api.py
View file

@ -170,7 +170,11 @@ class Inference(Protocol):
@webmethod(route="/inference/completion") @webmethod(route="/inference/completion")
async def completion( async def completion(
self, self,
request: CompletionRequest, model: str
content: InterleavedTextMedia,
sampling_params: Optional[SamplingParams] = SamplingParams(),
stream: Optional[bool] = False,
logprobs: Optional[LogProbConfig] = None,
) -> Union[CompletionResponse, CompletionResponseStreamChunk]: ... ) -> Union[CompletionResponse, CompletionResponseStreamChunk]: ...
@webmethod(route="/inference/chat_completion") @webmethod(route="/inference/chat_completion")

1
llama_toolchain/inference/meta_reference/inference.py
View file

@ -65,6 +65,7 @@ class MetaReferenceInferenceImpl(Inference):
) -> AsyncIterator[ ) -> AsyncIterator[
Union[ChatCompletionResponseStreamChunk, ChatCompletionResponse] Union[ChatCompletionResponseStreamChunk, ChatCompletionResponse]
]: ]:
# wrapper request to make it easier to pass around (internal only, not exposed to API)
request = ChatCompletionRequest( request = ChatCompletionRequest(
model=model, model=model,
messages=messages, messages=messages,