phoenix-oss/llama-stack-mirror
0
0
Fork 1
mirror of https://github.com/meta-llama/llama-stack.git synced 2025-12-13 13:02:38 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

completion() for tgi

Browse source
This commit is contained in:
Dinesh Yeduguru 2024-10-23 12:06:25 -07:00
parent 21f2e9adf5
commit 5570a63248
4 changed files with 100 additions and 8 deletions
Download patch file Download diff file Expand all files Collapse all files

13
llama_stack/providers/utils/inference/openai_compat.py
View file

@ -95,13 +95,6 @@ async def process_completion_stream_response(
choice = chunk.choices[0]
finish_reason = choice.finish_reason
if finish_reason:
if finish_reason in ["stop", "eos", "eos_token"]:
stop_reason = StopReason.end_of_turn
elif finish_reason == "length":
stop_reason = StopReason.out_of_tokens
break
text = text_from_choice(choice)
if text == "<|eot_id|>":
stop_reason = StopReason.end_of_turn
@ -115,6 +108,12 @@ async def process_completion_stream_response(
delta=text,
stop_reason=stop_reason,
)
if finish_reason:
if finish_reason in ["stop", "eos", "eos_token"]:
stop_reason = StopReason.end_of_turn
elif finish_reason == "length":
stop_reason = StopReason.out_of_tokens
break
yield CompletionResponseStreamChunk(
delta="",