phoenix-oss/llama-stack-mirror
0
0
Fork 1
mirror of https://github.com/meta-llama/llama-stack.git synced 2025-10-04 04:04:14 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

add EventLogger for inference

Browse source
This commit is contained in:
Ashwin Bharambe 2024-07-22 15:11:34 -07:00
parent 7574ffb25f
commit bbfd8a587e
2 changed files with 38 additions and 3 deletions
Download patch file Download diff file Expand all files Collapse all files

8
llama_toolchain/inference/client.py
View file

@ -13,6 +13,7 @@ from .api import (
Inference,
UserMessage,
)
from .event_logger import EventLogger
class InferenceClient(Inference):
@ -56,14 +57,15 @@ async def run_main(host: str, port: int):
messages=[message],
stream=True,
)
async for event in client.chat_completion(
iterator = client.chat_completion(
ChatCompletionRequest(
model=InstructModel.llama3_8b_chat,
messages=[message],
stream=True,
)
):
print(event)
)
async for log in EventLogger().log(iterator):
log.print()
def main(host: str, port: int):