phoenix-oss/llama-stack-mirror
0
0
Fork 1
mirror of https://github.com/meta-llama/llama-stack.git synced 2025-10-04 20:14:13 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

skeleton unified routing table, api routers

Browse source
This commit is contained in:
Xi Yan 2024-09-21 13:44:33 -07:00
parent 2dc14cba2c
commit 85d927adde
11 changed files with 210 additions and 231 deletions
Download patch file Download diff file Expand all files Collapse all files

12
llama_stack/apis/inference/client.py
View file

@ -10,10 +10,10 @@ from typing import Any, AsyncGenerator
import fire
import httpx
from pydantic import BaseModel
from termcolor import cprint
from llama_stack.distribution.datatypes import RemoteProviderConfig
from pydantic import BaseModel
from termcolor import cprint
from .event_logger import EventLogger
@ -104,11 +104,9 @@ async def run_main(host: str, port: int, stream: bool):
)
cprint(f"User>{message.content}", "green")
iterator = client.chat_completion(
ChatCompletionRequest(
model="Meta-Llama3.1-8B-Instruct",
messages=[message],
stream=stream,
)
model="Meta-Llama3.1-8B-Instruct",
messages=[message],
stream=stream,
)
async for log in EventLogger().log(iterator):
log.print()