phoenix-oss/llama-stack-mirror
0
0
Fork 1
mirror of https://github.com/meta-llama/llama-stack.git synced 2025-07-30 07:39:38 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

2 models routing client

Browse source
This commit is contained in:
Xi Yan 2024-09-19 21:44:12 -07:00
parent 4b083eec03
commit f6146f8e58
1 changed files with 3 additions and 2 deletions
Download patch file Download diff file Expand all files Collapse all files

5
llama_stack/apis/inference/client.py
View file

@ -89,10 +89,11 @@ async def run_main(host: str, port: int, stream: bool):
message = UserMessage( message = UserMessage(
content="hello world, write me a 2 sentence poem about the moon" content="hello world, write me a 2 sentence poem about the moon"
) )
cprint(f"User>{message.content}", "green") cprint(f"User>{message.content}", "green")
iterator = client.chat_completion( iterator = client.chat_completion(
ChatCompletionRequest( ChatCompletionRequest(
model="Meta-Llama3.1-8B-Instruct", model="Meta-Llama3.1-8B",
messages=[message], messages=[message],
stream=stream, stream=stream,
) )
@ -103,7 +104,7 @@ async def run_main(host: str, port: int, stream: bool):
cprint(f"User>{message.content}", "green") cprint(f"User>{message.content}", "green")
iterator = client.chat_completion( iterator = client.chat_completion(
ChatCompletionRequest( ChatCompletionRequest(
model="Meta-Llama3.1-8B", model="Meta-Llama3.1-8B-Instruct",
messages=[message], messages=[message],
stream=stream, stream=stream,
) )