phoenix-oss/llama-stack-mirror
0
0
Fork 1
mirror of https://github.com/meta-llama/llama-stack.git synced 2025-12-08 19:10:56 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

tasks registry

Browse source
This commit is contained in:
Xi Yan 2024-10-07 15:57:39 -07:00
parent 041634192a
commit 4764762dd4
9 changed files with 74 additions and 35 deletions
Download patch file Download diff file Expand all files Collapse all files

20
llama_stack/apis/evals/client.py
View file

@ -43,20 +43,20 @@ async def run_main(host: str, port: int):
client = EvaluationClient(f"http://{host}:{port}")
# CustomDataset
# response = await client.run_evals(
# "Llama3.1-8B-Instruct",
# "mmlu-simple-eval-en",
# "mmlu",
# )
# cprint(f"evaluate response={response}", "green")
# Eleuther Eval
response = await client.run_evals(
"Llama3.1-8B-Instruct",
"PLACEHOLDER_DATASET_NAME",
"mmlu-simple-eval-en",
"mmlu",
)
cprint(response.metrics["metrics_table"], "red")
cprint(f"evaluate response={response}", "green")
# Eleuther Eval
# response = await client.run_evals(
# "Llama3.1-8B-Instruct",
# "PLACEHOLDER_DATASET_NAME",
# "mmlu",
# )
# cprint(response.metrics["metrics_table"], "red")
def main(host: str, port: int):