phoenix-oss/llama-stack-mirror
0
0
Fork 1
mirror of https://github.com/meta-llama/llama-stack.git synced 2025-12-09 19:29:18 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

rag correctness scorer w/ custom dataset

Browse source
This commit is contained in:
Xi Yan 2024-10-15 00:42:03 -07:00
parent ec6c63ba57
commit 9cc0a54f0b
4 changed files with 18 additions and 4 deletions
Download patch file Download diff file Expand all files Collapse all files

6
llama_stack/apis/evals/client.py
View file

@ -136,6 +136,9 @@ async def run_main(host: str, port: int, eval_dataset_path: str = ""):
dataset_def=CustomDatasetDef(
identifier="rag-evals",
url=data_url_from_file(eval_dataset_path),
rename_columns_map={
"query": "input_query",
},
)
)
cprint(response, "cyan")
@ -150,6 +153,9 @@ async def run_main(host: str, port: int, eval_dataset_path: str = ""):
eval_scoring_config=EvaluateScoringConfig(
scorer_config_list=[
EvaluateSingleScorerConfig(scorer_name="accuracy"),
EvaluateSingleScorerConfig(
scorer_name="braintrust::answer-correctness"
),
]
),
)