rag correctness scorer w/ custom dataset

llama_stack/apis/datasets/datasets.py

@@ -103,6 +103,7 @@ class HuggingfaceDatasetDef(BaseModel):
     )
     rename_columns_map: Optional[Dict[str, str]] = Field(
         description="A map of column names to rename to fit the schema of eval dataset for scoring",
+        default=None,
     )
     kwargs: Dict[str, Any] = Field(
         description="Any additional arguments to get Huggingface (e.g. split, trust_remote_code)",
@@ -119,6 +120,10 @@ class CustomDatasetDef(BaseModel):
     url: str = Field(
         description="The URL to the dataset",
     )
+    rename_columns_map: Optional[Dict[str, str]] = Field(
+        description="A map of column names to rename to fit the schema of eval dataset for scoring",
+        default=None,
+    )
 
 
 DatasetDef = Annotated[

llama_stack/apis/evals/client.py

@@ -136,6 +136,9 @@ async def run_main(host: str, port: int, eval_dataset_path: str = ""):
         dataset_def=CustomDatasetDef(
             identifier="rag-evals",
             url=data_url_from_file(eval_dataset_path),
+            rename_columns_map={
+                "query": "input_query",
+            },
         )
     )
     cprint(response, "cyan")
@@ -150,6 +153,9 @@ async def run_main(host: str, port: int, eval_dataset_path: str = ""):
         eval_scoring_config=EvaluateScoringConfig(
             scorer_config_list=[
                 EvaluateSingleScorerConfig(scorer_name="accuracy"),
+                EvaluateSingleScorerConfig(
+                    scorer_name="braintrust::answer-correctness"
+                ),
             ]
         ),
     )