From 5ebcde3042e7209f052771460942cb3808f09e14 Mon Sep 17 00:00:00 2001 From: Derek Higgins Date: Fri, 19 Dec 2025 23:52:52 +0000 Subject: [PATCH] fix(scoring): remove broken dataset validation in score_batch methods (#4420) The Dataset model no longer has a dataset_schema attribute it was remove during a refactor (5287b437a) so this validation can no longer run. Changes: o basic scoring: removed validate_dataset_schema call and related imports o llm_as_judge scoring: removed validate_dataset_schema call and related imports o braintrust scoring: removed validate_dataset_schema call and related imports Validation is no longer needed at the dataset level since: o Dataset model changed from having dataset_schema to purpose/source fields o Scoring functions validate required fields when processing rows o Invalid data will fail naturally with clear error messages Fixes: #4419 Signed-off-by: Derek Higgins --- src/llama_stack/providers/inline/scoring/basic/scoring.py | 8 -------- .../providers/inline/scoring/braintrust/braintrust.py | 4 ---- .../providers/inline/scoring/llm_as_judge/scoring.py | 8 -------- 3 files changed, 20 deletions(-) diff --git a/src/llama_stack/providers/inline/scoring/basic/scoring.py b/src/llama_stack/providers/inline/scoring/basic/scoring.py index cf5cb79ba..55ce5ee8b 100644 --- a/src/llama_stack/providers/inline/scoring/basic/scoring.py +++ b/src/llama_stack/providers/inline/scoring/basic/scoring.py @@ -5,11 +5,6 @@ # the root directory of this source tree. from typing import Any -from llama_stack.core.datatypes import Api -from llama_stack.providers.utils.common.data_schema_validator import ( - get_valid_schemas, - validate_dataset_schema, -) from llama_stack_api import ( DatasetIO, Datasets, @@ -84,9 +79,6 @@ class BasicScoringImpl( scoring_functions: dict[str, ScoringFnParams | None] = None, save_results_dataset: bool = False, ) -> ScoreBatchResponse: - dataset_def = await self.datasets_api.get_dataset(dataset_id=dataset_id) - validate_dataset_schema(dataset_def.dataset_schema, get_valid_schemas(Api.scoring.value)) - all_rows = await self.datasetio_api.iterrows( dataset_id=dataset_id, limit=-1, diff --git a/src/llama_stack/providers/inline/scoring/braintrust/braintrust.py b/src/llama_stack/providers/inline/scoring/braintrust/braintrust.py index cfa35547b..0fa10f208 100644 --- a/src/llama_stack/providers/inline/scoring/braintrust/braintrust.py +++ b/src/llama_stack/providers/inline/scoring/braintrust/braintrust.py @@ -23,7 +23,6 @@ from llama_stack.core.datatypes import Api from llama_stack.core.request_headers import NeedsRequestProviderData from llama_stack.providers.utils.common.data_schema_validator import ( get_valid_schemas, - validate_dataset_schema, validate_row_schema, ) from llama_stack.providers.utils.scoring.aggregation_utils import aggregate_metrics @@ -165,9 +164,6 @@ class BraintrustScoringImpl( ) -> ScoreBatchResponse: await self.set_api_key() - dataset_def = await self.datasets_api.get_dataset(dataset_id=dataset_id) - validate_dataset_schema(dataset_def.dataset_schema, get_valid_schemas(Api.scoring.value)) - all_rows = await self.datasetio_api.iterrows( dataset_id=dataset_id, limit=-1, diff --git a/src/llama_stack/providers/inline/scoring/llm_as_judge/scoring.py b/src/llama_stack/providers/inline/scoring/llm_as_judge/scoring.py index 23e6ad705..afa22082a 100644 --- a/src/llama_stack/providers/inline/scoring/llm_as_judge/scoring.py +++ b/src/llama_stack/providers/inline/scoring/llm_as_judge/scoring.py @@ -5,11 +5,6 @@ # the root directory of this source tree. from typing import Any -from llama_stack.core.datatypes import Api -from llama_stack.providers.utils.common.data_schema_validator import ( - get_valid_schemas, - validate_dataset_schema, -) from llama_stack_api import ( DatasetIO, Datasets, @@ -73,9 +68,6 @@ class LlmAsJudgeScoringImpl( scoring_functions: dict[str, ScoringFnParams | None] = None, save_results_dataset: bool = False, ) -> ScoreBatchResponse: - dataset_def = await self.datasets_api.get_dataset(dataset_id=dataset_id) - validate_dataset_schema(dataset_def.dataset_schema, get_valid_schemas(Api.scoring.value)) - all_rows = await self.datasetio_api.iterrows( dataset_id=dataset_id, limit=-1,