phoenix-oss/llama-stack-mirror
0
0
Fork 1
mirror of https://github.com/meta-llama/llama-stack.git synced 2025-07-29 15:23:51 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

refactor

Browse source
This commit is contained in:
Xi Yan 2024-10-23 17:30:10 -07:00
parent 7c803cef86
commit 0ee82571a8
1 changed files with 3 additions and 9 deletions
Download patch file Download diff file Expand all files Collapse all files

12
llama_stack/providers/impls/meta_reference/datasetio/datasetio.py
View file

@ -61,18 +61,12 @@ class PandasDataframeDataset(BaseDataset):
else: else:
return self.df.iloc[idx].to_dict() return self.df.iloc[idx].to_dict()
def validate_dataset_schema(self) -> None: def _validate_dataset_schema(self) -> None:
if self.df is None: assert self.df is not None, "Dataset not loaded. Please call .load() first"
self.load()
assert self.df is not None, "Dataset loading failed. Please check logs."
# note that we will drop any columns in dataset that are not in the schema # note that we will drop any columns in dataset that are not in the schema
self.df = self.df[self.dataset_def.dataset_schema.keys()] self.df = self.df[self.dataset_def.dataset_schema.keys()]
# check all columns in dataset schema are present # check all columns in dataset schema are present
assert len(self.df.columns) == len(self.dataset_def.dataset_schema) assert len(self.df.columns) == len(self.dataset_def.dataset_schema)
# TODO: type checking against column types in dataset schema # TODO: type checking against column types in dataset schema
def load(self) -> None: def load(self) -> None:
@ -106,6 +100,7 @@ class PandasDataframeDataset(BaseDataset):
raise ValueError(f"Unsupported file type: {self.dataset_def.url}") raise ValueError(f"Unsupported file type: {self.dataset_def.url}")
self.df = df self.df = df
self._validate_dataset_schema()
class MetaReferenceDatasetIOImpl(DatasetIO, DatasetsProtocolPrivate): class MetaReferenceDatasetIOImpl(DatasetIO, DatasetsProtocolPrivate):
@ -123,7 +118,6 @@ class MetaReferenceDatasetIOImpl(DatasetIO, DatasetsProtocolPrivate):
dataset_def: DatasetDef, dataset_def: DatasetDef,
) -> None: ) -> None:
dataset_impl = PandasDataframeDataset(dataset_def) dataset_impl = PandasDataframeDataset(dataset_def)
dataset_impl.validate_dataset_schema()
self.dataset_infos[dataset_def.identifier] = DatasetInfo( self.dataset_infos[dataset_def.identifier] = DatasetInfo(
dataset_def=dataset_def, dataset_def=dataset_def,
dataset_impl=dataset_impl, dataset_impl=dataset_impl,