openml/datasets/dataset.py
def _cache_compressed_file_from_file(
        self, data_file: str
    ) -> Tuple[Union[pd.DataFrame, scipy.sparse.csr_matrix], List[bool], List[str]]:
        """ Store data from the local file in compressed format.

        If a local parquet file is present it will be used instead of the arff file.
        Sets cache_format to 'pickle' if data is sparse.
        """
        (
            data_pickle_file,
            data_feather_file,
            feather_attribute_file,
        ) = self._compressed_cache_file_paths(data_file)

        if data_file.endswith(".arff"):
            data, categorical, attribute_names = self._parse_data_from_arff(data_file)
        elif data_file.endswith(".pq"):
            try:
                data = pd.read_parquet(data_file)
            except Exception as e:
                raise Exception(f"File: {data_file}") from e

            categorical = [data[c].dtype.name == "category" for c in data.columns]
            attribute_names = list(data.columns)
        else:
            raise ValueError(f"Unknown file type for file '{data_file}'.")

        # Feather format does not work for sparse datasets, so we use pickle for sparse datasets
        if scipy.sparse.issparse(data):
            self.cache_format = "pickle"

        logger.info(f"{self.cache_format} write {self.name}")
        if self.cache_format == "feather":
            data.to_feather(data_feather_file)
            with open(feather_attribute_file, "wb") as fh:
                pickle.dump((categorical, attribute_names), fh, pickle.HIGHEST_PROTOCOL)
            self.data_feather_file = data_feather_file
            self.feather_attribute_file = feather_attribute_file
        else:
            with open(data_pickle_file, "wb") as fh:
                pickle.dump((data, categorical, attribute_names), fh, pickle.HIGHEST_PROTOCOL)
            self.data_pickle_file = data_pickle_file

        data_file = data_pickle_file if self.cache_format == "pickle" else data_feather_file
        logger.debug(f"Saved dataset {int(self.dataset_id or -1)}: {self.name} to file {data_file}")

        return data, categorical, attribute_names
 Store data from the local file in compressed format.

        If a local parquet file is present it will be used instead of the arff file.
        Sets cache_format to 'pickle' if data is sparse.
        
Store data from the local file in compressed format.
If a local parquet file is present it will be used instead of the arff file.
Sets cache_format to 'pickle' if data is sparse.
Store data from the local file in compressed format . If a local parquet file is present it will be used instead of the arff file . Sets cache_format to ' pickle ' if data is sparse .
def _cache_compressed_file_from_file(
        self, data_file: str
    ) -> Tuple[Union[pd.DataFrame, scipy.sparse.csr_matrix], List[bool], List[str]]:
        (
            data_pickle_file,
            data_feather_file,
            feather_attribute_file,
        ) = self._compressed_cache_file_paths(data_file)
        if data_file.endswith(".arff"):
            data, categorical, attribute_names = self._parse_data_from_arff(data_file)
        elif data_file.endswith(".pq"):
            try:
                data = pd.read_parquet(data_file)
            except Exception as e:
                raise Exception(f"File: {data_file}") from e
            categorical = [data[c].dtype.name == "category" for c in data.columns]
            attribute_names = list(data.columns)
        else:
            raise ValueError(f"Unknown file type for file '{data_file}'.")
        if scipy.sparse.issparse(data):
            self.cache_format = "pickle"
        logger.info(f"{self.cache_format} write {self.name}")
        if self.cache_format == "feather":
            data.to_feather(data_feather_file)
            with open(feather_attribute_file, "wb") as fh:
                pickle.dump((categorical, attribute_names), fh, pickle.HIGHEST_PROTOCOL)
            self.data_feather_file = data_feather_file
            self.feather_attribute_file = feather_attribute_file
        else:
            with open(data_pickle_file, "wb") as fh:
                pickle.dump((data, categorical, attribute_names), fh, pickle.HIGHEST_PROTOCOL)
            self.data_pickle_file = data_pickle_file
        data_file = data_pickle_file if self.cache_format == "pickle" else data_feather_file
        logger.debug(f"Saved dataset {int(self.dataset_id or -1)}: {self.name} to file {data_file}")
        return data, categorical, attribute_names
def _cache_compressed_file_from_file ( self , data_file : str ) -> Tuple [ Union [ pd . DataFrame , scipy . sparse . csr_matrix ] , List [ bool ] , List [ str ] ] : ( data_pickle_file , data_feather_file , feather_attribute_file , ) = self . _compressed_cache_file_paths ( data_file ) if data_file . endswith ( ".arff" ) : data , categorical , attribute_names = self . _parse_data_from_arff ( data_file ) elif data_file . endswith ( ".pq" ) : try : data = pd . read_parquet ( data_file ) except Exception as e : raise Exception ( f"File: {data_file}" ) from e categorical = [ data [ c ] . dtype . name == "category" for c in data . columns ] attribute_names = list ( data . columns ) else : raise ValueError ( f"Unknown file type for file '{data_file}'." ) if scipy . sparse . issparse ( data ) : self . cache_format = "pickle" logger . info ( f"{self.cache_format} write {self.name}" ) if self . cache_format == "feather" : data . to_feather ( data_feather_file ) with open ( feather_attribute_file , "wb" ) as fh : pickle . dump ( ( categorical , attribute_names ) , fh , pickle . HIGHEST_PROTOCOL ) self . data_feather_file = data_feather_file self . feather_attribute_file = feather_attribute_file else : with open ( data_pickle_file , "wb" ) as fh : pickle . dump ( ( data , categorical , attribute_names ) , fh , pickle . HIGHEST_PROTOCOL ) self . data_pickle_file = data_pickle_file data_file = data_pickle_file if self . cache_format == "pickle" else data_feather_file logger . debug ( f"Saved dataset {int(self.dataset_id or -1)}: {self.name} to file {data_file}" ) return data , categorical , attribute_names
Store data from the local file in compressed format.
Store data from the local file in compressed format .
""" Store data from the local file in compressed format.

        If a local parquet file is present it will be used instead of the arff file.
        Sets cache_format to 'pickle' if data is sparse.
        """ # Feather format does not work for sparse datasets, so we use pickle for sparse datasets
self
data_file
str