openml/datasets/dataset.py
def _load_data(self):
        """ Load data from compressed format or arff. Download data if not present on disk. """
        need_to_create_pickle = self.cache_format == "pickle" and self.data_pickle_file is None
        need_to_create_feather = self.cache_format == "feather" and self.data_feather_file is None

        if need_to_create_pickle or need_to_create_feather:
            if self.data_file is None:
                self._download_data()

            file_to_load = self.data_file if self.parquet_file is None else self.parquet_file
            return self._cache_compressed_file_from_file(file_to_load)

        # helper variable to help identify where errors occur
        fpath = self.data_feather_file if self.cache_format == "feather" else self.data_pickle_file
        logger.info(f"{self.cache_format} load data {self.name}")
        try:
            if self.cache_format == "feather":
                data = pd.read_feather(self.data_feather_file)
                fpath = self.feather_attribute_file
                with open(self.feather_attribute_file, "rb") as fh:
                    categorical, attribute_names = pickle.load(fh)
            else:
                with open(self.data_pickle_file, "rb") as fh:
                    data, categorical, attribute_names = pickle.load(fh)
        except FileNotFoundError:
            raise ValueError(f"Cannot find file for dataset {self.name} at location '{fpath}'.")
        except (EOFError, ModuleNotFoundError, ValueError) as e:
            error_message = e.message if hasattr(e, "message") else e.args[0]
            hint = ""

            if isinstance(e, EOFError):
                readable_error = "Detected a corrupt cache file"
            elif isinstance(e, ModuleNotFoundError):
                readable_error = "Detected likely dependency issues"
                hint = "This is most likely due to https://github.com/openml/openml-python/issues/918. "  # noqa: 501
            elif isinstance(e, ValueError) and "unsupported pickle protocol" in e.args[0]:
                readable_error = "Encountered unsupported pickle protocol"
            else:
                raise  # an unknown ValueError is raised, should crash and file bug report

            logger.warning(
                f"{readable_error} when loading dataset {self.id} from '{fpath}'. "
                f"{hint}"
                f"Error message was: {error_message}. "
                "We will continue loading data from the arff-file, "
                "but this will be much slower for big datasets. "
                "Please manually delete the cache file if you want OpenML-Python "
                "to attempt to reconstruct it."
            )
            data, categorical, attribute_names = self._parse_data_from_arff(self.data_file)

        data_up_to_date = isinstance(data, pd.DataFrame) or scipy.sparse.issparse(data)
        if self.cache_format == "pickle" and not data_up_to_date:
            logger.info("Updating outdated pickle file.")
            file_to_load = self.data_file if self.parquet_file is None else self.parquet_file
            return self._cache_compressed_file_from_file(file_to_load)
        return data, categorical, attribute_names
 Load data from compressed format or arff. Download data if not present on disk. 
Load data from compressed format or arff. Download data if not present on disk.
Load data from compressed format or arff . Download data if not present on disk .
def _load_data(self):
        need_to_create_pickle = self.cache_format == "pickle" and self.data_pickle_file is None
        need_to_create_feather = self.cache_format == "feather" and self.data_feather_file is None
        if need_to_create_pickle or need_to_create_feather:
            if self.data_file is None:
                self._download_data()
            file_to_load = self.data_file if self.parquet_file is None else self.parquet_file
            return self._cache_compressed_file_from_file(file_to_load)
        fpath = self.data_feather_file if self.cache_format == "feather" else self.data_pickle_file
        logger.info(f"{self.cache_format} load data {self.name}")
        try:
            if self.cache_format == "feather":
                data = pd.read_feather(self.data_feather_file)
                fpath = self.feather_attribute_file
                with open(self.feather_attribute_file, "rb") as fh:
                    categorical, attribute_names = pickle.load(fh)
            else:
                with open(self.data_pickle_file, "rb") as fh:
                    data, categorical, attribute_names = pickle.load(fh)
        except FileNotFoundError:
            raise ValueError(f"Cannot find file for dataset {self.name} at location '{fpath}'.")
        except (EOFError, ModuleNotFoundError, ValueError) as e:
            error_message = e.message if hasattr(e, "message") else e.args[0]
            hint = ""
            if isinstance(e, EOFError):
                readable_error = "Detected a corrupt cache file"
            elif isinstance(e, ModuleNotFoundError):
                readable_error = "Detected likely dependency issues"
                hint = "This is most likely due to https://github.com/openml/openml-python/issues/918. "  
            elif isinstance(e, ValueError) and "unsupported pickle protocol" in e.args[0]:
                readable_error = "Encountered unsupported pickle protocol"
            else:
                raise  
            logger.warning(
                f"{readable_error} when loading dataset {self.id} from '{fpath}'. "
                f"{hint}"
                f"Error message was: {error_message}. "
                "We will continue loading data from the arff-file, "
                "but this will be much slower for big datasets. "
                "Please manually delete the cache file if you want OpenML-Python "
                "to attempt to reconstruct it."
            )
            data, categorical, attribute_names = self._parse_data_from_arff(self.data_file)
        data_up_to_date = isinstance(data, pd.DataFrame) or scipy.sparse.issparse(data)
        if self.cache_format == "pickle" and not data_up_to_date:
            logger.info("Updating outdated pickle file.")
            file_to_load = self.data_file if self.parquet_file is None else self.parquet_file
            return self._cache_compressed_file_from_file(file_to_load)
        return data, categorical, attribute_names
def _load_data ( self ) : need_to_create_pickle = self . cache_format == "pickle" and self . data_pickle_file is None need_to_create_feather = self . cache_format == "feather" and self . data_feather_file is None if need_to_create_pickle or need_to_create_feather : if self . data_file is None : self . _download_data ( ) file_to_load = self . data_file if self . parquet_file is None else self . parquet_file return self . _cache_compressed_file_from_file ( file_to_load ) fpath = self . data_feather_file if self . cache_format == "feather" else self . data_pickle_file logger . info ( f"{self.cache_format} load data {self.name}" ) try : if self . cache_format == "feather" : data = pd . read_feather ( self . data_feather_file ) fpath = self . feather_attribute_file with open ( self . feather_attribute_file , "rb" ) as fh : categorical , attribute_names = pickle . load ( fh ) else : with open ( self . data_pickle_file , "rb" ) as fh : data , categorical , attribute_names = pickle . load ( fh ) except FileNotFoundError : raise ValueError ( f"Cannot find file for dataset {self.name} at location '{fpath}'." ) except ( EOFError , ModuleNotFoundError , ValueError ) as e : error_message = e . message if hasattr ( e , "message" ) else e . args [ 0 ] hint = "" if isinstance ( e , EOFError ) : readable_error = "Detected a corrupt cache file" elif isinstance ( e , ModuleNotFoundError ) : readable_error = "Detected likely dependency issues" hint = "This is most likely due to https://github.com/openml/openml-python/issues/918. " elif isinstance ( e , ValueError ) and "unsupported pickle protocol" in e . args [ 0 ] : readable_error = "Encountered unsupported pickle protocol" else : raise logger . warning ( f"{readable_error} when loading dataset {self.id} from '{fpath}'. " f"{hint}" f"Error message was: {error_message}. " "We will continue loading data from the arff-file, " "but this will be much slower for big datasets. " "Please manually delete the cache file if you want OpenML-Python " "to attempt to reconstruct it." ) data , categorical , attribute_names = self . _parse_data_from_arff ( self . data_file ) data_up_to_date = isinstance ( data , pd . DataFrame ) or scipy . sparse . issparse ( data ) if self . cache_format == "pickle" and not data_up_to_date : logger . info ( "Updating outdated pickle file." ) file_to_load = self . data_file if self . parquet_file is None else self . parquet_file return self . _cache_compressed_file_from_file ( file_to_load ) return data , categorical , attribute_names
Load data from compressed format or arff.
Load data from compressed format or arff .
""" Load data from compressed format or arff. Download data if not present on disk. """ # helper variable to help identify where errors occur # noqa: 501 # an unknown ValueError is raised, should crash and file bug report
self