graphinvent/Workflow.py
def preprocess_phase(self) -> None:
        """
        Preprocesses all the datasets (validation, training, and testing).
        """
        if not self.constants.restart:
            # start preprocessing job from scratch
            hdf_files_in_data_dir = bool(os.path.exists(self.valid_h5_path)
                                         or os.path.exists(self.test_h5_path)
                                         or os.path.exists(self.train_h5_path))
            if hdf_files_in_data_dir:
                raise OSError(
                    "There currently exist(s) pre-created *.h5 file(s) in the "
                    "dataset directory. If you would like to proceed with "
                    "creating new ones, please delete them and rerun the "
                    "program. Otherwise, check your input file."
                )
            self.preprocess_valid_data()
            self.preprocess_test_data()
            self.preprocess_train_data()

        else:  # restart existing preprocessing job

            # first determine where to restart based on which HDF files have been created
            if (os.path.exists(self.train_h5_path + ".chunked") or
                os.path.exists(self.test_h5_path)):
                print(
                    "-- Restarting preprocessing job from 'train.h5' (skipping "
                    "over 'test.h5' and 'valid.h5' as they seem to be finished).",
                    flush=True,
                )
                self.preprocess_train_data()
            elif (os.path.exists(self.test_h5_path + ".chunked") or
                  os.path.exists(self.valid_h5_path)):
                print(
                    "-- Restarting preprocessing job from 'test.h5' (skipping "
                    "over 'valid.h5' as it appears to be finished).",
                    flush=True,
                )
                self.preprocess_test_data()
                self.preprocess_train_data()
            elif os.path.exists(self.valid_h5_path + ".chunked"):
                print("-- Restarting preprocessing job from 'valid.h5'",
                      flush=True)
                self.preprocess_valid_data()
                self.preprocess_test_data()
                self.preprocess_train_data()
            else:
                raise ValueError(
                    "Warning: Nothing to restart! Check input file and/or "
                    "submission script."
                )

        Preprocesses all the datasets (validation, training, and testing).
        
Preprocesses all the datasets (validation, training, and testing).
Preprocesses all the datasets ( validation training and testing ) .
def preprocess_phase(self) -> None:
        if not self.constants.restart:
            hdf_files_in_data_dir = bool(os.path.exists(self.valid_h5_path)
                                         or os.path.exists(self.test_h5_path)
                                         or os.path.exists(self.train_h5_path))
            if hdf_files_in_data_dir:
                raise OSError(
                    "There currently exist(s) pre-created *.h5 file(s) in the "
                    "dataset directory. If you would like to proceed with "
                    "creating new ones, please delete them and rerun the "
                    "program. Otherwise, check your input file."
                )
            self.preprocess_valid_data()
            self.preprocess_test_data()
            self.preprocess_train_data()
        else:  
            if (os.path.exists(self.train_h5_path + ".chunked") or
                os.path.exists(self.test_h5_path)):
                print(
                    "-- Restarting preprocessing job from 'train.h5' (skipping "
                    "over 'test.h5' and 'valid.h5' as they seem to be finished).",
                    flush=True,
                )
                self.preprocess_train_data()
            elif (os.path.exists(self.test_h5_path + ".chunked") or
                  os.path.exists(self.valid_h5_path)):
                print(
                    "-- Restarting preprocessing job from 'test.h5' (skipping "
                    "over 'valid.h5' as it appears to be finished).",
                    flush=True,
                )
                self.preprocess_test_data()
                self.preprocess_train_data()
            elif os.path.exists(self.valid_h5_path + ".chunked"):
                print("-- Restarting preprocessing job from 'valid.h5'",
                      flush=True)
                self.preprocess_valid_data()
                self.preprocess_test_data()
                self.preprocess_train_data()
            else:
                raise ValueError(
                    "Warning: Nothing to restart! Check input file and/or "
                    "submission script."
                )
def preprocess_phase ( self ) -> None : if not self . constants . restart : hdf_files_in_data_dir = bool ( os . path . exists ( self . valid_h5_path ) or os . path . exists ( self . test_h5_path ) or os . path . exists ( self . train_h5_path ) ) if hdf_files_in_data_dir : raise OSError ( "There currently exist(s) pre-created *.h5 file(s) in the " "dataset directory. If you would like to proceed with " "creating new ones, please delete them and rerun the " "program. Otherwise, check your input file." ) self . preprocess_valid_data ( ) self . preprocess_test_data ( ) self . preprocess_train_data ( ) else : if ( os . path . exists ( self . train_h5_path + ".chunked" ) or os . path . exists ( self . test_h5_path ) ) : print ( "-- Restarting preprocessing job from 'train.h5' (skipping " "over 'test.h5' and 'valid.h5' as they seem to be finished)." , flush = True , ) self . preprocess_train_data ( ) elif ( os . path . exists ( self . test_h5_path + ".chunked" ) or os . path . exists ( self . valid_h5_path ) ) : print ( "-- Restarting preprocessing job from 'test.h5' (skipping " "over 'valid.h5' as it appears to be finished)." , flush = True , ) self . preprocess_test_data ( ) self . preprocess_train_data ( ) elif os . path . exists ( self . valid_h5_path + ".chunked" ) : print ( "-- Restarting preprocessing job from 'valid.h5'" , flush = True ) self . preprocess_valid_data ( ) self . preprocess_test_data ( ) self . preprocess_train_data ( ) else : raise ValueError ( "Warning: Nothing to restart! Check input file and/or " "submission script." )
Preprocesses all the datasets (validation, training, and testing).
Preprocesses all the datasets ( validation training and testing ) .
"""
        Preprocesses all the datasets (validation, training, and testing).
        """ # start preprocessing job from scratch # restart existing preprocessing job # first determine where to restart based on which HDF files have been created
self