src/proteingnn/data.py
def create_dataset(self, naming_func: Optional[Callable] = None, overwrite: bool = False,
                       shared_pose: bool = False, n_processes: int = 1,
                       data_attr_dict: Optional[Dict[str, Callable]] = None, **kwargs) -> None:
        """Iterate through, process and save all samples under predataset."""
        create_dir(self.dataset_path, overwrite)

        inputs = []
        for pdb in get_directory_pdb(src_dir=self.predataset_path, deep=True):
            if naming_func:
                path = naming_func(str(pdb))
            else:
                path = pdb.stem + '.pt'
            path = self.dataset_path / path

            if not path.exists():
                inputs.append((pdb, path))

        if shared_pose:  # assume all poses the same to speed up
            if n_processes == 1:
                self.set_shared(inputs[0][0])
            else:
                raise NotImplementedError('Sharing pose object not allowed by multiprocessing.')

        if n_processes == 1:
            for i, (pdb, path) in enumerate(inputs, 1):
                print(f'\rDataset processing [{i}|{len(inputs)}]'.ljust(75, ' '), end='')
                self.save_graph(pdb, path, shared_pose, data_attr_dict, **kwargs)

        elif n_processes > 1:
            pool = mlp.Pool(n_processes)
            pbar = tqdm(desc=self.name, total=len(inputs), ncols=75)

            for i, (pdb, path) in enumerate(inputs, 1):
                pool.apply_async(self.save_graph, args=(pdb, path, shared_pose, data_attr_dict), kwds=kwargs,
                                 callback=lambda x: pbar.update())

            pool.close()
            pool.join()
Iterate through, process and save all samples under predataset.
Iterate through, process and save all samples under predataset.
Iterate through process and save all samples under predataset .
def create_dataset(self, naming_func: Optional[Callable] = None, overwrite: bool = False,
                       shared_pose: bool = False, n_processes: int = 1,
                       data_attr_dict: Optional[Dict[str, Callable]] = None, **kwargs) -> None:
        create_dir(self.dataset_path, overwrite)
        inputs = []
        for pdb in get_directory_pdb(src_dir=self.predataset_path, deep=True):
            if naming_func:
                path = naming_func(str(pdb))
            else:
                path = pdb.stem + '.pt'
            path = self.dataset_path / path
            if not path.exists():
                inputs.append((pdb, path))
        if shared_pose:  
            if n_processes == 1:
                self.set_shared(inputs[0][0])
            else:
                raise NotImplementedError('Sharing pose object not allowed by multiprocessing.')
        if n_processes == 1:
            for i, (pdb, path) in enumerate(inputs, 1):
                print(f'\rDataset processing [{i}|{len(inputs)}]'.ljust(75, ' '), end='')
                self.save_graph(pdb, path, shared_pose, data_attr_dict, **kwargs)
        elif n_processes > 1:
            pool = mlp.Pool(n_processes)
            pbar = tqdm(desc=self.name, total=len(inputs), ncols=75)
            for i, (pdb, path) in enumerate(inputs, 1):
                pool.apply_async(self.save_graph, args=(pdb, path, shared_pose, data_attr_dict), kwds=kwargs,
                                 callback=lambda x: pbar.update())
            pool.close()
            pool.join()
def create_dataset ( self , naming_func : Optional [ Callable ] = None , overwrite : bool = False , shared_pose : bool = False , n_processes : int = 1 , data_attr_dict : Optional [ Dict [ str , Callable ] ] = None , ** kwargs ) -> None : create_dir ( self . dataset_path , overwrite ) inputs = [ ] for pdb in get_directory_pdb ( src_dir = self . predataset_path , deep = True ) : if naming_func : path = naming_func ( str ( pdb ) ) else : path = pdb . stem + '.pt' path = self . dataset_path / path if not path . exists ( ) : inputs . append ( ( pdb , path ) ) if shared_pose : if n_processes == 1 : self . set_shared ( inputs [ 0 ] [ 0 ] ) else : raise NotImplementedError ( 'Sharing pose object not allowed by multiprocessing.' ) if n_processes == 1 : for i , ( pdb , path ) in enumerate ( inputs , 1 ) : print ( f'\rDataset processing [{i}|{len(inputs)}]' . ljust ( 75 , ' ' ) , end = '' ) self . save_graph ( pdb , path , shared_pose , data_attr_dict , ** kwargs ) elif n_processes > 1 : pool = mlp . Pool ( n_processes ) pbar = tqdm ( desc = self . name , total = len ( inputs ) , ncols = 75 ) for i , ( pdb , path ) in enumerate ( inputs , 1 ) : pool . apply_async ( self . save_graph , args = ( pdb , path , shared_pose , data_attr_dict ) , kwds = kwargs , callback = lambda x : pbar . update ( ) ) pool . close ( ) pool . join ( )
Iterate through, process and save all samples under predataset.
Iterate through process and save all samples under predataset .
"""Iterate through, process and save all samples under predataset.""" # assume all poses the same to speed up
self
naming_func
Optional[Callable]
overwrite
bool
shared_pose
bool
n_processes
int
data_attr_dict
Optional[Dict[str, Callable]]