localpdb/PDBDownloader.py
def rsync_pdb_mirror(self, format='pdb', update=False):
        """
        Handles the RSYNC session with the PDB servers to download files in the selected format
        @param format: file format to download ('pdb' or 'mmCIF')
        @param update: denotes whether rsync will be in the update mode
        @return: exit code from the RSYNC (0 if run completed without errors)
        """

        # Set the urls and paths based on the configs
        url = self.config['rsync_url']
        add_opts = self.config['rsync_opts']
        local_mirror = self.db_path / 'mirror' / format

        if format not in ['pdb', 'mmCIF']:
            raise ValueError(f'Format \'{format}\' is not a valid. Only \'pdb\' and \'mmCIF\' formats are allowed!')

        # First make rsync dry run (-n) to check number of structures to be synced.
        # This also checks connectivity with the selected mirror.
        rsync_dry_cmd = f'rsync -nrlpt -v {add_opts} {url}/{format}/ {local_mirror}/'
        result, (stdout, _) = os_cmd(rsync_dry_cmd)
        n_structs = len([line for line in stdout])
        if update:
            n_structs += 2
        if result != 0:
            return 1

        fn_modified = f'{self.db_path}/data/{self.version}/modified_major.txt' if self.versions is None \
            else f'{self.db_path}/data/{self.version}/modified_major_merged.txt'
        modified = parse_simple(fn_modified)
        fn_obsolete = f'{self.db_path}/data/{self.version}/obsolete.txt' if self.versions is None \
            else f'{self.db_path}/data/{self.version}/obsolete_merged.txt'
        obsolete = parse_simple(fn_obsolete)
        bundles = parse_simple(f'{self.db_path}/data/{self.version}/pdb_bundles.txt')

        ids = list(modified-bundles-obsolete) if format == 'pdb' else list(modified-obsolete)
        _, map_dict = self.pdbv.adjust_pdb_ids({id_: id_ for id_ in ids}, self.version, mode='setup')
        if len(map_dict) > 0:
            logger.info(f'INFO: {len(map_dict)} entries in the {format} format had major coordinate revision without PDB id changes. '
                        f'Accounting for this during the mirror update.')
        for pdb_id in map_dict.keys():
            if format == 'pdb':
                org_fn = f'{self.db_path}/mirror/pdb/{pdb_id[1:3]}/pdb{pdb_id}.ent.gz'
                dest_fn = f'{self.db_path}/mirror/pdb/{pdb_id[1:3]}/pdb{map_dict[pdb_id]}.ent.gz'
            else:
                org_fn = f'{self.db_path}/mirror/mmCIF/{pdb_id[1:3]}/{pdb_id}.cif.gz'
                dest_fn = f'{self.db_path}/mirror/mmCIF/{pdb_id[1:3]}/{map_dict[pdb_id]}.cif.gz'
            try:
                shutil.copy2(org_fn, dest_fn)
                self.cp_files.append((org_fn, dest_fn))
                logger.debug(f'Moved file\'{org_fn}\' to \'{dest_fn}\'.')
            except:
                logger.debug(f'File \'{org_fn}\' that was supposed to be moved (versioning) does not exist.')

        # Now run main rsync process with the tqdm progress bar
        tqdm_str = f'tqdm --unit_scale --unit=item --dynamic_ncols=True --total={n_structs} >> /dev/null '
        rsync_cmd = f'rsync -rlpt -v {add_opts} {url}/{format}/ {local_mirror}/ | {tqdm_str}'
        result = os.system(rsync_cmd)
        return result

        Handles the RSYNC session with the PDB servers to download files in the selected format
        @param format: file format to download ('pdb' or 'mmCIF')
        @param update: denotes whether rsync will be in the update mode
        @return: exit code from the RSYNC (0 if run completed without errors)
        
Handles the RSYNC session with the PDB servers to download files in the selected format
Handles the RSYNC session with the PDB servers to download files in the selected format
def rsync_pdb_mirror(self, format='pdb', update=False):
        url = self.config['rsync_url']
        add_opts = self.config['rsync_opts']
        local_mirror = self.db_path / 'mirror' / format
        if format not in ['pdb', 'mmCIF']:
            raise ValueError(f'Format \'{format}\' is not a valid. Only \'pdb\' and \'mmCIF\' formats are allowed!')
        rsync_dry_cmd = f'rsync -nrlpt -v {add_opts} {url}/{format}/ {local_mirror}/'
        result, (stdout, _) = os_cmd(rsync_dry_cmd)
        n_structs = len([line for line in stdout])
        if update:
            n_structs += 2
        if result != 0:
            return 1
        fn_modified = f'{self.db_path}/data/{self.version}/modified_major.txt' if self.versions is None \
            else f'{self.db_path}/data/{self.version}/modified_major_merged.txt'
        modified = parse_simple(fn_modified)
        fn_obsolete = f'{self.db_path}/data/{self.version}/obsolete.txt' if self.versions is None \
            else f'{self.db_path}/data/{self.version}/obsolete_merged.txt'
        obsolete = parse_simple(fn_obsolete)
        bundles = parse_simple(f'{self.db_path}/data/{self.version}/pdb_bundles.txt')
        ids = list(modified-bundles-obsolete) if format == 'pdb' else list(modified-obsolete)
        _, map_dict = self.pdbv.adjust_pdb_ids({id_: id_ for id_ in ids}, self.version, mode='setup')
        if len(map_dict) > 0:
            logger.info(f'INFO: {len(map_dict)} entries in the {format} format had major coordinate revision without PDB id changes. '
                        f'Accounting for this during the mirror update.')
        for pdb_id in map_dict.keys():
            if format == 'pdb':
                org_fn = f'{self.db_path}/mirror/pdb/{pdb_id[1:3]}/pdb{pdb_id}.ent.gz'
                dest_fn = f'{self.db_path}/mirror/pdb/{pdb_id[1:3]}/pdb{map_dict[pdb_id]}.ent.gz'
            else:
                org_fn = f'{self.db_path}/mirror/mmCIF/{pdb_id[1:3]}/{pdb_id}.cif.gz'
                dest_fn = f'{self.db_path}/mirror/mmCIF/{pdb_id[1:3]}/{map_dict[pdb_id]}.cif.gz'
            try:
                shutil.copy2(org_fn, dest_fn)
                self.cp_files.append((org_fn, dest_fn))
                logger.debug(f'Moved file\'{org_fn}\' to \'{dest_fn}\'.')
            except:
                logger.debug(f'File \'{org_fn}\' that was supposed to be moved (versioning) does not exist.')
        tqdm_str = f'tqdm --unit_scale --unit=item --dynamic_ncols=True --total={n_structs} >> /dev/null '
        rsync_cmd = f'rsync -rlpt -v {add_opts} {url}/{format}/ {local_mirror}/ | {tqdm_str}'
        result = os.system(rsync_cmd)
        return result
def rsync_pdb_mirror ( self , format = 'pdb' , update = False ) : url = self . config [ 'rsync_url' ] add_opts = self . config [ 'rsync_opts' ] local_mirror = self . db_path / 'mirror' / format if format not in [ 'pdb' , 'mmCIF' ] : raise ValueError ( f'Format \'{format}\' is not a valid. Only \'pdb\' and \'mmCIF\' formats are allowed!' ) rsync_dry_cmd = f'rsync -nrlpt -v {add_opts} {url}/{format}/ {local_mirror}/' result , ( stdout , _ ) = os_cmd ( rsync_dry_cmd ) n_structs = len ( [ line for line in stdout ] ) if update : n_structs += 2 if result != 0 : return 1 fn_modified = f'{self.db_path}/data/{self.version}/modified_major.txt' if self . versions is None else f'{self.db_path}/data/{self.version}/modified_major_merged.txt' modified = parse_simple ( fn_modified ) fn_obsolete = f'{self.db_path}/data/{self.version}/obsolete.txt' if self . versions is None else f'{self.db_path}/data/{self.version}/obsolete_merged.txt' obsolete = parse_simple ( fn_obsolete ) bundles = parse_simple ( f'{self.db_path}/data/{self.version}/pdb_bundles.txt' ) ids = list ( modified - bundles - obsolete ) if format == 'pdb' else list ( modified - obsolete ) _ , map_dict = self . pdbv . adjust_pdb_ids ( { id_ : id_ for id_ in ids } , self . version , mode = 'setup' ) if len ( map_dict ) > 0 : logger . info ( f'INFO: {len(map_dict)} entries in the {format} format had major coordinate revision without PDB id changes. ' f'Accounting for this during the mirror update.' ) for pdb_id in map_dict . keys ( ) : if format == 'pdb' : org_fn = f'{self.db_path}/mirror/pdb/{pdb_id[1:3]}/pdb{pdb_id}.ent.gz' dest_fn = f'{self.db_path}/mirror/pdb/{pdb_id[1:3]}/pdb{map_dict[pdb_id]}.ent.gz' else : org_fn = f'{self.db_path}/mirror/mmCIF/{pdb_id[1:3]}/{pdb_id}.cif.gz' dest_fn = f'{self.db_path}/mirror/mmCIF/{pdb_id[1:3]}/{map_dict[pdb_id]}.cif.gz' try : shutil . copy2 ( org_fn , dest_fn ) self . cp_files . append ( ( org_fn , dest_fn ) ) logger . debug ( f'Moved file\'{org_fn}\' to \'{dest_fn}\'.' ) except : logger . debug ( f'File \'{org_fn}\' that was supposed to be moved (versioning) does not exist.' ) tqdm_str = f'tqdm --unit_scale --unit=item --dynamic_ncols=True --total={n_structs} >> /dev/null ' rsync_cmd = f'rsync -rlpt -v {add_opts} {url}/{format}/ {local_mirror}/ | {tqdm_str}' result = os . system ( rsync_cmd ) return result
Handles the RSYNC session with the PDB servers to download files in the selected format
Handles the RSYNC session with the PDB servers to download files in the selected format
"""
        Handles the RSYNC session with the PDB servers to download files in the selected format
        @param format: file format to download ('pdb' or 'mmCIF')
        @param update: denotes whether rsync will be in the update mode
        @return: exit code from the RSYNC (0 if run completed without errors)
        """ # Set the urls and paths based on the configs # First make rsync dry run (-n) to check number of structures to be synced. # This also checks connectivity with the selected mirror. # Now run main rsync process with the tqdm progress bar
exit code from the RSYNC (0 if run completed without errors)
exit code from the RSYNC ( 0 if run completed without errors )
self
format
file format to download ('pdb' or 'mmCIF')
file format to download ( ' pdb ' or ' mmCIF ' )
update
denotes whether rsync will be in the update mode
denotes whether rsync will be in the update mode