bin/bactopia/bactopia-datasets.py
def create_summary(outdir, training_set=False):
    """Create a summary of available datasets in JSON format."""
    from collections import OrderedDict
    available_datasets = OrderedDict()

    available_datasets['antimicrobial-resistance'] = []
    available_datasets['ariba'] = []
    available_datasets['minmer'] = {'sketches': [], 'last_update': None}
    available_datasets['plasmid'] = {'sketches': None, 'blastdb': None, 'last_update': None}

    # Antimicrobial Resistance
    if os.path.exists(f'{outdir}/antimicrobial-resistance'):
        for db in sorted(os.listdir(f'{outdir}/antimicrobial-resistance')):
            if db.endswith(".tar.gz"):
                if db != 'EMPTY.tar.gz':
                    name = db.replace(".tar.gz", "")
                    available_datasets['antimicrobial-resistance'].append({
                        'name': db,
                        'last_update': execute(
                            f'head -n 1 {outdir}/antimicrobial-resistance/{name}-updated.txt', capture=True
                        ).rstrip()
                    })

    # Ariba
    if os.path.exists(f'{outdir}/ariba'):
        for db in sorted(os.listdir(f'{outdir}/ariba')):
            if db.endswith(".tar.gz"):
                if db != 'EMPTY.tar.gz':
                    name = db.replace(".tar.gz", "")
                    available_datasets['ariba'].append({
                        'name': db,
                        'last_update': execute(
                            f'head -n 1 {outdir}/ariba/{name}-updated.txt', capture=True
                        ).rstrip()
                    })

    # Minmers
    if os.path.exists(f'{outdir}/minmer/minmer-updated.txt'):
        available_datasets['minmer'] = {
            'sketches': [],
            'last_update': execute(
                f'head -n 1 {outdir}/minmer/minmer-updated.txt', capture=True
            ).rstrip()
        }
        for sketch in sorted(os.listdir(f'{outdir}/minmer')):
            if sketch != 'minmer-updated.txt':
                available_datasets['minmer']['sketches'].append(sketch)

    # Organism Specific
    if os.path.exists(f'{outdir}/species-specific'):
        available_datasets['species-specific'] = OrderedDict()
        for species in sorted(os.listdir(f'{outdir}/species-specific')):
            if species.startswith("."):
                # Skip hidden files like .DS_Store
                continue
            new_species = OrderedDict()
            species_dir = f'{outdir}/species-specific/{species}'

            minmer = f'{species_dir}/minmer'
            new_species['minmer'] = {'mash': None, 'last_updated': None}
            if os.path.exists(f'{minmer}/refseq-genomes.msh'):
                new_species['minmer'] = {
                    'mash': f'species-specific/{species}/minmer/refseq-genomes.msh',
                    'last_updated': execute(
                        f'head -n 1 {minmer}/minmer-updated.txt',
                        capture=True
                    ).rstrip()
                }

            prokka = f'{species_dir}/annotation'
            new_species['annotation'] = { 'proteins': None, 'training_set': None, 'last_updated': None}
            if os.path.exists(f'{prokka}/{species}.faa'):
                new_species['annotation'] = {
                    'proteins': f'species-specific/{species}/annotation/{species}.faa',
                    'last_updated': execute(
                        f'head -n 1 {prokka}/proteins-updated.txt',
                        capture=True
                    ).rstrip()
                }

            if training_set or os.path.exists(f'{prokka}/prodigal.tf'):
                if not os.path.exists(prokka):
                    execute(f'mkdir -p {prokka}')
                if training_set:
                    execute(f'cp {training_set} {prokka}/prodigal.tf')
                new_species['annotation']['training_set'] = f'species-specific/{species}/annotation/prodigal.tf'

            new_species['genome_size'] = {'min': None, 'median': None, 'mean': None, 'max': None}
            if os.path.exists(f'{prokka}/genome_size.json'):
                with open(f'{prokka}/genome_size.json', 'r') as gs_fh:
                    json_data = json.load(gs_fh)
                    new_species['genome_size'] = json_data

            mlst = f'{species_dir}/mlst'
            new_species['mlst'] = {} 
            if os.path.exists(f'{mlst}'):
                for mlst_file in sorted(os.listdir(f'{mlst}')):
                    if mlst_file.endswith("tar.gz"):
                        schema = mlst_file.replace(".tar.gz", "")
                        new_species['mlst'][schema] = {
                            'data': f'species-specific/{species}/mlst/{schema}.tar.gz',
                            'last_updated': execute(f'head -n 1 {mlst}/{schema}-updated.txt', capture=True).rstrip()
                        }

            optionals = sorted([
                'reference-genomes', 'mapping-sequences', 'blast'
            ])
            new_species['optional'] = OrderedDict()
            for optional in optionals:
                # These are optional directories users can add data to
                optional_dir = f'species-specific/{species}/optional/{optional}'
                if not os.path.exists(optional_dir):
                    execute(f'mkdir -p {optional_dir}', directory=outdir)
                if optional == 'blast':
                    new_species['optional'][optional] = [
                        f'{optional_dir}/genes',
                        f'{optional_dir}/primers',
                        f'{optional_dir}/proteins',
                    ]
                    for blast_dir in new_species['optional'][optional]:
                        execute(f'mkdir -p {blast_dir}', directory=outdir)
                else:
                    new_species['optional'][optional] = f'{optional_dir}'

            available_datasets['species-specific'][species] = new_species

    with open(f'{outdir}/summary.json', 'w') as json_handle:
        logging.info(f'Writing summary of available datasets')
        json.dump(available_datasets, json_handle, indent=4)
        logging.debug(json.dumps(available_datasets, indent=4))
Create a summary of available datasets in JSON format.
Create a summary of available datasets in JSON format.
Create a summary of available datasets in JSON format .
def create_summary(outdir, training_set=False):
    from collections import OrderedDict
    available_datasets = OrderedDict()
    available_datasets['antimicrobial-resistance'] = []
    available_datasets['ariba'] = []
    available_datasets['minmer'] = {'sketches': [], 'last_update': None}
    available_datasets['plasmid'] = {'sketches': None, 'blastdb': None, 'last_update': None}
    if os.path.exists(f'{outdir}/antimicrobial-resistance'):
        for db in sorted(os.listdir(f'{outdir}/antimicrobial-resistance')):
            if db.endswith(".tar.gz"):
                if db != 'EMPTY.tar.gz':
                    name = db.replace(".tar.gz", "")
                    available_datasets['antimicrobial-resistance'].append({
                        'name': db,
                        'last_update': execute(
                            f'head -n 1 {outdir}/antimicrobial-resistance/{name}-updated.txt', capture=True
                        ).rstrip()
                    })
    if os.path.exists(f'{outdir}/ariba'):
        for db in sorted(os.listdir(f'{outdir}/ariba')):
            if db.endswith(".tar.gz"):
                if db != 'EMPTY.tar.gz':
                    name = db.replace(".tar.gz", "")
                    available_datasets['ariba'].append({
                        'name': db,
                        'last_update': execute(
                            f'head -n 1 {outdir}/ariba/{name}-updated.txt', capture=True
                        ).rstrip()
                    })
    if os.path.exists(f'{outdir}/minmer/minmer-updated.txt'):
        available_datasets['minmer'] = {
            'sketches': [],
            'last_update': execute(
                f'head -n 1 {outdir}/minmer/minmer-updated.txt', capture=True
            ).rstrip()
        }
        for sketch in sorted(os.listdir(f'{outdir}/minmer')):
            if sketch != 'minmer-updated.txt':
                available_datasets['minmer']['sketches'].append(sketch)
    if os.path.exists(f'{outdir}/species-specific'):
        available_datasets['species-specific'] = OrderedDict()
        for species in sorted(os.listdir(f'{outdir}/species-specific')):
            if species.startswith("."):
                continue
            new_species = OrderedDict()
            species_dir = f'{outdir}/species-specific/{species}'
            minmer = f'{species_dir}/minmer'
            new_species['minmer'] = {'mash': None, 'last_updated': None}
            if os.path.exists(f'{minmer}/refseq-genomes.msh'):
                new_species['minmer'] = {
                    'mash': f'species-specific/{species}/minmer/refseq-genomes.msh',
                    'last_updated': execute(
                        f'head -n 1 {minmer}/minmer-updated.txt',
                        capture=True
                    ).rstrip()
                }
            prokka = f'{species_dir}/annotation'
            new_species['annotation'] = { 'proteins': None, 'training_set': None, 'last_updated': None}
            if os.path.exists(f'{prokka}/{species}.faa'):
                new_species['annotation'] = {
                    'proteins': f'species-specific/{species}/annotation/{species}.faa',
                    'last_updated': execute(
                        f'head -n 1 {prokka}/proteins-updated.txt',
                        capture=True
                    ).rstrip()
                }
            if training_set or os.path.exists(f'{prokka}/prodigal.tf'):
                if not os.path.exists(prokka):
                    execute(f'mkdir -p {prokka}')
                if training_set:
                    execute(f'cp {training_set} {prokka}/prodigal.tf')
                new_species['annotation']['training_set'] = f'species-specific/{species}/annotation/prodigal.tf'
            new_species['genome_size'] = {'min': None, 'median': None, 'mean': None, 'max': None}
            if os.path.exists(f'{prokka}/genome_size.json'):
                with open(f'{prokka}/genome_size.json', 'r') as gs_fh:
                    json_data = json.load(gs_fh)
                    new_species['genome_size'] = json_data
            mlst = f'{species_dir}/mlst'
            new_species['mlst'] = {} 
            if os.path.exists(f'{mlst}'):
                for mlst_file in sorted(os.listdir(f'{mlst}')):
                    if mlst_file.endswith("tar.gz"):
                        schema = mlst_file.replace(".tar.gz", "")
                        new_species['mlst'][schema] = {
                            'data': f'species-specific/{species}/mlst/{schema}.tar.gz',
                            'last_updated': execute(f'head -n 1 {mlst}/{schema}-updated.txt', capture=True).rstrip()
                        }
            optionals = sorted([
                'reference-genomes', 'mapping-sequences', 'blast'
            ])
            new_species['optional'] = OrderedDict()
            for optional in optionals:
                optional_dir = f'species-specific/{species}/optional/{optional}'
                if not os.path.exists(optional_dir):
                    execute(f'mkdir -p {optional_dir}', directory=outdir)
                if optional == 'blast':
                    new_species['optional'][optional] = [
                        f'{optional_dir}/genes',
                        f'{optional_dir}/primers',
                        f'{optional_dir}/proteins',
                    ]
                    for blast_dir in new_species['optional'][optional]:
                        execute(f'mkdir -p {blast_dir}', directory=outdir)
                else:
                    new_species['optional'][optional] = f'{optional_dir}'
            available_datasets['species-specific'][species] = new_species
    with open(f'{outdir}/summary.json', 'w') as json_handle:
        logging.info(f'Writing summary of available datasets')
        json.dump(available_datasets, json_handle, indent=4)
        logging.debug(json.dumps(available_datasets, indent=4))
def create_summary ( outdir , training_set = False ) : from collections import OrderedDict available_datasets = OrderedDict ( ) available_datasets [ 'antimicrobial-resistance' ] = [ ] available_datasets [ 'ariba' ] = [ ] available_datasets [ 'minmer' ] = { 'sketches' : [ ] , 'last_update' : None } available_datasets [ 'plasmid' ] = { 'sketches' : None , 'blastdb' : None , 'last_update' : None } if os . path . exists ( f'{outdir}/antimicrobial-resistance' ) : for db in sorted ( os . listdir ( f'{outdir}/antimicrobial-resistance' ) ) : if db . endswith ( ".tar.gz" ) : if db != 'EMPTY.tar.gz' : name = db . replace ( ".tar.gz" , "" ) available_datasets [ 'antimicrobial-resistance' ] . append ( { 'name' : db , 'last_update' : execute ( f'head -n 1 {outdir}/antimicrobial-resistance/{name}-updated.txt' , capture = True ) . rstrip ( ) } ) if os . path . exists ( f'{outdir}/ariba' ) : for db in sorted ( os . listdir ( f'{outdir}/ariba' ) ) : if db . endswith ( ".tar.gz" ) : if db != 'EMPTY.tar.gz' : name = db . replace ( ".tar.gz" , "" ) available_datasets [ 'ariba' ] . append ( { 'name' : db , 'last_update' : execute ( f'head -n 1 {outdir}/ariba/{name}-updated.txt' , capture = True ) . rstrip ( ) } ) if os . path . exists ( f'{outdir}/minmer/minmer-updated.txt' ) : available_datasets [ 'minmer' ] = { 'sketches' : [ ] , 'last_update' : execute ( f'head -n 1 {outdir}/minmer/minmer-updated.txt' , capture = True ) . rstrip ( ) } for sketch in sorted ( os . listdir ( f'{outdir}/minmer' ) ) : if sketch != 'minmer-updated.txt' : available_datasets [ 'minmer' ] [ 'sketches' ] . append ( sketch ) if os . path . exists ( f'{outdir}/species-specific' ) : available_datasets [ 'species-specific' ] = OrderedDict ( ) for species in sorted ( os . listdir ( f'{outdir}/species-specific' ) ) : if species . startswith ( "." ) : continue new_species = OrderedDict ( ) species_dir = f'{outdir}/species-specific/{species}' minmer = f'{species_dir}/minmer' new_species [ 'minmer' ] = { 'mash' : None , 'last_updated' : None } if os . path . exists ( f'{minmer}/refseq-genomes.msh' ) : new_species [ 'minmer' ] = { 'mash' : f'species-specific/{species}/minmer/refseq-genomes.msh' , 'last_updated' : execute ( f'head -n 1 {minmer}/minmer-updated.txt' , capture = True ) . rstrip ( ) } prokka = f'{species_dir}/annotation' new_species [ 'annotation' ] = { 'proteins' : None , 'training_set' : None , 'last_updated' : None } if os . path . exists ( f'{prokka}/{species}.faa' ) : new_species [ 'annotation' ] = { 'proteins' : f'species-specific/{species}/annotation/{species}.faa' , 'last_updated' : execute ( f'head -n 1 {prokka}/proteins-updated.txt' , capture = True ) . rstrip ( ) } if training_set or os . path . exists ( f'{prokka}/prodigal.tf' ) : if not os . path . exists ( prokka ) : execute ( f'mkdir -p {prokka}' ) if training_set : execute ( f'cp {training_set} {prokka}/prodigal.tf' ) new_species [ 'annotation' ] [ 'training_set' ] = f'species-specific/{species}/annotation/prodigal.tf' new_species [ 'genome_size' ] = { 'min' : None , 'median' : None , 'mean' : None , 'max' : None } if os . path . exists ( f'{prokka}/genome_size.json' ) : with open ( f'{prokka}/genome_size.json' , 'r' ) as gs_fh : json_data = json . load ( gs_fh ) new_species [ 'genome_size' ] = json_data mlst = f'{species_dir}/mlst' new_species [ 'mlst' ] = { } if os . path . exists ( f'{mlst}' ) : for mlst_file in sorted ( os . listdir ( f'{mlst}' ) ) : if mlst_file . endswith ( "tar.gz" ) : schema = mlst_file . replace ( ".tar.gz" , "" ) new_species [ 'mlst' ] [ schema ] = { 'data' : f'species-specific/{species}/mlst/{schema}.tar.gz' , 'last_updated' : execute ( f'head -n 1 {mlst}/{schema}-updated.txt' , capture = True ) . rstrip ( ) } optionals = sorted ( [ 'reference-genomes' , 'mapping-sequences' , 'blast' ] ) new_species [ 'optional' ] = OrderedDict ( ) for optional in optionals : optional_dir = f'species-specific/{species}/optional/{optional}' if not os . path . exists ( optional_dir ) : execute ( f'mkdir -p {optional_dir}' , directory = outdir ) if optional == 'blast' : new_species [ 'optional' ] [ optional ] = [ f'{optional_dir}/genes' , f'{optional_dir}/primers' , f'{optional_dir}/proteins' , ] for blast_dir in new_species [ 'optional' ] [ optional ] : execute ( f'mkdir -p {blast_dir}' , directory = outdir ) else : new_species [ 'optional' ] [ optional ] = f'{optional_dir}' available_datasets [ 'species-specific' ] [ species ] = new_species with open ( f'{outdir}/summary.json' , 'w' ) as json_handle : logging . info ( f'Writing summary of available datasets' ) json . dump ( available_datasets , json_handle , indent = 4 ) logging . debug ( json . dumps ( available_datasets , indent = 4 ) )
Create a summary of available datasets in JSON format.
Create a summary of available datasets in JSON format .
"""Create a summary of available datasets in JSON format.""" # Antimicrobial Resistance # Ariba # Minmers # Organism Specific # Skip hidden files like .DS_Store # These are optional directories users can add data to
outdir
training_set