scripts/validation/genome_download_validator.py
def check_all_genome_files_exist(project_dir, upid_gca_file=None):
    """
    This function will extract all accessions per genome and check that all files
    exist. A json file will be generated with all missing accessions so that they
    can be downloaded using restore_gen_download. It reports download status for
    all domain subdirectories and mark it as "Success" or "Failure". In case of
    failure it will generate a json file with all accessions to restore

    project_dir: A project directory as generated by genome_downloader pipeline
    upid_gca_file: All upid-gca pairs either in json or tsv format. None by
    default. If None it will use the json file produced in project_dir during
    genome download
    """

    # list all domain directories in project
    domain_dirs = [x for x in os.listdir(project_dir) if x in gc.DOMAINS]
    upid_gca_pairs = None

    # load upid_gca pairs from project directory (.json)
    if upid_gca_file is None:
        upid_gca_fp = open(os.path.join(project_dir, "upid_gca_dict.json"), 'r')
        upid_gca_pairs = json.load(upid_gca_fp)
        upid_gca_fp.close()

    for domain in domain_dirs:
        domain_missing_accs = {}
        domain_dir_loc = os.path.join(project_dir, domain)

        upids = [x for x in os.listdir(domain_dir_loc)
                 if os.path.isdir(os.path.join(domain_dir_loc, x))]

        gen_missing_accs = []
        for upid in upids:
            upid_dir = os.path.join(domain_dir_loc, upid)

            gca_acc = upid_gca_pairs[upid]["GCA"]
            accessions = gf.fetch_genome_accessions(upid, gca_acc)

            # assuming that all files are decompressed and should be to avoid problems
            for accession in accessions:
                if accession is not None:
                    if check_file_format(os.path.join(upid_dir, accession + ".fa")) is False:
                        gen_missing_accs.append(accession)
                else:
                    print upid

            if len(gen_missing_accs) > 0:
                domain_missing_accs[upid] = gen_missing_accs

        if len(domain_missing_accs.keys()) > 0:
            fp_out = open(os.path.join(domain_dir_loc, domain + "acc_recovery.json"), 'w')
            json.dump(domain_missing_accs, fp_out)
            fp_out.close()

            print "%s Validation: Failure" % domain

        else:
            print "%s Validation: Success" % domain

    This function will extract all accessions per genome and check that all files
    exist. A json file will be generated with all missing accessions so that they
    can be downloaded using restore_gen_download. It reports download status for
    all domain subdirectories and mark it as "Success" or "Failure". In case of
    failure it will generate a json file with all accessions to restore

    project_dir: A project directory as generated by genome_downloader pipeline
    upid_gca_file: All upid-gca pairs either in json or tsv format. None by
    default. If None it will use the json file produced in project_dir during
    genome download
    
This function will extract all accessions per genome and check that all files
exist. A json file will be generated with all missing accessions so that they
can be downloaded using restore_gen_download. It reports download status for
all domain subdirectories and mark it as "Success" or "Failure". In case of
failure it will generate a json file with all accessions to restore

A project directory as generated by genome_downloader pipeline
upid_gca_file: All upid-gca pairs either in json or tsv format. None by
default. If None it will use the json file produced in project_dir during
genome download
This function will extract all accessions per genome and check that all files exist . A json file will be generated with all missing accessions so that they can be downloaded using restore_gen_download . It reports download status for all domain subdirectories and mark it as " Success " or " Failure " . In case of failure it will generate a json file with all accessions to restore A project directory as generated by genome_downloader pipeline upid_gca_file : All upid - gca pairs either in json or tsv format . None by default . If None it will use the json file produced in project_dir during genome download
def check_all_genome_files_exist(project_dir, upid_gca_file=None):
    domain_dirs = [x for x in os.listdir(project_dir) if x in gc.DOMAINS]
    upid_gca_pairs = None
    if upid_gca_file is None:
        upid_gca_fp = open(os.path.join(project_dir, "upid_gca_dict.json"), 'r')
        upid_gca_pairs = json.load(upid_gca_fp)
        upid_gca_fp.close()
    for domain in domain_dirs:
        domain_missing_accs = {}
        domain_dir_loc = os.path.join(project_dir, domain)
        upids = [x for x in os.listdir(domain_dir_loc)
                 if os.path.isdir(os.path.join(domain_dir_loc, x))]
        gen_missing_accs = []
        for upid in upids:
            upid_dir = os.path.join(domain_dir_loc, upid)
            gca_acc = upid_gca_pairs[upid]["GCA"]
            accessions = gf.fetch_genome_accessions(upid, gca_acc)
            for accession in accessions:
                if accession is not None:
                    if check_file_format(os.path.join(upid_dir, accession + ".fa")) is False:
                        gen_missing_accs.append(accession)
                else:
                    print upid
            if len(gen_missing_accs) > 0:
                domain_missing_accs[upid] = gen_missing_accs
        if len(domain_missing_accs.keys()) > 0:
            fp_out = open(os.path.join(domain_dir_loc, domain + "acc_recovery.json"), 'w')
            json.dump(domain_missing_accs, fp_out)
            fp_out.close()
            print "%s Validation: Failure" % domain
        else:
            print "%s Validation: Success" % domain
def check_all_genome_files_exist ( project_dir , upid_gca_file = None ) : domain_dirs = [ x for x in os . listdir ( project_dir ) if x in gc . DOMAINS ] upid_gca_pairs = None if upid_gca_file is None : upid_gca_fp = open ( os . path . join ( project_dir , "upid_gca_dict.json" ) , 'r' ) upid_gca_pairs = json . load ( upid_gca_fp ) upid_gca_fp . close ( ) for domain in domain_dirs : domain_missing_accs = { } domain_dir_loc = os . path . join ( project_dir , domain ) upids = [ x for x in os . listdir ( domain_dir_loc ) if os . path . isdir ( os . path . join ( domain_dir_loc , x ) ) ] gen_missing_accs = [ ] for upid in upids : upid_dir = os . path . join ( domain_dir_loc , upid ) gca_acc = upid_gca_pairs [ upid ] [ "GCA" ] accessions = gf . fetch_genome_accessions ( upid , gca_acc ) for accession in accessions : if accession is not None : if check_file_format ( os . path . join ( upid_dir , accession + ".fa" ) ) is False : gen_missing_accs . append ( accession ) else : print upid if len ( gen_missing_accs ) > 0 : domain_missing_accs [ upid ] = gen_missing_accs if len ( domain_missing_accs . keys ( ) ) > 0 : fp_out = open ( os . path . join ( domain_dir_loc , domain + "acc_recovery.json" ) , 'w' ) json . dump ( domain_missing_accs , fp_out ) fp_out . close ( ) print "%s Validation: Failure" % domain else : print "%s Validation: Success" % domain
This function will extract all accessions per genome and check that all files
exist.
This function will extract all accessions per genome and check that all files exist .
"""
    This function will extract all accessions per genome and check that all files
    exist. A json file will be generated with all missing accessions so that they
    can be downloaded using restore_gen_download. It reports download status for
    all domain subdirectories and mark it as "Success" or "Failure". In case of
    failure it will generate a json file with all accessions to restore

    project_dir: A project directory as generated by genome_downloader pipeline
    upid_gca_file: All upid-gca pairs either in json or tsv format. None by
    default. If None it will use the json file produced in project_dir during
    genome download
    """ # list all domain directories in project # load upid_gca pairs from project directory (.json) # assuming that all files are decompressed and should be to avoid problems
project_dir
upid_gca_file