scripts/export/genomes/fetch_gen_metadata.py
def fetch_assembly_accessions(upid, gca_acc, acc_ftp_link, reg_ftp_link=None):
    """
    Parses assembly report file and exports all assembly accessions in a
    dict format to be easily loaded via Django ORM

    upid: A valid Uniprot proteome accession (e.g. UP000005640 - Homo Sapiens)
    gca_acc: A valid ENA GCA accession
    acc_ftp_link: Assembly report file ftp url (as retrieved from GCA xml file)
    reg_ftp_link: Assembly region file ftp url (as retrieved from GCA xml file)
    """

    assembly_accs = []
    fields = {}
    entry = {}
    regions = None

    # load accession regions
    if reg_ftp_link is not None:
        regions = region_loader(reg_ftp_link)

    http_link = acc_ftp_link.replace("ftp://", "http://")
    response = requests.get(http_link).content

    acc_lines = response.strip().split('\n')

    # remove header line
    acc_lines.pop(0)

    acc_attributes = ''
    for acc_line in acc_lines:
        acc_attributes = acc_line.strip().split('\t')

        if acc_attributes[0].find('.') != -1:
            entry["model"] = gc.GENSEQ_MODEL
            entry["pk"] = str(acc_attributes[0])  # seq_acc

            if len(acc_attributes) == 7:
                acc_meta = fetch_gca_acc_metadata(acc_attributes[0])

                # skip if accession was unavailable (e.g. obsolete)
                if len(acc_meta.keys()) == 0:
                    # should log upid and accession
                    continue

                fields["ncbi_id"] = acc_meta["ncbi_id"]
                fields["description"] = acc_meta["description"]
                fields["upid"] = upid
                fields["gen_acc"] = gca_acc
                fields["seq_version"] = int(acc_attributes[0].partition('.')[2])
                if acc_attributes[2] != '':
                    fields["seq_length"] = int(acc_attributes[2])
                else:
                    fields["seq_length"] = 0
                fields["seq_role"] = acc_attributes[3]  # patch, loci etc

                # need to parse the regions file for these fields
                if reg_ftp_link is not None and (acc_attributes[0] in regions.keys()):
                    fields["seq_start"] = int(regions[acc_attributes[0]][0])
                    fields["seq_end"] = int(regions[acc_attributes[0]][1])
                else:
                    fields["seq_start"] = 0
                    fields["seq_end"] = 0

                fields["mol_type"] = None
                if acc_meta.has_key("mol_type"):
                    fields["mol_type"] = acc_meta["mol_type"]

                # primary, PATCHES, ALT_REF_LOCI_1
                fields["assembly_unit"] = acc_attributes[6]
                # this takes the date of the entry is created
                entry_date = datetime.datetime.now().strftime(
                    "%Y-%m-%d %H:%M:%S")
                fields["created"] = entry_date
                fields["updated"] = entry_date

            # need to review this..
            elif len(acc_attributes) < 7 and acc_attributes[0].find('.') != -1:
                fields = fetch_wgs_acc_metadata(acc_attributes[0])

        else:
            # skip cases no accession is provided
            pass

        # make sure to skip all cases of missing accessions
        if len(fields.keys()) == 0:
            continue

        entry["fields"] = fields
        assembly_accs.append(entry)
        fields = {}
        entry = {}

    return assembly_accs

    Parses assembly report file and exports all assembly accessions in a
    dict format to be easily loaded via Django ORM

    upid: A valid Uniprot proteome accession (e.g. UP000005640 - Homo Sapiens)
    gca_acc: A valid ENA GCA accession
    acc_ftp_link: Assembly report file ftp url (as retrieved from GCA xml file)
    reg_ftp_link: Assembly region file ftp url (as retrieved from GCA xml file)
    
Parses assembly report file and exports all assembly accessions in a
dict format to be easily loaded via Django ORM

A valid Uniprot proteome accession
gca_acc: A valid ENA GCA accession
acc_ftp_link: Assembly report file ftp url (as retrieved from GCA xml file)
reg_ftp_link: Assembly region file ftp url (as retrieved from GCA xml file)
Parses assembly report file and exports all assembly accessions in a dict format to be easily loaded via Django ORM A valid Uniprot proteome accession gca_acc : A valid ENA GCA accession acc_ftp_link : Assembly report file ftp url ( as retrieved from GCA xml file ) reg_ftp_link : Assembly region file ftp url ( as retrieved from GCA xml file )
def fetch_assembly_accessions(upid, gca_acc, acc_ftp_link, reg_ftp_link=None):
    assembly_accs = []
    fields = {}
    entry = {}
    regions = None
    if reg_ftp_link is not None:
        regions = region_loader(reg_ftp_link)
    http_link = acc_ftp_link.replace("ftp://", "http://")
    response = requests.get(http_link).content
    acc_lines = response.strip().split('\n')
    acc_lines.pop(0)
    acc_attributes = ''
    for acc_line in acc_lines:
        acc_attributes = acc_line.strip().split('\t')
        if acc_attributes[0].find('.') != -1:
            entry["model"] = gc.GENSEQ_MODEL
            entry["pk"] = str(acc_attributes[0])  
            if len(acc_attributes) == 7:
                acc_meta = fetch_gca_acc_metadata(acc_attributes[0])
                if len(acc_meta.keys()) == 0:
                    continue
                fields["ncbi_id"] = acc_meta["ncbi_id"]
                fields["description"] = acc_meta["description"]
                fields["upid"] = upid
                fields["gen_acc"] = gca_acc
                fields["seq_version"] = int(acc_attributes[0].partition('.')[2])
                if acc_attributes[2] != '':
                    fields["seq_length"] = int(acc_attributes[2])
                else:
                    fields["seq_length"] = 0
                fields["seq_role"] = acc_attributes[3]  
                if reg_ftp_link is not None and (acc_attributes[0] in regions.keys()):
                    fields["seq_start"] = int(regions[acc_attributes[0]][0])
                    fields["seq_end"] = int(regions[acc_attributes[0]][1])
                else:
                    fields["seq_start"] = 0
                    fields["seq_end"] = 0
                fields["mol_type"] = None
                if acc_meta.has_key("mol_type"):
                    fields["mol_type"] = acc_meta["mol_type"]
                fields["assembly_unit"] = acc_attributes[6]
                entry_date = datetime.datetime.now().strftime(
                    "%Y-%m-%d %H:%M:%S")
                fields["created"] = entry_date
                fields["updated"] = entry_date
            elif len(acc_attributes) < 7 and acc_attributes[0].find('.') != -1:
                fields = fetch_wgs_acc_metadata(acc_attributes[0])
        else:
            pass
        if len(fields.keys()) == 0:
            continue
        entry["fields"] = fields
        assembly_accs.append(entry)
        fields = {}
        entry = {}
    return assembly_accs
def fetch_assembly_accessions ( upid , gca_acc , acc_ftp_link , reg_ftp_link = None ) : assembly_accs = [ ] fields = { } entry = { } regions = None if reg_ftp_link is not None : regions = region_loader ( reg_ftp_link ) http_link = acc_ftp_link . replace ( "ftp://" , "http://" ) response = requests . get ( http_link ) . content acc_lines = response . strip ( ) . split ( '\n' ) acc_lines . pop ( 0 ) acc_attributes = '' for acc_line in acc_lines : acc_attributes = acc_line . strip ( ) . split ( '\t' ) if acc_attributes [ 0 ] . find ( '.' ) != - 1 : entry [ "model" ] = gc . GENSEQ_MODEL entry [ "pk" ] = str ( acc_attributes [ 0 ] ) if len ( acc_attributes ) == 7 : acc_meta = fetch_gca_acc_metadata ( acc_attributes [ 0 ] ) if len ( acc_meta . keys ( ) ) == 0 : continue fields [ "ncbi_id" ] = acc_meta [ "ncbi_id" ] fields [ "description" ] = acc_meta [ "description" ] fields [ "upid" ] = upid fields [ "gen_acc" ] = gca_acc fields [ "seq_version" ] = int ( acc_attributes [ 0 ] . partition ( '.' ) [ 2 ] ) if acc_attributes [ 2 ] != '' : fields [ "seq_length" ] = int ( acc_attributes [ 2 ] ) else : fields [ "seq_length" ] = 0 fields [ "seq_role" ] = acc_attributes [ 3 ] if reg_ftp_link is not None and ( acc_attributes [ 0 ] in regions . keys ( ) ) : fields [ "seq_start" ] = int ( regions [ acc_attributes [ 0 ] ] [ 0 ] ) fields [ "seq_end" ] = int ( regions [ acc_attributes [ 0 ] ] [ 1 ] ) else : fields [ "seq_start" ] = 0 fields [ "seq_end" ] = 0 fields [ "mol_type" ] = None if acc_meta . has_key ( "mol_type" ) : fields [ "mol_type" ] = acc_meta [ "mol_type" ] fields [ "assembly_unit" ] = acc_attributes [ 6 ] entry_date = datetime . datetime . now ( ) . strftime ( "%Y-%m-%d %H:%M:%S" ) fields [ "created" ] = entry_date fields [ "updated" ] = entry_date elif len ( acc_attributes ) < 7 and acc_attributes [ 0 ] . find ( '.' ) != - 1 : fields = fetch_wgs_acc_metadata ( acc_attributes [ 0 ] ) else : pass if len ( fields . keys ( ) ) == 0 : continue entry [ "fields" ] = fields assembly_accs . append ( entry ) fields = { } entry = { } return assembly_accs
Parses assembly report file and exports all assembly accessions in a
dict format to be easily loaded via Django ORM
Parses assembly report file and exports all assembly accessions in a dict format to be easily loaded via Django ORM
"""
    Parses assembly report file and exports all assembly accessions in a
    dict format to be easily loaded via Django ORM

    upid: A valid Uniprot proteome accession (e.g. UP000005640 - Homo Sapiens)
    gca_acc: A valid ENA GCA accession
    acc_ftp_link: Assembly report file ftp url (as retrieved from GCA xml file)
    reg_ftp_link: Assembly region file ftp url (as retrieved from GCA xml file)
    """ # load accession regions # remove header line # seq_acc # skip if accession was unavailable (e.g. obsolete) # should log upid and accession # patch, loci etc # need to parse the regions file for these fields # primary, PATCHES, ALT_REF_LOCI_1 # this takes the date of the entry is created # need to review this.. # skip cases no accession is provided # make sure to skip all cases of missing accessions
upid
gca_acc
acc_ftp_link
reg_ftp_link