scripts/export/genomes/fetch_gen_metadata.py
def fetch_gca_data(upid, assembly_acc, kingdom):
    """
    Parses ENA GCA accession xml, and returns the accession's data in the
    form of a dictionary

    upid: A valid Uniprot proteome accession (e.g. UP000005640 - Homo Sapiens)
    assembly_acc: A valid ENA GCA accession
    kingdom: The corresponding species kingdom
    """

    genome_entry = {}
    fields = {}
    tmp_acc = assembly_acc

    # check status
    response = requests.get(gc.ENA_XML_URL % tmp_acc)

    # need to do this repeatedly
    if response.status_code == httplib.OK:
        assembly_xml = response.content

        root = ET.fromstring(assembly_xml)
        assembly = root.find("ASSEMBLY")

        if assembly is None:
            return genome_entry

        primary_id = assembly.find("IDENTIFIERS").find("PRIMARY_ID")

        chromosomes = assembly.find("CHROMOSOMES")
        fields['chromosomes'] = []
        if chromosomes is not None:
            for chromosome in chromosomes:
                fields['chromosomes'].append({
                    'accession': chromosome.attrib['accession'],
                    'name': chromosome.find('NAME').text if chromosome.find('NAME') is not None else None,
                    'type': chromosome.find('TYPE').text if chromosome.find('TYPE') is not None else None,
                })

        if primary_id is not None:
            fields["gca_acc"] = primary_id.text
        else:
            fields["gca_acc"] = primary_id

        version = fields["gca_acc"].partition('.')[2]
        fields["gca_version"] = int(version)

        # add ensembl fields as a post-processing step
        fields["ensembl_id"] = None
        fields["ensembl_source"] = None

        fields["assembly_name"] = assembly.find("NAME").text

        assembly_links = None
        assembly_level = assembly.find("ASSEMBLY_LEVEL").text
        assembly_links = assembly.find("ASSEMBLY_LINKS")

        if assembly_level == "complete genome":
            assembly_level = assembly_level.replace(' ', '-')

        fields["assembly_level"] = assembly_level

        if assembly_level == "contig" and assembly_links is None:
            wgs_fields = assembly.find("WGS_SET")
            wgs_acc = gf.get_wgs_set_accession(
                wgs_fields.find("PREFIX").text, wgs_fields.find("VERSION").text)

            fields["wgs_acc"] = wgs_acc
            fields["wgs_version"] = int(wgs_fields.find("VERSION").text)
        else:
            fields["wgs_acc"] = None
            fields["wgs_version"] = None

        fields["study_ref"] = None
        if assembly.find("STUDY_REF") is not None:
            fields["study_ref"] = assembly.find("STUDY_REF").find("IDENTIFIERS").find("PRIMARY_ID").text

        # description can be very long, using title instead as short description
        fields["description"] = assembly.find("TITLE").text

        attributes = fetch_assembly_attributes(
            assembly.find("ASSEMBLY_ATTRIBUTES"))

        fields["total_length"] = int(attributes["total-length"])
        fields["ungapped_length"] = int(attributes["ungapped-length"])

        genome_desc = assembly.find("DESCRIPTION")

        if genome_desc is not None:
            # try to fetch description text
            genome_desc = genome_desc.text
            if genome_desc is not None:
                if genome_desc.find("circular") != -1:
                    fields["circular"] = 1
                else:
                    fields["circular"] = 0
        else:
            fields["circular"] = 0

        taxid = assembly.find("TAXON")
        fields["ncbi_id"] = int(taxid.find("TAXON_ID").text)

        fields["scientific_name"] = taxid.find("SCIENTIFIC_NAME").text

        common_name = None
        common_name = taxid.find("COMMON_NAME")
        if common_name is not None:
            fields["common_name"] = taxid.find("COMMON_NAME").text
        fields["kingdom"] = kingdom

        fields["num_gen_regions"] = int(attributes["count-regions"])
        fields["num_rfam_regions"] = None
        fields["num_families"] = None

        # this takes the date of the entry is created
        entry_date = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
        fields["created"] = entry_date
        fields["updated"] = entry_date

        genome_entry["model"] = gc.GENOME_MODEL
        genome_entry["pk"] = upid
        genome_entry["fields"] = fields

    else:
        print ("GCA xml unavailable for %s" % upid)

    return genome_entry

    Parses ENA GCA accession xml, and returns the accession's data in the
    form of a dictionary

    upid: A valid Uniprot proteome accession (e.g. UP000005640 - Homo Sapiens)
    assembly_acc: A valid ENA GCA accession
    kingdom: The corresponding species kingdom
    
Parses ENA GCA accession xml, and returns the accession's data in the
form of a dictionary

A valid Uniprot proteome accession
assembly_acc: A valid ENA GCA accession
kingdom: The corresponding species kingdom
Parses ENA GCA accession xml and returns the accession ' s data in the form of a dictionary A valid Uniprot proteome accession assembly_acc : A valid ENA GCA accession kingdom : The corresponding species kingdom
def fetch_gca_data(upid, assembly_acc, kingdom):
    genome_entry = {}
    fields = {}
    tmp_acc = assembly_acc
    response = requests.get(gc.ENA_XML_URL % tmp_acc)
    if response.status_code == httplib.OK:
        assembly_xml = response.content
        root = ET.fromstring(assembly_xml)
        assembly = root.find("ASSEMBLY")
        if assembly is None:
            return genome_entry
        primary_id = assembly.find("IDENTIFIERS").find("PRIMARY_ID")
        chromosomes = assembly.find("CHROMOSOMES")
        fields['chromosomes'] = []
        if chromosomes is not None:
            for chromosome in chromosomes:
                fields['chromosomes'].append({
                    'accession': chromosome.attrib['accession'],
                    'name': chromosome.find('NAME').text if chromosome.find('NAME') is not None else None,
                    'type': chromosome.find('TYPE').text if chromosome.find('TYPE') is not None else None,
                })
        if primary_id is not None:
            fields["gca_acc"] = primary_id.text
        else:
            fields["gca_acc"] = primary_id
        version = fields["gca_acc"].partition('.')[2]
        fields["gca_version"] = int(version)
        fields["ensembl_id"] = None
        fields["ensembl_source"] = None
        fields["assembly_name"] = assembly.find("NAME").text
        assembly_links = None
        assembly_level = assembly.find("ASSEMBLY_LEVEL").text
        assembly_links = assembly.find("ASSEMBLY_LINKS")
        if assembly_level == "complete genome":
            assembly_level = assembly_level.replace(' ', '-')
        fields["assembly_level"] = assembly_level
        if assembly_level == "contig" and assembly_links is None:
            wgs_fields = assembly.find("WGS_SET")
            wgs_acc = gf.get_wgs_set_accession(
                wgs_fields.find("PREFIX").text, wgs_fields.find("VERSION").text)
            fields["wgs_acc"] = wgs_acc
            fields["wgs_version"] = int(wgs_fields.find("VERSION").text)
        else:
            fields["wgs_acc"] = None
            fields["wgs_version"] = None
        fields["study_ref"] = None
        if assembly.find("STUDY_REF") is not None:
            fields["study_ref"] = assembly.find("STUDY_REF").find("IDENTIFIERS").find("PRIMARY_ID").text
        fields["description"] = assembly.find("TITLE").text
        attributes = fetch_assembly_attributes(
            assembly.find("ASSEMBLY_ATTRIBUTES"))
        fields["total_length"] = int(attributes["total-length"])
        fields["ungapped_length"] = int(attributes["ungapped-length"])
        genome_desc = assembly.find("DESCRIPTION")
        if genome_desc is not None:
            genome_desc = genome_desc.text
            if genome_desc is not None:
                if genome_desc.find("circular") != -1:
                    fields["circular"] = 1
                else:
                    fields["circular"] = 0
        else:
            fields["circular"] = 0
        taxid = assembly.find("TAXON")
        fields["ncbi_id"] = int(taxid.find("TAXON_ID").text)
        fields["scientific_name"] = taxid.find("SCIENTIFIC_NAME").text
        common_name = None
        common_name = taxid.find("COMMON_NAME")
        if common_name is not None:
            fields["common_name"] = taxid.find("COMMON_NAME").text
        fields["kingdom"] = kingdom
        fields["num_gen_regions"] = int(attributes["count-regions"])
        fields["num_rfam_regions"] = None
        fields["num_families"] = None
        entry_date = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
        fields["created"] = entry_date
        fields["updated"] = entry_date
        genome_entry["model"] = gc.GENOME_MODEL
        genome_entry["pk"] = upid
        genome_entry["fields"] = fields
    else:
        print ("GCA xml unavailable for %s" % upid)
    return genome_entry
def fetch_gca_data ( upid , assembly_acc , kingdom ) : genome_entry = { } fields = { } tmp_acc = assembly_acc response = requests . get ( gc . ENA_XML_URL % tmp_acc ) if response . status_code == httplib . OK : assembly_xml = response . content root = ET . fromstring ( assembly_xml ) assembly = root . find ( "ASSEMBLY" ) if assembly is None : return genome_entry primary_id = assembly . find ( "IDENTIFIERS" ) . find ( "PRIMARY_ID" ) chromosomes = assembly . find ( "CHROMOSOMES" ) fields [ 'chromosomes' ] = [ ] if chromosomes is not None : for chromosome in chromosomes : fields [ 'chromosomes' ] . append ( { 'accession' : chromosome . attrib [ 'accession' ] , 'name' : chromosome . find ( 'NAME' ) . text if chromosome . find ( 'NAME' ) is not None else None , 'type' : chromosome . find ( 'TYPE' ) . text if chromosome . find ( 'TYPE' ) is not None else None , } ) if primary_id is not None : fields [ "gca_acc" ] = primary_id . text else : fields [ "gca_acc" ] = primary_id version = fields [ "gca_acc" ] . partition ( '.' ) [ 2 ] fields [ "gca_version" ] = int ( version ) fields [ "ensembl_id" ] = None fields [ "ensembl_source" ] = None fields [ "assembly_name" ] = assembly . find ( "NAME" ) . text assembly_links = None assembly_level = assembly . find ( "ASSEMBLY_LEVEL" ) . text assembly_links = assembly . find ( "ASSEMBLY_LINKS" ) if assembly_level == "complete genome" : assembly_level = assembly_level . replace ( ' ' , '-' ) fields [ "assembly_level" ] = assembly_level if assembly_level == "contig" and assembly_links is None : wgs_fields = assembly . find ( "WGS_SET" ) wgs_acc = gf . get_wgs_set_accession ( wgs_fields . find ( "PREFIX" ) . text , wgs_fields . find ( "VERSION" ) . text ) fields [ "wgs_acc" ] = wgs_acc fields [ "wgs_version" ] = int ( wgs_fields . find ( "VERSION" ) . text ) else : fields [ "wgs_acc" ] = None fields [ "wgs_version" ] = None fields [ "study_ref" ] = None if assembly . find ( "STUDY_REF" ) is not None : fields [ "study_ref" ] = assembly . find ( "STUDY_REF" ) . find ( "IDENTIFIERS" ) . find ( "PRIMARY_ID" ) . text fields [ "description" ] = assembly . find ( "TITLE" ) . text attributes = fetch_assembly_attributes ( assembly . find ( "ASSEMBLY_ATTRIBUTES" ) ) fields [ "total_length" ] = int ( attributes [ "total-length" ] ) fields [ "ungapped_length" ] = int ( attributes [ "ungapped-length" ] ) genome_desc = assembly . find ( "DESCRIPTION" ) if genome_desc is not None : genome_desc = genome_desc . text if genome_desc is not None : if genome_desc . find ( "circular" ) != - 1 : fields [ "circular" ] = 1 else : fields [ "circular" ] = 0 else : fields [ "circular" ] = 0 taxid = assembly . find ( "TAXON" ) fields [ "ncbi_id" ] = int ( taxid . find ( "TAXON_ID" ) . text ) fields [ "scientific_name" ] = taxid . find ( "SCIENTIFIC_NAME" ) . text common_name = None common_name = taxid . find ( "COMMON_NAME" ) if common_name is not None : fields [ "common_name" ] = taxid . find ( "COMMON_NAME" ) . text fields [ "kingdom" ] = kingdom fields [ "num_gen_regions" ] = int ( attributes [ "count-regions" ] ) fields [ "num_rfam_regions" ] = None fields [ "num_families" ] = None entry_date = datetime . datetime . now ( ) . strftime ( "%Y-%m-%d %H:%M:%S" ) fields [ "created" ] = entry_date fields [ "updated" ] = entry_date genome_entry [ "model" ] = gc . GENOME_MODEL genome_entry [ "pk" ] = upid genome_entry [ "fields" ] = fields else : print ( "GCA xml unavailable for %s" % upid ) return genome_entry
Parses ENA GCA accession xml, and returns the accession's data in the
form of a dictionary
Parses ENA GCA accession xml and returns the accession ' s data in the form of a dictionary
"""
    Parses ENA GCA accession xml, and returns the accession's data in the
    form of a dictionary

    upid: A valid Uniprot proteome accession (e.g. UP000005640 - Homo Sapiens)
    assembly_acc: A valid ENA GCA accession
    kingdom: The corresponding species kingdom
    """ # check status # need to do this repeatedly # add ensembl fields as a post-processing step # description can be very long, using title instead as short description # try to fetch description text # this takes the date of the entry is created
upid
assembly_acc
kingdom