JoshuaTree2.py
def circos_dropper(fasta_path, gff_path, synteny_path, bed_path, circos_inputs, circos_outputs, loci_threshold,gene_info, n_cpus):
    """Visualize many pairwise synteny results. If synteny files are not supplied, conduct pairwise synteny between all included strains."""
    fasta_files = {fasta.split('_')[-2] : fasta for fasta in glob.glob(fasta_path+'/*.fa')+glob.glob(fasta_path+'/*.fasta')}
    gff_files = {gff.split('.')[-2] : gff for gff in glob.glob(gff_path+'/*.gff')+glob.glob(gff_path+'/*.gff3') }
    intersect_keys = set(fasta_files.keys()) & set(gff_files.keys())
    fasta_files = {protID:fasta for protID,fasta in fasta_files.items() if protID in intersect_keys}
    gff_files = {protID:gff for protID,gff in gff_files.items() if protID in intersect_keys}
    genomes = {}
    pairwise_syntenies = []
    print gff_files, fasta_files
    for protID in intersect_keys:
        genomes[protID] = Genome(fasta_files[protID],bed_path+'/'+protID+'.bed',protID,gff_files[protID],gene_info)
        genomes[protID].export_karyotype(circos_inputs+'/'+protID+'.karyotype.txt')
    print genomes
    synteny_files = glob.glob(synteny_path+'/*.unout')+glob.glob(synteny_path+'/*.lifted.anchors')
    synteny_protIDs = []
    if synteny_files:
        for synteny_file in synteny_files:
            if synteny_file.endswith('.unout'):
                coords = reduce(lambda x,y: x+y, sorted([[m.start(0),m.end(0)] for m in re.finditer('PAC2_0|PAC4GC',synteny_file)]))[1::2]
                q_protID, s_prot_ID = map(lambda x: synteny_file[x+1:x+4],coords)
            else:
                q_protID, s_prot_ID = tuple(synteny_file[synteny_file.rfind('/')+1:].split('.')[:2])
            synteny_protIDs.extend([(q_protID, s_prot_ID),(s_prot_ID, q_protID)])
            pairwise_synteny = PairwiseSynteny(genomes[q_protID],genomes[s_prot_ID],synteny_file,loci_threshold=loci_threshold)
            pairwise_synteny.generate_synteny_structure(synteny_path)
            pairwise_syntenies.append(pairwise_synteny)
        remaining_synteny = set(list(combinations(intersect_keys,r=2))) - set(synteny_protIDs)
    else:
        remaining_synteny = list(combinations(intersect_keys,r=2))
    if remaining_synteny:
        #print remaining_synteny

        def generate_CDS(protID):
            print(protID)
            genomes[protID].extract_CDS()
            return protID
        p = mp.ProcessingPool(n_cpus)
        r = p.amap(generate_CDS,list(set(reduce(lambda x,y: list(x)+list(y),remaining_synteny))))
        r.wait()
        protIDs = r.get()
        print(protIDs)
        #p = mp.ProcessingPool(ncpus=n_cpus)
        #p.daemon = True
        #r = p.amap(generate_CDS,list(set(reduce(lambda x,y: list(x)+list(y),remaining_synteny)))) # get pathos multiprocessing https://github.com/uqfoundation/pathos
        #while not r.ready():
        #    sleep(5)
        #r.wait()
        """
        for protID in set(reduce(lambda x,y: list(x)+list(y),remaining_synteny)):
            proc = mp.Process(target=lambda: genomes[protID].extract_CDS, args=None)
            proc.daemon = True
            proc.start()
            while len(mp.active_childern()) > n_cpus:
                sleep(1)
        while len(mp.active_childern()) > 0:
            sleep(1)"""
        # fixme add remaining prot ID feauture

        #def mycallback(x):
        #    pairwise_syntenies.extend(x)

        def p_synteny(protIDs):
            print(protIDs)
            q_protID, s_prot_ID = protIDs
            pairwise_synteny = PairwiseSynteny(genomes[q_protID],genomes[s_prot_ID],loci_threshold=loci_threshold)
            pairwise_synteny.generate_synteny_structure(synteny_path)
            return pairwise_synteny

        """
        for q_protID, s_prot_ID in combinations(genomes.keys(),r=2):
            pairwise_synteny = PairwiseSynteny(genomes[q_protID],genomes[s_prot_ID],loci_threshold=loci_threshold)
            pairwise_synteny.generate_synteny_structure(synteny_path)
            pairwise_syntenies.append(pairwise_synteny)"""
        r = p.amap(p_synteny,remaining_synteny)#,callback=mycallback) # _async
        r.wait()
        pairwise_syntenies.extend(r.get())
        p.close()


    for pairwise_synteny in pairwise_syntenies:
        pairwise_synteny.synteny_structure_2_link(circos_inputs+'/%s.%s.link.txt'%(pairwise_synteny.q_genome.protID,pairwise_synteny.s_genome.protID))
        circos_obj = Circos(pairwise_synteny)
        circos_obj.generate_config(ticks = circos_inputs+'/txticks.conf', ideogram = circos_inputs+'/txideogram.conf', links_and_rules = circos_inputs+'/linksAndrules.conf', config=circos_inputs+'/circos.conf')
        circos_obj.run_circos(circos_outputs+'/')
Visualize many pairwise synteny results. If synteny files are not supplied, conduct pairwise synteny between all included strains.
Visualize many pairwise synteny results. If synteny files are not supplied, conduct pairwise synteny between all included strains.
Visualize many pairwise synteny results . If synteny files are not supplied conduct pairwise synteny between all included strains .
def circos_dropper(fasta_path, gff_path, synteny_path, bed_path, circos_inputs, circos_outputs, loci_threshold,gene_info, n_cpus):
    fasta_files = {fasta.split('_')[-2] : fasta for fasta in glob.glob(fasta_path+'/*.fa')+glob.glob(fasta_path+'/*.fasta')}
    gff_files = {gff.split('.')[-2] : gff for gff in glob.glob(gff_path+'/*.gff')+glob.glob(gff_path+'/*.gff3') }
    intersect_keys = set(fasta_files.keys()) & set(gff_files.keys())
    fasta_files = {protID:fasta for protID,fasta in fasta_files.items() if protID in intersect_keys}
    gff_files = {protID:gff for protID,gff in gff_files.items() if protID in intersect_keys}
    genomes = {}
    pairwise_syntenies = []
    print gff_files, fasta_files
    for protID in intersect_keys:
        genomes[protID] = Genome(fasta_files[protID],bed_path+'/'+protID+'.bed',protID,gff_files[protID],gene_info)
        genomes[protID].export_karyotype(circos_inputs+'/'+protID+'.karyotype.txt')
    print genomes
    synteny_files = glob.glob(synteny_path+'/*.unout')+glob.glob(synteny_path+'/*.lifted.anchors')
    synteny_protIDs = []
    if synteny_files:
        for synteny_file in synteny_files:
            if synteny_file.endswith('.unout'):
                coords = reduce(lambda x,y: x+y, sorted([[m.start(0),m.end(0)] for m in re.finditer('PAC2_0|PAC4GC',synteny_file)]))[1::2]
                q_protID, s_prot_ID = map(lambda x: synteny_file[x+1:x+4],coords)
            else:
                q_protID, s_prot_ID = tuple(synteny_file[synteny_file.rfind('/')+1:].split('.')[:2])
            synteny_protIDs.extend([(q_protID, s_prot_ID),(s_prot_ID, q_protID)])
            pairwise_synteny = PairwiseSynteny(genomes[q_protID],genomes[s_prot_ID],synteny_file,loci_threshold=loci_threshold)
            pairwise_synteny.generate_synteny_structure(synteny_path)
            pairwise_syntenies.append(pairwise_synteny)
        remaining_synteny = set(list(combinations(intersect_keys,r=2))) - set(synteny_protIDs)
    else:
        remaining_synteny = list(combinations(intersect_keys,r=2))
    if remaining_synteny:
        def generate_CDS(protID):
            print(protID)
            genomes[protID].extract_CDS()
            return protID
        p = mp.ProcessingPool(n_cpus)
        r = p.amap(generate_CDS,list(set(reduce(lambda x,y: list(x)+list(y),remaining_synteny))))
        r.wait()
        protIDs = r.get()
        print(protIDs)
        def p_synteny(protIDs):
            print(protIDs)
            q_protID, s_prot_ID = protIDs
            pairwise_synteny = PairwiseSynteny(genomes[q_protID],genomes[s_prot_ID],loci_threshold=loci_threshold)
            pairwise_synteny.generate_synteny_structure(synteny_path)
            return pairwise_synteny
        r = p.amap(p_synteny,remaining_synteny)
        r.wait()
        pairwise_syntenies.extend(r.get())
        p.close()
    for pairwise_synteny in pairwise_syntenies:
        pairwise_synteny.synteny_structure_2_link(circos_inputs+'/%s.%s.link.txt'%(pairwise_synteny.q_genome.protID,pairwise_synteny.s_genome.protID))
        circos_obj = Circos(pairwise_synteny)
        circos_obj.generate_config(ticks = circos_inputs+'/txticks.conf', ideogram = circos_inputs+'/txideogram.conf', links_and_rules = circos_inputs+'/linksAndrules.conf', config=circos_inputs+'/circos.conf')
        circos_obj.run_circos(circos_outputs+'/')
def circos_dropper ( fasta_path , gff_path , synteny_path , bed_path , circos_inputs , circos_outputs , loci_threshold , gene_info , n_cpus ) : fasta_files = { fasta . split ( '_' ) [ - 2 ] : fasta for fasta in glob . glob ( fasta_path + '/*.fa' ) + glob . glob ( fasta_path + '/*.fasta' ) } gff_files = { gff . split ( '.' ) [ - 2 ] : gff for gff in glob . glob ( gff_path + '/*.gff' ) + glob . glob ( gff_path + '/*.gff3' ) } intersect_keys = set ( fasta_files . keys ( ) ) & set ( gff_files . keys ( ) ) fasta_files = { protID : fasta for protID , fasta in fasta_files . items ( ) if protID in intersect_keys } gff_files = { protID : gff for protID , gff in gff_files . items ( ) if protID in intersect_keys } genomes = { } pairwise_syntenies = [ ] print gff_files , fasta_files for protID in intersect_keys : genomes [ protID ] = Genome ( fasta_files [ protID ] , bed_path + '/' + protID + '.bed' , protID , gff_files [ protID ] , gene_info ) genomes [ protID ] . export_karyotype ( circos_inputs + '/' + protID + '.karyotype.txt' ) print genomes synteny_files = glob . glob ( synteny_path + '/*.unout' ) + glob . glob ( synteny_path + '/*.lifted.anchors' ) synteny_protIDs = [ ] if synteny_files : for synteny_file in synteny_files : if synteny_file . endswith ( '.unout' ) : coords = reduce ( lambda x , y : x + y , sorted ( [ [ m . start ( 0 ) , m . end ( 0 ) ] for m in re . finditer ( 'PAC2_0|PAC4GC' , synteny_file ) ] ) ) [ 1 : : 2 ] q_protID , s_prot_ID = map ( lambda x : synteny_file [ x + 1 : x + 4 ] , coords ) else : q_protID , s_prot_ID = tuple ( synteny_file [ synteny_file . rfind ( '/' ) + 1 : ] . split ( '.' ) [ : 2 ] ) synteny_protIDs . extend ( [ ( q_protID , s_prot_ID ) , ( s_prot_ID , q_protID ) ] ) pairwise_synteny = PairwiseSynteny ( genomes [ q_protID ] , genomes [ s_prot_ID ] , synteny_file , loci_threshold = loci_threshold ) pairwise_synteny . generate_synteny_structure ( synteny_path ) pairwise_syntenies . append ( pairwise_synteny ) remaining_synteny = set ( list ( combinations ( intersect_keys , r = 2 ) ) ) - set ( synteny_protIDs ) else : remaining_synteny = list ( combinations ( intersect_keys , r = 2 ) ) if remaining_synteny : def generate_CDS ( protID ) : print ( protID ) genomes [ protID ] . extract_CDS ( ) return protID p = mp . ProcessingPool ( n_cpus ) r = p . amap ( generate_CDS , list ( set ( reduce ( lambda x , y : list ( x ) + list ( y ) , remaining_synteny ) ) ) ) r . wait ( ) protIDs = r . get ( ) print ( protIDs ) """
        for protID in set(reduce(lambda x,y: list(x)+list(y),remaining_synteny)):
            proc = mp.Process(target=lambda: genomes[protID].extract_CDS, args=None)
            proc.daemon = True
            proc.start()
            while len(mp.active_childern()) > n_cpus:
                sleep(1)
        while len(mp.active_childern()) > 0:
            sleep(1)""" def p_synteny ( protIDs ) : print ( protIDs ) q_protID , s_prot_ID = protIDs pairwise_synteny = PairwiseSynteny ( genomes [ q_protID ] , genomes [ s_prot_ID ] , loci_threshold = loci_threshold ) pairwise_synteny . generate_synteny_structure ( synteny_path ) return pairwise_synteny """
        for q_protID, s_prot_ID in combinations(genomes.keys(),r=2):
            pairwise_synteny = PairwiseSynteny(genomes[q_protID],genomes[s_prot_ID],loci_threshold=loci_threshold)
            pairwise_synteny.generate_synteny_structure(synteny_path)
            pairwise_syntenies.append(pairwise_synteny)""" r = p . amap ( p_synteny , remaining_synteny ) r . wait ( ) pairwise_syntenies . extend ( r . get ( ) ) p . close ( ) for pairwise_synteny in pairwise_syntenies : pairwise_synteny . synteny_structure_2_link ( circos_inputs + '/%s.%s.link.txt' % ( pairwise_synteny . q_genome . protID , pairwise_synteny . s_genome . protID ) ) circos_obj = Circos ( pairwise_synteny ) circos_obj . generate_config ( ticks = circos_inputs + '/txticks.conf' , ideogram = circos_inputs + '/txideogram.conf' , links_and_rules = circos_inputs + '/linksAndrules.conf' , config = circos_inputs + '/circos.conf' ) circos_obj . run_circos ( circos_outputs + '/' )
Visualize many pairwise synteny results.
Visualize many pairwise synteny results .
"""Visualize many pairwise synteny results. If synteny files are not supplied, conduct pairwise synteny between all included strains.""" #print remaining_synteny #p = mp.ProcessingPool(ncpus=n_cpus) #p.daemon = True #r = p.amap(generate_CDS,list(set(reduce(lambda x,y: list(x)+list(y),remaining_synteny)))) # get pathos multiprocessing https://github.com/uqfoundation/pathos #while not r.ready(): #    sleep(5) #r.wait() """
        for protID in set(reduce(lambda x,y: list(x)+list(y),remaining_synteny)):
            proc = mp.Process(target=lambda: genomes[protID].extract_CDS, args=None)
            proc.daemon = True
            proc.start()
            while len(mp.active_childern()) > n_cpus:
                sleep(1)
        while len(mp.active_childern()) > 0:
            sleep(1)""" # fixme add remaining prot ID feauture #def mycallback(x): #    pairwise_syntenies.extend(x) """
        for q_protID, s_prot_ID in combinations(genomes.keys(),r=2):
            pairwise_synteny = PairwiseSynteny(genomes[q_protID],genomes[s_prot_ID],loci_threshold=loci_threshold)
            pairwise_synteny.generate_synteny_structure(synteny_path)
            pairwise_syntenies.append(pairwise_synteny)""" #,callback=mycallback) # _async
fasta_path
gff_path
synteny_path
bed_path
circos_inputs
circos_outputs
loci_threshold
gene_info
n_cpus