def make_loci_and_stats(data, samples, ipyclient):
    """
    Makes the .loci file from h5 data base. Iterates by optim loci at a
    time and write to file. Also makes alleles file if requested.
    """
    ## start vcf progress bar
    start = time.time()
    printstr = " building loci/stats   | {} | s7 |"
    elapsed = datetime.timedelta(seconds=int(time.time()-start))
    progressbar(20, 0, printstr.format(elapsed), spacer=data._spacer)

    ## get some db info
    with h5py.File(data.clust_database, 'r') as io5:
        ## will iterate optim loci at a time
        optim = io5["seqs"].attrs["chunksize"][0]
        nloci = io5["seqs"].shape[0]
        anames = io5["seqs"].attrs["samples"]

    ## get name and snp padding
    pnames, snppad = padnames(anames)
    snames = [i.name for i in samples]
    smask = np.array([i not in snames for i in anames])

    ## keep track of how many loci from each sample pass all filters
    samplecov = np.zeros(len(anames), dtype=np.int32)

    ## set initial value to zero for all values above min_samples_locus
    #for cov in range(data.paramsdict["min_samples_locus"], len(anames)+1):
    locuscov = Counter()
    for cov in range(len(anames)+1):
        locuscov[cov] = 0

    ## client for sending jobs to parallel engines
    lbview = ipyclient.load_balanced_view()

    ## send jobs in chunks
    loci_asyncs = {}
    for istart in xrange(0, nloci, optim):
        args = [data, optim, pnames, snppad, smask, istart, samplecov, locuscov, 1]
        loci_asyncs[istart] = lbview.apply(locichunk, args)

    while 1:
        done = [i.ready() for i in loci_asyncs.values()]
        elapsed = datetime.timedelta(seconds=int(time.time()-start))
        progressbar(len(done), sum(done), printstr.format(elapsed), spacer=data._spacer)
        time.sleep(0.1)
        if len(done) == sum(done):
            print("")
            break

    ## check for errors
    for job in loci_asyncs:
        if loci_asyncs[job].ready() and not loci_asyncs[job].successful():
            LOGGER.error("error in building loci [%s]: %s",
                         job, loci_asyncs[job].exception())
            raise IPyradWarningExit(loci_asyncs[job].exception())

    ## concat and cleanup
    results = [i.get() for i in loci_asyncs.values()]
    ## update dictionaries
    for chunk in results:
        samplecov += chunk[0]
        locuscov.update(chunk[1])

    ## get all chunk files
    tmploci = glob.glob(data.outfiles.loci+".[0-9]*")
    ## sort by start value
    tmploci.sort(key=lambda x: int(x.split(".")[-1]))

    ## write tmpchunks to locus file
    locifile = open(data.outfiles.loci, 'w')
    for tmploc in tmploci:
        with open(tmploc, 'r') as inloc:
            locdat = inloc.read()
            locifile.write(locdat)
            os.remove(tmploc)
    locifile.close()

    ## make stats file from data
    make_stats(data, samples, samplecov, locuscov)

    ## repeat for alleles output
    if "a" in data.paramsdict["output_formats"]:

        loci_asyncs = {}
        for istart in xrange(0, nloci, optim):
            args = [data, optim, pnames, snppad, smask, istart, samplecov, locuscov, 0]
            loci_asyncs[istart] = lbview.apply(locichunk, args)

        while 1:
            done = [i.ready() for i in loci_asyncs.values()]
            elapsed = datetime.timedelta(seconds=int(time.time()-start))
            progressbar(len(done), sum(done),
                " building alleles      | {} | s7 |".format(elapsed), 
                spacer=data._spacer)
            time.sleep(0.1)
            if len(done) == sum(done):
                print("")
                break

        ## check for errors
        for job in loci_asyncs:
            if loci_asyncs[job].ready() and not loci_asyncs[job].successful():
                LOGGER.error("error in building alleles [%s]: %s",
                             job, loci_asyncs[job].exception())
                raise IPyradWarningExit(loci_asyncs[job].exception())

        ## concat and cleanup
        #results = [i.get() for i in loci_asyncs.values()]

        ## get all chunk files
        tmploci = glob.glob(data.outfiles.loci+".[0-9]*")
        ## sort by start value
        tmploci.sort(key=lambda x: int(x.split(".")[-1]))

        ## write tmpchunks to locus file
        locifile = open(data.outfiles.alleles, 'w')
        for tmploc in tmploci:
            with open(tmploc, 'r') as inloc:
                locdat = inloc.read()
                inalleles = get_alleles(locdat)
                locifile.write(inalleles)
                os.remove(tmploc)
        locifile.close()
def make_loci_and_stats(data, samples, ipyclient):
    """
    Makes the .loci file from h5 data base. Iterates by optim loci at a
    time and write to file. Also makes alleles file if requested.
    """
    ## start vcf progress bar
    start = time.time()
    printstr = " building loci/stats   | {} | s7 |"
    elapsed = datetime.timedelta(seconds=int(time.time()-start))
    progressbar(20, 0, printstr.format(elapsed), spacer=data._spacer)

    ## get some db info
    with h5py.File(data.clust_database, 'r') as io5:
        ## will iterate optim loci at a time
        optim = io5["seqs"].attrs["chunksize"][0]
        nloci = io5["seqs"].shape[0]
        anames = io5["seqs"].attrs["samples"]

    ## get name and snp padding
    pnames, snppad = padnames(anames)
    snames = [i.name for i in samples]
    smask = np.array([i not in snames for i in anames])

    ## keep track of how many loci from each sample pass all filters
    samplecov = np.zeros(len(anames), dtype=np.int32)

    ## set initial value to zero for all values above min_samples_locus
    #for cov in range(data.paramsdict["min_samples_locus"], len(anames)+1):
    locuscov = Counter()
    for cov in range(len(anames)+1):
        locuscov[cov] = 0

    ## client for sending jobs to parallel engines
    lbview = ipyclient.load_balanced_view()

    ## send jobs in chunks
    loci_asyncs = {}
    for istart in xrange(0, nloci, optim):
        args = [data, optim, pnames, snppad, smask, istart, samplecov, locuscov, 1]
        loci_asyncs[istart] = lbview.apply(locichunk, args)

    while 1:
        done = [i.ready() for i in loci_asyncs.values()]
        elapsed = datetime.timedelta(seconds=int(time.time()-start))
        progressbar(len(done), sum(done), printstr.format(elapsed), spacer=data._spacer)
        time.sleep(0.1)
        if len(done) == sum(done):
            print("")
            break

    ## check for errors
    for job in loci_asyncs:
        if loci_asyncs[job].ready() and not loci_asyncs[job].successful():
            LOGGER.error("error in building loci [%s]: %s",
                         job, loci_asyncs[job].exception())
            raise IPyradWarningExit(loci_asyncs[job].exception())

    ## concat and cleanup
    results = [i.get() for i in loci_asyncs.values()]
    ## update dictionaries
    for chunk in results:
        samplecov += chunk[0]
        locuscov.update(chunk[1])

    ## get all chunk files
    tmploci = glob.glob(data.outfiles.loci+".[0-9]*")
    ## sort by start value
    tmploci.sort(key=lambda x: int(x.split(".")[-1]))

    ## write tmpchunks to locus file
    locifile = open(data.outfiles.loci, 'w')
    for tmploc in tmploci:
        with open(tmploc, 'r') as inloc:
            locdat = inloc.read()
            locifile.write(locdat)
            os.remove(tmploc)
    locifile.close()

    ## make stats file from data
    make_stats(data, samples, samplecov, locuscov)

    ## repeat for alleles output
    if "a" in data.paramsdict["output_formats"]:

        loci_asyncs = {}
        for istart in xrange(0, nloci, optim):
            args = [data, optim, pnames, snppad, smask, istart, samplecov, locuscov, 0]
            loci_asyncs[istart] = lbview.apply(locichunk, args)

        while 1:
            done = [i.ready() for i in loci_asyncs.values()]
            elapsed = datetime.timedelta(seconds=int(time.time()-start))
            progressbar(len(done), sum(done),
                " building alleles      | {} | s7 |".format(elapsed), 
                spacer=data._spacer)
            time.sleep(0.1)
            if len(done) == sum(done):
                print("")
                break

        ## check for errors
        for job in loci_asyncs:
            if loci_asyncs[job].ready() and not loci_asyncs[job].successful():
                LOGGER.error("error in building alleles [%s]: %s",
                             job, loci_asyncs[job].exception())
                raise IPyradWarningExit(loci_asyncs[job].exception())

        ## concat and cleanup
        #results = [i.get() for i in loci_asyncs.values()]

        ## get all chunk files
        tmploci = glob.glob(data.outfiles.loci+".[0-9]*")
        ## sort by start value
        tmploci.sort(key=lambda x: int(x.split(".")[-1]))

        ## write tmpchunks to locus file
        locifile = open(data.outfiles.alleles, 'w')
        for tmploc in tmploci:
            with open(tmploc, 'r') as inloc:
                locdat = inloc.read()
                inalleles = get_alleles(locdat)
                locifile.write(inalleles)
                os.remove(tmploc)
        locifile.close()
def make_loci_and_stats ( data , samples , ipyclient ) : ## start vcf progress bar start = time . time ( ) printstr = " building loci/stats   | {} | s7 |" elapsed = datetime . timedelta ( seconds = int ( time . time ( ) - start ) ) progressbar ( 20 , 0 , printstr . format ( elapsed ) , spacer = data . _spacer ) ## get some db info with h5py . File ( data . clust_database , 'r' ) as io5 : ## will iterate optim loci at a time optim = io5 [ "seqs" ] . attrs [ "chunksize" ] [ 0 ] nloci = io5 [ "seqs" ] . shape [ 0 ] anames = io5 [ "seqs" ] . attrs [ "samples" ] ## get name and snp padding pnames , snppad = padnames ( anames ) snames = [ i . name for i in samples ] smask = np . array ( [ i not in snames for i in anames ] ) ## keep track of how many loci from each sample pass all filters samplecov = np . zeros ( len ( anames ) , dtype = np . int32 ) ## set initial value to zero for all values above min_samples_locus #for cov in range(data.paramsdict["min_samples_locus"], len(anames)+1): locuscov = Counter ( ) for cov in range ( len ( anames ) + 1 ) : locuscov [ cov ] = 0 ## client for sending jobs to parallel engines lbview = ipyclient . load_balanced_view ( ) ## send jobs in chunks loci_asyncs = { } for istart in xrange ( 0 , nloci , optim ) : args = [ data , optim , pnames , snppad , smask , istart , samplecov , locuscov , 1 ] loci_asyncs [ istart ] = lbview . apply ( locichunk , args ) while 1 : done = [ i . ready ( ) for i in loci_asyncs . values ( ) ] elapsed = datetime . timedelta ( seconds = int ( time . time ( ) - start ) ) progressbar ( len ( done ) , sum ( done ) , printstr . format ( elapsed ) , spacer = data . _spacer ) time . sleep ( 0.1 ) if len ( done ) == sum ( done ) : print ( "" ) break ## check for errors for job in loci_asyncs : if loci_asyncs [ job ] . ready ( ) and not loci_asyncs [ job ] . successful ( ) : LOGGER . error ( "error in building loci [%s]: %s" , job , loci_asyncs [ job ] . exception ( ) ) raise IPyradWarningExit ( loci_asyncs [ job ] . exception ( ) ) ## concat and cleanup results = [ i . get ( ) for i in loci_asyncs . values ( ) ] ## update dictionaries for chunk in results : samplecov += chunk [ 0 ] locuscov . update ( chunk [ 1 ] ) ## get all chunk files tmploci = glob . glob ( data . outfiles . loci + ".[0-9]*" ) ## sort by start value tmploci . sort ( key = lambda x : int ( x . split ( "." ) [ - 1 ] ) ) ## write tmpchunks to locus file locifile = open ( data . outfiles . loci , 'w' ) for tmploc in tmploci : with open ( tmploc , 'r' ) as inloc : locdat = inloc . read ( ) locifile . write ( locdat ) os . remove ( tmploc ) locifile . close ( ) ## make stats file from data make_stats ( data , samples , samplecov , locuscov ) ## repeat for alleles output if "a" in data . paramsdict [ "output_formats" ] : loci_asyncs = { } for istart in xrange ( 0 , nloci , optim ) : args = [ data , optim , pnames , snppad , smask , istart , samplecov , locuscov , 0 ] loci_asyncs [ istart ] = lbview . apply ( locichunk , args ) while 1 : done = [ i . ready ( ) for i in loci_asyncs . values ( ) ] elapsed = datetime . timedelta ( seconds = int ( time . time ( ) - start ) ) progressbar ( len ( done ) , sum ( done ) , " building alleles      | {} | s7 |" . format ( elapsed ) , spacer = data . _spacer ) time . sleep ( 0.1 ) if len ( done ) == sum ( done ) : print ( "" ) break ## check for errors for job in loci_asyncs : if loci_asyncs [ job ] . ready ( ) and not loci_asyncs [ job ] . successful ( ) : LOGGER . error ( "error in building alleles [%s]: %s" , job , loci_asyncs [ job ] . exception ( ) ) raise IPyradWarningExit ( loci_asyncs [ job ] . exception ( ) ) ## concat and cleanup #results = [i.get() for i in loci_asyncs.values()] ## get all chunk files tmploci = glob . glob ( data . outfiles . loci + ".[0-9]*" ) ## sort by start value tmploci . sort ( key = lambda x : int ( x . split ( "." ) [ - 1 ] ) ) ## write tmpchunks to locus file locifile = open ( data . outfiles . alleles , 'w' ) for tmploc in tmploci : with open ( tmploc , 'r' ) as inloc : locdat = inloc . read ( ) inalleles = get_alleles ( locdat ) locifile . write ( inalleles ) os . remove ( tmploc ) locifile . close ( )
Makes the .loci file from h5 data base. Iterates by optim loci at a
    time and write to file. Also makes alleles file if requested.
Makes the . loci file from h5 data base . Iterates by optim loci at a time and write to file . Also makes alleles file if requested .