ultraplex/__main__.py
def concatenate_files(save_name, ultra_mode,
                      sbatch_compression,
                      output_directory,
                      sample_names,
                      compression_threads=8):
    """
	this function concatenates all the files produced by the 
	different workers, then sends an sbatch command to compress
	them all to fastqs.
	"""
    # First, file all the unique file names we have, ignoring threads
    all_names = glob.glob(output_directory + "ultraplex_" + save_name + '*')

    all_types = []  # ignoring threads
    for name in all_names:
        this_type = name.split("_tmp_thread_")[0]

        if this_type not in all_types:
            all_types.append(this_type)  # this_type contains directory if applicable

    # now concatenate them
    if ultra_mode:
        for this_type in all_types:
            # find all files with this barcode (or barcode combination)
            filenames = sorted(glob.glob(this_type + '*'))  # this type already has output directory
            # then concatenate
            command = ''
            for name in filenames:
                command = command + name + ' '
            command = 'cat ' + command + ' > ' + this_type + '.fastq'
            os.system(command)

            for name in filenames:
                os.remove(name)

            print("Compressing with pigz...")
            c_thread_n = '-p' + str(compression_threads)
            if sbatch_compression:
                os.system(
                    'sbatch -J compression -c ' + str(
                        compression_threads) + ' --time 4:00:00 --wrap="pigz ' + c_thread_n + ' ' + this_type + '.fastq"')
            else:
                os.system('pigz ' + c_thread_n + ' ' + this_type + '.fastq')

        # check if compression is complete
        if sbatch_compression:
            finished = False
            print("Compressing....")
            while not finished:
                # assume it's complete
                complete = True
                # now actually check
                for this_type in all_types:
                    filename = glob.glob(this_type + '*')

                    if '.gz' not in filename[0]:
                        complete = False

                if complete:
                    finished = True
                    print("Compression complete!")
                else:
                    time.sleep(1)
    else:  # if not ultra_mode
        for this_type in all_types:
            # find all files with this barcode (or barcode combination)
            filenames = sorted(glob.glob(this_type + '*'))
            # then concatenate
            command = ''
            for name in filenames:
                command = command + name + ' '
            command = 'cat ' + command + ' > ' + this_type + '.fastq.gz'
            os.system(command)

            for name in filenames:
                os.remove(name)

    if len(sample_names) > 0:  # ie the user has specified at least one sample name
        for this_type in all_types:
            sample_name_list = []
            # check whether any of the keys in the dictionary match this type
            for key, value in sample_names.items():
                if key in this_type and "no_match" not in this_type:
                    # check that if key doesn't contain 3' barcode, then the type does either
                    if "_3bc_" in this_type:
                        if "_3bc_" in key:
                            sample_name_list.append(key)
                    else:  # if 3' bc is not in the type, the by definition if won't be in the key
                        sample_name_list.append(key)

            if len(sample_name_list) == 1:  # check that only one name matches
                if "Fwd" in this_type:
                    os.system("mv " + this_type + ".fastq.gz " + output_directory + "ultraplex_" + save_name + "_" +
                              sample_names[sample_name_list[0]] + "_Fwd.fastq.gz")
                elif "Rev" in this_type:
                    os.system("mv " + this_type + ".fastq.gz " + output_directory + "ultraplex_" + save_name + "_" +
                              sample_names[sample_name_list[0]] + "_Rev.fastq.gz")
                else:
                    os.system("mv " + this_type + ".fastq.gz " + output_directory + "ultraplex_" + save_name + "_" +
                              sample_names[sample_name_list[0]] + ".fastq.gz")
            if len(sample_name_list) > 1:
                print("Warning - unable to rename file as filename-sample combination was not unique")

	this function concatenates all the files produced by the 
	different workers, then sends an sbatch command to compress
	them all to fastqs.
	
this function concatenates all the files produced by the
different workers, then sends an sbatch command to compress
them all to fastqs.
this function concatenates all the files produced by the different workers then sends an sbatch command to compress them all to fastqs .
def concatenate_files(save_name, ultra_mode,
                      sbatch_compression,
                      output_directory,
                      sample_names,
                      compression_threads=8):
    all_names = glob.glob(output_directory + "ultraplex_" + save_name + '*')
    all_types = []  
    for name in all_names:
        this_type = name.split("_tmp_thread_")[0]
        if this_type not in all_types:
            all_types.append(this_type)  
    if ultra_mode:
        for this_type in all_types:
            filenames = sorted(glob.glob(this_type + '*'))  
            command = ''
            for name in filenames:
                command = command + name + ' '
            command = 'cat ' + command + ' > ' + this_type + '.fastq'
            os.system(command)
            for name in filenames:
                os.remove(name)
            print("Compressing with pigz...")
            c_thread_n = '-p' + str(compression_threads)
            if sbatch_compression:
                os.system(
                    'sbatch -J compression -c ' + str(
                        compression_threads) + ' --time 4:00:00 --wrap="pigz ' + c_thread_n + ' ' + this_type + '.fastq"')
            else:
                os.system('pigz ' + c_thread_n + ' ' + this_type + '.fastq')
        if sbatch_compression:
            finished = False
            print("Compressing....")
            while not finished:
                complete = True
                for this_type in all_types:
                    filename = glob.glob(this_type + '*')
                    if '.gz' not in filename[0]:
                        complete = False
                if complete:
                    finished = True
                    print("Compression complete!")
                else:
                    time.sleep(1)
    else:  
        for this_type in all_types:
            filenames = sorted(glob.glob(this_type + '*'))
            command = ''
            for name in filenames:
                command = command + name + ' '
            command = 'cat ' + command + ' > ' + this_type + '.fastq.gz'
            os.system(command)
            for name in filenames:
                os.remove(name)
    if len(sample_names) > 0:  
        for this_type in all_types:
            sample_name_list = []
            for key, value in sample_names.items():
                if key in this_type and "no_match" not in this_type:
                    if "_3bc_" in this_type:
                        if "_3bc_" in key:
                            sample_name_list.append(key)
                    else:  
                        sample_name_list.append(key)
            if len(sample_name_list) == 1:  
                if "Fwd" in this_type:
                    os.system("mv " + this_type + ".fastq.gz " + output_directory + "ultraplex_" + save_name + "_" +
                              sample_names[sample_name_list[0]] + "_Fwd.fastq.gz")
                elif "Rev" in this_type:
                    os.system("mv " + this_type + ".fastq.gz " + output_directory + "ultraplex_" + save_name + "_" +
                              sample_names[sample_name_list[0]] + "_Rev.fastq.gz")
                else:
                    os.system("mv " + this_type + ".fastq.gz " + output_directory + "ultraplex_" + save_name + "_" +
                              sample_names[sample_name_list[0]] + ".fastq.gz")
            if len(sample_name_list) > 1:
                print("Warning - unable to rename file as filename-sample combination was not unique")
def concatenate_files ( save_name , ultra_mode , sbatch_compression , output_directory , sample_names , compression_threads = 8 ) : all_names = glob . glob ( output_directory + "ultraplex_" + save_name + '*' ) all_types = [ ] for name in all_names : this_type = name . split ( "_tmp_thread_" ) [ 0 ] if this_type not in all_types : all_types . append ( this_type ) if ultra_mode : for this_type in all_types : filenames = sorted ( glob . glob ( this_type + '*' ) ) command = '' for name in filenames : command = command + name + ' ' command = 'cat ' + command + ' > ' + this_type + '.fastq' os . system ( command ) for name in filenames : os . remove ( name ) print ( "Compressing with pigz..." ) c_thread_n = '-p' + str ( compression_threads ) if sbatch_compression : os . system ( 'sbatch -J compression -c ' + str ( compression_threads ) + ' --time 4:00:00 --wrap="pigz ' + c_thread_n + ' ' + this_type + '.fastq"' ) else : os . system ( 'pigz ' + c_thread_n + ' ' + this_type + '.fastq' ) if sbatch_compression : finished = False print ( "Compressing...." ) while not finished : complete = True for this_type in all_types : filename = glob . glob ( this_type + '*' ) if '.gz' not in filename [ 0 ] : complete = False if complete : finished = True print ( "Compression complete!" ) else : time . sleep ( 1 ) else : for this_type in all_types : filenames = sorted ( glob . glob ( this_type + '*' ) ) command = '' for name in filenames : command = command + name + ' ' command = 'cat ' + command + ' > ' + this_type + '.fastq.gz' os . system ( command ) for name in filenames : os . remove ( name ) if len ( sample_names ) > 0 : for this_type in all_types : sample_name_list = [ ] for key , value in sample_names . items ( ) : if key in this_type and "no_match" not in this_type : if "_3bc_" in this_type : if "_3bc_" in key : sample_name_list . append ( key ) else : sample_name_list . append ( key ) if len ( sample_name_list ) == 1 : if "Fwd" in this_type : os . system ( "mv " + this_type + ".fastq.gz " + output_directory + "ultraplex_" + save_name + "_" + sample_names [ sample_name_list [ 0 ] ] + "_Fwd.fastq.gz" ) elif "Rev" in this_type : os . system ( "mv " + this_type + ".fastq.gz " + output_directory + "ultraplex_" + save_name + "_" + sample_names [ sample_name_list [ 0 ] ] + "_Rev.fastq.gz" ) else : os . system ( "mv " + this_type + ".fastq.gz " + output_directory + "ultraplex_" + save_name + "_" + sample_names [ sample_name_list [ 0 ] ] + ".fastq.gz" ) if len ( sample_name_list ) > 1 : print ( "Warning - unable to rename file as filename-sample combination was not unique" )
this function concatenates all the files produced by the
different workers, then sends an sbatch command to compress
them all to fastqs.
this function concatenates all the files produced by the different workers then sends an sbatch command to compress them all to fastqs .
"""
	this function concatenates all the files produced by the 
	different workers, then sends an sbatch command to compress
	them all to fastqs.
	""" # First, file all the unique file names we have, ignoring threads
 # ignoring threads
 # this_type contains directory if applicable
 # now concatenate them
 # find all files with this barcode (or barcode combination)
 # this type already has output directory
 # then concatenate
 # check if compression is complete
 # assume it's complete
 # now actually check
 # if not ultra_mode
 # find all files with this barcode (or barcode combination)
 # then concatenate
 # ie the user has specified at least one sample name
 # check whether any of the keys in the dictionary match this type
 # check that if key doesn't contain 3' barcode, then the type does either
 # if 3' bc is not in the type, the by definition if won't be in the key
 # check that only one name matches

save_name
ultra_mode
sbatch_compression
output_directory
sample_names
compression_threads