python/pymethylprocess/preprocess.py
def batch_deploy_preprocess(n_cores,subtype_output_dir,meffil,torque,run,series, pc_qc_parameters_csv, use_cache, qc_only, chunk_size, bmiq, additional_command):
    """Deploy multiple preprocessing jobs in series or parallel."""
    pheno_csvs = glob.glob(os.path.join(subtype_output_dir,'*','*.csv'))
    opts = {'-n':n_cores}
    try:
        pc_qc_parameters = pd.read_csv(pc_qc_parameters).drop_duplicates().set_index('subtype')
    except:
        pc_qc_parameters = pd.DataFrame([[name,-1,0.05,0.05,0.05,0.05,5,-2] for name in np.vectorize(lambda x: x.split('/')[-2])(pheno_csvs)],
                        columns=['subtype','n_pcs','p_beadnum_samples','p_detection_samples','p_beadnum_cpgs','p_detection_cpgs','sex_sd','sex_cutoff']).drop_duplicates().set_index('subtype')
    if meffil:
        opts['-m']=''
    if use_cache:
        opts['-u']=''
    if qc_only:
        opts['-qc']=''
    if bmiq:
        opts['-b']=''
    commands=[]
    for pheno_csv in pheno_csvs:
        pheno_path = os.path.abspath(pheno_csv)
        subtype=pheno_path.split('/')[-2]
        opts['-pc'] = int(pc_qc_parameters.loc[subtype,'n_pcs'])
        opts['-bns'] = pc_qc_parameters.loc[subtype,'p_beadnum_samples']
        opts['-pds'] = pc_qc_parameters.loc[subtype,'p_detection_samples']
        opts['-bnc'] = pc_qc_parameters.loc[subtype,'p_beadnum_cpgs']
        opts['-pdc'] = pc_qc_parameters.loc[subtype,'p_detection_cpgs']
        opts['-sc'] = pc_qc_parameters.loc[subtype,'sex_cutoff']
        opts['-sd'] = pc_qc_parameters.loc[subtype,'sex_sd']
        opts['-i']=pheno_path[:pheno_path.rfind('/')+1]
        opts['-o']=pheno_path[:pheno_path.rfind('/')+1]+'methyl_array.pkl'
        command='pymethyl-preprocess preprocess_pipeline {}'.format(' '.join('{} {}'.format(k,v) for k,v in opts.items()))
        commands.append(command)
    if not torque:
        if not series and chunk_size != -1:
            #commands = np.array_split(commands,len(commands)//chunk_size)
            print(commands)
            with open('commands.txt','w') as f:
                f.write('\n'.join(commands))
            subprocess.call('cat commands.txt | xargs -L 1 -I CMD -P {} bash -c CMD'.format(chunk_size),shell=True) # https://www.gnu.org/software/parallel/sem.html
            """for command_list in commands:
                subprocess.call('run_parallel {}'.format(' '.join(['"{}"'.format(command) for command in command_list])),shell=True)"""
        else:
            for command in commands:
                if not series:
                    command="nohup {} &".format(command)
                if not run:
                    click.echo(command)
                else:
                    subprocess.call(command,shell=True)
    else:
        from methylnet.torque_jobs import assemble_run_torque
        for command in commands:
            job=assemble_run_torque(command, use_gpu=False, additions=additional_command, queue="default", time=4, ngpu=0, additional_options="")
Deploy multiple preprocessing jobs in series or parallel.
Deploy multiple preprocessing jobs in series or parallel.
Deploy multiple preprocessing jobs in series or parallel .
def batch_deploy_preprocess(n_cores,subtype_output_dir,meffil,torque,run,series, pc_qc_parameters_csv, use_cache, qc_only, chunk_size, bmiq, additional_command):
    pheno_csvs = glob.glob(os.path.join(subtype_output_dir,'*','*.csv'))
    opts = {'-n':n_cores}
    try:
        pc_qc_parameters = pd.read_csv(pc_qc_parameters).drop_duplicates().set_index('subtype')
    except:
        pc_qc_parameters = pd.DataFrame([[name,-1,0.05,0.05,0.05,0.05,5,-2] for name in np.vectorize(lambda x: x.split('/')[-2])(pheno_csvs)],
                        columns=['subtype','n_pcs','p_beadnum_samples','p_detection_samples','p_beadnum_cpgs','p_detection_cpgs','sex_sd','sex_cutoff']).drop_duplicates().set_index('subtype')
    if meffil:
        opts['-m']=''
    if use_cache:
        opts['-u']=''
    if qc_only:
        opts['-qc']=''
    if bmiq:
        opts['-b']=''
    commands=[]
    for pheno_csv in pheno_csvs:
        pheno_path = os.path.abspath(pheno_csv)
        subtype=pheno_path.split('/')[-2]
        opts['-pc'] = int(pc_qc_parameters.loc[subtype,'n_pcs'])
        opts['-bns'] = pc_qc_parameters.loc[subtype,'p_beadnum_samples']
        opts['-pds'] = pc_qc_parameters.loc[subtype,'p_detection_samples']
        opts['-bnc'] = pc_qc_parameters.loc[subtype,'p_beadnum_cpgs']
        opts['-pdc'] = pc_qc_parameters.loc[subtype,'p_detection_cpgs']
        opts['-sc'] = pc_qc_parameters.loc[subtype,'sex_cutoff']
        opts['-sd'] = pc_qc_parameters.loc[subtype,'sex_sd']
        opts['-i']=pheno_path[:pheno_path.rfind('/')+1]
        opts['-o']=pheno_path[:pheno_path.rfind('/')+1]+'methyl_array.pkl'
        command='pymethyl-preprocess preprocess_pipeline {}'.format(' '.join('{} {}'.format(k,v) for k,v in opts.items()))
        commands.append(command)
    if not torque:
        if not series and chunk_size != -1:
            print(commands)
            with open('commands.txt','w') as f:
                f.write('\n'.join(commands))
            subprocess.call('cat commands.txt | xargs -L 1 -I CMD -P {} bash -c CMD'.format(chunk_size),shell=True) 
        else:
            for command in commands:
                if not series:
                    command="nohup {} &".format(command)
                if not run:
                    click.echo(command)
                else:
                    subprocess.call(command,shell=True)
    else:
        from methylnet.torque_jobs import assemble_run_torque
        for command in commands:
            job=assemble_run_torque(command, use_gpu=False, additions=additional_command, queue="default", time=4, ngpu=0, additional_options="")
def batch_deploy_preprocess ( n_cores , subtype_output_dir , meffil , torque , run , series , pc_qc_parameters_csv , use_cache , qc_only , chunk_size , bmiq , additional_command ) : pheno_csvs = glob . glob ( os . path . join ( subtype_output_dir , '*' , '*.csv' ) ) opts = { '-n' : n_cores } try : pc_qc_parameters = pd . read_csv ( pc_qc_parameters ) . drop_duplicates ( ) . set_index ( 'subtype' ) except : pc_qc_parameters = pd . DataFrame ( [ [ name , - 1 , 0.05 , 0.05 , 0.05 , 0.05 , 5 , - 2 ] for name in np . vectorize ( lambda x : x . split ( '/' ) [ - 2 ] ) ( pheno_csvs ) ] , columns = [ 'subtype' , 'n_pcs' , 'p_beadnum_samples' , 'p_detection_samples' , 'p_beadnum_cpgs' , 'p_detection_cpgs' , 'sex_sd' , 'sex_cutoff' ] ) . drop_duplicates ( ) . set_index ( 'subtype' ) if meffil : opts [ '-m' ] = '' if use_cache : opts [ '-u' ] = '' if qc_only : opts [ '-qc' ] = '' if bmiq : opts [ '-b' ] = '' commands = [ ] for pheno_csv in pheno_csvs : pheno_path = os . path . abspath ( pheno_csv ) subtype = pheno_path . split ( '/' ) [ - 2 ] opts [ '-pc' ] = int ( pc_qc_parameters . loc [ subtype , 'n_pcs' ] ) opts [ '-bns' ] = pc_qc_parameters . loc [ subtype , 'p_beadnum_samples' ] opts [ '-pds' ] = pc_qc_parameters . loc [ subtype , 'p_detection_samples' ] opts [ '-bnc' ] = pc_qc_parameters . loc [ subtype , 'p_beadnum_cpgs' ] opts [ '-pdc' ] = pc_qc_parameters . loc [ subtype , 'p_detection_cpgs' ] opts [ '-sc' ] = pc_qc_parameters . loc [ subtype , 'sex_cutoff' ] opts [ '-sd' ] = pc_qc_parameters . loc [ subtype , 'sex_sd' ] opts [ '-i' ] = pheno_path [ : pheno_path . rfind ( '/' ) + 1 ] opts [ '-o' ] = pheno_path [ : pheno_path . rfind ( '/' ) + 1 ] + 'methyl_array.pkl' command = 'pymethyl-preprocess preprocess_pipeline {}' . format ( ' ' . join ( '{} {}' . format ( k , v ) for k , v in opts . items ( ) ) ) commands . append ( command ) if not torque : if not series and chunk_size != - 1 : print ( commands ) with open ( 'commands.txt' , 'w' ) as f : f . write ( '\n' . join ( commands ) ) subprocess . call ( 'cat commands.txt | xargs -L 1 -I CMD -P {} bash -c CMD' . format ( chunk_size ) , shell = True ) """for command_list in commands:
                subprocess.call('run_parallel {}'.format(' '.join(['"{}"'.format(command) for command in command_list])),shell=True)""" else : for command in commands : if not series : command = "nohup {} &" . format ( command ) if not run : click . echo ( command ) else : subprocess . call ( command , shell = True ) else : from methylnet . torque_jobs import assemble_run_torque for command in commands : job = assemble_run_torque ( command , use_gpu = False , additions = additional_command , queue = "default" , time = 4 , ngpu = 0 , additional_options = "" )
Deploy multiple preprocessing jobs in series or parallel.
Deploy multiple preprocessing jobs in series or parallel .
"""Deploy multiple preprocessing jobs in series or parallel.""" #commands = np.array_split(commands,len(commands)//chunk_size) # https://www.gnu.org/software/parallel/sem.html """for command_list in commands:
                subprocess.call('run_parallel {}'.format(' '.join(['"{}"'.format(command) for command in command_list])),shell=True)"""
n_cores
subtype_output_dir
meffil
torque
run
series
pc_qc_parameters_csv
use_cache
qc_only
chunk_size
bmiq
additional_command