python/pymethylprocess/preprocess.py
def na_report(input_pkl, output_dir, head_directory):
    """Print proportion of missing values throughout dataset."""
    import matplotlib.pyplot as plt
    if head_directory:
        input_pickles=glob.glob(os.path.join(input_pkl,'*/*.pkl'))
        output_dirs=np.vectorize(lambda x: x.replace('.pkl','.na_report'))(input_pickles).tolist()
    else:
        input_pickles=[input_pkl]
        output_dirs=[output_dir]
    for input_pkl,output_dir in zip(input_pickles,output_dirs):
        os.makedirs(output_dir,exist_ok=True)
        df=pickle.load(open(input_pkl,'rb'))['beta']
        na_frame = pd.isna(df).astype(int)
        na_frame = na_frame.iloc[np.argsort(na_frame.sum(axis=1)),:]
        print('{} NA Rate is on average: {}%'.format(input_pkl,sum(na_frame.values.flatten())/float(df.shape[0]*df.shape[1])*100.))
        plt.figure()
        pd.DataFrame(na_frame.sum(axis=1)).apply(lambda x: x/float(df.shape[1])).hist()
        plt.savefig(os.path.join(output_dir,'sample_missingness.png'))
        plt.figure()
        pd.DataFrame(na_frame.sum(axis=0)).apply(lambda x: x/float(df.shape[0])).hist()
        plt.savefig(os.path.join(output_dir,'cpg_missingness.png'))
        xy=np.transpose(np.nonzero(na_frame.values))
        if 0:
            plt.figure()
            plt.scatter(xy[:,1],xy[:,0])
            plt.xlim([0,df.shape[0]])
            plt.ylim([0,df.shape[1]])
            plt.xlabel('CpGs')
            plt.ylabel('Samples')
            plt.savefig(os.path.join(output_dir,'array_missingness.png'))
Print proportion of missing values throughout dataset.
Print proportion of missing values throughout dataset.
Print proportion of missing values throughout dataset .
def na_report(input_pkl, output_dir, head_directory):
    import matplotlib.pyplot as plt
    if head_directory:
        input_pickles=glob.glob(os.path.join(input_pkl,'*/*.pkl'))
        output_dirs=np.vectorize(lambda x: x.replace('.pkl','.na_report'))(input_pickles).tolist()
    else:
        input_pickles=[input_pkl]
        output_dirs=[output_dir]
    for input_pkl,output_dir in zip(input_pickles,output_dirs):
        os.makedirs(output_dir,exist_ok=True)
        df=pickle.load(open(input_pkl,'rb'))['beta']
        na_frame = pd.isna(df).astype(int)
        na_frame = na_frame.iloc[np.argsort(na_frame.sum(axis=1)),:]
        print('{} NA Rate is on average: {}%'.format(input_pkl,sum(na_frame.values.flatten())/float(df.shape[0]*df.shape[1])*100.))
        plt.figure()
        pd.DataFrame(na_frame.sum(axis=1)).apply(lambda x: x/float(df.shape[1])).hist()
        plt.savefig(os.path.join(output_dir,'sample_missingness.png'))
        plt.figure()
        pd.DataFrame(na_frame.sum(axis=0)).apply(lambda x: x/float(df.shape[0])).hist()
        plt.savefig(os.path.join(output_dir,'cpg_missingness.png'))
        xy=np.transpose(np.nonzero(na_frame.values))
        if 0:
            plt.figure()
            plt.scatter(xy[:,1],xy[:,0])
            plt.xlim([0,df.shape[0]])
            plt.ylim([0,df.shape[1]])
            plt.xlabel('CpGs')
            plt.ylabel('Samples')
            plt.savefig(os.path.join(output_dir,'array_missingness.png'))
def na_report ( input_pkl , output_dir , head_directory ) : import matplotlib . pyplot as plt if head_directory : input_pickles = glob . glob ( os . path . join ( input_pkl , '*/*.pkl' ) ) output_dirs = np . vectorize ( lambda x : x . replace ( '.pkl' , '.na_report' ) ) ( input_pickles ) . tolist ( ) else : input_pickles = [ input_pkl ] output_dirs = [ output_dir ] for input_pkl , output_dir in zip ( input_pickles , output_dirs ) : os . makedirs ( output_dir , exist_ok = True ) df = pickle . load ( open ( input_pkl , 'rb' ) ) [ 'beta' ] na_frame = pd . isna ( df ) . astype ( int ) na_frame = na_frame . iloc [ np . argsort ( na_frame . sum ( axis = 1 ) ) , : ] print ( '{} NA Rate is on average: {}%' . format ( input_pkl , sum ( na_frame . values . flatten ( ) ) / float ( df . shape [ 0 ] * df . shape [ 1 ] ) * 100. ) ) plt . figure ( ) pd . DataFrame ( na_frame . sum ( axis = 1 ) ) . apply ( lambda x : x / float ( df . shape [ 1 ] ) ) . hist ( ) plt . savefig ( os . path . join ( output_dir , 'sample_missingness.png' ) ) plt . figure ( ) pd . DataFrame ( na_frame . sum ( axis = 0 ) ) . apply ( lambda x : x / float ( df . shape [ 0 ] ) ) . hist ( ) plt . savefig ( os . path . join ( output_dir , 'cpg_missingness.png' ) ) xy = np . transpose ( np . nonzero ( na_frame . values ) ) if 0 : plt . figure ( ) plt . scatter ( xy [ : , 1 ] , xy [ : , 0 ] ) plt . xlim ( [ 0 , df . shape [ 0 ] ] ) plt . ylim ( [ 0 , df . shape [ 1 ] ] ) plt . xlabel ( 'CpGs' ) plt . ylabel ( 'Samples' ) plt . savefig ( os . path . join ( output_dir , 'array_missingness.png' ) )
Print proportion of missing values throughout dataset.
Print proportion of missing values throughout dataset .
"""Print proportion of missing values throughout dataset."""
input_pkl
output_dir
head_directory