combine_fold_results.py
def combine_fold_results(test_metrics, train_metrics, test_labels, test_preds, train_labels, train_preds,
                         outprefix, test_interval, file_type, second_data_source=False,
                         filter_actives_test=None, filter_actives_train=None):
    '''Make results files and graphs combined from results for
    separate crossvalidation folds. test_metrics and train_metrics
    are lists of lists of AUCs or RMSDs for each fold, for each
    test_interval. test_labels and test_preds are labels and final
    test predictions for each test fold, in a single list. train_labels
    and train_preds are labels and predictions for each train fold in
    a single list. affinity says whether to interpret the labels and
    predictions as affinities, and the metric as RMSD instead of AUC.
    second_data_source sets the output file names to reflect whether
    the results are for the second data source of a combined data model.'''
    if file_type=='affinity':
      metric = 'rmsd'
    elif file_type=='pose':
      metric='auc'
    elif file_type=='rmsd':
      metric = 'rmsd_rmse'
    two = '2' if second_data_source else ''

    #test and train metrics are organized by fold iteration, which with dynamic
    #stepping can be different lengths; pad them out to the max length with 
    #the last value
    test_metrics = make_uniform_array(test_metrics)
    train_metrics = make_uniform_array(train_metrics)
    
    #average metric across folds
    mean_test_metrics = np.mean(test_metrics, axis=0)
    mean_train_metrics = np.mean(train_metrics, axis=0)

    #write test and train metrics (mean and for each fold)
    write_results_file('%s.%s.test%s' % (outprefix, metric, two), mean_test_metrics, *test_metrics)
    write_results_file('%s.%s.train%s' % (outprefix, metric, two), mean_train_metrics, *train_metrics)

    #training plot of mean test and train metric across folds
    training_plot('%s_%s_train%s.pdf' % (outprefix, metric, two), mean_train_metrics, mean_test_metrics)
#added line
    
    if metric=='rmsd_rmse':

        if filter_actives_test:
            test_preds = np.array(test_preds)[np.array(test_labels)>0]
            test_labels = np.array(test_labels)[np.array(test_labels)>0]

        #correlation plots for last test iteration
        rmse = np.sqrt(sklearn.metrics.mean_squared_error(test_preds, test_labels))
        r2 = scipy.stats.pearsonr(test_preds, test_labels)[0]**2
        write_results_file('%s.rmsd_rmse.finaltest%s' % (outprefix, two), test_labels, test_preds, footer='RMSE,R %f %f\n' % (rmse, r2))
        makejoint(test_labels, test_preds, 'royalblue','%s_corr_test%s' % (outprefix, two))
        #plot_correlation('%s_corr_test%s.pdf' % (outprefix, two), test_preds, test_labels, rmse, r2)

        if filter_actives_train:
            train_preds = np.array(train_preds)[np.array(train_labels)>0]
            train_labels = np.array(train_labels)[np.array(train_labels)>0]

        rmse = np.sqrt(sklearn.metrics.mean_squared_error(train_preds, train_labels))
        r2 = scipy.stats.pearsonr(train_preds, train_labels)[0]**2
        write_results_file('%s.rmsd_rmse.finaltrain%s' % (outprefix, two), train_labels, train_preds, footer='RMSE,R %f %f\n' % (rmse, r2))
        makejoint(train_labels, train_preds, 'orangered','%s_corr_train%s' % (outprefix, two))        
        #plot_correlation('%s_corr_train%s.pdf' % (outprefix, two), train_preds, train_labels, rmse, r2)
    
    elif metric=='rmsd':

        if filter_actives_test:
            test_preds = np.array(test_preds)[np.array(test_labels)>0]
            test_labels = np.array(test_labels)[np.array(test_labels)>0]

        #correlation plots for last test iteration
        rmsd = np.sqrt(sklearn.metrics.mean_squared_error(test_preds, test_labels))
        r2 = scipy.stats.pearsonr(test_preds, test_labels)[0]**2
        write_results_file('%s.rmsd.finaltest%s' % (outprefix, two), test_labels, test_preds, footer='RMSD,R %f %f\n' % (rmsd, r2))
        makejoint(test_labels, test_preds, 'royalblue','%s_corr_test%s' % (outprefix, two))
        #plot_correlation('%s_corr_test%s.pdf' % (outprefix, two), test_preds, test_labels, rmsd, r2)

        if filter_actives_train:
            train_preds = np.array(train_preds)[np.array(train_labels)>0]
            train_labels = np.array(train_labels)[np.array(train_labels)>0]

        rmsd = np.sqrt(sklearn.metrics.mean_squared_error(train_preds, train_labels))
        r2 = scipy.stats.pearsonr(train_preds, train_labels)[0]**2
        write_results_file('%s.rmsd.finaltrain%s' % (outprefix, two), train_labels, train_preds, footer='RMSD,R %f %f\n' % (rmsd, r2))
        makejoint(train_labels, train_preds, 'orangered','%s_corr_train%s' % (outprefix, two))        
        #plot_correlation('%s_corr_train%s.pdf' % (outprefix, two), train_preds, train_labels, rmsd, r2)
    
    elif metric=='auc':

        #roc curves for the last test iteration
        if len(np.unique(test_labels)) > 1:
            last_iters = 1000
            avg_auc, max_auc, min_auc = last_iters_statistics(test_metrics, test_interval, last_iters)
            txt = 'For the last %s iterations:\nmean AUC=%.2f  max AUC=%.2f  min AUC=%.2f' % (last_iters, avg_auc, max_auc, min_auc)
            fpr, tpr, _ = sklearn.metrics.roc_curve(test_labels, test_preds)
            auc = sklearn.metrics.roc_auc_score(test_labels, test_preds)
            precscore = sklearn.metrics.average_precision_score(test_labels, test_preds)
            write_results_file('%s.auc.finaltest%s' % (outprefix, two), test_labels, test_preds, footer='AUC, precision score %f %f\n' % (auc,precscore))
            plot_roc_curve('%s_roc_test%s.pdf' % (outprefix, two), fpr, tpr, auc, txt)
            plot_prc('%s_prc_test%s.pdf' % (outprefix, two),test_labels, test_preds, precscore, '%s.auc.finaltest%s' % (outprefix, two))

        if len(np.unique(train_labels)) > 1:
            last_iters = 1000
            avg_auc, max_auc, min_auc = last_iters_statistics(train_metrics, test_interval, last_iters)
            txt = 'For the last %s iterations:\nmean AUC=%.2f  max AUC=%.2f  min AUC=%.2f' % (last_iters, avg_auc, max_auc, min_auc)
            fpr, tpr, _ = sklearn.metrics.roc_curve(train_labels, train_preds)
            auc = sklearn.metrics.roc_auc_score(train_labels, train_preds)
            precscore = sklearn.metrics.average_precision_score(train_labels, train_preds)
            write_results_file('%s.auc.finaltrain%s' % (outprefix, two), train_labels, train_preds, footer='AUC, precision score %f %f\n' % (auc,precscore))
            plot_roc_curve('%s_roc_train%s.pdf' % (outprefix, two), fpr, tpr, auc, txt)
            plot_prc('%s_prc_train%s.pdf' % (outprefix, two),train_labels, train_preds, precscore, '%s.auc.finaltrain%s' % (outprefix, two))
Make results files and graphs combined from results for
    separate crossvalidation folds. test_metrics and train_metrics
    are lists of lists of AUCs or RMSDs for each fold, for each
    test_interval. test_labels and test_preds are labels and final
    test predictions for each test fold, in a single list. train_labels
    and train_preds are labels and predictions for each train fold in
    a single list. affinity says whether to interpret the labels and
    predictions as affinities, and the metric as RMSD instead of AUC.
    second_data_source sets the output file names to reflect whether
    the results are for the second data source of a combined data model.
Make results files and graphs combined from results for
separate crossvalidation folds. test_metrics and train_metrics
are lists of lists of AUCs or RMSDs for each fold, for each
test_interval. test_labels and test_preds are labels and final
test predictions for each test fold, in a single list. train_labels
and train_preds are labels and predictions for each train fold in
a single list. affinity says whether to interpret the labels and
predictions as affinities, and the metric as RMSD instead of AUC.
second_data_source sets the output file names to reflect whether
the results are for the second data source of a combined data model.
Make results files and graphs combined from results for separate crossvalidation folds . test_metrics and train_metrics are lists of lists of AUCs or RMSDs for each fold for each test_interval . test_labels and test_preds are labels and final test predictions for each test fold in a single list . train_labels and train_preds are labels and predictions for each train fold in a single list . affinity says whether to interpret the labels and predictions as affinities and the metric as RMSD instead of AUC . second_data_source sets the output file names to reflect whether the results are for the second data source of a combined data model .
def combine_fold_results(test_metrics, train_metrics, test_labels, test_preds, train_labels, train_preds,
                         outprefix, test_interval, file_type, second_data_source=False,
                         filter_actives_test=None, filter_actives_train=None):
    if file_type=='affinity':
      metric = 'rmsd'
    elif file_type=='pose':
      metric='auc'
    elif file_type=='rmsd':
      metric = 'rmsd_rmse'
    two = '2' if second_data_source else ''
    test_metrics = make_uniform_array(test_metrics)
    train_metrics = make_uniform_array(train_metrics)
    mean_test_metrics = np.mean(test_metrics, axis=0)
    mean_train_metrics = np.mean(train_metrics, axis=0)
    write_results_file('%s.%s.test%s' % (outprefix, metric, two), mean_test_metrics, *test_metrics)
    write_results_file('%s.%s.train%s' % (outprefix, metric, two), mean_train_metrics, *train_metrics)
    training_plot('%s_%s_train%s.pdf' % (outprefix, metric, two), mean_train_metrics, mean_test_metrics)
    if metric=='rmsd_rmse':
        if filter_actives_test:
            test_preds = np.array(test_preds)[np.array(test_labels)>0]
            test_labels = np.array(test_labels)[np.array(test_labels)>0]
        rmse = np.sqrt(sklearn.metrics.mean_squared_error(test_preds, test_labels))
        r2 = scipy.stats.pearsonr(test_preds, test_labels)[0]**2
        write_results_file('%s.rmsd_rmse.finaltest%s' % (outprefix, two), test_labels, test_preds, footer='RMSE,R %f %f\n' % (rmse, r2))
        makejoint(test_labels, test_preds, 'royalblue','%s_corr_test%s' % (outprefix, two))
        if filter_actives_train:
            train_preds = np.array(train_preds)[np.array(train_labels)>0]
            train_labels = np.array(train_labels)[np.array(train_labels)>0]
        rmse = np.sqrt(sklearn.metrics.mean_squared_error(train_preds, train_labels))
        r2 = scipy.stats.pearsonr(train_preds, train_labels)[0]**2
        write_results_file('%s.rmsd_rmse.finaltrain%s' % (outprefix, two), train_labels, train_preds, footer='RMSE,R %f %f\n' % (rmse, r2))
        makejoint(train_labels, train_preds, 'orangered','%s_corr_train%s' % (outprefix, two))        
    elif metric=='rmsd':
        if filter_actives_test:
            test_preds = np.array(test_preds)[np.array(test_labels)>0]
            test_labels = np.array(test_labels)[np.array(test_labels)>0]
        rmsd = np.sqrt(sklearn.metrics.mean_squared_error(test_preds, test_labels))
        r2 = scipy.stats.pearsonr(test_preds, test_labels)[0]**2
        write_results_file('%s.rmsd.finaltest%s' % (outprefix, two), test_labels, test_preds, footer='RMSD,R %f %f\n' % (rmsd, r2))
        makejoint(test_labels, test_preds, 'royalblue','%s_corr_test%s' % (outprefix, two))
        if filter_actives_train:
            train_preds = np.array(train_preds)[np.array(train_labels)>0]
            train_labels = np.array(train_labels)[np.array(train_labels)>0]
        rmsd = np.sqrt(sklearn.metrics.mean_squared_error(train_preds, train_labels))
        r2 = scipy.stats.pearsonr(train_preds, train_labels)[0]**2
        write_results_file('%s.rmsd.finaltrain%s' % (outprefix, two), train_labels, train_preds, footer='RMSD,R %f %f\n' % (rmsd, r2))
        makejoint(train_labels, train_preds, 'orangered','%s_corr_train%s' % (outprefix, two))        
    elif metric=='auc':
        if len(np.unique(test_labels)) > 1:
            last_iters = 1000
            avg_auc, max_auc, min_auc = last_iters_statistics(test_metrics, test_interval, last_iters)
            txt = 'For the last %s iterations:\nmean AUC=%.2f  max AUC=%.2f  min AUC=%.2f' % (last_iters, avg_auc, max_auc, min_auc)
            fpr, tpr, _ = sklearn.metrics.roc_curve(test_labels, test_preds)
            auc = sklearn.metrics.roc_auc_score(test_labels, test_preds)
            precscore = sklearn.metrics.average_precision_score(test_labels, test_preds)
            write_results_file('%s.auc.finaltest%s' % (outprefix, two), test_labels, test_preds, footer='AUC, precision score %f %f\n' % (auc,precscore))
            plot_roc_curve('%s_roc_test%s.pdf' % (outprefix, two), fpr, tpr, auc, txt)
            plot_prc('%s_prc_test%s.pdf' % (outprefix, two),test_labels, test_preds, precscore, '%s.auc.finaltest%s' % (outprefix, two))
        if len(np.unique(train_labels)) > 1:
            last_iters = 1000
            avg_auc, max_auc, min_auc = last_iters_statistics(train_metrics, test_interval, last_iters)
            txt = 'For the last %s iterations:\nmean AUC=%.2f  max AUC=%.2f  min AUC=%.2f' % (last_iters, avg_auc, max_auc, min_auc)
            fpr, tpr, _ = sklearn.metrics.roc_curve(train_labels, train_preds)
            auc = sklearn.metrics.roc_auc_score(train_labels, train_preds)
            precscore = sklearn.metrics.average_precision_score(train_labels, train_preds)
            write_results_file('%s.auc.finaltrain%s' % (outprefix, two), train_labels, train_preds, footer='AUC, precision score %f %f\n' % (auc,precscore))
            plot_roc_curve('%s_roc_train%s.pdf' % (outprefix, two), fpr, tpr, auc, txt)
            plot_prc('%s_prc_train%s.pdf' % (outprefix, two),train_labels, train_preds, precscore, '%s.auc.finaltrain%s' % (outprefix, two))
def combine_fold_results ( test_metrics , train_metrics , test_labels , test_preds , train_labels , train_preds , outprefix , test_interval , file_type , second_data_source = False , filter_actives_test = None , filter_actives_train = None ) : if file_type == 'affinity' : metric = 'rmsd' elif file_type == 'pose' : metric = 'auc' elif file_type == 'rmsd' : metric = 'rmsd_rmse' two = '2' if second_data_source else '' test_metrics = make_uniform_array ( test_metrics ) train_metrics = make_uniform_array ( train_metrics ) mean_test_metrics = np . mean ( test_metrics , axis = 0 ) mean_train_metrics = np . mean ( train_metrics , axis = 0 ) write_results_file ( '%s.%s.test%s' % ( outprefix , metric , two ) , mean_test_metrics , * test_metrics ) write_results_file ( '%s.%s.train%s' % ( outprefix , metric , two ) , mean_train_metrics , * train_metrics ) training_plot ( '%s_%s_train%s.pdf' % ( outprefix , metric , two ) , mean_train_metrics , mean_test_metrics ) if metric == 'rmsd_rmse' : if filter_actives_test : test_preds = np . array ( test_preds ) [ np . array ( test_labels ) > 0 ] test_labels = np . array ( test_labels ) [ np . array ( test_labels ) > 0 ] rmse = np . sqrt ( sklearn . metrics . mean_squared_error ( test_preds , test_labels ) ) r2 = scipy . stats . pearsonr ( test_preds , test_labels ) [ 0 ] ** 2 write_results_file ( '%s.rmsd_rmse.finaltest%s' % ( outprefix , two ) , test_labels , test_preds , footer = 'RMSE,R %f %f\n' % ( rmse , r2 ) ) makejoint ( test_labels , test_preds , 'royalblue' , '%s_corr_test%s' % ( outprefix , two ) ) if filter_actives_train : train_preds = np . array ( train_preds ) [ np . array ( train_labels ) > 0 ] train_labels = np . array ( train_labels ) [ np . array ( train_labels ) > 0 ] rmse = np . sqrt ( sklearn . metrics . mean_squared_error ( train_preds , train_labels ) ) r2 = scipy . stats . pearsonr ( train_preds , train_labels ) [ 0 ] ** 2 write_results_file ( '%s.rmsd_rmse.finaltrain%s' % ( outprefix , two ) , train_labels , train_preds , footer = 'RMSE,R %f %f\n' % ( rmse , r2 ) ) makejoint ( train_labels , train_preds , 'orangered' , '%s_corr_train%s' % ( outprefix , two ) ) elif metric == 'rmsd' : if filter_actives_test : test_preds = np . array ( test_preds ) [ np . array ( test_labels ) > 0 ] test_labels = np . array ( test_labels ) [ np . array ( test_labels ) > 0 ] rmsd = np . sqrt ( sklearn . metrics . mean_squared_error ( test_preds , test_labels ) ) r2 = scipy . stats . pearsonr ( test_preds , test_labels ) [ 0 ] ** 2 write_results_file ( '%s.rmsd.finaltest%s' % ( outprefix , two ) , test_labels , test_preds , footer = 'RMSD,R %f %f\n' % ( rmsd , r2 ) ) makejoint ( test_labels , test_preds , 'royalblue' , '%s_corr_test%s' % ( outprefix , two ) ) if filter_actives_train : train_preds = np . array ( train_preds ) [ np . array ( train_labels ) > 0 ] train_labels = np . array ( train_labels ) [ np . array ( train_labels ) > 0 ] rmsd = np . sqrt ( sklearn . metrics . mean_squared_error ( train_preds , train_labels ) ) r2 = scipy . stats . pearsonr ( train_preds , train_labels ) [ 0 ] ** 2 write_results_file ( '%s.rmsd.finaltrain%s' % ( outprefix , two ) , train_labels , train_preds , footer = 'RMSD,R %f %f\n' % ( rmsd , r2 ) ) makejoint ( train_labels , train_preds , 'orangered' , '%s_corr_train%s' % ( outprefix , two ) ) elif metric == 'auc' : if len ( np . unique ( test_labels ) ) > 1 : last_iters = 1000 avg_auc , max_auc , min_auc = last_iters_statistics ( test_metrics , test_interval , last_iters ) txt = 'For the last %s iterations:\nmean AUC=%.2f  max AUC=%.2f  min AUC=%.2f' % ( last_iters , avg_auc , max_auc , min_auc ) fpr , tpr , _ = sklearn . metrics . roc_curve ( test_labels , test_preds ) auc = sklearn . metrics . roc_auc_score ( test_labels , test_preds ) precscore = sklearn . metrics . average_precision_score ( test_labels , test_preds ) write_results_file ( '%s.auc.finaltest%s' % ( outprefix , two ) , test_labels , test_preds , footer = 'AUC, precision score %f %f\n' % ( auc , precscore ) ) plot_roc_curve ( '%s_roc_test%s.pdf' % ( outprefix , two ) , fpr , tpr , auc , txt ) plot_prc ( '%s_prc_test%s.pdf' % ( outprefix , two ) , test_labels , test_preds , precscore , '%s.auc.finaltest%s' % ( outprefix , two ) ) if len ( np . unique ( train_labels ) ) > 1 : last_iters = 1000 avg_auc , max_auc , min_auc = last_iters_statistics ( train_metrics , test_interval , last_iters ) txt = 'For the last %s iterations:\nmean AUC=%.2f  max AUC=%.2f  min AUC=%.2f' % ( last_iters , avg_auc , max_auc , min_auc ) fpr , tpr , _ = sklearn . metrics . roc_curve ( train_labels , train_preds ) auc = sklearn . metrics . roc_auc_score ( train_labels , train_preds ) precscore = sklearn . metrics . average_precision_score ( train_labels , train_preds ) write_results_file ( '%s.auc.finaltrain%s' % ( outprefix , two ) , train_labels , train_preds , footer = 'AUC, precision score %f %f\n' % ( auc , precscore ) ) plot_roc_curve ( '%s_roc_train%s.pdf' % ( outprefix , two ) , fpr , tpr , auc , txt ) plot_prc ( '%s_prc_train%s.pdf' % ( outprefix , two ) , train_labels , train_preds , precscore , '%s.auc.finaltrain%s' % ( outprefix , two ) )
Make results files and graphs combined from results for
separate crossvalidation folds.
Make results files and graphs combined from results for separate crossvalidation folds .
'''Make results files and graphs combined from results for
    separate crossvalidation folds. test_metrics and train_metrics
    are lists of lists of AUCs or RMSDs for each fold, for each
    test_interval. test_labels and test_preds are labels and final
    test predictions for each test fold, in a single list. train_labels
    and train_preds are labels and predictions for each train fold in
    a single list. affinity says whether to interpret the labels and
    predictions as affinities, and the metric as RMSD instead of AUC.
    second_data_source sets the output file names to reflect whether
    the results are for the second data source of a combined data model.''' #test and train metrics are organized by fold iteration, which with dynamic #stepping can be different lengths; pad them out to the max length with  #the last value #average metric across folds #write test and train metrics (mean and for each fold) #training plot of mean test and train metric across folds #added line #correlation plots for last test iteration #plot_correlation('%s_corr_test%s.pdf' % (outprefix, two), test_preds, test_labels, rmse, r2) #plot_correlation('%s_corr_train%s.pdf' % (outprefix, two), train_preds, train_labels, rmse, r2) #correlation plots for last test iteration #plot_correlation('%s_corr_test%s.pdf' % (outprefix, two), test_preds, test_labels, rmsd, r2) #plot_correlation('%s_corr_train%s.pdf' % (outprefix, two), train_preds, train_labels, rmsd, r2) #roc curves for the last test iteration
test_metrics
train_metrics
test_labels
test_preds
train_labels
train_preds
outprefix
test_interval
file_type
second_data_source
filter_actives_test
filter_actives_train