python/methylnet/model_interpretability.py
def return_shap_values(shapley_data,output_dir,individuals,classes, output_histogram, absolute, log_transform):
    """Return matrix of shapley values per class, with option to, classes/individuals are columns, CpGs are rows, option to plot multiple histograms/density plots."""
    from sklearn.metrics import pairwise_distances
    os.makedirs(output_dir,exist_ok=True)
    shapley_data=ShapleyData.from_pickle(shapley_data)
    shapley_data_explorer=ShapleyDataExplorer(shapley_data)
    individuals=list(filter(None,individuals))
    classes=list(filter(None,classes))
    if absolute:
        shapley_data_explorer.make_shap_scores_abs()
    if classes and classes[0]=='all':
        classes = shapley_data_explorer.list_classes()
    if individuals and individuals[0]=='all':
        individuals = shapley_data_explorer.list_individuals(return_list=True)
    concat_list=[]
    if classes:
        for class_name in classes:
            concat_list.append(shapley_data_explorer.extract_class(class_name,get_shap_values=True))
    if individuals:
        for individual in individuals:
            concat_list.append(shapley_data_explorer.extract_individual(individual,get_shap_values=True))
    df=pd.concat(concat_list,axis=1,keys=classes+individuals)
    df.to_csv(join(output_dir,'returned_shap_values.csv'))
    pd.DataFrame(pairwise_distances(df.T.values,metric='correlation'),index=list(df),columns=list(df)).to_csv(join(output_dir,'returned_shap_values_corr_dist.csv'))
    #df.corr().to_csv(join(output_dir,'returned_shap_values_corr_dist.csv'))
    if output_histogram:
        import matplotlib
        matplotlib.use('Agg')
        import matplotlib.pyplot as plt
        import seaborn as sns
        sns.set()
        for entity in (classes+individuals):
            shap_scores = df[entity]
            plt.figure()
            ax=sns.distplot(shap_scores, kde=(log_transform==False))
            if log_transform:
                ax.set_xscale('symlog')
            plt.xlabel('{} SHAP value'.format("Log" if log_transform else ""))
            plt.ylabel('Frequency')
            plt.savefig(join(output_dir,'{}_shap_values.png'.format(entity)),dpi=300)
            top_10_shaps = shap_scores.abs().sort_values(ascending=False).iloc[:10]
            top_10_shaps=pd.DataFrame({'cpgs':top_10_shaps.index, '|SHAP|':top_10_shaps.values})
            plt.figure()
            ax=sns.barplot('|SHAP|','cpgs',orient='h',data=top_10_shaps)
            ax.tick_params(labelsize=4)
            plt.savefig(join(output_dir,'{}_top_shap_values.png'.format(entity)),dpi=300)
    if not absolute:
        print("All saved values reflect absolute values of sums, not sums of absolute values, if CpG SHAP scores are opposite signs across individuals, this will reduce the score of the resulting SHAP estimate.")
Return matrix of shapley values per class, with option to, classes/individuals are columns, CpGs are rows, option to plot multiple histograms/density plots.
Return matrix of shapley values per class, with option to, classes/individuals are columns, CpGs are rows, option to plot multiple histograms/density plots.
Return matrix of shapley values per class with option to classes / individuals are columns CpGs are rows option to plot multiple histograms / density plots .
def return_shap_values(shapley_data,output_dir,individuals,classes, output_histogram, absolute, log_transform):
    from sklearn.metrics import pairwise_distances
    os.makedirs(output_dir,exist_ok=True)
    shapley_data=ShapleyData.from_pickle(shapley_data)
    shapley_data_explorer=ShapleyDataExplorer(shapley_data)
    individuals=list(filter(None,individuals))
    classes=list(filter(None,classes))
    if absolute:
        shapley_data_explorer.make_shap_scores_abs()
    if classes and classes[0]=='all':
        classes = shapley_data_explorer.list_classes()
    if individuals and individuals[0]=='all':
        individuals = shapley_data_explorer.list_individuals(return_list=True)
    concat_list=[]
    if classes:
        for class_name in classes:
            concat_list.append(shapley_data_explorer.extract_class(class_name,get_shap_values=True))
    if individuals:
        for individual in individuals:
            concat_list.append(shapley_data_explorer.extract_individual(individual,get_shap_values=True))
    df=pd.concat(concat_list,axis=1,keys=classes+individuals)
    df.to_csv(join(output_dir,'returned_shap_values.csv'))
    pd.DataFrame(pairwise_distances(df.T.values,metric='correlation'),index=list(df),columns=list(df)).to_csv(join(output_dir,'returned_shap_values_corr_dist.csv'))
    if output_histogram:
        import matplotlib
        matplotlib.use('Agg')
        import matplotlib.pyplot as plt
        import seaborn as sns
        sns.set()
        for entity in (classes+individuals):
            shap_scores = df[entity]
            plt.figure()
            ax=sns.distplot(shap_scores, kde=(log_transform==False))
            if log_transform:
                ax.set_xscale('symlog')
            plt.xlabel('{} SHAP value'.format("Log" if log_transform else ""))
            plt.ylabel('Frequency')
            plt.savefig(join(output_dir,'{}_shap_values.png'.format(entity)),dpi=300)
            top_10_shaps = shap_scores.abs().sort_values(ascending=False).iloc[:10]
            top_10_shaps=pd.DataFrame({'cpgs':top_10_shaps.index, '|SHAP|':top_10_shaps.values})
            plt.figure()
            ax=sns.barplot('|SHAP|','cpgs',orient='h',data=top_10_shaps)
            ax.tick_params(labelsize=4)
            plt.savefig(join(output_dir,'{}_top_shap_values.png'.format(entity)),dpi=300)
    if not absolute:
        print("All saved values reflect absolute values of sums, not sums of absolute values, if CpG SHAP scores are opposite signs across individuals, this will reduce the score of the resulting SHAP estimate.")
def return_shap_values ( shapley_data , output_dir , individuals , classes , output_histogram , absolute , log_transform ) : from sklearn . metrics import pairwise_distances os . makedirs ( output_dir , exist_ok = True ) shapley_data = ShapleyData . from_pickle ( shapley_data ) shapley_data_explorer = ShapleyDataExplorer ( shapley_data ) individuals = list ( filter ( None , individuals ) ) classes = list ( filter ( None , classes ) ) if absolute : shapley_data_explorer . make_shap_scores_abs ( ) if classes and classes [ 0 ] == 'all' : classes = shapley_data_explorer . list_classes ( ) if individuals and individuals [ 0 ] == 'all' : individuals = shapley_data_explorer . list_individuals ( return_list = True ) concat_list = [ ] if classes : for class_name in classes : concat_list . append ( shapley_data_explorer . extract_class ( class_name , get_shap_values = True ) ) if individuals : for individual in individuals : concat_list . append ( shapley_data_explorer . extract_individual ( individual , get_shap_values = True ) ) df = pd . concat ( concat_list , axis = 1 , keys = classes + individuals ) df . to_csv ( join ( output_dir , 'returned_shap_values.csv' ) ) pd . DataFrame ( pairwise_distances ( df . T . values , metric = 'correlation' ) , index = list ( df ) , columns = list ( df ) ) . to_csv ( join ( output_dir , 'returned_shap_values_corr_dist.csv' ) ) if output_histogram : import matplotlib matplotlib . use ( 'Agg' ) import matplotlib . pyplot as plt import seaborn as sns sns . set ( ) for entity in ( classes + individuals ) : shap_scores = df [ entity ] plt . figure ( ) ax = sns . distplot ( shap_scores , kde = ( log_transform == False ) ) if log_transform : ax . set_xscale ( 'symlog' ) plt . xlabel ( '{} SHAP value' . format ( "Log" if log_transform else "" ) ) plt . ylabel ( 'Frequency' ) plt . savefig ( join ( output_dir , '{}_shap_values.png' . format ( entity ) ) , dpi = 300 ) top_10_shaps = shap_scores . abs ( ) . sort_values ( ascending = False ) . iloc [ : 10 ] top_10_shaps = pd . DataFrame ( { 'cpgs' : top_10_shaps . index , '|SHAP|' : top_10_shaps . values } ) plt . figure ( ) ax = sns . barplot ( '|SHAP|' , 'cpgs' , orient = 'h' , data = top_10_shaps ) ax . tick_params ( labelsize = 4 ) plt . savefig ( join ( output_dir , '{}_top_shap_values.png' . format ( entity ) ) , dpi = 300 ) if not absolute : print ( "All saved values reflect absolute values of sums, not sums of absolute values, if CpG SHAP scores are opposite signs across individuals, this will reduce the score of the resulting SHAP estimate." )
Return matrix of shapley values per class, with option to, classes/individuals are columns, CpGs are rows, option to plot multiple histograms/density plots.
Return matrix of shapley values per class with option to classes / individuals are columns CpGs are rows option to plot multiple histograms / density plots .
"""Return matrix of shapley values per class, with option to, classes/individuals are columns, CpGs are rows, option to plot multiple histograms/density plots.""" #df.corr().to_csv(join(output_dir,'returned_shap_values_corr_dist.csv'))
shapley_data
output_dir
individuals
classes
output_histogram
absolute
log_transform