<filename>src/plot.py<gh_stars>0
import glob, os
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
import argparse


parser = argparse.ArgumentParser()

parser.add_argument('--dir', type=str, default='analysis')
parser.add_argument('--task', type=str, default='cfever')
parser.add_argument('--seed', type=str, default=42)
parser.add_argument('--dir_tables', type=str, default=['analysis'], nargs='+')
parser.add_argument('--dir_rand', default=None)
parser.add_argument('--dir_bench', default=None)
parser.add_argument('--pooling', default=None)







args = parser.parse_args()


def plot(df, hue = "Strategy", hue_order = None):
    if args.dir_rand:
        # if use averaged results, remove the single run results
        df = df[df['Strategy'] != 'rand']
    unwanted = ['bertKM', 'FTbertKM', 'densityc', 'densitye', 'margin', 'least', 'alps-cal_seed',
                # 'alps-seedme', 'alps-seedmc',   'alps-seedmcl', 'alps-seedmel',
                # 'alps-seedcl',
                # 'alps-seedc',  'alps-seede', 'alps-seedel',
                'commitee_weighted_KL', 'commitee_weighted_KL-fake_balance',
                'commitee_weighted_vote-seedme', 'commitee_weighted_vote-seedmc',  'commitee_weighted_vote-seedmcl', 'commitee_weighted_vote-seedmel',
                'commitee_weighted_vote-seedmcel', 'commitee_weighted_vote-seedmce',
                'commitee_weighted_vote-seedcl', 'commitee_weighted_vote-seedc', 'commitee_weighted_vote-seede', 'commitee_weighted_vote-seedel',
                'alps-seedme-fake_balance', 'alps-seedmcl-fake_balance', 'alps-seedmel-fake_balance',
                'alps-seedc-fake_balance', 'alps-seede-fake_balance', 'alps-seedcl-fake_balance', 'alps-seedel-fake_balance'
                ]
    # unwanted = ['bertKM', 'FTbertKM', 'margin', 'least', 'densityc', 'densitye', 'densityes', 'commitee_vote', 'commitee_KL', 'commitee_vote-inverse', 'commitee_KL-inverse',
    #             'commitee_weighted_KL-inverse']
    # unwanted = ['commitee_weighted_KL', 'commitee_weighted_vote', 'commitee_vote', 'commitee_KL', 'bertKM', 'FTbertKM', 'margin', 'least', 'densityc', 'densitye', 'densityes']
    df = df[~df['Strategy'].isin(unwanted)]

    # value ='Acc'
    # value='F1'
    for value in ['Acc', 'F1']:
        key='Instance'
        title = '{}: Few-Shot Veracity Classification {} Performance'.format(args.task, value)
        # col_orders =['albert', 'bert', 'roberta', 'xlnet', 'bart', 'deberta']
        # col_orders = list(set(df['Model']))
        if len(set(df['Model'])) ==2:
            # col_orders =['longformer-base', 'longformer-large']
            col_orders =None
            # col_orders =['deberta-base', 'deberta-large']
        # elif 'bert-base' in set(df['Model']) and 'bert-large' in set(df['Model']):
        #     col_orders =['bert-base', 'roberta-base', 'deberta-base', 'bert-large', 'roberta-large', 'deberta-large']
        elif 'ensemble' in set(df['Model']):
            col_orders =['bert-base', 'roberta-base', 'deberta-base', 'bert-large', 'roberta-large', 'deberta-large', 'ensemble']
        elif 'ensemble_weighted' in set(df['Model']):
            col_orders =['bert-base', 'roberta-base', 'deberta-base', 'bert-large', 'roberta-large', 'deberta-large', 'ensemble_weighted']
        elif len(set(df['Model'])) ==6:
            col_orders =['bert-base', 'roberta-base', 'deberta-base', 'bert-large', 'roberta-large', 'deberta-large']
        else:
            col_orders =None
        print(col_orders)

        print(set(df['Strategy']))
        if 'alps-cal-inverse' in set(df['Strategy']):
            # hue_order = ['alps', 'alps-cal', 'densityes', 'densitycs', 'commitee_vote', 'commitee_KL']
            hue_order = ['alps-cal', 'commitee_weighted_KL', 'commitee_weighted_vote', 'alps-cal-inverse', 'commitee_weighted_KL-inverse', 'commitee_weighted_vote-inverse']
        # elif 'alps-cal-fake_balance' in set(df['Strategy']):
            # hue_order = ['alps', 'alps-cal', 'densityes', 'densitycs', 'commitee_vote', 'commitee_KL']
            # hue_order = ['alps-cal', 'commitee_weighted_KL', 'commitee_weighted_vote', 'alps-cal-fake_balance', 'commitee_weighted_KL-fake_balance', 'commitee_weighted_vote-fake_balance']
        elif len(set(df['Strategy'])) == 3 and 'commitee_weighted_vote-o' in set(df['Strategy']):
            hue_order = ['alps', 'commitee_weighted_vote', 'commitee_weighted_vote-o']
        elif len(set(df['Strategy'])) == 4 and 'alps-rerank' in set(df['Strategy']):
            hue_order = ['alps', 'commitee_weighted_vote', 'alps-o', 'alps-rerank']
        elif len(set(df['Strategy'])) == 4 and 'alps-o' in set(df['Strategy']):
            hue_order = ['alps', 'commitee_weighted_vote', 'commitee_weighted_vote-o', 'alps-o']
        elif len(set(df['Strategy'])) == 4 and 'commitee_weighted_vote-rerank' in set(df['Strategy']):
            hue_order = ['alps', 'commitee_weighted_vote', 'commitee_weighted_vote-o', 'commitee_weighted_vote-rerank']
        elif len(set(df['Strategy'])) == 5 and 'alps-rerank' in set(df['Strategy']):
            hue_order = ['alps', 'commitee_weighted_vote', 'commitee_weighted_vote-o', 'alps-o', 'alps-rerank']
        elif len(set(df['Strategy'])) == 5 and 'commitee_weighted_vote-rerank' in set(df['Strategy']):
            hue_order = ['alps', 'commitee_weighted_vote', 'commitee_weighted_vote-o', 'alps-o', 'commitee_weighted_vote-rerank']
        elif len(set(df['Strategy'])) == 6 and 'commitee_weighted_vote-rerank' in set(df['Strategy']):
            hue_order = ['alps', 'commitee_weighted_vote', 'commitee_weighted_vote-o', 'alps-o', 'commitee_weighted_vote-rerank', 'alps-rerank']
        # elif len(set(df['Strategy'])) == 9:
        #     hue_order = ['alps', 'alps-cal', 'densityes', 'densitycs', 'weighted', 'commitee_vote', 'commitee_KL', 'commitee_weighted_vote', 'commitee_weighted_KL']
        # elif len(set(df['Strategy'])) >5:
        #     hue_order = ['alps', 'alps-cal', 'alps-seede', 'alps-seedc', 'alps-seedme',  'alps-seedmc', 'alps-seedmce', 'alps-seedmec',  'alps-seedel', 'alps-seedcl', 'alps-seedmel', 'alps-seedmcl',  'alps-seedmcel', 'alps-seedmecl']
        #     hue_order = ['alps', 'alps-cal', 'alps-seedmet',  'alps-seedmct', 'alps-seedmetl', 'alps-seedmctl']
        #     hue_order = ['alps', 'alps-cal', 'alps-seedmep', 'alps-seedmcp', 'alps-seedmepl', 'alps-seedmcpl']
        #     hue_order = ['alps', 'alps-cal', 'alps-seedmep', 'alps-seedmcp', 'alps-seedmet',  'alps-seedmct']
        #     hue_order = ['alps', 'alps-cal', 'alps-seedmepl', 'alps-seedmcpl', 'alps-seedmetl', 'alps-seedmctl']
        #     hue_order = ['alps', 'alps-cal', 'alps-seedmep_', 'alps-seedmcp_', 'alps-seedmepl_', 'alps-seedmcpl_']
        #     hue_order = ['alps', 'alps-cal', 'alps-seedme', 'alps-seedmet', 'alps-seedmep_']

        # hue_order = ['alps', 'alps-cal', 'alps-seedmep', 'alps-seedmcp', 'alps-seedmet', 'alps-seedmct', 'alps-seedmepl', 'alps-seedmcpl', 'alps-seedmetl', 'alps-seedmctl']

        #     hue_order = ['entropy', 'alps-cal', 'entropy-seedme',  'entropy-seedmce', 'entropy-seedmec', 'entropy-seedcl']
        #     hue_order = ['densitycs', 'alps-cal', 'densitycs-seedme', 'densitycs-seedmce', 'densitycs-seedmec', 'densitycs-seedcl']

        else:
            hue_order=None
        print(hue_order)

        if 'Pattern' in df.keys():
            row_orders =set(df['Pattern'])
            g = sns.FacetGrid(df, hue=hue, hue_order=hue_order,
                              col='Model', col_order=col_orders,
                              row='Pattern', row_order=row_orders,
                              margin_titles=True)
        elif 'Pooling' in df.keys():
            row_orders =['cls', 'mean', 'max']
            g = sns.FacetGrid(df, hue=hue, hue_order=hue_order,
                              col='Model', col_order=col_orders,
                              row='Pooling', row_order=row_orders,
                              margin_titles=True)
        else:
            g = sns.FacetGrid(df, hue=hue, hue_order=hue_order,
                              col='Model', col_order=col_orders,
                              margin_titles=True)
        g.map(sns.lineplot, key, value)
        g.fig.suptitle(title, fontsize='large', fontweight='bold', y=1.01)
        g.set_axis_labels(key, value)
        axes = g.fig.axes

        if args.dir_rand:
            df_ = pd.read_table("{}/{}.csv".format(args.dir_rand, args.seed))
            df_['Model'] = df_['Model'].map(mapping)

            df_=df_.groupby(['Model', 'Instance', 'Strategy']).mean().reset_index()
            # print(df_)


            if 'Pattern' in df.keys():
                for ax, m in zip(axes, col_orders*len(row_orders)):
                    ax.plot(df_[df_['Model'] == m][key], df_[df_['Model'] == m][value], '--', C='k', label='rand')
            elif 'Pooling' in df.keys():
                for ax, m in zip(axes, col_orders*len(row_orders)):
                    ax.plot(df_[df_['Model'] == m][key], df_[df_['Model'] == m][value], '--', C='k', label='rand')
            else:
                for ax, m in zip(axes, col_orders):
                    # m=m.split("-")[0]
                    ax.plot(df_[df_['Model']==m][key], df_[df_['Model']==m][value], '--', C='k', label = 'rand')

        if args.dir_bench:
            df_ = pd.read_table("{}/{}.csv".format(args.dir_bench, args.seed))
            bench = 'alps-cal'
            df_=df_[df_['Strategy']==bench]

            df_=df_.groupby(['Model', 'Instance', 'Strategy']).mean().reset_index()
            # df_['Model'] = df_['Model'].map(mapping)
            # print(df_)

            if 'Pattern' in df.keys():
                for ax, m in zip(axes, col_orders*len(row_orders)):
                    ax.plot(df_[df_['Model'] == m][key], df_[df_['Model'] == m][value], '--', C='r', label='bench')
            elif 'Pooling' in df.keys():
                for ax, m in zip(axes, col_orders*len(row_orders)):
                    ax.plot(df_[df_['Model'] == m][key], df_[df_['Model'] == m][value], '--', C='r', label='bench')
            else:
                for ax, m in zip(axes, col_orders):
                    ax.plot(df_[df_['Model']==m][key], df_[df_['Model']==m][value], '--', C='r', label = 'bench')
        g.add_legend()


        plt.savefig('{}/{}.png'.format(args.dir, title), dpi=500)




if __name__ == '__main__':
    mapping = {'bert':'bert-large', 'roberta':'roberta-large', 'deberta':'deberta-large',
               'bert-base':'bert-base', 'roberta-base':'roberta-base', 'deberta-base':'deberta-base',
               'bert-large':'bert-large', 'roberta-large':'roberta-large', 'deberta-large':'deberta-large',
               'ensemble':'ensemble', 'ensemble_weighted':'ensemble_weighted',
               'longformer-base':'longformer-base', 'longformer-large':'longformer-large',
               'deberta-xxlarge':'deberta-xxlarge', 'albert-base':'albert-base'}
    pooling = {'bert-base':'cls', 'roberta-base':'cls', 'deberta-base':'mean',
               'bert-large':'cls', 'roberta-large':'cls', 'deberta-large':'mean'}
    pooling_cls = {'bert-base':'cls', 'roberta-base':'cls', 'deberta-base':'cls',
               'bert-large':'cls', 'roberta-large':'cls', 'deberta-large':'cls',
                   'longformer-base':'cls', 'longformer-large':'cls'}
    df=pd.DataFrame()
    for d in args.dir_tables:
        print(d)
        df_= pd.read_table("{}/{}.csv".format(d, args.seed))
        print(df_)
        df_['Model']=df_['Model'].map(mapping)
        df_ =df_[df_.Model.isin(mapping.values())]
        if set(df_['Strategy']) == set(['alps', 'alps-cal']):
            df_['Strategy']=df_['Strategy'].map({'alps':'alps-cal', 'alps-cal':'alps-cal'})
            print(df_)
        if 'Pattern' in df_.keys():
            df_ = df_[df_['Pattern'] == 0].drop(columns=['Pattern'])
        if 'Pooling' in df_.keys() and args.pooling == 'drop':
            df_ = df_.drop(columns=['Pooling'])
        elif 'Pooling' in df_.keys() and args.pooling == 'customize':
            data_=pd.DataFrame()
            for m in set(df_['Model']):
                data = df_[(df_['Pooling'] == pooling[m]) & (df_['Model'] == m)]
                data_=pd.concat([data_, data], ignore_index=True)
            df_ = data_.drop(columns=['Pooling'])
        elif 'Pooling' in df_.keys() and args.pooling == 'cls':
            data_=pd.DataFrame()
            for m in set(df_['Model']):
                data = df_[(df_['Pooling'] == pooling_cls[m]) & (df_['Model'] == m)]
                data_=pd.concat([data_, data], ignore_index=True)
            df_ = data_.drop(columns=['Pooling'])
        if 'inverse' in d.split('/'):
            print(df_['Strategy'])
            df_['Strategy']=df_['Strategy'].map({'alps-cal':'alps-cal-inverse',
                                                 'commitee_weighted_KL':'commitee_weighted_KL-inverse',
                                                 'commitee_weighted_vote':'commitee_weighted_vote-inverse',
                                                 'commitee_KL': 'commitee_KL-inverse',
                                                 'commitee_vote': 'commitee_vote-inverse'
                                                 })
            print(df_['Strategy'])
        if 'rerank' in d.split('/') or 'single_dup' in d.split('/'):
            print('rerank', df_['Strategy'])
            df_['Strategy'] = df_['Strategy'].map({
                'commitee_weighted_vote': 'commitee_weighted_vote-rerank',
                'alps': 'alps-rerank'})
        if 'fake_balance' in d.split('/') or 'o' in d.split('/'):
            print(df_['Strategy'])
            df_['Strategy']=df_['Strategy'].map({'alps-cal':'alps-cal-fake_balance',
                                                 'commitee_weighted_KL':'commitee_weighted_KL-fake_balance',
                                                 'commitee_weighted_vote':'commitee_weighted_vote-o',
                                                 'alps':'alps-o',
                                                 'commitee_KL': 'commitee_KL-fake_balance',
                                                 'commitee_vote': 'commitee_vote-fake_balance',
                                                 'alps-seedc':'alps-seedc-fake_balance',
                                                 'alps-seede': 'alps-seede-fake_balance',
                                                 'alps-seedcl': 'alps-seedcl-fake_balance',
                                                 'alps-seedel': 'alps-seedel-fake_balance',
                                                 'alps-seedmc': 'alps-seedmc-fake_balance',
                                                 'alps-seedme': 'alps-seedme-fake_balance',
                                                 'alps-seedmcl': 'alps-seedmcl-fake_balance',
                                                 'alps-seedmel': 'alps-seedmel-fake_balance',
                                                 'alps-cal_seed': 'alps-cal_seed-fake_balance',
                                                 })

            print(df_['Strategy'])
        print(df_)

        df= pd.concat([df, df_], ignore_index=True)
    # df.reset_index(drop=True)
    os.makedirs(args.dir, exist_ok=True)

    df.to_csv("{}/overview.csv".format(args.dir),
                                        sep='\t', encoding='utf-8', float_format='%.3f')


    print(df)

    plot(df)