text: import matplotlib.pyplot as plt
  import numpy as np
  import pandas as pd
  from get_ner_results_df import get_results_df
  
  def init_plot():
      plt.figure(figsize=(20,30))
      fig, (ax_f1, ax_em) = plt.subplots(2)
      ax_em.set_xscale("log")
      ax_f1.set_xscale("log")
      fig.suptitle('Experiment Results')
      return fig, ax_f1, ax_em
  
  def plot_f1_em_dict(res_dict):
  
      fig, ax_f1, ax_em = init_plot()
  
      line_name = 'splinter_base'
      x = [16,32,64,128, 256]
      y_f1 = []
      y_em = []
      yerr_f1_min = []
      yerr_em_min = []
      yerr_f1_max = []
      yerr_em_max = []
  
      for k in x:
  
  
          k_runs = [x for x in res_dict.keys() if x.startswith(str(k))]
          f1s = np.array([float(res_dict[run]['f1'][0]) for run in k_runs])
          ems = np.array([float(res_dict[run]['exact'][0]) for run in k_runs])
  
          if len(f1s) + len(ems) == 0:
              f1s = np.array([0] * len(x))
              ems = np.array([0] * len(x))
  
          y_f1.append(f1s.mean())
          y_em.append(ems.mean())
  
          yerr_f1_min.append(f1s.min())
          yerr_em_min.append(ems.min())
          yerr_f1_max.append(f1s.max())
          yerr_em_max.append(ems.max())
  
  
      ax_f1.plot(x, y_f1, label=line_name)
      ax_f1.fill_between(x, yerr_f1_min, yerr_f1_max, alpha=0.5)
      ax_em.plot(x, y_em, label=line_name)
      ax_em.fill_between(x, yerr_em_min, yerr_em_max, alpha=0.5)
  
      xrange = [2 ** x for x in range(4, 9)]
      xrange_text = [str(x) for x in xrange]
  
      ax_f1.set_title('F1 vs. #QA pairs')
      ax_f1.legend(prop={'size':6}, loc='upper left')
      ax_f1.set_xticks(xrange)
      ax_f1.set_xticklabels(xrange_text)
      ax_em.set_title('EM vs. #QA pairs')
      ax_em.legend(prop={'size':6}, loc='upper left')
      ax_em.set_xticks(xrange)
      ax_em.set_xticklabels(xrange_text)
      fig.tight_layout()
      plt.show()
  
  
  def plot_f1_em_dicts(names, dicts, show_min_max=True):
  
      fig, ax_f1, ax_em = init_plot()
  
      for line_name, d in zip(names, dicts):
          x = [16,32,64,128, 256]
          y_f1 = []
          y_em = []
          yerr_f1_min = []
          yerr_em_min = []
          yerr_f1_max = []
          yerr_em_max = []
  
          for k in x:
  
  
              k_runs = [x for x in d.keys() if x.startswith(str(k))]
              try:
                  f1s = np.array([float(d[run]['f1'][0]) for run in k_runs])
                  ems = np.array([float(d[run]['exact'][0]) for run in k_runs])
              except:
                  f1s = np.array([float(d[run]['f1']) for run in k_runs])
                  ems = np.array([float(d[run]['exact']) for run in k_runs])
  
              if len(f1s) + len(ems) == 0:
                  f1s = np.array([0] * len(x))
                  ems = np.array([0] * len(x))
  
              y_f1.append(f1s.mean())
              y_em.append(ems.mean())
  
              yerr_f1_min.append(f1s.min())
              yerr_em_min.append(ems.min())
              yerr_f1_max.append(f1s.max())
              yerr_em_max.append(ems.max())
  
  
          ax_f1.plot(x, y_f1, label=line_name)
          ax_em.plot(x, y_em, label=line_name)
  
          if show_min_max:
              ax_f1.fill_between(x, yerr_f1_min, yerr_f1_max, alpha=0.5)
              ax_em.fill_between(x, yerr_em_min, yerr_em_max, alpha=0.5)
  
          xrange = [2 ** x for x in range(4, 9)]
          xrange_text = [str(x) for x in xrange]
  
      ax_f1.set_title('F1 vs. #QA pairs')
      ax_f1.legend(prop={'size':6}, loc='upper left')
      ax_f1.set_xticks(xrange)
      ax_f1.set_xticklabels(xrange_text)
      ax_em.set_title('EM vs. #QA pairs')
      ax_em.legend(prop={'size':6}, loc='upper left')
      ax_em.set_xticks(xrange)
      ax_em.set_xticklabels(xrange_text)
      fig.tight_layout()
      plt.show()
  
  def plot_june_ninth_res():
      insert_bert = {'16-42': {'exact': ['4.673'], 'f1': ['7.727']}, '16-43': {'exact': ['6.262'], 'f1': ['10.371']},
       '16-44': {'exact': ['5.815'], 'f1': ['11.253']}, '16-45': {'exact': ['1.485'], 'f1': ['4.081']},
       '16-46': {'exact': ['3.207'], 'f1': ['4.574']}, '32-42': {'exact': ['9.146'], 'f1': ['13.918']},
       '32-43': {'exact': ['10.574'], 'f1': ['16.517']}, '32-44': {'exact': ['8.927'], 'f1': ['12.375']},
       '32-45': {'exact': ['10.859'], 'f1': ['15.279']}, '32-46': {'exact': ['12.763'], 'f1': ['19.023']},
       '64-42': {'exact': ['17.912'], 'f1': ['22.899']}, '64-43': {'exact': ['23.575'], 'f1': ['31.557']},
       '64-44': {'exact': ['28.629'], 'f1': ['36.973']}, '64-45': {'exact': ['8.118'], 'f1': ['11.196']},
       '64-46': {'exact': ['24.450'], 'f1': ['32.288']}, '128-42': {'exact': ['30.894'], 'f1': ['38.574']},
       '128-43': {'exact': ['36.861'], 'f1': ['44.864']}, '128-44': {'exact': ['44.608'], 'f1': ['53.243']},
       '128-45': {'exact': ['27.382'], 'f1': ['32.493']}, '128-46': {'exact': ['42.762'], 'f1': ['51.470']},
       '256-42': {'exact': ['42.239'], 'f1': ['49.223']}, '256-43': {'exact': ['49.281'], 'f1': ['57.320']},
       '256-44': {'exact': ['51.870'], 'f1': ['61.071']}, '256-45': {'exact': ['43.856'], 'f1': ['51.017']},
       '256-46': {'exact': ['47.759'], 'f1': ['56.813']}}
  
      return
  
  def plot_june_first_res():
      insert_word = {'16-42': {'exact': ['4.207'], 'f1': ['6.516']}, '16-43': {'exact': ['5.730'], 'f1': ['9.978']},
       '16-44': {'exact': ['9.042'], 'f1': ['16.027']}, '16-45': {'exact': ['2.170'], 'f1': ['5.289']},
       '16-46': {'exact': ['5.768'], 'f1': ['7.932']}, '32-42': {'exact': ['6.424'], 'f1': ['10.346']},
       '32-43': {'exact': ['14.067'], 'f1': ['20.653']}, '32-44': {'exact': ['19.844'], 'f1': ['28.444']},
       '32-45': {'exact': ['15.228'], 'f1': ['20.827']}, '32-46': {'exact': ['15.333'], 'f1': ['22.139']},
       '64-42': {'exact': ['23.156'], 'f1': ['29.377']}, '64-43': {'exact': ['26.801'], 'f1': ['35.368']},
       '64-44': {'exact': ['26.754'], 'f1': ['35.817']}, '64-45': {'exact': ['10.279'], 'f1': ['13.740']},
       '64-46': {'exact': ['24.831'], 'f1': ['31.113']}, '128-42': {'exact': ['32.521'], 'f1': ['41.913']},
       '128-43': {'exact': ['36.442'], 'f1': ['43.836']}, '128-44': {'exact': ['44.028'], 'f1': ['52.476']},
       '128-45': {'exact': ['34.253'], 'f1': ['40.423']}, '128-46': {'exact': ['39.907'], 'f1': ['47.744']}}
  
      sub_word = {'16-42': {'exact': ['3.569'], 'f1': ['5.851']}, '16-43': {'exact': ['4.930'], 'f1': ['8.387']}, '16-44': {'exact': ['5.568'], 'f1': ['9.988']}, '16-45': {'exact': ['1.770'], 'f1': ['4.554']}, '16-46': {'exact': ['4.112'], 'f1': ['6.375']}, '32-42': {'exact': ['8.214'], 'f1': ['12.589']}, '32-43': {'exact': ['10.593'], 'f1': ['16.887']}, '32-44': {'exact': ['20.367'], 'f1': ['29.290']}, '32-45': {'exact': ['10.707'], 'f1': ['15.389']}, '32-46': {'exact': ['14.942'], 'f1': ['21.200']}, '64-42': {'exact': ['17.255'], 'f1': ['21.947']}, '64-43': {'exact': ['25.193'], 'f1': ['33.124']}, '64-44': {'exact': ['22.395'], 'f1': ['30.204']}, '64-45': {'exact': ['17.331'], 'f1': ['21.969']}, '64-46': {'exact': ['26.068'], 'f1': ['33.391']}, '128-42': {'exact': ['26.677'], 'f1': ['32.834']}, '128-43': {'exact': ['34.415'], 'f1': ['41.417']}, '128-44': {'exact': ['39.250'], 'f1': ['47.647']}, '128-45': {'exact': ['17.588'], 'f1': ['20.865']}, '128-46': {'exact': ['40.992'], 'f1': ['49.047']}}
      insert_bert = {'16-42': {'exact': ['5.644'], 'f1': ['8.980']}, '16-43': {'exact': ['3.065'], 'f1': ['4.922']}, '16-44': {'exact': ['9.299'], 'f1': ['16.657']}, '16-45': {'exact': ['1.428'], 'f1': ['3.905']}, '16-46': {'exact': ['4.311'], 'f1': ['5.838']}, '32-42': {'exact': ['7.757'], 'f1': ['11.775']}, '32-43': {'exact': ['9.965'], 'f1': ['15.500']}, '32-44': {'exact': ['13.962'], 'f1': ['18.250']}, '32-45': {'exact': ['12.097'], 'f1': ['17.732']}, '32-46': {'exact': ['13.677'], 'f1': ['19.597']}, '64-42': {'exact': ['19.730'], 'f1': ['25.524']}, '64-43': {'exact': ['27.287'], 'f1': ['35.399']}, '64-44': {'exact': ['26.877'], 'f1': ['35.298']}, '64-45': {'exact': ['16.180'], 'f1': ['20.604']}, '64-46': {'exact': ['27.401'], 'f1': ['35.221']}, '128-43': {'exact': ['38.184'], 'f1': ['46.511']}, '128-44': {'exact': ['44.884'], 'f1': ['53.927']}, '128-45': {'exact': ['34.053'], 'f1': ['40.989']}, '128-46': {'exact': ['43.590'], 'f1': ['51.693']}}
      sub_bert = {'16-42': {'exact': ['3.483'], 'f1': ['5.999']}, '16-43': {'exact': ['3.883'], 'f1': ['7.038']}, '16-45': {'exact': ['1.351'], 'f1': ['3.638']}, '16-46': {'exact': ['3.845'], 'f1': ['5.954']}, '32-42': {'exact': ['5.206'], 'f1': ['8.827']}, '32-43': {'exact': ['8.699'], 'f1': ['14.026']}, '32-44': {'exact': ['16.608'], 'f1': ['24.796']}, '32-45': {'exact': ['6.748'], 'f1': ['10.752']}, '32-46': {'exact': ['11.116'], 'f1': ['16.437']}, '64-42': {'exact': ['15.247'], 'f1': ['19.749']}, '64-43': {'exact': ['19.359'], 'f1': ['26.196']}, '64-44': {'exact': ['21.852'], 'f1': ['29.529']}, '64-45': {'exact': ['19.330'], 'f1': ['24.742']}, '64-46': {'exact': ['22.014'], 'f1': ['30.077']}, '128-42': {'exact': ['29.704'], 'f1': ['38.446']}, '128-43': {'exact': ['29.656'], 'f1': ['36.370']}, '128-44': {'exact': ['39.916'], 'f1': ['48.002']}, '128-45': {'exact': ['10.488'], 'f1': ['12.426']}, '128-46': {'exact': ['37.803'], 'f1': ['45.590']}}
  
      names = ['insert_word', 'sub_word', 'insert_bert', 'sub_bert']
      dicts = [insert_word, sub_word, insert_bert, sub_bert]
      plot_f1_em_dicts(names, dicts)
  
  def deep_mosaic_vs_insert():
  
      baseline = {'64-43': {'exact': 23.289, 'f1': 31.024}, '16-46': {'exact': 4.064, 'f1': 5.716},
                  '16-43': {'exact': 5.511, 'f1': 9.563}, '64-45': {'exact': 22.004, 'f1': 28.248},
                  '32-42': {'exact': 7.052, 'f1': 11.504}, '256-46': {'exact': 47.464, 'f1': 57.048},
                  '128-43': {'exact': 30.494, 'f1': 39.791}, '64-46': {'exact': 25.136, 'f1': 33.040},
                  '128-44': {'exact': 35.719, 'f1': 43.534}, '32-44': {'exact': 19.378, 'f1': 27.676},
                  '32-43': {'exact': 11.402, 'f1': 17.522}, '64-42': {'exact': 21.662, 'f1': 26.912},
                  '16-44': {'exact': 9.089, 'f1': 16.428}, '16-42': {'exact': 4.683, 'f1': 6.779},
                  '256-43': {'exact': 52.213, 'f1': 61.855}, '256-45': {'exact': 40.973, 'f1': 50.231},
                  '128-45': {'exact': 29.219, 'f1': 35.873}, '256-42': {'exact': 41.515, 'f1': 50.887},
                  '16-45': {'exact': 1.713, 'f1': 4.461}, '128-46': {'exact': 39.688, 'f1': 49.168},
                  '64-44': {'exact': 25.707, 'f1': 33.754}, '256-44': {'exact': 48.568, 'f1': 58.361},
                  '128-42': {'exact': 27.163, 'f1': 34.966}}
  
      insert_bert_1_aug = {'16-42': {'exact': ['5.387'], 'f1': ['8.445']}, '16-43': {'exact': ['4.616'], 'f1': ['7.845']},
       '16-44': {'exact': ['6.177'], 'f1': ['10.417']}, '16-45': {'exact': ['1.723'], 'f1': ['4.527']},
       '16-46': {'exact': ['8.109'], 'f1': ['11.632']}, '32-42': {'exact': ['7.081'], 'f1': ['11.242']},
       '32-43': {'exact': ['13.876'], 'f1': ['20.547']}, '32-44': {'exact': ['19.397'], 'f1': ['27.645']},
       '32-45': {'exact': ['11.954'], 'f1': ['17.526']}, '32-46': {'exact': ['13.324'], 'f1': ['19.597']},
       '64-42': {'exact': ['21.605'], 'f1': ['28.500']}, '64-43': {'exact': ['25.859'], 'f1': ['34.296']},
       '64-44': {'exact': ['27.981'], 'f1': ['36.544']}, '64-45': {'exact': ['13.610'], 'f1': ['18.535']},
       '64-46': {'exact': ['21.709'], 'f1': ['28.470']}, '128-42': {'exact': ['29.542'], 'f1': ['37.570']},
       '128-43': {'exact': ['35.729'], 'f1': ['45.586']}, '128-44': {'exact': ['34.901'], 'f1': ['41.902']},
       '128-45': {'exact': ['29.257'], 'f1': ['35.812']}, '128-46': {'exact': ['39.488'], 'f1': ['48.145']},
       '256-42': {'exact': ['27.791'], 'f1': ['33.611']}, '256-43': {'exact': ['52.041'], 'f1': ['61.539']},
       '256-44': {'exact': ['48.796'], 'f1': ['57.675']}, '256-45': {'exact': ['45.084'], 'f1': ['53.613']},
       '256-46': {'exact': ['49.748'], 'f1': ['58.884']}}
  
      insert_bert_2_aug = {'16-42': {'exact': ['4.854'], 'f1': ['8.481']}, '16-43': {'exact': ['6.386'], 'f1': ['10.300']},
       '16-44': {'exact': ['6.786'], 'f1': ['13.298']}, '16-45': {'exact': ['1.837'], 'f1': ['4.799']},
       '16-46': {'exact': ['4.797'], 'f1': ['6.461']}, '32-42': {'exact': ['6.215'], 'f1': ['9.978']},
       '32-43': {'exact': ['10.888'], 'f1': ['17.446']}, '32-44': {'exact': ['13.239'], 'f1': ['19.302']},
       '32-45': {'exact': ['11.411'], 'f1': ['16.763']}, '32-46': {'exact': ['12.287'], 'f1': ['18.741']},
       '64-42': {'exact': ['12.620'], 'f1': ['16.471']}, '64-43': {'exact': ['23.974'], 'f1': ['31.751']},
       '64-44': {'exact': ['23.327'], 'f1': ['31.544']}, '64-45': {'exact': ['23.708'], 'f1': ['30.206']},
       '64-46': {'exact': ['22.461'], 'f1': ['29.837']}, '128-42': {'exact': ['33.644'], 'f1': ['41.888']},
       '128-43': {'exact': ['36.766'], 'f1': ['44.247']}, '128-44': {'exact': ['37.147'], 'f1': ['45.197']},
       '128-45': {'exact': ['8.670'], 'f1': ['10.896']}, '128-46': {'exact': ['40.487'], 'f1': ['48.716']},
       '256-42': {'exact': ['44.190'], 'f1': ['51.662']}, '256-43': {'exact': ['49.795'], 'f1': ['58.995']},
       '256-44': {'exact': ['49.910'], 'f1': ['58.695']}, '256-45': {'exact': ['43.466'], 'f1': ['51.101']},
       '256-46': {'exact': ['49.329'], 'f1': ['59.208']}}
  
      insert_bert_3_aug = {'16-42': {'exact': ['4.673'], 'f1': ['7.727']}, '16-43': {'exact': ['6.262'], 'f1': ['10.371']},
       '16-44': {'exact': ['5.815'], 'f1': ['11.253']}, '16-45': {'exact': ['1.485'], 'f1': ['4.081']},
       '16-46': {'exact': ['3.207'], 'f1': ['4.574']}, '32-42': {'exact': ['9.146'], 'f1': ['13.918']},
       '32-43': {'exact': ['10.574'], 'f1': ['16.517']}, '32-44': {'exact': ['8.927'], 'f1': ['12.375']},
       '32-45': {'exact': ['10.859'], 'f1': ['15.279']}, '32-46': {'exact': ['12.763'], 'f1': ['19.023']},
       '64-42': {'exact': ['17.912'], 'f1': ['22.899']}, '64-43': {'exact': ['23.575'], 'f1': ['31.557']},
       '64-44': {'exact': ['28.629'], 'f1': ['36.973']}, '64-45': {'exact': ['8.118'], 'f1': ['11.196']},
       '64-46': {'exact': ['24.450'], 'f1': ['32.288']}, '128-42': {'exact': ['30.894'], 'f1': ['38.574']},
       '128-43': {'exact': ['36.861'], 'f1': ['44.864']}, '128-44': {'exact': ['44.608'], 'f1': ['53.243']},
       '128-45': {'exact': ['27.382'], 'f1': ['32.493']}, '128-46': {'exact': ['42.762'], 'f1': ['51.470']},
       '256-42': {'exact': ['42.239'], 'f1': ['49.223']}, '256-43': {'exact': ['49.281'], 'f1': ['57.320']},
       '256-44': {'exact': ['51.870'], 'f1': ['61.071']}, '256-45': {'exact': ['43.856'], 'f1': ['51.017']},
       '256-46': {'exact': ['47.759'], 'f1': ['56.813']}}
  
      insert_bert_4_aug = {'16-42': {'exact': ['5.644'], 'f1': ['8.980']}, '16-43': {'exact': ['3.065'], 'f1': ['4.922']},
         '16-44': {'exact': ['9.299'], 'f1': ['16.657']}, '16-45': {'exact': ['1.428'], 'f1': ['3.905']},
         '16-46': {'exact': ['4.311'], 'f1': ['5.838']}, '32-42': {'exact': ['7.757'], 'f1': ['11.775']},
         '32-43': {'exact': ['9.965'], 'f1': ['15.500']}, '32-44': {'exact': ['13.962'], 'f1': ['18.250']},
         '32-45': {'exact': ['12.097'], 'f1': ['17.732']}, '32-46': {'exact': ['13.677'], 'f1': ['19.597']},
         '64-42': {'exact': ['19.730'], 'f1': ['25.524']}, '64-43': {'exact': ['27.287'], 'f1': ['35.399']},
         '64-44': {'exact': ['26.877'], 'f1': ['35.298']}, '64-45': {'exact': ['16.180'], 'f1': ['20.604']},
         '64-46': {'exact': ['27.401'], 'f1': ['35.221']}, '128-43': {'exact': ['38.184'], 'f1': ['46.511']},
         '128-44': {'exact': ['44.884'], 'f1': ['53.927']}, '128-45': {'exact': ['34.053'], 'f1': ['40.989']},
         '128-46': {'exact': ['43.590'], 'f1': ['51.693']}}
  
      mosaic2 = {'16-42': {'exact': ['6.243'], 'f1': ['8.402']}, '16-43': {'exact': ['10.821'], 'f1': ['16.966']}, '16-44': {'exact': ['8.718'], 'f1': ['14.004']}, '16-45': {'exact': ['4.131'], 'f1': ['9.880']}, '16-46': {'exact': ['5.273'], 'f1': ['7.045']}, '32-42': {'exact': ['14.685'], 'f1': ['20.597']}, '32-43': {'exact': ['12.972'], 'f1': ['19.545']}, '32-44': {'exact': ['21.233'], 'f1': ['29.625']}, '32-45': {'exact': ['17.398'], 'f1': ['24.793']}, '32-46': {'exact': ['15.837'], 'f1': ['22.776']}, '64-42': {'exact': ['27.258'], 'f1': ['34.719']}, '64-43': {'exact': ['26.354'], 'f1': ['34.528']}, '64-44': {'exact': ['30.484'], 'f1': ['39.154']}, '64-45': {'exact': ['27.287'], 'f1': ['35.167']}, '64-46': {'exact': ['24.936'], 'f1': ['30.956']}, '128-42': {'exact': ['37.565'], 'f1': ['46.523']}, '128-43': {'exact': ['38.555'], 'f1': ['48.459']}, '128-44': {'exact': ['43.837'], 'f1': ['52.290']}, '128-45': {'exact': ['32.264'], 'f1': ['39.862']}, '128-46': {'exact': ['43.676'], 'f1': ['52.595']}, '256-42': {'exact': ['47.473'], 'f1': ['55.532']}, '256-43': {'exact': ['52.613'], 'f1': ['62.630']}, '256-44': {'exact': ['49.043'], 'f1': ['58.062']}, '256-45': {'exact': ['43.295'], 'f1': ['52.298']}, '256-46': {'exact': ['51.404'], 'f1': ['60.902']}}
      mosaic4 = {'16-42': {'exact': ['9.917'], 'f1': ['13.520']}, '16-43': {'exact': ['6.948'], 'f1': ['10.071']}, '16-45': {'exact': ['6.158'], 'f1': ['11.808']}, '16-46': {'exact': ['4.435'], 'f1': ['6.100']}, '32-42': {'exact': ['23.679'], 'f1': ['32.046']}, '32-44': {'exact': ['21.262'], 'f1': ['29.573']}, '32-45': {'exact': ['19.206'], 'f1': ['27.152']}, '32-46': {'exact': ['19.435'], 'f1': ['26.089']}, '64-42': {'exact': ['28.467'], 'f1': ['35.044']}, '64-43': {'exact': ['28.609'], 'f1': ['36.021']}, '64-44': {'exact': ['27.820'], 'f1': ['35.246']}, '64-45': {'exact': ['29.742'], 'f1': ['36.741']}, '64-46': {'exact': ['29.990'], 'f1': ['35.766']}, '128-42': {'exact': ['38.222'], 'f1': ['47.068']}, '128-43': {'exact': ['39.555'], 'f1': ['47.748']}, '128-44': {'exact': ['41.658'], 'f1': ['47.879']}, '128-45': {'exact': ['37.851'], 'f1': ['45.811']}, '128-46': {'exact': ['44.256'], 'f1': ['53.318']}, '256-42': {'exact': ['50.719'], 'f1': ['58.678']}, '256-43': {'exact': ['55.087'], 'f1': ['64.735']}, '256-44': {'exact': ['54.763'], 'f1': ['65.191']}, '256-45': {'exact': ['47.064'], 'f1': ['54.730']}, '256-46': {'exact': ['49.691'], 'f1': ['56.455']}}
      names = ['baseline', 'insert_bert1', 'insert_bert2', 'insert_bert3', 'insert_bert4', 'mosaic2', 'mosaic4']
      dicts = [baseline, insert_bert_1_aug, insert_bert_2_aug, insert_bert_3_aug, insert_bert_4_aug, mosaic2, mosaic4]
      plot_f1_em_dicts(names, dicts)
  
  def plot_june_first_res():
  
      baseline = {'64-43': {'exact': 23.289, 'f1': 31.024}, '16-46': {'exact': 4.064, 'f1': 5.716},
                  '16-43': {'exact': 5.511, 'f1': 9.563}, '64-45': {'exact': 22.004, 'f1': 28.248},
                  '32-42': {'exact': 7.052, 'f1': 11.504}, '256-46': {'exact': 47.464, 'f1': 57.048},
                  '128-43': {'exact': 30.494, 'f1': 39.791}, '64-46': {'exact': 25.136, 'f1': 33.040},
                  '128-44': {'exact': 35.719, 'f1': 43.534}, '32-44': {'exact': 19.378, 'f1': 27.676},
                  '32-43': {'exact': 11.402, 'f1': 17.522}, '64-42': {'exact': 21.662, 'f1': 26.912},
                  '16-44': {'exact': 9.089, 'f1': 16.428}, '16-42': {'exact': 4.683, 'f1': 6.779},
                  '256-43': {'exact': 52.213, 'f1': 61.855}, '256-45': {'exact': 40.973, 'f1': 50.231},
                  '128-45': {'exact': 29.219, 'f1': 35.873}, '256-42': {'exact': 41.515, 'f1': 50.887},
                  '16-45': {'exact': 1.713, 'f1': 4.461}, '128-46': {'exact': 39.688, 'f1': 49.168},
                  '64-44': {'exact': 25.707, 'f1': 33.754}, '256-44': {'exact': 48.568, 'f1': 58.361},
                  '128-42': {'exact': 27.163, 'f1': 34.966}}
  
      insert_word = {'16-42': {'exact': ['4.207'], 'f1': ['6.516']}, '16-43': {'exact': ['5.730'], 'f1': ['9.978']},
       '16-44': {'exact': ['9.042'], 'f1': ['16.027']}, '16-45': {'exact': ['2.170'], 'f1': ['5.289']},
       '16-46': {'exact': ['5.768'], 'f1': ['7.932']}, '32-42': {'exact': ['6.424'], 'f1': ['10.346']},
       '32-43': {'exact': ['14.067'], 'f1': ['20.653']}, '32-44': {'exact': ['19.844'], 'f1': ['28.444']},
       '32-45': {'exact': ['15.228'], 'f1': ['20.827']}, '32-46': {'exact': ['15.333'], 'f1': ['22.139']},
       '64-42': {'exact': ['23.156'], 'f1': ['29.377']}, '64-43': {'exact': ['26.801'], 'f1': ['35.368']},
       '64-44': {'exact': ['26.754'], 'f1': ['35.817']}, '64-45': {'exact': ['10.279'], 'f1': ['13.740']},
       '64-46': {'exact': ['24.831'], 'f1': ['31.113']}, '128-42': {'exact': ['32.521'], 'f1': ['41.913']},
       '128-43': {'exact': ['36.442'], 'f1': ['43.836']}, '128-44': {'exact': ['44.028'], 'f1': ['52.476']},
       '128-45': {'exact': ['34.253'], 'f1': ['40.423']}, '128-46': {'exact': ['39.907'], 'f1': ['47.744']}}
  
      sub_word = {'16-42': {'exact': ['3.569'], 'f1': ['5.851']}, '16-43': {'exact': ['4.930'], 'f1': ['8.387']}, '16-44': {'exact': ['5.568'], 'f1': ['9.988']}, '16-45': {'exact': ['1.770'], 'f1': ['4.554']}, '16-46': {'exact': ['4.112'], 'f1': ['6.375']}, '32-42': {'exact': ['8.214'], 'f1': ['12.589']}, '32-43': {'exact': ['10.593'], 'f1': ['16.887']}, '32-44': {'exact': ['20.367'], 'f1': ['29.290']}, '32-45': {'exact': ['10.707'], 'f1': ['15.389']}, '32-46': {'exact': ['14.942'], 'f1': ['21.200']}, '64-42': {'exact': ['17.255'], 'f1': ['21.947']}, '64-43': {'exact': ['25.193'], 'f1': ['33.124']}, '64-44': {'exact': ['22.395'], 'f1': ['30.204']}, '64-45': {'exact': ['17.331'], 'f1': ['21.969']}, '64-46': {'exact': ['26.068'], 'f1': ['33.391']}, '128-42': {'exact': ['26.677'], 'f1': ['32.834']}, '128-43': {'exact': ['34.415'], 'f1': ['41.417']}, '128-44': {'exact': ['39.250'], 'f1': ['47.647']}, '128-45': {'exact': ['17.588'], 'f1': ['20.865']}, '128-46': {'exact': ['40.992'], 'f1': ['49.047']}}
      insert_bert = {'16-42': {'exact': ['5.644'], 'f1': ['8.980']}, '16-43': {'exact': ['3.065'], 'f1': ['4.922']}, '16-44': {'exact': ['9.299'], 'f1': ['16.657']}, '16-45': {'exact': ['1.428'], 'f1': ['3.905']}, '16-46': {'exact': ['4.311'], 'f1': ['5.838']}, '32-42': {'exact': ['7.757'], 'f1': ['11.775']}, '32-43': {'exact': ['9.965'], 'f1': ['15.500']}, '32-44': {'exact': ['13.962'], 'f1': ['18.250']}, '32-45': {'exact': ['12.097'], 'f1': ['17.732']}, '32-46': {'exact': ['13.677'], 'f1': ['19.597']}, '64-42': {'exact': ['19.730'], 'f1': ['25.524']}, '64-43': {'exact': ['27.287'], 'f1': ['35.399']}, '64-44': {'exact': ['26.877'], 'f1': ['35.298']}, '64-45': {'exact': ['16.180'], 'f1': ['20.604']}, '64-46': {'exact': ['27.401'], 'f1': ['35.221']}, '128-43': {'exact': ['38.184'], 'f1': ['46.511']}, '128-44': {'exact': ['44.884'], 'f1': ['53.927']}, '128-45': {'exact': ['34.053'], 'f1': ['40.989']}, '128-46': {'exact': ['43.590'], 'f1': ['51.693']}}
  
      insert_bert_1_aug = {'16-42': {'exact': ['5.387'], 'f1': ['8.445']}, '16-43': {'exact': ['4.616'], 'f1': ['7.845']},'16-44': {'exact': ['6.177'], 'f1': ['10.417']}, '16-45': {'exact': ['1.723'], 'f1': ['4.527']},'16-46': {'exact': ['8.109'], 'f1': ['11.632']}, '32-42': {'exact': ['7.081'], 'f1': ['11.242']},'32-43': {'exact': ['13.876'], 'f1': ['20.547']}, '32-44': {'exact': ['19.397'], 'f1': ['27.645']},'32-45': {'exact': ['11.954'], 'f1': ['17.526']}, '32-46': {'exact': ['13.324'], 'f1': ['19.597']},'64-42': {'exact': ['21.605'], 'f1': ['28.500']}, '64-43': {'exact': ['25.859'], 'f1': ['34.296']},'64-44': {'exact': ['27.981'], 'f1': ['36.544']}, '64-45': {'exact': ['13.610'], 'f1': ['18.535']},'64-46': {'exact': ['21.709'], 'f1': ['28.470']}, '128-42': {'exact': ['29.542'], 'f1': ['37.570']},'128-43': {'exact': ['35.729'], 'f1': ['45.586']}, '128-44': {'exact': ['34.901'], 'f1': ['41.902']},'128-45': {'exact': ['29.257'], 'f1': ['35.812']}, '128-46': {'exact': ['39.488'], 'f1': ['48.145']},'256-42': {'exact': ['27.791'], 'f1': ['33.611']}, '256-43': {'exact': ['52.041'], 'f1': ['61.539']},'256-44': {'exact': ['48.796'], 'f1': ['57.675']}, '256-45': {'exact': ['45.084'], 'f1': ['53.613']},'256-46': {'exact': ['49.748'], 'f1': ['58.884']}}
  
      sub_bert = {'16-42': {'exact': ['3.483'], 'f1': ['5.999']}, '16-43': {'exact': ['3.883'], 'f1': ['7.038']}, '16-45': {'exact': ['1.351'], 'f1': ['3.638']}, '16-46': {'exact': ['3.845'], 'f1': ['5.954']}, '32-42': {'exact': ['5.206'], 'f1': ['8.827']}, '32-43': {'exact': ['8.699'], 'f1': ['14.026']}, '32-44': {'exact': ['16.608'], 'f1': ['24.796']}, '32-45': {'exact': ['6.748'], 'f1': ['10.752']}, '32-46': {'exact': ['11.116'], 'f1': ['16.437']}, '64-42': {'exact': ['15.247'], 'f1': ['19.749']}, '64-43': {'exact': ['19.359'], 'f1': ['26.196']}, '64-44': {'exact': ['21.852'], 'f1': ['29.529']}, '64-45': {'exact': ['19.330'], 'f1': ['24.742']}, '64-46': {'exact': ['22.014'], 'f1': ['30.077']}, '128-42': {'exact': ['29.704'], 'f1': ['38.446']}, '128-43': {'exact': ['29.656'], 'f1': ['36.370']}, '128-44': {'exact': ['39.916'], 'f1': ['48.002']}, '128-45': {'exact': ['10.488'], 'f1': ['12.426']}, '128-46': {'exact': ['37.803'], 'f1': ['45.590']}}
      mosaic2 = {'16-42': {'exact': ['6.243'], 'f1': ['8.402']}, '16-43': {'exact': ['10.821'], 'f1': ['16.966']}, '16-44': {'exact': ['8.718'], 'f1': ['14.004']}, '16-45': {'exact': ['4.131'], 'f1': ['9.880']}, '16-46': {'exact': ['5.273'], 'f1': ['7.045']}, '32-42': {'exact': ['14.685'], 'f1': ['20.597']}, '32-43': {'exact': ['12.972'], 'f1': ['19.545']}, '32-44': {'exact': ['21.233'], 'f1': ['29.625']}, '32-45': {'exact': ['17.398'], 'f1': ['24.793']}, '32-46': {'exact': ['15.837'], 'f1': ['22.776']}, '64-42': {'exact': ['27.258'], 'f1': ['34.719']}, '64-43': {'exact': ['26.354'], 'f1': ['34.528']}, '64-44': {'exact': ['30.484'], 'f1': ['39.154']}, '64-45': {'exact': ['27.287'], 'f1': ['35.167']}, '64-46': {'exact': ['24.936'], 'f1': ['30.956']}, '128-42': {'exact': ['37.565'], 'f1': ['46.523']}, '128-43': {'exact': ['38.555'], 'f1': ['48.459']}, '128-44': {'exact': ['43.837'], 'f1': ['52.290']}, '128-45': {'exact': ['32.264'], 'f1': ['39.862']}, '128-46': {'exact': ['43.676'], 'f1': ['52.595']}, '256-42': {'exact': ['47.473'], 'f1': ['55.532']}, '256-43': {'exact': ['52.613'], 'f1': ['62.630']}, '256-44': {'exact': ['49.043'], 'f1': ['58.062']}, '256-45': {'exact': ['43.295'], 'f1': ['52.298']}, '256-46': {'exact': ['51.404'], 'f1': ['60.902']}}
  
      names = ['baseline', 'insert_word', 'sub_word', 'insert_bert', 'insert_bert_1_aug', 'sub_bert', 'mosaic2']
      dicts = [
  
  
  
  
  
  
  
  
  
  
  
  
  
  
  
  
  
  
  
  
  
  
  
  
  
  
  
  
  
  
  
          baseline, insert_word, sub_word, insert_bert, sub_bert, mosaic2]
      plot_f1_em_dicts(names, dicts)
  
  def plot_mosaic_augs_exp_res():
      baseline = {'64-43': {'exact': 23.289, 'f1': 31.024}, '16-46': {'exact': 4.064, 'f1': 5.716},
                  '16-43': {'exact': 5.511, 'f1': 9.563}, '64-45': {'exact': 22.004, 'f1': 28.248},
                  '32-42': {'exact': 7.052, 'f1': 11.504}, '256-46': {'exact': 47.464, 'f1': 57.048},
                  '128-43': {'exact': 30.494, 'f1': 39.791}, '64-46': {'exact': 25.136, 'f1': 33.040},
                  '128-44': {'exact': 35.719, 'f1': 43.534}, '32-44': {'exact': 19.378, 'f1': 27.676},
                  '32-43': {'exact': 11.402, 'f1': 17.522}, '64-42': {'exact': 21.662, 'f1': 26.912},
                  '16-44': {'exact': 9.089, 'f1': 16.428}, '16-42': {'exact': 4.683, 'f1': 6.779},
                  '256-43': {'exact': 52.213, 'f1': 61.855}, '256-45': {'exact': 40.973, 'f1': 50.231},
                  '128-45': {'exact': 29.219, 'f1': 35.873}, '256-42': {'exact': 41.515, 'f1': 50.887},
                  '16-45': {'exact': 1.713, 'f1': 4.461}, '128-46': {'exact': 39.688, 'f1': 49.168},
                  '64-44': {'exact': 25.707, 'f1': 33.754}, '256-44': {'exact': 48.568, 'f1': 58.361},
                  '128-42': {'exact': 27.163, 'f1': 34.966}}
  
      mosaic_2pairs = {'16-44':{'exact':5.844, 'f1':10.028}, '64-42':{'exact':27.915, 'f1':36.496}, '16-45':{'exact':2.789, 'f1':5.664}, '128-44':{'exact':43.323, 'f1':52.249}, '16-42':{'exact':4.606, 'f1':6.448}, '256-45':{'exact':51.432, 'f1':61.101}, '128-45':{'exact':32.093, 'f1':40.903}, '64-43':{'exact':21.024, 'f1':29.088}, '64-46':{'exact':29.457, 'f1':38.085}, '64-45':{'exact':25.526, 'f1':34.967}, '16-43':{'exact':10.593, 'f1':15.511}, '16-46':{'exact':4.035, 'f1':5.851}, '128-43':{'exact':34.729, 'f1':45.188}, '32-45':{'exact':17.807, 'f1':25.613}, '32-44':{'exact':9.375, 'f1':12.329}, '256-44':{'exact':51.413, 'f1':61.543}, '32-42':{'exact':16.341, 'f1':22.559}, '256-42':{'exact':52.594, 'f1':61.348}, '256-46':{'exact':51.823, 'f1':61.318}, '32-43':{'exact':12.972, 'f1':21.194}, '256-43':{'exact':51.756, 'f1':61.193}, '128-42':{'exact':39.412, 'f1':49.808}, '128-46':{'exact':44.094, 'f1':53.837}, '64-44':{'exact':25.726, 'f1':34.979}, '32-46':{'exact':16.199, 'f1':22.795}}
  
      insert_bert_3_aug = {'16-42': {'exact': ['4.673'], 'f1': ['7.727']}, '16-43': {'exact': ['6.262'], 'f1': ['10.371']},
       '16-44': {'exact': ['5.815'], 'f1': ['11.253']}, '16-45': {'exact': ['1.485'], 'f1': ['4.081']},
       '16-46': {'exact': ['3.207'], 'f1': ['4.574']}, '32-42': {'exact': ['9.146'], 'f1': ['13.918']},
       '32-43': {'exact': ['10.574'], 'f1': ['16.517']}, '32-44': {'exact': ['8.927'], 'f1': ['12.375']},
       '32-45': {'exact': ['10.859'], 'f1': ['15.279']}, '32-46': {'exact': ['12.763'], 'f1': ['19.023']},
       '64-42': {'exact': ['17.912'], 'f1': ['22.899']}, '64-43': {'exact': ['23.575'], 'f1': ['31.557']},
       '64-44': {'exact': ['28.629'], 'f1': ['36.973']}, '64-45': {'exact': ['8.118'], 'f1': ['11.196']},
       '64-46': {'exact': ['24.450'], 'f1': ['32.288']}, '128-42': {'exact': ['30.894'], 'f1': ['38.574']},
       '128-43': {'exact': ['36.861'], 'f1': ['44.864']}, '128-44': {'exact': ['44.608'], 'f1': ['53.243']},
       '128-45': {'exact': ['27.382'], 'f1': ['32.493']}, '128-46': {'exact': ['42.762'], 'f1': ['51.470']},
       '256-42': {'exact': ['42.239'], 'f1': ['49.223']}, '256-43': {'exact': ['49.281'], 'f1': ['57.320']},
       '256-44': {'exact': ['51.870'], 'f1': ['61.071']}, '256-45': {'exact': ['43.856'], 'f1': ['51.017']},
       '256-46': {'exact': ['47.759'], 'f1': ['56.813']}}
  
      names = ['baseline', 'mosaic_2pairs', 'insert_bert_3augs']
      dicts = [baseline, mosaic_2pairs, insert_bert_3_aug]
      plot_f1_em_dicts(names, dicts)
  
  def plot_window_exps():
      baseline = {'64-43': {'exact': [23.289], 'f1': [31.024]}, '16-46': {'exact': [4.064], 'f1': [5.716]}, '16-43': {'exact': [5.511], 'f1': [9.563]}, '64-45': {'exact': [22.004], 'f1': [28.248]}, '32-42': {'exact': [7.052], 'f1': [11.504]}, '256-46': {'exact': [47.464], 'f1': [57.048]}, '128-43': {'exact': [30.494], 'f1': [39.791]}, '64-46': {'exact': [25.136], 'f1': [33.040]}, '128-44': {'exact': [35.719], 'f1': [43.534]}, '32-44': {'exact': [19.378], 'f1': [27.676]}, '32-43': {'exact': [11.402], 'f1': [17.522]}, '64-42': {'exact': [21.662], 'f1': [26.912]}, '16-44': {'exact': [9.089], 'f1': [16.428]}, '16-42': {'exact': [4.683], 'f1': [6.779]}, '256-43': {'exact': [52.213], 'f1': [61.855]}, '256-45': {'exact': [40.973], 'f1': [50.231]}, '128-45': {'exact': [29.219], 'f1': [35.873]}, '256-42': {'exact': [41.515], 'f1': [50.887]}, '16-45': {'exact': [1.713], 'f1': [4.461]}, '128-46': {'exact': [39.688], 'f1': [49.168]}, '64-44': {'exact': [25.707], 'f1': [33.754]}, '256-44': {'exact': [48.568], 'f1': [58.361]}, '128-42': {'exact': [27.163], 'f1': [34.966]}}
      mosaic_w_b100_a0 = {'256-42': {'exact': 45.39830589131056,'f1': 53.33480713251173} ,'256-43': {'exact': 45.19843913581422,'f1': 54.24567595493678} ,'256-44': {'exact': 45.188921671266776,'f1': 52.941394445339995} ,'256-45': {'exact': 41.97201865423051,'f1': 50.58784693799245} ,'256-46': {'exact': 48.443894546492814,'f1': 57.641861586572546} ,'128-42': {'exact': 32.273722280384504,'f1': 39.49114257707035} ,'128-43': {'exact': 28.466736461406683,'f1': 35.49169086295526} ,'128-44': {'exact': 36.43285428761778,'f1': 43.7871060231018} ,'128-45': {'exact': 28.914057295136576,'f1': 36.68206212127552} ,'128-46': {'exact': 38.488626629865806,'f1': 45.85942274878873} ,'64-42': {'exact': 26.487103835538214,'f1': 33.99132463295243} ,'64-43': {'exact': 24.107737698677074,'f1': 31.19299060214086} ,'64-44': {'exact': 23.43199771580851,'f1': 29.710236654206277} ,'64-45': {'exact': 24.069667840487295,'f1': 30.628461984533608} ,'64-46': {'exact': 20.61482820976492,'f1': 26.39676031924598} ,'32-42': {'exact': 12.534500808984486,'f1': 16.61039733524734} ,'32-43': {'exact': 7.566384315218426,'f1': 10.458187860293629} ,'32-44': {'exact': 10.145617207575901,'f1': 13.089699684848846} ,'32-45': {'exact': 12.02055772342248,'f1': 16.759457620001616} ,'32-46': {'exact': 11.030741410488245,'f1': 15.22456013282724} ,'16-42': {'exact': 3.178833158846483,'f1': 4.522820987250254} ,'16-43': {'exact': 3.188350623393928,'f1': 4.64414960777035} ,'16-44': {'exact': 5.520129437517845,'f1': 8.370993698625112} ,'16-45': {'exact': 1.1135433520510136,'f1': 2.2770985465834883} ,'16-46': {'exact': 3.683258779861045,'f1': 5.007969331515544}}
      mosaic_w_b100_a1 =  {'256-42':{'exact': 49.7858570476825, 'f1': 57.37079988271525} ,'256-43':{'exact': 51.16588940706196, 'f1': 60.46925574405229} ,'256-44':{'exact': 51.679832492623966, 'f1': 61.26100346922336} ,'256-45':{'exact': 45.99790615779956, 'f1': 54.874672485582984} ,'256-46':{'exact': 52.23184543637575, 'f1': 62.2724565539392} ,'128-42':{'exact': 39.00256971542781, 'f1': 47.80993776576865} ,'128-43':{'exact': 39.154849148186926, 'f1': 48.79718776436416} ,'128-44':{'exact': 44.16103550014276, 'f1': 52.948878124119126} ,'128-45':{'exact': 34.4722565908442, 'f1': 43.10585568774017} ,'128-46':{'exact': 41.5627676786904, 'f1': 50.64313709695853} ,'64-42':{'exact': 29.684971923479583, 'f1': 38.14466943498521} ,'64-43':{'exact': 26.696488055581995, 'f1': 34.82179166469516} ,'64-44':{'exact': 30.06567050537737, 'f1': 39.13880423458638} ,'64-45':{'exact': 26.182544970019986, 'f1': 35.60921361608066} ,'64-46':{'exact': 26.115922718187875, 'f1': 32.914291124533854} ,'32-42':{'exact': 17.559722090035216, 'f1': 24.69251221996142} ,'32-43':{'exact': 15.361187779575522, 'f1': 22.6670501979822} ,'32-44':{'exact': 18.749405158465784, 'f1': 26.506625950029786} ,'32-45':{'exact': 11.459027315123251, 'f1': 14.746904023443546} ,'32-46':{'exact': 17.959455601027887, 'f1': 26.151910776159003} ,'16-42':{'exact': 2.1699819168173597, 'f1': 2.9552092046843454} ,'16-43':{'exact': 10.631007899495575, 'f1': 16.129110216938653} ,'16-44':{'exact': 6.357666317692966, 'f1': 9.879965461009986} ,'16-45':{'exact': 4.092509755401161, 'f1': 8.407950885606468} ,'16-46':{'exact': 7.3094127724374225, 'f1': 10.528210911915822}}
      mosaic_2pairs = {'16-44': {'exact': 5.844, 'f1': 10.028}, '64-42': {'exact': 27.915, 'f1': 36.496},
                       '16-45': {'exact': 2.789, 'f1': 5.664}, '128-44': {'exact': 43.323, 'f1': 52.249},
                       '16-42': {'exact': 4.606, 'f1': 6.448}, '256-45': {'exact': 51.432, 'f1': 61.101},
                       '128-45': {'exact': 32.093, 'f1': 40.903}, '64-43': {'exact': 21.024, 'f1': 29.088},
                       '64-46': {'exact': 29.457, 'f1': 38.085}, '64-45': {'exact': 25.526, 'f1': 34.967},
                       '16-43': {'exact': 10.593, 'f1': 15.511}, '16-46': {'exact': 4.035, 'f1': 5.851},
                       '128-43': {'exact': 34.729, 'f1': 45.188}, '32-45': {'exact': 17.807, 'f1': 25.613},
                       '32-44': {'exact': 9.375, 'f1': 12.329}, '256-44': {'exact': 51.413, 'f1': 61.543},
                       '32-42': {'exact': 16.341, 'f1': 22.559}, '256-42': {'exact': 52.594, 'f1': 61.348},
                       '256-46': {'exact': 51.823, 'f1': 61.318}, '32-43': {'exact': 12.972, 'f1': 21.194},
                       '256-43': {'exact': 51.756, 'f1': 61.193}, '128-42': {'exact': 39.412, 'f1': 49.808},
                       '128-46': {'exact': 44.094, 'f1': 53.837}, '64-44': {'exact': 25.726, 'f1': 34.979},
                       '32-46': {'exact': 16.199, 'f1': 22.795}}
  
      names = ['baseline', 'mosaic_w_b100_a0', 'mosaic_w_b100_a1','mosaic_2pairs']
      dicts = [baseline, mosaic_w_b100_a0, mosaic_w_b100_a1, mosaic_2pairs]
      plot_f1_em_dicts(names, dicts)
  
  def plot_mosaic_unite():
  
      # get_f1_em_dict_mosaic_unite_vs_unite_single()
  
      baseline = {'64-43': {'exact': [23.289], 'f1': [31.024]}, '16-46': {'exact': [4.064], 'f1': [5.716]}, '16-43': {'exact': [5.511], 'f1': [9.563]}, '64-45': {'exact': [22.004], 'f1': [28.248]}, '32-42': {'exact': [7.052], 'f1': [11.504]}, '256-46': {'exact': [47.464], 'f1': [57.048]}, '128-43': {'exact': [30.494], 'f1': [39.791]}, '64-46': {'exact': [25.136], 'f1': [33.040]}, '128-44': {'exact': [35.719], 'f1': [43.534]}, '32-44': {'exact': [19.378], 'f1': [27.676]}, '32-43': {'exact': [11.402], 'f1': [17.522]}, '64-42': {'exact': [21.662], 'f1': [26.912]}, '16-44': {'exact': [9.089], 'f1': [16.428]}, '16-42': {'exact': [4.683], 'f1': [6.779]}, '256-43': {'exact': [52.213], 'f1': [61.855]}, '256-45': {'exact': [40.973], 'f1': [50.231]}, '128-45': {'exact': [29.219], 'f1': [35.873]}, '256-42': {'exact': [41.515], 'f1': [50.887]}, '16-45': {'exact': [1.713], 'f1': [4.461]}, '128-46': {'exact': [39.688], 'f1': [49.168]}, '64-44': {'exact': [25.707], 'f1': [33.754]}, '256-44': {'exact': [48.568], 'f1': [58.361]}, '128-42': {'exact': [27.163], 'f1': [34.966]}}
      mosaic_unite = {'16-42': {'exact': ['6.243'], 'f1': ['8.402']}, '16-43': {'exact': ['10.821'], 'f1': ['16.966']}, '16-44': {'exact': ['8.718'], 'f1': ['14.004']}, '16-45': {'exact': ['4.131'], 'f1': ['9.880']}, '16-46': {'exact': ['5.273'], 'f1': ['7.045']}, '32-42': {'exact': ['14.685'], 'f1': ['20.597']}, '32-43': {'exact': ['12.972'], 'f1': ['19.545']}, '32-44': {'exact': ['21.233'], 'f1': ['29.625']}, '32-45': {'exact': ['17.398'], 'f1': ['24.793']}, '32-46': {'exact': ['15.837'], 'f1': ['22.776']}, '64-42': {'exact': ['27.258'], 'f1': ['34.719']}, '64-43': {'exact': ['26.354'], 'f1': ['34.528']}, '64-44': {'exact': ['30.484'], 'f1': ['39.154']}, '64-45': {'exact': ['27.287'], 'f1': ['35.167']}, '64-46': {'exact': ['24.936'], 'f1': ['30.956']}, '128-42': {'exact': ['37.565'], 'f1': ['46.523']}, '128-43': {'exact': ['38.555'], 'f1': ['48.459']}, '128-44': {'exact': ['43.837'], 'f1': ['52.290']}, '128-45': {'exact': ['32.264'], 'f1': ['39.862']}, '128-46': {'exact': ['43.676'], 'f1': ['52.595']}, '256-42': {'exact': ['47.473'], 'f1': ['55.532']}, '256-43': {'exact': ['52.613'], 'f1': ['62.630']}, '256-44': {'exact': ['49.043'], 'f1': ['58.062']}, '256-45': {'exact': ['43.295'], 'f1': ['52.298']}, '256-46': {'exact': ['51.404'], 'f1': ['60.902']}}
      mosaic_unite_npairs_4 = {'16-42': {'exact': ['9.917'], 'f1': ['13.520']}, '16-43': {'exact': ['6.948'], 'f1': ['10.071']}, '16-45': {'exact': ['6.158'], 'f1': ['11.808']}, '16-46': {'exact': ['4.435'], 'f1': ['6.100']}, '32-42': {'exact': ['23.679'], 'f1': ['32.046']}, '32-44': {'exact': ['21.262'], 'f1': ['29.573']}, '32-45': {'exact': ['19.206'], 'f1': ['27.152']}, '32-46': {'exact': ['19.435'], 'f1': ['26.089']}, '64-42': {'exact': ['28.467'], 'f1': ['35.044']}, '64-43': {'exact': ['28.609'], 'f1': ['36.021']}, '64-44': {'exact': ['27.820'], 'f1': ['35.246']}, '64-45': {'exact': ['29.742'], 'f1': ['36.741']}, '64-46': {'exact': ['29.990'], 'f1': ['35.766']}, '128-42': {'exact': ['38.222'], 'f1': ['47.068']}, '128-43': {'exact': ['39.555'], 'f1': ['47.748']}, '128-44': {'exact': ['41.658'], 'f1': ['47.879']}, '128-45': {'exact': ['37.851'], 'f1': ['45.811']}, '128-46': {'exact': ['44.256'], 'f1': ['53.318']}, '256-42': {'exact': ['50.719'], 'f1': ['58.678']}, '256-43': {'exact': ['55.087'], 'f1': ['64.735']}, '256-44': {'exact': ['54.763'], 'f1': ['65.191']}, '256-45': {'exact': ['47.064'], 'f1': ['54.730']}, '256-46': {'exact': ['49.691'], 'f1': ['56.455']}}
      mosaic_unite_npairs_8 = {'16-42': {'exact': ['10.450'], 'f1': ['13.714']}, '16-43': {'exact': ['3.131'], 'f1': ['4.329']}, '16-45': {'exact': ['3.969'], 'f1': ['7.187']}, '16-46': {'exact': ['5.996'], 'f1': ['8.475']}, '32-42': {'exact': ['20.463'], 'f1': ['29.142']}, '32-43': {'exact': ['15.285'], 'f1': ['22.637']}, '32-44': {'exact': ['14.438'], 'f1': ['19.627']}, '32-45': {'exact': ['20.853'], 'f1': ['27.427']}, '32-46': {'exact': ['19.911'], 'f1': ['27.208']}, '64-42': {'exact': ['23.422'], 'f1': ['28.740']}, '64-43': {'exact': ['28.590'], 'f1': ['36.705']}, '64-44': {'exact': ['33.149'], 'f1': ['42.364']}, '64-45': {'exact': ['29.171'], 'f1': ['35.461']}, '64-46': {'exact': ['29.923'], 'f1': ['34.602']}, '128-42': {'exact': ['43.009'], 'f1': ['53.585']}, '128-43': {'exact': ['38.165'], 'f1': ['45.141']}, '128-44': {'exact': ['48.729'], 'f1': ['57.282']}, '128-45': {'exact': ['38.060'], 'f1': ['44.394']}, '128-46': {'exact': ['43.447'], 'f1': ['50.516']}, '256-42': {'exact': ['50.823'], 'f1': ['58.583']}, '256-43': {'exact': ['53.945'], 'f1': ['62.633']}, '256-44': {'exact': ['49.500'], 'f1': ['58.228']}, '256-45': {'exact': ['46.331'], 'f1': ['53.200']}, '256-46': {'exact': ['54.002'], 'f1': ['63.557']}}
  
      names = ['baseline', 'unite2', 'unite4', 'unite8']
      dicts = [baseline, mosaic_unite, mosaic_unite_npairs_4, mosaic_unite_npairs_8]
      plot_f1_em_dicts(names, dicts)
  
  
  # ======================== RESULTS ========================
  baseline_old = {'64-43': {'exact': ['23.289'], 'f1': ['31.024']}, '16-46': {'exact': ['4.064'], 'f1': ['5.716']}, '16-43': {'exact': ['5.511'], 'f1': ['9.563']}, '64-45': {'exact': ['22.004'], 'f1': ['28.248']}, '32-42': {'exact': ['7.052'], 'f1': ['11.504']}, '256-46': {'exact': ['47.464'], 'f1': ['57.048']}, '128-43': {'exact': ['30.494'], 'f1': ['39.791']}, '64-46': {'exact': ['25.136'], 'f1': ['33.040']}, '128-44': {'exact': ['35.719'], 'f1': ['43.534']}, '32-44': {'exact': ['19.378'], 'f1': ['27.676']}, '32-43': {'exact': ['11.402'], 'f1': ['17.522']}, '64-42': {'exact': ['21.662'], 'f1': ['26.912']}, '16-44': {'exact': ['9.089'], 'f1': ['16.428']}, '16-42': {'exact': ['4.683'], 'f1': ['6.779']}, '256-43': {'exact': ['52.213'], 'f1': ['61.855']}, '256-45': {'exact': ['40.973'], 'f1': ['50.231']}, '128-45': {'exact': ['29.219'], 'f1': ['35.873']}, '256-42': {'exact': ['41.515'], 'f1': ['50.887']}, '16-45': {'exact': ['1.713'], 'f1': ['4.461']}, '128-46': {'exact': ['39.688'], 'f1': ['49.168']}, '64-44': {'exact': ['25.707'], 'f1': ['33.754']}, '256-44': {'exact': ['48.568'], 'f1': ['58.361']}, '128-42': {'exact': ['27.163'], 'f1': ['34.966']}}
  baseline = {'16-42': {'exact': ['4.683'], 'f1': ['6.779']}, '16-43': {'exact': ['5.511'], 'f1': ['9.563']}, '16-44': {'exact': ['9.089'], 'f1': ['16.428']}, '16-45': {'exact': ['1.713'], 'f1': ['4.461']}, '16-46': {'exact': ['4.064'], 'f1': ['5.716']}, '32-42': {'exact': ['7.052'], 'f1': ['11.504']}, '32-43': {'exact': ['11.402'], 'f1': ['17.522']}, '32-44': {'exact': ['19.378'], 'f1': ['27.676']}, '32-45': {'exact': ['10.469'], 'f1': ['15.059']}, '32-46': {'exact': ['13.781'], 'f1': ['21.021']}, '64-42': {'exact': ['21.662'], 'f1': ['26.912']}, '64-43': {'exact': ['23.289'], 'f1': ['31.024']}, '64-44': {'exact': ['25.707'], 'f1': ['33.754']}, '64-45': {'exact': ['22.004'], 'f1': ['28.248']}, '64-46': {'exact': ['25.136'], 'f1': ['33.040']}, '128-42': {'exact': ['27.163'], 'f1': ['34.966']}, '128-43': {'exact': ['30.494'], 'f1': ['39.791']}, '128-44': {'exact': ['35.719'], 'f1': ['43.534']}, '128-45': {'exact': ['29.219'], 'f1': ['35.873']}, '128-46': {'exact': ['39.688'], 'f1': ['49.168']}, '256-42': {'exact': ['41.515'], 'f1': ['50.887']}, '256-43': {'exact': ['52.213'], 'f1': ['61.855']}, '256-44': {'exact': ['48.568'], 'f1': ['58.361']}, '256-45': {'exact': ['40.973'], 'f1': ['50.231']}, '256-46': {'exact': ['47.464'], 'f1': ['57.048']}}
  mosaic_2_False = {'16-42': {'exact': ['4.606'], 'f1': ['6.448']}, '16-43': {'exact': ['10.593'], 'f1': ['15.511']}, '16-44': {'exact': ['5.844'], 'f1': ['10.028']}, '16-45': {'exact': ['2.789'], 'f1': ['5.664']}, '16-46': {'exact': ['4.035'], 'f1': ['5.851']}, '32-42': {'exact': ['16.341'], 'f1': ['22.559']}, '32-43': {'exact': ['12.972'], 'f1': ['21.194']}, '32-44': {'exact': ['9.375'], 'f1': ['12.329']}, '32-45': {'exact': ['17.807'], 'f1': ['25.613']}, '32-46': {'exact': ['16.199'], 'f1': ['22.795']}, '64-42': {'exact': ['27.915'], 'f1': ['36.496']}, '64-43': {'exact': ['21.024'], 'f1': ['29.088']}, '64-44': {'exact': ['25.726'], 'f1': ['34.979']}, '64-45': {'exact': ['25.526'], 'f1': ['34.967']}, '64-46': {'exact': ['29.457'], 'f1': ['38.085']}, '128-42': {'exact': ['39.412'], 'f1': ['49.808']}, '128-43': {'exact': ['34.729'], 'f1': ['45.188']}, '128-44': {'exact': ['43.323'], 'f1': ['52.249']}, '128-45': {'exact': ['32.093'], 'f1': ['40.903']}, '128-46': {'exact': ['44.094'], 'f1': ['53.837']}, '256-42': {'exact': ['52.594'], 'f1': ['61.348']}, '256-43': {'exact': ['51.756'], 'f1': ['61.193']}, '256-44': {'exact': ['51.413'], 'f1': ['61.543']}, '256-45': {'exact': ['51.432'], 'f1': ['61.101']}, '256-46': {'exact': ['51.823'], 'f1': ['61.318']}}
  mosaic_2_False_old = {'16-44': {'exact': ['5.844'], 'f1': ['10.028']}, '64-42': {'exact': ['27.915'], 'f1': ['36.496']},'16-45': {'exact': ['2.789'], 'f1': ['5.664']}, '128-44': {'exact': ['43.323'], 'f1': ['52.249']},'16-42': {'exact': ['4.606'], 'f1': ['6.448']}, '256-45': {'exact': ['51.432'], 'f1': ['61.101']},'128-45': {'exact': ['32.093'], 'f1': ['40.903']}, '64-43': {'exact': ['21.024'], 'f1': ['29.088']},'64-46': {'exact': ['29.457'], 'f1': ['38.085']}, '64-45': {'exact': ['25.526'], 'f1': ['34.967']},'16-43': {'exact': ['10.593'], 'f1': ['15.511']}, '16-46': {'exact': ['4.035'], 'f1': ['5.851']},'128-43': {'exact': ['34.729'], 'f1': ['45.188']}, '32-45': {'exact': ['17.807'], 'f1': ['25.613']},'32-44': {'exact': ['9.375'], 'f1': ['12.329']}, '256-44': {'exact': ['51.413'], 'f1': ['61.543']},'32-42': {'exact': ['16.341'], 'f1': ['22.559']}, '256-42': {'exact': ['52.594'], 'f1': ['61.348']},'256-46': {'exact': ['51.823'], 'f1': ['61.318']}, '32-43': {'exact': ['12.972'], 'f1': ['21.194']},'256-43': {'exact': ['51.756'], 'f1': ['61.193']}, '128-42': {'exact': ['39.412'], 'f1': ['49.808']},'128-46': {'exact': ['44.094'], 'f1': ['53.837']}, '64-44': {'exact': ['25.726'], 'f1': ['34.979']},'32-46': {'exact': ['16.199'], 'f1': ['22.795']}}
  lorem_ipsum = {'16-42': {'exact': ['4.216'], 'f1': ['7.610']}, '16-43': {'exact': ['4.578'], 'f1': ['8.178']}, '16-44': {'exact': ['5.844'], 'f1': ['10.727']}, '16-45': {'exact': ['3.874'], 'f1': ['8.646']}, '16-46': {'exact': ['4.397'], 'f1': ['6.731']}, '32-42': {'exact': ['8.090'], 'f1': ['12.467']}, '32-43': {'exact': ['8.366'], 'f1': ['13.197']}, '32-44': {'exact': ['9.860'], 'f1': ['15.384']}, '32-45': {'exact': ['9.508'], 'f1': ['13.985']}, '32-46': {'exact': ['9.641'], 'f1': ['14.836']}, '64-42': {'exact': ['21.509'], 'f1': ['28.019']}, '64-44': {'exact': ['16.256'], 'f1': ['22.237']}, '64-45': {'exact': ['18.036'], 'f1': ['24.494']}, '64-46': {'exact': ['19.216'], 'f1': ['26.109']}, '128-42': {'exact': ['29.951'], 'f1': ['38.315']}, '128-43': {'exact': ['36.576'], 'f1': ['45.416']}, '128-44': {'exact': ['38.612'], 'f1': ['47.097']}, '128-45': {'exact': ['28.514'], 'f1': ['35.158']}, '128-46': {'exact': ['40.268'], 'f1': ['49.166']}, '256-42': {'exact': ['50.795'], 'f1': ['59.677']}, '256-43': {'exact': ['51.699'], 'f1': ['60.965']}, '256-44': {'exact': ['50.281'], 'f1': ['58.919']}, '256-45': {'exact': ['47.349'], 'f1': ['56.562']}, '256-46': {'exact': ['49.814'], 'f1': ['58.830']}}
  lorem_ipsum_double = {'128-42': {'exact': ['29.951'], 'f1': ['38.315']}, '128-43': {'exact': ['36.576'], 'f1': ['45.416']}, '128-44': {'exact': ['38.612'], 'f1': ['47.097']}, '128-45': {'exact': ['28.514'], 'f1': ['35.158']}, '256-42': {'exact': ['50.795'], 'f1': ['59.677']}, '256-43': {'exact': ['51.699'], 'f1': ['60.965']}, '256-44': {'exact': ['50.281'], 'f1': ['58.919']}, '256-45': {'exact': ['47.349'], 'f1': ['56.562']}, '256-46': {'exact': ['49.814'], 'f1': ['58.830']}, '16-42': {'exact': ['5.035'], 'f1': ['8.250']}, '16-43': {'exact': ['4.245'], 'f1': ['7.076']}, '16-44': {'exact': ['6.453'], 'f1': ['11.067']}, '16-45': {'exact': ['3.969'], 'f1': ['8.745']}, '16-46': {'exact': ['6.881'], 'f1': ['8.893']}, '32-42': {'exact': ['10.498'], 'f1': ['15.898']}, '32-43': {'exact': ['11.411'], 'f1': ['17.548']}, '32-44': {'exact': ['12.592'], 'f1': ['19.322']}, '32-45': {'exact': ['12.991'], 'f1': ['18.040']}, '32-46': {'exact': ['13.581'], 'f1': ['20.203']}, '64-42': {'exact': ['21.509'], 'f1': ['28.019']}, '64-43': {'exact': ['22.471'], 'f1': ['30.192']}, '64-44': {'exact': ['21.919'], 'f1': ['30.313']}, '64-45': {'exact': ['19.863'], 'f1': ['26.411']}, '64-46': {'exact': ['20.758'], 'f1': ['27.602']}}
  mosaic_context_shuffle = {'16-42': {'exact': ['3.674'], 'f1': ['7.621']}, '16-43': {'exact': ['4.882'], 'f1': ['8.826']}, '16-44': {'exact': ['5.682'], 'f1': ['10.412']}, '16-45': {'exact': ['0.143'], 'f1': ['0.344']}, '16-46': {'exact': ['5.101'], 'f1': ['8.131']}, '32-42': {'exact': ['7.814'], 'f1': ['12.753']}, '32-43': {'exact': ['7.271'], 'f1': ['11.348']}, '32-44': {'exact': ['9.917'], 'f1': ['13.277']}, '32-45': {'exact': ['8.613'], 'f1': ['12.620']}, '32-46': {'exact': ['11.288'], 'f1': ['16.314']}, '64-42': {'exact': ['17.845'], 'f1': ['24.019']}, '64-43': {'exact': ['22.537'], 'f1': ['30.352']}, '64-44': {'exact': ['23.356'], 'f1': ['29.715']}, '64-46': {'exact': ['16.741'], 'f1': ['22.481']}, '128-42': {'exact': ['24.764'], 'f1': ['32.411']}, '128-43': {'exact': ['29.885'], 'f1': ['38.282']}, '128-44': {'exact': ['35.633'], 'f1': ['43.382']}, '128-45': {'exact': ['25.326'], 'f1': ['31.737']}, '128-46': {'exact': ['36.680'], 'f1': ['43.972']}, '256-42': {'exact': ['41.734'], 'f1': ['50.528']}, '256-43': {'exact': ['50.719'], 'f1': ['60.414']}, '256-44': {'exact': ['46.436'], 'f1': ['56.072']}, '256-45': {'exact': ['34.263'], 'f1': ['42.644']}}
  concat_coherent_context = {'16-42': {'exact': ['6.824'], 'f1': ['9.516']}, '16-43': {'exact': ['5.339'], 'f1': ['8.015']}, '16-44': {'exact': ['4.102'], 'f1': ['6.860']}, '16-45': {'exact': ['2.998'], 'f1': ['6.846']}, '16-46': {'exact': ['4.016'], 'f1': ['5.198']}, '32-42': {'exact': ['7.110'], 'f1': ['10.870']}, '32-43': {'exact': ['12.535'], 'f1': ['19.784']}, '32-44': {'exact': ['10.688'], 'f1': ['15.802']}, '32-45': {'exact': ['11.373'], 'f1': ['17.982']}, '32-46': {'exact': ['8.432'], 'f1': ['11.071']}, '64-42': {'exact': ['23.508'], 'f1': ['31.057']}, '64-43': {'exact': ['22.385'], 'f1': ['30.163']}, '64-44': {'exact': ['21.738'], 'f1': ['30.133']}, '64-45': {'exact': ['22.947'], 'f1': ['30.457']}, '64-46': {'exact': ['23.698'], 'f1': ['32.458']}, '128-42': {'exact': ['39.517'], 'f1': ['49.815']}, '128-43': {'exact': ['35.795'], 'f1': ['45.079']}, '128-44': {'exact': ['43.923'], 'f1': ['52.914']}, '128-45': {'exact': ['36.005'], 'f1': ['44.411']}, '128-46': {'exact': ['35.814'], 'f1': ['43.218']}, '256-42': {'exact': ['53.231'], 'f1': ['61.717']}, '256-43': {'exact': ['52.860'], 'f1': ['61.835']}, '256-44': {'exact': ['52.241'], 'f1': ['62.174']}, '256-45': {'exact': ['50.585'], 'f1': ['59.245']}, '256-46': {'exact': ['49.681'], 'f1': ['59.012']}}
  
  # ======================== BASELINES ========================
  bioasq_mosaic_2_True = {'16-42': {'exact': ['12.500'], 'f1': ['16.186']}, '16-43': {'exact': ['5.000'], 'f1': ['7.878']}, '16-44': {'exact': ['8.500'], 'f1': ['13.105']}, '16-45': {'exact': ['12.500'], 'f1': ['17.224']}, '16-46': {'exact': ['9.500'], 'f1': ['11.075']}, '32-42': {'exact': ['17.000'], 'f1': ['24.348']}, '32-43': {'exact': ['20.750'], 'f1': ['27.920']}, '32-44': {'exact': ['19.750'], 'f1': ['27.332']}, '32-45': {'exact': ['22.250'], 'f1': ['29.845']}, '32-46': {'exact': ['16.500'], 'f1': ['23.501']}, '64-42': {'exact': ['36.250'], 'f1': ['42.877']}, '64-43': {'exact': ['27.250'], 'f1': ['38.972']}, '64-44': {'exact': ['34.250'], 'f1': ['48.702']}, '64-45': {'exact': ['33.250'], 'f1': ['37.906']}, '64-46': {'exact': ['31.250'], 'f1': ['40.694']}, '128-42': {'exact': ['50.500'], 'f1': ['58.831']}, '128-43': {'exact': ['43.000'], 'f1': ['54.251']}, '128-44': {'exact': ['44.250'], 'f1': ['56.272']}, '128-45': {'exact': ['48.500'], 'f1': ['56.978']}, '128-46': {'exact': ['44.250'], 'f1': ['54.670']}, '256-42': {'exact': ['60.750'], 'f1': ['71.549']}, '256-43': {'exact': ['60.000'], 'f1': ['68.469']}, '256-44': {'exact': ['61.750'], 'f1': ['70.714']}, '256-45': {'exact': ['61.750'], 'f1': ['70.150']}, '256-46': {'exact': ['58.750'], 'f1': ['70.954']}}
  bioasq_mosaic_2_False = {'16-42': {'exact': ['12.500'], 'f1': ['16.186']}, '16-43': {'exact': ['5.000'], 'f1': ['7.878']}, '16-44': {'exact': ['8.500'], 'f1': ['13.105']}, '16-45': {'exact': ['12.500'], 'f1': ['17.224']}, '16-46': {'exact': ['9.500'], 'f1': ['11.075']}, '32-42': {'exact': ['17.000'], 'f1': ['24.348']}, '32-43': {'exact': ['20.750'], 'f1': ['27.920']}, '32-44': {'exact': ['19.750'], 'f1': ['27.332']}, '32-45': {'exact': ['22.250'], 'f1': ['29.845']}, '32-46': {'exact': ['16.500'], 'f1': ['23.501']}, '64-42': {'exact': ['36.250'], 'f1': ['42.877']}, '64-43': {'exact': ['27.250'], 'f1': ['38.972']}, '64-44': {'exact': ['34.250'], 'f1': ['48.702']}, '64-45': {'exact': ['33.250'], 'f1': ['37.906']}, '64-46': {'exact': ['31.250'], 'f1': ['40.694']}, '128-42': {'exact': ['50.500'], 'f1': ['58.831']}, '128-43': {'exact': ['43.000'], 'f1': ['54.251']}, '128-44': {'exact': ['44.250'], 'f1': ['56.272']}, '128-45': {'exact': ['48.500'], 'f1': ['56.978']}, '128-46': {'exact': ['44.250'], 'f1': ['54.670']}, '256-42': {'exact': ['60.750'], 'f1': ['71.549']}, '256-43': {'exact': ['60.000'], 'f1': ['68.469']}, '256-44': {'exact': ['61.750'], 'f1': ['70.714']}, '256-45': {'exact': ['61.750'], 'f1': ['70.150']}, '256-46': {'exact': ['58.750'], 'f1': ['70.954']}}
  bioasq_lorem_ipsum_double = {'16-42': {'exact': ['12.000'], 'f1': ['15.780']}, '16-43': {'exact': ['8.500'], 'f1': ['12.375']}, '16-44': {'exact': ['13.750'], 'f1': ['19.841']}, '16-45': {'exact': ['12.750'], 'f1': ['15.539']}, '16-46': {'exact': ['16.500'], 'f1': ['20.411']}, '32-42': {'exact': ['15.250'], 'f1': ['22.050']}, '32-43': {'exact': ['14.000'], 'f1': ['16.483']}, '32-44': {'exact': ['25.250'], 'f1': ['34.005']}, '32-45': {'exact': ['20.500'], 'f1': ['25.901']}, '32-46': {'exact': ['16.500'], 'f1': ['20.846']}, '64-42': {'exact': ['30.250'], 'f1': ['37.294']}, '64-43': {'exact': ['23.000'], 'f1': ['31.781']}, '64-44': {'exact': ['29.500'], 'f1': ['39.861']}, '64-45': {'exact': ['31.250'], 'f1': ['36.090']}, '64-46': {'exact': ['27.000'], 'f1': ['33.820']}, '128-42': {'exact': ['46.750'], 'f1': ['55.308']}, '128-43': {'exact': ['40.750'], 'f1': ['49.806']}, '128-44': {'exact': ['40.250'], 'f1': ['50.338']}, '128-45': {'exact': ['43.500'], 'f1': ['51.047']}, '128-46': {'exact': ['37.250'], 'f1': ['45.854']}, '256-42': {'exact': ['54.250'], 'f1': ['65.183']}, '256-43': {'exact': ['55.750'], 'f1': ['65.699']}, '256-44': {'exact': ['62.000'], 'f1': ['70.601']}, '256-45': {'exact': ['56.250'], 'f1': ['62.549']}, '256-46': {'exact': ['53.750'], 'f1': ['62.320']}}
  bioasq_coherent_concat = {'16-42': {'exact': ['13.750'], 'f1': ['18.431']}, '16-43': {'exact': ['6.000'], 'f1': ['8.576']}, '16-44': {'exact': ['11.250'], 'f1': ['17.893']}, '16-45': {'exact': ['9.750'], 'f1': ['12.981']}, '16-46': {'exact': ['12.500'], 'f1': ['14.842']}, '32-42': {'exact': ['19.250'], 'f1': ['26.746']}, '32-43': {'exact': ['16.000'], 'f1': ['20.506']}, '32-44': {'exact': ['24.250'], 'f1': ['31.830']}, '32-45': {'exact': ['22.500'], 'f1': ['28.660']}, '32-46': {'exact': ['19.750'], 'f1': ['24.154']}, '64-42': {'exact': ['31.750'], 'f1': ['38.524']}, '64-43': {'exact': ['26.000'], 'f1': ['35.506']}, '64-44': {'exact': ['25.750'], 'f1': ['35.649']}, '64-45': {'exact': ['30.250'], 'f1': ['35.401']}, '64-46': {'exact': ['29.500'], 'f1': ['37.358']}, '128-42': {'exact': ['47.250'], 'f1': ['57.008']}, '128-43': {'exact': ['36.500'], 'f1': ['45.467']}, '128-44': {'exact': ['37.250'], 'f1': ['49.124']}, '128-45': {'exact': ['46.250'], 'f1': ['54.675']}, '128-46': {'exact': ['34.250'], 'f1': ['41.894']}, '256-42': {'exact': ['56.250'], 'f1': ['66.872']}, '256-43': {'exact': ['55.250'], 'f1': ['64.965']}, '256-44': {'exact': ['53.750'], 'f1': ['63.163']}, '256-45': {'exact': ['57.500'], 'f1': ['66.665']}, '256-46': {'exact': ['54.500'], 'f1': ['64.752']}}
  bioasq_baseline = {'16-42': {'exact': ['14.250'], 'f1': ['18.524']}, '16-43': {'exact': ['4.250'], 'f1': ['5.714']}, '16-44': {'exact': ['14.250'], 'f1': ['21.901']}, '16-45': {'exact': ['12.750'], 'f1': ['16.063']}, '16-46': {'exact': ['10.250'], 'f1': ['13.633']}, '32-42': {'exact': ['16.500'], 'f1': ['22.704']}, '32-43': {'exact': ['15.250'], 'f1': ['20.370']}, '32-44': {'exact': ['22.250'], 'f1': ['29.864']}, '32-45': {'exact': ['21.000'], 'f1': ['26.880']}, '32-46': {'exact': ['19.000'], 'f1': ['23.131']}, '64-42': {'exact': ['28.000'], 'f1': ['34.368']}, '64-43': {'exact': ['23.000'], 'f1': ['31.523']}, '64-44': {'exact': ['24.000'], 'f1': ['32.833']}, '64-45': {'exact': ['27.250'], 'f1': ['32.975']}, '64-46': {'exact': ['24.500'], 'f1': ['33.646']}, '128-42': {'exact': ['39.000'], 'f1': ['48.625']}, '128-43': {'exact': ['35.750'], 'f1': ['45.624']}, '128-44': {'exact': ['37.500'], 'f1': ['47.848']}, '128-45': {'exact': ['40.500'], 'f1': ['48.828']}, '128-46': {'exact': ['34.500'], 'f1': ['46.018']}, '256-42': {'exact': ['50.750'], 'f1': ['62.818']}, '256-43': {'exact': ['47.500'], 'f1': ['59.934']}, '256-44': {'exact': ['51.500'], 'f1': ['61.272']}, '256-45': {'exact': ['54.000'], 'f1': ['64.310']}, '256-46': {'exact': ['47.500'], 'f1': ['57.798']}}
  
  hotpotqa_baseline = {'16-42': {'exact': ['4.813'], 'f1': ['8.022']}, '16-43': {'exact': ['8.033'], 'f1': ['12.528']}, '16-44': {'exact': ['7.067'], 'f1': ['11.932']}, '16-45': {'exact': ['4.626'], 'f1': ['8.845']}, '16-46': {'exact': ['7.321'], 'f1': ['12.865']}, '32-42': {'exact': ['5.118'], 'f1': ['8.933']}, '32-43': {'exact': ['6.677'], 'f1': ['11.064']}, '32-44': {'exact': ['6.321'], 'f1': ['10.443']}, '32-45': {'exact': ['4.779'], 'f1': ['8.610']}, '32-46': {'exact': ['7.541'], 'f1': ['12.385']}, '64-42': {'exact': ['9.405'], 'f1': ['14.589']}, '64-43': {'exact': ['13.574'], 'f1': ['21.222']}, '64-44': {'exact': ['7.795'], 'f1': ['12.287']}, '64-45': {'exact': ['9.998'], 'f1': ['15.092']}, '64-46': {'exact': ['9.134'], 'f1': ['15.084']}, '128-42': {'exact': ['12.862'], 'f1': ['18.858']}, '128-43': {'exact': ['23.471'], 'f1': ['33.958']}, '128-44': {'exact': ['25.318'], 'f1': ['36.322']}, '128-45': {'exact': ['17.675'], 'f1': ['26.213']}, '128-46': {'exact': ['18.370'], 'f1': ['27.077']}, '256-42': {'exact': ['30.351'], 'f1': ['42.549']}, '256-43': {'exact': ['31.079'], 'f1': ['43.780']}, '256-44': {'exact': ['30.419'], 'f1': ['42.640']}, '256-45': {'exact': ['31.944'], 'f1': ['44.764']}, '256-46': {'exact': ['29.487'], 'f1': ['41.536']}}
  hotpotqa_mosaic_2_True = {'128-42': {'exact': ['29.317'], 'f1': ['41.985']}, '128-43': {'exact': ['27.775'], 'f1': ['39.199']}, '128-44': {'exact': ['29.927'], 'f1': ['42.447']}, '128-45': {'exact': ['27.826'], 'f1': ['39.469']}, '256-42': {'exact': ['38.705'], 'f1': ['53.002']}, '256-43': {'exact': ['36.485'], 'f1': ['50.085']}, '256-44': {'exact': ['36.740'], 'f1': ['50.196']}, '256-45': {'exact': ['36.858'], 'f1': ['50.663']}, '256-46': {'exact': ['33.367'], 'f1': ['46.825']}}
  hotpotqa_mosaic_2_False= {'16-42': {'exact': ['3.169'], 'f1': ['5.233']}, '16-43': {'exact': ['10.863'], 'f1': ['16.921']}, '16-44': {'exact': ['7.677'], 'f1': ['12.420']}, '16-45': {'exact': ['4.355'], 'f1': ['8.565']}, '16-46': {'exact': ['8.033'], 'f1': ['14.384']}, '32-42': {'exact': ['8.609'], 'f1': ['13.316']}, '32-43': {'exact': ['15.082'], 'f1': ['21.988']}, '32-44': {'exact': ['9.100'], 'f1': ['13.425']}, '32-45': {'exact': ['11.591'], 'f1': ['19.070']}, '32-46': {'exact': ['9.541'], 'f1': ['15.843']}, '64-42': {'exact': ['18.607'], 'f1': ['27.308']}, '64-43': {'exact': ['22.572'], 'f1': ['33.006']}, '64-44': {'exact': ['15.150'], 'f1': ['22.735']}, '64-45': {'exact': ['19.607'], 'f1': ['29.459']}, '64-46': {'exact': ['16.895'], 'f1': ['26.522']}, '128-42': {'exact': ['29.317'], 'f1': ['41.985']}, '128-43': {'exact': ['27.775'], 'f1': ['39.199']}, '128-44': {'exact': ['29.927'], 'f1': ['42.447']}, '128-45': {'exact': ['27.826'], 'f1': ['39.469']}, '128-46': {'exact': ['30.588'], 'f1': ['44.081']}, '256-42': {'exact': ['38.705'], 'f1': ['53.002']}, '256-43': {'exact': ['36.485'], 'f1': ['50.085']}, '256-44': {'exact': ['36.740'], 'f1': ['50.196']}, '256-45': {'exact': ['36.858'], 'f1': ['50.663']}, '256-46': {'exact': ['33.367'], 'f1': ['46.825']}}
  hotpotqa_lorem_ipsum_double = {'32-42': {'exact': ['4.847'], 'f1': ['7.541']}, '64-42': {'exact': ['6.829'], 'f1': ['11.119']}, '64-43': {'exact': ['15.404'], 'f1': ['22.707']}, '64-44': {'exact': ['13.659'], 'f1': ['20.319']}, '64-45': {'exact': ['10.625'], 'f1': ['15.836']}, '64-46': {'exact': ['9.846'], 'f1': ['15.139']}, '128-42': {'exact': ['17.607'], 'f1': ['25.254']}, '128-43': {'exact': ['22.505'], 'f1': ['32.483']}, '128-44': {'exact': ['23.318'], 'f1': ['32.943']}, '128-45': {'exact': ['17.929'], 'f1': ['25.982']}, '128-46': {'exact': ['16.929'], 'f1': ['24.032']}, '256-42': {'exact': ['30.825'], 'f1': ['43.334']}, '256-43': {'exact': ['32.469'], 'f1': ['44.458']}, '256-44': {'exact': ['35.130'], 'f1': ['48.064']}, '256-45': {'exact': ['33.757'], 'f1': ['46.173']}, '256-46': {'exact': ['34.740'], 'f1': ['47.507']}}
  hotpotqa_coherent_concat = {'64-42': {'exact': ['15.557'], 'f1': ['23.687']}, '64-43': {'exact': ['15.794'], 'f1': ['23.494']}, '64-44': {'exact': ['12.015'], 'f1': ['17.642']}, '64-45': {'exact': ['11.676'], 'f1': ['17.591']}, '128-42': {'exact': ['21.945'], 'f1': ['32.498']}, '128-43': {'exact': ['25.182'], 'f1': ['35.156']}, '128-44': {'exact': ['25.538'], 'f1': ['36.460']}, '128-45': {'exact': ['23.454'], 'f1': ['33.184']}, '128-46': {'exact': ['23.589'], 'f1': ['33.287']}, '256-42': {'exact': ['31.927'], 'f1': ['44.094']}, '256-43': {'exact': ['33.825'], 'f1': ['46.023']}, '256-44': {'exact': ['35.774'], 'f1': ['49.886']}, '256-45': {'exact': ['35.486'], 'f1': ['48.652']}, '256-46': {'exact': ['33.215'], 'f1': ['45.662']}}
  
  naturalquestions_baseline = {'16-42': {'exact': ['15.386'], 'f1': ['22.254']}, '16-43': {'exact': ['9.777'], 'f1': ['16.167']}, '16-44': {'exact': ['8.227'], 'f1': ['14.950']}, '16-45': {'exact': ['15.293'], 'f1': ['22.244']}, '16-46': {'exact': ['13.376'], 'f1': ['19.186']}, '32-42': {'exact': ['15.223'], 'f1': ['21.601']}, '32-43': {'exact': ['16.142'], 'f1': ['22.455']}, '32-44': {'exact': ['15.394'], 'f1': ['22.674']}, '32-45': {'exact': ['17.061'], 'f1': ['23.091']}, '32-46': {'exact': ['16.586'], 'f1': ['23.250']}, '64-42': {'exact': ['17.583'], 'f1': ['24.542']}, '64-43': {'exact': ['19.734'], 'f1': ['26.022']}, '64-44': {'exact': ['16.275'], 'f1': ['23.259']}, '64-45': {'exact': ['18.853'], 'f1': ['25.649']}, '64-46': {'exact': ['17.287'], 'f1': ['23.554']}, '128-42': {'exact': ['20.505'], 'f1': ['27.444']}, '128-43': {'exact': ['21.938'], 'f1': ['28.444']}, '128-44': {'exact': ['24.338'], 'f1': ['32.105']}, '128-45': {'exact': ['25.779'], 'f1': ['34.280']}, '128-46': {'exact': ['21.611'], 'f1': ['27.992']}, '256-42': {'exact': ['30.126'], 'f1': ['38.897']}, '256-43': {'exact': ['26.356'], 'f1': ['34.068']}, '256-44': {'exact': ['29.955'], 'f1': ['38.736']}, '256-45': {'exact': ['29.596'], 'f1': ['38.481']}, '256-46': {'exact': ['25.577'], 'f1': ['32.477']}}
  
  newsqa_baseline = {'16-42': {'exact': ['0.712'], 'f1': ['1.798']}, '16-43': {'exact': ['0.332'], 'f1': ['0.705']}, '16-44': {'exact': ['1.163'], 'f1': ['3.225']}, '16-45': {'exact': ['1.187'], 'f1': ['3.021']}, '16-46': {'exact': ['0.404'], 'f1': ['1.122']}, '32-42': {'exact': ['0.926'], 'f1': ['1.897']}, '32-43': {'exact': ['0.665'], 'f1': ['1.853']}, '32-44': {'exact': ['2.066'], 'f1': ['5.578']}, '32-45': {'exact': ['3.015'], 'f1': ['6.966']}, '32-46': {'exact': ['1.045'], 'f1': ['3.567']}, '64-42': {'exact': ['1.401'], 'f1': ['3.074']}, '64-43': {'exact': ['0.309'], 'f1': ['1.592']}, '64-44': {'exact': ['1.828'], 'f1': ['5.843']}, '64-45': {'exact': ['2.588'], 'f1': ['6.492']}, '64-46': {'exact': ['1.353'], 'f1': ['3.879']}, '128-42': {'exact': ['8.642'], 'f1': ['15.563']}, '128-43': {'exact': ['14.245'], 'f1': ['24.020']}, '128-44': {'exact': ['8.903'], 'f1': ['16.222']}, '128-45': {'exact': ['11.990'], 'f1': ['21.290']}, '128-46': {'exact': ['6.030'], 'f1': ['11.488']}, '256-42': {'exact': ['19.919'], 'f1': ['30.969']}, '256-43': {'exact': ['19.539'], 'f1': ['32.598']}, '256-44': {'exact': ['19.017'], 'f1': ['30.411']}, '256-45': {'exact': ['18.447'], 'f1': ['31.378']}, '256-46': {'exact': ['22.483'], 'f1': ['34.846']}}
  newsqa_mosaic_2_True = {'16-42': {'exact': ['0.309'], 'f1': ['1.153']}, '16-43': {'exact': ['0.855'], 'f1': ['1.604']}, '16-44': {'exact': ['0.522'], 'f1': ['1.270']}, '16-45': {'exact': ['0.451'], 'f1': ['1.231']}, '16-46': {'exact': ['0.166'], 'f1': ['0.773']}, '32-42': {'exact': ['2.018'], 'f1': ['5.983']}, '32-43': {'exact': ['0.499'], 'f1': ['1.293']}, '32-44': {'exact': ['0.878'], 'f1': ['3.030']}, '32-45': {'exact': ['1.353'], 'f1': ['3.643']}, '32-46': {'exact': ['2.540'], 'f1': ['7.377']}, '64-42': {'exact': ['6.363'], 'f1': ['13.798']}, '64-43': {'exact': ['5.556'], 'f1': ['13.505']}, '64-44': {'exact': ['8.381'], 'f1': ['15.926']}, '64-45': {'exact': ['5.745'], 'f1': ['10.744']}, '64-46': {'exact': ['7.764'], 'f1': ['13.480']}, '128-42': {'exact': ['12.488'], 'f1': ['21.688']}, '128-43': {'exact': ['13.913'], 'f1': ['20.813']}, '128-44': {'exact': ['15.907'], 'f1': ['24.769']}, '128-45': {'exact': ['15.812'], 'f1': ['27.498']}, '128-46': {'exact': ['14.055'], 'f1': ['24.307']}, '256-42': {'exact': ['20.513'], 'f1': ['30.201']}, '256-43': {'exact': ['18.519'], 'f1': ['29.146']}, '256-44': {'exact': ['21.391'], 'f1': ['32.927']}, '256-45': {'exact': ['17.759'], 'f1': ['27.301']}, '256-46': {'exact': ['23.575'], 'f1': ['33.840']}}
  
  # def plot_table():
  #     # Add a table at the bottom of the axes
  #     the_table = plt.table(cellText=cell_text,
  #                           rowLabels=rows,
  #                           rowColours=colors,
  #                           colLabels=columns,
  #                           loc='bottom')
  
  ####### NER #####
  
  
  def get_average_ner_res(ner_results_path):
      ner_results_df = get_results_df(ner_results_path)
      ner_results_dict = ner_results_df.to_dict()
  
      averages_df = pd.DataFrame()
      df = pd.DataFrame(ner_results_dict)
      for exp in df['exp'].unique():
          df_exp = df[df['exp'] == exp]
          for examples in df['examples'].unique():
              df_exp_examples = df_exp[df_exp['examples'] == examples]
              df_exp_examples_mean = df_exp_examples.mean(axis=0)
              df_exp_examples_mean['exp'] = exp
              df_exp_examples_mean['examples'] = examples
              averages_df = averages_df.append(df_exp_examples_mean, ignore_index=True)
      return averages_df
  
  def get_average_across_dataests(df):
      averages_df = pd.DataFrame()
      for aug in df['aug'].unique():
          for examples in df['examples'].unique():
              aug_examples_df = df[(df['examples'] == examples) & (df['aug'] == aug)]
              df_exp_examples_mean = aug_examples_df.mean(axis=0)
              df_exp_examples_mean['aug'] = aug
              averages_df = averages_df.append(df_exp_examples_mean, ignore_index=True)
      averages_df = averages_df.round(3)
      return averages_df
  
  def print_overleaf_style(df):
      for dataset in df['dataset'].unique():
          for ex_num in df['examples'].unique():
              for aug in df['aug'].unique():
                  row = df[(df['aug']==aug) & (df['dataset'] == dataset) & (df['examples'] == ex_num)]
                  latex_line = f"\\verb|{row['dataset'].values[0]}| & {row['examples'].values[0]} & \\verb|{row['aug'].values[0]}| & {row['accuracy'].values[0]} & {row['recall'].values[0]} & {row['precision'].values[0]} & {row['f1'].values[0]}\\\\"
                  print(latex_line)
  
  def get_average_over_seeds_df(ner_results_path):
      averages_df = get_average_ner_res(ner_results_path)
      print('averages_df')
      print(averages_df)
      averages_df = averages_df.round(3) # Average to 3rd decimal
      averages_df['aug'] =  [x.split("-")[-1] for x in averages_df['exp']]
      averages_df['dataset'] = [x.split("-")[0] for x in averages_df['exp']]
  
      ### PRINT FULL AVERAGES OVER SEED TABLE
      print_overleaf_style(averages_df)
      print('FULL AVERAGES OVER SEED TABLE')
      return averages_df
  
  def get_deltas_df(averages_df):
      delta_df = pd.DataFrame() #how much better / wrost is mosaic to baseline
      for dataset in averages_df['dataset'].unique():
          for examples in averages_df['examples'].unique():
              baseline = averages_df[(averages_df['aug']=='baseline') & (averages_df['dataset'] == dataset) & (averages_df['examples'] == examples)]
              if baseline.empty:
                  print(f'Skipping - empty baseline for dataset {dataset}, examples {examples}')
                  import pdb; pdb.set_trace()
                  continue
  
              for aug in set(averages_df['aug'].unique()) - set(['baseline']):
                  print(aug)
                  aug_df = averages_df[(averages_df['aug'] == aug) & (averages_df['dataset'] == dataset) & (averages_df['examples'] == examples)]
                  if aug_df.empty:
                      print(f'Skipping - empty aug_df for dataset {dataset}, examples {examples}, aug {aug}')
                      import pdb; pdb.set_trace()
                      continue
  
                  delta_dict = {"dataset":dataset, 'examples': examples, 'aug': aug,
                                'accuracy': float(aug_df['accuracy']) - float(baseline['accuracy']),
                                'recall': float(aug_df['recall']) - float(baseline['recall']),
                                'precision': float(aug_df['precision']) - float(baseline['precision']),
                                'f1': float(aug_df['f1']) - float(baseline['f1'])}
                  delta_df = delta_df.append(delta_dict, ignore_index=True)
  
      print(delta_df)
      return delta_df
  
  
  if __name__ == '__main__':
      ner_results_path = 'results_ner'
      ner_results_path='results'
      averages_df = get_average_over_seeds_df(ner_results_path)
      delta_df = get_deltas_df(averages_df)
      average_across_dataests_df = get_average_across_dataests(delta_df)
      for ex_num in average_across_dataests_df['examples'].unique():
          for aug in average_across_dataests_df['aug'].unique():
              row = average_across_dataests_df[(average_across_dataests_df['aug']==aug) & (average_across_dataests_df['examples'] == ex_num)]
              latex_line = f"{row['examples'].values[0]} & \\verb|{row['aug'].values[0]}| & {row['accuracy'].values[0]} & {row['recall'].values[0]} & {row['precision'].values[0]} & {row['f1'].values[0]}\\\\"
              print(latex_line)
  
      import pdb; pdb.set_trace()
      print('Average Over All datasets, deltas from baseline, Model:bert-base-uncased, Averaged over 5 seeds (42-46)')
  
      base = np.array([0, 7 ,1 ,8, 2, 9, 3, 10, 4, 11, 5, 12, 6, 13])
      df_range = []
      df_range.append(list(base))
      df_range.append(list(base+14))
      df_range.append(list(base+28))
      df_range.append(list(base+42))
      df_range.append(list(base + 56))
      for i in [item for sublist in df_range for item in sublist]:
          row = averages_df.iloc[i]
          latex_line = f"\\verb|{row['dataset']}| & roberta-base & \\verb|{row['aug']}| & {row['examples']} & {row['accuracy']} & {row['recall']} & {row['precision']} & {row['f1']}\\\\"
          print(latex_line)
      import pdb; pdb.set_trace()
  
      # Average of improve
  
      names = ['baseline', 'baseline_old','mosaic_2_False', 'mosaic_2_False_old', 'lorem_ipsum']
      dicts = [baseline, baseline_old, mosaic_2_False, mosaic_2_False_old, lorem_ipsum_double]
  
      #SQUAD
      names = ['baseline', 'lorem_ipsum', 'lorem_ipsum_double', 'concat_coherent_context', 'mosaic_2_same-aug']
      dicts = [baseline, lorem_ipsum, lorem_ipsum_double, concat_coherent_context, mosaic_2_False]
  
      # # BIOASQ
      names = ['bioasq_baseline', 'lorem_ipsum_double', 'concat_coherent_context', 'mosaic_2_same-aug', 'mosaic_2_dynamic-aug']
      dicts = [bioasq_baseline, bioasq_lorem_ipsum_double, bioasq_coherent_concat, bioasq_mosaic_2_False, bioasq_mosaic_2_True]
  
      # # HOTPOTQA
      names = ['hotpotqa_baseline', 'lorem_ipsum_double', 'concat_coherent_context', 'mosaic_2_same-aug', 'mosaic_2_dynamic-aug']
      dicts = [hotpotqa_baseline, hotpotqa_lorem_ipsum_double, hotpotqa_coherent_concat, hotpotqa_mosaic_2_False, hotpotqa_mosaic_2_True]
  
      # NATURAL QUESTION
      # names = ['naturalquestions_baseline', 'lorem_ipsum_double', 'concat_coherent_context', 'mosaic_2_same-aug', 'mosaic_2_dynamic-aug']
      # dicts = [naturalquestions_baseline, naturalquestions_lorem_ipsum_double, naturalquestions_coherent_concat, naturalquestions_mosaic_2_False, naturalquestions_mosaic_2_True]
  
      # NEWS QA
      # names = ['newsqa_baseline', 'lorem_ipsum_double', 'concat_coherent_context', 'mosaic_2_same-aug', 'mosaic_2_dynamic-aug']
      # dicts = [newsqa_baseline, newsqa_lorem_ipsum_double, newsqa_coherent_concat, newsqa_mosaic_2_False, newsqa_mosaic_2_True]
      names = ['newsqa_baseline', 'mosaic_2_dynamic-aug']
      dicts = [newsqa_baseline, newsqa_mosaic_2_True]
  
      plot_f1_em_dicts(names, dicts, show_min_max=False)
  
  