text: import json, os, sys, random
  import numpy as np
  import matplotlib
  import matplotlib.pyplot as plt
  from sklearn.metrics import roc_curve, auc
  from matplotlib.backends.backend_pdf import PdfPages
  
  def read_data_customized_cv(folder_path):
      #customized_cross_validation/1/classification_report.txt
      data = {}
      total_data = {}
      for root, dirs, files in os.walk(folder_path):
          for filename in files:
              if filename == 'classification_report.txt':
                  file_full_path  = os.path.join(root, filename)
  
                  print "Reading " + file_full_path + " ..."
                  with open(file_full_path, 'r') as input_file:
                      data_per_file = json.load(input_file)
                  input_file.close()
  
                  for models in data_per_file.keys():
                      if models not in data:
                          data[models]    = {}
                      
                      experiment_times = file_full_path.split('/')[1]
                      if models not in total_data:
                          total_data[models]    = {}
                      if experiment_times not in total_data[models]:
                          total_data[models][experiment_times]  = {}
  
                      for specs in data_per_file[models].keys():
                          if specs != 'support':
                              if specs not in data[models]:
                                  data[models][specs]    = []
                              else:
                                  data[models][specs].append(data_per_file[models][specs])
  
                              if specs not in total_data[models][experiment_times]:
                                  total_data[models][experiment_times][specs] = \
                                      data_per_file[models][specs]
  
      with open(folder_path + '/total_data.txt', 'w') as output_file:
          json.dump(total_data, output_file, indent=2)
      output_file.close()
  
      with open(folder_path + '/summary.txt', 'w') as output_file:
          json.dump(data, output_file, indent=2)
      output_file.close()
  
  def format_data_customized_cv():
      # format data for selecting a specific data
      # not finished.
      data = {}
      for root, dirs, files in os.walk('customized_cross_validation'):
          for filename in files:
              if filename == 'classification_report.txt':
                  file_full_path  = os.path.join(root, filename)
  
                  print "Reading " + file_full_path + " ..."
                  with open(file_full_path, 'r') as input_file:
                      data_per_file = json.load(input_file)
                  input_file.close()
                  
  
  def read_data_classic_cv():
  
      data = {}
      with open('classic_cross_validation/cv_classification_report.txt', 'r') as input_file:
          data_per_file = json.load(input_file)
      input_file.close()
  
      for models in data_per_file.keys():
          if models not in data:
              data[models]    = {}
  
          for times in data_per_file[models].keys():
              for specs in data_per_file[models][times]:
                  if specs != 'support':
                      if specs not in data[models]:
                          data[models][specs]    = []
                      else:
                          data[models][specs].append(data_per_file[models][times][specs])
  
      with open('classic_cross_validation/summary.txt', 'w') as output_file:
          json.dump(data, output_file, indent=2)
      output_file.close()
  
  def stat_cv(summary_file_path, method='classic'):
      print "Loading data " + summary_file_path + " ..." 
      with open(summary_file_path, 'r') as input_file:
          data = json.load(input_file)
      input_file.close()
      print "Loading complete ..."
  
      data_for_tex = {}
      for models in data:
          for specs in ["precision", "recall", "f1_score"]:
              print " {0:20} {1:10} mean:{2:.9f} std:{3:.9f}"\
                  .format(\
                          models,\
                          specs,\
                          np.array(data[models][specs]).mean(),\
                          np.array(data[models][specs]).std() * 3,\
                          )
  
              
              if models not in data_for_tex:
                  data_for_tex[models] = {}
              for stat_specs in ['mean', 'std', 'max', 'min', 'range']:
                  if stat_specs not in data_for_tex[models]:
                      data_for_tex[models][stat_specs] = ''
                  if stat_specs == 'mean':
                      data_for_tex[models][stat_specs] \
                          = "{0:.4g}".format(np.array(data[models][specs]).mean() * 100)
                  if stat_specs == 'std':
                      data_for_tex[models][stat_specs] \
                          = "{0:.4g}".format(np.array(data[models][specs]).std() * 100 * 3)
                  if stat_specs == 'max':
                      data_for_tex[models][stat_specs] \
                          = "{0:.4g}".format(max(np.array(data[models][specs])) * 100)
                  if stat_specs == 'min':
                      data_for_tex[models][stat_specs] \
                          = "{0:.4g}".format(min(np.array(data[models][specs])) * 100)
                  if stat_specs == 'range':
                      data_for_tex[models][stat_specs] \
                          = "{0:.4g}".format((max(np.array(data[models][specs])) \
                                      - min(np.array(data[models][specs]))) * 100)
  
      if method == 'classic':
          spec = 'classic'
      else:
          spec = 'customized'
      numbers_path = 'numbers/'
      for models in data_for_tex:
          for stat_specs in data_for_tex[models]:
              # range
              filename = numbers_path + "__" + spec + "_"\
                              + models.lower().replace(' ', '_')\
                              + "_" + stat_specs  + "__.tex"
              print filename
              writefile = open(filename, 'w')
              writefile.write(data_for_tex[models][stat_specs])
              writefile.close()
      return data_for_tex
  
  def stat_cv_all(path1, path2):
      # std_trend('customized_cross_validation/summary.txt')
      #classic_data = stat_cv('classic_cross_validation/summary.txt')
      classic_data = stat_cv(path1 + '/summary.txt')
      customized_data = stat_cv(path2 + '/summary.txt', method='customized')
      times           = {}
      times['std']    = {}
      times['range']  = {}
      numbers_path = 'numbers/'
      for models in classic_data:
          for stat_specs in ['std', 'range']:
              times[stat_specs][models] = float(customized_data[models][stat_specs]) /\
                                          float(classic_data[models][stat_specs])
  
              filename = numbers_path + "__comparison_"\
                              + models.lower().replace(' ', '_')\
                              + "_" + stat_specs  + "__.tex"
              print filename
              writefile = open(filename, 'w')
              writefile.write("{0:.4g}".format(times[stat_specs][models]))
              writefile.close()
  
  def std_trend(summary_file_path):
      print "Loading data " + summary_file_path + " ..." 
      with open(summary_file_path, 'r') as input_file:
          data = json.load(input_file)
      input_file.close()
      print "Loading complete ..."
  
      for models in data:
          print "Size of data: " + str(len(data[models]['f1_score']))
          for sample_size in [10, 100, 1000, len(data[models]['f1_score'])]:
              print " {0:20} {1:10d} mean:{2:.9f} std:{3:.9f}"\
                  .format(\
                          models,\
                          sample_size,\
                          np.array(random.sample(data[models]['f1_score'],
                          sample_size)).mean() * 100,\
                          np.array(random.sample(data[models]['f1_score'],
                          sample_size)).std() * 3 * 100,\
                          )
          
      
  
  def plot_cv(summary_file_path, figure_save_path):
      print "Loading data ..."
      with open(summary_file_path, 'r') as input_file:
          data = json.load(input_file)
      input_file.close()
      print "Loading complete ..."
      # analyze size of the data
      num_models      = len(data.keys())
          
      fig, ax= plt.subplots(nrows = 1, ncols=num_models,\
                  figsize=(num_models * 2.5, 5), sharey=True)
  
      boxprops        = dict(linewidth=3, color='blue')
      flierprops      = dict(marker='o', markerfacecolor='blue', markersize=8,\
                          markeredgecolor='k',\
                          linestyle='none', linewidth=2)
      capprops        = dict(color='k', linewidth=2, linestyle='-')
      whiskerprops    = dict(color='k', linewidth=2, linestyle='-.')
      meanpointprops  = dict(marker='D', markeredgecolor='black',
                            markerfacecolor='firebrick', markersize=8)
      sigma_3pt5      = 99.7
      
      for idx, model in enumerate(data.keys()):
          display_data    = []
          for specs in ["precision", "recall", "f1_score", "roc_auc"]:
              print "Stacking data in " + model + " " + specs
              print "Size of samples: " + str(len(data[model][specs]))
              if display_data == []:
                  display_data = np.array(data[model][specs])
              else:
                  display_data = np.vstack((\
                                      display_data,\
                                      np.array(data[model][specs])\
                                      ))
          print "Plotting data in " + model + "..."
          ax[idx].boxplot(np.transpose(display_data),\
                      boxprops=boxprops,\
                      meanprops=meanpointprops,\
                      capprops=capprops,\
                      whiskerprops=whiskerprops,\
                      #medianprops=medianprops, \
                      showmeans=True, meanline=False,\
                      flierprops=flierprops,\
                      whis=[100 - sigma_3pt5, sigma_3pt5])
          ax[idx].set_title(model.replace(' ', '\n'))
          ax[0].set_ylabel("Percentage[%]")
  
          ax[idx].set_yticklabels(\
                              ["0", "20", "40", "60", "80", "100"]\
                              )
          ax[idx].set_xticklabels(\
                              ["Prec", "Rec", "F1", "AUC"],\
                              rotation=45)
  
      #plt.show()
      savefig_name = PdfPages(figure_save_path)
      plt.savefig(savefig_name, format='pdf', bbox_inches='tight')
      savefig_name.close()
      plt.close()
  
  if __name__ == "__main__":
  
      matplotlib.rcParams['pdf.use14corefonts'] = True
      font = {\
              'family' : 'serif',\
              'size'   : 20}
      plt.rc('font', **font)
  
      # read_data_classic_cv()
      # plot_cv('classic_cross_validation/summary.txt',\
      #         'figures/classic_cv_boxplot.pdf')
      # read_data_customized_cv('learn_traces')
      # plot_cv('learn_traces/summary.txt',\
      #         'figures/classic_cv_boxplot.pdf')
      # read_data_customized_cv('customized_cross_validation')
      plot_cv('customized_cross_validation_old_data/summary.txt',\
              'figures/customized_cv_boxplot.pdf')
      # stat_cv_all('learn_traces', 'customized_cross_validation')
  
  