code: # -*- coding: utf-8 -*-
  """
  Created on Wed Jun 20 10:59:28 2018
  
  @author: Thomas Schatz
  
  Code for managing the analysis of a set of ABXpy minimal-pair scores 
  for ABX tasks with the following structure:
    ON phone BY speaker, previous and following phonetic context)
  
  The main function is apply_analysis, see readme.md for usage example.
  
  In the current implementation, the metadata associated with each minimal-pair
  file can be stored directly in the name of the file, following a scheme described below.
  Note that this scheme is parameterized by the 'primary-metadata' section of
  the config file.
  At minima, the filename should be sufficient to deduce all relevant metadata for carrying
  out the analyses and plots.
  The scone_phobia/metadata folder can be used to store information associating filename
  components with further metadata. It can be useful to keep filenames from getting too long.
  
  This code requires a folder where pickles containing the minimal-pair scores are stored.
  If resampling of the scores is needed (e.g. to obtain estimate of variability for
  the analysis results), this folder should also contain a 'resampling' subfolder
  where pickles containing resample of the minimal-pair scores are stored.
  
  These minimal-pair scores pickles can be obtained with precompute_mp_scores.py
  and resample of those with resample_mp_scores.py. Note that both these scripts
  will name pickles based on the name of the original ABXpy results filename,
  so it's probably a good idea to name those original results files in accordance
  with the naming scheme described below.
  
  Part of this code could probably be generalized to analysing results from 
  other ABX tasks. If we need to do that, Not sure if we should try to increase
  the scope of the current library, if we should do two independent libraries with
  some (a lot of?) redundant code or if we should have an independent abstract
  library being called by several libraries applied to particular tasks.
  """
  
  import pandas
  import os
  import os.path as path
  import scone_phobia.utils.mp_scores as mp_scores
  import yaml
  
  
  def load_cfg_from_file(f):
      # decorator that will load keyword cfg argument
      # from "../config.yml" unless it is specified explicitly
      def wrapper(*args, **kwargs):
          if not('cfg' in kwargs) or (kwargs['cfg'] is None):
              dir = path.dirname(path.realpath(__file__))
              cfg_file = path.join(dir, "..", "config.yml")
              with open(cfg_file, 'r') as ymlfile:
                  kwargs['cfg'] = yaml.load(ymlfile, Loader=yaml.Loader)['primary-metadata']
          return f(*args, **kwargs)
      return wrapper
  
  
  """
  Filename parsing utilities
  
  Parse filenames for ABX results files and derivatives based on the 'primary-metadata'
  specified in '../config.yml'.
  
  Filenames (without the extension) should be of the form:
  
      Property1valueProperty1key__Property2valueProperty2key__...___PropertyNvaluePropertyNkey.extension
  
  where the property values and keys should not contain any double underscores and
  where the property keys should correspond to the keys in the 'primary-metadata' 
  section of the '../config.yml' file. The extension can be whatever file extension
  is appropriate. For example the following would be valid filenames for the 'primary-metadata'
  section of the config file template ('../config.yml.example'):
  
      HMM-GMMmodel__WSJtrain__CSJtest__KLdis.pickle
      HMM-GMMmodel__WSJtrain__CSJtest__KLdis.txt
      MFCCmodel__Nonetrain__CSJtest__COSdis.pickle
  
  The first one, for example, would be parsed into the following list of pairs:
  
      [('model type', 'HMM-GMM'),
       ('training set', 'WSJ'),
       ('test set', 'CSJ'),
       ('dissimilarity', 'KL')]
  
  For bootstrap related files, filenames will look like:
  
      HMM-GMMmodel__WSJtrain__CSJtest__KLdis__batchsize50__batch3.pickle
  
  where the number after 'batchsize' indicates the size of the resampling
  batches and the number after 'batch' is the batch ID for this particular file.
  For these files, the resampling batch size and batch ID are also
  returned.
  """
  
  def suffix_split(token, cfg, err_message):
      """
      Helper function for parsing filenames.
      Looking for a key from cfg that would be
      a suffix of token. There should be one
      and only one.
      """
      matches = []
      for key in cfg:
          if len(token) >= len(key):
              if token[-len(key):] == key:
                  matches.append((key, token[:-len(key)]))
      assert len(matches) == 1, err_message
      return matches[0]
  
  
  @load_cfg_from_file
  def parse_res_fname(fpath, cfg=None):
      name, _ = path.splitext(path.split(fpath)[1])
      err_message = ("Results filename {} is not correctly formatted."
                     " Check your config file and "
                     "formatting instructions in analyze_mp_scores.py."
                    ).format(name)
      N = len(cfg)
      tokens = name.split('__')
      assert len(tokens) == N, err_message
      used_keys = []
      res = []
      for token in tokens:
          key, value = suffix_split(token, cfg, err_message)
          assert not key in used_keys, err_message
          used_keys.append(key)
          res.append((cfg[key], value))
      return res
  
  
  @load_cfg_from_file
  def parse_bootres_fname(fpath, cfg=None):
      name, _ = path.splitext(path.split(fpath)[1])
      err_message = ("Bootstrap results filename filename {} is not correctly"
                     " formatted. Check your config file and "
                     "formatting instructions in analyze_mp_scores.py."
                    ).format(name)
      N = len(cfg)
      tokens = name.split('__')
      assert len(tokens) == N+2, err_message
      properties = parse_res_fname('__'.join(tokens[:N]), cfg=cfg)
      batch = tokens[-1]
      assert len(batch) >= 5 and batch[:5] == 'batch', batch
      batch = int(batch[5:])
      batchsize = tokens[-2]
      assert len(batchsize) >= 9 and batchsize[:9] == 'batchsize', batchsize
      batchsize = int(batchsize[9:])
      properties.append(('batch ID', batch))
      properties.append(('batch size', batchsize))
      return properties
  
  
  
  ############################
  ## Fetch and analyse data  #
  ############################
  
  def fetch_data(analysis, mp_folder, filt=None, encoding=None,
                 add_metadata=None):
      """Use the above to get just the right data"""
      get_metadata = lambda x, parse=parse_res_fname: parse(x)
      df = mp_scores.load_mp_errors(mp_folder,
                                    get_metadata,
                                    filt=filt,
                                    encoding=encoding) # load all mp scores in a big df
      if not(add_metadata is None):
          df = add_metadata(df)
      df = analysis(df)
      return df
  
  
  def fetch_resampled_data(analysis,
                           resampling_file=None,
                           resampled_mp_folder=None,
                           filt=None,
                           encoding=None,
                           add_metadata=None,
                           verbose=0):
      # Getting resampled minimal-pair scores to estimate variability.
      # This can take time so if resampling_file is not None,
      # results are saved once they are computed
      get_metadata = lambda x, parse=parse_bootres_fname: parse(x)
      if resampling_file is None:
          boot_dfs = mp_scores.resample_analysis(analysis,
                                                 resampled_mp_folder,
                                                 get_metadata,
                                                 filt=filt,
                                                 encoding=encoding,
                                                 add_metadata=add_metadata,
                                                 verbose=verbose)
      else:
          boot_dfs = mp_scores.resample_analysis_cached(resampling_file,
                                                        analysis,
                                                        resampled_mp_folder,
                                                        get_metadata,
                                                        filt=filt,
                                                        encoding=encoding,
                                                        add_metadata=add_metadata,
                                                        verbose=verbose)
      boot_df = pandas.concat(boot_dfs)
      return boot_df
  
  
  def resampling_filts(resample_caching_scheme, mp_folder, user_filt=None):
      """
      Function used to specify various way of caching resamples of analysis
      results.
      It is the responsibility of this function to ensure that all
      caching filters are consistent with user_filt.
      See apply_analysis below.
      TODO? Could add a scheme where caching is done by type of model.
      """
      caching_filts = []
      mp_files = [path.splitext(e)[0] for e in os.listdir(mp_folder)
                  if path.splitext(e)[1] == '.pickle']
      if resample_caching_scheme == 'mp_file':
          for mp_fname in mp_files:
              # only use caching filters useful given user-provided filt
              if (user_filt is None) or user_filt(mp_fname):
                  # use second arg default value to avoid scope issues 
                  filt = lambda boot_mp_fname, mp_fname=mp_fname:\
                              mp_fname in boot_mp_fname
                  caching_filts.append((mp_fname, filt))
      elif resample_caching_scheme == 'sametestset_mp_filepairs':
          # analysis not assumed symmetric, so we loop over all
          # pairs
          for mp_fname1 in mp_files:
              # only use caching filters useful given user-provided filt
              if user_filt(mp_fname1):
                  metadata1 = dict(parse_res_fname(mp_fname1))
                  for mp_fname2 in mp_files:
                      # only use caching filters useful given user-provided filt
                      if user_filt(mp_fname2):
                          metadata2 = dict(parse_res_fname(mp_fname2))
                          if metadata1['test set'] == metadata2['test set']:
                              filt_name = mp_fname1 + '___' + mp_fname2  # hacky
                              # use args default values to avoid scope issues
                              filt = lambda bname, n1=mp_fname1, n2=mp_fname2: \
                                      n1 in bname or n2 in bname
                              caching_filts.append((filt_name, filt))
      else:
          raise ValueError(('Unsupported resample caching scheme '
                            '{}'.format(resample_caching_scheme)))
      return caching_filts
  
  
  def apply_analysis(analysis, mp_folder,
                     filt=None,
                     add_metadata=None,
                     resampling=False,
                     resample_caching_scheme=None,
                     analysis_folder=None,
                     pickle_encoding=None,
                     resampled_pickle_encoding="latin1",
                     verbose=0):
      """
      analysis: function that takes a pandas dataframe containing all
          required minimal-pair scores and returns the analysis results
          of interest (in a pandas dataframe if resampling=True).
      mp_folder: folder where the pickles containing minimal-pair scores are stored
          if resampling=True, mp_folder should also contain a 'resampling' subfolder
          where pickles containing resampled versions of the minimal-pair scores
          are stored.
      filt: string -> bool function, that takes the name of a file in mp_folder
          and returns True iff that file should be included in the analysis. If
          set to None, all available files are included.
      add_metadata: pandas.Dataframe -> pandas.Dataframe function, that takes a
          raw mp_scores Dataframe (containing only 'contrast', 'error' and primary
          metadata columns, where primary metadata is as specified in the
          config.yml file) and adds some additional metadata columns to it
      resampling: whether or not to use resampling. Currently, this is only
          supported for minimal pairs averaged on speaker first then on context.
          This returns the full bootstrapped data and also adds resampling-based
          standard deviation estimate to the analysis results.
      resample_caching_scheme: if resampling is True, determines whether and how
          to cache resampled analysis results. Caching results on disk is useful:
              - if applying the analysis on resamples takes too long (if there are
                N resamples, the duration required for the analysis will be
                multiplied by N compared to applying the analysis without
                resampling)
              - if loading resampled minimal pair scores at once for all relevant
                files in mp_folder (as determined by filt) exhausts the available
                memory
          Currently there is only three supported values for
          resample_caching_scheme:
              - None: no caching
              - 'mp_file': will create one cache file per (non-resampled)
                 minimal-pair scores file.
                ** This should only be used for analyses which can be applied
                   independently for each set of minimal pair scores obtained
                   in the same ABX task with the same features
                   and dissimilarity function **
              - 'sametestset_mp_filepairs': will create one cache file
                  per (ordered) pair of (non-resampled) minimal-pair scores
                  files sharing the same test set.
                ** This should only be used for analyses comparing patterns of
                  discriminability in the same ABX task for pairs of 
                  (features/dissimilarity function couples). **
      analysis_folder: currently only used if resampling=True and
          resample_caching_scheme is not None, to specify where to store cached
          analysis resamples.
      pickle_encoding and resampled_pickle_encoding: useful to ensure pickles
          containing minimal pair scores, resp. resampled versions of those, will be
          read correctly, for example if they have been computed under a different 
          python environment than the current one.
      """
      if filt is None:
          filt = lambda mp_fname: True 
      df = fetch_data(analysis, mp_folder, filt=filt, encoding=pickle_encoding,
                      add_metadata=add_metadata)
      if resampling:
          boot_dfs = []
          resampled_mp_folder = path.join(mp_folder, 'resampling')
          if resample_caching_scheme is None:
              resampling_file = None
              boot_dfs.append(
                  fetch_resampled_data(analysis, resampling_file,
                                       resampled_mp_folder,
                                       filt=filt,
                                       encoding=resampled_pickle_encoding,
                                       add_metadata=add_metadata,
                                       verbose=verbose))
          else:
              caching_filts = resampling_filts(resample_caching_scheme,
                                               mp_folder,
                                               user_filt=filt)
              assert not(analysis_folder is None)
              for filt_name, caching_filt in caching_filts:
                  # keep and_filt in case we add other resampling caching schemes
                  # where the caching filts are defined more coarsely than some
                  # possible user-provided filters.
                  and_filt = lambda mp_fname, f1=filt, f2=caching_filt:\
                                  f1(mp_fname) and f2(mp_fname)
                  resampling_file = path.join(analysis_folder,
                                              '{}.pickle'.format(filt_name))
                  boot_dfs.append(
                      fetch_resampled_data(analysis, resampling_file,
                                           resampled_mp_folder,
                                           filt=and_filt,
                                           encoding=resampled_pickle_encoding,
                                           add_metadata=add_metadata,
                                           verbose=verbose))
          boot_df = pandas.concat(boot_dfs)
          # Add resulting standard deviation estimates to main dataframe 
          df = mp_scores.estimate_std(df, boot_df)
          # TODO: permutation tests
          return df, boot_df
      else:
          return df
  
apis: pandas.concat