content: #!/usr/bin/env python
  # -*- coding: utf-8 -*-
  #
  # DCASE 2016::Sound Event Detection in Real-life Audio / Baseline System
  
  import argparse
  import csv
  import math
  import numpy
  import textwrap
  import warnings
  
  from sklearn import mixture
  
  from src.dataset import *
  from src.evaluation import *
  from src.features import *
  from src.sound_event_detection import *
  
  __version_info__ = ('1', '0', '1')
  __version__ = '.'.join(__version_info__)
  
  
  def main(argv):
      numpy.random.seed(123456)  # let's make randomization predictable
  
      parser = argparse.ArgumentParser(
          prefix_chars='-+',
          formatter_class=argparse.RawDescriptionHelpFormatter,
          description=textwrap.dedent('''\
              DCASE 2016
              Task 3: Sound Event Detection in Real-life Audio 
              Baseline System
              ---------------------------------------------
                  Tampere University of Technology / Audio Research Group
                  Author:  Toni Heittola ( toni.heittola@tut.fi )
  
              System description
                  This is an baseline implementation for the D-CASE 2016, task 3 - Sound event detection in real life audio.
                  The system has binary classifier for each included sound event class. The GMM classifier is trained with
                  the positive and negative examples from the mixture signals, and classification is done between these
                  two models as likelihood ratio. Acoustic features are MFCC+Delta+Acceleration (MFCC0 omitted).
  
          '''))
  
      parser.add_argument("-development", help="Use the system in the development mode", action='store_true',
                          default=False, dest='development')
      parser.add_argument("-challenge", help="Use the system in the challenge mode", action='store_true',
                          default=False, dest='challenge')
  
      parser.add_argument('-v', '--version', action='version', version='%(prog)s ' + __version__)
      args = parser.parse_args()
  
      # Load parameters from config file
      parameter_file = os.path.join(os.path.dirname(os.path.realpath(__file__)),
                                    os.path.splitext(os.path.basename(__file__))[0] + '.yaml')
      params = load_parameters(parameter_file)
      params = process_parameters(params)
      make_folders(params)
  
      title("DCASE 2016::Sound Event Detection in Real-life Audio / Baseline System")
  
      # Check if mode is defined
      if not (args.development or args.challenge):
          args.development = True
          args.challenge = False
  
      dataset_evaluation_mode = 'folds'
      if args.development and not args.challenge:
          print "Running system in development mode"
          dataset_evaluation_mode = 'folds'
      elif not args.development and args.challenge:
          print "Running system in challenge mode"
          dataset_evaluation_mode = 'full'
  
      # Get dataset container class
      dataset = eval(params['general']['development_dataset'])(data_path=params['path']['data'])
  
      # Fetch data over internet and setup the data
      # ==================================================
      if params['flow']['initialize']:
          dataset.fetch()
  
      # Extract features for all audio files in the dataset
      # ==================================================
      if params['flow']['extract_features']:
          section_header('Feature extraction [Development data]')
  
          # Collect files from evaluation sets
          files = []
          for fold in dataset.folds(mode=dataset_evaluation_mode):
              for item_id, item in enumerate(dataset.train(fold)):
                  if item['file'] not in files:
                      files.append(item['file'])
              for item_id, item in enumerate(dataset.test(fold)):
                  if item['file'] not in files:
                      files.append(item['file'])
  
          # Go through files and make sure all features are extracted
          do_feature_extraction(files=files,
                                dataset=dataset,
                                feature_path=params['path']['features'],
                                params=params['features'],
                                overwrite=params['general']['overwrite'])
  
          foot()
  
      # Prepare feature normalizers
      # ==================================================
      if params['flow']['feature_normalizer']:
          section_header('Feature normalizer [Development data]')
  
          do_feature_normalization(dataset=dataset,
                                   feature_normalizer_path=params['path']['feature_normalizers'],
                                   feature_path=params['path']['features'],
                                   dataset_evaluation_mode=dataset_evaluation_mode,
                                   overwrite=params['general']['overwrite'])
  
          foot()
  
      # System training
      # ==================================================
      if params['flow']['train_system']:
          section_header('System training    [Development data]')
  
          do_system_training(dataset=dataset,
                             model_path=params['path']['models'],
                             feature_normalizer_path=params['path']['feature_normalizers'],
                             feature_path=params['path']['features'],
                             hop_length_seconds=params['features']['hop_length_seconds'],
                             classifier_params=params['classifier']['parameters'],
                             dataset_evaluation_mode=dataset_evaluation_mode,
                             classifier_method=params['classifier']['method'],
                             overwrite=params['general']['overwrite']
                             )
  
          foot()
  
      # System evaluation in development mode
      if args.development and not args.challenge:
  
          # System testing
          # ==================================================
          if params['flow']['test_system']:
              section_header('System testing     [Development data]')
  
              do_system_testing(dataset=dataset,
                                result_path=params['path']['results'],
                                feature_path=params['path']['features'],
                                model_path=params['path']['models'],
                                feature_params=params['features'],
                                detector_params=params['detector'],
                                dataset_evaluation_mode=dataset_evaluation_mode,
                                classifier_method=params['classifier']['method'],
                                overwrite=params['general']['overwrite']
                                )
              foot()
  
          # System evaluation
          # ==================================================
          if params['flow']['evaluate_system']:
              section_header('System evaluation  [Development data]')
  
              do_system_evaluation(dataset=dataset,
                                   dataset_evaluation_mode=dataset_evaluation_mode,
                                   result_path=params['path']['results'])
  
              foot()
  
      # System evaluation with challenge data
      elif not args.development and args.challenge:
          # Fetch data over internet and setup the data
          challenge_dataset = eval(params['general']['challenge_dataset'])()
  
          if params['flow']['initialize']:
              challenge_dataset.fetch()
  
          # System testing
          if params['flow']['test_system']:
              section_header('System testing     [Challenge data]')
  
              do_system_testing(dataset=challenge_dataset,
                                result_path=params['path']['challenge_results'],
                                feature_path=params['path']['features'],
                                model_path=params['path']['models'],
                                feature_params=params['features'],
                                detector_params=params['detector'],
                                dataset_evaluation_mode=dataset_evaluation_mode,
                                classifier_method=params['classifier']['method'],
                                overwrite=True
                                )
              foot()
  
              print " "
              print "Your results for the challenge data are stored at [" + params['path']['challenge_results'] + "]"
              print " "
  
  
  def process_parameters(params):
      """Parameter post-processing.
  
      Parameters
      ----------
      params : dict
          parameters in dict
  
      Returns
      -------
      params : dict
          processed parameters
  
      """
  
      params['features']['mfcc']['win_length'] = int(params['features']['win_length_seconds'] * params['features']['fs'])
      params['features']['mfcc']['hop_length'] = int(params['features']['hop_length_seconds'] * params['features']['fs'])
  
      # Copy parameters for current classifier method
      params['classifier']['parameters'] = params['classifier_parameters'][params['classifier']['method']]
  
      # Hash
      params['features']['hash'] = get_parameter_hash(params['features'])
      params['classifier']['hash'] = get_parameter_hash(params['classifier'])
      params['detector']['hash'] = get_parameter_hash(params['detector'])
  
      # Paths
      params['path']['data'] = os.path.join(os.path.dirname(os.path.realpath(__file__)), params['path']['data'])
      params['path']['base'] = os.path.join(os.path.dirname(os.path.realpath(__file__)), params['path']['base'])
  
      # Features
      params['path']['features_'] = params['path']['features']
      params['path']['features'] = os.path.join(params['path']['base'],
                                                params['path']['features'],
                                                params['features']['hash'])
  
      # Feature normalizers
      params['path']['feature_normalizers_'] = params['path']['feature_normalizers']
      params['path']['feature_normalizers'] = os.path.join(params['path']['base'],
                                                           params['path']['feature_normalizers'],
                                                           params['features']['hash'])
  
      # Models
      # Save parameters into folders to help manual browsing of files.
      params['path']['models_'] = params['path']['models']
      params['path']['models'] = os.path.join(params['path']['base'],
                                              params['path']['models'],
                                              params['features']['hash'],
                                              params['classifier']['hash'])
  
      # Results
      params['path']['results_'] = params['path']['results']
      params['path']['results'] = os.path.join(params['path']['base'],
                                               params['path']['results'],
                                               params['features']['hash'],
                                               params['classifier']['hash'],
                                               params['detector']['hash'])
      return params
  
  
  def make_folders(params, parameter_filename='parameters.yaml'):
      """Create all needed folders, and saves parameters in yaml-file for easier manual browsing of data.
  
      Parameters
      ----------
      params : dict
          parameters in dict
  
      parameter_filename : str
          filename to save parameters used to generate the folder name
  
      Returns
      -------
      nothing
  
      """
  
      # Check that target path exists, create if not
      check_path(params['path']['features'])
      check_path(params['path']['feature_normalizers'])
      check_path(params['path']['models'])
      check_path(params['path']['results'])
  
      # Save parameters into folders to help manual browsing of files.
  
      # Features
      feature_parameter_filename = os.path.join(params['path']['features'], parameter_filename)
      if not os.path.isfile(feature_parameter_filename):
          save_parameters(feature_parameter_filename, params['features'])
  
      # Feature normalizers
      feature_normalizer_parameter_filename = os.path.join(params['path']['feature_normalizers'], parameter_filename)
      if not os.path.isfile(feature_normalizer_parameter_filename):
          save_parameters(feature_normalizer_parameter_filename, params['features'])
  
      # Models
      model_features_parameter_filename = os.path.join(params['path']['base'],
                                                       params['path']['models_'],
                                                       params['features']['hash'],
                                                       parameter_filename)
      if not os.path.isfile(model_features_parameter_filename):
          save_parameters(model_features_parameter_filename, params['features'])
  
      model_models_parameter_filename = os.path.join(params['path']['base'],
                                                     params['path']['models_'],
                                                     params['features']['hash'],
                                                     params['classifier']['hash'],
                                                     parameter_filename)
      if not os.path.isfile(model_models_parameter_filename):
          save_parameters(model_models_parameter_filename, params['classifier'])
  
      # Results
      # Save parameters into folders to help manual browsing of files.
      result_features_parameter_filename = os.path.join(params['path']['base'],
                                                        params['path']['results_'],
                                                        params['features']['hash'],
                                                        parameter_filename)
      if not os.path.isfile(result_features_parameter_filename):
          save_parameters(result_features_parameter_filename, params['features'])
  
      result_models_parameter_filename = os.path.join(params['path']['base'],
                                                      params['path']['results_'],
                                                      params['features']['hash'],
                                                      params['classifier']['hash'],
                                                      parameter_filename)
      if not os.path.isfile(result_models_parameter_filename):
          save_parameters(result_models_parameter_filename, params['classifier'])
  
      result_detector_parameter_filename = os.path.join(params['path']['base'],
                                                        params['path']['results_'],
                                                        params['features']['hash'],
                                                        params['classifier']['hash'],
                                                        params['detector']['hash'],
                                                        parameter_filename)
      if not os.path.isfile(result_detector_parameter_filename):
          save_parameters(result_detector_parameter_filename, params['detector'])
  
  
  def get_feature_filename(audio_file, path, extension='cpickle'):
      """Get feature filename
  
      Parameters
      ----------
      audio_file : str
          audio file name from which the features are extracted
  
      path :  str
          feature path
  
      extension : str
          file extension
          (Default value='cpickle')
  
      Returns
      -------
      feature_filename : str
          full feature filename
  
      """
  
      return os.path.join(path, 'sequence_' + os.path.splitext(audio_file)[0] + '.' + extension)
  
  
  def get_feature_normalizer_filename(fold, scene_label, path, extension='cpickle'):
      """Get normalizer filename
  
      Parameters
      ----------
      fold : int >= 0
          evaluation fold number
  
      scene_label : str
          scene label
  
      path :  str
          normalizer path
  
      extension : str
          file extension
          (Default value='cpickle')
  
      Returns
      -------
      normalizer_filename : str
          full normalizer filename
  
      """
  
      return os.path.join(path, 'scale_fold' + str(fold) + '_' + str(scene_label) + '.' + extension)
  
  
  def get_model_filename(fold, scene_label, path, extension='cpickle'):
      """Get model filename
  
      Parameters
      ----------
      fold : int >= 0
          evaluation fold number
  
      scene_label : str
          scene label
  
      path :  str
          model path
  
      extension : str
          file extension
          (Default value='cpickle')
  
      Returns
      -------
      model_filename : str
          full model filename
  
      """
  
      return os.path.join(path, 'model_fold' + str(fold) + '_' + str(scene_label) + '.' + extension)
  
  
  def get_result_filename(fold, scene_label, path, extension='txt'):
      """Get result filename
  
      Parameters
      ----------
      fold : int >= 0
          evaluation fold number
  
      scene_label : str
          scene label
  
      path :  str
          result path
  
      extension : str
          file extension
          (Default value='cpickle')
  
      Returns
      -------
      result_filename : str
          full result filename
  
      """
  
      if fold == 0:
          return os.path.join(path, 'results_' + str(scene_label) + '.' + extension)
      else:
          return os.path.join(path, 'results_fold' + str(fold) + '_' + str(scene_label) + '.' + extension)
  
  
  def do_feature_extraction(files, dataset, feature_path, params, overwrite=False):
      """Feature extraction
  
      Parameters
      ----------
      files : list
          file list
  
      dataset : class
          dataset class
  
      feature_path : str
          path where the features are saved
  
      params : dict
          parameter dict
  
      overwrite : bool
          overwrite existing feature files
          (Default value=False)
  
      Returns
      -------
      nothing
  
      Raises
      -------
      IOError
          Audio file not found.
  
      """
  
      for file_id, audio_filename in enumerate(files):
          # Get feature filename
          current_feature_file = get_feature_filename(audio_file=os.path.split(audio_filename)[1], path=feature_path)
  
          progress(title_text='Extracting [sequences]',
                   percentage=(float(file_id) / len(files)),
                   note=os.path.split(audio_filename)[1])
  
          if not os.path.isfile(current_feature_file) or overwrite:
              # Load audio
              if os.path.isfile(dataset.relative_to_absolute_path(audio_filename)):
                  y, fs = load_audio(filename=dataset.relative_to_absolute_path(audio_filename), mono=True,
                                     fs=params['fs'])
              else:
                  raise IOError("Audio file not found [%s]" % audio_filename)
  
              # Extract features
              feature_data = feature_extraction(y=y,
                                                fs=fs,
                                                include_mfcc0=params['include_mfcc0'],
                                                include_delta=params['include_delta'],
                                                include_acceleration=params['include_acceleration'],
                                                mfcc_params=params['mfcc'],
                                                delta_params=params['mfcc_delta'],
                                                acceleration_params=params['mfcc_acceleration'])
              # Save
              save_data(current_feature_file, feature_data)
  
  
  def do_feature_normalization(dataset, feature_normalizer_path, feature_path, dataset_evaluation_mode='folds',
                               overwrite=False):
      """Feature normalization
  
      Calculated normalization factors for each evaluation fold based on the training material available.
  
      Parameters
      ----------
      dataset : class
          dataset class
  
      feature_normalizer_path : str
          path where the feature normalizers are saved.
  
      feature_path : str
          path where the features are saved.
  
      dataset_evaluation_mode : str ['folds', 'full']
          evaluation mode, 'full' all material available is considered to belong to one fold.
          (Default value='folds')
  
      overwrite : bool
          overwrite existing normalizers
          (Default value=False)
  
      Returns
      -------
      nothing
  
      Raises
      -------
      IOError
          Feature file not found.
  
      """
  
      for fold in dataset.folds(mode=dataset_evaluation_mode):
          for scene_id, scene_label in enumerate(dataset.scene_labels):
              current_normalizer_file = get_feature_normalizer_filename(fold=fold, scene_label=scene_label,
                                                                        path=feature_normalizer_path)
  
              if not os.path.isfile(current_normalizer_file) or overwrite:
                  # Collect sequence files from scene class
                  files = []
                  for item_id, item in enumerate(dataset.train(fold, scene_label=scene_label)):
                      if item['file'] not in files:
                          files.append(item['file'])
  
                  file_count = len(files)
  
                  # Initialize statistics
                  normalizer = FeatureNormalizer()
  
                  for file_id, audio_filename in enumerate(files):
                      progress(title_text='Collecting data',
                               fold=fold,
                               percentage=(float(file_id) / file_count),
                               note=os.path.split(audio_filename)[1])
  
                      # Load features
                      feature_filename = get_feature_filename(audio_file=os.path.split(audio_filename)[1],
                                                              path=feature_path)
                      if os.path.isfile(feature_filename):
                          feature_data = load_data(feature_filename)['stat']
                      else:
                          raise IOError("Feature file not found [%s]" % audio_filename)
  
                      # Accumulate statistics
                      normalizer.accumulate(feature_data)
  
                  # Calculate normalization factors
                  normalizer.finalize()
  
                  # Save
                  save_data(current_normalizer_file, normalizer)
  
  
  def do_system_training(dataset, model_path, feature_normalizer_path, feature_path, hop_length_seconds,
                         classifier_params,
                         dataset_evaluation_mode='folds', classifier_method='gmm', overwrite=False):
      """System training
  
      Train a model pair for each sound event class, one for activity and one for inactivity.
  
      model container format:
  
      {
          'normalizer': normalizer class
          'models' :
              {
                  'mouse click' :
                      {
                          'positive': mixture.GMM class,
                          'negative': mixture.GMM class
                      }
                  'keyboard typing' :
                      {
                          'positive': mixture.GMM class,
                          'negative': mixture.GMM class
                      }
                  ...
              }
      }
  
      Parameters
      ----------
      dataset : class
          dataset class
  
      model_path : str
          path where the models are saved.
  
      feature_normalizer_path : str
          path where the feature normalizers are saved.
  
      feature_path : str
          path where the features are saved.
  
      hop_length_seconds : float > 0
          feature frame hop length in seconds
  
      classifier_params : dict
          parameter dict
  
      dataset_evaluation_mode : str ['folds', 'full']
          evaluation mode, 'full' all material available is considered to belong to one fold.
          (Default value='folds')
  
      classifier_method : str ['gmm']
          classifier method, currently only GMM supported
          (Default value='gmm')
  
      overwrite : bool
          overwrite existing models
          (Default value=False)
  
      Returns
      -------
      nothing
  
      Raises
      -------
      ValueError
          classifier_method is unknown.
  
      IOError
          Feature normalizer not found.
          Feature file not found.
  
      """
  
      if classifier_method != 'gmm':
          raise ValueError("Unknown classifier method [" + classifier_method + "]")
  
      for fold in dataset.folds(mode=dataset_evaluation_mode):
          for scene_id, scene_label in enumerate(dataset.scene_labels):
              current_model_file = get_model_filename(fold=fold, scene_label=scene_label, path=model_path)
              if not os.path.isfile(current_model_file) or overwrite:
  
                  # Load normalizer
                  feature_normalizer_filename = get_feature_normalizer_filename(fold=fold, scene_label=scene_label,
                                                                                path=feature_normalizer_path)
                  if os.path.isfile(feature_normalizer_filename):
                      normalizer = load_data(feature_normalizer_filename)
                  else:
                      raise IOError("Feature normalizer not found [%s]" % feature_normalizer_filename)
  
                  # Initialize model container
                  model_container = {'normalizer': normalizer, 'models': {}}
  
                  # Restructure training data in to structure[files][events]
                  ann = {}
                  for item_id, item in enumerate(dataset.train(fold=fold, scene_label=scene_label)):
                      filename = os.path.split(item['file'])[1]
                      if filename not in ann:
                          ann[filename] = {}
                      if item['event_label'] not in ann[filename]:
                          ann[filename][item['event_label']] = []
                      ann[filename][item['event_label']].append((item['event_onset'], item['event_offset']))
  
                  # Collect training examples
                  data_positive = {}
                  data_negative = {}
                  file_count = len(ann)
                  for item_id, audio_filename in enumerate(ann):
                      progress(title_text='Collecting data',
                               fold=fold,
                               percentage=(float(item_id) / file_count),
                               note=scene_label + " / " + os.path.split(audio_filename)[1])
  
                      # Load features
                      feature_filename = get_feature_filename(audio_file=audio_filename, path=feature_path)
                      if os.path.isfile(feature_filename):
                          feature_data = load_data(feature_filename)['feat']
                      else:
                          raise IOError("Feature file not found [%s]" % feature_filename)
  
                      # Normalize features
                      feature_data = model_container['normalizer'].normalize(feature_data)
  
                      for event_label in ann[audio_filename]:
                          positive_mask = numpy.zeros((feature_data.shape[0]), dtype=bool)
  
                          for event in ann[audio_filename][event_label]:
                              start_frame = int(math.floor(event[0] / hop_length_seconds))
                              stop_frame = int(math.ceil(event[1] / hop_length_seconds))
  
                              if stop_frame > feature_data.shape[0]:
                                  stop_frame = feature_data.shape[0]
  
                              positive_mask[start_frame:stop_frame] = True
  
                          # Store positive examples
                          if event_label not in data_positive:
                              data_positive[event_label] = feature_data[positive_mask, :]
                          else:
                              data_positive[event_label] = numpy.vstack(
                                  (data_positive[event_label], feature_data[positive_mask, :]))
  
                          # Store negative examples
                          if event_label not in data_negative:
                              data_negative[event_label] = feature_data[~positive_mask, :]
                          else:
                              data_negative[event_label] = numpy.vstack(
                                  (data_negative[event_label], feature_data[~positive_mask, :]))
  
                  # Train models for each class
                  for event_label in data_positive:
                      progress(title_text='Train models',
                               fold=fold,
                               note=scene_label + " / " + event_label)
                      if classifier_method == 'gmm':
                          model_container['models'][event_label] = {}
                          model_container['models'][event_label]['positive'] = mixture.GMM(**classifier_params).fit(
                              data_positive[event_label])
                          model_container['models'][event_label]['negative'] = mixture.GMM(**classifier_params).fit(
                              data_negative[event_label])
                      else:
                          raise ValueError("Unknown classifier method [" + classifier_method + "]")
  
                  # Save models
                  save_data(current_model_file, model_container)
  
  
  def do_system_testing(dataset, result_path, feature_path, model_path, feature_params, detector_params,
                        dataset_evaluation_mode='folds', classifier_method='gmm', overwrite=False):
      """System testing.
  
      If extracted features are not found from disk, they are extracted but not saved.
  
      Parameters
      ----------
      dataset : class
          dataset class
  
      result_path : str
          path where the results are saved.
  
      feature_path : str
          path where the features are saved.
  
      model_path : str
          path where the models are saved.
  
      feature_params : dict
          parameter dict
  
      dataset_evaluation_mode : str ['folds', 'full']
          evaluation mode, 'full' all material available is considered to belong to one fold.
          (Default value='folds')
  
      classifier_method : str ['gmm']
          classifier method, currently only GMM supported
          (Default value='gmm')
  
      overwrite : bool
          overwrite existing models
          (Default value=False)
  
      Returns
      -------
      nothing
  
      Raises
      -------
      ValueError
          classifier_method is unknown.
  
      IOError
          Model file not found.
          Audio file not found.
  
      """
  
      if classifier_method != 'gmm':
          raise ValueError("Unknown classifier method [" + classifier_method + "]")
  
      for fold in dataset.folds(mode=dataset_evaluation_mode):
          for scene_id, scene_label in enumerate(dataset.scene_labels):
              current_result_file = get_result_filename(fold=fold, scene_label=scene_label, path=result_path)
              if not os.path.isfile(current_result_file) or overwrite:
                  results = []
  
                  # Load class model container
                  model_filename = get_model_filename(fold=fold, scene_label=scene_label, path=model_path)
                  if os.path.isfile(model_filename):
                      model_container = load_data(model_filename)
                  else:
                      raise IOError("Model file not found [%s]" % model_filename)
  
                  file_count = len(dataset.test(fold, scene_label=scene_label))
                  for file_id, item in enumerate(dataset.test(fold=fold, scene_label=scene_label)):
                      progress(title_text='Testing',
                               fold=fold,
                               percentage=(float(file_id) / file_count),
                               note=scene_label + " / " + os.path.split(item['file'])[1])
  
                      # Load features
                      feature_filename = get_feature_filename(audio_file=item['file'], path=feature_path)
  
                      if os.path.isfile(feature_filename):
                          feature_data = load_data(feature_filename)['feat']
                      else:
                          # Load audio
                          if os.path.isfile(dataset.relative_to_absolute_path(item['file'])):
                              y, fs = load_audio(filename=item['file'], mono=True, fs=feature_params['fs'])
                          else:
                              raise IOError("Audio file not found [%s]" % item['file'])
  
                          # Extract features
                          feature_data = feature_extraction(y=y,
                                                            fs=fs,
                                                            include_mfcc0=feature_params['include_mfcc0'],
                                                            include_delta=feature_params['include_delta'],
                                                            include_acceleration=feature_params['include_acceleration'],
                                                            mfcc_params=feature_params['mfcc'],
                                                            delta_params=feature_params['mfcc_delta'],
                                                            acceleration_params=feature_params['mfcc_acceleration'],
                                                            statistics=False)['feat']
  
                      # Normalize features
                      feature_data = model_container['normalizer'].normalize(feature_data)
  
                      current_results = event_detection(feature_data=feature_data,
                                                        model_container=model_container,
                                                        hop_length_seconds=feature_params['hop_length_seconds'],
                                                        smoothing_window_length_seconds=detector_params[
                                                            'smoothing_window_length'],
                                                        decision_threshold=detector_params['decision_threshold'],
                                                        minimum_event_length=detector_params['minimum_event_length'],
                                                        minimum_event_gap=detector_params['minimum_event_gap'])
  
                      # Store the result
                      for event in current_results:
                          results.append((dataset.absolute_to_relative(item['file']), event[0], event[1], event[2]))
  
                  # Save testing results
                  with open(current_result_file, 'wt') as f:
                      writer = csv.writer(f, delimiter='\t')
                      for result_item in results:
                          writer.writerow(result_item)
  
  
  def do_system_evaluation(dataset, result_path, dataset_evaluation_mode='folds'):
      """System evaluation. Testing outputs are collected and evaluated. Evaluation results are printed.
  
      Parameters
      ----------
      dataset : class
          dataset class
  
      result_path : str
          path where the results are saved.
  
      dataset_evaluation_mode : str ['folds', 'full']
          evaluation mode, 'full' all material available is considered to belong to one fold.
          (Default value='folds')
  
      Returns
      -------
      nothing
  
      Raises
      -------
      IOError
          Result file not found
  
      """
  
      # Set warnings off, sklearn metrics will trigger warning for classes without
      # predicted samples in F1-scoring. This is just to keep printing clean.
      warnings.simplefilter("ignore")
  
      overall_metrics_per_scene = {}
  
      for scene_id, scene_label in enumerate(dataset.scene_labels):
          if scene_label not in overall_metrics_per_scene:
              overall_metrics_per_scene[scene_label] = {}
  
          dcase2016_segment_based_metric = DCASE2016_EventDetection_SegmentBasedMetrics(
              class_list=dataset.event_labels(scene_label=scene_label))
          dcase2016_event_based_metric = DCASE2016_EventDetection_EventBasedMetrics(
              class_list=dataset.event_labels(scene_label=scene_label))
  
          for fold in dataset.folds(mode=dataset_evaluation_mode):
              results = []
              result_filename = get_result_filename(fold=fold, scene_label=scene_label, path=result_path)
  
              if os.path.isfile(result_filename):
                  with open(result_filename, 'rt') as f:
                      for row in csv.reader(f, delimiter='\t'):
                          results.append(row)
              else:
                  raise IOError("Result file not found [%s]" % result_filename)
  
              for file_id, item in enumerate(dataset.test(fold, scene_label=scene_label)):
                  current_file_results = []
                  for result_line in results:
                      if len(result_line) != 0 and result_line[0] == dataset.absolute_to_relative(item['file']):
                          current_file_results.append(
                              {'file': result_line[0],
                               'event_onset': float(result_line[1]),
                               'event_offset': float(result_line[2]),
                               'event_label': result_line[3].rstrip()
                               }
                          )
                  meta = dataset.file_meta(dataset.absolute_to_relative(item['file']))
  
                  dcase2016_segment_based_metric.evaluate(system_output=current_file_results, annotated_ground_truth=meta)
                  dcase2016_event_based_metric.evaluate(system_output=current_file_results, annotated_ground_truth=meta)
  
          overall_metrics_per_scene[scene_label]['segment_based_metrics'] = dcase2016_segment_based_metric.results()
          overall_metrics_per_scene[scene_label]['event_based_metrics'] = dcase2016_event_based_metric.results()
  
      print "  Evaluation over %d folds" % dataset.fold_count
      print " "
      print "  Results per scene "
      print "  {:18s} | {:5s} |  | {:39s}  ".format('', 'Main', 'Secondary metrics')
      print "  {:18s} | {:5s} |  | {:38s} | {:14s} | {:14s} |  {:14s} ".format('', '', 'Seg/Overall', 'Seg/Class',
                                                                               'Event/Overall', 'Event/Class')
      print "  {:18s} | {:5s} |  | {:6s} : {:5s} : {:5s} : {:5s} : {:5s} | {:6s} : {:5s} | {:6s} : {:5s} | {:6s} : {:5s} |".format(
          'Scene', 'ER', 'F1', 'ER', 'ER/S', 'ER/D', 'ER/I', 'F1', 'ER', 'F1', 'ER', 'F1', 'ER')
      print "  -------------------+-------+  +--------+-------+-------+-------+-------+--------+-------+--------+-------+--------+-------+"
      averages = {
          'segment_based_metrics': {
              'overall': {
                  'ER': [],
                  'F': [],
              },
              'class_wise_average': {
                  'ER': [],
                  'F': [],
              }
          },
          'event_based_metrics': {
              'overall': {
                  'ER': [],
                  'F': [],
              },
              'class_wise_average': {
                  'ER': [],
                  'F': [],
              }
          },
      }
      for scene_id, scene_label in enumerate(dataset.scene_labels):
          print "  {:18s} | {:5.2f} |  | {:4.1f} % : {:5.2f} : {:5.2f} : {:5.2f} : {:5.2f} | {:4.1f} % : {:5.2f} | {:4.1f} % : {:5.2f} | {:4.1f} % : {:5.2f} |".format(
              scene_label,
              overall_metrics_per_scene[scene_label]['segment_based_metrics']['overall']['ER'],
              overall_metrics_per_scene[scene_label]['segment_based_metrics']['overall']['F'] * 100,
              overall_metrics_per_scene[scene_label]['segment_based_metrics']['overall']['ER'],
              overall_metrics_per_scene[scene_label]['segment_based_metrics']['overall']['S'],
              overall_metrics_per_scene[scene_label]['segment_based_metrics']['overall']['D'],
              overall_metrics_per_scene[scene_label]['segment_based_metrics']['overall']['I'],
              overall_metrics_per_scene[scene_label]['segment_based_metrics']['class_wise_average']['F'] * 100,
              overall_metrics_per_scene[scene_label]['segment_based_metrics']['class_wise_average']['ER'],
              overall_metrics_per_scene[scene_label]['event_based_metrics']['overall']['F'] * 100,
              overall_metrics_per_scene[scene_label]['event_based_metrics']['overall']['ER'],
              overall_metrics_per_scene[scene_label]['event_based_metrics']['class_wise_average']['F'] * 100,
              overall_metrics_per_scene[scene_label]['event_based_metrics']['class_wise_average']['ER'],
              )
          averages['segment_based_metrics']['overall']['ER'].append(
              overall_metrics_per_scene[scene_label]['segment_based_metrics']['overall']['ER'])
          averages['segment_based_metrics']['overall']['F'].append(
              overall_metrics_per_scene[scene_label]['segment_based_metrics']['overall']['F'])
          averages['segment_based_metrics']['class_wise_average']['ER'].append(
              overall_metrics_per_scene[scene_label]['segment_based_metrics']['class_wise_average']['ER'])
          averages['segment_based_metrics']['class_wise_average']['F'].append(
              overall_metrics_per_scene[scene_label]['segment_based_metrics']['class_wise_average']['F'])
          averages['event_based_metrics']['overall']['ER'].append(
              overall_metrics_per_scene[scene_label]['event_based_metrics']['overall']['ER'])
          averages['event_based_metrics']['overall']['F'].append(
              overall_metrics_per_scene[scene_label]['event_based_metrics']['overall']['F'])
          averages['event_based_metrics']['class_wise_average']['ER'].append(
              overall_metrics_per_scene[scene_label]['event_based_metrics']['class_wise_average']['ER'])
          averages['event_based_metrics']['class_wise_average']['F'].append(
              overall_metrics_per_scene[scene_label]['event_based_metrics']['class_wise_average']['F'])
  
      print "  -------------------+-------+  +--------+-------+-------+-------+-------+--------+-------+--------+-------+--------+-------+"
      print "  {:18s} | {:5.2f} |  | {:4.1f} % : {:5.2f} : {:21s} | {:4.1f} % : {:5.2f} | {:4.1f} % : {:5.2f} | {:4.1f} % : {:5.2f} |".format(
          'Average',
          numpy.mean(averages['segment_based_metrics']['overall']['ER']),
          numpy.mean(averages['segment_based_metrics']['overall']['F']) * 100,
          numpy.mean(averages['segment_based_metrics']['overall']['ER']),
          ' ',
          numpy.mean(averages['segment_based_metrics']['class_wise_average']['F']) * 100,
          numpy.mean(averages['segment_based_metrics']['class_wise_average']['ER']),
          numpy.mean(averages['event_based_metrics']['overall']['F']) * 100,
          numpy.mean(averages['event_based_metrics']['overall']['ER']),
          numpy.mean(averages['event_based_metrics']['class_wise_average']['F']) * 100,
          numpy.mean(averages['event_based_metrics']['class_wise_average']['ER']),
          )
  
      print "  "
      # Restore warnings to default settings
      warnings.simplefilter("default")
      print "  Results per events "
  
      for scene_id, scene_label in enumerate(dataset.scene_labels):
          print "  "
          print "  " + scene_label.upper()
          print "  {:20s} | {:30s} |  | {:15s} ".format('', 'Segment-based', 'Event-based')
          print "  {:20s} | {:5s} : {:5s} : {:6s} : {:5s} |  | {:5s} : {:5s} : {:6s} : {:5s} |".format('Event', 'Nref',
                                                                                                       'Nsys', 'F1', 'ER',
                                                                                                       'Nref', 'Nsys',
                                                                                                       'F1', 'ER')
          print "  ---------------------+-------+-------+--------+-------+  +-------+-------+--------+-------+"
          seg_Nref = 0
          seg_Nsys = 0
  
          event_Nref = 0
          event_Nsys = 0
          for event_label in sorted(overall_metrics_per_scene[scene_label]['segment_based_metrics']['class_wise']):
              print "  {:20s} | {:5d} : {:5d} : {:4.1f} % : {:5.2f} |  | {:5d} : {:5d} : {:4.1f} % : {:5.2f} |".format(
                  event_label,
                  int(overall_metrics_per_scene[scene_label]['segment_based_metrics']['class_wise'][event_label]['Nref']),
                  int(overall_metrics_per_scene[scene_label]['segment_based_metrics']['class_wise'][event_label]['Nsys']),
                  overall_metrics_per_scene[scene_label]['segment_based_metrics']['class_wise'][event_label]['F'] * 100,
                  overall_metrics_per_scene[scene_label]['segment_based_metrics']['class_wise'][event_label]['ER'],
                  int(overall_metrics_per_scene[scene_label]['event_based_metrics']['class_wise'][event_label]['Nref']),
                  int(overall_metrics_per_scene[scene_label]['event_based_metrics']['class_wise'][event_label]['Nsys']),
                  overall_metrics_per_scene[scene_label]['event_based_metrics']['class_wise'][event_label]['F'] * 100,
                  overall_metrics_per_scene[scene_label]['event_based_metrics']['class_wise'][event_label]['ER'])
              seg_Nref += int(
                  overall_metrics_per_scene[scene_label]['segment_based_metrics']['class_wise'][event_label]['Nref'])
              seg_Nsys += int(
                  overall_metrics_per_scene[scene_label]['segment_based_metrics']['class_wise'][event_label]['Nsys'])
  
              event_Nref += int(
                  overall_metrics_per_scene[scene_label]['event_based_metrics']['class_wise'][event_label]['Nref'])
              event_Nsys += int(
                  overall_metrics_per_scene[scene_label]['event_based_metrics']['class_wise'][event_label]['Nsys'])
          print "  ---------------------+-------+-------+--------+-------+  +-------+-------+--------+-------+"
          print "  {:20s} | {:5d} : {:5d} : {:14s} |  | {:5d} : {:5d} : {:14s} |".format('Sum',
                                                                                         seg_Nref,
                                                                                         seg_Nsys,
                                                                                         '',
                                                                                         event_Nref,
                                                                                         event_Nsys,
                                                                                         '')
          print "  {:20s} | {:5s}   {:5s} : {:4.1f} % : {:5.2f} |  | {:5s}   {:5s} : {:4.1f} % : {:5.2f} |".format(
              'Average',
              '', '',
              overall_metrics_per_scene[scene_label]['segment_based_metrics']['class_wise_average']['F'] * 100,
              overall_metrics_per_scene[scene_label]['segment_based_metrics']['class_wise_average']['ER'],
              '', '',
              overall_metrics_per_scene[scene_label]['event_based_metrics']['class_wise_average']['F'] * 100,
              overall_metrics_per_scene[scene_label]['event_based_metrics']['class_wise_average']['ER'])
          print "  "
  
  
  if __name__ == "__main__":
      try:
          sys.exit(main(sys.argv))
      except (ValueError, IOError) as e:
          sys.exit(e)
  