text: # BSD 3-Clause License
  #
  # This file is part of the DM-VIO-Python-Tools.
  # https://github.com/lukasvst/dm-vio-python-tools
  #
  # Copyright (c) 2022, Lukas von Stumberg, TUM
  # All rights reserved.
  #
  # Redistribution and use in source and binary forms, with or without modification, are permitted provided that the
  # following conditions are met:
  #
  # 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following
  # disclaimer.
  #
  # 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the
  # following disclaimer in the documentation and/or other materials provided with the distribution.
  #
  # 3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote
  # products derived from this software without specific prior written permission.
  #
  # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
  # INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
  # DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
  # SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
  # WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
  # USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  
  import argparse
  from utils.config_utils import read_config, read_all_configs
  import subprocess
  from pathlib import Path
  from ruamel.yaml import YAML
  from datetime import datetime
  import sys
  from tqdm import tqdm
  
  
  class ResultsSorter:
      def __init__(self, use_commit_date=False):
          """
          :param use_commit_date: If false the time of run is used, otherwise the commit time.
          """
          self.use_commit_date = use_commit_date
          if use_commit_date:
              self.key = 'commit_time'
          else:
              self.key = 'date_run'
  
      def __call__(self, pair):
          setup = pair[1]
          if self.use_commit_date:
              # in this case we want to return (commit_date, run_date)
              return setup[self.key], setup['date_run']
  
          if self.key in setup:
              return setup[self.key]
          else:
              # Shouldn't happen.
              raise ValueError("Key does not exist.")
  
  
  def create_evaluation_file(config_name, outfile, no_download, sorter, filters):
      """ Create a python evaluation file with all results from the results_folder defined in the config with the passed
      name.
      :param config_name: Name of the configuration (in configs.yaml) to use.
      :param outfile: name to evaluation file which will be created by this method.
      :param no_download:if False this method will also download results from the central server (if rsync_command is
      set in the config).
      :param sorter: Used to sort the results before writing them to file.
      :param filters: List of filters which are used to filter the results written to the evaluation file.
      """
      # Read config.
      config, config_name, general_config, _ = read_config(config_name)
      if config is None:
          print('Error: config has to specified.')
          sys.exit(1)
  
      general_save_folder = config['results_path']
  
      # Download all results from central machine.
      if not no_download:
          if 'rsync_command' in config:
              rsync_command = config['rsync_command']
              rsync_target = config['rsync_command_target']
              full_rsync_command = '{} {}/* {}/'.format(rsync_command, rsync_target, general_save_folder)
              print(full_rsync_command)
              subprocess.run(full_rsync_command, shell=True)
  
      # Load all results folders and read yaml files.
      all_results = load_result_yamls(Path(general_save_folder))
  
      # Filter and sort results based on parameters
      all_results.sort(key=sorter)
  
      print('There are {} results before filtering.'.format(len(all_results)))
  
      # Apply filters.
      filtered = all_results
      for filter_fun in filters:
          filtered = filter(filter_fun, filtered)
      filtered_results = list(filtered)
  
      print('There are {} results after filtering.'.format(len(filtered_results)))
  
      # Generate Python evaluation script.
      out_path = Path(outfile)
      write_python_eval_file(filtered_results, out_path)
  
      return all_results
  
  
  def load_result_yamls(result_folder: Path):
      """Load the setup.yaml files for all results in the given folder."""
      yaml = YAML(typ='safe')
      all_results = []
      for child in result_folder.iterdir():
          yaml_file = child / 'setup' / 'setup.yaml'
          finished_file = child / 'setup' / 'Finished.txt'
          if not yaml_file.exists():
              print('WARNING: Skipping {}, because the setup file does not exist'.format(child))
              continue
          with open(yaml_file, 'r') as yaml_file_handle:
              settings = yaml.load(yaml_file_handle)
          finished = finished_file.exists()
          settings['finished'] = finished
          all_results.append((child, settings))
      return all_results
  
  
  def write_python_eval_file(results, outfilename):
      """Write a python evaluation file with all passed results to the file with the passed name."""
      if len(results) == 0:
          return
      # Check that first line of file contains 'autogenerated' before overwriting!
      autogen_string = '# AUTOGENERATED'
      if Path(outfilename).exists():
          with open(outfilename, 'r') as outfileread:
              if not autogen_string in outfileread.readline():
                  print('ERROR: Trying to overwrite file without AUTOGENERATED mark.')
                  sys.exit(1)
  
      with open(outfilename, 'w') as outfile:
          print('Writing Python eval file.')
          outfile.write(autogen_string + '\n')
  
          # Init variables.
          outfile.write('from trajectory_evaluation.evaluate import evaluate_run, Dataset\n'
                        'from pathlib import Path\n'
                        'from trajectory_evaluation.plots import square_plot, results_table, line_plot\n')
  
          all_configs = read_all_configs()
  
          parent_folder = results[0][0].parent
          outfile.write("folder = '{}'\n\n".format(parent_folder))
  
          outfile.write(
              "print('Remember that the previous results can change names, when new results are added to the folder '\n"
              "      'and this file is recreated. It might be better to copy specific lines from this file rather '\n"
              "      'than importing the whole file.')\n\n")
  
          prev_setup = None
  
          for i, result in enumerate(results):
              # We put some information about the run in the name and comment of the result.
              # - specified name: in name (but maybe not if standard one)
              # - custom comment: in comment!
              # - Git log: In comment.
              # - Realtime result: as -RT in name
              # - dataset: in the name
              # - diff_empty -> Before Git log if not empty
              # - num_iter, only_seq -> write in name if not full run
              # - unfinished -> write in name
              folder_path, setup = result
  
              custom_name = '_' + setup['name']
              if custom_name == '_dsoresult' or custom_name == '_dmvioresult' or custom_name == '_orbresult' or \
                      custom_name == '_basaltresult':  # remove standard names.
                  custom_name = ''
              custom_comment = ' ' + setup['comment'] if 'comment' in setup else ''
              commit_message = setup['commit_message']
              commit_message_first_line = commit_message.split('\n')[0]  # Commit message could contain line breaks.
              noimu_bool = 'noimu' in setup and setup['noimu']
              noimu = '_noimu' if noimu_bool else ''
              realtime = '_RT' if 'realtime' in setup and setup['realtime'] else ''
              config_name = setup['config_name']
              short_name = all_configs[config_name]['short_name'] if config_name in all_configs else config_name
              dataset_name = setup['dataset']
              dataset_config = all_configs['config_general'][dataset_name]
              diff_string = '+DIFF ' if not setup['diff_empty'] else ''
              default_iter = dataset_config['default_iter']
              partial_string = '_part' if setup['num_iter'] < default_iter or not setup['only_seq'] is None else ''
              unfinished_string = '_unfin' if setup['finished'] is False else ''
              withgui_string = '_withgui' if 'withgui' in setup and setup['withgui'] else ''
              quiet_string = '_quiet' if 'quiet' in setup and setup['quiet'] else ''
              output_type_string = ' output=' + setup['output_type'] if 'output_type' in setup else ''
              custom_args_string = setup['custom_dso_args'] if 'custom_dso_args' in setup else ''
              if 'custom_dmvio_args' in setup:
                  custom_args_string = setup['custom_dmvio_args']
              is_orb_result = 'orbslam' in setup and setup['orbslam']
              if custom_args_string != '':
                  custom_args_string = ' ' + custom_args_string
              settings_string = setup['dso_settings'] if 'dso_settings' in setup else ''
              if 'dmvio_settings' in setup:
                  settings_string = setup['dmvio_settings']
              if settings_string != '':
                  settings_string = ' settings=' + settings_string
              is_basalt_result = 'basalt' in setup and setup['basalt']
              if is_basalt_result:
                  settings_string = ' config={} calib={}'.format(setup['basalt_config'], setup['basalt_calib'])
              build_type_string = ''
              if 'build_type' in setup:
                  build_type = setup['build_type']
                  if build_type != 'RelWithDebInfo':
                      build_type_string = '_' + build_type
  
              res_name = str(
                  i) + custom_name + noimu + realtime + '_' + dataset_name + '_' + short_name + build_type_string + \
                         partial_string + unfinished_string
              if is_orb_result:
                  res_name = 'orb' + res_name
              elif is_basalt_result:
                  res_name = 'basalt' + res_name
              comment = custom_comment + diff_string + settings_string + custom_args_string + ' ' + \
                        commit_message_first_line
              visname = res_name + custom_comment + ':' + output_type_string + quiet_string + withgui_string + \
                        settings_string + custom_args_string + ' ' + diff_string + commit_message_first_line
              visname = visname.replace("'", '"')
  
              if 'tumvi' in dataset_name:
                  dataset_arg = 'Dataset.tumvi'
              elif '4seasons' in dataset_name:
                  dataset_arg = 'Dataset.four_seasons'
              elif 'euroc' in dataset_name:
                  dataset_arg = 'Dataset.euroc'
              else:
                  raise ValueError("ERROR: Unknown dataset")
  
              # Write newline if different commit hash than previous.
              if not prev_setup is None and setup['git_hash'] != prev_setup['git_hash']:
                  outfile.write('\n')
  
              outfile.write("#{}\n".format(comment))
              outfile.write(
                  "res{}, gtscale_{} = evaluate_run(Path(folder) / '{}', {}, {}, '{}')\n".format(res_name, res_name,
                                                                                                 folder_path.name,
                                                                                                 dataset_arg,
                                                                                                 setup['num_iter'],
                                                                                                 visname))
  
              prev_setup = setup
  
  
  def finished_or_temp_filter(pair):
      setup = pair[1]
      if setup['finished']:
          return True
      if 'temporary' in setup and setup['temporary'] is True:
          return True
      return False
  
  
  def full_filter(pair):
      setup = pair[1]
      all_configs = read_all_configs()
      dataset_name = setup['dataset']
      dataset_config = all_configs['config_general'][dataset_name]
      default_iter = dataset_config['default_iter']
      if setup['num_iter'] < default_iter or not setup['only_seq'] is None:
          return False
      return True
  
  
  def main():
      parser = argparse.ArgumentParser(
          description='Create Python evaluation file. It will also evaluate all results which have not been evaluated '
                      'yet.')
      parser.add_argument('--config', type=str, default=None, help="Config to use.")
      parser.add_argument('--outfile', type=str, default='evaluations.py',
                          help='Name of the resulting python evalution file. By default stored in the folder specified '
                               'in configs.yaml')
      parser.add_argument('--no_download', default=False, action='store_true',
                          help="Don't download from central storage before creating the evaluation file.")
      parser.add_argument('--evaluate', default=False, action='store_true',
                          help="If passed this script will compute all results (which can take a while). Otherwise "
                               "results will only be computed on demand when using the written evaluation script.")
      parser.add_argument('--force_evaluate', default=False, action='store_true',
                          help="Re-evaluate all results even if they have already been evaluated before")
      args = parser.parse_args()
  
      sorter = ResultsSorter(use_commit_date=True)  # You can first use commit date for sorting or just the run date.
      finished_filter = lambda pair: pair[1]['finished'] is True
  
      # Example date filter below.
      date = datetime.strptime('03.12.20 17:25:00', '%d.%m.%y %H:%M:%S')
      date_filter = lambda pair: pair[1]['date_run'] > date
  
      all_results = create_evaluation_file(args.config, args.outfile, args.no_download, sorter, [finished_filter])
  
      # Example: Use this to save only full evaluations.
      # all_results = create_evaluation_file(args.config, 'evaluations_only_full.py', True, sorter,
      #                                      [finished_filter, full_filter])
  
      if args.evaluate or args.force_evaluate:
          print("Pre-evaluating all results which have not been evaluated yet.")
          for pair in tqdm(all_results, leave=True):
              from trajectory_evaluation.evaluate import evaluate_with_config
              result, result_gt_scaled = evaluate_with_config(pair, args.force_evaluate)
  
  
  if __name__ == "__main__":
      main()
  