text: # -*- coding: utf-8 -*-
  from __future__ import print_function
  
  import ast
  import copy
  import fnmatch
  import logging
  import ntpath
  import os
  import random
  import sys
  from functools import reduce
  from six import iteritems
  
  if sys.version_info.major == 3:
      import _pickle as pk
  else:
      import cPickle as pk
      from itertools import izip as zip
  import codecs
  from collections import Counter, defaultdict
  from operator import add
  import numpy as np
  from keras_wrapper.extra.read_write import create_dir_if_not_exists
  from keras_wrapper.extra.tokenizers import *
  from .utils import bbox, to_categorical
  from .utils import MultiprocessQueue
  import multiprocessing
  
  logging.basicConfig(level=logging.INFO, format='[%(asctime)s] %(message)s', datefmt='%d/%m/%Y %H:%M:%S')
  logger = logging.getLogger(__name__)
  
  # ------------------------------------------------------- #
  #       SAVE/LOAD
  #           External functions for saving and loading Dataset instances
  # ------------------------------------------------------- #
  
  
  def saveDataset(dataset, store_path):
      """
      Saves a backup of the current Dataset object.
  
      :param dataset: Dataset object to save
      :param store_path: Saving path
      :return: None
      """
      create_dir_if_not_exists(store_path)
      store_path = store_path + '/Dataset_' + dataset.name + '.pkl'
      if not dataset.silence:
          logger.info("<<< Saving Dataset instance to " + store_path + " ... >>>")
  
      pk.dump(dataset, open(store_path, 'wb'), protocol=-1)
  
      if not dataset.silence:
          logger.info("<<< Dataset instance saved >>>")
  
  
  def loadDataset(dataset_path):
      """
      Loads a previously saved Dataset object.
  
      :param dataset_path: Path to the stored Dataset to load
      :return: Loaded Dataset object
      """
  
      logger.info("<<< Loading Dataset instance from " + dataset_path + " ... >>>")
      if sys.version_info.major == 3:
          dataset = pk.load(open(dataset_path, 'rb'), encoding='utf-8')
      else:
          dataset = pk.load(open(dataset_path, 'rb'))
  
      if not hasattr(dataset, 'pad_symbol'):
          dataset.pad_symbol = '<pad>'
      if not hasattr(dataset, 'unk_symbol'):
          dataset.unk_symbol = '<unk>'
      if not hasattr(dataset, 'null_symbol'):
          dataset.null_symbol = '<null>'
  
      logger.info("<<< Dataset instance loaded >>>")
      return dataset
  
  
  # ------------------------------------------------------- #
  #       DATA BATCH GENERATOR CLASS
  # ------------------------------------------------------- #
  
  def dataLoad(process_name, net, dataset, max_queue_len, queues):
      """
      Parallel data loader. Risky and untested!
      :param process_name:
      :param net:
      :param dataset:
      :param max_queue_len:
      :param queues:
      :return:
      """
      logger.info("Starting " + process_name + "...")
      in_queue, out_queue = queues
  
      while True:
          while out_queue.qsize() > max_queue_len:
              pass
  
          # available modes are 'indices' and 'consecutive'
          data_queue = in_queue.get()
  
          [mode, predict, set_split, ind, normalization, normalization_type, mean_substraction, data_augmentation] = data_queue
  
          # Recovers a batch of data
          if predict:
              if mode == 'indices':
                  X_batch = dataset.getX_FromIndices(set_split,
                                                     ind[0],
                                                     normalization=normalization,
                                                     normalization_type=normalization_type,
                                                     meanSubstraction=mean_substraction,
                                                     dataAugmentation=data_augmentation)
              elif mode == 'consecutive':
                  X_batch = dataset.getX(set_split,
                                         ind[0], ind[1],
                                         normalization=normalization,
                                         normalization_type=normalization_type,
                                         meanSubstraction=mean_substraction,
                                         dataAugmentation=data_augmentation)
              else:
                  raise NotImplementedError("Data retrieval mode '" + mode + "' is not implemented.")
              data = net.prepareData(X_batch, None)[0]
          else:
              X_batch, Y_batch = dataset.getXY_FromIndices(set_split,
                                                           ind[0],
                                                           normalization=normalization,
                                                           normalization_type=normalization_type,
                                                           meanSubstraction=mean_substraction,
                                                           dataAugmentation=data_augmentation)
              data = net.prepareData(X_batch, Y_batch)
  
          out_queue.put(data)
  
  
  class Parallel_Data_Batch_Generator(object):
      """
      Batch generator class. Retrieves batches of data.
      """
  
      def __init__(self,
                   set_split,
                   net,
                   dataset,
                   num_iterations,
                   batch_size=50,
                   normalization=False,
                   normalization_type=None,
                   data_augmentation=True,
                   wo_da_patch_type='whole',
                   da_patch_type='resize_and_rndcrop',
                   da_enhance_list=None,
                   mean_substraction=False,
                   predict=False,
                   random_samples=-1,
                   shuffle=True,
                   temporally_linked=False,
                   init_sample=-1,
                   final_sample=-1,
                   n_parallel_loaders=1):
          """
          Initializes the Data_Batch_Generator
          :param set_split: Split (train, val, test) to retrieve data
          :param net: Net which use the data
          :param dataset: Dataset instance
          :param num_iterations: Maximum number of iterations
          :param batch_size: Size of the minibatch
          :param normalization: Switches on/off the normalization of images
          :param data_augmentation: Switches on/off the data augmentation of the input
          :param mean_substraction: Switches on/off the mean substraction for images
          :param predict: Whether we are predicting or training
          :param random_samples: Retrieves this number of training samples
          :param shuffle: Shuffle the training dataset
          :param temporally_linked: Indicates if we are using a temporally-linked model
          :param n_parallel_loaders: Number of parallel loaders that will be used.
          """
  
          if da_enhance_list is None:
              da_enhance_list = []
  
          self.set_split = set_split
          self.dataset = dataset
          self.net = net
          self.predict = predict
          self.temporally_linked = temporally_linked
          self.first_idx = -1
          self.init_sample = init_sample
          self.final_sample = final_sample
          self.next_idx = None
          self.thread_list = []
  
          # Several parameters
          self.params = {'batch_size': batch_size,
                         'data_augmentation': data_augmentation,
                         'wo_da_patch_type': wo_da_patch_type,
                         'da_patch_type': da_patch_type,
                         'da_enhance_list': da_enhance_list,
                         'mean_substraction': mean_substraction,
                         'normalization': normalization,
                         'normalization_type': normalization_type,
                         'num_iterations': num_iterations,
                         'random_samples': random_samples,
                         'shuffle': shuffle,
                         'n_parallel_loaders': n_parallel_loaders}
  
      def __del__(self):
          self.terminateThreads()
  
      def terminateThreads(self):
          for t in self.thread_list:
              t.terminate()
  
      def generator(self):
          """
          Gets and processes the data
          :return: generator with the data
          """
  
          self.terminateThreads()
  
          if self.set_split == 'train' and not self.predict:
              data_augmentation = self.params['data_augmentation']
          else:
              data_augmentation = False
  
          # Initialize list of parallel data loaders
          thread_mngr = multiprocessing.Manager()
          in_queue = MultiprocessQueue(thread_mngr, multiprocess_type='Queue')  # if self.params['n_parallel_loaders'] > 1 else 'Pipe')
          out_queue = MultiprocessQueue(thread_mngr, multiprocess_type='Queue')  # if self.params['n_parallel_loaders'] > 1 else 'Pipe')
          # Create a queue per function
          for i in range(self.params['n_parallel_loaders']):
              # create process
              new_process = multiprocessing.Process(target=dataLoad,
                                                    args=('dataLoad_process_' + str(i),
                                                          self.net, self.dataset, int(self.params['n_parallel_loaders'] * 1.5), [in_queue, out_queue]))
              self.thread_list.append(new_process)  # store processes for terminating later
              new_process.start()
  
          it = 0
          while True:
              if self.set_split == 'train' and it % self.params['num_iterations'] == 0 and \
                      not self.predict and self.params['random_samples'] == -1 and self.params['shuffle']:
                  silence = self.dataset.silence
                  self.dataset.silence = True
                  self.dataset.shuffleTraining()
                  self.dataset.silence = silence
              if it % self.params['num_iterations'] == 0 and self.params['random_samples'] == -1:
                  self.dataset.resetCounters(set_name=self.set_split)
              it += 1
  
              # Checks if we are finishing processing the data split
              init_sample = (it - 1) * self.params['batch_size']
              final_sample = it * self.params['batch_size']
              n_samples_split = getattr(self.dataset, "len_" + self.set_split)
              if final_sample >= n_samples_split:
                  final_sample = n_samples_split
                  # batch_size = final_sample - init_sample
                  it = 0
  
              # Recovers a batch of data
              # random data selection
              if self.params['random_samples'] > 0:
                  num_retrieve = min(self.params['random_samples'], self.params['batch_size'])
                  if self.temporally_linked:
                      if self.first_idx == -1:
                          self.first_idx = np.random.randint(0, n_samples_split - self.params['random_samples'], 1)[0]
                          self.next_idx = self.first_idx
                      indices = range(self.next_idx, self.next_idx + num_retrieve)
                      self.next_idx += num_retrieve
                  else:
                      indices = np.random.randint(0, n_samples_split, num_retrieve)
                  self.params['random_samples'] -= num_retrieve
  
                  # Prepare query data for parallel data loaders
                  query_data = ['indices', self.predict, self.set_split, [indices],
                                self.params['normalization'], self.params['normalization_type'],
                                self.params['mean_substraction'], data_augmentation]
  
              # specific data selection
              elif self.init_sample > -1 and self.final_sample > -1:
                  indices = range(self.init_sample, self.final_sample)
  
                  # Prepare query data for parallel data loaders
                  query_data = ['indices', self.predict, self.set_split, [indices],
                                self.params['normalization'], self.params['normalization_type'],
                                self.params['mean_substraction'], data_augmentation]
  
              # consecutive data selection
              else:
                  if self.predict:
                      query_data = ['consecutive', self.predict, self.set_split, [init_sample, final_sample],
                                    self.params['normalization'], self.params['normalization_type'],
                                    self.params['mean_substraction'], data_augmentation]
                  else:
                      query_data = ['consecutive', self.predict, self.set_split, [range(init_sample, final_sample)],
                                    self.params['normalization'], self.params['normalization_type'],
                                    self.params['mean_substraction'], data_augmentation]
  
              # Insert data in queue
              in_queue.put(query_data)
  
              # Check if there is processed data in queue
              while out_queue.qsize() > 0:
                  data = out_queue.get()
                  yield (data)
  
  
  class Data_Batch_Generator(object):
      """
      Batch generator class. Retrieves batches of data.
      """
  
      def __init__(self,
                   set_split,
                   net,
                   dataset,
                   num_iterations,
                   batch_size=50,
                   normalization=False,
                   normalization_type=None,
                   data_augmentation=True,
                   wo_da_patch_type='whole',
                   da_patch_type='resize_and_rndcrop',
                   da_enhance_list=None,
                   mean_substraction=False,
                   predict=False,
                   random_samples=-1,
                   shuffle=True,
                   temporally_linked=False,
                   init_sample=-1,
                   final_sample=-1):
          """
          Initializes the Data_Batch_Generator
          :param set_split: Split (train, val, test) to retrieve data
          :param net: Net which use the data
          :param dataset: Dataset instance
          :param num_iterations: Maximum number of iterations
          :param batch_size: Size of the minibatch
          :param normalization: Switches on/off the normalization of images
          :param data_augmentation: Switches on/off the data augmentation of the input
          :param mean_substraction: Switches on/off the mean substraction for images
          :param predict: Whether we are predicting or training
          :param random_samples: Retrieves this number of training samples
          :param shuffle: Shuffle the training dataset
          :param temporally_linked: Indicates if we are using a temporally-linked model
          """
          if da_enhance_list is None:
              da_enhance_list = []
          self.set_split = set_split
          self.dataset = dataset
          self.net = net
          self.predict = predict
          self.temporally_linked = temporally_linked
          self.first_idx = -1
          self.init_sample = init_sample
          self.final_sample = final_sample
          self.next_idx = None
  
          # Several parameters
          self.params = {'batch_size': batch_size,
                         'data_augmentation': data_augmentation,
                         'wo_da_patch_type': wo_da_patch_type,
                         'da_patch_type': da_patch_type,
                         'da_enhance_list': da_enhance_list,
                         'mean_substraction': mean_substraction,
                         'normalization': normalization,
                         'normalization_type': normalization_type,
                         'num_iterations': num_iterations,
                         'random_samples': random_samples,
                         'shuffle': shuffle}
  
      def generator(self):
          """
          Gets and processes the data
          :return: generator with the data
          """
  
          if self.set_split == 'train' and not self.predict:
              data_augmentation = self.params['data_augmentation']
          else:
              data_augmentation = False
  
          it = 0
          while 1:
              if self.set_split == 'train' and it % self.params['num_iterations'] == 0 and \
                      not self.predict and self.params['random_samples'] == -1 and self.params['shuffle']:
                  silence = self.dataset.silence
                  self.dataset.silence = True
                  self.dataset.shuffleTraining()
                  self.dataset.silence = silence
              if it % self.params['num_iterations'] == 0 and self.params['random_samples'] == -1:
                  self.dataset.resetCounters(set_name=self.set_split)
              it += 1
  
              # Checks if we are finishing processing the data split
              init_sample = (it - 1) * self.params['batch_size']
              final_sample = it * self.params['batch_size']
              batch_size = self.params['batch_size']
              n_samples_split = getattr(self.dataset, "len_" + self.set_split)
              if final_sample >= n_samples_split:
                  final_sample = n_samples_split
                  batch_size = final_sample - init_sample
                  it = 0
  
              # Recovers a batch of data
              if self.params['random_samples'] > 0:
                  num_retrieve = min(self.params['random_samples'], self.params['batch_size'])
                  if self.temporally_linked:
                      if self.first_idx == -1:
                          self.first_idx = np.random.randint(0, n_samples_split - self.params['random_samples'], 1)[0]
                          self.next_idx = self.first_idx
                      indices = list(range(self.next_idx, self.next_idx + num_retrieve))
                      self.next_idx += num_retrieve
                  else:
                      indices = np.random.randint(0, n_samples_split, num_retrieve)
                  self.params['random_samples'] -= num_retrieve
  
                  # At sampling from train/val, we always have Y
                  if self.predict:
                      X_batch = self.dataset.getX_FromIndices(self.set_split,
                                                              indices,
                                                              normalization=self.params['normalization'],
                                                              normalization_type=self.params['normalization_type'],
                                                              meanSubstraction=self.params['mean_substraction'],
                                                              dataAugmentation=data_augmentation,
                                                              wo_da_patch_type=self.params['wo_da_patch_type'],
                                                              da_patch_type=self.params['da_patch_type'],
                                                              da_enhance_list=self.params['da_enhance_list']
                                                              )
                      data = self.net.prepareData(X_batch, None)[0]
  
                  else:
                      X_batch, Y_batch = self.dataset.getXY_FromIndices(self.set_split,
                                                                        indices,
                                                                        normalization=self.params['normalization'],
                                                                        normalization_type=self.params['normalization_type'],
                                                                        meanSubstraction=self.params['mean_substraction'],
                                                                        dataAugmentation=data_augmentation,
                                                                        wo_da_patch_type=self.params['wo_da_patch_type'],
                                                                        da_patch_type=self.params['da_patch_type'],
                                                                        da_enhance_list=self.params['da_enhance_list'])
                      data = self.net.prepareData(X_batch, Y_batch)
  
              elif self.init_sample > -1 and self.final_sample > -1:
                  indices = list(range(self.init_sample, self.final_sample))
                  if self.predict:
                      X_batch = self.dataset.getX_FromIndices(self.set_split,
                                                              indices,
                                                              normalization=self.params['normalization'],
                                                              normalization_type=self.params['normalization_type'],
                                                              meanSubstraction=self.params['mean_substraction'],
                                                              dataAugmentation=data_augmentation,
                                                              wo_da_patch_type=self.params['wo_da_patch_type'],
                                                              da_patch_type=self.params['da_patch_type'],
                                                              da_enhance_list=self.params['da_enhance_list'])
                      data = self.net.prepareData(X_batch, None)[0]
  
                  else:
                      X_batch, Y_batch = self.dataset.getXY_FromIndices(self.set_split,
                                                                        indices,
                                                                        normalization=self.params['normalization'],
                                                                        normalization_type=self.params[
                                                                            'normalization_type'],
                                                                        meanSubstraction=self.params['mean_substraction'],
                                                                        dataAugmentation=data_augmentation,
                                                                        wo_da_patch_type=self.params['wo_da_patch_type'],
                                                                        da_patch_type=self.params['da_patch_type'],
                                                                        da_enhance_list=self.params['da_enhance_list'])
                      data = self.net.prepareData(X_batch, Y_batch)
  
              else:
                  if self.predict:
                      X_batch = self.dataset.getX(self.set_split,
                                                  init_sample,
                                                  final_sample,
                                                  normalization=self.params['normalization'],
                                                  normalization_type=self.params['normalization_type'],
                                                  meanSubstraction=self.params['mean_substraction'],
                                                  dataAugmentation=False,
                                                  wo_da_patch_type=self.params['wo_da_patch_type'],
                                                  da_patch_type=self.params['da_patch_type'],
                                                  da_enhance_list=self.params['da_enhance_list'])
                      data = self.net.prepareData(X_batch, None)[0]
                  else:
                      X_batch, Y_batch = self.dataset.getXY(self.set_split,
                                                            batch_size,
                                                            normalization=self.params['normalization'],
                                                            normalization_type=self.params['normalization_type'],
                                                            meanSubstraction=self.params['mean_substraction'],
                                                            dataAugmentation=data_augmentation,
                                                            wo_da_patch_type=self.params['wo_da_patch_type'],
                                                            da_patch_type=self.params['da_patch_type'],
                                                            da_enhance_list=self.params['da_enhance_list'])
                      data = self.net.prepareData(X_batch, Y_batch)
  
              yield (data)
  
  
  class Homogeneous_Data_Batch_Generator(object):
      """
      Batch generator class. Retrieves batches of data.
      """
  
      def __init__(self,
                   set_split,
                   net,
                   dataset,
                   num_iterations,
                   batch_size=50,
                   joint_batches=20,
                   normalization=False,
                   normalization_type=None,
                   data_augmentation=True,
                   wo_da_patch_type='whole',
                   da_patch_type='resize_and_rndcrop',
                   da_enhance_list=None,
                   mean_substraction=False,
                   predict=False,
                   random_samples=-1,
                   shuffle=True):
          """
          Initializes the Data_Batch_Generator
          :param set_split: Split (train, val, test) to retrieve data
          :param net: Net which use the data
          :param dataset: Dataset instance
          :param num_iterations: Maximum number of iterations
          :param batch_size: Size of the minibatch
          :param normalization: Switches on/off the normalization of images
          :param data_augmentation: Switches on/off the data augmentation of the input
          :param mean_substraction: Switches on/off the mean substraction for images
          :param predict: Whether we are predicting or training
          :param random_samples: Retrieves this number of training samples
          :param shuffle: Shuffle the training dataset
          :param temporally_linked: Indicates if we are using a temporally-linked model
          """
          if da_enhance_list is None:
              da_enhance_list = []
  
          self.set_split = set_split
          self.dataset = dataset
          self.net = net
          self.predict = predict
          self.first_idx = -1
          self.batch_size = batch_size
          self.it = 0
  
          self.X_maxibatch = None
          self.Y_maxibatch = None
          self.tidx = None
          self.curr_idx = None
          self.batch_idx = None
          self.batch_tidx = None
          # Several parameters
          self.params = {'data_augmentation': data_augmentation,
                         'wo_da_patch_type': wo_da_patch_type,
                         'da_patch_type': da_patch_type,
                         'da_enhance_list': da_enhance_list,
                         'mean_substraction': mean_substraction,
                         'normalization': normalization,
                         'normalization_type': normalization_type,
                         'num_iterations': num_iterations / joint_batches,
                         'random_samples': random_samples,
                         'shuffle': shuffle,
                         'joint_batches': joint_batches}
          self.reset()
  
      def retrieve_maxibatch(self):
          """
          Gets a maxibatch of self.params['joint_batches'] * self.batch_size samples.
          :return:
          """
          if self.set_split == 'train' and not self.predict:
              data_augmentation = self.params['data_augmentation']
          else:
              data_augmentation = False
  
          if self.set_split == 'train' and self.it % self.params['num_iterations'] == 0 and \
                  not self.predict and self.params['random_samples'] == -1 and self.params['shuffle']:
              silence = self.dataset.silence
              self.dataset.silence = True
              self.dataset.shuffleTraining()
              self.dataset.silence = silence
          if self.it % self.params['num_iterations'] == 0 and self.params['random_samples'] == -1:
              self.dataset.resetCounters(set_name=self.set_split)
          self.it += 1
  
          # Checks if we are finishing processing the data split
          joint_batches = self.params['joint_batches']
          batch_size = self.batch_size * joint_batches
          init_sample = (self.it - 1) * batch_size
          final_sample = self.it * batch_size
          n_samples_split = getattr(self.dataset, "len_" + self.set_split)
  
          if final_sample >= n_samples_split:
              final_sample = n_samples_split
              batch_size = final_sample - init_sample
              self.it = 0
          # Recovers a batch of data
          X_batch, Y_batch = self.dataset.getXY(self.set_split,
                                                batch_size,  # This batch_size value is self.batch_size * joint_batches
                                                normalization_type=self.params['normalization_type'],
                                                normalization=self.params['normalization'],
                                                meanSubstraction=self.params['mean_substraction'],
                                                dataAugmentation=data_augmentation,
                                                wo_da_patch_type=self.params['wo_da_patch_type'],
                                                da_patch_type=self.params['da_patch_type'],
                                                da_enhance_list=self.params['da_enhance_list'])
  
          self.X_maxibatch = X_batch
          self.Y_maxibatch = Y_batch
  
      def reset(self):
          """
          Resets the counters.
          :return:
          """
          self.retrieve_maxibatch()
          text_Y_batch = self.Y_maxibatch[0][1]  # just use mask
          batch_lengths = np.asarray([int(np.sum(cc)) for cc in text_Y_batch])
          self.tidx = batch_lengths.argsort()
          self.curr_idx = 0
  
      def generator(self):
          """
          Gets and processes the data
          :return: generator with the data
          """
          while True:
              new_X = []
              new_Y = []
              next_idx = min(self.curr_idx + self.batch_size, len(self.tidx))
              self.batch_tidx = self.tidx[self.curr_idx:next_idx]
              for x_input_idx in range(len(self.X_maxibatch)):
                  x_to_add = [self.X_maxibatch[x_input_idx][i] for i in self.batch_tidx]
                  new_X.append(np.asarray(x_to_add))
  
              for y_input_idx in range(len(self.Y_maxibatch)):
                  Y_batch_ = []
                  for data_mask_idx in range(len(self.Y_maxibatch[y_input_idx])):
                      y_to_add = np.asarray([self.Y_maxibatch[y_input_idx][data_mask_idx][i] for i in self.batch_tidx])
                      Y_batch_.append(y_to_add)
                  new_Y.append(tuple(Y_batch_))
              data = self.net.prepareData(new_X, new_Y)
              self.curr_idx = next_idx
              if self.curr_idx >= len(self.tidx):
                  self.reset()
              yield (data)
  
  
  # ------------------------------------------------------- #
  #       MAIN CLASS
  # ------------------------------------------------------- #
  class Dataset(object):
      """
      Class for defining instances of databases adapted for Keras. It includes several utility functions for
      easily managing data splits, image loading, mean calculation, etc.
      """
  
      def __init__(self, name, path, pad_symbol='<pad>', unk_symbol='<unk>', null_symbol='<null>', silence=False):
          """
          Dataset initializer
          :param name: Dataset name
          :param path: Path to the folder where the images are stored
          :param silence: Verbosity
          """
          # Dataset name
          self.name = name
          # Path to the folder where the images are stored
          self.path = path
  
          # If silence = False, some informative sentences will be printed while using the "Dataset" object instance
          self.silence = silence
  
          self.pad_symbol = pad_symbol
          self.unk_symbol = unk_symbol
          self.null_symbol = null_symbol
  
          # Variable for storing external extra variables
          self.extra_variables = dict()
  
          # Data loading parameters
          # Lock for threads synchronization
          # self.__lock_read = threading.Lock()
  
          # Indicators for knowing if the data [X, Y] has been loaded for each data split
          self.loaded_train = [False, False]
          self.loaded_val = [False, False]
          self.loaded_test = [False, False]
          self.len_train = 0
          self.len_val = 0
          self.len_test = 0
  
          # Initialize dictionaries of samples
          self.X_train = dict()
          self.X_val = dict()
          self.X_test = dict()
          self.Y_train = dict()
          self.Y_val = dict()
          self.Y_test = dict()
  
          # Optionally, we point to the raw files. Note that these are not inputs/outputs of the dataset.
          # That means, we won't pre/post process the content of these files in the Dataset class.
          self.loaded_raw_train = [False, False]
          self.loaded_raw_val = [False, False]
          self.loaded_raw_test = [False, False]
  
          self.X_raw_train = dict()
          self.X_raw_val = dict()
          self.X_raw_test = dict()
          self.Y_raw_train = dict()
          self.Y_raw_val = dict()
          self.Y_raw_test = dict()
  
          #################################################
  
          # Parameters for managing all the inputs and outputs
          # List of identifiers for the inputs and outputs and their respective types
          # (which will define the preprocessing applied)
          self.ids_inputs = []
          self.types_inputs = defaultdict()  # see accepted types in self.__accepted_types_inputs
          self.inputs_data_augmentation_types = dict()  # see accepted types in self._available_augm_<input_type>
          self.optional_inputs = []
  
          self.ids_outputs = []
          self.types_outputs = defaultdict()  # see accepted types in self.__accepted_types_outputs
          self.sample_weights = dict()  # Choose whether we should compute output masks or not
  
          # List of implemented input and output data types
          self.__accepted_types_inputs = ['raw-image', 'image-features',
                                          'video', 'video-features',
                                          'text', 'text-features',
                                          'categorical', 'categorical_raw', 'binary',
                                          'id', 'ghost', 'file-name']
          self.__accepted_types_outputs = ['categorical', 'binary',
                                           'real',
                                           'text', 'dense-text', 'text-features',  # Dense text is just like text,
                                                                                   # but directly storing indices.
                                           '3DLabel', '3DSemanticLabel',
                                           'id', 'file-name']
          #    inputs/outputs with type 'id' are only used for storing external identifiers for your data
          #    they will not be used in any way. IDs must be stored in text files with a single id per line
  
          # List of implemented input normalization functions
          self.__available_norm_im_vid = ['0-1', '(-1)-1', 'inception']  # 'image' and 'video' only
          self.__available_norm_feat = ['L2']  # 'image-features' and 'video-features' only
  
          # List of implemented input data augmentation functions
          self.__available_augm_vid_feat = ['random_selection', 'noise']  # 'video-features' only
          #################################################
  
          # Parameters used for inputs/outputs of type 'text'
          self.extra_words = {self.pad_symbol: 0, self.unk_symbol: 1, self.null_symbol: 2}  # extra words introduced in all vocabularies
          self.vocabulary = dict()  # vocabularies (words2idx and idx2words)
          self.max_text_len = dict()  # number of words accepted in a 'text' sample
          self.vocabulary_len = dict()  # number of words in the vocabulary
          self.text_offset = dict()  # number of timesteps that the text is shifted (to the right)
          self.fill_text = dict()  # text padding mode
          self.label_smoothing = dict()  # Epsilon value for label smoothing. See arxiv.org/abs/1512.00567.
          self.pad_on_batch = dict()  # text padding mode: If pad_on_batch, the sample will have the maximum length
          # of the current batch. Else, it will have a fixed length (max_text_len)
          self.words_so_far = dict()  # if True, each sample will be represented as the complete set of words until
          # the point defined by the timestep dimension
          # (e.g. t=0 'a', t=1 'a dog', t=2 'a dog is', etc.)
          self.mapping = dict()  # Source -- Target predefined word mapping
          self.BPE = None  # Byte Pair Encoding instance
          self.BPE_separator = '@@'
          self.BPE_built = False
          self.moses_tokenizer = None
          self.moses_detokenizer = False
          self.moses_tokenizer_built = None
          self.moses_detokenizer_built = False
          #################################################
  
          # Parameters used for inputs of type 'video' or 'video-features'
          self.counts_frames = dict()
          self.paths_frames = dict()
          self.max_video_len = dict()
          #################################################
  
          # Parameters used for inputs of type 'image-features' or 'video-features'
          self.features_lengths = dict()
          #################################################
  
          # Parameters used for inputs of type 'raw-image'
          # Image resize dimensions used for all the returned images
          self.img_size = dict()
          # Image crop dimensions for the returned images
          self.img_size_crop = dict()
          # Training mean image
          self.train_mean = dict()
          # Whether they are RGB images (or grayscale)
          self.use_RGB = dict()
          #################################################
  
          # Parameters used for outputs of type 'categorical', '3DLabels' or '3DSemanticLabel'
          self.classes = dict()
          self.dic_classes = dict()
          #################################################
  
          # Parameters used for outputs of type '3DLabels' or '3DSemanticLabel'
          self.id_in_3DLabel = dict()
          self.num_poolings_model = dict()
          #################################################
  
          # Parameters used for outputs of type '3DSemanticLabel'
          self.semantic_classes = dict()
          #################################################
  
          # Parameters used for outputs of type 'sparse'
          self.sparse_binary = dict()
          #################################################
          # Set and reset counters to start loading data in batches
          self.last_train = 0
          self.last_val = 0
          self.last_test = 0
          self.resetCounters()
  
      def shuffleTraining(self):
          """
          Applies a random shuffling to the training samples.
          """
          if not self.silence:
              logger.info("Shuffling training samples.")
  
          # Shuffle
          num = self.len_train
          shuffled_order = random.sample([i for i in range(num)], num)
  
          # Process each input sample
          for sample_id in list(self.X_train):
              self.X_train[sample_id] = [self.X_train[sample_id][s] for s in shuffled_order]
          # Process each output sample
          for sample_id in list(self.Y_train):
              self.Y_train[sample_id] = [self.Y_train[sample_id][s] for s in shuffled_order]
  
          if not self.silence:
              logger.info("Shuffling training done.")
  
      def keepTopOutputs(self, set_name, id_out, n_top):
          """
          Keep the most frequent outputs from a set_name.
          :param set_name: Set name to modify.
          :param id_out: Id.
          :param n_top: Number of elements to keep.
          :return:
          """
          self.__checkSetName(set_name)
  
          if id_out not in self.ids_outputs:
              raise Exception("The parameter 'id_out' must specify a valid id for an output of the dataset.\n"
                              "Error produced because parameter %s was not in %s" % (id_out, self.ids_outputs))
  
          logger.info('Keeping top ' + str(n_top) + ' outputs from the ' + set_name + ' set and removing the rest.')
  
          # Sort outputs by number of occurrences
          samples = None
          samples = getattr(self, 'Y_' + set_name)
          count = Counter(samples[id_out])
          most_frequent = sorted(list(iteritems(count)), key=lambda x: x[1], reverse=True)[:n_top]
          most_frequent = [m[0] for m in most_frequent]
  
          # Select top samples
          kept = []
          for i, s in list(enumerate(samples[id_out])):
              if s in most_frequent:
                  kept.append(i)
  
          # Remove non-top samples
          # Inputs
          ids = None
          ids = list(getattr(self, 'X_' + set_name))
          for sample_id in ids:
              setattr(self, 'X_' + set_name + '[' + sample_id + ']', [getattr(self, 'X_' + set_name + '[' + sample_id + '][' + k + ']') for k in kept])
          # Outputs
          ids = list(getattr(self, 'Y_' + set_name))
          for sample_id in ids:
              setattr(self, 'Y_' + set_name + '[' + sample_id + ']', [getattr(self, 'Y_' + set_name + '[' + sample_id + '][' + k + ']') for k in kept])
  
          new_len = len(samples[id_out])
          setattr(self, 'len_' + set_name, new_len)
          self.__checkLengthSet(set_name)
  
          logger.info(str(new_len) + ' samples remaining after removal.')
  
      # ------------------------------------------------------- #
      #       GENERAL SETTERS
      #           classes list, train, val and test set, etc.
      # ------------------------------------------------------- #
  
      def resetCounters(self, set_name="all"):
          """
          Resets some basic counter indices for the next samples to read.
          """
          if set_name == "all":
              self.last_train = 0
              self.last_val = 0
              self.last_test = 0
          else:
              self.__checkSetName(set_name)
              setattr(self, 'last_' + set_name, 0)
  
      def setSilence(self, silence):
          """
          Changes the silence mode of the 'Dataset' instance.
          """
          self.silence = silence
  
      def setRawInput(self, path_list, set_name, type='file-name', id='raw-text', overwrite_split=False):
          """
          Loads a list which can contain all samples from either the 'train', 'val', or
          'test' set splits (specified by set_name).
  
          # General parameters
          :param overwrite_split:
          :param path_list: Path to a text file containing the paths to the images or a python list of paths
          :param set_name: identifier of the set split loaded ('train', 'val' or 'test')
          :param type: identifier of the type of input we are loading
                       (see self.__accepted_types_inputs for accepted types)
          :param id: identifier of the input data loaded
          """
          self.__checkSetName(set_name)
  
          # Insert type and id of input data
          keys_X_set = list(getattr(self, 'X_raw_' + set_name))
          if id not in self.ids_inputs or overwrite_split:
              self.ids_inputs.append(id)
              if id not in self.optional_inputs:
                  self.optional_inputs.append(id)  # This is always optional
          elif id in keys_X_set and not overwrite_split:
              raise Exception('An input with id "' + id + '" is already loaded into the Database.')
  
          if type not in self.__accepted_types_inputs:
              raise NotImplementedError(
                  'The input type "' + type + '" is not implemented. The list of valid types are the following: ' + str(
                      self.__accepted_types_inputs))
  
          if self.types_inputs.get(set_name) is None:
              self.types_inputs[set_name] = [type]
          else:
              self.types_inputs[set_name].append(type)
          aux_dict = getattr(self, 'X_raw_' + set_name)
          aux_dict[id] = path_list
          setattr(self, 'X_raw_' + set_name, aux_dict)
          del aux_dict
  
          aux_list = getattr(self, 'loaded_raw_' + set_name)
          aux_list[0] = True
          setattr(self, 'loaded_raw_' + set_name, aux_list)
          del aux_list
          if not self.silence:
              logger.info('Loaded "' + set_name + '" set inputs of type "' + type + '" with id "' + id + '".')
  
      def setInput(self, path_list, set_name, type='raw-image', id='image', repeat_set=1, required=True,
                   overwrite_split=False, normalization_types=None, data_augmentation_types=None,
                   add_additional=False,
                   img_size=None, img_size_crop=None, use_RGB=True,
                   # 'raw-image' / 'video'   (height, width, depth)
                   max_text_len=35, tokenization='tokenize_none', offset=0, fill='end', min_occ=0,  # 'text'
                   pad_on_batch=True, build_vocabulary=False, max_words=0, words_so_far=False,  # 'text'
                   bpe_codes=None, separator='@@', use_unk_class=False,  # 'text'
                   feat_len=1024,  # 'image-features' / 'video-features'
                   max_video_len=26,  # 'video'
                   sparse=False,  # 'binary'
                   ):
          """
          Loads a list which can contain all samples from either the 'train', 'val', or
          'test' set splits (specified by set_name).
  
          # General parameters
  
          :param use_RGB:
          :param path_list: can either be a path to a text file containing the
                            paths to the images or a python list of paths
          :param set_name: identifier of the set split loaded ('train', 'val' or 'test')
          :param type: identifier of the type of input we are loading
                       (accepted types can be seen in self.__accepted_types_inputs)
          :param id: identifier of the input data loaded
          :param repeat_set: repeats the inputs given (useful when we have more outputs than inputs).
                             Int or array of ints.
          :param required: flag for optional inputs
          :param overwrite_split: indicates that we want to overwrite the data with
                                  id that was already declared in the dataset
          :param normalization_types: type of normalization applied to the current input
                                      if we activate the data normalization while loading
          :param data_augmentation_types: type of data augmentation applied to the current
                                          input if we activate the data augmentation while loading
          :param add_additional: adds additional data to an already existent input ID
  
  
          # 'raw-image'-related parameters
  
          :param img_size: size of the input images (any input image will be resized to this)
          :param img_size_crop: size of the cropped zone (when dataAugmentation=False the central crop will be used)
  
  
          # 'text'-related parameters
  
          :param tokenization: type of tokenization applied (must be declared as a method of this class)
                               (only applicable when type=='text').
          :param build_vocabulary: whether a new vocabulary will be built from the loaded data or not
                                   (only applicable when type=='text'). A previously calculated vocabulary will be used
                                   if build_vocabulary is an 'id' from a previously loaded input/output
          :param max_text_len: maximum text length, the rest of the data will be padded with 0s
                              (only applicable if the output data is of type 'text').
          :param max_words: a maximum of 'max_words' words from the whole vocabulary will
                            be chosen by number or occurrences
          :param offset: number of timesteps that the text is shifted to the right
                        (for sequential conditional models, which take as input the previous output)
          :param fill: select whether padding before or after the sequence
          :param min_occ: minimum number of occurrences allowed for the words in the vocabulary. (default = 0)
          :param pad_on_batch: the batch timesteps size will be set to the length of the largest sample +1 if
                              True, max_len will be used as the fixed length otherwise
          :param words_so_far: if True, each sample will be represented as the complete set of words until the point
                              defined by the timestep dimension (e.g. t=0 'a', t=1 'a dog', t=2 'a dog is', etc.)
          :param bpe_codes: Codes used for applying BPE encoding.
          :param separator: BPE encoding separator.
  
          # 'image-features' and 'video-features'- related parameters
  
          :param feat_len: size of the feature vectors for each dimension.
                           We must provide a list if the features are not vectors.
  
  
          # 'video'-related parameters
          :param max_video_len: maximum video length, the rest of the data will be padded with 0s
                                (only applicable if the input data is of type 'video' or video-features').
          """
          self.__checkSetName(set_name)
          if img_size is None:
              img_size = [256, 256, 3]
  
          if img_size_crop is None:
              img_size_crop = [227, 227, 3]
          # Insert type and id of input data
          keys_X_set = list(getattr(self, 'X_' + set_name))
          if id not in self.ids_inputs:
              self.ids_inputs.append(id)
          elif id in keys_X_set and not overwrite_split and not add_additional:
              raise Exception('An input with id "' + id + '" is already loaded into the Database.')
  
          if not required and id not in self.optional_inputs:
              self.optional_inputs.append(id)
  
          if type not in self.__accepted_types_inputs:
              raise NotImplementedError('The input type "' + type +
                                        '" is not implemented. '
                                        'The list of valid types are the following: ' + str(self.__accepted_types_inputs))
          if self.types_inputs.get(set_name) is None:
              self.types_inputs[set_name] = [type]
          else:
              self.types_inputs[set_name].append(type)
  
          # Preprocess the input data depending on its type
          if type == 'raw-image':
              data = self.preprocessImages(path_list, id, set_name, img_size, img_size_crop, use_RGB)
          elif type == 'video':
              data = self.preprocessVideos(path_list, id, set_name, max_video_len, img_size, img_size_crop)
          elif type == 'text' or type == 'dense-text':
              if self.max_text_len.get(id) is None:
                  self.max_text_len[id] = dict()
              data = self.preprocessText(path_list, id, set_name, tokenization, build_vocabulary, max_text_len,
                                         max_words, offset, fill, min_occ, pad_on_batch, words_so_far,
                                         bpe_codes=bpe_codes, separator=separator, use_unk_class=use_unk_class)
          elif type == 'text-features':
              if self.max_text_len.get(id) is None:
                  self.max_text_len[id] = dict()
              data = self.preprocessTextFeatures(path_list, id, set_name, tokenization, build_vocabulary, max_text_len,
                                                 max_words, offset, fill, min_occ, pad_on_batch, words_so_far,
                                                 bpe_codes=bpe_codes, separator=separator, use_unk_class=use_unk_class)
          elif type == 'image-features':
              data = self.preprocessFeatures(path_list, id, set_name, feat_len)
          elif type == 'video-features':
              # Check if the chosen data augmentation types exists
              if data_augmentation_types is not None:
                  for da in data_augmentation_types:
                      if da not in self.__available_augm_vid_feat:
                          raise NotImplementedError(
                              'The chosen data augmentation type ' + da +
                              ' is not implemented for the type "video-features".')
              self.inputs_data_augmentation_types[id] = data_augmentation_types
              data = self.preprocessVideoFeatures(path_list, id, set_name, max_video_len, img_size, img_size_crop, feat_len)
          elif type == 'categorical':
              if build_vocabulary:
                  self.setClasses(path_list, id)
              data = self.preprocessCategorical(path_list, id)
          elif type == 'categorical_raw':
              data = self.preprocessIDs(path_list, id, set_name)
          elif type == 'binary':
              data = self.preprocessBinary(path_list, id, sparse)
          elif type == 'id':
              data = self.preprocessIDs(path_list, id, set_name)
          elif type == 'ghost':
              data = []
  
          if isinstance(repeat_set, (np.ndarray, np.generic, list)) or repeat_set > 1:
              data = list(np.repeat(data, repeat_set))
  
          self.__setInput(data, set_name, type, id, overwrite_split, add_additional)
  
      def __setInput(self, set_data, set_name, data_type, data_id, overwrite_split, add_additional):
          if add_additional:
              aux_dict = getattr(self, 'X_' + set_name)
              aux_dict[data_id] += set_data
              setattr(self, 'X_' + set_name, aux_dict)
          else:
              aux_dict = getattr(self, 'X_' + set_name)
              aux_dict[data_id] = set_data
              setattr(self, 'X_' + set_name, aux_dict)
          del aux_dict
  
          aux_list = getattr(self, 'loaded_' + set_name)
          aux_list[0] = True
          setattr(self, 'loaded_' + set_name, aux_list)
          del aux_list
  
          if data_id not in self.optional_inputs:
              setattr(self, 'len_' + set_name, len(getattr(self, 'X_' + set_name)[data_id]))
              if not overwrite_split and not add_additional:
                  self.__checkLengthSet(set_name)
  
          if not self.silence:
              logger.info(
                  'Loaded "' + set_name + '" set inputs of data_type "' + data_type + '" with data_id "' + data_id + '" and length ' + str(getattr(self, 'len_' + set_name)) + '.')
  
      def replaceInput(self, data, set_name, data_type, data_id):
          """
              Replaces the data in a certain set_name and for a given data_id
          """
          self.__setInput(data, set_name, data_type, data_id, True, False)
  
      def removeInput(self, set_name, id='label', type='categorical'):
          """
          Deletes an input from the dataset.
          :param set_name: Set name to remove.
          :param id: Input to remove id.
          :param type: Type of the input to remove.
          :return:
          """
          # Ensure that the output exists before removing it
          keys_X_set = getattr(self, 'X_' + set_name)
          if id in self.ids_inputs:
              ind_remove = self.ids_inputs.index(id)
              del self.ids_inputs[ind_remove]
              del self.types_inputs[set_name][ind_remove]
              aux_dict = getattr(self, 'X_' + set_name)
              del aux_dict[id]
              setattr(self, 'X_' + set_name, aux_dict)
              del aux_dict
  
          elif id not in keys_X_set:
              raise Exception('An input with id "' + id + '" does not exist in the Database.')
          if not self.silence:
              logger.info('Removed "' + set_name + '" set input of type "' + type + '" with id "' + id + '.')
  
      def setRawOutput(self, path_list, set_name, type='file-name', id='raw-text', overwrite_split=False,
                       add_additional=False):
          """
          Loads a list which can contain all samples from either the 'train', 'val', or
          'test' set splits (specified by set_name).
  
          # General parameters
          :param overwrite_split:
          :param add_additional:
          :param path_list: can either be a path to a text file containing the paths to
                                the images or a python list of paths
          :param set_name: identifier of the set split loaded ('train', 'val' or 'test')
          :param type: identifier of the type of input we are loading
                           (accepted types can be seen in self.__accepted_types_inputs)
          :param id: identifier of the input data loaded
          """
          self.__checkSetName(set_name)
  
          # Insert type and id of input data
          keys_Y_set = list(getattr(self, 'Y_raw_' + set_name))
          if id not in self.ids_inputs:
              self.ids_inputs.append(id)
              if id not in self.optional_inputs:
                  self.optional_inputs.append(id)  # This is always optional
  
          elif id in keys_Y_set and (not overwrite_split or not add_additional):
              raise Exception('An input with id "' + id + '" is already loaded into the Database.')
  
          if type not in self.__accepted_types_inputs:
              raise NotImplementedError(
                  'The input type "' + type + '" is not implemented. The list of valid types are the following: ' + str(
                      self.__accepted_types_inputs))
  
          if self.types_inputs.get(set_name) is None:
              self.types_inputs[set_name] = [type]
          else:
              self.types_inputs[set_name].append(type)
  
          aux_dict = getattr(self, 'Y_raw_' + set_name)
          aux_dict[id] = path_list
          setattr(self, 'Y_raw_' + set_name, aux_dict)
          del aux_dict
  
          aux_list = getattr(self, 'loaded_raw_' + set_name)
          aux_list[1] = True
          setattr(self, 'loaded_raw_' + set_name, aux_list)
          del aux_list
  
          if not self.silence:
              logger.info('Loaded "' + set_name + '" set inputs of type "' + type + '" with id "' + id + '".')
  
      def setOutput(self, path_list, set_name, type='categorical', id='label', repeat_set=1, overwrite_split=False,
                    add_additional=False, sample_weights=False, label_smoothing=0.,
                    tokenization='tokenize_none', max_text_len=0, offset=0, fill='end', min_occ=0,  # 'text'
                    pad_on_batch=True, words_so_far=False, build_vocabulary=False, max_words=0,  # 'text'
                    bpe_codes=None, separator='@@', use_unk_class=False,  # 'text'
                    associated_id_in=None, num_poolings=None,  # '3DLabel' or '3DSemanticLabel'
                    sparse=False,  # 'binary'
                    ):
          """
          Loads a set of output data.
  
          # General parameters
  
          :param path_list: can either be a path to a text file containing the labels or a python list of labels.
          :param set_name: identifier of the set split loaded ('train', 'val' or 'test').
          :param type: identifier of the type of input we are loading
                       (accepted types can be seen in self.__accepted_types_outputs).
          :param id: identifier of the input data loaded.
          :param repeat_set: repeats the outputs given
                             (useful when we have more inputs than outputs). Int or array of ints.
          :param overwrite_split: indicates that we want to overwrite
                                  the data with id that was already declared in the dataset
          :param add_additional: adds additional data to an already existent output ID
          :param sample_weights: switch on/off sample weights usage for the current output
          :param label_smoothing: epsilon value for label smoothing. See arxiv.org/abs/1512.00567.
              # 'text'-related parameters
  
          :param tokenization: type of tokenization applied (must be declared as a method of this class)
                               (only applicable when type=='text').
          :param build_vocabulary: whether a new vocabulary will be built from the loaded data or not
                                  (only applicable when type=='text').
          :param max_text_len: maximum text length, the rest of the data will be padded with 0s
                              (only applicable if the output data is of type 'text').
                               Set to 0 if the whole sentence will be used as an output class.
          :param max_words: a maximum of 'max_words' words from the whole vocabulary will
                            be chosen by number or occurrences
          :param offset: number of timesteps that the text is shifted to the right
                         (for sequential conditional models, which take as input the previous output)
          :param fill: select whether padding before or after the sequence
          :param min_occ: minimum number of occurrences allowed for the words in the vocabulary. (default = 0)
          :param pad_on_batch: the batch timesteps size will be set to the length of the largest sample +1
                               if True, max_len will be used as the fixed length otherwise
          :param words_so_far: if True, each sample will be represented as the complete set of words until the point
                               defined by the timestep dimension (e.g. t=0 'a', t=1 'a dog', t=2 'a dog is', etc.)
          :param bpe_codes: Codes used for applying BPE encoding.
          :param separator: BPE encoding separator.
  
              # '3DLabel' or '3DSemanticLabel'-related parameters
  
          :param associated_id_in: id of the input 'raw-image' associated to the inputted 3DLabels or 3DSemanticLabel
          :param num_poolings: number of pooling layers used in the model (used for calculating output dimensions)
  
              # 'binary'-related parameters
  
          :param sparse: indicates if the data is stored as a list of lists with class indices,
                         e.g. [[4, 234],[87, 222, 4568],[3],...]
  
          """
          self.__checkSetName(set_name)
  
          # Insert type and id of output data
          keys_Y_set = list(getattr(self, 'Y_' + set_name))
          if id not in self.ids_outputs:
              self.ids_outputs.append(id)
          elif id in keys_Y_set and not overwrite_split and not add_additional:
              raise Exception('An output with id "' + id + '" is already loaded into the Database.')
  
          if type not in self.__accepted_types_outputs:
              raise NotImplementedError('The output type "' + type +
                                        '" is not implemented. '
                                        'The list of valid types are the following: ' + str(self.__accepted_types_outputs))
          if self.types_outputs.get(set_name) is None:
              self.types_outputs[set_name] = [type]
          else:
              self.types_outputs[set_name].append(type)
  
          if hasattr(self, 'label_smoothing'):
              if self.label_smoothing.get(id) is None:
                  self.label_smoothing[id] = dict()
              self.label_smoothing[id][set_name] = label_smoothing
  
          # Preprocess the output data depending on its type
          if type == 'categorical':
              if build_vocabulary:
                  self.setClasses(path_list, id)
              data = self.preprocessCategorical(path_list, id,
                                                sample_weights=True if sample_weights and set_name == 'train' else False)
          elif type == 'text' or type == 'dense-text':
              if self.max_text_len.get(id) is None:
                  self.max_text_len[id] = dict()
              data = self.preprocessText(path_list, id, set_name, tokenization, build_vocabulary, max_text_len,
                                         max_words, offset, fill, min_occ, pad_on_batch, words_so_far,
                                         bpe_codes=bpe_codes, separator=separator, use_unk_class=use_unk_class)
          elif type == 'text-features':
              if self.max_text_len.get(id) is None:
                  self.max_text_len[id] = dict()
              data = self.preprocessTextFeatures(path_list, id, set_name, tokenization, build_vocabulary, max_text_len,
                                                 max_words, offset, fill, min_occ, pad_on_batch, words_so_far,
                                                 bpe_codes=bpe_codes, separator=separator, use_unk_class=use_unk_class)
          elif type == 'binary':
              data = self.preprocessBinary(path_list, id, sparse)
          elif type == 'real':
              data = self.preprocessReal(path_list)
          elif type == 'id':
              data = self.preprocessIDs(path_list, id, set_name)
          elif type == '3DLabel':
              data = self.preprocess3DLabel(path_list, id, associated_id_in, num_poolings)
          elif type == '3DSemanticLabel':
              data = self.preprocess3DSemanticLabel(path_list, id, associated_id_in, num_poolings)
  
          if isinstance(repeat_set, (np.ndarray, np.generic, list)) or repeat_set > 1:
              data = list(np.repeat(data, repeat_set))
          if self.sample_weights.get(id) is None:
              self.sample_weights[id] = dict()
          self.sample_weights[id][set_name] = sample_weights
          self.__setOutput(data, set_name, type, id, overwrite_split, add_additional)
  
      def __setOutput(self, labels, set_name, data_type, data_id, overwrite_split, add_additional):
          if add_additional:
              aux_dict = getattr(self, 'Y_' + set_name)
              aux_dict[data_id] += labels
              setattr(self, 'Y_' + set_name, aux_dict)
          else:
              aux_dict = getattr(self, 'Y_' + set_name)
              aux_dict[data_id] = labels
              setattr(self, 'Y_' + set_name, aux_dict)
          del aux_dict
  
          aux_list = getattr(self, 'loaded_' + set_name)
          aux_list[1] = True
          del aux_list
          setattr(self, 'len_' + set_name, len(getattr(self, 'Y_' + set_name)[data_id]))
          if not overwrite_split and not add_additional:
              self.__checkLengthSet(set_name)
  
          if not self.silence:
              logger.info(
                  'Loaded "' + set_name + '" set outputs of data_type "' + data_type + '" with data_id "' + data_id + '" and length ' + str(getattr(self, 'len_' + set_name)) + '.')
  
      def removeOutput(self, set_name, id='label', type='categorical'):
          """
          Deletes an output from the dataset.
          :param set_name: Set name to remove.
          :param id: Output to remove id.
          :param type: Type of the output to remove.
          :return:
          """
          # Ensure that the output exists before removing it
          keys_Y_set = list(getattr(self, 'Y_' + set_name))
          if id in self.ids_outputs:
              ind_remove = self.ids_outputs.index(id)
              del self.ids_outputs[ind_remove]
              del self.types_outputs[set_name][ind_remove]
              aux_dict = getattr(self, 'Y_' + set_name)
              del aux_dict[id]
              setattr(self, 'Y_' + set_name, aux_dict)
              del aux_dict
  
          elif id not in keys_Y_set:
              raise Exception('An output with id "' + id + '" does not exist in the Database.')
          if not self.silence:
              logger.info('Removed "' + set_name + '" set output with id "' + id + '.')
  
      # ------------------------------------------------------- #
      #       TYPE 'categorical' SPECIFIC FUNCTIONS
      # ------------------------------------------------------- #
  
      def setClasses(self, path_classes, data_id):
          """
          Loads the list of classes of the dataset.
          Each line must contain a unique identifier of the class.
  
          :param path_classes: Path to a text file with the classes or an instance of the class list.
          :param data_id: Dataset id
  
          :return: None
          """
  
          if isinstance(path_classes, str) and os.path.isfile(path_classes):
              classes = []
              with codecs.open(path_classes, 'r', encoding='utf-8') as list_:
                  for line in list_:
                      classes.append(line.rstrip('\n'))
              self.classes[data_id] = classes
          elif isinstance(path_classes, list):
              self.classes[data_id] = path_classes
          else:
              raise Exception('Wrong type for "path_classes".'
                              ' It must be a path to a text file with the classes or an instance of the class list.\n'
                              'It currently is: %s' % str(path_classes))
  
          self.dic_classes[data_id] = dict()
          for c in range(len(self.classes[data_id])):
              self.dic_classes[data_id][self.classes[data_id][c]] = c
  
          if not self.silence:
              logger.info('Loaded classes list with ' + str(len(self.dic_classes[data_id])) + " different labels.")
  
      def preprocessCategorical(self, labels_list, data_id, sample_weights=False):
          """
          Preprocesses categorical data.
  
          :param data_id:
          :param sample_weights:
          :param labels_list: Label list. Given as a path to a file or as an instance of the class list.
  
          :return: Preprocessed labels.
          """
  
          if isinstance(labels_list, str) and os.path.isfile(labels_list):
              labels = []
              with codecs.open(labels_list, 'r', encoding='utf-8') as list_:
                  for line in list_:
                      labels.append(int(line.rstrip('\n')))
          elif isinstance(labels_list, list):
              labels = labels_list
          else:
              raise Exception('Wrong type for "labels_list". '
                              'It must be a path to a text file with the labels or an instance of the class list.\n'
                              'It currently is: %s' % str(labels_list))
  
          if sample_weights:
              n_classes = len(set(labels))
              counts_per_class = np.zeros((n_classes,))
              for lab in labels:
                  counts_per_class[lab] += 1
  
              # Apply balanced weights per class
              inverse_counts_per_class = [sum(counts_per_class) - c_i for c_i in counts_per_class]
              weights_per_class = [float(c_i) / sum(inverse_counts_per_class) for c_i in inverse_counts_per_class]
              self.extra_variables['class_weights_' + data_id] = weights_per_class
  
          return labels
  
      @staticmethod
      def loadCategorical(y_raw, nClasses):
          """
          Converts a class vector (integers) to binary class matrix. From utils.
          :param y_raw: class vector to be converted into a matrix (integers from 0 to num_classes).
          :param nClasses: total number of classes.
          :return:
          """
          y = to_categorical(y_raw, nClasses).astype(np.uint8)
          return y
  
      # ------------------------------------------------------- #
      #       TYPE 'binary' SPECIFIC FUNCTIONS
      # ------------------------------------------------------- #
  
      def preprocessBinary(self, labels_list, data_id, sparse):
          """
          Preprocesses binary classes.
  
          :param data_id:
          :param labels_list: Binary label list given as an instance of the class list.
          :param sparse: indicates if the data is stored as a list of lists with class indices,
                          e.g. [[4, 234],[87, 222, 4568],[3],...]
  
          :return: Preprocessed labels.
          """
          if not isinstance(labels_list, list):
              raise Exception('Wrong type for "path_list". It must be an instance of the class list.')
  
          if sparse:
              labels = labels_list
          else:  # convert to sparse representation
              labels = [[str(i) for i, x in list(enumerate(y)) if x == 1] for y in labels_list]
          self.sparse_binary[data_id] = True
  
          unique_label_set = []
          for sample in labels:
              if sample not in unique_label_set:
                  unique_label_set.append(sample)
          y_vocab = ['::'.join(sample) for sample in unique_label_set]
  
          self.build_vocabulary(y_vocab, data_id, split_symbol='::', use_extra_words=False, is_val=True)
  
          return labels
  
      def loadBinary(self, y_raw, data_id):
          """
          Load a binary vector. May be of type 'sparse'
          :param y_raw: Vector to load.
          :param data_id: Id to load.
          :return:
          """
          try:
              sparse = self.sparse_binary[data_id]
          except Exception:  # allows backwards compatibility
              sparse = False
  
          if sparse:  # convert sparse into numpy array
              n_samples = len(y_raw)
              voc = self.vocabulary[data_id]['words2idx']
              num_words = len(list(voc))
              y = np.zeros((n_samples, num_words), dtype=np.uint8)
              for i, y_ in list(enumerate(y_raw)):
                  for elem in y_:
                      y[i, voc[elem]] = 1
          else:
              y = np.array(y_raw).astype(np.uint8)
  
          return y
  
      # ------------------------------------------------------- #
      #       TYPE 'real' SPECIFIC FUNCTIONS
      # ------------------------------------------------------- #
  
      @staticmethod
      def preprocessReal(labels_list):
          """
          Preprocesses real classes.
  
          :param labels_list: Label list. Given as a path to a file or as an instance of the class list.
  
          :return: Preprocessed labels.
          """
          if isinstance(labels_list, str) and os.path.isfile(labels_list):
              labels = []
              with codecs.open(labels_list, 'r', encoding='utf-8') as list_:
                  for line in list_:
                      labels.append(int(line.rstrip('\n')))
          elif isinstance(labels_list, list):
              labels = labels_list
          else:
              raise Exception('Wrong type for "labels_list". '
                              'It must be a path to a text file with real values or an instance of the class list.\n'
                              'It currently is: %s' % str(labels_list))
  
          return labels
  
      # ------------------------------------------------------- #
      #       TYPE 'features' SPECIFIC FUNCTIONS
      # ------------------------------------------------------- #
  
      def preprocessFeatures(self, path_list, data_id, set_name, feat_len):
          """
          Preprocesses features. We should give a path to a text file where each line must contain a path to a .npy file
          storing a feature vector. Alternatively "path_list" can be an instance of the class list.
  
          :param path_list: Path to a text file where each line must contain a path to a .npy file
                            storing a feature vector. Alternatively, instance of the class list.
          :param data_id: Dataset id
          :param set_name: Used?
          :param feat_len: Length of features. If all features have the same length, given as a number. Otherwise, list.
  
          :return: Preprocessed features
          """
          # file with a list, each line being a path to a .npy file with a feature vector
          if isinstance(path_list, str) and os.path.isfile(path_list):
              data = []
              with open(path_list, 'r') as list_:
                  for line in list_:
                      # data.append(np.fromstring(line.rstrip('\n'), sep=','))
                      data.append(line.rstrip('\n'))
          elif isinstance(path_list, list):
              data = path_list
          else:
              raise Exception(
                  'Wrong type for "path_list". It must be a path to a text file. Each line must contain a path'
                  ' to a .npy file storing a feature vector. Alternatively "path_list"'
                  ' can be an instance of the class list.\n'
                  'Currently it is: %s .' % str(path_list))
  
          if not isinstance(feat_len, list):
              feat_len = [feat_len]
          self.features_lengths[data_id] = feat_len
  
          return data
  
      def loadFeatures(self, X, feat_len, normalization_type='L2', normalization=False, loaded=False, external=False,
                       data_augmentation=True):
          """
          Loads and normalizes features.
  
          :param X: Features to load.
          :param feat_len: Length of the features.
          :param normalization_type: Normalization to perform to the features (see: self.__available_norm_feat)
          :param normalization: Whether to normalize or not the features.
          :param loaded: Flag that indicates if these features have been already loaded.
          :param external: Boolean indicating if the paths provided in 'X' are absolute paths to external images
          :param data_augmentation: Perform data augmentation (with mean=0.0, std_dev=0.01)
  
          :return: Loaded features as numpy array
          """
          if normalization and normalization_type not in self.__available_norm_feat:
              raise NotImplementedError(
                  'The chosen normalization type ' + normalization_type +
                  ' is not implemented for the type "image-features" and "video-features".')
  
          n_batch = len(X)
          features = np.zeros(tuple([n_batch] + feat_len))
  
          for i, feat in list(enumerate(X)):
              if not external:
                  feat = self.path + '/' + feat
  
              feat = np.load(feat)
  
              if data_augmentation:
                  noise_mean = 0.0
                  noise_dev = 0.01
                  noise = np.random.normal(noise_mean, noise_dev, feat.shape)
                  feat += noise
  
              if normalization:
                  if normalization_type == 'L2':
                      feat /= np.linalg.norm(feat, ord=2)
  
              features[i] = feat
  
          return np.array(features)
  
      # ------------------------------------------------------- #
      #       TYPE 'text' SPECIFIC FUNCTIONS
      # ------------------------------------------------------- #
  
      def preprocessText(self, annotations_list, data_id, set_name, tokenization, build_vocabulary, max_text_len,
                         max_words, offset, fill, min_occ, pad_on_batch, words_so_far,
                         bpe_codes=None, separator='@@', use_unk_class=False):
          """
          Preprocess 'text' data type: Builds vocabulary (if necessary) and preprocesses the sentences.
          Also sets Dataset parameters.
  
          :param annotations_list: Path to the sentences to process.
          :param data_id: Dataset id of the data.
          :param set_name: Name of the current set ('train', 'val', 'test')
          :param tokenization: Tokenization to perform.
          :param build_vocabulary: Whether we should build a vocabulary for this text or not.
          :param max_text_len: Maximum length of the text. If max_text_len == 0, we treat the full sentence as a class.
          :param max_words: Maximum number of words to include in the dictionary.
          :param offset: Text shifting.
          :param fill: Whether we path with zeros at the beginning or at the end of the sentences.
          :param min_occ: Minimum occurrences of each word to be included in the dictionary.
          :param pad_on_batch: Whether we get sentences with length of the maximum length of the
                               minibatch or sentences with a fixed (max_text_length) length.
          :param words_so_far: Experimental feature. Should be ignored.
          :param bpe_codes: Codes used for applying BPE encoding.
          :param separator: BPE encoding separator.
          :param use_unk_class: Add a special class for the unknown word when maxt_text_len == 0.
  
          :return: Preprocessed sentences.
          """
          sentences = []
          if isinstance(annotations_list, str) and os.path.isfile(annotations_list):
              with codecs.open(annotations_list, 'r', encoding='utf-8') as list_:
                  for line in list_:
                      sentences.append(line.rstrip('\n'))
          elif isinstance(annotations_list, list):
              sentences = annotations_list
          else:
              raise Exception(
                  'Wrong type for "annotations_list". '
                  'It must be a path to a text file with the sentences or a list of sentences. '
                  'It currently is: %s' % (str(annotations_list)))
  
          # Tokenize sentences
          if max_text_len != 0:  # will only tokenize if we are not using the whole sentence as a class
              # Check if tokenization method exists
              if hasattr(self, tokenization):
                  if 'bpe' in tokenization.lower():
                      if bpe_codes is None:
                          raise AssertionError('bpe_codes must be specified when applying a BPE tokenization.')
                      self.build_bpe(bpe_codes, separator=separator)
                  tokfun = eval('self.' + tokenization)
                  if not self.silence:
                      logger.info('\tApplying tokenization function: "' + tokenization + '".')
              else:
                  raise Exception('Tokenization procedure "' + tokenization + '" is not implemented.')
  
              for i, sentence in enumerate(sentences):
                  sentences[i] = tokfun(sentence)
          else:
              tokfun = None
  
          # Build vocabulary
          if build_vocabulary:
              self.build_vocabulary(sentences, data_id,
                                    max_text_len != 0,
                                    min_occ=min_occ,
                                    n_words=max_words,
                                    use_extra_words=(max_text_len != 0),
                                    use_unk_class=use_unk_class)
          elif isinstance(build_vocabulary, str):
              if build_vocabulary in self.vocabulary:
                  self.vocabulary[data_id] = self.vocabulary[build_vocabulary]
                  self.vocabulary_len[data_id] = self.vocabulary_len[build_vocabulary]
                  if not self.silence:
                      logger.info('\tReusing vocabulary named "' + build_vocabulary + '" for data with data_id "' + data_id + '".')
              else:
                  raise Exception('The parameter "build_vocabulary" must be a boolean '
                                  'or a str containing an data_id of the vocabulary we want to copy.\n'
                                  'It currently is: %s' % str(build_vocabulary))
  
          elif isinstance(build_vocabulary, dict):
              self.vocabulary[data_id] = build_vocabulary
              if not self.silence:
                  logger.info('\tReusing vocabulary from dictionary for data with data_id "' + data_id + '".')
  
          if data_id not in self.vocabulary:
              raise Exception('The dataset must include a vocabulary with data_id "' + data_id +
                              '" in order to process the type "text" data. Set "build_vocabulary" to True if you want to use the current data for building the vocabulary.')
  
          # Store max text len
          self.max_text_len[data_id][set_name] = max_text_len
          self.text_offset[data_id] = offset
          self.fill_text[data_id] = fill
          self.pad_on_batch[data_id] = pad_on_batch
          self.words_so_far[data_id] = words_so_far
  
          return sentences
  
      def preprocessTextFeatures(self, annotations_list, data_id, set_name, tokenization, build_vocabulary, max_text_len,
                                 max_words, offset, fill, min_occ, pad_on_batch, words_so_far, bpe_codes=None, separator='@@', use_unk_class=False):
          """
          Preprocess 'text' data type: Builds vocabulary (if necessary) and preprocesses the sentences.
          Also sets Dataset parameters.
  
          :param annotations_list: Path to the sentences to process.
          :param data_id: Dataset id of the data.
          :param set_name: Name of the current set ('train', 'val', 'test')
          :param tokenization: Tokenization to perform.
          :param build_vocabulary: Whether we should build a vocabulary for this text or not.
          :param max_text_len: Maximum length of the text. If max_text_len == 0, we treat the full sentence as a class.
          :param max_words: Maximum number of words to include in the dictionary.
          :param offset: Text shifting.
          :param fill: Whether we path with zeros at the beginning or at the end of the sentences.
          :param min_occ: Minimum occurrences of each word to be included in the dictionary.
          :param pad_on_batch: Whether we get sentences with length of the maximum length of the
                               minibatch or sentences with a fixed (max_text_length) length.
          :param words_so_far: Experimental feature. Should be ignored.
          :param bpe_codes: Codes used for applying BPE encoding.
          :param separator: BPE encoding separator.
  
          :return: Preprocessed sentences.
          """
          sentences = []
          if isinstance(annotations_list, str) and os.path.isfile(annotations_list):
              with codecs.open(annotations_list, 'r', encoding='utf-8') as list_:
                  for line in list_:
                      sentences.append(line.rstrip('\n'))
          elif isinstance(annotations_list, list):
              sentences = annotations_list
          else:
              raise Exception(
                  'Wrong type for "annotations_list". '
                  'It must be a path to a text file with the sentences or a list of sentences. '
                  'It currently is: %s' % (str(annotations_list)))
  
          # Tokenize sentences
          if max_text_len != 0:  # will only tokenize if we are not using the whole sentence as a class
              # Check if tokenization method exists
              if hasattr(self, tokenization):
                  if 'bpe' in tokenization.lower():
                      if bpe_codes is None:
                          raise AssertionError('bpe_codes must be specified when applying a BPE tokenization.')
                      self.build_bpe(bpe_codes, separator=separator)
                  tokfun = eval('self.' + tokenization)
                  if not self.silence:
                      logger.info('\tApplying tokenization function: "' + tokenization + '".')
              else:
                  raise Exception('Tokenization procedure "' + tokenization + '" is not implemented.')
  
              for sentence_idx, sentence in enumerate(sentences):
                  sentences[sentence_idx] = tokfun(sentence)
          else:
              tokfun = None
  
          # Build vocabulary
          if build_vocabulary:
              self.build_vocabulary(sentences, data_id, max_text_len != 0,
                                    min_occ=min_occ,
                                    n_words=max_words,
                                    use_extra_words=(max_text_len != 0),
                                    use_unk_class=use_unk_class)
          elif isinstance(build_vocabulary, str):
              if build_vocabulary in self.vocabulary:
                  self.vocabulary[data_id] = self.vocabulary[build_vocabulary]
                  self.vocabulary_len[data_id] = self.vocabulary_len[build_vocabulary]
                  if not self.silence:
                      logger.info('\tReusing vocabulary named "' + build_vocabulary + '" for data with data_id "' + data_id + '".')
              else:
                  raise Exception('The parameter "build_vocabulary" must be a boolean '
                                  'or a str containing an data_id of the vocabulary we want to copy.\n'
                                  'It currently is: %s' % str(build_vocabulary))
  
          elif isinstance(build_vocabulary, dict):
              self.vocabulary[data_id] = build_vocabulary
              if not self.silence:
                  logger.info('\tReusing vocabulary from dictionary for data with data_id "' + data_id + '".')
  
          if data_id not in self.vocabulary:
              raise Exception('The dataset must include a vocabulary with data_id "' + data_id +
                              '" in order to process the type "text" data. Set "build_vocabulary" to True if you want to use the current data for building the vocabulary.')
  
          # Store max text len
          self.max_text_len[data_id][set_name] = max_text_len
          self.text_offset[data_id] = offset
          self.fill_text[data_id] = fill
          self.pad_on_batch[data_id] = pad_on_batch
          self.words_so_far[data_id] = words_so_far
  
          # Max values per uint type:
          # uint8.max: 255
          # uint16.max: 65535
          # uint32.max: 4294967295
          if self.vocabulary_len[data_id] < 255:
              dtype_text = 'uint8'
          elif self.vocabulary_len[data_id] < 65535:
              dtype_text = 'uint16'
          else:
              dtype_text = 'uint32'
          vocab = self.vocabulary[data_id]['words2idx']
          sentence_features = np.ones((len(sentences), max_text_len)).astype(dtype_text) * self.extra_words[self.pad_symbol]
          max_text_len -= 1  # always leave space for <eos> symbol
          for sentence_idx, sentence in enumerate(sentences):
              words = sentence.strip().split()
              len_j = len(words)
              if fill == 'start':
                  offset_j = max_text_len - len_j - 1
              elif fill == 'center':
                  offset_j = (max_text_len - len_j) / 2
                  len_j += offset_j
              else:
                  offset_j = 0
                  len_j = min(len_j, max_text_len)
              if offset_j < 0:
                  len_j += offset_j
                  offset_j = 0
  
              for word_idx, word in list(zip(range(len_j), words[:len_j])):
                  sentence_features[sentence_idx, word_idx + offset_j] = vocab.get(word, vocab[self.unk_symbol])
              if offset > 0:  # Move the text to the right -> null symbol
                  sentence_features[sentence_idx] = np.append([vocab[self.null_symbol]] * offset, sentence_features[sentence_idx, :-offset])
          return sentence_features
  
      def build_vocabulary(self, captions, data_id, do_split=True, min_occ=0, n_words=0, split_symbol=' ',
                           use_extra_words=True, use_unk_class=False, is_val=False):
          """
          Vocabulary builder for data of type 'text'
  
          :param use_extra_words:
          :param captions: Corpus sentences
          :param data_id: Dataset id of the text
          :param do_split: Split sentence by words or use the full sentence as a class.
          :param split_symbol: symbol used for separating the elements in each sentence
          :param min_occ: Minimum occurrences of each word to be included in the dictionary.
          :param n_words: Maximum number of words to include in the dictionary.
          :param is_val: Set to True if the input 'captions' are values and we want to keep them sorted
          :return: None.
          """
          if not self.silence:
              logger.info("Creating vocabulary for data with data_id '" + data_id + "'.")
  
          counters = []
          sentence_counts = []
          counter = Counter()
          sentence_count = 0
          for line in captions:
              if do_split:
                  words = line.strip().split(split_symbol)
                  counter.update(words)
              else:
                  counter.update([line])
              sentence_count += 1
  
          if not do_split and not self.silence:
              logger.info('Using whole sentence as a single word.')
  
          counters.append(counter)
          sentence_counts.append(sentence_count)
          combined_counter = reduce(add, counters)
          if not self.silence:
              logger.info("\t Total: %d unique words in %d sentences with a total of %d words." %
                          (len(combined_counter), sum(sentence_counts), sum(list(combined_counter.values()))))
  
          # keep only words with less than 'min_occ' occurrences
          if min_occ > 1:
              removed = 0
              for k in list(combined_counter):
                  if combined_counter[k] < min_occ:
                      del combined_counter[k]
                      removed += 1
              if not self.silence:
                  logger.info("\t Removed %d words with less than %d occurrences. New total: %d." %
                              (removed, min_occ, len(combined_counter)))
  
          # keep only top 'n_words'
          if n_words > 0:
              if use_extra_words:
                  vocab_count = combined_counter.most_common(n_words - len(self.extra_words))
              else:
                  vocab_count = combined_counter.most_common(n_words)
              if not self.silence:
                  logger.info("Creating dictionary of %s most common words, covering %2.1f%% of the text."
                              % (n_words, 100.0 * sum([count for word, count in vocab_count]) / sum(list(combined_counter.values()))))
          else:
              if not self.silence:
                  logger.info("Creating dictionary of all words")
              vocab_count = counter.most_common()
  
          dictionary = {}
          for i, (word, count) in list(enumerate(vocab_count)):
              if is_val:
                  dictionary[word] = int(word)
              else:
                  dictionary[word] = i
              if use_extra_words:
                  dictionary[word] += len(self.extra_words)
              elif use_unk_class:
                  if i >= self.extra_words[self.unk_symbol]:
                      dictionary[word] += 1
  
          if use_extra_words:
              for w, k in iteritems(self.extra_words):
                  dictionary[w] = k
          elif use_unk_class:
              if not self.silence:
                  logger.info("\tAdding an additional unknown class")
              dictionary[self.unk_symbol] = self.extra_words[self.unk_symbol]
  
          # Store dictionary and append to previously existent if needed.
          if data_id not in self.vocabulary:
              self.vocabulary[data_id] = dict()
              self.vocabulary[data_id]['words2idx'] = dictionary
              inv_dictionary = {v: k for k, v in list(iteritems(dictionary))}
              self.vocabulary[data_id]['idx2words'] = inv_dictionary
  
              self.vocabulary_len[data_id] = len(vocab_count)
              if use_extra_words:
                  self.vocabulary_len[data_id] += len(self.extra_words)
              elif use_unk_class:
                  self.vocabulary_len[data_id] += 1
  
          else:
              old_keys = list(self.vocabulary[data_id]['words2idx'])
              new_keys = list(dictionary)
              added = 0
              for key in new_keys:
                  if key not in old_keys:
                      self.vocabulary[data_id]['words2idx'][key] = self.vocabulary_len[data_id]
                      self.vocabulary_len[data_id] += 1
                      added += 1
  
              inv_dictionary = {v: k for k, v in list(iteritems(self.vocabulary[data_id]['words2idx']))}
              self.vocabulary[data_id]['idx2words'] = inv_dictionary
  
              if not self.silence:
                  logger.info('Appending ' + str(added) + ' words to dictionary with data_id "' + data_id + '".')
                  logger.info('\tThe new total is ' + str(self.vocabulary_len[data_id]) + '.')
  
      def merge_vocabularies(self, ids):
          """
          Merges the vocabularies from a set of text inputs/outputs into a single one.
  
          :param ids: identifiers of the inputs/outputs whose vocabularies will be merged
          :return: None
          """
          if not isinstance(ids, list):
              raise AssertionError('ids must be a list of inputs/outputs identifiers of type text')
          if not self.silence:
              logger.info('Merging vocabularies of the following ids: ' + str(ids))
  
          # Pick the first vocabulary as reference
          vocab_ref = self.vocabulary[ids[0]]['words2idx']
          next_idx = max(list(vocab_ref.values())) + 1
  
          # Merge all vocabularies to the reference
          for i in range(1, len(ids)):
              current_data_id = ids[i]
              vocab = self.vocabulary[current_data_id]['words2idx']
              for w in list(vocab):
                  if w not in list(vocab_ref):
                      vocab_ref[w] = next_idx
                      next_idx += 1
  
          # Also build idx2words
          self.vocabulary[ids[0]]['words2idx'] = vocab_ref
          inv_dictionary = {v: k for k, v in list(iteritems(vocab_ref))}
          self.vocabulary[ids[0]]['idx2words'] = inv_dictionary
          self.vocabulary_len[ids[0]] = len(list(self.vocabulary[ids[0]]['words2idx']))
  
          # Insert in all ids
          for i in range(1, len(ids)):
              self.vocabulary[ids[i]]['words2idx'] = self.vocabulary[ids[0]]['words2idx']
              self.vocabulary[ids[i]]['idx2words'] = self.vocabulary[ids[0]]['idx2words']
              self.vocabulary_len[ids[i]] = self.vocabulary_len[ids[0]]
  
          if not self.silence:
              logger.info('\tThe new total is ' + str(self.vocabulary_len[ids[0]]) + '.')
  
      def build_bpe(self, codes, merges=-1, separator=u'@@', vocabulary=None, glossaries=None):
          """
          Constructs a BPE encoder instance. Currently, vocabulary and glossaries options are not implemented.
          :param codes: File with BPE codes (created by learn_bpe.py)
          :param separator: Separator between non-final subword units (default: '@@'))
          :param vocabulary: Vocabulary file. If provided, this script reverts any merge operations that produce an OOV.
          :param glossaries: The strings provided in glossaries will not be affected
                             by the BPE (i.e. they will neither be broken into subwords,
                             nor concatenated with other subwords.
          :return: None
          """
          from keras_wrapper.extra.external import BPE
          with codecs.open(codes, 'rb', encoding='utf-8') as cods:
              self.BPE = BPE(cods, merges=merges, separator=separator, vocab=vocabulary, glossaries=glossaries)
          self.BPE_separator = separator
          self.BPE_built = True
  
      def build_moses_tokenizer(self, language='en'):
          """
          Constructs a Moses tokenizer instance.
          :param language: Tokenizer language.
          :return: None
          """
          from sacremoses import MosesTokenizer
          self.moses_tokenizer = MosesTokenizer(lang=language)
          self.moses_tokenizer_built = True
  
      def build_moses_detokenizer(self, language='en'):
          """
          Constructs a BPE encoder instance. Currently, vocabulary and glossaries options are not implemented.
          :param codes: File with BPE codes (created by learn_bpe.py)
          :param separator: Separator between non-final subword units (default: '@@'))
          :param vocabulary: Vocabulary file. If provided, this script reverts any merge operations that produce an OOV.
          :param glossaries: The strings provided in glossaries will not be affected
                             by the BPE (i.e. they will neither be broken into subwords,
                             nor concatenated with other subwords.
          :return: None
          """
          from sacremoses import MosesDetokenizer
          self.moses_detokenizer = MosesDetokenizer(lang=language)
          self.moses_detokenizer_built = True
  
      def apply_label_smoothing(self, y, discount, vocabulary_len, discount_type='uniform'):
          """
          Applies label smoothing to a one-hot codified vector.
          :param y_text: Input to smooth
          :param discount: Discount to apply
          :param vocabulary_len: Length of the one-hot vectors
          :param discount_type: Type of smoothing. Types supported:
              'uniform': Subtract a 'label_smoothing_discount' from the label and distribute it uniformly among all labels.
          :return:
          """
          # if discount_type == 'uniform': # Currently, only 'uniform' discount_type is implemented.
          y = ((1 - discount) * y + (discount / vocabulary_len))
          return y
  
      @staticmethod
      def load3DLabels(bbox_list, nClasses, dataAugmentation, daRandomParams, img_size, size_crop, image_list):
          """
          Loads a set of outputs of the type 3DLabel (used for detection)
  
          :param bbox_list: list of bboxes, labels and original sizes
          :param nClasses: number of different classes to be detected
          :param dataAugmentation: are we applying data augmentation?
          :param daRandomParams: random parameters applied on data augmentation (vflip, hflip and random crop)
          :param img_size: resized applied to input images
          :param size_crop: crop size applied to input images
          :param image_list: list of input images used as identifiers to 'daRandomParams'
          :return: 3DLabels with shape (batch_size, width*height, classes)
          """
          from scipy import misc
  
          n_samples = len(bbox_list)
          h, w, d = img_size
          h_crop, w_crop, d_crop = size_crop
          labels = np.zeros((n_samples, nClasses, h_crop, w_crop), dtype=np.float32)
  
          for i in range(n_samples):
              line = bbox_list[i]
              arrayLine = line.split(';')
              arrayBndBox = arrayLine[:-1]
              w_original, h_original, d_original = eval(arrayLine[-1])
  
              label3D = np.zeros((nClasses, h_original, w_original), dtype=np.float32)
  
              for array in arrayBndBox:
                  bndbox = eval(array)[0]
                  idxclass = eval(array)[1]
  
                  # bndbox_ones = np.ones((bndbox[3] - bndbox[1] + 1, bndbox[2] - bndbox[0] + 1))
                  # label3D[idxclass, bndbox[1] - 1:bndbox[3], bndbox[0] - 1:bndbox[2]] = bndbox_ones
  
                  bndbox_ones = np.ones((bndbox[2] - bndbox[0] + 1, bndbox[3] - bndbox[1] + 1))
                  label3D[idxclass, bndbox[0] - 1:bndbox[2], bndbox[1] - 1:bndbox[3]] = bndbox_ones
  
              if not dataAugmentation or daRandomParams is None:
                  # Resize 3DLabel to crop size.
                  for j in range(nClasses):
                      label2D = misc.imresize(label3D[j], (h_crop, w_crop))
                      maxval = np.max(label2D)
                      if maxval > 0:
                          label2D /= maxval
                      labels[i, j] = label2D
              else:
                  label3D_rs = np.zeros((nClasses, h_crop, w_crop), dtype=np.float32)
                  # Crop the labels (random crop)
                  for j in range(nClasses):
                      label2D = misc.imresize(label3D[j], (h, w))
                      maxval = np.max(label2D)
                      if maxval > 0:
                          label2D /= maxval
                      randomParams = daRandomParams[image_list[i]]
                      # Take random crop
                      left = randomParams["left"]
                      right = np.add(left, size_crop[0:2])
  
                      label2D = label2D[left[0]:right[0], left[1]:right[1]]
  
                      # Randomly flip (with a certain probability)
                      flip = randomParams["hflip"]
                      prob_flip_horizontal = randomParams["prob_flip_horizontal"]
                      if flip < prob_flip_horizontal:  # horizontal flip
                          label2D = np.fliplr(label2D)
                      flip = randomParams["vflip"]
                      prob_flip_vertical = randomParams["prob_flip_vertical"]
                      if flip < prob_flip_vertical:  # vertical flip
                          label2D = np.flipud(label2D)
  
                      label3D_rs[j] = label2D
  
                  labels[i] = label3D_rs
  
          # Reshape labels to (batch_size, width*height, classes) before returning
          labels = np.reshape(labels, (n_samples, nClasses, w_crop * h_crop))
          labels = np.transpose(labels, (0, 2, 1))
  
          return labels
  
      def load3DSemanticLabels(self, labeled_images_list, nClasses, classes_to_colour, dataAugmentation, daRandomParams,
                               img_size, size_crop, image_list):
          """
          Loads a set of outputs of the type 3DSemanticLabel (used for semantic segmentation TRAINING)
  
          :param labeled_images_list: list of labeled images
          :param nClasses: number of different classes to be detected
          :param classes_to_colour: dictionary relating each class id to their corresponding colour in the labeled image
          :param dataAugmentation: are we applying data augmentation?
          :param daRandomParams: random parameters applied on data augmentation (vflip, hflip and random crop)
          :param img_size: resized applied to input images
          :param size_crop: crop size applied to input images
          :param image_list: list of input images used as identifiers to 'daRandomParams'
          :return: 3DSemanticLabels with shape (batch_size, width*height, classes)
          """
          from PIL import Image as pilimage
          from scipy import misc
  
          n_samples = len(labeled_images_list)
          h, w, d = img_size
          h_crop, w_crop, d_crop = size_crop
          labels = np.zeros((n_samples, nClasses, h_crop, w_crop), dtype=np.float32)
  
          for i in range(n_samples):
              line = labeled_images_list[i].rstrip('\n')
  
              # Load labeled GT image
              labeled_im = self.path + '/' + line
              # Check if the filename includes the extension
              [path, filename] = ntpath.split(labeled_im)
              [filename, ext] = os.path.splitext(filename)
              # If it doesn't then we find it
              if not ext:
                  filename = fnmatch.filter(os.listdir(path), filename + '*')
                  if not filename:
                      raise Exception('Non existent image ' + labeled_im)
                  else:
                      labeled_im = path + '/' + filename[0]
              # Read image
              try:
                  logging.disable(logging.CRITICAL)
                  labeled_im = pilimage.open(labeled_im)
                  labeled_im = np.asarray(labeled_im)
                  logging.disable(logging.NOTSET)
                  labeled_im = misc.imresize(labeled_im, (h, w))
              except Exception:
                  logger.info(labeled_im)
                  logger.warning("WARNING!")
                  logger.warning("Can't load image " + labeled_im)
                  labeled_im = np.zeros((h, w))
  
              label3D = np.zeros((nClasses, h, w), dtype=np.float32)
  
              # Insert 1s in the corresponding positions for each class
              for class_id, colour in iteritems(classes_to_colour):
                  # indices = np.where(np.all(labeled_im == colour, axis=-1))
                  indices = np.where(labeled_im == class_id)
                  num_vals = len(indices[0])
                  if num_vals > 0:
                      for idx_pos in range(num_vals):
                          x, y = indices[0][idx_pos], indices[1][idx_pos]
                          label3D[class_id, x, y] = 1.
  
              if not dataAugmentation or daRandomParams is None:
                  # Resize 3DLabel to crop size.
                  for j in range(nClasses):
                      label2D = misc.imresize(label3D[j], (h_crop, w_crop))
                      maxval = np.max(label2D)
                      if maxval > 0:
                          label2D /= maxval
                      labels[i, j] = label2D
              else:
                  label3D_rs = np.zeros((nClasses, h_crop, w_crop), dtype=np.float32)
                  # Crop the labels (random crop)
                  for j in range(nClasses):
                      label2D = misc.imresize(label3D[j], (h, w))
                      maxval = np.max(label2D)
                      if maxval > 0:
                          label2D /= maxval
                      randomParams = daRandomParams[image_list[i]]
                      # Take random crop
                      left = randomParams["left"]
                      right = np.add(left, size_crop[0:2])
  
                      label2D = label2D[left[0]:right[0], left[1]:right[1]]
  
                      # Randomly flip (with a certain probability)
                      flip = randomParams["hflip"]
                      prob_flip_horizontal = randomParams["prob_flip_horizontal"]
                      if flip < prob_flip_horizontal:  # horizontal flip
                          label2D = np.fliplr(label2D)
                      flip = randomParams["vflip"]
                      prob_flip_vertical = randomParams["prob_flip_vertical"]
                      if flip < prob_flip_vertical:  # vertical flip
                          label2D = np.flipud(label2D)
  
                      label3D_rs[j] = label2D
  
                  labels[i] = label3D_rs
  
          # Reshape labels to (batch_size, width*height, classes) before returning
          labels = np.reshape(labels, (n_samples, nClasses, w_crop * h_crop))
          labels = np.transpose(labels, (0, 2, 1))
  
          return labels
  
      def loadText(self, X, vocabularies, max_len, offset, fill, pad_on_batch, words_so_far, loading_X=False):
          """
          Text encoder: Transforms samples from a text representation into a numerical one. It also masks the text.
  
          :param X: Text to encode.
          :param vocabularies: Mapping word -> index
          :param max_len: Maximum length of the text.
          :param offset: Shifts the text to the right, adding null symbol at the start
          :param fill: 'start': the resulting vector will be filled with 0s at the beginning.
                       'end': it will be filled with 0s at the end.
                       'center': the vector will be surrounded by 0s, both at beginning and end.
          :param pad_on_batch: Whether we get sentences with length of the maximum length of the minibatch or
                               sentences with a fixed (max_text_length) length.
          :param words_so_far: Experimental feature. Use with caution.
          :param loading_X: Whether we are loading an input or an output of the model
          :return: Text as sequence of number. Mask for each sentence.
          """
          vocab = vocabularies['words2idx']
          n_batch = len(X)
  
          # Max values per uint type:
          # uint8.max: 255
          # uint16.max: 65535
          # uint32.max: 4294967295
          vocabulary_size = len(list(vocab))
  
          if vocabulary_size < 255:
              dtype_text = 'uint8'
          elif vocabulary_size < 65535:
              dtype_text = 'uint16'
          else:
              dtype_text = 'uint32'
  
          if max_len == 0:  # use whole sentence as class
              X_out = np.zeros(n_batch).astype(dtype_text)
              for sentence_idx in range(n_batch):
                  word = X[sentence_idx]
                  if self.unk_symbol in vocab:
                      X_out[sentence_idx] = vocab.get(word, vocab[self.unk_symbol])
                  else:
                      X_out[sentence_idx] = vocab[word]
              if loading_X:
                  X_out = (X_out, None)  # This None simulates a mask
          else:  # process text as a sequence of words
              if pad_on_batch:
                  max_len_batch = min(max([len(words.split(' ')) for words in X]) + 1, max_len)
              else:
                  max_len_batch = max_len
  
              if words_so_far:
                  X_out = np.ones((n_batch, max_len_batch, max_len_batch)).astype(dtype_text) * self.extra_words[self.pad_symbol]
                  X_mask = np.zeros((n_batch, max_len_batch, max_len_batch)).astype('int8')
                  null_row = np.ones((1, max_len_batch)).astype(dtype_text) * self.extra_words[self.pad_symbol]
                  zero_row = np.zeros((1, max_len_batch)).astype('int8')
                  if offset > 0:
                      null_row[0] = np.append([vocab[self.null_symbol]] * offset, null_row[0, :-offset])
              else:
                  X_out = np.ones((n_batch, max_len_batch)).astype(dtype_text) * self.extra_words[self.pad_symbol]
                  X_mask = np.zeros((n_batch, max_len_batch)).astype('int8')
  
              max_len_batch -= 1  # always leave space for <eos> symbol
  
              # fills text vectors with each word (fills with 0s or removes remaining words w.r.t. max_len)
              for sentence_idx in range(n_batch):
                  words = X[sentence_idx].strip().split(' ')
                  len_j = len(words)
                  if fill == 'start':
                      offset_j = max_len_batch - len_j - 1
                  elif fill == 'center':
                      offset_j = (max_len_batch - len_j) / 2
                      len_j += offset_j
                  else:
                      offset_j = 0
                      len_j = min(len_j, max_len_batch)
                  if offset_j < 0:
                      len_j += offset_j
                      offset_j = 0
  
                  if words_so_far:
                      for word_idx, word in list(zip(range(len_j), words[:len_j])):
                          next_w = vocab.get(word, next_w=vocab[self.unk_symbol])
                          for k in range(word_idx, len_j):
                              X_out[sentence_idx, k + offset_j, word_idx + offset_j] = next_w
                              X_mask[sentence_idx, k + offset_j, word_idx + offset_j] = 1  # fill mask
                          X_mask[sentence_idx, word_idx + offset_j, word_idx + 1 + offset_j] = 1  # add additional 1 for the <eos> symbol
  
                  else:
                      for word_idx, word in list(zip(range(len_j), words[:len_j])):
                          X_out[sentence_idx, word_idx + offset_j] = vocab.get(word, vocab[self.unk_symbol])
                          X_mask[sentence_idx, word_idx + offset_j] = 1  # fill mask
                      X_mask[sentence_idx, len_j + offset_j] = 1  # add additional 1 for the <eos> symbol
  
                  if offset > 0:  # Move the text to the right -> null symbol
                      if words_so_far:
                          for k in range(len_j):
                              X_out[sentence_idx, k] = np.append([vocab[self.null_symbol]] * offset, X_out[sentence_idx, k, :-offset])
                              X_mask[sentence_idx, k] = np.append([0] * offset, X_mask[sentence_idx, k, :-offset])
                          X_out[sentence_idx] = np.append(null_row, X_out[sentence_idx, :-offset], axis=0)
                          X_mask[sentence_idx] = np.append(zero_row, X_mask[sentence_idx, :-offset], axis=0)
                      else:
                          X_out[sentence_idx] = np.append([vocab[self.null_symbol]] * offset, X_out[sentence_idx, :-offset])
                          X_mask[sentence_idx] = np.append([1] * offset, X_mask[sentence_idx, :-offset])
              X_out = (np.asarray(X_out, dtype=dtype_text), np.asarray(X_mask, dtype='int8'))
  
          return X_out
  
      def loadTextOneHot(self,
                         X,
                         vocabularies,
                         vocabulary_len,
                         max_len,
                         offset,
                         fill,
                         pad_on_batch,
                         words_so_far,
                         sample_weights=False,
                         loading_X=False,
                         label_smoothing=0.):
  
          """
          Text encoder: Transforms samples from a text representation into a one-hot. It also masks the text.
  
          :param vocabulary_len:
          :param sample_weights:
          :param X: Text to encode.
          :param vocabularies: Mapping word -> index
          :param max_len: Maximum length of the text.
          :param offset: Shifts the text to the right, adding null symbol at the start
          :param fill: 'start': the resulting vector will be filled with 0s at the beginning.
                       'end': it will be filled with 0s at the end.
                       'center': the vector will be surrounded by 0s, both at beginning and end.
          :param pad_on_batch: Whether we get sentences with length of the maximum length of
                               the minibatch or sentences with a fixed (max_text_length) length.
          :param words_so_far: Experimental feature. Use with caution.
          :param loading_X: Whether we are loading an input or an output of the model
          :return: Text as sequence of one-hot vectors. Mask for each sentence.
          """
  
          y = self.loadText(X, vocabularies, max_len, offset, fill, pad_on_batch,
                            words_so_far, loading_X=loading_X)
          # Use whole sentence as class (classifier model)
          if max_len == 0:
              y_aux = to_categorical(y, vocabulary_len).astype(np.uint8)
          # Use words separately (generator model)
          else:
              if label_smoothing > 0.:
                  y_aux_type = np.float32
              else:
                  y_aux_type = np.uint8
              y_aux = np.zeros(list(y[0].shape) + [vocabulary_len]).astype(y_aux_type)
              for idx in range(y[0].shape[0]):
                  y_aux[idx] = to_categorical(y[0][idx], vocabulary_len).astype(y_aux_type)
              if label_smoothing > 0.:
                  y_aux = self.apply_label_smoothing(y_aux, label_smoothing, vocabulary_len)
              if sample_weights:
                  y_aux = (y_aux, y[1])  # join data and mask
          return y_aux
  
      def loadTextFeatures(self, X, max_len, pad_on_batch, offset):
          """
          Text encoder: Transforms samples from a text representation into a numerical one. It also masks the text.
  
          :param X: Encoded text.
          :param max_len: Maximum length of the text.
          :param pad_on_batch: Whether we get sentences with length of the maximum length of the minibatch or
                               sentences with a fixed (max_text_length) length.
          :return: Text as sequence of numbers. Mask for each sentence.
          """
          X_out = np.asarray(X)
          max_len_X = max(np.sum(X_out != 0, axis=-1))
          max_len_out = min(max_len_X + 1 - offset, max_len) if pad_on_batch else max_len
          X_out_aux = np.zeros((X_out.shape[0], max_len_out), dtype='int64')
          X_out_aux[:, :max_len_X] = X_out[:, :max_len_X].astype('int64')
          # Mask all zero-values
          X_mask = (np.ma.make_mask(X_out_aux, dtype='int') * 1).astype('int8')
          # But we keep with the first one, as it indicates the <pad> symbol
          X_mask = np.hstack((np.ones((X_mask.shape[0], 1)), X_mask[:, :-1]))
          X_out = (X_out_aux, X_mask)
          return X_out
  
      def loadTextFeaturesOneHot(self, X,
                                 vocabulary_len,
                                 max_len,
                                 pad_on_batch,
                                 offset,
                                 sample_weights=False,
                                 label_smoothing=0.):
  
          """
          Text encoder: Transforms samples from a text representation into a one-hot. It also masks the text.
          :param X: Encoded text.
          :param vocabulary_len: Length of the vocabulary (size of the one-hot vector)
          :param sample_weights: If True, we also return the mask of the text.
          :param vocabularies: Mapping word -> index
          :param max_len: Maximum length of the text.
          :param offset: Shifts the text to the right, adding null symbol at the start
          :param fill: 'start': the resulting vector will be filled with 0s at the beginning.
                       'end': it will be filled with 0s at the end.
                       'center': the vector will be surrounded by 0s, both at beginning and end.
          :param pad_on_batch: Whether we get sentences with length of the maximum length of
                               the minibatch or sentences with a fixed (max_text_length) length.
          :param words_so_far: Experimental feature. Use with caution.
          :param loading_X: Whether we are loading an input or an output of the model
          :return: Text as sequence of one-hot vectors. Mask for each sentence.
          """
          y = self.loadTextFeatures(X, max_len, pad_on_batch, offset)
  
          # Use whole sentence as class (classifier model)
          if max_len == 0:
              y_aux = to_categorical(y, vocabulary_len).astype(np.uint8)
          # Use words separately (generator model)
          else:
              if label_smoothing > 0.:
                  y_aux_type = np.float32
              else:
                  y_aux_type = np.uint8
              y_aux = np.zeros(list(y[0].shape) + [vocabulary_len]).astype(y_aux_type)
              for idx in range(y[0].shape[0]):
                  y_aux[idx] = to_categorical(y[0][idx], vocabulary_len).astype(y_aux_type)
              if label_smoothing > 0.:
                  y_aux = self.apply_label_smoothing(y_aux, label_smoothing, vocabulary_len)
              if sample_weights:
                  y_aux = (y_aux, y[1])  # join data and mask
          return y_aux
  
      def loadMapping(self, path_list):
          """
          Loads a mapping of Source -- Target words.
          :param path_list: Pickle object with the mapping
          :return: None
          """
          if not self.silence:
              logger.info("Loading source -- target mapping.")
          if sys.version_info.major == 3:
              self.mapping = pk.load(open(path_list, 'rb'), encoding='utf-8')
          else:
              self.mapping = pk.load(open(path_list, 'rb'))
          if not self.silence:
              logger.info("Source -- target mapping loaded with a total of %d words." % len(list(self.mapping)))
  
      # ------------------------------------------------------- #
      #       Tokenizing functions
      # ------------------------------------------------------- #
  
      @staticmethod
      def tokenize_basic(caption, lowercase=True):
          """
          Basic tokenizer for the input/output data of type 'text':
             * Splits punctuation
             * Optional lowercasing
  
          :param caption: String to tokenize
          :param lowercase: Whether to lowercase the caption or not
          :return: Tokenized version of caption
          """
          return tokenize_basic(caption, lowercase=lowercase)
  
      @staticmethod
      def tokenize_aggressive(caption, lowercase=True):
          """
          Aggressive tokenizer for the input/output data of type 'text':
          * Removes punctuation
          * Optional lowercasing
  
          :param caption: String to tokenize
          :param lowercase: Whether to lowercase the caption or not
          :return: Tokenized version of caption
          """
          return tokenize_aggressive(caption, lowercase=lowercase)
  
      @staticmethod
      def tokenize_icann(caption):
          """
          Tokenization used for the icann paper:
          * Removes some punctuation (. , ")
          * Lowercasing
  
          :param caption: String to tokenize
          :return: Tokenized version of caption
          """
          return tokenize_icann(caption)
  
      @staticmethod
      def tokenize_montreal(caption):
          """
          Similar to tokenize_icann
              * Removes some punctuation
              * Lowercase
  
          :param caption: String to tokenize
          :return: Tokenized version of caption
          """
          return tokenize_montreal(caption)
  
      @staticmethod
      def tokenize_soft(caption, lowercase=True):
          """
          Tokenization used for the icann paper:
              * Removes very little punctuation
              * Lowercase
          :param caption: String to tokenize
          :param lowercase: Whether to lowercase the caption or not
          :return: Tokenized version of caption
          """
          return tokenize_soft(caption, lowercase=lowercase)
  
      @staticmethod
      def tokenize_none(caption):
          """
          Does not tokenizes the sentences. Only performs a stripping
  
          :param caption: String to tokenize
          :return: Tokenized version of caption
          """
          return tokenize_none(caption)
  
      @staticmethod
      def tokenize_none_char(caption):
          """
          Character-level tokenization. Respects all symbols. Separates chars. Inserts <space> sybmol for spaces.
          If found an escaped char, "&apos;" symbol, it is converted to the original one
          # List of escaped chars (by moses tokenizer)
          & ->  &amp;
          | ->  &#124;
          < ->  &lt;
          > ->  &gt;
          ' ->  &apos;
          " ->  &quot;
          [ ->  &#91;
          ] ->  &#93;
          :param caption: String to tokenize
          :return: Tokenized version of caption
          """
          return tokenize_none_char(caption)
  
      @staticmethod
      def tokenize_CNN_sentence(caption):
          """
          Tokenization employed in the CNN_sentence package
          (https://github.com/yoonkim/CNN_sentence/blob/master/process_data.py#L97).
          :param caption: String to tokenize
          :return: Tokenized version of caption
          """
          return tokenize_CNN_sentence(caption)
  
      @staticmethod
      def tokenize_questions(caption):
          """
          Basic tokenizer for VQA questions:
              * Lowercasing
              * Splits contractions
              * Removes punctuation
              * Numbers to digits
  
          :param caption: String to tokenize
          :return: Tokenized version of caption
          """
          return tokenize_questions(caption)
  
      def tokenize_bpe(self, caption):
          """
          Applies BPE segmentation (https://github.com/rsennrich/subword-nmt)
          :param caption: Caption to detokenize.
          :return: Encoded version of caption.
          """
          if not self.BPE_built:
              raise Exception('Prior to use the "tokenize_bpe" method, you should invoke "build_BPE"')
          if isinstance(caption, str) and sys.version_info < (3, 0):
              caption = caption.decode('utf-8')
          tokenized = re.sub(u'[\n\t]+', u'', caption)
          tokenized = self.BPE.segment(tokenized).strip()
          return tokenized
  
      @staticmethod
      def detokenize_none(caption):
          """
          Dummy function: Keeps the caption as it is.
          :param caption: String to de-tokenize.
          :return: Same caption.
          """
          return detokenize_none(caption)
  
      @staticmethod
      def detokenize_bpe(caption, separator=u'@@'):
          """
          Reverts BPE segmentation (https://github.com/rsennrich/subword-nmt)
          :param caption: Caption to detokenize.
          :param separator: BPE separator.
          :return: Detokenized version of caption.
          """
          return detokenize_bpe(caption, separator=separator)
  
      @staticmethod
      def detokenize_none_char(caption):
          """
          Character-level detokenization. Respects all symbols. Joins chars into words. Words are delimited by
          the <space> token. If found an special character is converted to the escaped char.
          # List of escaped chars (by moses tokenizer)
              & ->  &amp;
              | ->  &#124;
              < ->  &lt;
              > ->  &gt;
              ' ->  &apos;
              " ->  &quot;
              [ ->  &#91;
              ] ->  &#93;
          :param caption: String to de-tokenize.
              :return: Detokenized version of caption.
          """
          return detokenize_none_char(caption)
  
      def tokenize_moses(self, caption, language='en', lowercase=False, aggressive_dash_splits=False, return_str=True, escape=False):
          """
          Applies the Moses tokenization. Relying on sacremoses' implementation of the Moses tokenizer.
  
          :param caption: Sentence to tokenize
          :param language: Language (will build the tokenizer for this language)
          :param lowercase: Whether to lowercase or not the sentence
          :param agressive_dash_splits: Option to trigger dash split rules .
          :param return_str: Return string or list
          :param escape: Escape HTML special chars
          :return:
          """
          # Compatibility with old Datasets instances:
          if not hasattr(self, 'moses_tokenizer_built'):
              self.moses_tokenizer_built = False
          if not self.moses_tokenizer_built:
              self.build_moses_tokenizer(language=language)
          if isinstance(caption, str) and sys.version_info < (3, 0):
              caption = caption.decode('utf-8')
          tokenized = re.sub(u'[\n\t]+', u'', caption)
          if lowercase:
              tokenized = tokenized.lower()
          return self.moses_tokenizer.tokenize(tokenized, aggressive_dash_splits=aggressive_dash_splits,
                                               return_str=return_str, escape=escape)
  
      def detokenize_moses(self, caption, language='en', lowercase=False, return_str=True, unescape=True):
          """
          Applies the Moses detokenization. Relying on sacremoses' implementation of the Moses tokenizer.
  
          :param caption: Sentence to tokenize
          :param language: Language (will build the tokenizer for this language)
          :param lowercase: Whether to lowercase or not the sentence
          :param agressive_dash_splits: Option to trigger dash split rules .
          :param return_str: Return string or list
          :param escape: Escape HTML special chars
          :return:
          """
          # Compatibility with old Datasets instances:
          if not hasattr(self, 'moses_detokenizer_built'):
              self.moses_detokenizer_built = False
          if not self.moses_detokenizer_built:
              self.build_moses_detokenizer(language=language)
          if isinstance(caption, str) and sys.version_info < (3, 0):
              caption = caption.decode('utf-8')
          tokenized = re.sub(u'[\n\t]+', u'', caption)
          if lowercase:
              tokenized = tokenized.lower()
          return self.moses_detokenizer.detokenize(tokenized.split(), return_str=return_str, unescape=unescape)
  
      # ------------------------------------------------------- #
      #       TYPE 'video' and 'video-features' SPECIFIC FUNCTIONS
      # ------------------------------------------------------- #
  
      def preprocessVideos(self, path_list, data_id, set_name, max_video_len, img_size, img_size_crop):
          """
          Preprocess videos. Subsample and crop frames.
          :param path_list: path to all images in all videos
          :param data_id: Data id to be processed.
          :param set_name: Set name to be processed.
          :param max_video_len: Maximum number of subsampled video frames.
          :param img_size: Size of each frame.
          :param img_size_crop: Size of each image crop.
          :return:
          """
          if isinstance(path_list, list) and len(path_list) == 2:
              # path to all images in all videos
              data = []
              with open(path_list[0], 'r') as list_:
                  for line in list_:
                      data.append(line.rstrip('\n'))
              # frame counts
              counts_frames = []
              with open(path_list[1], 'r') as list_:
                  for line in list_:
                      counts_frames.append(int(line.rstrip('\n')))
  
              if data_id not in self.paths_frames:
                  self.paths_frames[data_id] = dict()
              self.paths_frames[data_id][set_name] = data
              self.max_video_len[data_id] = max_video_len
              self.img_size[data_id] = img_size
              self.img_size_crop[data_id] = img_size_crop
          else:
              raise Exception('Wrong type for "path_list". It must be a list containing two paths: '
                              'a path to a text file with the paths to all images in all videos in '
                              '[0] and a path to another text file with the number of frames of '
                              'each video in each line in [1] (which will index the paths in the first file).\n'
                              'It currently is: %s' % str(path_list))
  
          return counts_frames
  
      def preprocessVideoFeatures(self, path_list, data_id, set_name, max_video_len, img_size, img_size_crop, feat_len):
          """
          Preprocess already extracted features from video frames.
          :param path_list: path to all features in all videos
          :param data_id: Data id to be processed.
          :param set_name: Set name to be processed.
          :param max_video_len: Maximum number of subsampled video features.
          :param img_size: Size of each frame.
          :param img_size_crop: Size of each image crop.
          :param feat_len: Length of each feature.
          :return:
          """
          if isinstance(path_list, list) and len(path_list) == 2:
              if isinstance(path_list[0], str):
                  # path to all images in all videos
                  paths_frames = []
                  with open(path_list[0], 'r') as list_:
                      for line in list_:
                          paths_frames.append(line.rstrip('\n'))
              elif isinstance(path_list[0], list):
                  paths_frames = path_list[0]
              else:
                  raise Exception('Wrong type for "path_frames". It must be a path to a file containing a'
                                  ' list of frames or directly a list of frames.\n'
                                  'It currently is: %s' % str(path_list[0]))
  
              if isinstance(path_list[1], str):
                  # frame counts
                  counts_frames = []
                  with open(path_list[1], 'r') as list_:
                      for line in list_:
                          counts_frames.append(int(line.rstrip('\n')))
              elif isinstance(path_list[1], list):
                  counts_frames = path_list[1]
              else:
                  raise Exception(
                      'Wrong type for "counts_frames".'
                      ' It must be a path to a file containing a list of counts or directly a list of counts.\n'
                      'It currently is: %s' % str(path_list[1]))
  
              # video indices
              video_indices = range(len(counts_frames))
  
              if data_id not in self.paths_frames:
                  self.paths_frames[data_id] = dict()
              if data_id not in self.counts_frames:
                  self.counts_frames[data_id] = dict()
  
              self.paths_frames[data_id][set_name] = paths_frames
              self.counts_frames[data_id][set_name] = counts_frames
              self.max_video_len[data_id] = max_video_len
              self.img_size[data_id] = img_size
              self.img_size_crop[data_id] = img_size_crop
          else:
              raise Exception('Wrong type for "path_list". '
                              'It must be a list containing two paths: a path to a text file with the paths to all '
                              'images in all videos in [0] and a path to another text file with the number of frames '
                              'of each video in each line in [1] (which will index the paths in the first file).'
                              'It currently is: %s' % str(path_list[1]))
  
          if feat_len is not None:
              if not isinstance(feat_len, list):
                  feat_len = [feat_len]
              self.features_lengths[data_id] = feat_len
  
          return video_indices
  
      def loadVideos(self, n_frames, data_id, last, set_name, max_len, normalization_type, normalization, meanSubstraction,
                     dataAugmentation):
          """
           Loads a set of videos from disk. (Untested!)
  
          :param n_frames: Number of frames per video
          :param data_id: Id to load
          :param last: Last video loaded
          :param set_name:  'train', 'val', 'test'
          :param max_len: Maximum length of videos
          :param normalization_type:  Type of normalization applied
          :param normalization: Whether we apply a 0-1 normalization to the images
          :param meanSubstraction:  Whether we are removing the training mean
          :param dataAugmentation:  Whether we are applying dataAugmentatino (random cropping and horizontal flip)
          """
  
          n_videos = len(n_frames)
          V = np.zeros((n_videos, max_len * 3, self.img_size_crop[data_id][0], self.img_size_crop[data_id][1]))
  
          idx = [0 for i in range(n_videos)]
          # recover all indices from image's paths of all videos
          for v in range(n_videos):
              this_last = last + v
              if this_last >= n_videos:
                  v = this_last % n_videos
                  this_last = v
              idx[v] = int(sum(getattr(self, 'Y_' + set_name)[data_id][:this_last]))
  
          # load images from each video
          for enum, (n, i) in list(enumerate(zip(n_frames, idx))):
              paths = self.paths_frames[data_id][set_name][i:i + n]
              daRandomParams = None
              if dataAugmentation:
                  daRandomParams = self.getDataAugmentationRandomParams(paths, data_id)
              # returns numpy array with dimensions (batch, channels, height, width)
              images = self.loadImages(paths, data_id, normalization_type, normalization, meanSubstraction, dataAugmentation,
                                       daRandomParams)
              # fills video matrix with each frame (fills with 0s or removes remaining frames w.r.t. max_len)
              len_j = images.shape[0]
              offset_j = max_len - len_j
              if offset_j < 0:
                  len_j = len_j + offset_j
                  offset_j = 0
              for j in range(len_j):
                  V[enum, (j + offset_j) * 3:(j + offset_j + 1) * 3] = images[j]
  
          return V
  
      def loadVideoFeatures(self, idx_videos, data_id, set_name, max_len, normalization_type, normalization, feat_len,
                            external=False, data_augmentation=True):
          """
  
          :param idx_videos: indices of the videos in the complete list of the current set_name
          :param data_id: identifier of the input/output that we are loading
          :param set_name: 'train', 'val' or 'test'
          :param max_len: maximum video length (number of frames)
          :param normalization_type: type of data normalization applied
          :param normalization: Switch on/off data normalization
          :param feat_len: length of the features about to load
          :param external: Switch on/off data loading from external dataset (not sharing self.path)
          :param data_augmentation: Switch on/off data augmentation
          :return:
          """
  
          n_videos = len(idx_videos)
          if isinstance(feat_len, list):
              feat_len = feat_len[0]
          features = np.zeros((n_videos, max_len, feat_len))
  
          selected_frames = self.getFramesPaths(idx_videos, data_id, set_name, max_len, data_augmentation)
          data_augmentation_types = self.inputs_data_augmentation_types[data_id]
  
          # load features from selected paths
          for i, vid_paths in list(enumerate(selected_frames)):
              for j, feat in list(enumerate(vid_paths)):
                  if not external:
                      feat = self.path + '/' + feat
  
                  # Check if the filename includes the extension
                  feat = np.load(feat)
  
                  if data_augmentation and 'noise' in data_augmentation_types:
                      noise_mean = 0.0
                      noise_dev = 0.01
                      noise = np.random.normal(noise_mean, noise_dev, feat.shape)
                      feat += noise
  
                  if normalization:
                      if normalization_type == 'L2':
                          feat /= np.linalg.norm(feat, ord=2)
  
                  features[i, j] = feat
  
          return np.array(features)
  
      def getFramesPaths(self, idx_videos, data_id, set_name, max_len, data_augmentation):
          """
          Recovers the paths from the selected video frames.
          """
  
          # recover chosen data augmentation types
          data_augmentation_types = self.inputs_data_augmentation_types[data_id]
          if data_augmentation_types is None:
              data_augmentation_types = []
  
          n_frames = [self.counts_frames[data_id][set_name][i_idx_vid] for i_idx_vid in idx_videos]
  
          n_videos = len(idx_videos)
          idx = [0 for i_nvid in range(n_videos)]
          # recover all initial indices from image's paths of all videos
          for v in range(n_videos):
              last_idx = idx_videos[v]
              idx[v] = int(sum(self.counts_frames[data_id][set_name][:last_idx]))
  
          # select subset of max_len from n_frames[i]
          selected_frames = [0 for i_nvid in range(n_videos)]
          for enum, (n, i) in list(enumerate(zip(n_frames, idx))):
              paths = self.paths_frames[data_id][set_name][i:i + n]
  
              if data_augmentation and 'random_selection' in data_augmentation_types:  # apply random frames selection
                  selected_idx = sorted(random.sample(range(n), min(max_len, n)))
              else:  # apply equidistant frames selection
                  selected_idx = np.round(np.linspace(0, n - 1, min(max_len, n)))
                  # splits = np.array_split(range(n), min(max_len, n))
                  # selected_idx = [s[0] for s in splits]
  
              selected_paths = [paths[int(idx)] for idx in selected_idx]
              selected_frames[enum] = selected_paths
  
          return selected_frames
  
      def loadVideosByIndex(self, n_frames, data_id, indices, set_name, max_len, normalization_type, normalization,
                            meanSubstraction, dataAugmentation):
          """
          Get videos by indices.
          :param n_frames: Indices of the frames to load from each video.
          :param data_id: Data id to be processed.
          :param indices: Indices of the videos to load.
          :param set_name: Set name to be processed.
          :param max_len: Maximum length of each video.
          :param normalization_type: Normalization type applied to the frames.
          :param normalization: Normalization applied to the frames.
          :param meanSubstraction: Mean subtraction applied to the frames.
          :param dataAugmentation: Whether apply data augmentation.
          :return:
          """
          n_videos = len(indices)
          V = np.zeros((n_videos, max_len * 3, self.img_size_crop[data_id][0], self.img_size_crop[data_id][1]))
  
          idx = [0 for i in range(n_videos)]
          # recover all indices from image's paths of all videos
          for v in range(n_videos):
              idx[v] = int(sum(eval('self.X_' + set_name + '[data_id][indices[v]]')))
  
          # load images from each video
          for enum, (n, i) in list(enumerate(zip(n_frames, idx))):
              paths = self.paths_frames[data_id][set_name][i:i + n]
              daRandomParams = None
              if dataAugmentation:
                  daRandomParams = self.getDataAugmentationRandomParams(paths, data_id)
              # returns numpy array with dimensions (batch, channels, height, width)
              images = self.loadImages(paths, data_id, normalization_type, normalization, meanSubstraction, dataAugmentation,
                                       daRandomParams)
              # fills video matrix with each frame (fills with 0s or removes remaining frames w.r.t. max_len)
              len_j = images.shape[0]
              offset_j = max_len - len_j
              if offset_j < 0:
                  len_j = len_j + offset_j
                  offset_j = 0
              for j in range(len_j):
                  V[enum, (j + offset_j) * 3:(j + offset_j + 1) * 3] = images[j]
  
          return V
  
      # ------------------------------------------------------- #
      #       TYPE 'id' SPECIFIC FUNCTIONS
      # ------------------------------------------------------- #
  
      @staticmethod
      def preprocessIDs(path_list, data_id, set_name):
          """
          Preprocess ID outputs: Strip and put each ID in a line.
          """
          logger.info('WARNING: inputs or outputs with type "id" will not be treated in any way by the dataset.')
          if isinstance(path_list, str) and os.path.isfile(path_list):  # path to list of IDs
              data = []
              with codecs.open(path_list, 'r', encoding='utf-8') as list_:
                  for line in list_:
                      data.append(line.rstrip('\n'))
          elif isinstance(path_list, list):
              data = path_list
          else:
              raise Exception('Wrong type for "path_list". '
                              'It must be a path to a text file with an data_id in each line'
                              ' or an instance of the class list with an data_id in each position.'
                              'It currently is: %s' % str(path_list))
  
          return data
  
      # ------------------------------------------------------- #
      #       TYPE '3DSemanticLabel' SPECIFIC FUNCTIONS
      # ------------------------------------------------------- #
  
      def getImageFromPrediction_3DSemanticLabel(self, img, n_classes):
          """
          Get the segmented image from the prediction of the model using the semantic classes of the dataset together with their corresponding colours.
  
          :param img: Prediction of the model.
          :param n_classes: Number of semantic classes.
          :return: out_img: The segmented image with the class colours.
          """
  
          h_crop, w_crop, d_crop = self.img_size_crop[self.id_in_3DLabel[self.ids_outputs[0]]]
          output_id = ''.join(self.ids_outputs)
  
          # prepare the segmented image
          pred_labels = np.reshape(img, (h_crop, w_crop, n_classes))
          # out_img = np.zeros((h_crop, w_crop, d_crop))
          out_img = np.zeros((h_crop, w_crop, 3))  # predictions saved as RGB images (3 channels)
  
          for ih in range(h_crop):
              for iw in range(w_crop):
                  lab = np.argmax(pred_labels[ih, iw])
                  out_img[ih, iw, :] = self.semantic_classes[output_id][lab]
  
          return out_img
  
      def preprocess3DSemanticLabel(self, path_list, data_id, associated_id_in, num_poolings):
          """
          Preprocess 3D Semantic labels
          """
          return self.preprocess3DLabel(path_list, data_id, associated_id_in, num_poolings)
  
      def setSemanticClasses(self, path_classes, data_id):
          """
          Loads the list of semantic classes of the dataset together with their corresponding colours in the GT image.
          Each line must contain a unique identifier of the class and its associated RGB colour representation
           separated by commas.
  
          :param path_classes: Path to a text file with the classes and their colours.
          :param data_id: input/output id
  
          :return: None
          """
          if isinstance(path_classes, str) and os.path.isfile(path_classes):
              semantic_classes = dict()
              with codecs.open(path_classes, 'r', encoding='utf-8') as list_:
                  for line in list_:
                      line = line.rstrip('\n').split(',')
                      if len(line) != 4:
                          raise Exception('Wrong format for semantic classes.'
                                          ' Must contain a class name followed by the '
                                          'RGB colour values separated by commas.'
                                          'It currently has a line of length: %s' % str(len(line)))
  
                      class_id = self.dic_classes[data_id][line[0]]
                      semantic_classes[int(class_id)] = [int(line[1]), int(line[2]), int(line[3])]
              self.semantic_classes[data_id] = semantic_classes
          else:
              raise Exception('Wrong type for "path_classes".'
                              ' It must be a path to a text file with the classes '
                              'and their associated colour in the GT image.'
                              'It currently is: %s' % str(path_classes))
  
          if not self.silence:
              logger.info('Loaded semantic classes list for data with data_id: ' + data_id)
  
      def load_GT_3DSemanticLabels(self, gt, data_id):
          """
          Loads a GT list of 3DSemanticLabels in a 2D matrix and reshapes them to an Nx1 array (EVALUATION)
  
          :param gt: list of Dataset output of type 3DSemanticLabels
          :param data_id: id of the input/output we are processing
          :return: out_list: containing a list of label images reshaped as an Nx1 array
          """
          from PIL import Image as pilimage
          from scipy import misc
  
          out_list = []
  
          assoc_id_in = self.id_in_3DLabel[data_id]
          classes_to_colour = self.semantic_classes[data_id]
          nClasses = len(list(classes_to_colour))
          img_size = self.img_size[assoc_id_in]
          size_crop = self.img_size_crop[assoc_id_in]
          num_poolings = self.num_poolings_model[data_id]
  
          n_samples = len(gt)
          h, w, d = img_size
          h_crop, w_crop, d_crop = size_crop
  
          # Modify output dimensions depending on number of poolings applied
          if num_poolings is not None:
              h_crop = int(np.floor(h_crop / np.power(2, num_poolings)))
              w_crop = int(np.floor(w_crop / np.power(2, num_poolings)))
  
          for i in range(n_samples):
              pre_labels = np.zeros((nClasses, h_crop, w_crop), dtype=np.float32)
              # labels = np.zeros((h_crop, w_crop), dtype=np.uint8)
              line = gt[i]
  
              # Load labeled GT image
              labeled_im = self.path + '/' + line
              # Check if the filename includes the extension
              [path, filename] = ntpath.split(labeled_im)
              [filename, ext] = os.path.splitext(filename)
              # If it doesn't then we find it
              if not ext:
                  filename = fnmatch.filter(os.listdir(path), filename + '*')
                  if not filename:
                      raise Exception('Non existent image ' + labeled_im)
                  else:
                      labeled_im = path + '/' + filename[0]
              # Read image
              try:
                  logging.disable(logging.CRITICAL)
                  labeled_im = pilimage.open(labeled_im)
                  labeled_im = np.asarray(labeled_im)
                  logging.disable(logging.NOTSET)
                  labeled_im = misc.imresize(labeled_im, (h, w))
              except Exception:
                  logger.warning("WARNING!")
                  logger.warning("Can't load image " + labeled_im)
                  labeled_im = np.zeros((h, w))
  
              label3D = np.zeros((nClasses, h, w), dtype=np.float32)
  
              # Insert 1s in the corresponding positions for each class
              for class_id, colour in iteritems(classes_to_colour):
                  # indices = np.where(np.all(labeled_im == colour, axis=-1))
                  indices = np.where(labeled_im == class_id)
                  num_vals = len(indices[0])
                  if num_vals > 0:
                      for idx_pos in range(num_vals):
                          x, y = indices[0][idx_pos], indices[1][idx_pos]
                          label3D[class_id, x, y] = 1.
  
              # Resize 3DLabel to crop size.
              for j in range(nClasses):
                  label2D = misc.imresize(label3D[j], (h_crop, w_crop))
                  maxval = np.max(label2D)
                  if maxval > 0:
                      label2D /= maxval
                  pre_labels[j] = label2D
  
              # Convert to single matrix with class IDs
              labels = np.argmax(pre_labels, axis=0)
              labels = np.reshape(labels, (w_crop * h_crop))
  
              out_list.append(labels)
  
          return out_list
  
      def resize_semantic_output(self, predictions, ids_out):
          """
          Resize semantic output.
          """
          from scipy import misc
  
          out_pred = []
  
          for pred, id_out in list(zip(predictions, ids_out)):
  
              assoc_id_in = self.id_in_3DLabel[id_out]
              in_size = self.img_size_crop[assoc_id_in]
              out_size = self.img_size[assoc_id_in]
              n_classes = len(self.classes[id_out])
  
              pred = np.transpose(pred, [1, 0])
              pred = np.reshape(pred, (-1, in_size[0], in_size[1]))
  
              new_pred = np.zeros(tuple([n_classes] + out_size[0:2]))
              for pos, p in list(enumerate(pred)):
                  new_pred[pos] = misc.imresize(p, tuple(out_size[0:2]))
  
              new_pred = np.reshape(new_pred, (-1, out_size[0] * out_size[1]))
              new_pred = np.transpose(new_pred, [1, 0])
  
              out_pred.append(new_pred)
  
          return out_pred
  
      # ------------------------------------------------------- #
      #       TYPE '3DLabel' SPECIFIC FUNCTIONS
      # ------------------------------------------------------- #
  
      def preprocess3DLabel(self, path_list, label_id, associated_id_in, num_poolings):
          if isinstance(path_list, str) and os.path.isfile(path_list):
              path_list_3DLabel = []
              with codecs.open(path_list, 'r', encoding='utf-8') as list_:
                  for line in list_:
                      path_list_3DLabel.append(line.strip())
          else:
              raise Exception('Wrong type for "path_list". '
                              'It must be a path to a text file with the path to 3DLabel files.'
                              'It currently is: %s' % str(path_list))
  
          self.num_poolings_model[label_id] = num_poolings
          self.id_in_3DLabel[label_id] = associated_id_in
  
          return path_list_3DLabel
  
      def convert_3DLabels_to_bboxes(self, predictions, original_sizes, threshold=0.5, idx_3DLabel=0,
                                     size_restriction=0.001):
          """
          Converts a set of predictions of type 3DLabel to their corresponding bounding boxes.
  
          :param idx_3DLabel:
          :param size_restriction:
          :param predictions: 3DLabels predicted by Model_Wrapper.
                              If type is list it will be assumed that position 0 corresponds to 3DLabels
          :param original_sizes: original sizes of the predicted images width and height
          :param threshold: minimum overlapping threshold for considering a prediction valid
          :return: predicted_bboxes, predicted_Y, predicted_scores for each image
          """
          from scipy import ndimage
          out_list = []
  
          # if type is list it will be assumed that position 0 corresponds to 3DLabels
          if isinstance(predictions, list):
              predict_3dLabels = predictions[idx_3DLabel]
          else:
              predict_3dLabels = predictions
  
          # Reshape from (n_samples, width*height, nClasses) to (n_samples, nClasses, width, height)
          n_samples, _, n_classes = predict_3dLabels.shape
          w, h, _ = self.img_size_crop[self.id_in_3DLabel[self.ids_outputs[idx_3DLabel]]]
          predict_3dLabels = np.transpose(predict_3dLabels, (0, 2, 1))
          predict_3dLabels = np.reshape(predict_3dLabels, (n_samples, n_classes, w, h))
  
          # list of hypotheses with the following info [predicted_bboxes, predicted_Y, predicted_scores]
          for s in range(n_samples):
              bboxes = []
              Y = []
              scores = []
              orig_h, orig_w = original_sizes[s]
              wratio = float(orig_w) / w
              hratio = float(orig_h) / h
              for c in range(n_classes):
                  map = predict_3dLabels[s][c]
  
                  # Compute binary selected region
                  binary_heat = map
                  binary_heat = np.where(binary_heat >= threshold, 255, 0)
  
                  # Get biggest connected component
                  min_size = map.shape[0] * map.shape[1] * size_restriction
                  labeled, nr_objects = ndimage.label(binary_heat)  # get connected components
                  [objects, counts] = np.unique(labeled, return_counts=True)  # count occurrences
                  # biggest_components = np.argsort(counts[1:])[::-1]
                  # selected_components = [1 if counts[i+1] >= min_size else 0
                  # for i in biggest_components] # check minimum size restriction
                  # selected_components = [1 for i in range(len(objects))]
                  # biggest_components = biggest_components[:min([np.sum(selected_components), 9999])] # get all bboxes
  
                  for obj in objects[1:]:
                      current_obj = np.where(labeled == obj, 255, 0)  # get the biggest
  
                      # Draw bounding box on original image
                      box = list(bbox(current_obj))
                      current_obj = np.nonzero(current_obj)
                      if len(current_obj) > min_size:  # filter too small bboxes
  
                          # expand box before final detection
                          # x_exp = box[2]# * box_expansion
                          # y_exp = box[3]# * box_expansion
                          # box[0] = max([0, box[0]-x_exp/2])
                          # box[1] = max([0, box[1]-y_exp/2])
                          # change width and height by xmax and ymax
                          # box[2] += box[0]
                          # box[3] += box[1]
                          # box[2] = min([new_reshape_size[1]-1, box[2] + x_exp])
                          # box[3] = min([new_reshape_size[0]-1, box[3] + y_exp])
  
                          # get score of the region
                          score = np.mean([map[point[0], point[1]] for point in current_obj])
  
                          # convert bbox to original size
                          box = np.array([box[0] * wratio, box[1] * hratio, box[2] * wratio, box[3] * hratio])
                          box = box.astype(int)
  
                          bboxes.append(box)
                          Y.append(c)
                          scores.append(score)
  
              out_list.append([bboxes, Y, scores])
  
          return out_list
  
      @staticmethod
      def convert_GT_3DLabels_to_bboxes(gt):
          """
          Converts a GT list of 3DLabels to a set of bboxes.
  
          :param gt: list of Dataset output of type 3DLabels
          :return: [out_list, original_sizes], where out_list contains a list of samples with the following info
                   [GT_bboxes, GT_Y], and original_sizes contains the original width and height for each image
          """
          out_list = []
          original_sizes = []
          # extra_vars[split]['references'] - list of samples with the following info [GT_bboxes, GT_Y]
  
          n_samples = len(gt)
          for i in range(n_samples):
              bboxes = []
              Y = []
  
              line = gt[i]
              arrayLine = line.split(';')
              arrayBndBox = arrayLine[:-1]
              w_original, h_original, d_original = eval(arrayLine[-1])
              original_sizes.append([h_original, w_original])
  
              for array in arrayBndBox:
                  bndbox = eval(array)[0]
                  # bndbox = [bndbox[1], bndbox[0], bndbox[3], bndbox[2]]
                  idxclass = eval(array)[1]
                  Y.append(idxclass)
                  bboxes.append(bndbox)
                  # bndbox_ones = np.ones((bndbox[2] - bndbox[0] + 1, bndbox[3] - bndbox[1] + 1))
                  # label3D[idxclass, bndbox[0] - 1:bndbox[2], bndbox[1] - 1:bndbox[3]] = bndbox_ones
  
              out_list.append([bboxes, Y])
  
          return [out_list, original_sizes]
  
      # ------------------------------------------------------- #
      #       TYPE 'raw-image' SPECIFIC FUNCTIONS
      # ------------------------------------------------------- #
  
      def preprocessImages(self, path_list, data_id, set_name, img_size, img_size_crop, use_RGB):
          """
          Image preprocessing function.
          :param path_list: Path to the images.
          :param data_id: Data id.
          :param set_name: Set name.
          :param img_size: Size of the images to process.
          :param img_size_crop: Size of the image crops.
          :param use_RGB: Whether use RGB color encoding.
          :return:
          """
          if isinstance(path_list, str) and os.path.isfile(path_list):  # path to list of images' paths
              data = []
              with open(path_list, 'r') as list_:
                  for line in list_:
                      data.append(line.rstrip('\n'))
          elif isinstance(path_list, list):
              data = path_list
          else:
              raise Exception('Wrong type for "path_list". It must be a path to a text file with an image '
                              'path in each line or an instance of the class list with an image path in each position.'
                              'It currently is: %s' % str(path_list))
  
          self.img_size[data_id] = img_size
          self.img_size_crop[data_id] = img_size_crop
          self.use_RGB[data_id] = use_RGB
  
          # Tries to load a train_mean file from the dataset folder if exists
          mean_file_path = self.path + '/train_mean'
          for s in range(len(self.img_size[data_id])):
              mean_file_path += '_' + str(self.img_size[data_id][s])
          mean_file_path += '_' + data_id + '_.jpg'
          if os.path.isfile(mean_file_path):
              self.setTrainMean(mean_file_path, data_id)
  
          return data
  
      def setTrainMean(self, mean_image, data_id, normalization=False):
          """
              Loads a pre-calculated training mean image, 'mean_image' can either be:
  
              - numpy.array (complete image)
              - list with a value per channel
              - string with the path to the stored image.
  
          :param mean_image:
          :param normalization:
          :param data_id: identifier of the type of input whose train mean is being introduced.
          """
          from scipy import misc
  
          if isinstance(mean_image, str):
              if not self.silence:
                  logger.info("Loading train mean image from file.")
              mean_image = misc.imread(mean_image)
          elif isinstance(mean_image, list):
              mean_image = np.array(mean_image, np.float64)
          self.train_mean[data_id] = mean_image.astype(np.float64)
  
          if normalization:
              self.train_mean[data_id] /= 255.0
  
          if self.train_mean[data_id].shape != tuple(self.img_size_crop[data_id]):
              # if not use_RGB:
              #     if len(self.train_mean[data_id].shape) == 1:
              #         if not self.silence:
              #             logger.info("Converting input train mean pixels into mean image.")
              #         mean_image = np.zeros(tuple(self.img_size_crop[data_id]), np.float64)
              #         mean_image[:, :] = self.train_mean[data_id]
              #         self.train_mean[data_id] = mean_image
              if len(self.train_mean[data_id].shape) == 1 and self.train_mean[data_id].shape[0] == self.img_size_crop[data_id][2]:
                  if not self.silence:
                      logger.info("Converting input train mean pixels into mean image.")
                  mean_image = np.zeros(tuple(self.img_size_crop[data_id]), np.float64)
                  for c in range(self.img_size_crop[data_id][2]):
                      mean_image[:, :, c] = self.train_mean[data_id][c]
                  self.train_mean[data_id] = mean_image
              else:
                  logger.warning(
                      "The loaded training mean size does not match the desired images size.\n"
                      "Change the images size with setImageSize(size) or "
                      "recalculate the training mean with calculateTrainMean().")
  
      def calculateTrainMean(self, data_id):
          """
              Calculates the mean of the data belonging to the training set split in each channel.
          """
          from scipy import misc
  
          calculate = False
          if data_id not in self.train_mean or not isinstance(self.train_mean[data_id], np.ndarray):
              calculate = True
          elif self.train_mean[data_id].shape != tuple(self.img_size[data_id]):
              calculate = True
              if not self.silence:
                  logger.warning(
                      "The loaded training mean size does not match the desired images size. Recalculating mean...")
  
          if calculate:
              if not self.silence:
                  logger.info("Start training set mean calculation...")
  
              I_sum = np.zeros(self.img_size_crop[data_id], dtype=np.float64)
  
              # Load images in batches and sum all of them
              init = 0
              batch = 200
              for current_image in range(batch, self.len_train, batch):
                  I = self.getX('train', init, current_image, meanSubstraction=False)[self.ids_inputs.index(data_id)]
                  for im in I:
                      I_sum += im
                  if not self.silence:
                      sys.stdout.write('\r')
                      sys.stdout.write("Processed %d/%d images..." % (current_image, self.len_train))
                      sys.stdout.flush()
                  init = current_image
              I = self.getX('train', init, self.len_train, meanSubstraction=False)[self.ids_inputs.index(data_id)]
              for im in I:
                  I_sum += im
              if not self.silence:
                  sys.stdout.write('\r')
                  sys.stdout.write("Processed %d/%d images..." % (current_image, self.len_train))
                  sys.stdout.flush()
  
              # Mean calculation
              self.train_mean[data_id] = I_sum / self.len_train
  
              # Store the calculated mean
              mean_name = '/train_mean'
              for s in range(len(self.img_size[data_id])):
                  mean_name += '_' + str(self.img_size[data_id][s])
              mean_name += '_' + data_id + '_.jpg'
              store_path = self.path + '/' + mean_name
              misc.imsave(store_path, self.train_mean[data_id])
  
              # self.train_mean[data_id] = self.train_mean[data_id].astype(np.float32)/255.0
  
              if not self.silence:
                  logger.info("Image mean stored in " + store_path)
  
          # Return the mean
          return self.train_mean[data_id]
  
      def loadImages(self, images, data_id, normalization_type='(-1)-1',
                     normalization=False, meanSubstraction=False,
                     dataAugmentation=False, daRandomParams=None,
                     wo_da_patch_type='whole',
                     da_patch_type='resize_and_rndcrop',
                     da_enhance_list=None,
                     useBGR=False,
                     external=False, loaded=False):
          """
          Loads a set of images from disk.
  
          :param images : list of image string names or list of matrices representing images (only if loaded==True)
          :param data_id : identifier in the Dataset object of the data we are loading
          :param normalization_type: type of normalization applied
          :param normalization : whether we applying a '0-1' or '(-1)-1' normalization to the images
          :param meanSubstraction : whether we are removing the training mean
          :param dataAugmentation : whether we are applying dataAugmentatino (random cropping and horizontal flip)
          :param daRandomParams : dictionary with results of random data augmentation provided by
                                  self.getDataAugmentationRandomParams()
          :param external : if True the images will be loaded from an external database, in this case the list of
                            images must be absolute paths
          :param loaded : set this option to True if images is a list of matricies instead of a list of strings
          """
          # Check if the chosen normalization type exists
          from PIL import Image as pilimage
          from scipy import misc
          import keras
  
          if normalization_type is None:
              normalization_type = '(-1)-1'
          if normalization and normalization_type not in self.__available_norm_im_vid:
              raise NotImplementedError(
                  'The chosen normalization type ' + str(normalization_type) +
                  ' is not implemented for the type "raw-image" and "video".')
          if da_enhance_list is None:
              da_enhance_list = []
          # Prepare the training mean image
          if meanSubstraction:  # remove mean
  
              if data_id not in self.train_mean:
                  raise Exception('Training mean is not loaded or calculated yet for the input with data_id "' + data_id + '".')
              train_mean = copy.copy(self.train_mean[data_id])
              train_mean = misc.imresize(train_mean, self.img_size_crop[data_id][0:2])
              train_mean = train_mean.astype(np.float64)
  
              # Transpose dimensions
              if len(self.img_size[data_id]) == 3:  # if it is a 3D image
                  # Convert RGB to BGR
                  if useBGR:
                      if self.img_size[data_id][2] == 3:  # if has 3 channels
                          train_mean = train_mean[:, :, ::-1]
                  if keras.backend.image_data_format() == 'channels_first':
                      train_mean = train_mean.transpose(2, 0, 1)
  
              # Also normalize training mean image if we are applying normalization to images
              if normalization:
                  if normalization_type == '0-1':
                      train_mean /= 255.0
                  elif normalization_type == '(-1)-1':
                      train_mean /= 127.5
                      train_mean -= 1.
  
          nImages = len(images)
  
          type_imgs = np.float64
          if len(self.img_size[data_id]) == 3:
              if keras.backend.image_data_format() == 'channels_first':
                  I = np.zeros([nImages] + [self.img_size_crop[data_id][2]] + self.img_size_crop[data_id][0:2], dtype=type_imgs)
              else:
                  I = np.zeros([nImages] + self.img_size_crop[data_id][0:2] + [self.img_size_crop[data_id][2]], dtype=type_imgs)
          else:
              I = np.zeros([nImages] + self.img_size_crop[data_id], dtype=type_imgs)
  
          # Process each image separately
          for i in range(nImages):
              im = images[i]
  
              if not loaded:
                  if not external:
                      im = self.path + '/' + im
  
                  # Check if the filename includes the extension
                  [path, filename] = ntpath.split(im)
                  [filename, ext] = os.path.splitext(filename)
  
                  # If it doesn't then we find it
                  if not ext:
                      filename = fnmatch.filter(os.listdir(path), filename + '*')
                      if not filename:
                          raise Exception('Non existent image ' + im)
                      else:
                          im = path + '/' + filename[0]
                  imname = im
  
                  # Read image
                  try:
                      logging.disable(logging.CRITICAL)
                      im = pilimage.open(im)
                      logging.disable(logging.NOTSET)
  
                  except Exception:
                      logger.warning("WARNING!")
                      logger.warning("Can't load image " + im)
                      im = np.zeros(tuple(self.img_size[data_id]))
  
              # Convert to RGB
              if not type(im).__module__ == np.__name__:
                  if self.use_RGB[data_id]:
                      im = im.convert('RGB')
                  else:
                      im = im.convert('L')
                  im = np.asarray(im, dtype=type_imgs)
  
              # Data augmentation
              if not dataAugmentation:
                  # TODO:
                  # wo_da_patch_type = central_crop, whole.
                  if wo_da_patch_type == 'central_crop':
                      # Use central crop.
                      im = self.getResizeImageWODistorsion(im, data_id)
                      im = np.asarray(im, dtype=type_imgs)
  
                      centerw, centerh = np.floor(np.shape(im)[0] * 0.5), np.floor(np.shape(im)[1] * 0.5)
                      halfw, halfh = np.floor(self.img_size_crop[data_id][0] * 0.5), np.floor(self.img_size_crop[data_id][1] * 0.5)
  
                      if self.img_size_crop[data_id][0] % 2 == 0:
                          im = im[centerw - halfw:centerw + halfw, centerh - halfh:centerh + halfh, :]
                      else:
                          im = im[centerw - halfw:centerw + halfw + 1, centerh - halfh:centerh + halfh + 1, :]
                  elif wo_da_patch_type == 'whole':
                      # Use whole image
                      im = misc.imresize(im, (self.img_size_crop[data_id][0], self.img_size_crop[data_id][1]))
                      im = np.asarray(im, dtype=type_imgs)
  
                  if not self.use_RGB[data_id]:
                      im = np.expand_dims(im, 2)
  
              else:
                  # TODO:
                  # da_patch_type: resize_and_rndcrop, rndcrop_and_resize, resizekp_and_rndcrop.
                  # da_enhance_list: brightness, color, sharpness, contrast.
                  min_value_enhance = 0.25
                  im = pilimage.fromarray(im.astype(np.uint8))
                  image_enhance_dict = {'brightness': 'ImageEnhance.Brightness(im)', 'color': 'ImageEnhance.Color(im)',
                                        'sharpness': 'ImageEnhance.Sharpness(im)',
                                        'contrast': 'ImageEnhance.Contrast(im)'}
  
                  for da_enhance in da_enhance_list:
                      image_enhance = eval(image_enhance_dict[da_enhance])
                      im = image_enhance.enhance((1 - min_value_enhance) + np.random.rand() * min_value_enhance * 2)
  
                  randomParams = daRandomParams[images[i]]
  
                  if da_patch_type == "rndcrop_and_resize":
                      w, h, d = np.shape(im)
                      mins = w if w < h else h
                      mincropfactor = 0.5
                      maxcropfactor = 1.0
  
                      random_factor = (maxcropfactor - np.random.rand() * (maxcropfactor - mincropfactor))
                      nw = int(random_factor * mins)
                      random_factor = (maxcropfactor - np.random.rand() * (maxcropfactor - mincropfactor))
                      nh = int(random_factor * mins)
  
                      iw = int((w - nw) * np.random.rand())
                      ih = int((h - nh) * np.random.rand())
  
                      im = im.crop((ih, iw, ih + nh, iw + nw))
                      im = im.resize((self.img_size_crop[data_id][1], self.img_size_crop[data_id][0]))
                      im = np.asarray(im, dtype=type_imgs)
                  elif da_patch_type == "resizekp_and_rndcrop":
                      im = self.getResizeImageWODistorsion(im, data_id)
                      # Take random crop
                      left = randomParams["left"]
                      right = np.add(left, self.img_size_crop[data_id][0:2])
  
                      iw, fw = 0, self.img_size_crop[data_id][0]
                      ih, fh = 0, self.img_size_crop[data_id][1]
  
                      if np.shape(im)[0] >= self.img_size[data_id][0] and np.shape(im)[1] >= self.img_size[data_id][1]:
                          iw, fw = left[0], right[0]
                          ih, fh = left[1], right[1]
                      elif np.shape(im)[0] >= self.img_size[data_id][0]:
                          iw, fw = left[0], right[0]
                      elif np.shape(im)[1] >= self.img_size[data_id][1]:
                          ih, fh = left[1], right[1]
  
                      offset_w = 0
                      offset_h = 0
  
                      w, h = np.shape(im)[0:2]
                      delta_w = np.floor((w - fw) * 0.5)
                      delta_h = np.floor((h - fh) * 0.5)
  
                      if delta_w > 0:
                          offset_w = int(np.random.rand() * delta_w)
                      if delta_h > 0:
                          offset_h = int(np.random.rand() * delta_h)
  
                      iw += offset_w
                      fw += offset_w
                      ih += offset_h
                      fh += offset_h
  
                      if self.use_RGB[data_id]:
                          im = im[iw:fw, ih:fh, :]
                      else:
                          im = im[iw:fw, ih:fh]
                  elif da_patch_type == 'resize_and_rndcrop':
                      # Resize
                      im = misc.imresize(im, (self.img_size[data_id][0], self.img_size[data_id][1]))
                      im = np.asarray(im, dtype=type_imgs)
                      if not self.use_RGB[data_id]:
                          im = np.expand_dims(im, 2)
  
                      # Take random crop
                      left = randomParams["left"]
                      right = np.add(left, self.img_size_crop[data_id][0:2])
  
                      try:
                          im = im[left[0]:right[0], left[1]:right[1], :]
                      except Exception:
                          logger.error('------- ERROR -------')
                          logger.error(left)
                          logger.error(right)
                          logger.error(im.shape)
                          logger.error(imname)
                          raise Exception('Error with image ' + imname)
  
                  # Randomly flip (with a certain probability)
                  flip = randomParams["hflip"]
                  prob_flip_horizontal = randomParams["prob_flip_horizontal"]
                  if flip < prob_flip_horizontal:  # horizontal flip
                      im = np.fliplr(im)
                  prob_flip_vertical = randomParams["prob_flip_vertical"]
                  flip = randomParams["vflip"]
                  if flip < prob_flip_vertical:  # vertical flip
                      im = np.flipud(im)
  
              # Normalize
              if normalization:
                  if normalization_type == '0-1':
                      im /= 255.0
                  elif normalization_type == '(-1)-1':
                      im /= 127.5
                      im -= 1.
                  elif normalization_type == 'inception':
                      im /= 255.
                      im -= 0.5
                      im *= 2.
  
              # Permute dimensions
              if len(self.img_size[data_id]) == 3:
                  # Convert RGB to BGR
                  if useBGR:
                      if self.img_size[data_id][2] == 3:  # if has 3 channels
                          im = im[:, :, ::-1]
                  if keras.backend.image_data_format() == 'channels_first':
                      im = im.transpose(2, 0, 1)
              else:
                  pass
  
              # Substract training images mean
              if meanSubstraction:  # remove mean
                  im = im - train_mean
  
              I[i] = im
  
          return I
  
      def getResizeImageWODistorsion(self, image, data_id):
          w, h = np.shape(image)[0:2]
          if w < h and (w < self.img_size_crop[data_id][0] or h > self.img_size[data_id][1]):
              w_size = self.img_size_crop[data_id][0]
              w_ratio = (w_size / float(w))
              h_size = int(h * w_ratio)
  
              if h > self.img_size[data_id][1]:
                  h_ratio = (self.img_size[data_id][1] / float(h))
                  if h_ratio > w_ratio:
                      w_size = int(w * h_ratio)
                      h_size = self.img_size[data_id][1]
          elif h < w and (h < self.img_size_crop[data_id][1] or w > self.img_size[data_id][0]):
              h_size = self.img_size_crop[data_id][1]
              h_ratio = (h_size / float(h))
              w_size = int(w * h_ratio)
  
              if w > self.img_size[data_id][0]:
                  w_ratio = (self.img_size[data_id][0] / float(w))
                  if w_ratio > h_ratio:
                      h_size = int(h * w_ratio)
                      w_size = self.img_size[data_id][0]
          else:
              w_size, h_size = self.img_size[data_id][0], self.img_size[data_id][1]
  
          # Resize
          img = copy.copy(image)
          img = img.resize((h_size, w_size))
          return img
  
      def getDataAugmentationRandomParams(self, images, data_id, prob_flip_horizontal=0.5, prob_flip_vertical=0.0):
          daRandomParams = dict()
          for i, image in enumerate(images):
              # Random crop
              margin = [self.img_size[data_id][0] - self.img_size_crop[data_id][0],
                        self.img_size[data_id][1] - self.img_size_crop[data_id][1]]
  
              if margin[0] > 0:
                  left = random.sample([k_ for k_ in range(margin[0])], 1)
              else:
                  left = [0]
              if margin[1] > 0:
                  left += random.sample([k for k in range(margin[1])], 1)
              else:
                  left += [0]
  
              # Randomly flip (with a certain probability)
              hflip = np.random.rand()
              vflip = np.random.rand()
  
              randomParams = dict()
              randomParams["left"] = left
              randomParams["hflip"] = hflip
              randomParams["vflip"] = vflip
              randomParams["prob_flip_horizontal"] = prob_flip_horizontal
              randomParams["prob_flip_vertical"] = prob_flip_vertical
  
              daRandomParams[image] = randomParams
  
          return daRandomParams
  
      def getClassID(self, class_name, data_id):
          """
              :return: the class data_id (int) for a given class string.
          """
          return self.dic_classes[data_id][class_name]
  
      # ------------------------------------------------------- #
      #       GETTERS
      #           [X,Y] pairs or X only
      # ------------------------------------------------------- #
  
      def getX(self, set_name, init, final, normalization_type='(-1)-1',
               normalization=False, meanSubstraction=False,
               dataAugmentation=False,
               wo_da_patch_type='whole', da_patch_type='resize_and_rndcrop', da_enhance_list=None,
               get_only_ids=False):
          """
          Gets all the data samples stored between the positions init to final
  
          :param set_name: 'train', 'val' or 'test' set
          :param init: initial position in the corresponding set split.
                       Must be bigger or equal than 0 and smaller than final.
          :param final: final position in the corresponding set split.
          # 'raw-image', 'video', 'image-features' and 'video-features'-related parameters
          :param normalization: indicates if we want to normalize the data.
          # 'image-features' and 'video-features'-related parameters
          :param normalization_type: indicates the type of normalization applied.
                                     See available types in self.__available_norm_im_vid for 'raw-image' and 'video'
                                     and self.__available_norm_feat for 'image-features' and 'video-features'.
          # 'raw-image' and 'video'-related parameters
          :param meanSubstraction: indicates if we want to substract the training mean from the returned images
                                   (only applicable if normalization=True)
          :param dataAugmentation: indicates if we want to apply data augmentation to the loaded images
                                  (random flip and cropping)
          :return: X, list of input data variables from sample 'init' to 'final' belonging to the chosen 'set_name'
          """
          self.__checkSetName(set_name)
          self.__isLoaded(set_name, 0)
          if da_enhance_list is None:
              da_enhance_list = []
          if final > getattr(self, 'len_' + set_name):
              raise Exception('"final" index must be smaller than the number of samples in the set.')
          if init < 0:
              raise Exception('"init" index must be equal or greater than 0.')
          if init >= final:
              raise Exception('"init" index must be smaller than "final" index.')
  
          X = []
          for id_in in list(self.ids_inputs):
              types_index = self.ids_inputs.index(id_in)
              type_in = self.types_inputs[set_name][types_index]
              ghost_x = False
              if id_in in self.optional_inputs:
                  try:
                      x = getattr(self, 'X_' + set_name)[id_in][init:final]
                      if len(x) != (final - init):
                          raise AssertionError('Retrieved a wrong number of samples.')
                  except Exception:
                      x = [[]] * (final - init)
                      ghost_x = True
              else:
                  x = getattr(self, 'X_' + set_name)[id_in][init:final]
  
              if not get_only_ids and not ghost_x:
                  if type_in == 'text-features':
                      x = self.loadTextFeatures(x,
                                                self.max_text_len[id_in][set_name],
                                                self.pad_on_batch[id_in],
                                                self.text_offset.get(id_in, 0))[0]
  
                  elif type_in == 'image-features':
                      x = self.loadFeatures(x,
                                            self.features_lengths[id_in],
                                            normalization_type,
                                            normalization,
                                            data_augmentation=dataAugmentation)
                  elif type_in == 'video-features':
                      x = self.loadVideoFeatures(x,
                                                 id_in,
                                                 set_name,
                                                 self.max_video_len[id_in],
                                                 normalization_type,
                                                 normalization,
                                                 self.features_lengths[id_in],
                                                 data_augmentation=dataAugmentation)
  
                  elif type_in == 'raw-image':
                      daRandomParams = None
                      if dataAugmentation:
                          daRandomParams = self.getDataAugmentationRandomParams(x, id_in)
                      x = self.loadImages(x,
                                          id_in,
                                          normalization_type,
                                          normalization,
                                          meanSubstraction,
                                          dataAugmentation,
                                          daRandomParams,
                                          wo_da_patch_type,
                                          da_patch_type,
                                          da_enhance_list)
                  elif type_in == 'video':
                      x = self.loadVideos(x,
                                          id_in,
                                          final,
                                          set_name,
                                          self.max_video_len[id_in],
                                          normalization_type,
                                          normalization,
                                          meanSubstraction,
                                          dataAugmentation)
                  elif type_in == 'text' or type_in == 'dense-text':
                      x = self.loadText(x,
                                        self.vocabulary[id_in],
                                        self.max_text_len[id_in][set_name],
                                        self.text_offset[id_in],
                                        fill=self.fill_text[id_in],
                                        pad_on_batch=self.pad_on_batch[id_in],
                                        words_so_far=self.words_so_far[id_in],
                                        loading_X=True)[0]
                  elif type_in == 'categorical':
                      nClasses = len(self.dic_classes[id_in])
                      x = self.loadCategorical(x,
                                               nClasses)
                  elif type_in == 'categorical_raw':
                      x = np.array(x)
                  elif type_in == 'binary':
                      x = self.loadBinary(x, id_in)
              X.append(x)
  
          return X
  
      def getY(self, set_name, init, final, dataAugmentation=False, get_only_ids=False):
          """
          Gets the [Y] samples for the FULL dataset
          :param set_name: 'train', 'val' or 'test' set
          :param init: initial position in the corresponding set split. Must be bigger or equal than 0 and smaller than
                       final.
          :param final: final position in the corresponding set split.
          :return: Y, list of output data variables from sample 'init' to 'final' belonging to the chosen 'set_name'
          """
          self.__checkSetName(set_name)
          self.__isLoaded(set_name, 1)
  
          if final > getattr(self, 'len_' + set_name):
              raise Exception('"final" index must be smaller than the number of samples in the set.')
          if init < 0:
              raise Exception('"init" index must be equal or greater than 0.')
          if init >= final:
              raise Exception('"init" index must be smaller than "final" index.')
  
          # Recover output samples
          Y = []
          for id_out in list(self.ids_outputs):
              types_index = self.ids_outputs.index(id_out)
              type_out = self.types_outputs[set_name][types_index]
              y = getattr(self, 'Y_' + set_name)[id_out][init:final]
              # Pre-process outputs
              if not get_only_ids:
                  if type_out == 'categorical':
                      nClasses = len(self.dic_classes[id_out])
                      y = self.loadCategorical(y, nClasses)
                  elif type_out == 'binary':
                      y = self.loadBinary(y, id_out)
                  elif type_out == 'real':
                      y = np.array(y).astype(np.float32)
                  elif type_out == '3DLabel':
                      nClasses = len(self.classes[id_out])
                      assoc_id_in = self.id_in_3DLabel[id_out]
                      imlist = getattr(self, 'Y_' + set_name)[assoc_id_in][init:final]
  
                      y = self.load3DLabels(y,
                                            nClasses,
                                            dataAugmentation,
                                            None,
                                            self.img_size[assoc_id_in],
                                            self.img_size_crop[assoc_id_in],
                                            imlist)
                  elif type_out == '3DSemanticLabel':
                      nClasses = len(self.classes[id_out])
                      classes_to_colour = self.semantic_classes[id_out]
                      assoc_id_in = self.id_in_3DLabel[id_out]
                      imlist = getattr(self, 'Y_' + set_name)[assoc_id_in][init:final]
                      y = self.load3DSemanticLabels(y,
                                                    nClasses,
                                                    classes_to_colour,
                                                    dataAugmentation,
                                                    None,
                                                    self.img_size[assoc_id_in],
                                                    self.img_size_crop[assoc_id_in],
                                                    imlist)
                  elif type_out == 'text-features':
                      y = self.loadTextFeaturesOneHot(y,
                                                      self.vocabulary_len[id_out],
                                                      self.max_text_len[id_out][set_name],
                                                      self.pad_on_batch[id_out],
                                                      self.text_offset.get(id_out, 0),
                                                      sample_weights=self.sample_weights[id_out][set_name],
                                                      label_smoothing=self.label_smoothing[id_out][set_name])
  
                  elif type_out == 'text':
                      y = self.loadTextOneHot(y,
                                              self.vocabulary[id_out],
                                              self.vocabulary_len[id_out],
                                              self.max_text_len[id_out][set_name],
                                              self.text_offset[id_out],
                                              self.fill_text[id_out],
                                              self.pad_on_batch[id_out],
                                              self.words_so_far[id_out],
                                              sample_weights=self.sample_weights[id_out][set_name],
                                              loading_X=False,
                                              label_smoothing=self.label_smoothing[id_out][set_name])
                  elif type_out == 'dense-text':
                      y = self.loadText(y,
                                        self.vocabulary[id_out],
                                        self.max_text_len[id_out][set_name],
                                        self.text_offset[id_out],
                                        self.fill_text[id_out],
                                        self.pad_on_batch[id_out],
                                        self.words_so_far[id_out],
                                        loading_X=False)
  
                      y = (y[0][:, :, None], y[1])
  
              Y.append(y)
  
          return Y
  
      def getXY(self, set_name, k, normalization_type='(-1)-1',
                normalization=False, meanSubstraction=False,
                dataAugmentation=False,
                wo_da_patch_type='whole', da_patch_type='resize_and_rndcrop', da_enhance_list=None,
                get_only_ids=False):
          """
          Gets the [X,Y] pairs for the next 'k' samples in the desired set.
          :param set_name: 'train', 'val' or 'test' set
          :param k: number of consecutive samples retrieved from the corresponding set.
          # 'raw-image', 'video', 'image-features' and 'video-features'-related parameters
          :param normalization: indicates if we want to normalize the data.
          # 'image-features' and 'video-features'-related parameters
          :param normalization_type: indicates the type of normalization applied. See available types in
                                     self.__available_norm_im_vid for 'image' and 'video' and self.__available_norm_feat
                                     for 'image-features' and 'video-features'.
          # 'raw-image' and 'video'-related parameters
          :param meanSubstraction: indicates if we want to substract the training mean from the returned images
                                   (only applicable if normalization=True)
          :param dataAugmentation: indicates if we want to apply data augmentation to the loaded images
                                  (random flip and cropping)
          :return: [X,Y], list of input and output data variables of the next 'k' consecutive samples belonging to
                   the chosen 'set_name'
          """
          self.__checkSetName(set_name)
          self.__isLoaded(set_name, 0)
          self.__isLoaded(set_name, 1)
          if da_enhance_list is None:
              da_enhance_list = []
  
          [new_last, last, surpassed] = self.__getNextSamples(k, set_name)
  
          # Recover input samples
          X = []
  
          for id_in in list(self.ids_inputs):
              types_index = self.ids_inputs.index(id_in)
              type_in = self.types_inputs[set_name][types_index]
              if id_in in self.optional_inputs:
                  try:
                      if surpassed:
                          x = getattr(self, 'X_' + set_name)[id_in][last:] + getattr(self, 'X_' + set_name)[id_in][0:new_last]
                      else:
                          x = getattr(self, 'X_' + set_name)[id_in][last:new_last]
                  except Exception:
                      x = []
              else:
                  if surpassed:
                      x = getattr(self, 'X_' + set_name)[id_in][last:] + getattr(self, 'X_' + set_name)[id_in][0:new_last]
                  else:
                      x = getattr(self, 'X_' + set_name)[id_in][last:new_last]
  
              # Pre-process inputs
              if not get_only_ids:
                  if type_in == 'text-features':
                      x = self.loadTextFeatures(x,
                                                self.max_text_len[id_in][set_name],
                                                self.pad_on_batch[id_in],
                                                self.text_offset.get(id_in, 0))[0]
  
                  elif type_in == 'image-features':
                      x = self.loadFeatures(x,
                                            self.features_lengths[id_in],
                                            normalization_type,
                                            normalization,
                                            data_augmentation=dataAugmentation)
                  elif type_in == 'video-features':
                      x = self.loadVideoFeatures(x,
                                                 id_in,
                                                 set_name,
                                                 self.max_video_len[id_in],
                                                 normalization_type,
                                                 normalization,
                                                 self.features_lengths[id_in],
                                                 data_augmentation=dataAugmentation)
                  elif type_in == 'text' or type_in == 'dense-text':
                      x = self.loadText(x,
                                        self.vocabulary[id_in],
                                        self.max_text_len[id_in][set_name],
                                        self.text_offset[id_in],
                                        fill=self.fill_text[id_in],
                                        pad_on_batch=self.pad_on_batch[id_in],
                                        words_so_far=self.words_so_far[id_in],
                                        loading_X=True)[0]
                  elif type_in == 'raw-image':
                      daRandomParams = None
                      if dataAugmentation:
                          daRandomParams = self.getDataAugmentationRandomParams(x, id_in)
                      x = self.loadImages(x,
                                          id_in,
                                          normalization_type,
                                          normalization,
                                          meanSubstraction,
                                          dataAugmentation,
                                          daRandomParams,
                                          wo_da_patch_type,
                                          da_patch_type,
                                          da_enhance_list)
                  elif type_in == 'video':
                      x = self.loadVideos(x,
                                          id_in,
                                          last,
                                          set_name,
                                          self.max_video_len[id_in],
                                          normalization_type,
                                          normalization,
                                          meanSubstraction,
                                          dataAugmentation)
                  elif type_in == 'categorical':
                      nClasses = len(self.dic_classes[id_in])
                      # load_sample_weights = self.sample_weights[id_out][set_name]
                      x = self.loadCategorical(x,
                                               nClasses)
                  elif type_in == 'categorical_raw':
                      x = np.array(x)
                  elif type_in == 'binary':
                      x = self.loadBinary(x, id_in)
              X.append(x)
  
          # Recover output samples
          Y = []
          for id_out in list(self.ids_outputs):
              types_index = self.ids_outputs.index(id_out)
              type_out = self.types_outputs[set_name][types_index]
              if surpassed:
                  y = getattr(self, 'Y_' + set_name)[id_out][last:] + getattr(self, 'Y_' + set_name)[id_out][0:new_last]
  
              else:
                  y = getattr(self, 'Y_' + set_name)[id_out][last:new_last]
  
              # Pre-process outputs
              if not get_only_ids:
                  if type_out == 'categorical':
                      nClasses = len(self.dic_classes[id_out])
                      # load_sample_weights = self.sample_weights[id_out][set_name]
                      y = self.loadCategorical(y,
                                               nClasses)
                  elif type_out == 'binary':
                      y = self.loadBinary(y,
                                          id_out)
                  elif type_out == 'real':
                      y = np.array(y).astype(np.float32)
                  elif type_out == '3DLabel':
                      nClasses = len(self.classes[id_out])
                      assoc_id_in = self.id_in_3DLabel[id_out]
                      if surpassed:
                          imlist = getattr(self, 'X_' + set_name)[assoc_id_in][last:] + getattr(self, 'X_' + set_name)[assoc_id_in][0:new_last]
  
                      else:
                          imlist = getattr(self, 'X_' + set_name)[assoc_id_in][last:new_last]
  
                      y = self.load3DLabels(y,
                                            nClasses,
                                            dataAugmentation,
                                            daRandomParams,
                                            self.img_size[assoc_id_in],
                                            self.img_size_crop[assoc_id_in],
                                            imlist)
                  elif type_out == '3DSemanticLabel':
                      nClasses = len(self.classes[id_out])
                      classes_to_colour = self.semantic_classes[id_out]
                      assoc_id_in = self.id_in_3DLabel[id_out]
                      if surpassed:
                          imlist = getattr(self, 'X_' + set_name)[assoc_id_in][last:] + getattr(self, 'X_' + set_name)[assoc_id_in][0:new_last]
                      else:
                          imlist = getattr(self, 'X_' + set_name)[assoc_id_in][last:new_last]
  
                      y = self.load3DSemanticLabels(y,
                                                    nClasses,
                                                    classes_to_colour,
                                                    dataAugmentation,
                                                    daRandomParams,
                                                    self.img_size[assoc_id_in],
                                                    self.img_size_crop[assoc_id_in],
                                                    imlist)
  
                  elif type_out == 'text-features':
                      y = self.loadTextFeaturesOneHot(y,
                                                      self.vocabulary_len[id_out],
                                                      self.max_text_len[id_out][set_name],
                                                      self.pad_on_batch[id_out],
                                                      self.text_offset.get(id_out, 0),
                                                      sample_weights=self.sample_weights[id_out][set_name],
                                                      label_smoothing=self.label_smoothing[id_out][set_name])
  
                  elif type_out == 'text':
                      y = self.loadTextOneHot(y,
                                              self.vocabulary[id_out],
                                              self.vocabulary_len[id_out],
                                              self.max_text_len[id_out][set_name],
                                              self.text_offset[id_out],
                                              self.fill_text[id_out],
                                              self.pad_on_batch[id_out],
                                              self.words_so_far[id_out],
                                              sample_weights=self.sample_weights[id_out][set_name],
                                              loading_X=False,
                                              label_smoothing=self.label_smoothing[id_out][set_name])
  
                  elif type_out == 'dense-text':
                      y = self.loadText(y,
                                        self.vocabulary[id_out],
                                        self.max_text_len[id_out][set_name],
                                        self.text_offset[id_out],
                                        self.fill_text[id_out],
                                        self.pad_on_batch[id_out],
                                        self.words_so_far[id_out],
                                        loading_X=False)
  
                      if self.label_smoothing[id_out][set_name] > 0.:
                          y[0] = self.apply_label_smoothing(y[0],
                                                            self.label_smoothing[id_out][set_name],
                                                            self.vocabulary_len[id_out])
                      y = (y[0][:, :, None], y[1])
  
              Y.append(y)
  
          return [X, Y]
  
      def getXY_FromIndices(self, set_name, k, normalization_type='(-1)-1',
                            normalization=False, meanSubstraction=False,
                            dataAugmentation=False,
                            wo_da_patch_type='whole', da_patch_type='resize_and_rndcrop', da_enhance_list=None,
                            get_only_ids=False):
          """
          Gets the [X,Y] pairs for the samples in positions 'k' in the desired set.
          :param set_name: 'train', 'val' or 'test' set
          :param k: positions of the desired samples
          # 'raw-image', 'video', 'image-features' and 'video-features'-related parameters
          :param normalization: indicates if we want to normalize the data.
          # 'image-features' and 'video-features'-related parameters
          :param normalization_type: indicates the type of normalization applied. See available types in
                                      self.__available_norm_im_vid for 'raw-image' and 'video' and
                                      self.__available_norm_feat for 'image-features' and 'video-features'.
          # 'raw-image' and 'video'-related parameters
          :param meanSubstraction: indicates if we want to substract the training mean from the returned images
                                   (only applicable if normalization=True)
          :param dataAugmentation: indicates if we want to apply data augmentation to the loaded images
                                   (random flip and cropping)
          :return: [X,Y], list of input and output data variables of the samples identified by the indices in 'k'
                   samples belonging to the chosen 'set_name'
          """
  
          self.__checkSetName(set_name)
          self.__isLoaded(set_name, 0)
          self.__isLoaded(set_name, 1)
          if da_enhance_list is None:
              da_enhance_list = []
  
          # Recover input samples
          X = []
          k = list(k)
          for id_in in list(self.ids_inputs):
              types_index = self.ids_inputs.index(id_in)
              type_in = self.types_inputs[set_name][types_index]
              ghost_x = False
              if id_in in self.optional_inputs:
                  try:
                      x = [getattr(self, 'X_' + set_name)[id_in][index] for index in k]
  
                  except Exception:
                      x = [[]] * len(k)
                      ghost_x = True
              else:
                  x = [getattr(self, 'X_' + set_name)[id_in][index] for index in k]
  
              # Pre-process inputs
              if not get_only_ids and not ghost_x:
                  if type_in == 'text-features':
                      x = self.loadTextFeatures(x,
                                                self.max_text_len[id_in][set_name],
                                                self.pad_on_batch[id_in],
                                                self.text_offset.get(id_in, 0))[0]
                  elif type_in == 'image-features':
                      x = self.loadFeatures(x,
                                            self.features_lengths[id_in],
                                            normalization_type,
                                            normalization,
                                            data_augmentation=dataAugmentation)
                  elif type_in == 'video-features':
                      x = self.loadVideoFeatures(x,
                                                 id_in,
                                                 set_name,
                                                 self.max_video_len[id_in],
                                                 normalization_type,
                                                 normalization,
                                                 self.features_lengths[id_in],
                                                 data_augmentation=dataAugmentation)
                  if type_in == 'raw-image':
                      daRandomParams = None
                      if dataAugmentation:
                          daRandomParams = self.getDataAugmentationRandomParams(x, id_in)
                      x = self.loadImages(x,
                                          id_in,
                                          normalization_type,
                                          normalization,
                                          meanSubstraction,
                                          dataAugmentation,
                                          daRandomParams,
                                          wo_da_patch_type,
                                          da_patch_type,
                                          da_enhance_list)
                  elif type_in == 'video':
                      x = self.loadVideosByIndex(x,
                                                 id_in,
                                                 k,
                                                 set_name,
                                                 self.max_video_len[id_in],
                                                 normalization_type,
                                                 normalization,
                                                 meanSubstraction,
                                                 dataAugmentation)
                  elif type_in == 'text':
                      x = self.loadText(x,
                                        self.vocabulary[id_in],
                                        self.max_text_len[id_in][set_name],
                                        self.text_offset[id_in],
                                        fill=self.fill_text[id_in],
                                        pad_on_batch=self.pad_on_batch[id_in],
                                        words_so_far=self.words_so_far[id_in],
                                        loading_X=True)[0]
                  elif type_in == 'categorical':
                      nClasses = len(self.dic_classes[id_in])
                      # load_sample_weights = self.sample_weights[id_out][set_name]
                      x = self.loadCategorical(x,
                                               nClasses)
                  elif type_in == 'categorical_raw':
                      x = np.array(x)
                  elif type_in == 'binary':
                      x = self.loadBinary(x,
                                          id_in)
              X.append(x)
  
          # Recover output samples
          Y = []
          for id_out in list(self.ids_outputs):
              types_index = self.ids_outputs.index(id_out)
              type_out = self.types_outputs[set_name][types_index]
  
              y = [getattr(self, 'Y_' + set_name)[id_out][index] for index in k]
  
              # Pre-process outputs
              if not get_only_ids:
                  if type_out == 'categorical':
                      nClasses = len(self.dic_classes[id_out])
                      y = self.loadCategorical(y, nClasses)
                  elif type_out == 'binary':
                      y = self.loadBinary(y, id_out)
                  elif type_out == 'real':
                      y = np.array(y).astype(np.float32)
                  elif type_out == '3DLabel':
                      nClasses = len(self.classes[id_out])
                      assoc_id_in = self.id_in_3DLabel[id_out]
                      imlist = [getattr(self, 'X_' + set_name)[assoc_id_in][index] for index in k]
  
                      y = self.load3DLabels(y, nClasses, dataAugmentation, daRandomParams,
                                            self.img_size[assoc_id_in], self.img_size_crop[assoc_id_in],
                                            imlist)
                  elif type_out == '3DSemanticLabel':
                      nClasses = len(self.classes[id_out])
                      classes_to_colour = self.semantic_classes[id_out]
                      assoc_id_in = self.id_in_3DLabel[id_out]
                      imlist = [getattr(self, 'X_' + set_name)[assoc_id_in][index] for index in k]
                      y = self.load3DSemanticLabels(y, nClasses, classes_to_colour, dataAugmentation, daRandomParams,
                                                    self.img_size[assoc_id_in], self.img_size_crop[assoc_id_in],
                                                    imlist)
  
                  elif type_out == 'text-features':
                      y = self.loadTextFeaturesOneHot(y,
                                                      self.vocabulary_len[id_out],
                                                      self.max_text_len[id_out][set_name],
                                                      self.pad_on_batch[id_out],
                                                      self.text_offset.get(id_out, 0),
                                                      sample_weights=self.sample_weights[id_out][set_name],
                                                      label_smoothing=self.label_smoothing[id_out][set_name])
  
                  elif type_out == 'text':
                      y = self.loadTextOneHot(y,
                                              self.vocabulary[id_out],
                                              self.vocabulary_len[id_out],
                                              self.max_text_len[id_out][set_name],
                                              self.text_offset[id_out],
                                              self.fill_text[id_out],
                                              self.pad_on_batch[id_out],
                                              self.words_so_far[id_out],
                                              sample_weights=self.sample_weights[id_out][set_name],
                                              loading_X=False,
                                              label_smoothing=self.label_smoothing[id_out][set_name])
  
                  elif type_out == 'dense-text':
                      y = self.loadText(y,
                                        self.vocabulary[id_out],
                                        self.max_text_len[id_out][set_name],
                                        self.text_offset[id_out],
                                        self.fill_text[id_out],
                                        self.pad_on_batch[id_out],
                                        self.words_so_far[id_out],
                                        loading_X=False)
  
                      if self.label_smoothing[id_out][set_name] > 0.:
                          y[0] = self.apply_label_smoothing(y[0],
                                                            self.label_smoothing[id_out][set_name],
                                                            self.vocabulary_len[id_out])
                      y = (y[0][:, :, None], y[1])
  
              Y.append(y)
  
          return [X, Y]
  
      def getX_FromIndices(self, set_name, k, normalization_type='(-1)-1',
                           normalization=False, meanSubstraction=False,
                           dataAugmentation=False,
                           wo_da_patch_type='whole', da_patch_type='resize_and_rndcrop', da_enhance_list=None,
                           get_only_ids=False):
          """
          Gets the [X,Y] pairs for the samples in positions 'k' in the desired set.
          :param set_name: 'train', 'val' or 'test' set
          :param k: positions of the desired samples
          # 'raw-image', 'video', 'image-features' and 'video-features'-related parameters
          :param normalization: indicates if we want to normalize the data.
          # 'image-features' and 'video-features'-related parameters
          :param normalization_type: indicates the type of normalization applied. See available types in
                                     self.__available_norm_im_vid for 'raw-image' and 'video' and
                                     self.__available_norm_feat for 'image-features' and 'video-features'.
          # 'raw-image' and 'video'-related parameters
          :param meanSubstraction: indicates if we want to substract the training mean from the returned images
                                  (only applicable if normalization=True)
          :param dataAugmentation: indicates if we want to apply data augmentation to the loaded images
                                  (random flip and cropping)
          :return: [X,Y], list of input and output data variables of the samples identified by the indices in 'k'
                   samples belonging to the chosen 'set_name'
          """
  
          self.__checkSetName(set_name)
          self.__isLoaded(set_name, 0)
          if da_enhance_list is None:
              da_enhance_list = []
          # Recover input samples
          X = []
          for id_in in list(self.ids_inputs):
              types_index = self.ids_inputs.index(id_in)
              type_in = self.types_inputs[set_name][types_index]
              ghost_x = False
              if id_in in self.optional_inputs:
                  try:
                      x = [getattr(self, 'X_' + set_name)[id_in][index] for index in k]
                  except Exception:
                      x = [[]] * len(k)
                      ghost_x = True
              else:
                  x = [getattr(self, 'X_' + set_name)[id_in][index] for index in k]
  
              # Pre-process inputs
              if not get_only_ids and not ghost_x:
                  if type_in == 'text-features':
                      x = self.loadTextFeatures(x,
                                                self.max_text_len[id_in][set_name],
                                                self.pad_on_batch[id_in],
                                                self.text_offset[id_in])[0]
                  elif type_in == 'image-features':
                      x = self.loadFeatures(x,
                                            self.features_lengths[id_in],
                                            normalization_type,
                                            normalization,
                                            data_augmentation=dataAugmentation)
                  elif type_in == 'video-features':
                      x = self.loadVideoFeatures(x,
                                                 id_in,
                                                 set_name,
                                                 self.max_video_len[id_in],
                                                 normalization_type,
                                                 normalization,
                                                 self.features_lengths[id_in],
                                                 data_augmentation=dataAugmentation)
                  elif type_in == 'raw-image':
                      daRandomParams = None
                      if dataAugmentation:
                          daRandomParams = self.getDataAugmentationRandomParams(x, id_in)
                      x = self.loadImages(x,
                                          id_in,
                                          normalization_type,
                                          normalization,
                                          meanSubstraction,
                                          dataAugmentation,
                                          daRandomParams,
                                          wo_da_patch_type,
                                          da_patch_type,
                                          da_enhance_list)
                  elif type_in == 'video':
                      x = self.loadVideosByIndex(x,
                                                 id_in,
                                                 k,
                                                 set_name,
                                                 self.max_video_len[id_in],
                                                 normalization_type,
                                                 normalization,
                                                 meanSubstraction,
                                                 dataAugmentation)
                  elif type_in == 'text':
                      x = self.loadText(x,
                                        self.vocabulary[id_in],
                                        self.max_text_len[id_in][set_name],
                                        self.text_offset[id_in],
                                        fill=self.fill_text[id_in],
                                        pad_on_batch=self.pad_on_batch[id_in],
                                        words_so_far=self.words_so_far[id_in],
                                        loading_X=True)[0]
                  elif type_in == 'categorical':
                      nClasses = len(self.dic_classes[id_in])
                      x = self.loadCategorical(x,
                                               nClasses)
                  elif type_in == 'categorical_raw':
                      x = np.array(x)
                  elif type_in == 'binary':
                      x = self.loadBinary(x,
                                          id_in)
              X.append(x)
  
          return X
  
      def getY_FromIndices(self, set_name, k, dataAugmentation=False, return_mask=True,
                           wo_da_patch_type='whole', da_patch_type='resize_and_rndcrop', da_enhance_list=None,
                           get_only_ids=False):
          """
          Gets the [Y] pairs for the samples in positions 'k' in the desired set.
          :param set_name: 'train', 'val' or 'test' set
          :param k: positions of the desired samples
          # 'raw-image', 'video', 'image-features' and 'video-features'-related parameters
          :param normalization: indicates if we want to normalize the data.
          # 'image-features' and 'video-features'-related parameters
          :param normalization_type: indicates the type of normalization applied. See available types in
                                     self.__available_norm_im_vid for 'raw-image' and 'video' and
                                     self.__available_norm_feat for 'image-features' and 'video-features'.
          # 'raw-image' and 'video'-related parameters
          :param meanSubstraction: indicates if we want to substract the training mean from the returned images
                                  (only applicable if normalization=True)
          :param dataAugmentation: indicates if we want to apply data augmentation to the loaded images
                                  (random flip and cropping)
          :return: [X,Y], list of input and output data variables of the samples identified by the indices in 'k'
                   samples belonging to the chosen 'set_name'
          """
  
          self.__checkSetName(set_name)
          self.__isLoaded(set_name, 0)
          if da_enhance_list is None:
              da_enhance_list = []
  
          # Recover output samples
          Y = []
          for id_out in list(self.ids_outputs):
              types_index = self.ids_outputs.index(id_out)
              type_out = self.types_outputs[set_name][types_index]
  
              y = [getattr(self, 'Y_' + set_name)[id_out][index] for index in k]
  
              # Pre-process outputs
              if not get_only_ids:
                  if type_out == 'categorical':
                      nClasses = len(self.dic_classes[id_out])
                      y = self.loadCategorical(y, nClasses)
                  elif type_out == 'binary':
                      y = self.loadBinary(y, id_out)
                  elif type_out == 'real':
                      y = np.array(y).astype(np.float32)
                  elif type_out == '3DLabel':
                      nClasses = len(self.classes[id_out])
                      assoc_id_in = self.id_in_3DLabel[id_out]
                      imlist = [getattr(self, 'X_' + set_name)[assoc_id_in][index] for index in k]
  
                      y = self.load3DLabels(y, nClasses, dataAugmentation, daRandomParams,
                                            self.img_size[assoc_id_in], self.img_size_crop[assoc_id_in],
                                            imlist)
                  elif type_out == '3DSemanticLabel':
                      nClasses = len(self.classes[id_out])
                      classes_to_colour = self.semantic_classes[id_out]
                      assoc_id_in = self.id_in_3DLabel[id_out]
                      imlist = [getattr(self, 'X_' + set_name)[assoc_id_in][index] for index in k]
                      y = self.load3DSemanticLabels(y, nClasses, classes_to_colour, dataAugmentation, daRandomParams,
                                                    self.img_size[assoc_id_in], self.img_size_crop[assoc_id_in],
                                                    imlist)
  
                  elif type_out == 'text-features':
                      y = self.loadTextFeaturesOneHot(y,
                                                      self.vocabulary_len[id_out],
                                                      self.max_text_len[id_out][set_name],
                                                      self.pad_on_batch[id_out],
                                                      self.text_offset.get(id_out, 0),
                                                      sample_weights=self.sample_weights[id_out][set_name],
                                                      label_smoothing=self.label_smoothing[id_out][set_name])
  
                  elif type_out == 'text':
                      y = self.loadTextOneHot(y,
                                              self.vocabulary[id_out],
                                              self.vocabulary_len[id_out],
                                              self.max_text_len[id_out][set_name],
                                              self.text_offset[id_out],
                                              self.fill_text[id_out],
                                              self.pad_on_batch[id_out],
                                              self.words_so_far[id_out],
                                              sample_weights=self.sample_weights[id_out][set_name],
                                              loading_X=False,
                                              label_smoothing=self.label_smoothing[id_out][set_name])
                      if not return_mask:
                          y = y[0]
                  elif type_out == 'dense-text':
                      y = self.loadText(y,
                                        self.vocabulary[id_out],
                                        self.max_text_len[id_out][set_name],
                                        self.text_offset[id_out],
                                        self.fill_text[id_out],
                                        self.pad_on_batch[id_out],
                                        self.words_so_far[id_out],
                                        loading_X=False)
  
                      if self.label_smoothing[id_out][set_name] > 0.:
                          y[0] = self.apply_label_smoothing(y[0],
                                                            self.label_smoothing[id_out][set_name],
                                                            self.vocabulary_len[id_out])
                      y = (y[0][:, :, None], y[1]) if return_mask else y[0][:, :, None]
  
              Y.append(y)
          return Y
  
      # ------------------------------------------------------- #
      #       AUXILIARY FUNCTIONS
      #
      # ------------------------------------------------------- #
  
      def __str__(self):
          """
          Prints the basic input-output information of the Dataset instance.
  
          :return: String representation of the Dataset.
          """
  
          str_ = '---------------------------------------------\n'
          str_ += '\tDataset ' + self.name + '\n'
          str_ += '---------------------------------------------\n'
          str_ += 'store path: ' + self.path + '\n'
          str_ += 'data length: ' + '\n'
          str_ += '\ttrain - ' + str(self.len_train) + '\n'
          str_ += '\tval   - ' + str(self.len_val) + '\n'
          str_ += '\ttest  - ' + str(self.len_test) + '\n'
  
          str_ += '\n'
          str_ += '[ INPUTS ]\n'
          for id_in, type_in in list(zip(self.ids_inputs, self.types_inputs)):
              str_ += type_in + ': ' + id_in + '\n'
  
          str_ += '\n'
          str_ += '[ OUTPUTS ]\n'
          for id_out, type_out in list(zip(self.ids_outputs, self.types_outputs)):
              str_ += type_out + ': ' + id_out + '\n'
          str_ += '---------------------------------------------\n'
          return str_
  
      def __isLoaded(self, set_name, pos):
          """
          Checks if the data from set_name at pos is already loaded
          :param set_name:
          :param pos:
          :return:
          """
          if not getattr(self, 'loaded_' + set_name)[pos]:
              if pos == 0:
                  raise Exception('Set ' + set_name + ' samples are not loaded yet.')
              elif pos == 1:
                  raise Exception('Set ' + set_name + ' labels are not loaded yet.')
          return
  
      @staticmethod
      def __checkSetName(set_name):
          """
          Checks name of a split.
          Only "train", "val" or "test" are valid set names.
          :param set_name: Split name
          :return: Boolean specifying the validity of the name
          """
          if set_name != 'train' and set_name != 'val' and set_name != 'test':
              raise Exception(
                  'Incorrect set_name specified "' + set_name + '"\nOnly "train", "val" or "test" are valid set names.')
          return
  
      def __checkLengthSet(self, set_name):
          """
          Check that the length of the inputs and outputs match. Only checked if the input is not optional.
          :param set_name:
          :return:
          """
          if getattr(self, 'loaded_' + set_name)[0] and getattr(self, 'loaded_' + set_name)[1]:
              lengths = []
              plot_ids_in = []
              for id_in in self.ids_inputs:
                  if id_in not in self.optional_inputs:
                      plot_ids_in.append(id_in)
                      lengths.append(len(getattr(self, 'X_' + set_name)[id_in]))
              for id_out in self.ids_outputs:
                  lengths.append(len(getattr(self, 'Y_' + set_name)[id_out]))
  
              if lengths[1:] != lengths[:-1]:
                  raise Exception('Inputs and outputs size '
                                  '(' + str(lengths) + ') for "' +
                                  set_name + '" set do not match.\n \t Inputs:' +
                                  str(plot_ids_in) + '\t Outputs:' + str(self.ids_outputs))
  
      def __getNextSamples(self, k, set_name):
          """
              Gets the indices to the next K samples we are going to read.
          """
          new_last = getattr(self, 'last_' + set_name) + k
          last = getattr(self, 'last_' + set_name)
          length = getattr(self, 'len_' + set_name)
          if new_last > length:
              new_last = new_last - length
              surpassed = True
          else:
              surpassed = False
          setattr(self, 'last_' + set_name, new_last)
  
          # self.__lock_read.release()  # UNLOCK
  
          return [new_last, last, surpassed]
  
      def __getstate__(self):
          """
              Behaviour applied when pickling a Dataset instance.
          """
          obj_dict = self.__dict__.copy()
          # del obj_dict['_Dataset__lock_read']
          return obj_dict
  
      def __setstate__(self, new_state):
          """
              Behaviour applied when unpickling a Dataset instance.
          """
          # dict['_Dataset__lock_read'] = threading.Lock()
          self.__dict__ = new_state
  
      # Deprecated methods. Maintained for backwards compatibility
  
      def setListGeneral(self, path_list, split=None, shuffle=True, type='raw-image', id='image'):
          """
          Deprecated
          """
          if split is None:
              split = [0.8, 0.1, 0.1]
          logger.info("WARNING: The method setListGeneral() is deprecated, consider using setInput() instead.")
          self.setInput(path_list, split, type=type, id=id)
  
      def setList(self, path_list, set_name, type='raw-image', id='image'):
          """
          DEPRECATED
          """
          logger.info("WARNING: The method setList() is deprecated, consider using setInput() instead.")
          self.setInput(path_list, set_name, type, id)
  
      def setLabels(self, labels_list, set_name, type='categorical', id='label'):
          """
              DEPRECATED
          """
          logger.info("WARNING: The method setLabels() is deprecated, consider using setOutput() instead.")
          self.setOutput(labels_list, set_name, type=type, id=id)
  