text: from keras.models import Model
  from keras.applications import resnet50, vgg16
  from keras.utils import np_utils
  import numpy as np
  from keras.preprocessing import image
  from sklearn import metrics
  from keras.layers import *
  from config import batch_size, image_size_h_p, image_size_w_p, image_size_h_c, image_size_w_c, nchannels
  from tensorflow.python.keras.utils.data_utils import Sequence
  
  # global constants
  DIM_ORDERING = 'tf'
  CONCAT_AXIS = -1
  def inception_module(x, params, dim_ordering, concat_axis,
                       subsample=(1, 1), activation='relu',
                       border_mode='same', weight_decay=None):
  
      # https://gist.github.com/nervanazoo/2e5be01095e935e90dd8  #
      # file-googlenet_neon-py
  
      (branch1, branch2, branch3, branch4) = params
  
      if weight_decay:
          W_regularizer = regularizers.l2(weight_decay)
          b_regularizer = regularizers.l2(weight_decay)
      else:
          W_regularizer = None
          b_regularizer = None
  
      pathway1 = Convolution2D(branch1[0], 1, 1,
                               subsample=subsample,
                               activation=activation,
                               border_mode=border_mode,
                               W_regularizer=W_regularizer,
                               b_regularizer=b_regularizer,
                               bias=False,
                               dim_ordering=dim_ordering)(x)
  
      pathway2 = Convolution2D(branch2[0], 1, 1,
                               subsample=subsample,
                               activation=activation,
                               border_mode=border_mode,
                               W_regularizer=W_regularizer,
                               b_regularizer=b_regularizer,
                               bias=False,
                               dim_ordering=dim_ordering)(x)
      pathway2 = Convolution2D(branch2[1], 3, 3,
                               subsample=subsample,
                               activation=activation,
                               border_mode=border_mode,
                               W_regularizer=W_regularizer,
                               b_regularizer=b_regularizer,
                               bias=False,
                               dim_ordering=dim_ordering)(pathway2)
  
      pathway3 = Convolution2D(branch3[0], 1, 1,
                               subsample=subsample,
                               activation=activation,
                               border_mode=border_mode,
                               W_regularizer=W_regularizer,
                               b_regularizer=b_regularizer,
                               bias=False,
                               dim_ordering=dim_ordering)(x)
      pathway3 = Convolution2D(branch3[1], 5, 5,
                               subsample=subsample,
                               activation=activation,
                               border_mode=border_mode,
                               W_regularizer=W_regularizer,
                               b_regularizer=b_regularizer,
                               bias=False,
                               dim_ordering=dim_ordering)(pathway3)
  
      pathway4 = MaxPooling2D(pool_size=(1, 1), dim_ordering=DIM_ORDERING)(x)
      pathway4 = Convolution2D(branch4[0], 1, 1,
                               subsample=subsample,
                               activation=activation,
                               border_mode=border_mode,
                               W_regularizer=W_regularizer,
                               b_regularizer=b_regularizer,
                               bias=False,
                               dim_ordering=dim_ordering)(pathway4)
  
      return Concatenate()([pathway1, pathway2, pathway3, pathway4])
  
  
  def conv_layer(x, nb_filter, nb_row, nb_col, dim_ordering,
                 subsample=(1, 1), activation='relu',
                 border_mode='same', weight_decay=None, padding=None):
  
      if weight_decay:
          W_regularizer = regularizers.l2(weight_decay)
          b_regularizer = regularizers.l2(weight_decay)
      else:
          W_regularizer = None
          b_regularizer = None
  
      x = Convolution2D(nb_filter, nb_row, nb_col,
                        subsample=subsample,
                        activation=activation,
                        border_mode=border_mode,
                        W_regularizer=W_regularizer,
                        b_regularizer=b_regularizer,
                        bias=False,
                        dim_ordering=dim_ordering)(x)
  
      if padding:
          for i in range(padding):
              x = ZeroPadding2D(padding=(1, 1), dim_ordering=DIM_ORDERING)(x)
  
      return x
  
  
  def GoogLeNet(input_shape):
      img_input = Input(input_shape)
      x = conv_layer(img_input, nb_col=7, nb_filter=64,
                     nb_row=7, dim_ordering=DIM_ORDERING, padding=3)
      x = MaxPooling2D(
          strides=(
              3, 3), pool_size=(
                  2, 2), dim_ordering=DIM_ORDERING)(x)
  
      x = conv_layer(x, nb_col=1, nb_filter=64,
                     nb_row=1, dim_ordering=DIM_ORDERING)
      x = conv_layer(x, nb_col=3, nb_filter=192,
                     nb_row=3, dim_ordering=DIM_ORDERING, padding=1)
      x = MaxPooling2D(
          strides=(
              3, 3), pool_size=(
                  2, 2), dim_ordering=DIM_ORDERING)(x)
  
      x = inception_module(x, params=[(64, ), (96, 128), (16, 32), (32, )],
                           dim_ordering=DIM_ORDERING, concat_axis=CONCAT_AXIS)
      x = inception_module(x, params=[(128,), (128, 192), (32, 96), (64, )],
                           dim_ordering=DIM_ORDERING, concat_axis=CONCAT_AXIS)
  
      x = MaxPooling2D(
          strides=(
              1, 1), pool_size=(
                  1, 1), dim_ordering=DIM_ORDERING)(x)
      x = ZeroPadding2D(padding=(1, 1), dim_ordering=DIM_ORDERING)(x)
  
      x = inception_module(x, params=[(192,), (96, 208), (16, 48), (64, )],
                           dim_ordering=DIM_ORDERING, concat_axis=CONCAT_AXIS)
      # AUX 1 - Branch HERE
      x = inception_module(x, params=[(160,), (112, 224), (24, 64), (64, )],
                           dim_ordering=DIM_ORDERING, concat_axis=CONCAT_AXIS)
      x = inception_module(x, params=[(128,), (128, 256), (24, 64), (64, )],
                           dim_ordering=DIM_ORDERING, concat_axis=CONCAT_AXIS)
      x = inception_module(x, params=[(112,), (144, 288), (32, 64), (64, )],
                           dim_ordering=DIM_ORDERING, concat_axis=CONCAT_AXIS)
      # AUX 2 - Branch HERE
      x = inception_module(x, params=[(256,), (160, 320), (32, 128), (128,)],
                           dim_ordering=DIM_ORDERING, concat_axis=CONCAT_AXIS)
      x = MaxPooling2D(
          strides=(
              1, 1), pool_size=(
                  1, 1), dim_ordering=DIM_ORDERING)(x)
  
      x = inception_module(x, params=[(256,), (160, 320), (32, 128), (128,)],
                           dim_ordering=DIM_ORDERING, concat_axis=CONCAT_AXIS)
      x = inception_module(x, params=[(384,), (192, 384), (48, 128), (128,)],
                           dim_ordering=DIM_ORDERING, concat_axis=CONCAT_AXIS)
      x = AveragePooling2D(strides=(1, 1), dim_ordering=DIM_ORDERING)(x)
      x = Flatten()(x)
  
      return Model(img_input,x)
  
  
  
  def matchnet(input_shape):
      input1 = Input(input_shape) 
      x = Conv2D(24, kernel_size=(7, 7),activation='relu')(input1)
  
      # pool0
      x = MaxPooling2D(pool_size=(3, 3), strides=2)(x)
      # conv1
      x = Conv2D(64, kernel_size=(5, 5),
                  activation='relu')(x)
      # pool1
      x = MaxPooling2D(pool_size=(3, 3), strides=2)(x)
      # conv2
      x = Conv2D(96, kernel_size=(3, 3),
                  activation='relu')(x)
      # conv3
      x = Conv2D(96, kernel_size=(3, 3),
                  activation='relu')(x)
      # conv4
      x = Conv2D(64, kernel_size=(3, 3),
                  activation='relu')(x)
      # pool4
      x = MaxPooling2D(pool_size=(3, 3), strides=2)(x)
      # bottleneck
      x = Flatten()(x)
      return Model(input1,x)
  
  def lenet5(input_shape):
      input1 = Input(input_shape) 
      x = Conv2D(20, [5, 5], padding='same', activation='relu')(input1)
  
      x = MaxPooling2D((2, 2), strides=2)(x)
  
      x = Conv2D(50, (5, 5), padding='same', activation='relu')(x)
      x = MaxPooling2D((2, 2), strides=2)(x)
  
      x = Flatten()(x)
      return Model(input1,x)
  
  def mccnn(input_shape):
      input1 = Input(input_shape) 
      x = Conv2D(112, (3,3), activation='relu')(input1)
      x = Conv2D(112, (3,3), activation='relu' )(x)
      x = Conv2D(112, (3,3), activation='relu' )(x)
      x = Conv2D(112, (3,3), activation='relu' )(x)
      x = MaxPooling2D((2,2))(x)
      x = Flatten()(x)
      return Model(input1,x)
  
  def resnet6(input_shape):
  
      input1 = Input(input_shape) 
  
      x1 = Conv2D(32, (5, 5), strides=[2,2], padding='same')(input1)
      x1 = MaxPooling2D(pool_size=(3, 3), strides=[2,2])(x1)
  
      # First residual block
      x2 = BatchNormalization()(x1)
      x2 = Activation('relu')(x2)
      x2 = Conv2D(32, (3, 3), strides=[2,2], padding='same',
                  kernel_initializer="he_normal",
                  kernel_regularizer=l2(1e-4))(x2)
  
      x2 = BatchNormalization()(x2)
      x2 = Activation('relu')(x2)
      x2 = Conv2D(32, (3, 3), padding='same',
                  kernel_initializer="he_normal",
                  kernel_regularizer=l2(1e-4))(x2)
  
      x1 = Conv2D(32, (1, 1), strides=[2,2], padding='same')(x1)
      x3 = add([x1, x2])
  
      # Second residual block
      x4 = BatchNormalization()(x3)
      x4 = Activation('relu')(x4)
      x4 = Conv2D(64, (3, 3), strides=[2,2], padding='same',
                  kernel_initializer="he_normal",
                  kernel_regularizer=l2(1e-4))(x4)
  
      x4 = BatchNormalization()(x4)
      x4 = Activation('relu')(x4)
      x4 = Conv2D(64, (3, 3), padding='same',
                  kernel_initializer="he_normal",
                  kernel_regularizer=l2(1e-4))(x4)
  
      x3 = Conv2D(64, (1, 1), strides=[2,2], padding='same')(x3)
      x5 = add([x3, x4])
  
      x = Flatten()(x5)
  
      return Model(input1,x)
  
  def resnet8(input_shape):
  
      input1 = Input(input_shape)
      x1 = Conv2D(32, (5, 5), strides=[2,2], padding='same')(input1)
      x1 = MaxPooling2D(pool_size=(3, 3), strides=[2,2])(x1)
  
      # First residual block
      x2 = BatchNormalization()(x1)
      x2 = Activation('relu')(x2)
      x2 = Conv2D(32, (3, 3), strides=[2,2], padding='same',
                  kernel_initializer="he_normal",
                  kernel_regularizer=l2(1e-4))(x2)
  
      x2 = BatchNormalization()(x2)
      x2 = Activation('relu')(x2)
      x2 = Conv2D(32, (3, 3), padding='same',
                  kernel_initializer="he_normal",
                  kernel_regularizer=l2(1e-4))(x2)
  
      x1 = Conv2D(32, (1, 1), strides=[2,2], padding='same')(x1)
      x3 = add([x1, x2])
  
      # Second residual block
      x4 = BatchNormalization()(x3)
      x4 = Activation('relu')(x4)
      x4 = Conv2D(64, (3, 3), strides=[2,2], padding='same',
                  kernel_initializer="he_normal",
                  kernel_regularizer=l2(1e-4))(x4)
  
      x4 = BatchNormalization()(x4)
      x4 = Activation('relu')(x4)
      x4 = Conv2D(64, (3, 3), padding='same',
                  kernel_initializer="he_normal",
                  kernel_regularizer=l2(1e-4))(x4)
  
      x3 = Conv2D(64, (1, 1), strides=[2,2], padding='same')(x3)
      x5 = add([x3, x4])
  
      # Third residual block
      x6 = BatchNormalization()(x5)
      x6 = Activation('relu')(x6)
      x6 = Conv2D(128, (3, 3), strides=[2,2], padding='same',
                  kernel_initializer="he_normal",
                  kernel_regularizer=l2(1e-4))(x6)
  
      x6 = BatchNormalization()(x6)
      x6 = Activation('relu')(x6)
      x6 = Conv2D(128, (3, 3), padding='same',
                  kernel_initializer="he_normal",
                  kernel_regularizer=l2(1e-4))(x6)
  
      x5 = Conv2D(128, (1, 1), strides=[2,2], padding='same')(x5)
      x7 = add([x5, x6])
  
      x = Flatten()(x7)
  
      return Model(input1,x) 
  
  def resnet50_model(input_shape):
    model = resnet50.ResNet50(weights=None,include_top=False,input_shape=input_shape)
    x = Flatten()(model.output)
    baseModel = Model(inputs=model.input,outputs=x)
    #for layer in baseModel.layers:
    #  layer.trainable = False
    return baseModel
  
  def vgg16_model(input_shape):
    model = vgg16.VGG16(weights=None, include_top=False, input_shape=input_shape)
    x = Flatten()(model.output)
    baseModel = Model(inputs=model.input,outputs=x)
    #for layer in baseModel.layers:
    #  layer.trainable = False
    return baseModel
  
  
  def small_vgg_car(input_shape):
      input1 = Input(input_shape)
      x = Conv2D(64, (3, 3), activation='relu', padding='same',name='block1_conv1')(input1)
      x = MaxPooling2D((2, 2), strides=(2, 2), name='block1_pool')(x)
  
      x = Conv2D(128, (3, 3), activation='relu', padding='same',name='block2_conv1')(x)
      x = MaxPooling2D((2, 2), strides=(2, 2), name='block2_pool')(x)
  
      # Block 2
      x = Conv2D(128, (3, 3), activation='relu', padding='same',name='block3_conv1')(x)
      x = MaxPooling2D((2, 2), strides=(2, 2), name='block3_pool')(x)
  
      # Block 3
      x = Conv2D(256, (3, 3), activation='relu', padding='same',name='block4_conv1')(x)
      x = MaxPooling2D((2, 2), strides=(2, 2), name='block4_pool')(x)
  
      x = Conv2D(512, (3, 3), activation='relu', padding='same',name='block5_conv1')(x)
      x = MaxPooling2D((2, 2), strides=(2, 2), name='block5_pool')(x)
  
      x = Flatten()(x)
  
      return Model(input1,x)
  
  def small_vgg_plate(input_shape):
      input1 = Input(input_shape)
      x = Conv2D(64, (3, 3), activation='relu', padding='same')(input1)
      x = Conv2D(64, (3, 3), activation='relu', padding='same')(x)
      x = MaxPooling2D((2, 2), strides=(2, 2))(x)
  
      x = Conv2D(128, (3, 3), activation='relu', padding='same')(x)
      x = Conv2D(128, (3, 3), activation='relu', padding='same')(x)
      x = MaxPooling2D((2, 2), strides=(2, 2))(x)
  
      x = Conv2D(256, (3, 3), activation='relu', padding='same')(x)
      x = MaxPooling2D((2, 2), strides=(2, 2))(x)
  
      x = Conv2D(512, (3, 3), activation='relu', padding='same')(x)
      x = MaxPooling2D((2, 2), strides=(2, 2))(x)
  
      x = Conv2D(512, (3, 3), activation='relu', padding='same')(x)
      x = Flatten()(x)
  
      return Model(input1,x)
  
  def small_vgg3d(input_shape):
      input1 = Input(input_shape)
      x = Conv3D(64, (1, 3, 3), activation='relu', padding='same',name='block1_conv1')(input1)
  
      # Block 1
      x = MaxPooling3D((1, 2, 2), strides=(1, 2, 2), name='block1_pool')(x)
  
      # Block 2
      x = Conv3D(128, (1, 3, 3), activation='relu', padding='same',name='block2_conv1')(x)
      x = MaxPooling3D((1, 2, 2), strides=(1, 2, 2), name='block2_pool')(x)
      x = Conv3D(128, (1, 3, 3), activation='relu', padding='same',name='block2_conv2')(x)
      x = MaxPooling3D((1, 2, 2), strides=(1, 2, 2), name='block3_pool')(x)
  
      # Block 3
      x = Conv3D(256, (1, 3, 3), activation='relu', padding='same',name='block3_conv1')(x)
      x = MaxPooling3D((1, 2, 2), strides=(1, 2, 2), name='block4_pool')(x)
  
      x = Conv3D(512, (1, 3, 3), activation='relu', padding='same',name='block4_conv1')(x)
      x = MaxPooling3D((1, 2, 2), strides=(1, 2, 2), name='block5_pool')(x)
  
      x = Flatten()(x)
  
      return Model(input1,x)
  
  #------------------------------------------------------------------------------
  def fold(list1, ind, train=False):
      _list1 = list1.copy()
      _list1.pop(ind)
      if train:
          return [_list1[i % len(_list1)] for i in [ind+2,ind+3]]
      else:
          return [_list1[i % len(_list1)] for i in [ind+4,ind+5]]
  #------------------------------------------------------------------------------
  def calculate_metrics(ytrue1, ypred1):
      conf = metrics.confusion_matrix(ytrue1, ypred1, [0,1])
      maxres = (conf[1,1],
                conf[0,0],
                conf[0,1],
                conf[1,0],
          metrics.precision_score(ytrue1, ypred1) * 100,
          metrics.recall_score(ytrue1, ypred1) * 100,
          metrics.f1_score(ytrue1, ypred1) * 100,
          metrics.accuracy_score(ytrue1, ypred1) * 100)
      return maxres
  
  #------------------------------------------------------------------------------
  def test_report(model_name, model, test_gen):
      print("=== Evaluating model: {:s} ===".format(model_name))
      a = open("%s_inferences_output.txt" % (model_name), "w")
      ytrue, ypred = [], []
      for i in range(len(test_gen)):
        X, Y, paths = test_gen[i]
        Y_ = model.predict(X)
        for y1, yreg, y2, p0, p1 in zip(Y_[0].tolist(), Y_[1].tolist(), Y['class_output'].argmax(axis=-1).tolist(), paths[0], paths[1]):
          y1_class = np.argmax(y1)
          ypred.append(y1_class)
          ytrue.append(y2)
          a.write("%s;%s;%d;%d;%f;%s\n" % (p0, p1, y2, y1_class, yreg[0], str(y1)))
  
      a.write('tp: %d, tn: %d, fp: %d, fn: %d P:%0.2f R:%0.2f F:%0.2f A:%0.2f' % calculate_metrics(ytrue, ypred))
      a.close()
  #------------------------------------------------------------------------------
  def process_load(f1, vec_size):
      _i1 = image.load_img(f1, target_size=vec_size)
      _i1 = image.img_to_array(_i1, dtype='uint8')
      return _i1
  
  def load_img(img, vec_size, vec_size2, metadata_dict):
    iplt0 = process_load(img[0][0], vec_size)
    iplt1 = process_load(img[2][0], vec_size)
    iplt2 = process_load(img[1][0], vec_size2)
    iplt3 = process_load(img[3][0], vec_size2)
  
    d1 = {"i0":iplt0,
          "i1":iplt1,
          "i2":iplt2,
          "i3":iplt3,
          "l":img[4],
          "p1":img[0][0],
          "p2":img[2][0],
          "c1":img[5]['color'],
          "c2":img[5]['color']
          }
    if metadata_dict is not None:
      diff = abs(np.array(metadata_dict[img[0][0]][:7]) - np.array(metadata_dict[img[2][0]][:7])).tolist()
      diff = [1 if i else 0 for i in diff]
      d1['metadata'] = np.array(metadata_dict[img[0][0]] + metadata_dict[img[2][0]] + diff)
    return d1
  
  
  class SiameseSequence(Sequence):
      def __init__(self,features, 
                  augmentations,
                  batch_size=batch_size,
                  input1=(image_size_h_p,image_size_w_p,nchannels),
                  input2=(image_size_h_c,image_size_w_c,nchannels), 
                  type1=None,
                  metadata_dict=None, 
                  metadata_length=0, 
                  with_paths=False):
          self.features = features
          self.batch_size = batch_size
          self.vec_size = input1
          self.vec_size2 = input2
          self.type = type1
          self.metadata_dict = metadata_dict
          self.metadata_length = metadata_length
          self.augment = augmentations
          self.with_paths = with_paths
  
      def __len__(self):
          return int(np.ceil(len(self.features) / 
              float(self.batch_size)))
  
      def __getitem__(self, idx):
          start = idx * self.batch_size
          end = (idx + 1) * self.batch_size
          batch = self.features[start:end]
          futures = []
          _vec_size = (len(batch),) + self.vec_size
          b1 = np.zeros(_vec_size)
          b2 = np.zeros(_vec_size)
          _vec_size2 = (len(batch),) + self.vec_size2
          b3 = np.zeros(_vec_size2)
          b4 = np.zeros(_vec_size2)
          blabels = np.zeros((len(batch)))
          p1 = []
          p2 = []
          c1 = []
          c2 = []
          if self.metadata_length>0:
              metadata = np.zeros((len(batch),self.metadata_length))
  
          i1 = 0
          for _b in batch:
              res = load_img(_b, self.vec_size, self.vec_size2, self.metadata_dict)
              if self.augment is not None:
                  b1[i1,:,:,:] = self.augment[0][0](image=res['i0'])["image"]
                  b2[i1,:,:,:] = self.augment[1][0](image=res['i1'])["image"]
                  b3[i1,:,:,:] = self.augment[2][0](image=res['i2'])["image"]
                  b4[i1,:,:,:] = self.augment[3][0](image=res['i3'])["image"]
              else:
                  b1[i1,:,:,:] = res['i0']
                  b2[i1,:,:,:] = res['i1']
                  b3[i1,:,:,:] = res['i2']
                  b4[i1,:,:,:] = res['i3']
              blabels[i1] = res['l']
              p1.append(res['p1'])
              p2.append(res['p2'])
              c1.append(res['c1'])
              c2.append(res['c2'])
              if self.metadata_length>0:
                  metadata[i1,:] = res['metadata']
              i1+=1
          blabels2 = np.array(blabels).reshape(-1,1)
          blabels = np_utils.to_categorical(blabels2, 2)
          y = {"class_output":blabels, "reg_output":blabels2}
          if self.type is None:
              result = [[b1, b2, b3, b4], y]
          elif self.type == 'plate':
              result = [[b1, b2], y]
          elif self.type == 'car':
              result = [[b3, b4], y]
          if self.metadata_length>0:
              result[0].append(metadata)
          if self.with_paths:
              result += [[p1,p2]]
  
          return result
  
  def load_img_temporal(img, vec_size2, tam, metadata_dict):
    iplt2 = [process_load(img[1][i], vec_size2, None) for i in range(tam)]
    iplt3 = [process_load(img[3][i], vec_size2, None) for i in range(tam)]
  
    d1 = {"i2":iplt2,
          "i3":iplt3,
          "l":img[4],
          "p1":str(img[0]),
          "p2":str(img[2]),
          "c1":img[5]['color'],
          "c2":img[5]['color']
          }
  
    d1['metadata'] = []
    for i in range(tam):
      diff = abs(np.array(metadata_dict[img[0][i]][:7]) - np.array(metadata_dict[img[2][i]][:7])).tolist()
      for j in range(len(diff)):
        diff[j] = 1 if diff[j] else 0
      d1['metadata'] += metadata_dict[img[0][i]] + metadata_dict[img[2][i]] + diff
    d1['metadata'] = np.array(d1['metadata'])
    return d1
  
  class SiameseSequenceTemporal(Sequence):
      def __init__(self,features, 
                  augmentations,
                  tam, 
                  metadata_dict, 
                  metadata_length, 
                  batch_size,
                  with_paths=False):
          self.tam = tam
          self.features = features
          self.batch_size = batch_size
          self.vec_size2 = (image_size_h_c,image_size_w_c,nchannels)
          self.metadata_dict = metadata_dict
          self.metadata_length = metadata_length
          self.augment = augmentations
          self.with_paths = with_paths
  
      def __len__(self):
          return int(np.ceil(len(self.features) / 
              float(self.batch_size)))
  
      def __getitem__(self, idx):
          start = idx * self.batch_size
          end = (idx + 1) * self.batch_size
          batch = self.features[start:end]
          futures = []
          _vec_size2 = (len(batch),self.tam,) + self.vec_size2
          b3 = np.zeros(_vec_size2)
          b4 = np.zeros(_vec_size2)
          blabels = np.zeros((len(batch)))
          p1 = []
          p2 = []
          c1 = []
          c2 = []
          if self.metadata_length>0:
              metadata = np.zeros((len(batch),self.metadata_length))
          i = 0
          for _b in batch:
              r = load_img_temporal(_b, self.vec_size2, self.tam, self.metadata_dict)
              for j in range(self.tam):
                  b3[i,j,:,:,:] = self.augment[2][j](image=r['i2'][j])["image"]
                  b4[i,j,:,:,:] = self.augment[3][j](image=r['i3'][j])["image"]
              blabels[i] = r['l']
              p1.append(r['p1'])
              p2.append(r['p2'])
              c1.append(r['c1'])
              c2.append(r['c2'])
              if self.metadata_length>0:
                  metadata[i,:] = r['metadata']
              i+=1
          blabels2 = np.array(blabels).reshape(-1,1)
          blabels = np_utils.to_categorical(blabels2, 2)
          y = {"class_output":blabels, "reg_output":blabels2}
          result = [[b3, b4, metadata], y]
  
          if self.with_paths:
            result += [[p1,p2]]
  
          return result