code: #!/usr/bin/env python3
  # -*- coding: utf-8 -*-
  """
  Created on Wed Dec  6 15:40:14 2017
  
  Functions needed to read the data from different databases
  
  @author: anazabal, olmosUC3M, ivaleraM
  """
  
  import csv
  import os
  import torch
  import numpy as np
  from sklearn.metrics import mean_squared_error
  
  
  def read_data(data_file, types_file, miss_file, true_miss_file):
      # Read types of data from data file
      with open(types_file) as f:
          types_dict = [{k: v for k, v in row.items()}
                        for row in csv.DictReader(f, skipinitialspace=True)]
      for t in types_dict:
          t['dim'] = int(t['dim'])
  
      # Read data from input file
      with open(data_file, 'r') as f:
          data = [[float(x) for x in rec] for rec in csv.reader(f, delimiter=',')]
          data = np.array(data)
  
      # Substitute NaN values by something (we assume we have the real missing value mask)
      if true_miss_file:
          with open(true_miss_file, 'r') as f:
              missing_positions = [[int(x) for x in rec] for rec in csv.reader(f, delimiter=',')]
              missing_positions = np.array(missing_positions)
  
          true_miss_mask = np.ones([np.shape(data)[0], len(types_dict)])
          true_miss_mask[missing_positions[:, 0] - 1, missing_positions[:, 1] - 1] = 0  # Indexes in the csv start at 1
          data_masked = np.ma.masked_where(np.isnan(data), data)
          # We need to fill the data depending on the given data...
          data_filler = []
          for i in range(len(types_dict)):
              if types_dict[i]['type'] == 'cat' or types_dict[i]['type'] == 'ordinal':
                  aux = np.unique(data[:, i])
                  data_filler.append(aux[0])  # Fill with the first element of the cat (0, 1, or whatever)
              else:
                  data_filler.append(0.0)
  
          data = data_masked.filled(data_filler)
      else:
          true_miss_mask = np.ones([np.shape(data)[0], len(types_dict)])  # It doesn't affect our data
  
      # Construct the data matrices
      data_complete = []
      for i in range(np.shape(data)[1]):
  
          if types_dict[i]['type'] == 'cat':
              # Get categories
              cat_data = [int(x) for x in data[:, i]]
              categories, indexes = np.unique(cat_data, return_inverse=True)
              # Transform categories to a vector of 0:n_categories
              new_categories = np.arange(int(types_dict[i]['dim']))
              cat_data = new_categories[indexes]
              # Create one hot encoding for the categories
              aux = np.zeros([np.shape(data)[0], len(new_categories)])
              aux[np.arange(np.shape(data)[0]), cat_data] = 1
              data_complete.append(aux)
  
          elif types_dict[i]['type'] == 'ordinal':
              # Get categories
              cat_data = [int(x) for x in data[:, i]]
              categories, indexes = np.unique(cat_data, return_inverse=True)
              # Transform categories to a vector of 0:n_categories
              new_categories = np.arange(int(types_dict[i]['dim']))
              cat_data = new_categories[indexes]
              # Create thermometer encoding for the categories
              aux = np.zeros([np.shape(data)[0], 1 + len(new_categories)])
              aux[:, 0] = 1
              aux[np.arange(np.shape(data)[0]), 1 + cat_data] = -1
              aux = np.cumsum(aux, 1)
              data_complete.append(aux[:, :-1])
  
          elif types_dict[i]['type'] == 'count':
              if np.min(data[:, i]) == 0:
                  aux = data[:, i] + 1
                  data_complete.append(np.transpose([aux]))
              else:
                  data_complete.append(np.transpose([data[:, i]]))
          else:
              data_complete.append(np.transpose([data[:, i]]))
  
      data = np.concatenate(data_complete, 1)
  
      # Read Missing mask from csv (contains positions of missing values)
      n_samples = np.shape(data)[0]
      n_variables = len(types_dict)
      miss_mask = np.ones([np.shape(data)[0], n_variables])
      # If there is no mask, assume all data is observed
      if os.path.isfile(miss_file):
          with open(miss_file, 'r') as f:
              missing_positions = [[int(x) for x in rec] for rec in csv.reader(f, delimiter=',')]
              missing_positions = np.array(missing_positions)
          miss_mask[missing_positions[:, 0] - 1, missing_positions[:, 1] - 1] = 0  # Indexes in the csv start at 1
  
      return data, types_dict, miss_mask, true_miss_mask, n_samples
  
  
  def next_batch(data, labels, types_dict, miss_mask, batch_size, index_batch):
      # Create minibath
      batch_xs = data[index_batch * batch_size:(index_batch + 1) * batch_size, :]
  
      # Slipt variables of the batches
      data_list = []
      initial_index = 0
      for d in types_dict:
          dim = int(d['dim'])
          data_list.append(batch_xs[:, initial_index:initial_index + dim])
          initial_index += dim
  
      data_tensor = torch.cat(data_list, dim=1)
  
      # Missing data
      miss_list = miss_mask[index_batch * batch_size:(index_batch + 1) * batch_size, :]
  
      # Labels
      labels_batch = None
      if labels is not None:
          labels_batch = labels[index_batch * batch_size:(index_batch + 1) * batch_size]
  
      return data_tensor, labels_batch, miss_list
  
  
  def samples_concatenation(samples):
      for i, batch in enumerate(samples):
          if i == 0:
              samples_x = np.concatenate(batch['x'], 1)
              samples_y = batch['y']
              samples_z = batch['z']
              samples_s = batch['s']
          else:
              samples_x = np.concatenate([samples_x, np.concatenate(batch['x'], 1)], 0)
              samples_y = np.concatenate([samples_y, batch['y']], 0)
              samples_z = np.concatenate([samples_z, batch['z']], 0)
              samples_s = np.concatenate([samples_s, batch['s']], 0)
  
      return samples_s, samples_z, samples_y, samples_x
  
  
  def discrete_variables_transformation(data, types_dict):
      ind_ini = 0
      output = []
      for d in range(len(types_dict)):
          ind_end = ind_ini + int(types_dict[d]['dim'])
          if types_dict[d]['type'] == 'cat':
              output.append(np.reshape(np.argmax(data[:, ind_ini:ind_end], 1), [-1, 1]))
          elif types_dict[d]['type'] == 'ordinal':
              output.append(np.reshape(np.sum(data[:, ind_ini:ind_end], 1) - 1, [-1, 1]))
          else:
              output.append(data[:, ind_ini:ind_end])
          ind_ini = ind_end
  
      return np.concatenate(output, 1)
  
  
  # Several baselines
  def mean_imputation(train_data, miss_mask, types_dict):
      ind_ini = 0
      est_data = []
      for dd in range(len(types_dict)):
          # Imputation for cat and ordinal is done using the mode of the data
          if types_dict[dd]['type'] == 'cat' or types_dict[dd]['type'] == 'ordinal':
              ind_end = ind_ini + 1
              # The imputation is based on whatever is observed
              miss_pattern = (miss_mask[:, dd] == 1)
              values, counts = np.unique(train_data[miss_pattern, ind_ini:ind_end], return_counts=True)
              data_mode = np.argmax(counts)
              data_imputed = train_data[:, ind_ini:ind_end] * miss_mask[:, ind_ini:ind_end] + data_mode * (
                      1.0 - miss_mask[:, ind_ini:ind_end])
  
          # Imputation for the rest of the variables is done with the mean of the data
          else:
              ind_end = ind_ini + int(types_dict[dd]['dim'])
              miss_pattern = (miss_mask[:, dd] == 1)
              # The imputation is based on whatever is observed
              data_mean = np.mean(train_data[miss_pattern, ind_ini:ind_end], 0)
              data_imputed = train_data[:, ind_ini:ind_end] * miss_mask[:, ind_ini:ind_end] + data_mean * (
                      1.0 - miss_mask[:, ind_ini:ind_end])
  
          est_data.append(data_imputed)
          ind_ini = ind_end
  
      return np.concatenate(est_data, 1)
  
  
  def p_distribution_params_concatenation(params, types_dict, z_dim, s_dim):
      keys = params[0].keys()
      out_dict = {key: [] for key in keys}
  
      for i, batch in enumerate(params):
  
          for d, k in enumerate(keys):
  
              if k == 'z' or k == 'y':
                  if i == 0:
                      out_dict[k] = batch[k]
                  else:
                      out_dict[k] = np.concatenate([out_dict[k], batch[k]], 1)
  
              elif k == 'x':
                  if i == 0:
                      out_dict[k] = batch[k]
                  else:
                      for v in range(len(types_dict)):
                          if types_dict[v]['type'] == 'pos' or types_dict[v]['type'] == 'real':
                              out_dict[k][v] = np.concatenate([out_dict[k][v], batch[k][v]], 1)
                          else:
                              out_dict[k][v] = np.concatenate([out_dict[k][v], batch[k][v]], 0)
  
      return out_dict
  
  
  def q_distribution_params_concatenation(params, z_dim, s_dim):
      keys = params[0].keys()
      out_dict = {key: [] for key in keys}
  
      for i, batch in enumerate(params):
          for d, k in enumerate(keys):
              out_dict[k].append(batch[k])
  
      out_dict['z'] = np.concatenate(out_dict['z'], 1)
      if 's' in out_dict:
          out_dict['s'] = np.concatenate(out_dict['s'], 0)
  
      return out_dict
  
  
  def statistics(loglik_params, types_dict):
      loglik_mean = []
      loglik_mode = []
  
      for d, attrib in enumerate(loglik_params):
          if types_dict[d]['type'] == 'real':
              # Normal distribution (mean, sigma)
              loglik_mean.append(attrib[0])
              loglik_mode.append(attrib[0])
          # Only for log-normal
          elif types_dict[d]['type'] == 'pos':
              # Log-normal distribution (mean, sigma)
              loglik_mean.append(np.maximum(np.exp(attrib[0] + 0.5 * attrib[1]) - 1.0, 0.0))
              loglik_mode.append(np.maximum(np.exp(attrib[0] - attrib[1]) - 1.0, 0.0))
          elif types_dict[d]['type'] == 'count':
              # Poisson distribution (lambda)
              loglik_mean.append(attrib)
              loglik_mode.append(np.floor(attrib))
  
          else:
              # Categorical and ordinal (mode imputation for both)
              loglik_mean.append(np.reshape(np.argmax(attrib, 1), [-1, 1]))
              loglik_mode.append(np.reshape(np.argmax(attrib, 1), [-1, 1]))
  
      return np.transpose(np.squeeze(loglik_mean)), np.transpose(np.squeeze(loglik_mode))
  
  
  def error_computation(x_train, x_hat, types_dict, miss_mask):
      error_observed = []
      error_missing = []
      ind_ini = 0
      for dd in range(len(types_dict)):
          # Mean classification error
          if types_dict[dd]['type'] == 'cat':
              ind_end = ind_ini + 1
              error_observed.append(np.mean(
                  x_train[miss_mask[:, dd] == 1, ind_ini:ind_end] != x_hat[miss_mask[:, dd] == 1, ind_ini:ind_end]))
              if np.sum(miss_mask[:, dd] == 0, 0) == 0:
                  error_missing.append(0)
              else:
                  error_missing.append(np.mean(
                      x_train[miss_mask[:, dd] == 0, ind_ini:ind_end] != x_hat[miss_mask[:, dd] == 0, ind_ini:ind_end]))
          # Mean "shift" error
          elif types_dict[dd]['type'] == 'ordinal':
              ind_end = ind_ini + 1
              error_observed.append(np.mean(np.abs(
                  x_train[miss_mask[:, dd] == 1, ind_ini:ind_end] - x_hat[miss_mask[:, dd] == 1, ind_ini:ind_end])) / int(
                  types_dict[dd]['dim']))
              if np.sum(miss_mask[:, dd] == 0, 0) == 0:
                  error_missing.append(0)
              else:
                  error_missing.append(np.mean(np.abs(
                      x_train[miss_mask[:, dd] == 0, ind_ini:ind_end] - x_hat[miss_mask[:, dd] == 0,
                                                                        ind_ini:ind_end])) / int(types_dict[dd]['dim']))
          # Normalized root mean square error
          else:
              ind_end = ind_ini + int(types_dict[dd]['dim'])
              norm_term = np.max(x_train[:, dd]) - np.min(x_train[:, dd])
              error_observed.append(np.sqrt(mean_squared_error(x_train[miss_mask[:, dd] == 1, ind_ini:ind_end],
                                                               x_hat[miss_mask[:, dd] == 1,
                                                               ind_ini:ind_end])) / norm_term)
  
              if np.sum(miss_mask[:, dd] == 0, 0) == 0:
                  error_missing.append(0)
              else:
                  error_missing.append(np.sqrt(mean_squared_error(x_train[miss_mask[:, dd] == 0, ind_ini:ind_end],
                                                                  x_hat[miss_mask[:, dd] == 0,
                                                                  ind_ini:ind_end])) / norm_term)
  
          ind_ini = ind_end
  
      return error_observed, error_missing
  
apis: numpy.sum numpy.cumsum numpy.transpose numpy.squeeze sklearn.metrics.mean_squared_error numpy.floor numpy.abs numpy.argmax numpy.exp numpy.max numpy.shape numpy.min numpy.isnan numpy.array numpy.concatenate torch.cat numpy.unique numpy.mean