content: # Functions performing various input/output operations for the ChaLearn AutoML challenge
  
  # Main contributor: Arthur Pesah, August 2014
  # Edits: Isabelle Guyon, October 2014
  
  # ALL INFORMATION, SOFTWARE, DOCUMENTATION, AND DATA ARE PROVIDED "AS-IS".
  # ISABELLE GUYON, CHALEARN, AND/OR OTHER ORGANIZERS OR CODE AUTHORS DISCLAIM
  # ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
  # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR ANY PARTICULAR PURPOSE, AND THE
  # WARRANTY OF NON-INFRIGEMENT OF ANY THIRD PARTY'S INTELLECTUAL PROPERTY RIGHTS.
  # IN NO EVENT SHALL ISABELLE GUYON AND/OR OTHER ORGANIZERS BE LIABLE FOR ANY SPECIAL,
  # INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER ARISING OUT OF OR IN
  # CONNECTION WITH THE USE OR PERFORMANCE OF SOFTWARE, DOCUMENTS, MATERIALS,
  # PUBLICATIONS, OR INFORMATION MADE AVAILABLE FOR THE CHALLENGE.
  import numpy as np
  import os
  import re
  import time
  
  import scipy.sparse
  
  try:
      import autosklearn.data.competition_c_functions as competition_c_functions
      competition_c_functions_is_there = True
  except:
      competition_c_functions_is_there = False
      pass
  
  from autosklearn.data import util as data_util
  from autosklearn.data.data_manager import DataManager
  from autosklearn.constants import *
  
  
  def data_dense(filename, feat_type=None, verbose=False):
      # The 2nd parameter makes possible a using of the 3 functions of data
      # reading (data, data_sparse, data_binary_sparse) without changing
      # parameters
  
      # This code is based on scipy.io.arff.arff_load
      r_comment = re.compile(r'^%')
      # Match an empty line
      r_empty = re.compile(r'^\s+$')
      descr = [(str(i), np.float32) for i in range(len(feat_type))]
  
      def generator(row_iter, delim=','):
          # Copied from scipy.io.arff.arffread
          raw = next(row_iter)
          while r_empty.match(raw) or r_comment.match(raw):
              raw = next(row_iter)
  
          # 'compiling' the range since it does not change
          # Note, I have already tried zipping the converters and
          # row elements and got slightly worse performance.
          elems = list(range(len(feat_type)))
  
          row = raw.split(delim)
          # yield tuple([np.float64(row[i]) for i in elems])
          yield tuple([row[i] for i in elems])
          for raw in row_iter:
              while r_comment.match(raw) or r_empty.match(raw):
                  raw = next(row_iter)
              row = raw.split(delim)
              # yield tuple([np.float64(row[i]) for i in elems])
              yield tuple([row[i] for i in elems])
  
      with open(filename) as fh:
          a = generator(fh, delim=" ")
          # No error should happen here: it is a bug otherwise
          data = np.fromiter(a, descr)
  
          data = data.view(np.float32).reshape((len(data), -1))
          return data
  
  
  def data_sparse(filename, feat_type):
      # This function takes as argument a file representing a sparse matrix
      # sparse_matrix[i][j] = "a:b" means matrix[i][a] = b
      # It converts it into a numpy array, using sparse_list_to_array function,
      # and returns this array
      sparse_list = sparse_file_to_sparse_list(filename)
      return sparse_list_to_csr_sparse(sparse_list, len(feat_type))
  
  
  def data_binary_sparse(filename, feat_type):
      # This function takes as an argument a file representing a binary sparse
      # matrix
      # binary_sparse_matrix[i][j] = a means matrix[i][j] = 1
      # It converts it into a numpy array an returns this array.
  
      inner_data = file_to_array(filename)
      nbr_samples = len(inner_data)
      # the construction is easier w/ dok_sparse
      dok_sparse = scipy.sparse.dok_matrix((nbr_samples, len(feat_type)))
      print ("Converting {} to dok sparse matrix".format(filename))
      for row in range(nbr_samples):
          for feature in inner_data[row]:
              dok_sparse[row, int(feature) - 1] = 1
      print ("Converting {} to csr sparse matrix".format(filename))
      return dok_sparse.tocsr()
  
  
  def file_to_array(filename, verbose=False):
      # Converts a file to a list of list of STRING; It differs from
      # np.genfromtxt in that the number of columns doesn't need to be constant
      data = []
      with open(filename, "r") as data_file:
          if verbose:
              print ("Reading {}...".format(filename))
          lines = data_file.readlines()
          if verbose:
              print ("Converting {} to correct array...".format(filename))
          data = [lines[i].strip().split() for i in range(len(lines))]
      return data
  
  
  def read_first_line(filename):
      # Read fist line of file
      data = []
      with open(filename, "r") as data_file:
          line = data_file.readline()
          data = line.strip().split()
      return data
  
  
  def sparse_file_to_sparse_list(filename, verbose=True):
      # Converts a sparse data file to a sparse list, so that:
      # sparse_list[i][j] = (a,b) means matrix[i][a]=b
      data_file = open(filename, "r")
      if verbose:
          print ("Reading {}...".format(filename))
      lines = data_file.readlines()
      if verbose:
          print ("Converting {} to correct array")
      data = [lines[i].split(' ') for i in range(len(lines))]
      if verbose:
          print ("Converting {} to sparse list".format(filename))
  
      _converter = lambda a_: (int(a_[0]), np.float32(float(a_[1])))
      return [[_converter(data[i][j].rstrip().split(':'))
               for j in range(len(data[i])) if data[i][j] != '\n']
              for i in range(len(data))]
  
  
  def sparse_list_to_csr_sparse(sparse_list, nbr_features, verbose=True):
      # This function takes as argument a matrix of tuple representing a sparse
      # matrix and the number of features.
      # sparse_list[i][j] = (a,b) means matrix[i][a]=b
      # It converts it into a scipy csr sparse matrix
      nbr_samples = len(sparse_list)
      # construction easier w/ dok_sparse...
      dok_sparse = scipy.sparse.dok_matrix((nbr_samples, nbr_features),
                                           dtype=np.float32)
      if verbose:
          print ("\tConverting sparse list to dok sparse matrix")
      for row in range(nbr_samples):
          for column in range(len(sparse_list[row])):
              (feature, value) = sparse_list[row][column]
              dok_sparse[row, feature - 1] = value
      if verbose:
          print ("\tConverting dok sparse matrix to csr sparse matrix")
          # but csr better for shuffling data or other tricks
      return dok_sparse.tocsr()
  
  
  class CompetitionDataManager(DataManager):
      ''' This class aims at loading and saving data easily with a cache and at generating a dictionary (self.info) in which each key is a feature (e.g. : name, format, feat_num,...).
      Methods defined here are :
      __init__ (...)
          x.__init__([(feature, value)]) -> void
          Initialize the info dictionary with the tuples (feature, value) given as argument. It recognizes the type of value (int, string) and assign value to info[feature]. An unlimited number of tuple can be sent.
  
      getInfo (...)
          x.getInfo (filename) -> void
          Fill the dictionary with an info file. Each line of the info file must have this format 'feature' : value
          The information is obtained from the public.info file if it exists, or inferred from the data files
  
      getInfoFromFile (...)
          x.getInfoFromFile (filename) -> void
          Fill the dictionary with an info file. Each line of the info file must have this format 'feature' : value
      '''
  
      def __init__(self, basename, input_dir, verbose=False, encode_labels=True):
          super(CompetitionDataManager, self).__init__()
  
          self.basename = basename
          if basename in input_dir:
              self.input_dir = input_dir
          else:
              self.input_dir = input_dir + "/" + basename + "/"
  
          info_file = os.path.join(self.input_dir, basename + '_public.info')
          self.getInfo(info_file)
          self.feat_type = self.loadType(os.path.join(self.input_dir, basename + '_feat.type'), verbose=verbose)
  
  
          Xtr = self.loadData(os.path.join(self.input_dir, basename + '_train.data'),
                              self.info['train_num'], verbose=verbose)
          Ytr = self.loadLabel(os.path.join(self.input_dir, basename + '_train.solution'),
                               self.info['train_num'], verbose=verbose)
          Xva = self.loadData(os.path.join(self.input_dir, basename + '_valid.data'),
                              self.info['valid_num'], verbose=verbose)
          Xte = self.loadData(os.path.join(self.input_dir, basename + '_test.data'),
                              self.info['test_num'], verbose=verbose)
  
          self._data['X_train'] = Xtr
          self._data['Y_train'] = Ytr
          self._data['X_valid'] = Xva
          self._data['X_test'] = Xte
  
          p = os.path.join(self.input_dir, basename + '_valid.solution')
          if os.path.exists(p):
              try:
                  self._data['Y_valid'] = self.loadLabel(p,
                      self.info['valid_num'], verbose=verbose)
              except (IOError, OSError):
                  pass
  
          p = os.path.join(self.input_dir, basename + '_test.solution')
          if os.path.exists(p):
              try:
                  self.data['Y_test'] = self.loadLabel(p,
                      self.info['test_num'], verbose=verbose)
              except (IOError, OSError) as e:
                  pass
  
          if encode_labels:
              self.perform1HotEncoding()
  
      def loadData (self, filename, num_points, verbose=True):
          ''' Get the data from a text file in one of 3 formats: matrix, sparse, binary_sparse'''
          if verbose:  print("========= Reading " + filename)
          start = time.time()
  
          if 'format' not in self.info:
              self.getFormatData(filename)
          if competition_c_functions_is_there:
              data_func = {'dense': competition_c_functions.read_dense_file,
                           'sparse': competition_c_functions.read_sparse_file,
                           'sparse_binary': competition_c_functions.read_sparse_binary_file}
  
              data = data_func[self.info['format']](filename, num_points,
                                                    self.info['feat_num'])
  
              if scipy.sparse.issparse(data):
                  if not np.all(data.indices >= 0):
                      raise ValueError("Sparse data must be 1-indexed, "
                                       "not 0-indexed.")
          else:
              data_func = {'dense': data_dense,
                           'sparse': data_sparse,
                           'sparse_binary': data_binary_sparse}
  
              data = data_func[self.info['format']](filename, self.feat_type)
  
          end = time.time()
          if verbose:  print( "[+] Success in %5.2f sec" % (end - start))
          return data
  
      def loadLabel (self, filename, num_points, verbose=True):
          ''' Get the solution/truth values'''
          if verbose:  print("========= Reading " + filename)
          start = time.time()
  
          # IG: Here change to accommodate the new multiclass label format
          if competition_c_functions_is_there:
              if self.info['task'] == MULTILABEL_CLASSIFICATION:
                  # cast into ints
                  label = (competition_c_functions.read_dense_file_unknown_width(
                      filename, num_points)).astype(np.int)
              elif self.info['task'] == MULTICLASS_CLASSIFICATION:
                  label = competition_c_functions.read_dense_file_unknown_width(
                      filename, num_points)
                  # read the class from the only non zero entry in each line!
                  # should be ints right away
                  label = np.where(label != 0)[1];
              else:
                  label = competition_c_functions.read_dense_file_unknown_width(
                      filename, num_points)
          else:
              if self.info['task'] == MULTILABEL_CLASSIFICATION:
                  label = self._data(filename)
              elif self.info['task'] == MULTICLASS_CLASSIFICATION:
                  label = data_util.convert_to_num(self._data(filename))
              else:
                  label = np.ravel(data_util.data(filename)) # get a column vector
  
          end = time.time()
          if verbose:  print( "[+] Success in %5.2f sec" % (end - start))
          return label
  
      def loadType (self, filename, verbose=True):
          ''' Get the variable types'''
          if verbose:  print("========= Reading " + filename)
          start = time.time()
          type_list = []
          if os.path.isfile(filename):
              if competition_c_functions_is_there:
                  type_list = competition_c_functions.file_to_array(filename,
                                                              verbose=False)
              else:
                  type_list = file_to_array(filename, verbose=False)
          else:
              n=self.info['feat_num']
              type_list = [self.info['feat_type']]*n
          type_list = np.array(type_list).ravel()
          end = time.time()
          if verbose:  print( "[+] Success in %5.2f sec" % (end - start))
          return type_list
  
      def getInfo (self, filename, verbose=True):
          ''' Get all information {attribute = value} pairs from the filename (public.info file),
                if it exists, otherwise, output default values'''
          if filename==None:
              basename = self.basename
              input_dir = self.input_dir
          else:
              # Split away the _public.info (anyway, I don't know why its
              # there... the dataset name is known from the call)
              basename = "_".join(os.path.basename(filename).split('_')[:-1])
              input_dir = os.path.dirname(filename)
          if os.path.exists(filename):
              self.getInfoFromFile (filename)
              print "Info file found : " + os.path.abspath(filename)
              # Finds the data format ('dense', 'sparse', or 'sparse_binary')
              self.getFormatData(os.path.join(input_dir, basename + '_train.data'))
          else:
              raise NotImplementedError("The user must always provide an info "
                                        "file.")
  
          self.info['task'] = STRING_TO_TASK_TYPES[self.info['task']]
  
          return self.info
  
      def getInfoFromFile (self, filename):
          ''' Get all information {attribute = value} pairs from the public.info file'''
          with open (filename, "r") as info_file:
              lines = info_file.readlines()
              features_list = list(map(lambda x: tuple(x.strip("\'").split(" = ")), lines))
              for (key, value) in features_list:
                  self.info[key] = value.rstrip().strip("'").strip(' ')
                  if self.info[key].isdigit(): # if we have a number, we want it to be an integer
                      self.info[key] = int(self.info[key])
          return self.info
  
      def getFormatData(self,filename):
          ''' Get the data format directly from the data file (in case we do not have an info file)'''
          if 'format' in self.info.keys():
              return self.info['format']
          if 'is_sparse' in self.info.keys():
              if self.info['is_sparse'] == 0:
                  self.info['format'] = 'dense'
              else:
                  if competition_c_functions_is_there:
                      data = competition_c_functions.read_first_line(filename)
                  else:
                      data = data_util.read_first_line(filename)
                  if ':' in data[0]:
                      self.info['format'] = 'sparse'
                  else:
                      self.info['format'] = 'sparse_binary'
          else:
              if competition_c_functions_is_there:
                  data = competition_c_functions.file_to_array(filename)
              else:
                  data = data_util.file_to_array(filename)
              if ':' in data[0][0]:
                  self.info['is_sparse'] = 1
                  self.info['format'] = 'sparse'
              else:
                  nbr_columns = len(data[0])
                  for row in range (len(data)):
                      if len(data[row]) != nbr_columns:
                          self.info['format'] = 'sparse_binary'
                  if 'format' not in self.info.keys():
                      self.info['format'] = 'dense'
                      self.info['is_sparse'] = 0
          return self.info['format']
  
  