text: from __future__ import division
  import Data
  import h5py
  import re
  import numpy as np
  import random
  import ctypes
  import tempfile
  import os
  import copy
  from threading import Thread
  from Queue import Queue, Empty
  
  class Dataset(object):
      def __init__(self, data, block_length=1, use_blocks=None, offsets=None):
          """
          data can be a numpy array (in C order), a tuple of such arrays, or a list of such tuples or arrays
          """
          self.files = list()
          if isinstance(data, list): #Several files
              for file in data:
                  if isinstance(file, tuple):
                      for d in file:
                          assert(isinstance(d, np.ndarray) and not np.isfortran(d))
                      self.files.append(file)
          elif isinstance(data, tuple): #Just one file
              for d in data:
                  assert(isinstance(d, np.ndarray) and d.ndim == 2 and not np.isfortran(d))
              self.files.append(data)
          elif isinstance(data, np.ndarray): #One file with one kind of element only (not input-output)
              assert(isinstance(data, np.ndarray) and not np.isfortran(data))
              self.files.append(tuple([data]))
          # Support for block datapoints
          self.block_length = block_length
          if block_length == 1:
              self.block_lengths = [np.int(1)] * self.get_arity()
              self.offsets = [np.int(0)] * self.get_arity()
          elif block_length > 1:
              self.block_lengths = [np.int(block_length) if ub else np.int(1) for ub in use_blocks]  # np.asarray(dtype=np.int) and [np.int(x)] have elements with diff type. Careful!
              self.offsets = [np.int(off) for off in offsets]
              for ub, off in zip(use_blocks, offsets):
                  if off != 0 and ub:
                      raise Exception("Can't have both a block size greater than 1 and an offset.")
          else:
              raise Exception("Block size must be positive")
  
      def get_n_files(self):
          return len(self.files)
  
      def get_arity(self):
          return len(self.files[0])
  
      def get_dimensionality(self, element=0):
          return self.get_file(element, 0).shape[1] * self.block_lengths[element]
  
      def get_type(self, element=0):
          return self.get_file(element, 0).dtype
  
      def get_file(self, element, index):
          return self.files[index][element]
  
      def get_file_shape(self, element, index):
          return self.get_file(element, index).shape
  
      def get_n_blocks_in_file(self, index):
          return np.maximum(self.get_file_shape(0, index)[0] - self.block_length + 1, 0)
  
      def iterator(self, batch_size=1, get_smaller_final_batch=False, shuffle=False, just_input=False, n_batches=None):
          """
          Returns an minibatch iterator
          get_smaller_final_batch: If True, it will return a final minibatch of size different to the specified. E.g. if the dataset has 1037 datapoints and the minibatch size is 100, it would return 10 batches of size 100 and one of size 37.
          shuffle: if True datapoints are shuffled
          just_input:  deprecated, useless now.
          n_batches: (optional) Number of minibatches to iterate through. If None then it will iterate through the whole dataset.
          """
          return DatasetIterator(self, batch_size, get_smaller_final_batch, shuffle, n_batches=n_batches)
  
      def describe(self):
          # Create a better description
          return "Dataset %d files" % (self.get_n_files())
  
      def file_iterator(self, **kwargs):
          return DatasetFileIterator(self, **kwargs)
  
      def sample_data(self, n=None, proportion=None):
          """
          With replacement
          """
          if n is None:
              assert(proportion is not None)
              total = sum([self.get_file_shape(0, i)[0] for i in xrange(len(self.file_paths))])
              n = total * proportion
          data = tuple(np.empty((n, self.get_dimensionality(i)), dtype=self.get_type(i)) for i in xrange(self.get_arity()))
          row = 0
          while row < n:
              file_index = np.random.randint(0, self.get_n_files())
              index = np.random.randint(0, self.get_file_shape(0, file_index)[0] - (self.block_length - 1))
              for i in xrange(self.get_arity()):
                  data[i][row] = self.get_file(i, file_index)[index:index + self.block_lengths[i], :].flatten()
              row += 1
          return data
  
      def map(self, mapping_f):
          new_data = list()
          # Transform file by file
          for x in self.file_iterator():
              new_data.append(mapping_f(*x))
          return Dataset(new_data)
  
      def reduce(self, reduction_f, initial_accum):
          # Reduce file by file
          accum = initial_accum
          for x in self.file_iterator():
              accum = reduction_f(accum, *x)
          return accum
  
  
  class DatasetIterator(object):
      def __init__(self, dataset, batch_size, get_smaller_final_batch, shuffle, n_batches=None):
          self.dataset = dataset
          self.shuffle = shuffle
          self.n_batches = n_batches
          # # Store an order for the files
          self.files_order = range(self.dataset.get_n_files())
          self.arity = self.dataset.get_arity()
          # # When iterating over a dataset with arity greater than one, a tuple or arrays is returned, otherwise just the array
          if self.arity == 1:
              self.return_type = lambda x: x[0]
          else:
              self.return_type = tuple
          self.element_dimensionalities = [self.dataset.get_dimensionality(i) for i in xrange(self.arity)]
          self.element_sizes = [np.int(self.dataset.get_file(i, 0).strides[0]) for i in xrange(self.arity)]
          self.element_types = [self.dataset.get_type(i) for i in xrange(self.arity)]
          self.element_block_lengths = self.dataset.block_lengths
          self.element_offsets = self.dataset.offsets
          self.block_length = self.dataset.block_length
          self.set_batch_size(batch_size, get_smaller_final_batch)
          # srcs
          self.n_srcs = np.sum([np.int(self.dataset.get_n_blocks_in_file(i)) for i in xrange(self.dataset.get_n_files())])
          self.srcs = [np.zeros(self.n_srcs, dtype="int") for i in xrange(self.dataset.get_arity())]
          src_count = 0
          for f in xrange(self.dataset.get_n_files()):
              for i in xrange(self.dataset.get_n_blocks_in_file(f)):
                  for e in xrange(self.dataset.get_arity()):
                      self.srcs[e][src_count] = self.dataset.files[f][e].ctypes.data + self.element_sizes[e] * (self.element_offsets[e] +  i)
                  src_count += 1
          self.restart()
  
      def set_batch_size(self, n, get_smaller_final_batch):
          self.batch_size = np.int(n)
          self.get_smaller_final_batch = get_smaller_final_batch
          self.batches = [np.empty((self.batch_size, self.element_dimensionalities[i]), dtype=self.element_types[i]) for i in xrange(self.arity)]
  
      def get_arity(self):
          return self.arity
  
      def get_batch_size(self):
          return self.batch_size
  
      def get_datapoint_sizes(self):
          return np.array(self.element_sizes)
  
      def get_datapoint_dimensionalities(self):
          return self.element_dimensionalities
  
      def __iter__(self):
          self.batches_returned = 0
          self.finished = False
          self.restart()
          return self
  
      def restart(self):
          # Shuffle if appropiate
          if self.shuffle:
              self.srcs_order = np.random.permutation(self.n_srcs)
          else:
              self.srcs_order = np.arange(self.n_srcs, dtype="int")
          self.srcs_index = np.int(0)
  
      def next(self):
          if self.batches_returned == self.n_batches or self.finished:
              raise StopIteration
          elif self.srcs_index + self.batch_size < self.n_srcs:
              for j in xrange(self.arity):
                  dst = np.int(self.batches[j].ctypes.data)
                  size = self.element_sizes[j] * self.element_block_lengths[j]
                  for i in xrange(self.batch_size):
                      n = self.srcs_order[self.srcs_index + i]
                      ctypes.memmove(dst, np.int(self.srcs[j][n]), size)
                      dst += size
              self.srcs_index += self.batch_size
              self.batches_returned += 1
              return self.return_type(self.batches)
          elif self.n_batches is None:
              srcs_left = self.n_srcs - self.srcs_index
              if srcs_left ==0 or not self.get_smaller_final_batch:
                  raise StopIteration
              else:
                  for j in xrange(self.arity):
                      dst = np.int(self.batches[j].ctypes.data)
                      size = self.element_sizes[j] * self.element_block_lengths[j]
                      for i in xrange(srcs_left):
                          n = self.srcs_order[self.srcs_index + i]
                          ctypes.memmove(dst, np.int(self.srcs[j][n]), size)
                          dst += size
                  self.srcs_index += srcs_left
                  self.batches_returned += 1
                  self.finished = True
                  return self.return_type([b[0:srcs_left, :] for b in self.batches])
          else:
              self.restart()
              if self.batch_size >= self.n_srcs:
                  if not self.get_smaller_final_batch:
                      raise "Dataset doesn't even contain one minibatch"
                  else:
                      srcs_left = self.n_srcs - self.srcs_index
                      for j in xrange(self.arity):
                          dst = np.int(self.batches[j].ctypes.data)
                          size = self.element_sizes[j] * self.element_block_lengths[j]
                          for i in xrange(srcs_left):
                              n = self.srcs_order[self.srcs_index + i]
                              ctypes.memmove(dst, np.int(self.srcs[j][n]), size)
                              dst += size
                      self.srcs_index += srcs_left
                      self.batches_returned += 1
                      return self.return_type([b[0:srcs_left, :] for b in self.batches])
              else:
                  for j in xrange(self.arity):
                      dst = np.int(self.batches[j].ctypes.data)
                      size = self.element_sizes[j] * self.element_block_lengths[j]
                      for i in xrange(self.batch_size):
                          n = self.srcs_order[self.srcs_index + i]
                          ctypes.memmove(dst, np.int(self.srcs[j][n]), size)
                          dst += size
                  self.srcs_index += self.batch_size
                  self.batches_returned += 1
              return self.return_type(self.batches)
  
  
  class DatasetFileIterator(object):
      # TODO: it should be possible to obtain the files in a randomized order.
      def __init__(self, dataset, **kwargs):
          self.dataset = dataset
  
      def __iter__(self):
          self.f_index = 0
          return self
  
      def next(self):
          if self.f_index >= self.dataset.get_n_files():
              raise StopIteration
          else:
              self.f_index += 1
              x = [self.dataset.get_file(e, self.f_index - 1) for e in xrange(self.dataset.get_arity())]
              return tuple(x)
  
  def load_dataset_from_hdf5(filename, entries_regexp, element_names, block_length=1, use_blocks=None, offsets=None):
          """
          filename: hdf5 filename
          entries_regexp: Regular expresion for selecting files in the hdf5 file (e.g. /training/.*/.*/.*)
          element_names: Name of the hdf5 tensors with the actual data for each of the files. (e.g. tuple("acoustics", "phone_state") ).
          delete_after_use: If set to True the hdf5 file will be deleted from the hard drive when the object is destroyed. It is used to create temporary hdf5 files as a results of "mapping" a dataset.
          """
          f = h5py.File(filename, "r")
          element_names = element_names if isinstance(element_names, tuple) else (element_names,)
          # # Find the entries that satisfy the regexp
          # split the pattern by "/"
          pats = entries_regexp.split("/")
          try:
              pats.remove("")
          except:
              pass
          # traverse from the root
          entries = [f["/"]]
          for p in pats:
              entries = [v for r in entries for k, v in r.items() if re.match("^%s$" % p, str(k))]
          data = [tuple([np.array(e[dsn].value) for dsn in element_names]) for e in entries]
  
          return Dataset(data, block_length, use_blocks, offsets)