class_docstr: This class defines an iterator for loading and iterating the sentences in
  a structure to train the skip-thought vectors
class_docstr_tok: ["This","class","defines","an","iterator","for","loading","and","iterating","the","sentences","in","a","structure","to","train","the","skip-thought","vectors"]
docstr: Construct a sentence dataset object.
  Build the context using skip-thought model
  
  Aguments:
      data_file (str): path to hdf5 file containing sentences
      sent_name (str): name of tokenized dataset
      text_name (str): name of raw text dataset
      nwords (int): size of vocabulary
      max_len (int): maximum number of words per sentence
      index_from (int): index offset for padding (0) and OOV (1)
      eos (int): index of EOS token
docstr_tok: ["Construct","a","sentence","dataset","object",".","Build","the","context","using","skip-thought","model","Aguments",":","data_file","(","str",")",":","path","to","hdf5","file","containing","sentences","sent_name","(","str",")",":","name","of","tokenized","dataset","text_name","(","str",")",":","name","of","raw","text","dataset","nwords","(","int",")",":","size","of","vocabulary","max_len","(","int",")",":","maximum","number","of","words","per","sentence","index_from","(","int",")",":","index","offset","for","padding","(","0",")","and","OOV","(","1",")","eos","(","int",")",":","index","of","EOS","token"]
code: def __init__(
      self,
      data_file=None,
      sent_name=None,
      text_name=None,
      nwords=None,
      max_len=30,
      index_from=2,
      eos=3,
  ):
      """
      Construct a sentence dataset object.
      Build the context using skip-thought model
  
      Aguments:
          data_file (str): path to hdf5 file containing sentences
          sent_name (str): name of tokenized dataset
          text_name (str): name of raw text dataset
          nwords (int): size of vocabulary
          max_len (int): maximum number of words per sentence
          index_from (int): index offset for padding (0) and OOV (1)
          eos (int): index of EOS token
      """
      super(SentenceHomogenous, self).__init__(name=None)
      self.nwords = nwords
      self.batch_index = 0
      self.nbatches = 0
      self.max_len = max_len
      self.index_from = index_from
      self.eos = eos
      self.data_file = data_file
      self.sent_name = sent_name
      self.text_name = text_name
  
      h5f = h5py.File(self.data_file, "r+")
  
      # Extract sentences array from h5 file and make copy in memory
      sentences = h5f[self.sent_name][:]
  
      # Load sentence raw text if desired
      # sentence_text = h5f[self.text_name]
  
      self.nsamples = h5f[self.sent_name].attrs["nsample"] - 2
  
      # Use shifted view of in-memory copy of sentences to group sentences into triplets
      self.source = sentences[1:-1]
      self.forward = sentences[2:]
      self.backward = sentences[:-2]
  
      self.lengths = [len(cc) for cc in self.source]
      self.len_unique = np.unique(self.lengths)
      self.len_unique = [
          ll for ll in self.len_unique if ll <= self.max_len
      ]
  
      self.len_indicies = dict()
      self.len_counts = dict()
      for ll in self.len_unique:
          self.len_indicies[ll] = np.where(self.lengths == ll)[0]
          self.len_counts[ll] = len(self.len_indicies[ll])
  
      # Compute number of batches of homogenous lengths
      self.nbatches = 0
      for ll in self.len_unique:
          self.nbatches += int(
              np.ceil(self.len_counts[ll] / float(self.be.bsz))
          )
  
      # compute the total number of samples (including empty samples in minibatches)
      self.ndata = self.nbatches * self.be.bsz
  
      self.len_curr_counts = copy.copy(self.len_counts)
  
      # inputs using embeddings
      self.dev_X = self.be.iobuf(self.max_len, dtype=np.int32)
      self.dev_X_p = self.be.iobuf(
          self.max_len, dtype=np.int32
      )  # previous sentence as input
      self.dev_X_n = self.be.iobuf(
          self.max_len, dtype=np.int32
      )  # next sentence as input
  
      # the np array to put noncontiguous data in. use for entire minibatch
      self.X_np = np.empty((self.max_len, self.be.bsz), dtype=np.int32)
      self.X_p_np = np.empty(
          (self.max_len, self.be.bsz), dtype=np.int32
      )
      self.X_n_np = np.empty(
          (self.max_len, self.be.bsz), dtype=np.int32
      )
  
      # flat output labels, do one-hot on device
      self.dev_y_p_flat = self.be.iobuf(
          (1, self.max_len), dtype=np.int32
      )
      self.dev_y_n_flat = self.be.iobuf(
          (1, self.max_len), dtype=np.int32
      )
      self.dev_y_p = self.be.iobuf(
          (nwords, self.max_len), dtype=np.int32
      )
      self.dev_y_n = self.be.iobuf(
          (nwords, self.max_len), dtype=np.int32
      )
  
      # output labels and masks to deal with variable length sentences
      self.dev_y_p_mask = self.be.iobuf(
          (nwords, self.max_len), dtype=np.int32
      )
      self.dev_y_n_mask = self.be.iobuf(
          (nwords, self.max_len), dtype=np.int32
      )
      self.dev_y_p_mask_list = self.get_bsz(
          self.dev_y_p_mask, self.max_len
      )
      self.dev_y_n_mask_list = self.get_bsz(
          self.dev_y_n_mask, self.max_len
      )
  
      # for the flat label
      self.y_p_np = np.empty(
          (self.max_len, self.be.bsz), dtype=np.int32
      )
      self.y_n_np = np.empty(
          (self.max_len, self.be.bsz), dtype=np.int32
      )
  
      self.clear_list = [
          self.X_np,
          self.X_p_np,
          self.X_n_np,
          self.y_p_np,
          self.y_n_np,
          self.dev_y_p_mask,
          self.dev_y_n_mask,
      ]
      self.shape = [
          (self.max_len, 1),
          (self.max_len, 1),
          (self.max_len, 1),
      ]
  
      h5f.close()
      self.reset()
  
file_path: neon/examples/skip-thought/data_iterator.py
imports: from __future__ import division
  import numpy
  import h5py
  import copy
  from neon.data.dataiterator import NervanaDataIterator
label_desc: Use image node_id 1 to create a new SentenceHomogenous object from inherited base classes: NervanaDataIterator with example: obj = SentenceHomogenous(data_file, sent_name, text_name, nwords, max_len, index_from, eos)