code: def load_data(
      path,
      file_ext=["txt"],
      valid_split=None,
      vocab_file_name=None,
      max_vocab_size=None,
      max_len_w=None,
      output_path=None,
      subset_pct=100,
  ):
      """
      Given a path where data are saved, look for the ones with the right extensions
      If a split factor is given, it will split all the files into training and valid
      set. Then build vocabulary from the training and validation sets.
  
      Arguments:
          path: which directory to look for all the documents
          file_ext: what extension of the files to look for
          valid_split: to split the data into train/valid set. If None, no split
          vocab_file_name: optional file name. If None, the script will decide a name
                           given path and split
          max_vocab_size: maximum number of words to use in vocabulary (by most frequent)
          max_len_w: maximum length of sentences in words
          output_path: path used to save preprocessed data and resuts
          subset_pct: subset of dataset to load into H5 file (percentage)
  
      Returns:
          The function saves 2 files:
          h5 file with preprocessed data
          vocabulary file with: vocab, reverse_vocab, word_count
      """
      file_names = get_file_list(path, file_ext)
  
      file_str = get_file_str(
          path,
          len(file_names),
          labelled=False,
          valid_split=valid_split,
          subset_pct=subset_pct,
      )
  
      # create output dir if needed
      if not os.path.isdir(output_path):
          os.makedirs(output_path)
  
      # file name to store the vocabulary
      if vocab_file_name is None:
          vocab_file_name = file_str + ".vocab"
          vocab_file_name = os.path.join(output_path, vocab_file_name)
  
      # If max sizes arent set, assume no limit
      if not max_len_w:
          max_len_w = sys.maxsize
      if not max_vocab_size:
          max_vocab_size = sys.maxsize
  
      # file name to store the pre-processed train/valid dataset
      h5_file_name = os.path.join(output_path, file_str + ".h5")
  
      if os.path.exists(h5_file_name) and os.path.exists(
          vocab_file_name
      ):
          neon_logger.display(
              "dataset files {} and vocabulary file {} already exist. "
              "will use cached data. ".format(
                  h5_file_name, vocab_file_name
              )
          )
          return h5_file_name, vocab_file_name
  
      # split into training/valid set
      if valid_split is not None:
          if "json" in file_ext:
              # Split based on number of files
              train_split = int(
                  np.ceil(len(file_names) * (1 - valid_split))
              )
              train_files = file_names[:train_split]
              valid_files = file_names[train_split:]
  
              train_sent = load_json_sent(train_files, subset_pct)
              valid_sent = load_json_sent(valid_files, subset_pct)
              all_sent = train_sent + valid_sent
          elif "txt" in file_ext:
              # Split based on number of lines (since only 2 files)
              all_sent = load_txt_sent(file_names, subset_pct)
              train_split = int(
                  np.ceil(len(all_sent) * (1 - valid_split))
              )
  
              train_sent = all_sent[:train_split]
              valid_sent = all_sent[train_split:]
          else:
              neon_logger.display(
                  "Unsure how to load file_ext {}, please use 'json' or 'txt'.".format(
                      file_ext
                  )
              )
      else:
          train_files = file_names
          if "json" in file_ext:
              train_sent = load_json_sent(train_files, subset_pct)
          elif "txt" in file_ext:
              train_sent = load_txt_sent(train_files, subset_pct)
          else:
              neon_logger.display(
                  "Unsure how to load file_ext {}, please use 'json' or 'txt'.".format(
                      file_ext
                  )
              )
          all_sent = train_sent
  
      if os.path.exists(vocab_file_name):
          neon_logger.display(
              "open existing vocab file: {}".format(vocab_file_name)
          )
          vocab, rev_vocab, word_count = load_obj(vocab_file_name)
      else:
          neon_logger.display("Building  vocab file")
  
          # build vocab
          word_count = defaultdict(int)
          for sent in all_sent:
              sent_words = tokenize(sent)
  
              if len(sent_words) > max_len_w or len(sent_words) == 0:
                  continue
  
              for word in sent_words:
                  word_count[word] += 1
  
          # sort the word_count , re-assign ids by its frequency. Useful for downstream tasks
          # only done for train vocab
          vocab_sorted = sorted(
              word_count.items(), key=lambda kv: kv[1], reverse=True
          )
  
          vocab = OrderedDict()
  
          # get word count as array in same ordering as vocab (but with maximum length)
          word_count_ = np.zeros((len(word_count),), dtype=np.int64)
          for i, t in enumerate(
              list(zip(*vocab_sorted))[0][:max_vocab_size]
          ):
              word_count_[i] = word_count[t]
              vocab[t] = i
          word_count = word_count_
  
          # generate the reverse vocab
          rev_vocab = dict(
              (wrd_id, wrd) for wrd, wrd_id in vocab.items()
          )
  
          neon_logger.display(
              "vocabulary from {} is saved into {}".format(
                  path, vocab_file_name
              )
          )
          save_obj((vocab, rev_vocab, word_count), vocab_file_name)
  
      vocab_size = len(vocab)
      neon_logger.display(
          "\nVocab size from the dataset is: {}".format(vocab_size)
      )
  
      neon_logger.display(
          "\nProcessing and saving training data into {}".format(
              h5_file_name
          )
      )
  
      # now process and save the train/valid data
      h5f = h5py.File(h5_file_name, "w", libver="latest")
      shape, maxshape = (len(train_sent),), (None)
      dt = np.dtype(
          [
              ("text", h5py.special_dtype(vlen=str)),
              ("num_words", np.uint16),
          ]
      )
      report_text_train = h5f.create_dataset(
          "report_train",
          shape=shape,
          maxshape=maxshape,
          dtype=dt,
          compression="gzip",
      )
      report_train = h5f.create_dataset(
          "train",
          shape=shape,
          maxshape=maxshape,
          dtype=h5py.special_dtype(vlen=np.int32),
          compression="gzip",
      )
  
      # map text to integers
      wdata = np.zeros((1,), dtype=dt)
      ntrain = 0
      for sent in train_sent:
          text_int = [
              -1 if t not in vocab else vocab[t] for t in tokenize(sent)
          ]
  
          # enforce maximum sentence length
          if len(text_int) > max_len_w or len(text_int) == 0:
              continue
  
          report_train[ntrain] = text_int
  
          wdata["text"] = clean_string(sent)
          wdata["num_words"] = len(text_int)
          report_text_train[ntrain] = wdata
          ntrain += 1
  
      report_train.attrs["nsample"] = ntrain
      report_train.attrs["vocab_size"] = vocab_size
      report_text_train.attrs["nsample"] = ntrain
      report_text_train.attrs["vocab_size"] = vocab_size
  
      if valid_split:
          neon_logger.display(
              "\nProcessing and saving validation data into {}".format(
                  h5_file_name
              )
          )
          shape = (len(valid_sent),)
          report_text_valid = h5f.create_dataset(
              "report_valid",
              shape=shape,
              maxshape=maxshape,
              dtype=dt,
              compression="gzip",
          )
          report_valid = h5f.create_dataset(
              "valid",
              shape=shape,
              maxshape=maxshape,
              dtype=h5py.special_dtype(vlen=np.int32),
              compression="gzip",
          )
          nvalid = 0
          for sent in valid_sent:
              text_int = [
                  -1 if t not in vocab else vocab[t]
                  for t in tokenize(sent)
              ]
  
              # enforce maximum sentence length
              if len(text_int) > max_len_w or len(text_int) == 0:
                  continue
  
              report_valid[nvalid] = text_int
              wdata["text"] = clean_string(sent)
              wdata["num_words"] = len(text_int)
              report_text_valid[nvalid] = wdata
              nvalid += 1
  
          report_valid.attrs["nsample"] = nvalid
          report_valid.attrs["vocab_size"] = vocab_size
          report_text_valid.attrs["nsample"] = nvalid
          report_text_valid.attrs["vocab_size"] = vocab_size
  
      h5f.close()
  
      return h5_file_name, vocab_file_name
  
file_path: neon/examples/skip-thought/data_loader.py
imports: from __future__ import print_function
  import os
  import h5py
  import json
  import numpy
  from collections import defaultdict, OrderedDict
  import sys
  from neon import logger
  from neon.util.persist import load_obj, save_obj
  from util import tokenize, clean_string
label_desc: Use image node_id 6 for calling a global function with example usage: load_data(path, file_ext, valid_split, vocab_file_name, max_vocab_size, max_len_w, output_path, subset_pct) and returns: h5_file_name,  vocab_file_name, h5_file_name,  vocab_file_name