code: def make_dataset(path, w2id, tok=False):
      """generate dataset."""
      dataset, npy_dataset = [], []
      token_count, unknown_count = 0, 0
      for words in read_file(path, tok):
          array = make_array(w2id, words)
          npy_dataset.append(array)
          dataset.append(words)
          token_count += array.size
          unknown_count += (array == unk_token_id).sum()
      print("# of tokens:{}".format(token_count))
      print(
          "# of unknown {} {:.2}".format(
              unknown_count, 100.0 * unknown_count / token_count
          )
      )
      return dataset, npy_dataset
  
file_path: texar/examples/transformer/utils/preprocess.py
imports: import collections
  import re
  import json
  import os
  import numpy
  import pickle
  import argparse
  from io import open
label_desc: Use image node_id 6 for calling a global function with example usage: make_dataset(path, w2id, tok) and returns: dataset,  npy_dataset