text: import tensorflow as tf
  import random
  from tqdm import tqdm
  import spacy
  import ujson as json
  from collections import Counter
  import numpy as np
  from codecs import open
  import jieba
  
  '''
  This file is taken and modified from R-Net by HKUST-KnowComp
  https://github.com/HKUST-KnowComp/R-Net
  '''
  
  # nlp = spacy.blank("en")
  
  
  def word_tokenize(sent):
      # doc = nlp(sent)
      # return [token.text for token in doc]
      doc = jieba.cut(sent)
      return list(doc)
  
  def convert_idx(text, tokens):
      current = 0
      spans = []
      for token in tokens:
          current = text.find(token, current)
          if current < 0:
              print("Token {} cannot be found".format(token))
              raise Exception()
          spans.append((current, current + len(token)))
          current += len(token)
      return spans
  
  
  def process_file(filename, data_type, word_counter, char_counter):
      print("Generating {} examples...".format(data_type))
      examples = []
      eval_examples = {}
      total = 0
      with open(filename, "r") as fh:
          source = json.load(fh)
          for _id in range(len(source)):
              context = source[_id]['context'].replace("''", '" ').replace("``", '" ')
              context_tokens = list(context)
              context_chars = [list(token) for token in context_tokens]
              spans = convert_idx(context, context_tokens)
              
              ques = source[_id]["question"].replace("''", '" ').replace("``", '" ')
              ques_tokens = list(ques)
              ques_chars = [list(token) for token in ques_tokens]
  
              opts_tokens = list(source[_id]["options"])
  
              for token in context_tokens:
                  word_counter[token] += 1
                  for char in token:
                      char_counter[char] += 1
  
              for token in ques_tokens:
                  word_counter[token] += 1
                  for char in token:
                      char_counter[char] += 1
              
              for token in opts_tokens:
                  word_counter[token] += 1
                  for char in token:
                      char_counter[char] += 1
  
  
              example = {"context_tokens": context_tokens, "context_chars": context_chars, "ques_tokens": ques_tokens,
                         "ques_chars": ques_chars, "y1s": [-1], "y2s": [-1], "id": source[_id]['id']}
              examples.append(example)
  
              eval_examples[source[_id]['id']] = {
                  "context": context, "spans": spans, "answers": None, "uuid": source[_id]['id']}
  
          print("{} questions in total".format(len(examples)))
      return examples, eval_examples
  
  
  def get_embedding(counter, data_type, limit=-1, emb_file=None, size=None, vec_size=None):
      print("Generating {} embedding...".format(data_type))
      embedding_dict = {}
      filtered_elements = [k for k, v in counter.items() if v > limit]
      if emb_file is not None:
          assert size is not None
          assert vec_size is not None
          with open(emb_file, "r", encoding="utf-8") as fh:
              print('embedding file info : {}'.format(fh.readline()))
              for line in tqdm(fh, total=size):
                  array = line.split()
                  word = "".join(array[0:-vec_size])
                  vector = list(map(float, array[-vec_size:]))
                  if word in counter and counter[word] > limit:
                      embedding_dict[word] = vector
          print("{} / {} tokens have corresponding {} embedding vector".format(
              len(embedding_dict), len(filtered_elements), data_type))
      else:
          assert vec_size is not None
          for token in filtered_elements:
              embedding_dict[token] = [np.random.normal(
                  scale=0.1) for _ in range(vec_size)]
          print("{} tokens have corresponding embedding vector".format(
              len(filtered_elements)))
  
      NULL = "--NULL--"
      OOV = "--OOV--"
      token2idx_dict = {token: idx for idx,
                        token in enumerate(embedding_dict.keys(), 2)}
      token2idx_dict[NULL] = 0
      token2idx_dict[OOV] = 1
      embedding_dict[NULL] = [0. for _ in range(vec_size)]
      embedding_dict[OOV] = [0. for _ in range(vec_size)]
      idx2emb_dict = {idx: embedding_dict[token]
                      for token, idx in token2idx_dict.items()}
      emb_mat = [idx2emb_dict[idx] for idx in range(len(idx2emb_dict))]
      return emb_mat, token2idx_dict
  
  def convert_to_features(config, data, word2idx_dict, char2idx_dict):
  
      example = {}
      context, question = data
      context = context.replace("''", '" ').replace("``", '" ')
      question = question.replace("''", '" ').replace("``", '" ')
      example['context_tokens'] = list(context)
      example['ques_tokens'] = list(question)
      example['context_chars'] = [list(token) for token in example['context_tokens']]
      example['ques_chars'] = [list(token) for token in example['ques_tokens']]
  
      para_limit = config.test_para_limit
      ques_limit = config.test_ques_limit
      ans_limit = 100
      char_limit = config.char_limit
  
      def filter_func(example):
          return len(example["context_tokens"]) > para_limit or \
                 len(example["ques_tokens"]) > ques_limit
  
      if filter_func(example):
          raise ValueError("Context/Questions lengths are over the limit")
  
      context_idxs = np.zeros([para_limit], dtype=np.int32)
      context_char_idxs = np.zeros([para_limit, char_limit], dtype=np.int32)
      ques_idxs = np.zeros([ques_limit], dtype=np.int32)
      ques_char_idxs = np.zeros([ques_limit, char_limit], dtype=np.int32)
      y1 = np.zeros([para_limit], dtype=np.float32)
      y2 = np.zeros([para_limit], dtype=np.float32)
  
      def _get_word(word):
          for each in (word, word.lower(), word.capitalize(), word.upper()):
              if each in word2idx_dict:
                  return word2idx_dict[each]
          return 1
  
      def _get_char(char):
          if char in char2idx_dict:
              return char2idx_dict[char]
          return 1
  
      for i, token in enumerate(example["context_tokens"]):
          context_idxs[i] = _get_word(token)
  
      for i, token in enumerate(example["ques_tokens"]):
          ques_idxs[i] = _get_word(token)
  
      for i, token in enumerate(example["context_chars"]):
          for j, char in enumerate(token):
              if j == char_limit:
                  break
              context_char_idxs[i, j] = _get_char(char)
  
      for i, token in enumerate(example["ques_chars"]):
          for j, char in enumerate(token):
              if j == char_limit:
                  break
              ques_char_idxs[i, j] = _get_char(char)
  
      return context_idxs, context_char_idxs, ques_idxs, ques_char_idxs
  
  def build_features(config, examples, data_type, out_file, word2idx_dict, char2idx_dict, is_test=False):
  
      para_limit = config.test_para_limit if is_test else config.para_limit
      ques_limit = config.test_ques_limit if is_test else config.ques_limit
      ans_limit = 100 if is_test else config.ans_limit
      char_limit = config.char_limit
  
      def filter_func(example, is_test=False):
          return len(example["context_tokens"]) > para_limit or \
                 len(example["ques_tokens"]) > ques_limit or \
                 (example["y2s"][0] - example["y1s"][0]) > ans_limit
  
      print("Processing {} examples...".format(data_type))
      writer = tf.python_io.TFRecordWriter(out_file)
      total = 0
      total_ = 0
      meta = {}
      for example in tqdm(examples):
          total_ += 1
  
          total += 1
          context_idxs = np.zeros([para_limit], dtype=np.int32)
          context_char_idxs = np.zeros([para_limit, char_limit], dtype=np.int32)
          ques_idxs = np.zeros([ques_limit], dtype=np.int32)
          ques_char_idxs = np.zeros([ques_limit, char_limit], dtype=np.int32)
          y1 = np.zeros([para_limit], dtype=np.float32)
          y2 = np.zeros([para_limit], dtype=np.float32)
  
          def _get_word(word):
              for each in (word, word.lower(), word.capitalize(), word.upper()):
                  if each in word2idx_dict:
                      return word2idx_dict[each]
              return 1
  
          def _get_char(char):
              if char in char2idx_dict:
                  return char2idx_dict[char]
              return 1
  
          for i, token in enumerate(example["context_tokens"]):
              context_idxs[i] = _get_word(token)
  
          for i, token in enumerate(example["ques_tokens"]):
              ques_idxs[i] = _get_word(token)
  
          for i, token in enumerate(example["context_chars"]):
              for j, char in enumerate(token):
                  if j == char_limit:
                      break
                  context_char_idxs[i, j] = _get_char(char)
  
          for i, token in enumerate(example["ques_chars"]):
              for j, char in enumerate(token):
                  if j == char_limit:
                      break
                  ques_char_idxs[i, j] = _get_char(char)
  
          start, end = example["y1s"][-1], example["y2s"][-1]
          y1[start], y2[end] = 1.0, 1.0
  
          record = tf.train.Example(features=tf.train.Features(feature={
                                    "context_idxs": tf.train.Feature(bytes_list=tf.train.BytesList(value=[context_idxs.tostring()])),
                                    "ques_idxs": tf.train.Feature(bytes_list=tf.train.BytesList(value=[ques_idxs.tostring()])),
                                    "context_char_idxs": tf.train.Feature(bytes_list=tf.train.BytesList(value=[context_char_idxs.tostring()])),
                                    "ques_char_idxs": tf.train.Feature(bytes_list=tf.train.BytesList(value=[ques_char_idxs.tostring()])),
                                    "y1": tf.train.Feature(bytes_list=tf.train.BytesList(value=[y1.tostring()])),
                                    "y2": tf.train.Feature(bytes_list=tf.train.BytesList(value=[y2.tostring()])),
                                    "id": tf.train.Feature(int64_list=tf.train.Int64List(value=[example["id"]]))
                                    }))
          writer.write(record.SerializeToString())
      print("Built {} / {} instances of features in total".format(total, total_))
      meta["total"] = total
      writer.close()
      return meta
  
  
  def save(filename, obj, message=None):
      if message is not None:
          print("Saving {}...".format(message))
          with open(filename, "w") as fh:
              json.dump(obj, fh)
  
  
  def prepro(config):
      word_counter, char_counter = Counter(), Counter()
  
      test_examples, test_eval = process_file(
          config.test_file, "test", word_counter, char_counter)
      
      word_emb_file = config.tw_w2v
      char_emb_file = config.tw_c2v
      char_emb_size = config.tw_char_size
      char_emb_dim = config.tw_char_dim
  
      word_emb_mat, word2idx_dict = get_embedding(
          word_counter, "word", emb_file=word_emb_file, size=config.tw_word_size, vec_size=config.tw_word_dim)
      char_emb_mat, char2idx_dict = get_embedding(
          char_counter, "char", emb_file=char_emb_file, size=char_emb_size, vec_size=char_emb_dim)
      '''
      with open(config.char_emb_file, "r") as fh:
          char_mat = np.array(json.load(fh), dtype=np.float32)
      with open(config.char_dictionary, "r") as fh:
          char2idx_dict = json.load(fh)
      '''
      test_meta = build_features(config, test_examples, "test",
                                 config.test_record_file, char2idx_dict, char2idx_dict, is_test=True)
      
      save(config.test_eval_file, test_eval, message="test eval")
      save(config.test_meta, test_meta, message="test meta")
      save(config.word_emb_file, word_emb_mat, message="word embedding")
      save(config.word_dictionary, word2idx_dict, message="word dictionary")
      save(config.char_emb_file, char_emb_mat, message="char embedding")
      save(config.char_dictionary, char2idx_dict, message="char dictionary")
  