text: import gc
  import glob
  import json
  import os
  import torch
  import argparse
  from multiprocess import Pool
  from os.path import join as pjoin
  from others.logging import init_logger
  from others.logging import logger
  
  from others.tokenization import BertTokenizer
  
  parser = argparse.ArgumentParser()
  parser.add_argument("-pretrained_model", default='bert', type=str)
  
  #parser.add_argument("-mode", default='', type=str)
  #parser.add_argument("-select_mode", default='greedy', type=str)
  parser.add_argument("-map_path", default='../../data/')
  parser.add_argument("-raw_path", default='../../line_data')
  parser.add_argument("-save_path", default='../../data/')
  
  parser.add_argument("-shard_size", default=2000, type=int)
  parser.add_argument('-min_src_nsents', default=3, type=int)
  parser.add_argument('-max_src_nsents', default=100, type=int)
  parser.add_argument('-min_src_ntokens_per_sent', default=5, type=int)
  parser.add_argument('-max_src_ntokens_per_sent', default=200, type=int)
  parser.add_argument('-min_tgt_ntokens', default=5, type=int)
  parser.add_argument('-max_tgt_ntokens', default=500, type=int)
  
  parser.add_argument("-lower", type=str2bool, nargs='?',const=True,default=True)
  parser.add_argument("-use_bert_basic_tokenizer", type=str2bool, nargs='?',const=True,default=False)
  
  parser.add_argument('-log_file', default='../../logs/cnndm.log')
  
  parser.add_argument('-dataset', default='')
  
  parser.add_argument('-n_cpus', default=2, type=int)
  
  
  args = parser.parse_args()
  init_logger(args.log_file)
  
  
  class BertData():
      def __init__(self, args):
          self.args = args
          self.tokenizer = BertTokenizer.from_pretrained('bert-base-uncased', do_lower_case=True)
  
          self.sep_token = '[SEP]'
          self.cls_token = '[CLS]'
          self.pad_token = '[PAD]'
          self.tgt_bos = '[unused0]'
          self.tgt_eos = '[unused1]'
          self.tgt_sent_split = '[unused2]'
          self.sep_vid = self.tokenizer.vocab[self.sep_token]
          self.cls_vid = self.tokenizer.vocab[self.cls_token]
          self.pad_vid = self.tokenizer.vocab[self.pad_token]
  
      def preprocess(self, src, tgt, sent_labels, use_bert_basic_tokenizer=False, is_test=False):
  
          if ((not is_test) and len(src) == 0):
              return None
  
          original_src_txt = [' '.join(s) for s in src]
  
          idxs = [i for i, s in enumerate(src) if (len(s) > self.args.min_src_ntokens_per_sent)]
  
          _sent_labels = [0] * len(src)
          for l in sent_labels:
              _sent_labels[l] = 1
  
          src = [src[i][:self.args.max_src_ntokens_per_sent] for i in idxs]
          sent_labels = [_sent_labels[i] for i in idxs]
          src = src[:self.args.max_src_nsents]
          sent_labels = sent_labels[:self.args.max_src_nsents]
  
          if ((not is_test) and len(src) < self.args.min_src_nsents):
              return None
  
          src_txt = [' '.join(sent) for sent in src]
          text = ' {} {} '.format(self.sep_token, self.cls_token).join(src_txt)
  
          src_subtokens = self.tokenizer.tokenize(text)
  
          src_subtokens = [self.cls_token] + src_subtokens + [self.sep_token]
          src_subtoken_idxs = self.tokenizer.convert_tokens_to_ids(src_subtokens)
          _segs = [-1] + [i for i, t in enumerate(src_subtoken_idxs) if t == self.sep_vid]
          segs = [_segs[i] - _segs[i - 1] for i in range(1, len(_segs))]
          segments_ids = []
          for i, s in enumerate(segs):
              if (i % 2 == 0):
                  segments_ids += s * [0]
              else:
                  segments_ids += s * [1]
          cls_ids = [i for i, t in enumerate(src_subtoken_idxs) if t == self.cls_vid]
          sent_labels = sent_labels[:len(cls_ids)]
  
          tgt_subtokens_str = '[unused0] ' + ' [unused2] '.join(
              [' '.join(self.tokenizer.tokenize(' '.join(tt), use_bert_basic_tokenizer=use_bert_basic_tokenizer)) for tt in tgt]) + ' [unused1]'
          tgt_subtoken = tgt_subtokens_str.split()[:self.args.max_tgt_ntokens]
          if ((not is_test) and len(tgt_subtoken) < self.args.min_tgt_ntokens):
              return None
  
          tgt_subtoken_idxs = self.tokenizer.convert_tokens_to_ids(tgt_subtoken)
  
          tgt_txt = '<q>'.join([' '.join(tt) for tt in tgt])
          src_txt = [original_src_txt[i] for i in idxs]
  
          return src_subtoken_idxs, sent_labels, tgt_subtoken_idxs, segments_ids, cls_ids, src_txt, tgt_txt
  
  def format_to_bert(args, sent_labels):
      if (args.dataset != ''):
          datasets = [args.dataset]
      else:
          datasets = ['train', 'valid', 'test']
      for corpus_type in datasets:
          a_lst = []
          for json_f in glob.glob(pjoin(args.raw_path, '*' + corpus_type + '.*.json')):
              real_name = json_f.split('/')[-1]
              a_lst.append((corpus_type, json_f, args, pjoin(args.save_path, real_name.replace('json', 'bert.pt'))))
          print(a_lst)
          pool = Pool(args.n_cpus)
          for d in pool.imap(_format_to_bert, a_lst):
              pass
  
          pool.close()
          pool.join()
  
  
  def _format_to_bert(params, sent_labels):
      corpus_type, json_file, args, save_file = params
      is_test = corpus_type == 'test'
      if (os.path.exists(save_file)):
          logger.info('Ignore %s' % save_file)
          return
  
      bert = BertData(args)
  
      logger.info('Processing %s' % json_file)
      jobs = json.load(open(json_file))
      datasets = []
      for d in jobs:
          source, tgt = d['src'], d['tgt']
  
          if (args.lower):
              source = [' '.join(s).lower().split() for s in source]
              tgt = [' '.join(s).lower().split() for s in tgt]
          b_data = bert.preprocess(source, tgt, sent_labels, use_bert_basic_tokenizer=args.use_bert_basic_tokenizer,
                                   is_test=is_test)
          # b_data = bert.preprocess(source, tgt, sent_labels, use_bert_basic_tokenizer=args.use_bert_basic_tokenizer)
  
          if (b_data is None):
              continue
          src_subtoken_idxs, sent_labels, tgt_subtoken_idxs, segments_ids, cls_ids, src_txt, tgt_txt = b_data
          b_data_dict = {"src": src_subtoken_idxs, "tgt": tgt_subtoken_idxs,
                         "src_sent_labels": sent_labels, "segs": segments_ids, 'clss': cls_ids,
                         'src_txt': src_txt, "tgt_txt": tgt_txt}
          datasets.append(b_data_dict)
      logger.info('Processed instances %d' % len(datasets))
      logger.info('Saving to %s' % save_file)
      torch.save(datasets, save_file)
      datasets = []
      gc.collect()