code: def main(args):
      # For Chinese (Ro)Bert, the best result is from : RoBERTa-wwm-ext (https://github.com/ymcui/Chinese-BERT-wwm)
      # If we want to fine-tune these model, we have to use same tokenizer : LTP (https://github.com/HIT-SCIR/ltp)
      with open(args.file_name, "r", encoding="utf-8") as f:
          data = f.readlines()
      data = [
          line.strip()
          for line in data
          if len(line) > 0 and not line.isspace()
      ]  # avoid delimiter like '\u2029'
      ltp_tokenizer = LTP(args.ltp)  # faster in GPU device
      bert_tokenizer = BertTokenizer.from_pretrained(args.bert)
  
      ref_ids = prepare_ref(data, ltp_tokenizer, bert_tokenizer)
  
      with open(args.save_path, "w", encoding="utf-8") as f:
          data = [json.dumps(ref) + "\n" for ref in ref_ids]
          f.writelines(data)
  
file_path: H2O/h2o_flexgen/benchmark/third_party/transformers/examples/legacy/run_chinese_ref.py
imports: import argparse
  import json
  from typing import List
  from ltp import LTP
  from transformers import BertTokenizer
label_desc: Use image node_id 6 for calling a global function with example usage: main(args) without return types