code: def convert_examples_to_features(
      examples,
      label_list,
      max_seq_length,
      tokenizer,
      cls_token_at_end=False,
      cls_token="[CLS]",
      cls_token_segment_id=1,
      sep_token="[SEP]",
      sep_token_extra=False,
      pad_on_left=False,
      pad_token=0,
      pad_token_segment_id=0,
      pad_token_label_id=-100,
      sequence_a_segment_id=0,
      mask_padding_with_zero=True,
      model_name=None,
  ):
      """Loads a data file into a list of `InputBatch`s
      `cls_token_at_end` define the location of the CLS token:
          - False (Default, BERT/XLM pattern): [CLS] + A + [SEP] + B + [SEP]
          - True (XLNet/GPT pattern): A + [SEP] + B + [SEP] + [CLS]
      `cls_token_segment_id` define the segment id associated to the CLS token (0 for BERT, 2 for XLNet)
      """
  
      label_map = {label: i for i, label in enumerate(label_list)}
  
      # my logic in crf_padding requires this check. I create mask for crf by labels==pad_token_label_id to not include it
      # in loss and decoding
      assert pad_token_label_id not in label_map.values()
  
      features = []
      for ex_index, example in enumerate(examples):
          if ex_index % 10000 == 0:
              print("###############")
              logger.info(
                  "Writing example %d of %d", ex_index, len(examples)
              )
              print("###############")
  
          tokens = []
          label_ids = []
          token_type_ids = []
  
          assert len(example.words) == len(example.labels)
          if example.trigger_word_ids is not None:
              assert (
                  len(example.words)
                  == len(example.labels)
                  == len(example.trigger_word_ids)
              )
  
          # print("example.words:", example.words)
          # print("example.labels:", example.labels)
          # print("example.trigger_word_ids:", example.trigger_word_ids)
  
          for idx, word, label in zip(
              range(len(example.words)), example.words, example.labels
          ):
              word_tokens = tokenizer.tokenize(word)
              # clear out wrong chars
              if len(word_tokens) == 0:
                  continue
              tokens.append(word)
  
              # Use the real label id for the first token of the word, and padding ids for the remaining tokens
              if label != "X":
                  label_ids.extend(
                      [label_map[label]]
                      + [pad_token_label_id] * (len([word]) - 1)
                  )
              else:
                  label_ids.extend(
                      [pad_token_label_id]
                      + [pad_token_label_id] * (len([word]) - 1)
                  )
  
              if example.trigger_word_ids is not None:
                  trigger_word_id = example.trigger_word_ids[idx]
                  token_type = [0] if trigger_word_id == "O" else [1]
                  token_type_ids.extend(token_type)
              else:
                  token_type_ids.extend([0])
  
          # print("len(tokens):", len(tokens))
          # print("len(label_ids):", len(label_ids))
          # print("len(token_type_ids):", len(token_type_ids))
  
          # Account for [CLS] and [SEP] with "- 2" and with "- 3" for RoBERTa.
          special_tokens_count = 3 if sep_token_extra else 2
          if len(tokens) > max_seq_length - special_tokens_count:
              tokens = tokens[: (max_seq_length - special_tokens_count)]
              label_ids = label_ids[
                  : (max_seq_length - special_tokens_count)
              ]
              token_type_ids = token_type_ids[
                  : (max_seq_length - special_tokens_count)
              ]
  
          # The convention in BERT is:
          # (a) For sequence pairs:
          #  tokens:   [CLS] is this jack ##son ##ville ? [SEP] no it is not . [SEP]
          #  type_ids:   0   0  0    0    0     0       0   0   1  1  1  1   1   1
          # (b) For single sequences:
          #  tokens:   [CLS] the dog is hairy . [SEP]
          #  type_ids:   0   0   0   0  0     0   0
          #
          # Where "type_ids" are used to indicate whether this is the first
          # sequence or the second sequence. The embedding vectors for `type=0` and
          # `type=1` were learned during pre-training and are added to the wordpiece
          # embedding vector (and position vector). This is not *strictly* necessary
          # since the [SEP] token unambiguously separates the sequences, but it makes
          # it easier for the model to learn the concept of sequences.
          #
          # For classification tasks, the first vector (corresponding to [CLS]) is
          # used as as the "sentence vector". Note that this only makes sense because
          # the entire model is fine-tuned.
          tokens += [sep_token]
          label_ids += [pad_token_label_id]  # [label_map["X"]]
          token_type_ids += [0]
          if sep_token_extra:
              # roberta uses an extra separator b/w pairs of sentences
              tokens += [sep_token]
              label_ids += [pad_token_label_id]
              token_type_ids += [0]
          segment_ids = [sequence_a_segment_id] * len(tokens)
  
          if cls_token_at_end:
              tokens += [cls_token]
              label_ids += [pad_token_label_id]
              token_type_ids += [0]
              segment_ids += [cls_token_segment_id]
          else:
              tokens = [cls_token] + tokens
              token_type_ids += [0]
              label_ids = [pad_token_label_id] + label_ids
              segment_ids = [cls_token_segment_id] + segment_ids
  
          if model_name:
              if model_name == "xlm-roberta-base":
                  tokenizer = XLMRobertaTokenizer.from_pretrained(
                      model_name
                  )
                  input_ids = tokenizer.convert_tokens_to_ids(tokens)
              elif model_name.startswith("bert"):
                  tokenizer = BertTokenizer.from_pretrained(model_name)
                  input_ids = tokenizer.convert_tokens_to_ids(tokens)
              elif model_name == "roberta":
                  tokenizer = RobertaTokenizer.from_pretrained(
                      model_name
                  )
                  input_ids = tokenizer.convert_tokens_to_ids(tokens)
          else:
              input_ids = tokenizer.convert_tokens_to_ids(tokens)
  
          # The mask has 1 for real tokens and 0 for padding tokens. Only real
          # tokens are attended to.
          input_mask = [1 if mask_padding_with_zero else 0] * len(
              input_ids
          )
  
          # Zero-pad up to the sequence length.
          padding_length = max_seq_length - len(input_ids)
          if pad_on_left:
              input_ids = ([pad_token] * padding_length) + input_ids
              input_mask = (
                  [0 if mask_padding_with_zero else 1] * padding_length
              ) + input_mask
              segment_ids = (
                  [pad_token_segment_id] * padding_length
              ) + segment_ids
              label_ids = (
                  [pad_token_label_id] * padding_length
              ) + label_ids
              token_type_ids = ([0] * padding_length) + token_type_ids
          else:
              input_ids += [pad_token] * padding_length
              input_mask += [
                  0 if mask_padding_with_zero else 1
              ] * padding_length
              segment_ids += [pad_token_segment_id] * padding_length
              label_ids += [pad_token_label_id] * padding_length
              token_type_ids += [0] * padding_length
  
          # print("len(input_ids):", len(input_ids))
          # print("len(label_ids):", len(label_ids))
          # print("len(token_type_ids):", len(token_type_ids))
          assert len(input_ids) == max_seq_length
          assert len(input_mask) == max_seq_length
          assert len(segment_ids) == max_seq_length
          assert len(label_ids) == max_seq_length
          assert len(token_type_ids) == max_seq_length
  
          if ex_index < 0:
              logger.info("*** Example ***")
              logger.info("guid: %s", example.guid)
              logger.info(
                  "tokens: %s", " ".join([str(x) for x in tokens])
              )
              logger.info(
                  "input_ids: %s", " ".join([str(x) for x in input_ids])
              )
              logger.info(
                  "input_mask: %s",
                  " ".join([str(x) for x in input_mask]),
              )
              logger.info(
                  "segment_ids: %s",
                  " ".join([str(x) for x in segment_ids]),
              )
              logger.info(
                  "label_ids: %s", " ".join([str(x) for x in label_ids])
              )
              logger.info(
                  "token_type_ids: %s",
                  " ".join([str(x) for x in token_type_ids]),
              )
  
          features.append(
              InputFeatures(
                  input_ids=input_ids,
                  input_mask=input_mask,
                  segment_ids=segment_ids,
                  label_ids=label_ids,
                  token_type_ids=token_type_ids,
              )
          )
      return features
  
imports: from __future__ import absolute_import, division, print_function
  import json
  import logging
  import os
  import sys
  from copy import deepcopy
  from io import open
  from transformers import XLMRobertaTokenizer, BertTokenizer, RobertaTokenizer, AutoTokenizer
  from .utils_ee import read_by_lines, write_by_lines, load_dict, label_data, process_remained_pred_trigger, clear_wrong_tokens
  from torch.nn.utils.rnn import pad_sequence
  from tqdm import tqdm
label_desc: Use image node_id 1 for calling a global function with example usage: convert_examples_to_features(examples, label_list, max_seq_length, tokenizer, cls_token_at_end, cls_token, cls_token_segment_id, sep_token, sep_token_extra, pad_on_left, pad_token, pad_token_segment_id, pad_token_label_id, sequence_a_segment_id, mask_padding_with_zero, model_name) and returns: features