docstr: Preprocessing of input tokens:
      - add BERT sentence markers ([CLS] and [SEP])
      - map tokens to token indices in the BERT vocabulary
      - pad and truncate sequences
      - create an input_mask
      - create token type ids, aka. segment ids
  
  Args:
      tokens (list): List of token lists to preprocess.
      max_len (int, optional): Maximum number of tokens
                      (documents will be truncated or padded).
                      Defaults to 512.
  Returns:
      tuple: A tuple containing the following four lists
          list of preprocessed token lists
          list of input id lists
          list of input mask lists
          list of token type id lists
docstr_tok: ["Preprocessing","of","input","tokens",":","-","add","BERT","sentence","markers","(","[","CLS","]","and","[","SEP","]",")","-","map","tokens","to","token","indices","in","the","BERT","vocabulary","-","pad","and","truncate","sequences","-","create","an","input_mask","-","create","token","type","ids",",","aka",".","segment","ids","Args",":","tokens","(","list",")",":","List","of","token","lists","to","preprocess",".","max_len","(","int",",","optional",")",":","Maximum","number","of","tokens","(","documents","will","be","truncated","or","padded",")",".","Defaults","to","512",".","Returns",":","tuple",":","A","tuple","containing","the","following","four","lists","list","of","preprocessed","token","lists","list","of","input","id","lists","list","of","input","mask","lists","list","of","token","type","id","lists"]
code: def preprocess_encoder_tokens(self, tokens, max_len=BERT_MAX_LEN):
      """Preprocessing of input tokens:
          - add BERT sentence markers ([CLS] and [SEP])
          - map tokens to token indices in the BERT vocabulary
          - pad and truncate sequences
          - create an input_mask
          - create token type ids, aka. segment ids
  
      Args:
          tokens (list): List of token lists to preprocess.
          max_len (int, optional): Maximum number of tokens
                          (documents will be truncated or padded).
                          Defaults to 512.
      Returns:
          tuple: A tuple containing the following four lists
              list of preprocessed token lists
              list of input id lists
              list of input mask lists
              list of token type id lists
      """
      if max_len > BERT_MAX_LEN:
          logging.info(
              "setting max_len to max allowed tokens: {}".format(
                  BERT_MAX_LEN
              )
          )
          max_len = BERT_MAX_LEN
  
      if isinstance(tokens[0][0], str):
          tokens = [x[0 : max_len - 2] + ["[SEP]"] for x in tokens]
          token_type_ids = None
      else:
          # get tokens for each sentence [[t00, t01, ...] [t10, t11,... ]]
          tokens = [
              self._truncate_seq_pair(
                  sentence[0], sentence[1], max_len - 3
              )
              for sentence in tokens
          ]
  
          # construct token_type_ids
          # [[0, 0, 0, 0, ... 0, 1, 1, 1, ... 1], [0, 0, 0, ..., 1, 1, ]
          token_type_ids = [
              [
                  [i] * len(sentence)
                  for i, sentence in enumerate(example)
              ]
              for example in tokens
          ]
          # merge sentences
          tokens = [
              [token for sentence in example for token in sentence]
              for example in tokens
          ]
          # prefix with [0] for [CLS]
          token_type_ids = [
              [0] + [i for sentence in example for i in sentence]
              for example in token_type_ids
          ]
          # pad sequence
          token_type_ids = [
              x + [0] * (max_len - len(x)) for x in token_type_ids
          ]
  
      tokens = [[BertTokens.CLS] + x for x in tokens]
      # convert tokens to indices
      input_ids = [
          self.tokenizer.convert_tokens_to_ids(x) for x in tokens
      ]
      # pad sequence
      input_ids = [x + [0] * (max_len - len(x)) for x in input_ids]
      # create input mask
      input_mask = [[min(1, x) for x in y] for y in input_ids]
      return tokens, input_ids, input_mask, token_type_ids
  
imports: import logging
  import warnings
  from collections.abc import Iterable
  from enum import Enum
  import torch
  from pytorch_pretrained_bert.tokenization import BertTokenizer
  from tqdm import tqdm
  import numpy
  import torch.nn
  from torch.utils.data import DataLoader, RandomSampler, SequentialSampler, TensorDataset
  from pytorch_pretrained_bert.modeling import BertForSequenceClassification
  from pytorch_pretrained_bert.optimization import BertAdam
  from interpret_text.experimental.common.utils_pytorch import get_device, move_to_device
  from interpret_text.experimental.common.constants import BertTokens
  from collections import namedtuple
  from cached_property import cached_property
label_desc: Use image node_id 5 for calling the Tokenizer obj's underlying member method code with example usage: obj.preprocess_encoder_tokens(tokens, max_len) and returns: tokens,  input_ids,  input_mask,  token_type_ids