docstr: Tokenize and preprocesses input word lists, involving the following steps
      0. WordPiece tokenization.
      1. Convert string tokens to token ids.
      2. Convert input labels to label ids, if labels and label_map are
          provided.
      3. If a word is tokenized into multiple pieces of tokens by the
          WordPiece tokenizer, label the extra tokens with
          trailing_piece_tag.
      4. Pad or truncate input text according to max_seq_length
      5. Create input_mask for masking out padded tokens.
  
  Args:
      text (list): List of lists. Each sublist is a list of words in an
          input sentence.
      max_len (int, optional): Maximum length of the list of
          tokens. Lists longer than this are truncated and shorter
          ones are padded with "O"s. Default value is BERT_MAX_LEN=512.
      labels (list, optional): List of word label lists. Each sublist
          contains labels corresponding to the input word list. The lengths
          of the label list and word list must be the same. Default
          value is None.
      label_map (dict, optional): Dictionary for mapping original token
          labels (which may be string type) to integers. Default value
          is None.
      trailing_piece_tag (str, optional): Tag used to label trailing
          word pieces. For example, "criticize" is broken into "critic"
          and "##ize", "critic" preserves its original label and "##ize"
          is labeled as trailing_piece_tag. Default value is "X".
  
  Returns:
      tuple: A tuple containing the following four lists.
          1. input_ids_all: List of lists. Each sublist contains
              numerical values, i.e. token ids, corresponding to the
              tokens in the input text data.
          2. input_mask_all: List of lists. Each sublist
              contains the attention mask of the input token id list,
              1 for input tokens and 0 for padded tokens, so that
              padded tokens are not attended to.
          3. trailing_token_mask: List of lists. Each sublist is
              a boolean list, True for the first word piece of each
              original word, False for the trailing word pieces,
              e.g. "##ize". This mask is useful for removing the
              predictions on trailing word pieces, so that each
              original word in the input text has a unique predicted
              label.
          4. label_ids_all: List of lists of numerical labels,
              each sublist contains token labels of a input
              sentence/paragraph, if labels is provided. If the `labels`
              argument is not provided, the value of this is None.
docstr_tok: ["Tokenize","and","preprocesses","input","word","lists",",","involving","the","following","steps","0",".","WordPiece","tokenization",".","1",".","Convert","string","tokens","to","token","ids",".","2",".","Convert","input","labels","to","label","ids",",","if","labels","and","label_map","are","provided",".","3",".","If","a","word","is","tokenized","into","multiple","pieces","of","tokens","by","the","WordPiece","tokenizer",",","label","the","extra","tokens","with","trailing_piece_tag",".","4",".","Pad","or","truncate","input","text","according","to","max_seq_length","5",".","Create","input_mask","for","masking","out","padded","tokens",".","Args",":","text","(","list",")",":","List","of","lists",".","Each","sublist","is","a","list","of","words","in","an","input","sentence",".","max_len","(","int",",","optional",")",":","Maximum","length","of","the","list","of","tokens",".","Lists","longer","than","this","are","truncated","and","shorter","ones","are","padded","with","``","O","''","s.","Default","value","is","BERT_MAX_LEN=512",".","labels","(","list",",","optional",")",":","List","of","word","label","lists",".","Each","sublist","contains","labels","corresponding","to","the","input","word","list",".","The","lengths","of","the","label","list","and","word","list","must","be","the","same",".","Default","value","is","None",".","label_map","(","dict",",","optional",")",":","Dictionary","for","mapping","original","token","labels","(","which","may","be","string","type",")","to","integers",".","Default","value","is","None",".","trailing_piece_tag","(","str",",","optional",")",":","Tag","used","to","label","trailing","word","pieces",".","For","example",",","``","criticize","''","is","broken","into","``","critic","''","and","``","#","#","ize","''",",","``","critic","''","preserves","its","original","label","and","``","#","#","ize","''","is","labeled","as","trailing_piece_tag",".","Default","value","is","``","X","''",".","Returns",":","tuple",":","A","tuple","containing","the","following","four","lists",".","1.","input_ids_all",":","List","of","lists",".","Each","sublist","contains","numerical","values",",","i.e",".","token","ids",",","corresponding","to","the","tokens","in","the","input","text","data",".","2.","input_mask_all",":","List","of","lists",".","Each","sublist","contains","the","attention","mask","of","the","input","token","id","list",",","1","for","input","tokens","and","0","for","padded","tokens",",","so","that","padded","tokens","are","not","attended","to",".","3.","trailing_token_mask",":","List","of","lists",".","Each","sublist","is","a","boolean","list",",","True","for","the","first","word","piece","of","each","original","word",",","False","for","the","trailing","word","pieces",",","e.g",".","``","#","#","ize","''",".","This","mask","is","useful","for","removing","the","predictions","on","trailing","word","pieces",",","so","that","each","original","word","in","the","input","text","has","a","unique","predicted","label",".","4.","label_ids_all",":","List","of","lists","of","numerical","labels",",","each","sublist","contains","token","labels","of","a","input","sentence\/paragraph",",","if","labels","is","provided",".","If","the","`","labels","`","argument","is","not","provided",",","the","value","of","this","is","None","."]
code: def tokenize_ner(
      self,
      text,
      max_len=BERT_MAX_LEN,
      labels=None,
      label_map=None,
      trailing_piece_tag="X",
  ):
      """
      Tokenize and preprocesses input word lists, involving the following steps
          0. WordPiece tokenization.
          1. Convert string tokens to token ids.
          2. Convert input labels to label ids, if labels and label_map are
              provided.
          3. If a word is tokenized into multiple pieces of tokens by the
              WordPiece tokenizer, label the extra tokens with
              trailing_piece_tag.
          4. Pad or truncate input text according to max_seq_length
          5. Create input_mask for masking out padded tokens.
  
      Args:
          text (list): List of lists. Each sublist is a list of words in an
              input sentence.
          max_len (int, optional): Maximum length of the list of
              tokens. Lists longer than this are truncated and shorter
              ones are padded with "O"s. Default value is BERT_MAX_LEN=512.
          labels (list, optional): List of word label lists. Each sublist
              contains labels corresponding to the input word list. The lengths
              of the label list and word list must be the same. Default
              value is None.
          label_map (dict, optional): Dictionary for mapping original token
              labels (which may be string type) to integers. Default value
              is None.
          trailing_piece_tag (str, optional): Tag used to label trailing
              word pieces. For example, "criticize" is broken into "critic"
              and "##ize", "critic" preserves its original label and "##ize"
              is labeled as trailing_piece_tag. Default value is "X".
  
      Returns:
          tuple: A tuple containing the following four lists.
              1. input_ids_all: List of lists. Each sublist contains
                  numerical values, i.e. token ids, corresponding to the
                  tokens in the input text data.
              2. input_mask_all: List of lists. Each sublist
                  contains the attention mask of the input token id list,
                  1 for input tokens and 0 for padded tokens, so that
                  padded tokens are not attended to.
              3. trailing_token_mask: List of lists. Each sublist is
                  a boolean list, True for the first word piece of each
                  original word, False for the trailing word pieces,
                  e.g. "##ize". This mask is useful for removing the
                  predictions on trailing word pieces, so that each
                  original word in the input text has a unique predicted
                  label.
              4. label_ids_all: List of lists of numerical labels,
                  each sublist contains token labels of a input
                  sentence/paragraph, if labels is provided. If the `labels`
                  argument is not provided, the value of this is None.
      """
  
      def _is_iterable_but_not_string(obj):
          return isinstance(obj, Iterable) and not isinstance(obj, str)
  
      if max_len > BERT_MAX_LEN:
          warnings.warn(
              "setting max_len to max allowed tokens: {}".format(
                  BERT_MAX_LEN
              )
          )
          max_len = BERT_MAX_LEN
  
      if not _is_iterable_but_not_string(text):
          # The input text must be an non-string Iterable
          raise ValueError(
              "Input text must be an iterable and not a string."
          )
      else:
          # If the input text is a single list of words, convert it to
          # list of lists for later iteration
          if not _is_iterable_but_not_string(text[0]):
              text = [text]
      if labels is not None:
          if not _is_iterable_but_not_string(labels):
              raise ValueError(
                  "labels must be an iterable and not a string."
              )
          else:
              if not _is_iterable_but_not_string(labels[0]):
                  labels = [labels]
  
      label_available = True
      if labels is None:
          label_available = False
          # create an artificial label list for creating trailing token mask
          labels = [["O"] * len(t) for t in text]
  
      input_ids_all = []
      input_mask_all = []
      label_ids_all = []
      trailing_token_mask_all = []
      for t, t_labels in zip(text, labels):
          if len(t) != len(t_labels):
              raise ValueError(
                  "The number of words is {0}, but the number of labels is {1}.".format(
                      len(t), len(t_labels)
                  )
              )
  
          new_labels = []
          new_tokens = []
          if label_available:
              for word, tag in zip(t, t_labels):
                  sub_words = self.tokenizer.tokenize(word)
                  for count, sub_word in enumerate(sub_words):
                      if count > 0:
                          tag = trailing_piece_tag
                      new_labels.append(tag)
                      new_tokens.append(sub_word)
          else:
              for word in t:
                  sub_words = self.tokenizer.tokenize(word)
                  for count, sub_word in enumerate(sub_words):
                      if count > 0:
                          tag = trailing_piece_tag
                      else:
                          tag = "O"
                      new_labels.append(tag)
                      new_tokens.append(sub_word)
  
          if len(new_tokens) > max_len:
              new_tokens = new_tokens[:max_len]
              new_labels = new_labels[:max_len]
          input_ids = self.tokenizer.convert_tokens_to_ids(new_tokens)
  
          # The mask has 1 for real tokens and 0 for padding tokens.
          # Only real tokens are attended to.
          input_mask = [1.0] * len(input_ids)
  
          # Zero-pad up to the max sequence length.
          padding = [0.0] * (max_len - len(input_ids))
          label_padding = ["O"] * (max_len - len(input_ids))
  
          input_ids += padding
          input_mask += padding
          new_labels += label_padding
  
          trailing_token_mask_all.append(
              [
                  True if label != trailing_piece_tag else False
                  for label in new_labels
              ]
          )
  
          if label_map:
              label_ids = [label_map[label] for label in new_labels]
          else:
              label_ids = new_labels
  
          input_ids_all.append(input_ids)
          input_mask_all.append(input_mask)
          label_ids_all.append(label_ids)
  
      if label_available:
          return (
              input_ids_all,
              input_mask_all,
              trailing_token_mask_all,
              label_ids_all,
          )
      else:
          return (
              input_ids_all,
              input_mask_all,
              trailing_token_mask_all,
              None,
          )
  
imports: import logging
  import warnings
  from collections.abc import Iterable
  from enum import Enum
  import torch
  from pytorch_pretrained_bert.tokenization import BertTokenizer
  from tqdm import tqdm
  import numpy
  import torch.nn
  from torch.utils.data import DataLoader, RandomSampler, SequentialSampler, TensorDataset
  from pytorch_pretrained_bert.modeling import BertForSequenceClassification
  from pytorch_pretrained_bert.optimization import BertAdam
  from interpret_text.experimental.common.utils_pytorch import get_device, move_to_device
  from interpret_text.experimental.common.constants import BertTokens
  from collections import namedtuple
  from cached_property import cached_property
label_desc: Use image node_id 6 for calling the Tokenizer obj's underlying member method code with example usage: obj.tokenize_ner(text, max_len, labels, label_map, trailing_piece_tag) and returns: unknown, input_ids_all,  input_mask_all,  trailing_token_mask_all,  label_ids_all, input_ids_all,  input_mask_all,  trailing_token_mask_all,  None