class_docstr: Bert Tokenizer built ontop of tensorflow_text.BertTokenizer.
class_docstr_tok: ["Bert","Tokenizer","built","ontop","of","tensorflow_text.BertTokenizer","."]
docstr: Tokenize a sentence with the BERT model vocab file and without padding.
  
  Add special tokens according to config.
  
  Args:
    sequence: Tensor of shape [batch_size, 1].
    max_len: The number of tokens after padding and truncating.
    add_cls: Whether to add CLS token at the front of each sequence.
    add_sep: Whether to add SEP token at the end of each sequence.
  
  Returns:
    word_ids: Ragged tokenized sequences [batch_size, None].
docstr_tok: ["Tokenize","a","sentence","with","the","BERT","model","vocab","file","and","without","padding",".","Add","special","tokens","according","to","config",".","Args",":","sequence",":","Tensor","of","shape","[","batch_size",",","1","]",".","max_len",":","The","number","of","tokens","after","padding","and","truncating",".","add_cls",":","Whether","to","add","CLS","token","at","the","front","of","each","sequence",".","add_sep",":","Whether","to","add","SEP","token","at","the","end","of","each","sequence",".","Returns",":","word_ids",":","Ragged","tokenized","sequences","[","batch_size",",","None","]","."]
code: def tokenize_single_sentence_unpad(
      self,
      sequence: tf.Tensor,
      max_len: int = 128,
      add_cls: bool = True,
      add_sep: bool = True,
  ):
      """Tokenize a sentence with the BERT model vocab file and without padding.
  
      Add special tokens according to config.
  
      Args:
        sequence: Tensor of shape [batch_size, 1].
        max_len: The number of tokens after padding and truncating.
        add_cls: Whether to add CLS token at the front of each sequence.
        add_sep: Whether to add SEP token at the end of each sequence.
  
      Returns:
        word_ids: Ragged tokenized sequences [batch_size, None].
      """
      vocab_file_path = (
          self._model.resolved_object.vocab_file.asset_path
      )
      tokenizer = text.BertTokenizer(
          vocab_file_path,
          lower_case=self._do_lower_case,
          token_out_type=tf.int64,
      )
      word_ids = tokenizer.tokenize(sequence)
      # Tokenizer default puts tokens into array of size 1. merge_dims flattens it
      word_ids = word_ids.merge_dims(-2, -1)
      if add_cls:
          cls_token = tf.fill(
              [tf.shape(sequence)[0], 1],
              tf.constant(self._cls_id, dtype=tf.int64),
          )
  
          word_ids = tf.concat([cls_token, word_ids], 1)
  
      if add_sep:
          sep_token = tf.fill(
              [tf.shape(sequence)[0], 1],
              tf.constant(self._sep_id, dtype=tf.int64),
          )
  
          word_ids = word_ids[:, : max_len - 1]
          word_ids = tf.concat([word_ids, sep_token], 1)
  
      return word_ids
  
file_path: tfx/tfx/examples/bert/utils/bert_tokenizer_utils.py
imports: import os
  import tensorflow
  import tensorflow_hub
  import tensorflow_text
  from tensorflow.python.eager.context import eager_mode
label_desc: Use image node_id 3 for calling the BertPreprocessor obj's underlying member method code with example usage: obj.tokenize_single_sentence_unpad(sequence, max_len, add_cls, add_sep) and returns: word_ids