class_docstr: Bert Tokenizer built ontop of tensorflow_text.BertTokenizer.
class_docstr_tok: ["Bert","Tokenizer","built","ontop","of","tensorflow_text.BertTokenizer","."]
docstr: Tokenize a sequence pair.
  
  Tokenize each sequence with self.tokenize_single_sentence. Then add CLS
  token in front of the first sequence, add SEP tokens between the two
  sequences and at the end of the second sequence.
  
  Args:
    sequence_a: [batch_size, 1]
    sequence_b: [batch_size, 1]
    max_len: The length of the concatenated tokenized sentences.
  
  Returns:
    word_ids: Tokenized sequences [batch_size, max_len].
    input_mask: Mask padded tokens [batch_size, max_len].
    segment_ids: Distinguish multiple sequences [batch_size, max_len].
docstr_tok: ["Tokenize","a","sequence","pair",".","Tokenize","each","sequence","with","self.tokenize_single_sentence",".","Then","add","CLS","token","in","front","of","the","first","sequence",",","add","SEP","tokens","between","the","two","sequences","and","at","the","end","of","the","second","sequence",".","Args",":","sequence_a",":","[","batch_size",",","1","]","sequence_b",":","[","batch_size",",","1","]","max_len",":","The","length","of","the","concatenated","tokenized","sentences",".","Returns",":","word_ids",":","Tokenized","sequences","[","batch_size",",","max_len","]",".","input_mask",":","Mask","padded","tokens","[","batch_size",",","max_len","]",".","segment_ids",":","Distinguish","multiple","sequences","[","batch_size",",","max_len","]","."]
code: def tokenize_sentence_pair(
      self, sequence_a: tf.Tensor, sequence_b: tf.Tensor, max_len: int
  ):
      """Tokenize a sequence pair.
  
      Tokenize each sequence with self.tokenize_single_sentence. Then add CLS
      token in front of the first sequence, add SEP tokens between the two
      sequences and at the end of the second sequence.
  
      Args:
        sequence_a: [batch_size, 1]
        sequence_b: [batch_size, 1]
        max_len: The length of the concatenated tokenized sentences.
  
      Returns:
        word_ids: Tokenized sequences [batch_size, max_len].
        input_mask: Mask padded tokens [batch_size, max_len].
        segment_ids: Distinguish multiple sequences [batch_size, max_len].
      """
      # TODO(dzats): the issue here is nuanced. Depending on the dataset, one
      # might want to keep the entire first sentence, or the second. Consider
      # alternate truncate stratagies.
      sentence_len = max_len // 2
      word_id_a = self.tokenize_single_sentence_unpad(
          sequence_a,
          sentence_len,
          True,
          True,
      )
  
      word_id_b = self.tokenize_single_sentence_unpad(
          sequence_b,
          sentence_len,
          False,
          True,
      )
  
      word_ids = tf.concat([word_id_a, word_id_b], 1)
      word_ids = word_ids.to_tensor(
          shape=[None, max_len],
          default_value=tf.constant(self._pad_id, dtype=tf.int64),
      )
  
      input_mask = tf.cast(
          tf.not_equal(word_ids, self._pad_id), tf.int64
      )
      # Fill a ragged tensor of zero with word_id_a's shape
      segment_ids = tf.cast(word_id_a < 0, tf.int64)
      segment_ids = segment_ids.to_tensor(
          shape=[None, max_len],
          default_value=tf.constant(1, dtype=tf.int64),
      )
      return word_ids, input_mask, segment_ids
  
file_path: tfx/tfx/examples/bert/utils/bert_tokenizer_utils.py
imports: import os
  import tensorflow
  import tensorflow_hub
  import tensorflow_text
  from tensorflow.python.eager.context import eager_mode
label_desc: Use image node_id 5 for calling the BertPreprocessor obj's underlying member method code with example usage: obj.tokenize_sentence_pair(sequence_a, sequence_b, max_len) and returns: word_ids,  input_mask,  segment_ids