class_docstr: Tokenizes a tensor of UTF-8 string tokens into phrases.
docstr: Tokenizes a tensor of UTF-8 string tokens further into phrase tokens.
  
  ### Example, single string tokenization:
  >>> vocab = ["I", "have", "a", "dream", "a dream", "I have a", "<UNK>"]
  >>> tokenizer = PhraseTokenizer(vocab, token_out_type=tf.string)
  >>> tokens = [["I have a dream"]]
  >>> phrases = tokenizer.tokenize(tokens)
  >>> phrases
  <tf.RaggedTensor [[[b'I have a', b'dream']]]>
  
  Args:
    input: An N-dimensional `Tensor` or `RaggedTensor` of UTF-8 strings.
  
  Returns:
    tokens: is a `RaggedTensor`, where `tokens[i, j]` is the j-th token
        (i.e., phrase) for `input[i]` (i.e., the i-th input word). This
        token is either the actual token string content, or the corresponding
        integer id, i.e., the index of that token string in the vocabulary.
        This choice is controlled by the `token_out_type` parameter passed to
        the initializer method.
docstr_tok: ["Tokenizes","a","tensor","of","UTF-8","string","tokens","further","into","phrase","tokens",".","#","#","#","Example",",","single","string","tokenization",":",">",">",">","vocab","=","[","``","I","''",",","``","have","''",",","``","a","''",",","``","dream","''",",","``","a","dream","''",",","``","I","have","a","''",",","``","<","UNK",">","''","]",">",">",">","tokenizer","=","PhraseTokenizer","(","vocab",",","token_out_type=tf.string",")",">",">",">","tokens","=","[","[","``","I","have","a","dream","''","]","]",">",">",">","phrases","=","tokenizer.tokenize","(","tokens",")",">",">",">","phrases","<","tf.RaggedTensor","[","[","[","b","'","I","have","a","'",",","b'dream","'","]","]","]",">","Args",":","input",":","An","N-dimensional","`","Tensor","`","or","`","RaggedTensor","`","of","UTF-8","strings",".","Returns",":","tokens",":","is","a","`","RaggedTensor","`",",","where","`","tokens","[","i",",","j","]","`","is","the","j-th","token","(","i.e.",",","phrase",")","for","`","input","[","i","]","`","(","i.e.",",","the","i-th","input","word",")",".","This","token","is","either","the","actual","token","string","content",",","or","the","corresponding","integer","id",",","i.e.",",","the","index","of","that","token","string","in","the","vocabulary",".","This","choice","is","controlled","by","the","`","token_out_type","`","parameter","passed","to","the","initializer","method","."]
code: def tokenize(self, input):  # pylint: disable=redefined-builtin
      """Tokenizes a tensor of UTF-8 string tokens further into phrase tokens.
  
      ### Example, single string tokenization:
      >>> vocab = ["I", "have", "a", "dream", "a dream", "I have a", "<UNK>"]
      >>> tokenizer = PhraseTokenizer(vocab, token_out_type=tf.string)
      >>> tokens = [["I have a dream"]]
      >>> phrases = tokenizer.tokenize(tokens)
      >>> phrases
      <tf.RaggedTensor [[[b'I have a', b'dream']]]>
  
      Args:
        input: An N-dimensional `Tensor` or `RaggedTensor` of UTF-8 strings.
  
      Returns:
        tokens: is a `RaggedTensor`, where `tokens[i, j]` is the j-th token
            (i.e., phrase) for `input[i]` (i.e., the i-th input word). This
            token is either the actual token string content, or the corresponding
            integer id, i.e., the index of that token string in the vocabulary.
            This choice is controlled by the `token_out_type` parameter passed to
            the initializer method.
      """
      name = None
      with ops.name_scope(name, "PhraseTokenize", [input, self._model]):
          # Check that the types are expected and the ragged rank is appropriate.
          tokens = ragged_tensor.convert_to_tensor_or_ragged_tensor(
              input
          )
          rank = tokens.shape.ndims
          if rank is None:
              raise ValueError("input must have a known rank.")
  
          if rank == 0:
              phrases = self.tokenize(array_ops_stack.stack([tokens]))
              return phrases.values
  
          elif rank > 1:
              if not ragged_tensor.is_ragged(tokens):
                  tokens = ragged_tensor.RaggedTensor.from_tensor(
                      tokens, ragged_rank=rank - 1
                  )
              phrases = self.tokenize(tokens.flat_values)
              phrases = phrases.with_row_splits_dtype(
                  tokens.row_splits.dtype
              )
              return tokens.with_flat_values(phrases)
  
          # Tokenize the tokens into phrases.
          (
              subwords,
              phrase_ids,
              row_splits,
          ) = gen_phrase_tokenizer.phrase_tokenize(
              input_values=tokens, phrase_model=self._model
          )
  
          if self._token_out_type == dtypes.int64:
              values = math_ops.cast(phrase_ids, dtypes.int64)
          elif self._token_out_type == dtypes.int32:
              values = math_ops.cast(phrase_ids, dtypes.int32)
          else:
              values = subwords
  
          phrases = RaggedTensor.from_row_splits(
              values, row_splits, validate=False
          )
  
          return phrases
  
imports: from __future__ import absolute_import
  from __future__ import division
  from __future__ import print_function
  from tensorflow.python.eager import monitoring
  from tensorflow.python.framework import constant_op
  from tensorflow.python.framework import dtypes
  from tensorflow.python.framework import ops
  from tensorflow.python.framework import tensor
  from tensorflow.python.ops import array_ops_stack
  from tensorflow.python.ops import math_ops
  from tensorflow.python.ops.ragged import ragged_tensor
  from tensorflow.python.ops.ragged.ragged_tensor import RaggedTensor
  from tensorflow_text.core.pybinds import pywrap_phrase_tokenizer_model_builder
  from tensorflow_text.python.ops.tokenization import Detokenizer
  from tensorflow_text.python.ops.tokenization import Tokenizer
  from tensorflow.python.framework import load_library
  from tensorflow.python.platform import resource_loader
label_desc: Use image node_id 2 for calling the PhraseTokenizer obj's underlying member method code with example usage: obj.tokenize(input) and returns: phrases, phrases, tokens