class_docstr: Tokenizes a tensor of UTF-8 string tokens into phrases.
docstr: Initializes the PhraseTokenizer.
  
  Args:
    vocab: (optional) The list of tokens in the vocabulary.
    token_out_type: (optional) The type of the token to return. This can be
      `tf.int64` or `tf.int32` IDs, or `tf.string` subwords.
    unknown_token: (optional) The string value to substitute for an unknown
      token. It must be included in `vocab`.
    support_detokenization: (optional) Whether to make the tokenizer support
      doing detokenization. Setting it to true expands the size of the model
      flatbuffer.
    prob: Probability of emitting a phrase when there is a match.
    split_end_punctuation: Split the end punctuation.
    model_buffer: (optional) Bytes object (or a uint8 tf.Tenosr) that contains
      the phrase model in flatbuffer format (see phrase_tokenizer_model.fbs).
      If not `None`, all other arguments (except `token_output_type`) are
      ignored.
docstr_tok: ["Initializes","the","PhraseTokenizer",".","Args",":","vocab",":","(","optional",")","The","list","of","tokens","in","the","vocabulary",".","token_out_type",":","(","optional",")","The","type","of","the","token","to","return",".","This","can","be","`","tf.int64","`","or","`","tf.int32","`","IDs",",","or","`","tf.string","`","subwords",".","unknown_token",":","(","optional",")","The","string","value","to","substitute","for","an","unknown","token",".","It","must","be","included","in","`","vocab","`",".","support_detokenization",":","(","optional",")","Whether","to","make","the","tokenizer","support","doing","detokenization",".","Setting","it","to","true","expands","the","size","of","the","model","flatbuffer",".","prob",":","Probability","of","emitting","a","phrase","when","there","is","a","match",".","split_end_punctuation",":","Split","the","end","punctuation",".","model_buffer",":","(","optional",")","Bytes","object","(","or","a","uint8","tf.Tenosr",")","that","contains","the","phrase","model","in","flatbuffer","format","(","see","phrase_tokenizer_model.fbs",")",".","If","not","`","None","`",",","all","other","arguments","(","except","`","token_output_type","`",")","are","ignored","."]
code: def __init__(
      self,
      vocab=None,
      token_out_type=dtypes.int32,
      unknown_token="<UNK>",
      support_detokenization=True,
      prob=0,
      split_end_punctuation=False,
      model_buffer=None,
  ):
      """Initializes the PhraseTokenizer.
  
      Args:
        vocab: (optional) The list of tokens in the vocabulary.
        token_out_type: (optional) The type of the token to return. This can be
          `tf.int64` or `tf.int32` IDs, or `tf.string` subwords.
        unknown_token: (optional) The string value to substitute for an unknown
          token. It must be included in `vocab`.
        support_detokenization: (optional) Whether to make the tokenizer support
          doing detokenization. Setting it to true expands the size of the model
          flatbuffer.
        prob: Probability of emitting a phrase when there is a match.
        split_end_punctuation: Split the end punctuation.
        model_buffer: (optional) Bytes object (or a uint8 tf.Tenosr) that contains
          the phrase model in flatbuffer format (see phrase_tokenizer_model.fbs).
          If not `None`, all other arguments (except `token_output_type`) are
          ignored.
      """
      super().__init__()
      _tf_text_phrase_tokenizer_op_create_counter.get_cell().increase_by(
          1
      )
  
      if model_buffer is None:
          model_buffer = (
              pywrap_phrase_tokenizer_model_builder.build_phrase_model(
                  vocab,
                  unknown_token,
                  support_detokenization,
                  prob,
                  split_end_punctuation,
              )
          )
      # Use uint8 tensor as a buffer for the model to avoid any possible changes,
      # for example truncation by '\0'.
      if isinstance(model_buffer, tensor.Tensor):
          self._model = model_buffer
      else:
          self._model = constant_op.constant(
              list(model_buffer), dtype=dtypes.uint8
          )
  
      self._token_out_type = token_out_type
  
imports: from __future__ import absolute_import
  from __future__ import division
  from __future__ import print_function
  from tensorflow.python.eager import monitoring
  from tensorflow.python.framework import constant_op
  from tensorflow.python.framework import dtypes
  from tensorflow.python.framework import ops
  from tensorflow.python.framework import tensor
  from tensorflow.python.ops import array_ops_stack
  from tensorflow.python.ops import math_ops
  from tensorflow.python.ops.ragged import ragged_tensor
  from tensorflow.python.ops.ragged.ragged_tensor import RaggedTensor
  from tensorflow_text.core.pybinds import pywrap_phrase_tokenizer_model_builder
  from tensorflow_text.python.ops.tokenization import Detokenizer
  from tensorflow_text.python.ops.tokenization import Tokenizer
  from tensorflow.python.framework import load_library
  from tensorflow.python.platform import resource_loader
label_desc: Use image node_id 1 to create a new PhraseTokenizer object from inherited base classes: Tokenizer, Detokenizer with example: obj = PhraseTokenizer(vocab, token_out_type, unknown_token, support_detokenization, prob, split_end_punctuation, model_buffer)