class_docstr: Tokenizes a tensor of UTF-8 string into words according to labels.
class_docstr_tok: ["Tokenizes","a","tensor","of","UTF-8","string","into","words","according","to","labels","."]
docstr: Tokenizes a tensor of UTF-8 strings according to labels.
  
  ### Example:
  
  >>> strings = ["HelloMonday", "DearFriday"]
  >>> labels = [[0, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1],
  ...           [0, 1, 1, 1, 0, 1, 1, 1, 1, 1, 0]]
  >>> tokenizer = SplitMergeTokenizer()
  >>> tokenizer.tokenize(strings, labels)
  <tf.RaggedTensor [[b'Hello', b'Monday'], [b'Dear', b'Friday']]>
  
  Args:
    input: An N-dimensional `Tensor` or `RaggedTensor` of UTF-8 strings.
    labels: An (N+1)-dimensional `Tensor` or `RaggedTensor` of `int32`, with
      `labels[i1...iN, j]` being the split(0)/merge(1) label of the j-th
      character for `input[i1...iN]`.  Here split means create a new word with
      this character and merge means adding this character to the previous
      word.
    force_split_at_break_character: bool indicates whether to force start a
      new word after seeing a ICU defined whitespace character.  When seeing
      one or more ICU defined whitespace character:
      * if `force_split_at_break_character` is set true, then create a new
        word at the first non-space character, regardless of the label of that
        character, for instance:
  
        ```python
        input="New York"
        labels=[0, 1, 1, 0, 1, 1, 1, 1]
        output tokens=["New", "York"]
        ```
  
        ```python
        input="New York"
        labels=[0, 1, 1, 1, 1, 1, 1, 1]
        output tokens=["New", "York"]
        ```
  
        ```python
        input="New York",
        labels=[0, 1, 1, 1, 0, 1, 1, 1]
        output tokens=["New", "York"]
        ```
  
      * otherwise, whether to create a new word or not for the first non-space
        character depends on the label of that character, for instance:
  
        ```python
        input="New York",
        labels=[0, 1, 1, 0, 1, 1, 1, 1]
        output tokens=["NewYork"]
        ```
  
        ```python
        input="New York",
        labels=[0, 1, 1, 1, 1, 1, 1, 1]
        output tokens=["NewYork"]
        ```
  
        ```python
        input="New York",
        labels=[0, 1, 1, 1, 0, 1, 1, 1]
        output tokens=["New", "York"]
        ```
  
  Returns:
    A `RaggedTensor` of strings where `tokens[i1...iN, j]` is the string
    content of the `j-th` token in `input[i1...iN]`
docstr_tok: ["Tokenizes","a","tensor","of","UTF-8","strings","according","to","labels",".","#","#","#","Example",":",">",">",">","strings","=","[","``","HelloMonday","''",",","``","DearFriday","''","]",">",">",">","labels","=","[","[","0",",","1",",","1",",","1",",","1",",","0",",","1",",","1",",","1",",","1",",","1","]",",","...","[","0",",","1",",","1",",","1",",","0",",","1",",","1",",","1",",","1",",","1",",","0","]","]",">",">",">","tokenizer","=","SplitMergeTokenizer","(",")",">",">",">","tokenizer.tokenize","(","strings",",","labels",")","<","tf.RaggedTensor","[","[","b'Hello","'",",","b'Monday","'","]",",","[","b'Dear","'",",","b'Friday","'","]","]",">","Args",":","input",":","An","N-dimensional","`","Tensor","`","or","`","RaggedTensor","`","of","UTF-8","strings",".","labels",":","An","(","N+1",")","-dimensional","`","Tensor","`","or","`","RaggedTensor","`","of","`","int32","`",",","with","`","labels","[","i1","...","iN",",","j","]","`","being","the","split","(","0",")","\/merge","(","1",")","label","of","the","j-th","character","for","`","input","[","i1","...","iN","]","`",".","Here","split","means","create","a","new","word","with","this","character","and","merge","means","adding","this","character","to","the","previous","word",".","force_split_at_break_character",":","bool","indicates","whether","to","force","start","a","new","word","after","seeing","a","ICU","defined","whitespace","character",".","When","seeing","one","or","more","ICU","defined","whitespace","character",":","*","if","`","force_split_at_break_character","`","is","set","true",",","then","create","a","new","word","at","the","first","non-space","character",",","regardless","of","the","label","of","that","character",",","for","instance",":","``","`","python","input=","''","New","York","''","labels=","[","0",",","1",",","1",",","0",",","1",",","1",",","1",",","1","]","output","tokens=","[","``","New","''",",","``","York","''","]","``","`","``","`","python","input=","''","New","York","''","labels=","[","0",",","1",",","1",",","1",",","1",",","1",",","1",",","1","]","output","tokens=","[","``","New","''",",","``","York","''","]","``","`","``","`","python","input=","''","New","York","''",",","labels=","[","0",",","1",",","1",",","1",",","0",",","1",",","1",",","1","]","output","tokens=","[","``","New","''",",","``","York","''","]","``","`","*","otherwise",",","whether","to","create","a","new","word","or","not","for","the","first","non-space","character","depends","on","the","label","of","that","character",",","for","instance",":","``","`","python","input=","''","New","York","''",",","labels=","[","0",",","1",",","1",",","0",",","1",",","1",",","1",",","1","]","output","tokens=","[","``","NewYork","''","]","``","`","``","`","python","input=","''","New","York","''",",","labels=","[","0",",","1",",","1",",","1",",","1",",","1",",","1",",","1","]","output","tokens=","[","``","NewYork","''","]","``","`","``","`","python","input=","''","New","York","''",",","labels=","[","0",",","1",",","1",",","1",",","0",",","1",",","1",",","1","]","output","tokens=","[","``","New","''",",","``","York","''","]","``","`","Returns",":","A","`","RaggedTensor","`","of","strings","where","`","tokens","[","i1","...","iN",",","j","]","`","is","the","string","content","of","the","`","j-th","`","token","in","`","input","[","i1","...","iN","]","`"]
code: def tokenize(
      self,
      input,  # pylint: disable=redefined-builtin
      labels,
      force_split_at_break_character=True,
  ):
      """Tokenizes a tensor of UTF-8 strings according to labels.
  
      ### Example:
  
      >>> strings = ["HelloMonday", "DearFriday"]
      >>> labels = [[0, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1],
      ...           [0, 1, 1, 1, 0, 1, 1, 1, 1, 1, 0]]
      >>> tokenizer = SplitMergeTokenizer()
      >>> tokenizer.tokenize(strings, labels)
      <tf.RaggedTensor [[b'Hello', b'Monday'], [b'Dear', b'Friday']]>
  
      Args:
        input: An N-dimensional `Tensor` or `RaggedTensor` of UTF-8 strings.
        labels: An (N+1)-dimensional `Tensor` or `RaggedTensor` of `int32`, with
          `labels[i1...iN, j]` being the split(0)/merge(1) label of the j-th
          character for `input[i1...iN]`.  Here split means create a new word with
          this character and merge means adding this character to the previous
          word.
        force_split_at_break_character: bool indicates whether to force start a
          new word after seeing a ICU defined whitespace character.  When seeing
          one or more ICU defined whitespace character:
          * if `force_split_at_break_character` is set true, then create a new
            word at the first non-space character, regardless of the label of that
            character, for instance:
  
            ```python
            input="New York"
            labels=[0, 1, 1, 0, 1, 1, 1, 1]
            output tokens=["New", "York"]
            ```
  
            ```python
            input="New York"
            labels=[0, 1, 1, 1, 1, 1, 1, 1]
            output tokens=["New", "York"]
            ```
  
            ```python
            input="New York",
            labels=[0, 1, 1, 1, 0, 1, 1, 1]
            output tokens=["New", "York"]
            ```
  
          * otherwise, whether to create a new word or not for the first non-space
            character depends on the label of that character, for instance:
  
            ```python
            input="New York",
            labels=[0, 1, 1, 0, 1, 1, 1, 1]
            output tokens=["NewYork"]
            ```
  
            ```python
            input="New York",
            labels=[0, 1, 1, 1, 1, 1, 1, 1]
            output tokens=["NewYork"]
            ```
  
            ```python
            input="New York",
            labels=[0, 1, 1, 1, 0, 1, 1, 1]
            output tokens=["New", "York"]
            ```
  
      Returns:
        A `RaggedTensor` of strings where `tokens[i1...iN, j]` is the string
        content of the `j-th` token in `input[i1...iN]`
      """
      subword, _, _ = self.tokenize_with_offsets(
          input, labels, force_split_at_break_character
      )
      return subword
  
file_path: text/tensorflow_text/python/ops/split_merge_tokenizer.py
imports: from __future__ import absolute_import
  from __future__ import division
  from __future__ import print_function
  from tensorflow.python.eager import monitoring
  from tensorflow.python.framework import dtypes
  from tensorflow.python.framework import ops
  from tensorflow.python.ops import array_ops
  from tensorflow.python.ops import array_ops_stack
  from tensorflow.python.ops import math_ops
  from tensorflow.python.ops.ragged import ragged_tensor
  from tensorflow.python.ops.ragged.ragged_tensor import RaggedTensor
  from tensorflow_text.python.ops.tokenization import TokenizerWithOffsets
  from tensorflow.python.framework import load_library
  from tensorflow.python.platform import resource_loader
label_desc: Use image node_id 2 for calling the SplitMergeTokenizer obj's underlying member method code with example usage: obj.tokenize(input, labels, force_split_at_break_character) and returns: subword