class_docstr: Tokenizes a tensor of UTF-8 string into words according to logits.
docstr: Tokenizes a tensor of UTF-8 strings into tokens with [start,end) offsets.
  
  ### Example:
  
  >>> strings = ['IloveFlume!', 'and tensorflow']
  >>> logits = [
  ... [
  ...     # 'I'
  ...     [5.0, -3.2],  # I: split
  ...     # 'love'
  ...     [2.2, -1.0],  # l: split
  ...     [0.2, 12.0],  # o: merge
  ...     [0.0, 11.0],  # v: merge
  ...     [-3.0, 3.0],  # e: merge
  ...     # 'Flume'
  ...     [10.0, 0.0],  # F: split
  ...     [0.0, 11.0],  # l: merge
  ...     [0.0, 11.0],  # u: merge
  ...     [0.0, 12.0],  # m: merge
  ...     [0.0, 12.0],  # e: merge
  ...     # '!'
  ...     [5.2, -7.0],  # !: split
  ...     # padding:
  ...     [1.0, 0.0], [1.0, 1.0], [1.0, 0.0],
  ... ], [
  ...     # 'and'
  ...     [2.0, 0.7],  # a: split
  ...     [0.2, 1.5],  # n: merge
  ...     [0.5, 2.3],  # d: merge
  ...     # ' '
  ...     [1.7, 7.0],  # <space>: merge
  ...     # 'tensorflow'
  ...     [2.2, 0.1],  # t: split
  ...     [0.2, 3.1],  # e: merge
  ...     [1.1, 2.5],  # n: merge
  ...     [0.7, 0.9],  # s: merge
  ...     [0.6, 1.0],  # o: merge
  ...     [0.3, 1.0],  # r: merge
  ...     [0.2, 2.2],  # f: merge
  ...     [0.7, 3.1],  # l: merge
  ...     [0.4, 5.0],  # o: merge
  ...     [0.8, 6.0],  # w: merge
  ... ]]
  >>> tokenizer = SplitMergeFromLogitsTokenizer()
  >>> tokens, starts, ends = tokenizer.tokenize_with_offsets(strings, logits)
  >>> tokens
  <tf.RaggedTensor [[b'I', b'love', b'Flume', b'!'], [b'and', b'tensorflow']]>
  >>> starts
  <tf.RaggedTensor [[0, 1, 5, 10], [0, 4]]>
  >>> ends
  <tf.RaggedTensor [[1, 5, 10, 11], [3, 14]]>
  
  Args:
    strings: A 1D `Tensor` of UTF-8 strings.
    logits: 3D Tensor; logits[i,j,0] is the logit for the split action for
      j-th character of strings[i].  logits[i,j,1] is the logit for the merge
      action for that same character.  For each character, we pick the action
      with the greatest logit.  Split starts a new word at this character and
      merge adds this character to the previous word.  The shape of this
      tensor should be (n, m, 2) where n is the number of strings, and m is
      greater or equal with the number of characters from each strings[i].  As
      the elements of the strings tensor may have different lengths (in UTF-8
      chars), padding may be required to get a dense vector; for each row, the
      extra (padding) pairs of logits are ignored.
  
  Returns:
    A tuple `(tokens, start_offsets, end_offsets)` where:
      * `tokens` is a `RaggedTensor` of strings where `tokens[i, k]` is
        the string content of the `k-th` token in `strings[i]`
      * `start_offsets` is a `RaggedTensor` of int64s where
        `start_offsets[i, k]` is the byte offset for the start of the
        `k-th` token in `strings[i]`.
      * `end_offsets` is a `RaggedTensor` of int64s where
        `end_offsets[i, k]` is the byte offset immediately after the
        end of the `k-th` token in `strings[i]`.
  
  Raises:
    InvalidArgumentError: if one of the input Tensors has the wrong shape.
      E.g., if the tensor logits does not have enough elements for one of the
      strings.
docstr_tok: ["Tokenizes","a","tensor","of","UTF-8","strings","into","tokens","with","[","start",",","end",")","offsets",".","#","#","#","Example",":",">",">",">","strings","=","[","'IloveFlume","!","'",",","'and","tensorflow","'","]",">",">",">","logits","=","[","...","[","...","#","'","I'","...","[","5.0",",","-3.2","]",",","#","I",":","split","...","#","'love'","...","[","2.2",",","-1.0","]",",","#","l",":","split","...","[","0.2",",","12.0","]",",","#","o",":","merge","...","[","0.0",",","11.0","]",",","#","v",":","merge","...","[","-3.0",",","3.0","]",",","#","e",":","merge","...","#","'Flume'","...","[","10.0",",","0.0","]",",","#","F",":","split","...","[","0.0",",","11.0","]",",","#","l",":","merge","...","[","0.0",",","11.0","]",",","#","u",":","merge","...","[","0.0",",","12.0","]",",","#","m",":","merge","...","[","0.0",",","12.0","]",",","#","e",":","merge","...","#","'","!","'","...","[","5.2",",","-7.0","]",",","#","!",":","split","...","#","padding",":","...","[","1.0",",","0.0","]",",","[","1.0",",","1.0","]",",","[","1.0",",","0.0","]",",","...","]",",","[","...","#","'and'","...","[","2.0",",","0.7","]",",","#","a",":","split","...","[","0.2",",","1.5","]",",","#","n",":","merge","...","[","0.5",",","2.3","]",",","#","d",":","merge","...","#","'","'","...","[","1.7",",","7.0","]",",","#","<","space",">",":","merge","...","#","'tensorflow'","...","[","2.2",",","0.1","]",",","#","t",":","split","...","[","0.2",",","3.1","]",",","#","e",":","merge","...","[","1.1",",","2.5","]",",","#","n",":","merge","...","[","0.7",",","0.9","]",",","#","s",":","merge","...","[","0.6",",","1.0","]",",","#","o",":","merge","...","[","0.3",",","1.0","]",",","#","r",":","merge","...","[","0.2",",","2.2","]",",","#","f",":","merge","...","[","0.7",",","3.1","]",",","#","l",":","merge","...","[","0.4",",","5.0","]",",","#","o",":","merge","...","[","0.8",",","6.0","]",",","#","w",":","merge","...","]","]",">",">",">","tokenizer","=","SplitMergeFromLogitsTokenizer","(",")",">",">",">","tokens",",","starts",",","ends","=","tokenizer.tokenize_with_offsets","(","strings",",","logits",")",">",">",">","tokens","<","tf.RaggedTensor","[","[","b","'","I","'",",","b'love","'",",","b'Flume","'",",","b","'","!","'","]",",","[","b'and","'",",","b'tensorflow","'","]","]",">",">",">",">","starts","<","tf.RaggedTensor","[","[","0",",","1",",","5",",","10","]",",","[","0",",","4","]","]",">",">",">",">","ends","<","tf.RaggedTensor","[","[","1",",","5",",","10",",","11","]",",","[","3",",","14","]","]",">","Args",":","strings",":","A","1D","`","Tensor","`","of","UTF-8","strings",".","logits",":","3D","Tensor",";","logits","[","i",",","j,0","]","is","the","logit","for","the","split","action","for","j-th","character","of","strings","[","i","]",".","logits","[","i",",","j,1","]","is","the","logit","for","the","merge","action","for","that","same","character",".","For","each","character",",","we","pick","the","action","with","the","greatest","logit",".","Split","starts","a","new","word","at","this","character","and","merge","adds","this","character","to","the","previous","word",".","The","shape","of","this","tensor","should","be","(","n",",","m",",","2",")","where","n","is","the","number","of","strings",",","and","m","is","greater","or","equal","with","the","number","of","characters","from","each","strings","[","i","]",".","As","the","elements","of","the","strings","tensor","may","have","different","lengths","(","in","UTF-8","chars",")",",","padding","may","be","required","to","get","a","dense","vector",";","for","each","row",",","the","extra","(","padding",")","pairs","of","logits","are","ignored",".","Returns",":","A","tuple","`","(","tokens",",","start_offsets",",","end_offsets",")","`","where",":","*","`","tokens","`","is","a","`","RaggedTensor","`","of","strings","where","`","tokens","[","i",",","k","]","`","is","the","string","content","of","the","`","k-th","`","token","in","`","strings","[","i","]","`","*","`","start_offsets","`","is","a","`","RaggedTensor","`","of","int64s","where","`","start_offsets","[","i",",","k","]","`","is","the","byte","offset","for","the","start","of","the","`","k-th","`","token","in","`","strings","[","i","]","`",".","*","`","end_offsets","`","is","a","`","RaggedTensor","`","of","int64s","where","`","end_offsets","[","i",",","k","]","`","is","the","byte","offset","immediately","after","the","end","of","the","`","k-th","`","token","in","`","strings","[","i","]","`",".","Raises",":","InvalidArgumentError",":","if","one","of","the","input","Tensors","has","the","wrong","shape",".","E.g.",",","if","the","tensor","logits","does","not","have","enough","elements","for","one","of","the","strings","."]
code: def tokenize_with_offsets(self, strings, logits):
      """Tokenizes a tensor of UTF-8 strings into tokens with [start,end) offsets.
  
      ### Example:
  
      >>> strings = ['IloveFlume!', 'and tensorflow']
      >>> logits = [
      ... [
      ...     # 'I'
      ...     [5.0, -3.2],  # I: split
      ...     # 'love'
      ...     [2.2, -1.0],  # l: split
      ...     [0.2, 12.0],  # o: merge
      ...     [0.0, 11.0],  # v: merge
      ...     [-3.0, 3.0],  # e: merge
      ...     # 'Flume'
      ...     [10.0, 0.0],  # F: split
      ...     [0.0, 11.0],  # l: merge
      ...     [0.0, 11.0],  # u: merge
      ...     [0.0, 12.0],  # m: merge
      ...     [0.0, 12.0],  # e: merge
      ...     # '!'
      ...     [5.2, -7.0],  # !: split
      ...     # padding:
      ...     [1.0, 0.0], [1.0, 1.0], [1.0, 0.0],
      ... ], [
      ...     # 'and'
      ...     [2.0, 0.7],  # a: split
      ...     [0.2, 1.5],  # n: merge
      ...     [0.5, 2.3],  # d: merge
      ...     # ' '
      ...     [1.7, 7.0],  # <space>: merge
      ...     # 'tensorflow'
      ...     [2.2, 0.1],  # t: split
      ...     [0.2, 3.1],  # e: merge
      ...     [1.1, 2.5],  # n: merge
      ...     [0.7, 0.9],  # s: merge
      ...     [0.6, 1.0],  # o: merge
      ...     [0.3, 1.0],  # r: merge
      ...     [0.2, 2.2],  # f: merge
      ...     [0.7, 3.1],  # l: merge
      ...     [0.4, 5.0],  # o: merge
      ...     [0.8, 6.0],  # w: merge
      ... ]]
      >>> tokenizer = SplitMergeFromLogitsTokenizer()
      >>> tokens, starts, ends = tokenizer.tokenize_with_offsets(strings, logits)
      >>> tokens
      <tf.RaggedTensor [[b'I', b'love', b'Flume', b'!'], [b'and', b'tensorflow']]>
      >>> starts
      <tf.RaggedTensor [[0, 1, 5, 10], [0, 4]]>
      >>> ends
      <tf.RaggedTensor [[1, 5, 10, 11], [3, 14]]>
  
      Args:
        strings: A 1D `Tensor` of UTF-8 strings.
        logits: 3D Tensor; logits[i,j,0] is the logit for the split action for
          j-th character of strings[i].  logits[i,j,1] is the logit for the merge
          action for that same character.  For each character, we pick the action
          with the greatest logit.  Split starts a new word at this character and
          merge adds this character to the previous word.  The shape of this
          tensor should be (n, m, 2) where n is the number of strings, and m is
          greater or equal with the number of characters from each strings[i].  As
          the elements of the strings tensor may have different lengths (in UTF-8
          chars), padding may be required to get a dense vector; for each row, the
          extra (padding) pairs of logits are ignored.
  
      Returns:
        A tuple `(tokens, start_offsets, end_offsets)` where:
          * `tokens` is a `RaggedTensor` of strings where `tokens[i, k]` is
            the string content of the `k-th` token in `strings[i]`
          * `start_offsets` is a `RaggedTensor` of int64s where
            `start_offsets[i, k]` is the byte offset for the start of the
            `k-th` token in `strings[i]`.
          * `end_offsets` is a `RaggedTensor` of int64s where
            `end_offsets[i, k]` is the byte offset immediately after the
            end of the `k-th` token in `strings[i]`.
  
      Raises:
        InvalidArgumentError: if one of the input Tensors has the wrong shape.
          E.g., if the tensor logits does not have enough elements for one of the
          strings.
      """
      name = None
      with ops.name_scope(
          name, "SplitMergeFromLogitsTokenizer", [strings, logits]
      ):
          # Tokenize the strings into tokens.
          force_split = self._force_split_at_break_character
          (
              token_values,
              token_row_splits,
              start_values,
              end_values,
          ) = gen_split_merge_from_logits_tokenizer.tokenizer_from_logits(
              strings=strings,
              logits=logits,
              force_split_at_break_character=force_split,
          )
  
          # Put token info into RaggedTensors, as indicated by token_row_splits.
          def put_token_info_into_ragged_tensor(token_info_values):
              return RaggedTensor.from_row_splits(
                  token_info_values, token_row_splits, validate=False
              )
  
          tokens = put_token_info_into_ragged_tensor(token_values)
          start_offsets = put_token_info_into_ragged_tensor(
              start_values
          )
          end_offsets = put_token_info_into_ragged_tensor(end_values)
          return tokens, start_offsets, end_offsets
  
file_path: text/tensorflow_text/python/ops/split_merge_from_logits_tokenizer.py
imports: from __future__ import absolute_import
  from __future__ import division
  from __future__ import print_function
  from tensorflow.python.eager import monitoring
  from tensorflow.python.framework import ops
  from tensorflow.python.ops.ragged.ragged_tensor import RaggedTensor
  from tensorflow_text.python.ops.tokenization import TokenizerWithOffsets
  from tensorflow.python.framework import load_library
  from tensorflow.python.platform import resource_loader
label_desc: Use image node_id 3 for calling the SplitMergeFromLogitsTokenizer obj's underlying member method code with example usage: obj.tokenize_with_offsets(strings, logits) and returns: tokens,  start_offsets,  end_offsets, RaggedTensor