third_party/tflite_support/src/tensorflow_lite_support/custom_ops/python/sentencepiece_tokenizer.py
def detokenize(self, input):  # pylint: disable=redefined-builtin
    """Detokenizes tokens into preprocessed text.

    Args:
      input: A `RaggedTensor` or `Tensor` with int32 encoded text with rank >=
        1.

    Returns:
      A N-1 dimensional string Tensor or RaggedTensor of the detokenized text.
    """
    input_tensor = ragged_tensor.convert_to_tensor_or_ragged_tensor(input)
    if input_tensor.shape.ndims is None:
      raise ValueError("Rank of input_tensor must be statically known.")
    if input_tensor.shape.ndims == 0:
      raise ValueError("Rank of input_tensor must be at least 1.")
    if ragged_tensor.is_ragged(input_tensor):
      if input_tensor.flat_values.shape.ndims > 1:
        # If the flat_values of our ragged tensor is multi-dimensional, we can
        # process it separately and our output will have the same nested
        # splits as our input.
        tokens = self.detokenize(input_tensor.flat_values)
        return input_tensor.with_flat_values(tokens)
      elif input_tensor.ragged_rank > 1:
        # Recursively process the values of the ragged tensor.
        tokens = self.detokenize(input_tensor.values)
        return input_tensor.with_values(tokens)
      else:
        return gen_sentencepiece_detokenizer_op.tf_sentencepiece_detokenize_op(
            self._converted_model_detokenizer, input_tensor.flat_values,
            input_tensor.row_splits)
    else:
      if input_tensor.shape.ndims > 1:
        # Convert the input tensor to ragged and process it.
        return self.detokenize(
            tf.RaggedTensor.from_tensor(
                input_tensor, row_splits_dtype=tf.int32))
      else:
        tokens = self.detokenize(tf.stack([input_tensor]))
        return tf.reshape(tokens, [])
Detokenizes tokens into preprocessed text.

    Args:
      input: A `RaggedTensor` or `Tensor` with int32 encoded text with rank >=
        1.

    Returns:
      A N-1 dimensional string Tensor or RaggedTensor of the detokenized text.
    
Detokenizes tokens into preprocessed text.
Detokenizes tokens into preprocessed text .
def detokenize(self, input):  
    input_tensor = ragged_tensor.convert_to_tensor_or_ragged_tensor(input)
    if input_tensor.shape.ndims is None:
      raise ValueError("Rank of input_tensor must be statically known.")
    if input_tensor.shape.ndims == 0:
      raise ValueError("Rank of input_tensor must be at least 1.")
    if ragged_tensor.is_ragged(input_tensor):
      if input_tensor.flat_values.shape.ndims > 1:
        tokens = self.detokenize(input_tensor.flat_values)
        return input_tensor.with_flat_values(tokens)
      elif input_tensor.ragged_rank > 1:
        tokens = self.detokenize(input_tensor.values)
        return input_tensor.with_values(tokens)
      else:
        return gen_sentencepiece_detokenizer_op.tf_sentencepiece_detokenize_op(
            self._converted_model_detokenizer, input_tensor.flat_values,
            input_tensor.row_splits)
    else:
      if input_tensor.shape.ndims > 1:
        return self.detokenize(
            tf.RaggedTensor.from_tensor(
                input_tensor, row_splits_dtype=tf.int32))
      else:
        tokens = self.detokenize(tf.stack([input_tensor]))
        return tf.reshape(tokens, [])
def detokenize ( self , input ) : input_tensor = ragged_tensor . convert_to_tensor_or_ragged_tensor ( input ) if input_tensor . shape . ndims is None : raise ValueError ( "Rank of input_tensor must be statically known." ) if input_tensor . shape . ndims == 0 : raise ValueError ( "Rank of input_tensor must be at least 1." ) if ragged_tensor . is_ragged ( input_tensor ) : if input_tensor . flat_values . shape . ndims > 1 : tokens = self . detokenize ( input_tensor . flat_values ) return input_tensor . with_flat_values ( tokens ) elif input_tensor . ragged_rank > 1 : tokens = self . detokenize ( input_tensor . values ) return input_tensor . with_values ( tokens ) else : return gen_sentencepiece_detokenizer_op . tf_sentencepiece_detokenize_op ( self . _converted_model_detokenizer , input_tensor . flat_values , input_tensor . row_splits ) else : if input_tensor . shape . ndims > 1 : return self . detokenize ( tf . RaggedTensor . from_tensor ( input_tensor , row_splits_dtype = tf . int32 ) ) else : tokens = self . detokenize ( tf . stack ( [ input_tensor ] ) ) return tf . reshape ( tokens , [ ] )
Detokenizes tokens into preprocessed text.
Detokenizes tokens into preprocessed text .
# pylint: disable=redefined-builtin """Detokenizes tokens into preprocessed text.

    Args:
      input: A `RaggedTensor` or `Tensor` with int32 encoded text with rank >=
        1.

    Returns:
      A N-1 dimensional string Tensor or RaggedTensor of the detokenized text.
    """ # If the flat_values of our ragged tensor is multi-dimensional, we can # process it separately and our output will have the same nested # splits as our input. # Recursively process the values of the ragged tensor. # Convert the input tensor to ragged and process it.
A N-1 dimensional string Tensor or RaggedTensor of the detokenized text.
A N - 1 dimensional string Tensor or RaggedTensor of the detokenized text .
self
input
A `RaggedTensor` or `Tensor` with int32 encoded text with rank >
1.
A ` RaggedTensor ` or ` Tensor ` with int32 encoded text with rank > 1 .