docstr: Check that can convert a Keras model to TFLite and it produces the same result for tokenization.
docstr_tok: ["Check","that","can","convert","a","Keras","model","to","TFLite","and","it","produces","the","same","result","for","tokenization","."]
code: def test_tflite_opt_sentence_detokenizer(self):
      """Check that can convert a Keras model to TFLite and it produces the same result for tokenization."""
  
      class DeTokenizerLayer(tf.keras.layers.Layer):
          def __init__(self, sentencepiece_model, **kwargs):
              super(DeTokenizerLayer, self).__init__(**kwargs)
              self.sp = (
                  sentencepiece_tokenizer.FastSentencepieceTokenizer(
                      sentencepiece_model
                  )
              )
  
          def call(self, input_tensor, **kwargs):
              return self.sp.detokenize(input_tensor)
  
      model = tf.keras.models.Sequential(
          [DeTokenizerLayer(self.sentencepiece_model)]
      )
      input_data = np.array(
          [
              [
                  13,
                  36,
                  83,
                  131,
                  13,
                  36,
                  4,
                  3127,
                  152,
                  130,
                  30,
                  2424,
                  168,
                  1644,
                  1524,
                  4,
                  3127,
                  152,
                  130,
                  30,
                  2424,
                  168,
                  1644,
                  636,
              ]
          ],
          dtype=np.int32,
      )
      tf_result = model.predict(input_data)
      converter = tf.lite.TFLiteConverter.from_keras_model(model)
      supported_ops = [tf.lite.OpsSet.TFLITE_BUILTINS]
      converter.target_spec.supported_ops = supported_ops
      converter.allow_custom_ops = True
      tflite_model = converter.convert()
      interpreter = interpreter_wrapper.InterpreterWithCustomOps(
          model_content=tflite_model,
          custom_op_registerers=tf_text.tflite_registrar.SELECT_TFTEXT_OPS,
      )
      interpreter.allocate_tensors()
      input_details = interpreter.get_input_details()
  
      interpreter.set_tensor(input_details[0]["index"], input_data)
      interpreter.invoke()
      output_details = interpreter.get_output_details()
      expected_result = [
          "to be or not to be ignored by length text1 ignored by length text2"
      ]
      self.assertAllEqual(tf_result, expected_result)
      self.assertAllEqual(
          interpreter.get_tensor(output_details[0]["index"]),
          expected_result,
      )
  
imports: import time
  from absl import flags
  import numpy
  import tensorflow.compat.v2
  import tensorflow_text
  from tensorflow.lite.python import interpreter
  from tensorflow.python.platform import gfile
  from tensorflow_text.python.ops import fast_sentencepiece_tokenizer
label_desc: Use image node_id 6 for calling the FastSentencepieceTokenizerTest obj's underlying member method code with example usage: obj.test_tflite_opt_sentence_detokenizer() and returns: self