docstr: Check that can convert a Keras model to TFLite and it produces the same result for tokenization.
docstr_tok: ["Check","that","can","convert","a","Keras","model","to","TFLite","and","it","produces","the","same","result","for","tokenization","."]
code: def test_tflite_opt_sentence_tokenizer(self):
      """Check that can convert a Keras model to TFLite and it produces the same result for tokenization."""
  
      class TokenizerLayer(tf.keras.layers.Layer):
          def __init__(self, sentencepiece_model, **kwargs):
              super(TokenizerLayer, self).__init__(**kwargs)
              self.sp = (
                  sentencepiece_tokenizer.FastSentencepieceTokenizer(
                      sentencepiece_model
                  )
              )
  
          def call(self, input_tensor, **kwargs):
              return self.sp.tokenize(input_tensor).flat_values
  
      model = tf.keras.models.Sequential(
          [TokenizerLayer(self.sentencepiece_model)]
      )
      input_data = np.array(
          [
              [
                  " ",
                  "to be or not to be",
                  "ignored by length text1",
                  "ignored by length text2",
              ]
          ]
      )
      tf_result = model.predict(input_data)
      converter = tf.lite.TFLiteConverter.from_keras_model(model)
      supported_ops = [tf.lite.OpsSet.TFLITE_BUILTINS]
      converter.target_spec.supported_ops = supported_ops
      converter.allow_custom_ops = True
      tflite_model = converter.convert()
      interpreter = interpreter_wrapper.InterpreterWithCustomOps(
          model_content=tflite_model,
          custom_op_registerers=tf_text.tflite_registrar.SELECT_TFTEXT_OPS,
      )
      interpreter.allocate_tensors()
      input_details = interpreter.get_input_details()
  
      interpreter.set_tensor(input_details[0]["index"], input_data)
      interpreter.invoke()
      output_details = interpreter.get_output_details()
      expected_result = [
          13,
          36,
          83,
          131,
          13,
          36,
          4,
          3127,
          152,
          130,
          30,
          2424,
          168,
          1644,
          1524,
          4,
          3127,
          152,
          130,
          30,
          2424,
          168,
          1644,
          636,
      ]
      self.assertAllEqual(tf_result, expected_result)
      self.assertAllEqual(
          interpreter.get_tensor(output_details[0]["index"]),
          expected_result,
      )
  
imports: import time
  from absl import flags
  import numpy
  import tensorflow.compat.v2
  import tensorflow_text
  from tensorflow.lite.python import interpreter
  from tensorflow.python.platform import gfile
  from tensorflow_text.python.ops import fast_sentencepiece_tokenizer
label_desc: Use image node_id 5 for calling the FastSentencepieceTokenizerTest obj's underlying member method code with example usage: obj.test_tflite_opt_sentence_tokenizer() and returns: self