import tensorflow as tf

        _MergeCandidates, (tokens, candidates),
        parallel_iterations=1,
        back_prop=False)[0]

  def Encode(self, text):
    """Converts string `text` to integer ids and the encoded string.

    Encoding includes prefixing the beginning-of-word token to each word.

    Returns:
      ids: the encoded integer ids.
      tokens: the encoded string.
    """
    words = tf.sparse.to_dense(tf.strings.split([text]), default_value='')[0]
    num_words = tf.size(words)
    ids_ta = tf.TensorArray(tf.int32, 0, dynamic_size=True)

    def _WordsToIds(i, words, ids_ta):
      encoded_ids = self._EncodeToIds(BOW_STR + words[i])
      ids_ta = ids_ta.scatter(
          tf.range(ids_ta.size(),
                   ids_ta.size() + tf.size(encoded_ids)), encoded_ids)
      return i + 1, words, ids_ta

    _, _, ids_ta = tf.while_loop(
        lambda i, *_: i < num_words,
