import tensorflow as tf

        self._TokenToString(tokens[0]) + self._TokenToString(tokens[1]))

  def _EncodeToIds(self, word):
    # Below:
    #   * a token is a wordpiece ID.
    #   * the tokens array will be merged in-place.
    #   * the candidates array is an array of size len(tokens) - 1.
    #     It contains the token for the merged wordpiece, if it exists,
    #     -1 otherwise. For instance, candidate[3] = id(token[3] + token[4]).
    # First, split into basic UTF-8 characters (letters).
    chars = tf.strings.unicode_split(word, 'UTF-8')
    tokens = self._StringToToken(chars)
    tokens = tf.where(
        tf.equal(tokens, NO_TOKEN),
        # Unseen character.
        tf.broadcast_to(self.unk_id, tf.shape(tokens)),
        tokens)
    # Create initial candidate list.
    candidates = tf.map_fn(
        self._MergeTokens, (tokens[:-1], tokens[1:]), dtype=tokens.dtype)

    def _ShouldMerge(unused_tokens, candidates):
      """Merge until not possible, or we abort early according to merge_prob."""
      return tf.logical_and(
          tf.reduce_any(tf.not_equal(candidates, NO_TOKEN)),
          tf.random.uniform([]) < self._merge_prob)
