import tensorflow as tf


  def __init__(self, wpm_filepath, merge_prob=1.):
    """Create a WPM encoder.

    Args:
      wpm_filepath: a path to the file containing the vocabulary.
      merge_prob: the probability of merging tokens while encoding.
    """
    # Load vocabulary file.
    self._pieces = []
    with tf.gfile.Open(wpm_filepath, 'r') as f:
      for line in f.readlines():
        line = line.decode('utf-8')
        piece = line.strip().split('\t')[0]
        self._pieces.append(piece)
    self._merge_prob = merge_prob

  def _TokenToString(self, token):
    return py_x_ops.vocab_id_to_token(token, vocab=self._pieces)

