'Initialize vocabulary with tokens from token_generator.'

def _init_vocab(self, token_generator):

    self._id_to_token = {}
    self._id_to_token.update(dict(list(enumerate(RESERVED_TOKENS))))
    token_id = len(RESERVED_TOKENS)
    for token in token_generator:
        self._id_to_token[token_id] = token
        token_id += 1
    self._token_to_id = dict([(v, k) for (k, v) in six.iteritems(self._id_to_token)])
