'Converts a list of tokens to a list of subtoken ids.
Args:
tokens: a list of strings.
Returns:
a list of integers in the range [0, vocab_size)'

def _tokens_to_subtoken_ids(self, tokens):

    ret = []
    for token in tokens:
        ret.extend(self._escaped_token_to_subtoken_ids(_escape_token(token, self._alphabet)))
    return ret
