def METHOD_NAME(self):
    tokenizer = self.get_tokenizer()
    text = "lower newer"
    bpe_tokens = ["lo", "w", "er</w>", "n", "e", "w", "er</w>"]
    tokens = tokenizer.tokenize(text)
    self.assertListEqual(tokens, bpe_tokens)
    input_tokens = tokens + [tokenizer.unk_token]
    input_bpe_tokens = [10, 2, 16, 9, 3, 2, 16, 20]
    self.assertListEqual(tokenizer.convert_tokens_to_ids(input_tokens), input_bpe_tokens)