Use supplied UNK token even when vocab absent
If a vocab file isn't provided the supplied unk token (different from [UNK]) gets ignored and later throws an error:
Exception: WordPiece error: Missing [UNK] token from the vocabulary
when trying to encode an input string with an unknown token.
@@ -29,7 +29,7 @@ class BertWordPieceTokenizer(BaseTokenizer):
if vocab_file is not None:
tokenizer = Tokenizer(WordPiece(vocab_file, unk_token=str(unk_token)))
else:
- tokenizer = Tokenizer(WordPiece())
+ tokenizer = Tokenizer(WordPiece(unk_token=str(unk_token)))
# Let the tokenizer know about special tokens if they are part of the vocab
if tokenizer.token_to_id(str(unk_token)) is not None:
