[FIX] In SentencePieceBPETokenizer, when Vocab or merges is None, unk_token cannot be used.
* [fix] Use unk_token
In SentencePieceBPETokenizer, when Vocab or  merges is None, unk_token cannot be used.
* [fix] If unk_token is None, this case is also considered.
* Update bindings/python/py_src/tokenizers/implementations/sentencepiece_bpe.py
@@ -26,7 +26,7 @@ class SentencePieceBPETokenizer(BaseTokenizer):
if vocab is not None and merges is not None:
tokenizer = Tokenizer(BPE(vocab, merges, dropout=dropout, unk_token=unk_token, fuse_unk=fuse_unk))
else:
- tokenizer = Tokenizer(BPE())
+ tokenizer = Tokenizer(BPE(dropout=dropout, unk_token=unk_token, fuse_unk=fuse_unk))
if tokenizer.token_to_id(str(unk_token)) is not None:
tokenizer.add_special_tokens([str(unk_token)])
