BasicPreTokenizer handles do_basic_tokenize for Bert
@@ -58,6 +58,8 @@ fn is_chinese_char(c: char) -> bool {
}
pub struct BasicPreTokenizer {
+ /// Whether to do the basic tokenization
+ do_basic_tokenize: bool,
/// Whether to lower case the input.
do_lower_case: bool,
/// A list of token not to split.
@@ -68,11 +70,13 @@ pub struct BasicPreTokenizer {
impl BasicPreTokenizer {
pub fn new(
+ do_basic_tokenize: bool,
do_lower_case: bool,
never_split: HashSet<String>,
tokenize_chinese_chars: bool,
) -> Self {
BasicPreTokenizer {
+ do_basic_tokenize,
do_lower_case,
never_split,
tokenize_chinese_chars,
@@ -145,6 +149,12 @@ impl BasicPreTokenizer {
impl PreTokenizer for BasicPreTokenizer {
fn pre_tokenize(&self, s: &str) -> Result<Vec<String>> {
+ if !self.do_basic_tokenize {
+ Ok(whitespace_tokenize(&s)
+ .into_iter()
+ .map(|s| s.to_owned())
+ .collect())
+ } else {
let mut text = self.clean_text(s);
// This was added on November 1st, 2018 for the multilingual and Chinese
@@ -169,3 +179,4 @@ impl PreTokenizer for BasicPreTokenizer {
Ok(split_tokens)
}
}
+}
