Add WhitespaceSplit PreTokenizer
@@ -26,9 +26,36 @@ impl PreTokenizer for Whitespace {
}
}
+pub struct WhitespaceSplit;
+impl PreTokenizer for WhitespaceSplit {
+ fn pre_tokenize(&self, s: &str) -> Result<Vec<(String, Offsets)>> {
+ let mut words = vec![];
+ let mut word = Vec::with_capacity(1000);
+ let mut offset = 0;
+
+ s.chars().for_each(|c| {
+ if c.is_whitespace() {
+ if !word.is_empty() {
+ let offsets = (offset - word.len(), offset);
+ words.push((word.drain(0..).collect::<String>(), offsets));
+ }
+ } else {
+ word.push(c);
+ }
+ offset += 1;
+ });
+ if !word.is_empty() {
+ let offsets = (offset - word.len(), offset);
+ words.push((word.drain(0..).collect::<String>(), offsets));
+ }
+
+ Ok(words)
+ }
+}
+
#[cfg(test)]
mod tests {
- use super::Whitespace;
+ use super::*;
use crate::tokenizer::PreTokenizer;
#[test]
@@ -58,4 +85,26 @@ mod tests {
assert_eq!(pretok.pre_tokenize(s).unwrap(), res);
}
}
+
+ #[test]
+ fn whitespace_split() {
+ let tests = vec![
+ (
+ "Hey man!",
+ vec![("Hey".into(), (0, 3)), ("man!".into(), (4, 8))],
+ ),
+ (
+ "Hey, man, Good?",
+ vec![
+ ("Hey,".into(), (0, 4)),
+ ("man,".into(), (5, 9)),
+ ("Good?".into(), (10, 15)),
+ ],
+ ),
+ ];
+ let pretok = WhitespaceSplit;
+ for (s, res) in tests {
+ assert_eq!(pretok.pre_tokenize(s).unwrap(), res);
+ }
+ }
}
