pub fn tokenize(content: &str, ids: &HashMap<&str, usize>) -> Vec<usize> {
    /// Notice that we return indices (avoiding allocation)
    content
        // You can split on nonalphanumerics for a big speedup but the tokens are dubious.
        //.split(|c: char| ! c.is_alphanumeric())
        .split_word_bounds()
        .map(|mention| *ids.get(mention).unwrap_or(&0))
        // This will remove unknown words
        // but this is troublesome because it makes the context windows too wide if you have few words
        // because most words will be unknown
        //.filter(|i| *i > 0)
        .collect()
}
/// Tokenize a string according to a dictionary. Unknowns will be 0.
Tokenize a string according to a dictionary. Unknowns will be 0.
Tokenize a string according to a dictionary . Unknowns will be 0 .
pub fn tokenize(content: &str, ids: &HashMap<&str, usize>) -> Vec<usize> {
    /// Notice that we return indices (avoiding allocation)
    content
        // You can split on nonalphanumerics for a big speedup but the tokens are dubious.
        //.split(|c: char| ! c.is_alphanumeric())
        .split_word_bounds()
        .map(|mention| *ids.get(mention).unwrap_or(&0))
        // This will remove unknown words
        // but this is troublesome because it makes the context windows too wide if you have few words
        // because most words will be unknown
        //.filter(|i| *i > 0)
        .collect()
}
pub fn tokenize ( content : & str , ids : & HashMap < & str , usize > ) -> Vec < usize > { /// Notice that we return indices (avoiding allocation) content // You can split on nonalphanumerics for a big speedup but the tokens are dubious. //.split(|c: char| ! c.is_alphanumeric()) . split_word_bounds ( ) . map ( | mention | * ids . get ( mention ) . unwrap_or ( & 0 ) ) // This will remove unknown words // but this is troublesome because it makes the context windows too wide if you have few words // because most words will be unknown //.filter(|i| *i > 0) . collect ( ) }
Tokenize a string according to a dictionary.
Tokenize a string according to a dictionary .
content
&str
ids
&HashMap<&str, usize>
content
&str
ids
&HashMap<&str, usize>