pub fn tokenize(text: &str) -> Result<Vec<Token>, TokenizerError> {
    let mut tokenizer = Tokenizer::new(text);
    match tokenizer.tokenize() {
        Ok(_) => Ok(tokenizer.tokens),
        Err(error) => Err(error),
    }
}
/// Tokenize a body of text.
Tokenize a body of text.
Tokenize a body of text .
pub fn tokenize(text: &str) -> Result<Vec<Token>, TokenizerError> {
    let mut tokenizer = Tokenizer::new(text);
    match tokenizer.tokenize() {
        Ok(_) => Ok(tokenizer.tokens),
        Err(error) => Err(error),
    }
}
pub fn tokenize ( text : & str ) -> Result < Vec < Token > , TokenizerError > { let mut tokenizer = Tokenizer :: new ( text ) ; match tokenizer . tokenize ( ) { Ok ( _ ) => Ok ( tokenizer . tokens ) , Err ( error ) => Err ( error ) , } }
Tokenize a body of text.
Tokenize a body of text .
text
&str
text
&str