fn make_tokenizer(s: &str) -> impl std::iter::Iterator<Item = &str> + '_ {
    s.split_whitespace()
        .map(|token| {
            // poor's man split_including_delim() that keeps the paranthesis delimiters as values.
            let mut last_paren_idx = 0;
            let mut parens_and_tokens = token
                .match_indices(|c: char| is_paren(&c))
                .map(|(idx, matched)| {
                    let res = if last_paren_idx != idx {
                        vec![&token[last_paren_idx..idx], matched]
                    } else {
                        vec![matched]
                    };
                    last_paren_idx = idx + matched.len();
                    res
                })
                .flatten()
                .collect_vec();
            if last_paren_idx < token.len() {
                parens_and_tokens.push(&token[last_paren_idx..])
            }
            parens_and_tokens
        })
        .flatten()
}
/// Returns a token iterator for given string, essentially splitting at whitespace and parenthesis,
/// while keeping the parenthesis.
/// Takes   '1 + (2 * 3)'
/// Returns ['1', '+', '(', '2', '*', '3', ')']
Returns a token iterator for given string, essentially splitting at whitespace and parenthesis,
while keeping the parenthesis.
Returns a token iterator for given string essentially splitting at whitespace and parenthesis while keeping the parenthesis .
fn make_tokenizer(s: &str) -> impl std::iter::Iterator<Item = &str> + '_ {
    s.split_whitespace()
        .map(|token| {
            // poor's man split_including_delim() that keeps the paranthesis delimiters as values.
            let mut last_paren_idx = 0;
            let mut parens_and_tokens = token
                .match_indices(|c: char| is_paren(&c))
                .map(|(idx, matched)| {
                    let res = if last_paren_idx != idx {
                        vec![&token[last_paren_idx..idx], matched]
                    } else {
                        vec![matched]
                    };
                    last_paren_idx = idx + matched.len();
                    res
                })
                .flatten()
                .collect_vec();
            if last_paren_idx < token.len() {
                parens_and_tokens.push(&token[last_paren_idx..])
            }
            parens_and_tokens
        })
        .flatten()
}
fn make_tokenizer ( s : & str ) -> impl std :: iter :: Iterator < Item = & str > + ' _ { s . split_whitespace ( ) . map ( | token | { // poor's man split_including_delim() that keeps the paranthesis delimiters as values. let mut last_paren_idx = 0 ; let mut parens_and_tokens = token . match_indices ( | c : char | is_paren ( & c ) ) . map ( | ( idx , matched ) | { let res = if last_paren_idx != idx { vec ! [ token [ last_paren_idx idx ] matched ] } else { vec ! [ matched ] } ; last_paren_idx = idx + matched . len ( ) ; res } ) . flatten ( ) . collect_vec ( ) ; if last_paren_idx < token . len ( ) { parens_and_tokens . push ( & token [ last_paren_idx .. ] ) } parens_and_tokens } ) . flatten ( ) }
Returns a token iterator for given string, essentially splitting at whitespace and parenthesis,
while keeping the parenthesis.
Returns a token iterator for given string essentially splitting at whitespace and parenthesis while keeping the parenthesis .
s
&str
s
&str