pub fn tokenize(tokens: &mut Vec<Token>, iter: &mut std::str::Chars, curr: &Option<char>) {
    match curr {
        Some(c) => match c {
            'a' | 'A' => return tokenize_word(tokens, iter, *c),
            'd' | 'D' => return tokenize_word(tokens, iter, *c),
            '0'..='9' => return tokenize_num(tokens, iter, c.to_digit(RADIX).unwrap()),
            '%' => return tokenize_percent(tokens, iter, 2),
            '+' | '-' | '!' | '*' | '>' | '<' | '=' | '$' => return tokenize_op2(tokens, iter, *c),
            '~' | '`' | '^' | 'Y' | '/' => return tokenize_op(tokens, iter, *c),
            '(' | '{' | '[' => return tokenize_start(tokens, iter, *c),
            ')' | '}' | ']' => return tokenize_end(tokens, iter, *c),   
            ',' => tokenize_comma(tokens, iter),
            '\n' | '\t' | '\r' | ' ' => tokenize_whitespace(tokens, iter, *c),
            _ => println!("Unexpected token: char={}", c),
        },
        None => return,
    }
}
// Populate the vector of tokens by iterating through the
// source string one character at a time. If a function consumes
// the character then it must advance the iterator before recursively
// calling tokenize again. 
Populate the vector of tokens by iterating through the
source string one character at a time. If a function consumes
the character then it must advance the iterator before recursively
calling tokenize again.
Populate the vector of tokens by iterating through the source string one character at a time . If a function consumes the character then it must advance the iterator before recursively calling tokenize again .
pub fn tokenize(tokens: &mut Vec<Token>, iter: &mut std::str::Chars, curr: &Option<char>) {
    match curr {
        Some(c) => match c {
            'a' | 'A' => return tokenize_word(tokens, iter, *c),
            'd' | 'D' => return tokenize_word(tokens, iter, *c),
            '0'..='9' => return tokenize_num(tokens, iter, c.to_digit(RADIX).unwrap()),
            '%' => return tokenize_percent(tokens, iter, 2),
            '+' | '-' | '!' | '*' | '>' | '<' | '=' | '$' => return tokenize_op2(tokens, iter, *c),
            '~' | '`' | '^' | 'Y' | '/' => return tokenize_op(tokens, iter, *c),
            '(' | '{' | '[' => return tokenize_start(tokens, iter, *c),
            ')' | '}' | ']' => return tokenize_end(tokens, iter, *c),   
            ',' => tokenize_comma(tokens, iter),
            '\n' | '\t' | '\r' | ' ' => tokenize_whitespace(tokens, iter, *c),
            _ => println!("Unexpected token: char={}", c),
        },
        None => return,
    }
}
pub fn tokenize ( tokens : & mut Vec < Token > , iter : & mut std :: str :: Chars , curr : & Option < char > ) { match curr { Some ( c ) => match c { 'a' | 'A' => return tokenize_word ( tokens , iter , * c ) , 'd' | 'D' => return tokenize_word ( tokens , iter , * c ) , '0' ..= '9' => return tokenize_num ( tokens , iter , c . to_digit ( RADIX ) . unwrap ( ) ) , '%' => return tokenize_percent ( tokens , iter , 2 ) , '+' | '-' | '!' | '*' | '>' | '<' | '=' | '$' => return tokenize_op2 ( tokens , iter , * c ) , '~' | '`' | '^' | 'Y' | '/' => return tokenize_op ( tokens , iter , * c ) , '(' | '{' | '[' => return tokenize_start ( tokens , iter , * c ) , ')' | '}' | ']' => return tokenize_end ( tokens , iter , * c ) , ',' => tokenize_comma ( tokens , iter ) , '\n' | '\t' | '\r' | ' ' => tokenize_whitespace ( tokens , iter , * c ) , _ => println ! ( " " c ) , } , None => return , } }
Populate the vector of tokens by iterating through the
source string one character at a time.
Populate the vector of tokens by iterating through the source string one character at a time .
tokens
&mut Vec<Token>
iter
&mut std::str::Chars
curr
&Option<char>
tokens
&mut Vec<Token>
iter
&mut std::str::Chars
curr
&Option<char>