pub fn tokenize<Chars: Iterator<Item=char>>(buffer: &mut TokenReadBuffer<Chars>, location: FileLocation) -> (Token, String, FileLocation) {
    match buffer.read_next() {
        None            => (Token::EndOfFile, String::from(""), location),
        Some(';')       => read_comment(buffer, location),
        Some('(')       => (Token::OpenParen, buffer.read_characters(), buffer.update_location(location)),
        Some(')')       => (Token::CloseParen, buffer.read_characters(), buffer.update_location(location)),
        Some(' ')       |
        Some('\t')      |
        Some('\n')      |
        Some('\r')      => read_whitespace(buffer, location),
        Some('\'')      => read_character(buffer, location),
        Some('\"')      => read_string(buffer, location),
        Some('$')       => read_hex_number(buffer, location),
        Some('=')       => {
            let next_char = buffer.read_next();
            if next_char == Some('t') || next_char == Some('f') {
                (Token::Boolean, buffer.read_characters(), buffer.update_location(location))
            } else {
                if next_char.is_some() {
                    buffer.push_back();
                }
                (Token::Symbol('#'), buffer.read_characters(), buffer.update_location(location))
            }
        },

        Some(symbol)    => {
            if symbol.is_alphabetic() || symbol == '_' {
                read_atom(buffer, location)
            } else if symbol.is_numeric() {
                read_number(buffer, location)
            } else {
                // Repeated symbol values create a longer atom
                let mut symbol_string = symbol.to_string();

                loop {
                    // Collect as many copies of the symbol as we can
                    let next = buffer.read_next();
                    if next.is_none() {
                        break;
                    } else if let Some(next) = next {
                        if next != symbol {
                            // Allow for some symbols to be longer than one character (eg >=)
                            match next {
                                '=' => { },

                                _ => {
                                    buffer.push_back();
                                    break;
                                }
                            }
                        }
                    }

                    symbol_string.push(symbol);
                }

                if symbol_string.len() <= 1 {
                    (Token::Symbol(symbol), buffer.read_characters(), buffer.update_location(location))
                } else {
                    (Token::Atom, buffer.read_characters(), buffer.update_location(location))
                }
            }
        }
    }
}
///
/// Reads a token from a string, starting at the specified location. Returns the token, the text for the token and the updated location.
///
Reads a token from a string, starting at the specified location. Returns the token, the text for the token and the updated location.
Reads a token from a string starting at the specified location . Returns the token the text for the token and the updated location .
pub fn tokenize<Chars: Iterator<Item=char>>(buffer: &mut TokenReadBuffer<Chars>, location: FileLocation) -> (Token, String, FileLocation) {
    match buffer.read_next() {
        None            => (Token::EndOfFile, String::from(""), location),
        Some(';')       => read_comment(buffer, location),
        Some('(')       => (Token::OpenParen, buffer.read_characters(), buffer.update_location(location)),
        Some(')')       => (Token::CloseParen, buffer.read_characters(), buffer.update_location(location)),
        Some(' ')       |
        Some('\t')      |
        Some('\n')      |
        Some('\r')      => read_whitespace(buffer, location),
        Some('\'')      => read_character(buffer, location),
        Some('\"')      => read_string(buffer, location),
        Some('$')       => read_hex_number(buffer, location),
        Some('=')       => {
            let next_char = buffer.read_next();
            if next_char == Some('t') || next_char == Some('f') {
                (Token::Boolean, buffer.read_characters(), buffer.update_location(location))
            } else {
                if next_char.is_some() {
                    buffer.push_back();
                }
                (Token::Symbol('#'), buffer.read_characters(), buffer.update_location(location))
            }
        },
        Some(symbol)    => {
            if symbol.is_alphabetic() || symbol == '_' {
                read_atom(buffer, location)
            } else if symbol.is_numeric() {
                read_number(buffer, location)
            } else {
                // Repeated symbol values create a longer atom
                let mut symbol_string = symbol.to_string();
                loop {
                    // Collect as many copies of the symbol as we can
                    let next = buffer.read_next();
                    if next.is_none() {
                        break;
                    } else if let Some(next) = next {
                        if next != symbol {
                            // Allow for some symbols to be longer than one character (eg >=)
                            match next {
                                '=' => { },
                                _ => {
                                    buffer.push_back();
                                    break;
                                }
                            }
                        }
                    }
                    symbol_string.push(symbol);
                }
                if symbol_string.len() <= 1 {
                    (Token::Symbol(symbol), buffer.read_characters(), buffer.update_location(location))
                } else {
                    (Token::Atom, buffer.read_characters(), buffer.update_location(location))
                }
            }
        }
    }
}
pub fn tokenize < Chars : Iterator < Item = char > > ( buffer : & mut TokenReadBuffer < Chars > , location : FileLocation ) -> ( Token , String , FileLocation ) { match buffer . read_next ( ) { None => ( Token :: EndOfFile , String :: from ( " " ) , location ) , Some ( ';' ) => read_comment ( buffer , location ) , Some ( '(' ) => ( Token :: OpenParen , buffer . read_characters ( ) , buffer . update_location ( location ) ) , Some ( ')' ) => ( Token :: CloseParen , buffer . read_characters ( ) , buffer . update_location ( location ) ) , Some ( ' ' ) | Some ( '\t' ) | Some ( '\n' ) | Some ( '\r' ) => read_whitespace ( buffer , location ) , Some ( '\'' ) => read_character ( buffer , location ) , Some ( '\"' ) => read_string ( buffer , location ) , Some ( '$' ) => read_hex_number ( buffer , location ) , Some ( '=' ) => { let next_char = buffer . read_next ( ) ; if next_char == Some ( 't' ) || next_char == Some ( 'f' ) { ( Token :: Boolean , buffer . read_characters ( ) , buffer . update_location ( location ) ) } else { if next_char . is_some ( ) { buffer . push_back ( ) ; } ( Token :: Symbol ( '#' ) , buffer . read_characters ( ) , buffer . update_location ( location ) ) } } , Some ( symbol ) => { if symbol . is_alphabetic ( ) || symbol == '_' { read_atom ( buffer , location ) } else if symbol . is_numeric ( ) { read_number ( buffer , location ) } else { // Repeated symbol values create a longer atom let mut symbol_string = symbol . to_string ( ) ; loop { // Collect as many copies of the symbol as we can let next = buffer . read_next ( ) ; if next . is_none ( ) { break ; } else if let Some ( next ) = next { if next != symbol { // Allow for some symbols to be longer than one character (eg >=) match next { '=' => { } , _ => { buffer . push_back ( ) ; break ; } } } } symbol_string . push ( symbol ) ; } if symbol_string . len ( ) <= 1 { ( Token :: Symbol ( symbol ) , buffer . read_characters ( ) , buffer . update_location ( location ) ) } else { ( Token :: Atom , buffer . read_characters ( ) , buffer . update_location ( location ) ) } } } } }
Reads a token from a string, starting at the specified location.
Reads a token from a string starting at the specified location .
buffer
&mut TokenReadBuffer<Chars>
location
FileLocation
buffer
&mut TokenReadBuffer<Chars>
location
FileLocation