pub fn lex(source: &SourceFile) -> FileLexemes {
    //TODO performance: does this initial capacity make sense?
    let mut reader = SourceReader::new(source);
    let mut lexer = CodeLexer::new(source.len());
    //TODO: some of these need to be in specific order, like eof/unlexable last, and indent first - but for other the order could be optimized
    loop {
        try_lex!(lex_indents, reader, lexer);
        try_lex!(lex_grouping, reader, lexer);
        try_lex!(lex_operator, reader, lexer);
        try_lex!(lex_association, reader, lexer);
        try_lex!(lex_separators, reader, lexer);
        try_lex!(lex_literal, reader, lexer);
        try_lex!(lex_keyword_identifier, reader, lexer);
        if lex_eof(&mut reader) {
            break;
        }
        try_lex!(lex_unlexable, reader, lexer);
    }
    lexer.into_lexemes()
}
/// Lexes a whole source files and returns the lexemes.
Lexes a whole source files and returns the lexemes.
Lexes a whole source files and returns the lexemes .
pub fn lex(source: &SourceFile) -> FileLexemes {
    //TODO performance: does this initial capacity make sense?
    let mut reader = SourceReader::new(source);
    let mut lexer = CodeLexer::new(source.len());
    //TODO: some of these need to be in specific order, like eof/unlexable last, and indent first - but for other the order could be optimized
    loop {
        try_lex!(lex_indents, reader, lexer);
        try_lex!(lex_grouping, reader, lexer);
        try_lex!(lex_operator, reader, lexer);
        try_lex!(lex_association, reader, lexer);
        try_lex!(lex_separators, reader, lexer);
        try_lex!(lex_literal, reader, lexer);
        try_lex!(lex_keyword_identifier, reader, lexer);
        if lex_eof(&mut reader) {
            break;
        }
        try_lex!(lex_unlexable, reader, lexer);
    }
    lexer.into_lexemes()
}
pub fn lex ( source : & SourceFile ) -> FileLexemes { //TODO performance: does this initial capacity make sense? let mut reader = SourceReader :: new ( source ) ; let mut lexer = CodeLexer :: new ( source . len ( ) ) ; //TODO: some of these need to be in specific order, like eof/unlexable last, and indent first - but for other the order could be optimized loop { try_lex ! ( lex_indents reader lexer ) ; try_lex ! ( lex_grouping reader lexer ) ; try_lex ! ( lex_operator reader lexer ) ; try_lex ! ( lex_association reader lexer ) ; try_lex ! ( lex_separators reader lexer ) ; try_lex ! ( lex_literal reader lexer ) ; try_lex ! ( lex_keyword_identifier reader lexer ) ; if lex_eof ( & mut reader ) { break ; } try_lex ! ( lex_unlexable reader lexer ) ; } lexer . into_lexemes ( ) }
Lexes a whole source files and returns the lexemes.
Lexes a whole source files and returns the lexemes .
source
&SourceFile
source
&SourceFile