pub fn lex(query: &str) -> Result<Vec<(Token, usize)>, ParserError> {
    let buf = &mut LexBuf::new(query);
    let mut tokens = vec![];
    while let Some(ch) = buf.next() {
        let pos = buf.pos() - ch.len_utf8();
        let token = match ch {
            _ if ch.is_ascii_whitespace() => continue,
            '-' if buf.consume('-') => {
                lex_line_comment(buf);
                continue;
            }
            '/' if buf.consume('*') => {
                lex_multiline_comment(buf)?;
                continue;
            }
            '\'' => Token::String(lex_string(buf)?),
            'x' | 'X' if buf.consume('\'') => Token::HexString(lex_string(buf)?),
            'e' | 'E' if buf.consume('\'') => lex_extended_string(buf)?,
            'A'..='Z' | 'a'..='z' | '_' | '\u{80}'..=char::MAX => lex_ident(buf),
            '"' => lex_quoted_ident(buf)?,
            '0'..='9' => lex_number(buf)?,
            '.' if matches!(buf.peek(), Some('0'..='9')) => lex_number(buf)?,
            '$' if matches!(buf.peek(), Some('0'..='9')) => lex_parameter(buf)?,
            '$' => lex_dollar_string(buf)?,
            '(' => Token::LParen,
            ')' => Token::RParen,
            ',' => Token::Comma,
            '.' => Token::Dot,
            ':' if buf.consume(':') => Token::DoubleColon,
            ':' => Token::Colon,
            ';' => Token::Semicolon,
            '[' => Token::LBracket,
            ']' => Token::RBracket,
            #[rustfmt::skip]
            '+'|'-'|'*'|'/'|'<'|'>'|'='|'~'|'!'|'@'|'#'|'%'|'^'|'&'|'|'|'`'|'?' => lex_op(buf)?,
            _ => bail!(pos, "unexpected character in input: {}", ch),
        };
        tokens.push((token, pos))
    }

    #[cfg(debug_assertions)]
    for (_token, pos) in &tokens {
        assert!(query.is_char_boundary(*pos));
    }

    Ok(tokens)
}
/// Lexes a SQL query.
///
/// Returns a list of tokens alongside their corresponding byte offset in the
/// input string. Returns an error if the SQL query is lexically invalid.
///
/// See the module documentation for more information about the lexical
/// structure of SQL.
Lexes a SQL query.
Returns a list of tokens alongside their corresponding byte offset in the
input string. Returns an error if the SQL query is lexically invalid.

See the module documentation for more information about the lexical
structure of SQL.
Lexes a SQL query . Returns a list of tokens alongside their corresponding byte offset in the input string . Returns an error if the SQL query is lexically invalid . See the module documentation for more information about the lexical structure of SQL .
pub fn lex(query: &str) -> Result<Vec<(Token, usize)>, ParserError> {
    let buf = &mut LexBuf::new(query);
    let mut tokens = vec![];
    while let Some(ch) = buf.next() {
        let pos = buf.pos() - ch.len_utf8();
        let token = match ch {
            _ if ch.is_ascii_whitespace() => continue,
            '-' if buf.consume('-') => {
                lex_line_comment(buf);
                continue;
            }
            '/' if buf.consume('*') => {
                lex_multiline_comment(buf)?;
                continue;
            }
            '\'' => Token::String(lex_string(buf)?),
            'x' | 'X' if buf.consume('\'') => Token::HexString(lex_string(buf)?),
            'e' | 'E' if buf.consume('\'') => lex_extended_string(buf)?,
            'A'..='Z' | 'a'..='z' | '_' | '\u{80}'..=char::MAX => lex_ident(buf),
            '"' => lex_quoted_ident(buf)?,
            '0'..='9' => lex_number(buf)?,
            '.' if matches!(buf.peek(), Some('0'..='9')) => lex_number(buf)?,
            '$' if matches!(buf.peek(), Some('0'..='9')) => lex_parameter(buf)?,
            '$' => lex_dollar_string(buf)?,
            '(' => Token::LParen,
            ')' => Token::RParen,
            ',' => Token::Comma,
            '.' => Token::Dot,
            ':' if buf.consume(':') => Token::DoubleColon,
            ':' => Token::Colon,
            ';' => Token::Semicolon,
            '[' => Token::LBracket,
            ']' => Token::RBracket,
            #[rustfmt::skip]
            '+'|'-'|'*'|'/'|'<'|'>'|'='|'~'|'!'|'@'|'#'|'%'|'^'|'&'|'|'|'`'|'?' => lex_op(buf)?,
            _ => bail!(pos, "unexpected character in input: {}", ch),
        };
        tokens.push((token, pos))
    }
    #[cfg(debug_assertions)]
    for (_token, pos) in &tokens {
        assert!(query.is_char_boundary(*pos));
    }
    Ok(tokens)
}
pub fn lex ( query : & str ) -> Result < Vec < ( Token , usize ) > , ParserError > { let buf = & mut LexBuf :: new ( query ) ; let mut tokens = vec ! [ ] ; while let Some ( ch ) = buf . next ( ) { let pos = buf . pos ( ) - ch . len_utf8 ( ) ; let token = match ch { _ if ch . is_ascii_whitespace ( ) => continue , '-' if buf . consume ( '-' ) => { lex_line_comment ( buf ) ; continue ; } '/' if buf . consume ( '*' ) => { lex_multiline_comment ( buf ) ? ; continue ; } '\'' => Token :: String ( lex_string ( buf ) ? ) , 'x' | 'X' if buf . consume ( '\'' ) => Token :: HexString ( lex_string ( buf ) ? ) , 'e' | 'E' if buf . consume ( '\'' ) => lex_extended_string ( buf ) ? , 'A' ..= 'Z' | 'a' ..= 'z' | '_' | '\u{80}' ..= char :: MAX => lex_ident ( buf ) , '"' => lex_quoted_ident ( buf ) ? , '0' ..= '9' => lex_number ( buf ) ? , '.' if matches ! ( buf peek ( ) Some ( '0' '9' ) ) => lex_number ( buf ) ? , '$' if matches ! ( buf peek ( ) Some ( '0' '9' ) ) => lex_parameter ( buf ) ? , '$' => lex_dollar_string ( buf ) ? , '(' => Token :: LParen , ')' => Token :: RParen , ',' => Token :: Comma , '.' => Token :: Dot , ':' if buf . consume ( ':' ) => Token :: DoubleColon , ':' => Token :: Colon , ';' => Token :: Semicolon , '[' => Token :: LBracket , ']' => Token :: RBracket , # [ rustfmt :: skip ] '+' | '-' | '*' | '/' | '<' | '>' | '=' | '~' | '!' | '@' | '#' | '%' | '^' | '&' | '|' | '`' | '?' => lex_op ( buf ) ? , _ => bail ! ( pos " " ch ) , } ; tokens . push ( ( token , pos ) ) } # [ cfg ( debug_assertions ) ] for ( _token , pos ) in & tokens { assert ! ( query is_char_boundary ( pos ) ) ; } Ok ( tokens ) }
Lexes a SQL query.
Lexes a SQL query .
query
&str
query
&str