pub fn tokenize(&mut self) -> Result<Vec<Token>, TokenizerError> {
        let mut peekable = self.query.chars().peekable();

        let mut tokens: Vec<Token> = vec![];

        while let Some(token) = self.next_token(&mut peekable)? {
            match &token {
                Token::Whitespace(Whitespace::Newline) => {
                    self.line += 1;
                    self.col = 1;
                }

                Token::Whitespace(Whitespace::Tab) => self.col += 4,
                Token::Identifier(s) => self.col += s.len() as u64,
                Token::Keyword(s) => self.col += s.len() as u64,
                Token::Number(s) => self.col += s.len() as u64,
                Token::String(s) => self.col += s.len() as u64,
                Token::SingleQuotedString(s) => self.col += s.len() as u64,
                Token::DoubleQuotedString(s) => self.col += s.len() as u64,
                _ => self.col += 1,
            }

            tokens.push(token);
        }
        Ok(tokens)
    }
/// Tokenize the statement and produce a vector of tokens
Tokenize the statement and produce a vector of tokens
Tokenize the statement and produce a vector of tokens
pub fn tokenize(&mut self) -> Result<Vec<Token>, TokenizerError> {
        let mut peekable = self.query.chars().peekable();
        let mut tokens: Vec<Token> = vec![];
        while let Some(token) = self.next_token(&mut peekable)? {
            match &token {
                Token::Whitespace(Whitespace::Newline) => {
                    self.line += 1;
                    self.col = 1;
                }
                Token::Whitespace(Whitespace::Tab) => self.col += 4,
                Token::Identifier(s) => self.col += s.len() as u64,
                Token::Keyword(s) => self.col += s.len() as u64,
                Token::Number(s) => self.col += s.len() as u64,
                Token::String(s) => self.col += s.len() as u64,
                Token::SingleQuotedString(s) => self.col += s.len() as u64,
                Token::DoubleQuotedString(s) => self.col += s.len() as u64,
                _ => self.col += 1,
            }
            tokens.push(token);
        }
        Ok(tokens)
    }
pub fn tokenize ( & mut self ) -> Result < Vec < Token > , TokenizerError > { let mut peekable = self . query . chars ( ) . peekable ( ) ; let mut tokens : Vec < Token > = vec ! [ ] ; while let Some ( token ) = self . next_token ( & mut peekable ) ? { match & token { Token :: Whitespace ( Whitespace :: Newline ) => { self . line += 1 ; self . col = 1 ; } Token :: Whitespace ( Whitespace :: Tab ) => self . col += 4 , Token :: Identifier ( s ) => self . col += s . len ( ) as u64 , Token :: Keyword ( s ) => self . col += s . len ( ) as u64 , Token :: Number ( s ) => self . col += s . len ( ) as u64 , Token :: String ( s ) => self . col += s . len ( ) as u64 , Token :: SingleQuotedString ( s ) => self . col += s . len ( ) as u64 , Token :: DoubleQuotedString ( s ) => self . col += s . len ( ) as u64 , _ => self . col += 1 , } tokens . push ( token ) ; } Ok ( tokens ) }
Tokenize the statement and produce a vector of tokens
Tokenize the statement and produce a vector of tokens
&mut self
&mut self