pub(crate) fn tokenize(mut input: &str) -> crate::Result<Self> {
        let mut tokens = vec![];
        let mut loc = Location::default();
        while !input.is_empty() {
            let old_input = input;
            let mut chars = input.chars();
            let kind = match chars.next().unwrap() {
                // c if c.is_ascii_whitespace() => continue,
                ':' => TokenKind::Colon,
                '=' => TokenKind::Eq,
                '\\' => TokenKind::BSlash,
                '/' => TokenKind::FSlash,
                ';' => TokenKind::Semi,
                '[' => TokenKind::OpenBracket,
                ']' => TokenKind::CloseBracket,
                '{' => {
                    let mut buf = Vec::new();
                    // Read alphanumeric ASCII including whitespace until we find a closing curly.
                    loop {
                        match chars.next() {
                            None => bail!("unclosed escape literal"),
                            Some('}') => break,
                            Some(c) if c.is_ascii() => buf.push(c),
                            Some(c) => bail!("Invalid JDBC token `{}`", c),
                        }
                    }
                    TokenKind::Escaped(buf)
                }
                c if c.is_ascii() => TokenKind::Atom(c),
                c => bail!("Invalid JDBC token `{}`", c),
            };
            tokens.push(Token { kind, loc });
            input = chars.as_str();

            let consumed = old_input.len() - input.len();
            loc.advance(&old_input[..consumed]);
        }
        tokens.reverse();
        Ok(Self { tokens })
    }
/// Parse a string into a list of tokens.
Parse a string into a list of tokens.
Parse a string into a list of tokens .
pub(crate) fn tokenize(mut input: &str) -> crate::Result<Self> {
        let mut tokens = vec![];
        let mut loc = Location::default();
        while !input.is_empty() {
            let old_input = input;
            let mut chars = input.chars();
            let kind = match chars.next().unwrap() {
                // c if c.is_ascii_whitespace() => continue,
                ':' => TokenKind::Colon,
                '=' => TokenKind::Eq,
                '\\' => TokenKind::BSlash,
                '/' => TokenKind::FSlash,
                ';' => TokenKind::Semi,
                '[' => TokenKind::OpenBracket,
                ']' => TokenKind::CloseBracket,
                '{' => {
                    let mut buf = Vec::new();
                    // Read alphanumeric ASCII including whitespace until we find a closing curly.
                    loop {
                        match chars.next() {
                            None => bail!("unclosed escape literal"),
                            Some('}') => break,
                            Some(c) if c.is_ascii() => buf.push(c),
                            Some(c) => bail!("Invalid JDBC token `{}`", c),
                        }
                    }
                    TokenKind::Escaped(buf)
                }
                c if c.is_ascii() => TokenKind::Atom(c),
                c => bail!("Invalid JDBC token `{}`", c),
            };
            tokens.push(Token { kind, loc });
            input = chars.as_str();
            let consumed = old_input.len() - input.len();
            loc.advance(&old_input[..consumed]);
        }
        tokens.reverse();
        Ok(Self { tokens })
    }
pub ( crate ) fn tokenize ( mut input : & str ) -> crate :: Result < Self > { let mut tokens = vec ! [ ] ; let mut loc = Location :: default ( ) ; while ! input . is_empty ( ) { let old_input = input ; let mut chars = input . chars ( ) ; let kind = match chars . next ( ) . unwrap ( ) { // c if c.is_ascii_whitespace() => continue, ':' => TokenKind :: Colon , '=' => TokenKind :: Eq , '\\' => TokenKind :: BSlash , '/' => TokenKind :: FSlash , ';' => TokenKind :: Semi , '[' => TokenKind :: OpenBracket , ']' => TokenKind :: CloseBracket , '{' => { let mut buf = Vec :: new ( ) ; // Read alphanumeric ASCII including whitespace until we find a closing curly. loop { match chars . next ( ) { None => bail ! ( " " ) , Some ( '}' ) => break , Some ( c ) if c . is_ascii ( ) => buf . push ( c ) , Some ( c ) => bail ! ( " " c ) , } } TokenKind :: Escaped ( buf ) } c if c . is_ascii ( ) => TokenKind :: Atom ( c ) , c => bail ! ( " " c ) , } ; tokens . push ( Token { kind , loc } ) ; input = chars . as_str ( ) ; let consumed = old_input . len ( ) - input . len ( ) ; loc . advance ( & old_input [ .. consumed ] ) ; } tokens . reverse ( ) ; Ok ( Self { tokens } ) }
Parse a string into a list of tokens.
Parse a string into a list of tokens .
input
&str
input
&str