pub fn lex(input: &str, span_offset: usize) -> (Vec<Token>, Option<ParseError>) {
    let mut char_indices = input.char_indices().peekable();
    let mut error = None;

    let mut output = vec![];

    while let Some((idx, c)) = char_indices.peek() {
        if *c == '|' {
            let idx = *idx;
            let prev_idx = idx;
            let _ = char_indices.next();
            if let Some((idx, c)) = char_indices.peek() {
                if *c == '|' {
                    // we have '||' instead of '|'
                    let idx = *idx;
                    let _ = char_indices.next();
                    output.push(Token::new(
                        TokenContents::Bare("||".into()),
                        Span::new(span_offset + prev_idx, span_offset + idx + 1),
                    ));
                    continue;
                }
            }
            output.push(Token::new(
                TokenContents::Pipe,
                Span::new(span_offset + idx, span_offset + idx + 1),
            ));
        } else if *c == ';' {
            let idx = *idx;
            let _ = char_indices.next();
            output.push(Token::new(
                TokenContents::Semicolon,
                Span::new(span_offset + idx, span_offset + idx + 1),
            ));
        } else if *c == '\n' || *c == '\r' {
            let idx = *idx;
            let _ = char_indices.next();
            output.push(Token::new(
                TokenContents::EOL,
                Span::new(span_offset + idx, span_offset + idx + 1),
            ));
        } else if c.is_whitespace() {
            let _ = char_indices.next();
        } else {
            let (result, err) = bare(&mut char_indices, span_offset);
            if error.is_none() {
                error = err;
            }
            let Spanned { item, span } = result;
            output.push(Token::new(TokenContents::Bare(item), span));
        }
    }

    (output, error)
}
/// Breaks the input string into a vector of tokens. This tokenization only tries to classify separators like
/// semicolons, pipes, etc from external bare values (values that haven't been classified further)
/// Takes in a string and and offset, which is used to offset the spans created (for when this function is used to parse inner strings)
Breaks the input string into a vector of tokens. This tokenization only tries to classify separators like
semicolons, pipes, etc from external bare values (values that haven't been classified further)
Takes in a string and and offset, which is used to offset the spans created (for when this function is used to parse inner strings)
Breaks the input string into a vector of tokens . This tokenization only tries to classify separators like semicolons pipes etc from external bare values ( values that haven ' t been classified further ) Takes in a string and and offset which is used to offset the spans created ( for when this function is used to parse inner strings )
pub fn lex(input: &str, span_offset: usize) -> (Vec<Token>, Option<ParseError>) {
    let mut char_indices = input.char_indices().peekable();
    let mut error = None;
    let mut output = vec![];
    while let Some((idx, c)) = char_indices.peek() {
        if *c == '|' {
            let idx = *idx;
            let prev_idx = idx;
            let _ = char_indices.next();
            if let Some((idx, c)) = char_indices.peek() {
                if *c == '|' {
                    // we have '||' instead of '|'
                    let idx = *idx;
                    let _ = char_indices.next();
                    output.push(Token::new(
                        TokenContents::Bare("||".into()),
                        Span::new(span_offset + prev_idx, span_offset + idx + 1),
                    ));
                    continue;
                }
            }
            output.push(Token::new(
                TokenContents::Pipe,
                Span::new(span_offset + idx, span_offset + idx + 1),
            ));
        } else if *c == ';' {
            let idx = *idx;
            let _ = char_indices.next();
            output.push(Token::new(
                TokenContents::Semicolon,
                Span::new(span_offset + idx, span_offset + idx + 1),
            ));
        } else if *c == '\n' || *c == '\r' {
            let idx = *idx;
            let _ = char_indices.next();
            output.push(Token::new(
                TokenContents::EOL,
                Span::new(span_offset + idx, span_offset + idx + 1),
            ));
        } else if c.is_whitespace() {
            let _ = char_indices.next();
        } else {
            let (result, err) = bare(&mut char_indices, span_offset);
            if error.is_none() {
                error = err;
            }
            let Spanned { item, span } = result;
            output.push(Token::new(TokenContents::Bare(item), span));
        }
    }
    (output, error)
}
pub fn lex ( input : & str , span_offset : usize ) -> ( Vec < Token > , Option < ParseError > ) { let mut char_indices = input . char_indices ( ) . peekable ( ) ; let mut error = None ; let mut output = vec ! [ ] ; while let Some ( ( idx , c ) ) = char_indices . peek ( ) { if * c == '|' { let idx = * idx ; let prev_idx = idx ; let _ = char_indices . next ( ) ; if let Some ( ( idx , c ) ) = char_indices . peek ( ) { if * c == '|' { // we have '||' instead of '|' let idx = * idx ; let _ = char_indices . next ( ) ; output . push ( Token :: new ( TokenContents :: Bare ( " " . into ( ) ) , Span :: new ( span_offset + prev_idx , span_offset + idx + 1 ) , ) ) ; continue ; } } output . push ( Token :: new ( TokenContents :: Pipe , Span :: new ( span_offset + idx , span_offset + idx + 1 ) , ) ) ; } else if * c == ';' { let idx = * idx ; let _ = char_indices . next ( ) ; output . push ( Token :: new ( TokenContents :: Semicolon , Span :: new ( span_offset + idx , span_offset + idx + 1 ) , ) ) ; } else if * c == '\n' || * c == '\r' { let idx = * idx ; let _ = char_indices . next ( ) ; output . push ( Token :: new ( TokenContents :: EOL , Span :: new ( span_offset + idx , span_offset + idx + 1 ) , ) ) ; } else if c . is_whitespace ( ) { let _ = char_indices . next ( ) ; } else { let ( result , err ) = bare ( & mut char_indices , span_offset ) ; if error . is_none ( ) { error = err ; } let Spanned { item , span } = result ; output . push ( Token :: new ( TokenContents :: Bare ( item ) , span ) ) ; } } ( output , error ) }
Breaks the input string into a vector of tokens.
Breaks the input string into a vector of tokens .
input
&str
span_offset
usize
input
&str
span_offset
usize