fn lex_first_token(text: &str) -> Option<(Token, Option<SyntaxError>)> {
    // non-empty string is a precondtion of `rustc_lexer::first_token()`.
    if text.is_empty() {
        return None;
    }

    let rustc_token = rustc_lexer::first_token(text);
    let (syntax_kind, error) = rustc_token_kind_to_syntax_kind(&rustc_token.kind, text);

    let token = Token { kind: syntax_kind, len: TextUnit::from_usize(rustc_token.len) };
    let error = error.map(|error| {
        SyntaxError::new(
            SyntaxErrorKind::TokenizeError(error),
            TextRange::from_to(TextUnit::from(0), TextUnit::of_str(text)),
        )
    });

    Some((token, error))
}
/// Returns `SyntaxKind` and `Option<SyntaxError>` of the first token
/// encountered at the beginning of the string.
///
/// Returns `None` if the string contains zero tokens or if the token was parsed
/// with an error.
/// The token is malformed if the returned error is not `None`.
///
/// Beware that unescape errors are not checked at tokenization time.
Returns `SyntaxKind` and `Option` of the first token
encountered at the beginning of the string.

Returns `None` if the string contains zero tokens or if the token was parsed
with an error.
The token is malformed if the returned error is not `None`.

Beware that unescape errors are not checked at tokenization time.
Returns ` SyntaxKind ` and ` Option ` of the first token encountered at the beginning of the string . Returns ` None ` if the string contains zero tokens or if the token was parsed with an error . The token is malformed if the returned error is not ` None ` . Beware that unescape errors are not checked at tokenization time .
fn lex_first_token(text: &str) -> Option<(Token, Option<SyntaxError>)> {
    // non-empty string is a precondtion of `rustc_lexer::first_token()`.
    if text.is_empty() {
        return None;
    }
    let rustc_token = rustc_lexer::first_token(text);
    let (syntax_kind, error) = rustc_token_kind_to_syntax_kind(&rustc_token.kind, text);
    let token = Token { kind: syntax_kind, len: TextUnit::from_usize(rustc_token.len) };
    let error = error.map(|error| {
        SyntaxError::new(
            SyntaxErrorKind::TokenizeError(error),
            TextRange::from_to(TextUnit::from(0), TextUnit::of_str(text)),
        )
    });
    Some((token, error))
}
fn lex_first_token ( text : & str ) -> Option < ( Token , Option < SyntaxError > ) > { // non-empty string is a precondtion of `rustc_lexer::first_token()`. if text . is_empty ( ) { return None ; } let rustc_token = rustc_lexer :: first_token ( text ) ; let ( syntax_kind , error ) = rustc_token_kind_to_syntax_kind ( & rustc_token . kind , text ) ; let token = Token { kind : syntax_kind , len : TextUnit :: from_usize ( rustc_token . len ) } ; let error = error . map ( | error | { SyntaxError :: new ( SyntaxErrorKind :: TokenizeError ( error ) , TextRange :: from_to ( TextUnit :: from ( 0 ) , TextUnit :: of_str ( text ) ) , ) } ) ; Some ( ( token , error ) ) }
Returns `SyntaxKind` and `Option<SyntaxError>` of the first token
encountered at the beginning of the string.
Returns ` SyntaxKind ` and ` Option<SyntaxError > ` of the first token encountered at the beginning of the string .
text
&str