asm/lex.c
struct Token *lex(
        struct Context *ctx,
        void (*linecb)(struct Context*, struct Token*)
    ) {
    // first/current token and start of current line
    struct Token *first = asmalloc(sizeof(struct Token)),
                 *token = first;

    // first token is NOP, should be skipped on return
    *first = (struct Token) { .kind = TK_NOP };

    // pushes start to the token stream if it exists
    // TODO: this is kind of a garbage macro (complicated, modifies control flow,
    // etc.) but I can't think of a better solution at the moment.
#define lex_symbol()                                                    \
    if (start != NULL) {                                                \
        token = lex_data(                                               \
            ctx,                                                        \
            token,                                                      \
            TK_SYMBOL,                                                  \
            start,                                                      \
            ctx->current - start                                        \
        );                                                              \
        start = NULL;                                                   \
        if (expand_define(ctx, token)) {                                \
            token = token->prev;                                        \
            goto next_it;                                               \
        } else {                                                        \
            kind_symbol(token);                                         \
        }                                                               \
    }

    // start of current symbol being read
    const char *start = NULL;

    bool in_str = false, escaped = false;

    while (ctx->current != NULL) {
        // current character is escaped if prefixed with non-escaped backslash
        escaped =
            ctx->current != ctx->input->data &&
            ctx->current[-1] == '\\' &&
            (ctx->current == (ctx->input->data + 1) ||
             ctx->current[-2] != '\\');

        // read string literal if in_str
        if (in_str) {
            asmchk(*ctx->current != '\0', ctx, token, "Unfinished string");

            if (!escaped && *ctx->current == '\"') {
                in_str = false;
                token = lex_data(
                    ctx, token,
                    TK_STRING, start + 1, ctx->current - start - 1
                );
                start = NULL;
            }

            ctx->current++;
            continue;
        }

        switch (*ctx->current) {
            case '\0':
            case EOF:
                lex_symbol();
                pop_input(ctx);
                continue;
            case '\t':
            case ' ':
            case ',':
                lex_symbol();
                break;
            case ';':
                // EOL when comment (;) is encountered
                if (escaped) {
                    break;
                }
            case '\r':
            case '\n':
                lex_symbol();
                token = lex_value(ctx, token, TK_EOL);

                // seek to character after next newline
                ctx->current = strpbrk(ctx->current, "\n");
                ctx->current = ctx->current ?
                    (ctx->current + 1) :
                    (ctx->current + strlen(ctx->current) + 1);
                ctx->input->line = ctx->current;

                // trace backwards to find line start
                struct Token *line_start = token;
                while (line_start->prev->kind != TK_EOL &&
                        line_start->prev != first) {
                    line_start = line_start->prev;
                }

                if (linecb != NULL) {
                    linecb(ctx, line_start);
                }

                line_start = NULL;
                ctx->input->line_no++;
                continue;
            case '\"':
                if (!escaped) {
                    lex_symbol();
                    start = ctx->current;
                    in_str = true;
                    break;
                }
            case '\'':
                if (!escaped) {
                    lex_symbol();
                    const char *char_start = ctx->current;
                    ctx->current++;
                    asmchk(
                        isprint(*ctx->current), ctx, token,
                        "Expected continuation of character"
                    );
                    char v = lex_potentially_escaped(ctx, token);
                    asmchk(
                        *ctx->current == '\'', ctx, token,
                        "Expected end of character"
                    );
                    token = lex_push(ctx, token, (struct Token) {
                        .kind = TK_NUMBER,
                        .data = ctx->current,
                        .len = ctx->current - char_start + 1,
                        .line_index = char_start - ctx->input->line,
                        .value = v
                    });
                    break;
                }
            default:
                // try to read single character tokens
                if (!escaped) {
                    for (usize i = 0; i < TK_COUNT; i++) {
                        if (*ctx->current != TOKEN_VALUES[i]) {
                            continue;
                        }

                        lex_symbol();
                        token = lex_value(ctx, token, i);
                        start = NULL;
                        goto next_char;
                    }
                }

                // starting a new symbol
                if (start == NULL) {
                    start = ctx->current;
                }

                // check that this character is legal in a symbol
                asmchk(
                    isalpha(*ctx->current) ||
                        isdigit(*ctx->current) ||
                        *ctx->current == '_',
                    ctx,
                    token,
                    "Illegal character '%c' in symbol",
                    *ctx->current
                );
        }
next_char:
        ctx->current++;
next_it:
        continue;
    }

    // first->next to skip over initial NOP token
    return first->next;
}
// tokenizes whatever input is currently available in ctx, calling back into
// linecb() with the first token of every line read.
// returns a pointer to the start of the token stream, NULL if there are no
// tokens in the stream.
tokenizes whatever input is currently available in ctx, calling back into
linecb() with the first token of every line read.
returns a pointer to the start of the token stream, NULL if there are no
tokens in the stream.
tokenizes whatever input is currently available in ctx calling back into linecb () with the first token of every line read . returns a pointer to the start of the token stream NULL if there are no tokens in the stream .
struct Token *lex(
        struct Context *ctx,
        void (*linecb)(struct Context*, struct Token*)
    ) {
    struct Token *first = asmalloc(sizeof(struct Token)),
                 *token = first;
    *first = (struct Token) { .kind = TK_NOP };
#define lex_symbol()                                                    \
    if (start != NULL) {                                                \
        token = lex_data(                                               \
            ctx,                                                        \
            token,                                                      \
            TK_SYMBOL,                                                  \
            start,                                                      \
            ctx->current - start                                        \
        );                                                              \
        start = NULL;                                                   \
        if (expand_define(ctx, token)) {                                \
            token = token->prev;                                        \
            goto next_it;                                               \
        } else {                                                        \
            kind_symbol(token);                                         \
        }                                                               \
    }
    const char *start = NULL;
    bool in_str = false, escaped = false;
    while (ctx->current != NULL) {
        escaped =
            ctx->current != ctx->input->data &&
            ctx->current[-1] == '\\' &&
            (ctx->current == (ctx->input->data + 1) ||
             ctx->current[-2] != '\\');
        if (in_str) {
            asmchk(*ctx->current != '\0', ctx, token, "Unfinished string");
            if (!escaped && *ctx->current == '\"') {
                in_str = false;
                token = lex_data(
                    ctx, token,
                    TK_STRING, start + 1, ctx->current - start - 1
                );
                start = NULL;
            }
            ctx->current++;
            continue;
        }
        switch (*ctx->current) {
            case '\0':
            case EOF:
                lex_symbol();
                pop_input(ctx);
                continue;
            case '\t':
            case ' ':
            case ',':
                lex_symbol();
                break;
            case ';':
                if (escaped) {
                    break;
                }
            case '\r':
            case '\n':
                lex_symbol();
                token = lex_value(ctx, token, TK_EOL);
                ctx->current = strpbrk(ctx->current, "\n");
                ctx->current = ctx->current ?
                    (ctx->current + 1) :
                    (ctx->current + strlen(ctx->current) + 1);
                ctx->input->line = ctx->current;
                struct Token *line_start = token;
                while (line_start->prev->kind != TK_EOL &&
                        line_start->prev != first) {
                    line_start = line_start->prev;
                }
                if (linecb != NULL) {
                    linecb(ctx, line_start);
                }
                line_start = NULL;
                ctx->input->line_no++;
                continue;
            case '\"':
                if (!escaped) {
                    lex_symbol();
                    start = ctx->current;
                    in_str = true;
                    break;
                }
            case '\'':
                if (!escaped) {
                    lex_symbol();
                    const char *char_start = ctx->current;
                    ctx->current++;
                    asmchk(
                        isprint(*ctx->current), ctx, token,
                        "Expected continuation of character"
                    );
                    char v = lex_potentially_escaped(ctx, token);
                    asmchk(
                        *ctx->current == '\'', ctx, token,
                        "Expected end of character"
                    );
                    token = lex_push(ctx, token, (struct Token) {
                        .kind = TK_NUMBER,
                        .data = ctx->current,
                        .len = ctx->current - char_start + 1,
                        .line_index = char_start - ctx->input->line,
                        .value = v
                    });
                    break;
                }
            default:
                if (!escaped) {
                    for (usize i = 0; i < TK_COUNT; i++) {
                        if (*ctx->current != TOKEN_VALUES[i]) {
                            continue;
                        }
                        lex_symbol();
                        token = lex_value(ctx, token, i);
                        start = NULL;
                        goto next_char;
                    }
                }
                if (start == NULL) {
                    start = ctx->current;
                }
                asmchk(
                    isalpha(*ctx->current) ||
                        isdigit(*ctx->current) ||
                        *ctx->current == '_',
                    ctx,
                    token,
                    "Illegal character '%c' in symbol",
                    *ctx->current
                );
        }
next_char:
        ctx->current++;
next_it:
        continue;
    }
    return first->next;
}
struct Token * lex ( struct Context * ctx , void ( * linecb ) ( struct Context * , struct Token * ) ) { struct Token * first = asmalloc ( sizeof ( struct Token ) ) , * token = first ; * first = ( struct Token ) { . kind = TK_NOP } ; #define lex_symbol ( )                                                     \
    if (start != NULL) {                                                \
        token = lex_data(                                               \
            ctx,                                                        \
            token,                                                      \
            TK_SYMBOL,                                                  \
            start,                                                      \
            ctx->current - start                                        \
        );                                                              \
        start = NULL;                                                   \
        if (expand_define(ctx, token)) {                                \
            token = token->prev;                                        \
            goto next_it;                                               \
        } else {                                                        \
            kind_symbol(token);                                         \
        }                                                               \
    } const char * start = NULL ; bool in_str = false , escaped = false ; while ( ctx -> current != NULL ) { escaped = ctx -> current != ctx -> input -> data && ctx -> current [ -1 ] == ' \\ ' && ( ctx -> current == ( ctx -> input -> data + 1 ) || ctx -> current [ -2 ] != ' \\ ' ) ; if ( in_str ) { asmchk ( * ctx -> current != ' \0 ' , ctx , token , " " ) ; if ( ! escaped && * ctx -> current == ' \" ' ) { in_str = false ; token = lex_data ( ctx , token , TK_STRING , start + 1 , ctx -> current - start - 1 ) ; start = NULL ; } ctx -> current ++ ; continue ; } switch ( * ctx -> current ) { case ' \0 ' : case EOF : lex_symbol ( ) ; pop_input ( ctx ) ; continue ; case ' \t ' : case ' ' : case ' ' : lex_symbol ( ) ; break ; case ' ' : if ( escaped ) { break ; } case ' \r ' : case ' \n ' : lex_symbol ( ) ; token = lex_value ( ctx , token , TK_EOL ) ; ctx -> current = strpbrk ( ctx -> current , " \n " ) ; ctx -> current = ctx -> current ? ( ctx -> current + 1 ) : ( ctx -> current + strlen ( ctx -> current ) + 1 ) ; ctx -> input -> line = ctx -> current ; struct Token * line_start = token ; while ( line_start -> prev -> kind != TK_EOL && line_start -> prev != first ) { line_start = line_start -> prev ; } if ( linecb != NULL ) { linecb ( ctx , line_start ) ; } line_start = NULL ; ctx -> input -> line_no ++ ; continue ; case ' \" ' : if ( ! escaped ) { lex_symbol ( ) ; start = ctx -> current ; in_str = true ; break ; } case ' \' ' : if ( ! escaped ) { lex_symbol ( ) ; const char * char_start = ctx -> current ; ctx -> current ++ ; asmchk ( isprint ( * ctx -> current ) , ctx , token , " " ) ; char v = lex_potentially_escaped ( ctx , token ) ; asmchk ( * ctx -> current == ' \' ' , ctx , token , " " ) ; token = lex_push ( ctx , token , ( struct Token ) { . kind = TK_NUMBER , . data = ctx -> current , . len = ctx -> current - char_start + 1 , . line_index = char_start - ctx -> input -> line , . value = v } ) ; break ; } default : if ( ! escaped ) { for ( usize i = 0 ; i < TK_COUNT ; i ++ ) { if ( * ctx -> current != TOKEN_VALUES [ i ] ) { continue ; } lex_symbol ( ) ; token = lex_value ( ctx , token , i ) ; start = NULL ; goto next_char ; } } if ( start == NULL ) { start = ctx -> current ; } asmchk ( isalpha ( * ctx -> current ) || isdigit ( * ctx -> current ) || * ctx -> current == ' ' , ctx , token , " " , * ctx -> current ) ; } next_char : ctx -> current ++ ; next_it : continue ; } return first -> next ; }
tokenizes whatever input is currently available in ctx, calling back into
linecb() with the first token of every line read.
tokenizes whatever input is currently available in ctx calling back into linecb () with the first token of every line read .
// first/current token and start of current line // first token is NOP, should be skipped on return // pushes start to the token stream if it exists // TODO: this is kind of a garbage macro (complicated, modifies control flow, // etc.) but I can't think of a better solution at the moment. // start of current symbol being read // current character is escaped if prefixed with non-escaped backslash // read string literal if in_str // EOL when comment (;) is encountered // seek to character after next newline // trace backwards to find line start // try to read single character tokens // starting a new symbol // check that this character is legal in a symbol // first->next to skip over initial NOP token