* remove old code

* remove old code

(%
 % rowl - generation 1
 % Copyright (C) 2010 nineties
 %
 % $Id: regalloc.rl 2010-04-02 21:57:54 nineties $
 %);

(% Register allocation %);

include(stddef, code);
export(regalloc);

conflicts: NULL;
equivregs: NULL;
input_count: NULL;
output_count: NULL;

(% p0: register, p1: is output %);
compute_score: (p0) {
    allocate(1);
    x0 = num_conflicts(p0) * 5;
    x0 = x0 + equivreg_score(p0) * 3;
    x0 = x0 + vec_at(input_count, p0[1]);
    x0 = x0 + vec_at(output_count, p0[1]) * 2;
    return x0;
};

(% p0: set of location-ids, p1: location %);
add_conflicts: (p0, p1) {
    allocate(3);
    if (p1 == NULL) { return; };
    if (is_constant_operand(p1)) { return; };
    x0 = p1[1];
    while (p0 != NULL) {
        x1 = ls_value(p0);
        if (x0 != x1) {
            puts("conflict: ");
            emit_opd(stdout, get_reg(x0), 32);
            puts(" <-> ");
            emit_opd(stdout, get_reg(x1), 32);
            putc('\n');
            x2 = iset_add(vec_at(conflicts, x0), x1);
            vec_put(conflicts, x0, x2);
            x2 = iset_add(vec_at(conflicts, x1), x0);
            vec_put(conflicts, x1, x2);
        };
        p0 = ls_next(p0);
    };
};

num_conflicts: (p0) {
    allocate(1);
    x0 = p0[1]; (% index %);
    return iset_size(vec_at(conflicts, x0));
};

(% p1: registers assigned to p0 %);
update_tables: (p0, p1) {
    allocate(3);
    x0 = p1;
    (% registers conflicting with p1 now be conflict with registers in p1 %);
    while (x0 != NULL) {
        x1 = vec_at(conflicts, p0[1]);
        add_conflicts(x1, ls_value(x0));
        x0 = ls_next(x0);
    };
    (% remove entries of p0 %);
    x0 = 0;
    x1 = num_locations();
    while (x0 < x1) {
        x2 = vec_at(conflicts, x0);
        x2 = iset_del(x2, p0[1]);
        vec_put(conflicts, x0, x2);

        x2 = vec_at(equivregs, x0);
        x2 = iset_del(x2, p0[1]);
        vec_put(equivregs, x0, x2);

        x0 = x0 + 1;
    };
};

equivreg_score: (p0) {
    allocate(3);
    x0 = p0[1]; (% index %);
    x1 = vec_at(equivregs, x0);
    x2 = 0;
    while (x1 != NULL) {
        if (p0[0] == OPD_PSEUDO) {
            x2 = x2 + 2;
            goto &next;
        };
        if (p0[0] == OPD_REGISTER) {
            x2 = x2 + 1;
            goto &next;
        };
        if (p0[0] == OPD_STACK) {
            x2 = x2 + 4;
            goto &next;
        };
        if (p0[0] == OPD_ARG) {
            x2 = x2 + 4;
            goto &next;
        };
        label next;
        x1 = ls_next(x1);
    };
    return x2;
};

add_equivregs: (p0, p1) {
    allocate(1);
    if (is_constant_operand(p0)) { return; };
    if (is_constant_operand(p1)) { return; };
    x0 = vec_at(equivregs, p0[1]);
    x0 = iset_add(x0, p1[1]);
    vec_put(equivregs, p0[1], x0);
};

incr_output_count: (p0) {
    allocate(1);
    if (p0 == NULL) { return; };
    if (is_constant_operand(p0)) { return; };
    x0 = vec_at(output_count, p0[1]);
    vec_size(output_count, p0[1], x0 + 1);
    return;
};

incr_input_count: (p0) {
    allocate(1);
    if (p0 == NULL) { return; };
    if (is_constant_operand(p0)) { return; };
    x0 = vec_at(input_count, p0[1]);
    vec_size(input_count, p0[1], x0 + 1);
    return;
};

(% p0: instructions %);
compute_conflicts: (p0) {
    allocate(3);
    conflicts = mkvec(num_locations());
    equivregs = mkvec(num_locations());
    input_count = mkvec(num_locations());
    output_count = mkvec(num_locations());
    while (p0 != NULL) {
        x0 = ls_value(p0); (% instruction %);
        x1 = x0[INST_LIVE]; (% live locations %);

        add_conflicts(x1, x0[INST_OPERAND2]);
        add_conflicts(x1, x0[INST_OPERAND1]);

        (% live registers in x1 conflict each other %);
        x2 = x1;
        while (x2 != NULL) {
            add_conflicts(x1, get_reg(ls_value(x2)));
            x2 = ls_next(x2);
        };

        if (x0[INST_OPCODE] == INST_MOVL) {
            add_equivregs(x0[INST_OPERAND1], x0[INST_OPERAND2]);
            add_equivregs(x0[INST_OPERAND2], x0[INST_OPERAND1]);
        };
        incr_output_count(x0[INST_OPERAND1]);
        incr_input_count(x0[INST_OPERAND2]);
        p0 = ls_next(p0);
    };
};

(% pickup pseudo register which have maximal conflicts %);
pickup_pseudo_reg: () {
    allocate(4);
    x0 = NULL;
    x1 = num_pseudo();
    x2 = 0;
    while (x2 < x1) {
        x3 = get_pseudo(x2);
        if (x3 != NULL) {
            if (x0 == NULL) {
                x0 = x3;
            } else {
                if (compute_score(x0) < compute_score(x3)) {
                    x0 = x3;
                };
            };
        };
        x2 = x2 + 1;
    };
    assert(x0 != NULL);
    return x0;
};

move_cost: (p0) {
    if (p0[0] == OPD_PSEUDO) { return 2; };
    if (p0[0] == OPD_REGISTER) { return 1; };
    if (p0[0] == OPD_STACK) { return 3; };
    if (p0[0] == OPD_ARG) { return 3; };
    if (p0[0] == OPD_AT) { return 3; };
    fputs(stderr, "ERROR: not reachable here\n");
    exit(1);
};

is_memory_access: (p0) {
    if (p0[0] == OPD_STACK) { return TRUE; };
    if (p0[0] == OPD_ARG)   { return TRUE; };
    return FALSE;
};

select_best_equivreg: (p0) {
    allocate(6);
    x0 = vec_at(conflicts, p0[1]);
    x1 = vec_at(equivregs, p0[1]);
    x2 = NULL; (% location with maximum cost %);
    x3 = NULL; (% register with maximum cost %);
    x5 = FALSE; (% TRUE if it must be a register %);
    while (x1 != NULL) {
        if (iset_contains(x0, ls_value(x1)) == FALSE) {
            if (ls_value(x1) != p0[1]) {
                x4 = get_reg(ls_value(x1));
                if (x3 == NULL) {
                    if (x4[0] == OPD_REGISTER) {
                        x3 = x4;
                    };
                };
                if (x2 == NULL) {
                    x2 = x4;
                } else {
                    if (is_memory_access(x2)) {
                        if (is_memory_access(x4)) {
                            x5 = TRUE;
                            goto &skip;
                        };
                    };
                    if (move_cost(x2) < move_cost(x4)) {
                        x2 = x4;
                    };
                };
            };
        };
        label skip;
        x1 = ls_next(x1);
    };
    if (x5) { return x3; };
    if (p0[PSEUDO_TYPE] == LOCATION_REGISTER) { return x3; };
    return x2;
};

select_location: (p0) {
    allocate(4);
    if (p0[PSEUDO_TYPE] == LOCATION_MEMORY) {
	goto &alloc_stackmem;
    };

    x0 = vec_at(conflicts, p0[1]);

    (% try to allocate register from equivregs %);
    x1 = select_best_equivreg(p0);
    if (x1 != NULL) { return ls_singleton(x1); };

    (% try to allocation register %);
    x1 = 0;
    while (x1 < num_normal_regs()) {
        if (iset_contains(x0, x1) == FALSE) {
            if (p0[1] != x1) {
                return ls_singleton(get_reg(x1));
            };
        };
        x1 = x1 + 1;
    };

    if (p0[PSEUDO_TYPE] == LOCATION_REGISTER) {
        fputs(stderr, "ERROR: failed to allocate register\n");
        exit(1);
    };

    label alloc_stackmem;

    if (p0[PSEUDO_LENGTH] == 1) {
        (% try to reuse stack memory %);
        x1 = 0;
        while (x1 < num_stack()) {
            x2 = get_stack(x1);
            if (iset_contains(x0, x2[1]) == FALSE) {
                return ls_singleton(x2);
            };
            x1 = x1 + 1;
        };

        (% allocate new stack memory %);
        (% here, we must resize working tables. %);
        vec_pushback(conflicts, NULL);
        vec_pushback(equivregs, NULL);
        vec_pushback(input_count, NULL);
        vec_pushback(output_count, NULL);
        return ls_singleton(get_stack(x1));
    } else {
        (% returns continuous memories %);
        x0 = p0[PSEUDO_LENGTH];
        x1 = get_stack_array(num_stack(), x0);
        vec_resize(conflicts, vec_size(conflicts) + x0);
        vec_resize(equivregs, vec_size(equivregs) + x0);
        vec_resize(input_count, vec_size(input_count) + x0);
        vec_resize(output_count, vec_size(output_count) + x0);
        return x1;
    };
};

assign_location: (p0) {
    allocate(2);
    x0 = select_location(p0);
    (% update register table %);
    puts(">> assign: ");
    emit_opd(stdout, p0, 32);
    puts(" <-");
    x1 = x0;
    while (x1 != NULL) {
        putc(' ');
        emit_opd(stdout, ls_value(x1), 32);
        x1 = ls_next(x1);
    };
    putc('\n');

    assign_pseudo(p0, x0);
    (% update conflicts/equivregs %);
    update_tables(p0, x0);

    (% update output_count/input_count %);
    while (x0 != NULL) {
        vec_put(output_count, (ls_value(x0))[1],
            vec_at(output_count, p0[1]) + vec_at(output_count, (ls_value(x0))[1]));
        vec_put(input_count, (ls_value(x0))[1],
            vec_at(input_count, p0[1]) + vec_at(input_count, (ls_value(x0))[1]));
        x0 = ls_next(x0);
    };
};

assign_locations: () {
    allocate(2);
    x0 = 0;
    x1 = num_pseudo();
    while (x0 < x1) {
        assign_location(pickup_pseudo_reg());
	x0 = x0 + 1;
    };
};

(% replace pseudo-reg to physical register or stack memory %);
replace: (p0) {
    allocate(2);
    if (p0 == NULL) { return NULL; };
    if (p0[0] == OPD_PSEUDO) {
        return replace(ls_value(p0[PSEUDO_LOCATION]));
    };
    if (p0[0] == OPD_AT) {
        x0 = p0[2];
        x1 = p0[3]; (% index %);
        if (x0[0] == OPD_STACK) {
            return get_stack(x0[STACK_OFFSET] + x1);
        };
        if (x0[0] == OPD_PSEUDO) {
            return replace(ls_at(x0[PSEUDO_LOCATION], x1));
        };
        fputs(stderr, "ERROR: not reachable here\n");
        exit(1);
    };
    return p0;
};

update_instructions: (p0) {
    allocate(2);
    if (p0 == NULL) { return NULL; };
    x0 = ls_value(p0);
    x0[INST_OPERAND1] = replace(x0[INST_OPERAND1]);
    x0[INST_OPERAND2]  = replace(x0[INST_OPERAND2]);

    (% eliminate meaningless move %);
    if (x0[INST_OPCODE] == INST_MOVL) {
        if (x0[INST_OPERAND1] == x0[INST_OPERAND2]) {
            return update_instructions(ls_next(p0));
        };
    };

    x1 = ls_cons(x0, update_instructions(ls_next(p0)));

    (% insert leave instruction %);
    if (x0[INST_OPCODE] == INST_RET) {
        x1 = ls_cons(mkinst(INST_LEAVE, NULL, NULL), x1);
    };
    return x1;
};

allocate_stack_frame: (p0) {
    allocate(1);
    x0 = p0;
    if (num_stack() > 0) {
        x0 = ls_cons(mkinst(INST_SUBL, mktup2(OPD_INTEGER, 4*num_stack()), get_esp()), x0);
    };
    return
        ls_cons(mkinst(INST_PUSHL, get_ebp(), NULL),
        ls_cons(mkinst(INST_MOVL, get_esp(), get_ebp()),
            x0));
};

(% p0: TCODE_FUNC object %);
regalloc: (p0) {
    allocate(1);

    (% liveness analysis %);
    liveness(p0);

    x0 = p0[3]; (% instructions %);
    if (num_pseudo() > 0) {
        compute_conflicts(x0);
        assign_locations();
    };
    x0 = update_instructions(x0);
    x0 = allocate_stack_frame(x0);
    p0[3] = x0;
};

(%
 % rowl - generation 1
 % Copyright (C) 2010 nineties
 %
 % $Id: regalloc.rl 2010-04-02 16:40:39 nineties $
 %);

(% Register allocation %);

include(stddef, code);
export(regalloc);

conflicts: NULL;
equivregs: NULL;
input_count: NULL;
output_count: NULL;

(% p0: register, p1: is output %);
compute_score: (p0) {
    allocate(1);
    x0 = num_conflicts(p0) * 5;
    x0 = x0 + equivreg_score(p0) * 3;
    x0 = x0 + vec_at(input_count, p0[1]);
    x0 = x0 + vec_at(output_count, p0[1]) * 2;
    return x0;
};

(% p0: set of location-ids, p1: location %);
add_conflicts: (p0, p1) {
    allocate(3);
    if (p1 == NULL) { return; };
    if (is_constant_operand(p1)) { return; };
    x0 = p1[1];
    while (p0 != NULL) {
        x1 = ls_value(p0);
        if (x0 != x1) {
            puts("conflict: ");
            emit_opd(stdout, get_reg(x0), 32);
            puts(" <-> ");
            emit_opd(stdout, get_reg(x1), 32);
            putc('\n');
            x2 = iset_add(vec_at(conflicts, x0), x1);
            vec_put(conflicts, x0, x2);
            x2 = iset_add(vec_at(conflicts, x1), x0);
            vec_put(conflicts, x1, x2);
        };
        p0 = ls_next(p0);
    };
};

num_conflicts: (p0) {
    allocate(1);
    x0 = p0[1]; (% index %);
    return iset_size(vec_at(conflicts, x0));
};

(% p1: registers assigned to p0 %);
update_tables: (p0, p1) {
    allocate(3);
    x0 = p1;
    (% registers conflicting with p1 now be conflict with registers in p1 %);
    while (x0 != NULL) {
        x1 = vec_at(conflicts, p0[1]);
        add_conflicts(x1, ls_value(x0));
        x0 = ls_next(x0);
    };
    (% remove entries of p0 %);
    x0 = 0;
    x1 = num_locations();
    while (x0 < x1) {
        x2 = vec_at(conflicts, x0);
        x2 = iset_del(x2, p0[1]);
        vec_put(conflicts, x0, x2);

        x2 = vec_at(equivregs, x0);
        x2 = iset_del(x2, p0[1]);
        vec_put(equivregs, x0, x2);

        x0 = x0 + 1;
    };
};

equivreg_score: (p0) {
    allocate(3);
    x0 = p0[1]; (% index %);
    x1 = vec_at(equivregs, x0);
    x2 = 0;
    while (x1 != NULL) {
        if (p0[0] == OPD_PSEUDO) {
            x2 = x2 + 2;
            goto &next;
        };
        if (p0[0] == OPD_REGISTER) {
            x2 = x2 + 1;
            goto &next;
        };
        if (p0[0] == OPD_STACK) {
            x2 = x2 + 4;
            goto &next;
        };
        if (p0[0] == OPD_ARG) {
            x2 = x2 + 4;
            goto &next;
        };
        label next;
        x1 = ls_next(x1);
    };
    return x2;
};

add_equivregs: (p0, p1) {
    allocate(1);
    if (is_constant_operand(p0)) { return; };
    if (is_constant_operand(p1)) { return; };
    x0 = vec_at(equivregs, p0[1]);
    x0 = iset_add(x0, p1[1]);
    vec_put(equivregs, p0[1], x0);
};

incr_output_count: (p0) {
    allocate(1);
    if (p0 == NULL) { return; };
    if (is_constant_operand(p0)) { return; };
    x0 = vec_at(output_count, p0[1]);
    vec_size(output_count, p0[1], x0 + 1);
    return;
};

incr_input_count: (p0) {
    allocate(1);
    if (p0 == NULL) { return; };
    if (is_constant_operand(p0)) { return; };
    x0 = vec_at(input_count, p0[1]);
    vec_size(input_count, p0[1], x0 + 1);
    return;
};

(% p0: instructions %);
compute_conflicts: (p0) {
    allocate(3);
    conflicts = mkvec(num_locations());
    equivregs = mkvec(num_locations());
    input_count = mkvec(num_locations());
    output_count = mkvec(num_locations());
    while (p0 != NULL) {
        x0 = ls_value(p0); (% instruction %);
        x1 = x0[INST_LIVE]; (% live locations %);

        add_conflicts(x1, x0[INST_OPERAND2]);
        add_conflicts(x1, x0[INST_OPERAND1]);

        (% live registers in x1 conflict each other %);
        x2 = x1;
        while (x2 != NULL) {
            add_conflicts(x1, get_reg(ls_value(x2)));
            x2 = ls_next(x2);
        };

        if (x0[INST_OPCODE] == INST_MOVL) {
            add_equivregs(x0[INST_OPERAND1], x0[INST_OPERAND2]);
            add_equivregs(x0[INST_OPERAND2], x0[INST_OPERAND1]);
        };
        incr_output_count(x0[INST_OPERAND1]);
        incr_input_count(x0[INST_OPERAND2]);
        p0 = ls_next(p0);
    };
};

(% pickup pseudo register which have maximal conflicts %);
pickup_pseudo_reg: () {
    allocate(4);
    x0 = NULL;
    x1 = num_pseudo();
    x2 = 0;
    while (x2 < x1) {
        x3 = get_pseudo(x2);
        if (x3 != NULL) {
            if (x0 == NULL) {
                x0 = x3;
            } else {
                if (compute_score(x0) < compute_score(x3)) {
                    x0 = x3;
                };
            };
        };
        x2 = x2 + 1;
    };
    assert(x0 != NULL);
    return x0;
};

move_cost: (p0) {
    if (p0[0] == OPD_PSEUDO) { return 2; };
    if (p0[0] == OPD_REGISTER) { return 1; };
    if (p0[0] == OPD_STACK) { return 3; };
    if (p0[0] == OPD_ARG) { return 3; };
    if (p0[0] == OPD_AT) { return 3; };
    fputs(stderr, "ERROR: not reachable here\n");
    exit(1);
};

is_memory_access: (p0) {
    if (p0[0] == OPD_STACK) { return TRUE; };
    if (p0[0] == OPD_ARG)   { return TRUE; };
    return FALSE;
};

select_best_equivreg: (p0) {
    allocate(6);
    x0 = vec_at(conflicts, p0[1]);
    x1 = vec_at(equivregs, p0[1]);
    x2 = NULL; (% location with maximum cost %);
    x3 = NULL; (% register with maximum cost %);
    x5 = FALSE; (% TRUE if it must be a register %);
    while (x1 != NULL) {
        if (iset_contains(x0, ls_value(x1)) == FALSE) {
            if (ls_value(x1) != p0[1]) {
                x4 = get_reg(ls_value(x1));
                if (x3 == NULL) {
                    if (x4[0] == OPD_REGISTER) {
                        x3 = x4;
                    };
                };
                if (x2 == NULL) {
                    x2 = x4;
                } else {
                    if (is_memory_access(x2)) {
                        if (is_memory_access(x4)) {
                            x5 = TRUE;
                            goto &skip;
                        };
                    };
                    if (move_cost(x2) < move_cost(x4)) {
                        x2 = x4;
                    };
                };
            };
        };
        label skip;
        x1 = ls_next(x1);
    };
    if (x5) { return x3; };
    if (p0[PSEUDO_TYPE] == LOCATION_REGISTER) { return x3; };
    return x2;
};

select_location: (p0) {
    allocate(4);
    if (p0[PSEUDO_TYPE] == LOCATION_MEMORY) {
	goto &alloc_stackmem;
    };

    x0 = vec_at(conflicts, p0[1]);

    (% try to allocate register from equivregs %);
    x1 = select_best_equivreg(p0);
    if (x1 != NULL) { return ls_singleton(x1); };

    (% try to allocation register %);
    x1 = 0;
    while (x1 < num_normal_regs()) {
        if (iset_contains(x0, x1) == FALSE) {
            if (p0[1] != x1) {
                return ls_singleton(get_reg(x1));
            };
        };
        x1 = x1 + 1;
    };

    if (p0[PSEUDO_TYPE] == LOCATION_REGISTER) {
        fputs(stderr, "ERROR: failed to allocate register\n");
        exit(1);
    };

    label alloc_stackmem;

    if (p0[PSEUDO_LENGTH] == 1) {
        (% try to reuse stack memory %);
        x1 = 0;
        while (x1 < num_stack()) {
            x2 = get_stack(x1);
            if (iset_contains(x0, x2[1]) == FALSE) {
                return ls_singleton(x2);
            };
            x1 = x1 + 1;
        };

        (% allocate new stack memory %);
        (% here, we must resize working tables. %);
        vec_pushback(conflicts, NULL);
        vec_pushback(equivregs, NULL);
        vec_pushback(input_count, NULL);
        vec_pushback(output_count, NULL);
        return ls_singleton(get_stack(x1));
    } else {
        (% returns continuous memories %);
        x0 = p0[PSEUDO_LENGTH];
        x1 = get_stack_array(num_stack(), x0);
        vec_resize(conflicts, vec_size(conflicts) + x0);
        vec_resize(equivregs, vec_size(equivregs) + x0);
        vec_resize(input_count, vec_size(input_count) + x0);
        vec_resize(output_count, vec_size(output_count) + x0);
        return x1;
    };
};

assign_location: (p0) {
    allocate(5);
    x0 = select_location(p0);
    (% update register table %);
    puts(">> assign: ");
    emit_opd(stdout, p0, 32);
    puts(" <-");
    x1 = x0;
    while (x1 != NULL) {
        putc(' ');
        emit_opd(stdout, ls_value(x1), 32);
        x1 = ls_next(x1);
    };
    putc('\n');

    assign_pseudo(p0, x0);
    (% update conflicts/equivregs %);
    update_tables(p0, x0);
    (%
    x1 = 0;
    x2 = num_locations();
    while (x1 < x2) {
        x4 = x0;
        while (x4 != NULL) {
            x3 = vec_at(conflicts, x1);
            if (iset_contains(x3, p0[1])) {
                x3 = iset_del(x3, p0[1]);
                x3 = iset_add(x3, (ls_value(x4))[1]);
                vec_put(conflicts, x1, x3);

                puts("update conflict: ");
                emit_opd(stdout, ls_value(x4), 32);
                puts(" <-> ");
                emit_opd(stdout, get_reg(x1), 32);
                putc('\n');
            };

            x3 = vec_at(equivregs, x1);
            if (iset_contains(x3, p0[1])) {
                x3 = iset_del(x3, p0[1]);
                x3 = iset_add(x3, (ls_value(x4))[1]);
                vec_put(equivregs, x1, x3);
            };
            x4 = ls_next(x4);
        };

        x1 = x1 + 1;
    };
    %);

    (% update output_count/input_count %);
    while (x0 != NULL) {
        vec_put(output_count, (ls_value(x0))[1],
            vec_at(output_count, p0[1]) + vec_at(output_count, (ls_value(x0))[1]));
        vec_put(input_count, (ls_value(x0))[1],
            vec_at(input_count, p0[1]) + vec_at(input_count, (ls_value(x0))[1]));
        x0 = ls_next(x0);
    };
};

assign_locations: () {
    allocate(2);
    x0 = 0;
    x1 = num_pseudo();
    while (x0 < x1) {
        assign_location(pickup_pseudo_reg());
	x0 = x0 + 1;
    };
};

(% replace pseudo-reg to physical register or stack memory %);
replace: (p0) {
    allocate(2);
    if (p0 == NULL) { return NULL; };
    if (p0[0] == OPD_PSEUDO) {
        return replace(ls_value(p0[PSEUDO_LOCATION]));
    };
    if (p0[0] == OPD_AT) {
        x0 = p0[2];
        x1 = p0[3]; (% index %);
        if (x0[0] == OPD_STACK) {
            return get_stack(x0[STACK_OFFSET] + x1);
        };
        if (x0[0] == OPD_PSEUDO) {
            return replace(ls_at(x0[PSEUDO_LOCATION], x1));
        };
        fputs(stderr, "ERROR: not reachable here\n");
        exit(1);
    };
    return p0;
};

update_instructions: (p0) {
    allocate(2);
    if (p0 == NULL) { return NULL; };
    x0 = ls_value(p0);
    x0[INST_OPERAND1] = replace(x0[INST_OPERAND1]);
    x0[INST_OPERAND2]  = replace(x0[INST_OPERAND2]);

    (% eliminate meaningless move %);
    if (x0[INST_OPCODE] == INST_MOVL) {
        if (x0[INST_OPERAND1] == x0[INST_OPERAND2]) {
            return update_instructions(ls_next(p0));
        };
    };

    x1 = ls_cons(x0, update_instructions(ls_next(p0)));

    (% insert leave instruction %);
    if (x0[INST_OPCODE] == INST_RET) {
        x1 = ls_cons(mkinst(INST_LEAVE, NULL, NULL), x1);
    };
    return x1;
};

allocate_stack_frame: (p0) {
    allocate(1);
    x0 = p0;
    if (num_stack() > 0) {
        x0 = ls_cons(mkinst(INST_SUBL, mktup2(OPD_INTEGER, 4*num_stack()), get_esp()), x0);
    };
    return
        ls_cons(mkinst(INST_PUSHL, get_ebp(), NULL),
        ls_cons(mkinst(INST_MOVL, get_esp(), get_ebp()),
            x0));
};

(% p0: TCODE_FUNC object %);
regalloc: (p0) {
    allocate(1);

    (% liveness analysis %);
    liveness(p0);

    x0 = p0[3]; (% instructions %);
    if (num_pseudo() > 0) {
        compute_conflicts(x0);
        assign_locations();
    };
    x0 = update_instructions(x0);
    x0 = allocate_stack_frame(x0);
    p0[3] = x0;
};


Treat unexpected state as such

Treat unexpected state as such

– Currently, the parser should not end up with “both gaps” input when handling match.

/*
 * Copyright (c) 2019 Tuukka Norri
 * This code is licensed under MIT license (see LICENSE for details).
 */

#include <stdexcept>
#include "msa_combiner.hh"

%% machine alignment_parser;
%% write data;

namespace vcf2multialign {

	void msa_combiner::prepare_msa_parser()
	{
		%% write init;
	}

	void msa_combiner::parse_msa(aligned_character_pack const &pack)
	{
		%%{
			variable cs		m_fsm.cs;
			variable p		m_fsm.p;
			variable pe		m_fsm.pe;
			variable eof	m_fsm.pe;

			nt			= [ACGTN];	# Add [acgtn] to allow lowecase.
			ntg			= nt | '-';

			a_			= [A];		# Add [a] etc. to allow lowercase.
			c_			= [C];
			g_			= [G];
			t_			= [T];
			n_			= [N];
			not_a		= nt - a_;
			not_c		= nt - c_;
			not_g		= nt - g_;
			not_t		= nt - t_;
			not_n		= nt - n_;

			same		= /AA/ | /CC/ | /GG/ | /TT/ | /NN/;	# Add /i to allow lowercase.
			diff		= (a_ . not_a) | (c_ . not_c) | (g_ . not_g) | (t_ . not_t) | (n_ . not_n);
			both_nt		= nt{2};				# Any two non-gap.

			action deletion_continue_r {
				m_current_segment.ref.string += pack.ref.character;
				fnext handle_deletion;
				fbreak;
			}

			action main_new_deletion_r {
				m_current_segment.reset(pack, segment_type::DELETION);
				m_current_segment.ref.string += pack.ref.character;
				fnext handle_deletion;
				fbreak;
			}

			action main_new_mismatch_b {
				m_current_segment.reset(pack, segment_type::MISMATCH);
				m_current_segment.ref.string += pack.ref.character;
				m_current_segment.alt.string += pack.alt.character;
				fnext handle_mismatch;
				fbreak;
			}

			action main_new_mixed_b {
				m_current_segment.reset(pack, segment_type::MIXED);
				m_current_segment.ref.string += pack.ref.character;
				m_current_segment.alt.string += pack.alt.character;
				fnext handle_mixed;
				fbreak;
			}

			action main_new_mixed_r {
				m_current_segment.reset(pack, segment_type::MIXED);
				m_current_segment.ref.string += pack.ref.character;
				fnext handle_mixed;
				fbreak;
			}

			action main_new_match_b {
				m_current_segment.reset(pack, segment_type::MATCH);
				m_current_segment.ref.string += pack.ref.character;
				fnext handle_match;
				fbreak;
			}

			action match_continue_b {
				m_current_segment.ref.string += pack.ref.character;
				fnext handle_match;
				fbreak;
			}

			action match_continue_g {
				fnext handle_match;
				fbreak;
			}

			action mismatch_continue_b {
				m_current_segment.ref.string += pack.ref.character;
				m_current_segment.alt.string += pack.alt.character;
				fnext handle_mismatch;
				fbreak;
			}

			action mismatch_continue_g {
				fnext handle_mismatch;
				fbreak;
			}

			action mismatch_mark_mixed_a {
				m_current_segment.type = segment_type::MIXED;
				m_current_segment.alt.string += pack.alt.character;
				fnext handle_mixed;
				fbreak;
			}

			action mixed_continue_a {
				m_current_segment.alt.string += pack.alt.character;
				fnext handle_mixed;
				fbreak;
			}

			action mixed_continue_g {
				fnext handle_mixed;
				fbreak;
			}

			action mixed_continue_r {
				m_current_segment.ref.string += pack.ref.character;
				fnext handle_mixed;
				fbreak;
			}

			action start_new_del_r {
				push_current_segment();
				m_current_segment.reset(pack, segment_type::DELETION);
				m_current_segment.ref.string += pack.ref.character;
				fnext handle_deletion;
				fbreak;
			}

			action start_new_match_b {
				push_current_segment();
				m_current_segment.reset(pack, segment_type::MATCH);
				m_current_segment.ref.string += pack.ref.character;
				fnext handle_match;
				fbreak;
			}

			action start_new_mismatch_b {
				push_current_segment();
				m_current_segment.reset(pack, segment_type::MISMATCH);
				m_current_segment.ref.string += pack.ref.character;
				m_current_segment.alt.string += pack.alt.character;
				fnext handle_mismatch;
				fbreak;
			}

			action start_new_mixed_b {
				push_current_segment();
				m_current_segment.reset(pack, segment_type::MIXED);
				m_current_segment.ref.string += pack.ref.character;
				m_current_segment.alt.string += pack.alt.character;
				fnext handle_mixed;
				fbreak;
			}

			action start_new_mixed_r {
				push_current_segment();
				m_current_segment.reset(pack, segment_type::MIXED);
				m_current_segment.ref.string += pack.ref.character;
				fnext handle_mixed;
				fbreak;
			}

			action handle_error {
				throw std::runtime_error("Unexpected character");
			}

			action handle_unexpected_state {
				throw std::runtime_error("Unexpected state");
			}

			# The following machine definitions should handle all combinations of four characters.
			# “Same” or “matching” segment refers to the property that for every alt non-gap character there is a matching ref character.
			# Otherwise the segment is considered non-matching. In some cases, this property cannot be determined by looking ahead just one character.

			# Ref	X-	X-	X-	X-
			# Alt	X-	X+	Y*	-*
			same_next_both_g		= (same . '--');
			same_next_ref_g			= (same . '-' . nt);
			diff_next_ref_g			= (diff . '-' . ntg);
			alt_g_next_ref_g		= (nt . '--' . ntg);

			# Ref	X+
			# Alt	-*
			alt_g_next_ref_c		= (nt . '-' . nt . ntg);

			# Ref	X+	X+
			# Alt	X*	Y*
			# Non-gap alt characters consistently match / do not match ref.
			same_next_ref_nt		= (same . nt . ntg);
			diff_next_ref_nt		= (diff . nt . ntg);

			# Ref	-	-
			# Alt	-	X
			# ref_g may come after same_next_ref_g, diff_next_ref_g, alt_g_next_ref_g.
			both_g					= ('--');
			ref_g					= ('-' . nt);

			# Make a static assertion fail if there are any character combinations that were not listed above.
			action report_unhandled {
				static_assert(false, "Found a character combination that was not handled.");
			}
			unhandled = ((ntg{4}) - (
				same_next_both_g		|
				same_next_ref_g			|
				diff_next_ref_g			|
				alt_g_next_ref_g		|

				alt_g_next_ref_c		|

				same_next_ref_nt		|
				diff_next_ref_nt		|

				(both_g . ntg{2})		|	# Corresponds to both_g in the machines below but would match more without the ntg{2}.
				(ref_g . ntg{2})			# Corresponds to ref_g in the machines below but would match more without the ntg{2}.
			));
			fail_on_unhandled_combinations := unhandled $(report_unhandled);

			# In the following cases start a new segment at the marked position b.c.
			# the following characters cannot be known and this reduces the number
			# of special cases a bit (1).
			# Ref	ACG-	ACG-
			# Alt	ACG-	TTG-
			#		  ^		  ^
			# Action suffixes: _b: both have non-gap, _r: ref has non-gap, _a: alt has non-gap, _g: both have gap.

			common = (
				# Since the next ref character is a gap, a new segment needs to be created since we don’t know the subsequent characters.
				(same_next_both_g	@(start_new_mixed_b))		|	# We currently don’t change a matching segment to mixed, so mark the segment for post-processing here.
				(same_next_ref_g	@(start_new_mixed_b))		|
				(diff_next_ref_g	@(start_new_mismatch_b))	|
				(alt_g_next_ref_g	@(start_new_mixed_r))
			);

			handle_match := common | (
				(alt_g_next_ref_c	@(start_new_del_r))			|	# Deletion in alt w.r.t. ref.

				(same_next_ref_nt	@(match_continue_b))		|
				(diff_next_ref_nt	@(start_new_mismatch_b))	|

				(both_g				@(handle_unexpected_state))	|
				(ref_g				@(handle_unexpected_state))
			) $err(handle_error);

			handle_mismatch := common | (
				(alt_g_next_ref_c	@(start_new_del_r))			|	# Deletion in alt w.r.t. ref.

				(same_next_ref_nt	@(start_new_match_b))		|
				(diff_next_ref_nt	@(mismatch_continue_b))		|

				(both_g				@(mismatch_continue_g))		|
				(ref_g				@(mismatch_mark_mixed_a))
			) $err(handle_error);

			handle_mixed := common | (
				(alt_g_next_ref_c	@(start_new_del_r))			|	# Deletion in alt w.r.t. ref.

				(same_next_ref_nt	@(start_new_match_b))		|
				(diff_next_ref_nt	@(start_new_mismatch_b))	|

				(both_g				@(mixed_continue_g))		|
				(ref_g				@(mixed_continue_a))
			) $err(handle_error);

			handle_deletion := common | (
				(alt_g_next_ref_c	@(deletion_continue_r))		|

				(same_next_ref_nt	@(start_new_match_b))		|
				(diff_next_ref_nt	@(start_new_mismatch_b))	|

				(both_g				@(handle_unexpected_state))	|
				(ref_g				@(handle_unexpected_state))
			) $err(handle_error);

			# Starting machines. Ref is guaranteed to be non-gap. See common.
			# Ref	X-	X-	X-	X-	X+	X+	X+
			# Alt	X-	X+	Y*	-*	-*	X*	Y*
			#same_next_both_g
			#same_next_ref_g
			#diff_next_ref_g
			#alt_g_next_ref_g
			#alt_g_next_ref_c
			#same_next_ref_nt
			#diff_next_ref_nt
			main_unexpected = ntg{4} - (same_next_both_g | same_next_ref_g | diff_next_ref_g | alt_g_next_ref_g | alt_g_next_ref_c | same_next_ref_nt | diff_next_ref_nt);

			main := (
				(same_next_both_g	@(main_new_match_b))		|
				(same_next_ref_g	@(main_new_mixed_b))		|
				(diff_next_ref_g	@(main_new_mismatch_b))		|
				(alt_g_next_ref_g	@(main_new_mixed_r))		|
				(alt_g_next_ref_c	@(main_new_deletion_r))		|
				(same_next_ref_nt	@(main_new_match_b))		|
				(diff_next_ref_nt	@(main_new_mismatch_b))		|
				(main_unexpected	@(handle_unexpected_state))
			) $err(handle_error);

			write exec noend;
		}%%
	}
}

/*
 * Copyright (c) 2019 Tuukka Norri
 * This code is licensed under MIT license (see LICENSE for details).
 */

#include <stdexcept>
#include "msa_combiner.hh"

%% machine alignment_parser;
%% write data;

namespace vcf2multialign {

	void msa_combiner::prepare_msa_parser()
	{
		%% write init;
	}

	void msa_combiner::parse_msa(aligned_character_pack const &pack)
	{
		%%{
			variable cs		m_fsm.cs;
			variable p		m_fsm.p;
			variable pe		m_fsm.pe;
			variable eof	m_fsm.pe;

			nt			= [ACGTN];	# Add [acgtn] to allow lowecase.
			ntg			= nt | '-';

			a_			= [A];		# Add [a] etc. to allow lowercase.
			c_			= [C];
			g_			= [G];
			t_			= [T];
			n_			= [N];
			not_a		= nt - a_;
			not_c		= nt - c_;
			not_g		= nt - g_;
			not_t		= nt - t_;
			not_n		= nt - n_;

			same		= /AA/ | /CC/ | /GG/ | /TT/ | /NN/;	# Add /i to allow lowercase.
			diff		= (a_ . not_a) | (c_ . not_c) | (g_ . not_g) | (t_ . not_t) | (n_ . not_n);
			both_nt		= nt{2};				# Any two non-gap.

			action deletion_continue_r {
				m_current_segment.ref.string += pack.ref.character;
				fnext handle_deletion;
				fbreak;
			}

			action main_new_deletion_r {
				m_current_segment.reset(pack, segment_type::DELETION);
				m_current_segment.ref.string += pack.ref.character;
				fnext handle_deletion;
				fbreak;
			}

			action main_new_mismatch_b {
				m_current_segment.reset(pack, segment_type::MISMATCH);
				m_current_segment.ref.string += pack.ref.character;
				m_current_segment.alt.string += pack.alt.character;
				fnext handle_mismatch;
				fbreak;
			}

			action main_new_mixed_b {
				m_current_segment.reset(pack, segment_type::MIXED);
				m_current_segment.ref.string += pack.ref.character;
				m_current_segment.alt.string += pack.alt.character;
				fnext handle_mixed;
				fbreak;
			}

			action main_new_mixed_r {
				m_current_segment.reset(pack, segment_type::MIXED);
				m_current_segment.ref.string += pack.ref.character;
				fnext handle_mixed;
				fbreak;
			}

			action main_new_match_b {
				m_current_segment.reset(pack, segment_type::MATCH);
				m_current_segment.ref.string += pack.ref.character;
				fnext handle_match;
				fbreak;
			}

			action match_continue_b {
				m_current_segment.ref.string += pack.ref.character;
				fnext handle_match;
				fbreak;
			}

			action match_continue_g {
				fnext handle_match;
				fbreak;
			}

			action mismatch_continue_b {
				m_current_segment.ref.string += pack.ref.character;
				m_current_segment.alt.string += pack.alt.character;
				fnext handle_mismatch;
				fbreak;
			}

			action mismatch_continue_g {
				fnext handle_mismatch;
				fbreak;
			}

			action mismatch_mark_mixed_a {
				m_current_segment.type = segment_type::MIXED;
				m_current_segment.alt.string += pack.alt.character;
				fnext handle_mixed;
				fbreak;
			}

			action mixed_continue_a {
				m_current_segment.alt.string += pack.alt.character;
				fnext handle_mixed;
				fbreak;
			}

			action mixed_continue_g {
				fnext handle_mixed;
				fbreak;
			}

			action mixed_continue_r {
				m_current_segment.ref.string += pack.ref.character;
				fnext handle_mixed;
				fbreak;
			}

			action start_new_del_r {
				push_current_segment();
				m_current_segment.reset(pack, segment_type::DELETION);
				m_current_segment.ref.string += pack.ref.character;
				fnext handle_deletion;
				fbreak;
			}

			action start_new_match_b {
				push_current_segment();
				m_current_segment.reset(pack, segment_type::MATCH);
				m_current_segment.ref.string += pack.ref.character;
				fnext handle_match;
				fbreak;
			}

			action start_new_mismatch_b {
				push_current_segment();
				m_current_segment.reset(pack, segment_type::MISMATCH);
				m_current_segment.ref.string += pack.ref.character;
				m_current_segment.alt.string += pack.alt.character;
				fnext handle_mismatch;
				fbreak;
			}

			action start_new_mixed_b {
				push_current_segment();
				m_current_segment.reset(pack, segment_type::MIXED);
				m_current_segment.ref.string += pack.ref.character;
				m_current_segment.alt.string += pack.alt.character;
				fnext handle_mixed;
				fbreak;
			}

			action start_new_mixed_r {
				push_current_segment();
				m_current_segment.reset(pack, segment_type::MIXED);
				m_current_segment.ref.string += pack.ref.character;
				fnext handle_mixed;
				fbreak;
			}

			action handle_error {
				throw std::runtime_error("Unexpected character");
			}

			action handle_unexpected_state {
				throw std::runtime_error("Unexpected state");
			}

			# The following machine definitions should handle all combinations of four characters.
			# “Same” or “matching” segment refers to the property that for every alt non-gap character there is a matching ref character.
			# Otherwise the segment is considered non-matching. In some cases, this property cannot be determined by looking ahead just one character.

			# Ref	X-	X-	X-	X-
			# Alt	X-	X+	Y*	-*
			same_next_both_g		= (same . '--');
			same_next_ref_g			= (same . '-' . nt);
			diff_next_ref_g			= (diff . '-' . ntg);
			alt_g_next_ref_g		= (nt . '--' . ntg);

			# Ref	X+
			# Alt	-*
			alt_g_next_ref_c		= (nt . '-' . nt . ntg);

			# Ref	X+	X+
			# Alt	X*	Y*
			# Non-gap alt characters consistently match / do not match ref.
			same_next_ref_nt		= (same . nt . ntg);
			diff_next_ref_nt		= (diff . nt . ntg);

			# Ref	-	-
			# Alt	-	X
			# ref_g may come after same_next_ref_g, diff_next_ref_g, alt_g_next_ref_g.
			both_g					= ('--');
			ref_g					= ('-' . nt);

			# Make a static assertion fail if there are any character combinations that were not listed above.
			action report_unhandled {
				static_assert(false, "Found a character combination that was not handled.");
			}
			unhandled = ((ntg{4}) - (
				same_next_both_g		|
				same_next_ref_g			|
				diff_next_ref_g			|
				alt_g_next_ref_g		|

				alt_g_next_ref_c		|

				same_next_ref_nt		|
				diff_next_ref_nt		|

				(both_g . ntg{2})		|	# Corresponds to both_g in the machines below but would match more without the ntg{2}.
				(ref_g . ntg{2})			# Corresponds to ref_g in the machines below but would match more without the ntg{2}.
			));
			fail_on_unhandled_combinations := unhandled $(report_unhandled);

			# In the following cases start a new segment at the marked position b.c.
			# the following characters cannot be known and this reduces the number
			# of special cases a bit (1).
			# Ref	ACG-	ACG-
			# Alt	ACG-	TTG-
			#		  ^		  ^
			# Action suffixes: _b: both have non-gap, _r: ref has non-gap, _a: alt has non-gap, _g: both have gap.

			common = (
				# Since the next ref character is a gap, a new segment needs to be created since we don’t know the subsequent characters.
				(same_next_both_g	@(start_new_mixed_b))		|	# We currently don’t change a matching segment to mixed, so mark the segment for post-processing here.
				(same_next_ref_g	@(start_new_mixed_b))		|
				(diff_next_ref_g	@(start_new_mismatch_b))	|
				(alt_g_next_ref_g	@(start_new_mixed_r))
			);

			handle_match := common | (
				(alt_g_next_ref_c	@(start_new_del_r))			|	# Deletion in alt w.r.t. ref.

				(same_next_ref_nt	@(match_continue_b))		|
				(diff_next_ref_nt	@(start_new_mismatch_b))	|

				(both_g				@(match_continue_g))		|
				(ref_g				@(handle_unexpected_state))
			) $err(handle_error);

			handle_mismatch := common | (
				(alt_g_next_ref_c	@(start_new_del_r))			|	# Deletion in alt w.r.t. ref.

				(same_next_ref_nt	@(start_new_match_b))		|
				(diff_next_ref_nt	@(mismatch_continue_b))		|

				(both_g				@(mismatch_continue_g))		|
				(ref_g				@(mismatch_mark_mixed_a))
			) $err(handle_error);

			handle_mixed := common | (
				(alt_g_next_ref_c	@(start_new_del_r))			|	# Deletion in alt w.r.t. ref.

				(same_next_ref_nt	@(start_new_match_b))		|
				(diff_next_ref_nt	@(start_new_mismatch_b))	|

				(both_g				@(mixed_continue_g))		|
				(ref_g				@(mixed_continue_a))
			) $err(handle_error);

			handle_deletion := common | (
				(alt_g_next_ref_c	@(deletion_continue_r))		|

				(same_next_ref_nt	@(start_new_match_b))		|
				(diff_next_ref_nt	@(start_new_mismatch_b))	|

				(both_g				@(handle_unexpected_state))	|
				(ref_g				@(handle_unexpected_state))
			) $err(handle_error);

			# Starting machines. Ref is guaranteed to be non-gap. See common.
			# Ref	X-	X-	X-	X-	X+	X+	X+
			# Alt	X-	X+	Y*	-*	-*	X*	Y*
			#same_next_both_g
			#same_next_ref_g
			#diff_next_ref_g
			#alt_g_next_ref_g
			#alt_g_next_ref_c
			#same_next_ref_nt
			#diff_next_ref_nt
			main_unexpected = ntg{4} - (same_next_both_g | same_next_ref_g | diff_next_ref_g | alt_g_next_ref_g | alt_g_next_ref_c | same_next_ref_nt | diff_next_ref_nt);

			main := (
				(same_next_both_g	@(main_new_match_b))		|
				(same_next_ref_g	@(main_new_mixed_b))		|
				(diff_next_ref_g	@(main_new_mismatch_b))		|
				(alt_g_next_ref_g	@(main_new_mixed_r))		|
				(alt_g_next_ref_c	@(main_new_deletion_r))		|
				(same_next_ref_nt	@(main_new_match_b))		|
				(diff_next_ref_nt	@(main_new_mismatch_b))		|
				(main_unexpected	@(handle_unexpected_state))
			) $err(handle_error);

			write exec noend;
		}%%
	}
}


improved description of LX_TK_RESET

improved description of LX_TK_RESET

#ifndef HEXT_LEXER_H
#define HEXT_LEXER_H

#include <string>
#include <vector>
#include <sstream>
#include <stdexcept>
#include <utility>

#include "hext/token.h"
#include "hext/string-util.h"

/// Convenience macro to create a token and push it onto the token-queue.
#define LX_TK_START(tk_id) \
  token tok;               \
  tok.tid = tk_id;         \
  tok.begin = p;           \
  tok.end = nullptr;       \
  tokens.push_back(tok);

/// Convenience macro to change last inserted token.
#define LX_TK_RESET(tk_id)   \
  assert(tokens.size() > 0); \
  tokens.back().tid = tk_id; \
  tokens.back().end = p;

/// Convenience macro to mark the end of the last token in the token-queue.
#define LX_TK_STOP           \
  assert(tokens.size() > 0); \
  tokens.back().end = p;

namespace hext {

namespace ragel {
  %%{
    machine hext;
    include "hext.rl";
    write data;
  }%%
} // namespace ragel

/// lexer contains the ragel state machine and shields the user from its
/// details.
class lexer
{
public:
  /// Clang warns (-Wweak-vtables) that a vtable for lex_error may be placed
  /// in every translation unit, because lex_error doesn't have any
  /// 'out-of-line virtual method definitions', where it would normally put
  /// the vtable. But http://stackoverflow.com/a/23749273 suggests that this
  /// is a non-issue; the linker will clean it up.
  class lex_error : public std::runtime_error
  {
  public:
    explicit lex_error(const std::string& msg);
  };

  /// Setup ragel
  lexer(const char * begin, const char * end);

  /// Throws lexer::lex_error on invalid input (by calling lexer::throw_error)
  std::vector<token> lex();

private:
  /// Print diagnostics and throw lexer::lex_error.
  void throw_error() const;

  /// A pointer to the beginning of the overall input is needed to provide
  /// good error diagnostics.
  const char * p_begin;

  /// ragel's runtime state
  const char * p;
  const char * pe;
  const char * eof;
  int cs;
};

} // namespace hext

#endif // HEXT_LEXER_H

#ifndef HEXT_LEXER_H
#define HEXT_LEXER_H

#include <string>
#include <vector>
#include <sstream>
#include <stdexcept>
#include <utility>

#include "hext/token.h"
#include "hext/string-util.h"

/// Convenience macro to create a token and push it onto the token-queue.
#define LX_TK_START(tk_id) \
  token tok;               \
  tok.tid = tk_id;         \
  tok.begin = p;           \
  tok.end = nullptr;       \
  tokens.push_back(tok);

/// Convenience macro to reset last inserted token.
#define LX_TK_RESET(tk_id)   \
  assert(tokens.size() > 0); \
  tokens.back().tid = tk_id; \
  tokens.back().end = p;

/// Convenience macro to mark the end of the last token in the token-queue.
#define LX_TK_STOP           \
  assert(tokens.size() > 0); \
  tokens.back().end = p;

namespace hext {

namespace ragel {
  %%{
    machine hext;
    include "hext.rl";
    write data;
  }%%
} // namespace ragel

/// lexer contains the ragel state machine and shields the user from its
/// details.
class lexer
{
public:
  /// Clang warns (-Wweak-vtables) that a vtable for lex_error may be placed
  /// in every translation unit, because lex_error doesn't have any
  /// 'out-of-line virtual method definitions', where it would normally put
  /// the vtable. But http://stackoverflow.com/a/23749273 suggests that this
  /// is a non-issue; the linker will clean it up.
  class lex_error : public std::runtime_error
  {
  public:
    explicit lex_error(const std::string& msg);
  };

  /// Setup ragel
  lexer(const char * begin, const char * end);

  /// Throws lexer::lex_error on invalid input (by calling lexer::throw_error)
  std::vector<token> lex();

private:
  /// Print diagnostics and throw lexer::lex_error.
  void throw_error() const;

  /// A pointer to the beginning of the overall input is needed to provide
  /// good error diagnostics.
  const char * p_begin;

  /// ragel's runtime state
  const char * p;
  const char * pe;
  const char * eof;
  int cs;
};

} // namespace hext

#endif // HEXT_LEXER_H