fixed ascii immediate

fixed ascii immediate

package scanner

import (
    "fmt"
    "strconv"
    "math"
    "os"
    . "parser"
)

%%{
    machine asm;
    write data;
}%%

type forState struct {
    name string
    max int64
    p int
}

type ScannerState struct {
    cs, p, pe, ts, te, act, eof, line int
    data []byte
    consts map[string]int64
    fconsts map[string]float64
    forloops []forState
    forvars map[string]int64
}

func NewScanner(data []byte) *ScannerState {
    state := ScannerState{0, 0, len(data), 0, 0, 0, 0, 1, data, make(map[string]int64), make(map[string]float64), make([]forState, 0, 3), make(map[string]int64)}
    state.fconsts["M_PI"] = math.Pi
    %% write init;
    return &state
}

func (state *ScannerState) Lex(lval *ParserSymType) int {
    token := 0
    sa := 0
    var ok bool
    %%{
        variable cs state.cs;
        variable p state.p;
        variable pe state.pe;
        variable ts state.ts;
        variable te state.te;
        variable act state.act;
        variable eof state.eof;
        variable data state.data;

        action singleToken {
            token = int(state.data[state.p])
            fbreak;
        }
        action pos {
            sa = state.p
        }
        action storeString {
            lval.Id = string(state.data[sa:state.p])
        }
        action parseBin {
            lval.Num, _ = strconv.ParseInt(string(state.data[sa:state.p]), 2, 0)
        }
        action parseOct {
            lval.Num, _ = strconv.ParseInt(string(state.data[sa:state.p]), 8, 0)
        }
        action parseDec {
            lval.Num, _ = strconv.ParseInt(string(state.data[sa:state.p]), 10, 0)
        }
        action parseHex {
            lval.Num, _ = strconv.ParseInt(string(state.data[sa:state.p]), 16, 0)
        }
        action parseAscii {
            lval.Num = int64(state.data[state.p - 1])
        }
        action parseFloat {
            lval.Fnum, _ = strconv.ParseFloat(string(state.data[sa:state.p]), 64)
        }

        identifier = [a-zA-Z_] >pos [a-zA-Z0-9_]* %storeString;
        literal =  '0b'i %pos [01]+ %parseBin |
                   '0x'i %pos xdigit+ %parseHex |
                   '0' %pos digit+ %parseOct |
                         digit+ >pos %parseDec |
                        "'" extend %parseAscii "'" ;
        float = ( digit+ '.' digit* |
                digit* '.' digit+ ) >pos %parseFloat;
        fix = 'fix'i [1-7] %{ lval.Num = int64(state.data[state.p-1] - '0') };
        comment := any* '\n' @{ fhold; fnext main; };
        registerLH = '$' %pos digit+  %parseDec ('L'i | 'H'i %{ lval.Num |= 0x10 } );
        register = '$' %pos digit+  %parseDec;
        literalorconst = literal | identifier %{ lval.Num, ok = state.consts[lval.Id]; if ok == false { state.p = sa; fnext *asm_error; } };
        for := space* identifier %{ state.forloops = append(state.forloops, forState{lval.Id, 0, 0}) } space* 'in'i space* literalorconst %{ state.forvars[state.forloops[len(state.forloops)-1].name] = lval.Num } space* 'to'i space* literalorconst %{ state.forloops[len(state.forloops)-1].max = lval.Num } space* '{' @{ state.forloops[len(state.forloops)-1].p = state.p; fnext main; };

        main := |*
                 '\n' => {
                     state.line++
                     token = int(state.data[state.p])
                     fbreak;
                 };
                 ';' => { fnext comment; };
                 space;
                 'for'i => { fnext for; };
                 'const'i => {
                     token = CONST
                     fbreak;
                 };
                 float => {
                     token = FLOAT;
                     fbreak;
                 };
                 'define'i => {
                     token = DEFINE
                     fbreak;
                 };
                 'mem'i => {
                     token = MEM
                     fbreak;
                 };
                 'reg'i => {
                     token = REG
                     fbreak;
                 };
                 'last'i => {
                     token = LAST
                     fbreak;
                 };
                 'imm'i => {
                     token = IMM
                     fbreak;
                 };
                 'unsigned'i => {
                     token = UNSIGNED
                     fbreak;
                 };
                 'signed'i => {
                     token = SIGNED
                     fbreak;
                 };
                 'int'i => {
                     token = INT
                     fbreak;
                 };
                 fix => {
                     token = FIX
                     fbreak;
                 };
                 literal => {
                     token = LITERAL
                     fbreak;
                 };
                 '(' => singleToken;
                 ')' => singleToken;
                 '{' => singleToken;
                 '}' => {
                     if len(state.forloops) > 0 {
                         name := state.forloops[len(state.forloops)-1].name
                         state.forvars[name]++
                         if state.forvars[name] == state.forloops[len(state.forloops)-1].max {
                             state.forloops = state.forloops[:len(state.forloops)-1]
                             delete(state.forvars, name)
                         } else {
                             state.p = state.forloops[len(state.forloops)-1].p
                         }
                     } else {
                         token = int(state.data[state.p])
                         fbreak;
                     }
                 };
                 '=' => singleToken;
                 '+' => singleToken;
                 '-' => singleToken;
                 '*' => singleToken;
                 '/' => singleToken;
                 '%' => singleToken;
                 '&' => singleToken;
                 '|' => singleToken;
                 '>>' => {
                     token = RSHIFT
                     fbreak;
                 };
                 '<<' => {
                     token = LSHIFT
                     fbreak;
                 };
                 '[' => singleToken;
                 ']' => singleToken;
                 '^' => singleToken;
                 ',' => singleToken;
                 ':' => singleToken;
                 register => {
                     token = REGISTER
                     fbreak;
                 };
                 registerLH => {
                     token = REGISTERLH
                     fbreak;
                 };
                 'DW'i => {
                     token = DW
                     fbreak;
                 };
                 'NOP'i => {
                     token = OP
                     lval.Num = 0x0000
                     fbreak;
                 };
                 'JMP'i => {
                     token = OP1
                     lval.Num = 0x0010
                     fbreak;
                 };
                 'JZ'i => {
                     token = OP1
                     lval.Num = 0x0020
                     fbreak;
                 };
                 'JNE'i => {
                     token = OP1
                     lval.Num = 0x0030
                     fbreak;
                 };
                 'JNZ'i => {
                     token = OP1
                     lval.Num = 0x0030
                     fbreak;
                 };
                 'JLE'i => {
                     token = OP1
                     lval.Num = 0x0040
                     fbreak;
                 };
                 'JLT'i => {
                     token = OP1
                     lval.Num = 0x0050
                     fbreak;
                 };
                 'JGE'i => {
                     token = OP1
                     lval.Num = 0x0060
                     fbreak;
                 };
                 'JGT'i => {
                     token = OP1
                     lval.Num = 0x0070
                     fbreak;
                 };
                 'JULE'i => {
                     token = OP1
                     lval.Num = 0x0080
                     fbreak;
                 };
                 'JULT'i => {
                     token = OP1
                     lval.Num = 0x0090
                     fbreak;
                 };
                 'JUGE'i => {
                     token = OP1
                     lval.Num = 0x00A0
                     fbreak;
                 };
                 'JUGT'i => {
                     token = OP1
                     lval.Num = 0x00B0
                     fbreak;
                 };
                 'CMP'i => {
                     token = OP2
                     lval.Num = 0x0100
                     fbreak;
                 };
                 'MOV'i => {
                     token = MOV
                     lval.Num = 0xC300
                     fbreak;
                 };
                 'MOVH'i => {
                     token = MOV
                     lval.Num = 0xC200
                     fbreak;
                 };
                 'MOVL'i => {
                     token = MOV
                     lval.Num = 0xC100
                     fbreak;
                 };
                 'ADD'i => {
                     token = OP3
                     lval.Num = 0x1000
                     fbreak;
                 };
                 'ADDC'i => {
                     token = OP3
                     lval.Num = 0x2000
                     fbreak;
                 };
                 'SUB'i => {
                     token = OP3
                     lval.Num = 0x3000
                     fbreak;
                 };
                 'SUBB'i => {
                     token = OP3
                     lval.Num = 0x4000
                     fbreak;
                 };
                 'AND'i => {
                     token = OP3
                     lval.Num = 0x5000
                     fbreak;
                 };
                 'OR'i => {
                     token = OP3
                     lval.Num = 0x6000
                     fbreak;
                 };
                 'XOR'i => {
                     token = OP3
                     lval.Num = 0x7000
                     fbreak;
                 };
                 'SHL'i => {
                     token = OP3
                     lval.Num = 0x8000
                     fbreak;
                 };
                 'SHR'i => {
                     token = OP3
                     lval.Num = 0x9000
                     fbreak;
                 };
                 'SAR'i => {
                     token = OP3
                     lval.Num = 0xA000
                     fbreak;
                 };
                 identifier => {
                     token = IDENTIFIER
                     lval.Num, ok = state.forvars[lval.Id]
                     if ok == true {
                         token = LITERAL
                     } else {
                         lval.Num, ok = state.consts[lval.Id]
                         if ok == true {
                             token = LITERAL
                         } else {
                            lval.Fnum, ok = state.fconsts[lval.Id]
                            if ok == true {
                                token = FLOAT
                            }
                         }
                     }
                     fbreak;
                 };
                 *|;

        write exec;
    }%%
    return token
}

func (state *ScannerState) ScanOk() {
    if state.p != state.pe {
        last := state.p
        for ; last<=state.pe; last++ {
            if state.data[last] == '\n' {
                break
            }
        }
        fmt.Fprintln(os.Stderr, "Scan error at line", state.line, "near", string(state.data[state.p:last]))
    }
}

func (state *ScannerState) Error(s string) {
    fmt.Fprintf(os.Stderr, "syntax error: %s in line %d\n", s, state.line)
}

func (state *ScannerState) AddConst(id string, num int64) {
    state.consts[id] = num
}

func (state *ScannerState) AddConstf(id string, num float64) {
    state.fconsts[id] = num
}

package scanner

import (
    "fmt"
    "strconv"
    "math"
    "os"
    . "parser"
)

%%{
    machine asm;
    write data;
}%%

type forState struct {
    name string
    max int64
    p int
}

type ScannerState struct {
    cs, p, pe, ts, te, act, eof, line int
    data []byte
    consts map[string]int64
    fconsts map[string]float64
    forloops []forState
    forvars map[string]int64
}

func NewScanner(data []byte) *ScannerState {
    state := ScannerState{0, 0, len(data), 0, 0, 0, 0, 1, data, make(map[string]int64), make(map[string]float64), make([]forState, 0, 3), make(map[string]int64)}
    state.fconsts["M_PI"] = math.Pi
    %% write init;
    return &state
}

func (state *ScannerState) Lex(lval *ParserSymType) int {
    token := 0
    sa := 0
    var ok bool
    %%{
        variable cs state.cs;
        variable p state.p;
        variable pe state.pe;
        variable ts state.ts;
        variable te state.te;
        variable act state.act;
        variable eof state.eof;
        variable data state.data;

        action singleToken {
            token = int(state.data[state.p])
            fbreak;
        }
        action pos {
            sa = state.p
        }
        action storeString {
            lval.Id = string(state.data[sa:state.p])
        }
        action parseBin {
            lval.Num, _ = strconv.ParseInt(string(state.data[sa:state.p]), 2, 0)
        }
        action parseOct {
            lval.Num, _ = strconv.ParseInt(string(state.data[sa:state.p]), 8, 0)
        }
        action parseDec {
            lval.Num, _ = strconv.ParseInt(string(state.data[sa:state.p]), 10, 0)
        }
        action parseHex {
            lval.Num, _ = strconv.ParseInt(string(state.data[sa:state.p]), 16, 0)
        }
        action parseAscii {
            lval.Num = int64(state.data[sa])
        }
        action parseFloat {
            lval.Fnum, _ = strconv.ParseFloat(string(state.data[sa:state.p]), 64)
        }

        identifier = [a-zA-Z_] >pos [a-zA-Z0-9_]* %storeString;
        literal =  '0b'i %pos [01]+ %parseBin |
                   '0x'i %pos xdigit+ %parseHex |
                   '0' %pos digit+ %parseOct |
                         digit+ >pos %parseDec |
                        "'" extend %parseAscii "'" ;
        float = ( digit+ '.' digit* |
                digit* '.' digit+ ) >pos %parseFloat;
        fix = 'fix'i [1-7] %{ lval.Num = int64(state.data[state.p-1] - '0') };
        comment := any* '\n' @{ fhold; fnext main; };
        registerLH = '$' %pos digit+  %parseDec ('L'i | 'H'i %{ lval.Num |= 0x10 } );
        register = '$' %pos digit+  %parseDec;
        literalorconst = literal | identifier %{ lval.Num, ok = state.consts[lval.Id]; if ok == false { state.p = sa; fnext *asm_error; } };
        for := space* identifier %{ state.forloops = append(state.forloops, forState{lval.Id, 0, 0}) } space* 'in'i space* literalorconst %{ state.forvars[state.forloops[len(state.forloops)-1].name] = lval.Num } space* 'to'i space* literalorconst %{ state.forloops[len(state.forloops)-1].max = lval.Num } space* '{' @{ state.forloops[len(state.forloops)-1].p = state.p; fnext main; };

        main := |*
                 '\n' => {
                     state.line++
                     token = int(state.data[state.p])
                     fbreak;
                 };
                 ';' => { fnext comment; };
                 space;
                 'for'i => { fnext for; };
                 'const'i => {
                     token = CONST
                     fbreak;
                 };
                 float => {
                     token = FLOAT;
                     fbreak;
                 };
                 'define'i => {
                     token = DEFINE
                     fbreak;
                 };
                 'mem'i => {
                     token = MEM
                     fbreak;
                 };
                 'reg'i => {
                     token = REG
                     fbreak;
                 };
                 'last'i => {
                     token = LAST
                     fbreak;
                 };
                 'imm'i => {
                     token = IMM
                     fbreak;
                 };
                 'unsigned'i => {
                     token = UNSIGNED
                     fbreak;
                 };
                 'signed'i => {
                     token = SIGNED
                     fbreak;
                 };
                 'int'i => {
                     token = INT
                     fbreak;
                 };
                 fix => {
                     token = FIX
                     fbreak;
                 };
                 literal => {
                     token = LITERAL
                     fbreak;
                 };
                 '(' => singleToken;
                 ')' => singleToken;
                 '{' => singleToken;
                 '}' => {
                     if len(state.forloops) > 0 {
                         name := state.forloops[len(state.forloops)-1].name
                         state.forvars[name]++
                         if state.forvars[name] == state.forloops[len(state.forloops)-1].max {
                             state.forloops = state.forloops[:len(state.forloops)-1]
                             delete(state.forvars, name)
                         } else {
                             state.p = state.forloops[len(state.forloops)-1].p
                         }
                     } else {
                         token = int(state.data[state.p])
                         fbreak;
                     }
                 };
                 '=' => singleToken;
                 '+' => singleToken;
                 '-' => singleToken;
                 '*' => singleToken;
                 '/' => singleToken;
                 '%' => singleToken;
                 '&' => singleToken;
                 '|' => singleToken;
                 '>>' => {
                     token = RSHIFT
                     fbreak;
                 };
                 '<<' => {
                     token = LSHIFT
                     fbreak;
                 };
                 '[' => singleToken;
                 ']' => singleToken;
                 '^' => singleToken;
                 ',' => singleToken;
                 ':' => singleToken;
                 register => {
                     token = REGISTER
                     fbreak;
                 };
                 registerLH => {
                     token = REGISTERLH
                     fbreak;
                 };
                 'DW'i => {
                     token = DW
                     fbreak;
                 };
                 'NOP'i => {
                     token = OP
                     lval.Num = 0x0000
                     fbreak;
                 };
                 'JMP'i => {
                     token = OP1
                     lval.Num = 0x0010
                     fbreak;
                 };
                 'JZ'i => {
                     token = OP1
                     lval.Num = 0x0020
                     fbreak;
                 };
                 'JNE'i => {
                     token = OP1
                     lval.Num = 0x0030
                     fbreak;
                 };
                 'JNZ'i => {
                     token = OP1
                     lval.Num = 0x0030
                     fbreak;
                 };
                 'JLE'i => {
                     token = OP1
                     lval.Num = 0x0040
                     fbreak;
                 };
                 'JLT'i => {
                     token = OP1
                     lval.Num = 0x0050
                     fbreak;
                 };
                 'JGE'i => {
                     token = OP1
                     lval.Num = 0x0060
                     fbreak;
                 };
                 'JGT'i => {
                     token = OP1
                     lval.Num = 0x0070
                     fbreak;
                 };
                 'JULE'i => {
                     token = OP1
                     lval.Num = 0x0080
                     fbreak;
                 };
                 'JULT'i => {
                     token = OP1
                     lval.Num = 0x0090
                     fbreak;
                 };
                 'JUGE'i => {
                     token = OP1
                     lval.Num = 0x00A0
                     fbreak;
                 };
                 'JUGT'i => {
                     token = OP1
                     lval.Num = 0x00B0
                     fbreak;
                 };
                 'CMP'i => {
                     token = OP2
                     lval.Num = 0x0100
                     fbreak;
                 };
                 'MOV'i => {
                     token = MOV
                     lval.Num = 0xC300
                     fbreak;
                 };
                 'MOVH'i => {
                     token = MOV
                     lval.Num = 0xC200
                     fbreak;
                 };
                 'MOVL'i => {
                     token = MOV
                     lval.Num = 0xC100
                     fbreak;
                 };
                 'ADD'i => {
                     token = OP3
                     lval.Num = 0x1000
                     fbreak;
                 };
                 'ADDC'i => {
                     token = OP3
                     lval.Num = 0x2000
                     fbreak;
                 };
                 'SUB'i => {
                     token = OP3
                     lval.Num = 0x3000
                     fbreak;
                 };
                 'SUBB'i => {
                     token = OP3
                     lval.Num = 0x4000
                     fbreak;
                 };
                 'AND'i => {
                     token = OP3
                     lval.Num = 0x5000
                     fbreak;
                 };
                 'OR'i => {
                     token = OP3
                     lval.Num = 0x6000
                     fbreak;
                 };
                 'XOR'i => {
                     token = OP3
                     lval.Num = 0x7000
                     fbreak;
                 };
                 'SHL'i => {
                     token = OP3
                     lval.Num = 0x8000
                     fbreak;
                 };
                 'SHR'i => {
                     token = OP3
                     lval.Num = 0x9000
                     fbreak;
                 };
                 'SAR'i => {
                     token = OP3
                     lval.Num = 0xA000
                     fbreak;
                 };
                 identifier => {
                     token = IDENTIFIER
                     lval.Num, ok = state.forvars[lval.Id]
                     if ok == true {
                         token = LITERAL
                     } else {
                         lval.Num, ok = state.consts[lval.Id]
                         if ok == true {
                             token = LITERAL
                         } else {
                            lval.Fnum, ok = state.fconsts[lval.Id]
                            if ok == true {
                                token = FLOAT
                            }
                         }
                     }
                     fbreak;
                 };
                 *|;

        write exec;
    }%%
    return token
}

func (state *ScannerState) ScanOk() {
    if state.p != state.pe {
        last := state.p
        for ; last<=state.pe; last++ {
            if state.data[last] == '\n' {
                break
            }
        }
        fmt.Fprintln(os.Stderr, "Scan error at line", state.line, "near", string(state.data[state.p:last]))
    }
}

func (state *ScannerState) Error(s string) {
    fmt.Fprintf(os.Stderr, "syntax error: %s in line %d\n", s, state.line)
}

func (state *ScannerState) AddConst(id string, num int64) {
    state.consts[id] = num
}

func (state *ScannerState) AddConstf(id string, num float64) {
    state.fconsts[id] = num
}


Remove dedicated string machine in the XML lexer.

This removes the need for another fcall/fret combination.

%%machine base_lexer;

%%{
    ##
    # Base grammar for the XML lexer.
    #
    # This grammar is shared between the C and Java extensions. As a result of
    # this you should **not** include language specific code in Ragel
    # actions/callbacks.
    #
    # To call back in to Ruby you can use one of the following two functions:
    #
    # * callback
    # * callback_simple
    #
    # The first function takes 5 arguments:
    #
    # * The name of the Ruby method to call.
    # * The input data.
    # * The encoding of the input data.
    # * The start of the current buffer.
    # * The end of the current buffer.
    #
    # The function callback_simple only takes one argument: the name of the
    # method to call. This function should be used for callbacks that don't
    # require any values.
    #
    # When you call a method in Ruby make sure that said method is defined as
    # an instance method in the `Oga::XML::Lexer` class.

    newline    = '\n' | '\r\n';
    whitespace = [ \t];
    identifier = [a-zA-Z0-9\-_]+;
    attribute  = [a-zA-Z0-9\-_:]+;

    cdata_start = '<![CDATA[';
    cdata_end   = ']]>';

    comment_start = '<!--';
    comment_end   = '-->';

    # Strings
    #
    # Strings in HTML can either be single or double quoted. If a string
    # starts with one of these quotes it must be closed with the same type
    # of quote.
    dquote = '"';
    squote = "'";

    string_dquote = (dquote ^dquote+ dquote);
    string_squote = (squote ^squote+ squote);

    string = string_dquote | string_squote;

    action emit_string {
        callback("on_string", data, encoding, ts + 1, te - 1);
    }

    # DOCTYPES
    #
    # http://www.w3.org/TR/html-markup/syntax.html#doctype-syntax
    #
    # These rules support the 3 flavours of doctypes:
    #
    # 1. Normal doctypes, as introduced in the HTML5 specification.
    # 2. Deprecated doctypes, the more verbose ones used prior to HTML5.
    # 3. Legacy doctypes
    #
    doctype_start = '<!DOCTYPE'i whitespace+;

    action start_doctype {
        callback_simple("on_doctype_start");
        fcall doctype;
    }

    # Machine for processing doctypes. Doctype values such as the public
    # and system IDs are treated as T_STRING tokens.
    doctype := |*
        'PUBLIC' | 'SYSTEM' => {
            callback("on_doctype_type", data, encoding, ts, te);
        };

        # Consumes everything between the [ and ]. Due to the use of :> the ]
        # is not consumed by any+.
        '[' any+ :> ']' => {
            callback("on_doctype_inline", data, encoding, ts + 1, te - 1);
        };

        # Lex the public/system IDs as regular strings.
        string => emit_string;

        # Whitespace inside doctypes is ignored since there's no point in
        # including it.
        whitespace;

        identifier => {
            callback("on_doctype_name", data, encoding, ts, te);
        };

        '>' => {
            callback_simple("on_doctype_end");
            fret;
        };
    *|;

    # XML declaration tags
    #
    # http://www.w3.org/TR/REC-xml/#sec-prolog-dtd
    #
    xml_decl_start = '<?xml';
    xml_decl_end   = '?>';

    action start_xml_decl {
        callback_simple("on_xml_decl_start");
        fcall xml_decl;
    }

    # Machine that processes the contents of an XML declaration tag.
    xml_decl := |*
        xml_decl_end => {
            callback_simple("on_xml_decl_end");
            fret;
        };

        # Attributes and their values (e.g. version="1.0").
        attribute => {
            callback("on_attribute", data, encoding, ts, te);
        };

        string => emit_string;

        any;
    *|;

    # Elements
    #
    # http://www.w3.org/TR/html-markup/syntax.html#syntax-elements
    #

    # Action that creates the tokens for the opening tag, name and
    # namespace (if any). Remaining work is delegated to a dedicated
    # machine.
    action start_element {
        fhold;
        fcall element_head;
    }

    # Machine used for lexing the name/namespace of an element.
    element_name := |*
        identifier ':' => {
            callback("on_element_ns", data, encoding, ts, te - 1);
        };

        identifier => {
            callback("on_element_name", data, encoding, ts, te);
            fret;
        };
    *|;

    # Machine used for processing the characters inside a element head. An
    # element head is everything between `<NAME` (where NAME is the element
    # name) and `>`.
    #
    # For example, in `<p foo="bar">` the element head is ` foo="bar"`.
    #
    element_head := |*
        whitespace | '=';

        '<' => {
            callback_simple("on_element_start");
            fcall element_name;
        };

        newline => {
            callback_simple("on_newline");
        };

        # Attribute names.
        attribute => {
            callback("on_attribute", data, encoding, ts, te);
        };

        # Attribute values.
        string => emit_string;

        # The closing character of the open tag.
        ('>' | '/') => {
            fhold;
            fret;
        };
    *|;

    main := |*
        '<'            => start_element;
        doctype_start  => start_doctype;
        xml_decl_start => start_xml_decl;

        # Comments
        #
        # http://www.w3.org/TR/html-markup/syntax.html#comments
        #
        # Unlike the W3 specification these rules *do* allow character
        # sequences such as `--` and `->`. Putting extra checks in for these
        # sequences would actually make the rules/actions more complex.
        #
        comment_start any* comment_end => {
            callback("on_comment", data, encoding, ts + 4, te - 3);
        };

        # CDATA
        #
        # http://www.w3.org/TR/html-markup/syntax.html#cdata-sections
        #
        # In HTML CDATA tags have no meaning/are not supported. Oga does
        # support them but treats their contents as plain text.
        #
        cdata_start any* cdata_end => {
            callback("on_cdata", data, encoding, ts + 9, te - 3);
        };

        # Enter the body of the tag. If HTML mode is enabled and the current
        # element is a void element we'll close it and bail out.
        '>' => {
            callback_simple("on_element_open_end");
        };

        # Regular closing tags.
        '</' identifier '>' => {
            callback_simple("on_element_end");
        };

        # Self closing elements that are not handled by the HTML mode.
        '/>' => {
            callback_simple("on_element_end");
        };

        # Note that this rule should be declared at the very bottom as it
        # will otherwise take precedence over the other rules.
        ^('<' | '>')+ => {
            callback("on_text", data, encoding, ts, te);
        };
    *|;
}%%

%%machine base_lexer;

%%{
    ##
    # Base grammar for the XML lexer.
    #
    # This grammar is shared between the C and Java extensions. As a result of
    # this you should **not** include language specific code in Ragel
    # actions/callbacks.
    #
    # To call back in to Ruby you can use one of the following two functions:
    #
    # * callback
    # * callback_simple
    #
    # The first function takes 5 arguments:
    #
    # * The name of the Ruby method to call.
    # * The input data.
    # * The encoding of the input data.
    # * The start of the current buffer.
    # * The end of the current buffer.
    #
    # The function callback_simple only takes one argument: the name of the
    # method to call. This function should be used for callbacks that don't
    # require any values.
    #
    # When you call a method in Ruby make sure that said method is defined as
    # an instance method in the `Oga::XML::Lexer` class.

    newline    = '\n' | '\r\n';
    whitespace = [ \t];
    identifier = [a-zA-Z0-9\-_]+;
    attribute  = [a-zA-Z0-9\-_:]+;

    cdata_start = '<![CDATA[';
    cdata_end   = ']]>';

    comment_start = '<!--';
    comment_end   = '-->';

    # Strings
    #
    # Strings in HTML can either be single or double quoted. If a string
    # starts with one of these quotes it must be closed with the same type
    # of quote.
    dquote = '"';
    squote = "'";

    # Machine for processing double quoted strings.
    string_dquote := |*
        ^dquote+ => {
            callback("on_string", data, encoding, ts, te);
        };

        dquote => { fret; };
    *|;

    # Machine for processing single quoted strings.
    string_squote := |*
        ^squote+ => {
            callback("on_string", data, encoding, ts, te);
        };

        squote => { fret; };
    *|;

    # DOCTYPES
    #
    # http://www.w3.org/TR/html-markup/syntax.html#doctype-syntax
    #
    # These rules support the 3 flavours of doctypes:
    #
    # 1. Normal doctypes, as introduced in the HTML5 specification.
    # 2. Deprecated doctypes, the more verbose ones used prior to HTML5.
    # 3. Legacy doctypes
    #
    doctype_start = '<!DOCTYPE'i whitespace+;

    action start_doctype {
        callback_simple("on_doctype_start");
        fcall doctype;
    }

    # Machine for processing doctypes. Doctype values such as the public
    # and system IDs are treated as T_STRING tokens.
    doctype := |*
        'PUBLIC' | 'SYSTEM' => {
            callback("on_doctype_type", data, encoding, ts, te);
        };

        # Consumes everything between the [ and ]. Due to the use of :> the ]
        # is not consumed by any+.
        '[' any+ :> ']' => {
            callback("on_doctype_inline", data, encoding, ts + 1, te - 1);
        };

        # Lex the public/system IDs as regular strings.
        dquote => { fcall string_dquote; };
        squote => { fcall string_squote; };

        # Whitespace inside doctypes is ignored since there's no point in
        # including it.
        whitespace;

        identifier => {
            callback("on_doctype_name", data, encoding, ts, te);
        };

        '>' => {
            callback_simple("on_doctype_end");
            fret;
        };
    *|;

    # XML declaration tags
    #
    # http://www.w3.org/TR/REC-xml/#sec-prolog-dtd
    #
    xml_decl_start = '<?xml';
    xml_decl_end   = '?>';

    action start_xml_decl {
        callback_simple("on_xml_decl_start");
        fcall xml_decl;
    }

    # Machine that processes the contents of an XML declaration tag.
    xml_decl := |*
        xml_decl_end => {
            callback_simple("on_xml_decl_end");
            fret;
        };

        # Attributes and their values (e.g. version="1.0").
        attribute => {
            callback("on_attribute", data, encoding, ts, te);
        };

        dquote => { fcall string_dquote; };
        squote => { fcall string_squote; };

        any;
    *|;

    # Elements
    #
    # http://www.w3.org/TR/html-markup/syntax.html#syntax-elements
    #

    # Action that creates the tokens for the opening tag, name and
    # namespace (if any). Remaining work is delegated to a dedicated
    # machine.
    action start_element {
        fhold;
        fcall element_head;
    }

    # Machine used for lexing the name/namespace of an element.
    element_name := |*
        identifier ':' => {
            callback("on_element_ns", data, encoding, ts, te - 1);
        };

        identifier => {
            callback("on_element_name", data, encoding, ts, te);
            fret;
        };
    *|;

    # Machine used for processing the characters inside a element head. An
    # element head is everything between `<NAME` (where NAME is the element
    # name) and `>`.
    #
    # For example, in `<p foo="bar">` the element head is ` foo="bar"`.
    #
    element_head := |*
        whitespace | '=';

        '<' => {
            callback_simple("on_element_start");
            fcall element_name;
        };

        newline => {
            callback_simple("on_newline");
        };

        # Attribute names.
        attribute => {
            callback("on_attribute", data, encoding, ts, te);
        };

        # Attribute values.
        dquote => { fcall string_dquote; };
        squote => { fcall string_squote; };

        # The closing character of the open tag.
        ('>' | '/') => {
            fhold;
            fret;
        };
    *|;

    main := |*
        '<'            => start_element;
        doctype_start  => start_doctype;
        xml_decl_start => start_xml_decl;

        # Comments
        #
        # http://www.w3.org/TR/html-markup/syntax.html#comments
        #
        # Unlike the W3 specification these rules *do* allow character
        # sequences such as `--` and `->`. Putting extra checks in for these
        # sequences would actually make the rules/actions more complex.
        #
        comment_start any* comment_end => {
            callback("on_comment", data, encoding, ts + 4, te - 3);
        };

        # CDATA
        #
        # http://www.w3.org/TR/html-markup/syntax.html#cdata-sections
        #
        # In HTML CDATA tags have no meaning/are not supported. Oga does
        # support them but treats their contents as plain text.
        #
        cdata_start any* cdata_end => {
            callback("on_cdata", data, encoding, ts + 9, te - 3);
        };

        # Enter the body of the tag. If HTML mode is enabled and the current
        # element is a void element we'll close it and bail out.
        '>' => {
            callback_simple("on_element_open_end");
        };

        # Regular closing tags.
        '</' identifier '>' => {
            callback_simple("on_element_end");
        };

        # Self closing elements that are not handled by the HTML mode.
        '/>' => {
            callback_simple("on_element_end");
        };

        # Note that this rule should be declared at the very bottom as it
        # will otherwise take precedence over the other rules.
        ^('<' | '>')+ => {
            callback("on_text", data, encoding, ts, te);
        };
    *|;
}%%