fod_lexer.rl: work even if 'lex->realloc==NULL'

Signed-off-by: Pavel Kretov <7cae407f1f3b56350827d1394be40a3e8f9cb303@gmail.com>

#include "fod_common.h"
#include "fod_lexer.h"
#include <string.h>
#include <stdio.h>

%% machine fod_lexer;
%% write data;

void fod_lexer_reset(
	struct fod_lexer *lex,
	char const       *input,
	fod_reallocator   realloc,
	void             *realloc_arg) {

    lex->realloc             = realloc;
    lex->realloc_arg         = realloc_arg;
    lex->input               = input;
    lex->input_end           = input + strlen(input);
    lex->is_eof              = 0;
    lex->is_error            = 0;
    lex->current_token_start = NULL;
    lex->current_token_end   = NULL;
    lex->current_position    = input;
    lex->current_state       = fod_lexer_start;
    lex->current_action      = 0;
}

int fod_lexer_tokenize(
        struct fod_lexer        *lex,
        enum fod_lexeme_major   *out_major,
        struct fod_lexeme_minor *out_minor) {

    /* Don't use ragel's 'write init;' construct. Instead, perform manual
       state restoration. */
    char const *p   = lex->current_position;
    char const *pe  = lex->input_end;
    char const *ts  = lex->current_token_start;
    char const *te  = lex->current_token_end;
    char const *eof = pe;

    int cs  = lex->current_state;
    int act = lex->current_action;

    /* In Lemon, valid token major code is never zero, 'cause zero
       means 'EOF'. Let's return it in case of errors. */
    *out_major = 0;

    #pragma GCC diagnostic push
    #pragma GCC diagnostic ignored "-Wswitch-default"
    #pragma GCC diagnostic ignored "-Wsign-conversion"

    %%{
    number = digit+;
    single_quoted_string = "'" ([^']|"\\'")* "'";
    double_quoted_string = '"' ([^"]|'\\"')* '"';
    main := |*
        # Skip any whitespaces which are not in string literal.
        space+ => { printf("SPACES\n"); };

        # Parse
        '=='  => { *out_major = LEX_OPERATOR_EQ; printf("==\n"); fbreak; };
        '!='  => { *out_major = LEX_OPERATOR_NE; printf("!=\n"); fbreak; };
        '<='  => { *out_major = LEX_OPERATOR_LE; printf("<=\n"); fbreak; };
        '>='  => { *out_major = LEX_OPERATOR_GE; printf(">=\n"); fbreak; };
        '<'   => { *out_major = LEX_OPERATOR_LT; printf("<\n");  fbreak; };
        '>'   => { *out_major = LEX_OPERATOR_GT; printf(">\n");  fbreak; };
        '('   => { *out_major = LEX_PAREN_LEFT;  printf("(\n");  fbreak; };
        ')'   => { *out_major = LEX_PAREN_RIGHT; printf(")\n");  fbreak; };

        #
        '&&'  => { *out_major = LEX_OPERATOR_AND; printf("&&\n"); fbreak; };
        '||'  => { *out_major = LEX_OPERATOR_OR;  printf("||\n"); fbreak; };
        '!'   => { *out_major = LEX_OPERATOR_NOT; printf("!\n");  fbreak; };

        # Unsigned integer literal.
        number => {
	    fod_longest_uint res;
	    if (!fod_substring_to_uint(&res, ts, te))
	       fbreak;

            *out_major = LEX_LITERAL_NUMBER;
            out_minor->literal_number = res;
            fbreak;
        };

        single_quoted_string |
        double_quoted_string => {
            char *str = NULL;
            if (lex->realloc) {
	            str = fod_substring_duplicate_and_unquote(ts, te,
                                            lex->realloc, lex->realloc_arg);
	            if (str == NULL)
    	            fbreak;
	        }

            *out_major = LEX_LITERAL_STRING;
            out_minor->literal_string = str;
            fbreak;
        };

        # Parameters returning cl_bool.

        'cl_device_available'
	| 'device_available'
	| 'deviceAvailable'
	| 'available' => {
                *out_major = LEX_PARAM_BOOL;
                out_minor->device_param_code = CL_DEVICE_AVAILABLE;
	        fbreak;
        };

        'cl_device_compiler_available'
	| 'device_compiler_available'
	| 'deviceCompilerAvailable'
	| 'compiler_available'
	| 'compilerAvailable' => {
                *out_major = LEX_PARAM_BOOL;
                out_minor->device_param_code = CL_DEVICE_COMPILER_AVAILABLE;
	        fbreak;
        };

        'cl_device_endian_little'
	| 'device_endian_little'
	| 'endian_little'
	| 'little_endian'
	| 'deviceEndianLittle'
	| 'deviceLittleEndian'
	| 'endianLittle'
	| 'littleEndian' => {
                *out_major = LEX_PARAM_BOOL;
                out_minor->device_param_code = CL_DEVICE_ENDIAN_LITTLE;
	        fbreak;
        };

	'cl_device_error_correction_support'
	| 'device_error_correction_support'
	| 'device_error_correction'
	| 'error_correction_support'
	| 'error_correction'
	| 'deviceErrorCorrectionSupport'
	| 'deviceErrorCorrection'
	| 'errorCorrectionSupport'
	| 'errorCorrection' => {
                *out_major = LEX_PARAM_BOOL;
                out_minor->device_param_code = CL_DEVICE_ERROR_CORRECTION_SUPPORT;
	        fbreak;
	};

	'cl_exec_kernel'
	| 'exec_kernel'
	| 'device_exec_kernel'
	| 'execKernel'
	| 'deviceExecKernel' => {
                *out_major = LEX_PARAM_BOOL;
                out_minor->device_param_code = CL_EXEC_KERNEL;
	        fbreak;
        };

	'cl_exec_native_kernel'
	| 'exec_native_kernel'
	| 'device_exec_native_kernel'
	| 'execNativeKernel'
	| 'deviceExecNativeKernel' => {
                *out_major = LEX_PARAM_BOOL;
                out_minor->device_param_code = CL_EXEC_NATIVE_KERNEL;
	        fbreak;
        };

	'cl_device_image_support'
	| 'device_image_support'
	| 'image_support'
	| 'deviceImageSupport'
	| 'imageSupport' => {
                *out_major = LEX_PARAM_BOOL;
                out_minor->device_param_code = CL_DEVICE_IMAGE_SUPPORT;
	        fbreak;
        };

        # Parameters returning cl_uint.

	'cl_device_address_bits'
	| 'device_address_bits'
	| 'address_bits'
	| 'deviceAddressBits'
	| 'addressBits'
	=> {
                *out_major = LEX_PARAM_UINT;
                out_minor->device_param_code = CL_DEVICE_ADDRESS_BITS;
	        fbreak;
        };

	'cl_device_global_mem_cacheline_size'
	| 'device_global_mem_cacheline_size'
	| 'global_mem_cacheline_size'
	| 'deviceglobalmemcachelinesize'
	| 'globalmemcachelinesize'
	=> {
                *out_major = LEX_PARAM_UINT;
                out_minor->device_param_code = CL_DEVICE_GLOBAL_MEM_CACHELINE_SIZE;
	        fbreak;
        };

	'cl_device_max_clock_frequency'
	| 'device_max_clock_frequency'
	| 'max_clock_frequency'
	| 'deviceMaxClockFrequency'
	| 'maxClockFrequency'
	=> {
                *out_major = LEX_PARAM_UINT;
                out_minor->device_param_code = CL_DEVICE_MAX_CLOCK_FREQUENCY;
	        fbreak;
        };

	'cl_device_max_compute_units'
	| 'device_max_compute_units'
	| 'max_compute_units'
	| 'deviceMaxComputeUnits'
	| 'maxComputeUnits'
	=> {
                *out_major = LEX_PARAM_UINT;
                out_minor->device_param_code = CL_DEVICE_MAX_COMPUTE_UNITS;
	        fbreak;
        };

	'cl_device_max_constant_args'
	| 'device_max_constant_args'
	| 'max_constant_args'
	| 'deviceMaxConstantArgs'
	| 'maxConstantArgs'
	=> {
                *out_major = LEX_PARAM_UINT;
                out_minor->device_param_code = CL_DEVICE_MAX_CONSTANT_ARGS;
	        fbreak;
        };

	'cl_device_max_read_image_args'
	| 'device_max_read_image_args'
	| 'max_read_image_args'
	| 'deviceMaxReadImageArgs'
	| 'maxReadImageArgs'
	=> {
                *out_major = LEX_PARAM_UINT;
                out_minor->device_param_code = CL_DEVICE_MAX_READ_IMAGE_ARGS;
	        fbreak;
        };

	'cl_device_global_mem_size'
	| 'device_global_mem_size'
	| 'global_mem_size'
	| 'deviceGlobalMemSize'
	| 'globalMemSize'
	=> {
                *out_major = LEX_PARAM_UINT;
                out_minor->device_param_code = CL_DEVICE_GLOBAL_MEM_SIZE;
	        fbreak;
        };

        # Parameters returning cl_uint.

	'cl_device_local_mem_size'
	| 'device_local_mem_size'
	| 'local_mem_size'
	| 'deviceLocalMemSize'
	| 'localMemSize'
	=> {
                *out_major = LEX_PARAM_ULONG;
                out_minor->device_param_code = CL_DEVICE_LOCAL_MEM_SIZE;
	        fbreak;
        };

	'cl_device_max_constant_buffer_size'
	| 'device_max_constant_buffer_size'
	| 'max_constant_buffer_size'
	| 'deviceMaxConstantBufferSize'
	| 'maxConstantBufferSize'
	=> {
                *out_major = LEX_PARAM_ULONG;
                out_minor->device_param_code = CL_DEVICE_MAX_CONSTANT_BUFFER_SIZE;
	        fbreak;
        };

	'cl_device_max_mem_alloc_size'
	| 'device_max_mem_alloc_size'
	| 'max_mem_alloc_size'
	| 'deviceMaxMemAllocSize'
	| 'maxMemAllocSize'
	=> {
                *out_major = LEX_PARAM_ULONG;
                out_minor->device_param_code = CL_DEVICE_MAX_MEM_ALLOC_SIZE;
	        fbreak;
        };

	'cl_device_global_mem_cache_size'
	| 'device_global_mem_cache_size'
	| 'global_mem_cache_size'
	| 'deviceGlobalMemCacheSize'
	| 'globalMemCacheSize'
	=> {
                *out_major = LEX_PARAM_ULONG;
                out_minor->device_param_code = CL_DEVICE_GLOBAL_MEM_CACHE_SIZE;
	        fbreak;
        };

	# Parameters returning size_t.

	# Parameters returning string.

    *|;

    write exec;

    }%%

    #pragma GCC diagnostic pop

    if (*out_major == 0) {
        if (p == pe)
	    lex->is_eof = 1;
	else
	    lex->is_error = 1;

        return 0;
    }

    /* No need to save 'pe' here. */
    lex->current_state       = cs;
    lex->current_position    = p;
    lex->current_token_start = ts;
    lex->current_token_end   = te;
    lex->current_action      = act;
    return 1;
}

#include "fod_common.h"
#include "fod_lexer.h"
#include <string.h>
#include <stdio.h>

%% machine fod_lexer;
%% write data;

void fod_lexer_reset(
	struct fod_lexer *lex,
	char const       *input,
	fod_reallocator   realloc,
	void             *realloc_arg) {

    lex->realloc             = realloc;
    lex->realloc_arg         = realloc_arg;
    lex->input               = input;
    lex->input_end           = input + strlen(input);
    lex->is_eof              = 0;
    lex->is_error            = 0;
    lex->current_token_start = NULL;
    lex->current_token_end   = NULL;
    lex->current_position    = input;
    lex->current_state       = fod_lexer_start;
    lex->current_action      = 0;
}

int fod_lexer_tokenize(
        struct fod_lexer        *lex,
        enum fod_lexeme_major   *out_major,
        struct fod_lexeme_minor *out_minor) {

    /* Don't use ragel's 'write init;' construct. Instead, perform manual
       state restoration. */
    char const *p   = lex->current_position;
    char const *pe  = lex->input_end;
    char const *ts  = lex->current_token_start;
    char const *te  = lex->current_token_end;
    char const *eof = pe;

    int cs  = lex->current_state;
    int act = lex->current_action;

    /* In Lemon, valid token major code is never zero, 'cause zero
       means 'EOF'. Let's return it in case of errors. */
    *out_major = 0;

    #pragma GCC diagnostic push
    #pragma GCC diagnostic ignored "-Wswitch-default"
    #pragma GCC diagnostic ignored "-Wsign-conversion"

    %%{
    number = digit+;
    single_quoted_string = "'" ([^']|"\\'")* "'";
    double_quoted_string = '"' ([^"]|'\\"')* '"';
    main := |*
        # Skip any whitespaces which are not in string literal.
        space+ => { printf("SPACES\n"); };

        # Parse
        '=='  => { *out_major = LEX_OPERATOR_EQ; printf("==\n"); fbreak; };
        '!='  => { *out_major = LEX_OPERATOR_NE; printf("!=\n"); fbreak; };
        '<='  => { *out_major = LEX_OPERATOR_LE; printf("<=\n"); fbreak; };
        '>='  => { *out_major = LEX_OPERATOR_GE; printf(">=\n"); fbreak; };
        '<'   => { *out_major = LEX_OPERATOR_LT; printf("<\n");  fbreak; };
        '>'   => { *out_major = LEX_OPERATOR_GT; printf(">\n");  fbreak; };
        '('   => { *out_major = LEX_PAREN_LEFT;  printf("(\n");  fbreak; };
        ')'   => { *out_major = LEX_PAREN_RIGHT; printf(")\n");  fbreak; };

        #
        '&&'  => { *out_major = LEX_OPERATOR_AND; printf("&&\n"); fbreak; };
        '||'  => { *out_major = LEX_OPERATOR_OR;  printf("||\n"); fbreak; };
        '!'   => { *out_major = LEX_OPERATOR_NOT; printf("!\n");  fbreak; };

        # Unsigned integer literal.
        number => {
	    fod_longest_uint res;
	    if (!fod_substring_to_uint(&res, ts, te))
	       fbreak;

            *out_major = LEX_LITERAL_NUMBER;
            out_minor->literal_number = res;
            fbreak;
        };

        single_quoted_string |
        double_quoted_string => {
	    char *str = fod_substring_duplicate_and_unquote(ts, te,
                                            lex->realloc, lex->realloc_arg);
	    if (str == NULL)
	        fbreak;

            *out_major = LEX_LITERAL_STRING;
            out_minor->literal_string = str;
            fbreak;
        };

        # Parameters returning cl_bool.

        'cl_device_available'
	| 'device_available'
	| 'deviceAvailable'
	| 'available' => {
                *out_major = LEX_PARAM_BOOL;
                out_minor->device_param_code = CL_DEVICE_AVAILABLE;
	        fbreak;
        };

        'cl_device_compiler_available'
	| 'device_compiler_available'
	| 'deviceCompilerAvailable'
	| 'compiler_available'
	| 'compilerAvailable' => {
                *out_major = LEX_PARAM_BOOL;
                out_minor->device_param_code = CL_DEVICE_COMPILER_AVAILABLE;
	        fbreak;
        };

        'cl_device_endian_little'
	| 'device_endian_little'
	| 'endian_little'
	| 'little_endian'
	| 'deviceEndianLittle'
	| 'deviceLittleEndian'
	| 'endianLittle'
	| 'littleEndian' => {
                *out_major = LEX_PARAM_BOOL;
                out_minor->device_param_code = CL_DEVICE_ENDIAN_LITTLE;
	        fbreak;
        };

	'cl_device_error_correction_support'
	| 'device_error_correction_support'
	| 'device_error_correction'
	| 'error_correction_support'
	| 'error_correction'
	| 'deviceErrorCorrectionSupport'
	| 'deviceErrorCorrection'
	| 'errorCorrectionSupport'
	| 'errorCorrection' => {
                *out_major = LEX_PARAM_BOOL;
                out_minor->device_param_code = CL_DEVICE_ERROR_CORRECTION_SUPPORT;
	        fbreak;
	};

	'cl_exec_kernel'
	| 'exec_kernel'
	| 'device_exec_kernel'
	| 'execKernel'
	| 'deviceExecKernel' => {
                *out_major = LEX_PARAM_BOOL;
                out_minor->device_param_code = CL_EXEC_KERNEL;
	        fbreak;
        };

	'cl_exec_native_kernel'
	| 'exec_native_kernel'
	| 'device_exec_native_kernel'
	| 'execNativeKernel'
	| 'deviceExecNativeKernel' => {
                *out_major = LEX_PARAM_BOOL;
                out_minor->device_param_code = CL_EXEC_NATIVE_KERNEL;
	        fbreak;
        };

	'cl_device_image_support'
	| 'device_image_support'
	| 'image_support'
	| 'deviceImageSupport'
	| 'imageSupport' => {
                *out_major = LEX_PARAM_BOOL;
                out_minor->device_param_code = CL_DEVICE_IMAGE_SUPPORT;
	        fbreak;
        };

        # Parameters returning cl_uint.

	'cl_device_address_bits'
	| 'device_address_bits'
	| 'address_bits'
	| 'deviceAddressBits'
	| 'addressBits'
	=> {
                *out_major = LEX_PARAM_UINT;
                out_minor->device_param_code = CL_DEVICE_ADDRESS_BITS;
	        fbreak;
        };

	'cl_device_global_mem_cacheline_size'
	| 'device_global_mem_cacheline_size'
	| 'global_mem_cacheline_size'
	| 'deviceglobalmemcachelinesize'
	| 'globalmemcachelinesize'
	=> {
                *out_major = LEX_PARAM_UINT;
                out_minor->device_param_code = CL_DEVICE_GLOBAL_MEM_CACHELINE_SIZE;
	        fbreak;
        };

	'cl_device_max_clock_frequency'
	| 'device_max_clock_frequency'
	| 'max_clock_frequency'
	| 'deviceMaxClockFrequency'
	| 'maxClockFrequency'
	=> {
                *out_major = LEX_PARAM_UINT;
                out_minor->device_param_code = CL_DEVICE_MAX_CLOCK_FREQUENCY;
	        fbreak;
        };

	'cl_device_max_compute_units'
	| 'device_max_compute_units'
	| 'max_compute_units'
	| 'deviceMaxComputeUnits'
	| 'maxComputeUnits'
	=> {
                *out_major = LEX_PARAM_UINT;
                out_minor->device_param_code = CL_DEVICE_MAX_COMPUTE_UNITS;
	        fbreak;
        };

	'cl_device_max_constant_args'
	| 'device_max_constant_args'
	| 'max_constant_args'
	| 'deviceMaxConstantArgs'
	| 'maxConstantArgs'
	=> {
                *out_major = LEX_PARAM_UINT;
                out_minor->device_param_code = CL_DEVICE_MAX_CONSTANT_ARGS;
	        fbreak;
        };

	'cl_device_max_read_image_args'
	| 'device_max_read_image_args'
	| 'max_read_image_args'
	| 'deviceMaxReadImageArgs'
	| 'maxReadImageArgs'
	=> {
                *out_major = LEX_PARAM_UINT;
                out_minor->device_param_code = CL_DEVICE_MAX_READ_IMAGE_ARGS;
	        fbreak;
        };

	'cl_device_global_mem_size'
	| 'device_global_mem_size'
	| 'global_mem_size'
	| 'deviceGlobalMemSize'
	| 'globalMemSize'
	=> {
                *out_major = LEX_PARAM_UINT;
                out_minor->device_param_code = CL_DEVICE_GLOBAL_MEM_SIZE;
	        fbreak;
        };

        # Parameters returning cl_uint.

	'cl_device_local_mem_size'
	| 'device_local_mem_size'
	| 'local_mem_size'
	| 'deviceLocalMemSize'
	| 'localMemSize'
	=> {
                *out_major = LEX_PARAM_ULONG;
                out_minor->device_param_code = CL_DEVICE_LOCAL_MEM_SIZE;
	        fbreak;
        };

	'cl_device_max_constant_buffer_size'
	| 'device_max_constant_buffer_size'
	| 'max_constant_buffer_size'
	| 'deviceMaxConstantBufferSize'
	| 'maxConstantBufferSize'
	=> {
                *out_major = LEX_PARAM_ULONG;
                out_minor->device_param_code = CL_DEVICE_MAX_CONSTANT_BUFFER_SIZE;
	        fbreak;
        };

	'cl_device_max_mem_alloc_size'
	| 'device_max_mem_alloc_size'
	| 'max_mem_alloc_size'
	| 'deviceMaxMemAllocSize'
	| 'maxMemAllocSize'
	=> {
                *out_major = LEX_PARAM_ULONG;
                out_minor->device_param_code = CL_DEVICE_MAX_MEM_ALLOC_SIZE;
	        fbreak;
        };

	'cl_device_global_mem_cache_size'
	| 'device_global_mem_cache_size'
	| 'global_mem_cache_size'
	| 'deviceGlobalMemCacheSize'
	| 'globalMemCacheSize'
	=> {
                *out_major = LEX_PARAM_ULONG;
                out_minor->device_param_code = CL_DEVICE_GLOBAL_MEM_CACHE_SIZE;
	        fbreak;
        };

	# Parameters returning size_t.

	# Parameters returning string.

    *|;

    write exec;

    }%%

    #pragma GCC diagnostic pop

    if (*out_major == 0) {
        if (p == pe)
	    lex->is_eof = 1;
	else
	    lex->is_error = 1;

        return 0;
    }

    /* No need to save 'pe' here. */
    lex->current_state       = cs;
    lex->current_position    = p;
    lex->current_token_start = ts;
    lex->current_token_end   = te;
    lex->current_action      = act;
    return 1;
}


cleanups

cleanups

package dns

// Parse RRs
// With the thankful help of gdnsd and the Go examples for Ragel.
//

import (
    "os"
    "io"
    "net"
    "strconv"
)

const _RDATAMAX = 7
const _IOBUF = 65365

// Save up tokens, after we've seen the entire rdata
// we can use this.
type token struct {
    T []string      // text
    N []int         // number
    ti int          // text counter
    ni int          // number counter
}

func newToken() *token {
    to := new(token)
    to.T = make([]string, _RDATAMAX)
    to.N = make([]int, _RDATAMAX)
    to.ni, to.ti = 0, 0
    return to
}

// Only push functions are provided. Reading is done, by directly
// accessing the members (T and N). See types.rl.
func (to *token) pushInt(s string) {
    i, err := strconv.Atoi(s)
    if err != nil {
        panic("Failure to parse to int: " + s)
    }
    to.N[to.ni] = i
    to.ni++
    if to.ni > _RDATAMAX {
        panic("Too much rdata (int)")
    }
}

func (to *token) pushString(s string) {
    to.T[to.ti] = s
    to.ti++
    if to.ti > _RDATAMAX {
        panic("Too much rdata (string)")
    }
}

func (to *token) reset() {
    to.ni, to.ti = 0, 0
}

%%{
        machine z;
        write data;
}%%

// SetString
// All the NewReader stuff is expensive...
// only works for short io.Readers as we put the whole thing
// in a string -- needs to be extended for large files (sliding window).
func Zparse(q io.Reader) (z *Zone, err os.Error) {
        buf := make([]byte, _IOBUF)
        n, err := q.Read(buf)
        if err != nil {
            return nil, err
        }
        buf = buf[:n]
        z = new(Zone)

        data := string(buf)
//        cs, p, pe, eof := 0, 0, len(data), len(data)
        cs, p, pe := 0, 0, len(data)
        brace := false
        lines := 0
        mark := 0
        hdr := new(RR_Header)
        tok := newToken()
        var rr RR

        %%{
                action mark       { mark = p }
                action qname      { hdr.Name = data[mark:p] }
                action qclass     { hdr.Class = Str_class[data[mark:p]] }
                action defTtl     { /* ... */ }
                action setTtl     { ttl, _ :=  strconv.Atoi(data[mark:p]); hdr.Ttl = uint32(ttl) }
                action number     { tok.pushInt(data[mark:p]) }
                action text       { tok.pushString(data[mark:p]) }
                action textblank  { tok.pushString(data[mark:p]) }
                action set        { z.Push(rr); tok.reset(); println("setting") }
                action openBrace  { if brace { println("Brace already open")} ; brace = true }
                action closeBrace { if !brace { println("Brace already closed")}; brace = false }
                action brace      { brace }
                action linecount  { lines++ }
                action qtype      {
                    i := Str_rr[data[mark:p]]
                    mk, known := rr_mk[int(i)]
                    if ! known {
                        println("Unknown type seen: " + data[mark:p])
                        // panic?
                    }
                    rr = mk()
                    hdr.Rrtype = i
                }

                # Newlines
                nl = [\n]+ $linecount;

                # Comments, entire line. Shorter comments are handled in the
                # 'bl' definition below.
                comment = ';' [^\n]*;

                bl = (
                    [ \t]+
                    | '(' $openBrace
                    | ')' $closeBrace
                    | (comment? nl)+ when brace
                )+ %mark;

                qclass      = ('IN'i|'CS'i|'CH'i|'HS'i|'ANY'i|'NONE'i) %qclass;
                ttl         = digit+ >mark;
                qname       = [\-a-zA-Z0-9.\\_]+ %qname;
                # If I use this in the definitions at the end, things break.
                # 6l seems to hang when compiling the resulting .go file...
                # tb is WITH the space
                tb          = [\-a-zA-Z0-9.\\/+=: ]+ $1 %0 %textblank;
                t           = [\-a-zA-Z0-9.\\/+=:]+ $1 %0 %text;
                n           = [0-9]+ $1 %0 %number;

                lhs = qname? bl %defTtl (
                      (ttl %setTtl bl (qclass bl)?)
                    | (qclass bl (ttl %setTtl bl)?)
                )?;

                # RDATA definitions.
                include "types.rl";

                # RR definitions.
                rhs = (
                       ('A'i         %qtype bl t) %rdata_a
                     | ('AAAA'i      %qtype bl t) %rdata_aaaa
                     | ('NS'i        %qtype bl t) %rdata_ns
                     | ('CNAME'i     %qtype bl t) %rdata_cname
                     | ('SOA'i       %qtype bl t bl t bl n bl n bl n bl n bl n) %rdata_soa
                     | ('MX'i        %qtype bl n bl t) %rdata_mx
                     | ('DS'i        %qtype bl n bl n bl n bl t) %rdata_ds
                     | ('DNSKEY'i    %qtype bl n bl n bl n bl t) %rdata_dnskey
                     | ('RRSIG'i     %qtype bl n bl n bl n bl n bl n bl n bl n bl t bl t) %rdata_rrsig
                );

                rr = lhs rhs %set;
                main := (rr? bl? ((comment? nl) when !brace))*;

                write init;
                write exec;
        }%%

/*
    This part needs work
        if cs < z_first_final {
                // No clue what I'm doing what so ever
                if p == pe {
                        println("unexpected eof")
                        return z, nil
                } else {
                        println("error at position ", p)
                        return z, nil
                }
        }
*/
        return z, nil
}

package dns

// Parse RRs
// With the thankful help of gdnsd and the Go examples for Ragel.
//

import (
    "os"
    "io"
    "net"
    "strconv"
)

const _RDATAMAX = 7
const _IOBUF = 65365

// Save up tokens, after we've seen the entire rdata
// we can use this.
type token struct {
    T []string      // text
    N []int         // number
    ti int          // text counter
    ni int          // number counter
}

func newToken() *token {
    to := new(token)
    to.T = make([]string, _RDATAMAX)
    to.N = make([]int, _RDATAMAX)
    to.ni, to.ti = 0, 0
    return to
}

// Only push functions are provided. Reading is done, by directly
// accessing the members (T and N). See types.rl.
func (to *token) pushInt(s string) {
    i, err := strconv.Atoi(s)
    if err != nil {
        panic("Failure to parse to int: " + s)
    }
    to.N[to.ni] = i
    to.ni++
    if to.ni > _RDATAMAX {
        panic("Too much rdata (int)")
    }
}

func (to *token) pushString(s string) {
    to.T[to.ti] = s
    to.ti++
    if to.ti > _RDATAMAX {
        panic("Too much rdata (string)")
    }
}

func (to *token) reset() {
    to.ni, to.ti = 0, 0
}

%%{
        machine z;
        write data;
}%%

// SetString
// All the NewReader stuff is expensive...
// only works for short io.Readers as we put the whole thing
// in a string -- needs to be extended for large files (sliding window).
func Zparse(q io.Reader) (z *Zone, err os.Error) {
        buf := make([]byte, _IOBUF)
        n, err := q.Read(buf)
        if err != nil {
            return nil, err
        }
        buf = buf[:n]
        z = new(Zone)

        data := string(buf)
//        cs, p, pe, eof := 0, 0, len(data), len(data)
        cs, p, pe := 0, 0, len(data)
        brace := false
        lines := 0
        mark := 0
        hdr := new(RR_Header)
        tok := newToken()
        var rr RR

        %%{
                action mark       { mark = p }
                action qname      { hdr.Name = data[mark:p] }
                action qclass     { hdr.Class = Str_class[data[mark:p]] }
                action defTtl     { /* ... */ }
                action setTtl     { ttl, _ :=  strconv.Atoi(data[mark:p]); hdr.Ttl = uint32(ttl) }
                action number     { tok.pushInt(data[mark:p]) }
                action text       { tok.pushString(data[mark:p]) }
                action textblank  { tok.pushString(data[mark:p]) }
                action set        { z.Push(rr); tok.reset(); println("setting") }
                action openBrace  { if brace { println("Brace already open")} ; brace = true }
                action closeBrace { if !brace { println("Brace already closed")}; brace = false }
                action brace      { brace }
                action linecount  { lines++ }

                # Newlines
                nl = [\n]+ $linecount;

                # Comments, entire line. Shorter comments are handled in the
                # 'bl' definition below.
                comment = ';' [^\n]*;

                bl = (
                    [ \t]+
                    | '(' $openBrace
                    | ')' $closeBrace
                    | (comment? nl)+ when brace
                )+ %mark;

                action qtype    {
                    i := Str_rr[data[mark:p]]
                    mk, known := rr_mk[int(i)]
                    if ! known {
                        println("Unknown type seen: " + data[mark:p])
                        // panic?
                    }
                    rr = mk()
                    hdr.Rrtype = i
                }

                qclass      = ('IN'i|'CS'i|'CH'i|'HS'i|'ANY'i|'NONE'i) %qclass;
                ttl         = digit+ >mark;
                qname       = [\-a-zA-Z0-9.\\_]+ %qname;
                # If I use this in the definitions at the end, things break.
                # 6l seems to hang when compiling the resulting .go file...
                # tb is WITH the space
                tb          = [\-a-zA-Z0-9.\\/+=: ]+ $1 %0 %textblank;
                t           = [\-a-zA-Z0-9.\\/+=:]+ $1 %0 %text;
                n           = [0-9]+ $1 %0 %number;

                lhs = qname? bl %defTtl (
                      (ttl %setTtl bl (qclass bl)?)
                    | (qclass bl (ttl %setTtl bl)?)
                )?;

                # RDATA definitions.
                include "types.rl";

                # RR definitions.
                rhs = (
                       ('A'i         %qtype bl t) %rdata_a
                     | ('AAAA'i      %qtype bl t) %rdata_aaaa
                     | ('NS'i        %qtype bl t) %rdata_ns
                     | ('CNAME'i     %qtype bl t) %rdata_cname
                     | ('SOA'i       %qtype bl t bl t bl n bl n bl n bl n bl n) %rdata_soa
                     | ('MX'i        %qtype bl n bl t) %rdata_mx
                     | ('DS'i        %qtype bl n bl n bl n bl t) %rdata_ds
                     | ('DNSKEY'i    %qtype bl n bl n bl n bl t) %rdata_dnskey
                     | ('RRSIG'i     %qtype bl n bl n bl n bl n bl n bl n bl n bl t bl t) %rdata_rrsig
                ) %set;

                rr = lhs rhs;
                main := (rr? bl? ((comment? nl) when !brace))*;

                write init;
                write exec;
        }%%

        if cs < z_first_final {
                // No clue what I'm doing what so ever
                if p == pe {
                        println("unexpected eof")
                        return z, nil
                } else {
                        println("error at position ", p)
                        return z, nil
                }
        }
        return z, nil
}


Short docs on lexer callback names.

Short docs on lexer callback names.

%%machine base_lexer;

%%{
    ##
    # Base grammar for the XML lexer.
    #
    # This grammar is shared between the C and Java extensions. As a result of
    # this you should **not** include language specific code in Ragel
    # actions/callbacks.
    #
    # To call back in to Ruby you can use one of the following two functions:
    #
    # * callback
    # * callback_simple
    #
    # The first function takes 5 arguments:
    #
    # * The name of the Ruby method to call.
    # * The input data.
    # * The encoding of the input data.
    # * The start of the current buffer.
    # * The end of the current buffer.
    #
    # The function callback_simple only takes one argument: the name of the
    # method to call. This function should be used for callbacks that don't
    # require any values.
    #
    # When you call a method in Ruby make sure that said method is defined as
    # an instance method in the `Oga::XML::Lexer` class.
    #
    # The name of the callback to invoke should be an identifier starting with
    # "id_". The identifier should be defined in the associated C and Java code.
    # In case of C code its value should be a Symbol as a VALUE object, for Java
    # it should be a String. For example:
    #
    #     VALUE id_foo = rb_intern("foo");
    #
    # And for Java:
    #
    #     String id_foo = "foo";
    #
    # ## Machine Transitions
    #
    # To transition from one machine to another always use `fnext` instead of
    # `fcall` and `fret`. This removes the need for the code to keep track of a
    # stack.
    #

    newline = '\n' | '\r\n';

    action count_newlines {
        if ( fc == '\n' ) lines++;
    }

    whitespace = [ \t];
    ident_char = [a-zA-Z0-9\-_];
    identifier = ident_char+;

    # Comments
    #
    # http://www.w3.org/TR/html-markup/syntax.html#comments
    #
    # Unlike the W3 specification these rules *do* allow character sequences
    # such as `--` and `->`. Putting extra checks in for these sequences would
    # actually make the rules/actions more complex.
    #

    comment_start = '<!--';
    comment_end   = '-->';
    comment       = comment_start (any* -- comment_end) comment_end;

    action start_comment {
        callback(id_on_comment, data, encoding, ts + 4, te - 3);
    }

    # CDATA
    #
    # http://www.w3.org/TR/html-markup/syntax.html#cdata-sections
    #
    # In HTML CDATA tags have no meaning/are not supported. Oga does
    # support them but treats their contents as plain text.
    #

    cdata_start = '<![CDATA[';
    cdata_end   = ']]>';
    cdata       = cdata_start (any* -- cdata_end) cdata_end;

    action start_cdata {
        callback(id_on_cdata, data, encoding, ts + 9, te - 3);
    }

    # Processing Instructions
    #
    # http://www.w3.org/TR/xpath/#section-Processing-Instruction-Nodes
    # http://en.wikipedia.org/wiki/Processing_Instruction
    #
    # These are tags meant to be used by parsers/libraries for custom behaviour.
    # One example are the tags used by PHP: <?php and ?>. Note that the XML
    # declaration tags (<?xml ?>) are not considered to be a processing
    # instruction.
    #

    proc_ins_start = '<?' identifier;
    proc_ins_end   = '?>';

    action start_proc_ins {
        callback_simple(id_on_proc_ins_start);
        callback(id_on_proc_ins_name, data, encoding, ts + 2, te);

        mark = te;

        fnext proc_ins_body;
    }

    proc_ins_body := |*
        proc_ins_end => {
            callback(id_on_text, data, encoding, mark, ts);
            callback_simple(id_on_proc_ins_end);

            mark = 0;

            fnext main;
        };

        any;
    *|;

    # Strings
    #
    # Strings in HTML can either be single or double quoted. If a string
    # starts with one of these quotes it must be closed with the same type
    # of quote.
    #
    dquote = '"';
    squote = "'";

    action emit_string {
        callback(id_on_string_body, data, encoding, ts, te);

        if ( lines > 0 )
        {
            advance_line(lines);

            lines = 0;
        }
    }

    action start_string_squote {
        callback_simple(id_on_string_squote);

        fcall string_squote;
    }

    action start_string_dquote {
        callback_simple(id_on_string_dquote);

        fcall string_dquote;
    }

    string_squote := |*
        ^squote* $count_newlines => emit_string;

        squote => {
            callback_simple(id_on_string_squote);

            fret;
        };
    *|;

    string_dquote := |*
        ^dquote* $count_newlines => emit_string;

        dquote => {
            callback_simple(id_on_string_dquote);

            fret;
        };
    *|;

    # DOCTYPES
    #
    # http://www.w3.org/TR/html-markup/syntax.html#doctype-syntax
    #
    # These rules support the 3 flavours of doctypes:
    #
    # 1. Normal doctypes, as introduced in the HTML5 specification.
    # 2. Deprecated doctypes, the more verbose ones used prior to HTML5.
    # 3. Legacy doctypes
    #
    doctype_start = '<!DOCTYPE'i whitespace+;

    action start_doctype {
        callback_simple(id_on_doctype_start);
        fnext doctype;
    }

    # Machine for processing inline rules of a doctype.
    doctype_inline := |*
        ^']'* $count_newlines => {
            callback(id_on_doctype_inline, data, encoding, ts, te);

            if ( lines > 0 )
            {
                advance_line(lines);

                lines = 0;
            }
        };

        ']' => { fnext doctype; };
    *|;

    # Machine for processing doctypes. Doctype values such as the public
    # and system IDs are treated as T_STRING tokens.
    doctype := |*
        'PUBLIC' | 'SYSTEM' => {
            callback(id_on_doctype_type, data, encoding, ts, te);
        };

        # Starts a set of inline doctype rules.
        '[' => { fnext doctype_inline; };

        # Lex the public/system IDs as regular strings.
        squote => start_string_squote;
        dquote => start_string_dquote;

        # Whitespace inside doctypes is ignored since there's no point in
        # including it.
        whitespace;

        identifier => {
            callback(id_on_doctype_name, data, encoding, ts, te);
        };

        '>' => {
            callback_simple(id_on_doctype_end);
            fnext main;
        };
    *|;

    # XML declaration tags
    #
    # http://www.w3.org/TR/REC-xml/#sec-prolog-dtd
    #
    xml_decl_start = '<?xml';
    xml_decl_end   = '?>';

    action start_xml_decl {
        callback_simple(id_on_xml_decl_start);
        fnext xml_decl;
    }

    # Machine that processes the contents of an XML declaration tag.
    xml_decl := |*
        xml_decl_end => {
            callback_simple(id_on_xml_decl_end);
            fnext main;
        };

        # Attributes and their values (e.g. version="1.0").
        identifier => {
            callback(id_on_attribute, data, encoding, ts, te);
        };

        squote => start_string_squote;
        dquote => start_string_dquote;

        any;
    *|;

    # Elements
    #
    # http://www.w3.org/TR/html-markup/syntax.html#syntax-elements
    #
    # Lexing of elements is broken up into different machines that handle the
    # name/namespace, contents of the open tag and the body of an element. The
    # body of an element is lexed using the `main` machine.
    #

    element_start = '<' ident_char;
    element_end   = '</' identifier (':' identifier)* '>';

    action start_element {
        callback_simple(id_on_element_start);
        fhold;
        fnext element_name;
    }

    action close_element {
        callback_simple(id_on_element_end);
    }

    # Machine used for lexing the name/namespace of an element.
    element_name := |*
        identifier ':' => {
            callback(id_on_element_ns, data, encoding, ts, te - 1);
        };

        identifier => {
            callback(id_on_element_name, data, encoding, ts, te);
            fnext element_head;
        };
    *|;

    # Machine used for processing the contents of an element's starting tag.
    # This includes the name, namespace and attributes.
    element_head := |*
        whitespace | '=';

        newline => {
            callback_simple(id_advance_line);
        };

        # Attribute names and namespaces.
        identifier ':' => {
            callback(id_on_attribute_ns, data, encoding, ts, te - 1);
        };

        identifier => {
            callback(id_on_attribute, data, encoding, ts, te);
        };

        # Attribute values.
        squote => start_string_squote;
        dquote => start_string_dquote;

        # We're done with the open tag of the element.
        '>' => {
            callback_simple(id_on_element_open_end);
            fnext main;
        };

        # Self closing tags.
        '/>' => {
            callback_simple(id_on_element_end);
            fnext main;
        };
    *|;

    # Text
    #
    # http://www.w3.org/TR/xml/#syntax
    # http://www.w3.org/TR/html-markup/syntax.html#text-syntax
    #
    # Text content is everything leading up to certain special tags such as "</"
    # and "<?".

    action start_text {
        fhold;
        fnext text;
    }

    # These characters terminate a T_TEXT sequence and instruct Ragel to jump
    # back to the main machine.
    #
    # Note that this only works if each sequence is exactly 2 characters
    # long. Because of this "<!" is used instead of "<!--".

    terminate_text = '</' | '<!' | '<?' | element_start;
    allowed_text   = (any* -- terminate_text) $count_newlines;

    text := |*
        terminate_text | allowed_text => {
            callback(id_on_text, data, encoding, ts, te);

            if ( lines > 0 )
            {
                advance_line(lines);

                lines = 0;
            }

            fnext main;
        };

        # Text followed by a special tag, such as "foo<!--"
        allowed_text %{ mark = p; } terminate_text => {
            callback(id_on_text, data, encoding, ts, mark);

            p    = mark - 1;
            mark = 0;

            if ( lines > 0 )
            {
                advance_line(lines);

                lines = 0;
            }

            fnext main;
        };
    *|;

    # The main machine aka the entry point of Ragel.
    main := |*
        doctype_start  => start_doctype;
        xml_decl_start => start_xml_decl;
        comment        => start_comment;
        cdata          => start_cdata;
        proc_ins_start => start_proc_ins;
        element_start  => start_element;
        element_end    => close_element;
        any            => start_text;
    *|;
}%%

%%machine base_lexer;

%%{
    ##
    # Base grammar for the XML lexer.
    #
    # This grammar is shared between the C and Java extensions. As a result of
    # this you should **not** include language specific code in Ragel
    # actions/callbacks.
    #
    # To call back in to Ruby you can use one of the following two functions:
    #
    # * callback
    # * callback_simple
    #
    # The first function takes 5 arguments:
    #
    # * The name of the Ruby method to call.
    # * The input data.
    # * The encoding of the input data.
    # * The start of the current buffer.
    # * The end of the current buffer.
    #
    # The function callback_simple only takes one argument: the name of the
    # method to call. This function should be used for callbacks that don't
    # require any values.
    #
    # When you call a method in Ruby make sure that said method is defined as
    # an instance method in the `Oga::XML::Lexer` class.
    #
    # ## Machine Transitions
    #
    # To transition from one machine to another always use `fnext` instead of
    # `fcall` and `fret`. This removes the need for the code to keep track of a
    # stack.
    #

    newline = '\n' | '\r\n';

    action count_newlines {
        if ( fc == '\n' ) lines++;
    }

    whitespace = [ \t];
    ident_char = [a-zA-Z0-9\-_];
    identifier = ident_char+;

    # Comments
    #
    # http://www.w3.org/TR/html-markup/syntax.html#comments
    #
    # Unlike the W3 specification these rules *do* allow character sequences
    # such as `--` and `->`. Putting extra checks in for these sequences would
    # actually make the rules/actions more complex.
    #

    comment_start = '<!--';
    comment_end   = '-->';
    comment       = comment_start (any* -- comment_end) comment_end;

    action start_comment {
        callback(id_on_comment, data, encoding, ts + 4, te - 3);
    }

    # CDATA
    #
    # http://www.w3.org/TR/html-markup/syntax.html#cdata-sections
    #
    # In HTML CDATA tags have no meaning/are not supported. Oga does
    # support them but treats their contents as plain text.
    #

    cdata_start = '<![CDATA[';
    cdata_end   = ']]>';
    cdata       = cdata_start (any* -- cdata_end) cdata_end;

    action start_cdata {
        callback(id_on_cdata, data, encoding, ts + 9, te - 3);
    }

    # Processing Instructions
    #
    # http://www.w3.org/TR/xpath/#section-Processing-Instruction-Nodes
    # http://en.wikipedia.org/wiki/Processing_Instruction
    #
    # These are tags meant to be used by parsers/libraries for custom behaviour.
    # One example are the tags used by PHP: <?php and ?>. Note that the XML
    # declaration tags (<?xml ?>) are not considered to be a processing
    # instruction.
    #

    proc_ins_start = '<?' identifier;
    proc_ins_end   = '?>';

    action start_proc_ins {
        callback_simple(id_on_proc_ins_start);
        callback(id_on_proc_ins_name, data, encoding, ts + 2, te);

        mark = te;

        fnext proc_ins_body;
    }

    proc_ins_body := |*
        proc_ins_end => {
            callback(id_on_text, data, encoding, mark, ts);
            callback_simple(id_on_proc_ins_end);

            mark = 0;

            fnext main;
        };

        any;
    *|;

    # Strings
    #
    # Strings in HTML can either be single or double quoted. If a string
    # starts with one of these quotes it must be closed with the same type
    # of quote.
    #
    dquote = '"';
    squote = "'";

    action emit_string {
        callback(id_on_string_body, data, encoding, ts, te);

        if ( lines > 0 )
        {
            advance_line(lines);

            lines = 0;
        }
    }

    action start_string_squote {
        callback_simple(id_on_string_squote);

        fcall string_squote;
    }

    action start_string_dquote {
        callback_simple(id_on_string_dquote);

        fcall string_dquote;
    }

    string_squote := |*
        ^squote* $count_newlines => emit_string;

        squote => {
            callback_simple(id_on_string_squote);

            fret;
        };
    *|;

    string_dquote := |*
        ^dquote* $count_newlines => emit_string;

        dquote => {
            callback_simple(id_on_string_dquote);

            fret;
        };
    *|;

    # DOCTYPES
    #
    # http://www.w3.org/TR/html-markup/syntax.html#doctype-syntax
    #
    # These rules support the 3 flavours of doctypes:
    #
    # 1. Normal doctypes, as introduced in the HTML5 specification.
    # 2. Deprecated doctypes, the more verbose ones used prior to HTML5.
    # 3. Legacy doctypes
    #
    doctype_start = '<!DOCTYPE'i whitespace+;

    action start_doctype {
        callback_simple(id_on_doctype_start);
        fnext doctype;
    }

    # Machine for processing inline rules of a doctype.
    doctype_inline := |*
        ^']'* $count_newlines => {
            callback(id_on_doctype_inline, data, encoding, ts, te);

            if ( lines > 0 )
            {
                advance_line(lines);

                lines = 0;
            }
        };

        ']' => { fnext doctype; };
    *|;

    # Machine for processing doctypes. Doctype values such as the public
    # and system IDs are treated as T_STRING tokens.
    doctype := |*
        'PUBLIC' | 'SYSTEM' => {
            callback(id_on_doctype_type, data, encoding, ts, te);
        };

        # Starts a set of inline doctype rules.
        '[' => { fnext doctype_inline; };

        # Lex the public/system IDs as regular strings.
        squote => start_string_squote;
        dquote => start_string_dquote;

        # Whitespace inside doctypes is ignored since there's no point in
        # including it.
        whitespace;

        identifier => {
            callback(id_on_doctype_name, data, encoding, ts, te);
        };

        '>' => {
            callback_simple(id_on_doctype_end);
            fnext main;
        };
    *|;

    # XML declaration tags
    #
    # http://www.w3.org/TR/REC-xml/#sec-prolog-dtd
    #
    xml_decl_start = '<?xml';
    xml_decl_end   = '?>';

    action start_xml_decl {
        callback_simple(id_on_xml_decl_start);
        fnext xml_decl;
    }

    # Machine that processes the contents of an XML declaration tag.
    xml_decl := |*
        xml_decl_end => {
            callback_simple(id_on_xml_decl_end);
            fnext main;
        };

        # Attributes and their values (e.g. version="1.0").
        identifier => {
            callback(id_on_attribute, data, encoding, ts, te);
        };

        squote => start_string_squote;
        dquote => start_string_dquote;

        any;
    *|;

    # Elements
    #
    # http://www.w3.org/TR/html-markup/syntax.html#syntax-elements
    #
    # Lexing of elements is broken up into different machines that handle the
    # name/namespace, contents of the open tag and the body of an element. The
    # body of an element is lexed using the `main` machine.
    #

    element_start = '<' ident_char;
    element_end   = '</' identifier (':' identifier)* '>';

    action start_element {
        callback_simple(id_on_element_start);
        fhold;
        fnext element_name;
    }

    action close_element {
        callback_simple(id_on_element_end);
    }

    # Machine used for lexing the name/namespace of an element.
    element_name := |*
        identifier ':' => {
            callback(id_on_element_ns, data, encoding, ts, te - 1);
        };

        identifier => {
            callback(id_on_element_name, data, encoding, ts, te);
            fnext element_head;
        };
    *|;

    # Machine used for processing the contents of an element's starting tag.
    # This includes the name, namespace and attributes.
    element_head := |*
        whitespace | '=';

        newline => {
            callback_simple(id_advance_line);
        };

        # Attribute names and namespaces.
        identifier ':' => {
            callback(id_on_attribute_ns, data, encoding, ts, te - 1);
        };

        identifier => {
            callback(id_on_attribute, data, encoding, ts, te);
        };

        # Attribute values.
        squote => start_string_squote;
        dquote => start_string_dquote;

        # We're done with the open tag of the element.
        '>' => {
            callback_simple(id_on_element_open_end);
            fnext main;
        };

        # Self closing tags.
        '/>' => {
            callback_simple(id_on_element_end);
            fnext main;
        };
    *|;

    # Text
    #
    # http://www.w3.org/TR/xml/#syntax
    # http://www.w3.org/TR/html-markup/syntax.html#text-syntax
    #
    # Text content is everything leading up to certain special tags such as "</"
    # and "<?".

    action start_text {
        fhold;
        fnext text;
    }

    # These characters terminate a T_TEXT sequence and instruct Ragel to jump
    # back to the main machine.
    #
    # Note that this only works if each sequence is exactly 2 characters
    # long. Because of this "<!" is used instead of "<!--".

    terminate_text = '</' | '<!' | '<?' | element_start;
    allowed_text   = (any* -- terminate_text) $count_newlines;

    text := |*
        terminate_text | allowed_text => {
            callback(id_on_text, data, encoding, ts, te);

            if ( lines > 0 )
            {
                advance_line(lines);

                lines = 0;
            }

            fnext main;
        };

        # Text followed by a special tag, such as "foo<!--"
        allowed_text %{ mark = p; } terminate_text => {
            callback(id_on_text, data, encoding, ts, mark);

            p    = mark - 1;
            mark = 0;

            if ( lines > 0 )
            {
                advance_line(lines);

                lines = 0;
            }

            fnext main;
        };
    *|;

    # The main machine aka the entry point of Ragel.
    main := |*
        doctype_start  => start_doctype;
        xml_decl_start => start_xml_decl;
        comment        => start_comment;
        cdata          => start_cdata;
        proc_ins_start => start_proc_ins;
        element_start  => start_element;
        element_end    => close_element;
        any            => start_text;
    *|;
}%%


Scan Audio tracks

Scan Audio tracks

Use a state variable to track which sub document you're in.
Treat each line as it's own scanner.

package handbrake

import (
	"fmt"
)

%%{
	machine handbrake;
	write data;
}%%

type Section int
const (
	NONE Section = iota
	CHAPTER
	AUDIO
	SUBTITLE
)

var section Section = NONE
func parseOutput(data string) (HandBrakeMeta) {
	cs, p, pe, eof := 0, 0, len(data), 0
	top, ts, te, act := 0,0,0,0
	_,_,_,_ = top, ts, te, act
	var stack = []int{0}
	_ = eof
	line := 1
	csp := 0
	meta := HandBrakeMeta{}
	fmt.Printf("%02d: ", line)
	_ = csp
	%%{
		action newline { line +=1; fmt.Printf("\n%02d: ", line) }
		newline = any* '\n' @ newline;
		stitle := |*
			(alnum|space)+[.]*alnum* => { fmt.Printf("%s", data[ts:te]); fret;};
			"\n" => { fret; };
		*|;
		sduration := |*
			space*;
			digit{2}[:]digit{2}[:]digit{2} => { fmt.Printf("%s:", data[ts:te]); fret;};
			"\n" => { fret; };
		*|;
		picture := |*
			space*;
			digit{3,4} "x" digit{3,4} => { fmt.Printf("%s:", data[ts:te]); fret;};
		*|;
		paspect := |*
			space*;
			digit{1,4} "/" digit{1,4} => { fmt.Printf("%s:", data[ts:te]); fret; };
		*|;
		daspect := |*
			space*;
			digit . "." . digit{1,3} => { fmt.Printf("%s:", data[ts:te]); fret; };
		*|;
		sfps := |*
			"\n" => { ts -= 10; fmt.Printf("%s", data[ts:te-5]); p -= 1; fret; };
		*|;
		crop := |*
			space*;
			digit{1,3} "/" digit{1,3} "/" digit{1,3} "/" digit{1,3} => { fmt.Printf("%s", data[ts:te]); fret; };
		*|;
		atrack := |*
#			space+ "+" space+;
# track
#			digit => { fmt.Printf("%s,", data[ts:te]); };
#			"," space+;
# Language
			[A-Z] alpha+ => { fmt.Printf("a-%s:", data[ts:te]); };
			space;
			"(";
# Codec
			"AC3" | "DTS" | "aac" => { fmt.Printf("b-%s:", data[ts:te]); };
			")";
			space;
			"(";
# Channels
			digit . "." . digit => { fmt.Printf("c-%s:", data[ts:te]) };
			space;
			"ch) ";
# Ignore this bit
			"(iso" digit{3} "-" digit ":" space lower{3} "),";
# Hertz
			space;
			digit+ "Hz," => { fmt.Printf("d-%s:", data[ts:te-3]) };
			space;
# Bps
			digit+ "bps" => { fmt.Printf("e-%s", data[ts:te-3]); fret; };
			*|;
		word = [a-z]+;
		prefix = space+ "+";
		prefixsp = prefix space;
		stream = prefixsp "stream:" space*;
		duration = prefixsp "duration:";
		size = prefixsp "size:";
		pixelaspect = prefix any+ "pixel" space+ "aspect:";
		displayaspect = prefix any+ "display" space+ "aspect:";
		fps = prefix any+ "fps";
		autocrop = prefixsp "autocrop:";
		audiotrack = prefixsp digit "," space;
		main := (
			newline |
			stream @{ fcall stitle; } |
			duration @{ fcall sduration; } |
			size @{ fcall picture; } |
			pixelaspect @{ fcall paspect; } |
			displayaspect @{ fcall daspect; } |
			fps @{ fcall sfps; } |
			autocrop @{ fcall crop; } |
			prefixsp "chapters:" @{ section = CHAPTER; fmt.Printf("chapter"); } |
			prefixsp "audio tracks:" @{ section = AUDIO; fmt.Printf("audio"); } |
			prefixsp "subtitle tracks:" @{ section = SUBTITLE; fmt.Printf("subtitle"); } |
			audiotrack @{ if section == AUDIO { fcall atrack; } }
		)*;
		write init;
		write exec;
	}%%
	return meta
}

package handbrake

import (
	"fmt"
)

%%{
	machine handbrake;
	write data;
}%%

func parseOutput(data string) (HandBrakeMeta) {
	cs, p, pe, eof := 0, 0, len(data), 0
	top, ts, te, act := 0,0,0,0
	_,_,_,_ = top, ts, te, act
	stack := []int{0}
	_ = eof
	line := 1
	capture := false
	csp := 0
	meta := HandBrakeMeta{}
	fmt.Printf("%02d: ", line)
	_ = capture
	_ = csp
	%%{
		action newline { line +=1; fmt.Printf("\n%02d: ", line) }
		newline = any* '\n' @ newline;
		title := |*
			(alnum|space)+[.]*alnum* => { fmt.Printf("%s", data[ts:te]); fret;};
			"\n" => { fret; };
		*|;
		duration := |*
			space*;
			digit{2}[:]digit{2}[:]digit{2} => { fmt.Printf("%s", data[ts:te]); fret;};
			"\n" => { fret; };
		*|;
		picture := |*
			space*;
			digit{3,4} "x" digit{3,4} => { fmt.Printf("%s", data[ts:te]); fret;};
		*|;
		paspect := |*
			space*;
			digit{1,4} "/" digit{1,4} => { fmt.Printf("%s", data[ts:te]); fret; };
		*|;
		daspect := |*
			space*;
			digit{1} . "." . digit{1,3} => { fmt.Printf("%s", data[ts:te]); fret; };
		*|;
		fps := |*
			"\n" => { ts -= 10; fmt.Printf("%s", data[ts:te-5]); p -= 1; fret; };
		*|;
		crop := |*
			space*;
			digit{1,3} "/" digit{1,3} "/" digit{1,3} "/" digit{1,3} => { fmt.Printf("%s", data[ts:te]); fret; };
		*|;
		main := (
			newline |
			space+ "+" space+ "stream:" space* @{ fcall title; } |
			space+ "+" space+ "duration:" @{ fcall duration; } |
			space+ "+" space+ "size:" @{ fcall picture; } |
			space+ "+" any+ "pixel" space+ "aspect:" @{ fcall paspect; } |
			space+ "+" any+ "display" space+ "aspect:" @{ fcall daspect; } |
			space+ "+" any+ "fps" @{ fcall fps; } |
			space+ "+" space+ "autocrop:" @{ fcall crop; }
		)*;
		write init;
		write exec;
	}%%
	return meta
}


libhext: Parser: explicitly move parsed Rule to caller to avoid copy

// Copyright 2015, 2016 Thomas Trapp
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

#include "Parser.h"

#include "hext/AppendPipe.h"
#include "hext/AttributeCapture.h"
#include "hext/AttributeCountMatch.h"
#include "hext/AttributeMatch.h"
#include "hext/BeginsWithTest.h"
#include "hext/Builtins.h"
#include "hext/Capture.h"
#include "hext/CasePipe.h"
#include "hext/ChildCountMatch.h"
#include "hext/CollapseWsPipe.h"
#include "hext/ContainsTest.h"
#include "hext/ContainsWordsTest.h"
#include "hext/EndsWithTest.h"
#include "hext/EqualsTest.h"
#include "hext/FunctionCapture.h"
#include "hext/FunctionValueMatch.h"
#include "hext/Match.h"
#include "hext/NegateMatch.h"
#include "hext/NegateTest.h"
#include "hext/NthChildMatch.h"
#include "hext/OnlyChildMatch.h"
#include "hext/PrependPipe.h"
#include "hext/RegexPipe.h"
#include "hext/RegexReplacePipe.h"
#include "hext/RegexTest.h"
#include "hext/TrimPipe.h"
#include "hext/ValueTest.h"

#include "PatternValues.h"
#include "StringUtil.h"

#include <vector>
#include <sstream>
#include <utility>
#include <algorithm>

#include <gumbo.h>

namespace hext {

/// The ragel namespace holds ragel's static data.
namespace ragel {
  /// Embed the ragel state machine.
  %%{
    machine hext;
    include "hext-machine.rl";
    write data;
  }%%
} // namespace ragel

Parser::Parser(const char * begin, const char * end) noexcept
: rule_stack_()
, top_rule_(nullptr)
, p_begin_(begin)
, p(begin)
, pe(end)
, eof(end)
, cs(0)
{
}

Parser::~Parser() = default;
Parser::Parser(Parser&&) = default;
Parser& Parser::operator=(Parser&&) = default;

Rule Parser::parse()
{
  // Allow ragel to access its namespace.
  using namespace ragel;

  // Hide clang's warning -Wunused-const-variable.
  (void)hext_first_final;
  (void)hext_error;
  (void)hext_en_main;

  // When calling Parser::parse repeatedly, ensure we are always in a valid
  // state.
  this->p = this->p_begin_;
  this->rule_stack_.clear();
  this->top_rule_ = nullptr;

  // All values required to construct Matches and Captures.
  PatternValues pv;

  const char * tok_begin = nullptr;
  const char * tok_end = nullptr;
  std::string tok = "";

  // Convenience lambda to store the start of a token. Used within the hext
  // machine definition.
  auto tk_start = [this, &tok_begin, &tok_end](){
    tok_begin = this->p;
    tok_end = nullptr;
  };

  // Convenience lambda to complete a token. Used within the hext
  // machine definition.
  auto tk_stop = [this, &tok, &tok_begin, &tok_end](){
    assert(tok_begin);
    assert(this->p);
    tok_end = this->p;
    tok = std::string(
      tok_begin,
      static_cast<std::string::size_type>(std::distance(tok_begin, tok_end))
    );
  };

  // Hide warnings generated by ragel code.
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wsign-conversion"
#pragma GCC diagnostic ignored "-Wold-style-cast"
#pragma GCC diagnostic ignored "-Wpragmas"
#pragma GCC diagnostic ignored "-Wunreachable-code-break"
  %%{
    machine hext;
    include "hext-machine.rl";
    write init;
    write exec;
  }%%
#pragma GCC diagnostic pop

  // Throw error if there are missing closing tags.
  if( this->rule_stack_.size() )
    this->throw_missing_tag(this->rule_stack_.back().get_tag());

  if( this->top_rule_ )
  {
    return std::move(*this->top_rule_);
  }
  else
  {
    return Rule();
  }
}

Rule& Parser::cur_rule()
{
  assert(this->rule_stack_.size());
  return this->rule_stack_.back();
}

void Parser::push_rule()
{
  this->rule_stack_.emplace_back();
}

void Parser::pop_rule()
{
  assert(this->rule_stack_.size());
  if( this->rule_stack_.empty() )
    return;

  Rule rule = std::move(this->rule_stack_.back());
  this->rule_stack_.pop_back();

  if( this->rule_stack_.empty() )
  {
    if( this->top_rule_ )
      this->top_rule_->append_next(std::move(rule));
    else
      this->top_rule_ = std::make_unique<Rule>(std::move(rule));
  }
  else
  {
    this->rule_stack_.back().append_child(std::move(rule));
  }
}

void Parser::set_open_tag_or_throw(const std::string& tag_name)
{
  assert(this->rule_stack_.size());
  if( this->rule_stack_.empty() )
    return;

  if( tag_name.size() == 1 && tag_name[0] == '*' )
  {
    this->rule_stack_.back().set_tag(HtmlTag::ANY);
    return;
  }

  GumboTag tag = gumbo_tag_enum(tag_name.c_str());
  if( tag == GUMBO_TAG_UNKNOWN )
    this->throw_invalid_tag(tag_name);
  else
    this->rule_stack_.back().set_tag(static_cast<HtmlTag>(tag));
}

void Parser::validate_close_tag_or_throw(const std::string& tag_name)
{
  if( this->rule_stack_.empty() )
    this->throw_unexpected_tag(tag_name, /* expected no tag: */ {});

  HtmlTag expected_tag = this->rule_stack_.back().get_tag();
  if( tag_name.size() == 1 && tag_name[0] == '*' )
  {
    if( expected_tag != HtmlTag::ANY )
      this->throw_unexpected_tag(tag_name, expected_tag);
  }
  else
  {
    GumboTag given_tag = gumbo_tag_enum(tag_name.c_str());
    if( given_tag == GUMBO_TAG_UNKNOWN
        || static_cast<HtmlTag>(given_tag) != expected_tag )
      this->throw_unexpected_tag(tag_name, expected_tag);
  }
}

void Parser::throw_unexpected() const
{
  assert(this->p && this->p_begin_ && this->pe);
  assert(this->p <= this->pe && this->p >= this->p_begin_);

  std::stringstream error_msg;
  if( this->p == this->pe || !this->p )
    error_msg << "Premature termination ";
  else
    error_msg << "Unexpected character '"
              << CharName(*(this->p))
              << "' ";

  if( this->p && this->pe )
    this->print_error_location(this->p, /* mark_len: */ 1, error_msg);

  throw SyntaxError(error_msg.str());
}

void Parser::throw_invalid_tag(const std::string& tag) const
{
  std::stringstream error_msg;
  error_msg << "Unknown HTML tag '" << tag << "' ";

  auto unexpected_char = this->p - 1;
  this->print_error_location(unexpected_char, tag.size(), error_msg);

  throw SyntaxError(error_msg.str());
}

void Parser::throw_regex_error(
  std::size_t mark_len,
  boost::regex_constants::error_type e_code
) const
{
  std::stringstream error_msg;
  error_msg << "In regular expression ";

  auto unexpected_char = this->p - 1;
  this->print_error_location(unexpected_char, mark_len, error_msg);

  // regex_error::what() not only contains an error message, but also adds the
  // error location. Therefore we use regex_traits::error_string to get a
  // shorter error description.
  boost::regex_traits<boost::regex::value_type> traits;
  error_msg << "\nError: "
            << traits.error_string(e_code);

  throw SyntaxError(error_msg.str());
}

void Parser::throw_missing_tag(HtmlTag missing) const
{
  std::stringstream error_msg;
  error_msg << "Missing closing tag '</"
            << ( missing == HtmlTag::ANY
                 ? "*"
                 : gumbo_normalized_tagname(static_cast<GumboTag>(missing)) )
            << ">' ";

  this->print_error_location(this->pe, /* mark_len: */ 0, error_msg);

  throw SyntaxError(error_msg.str());
}

void Parser::throw_unexpected_tag(
  const std::string& tag,
  boost::optional<HtmlTag> expected
) const
{
  std::stringstream error_msg;
  error_msg << "Unexpected closing tag '</"
            << tag
            << ">'";

  if( expected )
  {
    error_msg << ", expected '</"
              << ( *expected == HtmlTag::ANY
                   ? "*"
                   : gumbo_normalized_tagname(static_cast<GumboTag>(*expected)) )
              << ">'";
  }

  error_msg << " ";

  auto mark_len = tag.size() + 2; // strlen("</")
  auto unexpected_char = this->p - 1;
  this->print_error_location(unexpected_char, mark_len, error_msg);

  throw SyntaxError(error_msg.str());
}

void Parser::print_error_location(
  const char * uc,
  std::size_t mark_len,
  std::ostream& out
) const
{
  assert(uc && this->p_begin_ && this->pe);
  assert(uc <= this->pe && uc >= this->p_begin_);
  if( !uc || !this->p_begin_ || !this->pe || uc > this->pe )
    return;

  // The zero-based line and char offset of the unexpected character
  CharPosPair pos = CharPosition(this->p_begin_, uc);
  auto line_count = static_cast<std::size_t>(pos.first + 1);
  auto char_count = static_cast<std::size_t>(pos.second + 1);

  if( uc == this->pe )
    out << "at end of input:\n\n";
  else
    out << "at line " << line_count
        << ", char " << char_count << ":\n\n";

  // Don't print the unexpected character if it is a newline
  if( uc == this->pe || *uc == '\n' )
    PrintNumberedLines(this->p_begin_, uc, out);
  else
    PrintNumberedLines(this->p_begin_, uc + 1, out);

  if( mark_len < 1 )
    return;

  // If there are non-ascii characters in the input, lazily bail out.
  if( std::any_of(this->p_begin_, this->pe, [](signed char c){return c < 0;}) )
    return;

  // The longest the mark can be is the length of the last line.
  mark_len = std::min(char_count, mark_len);

  // Print a visual indicator directly under the unexpected token ('^').
  // The required amount of indentation must be known.
  std::size_t indent = static_cast<std::size_t>(DecimalWidth(line_count))
                     + 2            // ": "
                     + char_count   // position of the unexpected character from
                                    // the beginning of the line.
                     - mark_len;    // the length of the '^' mark

  out << std::string(indent, ' ')
      << std::string(mark_len, '^')
      << " here\n";
}

} // namespace hext

// Copyright 2015 Thomas Trapp
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

#include "Parser.h"

#include "hext/AppendPipe.h"
#include "hext/AttributeCapture.h"
#include "hext/AttributeCountMatch.h"
#include "hext/AttributeMatch.h"
#include "hext/BeginsWithTest.h"
#include "hext/Builtins.h"
#include "hext/Capture.h"
#include "hext/CasePipe.h"
#include "hext/ChildCountMatch.h"
#include "hext/CollapseWsPipe.h"
#include "hext/ContainsTest.h"
#include "hext/ContainsWordsTest.h"
#include "hext/EndsWithTest.h"
#include "hext/EqualsTest.h"
#include "hext/FunctionCapture.h"
#include "hext/FunctionValueMatch.h"
#include "hext/Match.h"
#include "hext/NegateMatch.h"
#include "hext/NegateTest.h"
#include "hext/NthChildMatch.h"
#include "hext/OnlyChildMatch.h"
#include "hext/PrependPipe.h"
#include "hext/RegexPipe.h"
#include "hext/RegexReplacePipe.h"
#include "hext/RegexTest.h"
#include "hext/TrimPipe.h"
#include "hext/ValueTest.h"

#include "PatternValues.h"
#include "StringUtil.h"

#include <vector>
#include <sstream>
#include <utility>
#include <algorithm>

#include <gumbo.h>

namespace hext {

/// The ragel namespace holds ragel's static data.
namespace ragel {
  /// Embed the ragel state machine.
  %%{
    machine hext;
    include "hext-machine.rl";
    write data;
  }%%
} // namespace ragel

Parser::Parser(const char * begin, const char * end) noexcept
: rule_stack_()
, top_rule_(nullptr)
, p_begin_(begin)
, p(begin)
, pe(end)
, eof(end)
, cs(0)
{
}

Parser::~Parser() = default;
Parser::Parser(Parser&&) = default;
Parser& Parser::operator=(Parser&&) = default;

Rule Parser::parse()
{
  // Allow ragel to access its namespace.
  using namespace ragel;

  // Hide clang's warning -Wunused-const-variable.
  (void)hext_first_final;
  (void)hext_error;
  (void)hext_en_main;

  // When calling Parser::parse repeatedly, ensure we are always in a valid
  // state.
  this->p = this->p_begin_;
  this->rule_stack_.clear();
  this->top_rule_ = nullptr;

  // All values required to construct Matches and Captures.
  PatternValues pv;

  const char * tok_begin = nullptr;
  const char * tok_end = nullptr;
  std::string tok = "";

  // Convenience lambda to store the start of a token. Used within the hext
  // machine definition.
  auto tk_start = [this, &tok_begin, &tok_end](){
    tok_begin = this->p;
    tok_end = nullptr;
  };

  // Convenience lambda to complete a token. Used within the hext
  // machine definition.
  auto tk_stop = [this, &tok, &tok_begin, &tok_end](){
    assert(tok_begin);
    assert(this->p);
    tok_end = this->p;
    tok = std::string(
      tok_begin,
      static_cast<std::string::size_type>(std::distance(tok_begin, tok_end))
    );
  };

  // Hide warnings generated by ragel code.
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wsign-conversion"
#pragma GCC diagnostic ignored "-Wold-style-cast"
#pragma GCC diagnostic ignored "-Wpragmas"
#pragma GCC diagnostic ignored "-Wunreachable-code-break"
  %%{
    machine hext;
    include "hext-machine.rl";
    write init;
    write exec;
  }%%
#pragma GCC diagnostic pop

  // Throw error if there are missing closing tags.
  if( this->rule_stack_.size() )
    this->throw_missing_tag(this->rule_stack_.back().get_tag());

  if( this->top_rule_ )
  {
    return *this->top_rule_;
  }
  else
  {
    return Rule();
  }
}

Rule& Parser::cur_rule()
{
  assert(this->rule_stack_.size());
  return this->rule_stack_.back();
}

void Parser::push_rule()
{
  this->rule_stack_.emplace_back();
}

void Parser::pop_rule()
{
  assert(this->rule_stack_.size());
  if( this->rule_stack_.empty() )
    return;

  Rule rule = std::move(this->rule_stack_.back());
  this->rule_stack_.pop_back();

  if( this->rule_stack_.empty() )
  {
    if( this->top_rule_ )
      this->top_rule_->append_next(std::move(rule));
    else
      this->top_rule_ = std::make_unique<Rule>(std::move(rule));
  }
  else
  {
    this->rule_stack_.back().append_child(std::move(rule));
  }
}

void Parser::set_open_tag_or_throw(const std::string& tag_name)
{
  assert(this->rule_stack_.size());
  if( this->rule_stack_.empty() )
    return;

  if( tag_name.size() == 1 && tag_name[0] == '*' )
  {
    this->rule_stack_.back().set_tag(HtmlTag::ANY);
    return;
  }

  GumboTag tag = gumbo_tag_enum(tag_name.c_str());
  if( tag == GUMBO_TAG_UNKNOWN )
    this->throw_invalid_tag(tag_name);
  else
    this->rule_stack_.back().set_tag(static_cast<HtmlTag>(tag));
}

void Parser::validate_close_tag_or_throw(const std::string& tag_name)
{
  if( this->rule_stack_.empty() )
    this->throw_unexpected_tag(tag_name, /* expected no tag: */ {});

  HtmlTag expected_tag = this->rule_stack_.back().get_tag();
  if( tag_name.size() == 1 && tag_name[0] == '*' )
  {
    if( expected_tag != HtmlTag::ANY )
      this->throw_unexpected_tag(tag_name, expected_tag);
  }
  else
  {
    GumboTag given_tag = gumbo_tag_enum(tag_name.c_str());
    if( given_tag == GUMBO_TAG_UNKNOWN
        || static_cast<HtmlTag>(given_tag) != expected_tag )
      this->throw_unexpected_tag(tag_name, expected_tag);
  }
}

void Parser::throw_unexpected() const
{
  assert(this->p && this->p_begin_ && this->pe);
  assert(this->p <= this->pe && this->p >= this->p_begin_);

  std::stringstream error_msg;
  if( this->p == this->pe || !this->p )
    error_msg << "Premature termination ";
  else
    error_msg << "Unexpected character '"
              << CharName(*(this->p))
              << "' ";

  if( this->p && this->pe )
    this->print_error_location(this->p, /* mark_len: */ 1, error_msg);

  throw SyntaxError(error_msg.str());
}

void Parser::throw_invalid_tag(const std::string& tag) const
{
  std::stringstream error_msg;
  error_msg << "Unknown HTML tag '" << tag << "' ";

  auto unexpected_char = this->p - 1;
  this->print_error_location(unexpected_char, tag.size(), error_msg);

  throw SyntaxError(error_msg.str());
}

void Parser::throw_regex_error(
  std::size_t mark_len,
  boost::regex_constants::error_type e_code
) const
{
  std::stringstream error_msg;
  error_msg << "In regular expression ";

  auto unexpected_char = this->p - 1;
  this->print_error_location(unexpected_char, mark_len, error_msg);

  // regex_error::what() not only contains an error message, but also adds the
  // error location. Therefore we use regex_traits::error_string to get a
  // shorter error description.
  boost::regex_traits<boost::regex::value_type> traits;
  error_msg << "\nError: "
            << traits.error_string(e_code);

  throw SyntaxError(error_msg.str());
}

void Parser::throw_missing_tag(HtmlTag missing) const
{
  std::stringstream error_msg;
  error_msg << "Missing closing tag '</"
            << ( missing == HtmlTag::ANY
                 ? "*"
                 : gumbo_normalized_tagname(static_cast<GumboTag>(missing)) )
            << ">' ";

  this->print_error_location(this->pe, /* mark_len: */ 0, error_msg);

  throw SyntaxError(error_msg.str());
}

void Parser::throw_unexpected_tag(
  const std::string& tag,
  boost::optional<HtmlTag> expected
) const
{
  std::stringstream error_msg;
  error_msg << "Unexpected closing tag '</"
            << tag
            << ">'";

  if( expected )
  {
    error_msg << ", expected '</"
              << ( *expected == HtmlTag::ANY
                   ? "*"
                   : gumbo_normalized_tagname(static_cast<GumboTag>(*expected)) )
              << ">'";
  }

  error_msg << " ";

  auto mark_len = tag.size() + 2; // strlen("</")
  auto unexpected_char = this->p - 1;
  this->print_error_location(unexpected_char, mark_len, error_msg);

  throw SyntaxError(error_msg.str());
}

void Parser::print_error_location(
  const char * uc,
  std::size_t mark_len,
  std::ostream& out
) const
{
  assert(uc && this->p_begin_ && this->pe);
  assert(uc <= this->pe && uc >= this->p_begin_);
  if( !uc || !this->p_begin_ || !this->pe || uc > this->pe )
    return;

  // The zero-based line and char offset of the unexpected character
  CharPosPair pos = CharPosition(this->p_begin_, uc);
  auto line_count = static_cast<std::size_t>(pos.first + 1);
  auto char_count = static_cast<std::size_t>(pos.second + 1);

  if( uc == this->pe )
    out << "at end of input:\n\n";
  else
    out << "at line " << line_count
        << ", char " << char_count << ":\n\n";

  // Don't print the unexpected character if it is a newline
  if( uc == this->pe || *uc == '\n' )
    PrintNumberedLines(this->p_begin_, uc, out);
  else
    PrintNumberedLines(this->p_begin_, uc + 1, out);

  if( mark_len < 1 )
    return;

  // If there are non-ascii characters in the input, lazily bail out.
  if( std::any_of(this->p_begin_, this->pe, [](signed char c){return c < 0;}) )
    return;

  // The longest the mark can be is the length of the last line.
  mark_len = std::min(char_count, mark_len);

  // Print a visual indicator directly under the unexpected token ('^').
  // The required amount of indentation must be known.
  std::size_t indent = static_cast<std::size_t>(DecimalWidth(line_count))
                     + 2            // ": "
                     + char_count   // position of the unexpected character from
                                    // the beginning of the line.
                     - mark_len;    // the length of the '^' mark

  out << std::string(indent, ' ')
      << std::string(mark_len, '^')
      << " here\n";
}

} // namespace hext