ifchd-parse: Simplify leftover-buffer copying.

/* ifchd-parse.rl - interface change daemon parser
 *
 * Copyright 2004-2018 Nicholas J. Kain <njkain at gmail dot com>
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are met:
 *
 * - Redistributions of source code must retain the above copyright notice,
 *   this list of conditions and the following disclaimer.
 *
 * - Redistributions in binary form must reproduce the above copyright notice,
 *   this list of conditions and the following disclaimer in the documentation
 *   and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 * POSSIBILITY OF SUCH DAMAGE.
 */

#include <stddef.h>
#include <stdbool.h>
#include <stdio.h>
#include <string.h>
#include <arpa/inet.h>
#include "nk/log.h"

#include "ifchd-parse.h"
#include "ifchd.h"
#include "ifset.h"
#include "ndhc.h"

%%{
    machine ipv4set_parser;

    action XSt { arg_start = p; }
    action IpEn {
        ptrdiff_t arg_len = p - arg_start;
        if (arg_len > 0 && (size_t)arg_len < sizeof ip4_addr) {
            have_ip = true;
            memcpy(ip4_addr, arg_start, (size_t)arg_len);
        }
        ip4_addr[arg_len] = 0;
    }
    action SnEn {
        ptrdiff_t arg_len = p - arg_start;
        if (arg_len > 0 && (size_t)arg_len < sizeof ip4_subnet) {
            have_subnet = true;
            memcpy(ip4_subnet, arg_start, (size_t)arg_len);
        }
        ip4_subnet[arg_len] = 0;
    }
    action BcEn {
        ptrdiff_t arg_len = p - arg_start;
        if (arg_len > 0 && (size_t)arg_len < sizeof ip4_bcast) {
            have_ip = true;
            memcpy(ip4_bcast, arg_start, (size_t)arg_len);
        }
        ip4_bcast[arg_len] = 0;
    }

    v4addr = digit{1,3} '.' digit{1,3} '.' digit{1,3} '.' digit{1,3};
    ip4_nobc = (v4addr > XSt % IpEn) ',' (v4addr > XSt % SnEn);
    ip4_bc = (v4addr > XSt % IpEn) ',' (v4addr > XSt % SnEn) ','
             (v4addr > XSt % BcEn);
    main := (ip4_bc|ip4_nobc);
}%%

%% write data;

static int perform_ip4set(const char buf[static 1], size_t len)
{
    char ip4_addr[INET_ADDRSTRLEN];
    char ip4_subnet[INET_ADDRSTRLEN];
    char ip4_bcast[INET_ADDRSTRLEN];
    const char *p = buf;
    const char *pe = p + len;
    const char *eof = pe;
    const char *arg_start = p;
    int cs = 0;
    bool have_ip = false;
    bool have_subnet = false;
    bool have_bcast = false;

    %% write init;
    %% write exec;

    if (cs < ipv4set_parser_first_final) {
        log_line("%s: received invalid arguments", __func__);
        return -1;
    }

    // These should never trigger because of the above check, but be safe...
    if (!have_ip) {
        log_line("%s: No IPv4 address specified.", __func__);
        return -1;
    }
    if (!have_subnet) {
        log_line("%s: No IPv4 subnet specified.", __func__);
        return -1;
    }

    return perform_ip_subnet_bcast(ip4_addr, ip4_subnet,
                                   have_bcast ? ip4_bcast : (char *)0);
}

%%{
    machine ifchd_parser;

    action Reset { cl.state = STATE_NOTHING; }
    action ArgSt { arg_start = p; }
    action ArgEn {
        ptrdiff_t al = p - arg_start;
        if (al < 0 || (size_t)al > sizeof tb - 1) {
            log_line("command argument would overflow");
            return -99;
        }
        arg_len = (size_t)al;
        memcpy(tb, arg_start, arg_len);
        tb[arg_len] = 0;
    }

    action Dispatch {
        int pr = 0;
        switch (cl.state) {
        case STATE_IP4SET: pr = perform_ip4set(tb, arg_len); break;
        case STATE_TIMEZONE: pr = perform_timezone( tb, arg_len); break;
        case STATE_ROUTER: pr = perform_router(tb, arg_len); break;
        case STATE_DNS: pr = perform_dns(tb, arg_len); break;
        case STATE_LPRSVR: pr = perform_lprsvr(tb, arg_len); break;
        case STATE_HOSTNAME: pr = perform_hostname(tb, arg_len); break;
        case STATE_DOMAIN: pr = perform_domain(tb, arg_len); break;
        case STATE_IPTTL: pr = perform_ipttl(tb, arg_len); break;
        case STATE_MTU: pr = perform_mtu(tb, arg_len); break;
        case STATE_NTPSVR: pr = perform_ntpsrv(tb, arg_len); break;
        case STATE_WINS: pr = perform_wins(tb, arg_len); break;
        case STATE_CARRIER: pr = perform_carrier(); break;
        default:
            arg_len = 0;
            log_line("error: invalid state in dispatch_work");
            return -99;
        }
        arg_len = 0;
        if (pr == -99)
            return -99;
        cmdf |= pr;
    }

    terminator = ';' > Dispatch;
    v4addr = digit{1,3} '.' digit{1,3} '.' digit{1,3} '.' digit{1,3};
    ip_arg = (v4addr > ArgSt % ArgEn) terminator;
    ip4set_arg = (((v4addr ','){1,2} v4addr) > ArgSt % ArgEn) terminator;
    iplist_arg = (((v4addr ',')* v4addr) > ArgSt % ArgEn) terminator;
    str_arg = ([^;\0]+ > ArgSt % ArgEn) terminator;
    s32_arg = (extend{4} > ArgSt % ArgEn) terminator;
    u16_arg = (extend{2} > ArgSt % ArgEn) terminator;
    u8_arg = (extend{1} > ArgSt % ArgEn) terminator;

    cmd_ip = ('routr:' % { cl.state = STATE_ROUTER; }) ip_arg;
    cmd_ip4set = ('ip4:' % { cl.state = STATE_IP4SET; }) ip4set_arg;
    cmd_iplist = ('dns:' % { cl.state = STATE_DNS; }
                 |'lpr:' % { cl.state = STATE_LPRSVR; }
                 |'ntp:' % { cl.state = STATE_NTPSVR; }
                 |'wins:' % { cl.state = STATE_WINS; }
                 ) iplist_arg;
    cmd_str = ('host:' % { cl.state = STATE_HOSTNAME; }
              |'dom:' % { cl.state = STATE_DOMAIN; }
              ) str_arg;
    cmd_s32 = ('tzone:' % { cl.state = STATE_TIMEZONE; }) s32_arg;
    cmd_u16 = ('mtu:' % { cl.state = STATE_MTU; }) u16_arg;
    cmd_u8  = ('ipttl:' % { cl.state = STATE_IPTTL; }) u8_arg;
    cmd_none = ('carrier:' % { cl.state = STATE_CARRIER; }) terminator;

    command = (cmd_ip|cmd_ip4set|cmd_iplist|cmd_str|cmd_s32|cmd_u16|cmd_u8|cmd_none);
    main := (command > Reset)+;
}%%

%% write data;

/*
 * Returns -99 on fatal error; that leads to peer connection being closed.
 * Returns -1 if one of the commands failed.
 * Returns 0 on success.
 */
int execute_buffer(const char newbuf[static 1])
{
    char buf[MAX_BUF * 2];
    char tb[MAX_BUF];
    int cmdf = 0;

    ssize_t buflen = snprintf(buf, sizeof buf, "%s%s", cl.ibuf, newbuf);
    if (buflen < 0) {
        log_line("%s: (%s) snprintf1 failed; your system is broken?",
                 client_config.interface, __func__);
        return -99;
    }
    if ((size_t)buflen >= sizeof buf) {
        log_line("%s: (%s) input is too long for buffer",
                 client_config.interface, __func__);
        return -99;
    }

    const char *p = buf;
    const char *pe = p + strlen(buf);
    const char *arg_start = p;
    size_t arg_len = 0;
    int cs = 0;

    %% write init;
    %% write exec;

    if (p != pe) {
        ssize_t ilen = snprintf(cl.ibuf, sizeof cl.ibuf, "%s", p);
        if (ilen < 0) {
            log_line("%s: (%s) snprintf2 failed; your system is broken?",
                     client_config.interface, __func__);
            return -99;
        }
        if ((size_t)ilen >= sizeof buf) {
            log_line("%s: (%s) unconsumed input too long for buffer",
                     client_config.interface, __func__);
            return -99;
        }
    }

    if (cs < ifchd_parser_first_final) {
        log_line("%s: ifch received invalid commands",
                 client_config.interface);
        return -99;
    }
    return !cmdf ? 0 : -1;
}

/* ifchd-parse.rl - interface change daemon parser
 *
 * Copyright 2004-2018 Nicholas J. Kain <njkain at gmail dot com>
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are met:
 *
 * - Redistributions of source code must retain the above copyright notice,
 *   this list of conditions and the following disclaimer.
 *
 * - Redistributions in binary form must reproduce the above copyright notice,
 *   this list of conditions and the following disclaimer in the documentation
 *   and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 * POSSIBILITY OF SUCH DAMAGE.
 */

#include <stddef.h>
#include <stdbool.h>
#include <stdio.h>
#include <string.h>
#include <arpa/inet.h>
#include "nk/log.h"

#include "ifchd-parse.h"
#include "ifchd.h"
#include "ifset.h"
#include "ndhc.h"

%%{
    machine ipv4set_parser;

    action XSt { arg_start = p; }
    action IpEn {
        ptrdiff_t arg_len = p - arg_start;
        if (arg_len > 0 && (size_t)arg_len < sizeof ip4_addr) {
            have_ip = true;
            memcpy(ip4_addr, arg_start, (size_t)arg_len);
        }
        ip4_addr[arg_len] = 0;
    }
    action SnEn {
        ptrdiff_t arg_len = p - arg_start;
        if (arg_len > 0 && (size_t)arg_len < sizeof ip4_subnet) {
            have_subnet = true;
            memcpy(ip4_subnet, arg_start, (size_t)arg_len);
        }
        ip4_subnet[arg_len] = 0;
    }
    action BcEn {
        ptrdiff_t arg_len = p - arg_start;
        if (arg_len > 0 && (size_t)arg_len < sizeof ip4_bcast) {
            have_ip = true;
            memcpy(ip4_bcast, arg_start, (size_t)arg_len);
        }
        ip4_bcast[arg_len] = 0;
    }

    v4addr = digit{1,3} '.' digit{1,3} '.' digit{1,3} '.' digit{1,3};
    ip4_nobc = (v4addr > XSt % IpEn) ',' (v4addr > XSt % SnEn);
    ip4_bc = (v4addr > XSt % IpEn) ',' (v4addr > XSt % SnEn) ','
             (v4addr > XSt % BcEn);
    main := (ip4_bc|ip4_nobc);
}%%

%% write data;

static int perform_ip4set(const char buf[static 1], size_t len)
{
    char ip4_addr[INET_ADDRSTRLEN];
    char ip4_subnet[INET_ADDRSTRLEN];
    char ip4_bcast[INET_ADDRSTRLEN];
    const char *p = buf;
    const char *pe = p + len;
    const char *eof = pe;
    const char *arg_start = p;
    int cs = 0;
    bool have_ip = false;
    bool have_subnet = false;
    bool have_bcast = false;

    %% write init;
    %% write exec;

    if (cs < ipv4set_parser_first_final) {
        log_line("%s: received invalid arguments", __func__);
        return -1;
    }

    // These should never trigger because of the above check, but be safe...
    if (!have_ip) {
        log_line("%s: No IPv4 address specified.", __func__);
        return -1;
    }
    if (!have_subnet) {
        log_line("%s: No IPv4 subnet specified.", __func__);
        return -1;
    }

    return perform_ip_subnet_bcast(ip4_addr, ip4_subnet,
                                   have_bcast ? ip4_bcast : (char *)0);
}

%%{
    machine ifchd_parser;

    action Reset { cl.state = STATE_NOTHING; }
    action ArgSt { arg_start = p; }
    action ArgEn {
        ptrdiff_t al = p - arg_start;
        if (al < 0 || (size_t)al > sizeof tb - 1) {
            log_line("command argument would overflow");
            return -99;
        }
        arg_len = (size_t)al;
        memcpy(tb, arg_start, arg_len);
        tb[arg_len] = 0;
    }

    action Dispatch {
        int pr = 0;
        switch (cl.state) {
        case STATE_IP4SET: pr = perform_ip4set(tb, arg_len); break;
        case STATE_TIMEZONE: pr = perform_timezone( tb, arg_len); break;
        case STATE_ROUTER: pr = perform_router(tb, arg_len); break;
        case STATE_DNS: pr = perform_dns(tb, arg_len); break;
        case STATE_LPRSVR: pr = perform_lprsvr(tb, arg_len); break;
        case STATE_HOSTNAME: pr = perform_hostname(tb, arg_len); break;
        case STATE_DOMAIN: pr = perform_domain(tb, arg_len); break;
        case STATE_IPTTL: pr = perform_ipttl(tb, arg_len); break;
        case STATE_MTU: pr = perform_mtu(tb, arg_len); break;
        case STATE_NTPSVR: pr = perform_ntpsrv(tb, arg_len); break;
        case STATE_WINS: pr = perform_wins(tb, arg_len); break;
        case STATE_CARRIER: pr = perform_carrier(); break;
        default:
            arg_len = 0;
            log_line("error: invalid state in dispatch_work");
            return -99;
        }
        arg_len = 0;
        if (pr == -99)
            return -99;
        cmdf |= pr;
    }

    terminator = ';' > Dispatch;
    v4addr = digit{1,3} '.' digit{1,3} '.' digit{1,3} '.' digit{1,3};
    ip_arg = (v4addr > ArgSt % ArgEn) terminator;
    ip4set_arg = (((v4addr ','){1,2} v4addr) > ArgSt % ArgEn) terminator;
    iplist_arg = (((v4addr ',')* v4addr) > ArgSt % ArgEn) terminator;
    str_arg = ([^;\0]+ > ArgSt % ArgEn) terminator;
    s32_arg = (extend{4} > ArgSt % ArgEn) terminator;
    u16_arg = (extend{2} > ArgSt % ArgEn) terminator;
    u8_arg = (extend{1} > ArgSt % ArgEn) terminator;

    cmd_ip = ('routr:' % { cl.state = STATE_ROUTER; }) ip_arg;
    cmd_ip4set = ('ip4:' % { cl.state = STATE_IP4SET; }) ip4set_arg;
    cmd_iplist = ('dns:' % { cl.state = STATE_DNS; }
                 |'lpr:' % { cl.state = STATE_LPRSVR; }
                 |'ntp:' % { cl.state = STATE_NTPSVR; }
                 |'wins:' % { cl.state = STATE_WINS; }
                 ) iplist_arg;
    cmd_str = ('host:' % { cl.state = STATE_HOSTNAME; }
              |'dom:' % { cl.state = STATE_DOMAIN; }
              ) str_arg;
    cmd_s32 = ('tzone:' % { cl.state = STATE_TIMEZONE; }) s32_arg;
    cmd_u16 = ('mtu:' % { cl.state = STATE_MTU; }) u16_arg;
    cmd_u8  = ('ipttl:' % { cl.state = STATE_IPTTL; }) u8_arg;
    cmd_none = ('carrier:' % { cl.state = STATE_CARRIER; }) terminator;

    command = (cmd_ip|cmd_ip4set|cmd_iplist|cmd_str|cmd_s32|cmd_u16|cmd_u8|cmd_none);
    main := (command > Reset)+;
}%%

%% write data;

/*
 * Returns -99 on fatal error; that leads to peer connection being closed.
 * Returns -1 if one of the commands failed.
 * Returns 0 on success.
 */
int execute_buffer(const char newbuf[static 1])
{
    char buf[MAX_BUF * 2];
    char tb[MAX_BUF];
    int cmdf = 0;

    ssize_t buflen = snprintf(buf, sizeof buf, "%s%s", cl.ibuf, newbuf);
    if (buflen < 0) {
        log_line("%s: (%s) snprintf1 failed; your system is broken?",
                 client_config.interface, __func__);
        return -99;
    }
    if ((size_t)buflen >= sizeof buf) {
        log_line("%s: (%s) input is too long for buffer",
                 client_config.interface, __func__);
        return -99;
    }

    size_t init_siz = strlen(buf);
    const char *p = buf;
    const char *pe = p + init_siz;
    const char *arg_start = p;
    size_t arg_len = 0;
    int cs = 0;

    %% write init;
    %% write exec;

    ptrdiff_t blt = pe - p;
    size_t bytes_left = blt >= 0 ? (size_t)blt : 0;
    if (bytes_left > 0) {
        size_t taken = init_siz - bytes_left;
        ssize_t ilen = snprintf(cl.ibuf, sizeof cl.ibuf, "%s", buf + taken);
        if (ilen < 0) {
            log_line("%s: (%s) snprintf2 failed; your system is broken?",
                     client_config.interface, __func__);
            return -99;
        }
        if ((size_t)ilen >= sizeof buf) {
            log_line("%s: (%s) unconsumed input too long for buffer",
                     client_config.interface, __func__);
            return -99;
        }
    }

    if (cs < ifchd_parser_first_final) {
        log_line("%s: ifch received invalid commands",
                 client_config.interface);
        return -99;
    }
    return !cmdf ? 0 : -1;
}


mark lex_error's single argument constructor explicit

#ifndef HEXT_LEXER_H
#define HEXT_LEXER_H

#include <string>
#include <sstream>
#include <stdexcept>
#include <utility>

#include "hext/token.h"
#include "hext/lexer-util.h"

namespace hext {

namespace ragel {
  %%{
    machine hext;
    include "hext.rl";
    write data;
  }%%
} // namespace ragel

class lexer
{
public:
  class lex_error : public std::runtime_error
  {
  public:
    explicit lex_error(const std::string& msg)
    : std::runtime_error(msg) {}
  };

  lexer(const char * begin, const char * end)
  : p_begin(begin),
    p(begin),
    pe(end),
    eof(end),
    cs(0)
  {
    using namespace ragel;
    %% write init;
  }

  std::vector<token> lex()
  {
    using namespace ragel;

    std::vector<token> tokens;

    // this throws on lexing error
    %% write exec;

    return tokens;
  }

  void throw_error() const
  {
    std::pair<size_t, size_t> pos =
      get_char_position(this->p, this->p_begin, this->pe);

    std::string char_name;
    if( this->p == this->pe )
    {
      char_name = "[eof]";
    }
    else
    {
      if( this->p != nullptr )
        char_name = get_char_name(*this->p);
      else
        char_name = "[unknown]";
    }

    std::stringstream error_msg;
    error_msg << "Error at line "
              << pos.first + 1
              << ", char "
              << pos.second + 1
              << ", unexpected character '"
              << char_name
              << "'";

    throw lex_error(error_msg.str());
  }

private:
  const char * p_begin;
  const char * p;
  const char * pe;
  const char * eof;
  int cs;
};

} // namespace hext

#endif // HEXT_LEXER_H

#ifndef HEXT_LEXER_H
#define HEXT_LEXER_H

#include <string>
#include <sstream>
#include <stdexcept>
#include <utility>

#include "hext/token.h"
#include "hext/lexer-util.h"

namespace hext {

namespace ragel {
  %%{
    machine hext;
    include "hext.rl";
    write data;
  }%%
} // namespace ragel

class lexer
{
public:
  class lex_error : public std::runtime_error
  {
  public:
    lex_error(const std::string& msg)
    : std::runtime_error(msg) {}
  };

  lexer(const char * begin, const char * end)
  : p_begin(begin),
    p(begin),
    pe(end),
    eof(end),
    cs(0)
  {
    using namespace ragel;
    %% write init;
  }

  std::vector<token> lex()
  {
    using namespace ragel;

    std::vector<token> tokens;

    // this throws on lexing error
    %% write exec;

    return tokens;
  }

  void throw_error() const
  {
    std::pair<size_t, size_t> pos =
      get_char_position(this->p, this->p_begin, this->pe);

    std::string char_name;
    if( this->p == this->pe )
    {
      char_name = "[eof]";
    }
    else
    {
      if( this->p != nullptr )
        char_name = get_char_name(*this->p);
      else
        char_name = "[unknown]";
    }

    std::stringstream error_msg;
    error_msg << "Error at line "
              << pos.first + 1
              << ", char "
              << pos.second + 1
              << ", unexpected character '"
              << char_name
              << "'";

    throw lex_error(error_msg.str());
  }

private:
  const char * p_begin;
  const char * p;
  const char * pe;
  const char * eof;
  int cs;
};

} // namespace hext

#endif // HEXT_LEXER_H


Use squote/dquote everywhere in the lexer.

%%machine lexer; # %

module Oga
  ##
  #
  class Lexer
    %% write data; # %

    # Lazy way of forwarding instance method calls used internally by Ragel to
    # their corresponding class methods.
    private_methods.grep(/^_lexer_/).each do |name|
      define_method(name) do
        return self.class.send(name)
      end

      private(name)
    end

    def initialize
      reset
    end

    def reset
      @line   = 1
      @column = 1
      @data   = nil
      @ts     = nil
      @te     = nil
      @tokens = []
      @stack  = []
      @top    = 0

      @string_buffer = ''
      @text_buffer   = ''
    end

    def lex(data)
      @data       = data
      lexer_start = self.class.lexer_start
      eof         = data.length

      %% write init;
      %% write exec;

      tokens = @tokens

      reset

      return tokens
    end

    private

    def advance_line
      @line  += 1
      @column = 1
    end

    def advance_column(length = 1)
      @column += length
    end

    def t(type, start = @ts, stop = @te)
      value = text(start, stop)

      add_token(type, value)
    end

    def text(start = @ts, stop = @te)
      return @data[start...stop]
    end

    def add_token(type, value)
      token = [type, value, @line, @column]

      advance_column(value.length)

      @tokens << token
    end

    def emit_text_buffer
      add_token(:T_TEXT, @text_buffer)

      @text_buffer = ''
    end

    def emit_string_buffer
      add_token(:T_STRING, @string_buffer)
      advance_column

      @string_buffer = ''
    end

    %%{
      # Use instance variables for `ts` and friends.
      access @;

      newline    = '\n' | '\r\n';
      whitespace = [ \t];

      action emit_space {
        t(:T_SPACE)
      }

      action emit_newline {
        t(:T_NEWLINE)
        advance_line
      }

      # String processing
      #
      # These actions/definitions can be used to process single and/or double
      # quoted strings (e.g. for tag attribute values).
      #
      # The string_dquote and string_squote machines should not be used
      # directly, instead the corresponding actions should be used.
      #
      dquote = '"';
      squote = "'";

      action buffer_text {
        @text_buffer << text
      }

      action buffer_string {
        @string_buffer << text
      }

      action string_dquote {
        advance_column
        fcall string_dquote;
      }

      action string_squote {
        advance_column
        fcall string_squote;
      }

      string_dquote := |*
        ^dquote => buffer_string;
        dquote  => {
          emit_string_buffer
          fret;
        };
      *|;

      string_squote := |*
        ^squote => buffer_string;
        squote  => {
          emit_string_buffer
          fret;
        };
      *|;

      # DOCTYPES
      #
      # http://www.w3.org/TR/html-markup/syntax.html#doctype-syntax
      #
      # These rules support the 3 flavours of doctypes:
      #
      # 1. Normal doctypes, as introduced in the HTML5 specification.
      # 2. Deprecated doctypes, the more verbose ones used prior to HTML5.
      # 3. Legacy doctypes
      #
      doctype_start = '<!DOCTYPE'i whitespace+ 'HTML'i;

      doctype := |*
        'PUBLIC' | 'SYSTEM' => { t(:T_DOCTYPE_TYPE) };

        # Lex the public/system IDs as regular strings.
        dquote => string_dquote;
        squote => string_squote;

        # Whitespace inside doctypes is ignored since there's no point in
        # including it.
        whitespace => { advance_column };

        '>' => {
          t(:T_DOCTYPE_END)
          fgoto main;
        };
      *|;

      # CDATA
      #
      # http://www.w3.org/TR/html-markup/syntax.html#cdata-sections
      #
      # CDATA tags are broken up into 3 parts: the start, the content and the
      # end tag.
      #
      # In HTML CDATA tags have no meaning/are not supported. Oga does
      # support them but treats their contents as plain text.
      #
      cdata_start = '<![CDATA[';
      cdata_end   = ']]>';

      cdata := |*
        cdata_end => {
          emit_text_buffer

          t(:T_CDATA_END)

          fgoto main;
        };

        # Consume everything else character by character and store it in a
        # separate buffer.
        any => buffer_text;
      *|;

      main := |*
        whitespace => emit_space;
        newline    => emit_newline;

        doctype_start => {
          t(:T_DOCTYPE_START)

          fgoto doctype;
        };

        # @cdata_buffer is used to store the content of the CDATA tag.
        cdata_start => {
          t(:T_CDATA_START)

          fgoto cdata;
        };

        # General rules and actions.
        '<' => { t(:T_SMALLER) };
        '>' => { t(:T_GREATER) };
        '/' => { t(:T_SLASH) };
        '-' => { t(:T_DASH) };
        ']' => { t(:T_RBRACKET) };
        '[' => { t(:T_LBRACKET) };
        ':' => { t(:T_COLON) };
        '!' => { t(:T_BANG) };
        '=' => { t(:T_EQUALS) };

        dquote => { t(:T_DQUOTE) };
        squote => { t(:T_SQUOTE) };
      *|;
    }%%
  end # Lexer
end # Oga

%%machine lexer; # %

module Oga
  ##
  #
  class Lexer
    %% write data; # %

    # Lazy way of forwarding instance method calls used internally by Ragel to
    # their corresponding class methods.
    private_methods.grep(/^_lexer_/).each do |name|
      define_method(name) do
        return self.class.send(name)
      end

      private(name)
    end

    def initialize
      reset
    end

    def reset
      @line   = 1
      @column = 1
      @data   = nil
      @ts     = nil
      @te     = nil
      @tokens = []
      @stack  = []
      @top    = 0

      @string_buffer = ''
      @text_buffer   = ''
    end

    def lex(data)
      @data       = data
      lexer_start = self.class.lexer_start
      eof         = data.length

      %% write init;
      %% write exec;

      tokens = @tokens

      reset

      return tokens
    end

    private

    def advance_line
      @line  += 1
      @column = 1
    end

    def advance_column(length = 1)
      @column += length
    end

    def t(type, start = @ts, stop = @te)
      value = text(start, stop)

      add_token(type, value)
    end

    def text(start = @ts, stop = @te)
      return @data[start...stop]
    end

    def add_token(type, value)
      token = [type, value, @line, @column]

      advance_column(value.length)

      @tokens << token
    end

    def emit_text_buffer
      add_token(:T_TEXT, @text_buffer)

      @text_buffer = ''
    end

    def emit_string_buffer
      add_token(:T_STRING, @string_buffer)
      advance_column

      @string_buffer = ''
    end

    %%{
      # Use instance variables for `ts` and friends.
      access @;

      newline    = '\n' | '\r\n';
      whitespace = [ \t];

      action emit_space {
        t(:T_SPACE)
      }

      action emit_newline {
        t(:T_NEWLINE)
        advance_line
      }

      # String processing
      #
      # These actions/definitions can be used to process single and/or double
      # quoted strings (e.g. for tag attribute values).
      #
      # The string_dquote and string_squote machines should not be used
      # directly, instead the corresponding actions should be used.
      #
      dquote = '"';
      squote = "'";

      action buffer_text {
        @text_buffer << text
      }

      action buffer_string {
        @string_buffer << text
      }

      action string_dquote {
        advance_column
        fcall string_dquote;
      }

      action string_squote {
        advance_column
        fcall string_squote;
      }

      string_dquote := |*
        ^dquote => buffer_string;
        dquote  => {
          emit_string_buffer
          fret;
        };
      *|;

      string_squote := |*
        ^squote => buffer_string;
        squote  => {
          emit_string_buffer
          fret;
        };
      *|;

      # DOCTYPES
      #
      # http://www.w3.org/TR/html-markup/syntax.html#doctype-syntax
      #
      # These rules support the 3 flavours of doctypes:
      #
      # 1. Normal doctypes, as introduced in the HTML5 specification.
      # 2. Deprecated doctypes, the more verbose ones used prior to HTML5.
      # 3. Legacy doctypes
      #
      doctype_start = '<!DOCTYPE'i whitespace+ 'HTML'i;

      doctype := |*
        'PUBLIC' | 'SYSTEM' => { t(:T_DOCTYPE_TYPE) };

        # Lex the public/system IDs as regular strings.
        dquote => string_dquote;
        squote => string_squote;

        # Whitespace inside doctypes is ignored since there's no point in
        # including it.
        whitespace => { advance_column };

        '>' => {
          t(:T_DOCTYPE_END)
          fgoto main;
        };
      *|;

      # CDATA
      #
      # http://www.w3.org/TR/html-markup/syntax.html#cdata-sections
      #
      # CDATA tags are broken up into 3 parts: the start, the content and the
      # end tag.
      #
      # In HTML CDATA tags have no meaning/are not supported. Oga does
      # support them but treats their contents as plain text.
      #
      cdata_start = '<![CDATA[';
      cdata_end   = ']]>';

      cdata := |*
        cdata_end => {
          emit_text_buffer

          t(:T_CDATA_END)

          fgoto main;
        };

        # Consume everything else character by character and store it in a
        # separate buffer.
        any => buffer_text;
      *|;

      main := |*
        whitespace => emit_space;
        newline    => emit_newline;

        doctype_start => {
          t(:T_DOCTYPE_START)

          fgoto doctype;
        };

        # @cdata_buffer is used to store the content of the CDATA tag.
        cdata_start => {
          t(:T_CDATA_START)

          fgoto cdata;
        };

        # General rules and actions.
        '<' => { t(:T_SMALLER) };
        '>' => { t(:T_GREATER) };
        '/' => { t(:T_SLASH) };
        '"' => { t(:T_DQUOTE) };
        "'" => { t(:T_SQUOTE) };
        '-' => { t(:T_DASH) };
        ']' => { t(:T_RBRACKET) };
        '[' => { t(:T_LBRACKET) };
        ':' => { t(:T_COLON) };
        '!' => { t(:T_BANG) };
        '=' => { t(:T_EQUALS) };
      *|;
    }%%
  end # Lexer
end # Oga


Removed the emit_text_buffer Ragel action.

%%machine lexer; # %

module Oga
  ##
  #
  class Lexer
    %% write data; # %

    attr_reader :html

    HTML_VOID_ELEMENTS = [
      'area',
      'base',
      'br',
      'col',
      'command',
      'embed',
      'hr',
      'img',
      'input',
      'keygen',
      'link',
      'meta',
      'param',
      'source',
      'track',
      'wbr'
    ]

    # Lazy way of forwarding instance method calls used internally by Ragel to
    # their corresponding class methods.
    private_methods.grep(/^_lexer_/).each do |name|
      define_method(name) do
        return self.class.send(name)
      end

      private(name)
    end

    def initialize(options = {})
      options.each do |key, value|
        instance_variable_set("@#{key}", value) if respond_to?(key)
      end

      reset
    end

    def reset
      @line     = 1
      @column   = 1
      @data     = nil
      @ts       = nil
      @te       = nil
      @tokens   = []
      @stack    = []
      @top      = 0
      @elements = []

      @string_buffer = ''
      @text_buffer   = ''
    end

    def lex(data)
      @data       = data
      lexer_start = self.class.lexer_start
      eof         = data.length

      %% write init;
      %% write exec;

      tokens = @tokens

      reset

      return tokens
    end

    def html?
      return !!html
    end

    private

    def advance_line(amount = 1)
      @line  += amount
      @column = 1
    end

    def advance_column(length = 1)
      @column += length
    end

    def t(type, start = @ts, stop = @te)
      value = text(start, stop)

      add_token(type, value)
    end

    def text(start = @ts, stop = @te)
      return @data[start...stop]
    end

    def add_token(type, value)
      token = [type, value, @line, @column]

      advance_column(value.length) if value

      @tokens << token
    end

    def emit_text_buffer
      return if @text_buffer.empty?

      add_token(:T_TEXT, @text_buffer)

      lines = @text_buffer.count("\n")

      advance_line(lines) if lines > 0

      @text_buffer = ''
    end

    def buffer_text_until_eof(eof)
      @text_buffer << text

      emit_text_buffer if @te == eof
    end

    def emit_string_buffer
      add_token(:T_STRING, @string_buffer)
      advance_column

      @string_buffer = ''
    end

    def current_element
      return @elements.last
    end

    %%{
      # Use instance variables for `ts` and friends.
      access @;

      newline    = '\n' | '\r\n';
      whitespace = [ \t];

      # String processing
      #
      # These actions/definitions can be used to process single and/or double
      # quoted strings (e.g. for tag attribute values).
      #
      # The string_dquote and string_squote machines should not be used
      # directly, instead the corresponding actions should be used.
      #
      dquote = '"';
      squote = "'";

      action buffer_text {
        @text_buffer << text
      }

      action buffer_string {
        @string_buffer << text
      }

      action string_dquote {
        fcall string_dquote;
      }

      action string_squote {
        fcall string_squote;
      }

      string_dquote := |*
        ^dquote => buffer_string;
        dquote  => {
          emit_string_buffer
          advance_column
          fret;
        };
      *|;

      string_squote := |*
        ^squote => buffer_string;
        squote  => {
          emit_string_buffer
          advance_column
          fret;
        };
      *|;

      # DOCTYPES
      #
      # http://www.w3.org/TR/html-markup/syntax.html#doctype-syntax
      #
      # These rules support the 3 flavours of doctypes:
      #
      # 1. Normal doctypes, as introduced in the HTML5 specification.
      # 2. Deprecated doctypes, the more verbose ones used prior to HTML5.
      # 3. Legacy doctypes
      #
      doctype_start = '<!DOCTYPE'i whitespace+ 'HTML'i;

      doctype := |*
        'PUBLIC' | 'SYSTEM' => { t(:T_DOCTYPE_TYPE) };

        # Lex the public/system IDs as regular strings.
        dquote => string_dquote;
        squote => string_squote;

        # Whitespace inside doctypes is ignored since there's no point in
        # including it.
        whitespace => { advance_column };

        '>' => {
          t(:T_DOCTYPE_END)
          fret;
        };
      *|;

      # CDATA
      #
      # http://www.w3.org/TR/html-markup/syntax.html#cdata-sections
      #
      # CDATA tags are broken up into 3 parts: the start, the content and the
      # end tag.
      #
      # In HTML CDATA tags have no meaning/are not supported. Oga does
      # support them but treats their contents as plain text.
      #
      cdata_start = '<![CDATA[';
      cdata_end   = ']]>';

      cdata := |*
        cdata_end => {
          emit_text_buffer
          t(:T_CDATA_END)
          fret;
        };

        any => buffer_text;
      *|;

      # Comments
      #
      # http://www.w3.org/TR/html-markup/syntax.html#comments
      #
      # Comments are lexed into 3 parts: the start tag, the content and the end
      # tag.
      #
      # Unlike the W3 specification these rules *do* allow character sequences
      # such as `--` and `->`. Putting extra checks in for these sequences
      # would actually make the rules/actions more complex.
      #
      comment_start = '<!--';
      comment_end   = '-->';

      comment := |*
        comment_end => {
          emit_text_buffer
          t(:T_COMMENT_END)
          fret;
        };

        any => buffer_text;
      *|;

      # Elements
      #
      # http://www.w3.org/TR/html-markup/syntax.html#syntax-elements
      #

      # Action that creates the tokens for the opening tag, name and namespace
      # (if any). Remaining work is delegated to a dedicated machine.
      action open_element {
        emit_text_buffer
        add_token(:T_ELEM_OPEN, nil)
        advance_column

        # Add the element name. If the name includes a namespace we'll break
        # the name up into two separate tokens.
        name = text(@ts + 1)

        if name.include?(':')
          ns, name = name.split(':')

          add_token(:T_ELEM_NS, ns)

          # Advance the column for the colon (:) that separates the namespace
          # and element name.
          advance_column
        end

        @elements << name

        add_token(:T_ELEM_NAME, name)

        fcall element;
      }

      element_name  = [a-zA-Z0-9\-_:]+;
      element_start = '<' element_name;

      element_text := |*
        ^'<' => {
          buffer_text_until_eof(eof)
        };

        '<' => {
          emit_text_buffer
          fhold;
          fret;
        };
      *|;

      element_closing_tag := |*
        whitespace => { advance_column };

        element_name => {
          emit_text_buffer
          add_token(:T_ELEM_CLOSE, nil)

          # Advance the column for the </
          advance_column(2)

          # Advance the column for the closing name.
          advance_column(text.length)

          fret;
        };

        '>' => { fret; };
      *|;

      element := |*
        whitespace => { advance_column };

        element_start => open_element;

        # Consume the text inside the element.
        '>' => {
          # If HTML lexing is enabled and we're in a void element we'll bail
          # out right away.
          if html? and HTML_VOID_ELEMENTS.include?(current_element)
            add_token(:T_ELEM_CLOSE, nil)
            @elements.pop
          end

          advance_column

          fcall element_text;
        };

        # Attributes and their values.
        element_name %{ t(:T_ATTR, @ts, p) }

        # The value of the attribute. Attribute values are not required. e.g.
        # in <p data-foo></p> data-foo would be a boolean attribute.
        (
          '=' >{ advance_column }

          # The value of the attribute, wrapped in either single or double
          # quotes.
          (dquote @string_dquote | squote @string_squote)
        )*;

        # Non self-closing elements.
        '</' => {
          fcall element_closing_tag;

          @elements.pop

          fret;
        };

        # self-closing / void elements.
        '/>' => {
          advance_column
          add_token(:T_ELEM_CLOSE, nil)

          @elements.pop

          fret;
        };
      *|;

      main := |*
        doctype_start => {
          emit_text_buffer
          t(:T_DOCTYPE_START)
          fcall doctype;
        };

        cdata_start => {
          emit_text_buffer
          t(:T_CDATA_START)
          fcall cdata;
        };

        comment_start => {
          emit_text_buffer
          t(:T_COMMENT_START)
          fcall comment;
        };

        element_start => open_element;

        any => {
          buffer_text_until_eof(eof)
        };
      *|;
    }%%
  end # Lexer
end # Oga

%%machine lexer; # %

module Oga
  ##
  #
  class Lexer
    %% write data; # %

    attr_reader :html

    HTML_VOID_ELEMENTS = [
      'area',
      'base',
      'br',
      'col',
      'command',
      'embed',
      'hr',
      'img',
      'input',
      'keygen',
      'link',
      'meta',
      'param',
      'source',
      'track',
      'wbr'
    ]

    # Lazy way of forwarding instance method calls used internally by Ragel to
    # their corresponding class methods.
    private_methods.grep(/^_lexer_/).each do |name|
      define_method(name) do
        return self.class.send(name)
      end

      private(name)
    end

    def initialize(options = {})
      options.each do |key, value|
        instance_variable_set("@#{key}", value) if respond_to?(key)
      end

      reset
    end

    def reset
      @line     = 1
      @column   = 1
      @data     = nil
      @ts       = nil
      @te       = nil
      @tokens   = []
      @stack    = []
      @top      = 0
      @elements = []

      @string_buffer = ''
      @text_buffer   = ''
    end

    def lex(data)
      @data       = data
      lexer_start = self.class.lexer_start
      eof         = data.length

      %% write init;
      %% write exec;

      tokens = @tokens

      reset

      return tokens
    end

    def html?
      return !!html
    end

    private

    def advance_line(amount = 1)
      @line  += amount
      @column = 1
    end

    def advance_column(length = 1)
      @column += length
    end

    def t(type, start = @ts, stop = @te)
      value = text(start, stop)

      add_token(type, value)
    end

    def text(start = @ts, stop = @te)
      return @data[start...stop]
    end

    def add_token(type, value)
      token = [type, value, @line, @column]

      advance_column(value.length) if value

      @tokens << token
    end

    def emit_text_buffer
      return if @text_buffer.empty?

      add_token(:T_TEXT, @text_buffer)

      lines = @text_buffer.count("\n")

      advance_line(lines) if lines > 0

      @text_buffer = ''
    end

    def buffer_text_until_eof(eof)
      @text_buffer << text

      emit_text_buffer if @te == eof
    end

    def emit_string_buffer
      add_token(:T_STRING, @string_buffer)
      advance_column

      @string_buffer = ''
    end

    def current_element
      return @elements.last
    end

    %%{
      # Use instance variables for `ts` and friends.
      access @;

      newline    = '\n' | '\r\n';
      whitespace = [ \t];

      # String processing
      #
      # These actions/definitions can be used to process single and/or double
      # quoted strings (e.g. for tag attribute values).
      #
      # The string_dquote and string_squote machines should not be used
      # directly, instead the corresponding actions should be used.
      #
      dquote = '"';
      squote = "'";

      action buffer_text {
        @text_buffer << text
      }

      action emit_text_buffer {
        emit_text_buffer
      }

      action buffer_string {
        @string_buffer << text
      }

      action string_dquote {
        fcall string_dquote;
      }

      action string_squote {
        fcall string_squote;
      }

      string_dquote := |*
        ^dquote => buffer_string;
        dquote  => {
          emit_string_buffer
          advance_column
          fret;
        };
      *|;

      string_squote := |*
        ^squote => buffer_string;
        squote  => {
          emit_string_buffer
          advance_column
          fret;
        };
      *|;

      # DOCTYPES
      #
      # http://www.w3.org/TR/html-markup/syntax.html#doctype-syntax
      #
      # These rules support the 3 flavours of doctypes:
      #
      # 1. Normal doctypes, as introduced in the HTML5 specification.
      # 2. Deprecated doctypes, the more verbose ones used prior to HTML5.
      # 3. Legacy doctypes
      #
      doctype_start = '<!DOCTYPE'i whitespace+ 'HTML'i;

      doctype := |*
        'PUBLIC' | 'SYSTEM' => { t(:T_DOCTYPE_TYPE) };

        # Lex the public/system IDs as regular strings.
        dquote => string_dquote;
        squote => string_squote;

        # Whitespace inside doctypes is ignored since there's no point in
        # including it.
        whitespace => { advance_column };

        '>' => {
          t(:T_DOCTYPE_END)
          fret;
        };
      *|;

      # CDATA
      #
      # http://www.w3.org/TR/html-markup/syntax.html#cdata-sections
      #
      # CDATA tags are broken up into 3 parts: the start, the content and the
      # end tag.
      #
      # In HTML CDATA tags have no meaning/are not supported. Oga does
      # support them but treats their contents as plain text.
      #
      cdata_start = '<![CDATA[';
      cdata_end   = ']]>';

      cdata := |*
        cdata_end => {
          emit_text_buffer
          t(:T_CDATA_END)
          fret;
        };

        any => buffer_text;
      *|;

      # Comments
      #
      # http://www.w3.org/TR/html-markup/syntax.html#comments
      #
      # Comments are lexed into 3 parts: the start tag, the content and the end
      # tag.
      #
      # Unlike the W3 specification these rules *do* allow character sequences
      # such as `--` and `->`. Putting extra checks in for these sequences
      # would actually make the rules/actions more complex.
      #
      comment_start = '<!--';
      comment_end   = '-->';

      comment := |*
        comment_end => {
          emit_text_buffer
          t(:T_COMMENT_END)
          fret;
        };

        any => buffer_text;
      *|;

      # Elements
      #
      # http://www.w3.org/TR/html-markup/syntax.html#syntax-elements
      #

      # Action that creates the tokens for the opening tag, name and namespace
      # (if any). Remaining work is delegated to a dedicated machine.
      action open_element {
        add_token(:T_ELEM_OPEN, nil)
        advance_column

        # Add the element name. If the name includes a namespace we'll break
        # the name up into two separate tokens.
        name = text(@ts + 1)

        if name.include?(':')
          ns, name = name.split(':')

          add_token(:T_ELEM_NS, ns)

          # Advance the column for the colon (:) that separates the namespace
          # and element name.
          advance_column
        end

        @elements << name

        add_token(:T_ELEM_NAME, name)

        fcall element;
      }

      element_name  = [a-zA-Z0-9\-_:]+;
      element_start = '<' element_name;

      element_text := |*
        ^'<' => {
          buffer_text_until_eof(eof)
        };

        '<' => {
          emit_text_buffer
          fhold;
          fret;
        };
      *|;

      element_closing_tag := |*
        whitespace => { advance_column };

        element_name => {
          emit_text_buffer
          add_token(:T_ELEM_CLOSE, nil)

          # Advance the column for the </
          advance_column(2)

          # Advance the column for the closing name.
          advance_column(text.length)

          fret;
        };

        '>' => { fret; };
      *|;

      element := |*
        whitespace => { advance_column };

        element_start => open_element;

        # Consume the text inside the element.
        '>' => {
          # If HTML lexing is enabled and we're in a void element we'll bail
          # out right away.
          if html? and HTML_VOID_ELEMENTS.include?(current_element)
            add_token(:T_ELEM_CLOSE, nil)
            @elements.pop
          end

          advance_column

          fcall element_text;
        };

        # Attributes and their values.
        element_name %{ t(:T_ATTR, @ts, p) }

        # The value of the attribute. Attribute values are not required. e.g.
        # in <p data-foo></p> data-foo would be a boolean attribute.
        (
          '=' >{ advance_column }

          # The value of the attribute, wrapped in either single or double
          # quotes.
          (dquote @string_dquote | squote @string_squote)
        )*;

        # Non self-closing elements.
        '</' => {
          fcall element_closing_tag;

          @elements.pop

          fret;
        };

        # self-closing / void elements.
        '/>' => {
          advance_column
          add_token(:T_ELEM_CLOSE, nil)

          @elements.pop

          fret;
        };
      *|;

      main := |*
        doctype_start @emit_text_buffer => {
          t(:T_DOCTYPE_START)
          fcall doctype;
        };

        cdata_start @emit_text_buffer => {
          t(:T_CDATA_START)
          fcall cdata;
        };

        comment_start @emit_text_buffer => {
          t(:T_COMMENT_START)
          fcall comment;
        };

        element_start @emit_text_buffer => open_element;

        any => {
          buffer_text_until_eof(eof)
        };
      *|;
    }%%
  end # Lexer
end # Oga


allow either one of direct-desc or nth-child, not both (every nth-child is also a direct-desc)

%%{
  machine hext;

  action error {
    this->throw_unexpected();
  }

  action act_newline {
    if( rule_start )
    {
      rule.consume_and_reset();
      rule_start = false;
    }
    else
    {
      rule.reset_indent();
    }
  }

  nth_child = ( [1-9][0-9]** );
  attr_name = ( alpha (alnum | '-' | '_')** );
  builtin_name = ( alpha (alnum | '-' | '_')** );
  cap_var = ( [^/{}][^/{}]** );
  match_literal = ( [^/{}"][^/{}"]** );
  cap_regex = ( ( [^/] | '\\/' )** );
  match_regex = ( ( [^/] | '\\/' )** );

  comment = (
    ' '*
    '#'
    (any - '\n')*
    (
      '\n' %act_newline
    )
  );

  attributes =
    (
      space+
      (
        (
          (
            '@'
            (
              builtin_name
              >{ LX_TK_START; }
              %{ LX_TK_STOP;
                 {
                   bi::builtin_func_ptr bf = bi::get_builtin_by_name(tok);
                   if( !bf ) this->throw_unknown_builtin(tok);
                   rule.pattern().set_builtin_function(bf);
                 }
              }
            )
          )
          |
          (
            attr_name >{ LX_TK_START; }
                      %{ LX_TK_STOP; rule.pattern().set_attr_name(tok); }
          )
        )
        (
          '='
          (
            (
              '{'
              (
                cap_var >{ LX_TK_START; }
                        %{ LX_TK_STOP; rule.pattern().set_cap_var(tok); }
              )
              (
                '/'
                (
                  cap_regex >{ LX_TK_START; }
                            %{ LX_TK_STOP;
                               rule.pattern().set_cap_regex(tok); }
                )
                '/'
              )?
              (
                '}' %{ rule.consume_capture_pattern(); }
              )
            )
            |
            (
              (
                '/'
                (
                  match_regex >{ LX_TK_START; }
                              %{ LX_TK_STOP;
                                 rule.pattern().set_attr_regex(tok);
                                 rule.consume_match_pattern(); }
                )
                '/'
              )
              |
              (
                '"'
                (
                  match_literal >{ LX_TK_START; }
                                %{ LX_TK_STOP;
                                   rule.pattern().set_attr_literal(tok);
                                   rule.consume_match_pattern(); }
                )
                '"'
              )
            )
          )
        )?
      )
    )+;

  main :=
    (
      comment
      |
      (
        (
          '  ' %{ rule.increment_indent(); }
        )*
        (
          '<' %{ rule_start = true; }
        )
        (
          (
            '!' %{ rule.set_direct_descendant(true); }
          )
          |
          (
            nth_child >{ LX_TK_START; }
                      %{ LX_TK_STOP;
                         rule.set_nth_child(std::stoi(tok)); }
          )
        )?
        (
          attr_name >{ LX_TK_START; }
                    %{ LX_TK_STOP;
                       rule.set_tag_name(tok); }
        )?
        attributes?
        (
          '>' %{ rule.set_closed(true); }
        )?
      ):>
      (
        '\n' %act_newline
      )
    )**
    $err(error) $/act_newline $/{ fbreak; };
}%%

%%{
  machine hext;

  action error {
    this->throw_unexpected();
  }

  action act_newline {
    if( rule_start )
    {
      rule.consume_and_reset();
      rule_start = false;
    }
    else
    {
      rule.reset_indent();
    }
  }

  nth_child = ( [1-9][0-9]** );
  attr_name = ( alpha (alnum | '-' | '_')** );
  builtin_name = ( alpha (alnum | '-' | '_')** );
  cap_var = ( [^/{}][^/{}]** );
  match_literal = ( [^/{}"][^/{}"]** );
  cap_regex = ( ( [^/] | '\\/' )** );
  match_regex = ( ( [^/] | '\\/' )** );

  comment = (
    ' '*
    '#'
    (any - '\n')*
    (
      '\n' %act_newline
    )
  );

  attributes =
    (
      space+
      (
        (
          (
            '@'
            (
              builtin_name
              >{ LX_TK_START; }
              %{ LX_TK_STOP;
                 {
                   bi::builtin_func_ptr bf = bi::get_builtin_by_name(tok);
                   if( !bf ) this->throw_unknown_builtin(tok);
                   rule.pattern().set_builtin_function(bf);
                 }
              }
            )
          )
          |
          (
            attr_name >{ LX_TK_START; }
                      %{ LX_TK_STOP; rule.pattern().set_attr_name(tok); }
          )
        )
        (
          '='
          (
            (
              '{'
              (
                cap_var >{ LX_TK_START; }
                        %{ LX_TK_STOP; rule.pattern().set_cap_var(tok); }
              )
              (
                '/'
                (
                  cap_regex >{ LX_TK_START; }
                            %{ LX_TK_STOP;
                               rule.pattern().set_cap_regex(tok); }
                )
                '/'
              )?
              (
                '}' %{ rule.consume_capture_pattern(); }
              )
            )
            |
            (
              (
                '/'
                (
                  match_regex >{ LX_TK_START; }
                              %{ LX_TK_STOP;
                                 rule.pattern().set_attr_regex(tok);
                                 rule.consume_match_pattern(); }
                )
                '/'
              )
              |
              (
                '"'
                (
                  match_literal >{ LX_TK_START; }
                                %{ LX_TK_STOP;
                                   rule.pattern().set_attr_literal(tok);
                                   rule.consume_match_pattern(); }
                )
                '"'
              )
            )
          )
        )?
      )
    )+;

  main :=
    (
      comment
      |
      (
        (
          '  ' %{ rule.increment_indent(); }
        )*
        (
          '<' %{ rule_start = true; }
        )
        (
          '!' %{ rule.set_direct_descendant(true); }
        )?
        (
          nth_child >{ LX_TK_START; }
                    %{ LX_TK_STOP;
                       rule.set_nth_child(std::stoi(tok)); }
        )?
        (
          attr_name >{ LX_TK_START; }
                    %{ LX_TK_STOP;
                       rule.set_tag_name(tok); }
        )?
        attributes?
        (
          '>' %{ rule.set_closed(true); }
        )?
      ):>
      (
        '\n' %act_newline
      )
    )**
    $err(error) $/act_newline $/{ fbreak; };
}%%


More cleanup of the ragel file

More cleanup of the ragel file

#include "kegogi_tokens.h"
#include "kegogi_parser.h"

#include <stdio.h>

#include <bstring.h>
#include <dbg.h>

%%{
    machine kegogi_lexer;
    defaults = 'defaults';
    send = 'send';
    expect = 'expect';

    comment = '#' [^\n]*;

    action mark { m = fpc; }
    action s1 { s1 = blk2bstr(m, fpc-m); }
    action s2 { s2 = blk2bstr(m, fpc-m); }
    action s3 { s3 = blk2bstr(m, fpc-m); }

    ustring = (alpha (alnum | '-')*) >mark %s1;
    number = (digit+) >mark %s1;
    qstring = '\"' (([^\\\"] | ('\\' any))*) >mark %s1 '\"';
    pattern = '(' (([^\\)] | ('\\' any))*) >mark %s1 ')';
    dict_start = '{';
    dict_end = '}';
    equals = [:=];

    host = ([^/:] - space)+ >mark %s2;
    uri = ('/' (any - space)*) >mark %s1;
    port = digit+ >mark %s3;
    url = ((alpha+ '://')? host (':' port)?)? uri;

    newline = '\n';

    main := |*
         defaults => {
             TokenList_append0(&token_list, TKDEFAULTS);
             s1 = s2 = s3 = NULL;
         };
         send => {
             TokenList_append0(&token_list, TKSEND);
             s1 = s2 = s3 = NULL;
         };
         expect => {
             TokenList_append0(&token_list, TKEXPECT);
             s1 = s2 = s3 = NULL;
         };
         (ustring|qstring) => {
             TokenList_append1(&token_list, TKSTRING, s1);
             s1 = s2 = s3 = NULL;
         };
         pattern => {
             TokenList_append1(&token_list, TKPATTERN, s1);
             s1 = s2 = s3 = NULL;
         };
         number => {
             TokenList_append1(&token_list, TKNUMBER, s1);
             s1 = s2 = s3 = NULL;
         };
         url => {
             TokenList_append3(&token_list, TKURL, s1, s2, s3);
             s1 = s2 = s3 = NULL;
         };
         newline => {
             TokenList_append0(&token_list, TKNEWLINE);
             s1 = s2 = s3 = NULL;
         };
         dict_start => {
             TokenList_append0(&token_list, TKDICT_START);
             s1 = s2 = s3 = NULL;
         };
         dict_end => {
             TokenList_append0(&token_list, TKDICT_END);
             s1 = s2 = s3 = NULL;
         };
         equals => {
             TokenList_append0(&token_list, TKEQUALS);
             s1 = s2 = s3 = NULL;
         };
         space => { };
         ',' => { };
         comment => { };
    *|;
}%%

%% write data;

TokenList *get_kegogi_tokens(bstring content) {
    TokenList *token_list = TokenList_create(1024);
    check_mem(token_list);

    char *p = bdata(content);
    char *pe = p + blength(content);
    char *m = NULL;
    char *eof = pe;
    int cs = -1;
    int act = -1;
    char *ts = NULL;
    char *te = NULL;

    bstring s1 = NULL;
    bstring s2 = NULL;
    bstring s3 = NULL;

    %% write init;
    %% write exec;

    return token_list;

error:
    TokenList_destroy(token_list);
    return NULL;
}

#include "kegogi_tokens.h"
#include "kegogi_parser.h"

#include <stdio.h>

#include <bstring.h>
#include <dbg.h>

%%{
    machine kegogi_lexer;
    defaults = 'defaults';
    send = 'send';
    expect = 'expect';

    comment = '#' [^\n]*;

    action mark { m = fpc; }
    action s1 { s1 = blk2bstr(m, fpc-m); }
    action s2 { s2 = blk2bstr(m, fpc-m); }
    action s3 { s3 = blk2bstr(m, fpc-m); }

    ustring = (alpha (alnum | '-')*) >mark %s1;
    number = (digit+) >mark %s1;
    qstring = '\"' (([^\\\"] | ('\\' any))*) >mark %s1 '\"';
    pattern = '(' (([^\\)] | ('\\' any))*) >mark %s1 ')';
    dict_start = '{';
    dict_end = '}';
    equals = [:=];

    host = [^/: ]+ >mark %s2;
    uri = ('/' (any - space)*) >mark %s1;
    port = digit+ >mark %s3;
    url = ((alpha+ '://')? host (':' port)?)? uri;

    newline = '\n';

    main := |*
         defaults => {
             TokenList_append0(&token_list, TKDEFAULTS);
             s1 = s2 = s3 = NULL;
         };
         send => {
             TokenList_append0(&token_list, TKSEND);
             s1 = s2 = s3 = NULL;
         };
         expect => {
             TokenList_append0(&token_list, TKEXPECT);
             s1 = s2 = s3 = NULL;
         };
         (ustring|qstring) => {
             TokenList_append1(&token_list, TKSTRING, s1);
             s1 = s2 = s3 = NULL;
         };
         pattern => {
             TokenList_append1(&token_list, TKPATTERN, s1);
             s1 = s2 = s3 = NULL;
         };
         number => {
             TokenList_append1(&token_list, TKNUMBER, s1);
             s1 = s2 = s3 = NULL;
         };
         url => {
             TokenList_append3(&token_list, TKURL, s1, s2, s3);
             s1 = s2 = s3 = NULL;
         };
         newline => {
             TokenList_append0(&token_list, TKNEWLINE);
             s1 = s2 = s3 = NULL;
         };
         dict_start => {
             TokenList_append0(&token_list, TKDICT_START);
             s1 = s2 = s3 = NULL;
         };
         dict_end => {
             TokenList_append0(&token_list, TKDICT_END);
             s1 = s2 = s3 = NULL;
         };
         equals => {
             TokenList_append0(&token_list, TKEQUALS);
             s1 = s2 = s3 = NULL;
         };
         space => { };
         ',' => { };
         comment => { };
    *|;
}%%

%% write data;

TokenList *get_kegogi_tokens(bstring content) {
    TokenList *token_list = TokenList_create(1024);
    check_mem(token_list);

    char *p = bdata(content);
    char *pe = p + blength(content);
    char *m = NULL;
    char *eof = pe;
    int cs = -1;
    int act = -1;
    char *ts = NULL;
    char *te = NULL;

    bstring s1 = NULL;
    bstring s2 = NULL;
    bstring s3 = NULL;

    %% write init;
    %% write exec;

    return token_list;

error:
    TokenList_destroy(token_list);
    return NULL;
}


- remove the old ragel grammar

- remove the old ragel grammar

require "logstash/namespace"

%%{
  machine logstash_config;

  action mark {
    @tokenstack.push(p)
    #puts "Mark: #{self.line(string, p)}##{self.column(string, p)}"
  }

  action stack_numeric {
    startpos = @tokenstack.pop
    endpos = p
    token = string[startpos ... endpos]
    #puts "numeric: #{token}"
    #puts "numeric?: #{string[startpos,50]}"
    #puts [startpos, endpos].join(",")
    # TODO(sissel): Don't do 'to_i' here. Type coersion is the job of the
    # plugin and the validator.
    @stack << token.to_i
  }

  action stack_string {
    startpos = @tokenstack.pop
    endpos = p
    token = string[startpos ... endpos]
    #puts "string: #{token}"
    @stack << token
  }

  action stack_quoted_string {
    startpos = @tokenstack.pop
    endpos = p
    token = string[startpos + 1 ... endpos - 1] # Skip quotations

    # Parse escapes.
    token.gsub(/\\./) { |m| m[1,1] }
    #puts "quotedstring: #{token}"
    @stack << token
  }

  action array_init {
    @array = []
    @stack << :array_init
  }

  action array_push {
    while @stack.last != :array_init
      @array.unshift @stack.pop
    end
    @stack.pop # pop :array_init

    @stack << @array
  }

  action parameter_init {
    # nothing
  }

  action parameter {
    value = @stack.pop
    name = @stack.pop
    #puts "parameter: #{name} => #{value}"
    if value.is_a?(Array)
      @parameters[name] += value
    else
      @parameters[name] << value
    end
  }

  action plugin {
    @components ||= []
    name = @stack.pop
    #@components << { :name => name, :parameters => @parameters }
    @components << { name => @parameters }
    @parameters = Hash.new { |h,k| h[k] = [] }
  }

  action component_init {
    @components = []
    @parameters = Hash.new { |h,k| h[k] = [] }
  }

  action component {
    name = @stack.pop
    @config ||= Hash.new { |h,k| h[k] = [] }
    @config[name] += @components
    #puts "Config component: #{name}"
  }

  #%{ e = @tokenstack.pop; puts "Comment: #{string[e ... p]}" };
  comment = "#" (any - [\r\n])* >mark ;
  ws = ([ \t\r\n] | comment)** ;
  #ws = ([ \t\n])** ;

  # TODO(sissel): Support floating point values?
  numeric = ( ("+" | "-")?  [0-9] :>> [0-9]** ) >mark %stack_numeric;
  quoted_string = (
    ( "\"" ( ( (any - [\\"\r\n]) | "\\" any )* ) "\"" )
    | ( "'" ( ( (any - [\\'\r\n]) | "\\" any )* ) "'" )
  ) >mark %stack_quoted_string ;
  naked_string = ( [A-Za-z_] :>> [A-Za-z0-9_]* ) >mark %stack_string ;
  string = ( quoted_string | naked_string ) ;

  # TODO(sissel): allow use of this.
  regexp_literal = ( "/" ( ( (any - [\\'\r\n]) | "\\" any )* ) "/" )  ;

  array = (
    ( "[" ( ws | "" ) "]" )
    | ( "[" ws ( string | numeric ) ws ("," ws (string | numeric ) ws)* "]" )
  ) >array_init %array_push;
  # TODO(sissel): Implement hash syntax { key => value, ... }
  # TODO(sissel): hashes should support arrays as values.

  parameter_value = ( numeric | string | array );
  parameter = ( string ws "=>" ws parameter_value ) %parameter ;
  parameters = ( parameter ( ws parameter )** ) >parameter_init ;

  # Statement:
  # component {
  #   plugin_name {
  #     bar => ...
  #     baz => ...
  #   }
  #   ...
  # }

  plugin = (
    (
      naked_string ws "{" ws
        parameters
      ws "}"
    ) | (
      naked_string ws "{" ws "}"
    )
  ) %plugin ;

  component = (
    naked_string ws "{"
      >component_init
      ( ws plugin )**
    ws "}"
  ) %component ;

  config = (ws component? )** ;

  main := config %{ puts "END" }
          $err {
            # Compute line and column of the cursor (p)
            $stderr.puts "Error at line #{self.line(string, p)}, column #{self.column(string, p)}: #{string[p .. -1].inspect}"
            # TODO(sissel): Note what we were expecting?
          } ;
}%%

class LogStash::Config::Grammar
  attr_accessor :eof
  attr_accessor :config

  def initialize
    # BEGIN RAGEL DATA
    %% write data;
    # END RAGEL DATA

    @tokenstack = Array.new
    @stack = Array.new

    @types = Hash.new { |h,k| h[k] = [] }
    @edges = []
  end

  def parse(string)
    # TODO(sissel): Due to a bug in my parser, we need one trailing whitespace
    # at the end of the string. I'll fix this later.
    string += "\n"

    data = string.unpack("c*")

    # BEGIN RAGEL INIT
    %% write init;
    # END RAGEL INIT

    begin
      # BEGIN RAGEL EXEC
      %% write exec;
      # END RAGEL EXEC
    rescue => e
      # Compute line and column of the cursor (p)
      raise e
    end

    if cs < self.logstash_config_first_final
      $stderr.puts "Error at line #{self.line(string, p)}, column #{self.column(string, p)}: #{string[p .. -1].inspect}"
      raise "Invalid Configuration. Check syntax of config file."
    end
    return cs
  end # def parse

  def line(str, pos)
    return str[0 .. pos].count("\n") + 1
  end

  def column(str, pos)
    return str[0 .. pos].split("\n").last.length
  end
end # class LogStash::Config::Grammar

#def parse(string)
  #cfgparser = LogStash::Config::Grammar.new
  #result = cfgparser.parse(string)
  #puts "result %s" % result
  #ap cfgparser.config
#end

#parse(File.open(ARGV[0]).read)


Added send token.  Got rid of util.h include

#include <stdio.h>
#include <stdlib.h>

#include "kegogi_parser.h"

#include <bstring.h>
#include "kegogi.h"
#include "httpclient.h"

%%{
    machine kegogi;
    action command {
        if(idx < max_commands && uri != NULL && method != NULL) {
            host = host ? host : bstrcpy(default_host);
	    int numPort = port ? atoi(bdata(port)) : default_port;
	    if(port) bdestroy(port);
	    Request *req = Request_create(host, numPort, method, uri);
	    commands[idx++].request = req;
        }
	else
	    printf("error\n");
    }

    action clear {
        method = NULL;
 	port = NULL;
	host = NULL;
	uri = NULL;
    }

    action mark {
        mark = fpc;
    }

    action method {
        method = blk2bstr(mark, fpc - mark);
    }

    action host {
        host = blk2bstr(mark, fpc - mark);
    }

    action port {
        port = blk2bstr(mark, fpc - mark);
    }

    action uri {
        uri = blk2bstr(mark, fpc - mark);
    }

    action status_code {
        status_code = atoin(mark, fpc - mark);
    }

    action param_name {
        param_name = blk2bstr(mark, fpc - mark);
    }

    action dictionary {

    }

    action pattern {
        pattern = blk2bstr(mark, fpc - mark);
    }

    ws = (space - '\n')+;

    method = (upper | digit){1,20} >mark %method;

    protocol = alpha+ '://';
    host = ([^:/]+) >mark %host;
    port = ':' (digit+) >mark %port;
    uri = ('/' (^space)*) >mark %uri;

    url = ((protocol? host port?)? uri);

    comment = '#' [^\n]*;

    command = (ws? "send" ws+ method ws+ url ws? comment? '\n') >clear @command;
    empty_line = ws? comment? '\n';

    main := (command | empty_line)*;
}%%

%% write data;

int parse_kegogi_file(const char *path, Command commands[], int max_commands) {
    FILE *f = fopen(path, "r");
    fseek(f, 0L, SEEK_END);
    int size = ftell(f);
    fseek(f, 0L, SEEK_SET);
    char *buffer = malloc(size);
    fread(buffer, 1, size, f);

    int idx = 0;

    bstring default_host = bfromcstr("localhost");
    int default_port = 80;

    bstring host, port, uri, method;
    char *mark, *p = buffer, *pe = buffer + size, *eof = pe;
    int cs;

    %% write init;
    %% write exec;

    return idx;
}

#include <stdio.h>
#include <stdlib.h>

#include "kegogi_parser.h"

#include <bstring.h>
#include "kegogi.h"
#include "httpclient.h"
#include "util.h"

%%{
    machine kegogi;
    action command {
        if(idx < max_commands && uri != NULL && method != NULL) {
            host = host ? host : bstrcpy(default_host);
	    int numPort = port ? atoi(bdata(port)) : default_port;
	    if(port) bdestroy(port);
	    Request *req = Request_create(host, numPort, method, uri);
	    commands[idx++].request = req;
        }
	else
	    printf("error\n");
    }

    action clear {
        method = NULL;
 	port = NULL;
	host = NULL;
	uri = NULL;
    }

    action mark {
        mark = fpc;
    }

    action method {
        method = blk2bstr(mark, fpc - mark);
    }

    action host {
        host = blk2bstr(mark, fpc - mark);
    }

    action port {
        port = blk2bstr(mark, fpc - mark);
    }

    action uri {
        uri = blk2bstr(mark, fpc - mark);
    }

    action status_code {
        status_code = atoin(mark, fpc - mark);
    }

    action param_name {
        param_name = blk2bstr(mark, fpc - mark);
    }

    action dictionary {

    }

    action pattern {
        pattern = blk2bstr(mark, fpc - mark);
    }

    ws = (space - '\n')+;

    method = (upper | digit){1,20} >mark %method;

    protocol = alpha+ '://';
    host = ([^:/]+) >mark %host;
    port = ':' (digit+) >mark %port;
    uri = ('/' (^space)*) >mark %uri;

    url = ((protocol? host port?)? uri);

    comment = '#' [^\n]*;

    command = (ws? method ws url ws? comment? '\n') >clear @command;
    empty_line = ws? comment? '\n';

    main := (command | empty_line)*;
}%%

%% write data;

int parse_kegogi_file(const char *path, Command commands[], int max_commands) {
    FILE *f = fopen(path, "r");
    fseek(f, 0L, SEEK_END);
    int size = ftell(f);
    fseek(f, 0L, SEEK_SET);
    char *buffer = malloc(size);
    fread(buffer, 1, size, f);

    int idx = 0;

    bstring default_host = bfromcstr("localhost");
    int default_port = 80;

    bstring host, port, uri, method;
    char *mark, *p = buffer, *pe = buffer + size, *eof = pe;
    int cs;

    %% write init;
    %% write exec;

    return idx;
}