update postprocessor

update postprocessor

#include "config.h"

#include <string>
#include <fstream>
#include <iostream>
#include <cstdlib>
// New for directory recursion
#ifdef WIN32
	#include <dirent_msvc.h>
#else
	#include <dirent.h>
#endif

using std::cin;
using std::cout;
using std::cerr;
using std::ifstream;
using std::ofstream;
using std::string;
using std::ios_base;
using std::endl;

%%{
  machine PostProcess;
  write data;

  # end of file character
  EOF = 0;

  # all but end of file character
  default = ^0;

  # end of line
  EOL = ('\r'? . '\n') @{line++;};

  # matlab identifier
  IDENT = [A-Z\\a-z\-_][A-Z\\a-z0-9\-_]*;

  # matlab ifdentifier without underscore characters
  IDENT_WO_US = [A-Z\\a-z\-][A-Z\\a-z0-9\-]*;

  ANY_TAG = '<' . [^>]* . '>';

  action echo {
    fout << fpc;
    cout << fpc;
  }

  action st_tok { tmp_p = p; }

  action echo_tok { fout.write(tmp_p, p - tmp_p); cout.write(tmp_p, p-tmp_p); }

  rettype:= |*

    # matlab is typeless, so discard the type
    ('matlabtypesubstitute') => {fout << " ";};

    # replace all "::" by "."
    ('::' . [A-Z\\a-z\-_]) => { fout << '.' << *(te-1); };

    # a word
    (any - [\n <>\&:,\t])+ => { fout.write(ts, te-ts); };

    # word separators
    ([\n <>\&:\t]) => {fout << *ts;};

    # comma or > end the type
    (',') => { fhold; fret; };
    ('>') => { p -=4; fret; };
  *|;

  # reconstruct return values
  retval:= |*
    ('<' . ('::')?) => { fcall rettype; };

    # matlab identifier (1 return value)
#    ('<' . (default - [,&])*) => { cerr.write(ts+4, te - ts-4); cerr << std::endl; fout.write(ts+4, te - ts-4); };

    (',' . (default - '\&')*) => { fout << " "; fout.write(ts+1, te-ts-1); fout << ""; };

    # end of return value
    ('>') => {
                  if(only_retval) { fout << " ="; }
                  fret;
                };

    # white spaces
    ([ \t\n]*) => { fout.write(ts, te - ts); };

    # typebreak
    ('
') => {};

    # other tags
    #(ANY_TAG) => { fout.write(ts, te-ts); };

  *|;

  retvals := |*

    ('<') => {};

    ('mlhsInnerSubst') => { fcall retval; };

    ('>') => { fout << "] ="; fgoto main; };

    # white spaces and commata
    ([ \t\n]*) => { fout.write(ts, te - ts); };
    ('
' . [ \t\n]* . ',') => { fout << ",
\n"; };
    (',') => { fout << ", "; };

    # other tags
    (ANY_TAG) => { fout.write(ts, te-ts); };

  *|;

  # function name
  mtocsubst:= |*
    '_';

    (IDENT_WO_US);

    '_tsbus_cotm' => { fgoto main; };
  *|;

  main:= |*
   # list of return values
   ('mlhsSubst') => { fout << "function ["; only_retval = false; fgoto retvals; };

   # one return value
   ('mlhsInnerSubst') => { fout << "function "; only_retval = true; fcall retval; fout << " ="; };

   # no return values
   ('noret::substitute') => {fout << "function ";};

   # function name
   ('mtoc_subst_') => { fgoto mtocsubst; };
   ('::mtoc_subst_') => { fout << '.'; fgoto mtocsubst; };

   # matlab is typeless, so discard the type
   ('matlabtypesubstitute') => {fout << " ";};

   # remove leading "::" (global namespace identifier)
   ([(,>] . '::') => { fout.write(ts, 1); fout << " "; };
   ('<' . '::') => { fout.write(ts, 4); fout << " ";};

   # replace all "::" by "."
   ('::' . [A-Z\\a-z\-_]) => { fout << '.' << *(te-1); };

   # a word
   (any - [\n <>()[\]{}\&:.,;_\t])+ => { fout.write(ts, te-ts); };

   # word separators
   ([\n <>()[\]{}\t:.;,_\&]) => {fout << *ts;};

   # a single dot stays a dot
   ('.') => {fout << '.';};
   *|;
}%%

class PostProcess {

private:
  string docdir_;
  int          line            , col;
  char        *ts              , *te;
  int          act             , have;
  int          cs;
  int          top;
  int          stack[5];
  bool         only_retval;

public:
  /**
   * @class PostProcess
   *
   * @change{1,2,dw,2011-11-04} Changed the postprocessor interface from taking a single file argument to
   * assuming the passed string to be a folder whos contents are to be postprocessed.
   */
  // constructor
  PostProcess(const string &docdir) :
    docdir_(docdir),
    line(1),
    ts(0), te(0), have(0),
    top(0), only_retval(false)
  { }

  int execute()
  {
    DIR *dp;
    if ((dp  = opendir(docdir_.c_str())) == NULL) {
      cerr << "Error opening directory " << docdir_ << endl;
      return -1;
    }

    struct dirent* dirp;
    string file;
    while ((dirp = readdir(dp)) != NULL) {
      file = string(dirp->d_name);
      // Process only html files
      if (file.substr(file.find_last_of(".") + 1) == "html" && file.find("8rl") == string::npos) {
        postprocess(docdir_ + string("/") + file);
      }
    }
    closedir(dp);
    return 0;
  }

  // run postprocessor
  int postprocess(string file)
  {
    std::ios::sync_with_stdio(false);

    %% write init;

    ifstream is;
    try {
      is.open(file.c_str());
    } catch (std::ifstream::failure e) {
      cerr << "Exception opening/reading file";
      exit(-1);
    }

    is.seekg(0, ios_base::end);
    int length = is.tellg();
    is.seekg(0, ios_base::beg);

    char* buf = new char[(int)(1.1*length)];
    char* p = buf;
//    char * tmp_p = p;

    is.read(buf, length);
    is.close();

    ofstream fout;
    try {
      fout.open(file.c_str(), ios_base::trunc);
    } catch (std::ofstream::failure e) {
      cerr << "Exception opening/writing file";
      exit(-1);
    }

    int len = is.gcount();
    char *pe = p + len;
    char *eof = pe;

    %% write exec;

    /* Check if we failed. */
    if ( cs == PostProcess_error )
    {
      /* Machine failed before finding a token. */
      cerr << file << ": PARSE ERROR in line " << line << endl;
      cerr.write(p, 100);
      exit(-1);
    }

    fout.close();
    delete buf;

    return 0;
  }
};

void usage()
{
  cout << "postprocess Version " << MTOCPP_VERSION_MAJOR << "."
    << MTOCPP_VERSION_MINOR << endl;
  cout << "Usage: ./postprocess filename" << endl;
}

int main(int argc, char ** argv)
{
  string docdir;
  if(argc >= 2)
  {
    if (std::string("--help") == std::string(argv[1]))
    {
      usage();
      return 0;
    }
    docdir = argv[1];
  }
  else
  {
    cerr << "wrong number of arguments!" << endl;
    exit(-2);
  }

  cout << "Running mtoc++ postprocessor on directory " << docdir << endl;

  PostProcess scanner(docdir);
  scanner.execute();
  return 0;
}

/* vim: set et sw=2 ft=ragel: */

#include "config.h"

#include <string>
#include <fstream>
#include <iostream>
#include <cstdlib>
// New for directory recursion
#ifdef WIN32
	#include <dirent_msvc.h>
#else
	#include <dirent.h>
#endif

using std::cin;
using std::cout;
using std::cerr;
using std::ifstream;
using std::ofstream;
using std::string;
using std::ios_base;
using std::endl;

%%{
  machine PostProcess;
  write data;

  # end of file character
  EOF = 0;

  # all but end of file character
  default = ^0;

  # end of line
  EOL = ('\r'? . '\n') @{line++;};

  # matlab identifier
  IDENT = [A-Z\\a-z\-_][A-Z\\a-z0-9\-_]*;

  # matlab ifdentifier without underscore characters
  IDENT_WO_US = [A-Z\\a-z\-][A-Z\\a-z0-9\-]*;

  ANY_TAG = '<' . [^>]* . '>';

  action echo {
    fout << fpc;
    cout << fpc;
  }

  action st_tok { tmp_p = p; }

  action echo_tok { fout.write(tmp_p, p - tmp_p); cout.write(tmp_p, p-tmp_p); }

  rettype:= |*

    # matlab is typeless, so discard the type
    ('matlabtypesubstitute') => {fout << " ";};

    # replace all "::" by "."
    ('::' . [A-Z\\a-z\-_]) => { fout << '.' << *(te-1); };

    # a word
    (any - [\n <>\&:,\t])+ => { fout.write(ts, te-ts); };

    # word separators
    ([\n <>\&:\t]) => {fout << *ts;};

    # comma or > end the type
    (',') => { fhold; fret; };
    ('>') => { p -=4; fret; };
  *|;

  # reconstruct return values
  retval:= |*
    ('<' . ('::')?) => { fcall rettype; };

    # matlab identifier (1 return value)
#    ('<' . (default - [,&])*) => { cerr.write(ts+4, te - ts-4); cerr << std::endl; fout.write(ts+4, te - ts-4); };

    (',' . (default - '\&')*) => { fout << " "; fout.write(ts+1, te-ts-1); fout << ""; };

    # end of return value
    ('>') => {
                  if(only_retval) { fout << " ="; }
                  fret;
                };

    # white spaces
    ([ \t\n]*) => { fout.write(ts, te - ts); };

    # typebreak
    ('
') => {};

    # other tags
    #(ANY_TAG) => { fout.write(ts, te-ts); };

  *|;

  retvals := |*

    ('<') => {};

    ('mlhsInnerSubst') => { fcall retval; };

    ('>') => { fout << "] ="; fgoto main; };

    # white spaces and commata
    ([ \t\n]*) => { fout.write(ts, te - ts); };
    ('
' . [ \t\n]* . ',') => { fout << ",
\n"; };
    (',') => { fout << ", "; };

    # other tags
    (ANY_TAG) => { fout.write(ts, te-ts); };

  *|;

  # function name
  mtocsubst:= |*
    '_';

    (IDENT_WO_US);

    '_tsbus_cotm' => { fgoto main; };
  *|;

  main:= |*
   # list of return values
   ('mlhsSubst') => { fout << "function ["; only_retval = false; fgoto retvals; };

   # one return value
   ('mlhsInnerSubst') => { fout << "function "; only_retval = true; fcall retval; fout << " ="; };

   # no return values
   ('noret::substitute') => {fout << "function ";};

   # function name
   ('mtoc_subst_') => { fgoto mtocsubst; };

   # matlab is typeless, so discard the type
   ('matlabtypesubstitute') => {fout << " ";};

   # remove leading "::" (global namespace identifier)
   ([(,>] . '::') => { fout.write(ts, 1); fout << " "; };
   ('<' . '::') => { fout.write(ts, 4); fout << " ";};

   # replace all "::" by "."
   ('::' . [A-Z\\a-z\-_]) => { fout << '.' << *(te-1); };

   # a word
   (any - [\n <>()[\]{}\&:,;_\t])+ => { fout.write(ts, te-ts); };

   # word separators
   ([\n <>()[\]{}\t:;,_\&]) => {fout << *ts;};
   *|;
}%%

class PostProcess {

private:
  string docdir_;
  int          line            , col;
  char        *ts              , *te;
  int          act             , have;
  int          cs;
  int          top;
  int          stack[5];
  bool         only_retval;

public:
  /**
   * @class PostProcess
   *
   * @change{1,2,dw,2011-11-04} Changed the postprocessor interface from taking a single file argument to
   * assuming the passed string to be a folder whos contents are to be postprocessed.
   */
  // constructor
  PostProcess(const string &docdir) :
    docdir_(docdir),
    line(1),
    ts(0), te(0), have(0),
    top(0), only_retval(false)
  { }

  int execute()
  {
    DIR *dp;
    if ((dp  = opendir(docdir_.c_str())) == NULL) {
      cerr << "Error opening directory " << docdir_ << endl;
      return -1;
    }

    struct dirent* dirp;
    string file;
    while ((dirp = readdir(dp)) != NULL) {
      file = string(dirp->d_name);
      // Process only html files
      if (file.substr(file.find_last_of(".") + 1) == "html" && file.find("8rl") == string::npos) {
        postprocess(docdir_ + string("/") + file);
      }
    }
    closedir(dp);
    return 0;
  }

  // run postprocessor
  int postprocess(string file)
  {
    std::ios::sync_with_stdio(false);

    %% write init;

    ifstream is;
    try {
      is.open(file.c_str());
    } catch (std::ifstream::failure e) {
      cerr << "Exception opening/reading file";
      exit(-1);
    }

    is.seekg(0, ios_base::end);
    int length = is.tellg();
    is.seekg(0, ios_base::beg);

    char* buf = new char[(int)(1.1*length)];
    char* p = buf;
//    char * tmp_p = p;

    is.read(buf, length);
    is.close();

    ofstream fout;
    try {
      fout.open(file.c_str(), ios_base::trunc);
    } catch (std::ofstream::failure e) {
      cerr << "Exception opening/writing file";
      exit(-1);
    }

    int len = is.gcount();
    char *pe = p + len;
    char *eof = pe;

    %% write exec;

    /* Check if we failed. */
    if ( cs == PostProcess_error )
    {
      /* Machine failed before finding a token. */
      cerr << file << ": PARSE ERROR in line " << line << endl;
      cerr.write(p, 100);
      exit(-1);
    }

    fout.close();
    delete buf;

    return 0;
  }
};

void usage()
{
  cout << "postprocess Version " << MTOCPP_VERSION_MAJOR << "."
    << MTOCPP_VERSION_MINOR << endl;
  cout << "Usage: ./postprocess filename" << endl;
}

int main(int argc, char ** argv)
{
  string docdir;
  if(argc >= 2)
  {
    if (std::string("--help") == std::string(argv[1]))
    {
      usage();
      return 0;
    }
    docdir = argv[1];
  }
  else
  {
    cerr << "wrong number of arguments!" << endl;
    exit(-2);
  }

  cout << "Running mtoc++ postprocessor on directory " << docdir << endl;

  PostProcess scanner(docdir);
  scanner.execute();
  return 0;
}

/* vim: set et sw=2 ft=ragel: */


fix ragel warning 'applying kleene star to a machine that accepts zero length word'

%%{
  machine hext;

  action error {
    throw_error();
  }

  cap_limit = ( [1-9][0-9]** );
  attr_name = ( alpha (alnum | '-' | '_')** );
  cap_var       = ( alpha (alnum | '-' | '_')** );
  match_literal = ( alpha (alnum | '-' | '_' | ' ')** );
  cap_regex   = ( [^/]+* );
  match_regex = ( [^/]+* );

  comment = (
    ' '*
    '#'
    (any - '\n')*
    (
      '\n' >{ LX_TK_START(TK_NEWLINE); }
           %{ LX_TK_STOP; }
    )
  );

  attributes =
    (
      space+
      (
        (
          '@' >{ LX_TK_START(TK_BUILTIN); }
              %{ LX_TK_STOP; }
        )?
        (
          attr_name >{ LX_TK_START(TK_ATTR_NAME); }
                    %{ LX_TK_STOP; }
        )
        <:
        (
          '="'
          (
            (
              (
                '{' >{ LX_TK_START(TK_CAP_BEGIN); }
                    %{ LX_TK_STOP; }
              )
              (
                cap_var** >{ LX_TK_START(TK_CAP_VAR); }
                          %{ LX_TK_STOP; }
              )
              (
                '/'
                (
                  cap_regex >{ LX_TK_START(TK_CAP_REGEX); }
                            %{ LX_TK_STOP; }
                )
                '/'
              )?
              (
                '}' >{ LX_TK_START(TK_CAP_END); }
                    %{ LX_TK_STOP; }
              )
            )
            |
            (
              (
                '/'
                (
                  match_regex >{ LX_TK_START(TK_MATCH_REGEX); }
                              %{ LX_TK_STOP; }
                )
                '/'
              )
              |
              (
                match_literal** >{ LX_TK_START(TK_MATCH_LITERAL); }
                                %{ LX_TK_STOP; }
              )
            )
          )
          '"'
        )?
      )
    )+;

  main :=
    (
      comment
      |
      (
        (
          '  ' >{ LX_TK_START(TK_INDENT); }
               %{ LX_TK_STOP; }
        )*
        (
          '<' >{ LX_TK_START(TK_RULE_BEGIN); }
              %{ LX_TK_STOP; }
        )
        (
          '!' >{ LX_TK_START(TK_DIRECT_DESC); }
              %{ LX_TK_STOP; }
        )?
        (
          cap_limit >{ LX_TK_START(TK_CAP_LIMIT); }
                    %{ LX_TK_STOP; }
        )?
        (
          attr_name >{ LX_TK_START(TK_TAG_NAME); }
                    %{ LX_TK_STOP; }
        )?
        attributes?
        (
          '>' >{ LX_TK_START(TK_RULE_END); }
              %{ LX_TK_STOP; }
        )
      )
      (
        '\n' >{ LX_TK_START(TK_NEWLINE); }
             %{ LX_TK_STOP; }
      )
    )**
    $err(error) $/{ LX_TK_START(TK_EOF); fbreak; };
}%%

%%{
  machine hext;

  action error {
    throw_error();
  }

  cap_limit = ( [1-9][0-9]** );
  attr_name = ( alpha (alnum | '-' | '_')** );
  cap_var       = ( alpha (alnum | '-' | '_')** );
  match_literal = ( alpha (alnum | '-' | '_' | ' ')** );
  cap_regex   = ( [^/]+* );
  match_regex = ( [^/]+* );

  comment = (
    ' '*
    '#'
    (any - '\n')*
    (
      '\n' >{ LX_TK_START(TK_NEWLINE); }
           %{ LX_TK_STOP; }
    )
  );

  attributes =
    (
      space+
      (
        (
          '@' >{ LX_TK_START(TK_BUILTIN); }
              %{ LX_TK_STOP; }
        )?
        (
          attr_name >{ LX_TK_START(TK_ATTR_NAME); }
                    %{ LX_TK_STOP; }
        )
        <:
        (
          '="'
          (
            (
              (
                '{' >{ LX_TK_START(TK_CAP_BEGIN); }
                    %{ LX_TK_STOP; }
              )
              (
                cap_var** >{ LX_TK_START(TK_CAP_VAR); }
                          %{ LX_TK_STOP; }
              )
              (
                '/'
                (
                  cap_regex >{ LX_TK_START(TK_CAP_REGEX); }
                            %{ LX_TK_STOP; }
                )
                '/'
              )?
              (
                '}' >{ LX_TK_START(TK_CAP_END); }
                    %{ LX_TK_STOP; }
              )
            )
            |
            (
              (
                '/'
                (
                  match_regex** >{ LX_TK_START(TK_MATCH_REGEX); }
                                %{ LX_TK_STOP; }
                )
                '/'
              )
              |
              (
                match_literal** >{ LX_TK_START(TK_MATCH_LITERAL); }
                                %{ LX_TK_STOP; }
              )
            )
          )
          '"'
        )?
      )
    )+;

  main :=
    (
      comment
      |
      (
        (
          '  ' >{ LX_TK_START(TK_INDENT); }
               %{ LX_TK_STOP; }
        )*
        (
          '<' >{ LX_TK_START(TK_RULE_BEGIN); }
              %{ LX_TK_STOP; }
        )
        (
          '!' >{ LX_TK_START(TK_DIRECT_DESC); }
              %{ LX_TK_STOP; }
        )?
        (
          cap_limit >{ LX_TK_START(TK_CAP_LIMIT); }
                    %{ LX_TK_STOP; }
        )?
        (
          attr_name >{ LX_TK_START(TK_TAG_NAME); }
                    %{ LX_TK_STOP; }
        )?
        attributes?
        (
          '>' >{ LX_TK_START(TK_RULE_END); }
              %{ LX_TK_STOP; }
        )
      )
      (
        '\n' >{ LX_TK_START(TK_NEWLINE); }
             %{ LX_TK_STOP; }
      )
    )**
    $err(error) $/{ LX_TK_START(TK_EOF); fbreak; };
}%%


Removed CSS lexer code that was commented out.

%%machine css_lexer; # %

module Oga
  module CSS
    ##
    # Lexer for turning CSS expressions into a sequence of tokens. Tokens are
    # returned as arrays with every array having two values:
    #
    # 1. The token type as a Symbol
    # 2. The token value, or nil if there is no value.
    #
    # ## Thread Safety
    #
    # Similar to the XPath lexer this lexer keeps track of an internal state. As
    # a result it's not safe to share the same instance of this lexer between
    # multiple threads. However, no global state is used so you can use separate
    # instances in threads just fine.
    #
    class Lexer
      %% write data;

      # % fix highlight

      ##
      # @param [String] data The data to lex.
      #
      def initialize(data)
        @data = data
      end

      ##
      # Gathers all the tokens for the input and returns them as an Array.
      #
      # @see [#advance]
      # @return [Array]
      #
      def lex
        tokens = []

        advance do |type, value|
          tokens << [type, value]
        end

        return tokens
      end

      ##
      # Advances through the input and generates the corresponding tokens. Each
      # token is yielded to the supplied block.
      #
      # This method stores the supplied block in `@block` and resets it after
      # the lexer loop has finished.
      #
      # @see [#add_token]
      #
      def advance(&block)
        @block = block

        data  = @data # saves ivar lookups while lexing.
        ts    = nil
        te    = nil
        stack = []
        top   = 0
        cs    = self.class.css_lexer_start
        act   = 0
        eof   = @data.bytesize
        p     = 0
        pe    = eof

        _css_lexer_eof_trans          = self.class.send(:_css_lexer_eof_trans)
        _css_lexer_from_state_actions = self.class.send(:_css_lexer_from_state_actions)
        _css_lexer_index_offsets      = self.class.send(:_css_lexer_index_offsets)
        _css_lexer_indicies           = self.class.send(:_css_lexer_indicies)
        _css_lexer_key_spans          = self.class.send(:_css_lexer_key_spans)
        _css_lexer_to_state_actions   = self.class.send(:_css_lexer_to_state_actions)
        _css_lexer_trans_actions      = self.class.send(:_css_lexer_trans_actions)
        _css_lexer_trans_keys         = self.class.send(:_css_lexer_trans_keys)
        _css_lexer_trans_targs        = self.class.send(:_css_lexer_trans_targs)

        %% write exec;

        # % fix highlight
      ensure
        @block = nil
      end

      private

      ##
      # Emits a token of which the value is based on the supplied start/stop
      # position.
      #
      # @param [Symbol] type The token type.
      # @param [Fixnum] start
      # @param [Fixnum] stop
      #
      # @see [#text]
      # @see [#add_token]
      #
      def emit(type, start, stop)
        value = slice_input(start, stop)

        add_token(type, value)
      end

      ##
      # Returns the text between the specified start and stop position.
      #
      # @param [Fixnum] start
      # @param [Fixnum] stop
      # @return [String]
      #
      def slice_input(start, stop)
        return @data.byteslice(start, stop - start)
      end

      ##
      # Yields a new token to the supplied block.
      #
      # @param [Symbol] type The token type.
      # @param [String] value The token value.
      #
      # @yieldparam [Symbol] type
      # @yieldparam [String|NilClass] value
      #
      def add_token(type, value = nil)
        @block.call(type, value)
      end

      %%{
        getkey (data.getbyte(p) || 0);

        whitespace = [\t ]+;

        comma  = ',' %{ add_token(:T_COMMA) };
        hash   = '#' %{ add_token(:T_HASH) };
        dot    = '.' %{ add_token(:T_DOT) };
        lbrack = '[' %{ add_token(:T_LBRACK) };
        rbrack = ']' %{ add_token(:T_RBRACK) };
        colon  = ':' %{ add_token(:T_COLON) };
        lparen = '(';
        rparen = ')';
        pipe   = '|';
        odd    = 'odd';
        even   = 'even';
        minus  = '-';
        nth    = 'n';

        # Identifiers
        #
        # Identifiers are used for element and attribute names. Identifiers have
        # to start with a letter.

        identifier = '*' | [a-zA-Z]+ [a-zA-Z\-_0-9]*;

        action emit_identifier {
          emit(:T_IDENT, ts, te)
        }

        # Operators
        #
        # Various operators that can be used for filtering nodes. For example,
        # "$=" can be used to select attribute values that end with a given
        # string.
        #
        # http://www.w3.org/TR/css3-selectors/#selectors

        op_eq          = '=';
        op_space_in    = '~=';
        op_starts_with = '^=';
        op_ends_with   = '$=';
        op_in          = '*=';
        op_hyphen_in   = '|=';
        op_child       = '>';
        op_fol_direct  = '+';
        op_fol         = '~';

        # Numbers
        #
        # CSS selectors only understand integers, floating points are not
        # supported.

        integer = ('-' | '+')* digit+;

        action emit_integer {
          value = slice_input(ts, te).to_i

          add_token(:T_INT, value)
        }

        # Strings
        #
        # Strings can be single or double quoted. They are mainly used for
        # attribute values.
        #
        dquote = '"';
        squote = "'";

        string_dquote = (dquote ^dquote* dquote);
        string_squote = (squote ^squote* squote);

        string = string_dquote | string_squote;

        action emit_string {
          emit(:T_STRING, ts + 1, te - 1)
        }

        # Pseudo Classes
        #
        # http://www.w3.org/TR/css3-selectors/#structural-pseudos

        action emit_lparen {
          add_token(:T_LPAREN)

          fnext pseudo_args;
        }

        action emit_rparen {
          add_token(:T_RPAREN)

          fnext main;
        }

        # Machine used for processing the arguments of a pseudo class. These are
        # handled in a separate machine as these arguments can contain data not
        # allowed elsewhere. For example, "2n" is not allowed to appear outside
        # of the arguments list.
        pseudo_args := |*
          nth     => { add_token(:T_NTH) };
          minus   => { add_token(:T_MINUS) };
          odd     => { add_token(:T_ODD) };
          even    => { add_token(:T_EVEN) };
          integer => emit_integer;
          rparen  => emit_rparen;
        *|;

        main := |*
          whitespace | comma | hash | dot | lbrack | rbrack | colon;

          # Some of the operators have similar characters (e.g. the "="). As a
          # result we can't use rules like the following:
          #
          #     '='  %{ ... };
          #     '*=' %{ ... };
          #
          # This would result in both machines being executed for the input
          # "*=". The syntax below ensures that only the first match is handled.

          op_eq          => { add_token(:T_EQ) };
          op_space_in    => { add_token(:T_SPACE_IN) };
          op_starts_with => { add_token(:T_STARTS_WITH) };
          op_ends_with   => { add_token(:T_ENDS_WITH) };
          op_in          => { add_token(:T_IN) };
          op_hyphen_in   => { add_token(:T_HYPHEN_IN) };
          op_child       => { add_token(:T_CHILD) };
          op_fol_direct  => { add_token(:T_FOLLOWING_DIRECT) };
          op_fol         => { add_token(:T_FOLLOWING) };

          # The pipe character is also used in the |= operator so the action for
          # this is handled separately.
          pipe => { add_token(:T_PIPE) };

          lparen     => emit_lparen;
          identifier => emit_identifier;
          integer    => emit_integer;
          string     => emit_string;

          any;
        *|;
      }%%
    end # Lexer
  end # CSS
end # Oga

%%machine css_lexer; # %

module Oga
  module CSS
    ##
    # Lexer for turning CSS expressions into a sequence of tokens. Tokens are
    # returned as arrays with every array having two values:
    #
    # 1. The token type as a Symbol
    # 2. The token value, or nil if there is no value.
    #
    # ## Thread Safety
    #
    # Similar to the XPath lexer this lexer keeps track of an internal state. As
    # a result it's not safe to share the same instance of this lexer between
    # multiple threads. However, no global state is used so you can use separate
    # instances in threads just fine.
    #
    class Lexer
      %% write data;

      # % fix highlight

      ##
      # @param [String] data The data to lex.
      #
      def initialize(data)
        @data = data
      end

      ##
      # Gathers all the tokens for the input and returns them as an Array.
      #
      # @see [#advance]
      # @return [Array]
      #
      def lex
        tokens = []

        advance do |type, value|
          tokens << [type, value]
        end

        return tokens
      end

      ##
      # Advances through the input and generates the corresponding tokens. Each
      # token is yielded to the supplied block.
      #
      # This method stores the supplied block in `@block` and resets it after
      # the lexer loop has finished.
      #
      # @see [#add_token]
      #
      def advance(&block)
        @block = block

        data  = @data # saves ivar lookups while lexing.
        ts    = nil
        te    = nil
        stack = []
        top   = 0
        cs    = self.class.css_lexer_start
        act   = 0
        eof   = @data.bytesize
        p     = 0
        pe    = eof

        _css_lexer_eof_trans          = self.class.send(:_css_lexer_eof_trans)
        _css_lexer_from_state_actions = self.class.send(:_css_lexer_from_state_actions)
        _css_lexer_index_offsets      = self.class.send(:_css_lexer_index_offsets)
        _css_lexer_indicies           = self.class.send(:_css_lexer_indicies)
        _css_lexer_key_spans          = self.class.send(:_css_lexer_key_spans)
        _css_lexer_to_state_actions   = self.class.send(:_css_lexer_to_state_actions)
        _css_lexer_trans_actions      = self.class.send(:_css_lexer_trans_actions)
        _css_lexer_trans_keys         = self.class.send(:_css_lexer_trans_keys)
        _css_lexer_trans_targs        = self.class.send(:_css_lexer_trans_targs)

        %% write exec;

        # % fix highlight
      ensure
        @block = nil
      end

      private

      ##
      # Emits a token of which the value is based on the supplied start/stop
      # position.
      #
      # @param [Symbol] type The token type.
      # @param [Fixnum] start
      # @param [Fixnum] stop
      #
      # @see [#text]
      # @see [#add_token]
      #
      def emit(type, start, stop)
        value = slice_input(start, stop)

        add_token(type, value)
      end

      ##
      # Returns the text between the specified start and stop position.
      #
      # @param [Fixnum] start
      # @param [Fixnum] stop
      # @return [String]
      #
      def slice_input(start, stop)
        return @data.byteslice(start, stop - start)
      end

      ##
      # Yields a new token to the supplied block.
      #
      # @param [Symbol] type The token type.
      # @param [String] value The token value.
      #
      # @yieldparam [Symbol] type
      # @yieldparam [String|NilClass] value
      #
      def add_token(type, value = nil)
        @block.call(type, value)
      end

      %%{
        getkey (data.getbyte(p) || 0);

        whitespace = [\t ]+;

        comma  = ',' %{ add_token(:T_COMMA) };
        hash   = '#' %{ add_token(:T_HASH) };
        dot    = '.' %{ add_token(:T_DOT) };
        lbrack = '[' %{ add_token(:T_LBRACK) };
        rbrack = ']' %{ add_token(:T_RBRACK) };
        colon  = ':' %{ add_token(:T_COLON) };
        lparen = '(';
        rparen = ')';
        pipe   = '|';
        odd    = 'odd';
        even   = 'even';
        minus  = '-';
        nth    = 'n';

        # Identifiers
        #
        # Identifiers are used for element and attribute names. Identifiers have
        # to start with a letter.

        identifier = '*' | [a-zA-Z]+ [a-zA-Z\-_0-9]*;

        action emit_identifier {
          emit(:T_IDENT, ts, te)
        }

        # Operators
        #
        # Various operators that can be used for filtering nodes. For example,
        # "$=" can be used to select attribute values that end with a given
        # string.
        #
        # http://www.w3.org/TR/css3-selectors/#selectors

        op_eq          = '=';
        op_space_in    = '~=';
        op_starts_with = '^=';
        op_ends_with   = '$=';
        op_in          = '*=';
        op_hyphen_in   = '|=';
        op_child       = '>';
        op_fol_direct  = '+';
        op_fol         = '~';

        # Numbers
        #
        # CSS selectors only understand integers, floating points are not
        # supported.

        integer = ('-' | '+')* digit+;

        action emit_integer {
          value = slice_input(ts, te).to_i

          add_token(:T_INT, value)
        }

        # Strings
        #
        # Strings can be single or double quoted. They are mainly used for
        # attribute values.
        #
        dquote = '"';
        squote = "'";

        string_dquote = (dquote ^dquote* dquote);
        string_squote = (squote ^squote* squote);

        string = string_dquote | string_squote;

        action emit_string {
          emit(:T_STRING, ts + 1, te - 1)
        }

        # Nth numbers
        #
        # These numbers are in the form of 2n+1 and are used for
        # pseudo-selectors such as nth-child(2n+1). The following parts such as
        # "-1" and "+1" are handled by the `integer` type and the corresponding
        # `emit_integer` action.

        #nth_integer    = integer 'n';
        #nth_identifier = '+n' | '-n';

        #action emit_nth_integer {
        #  value = slice_input(ts, te - 1).to_i

        #  add_token(:T_INT, value)
        #  add_token(:T_NTH, nil)
        #}

        #action emit_nth_identifier {
        #  emit(:T_NTH, ts, te)
        #}

        # Pseudo Classes
        #
        # http://www.w3.org/TR/css3-selectors/#structural-pseudos

        action emit_lparen {
          add_token(:T_LPAREN)

          fnext pseudo_args;
        }

        action emit_rparen {
          add_token(:T_RPAREN)

          fnext main;
        }

        # Machine used for processing the arguments of a pseudo class. These are
        # handled in a separate machine as these arguments can contain data not
        # allowed elsewhere. For example, "2n" is not allowed to appear outside
        # of the arguments list.
        pseudo_args := |*
          nth     => { add_token(:T_NTH) };
          minus   => { add_token(:T_MINUS) };
          odd     => { add_token(:T_ODD) };
          even    => { add_token(:T_EVEN) };
          integer => emit_integer;
          rparen  => emit_rparen;
        *|;

        main := |*
          whitespace | comma | hash | dot | lbrack | rbrack | colon;

          # Some of the operators have similar characters (e.g. the "="). As a
          # result we can't use rules like the following:
          #
          #     '='  %{ ... };
          #     '*=' %{ ... };
          #
          # This would result in both machines being executed for the input
          # "*=". The syntax below ensures that only the first match is handled.

          op_eq          => { add_token(:T_EQ) };
          op_space_in    => { add_token(:T_SPACE_IN) };
          op_starts_with => { add_token(:T_STARTS_WITH) };
          op_ends_with   => { add_token(:T_ENDS_WITH) };
          op_in          => { add_token(:T_IN) };
          op_hyphen_in   => { add_token(:T_HYPHEN_IN) };
          op_child       => { add_token(:T_CHILD) };
          op_fol_direct  => { add_token(:T_FOLLOWING_DIRECT) };
          op_fol         => { add_token(:T_FOLLOWING) };

          # The pipe character is also used in the |= operator so the action for
          # this is handled separately.
          pipe => { add_token(:T_PIPE) };

          lparen     => emit_lparen;
          identifier => emit_identifier;
          integer    => emit_integer;
          string     => emit_string;

          any;
        *|;
      }%%
    end # Lexer
  end # CSS
end # Oga


[testing] allow configurable IO chunk size

This allows the IO chunk size to be configured for the BEL Script
parser. It enables testing how the chunk size may impact parsing.

=begin
%%{
machine bel;

  include 'common.rl';
  include 'define.rl';
  include 'set.rl';
  include 'statement.rl';
  include 'term.rl';

  document_main :=
    (
		  SP* <: (
			  DEFINE_KW SP+ ANNOTATION_KW @call_define_annotation |
			  DEFINE_KW SP+ NAMESPACE_KW @call_define_namespace |
			  SET_KW @call_set |
			  COMMENT ^NL+ >s $n NL %comment |
			  UNSET_KW @call_unset |
			  ^('\r' | '\n' | '#' | 'D' | 'S' | 'U') >{fpc -= 1;} >statement_init >call_statement |
			  NL @newline
			)
    )+;
}%%
=end

require_relative 'language'
require_relative 'namespace'
require_relative 'evidence_model'
require_relative 'nonblocking_io_wrapper'

module BEL
  module Script

    class << self

      MAX_LENGTH = 1024 * 128 # 128K

      # Parse BEL Script +content+ into an enumerator of objects.
      #
			# @param [String, IO] content the BEL Script data to parse
			# @param [Hash]       namespaces the
			#        {BEL::Namespace::NamespaceDefinition} to use when matching
			#				 prefixes in BEL statements; defaults to empty meaning use the
			#				 +DEFINE NAMESPACE+ definitions in the content
			# @return [Enumerator] the parsed objects
      def parse(content, namespaces = {})
        return nil unless content

        parser =
          if content.is_a? String
					  if !content.end_with?("\n")
							content << "\n"
						end
            Parser.new(content, namespaces)
          elsif content.respond_to? :read
            IOParser.new(content, namespaces, MAX_LENGTH)
          else
            nil
          end

        unless parser
          fail ArgumentError, "content: expected string or io-like"
        end

        if block_given?
          parser.each do |obj|
            yield obj
          end
        else
          if parser.respond_to? :lazy
            parser.lazy
          else
            parser
          end
        end
      end

      # Parse BEL Script +content+, in chunks, into an enumerator of objects.
      # The length of the chunks can be configured with +chunk_length+.
      #
			# @param  [IO]         io the BEL Script data to parse; if a {String}
			#         then {#parse} will be called instead
			# @param  [Hash]       namespaces the
			#         {BEL::Namespace::NamespaceDefinition} to use when matching
			#				  prefixes in BEL statements; defaults to empty meaning use the
			#				  +DEFINE NAMESPACE+ definitions in the content
			# @param  [Integer]    chunk_length determines how many bytes are
			#         buffered at a time when +io+ is an {IO}
			# @return [Enumerator] the parsed objects
			def parse_chunked(io, namespaces = {}, chunk_length = MAX_LENGTH)
				parser =
					if io.is_a? String
						parse(io, namespaces)
					elsif io.respond_to? :read
						IOParser.new(io, namespaces, chunk_length)
					else
						nil
					end

				unless parser
					fail ArgumentError, "content: expected string or io-like"
				end

				if block_given?
					parser.each do |obj|
						yield obj
					end
				else
					if parser.respond_to? :lazy
						parser.lazy
					else
						parser
					end
				end
			end
    end

    private

    class Parser
      include Enumerable

      def initialize(content, namespaces = {})
        @content = content
        @namespaces =
          case namespaces
          when BEL::Namespace::ResourceIndex
            Hash[namespaces.namespaces.map { |ns| [ns.prefix, ns] }]
          when Hash
            namespaces
          end
        @annotations = {}
        @statement_group = nil
        %% write data;
      end

      def each
        eof = :ignored
        buffer = []
        stack = []
        data = @content.unpack('C*')

        %% write init;
        %% write exec;
      end
    end

    class IOParser
      include Enumerable

      def initialize(io, namespaces = {}, max_chunk_length)
        @io         = NonblockingIOWrapper.new(io, max_chunk_length)
        @namespaces =
          case namespaces
          when BEL::Namespace::ResourceIndex
            Hash[namespaces.namespaces.map { |ns| [ns.prefix, ns] }]
          when Hash
            namespaces
          end
        @annotations = {}
        @statement_group = nil
        %% write data;
      end

      def each
        pe = :ignored
        eof = :ignored
        buffer = []
        stack = []

        %% write init;

				@io.each do |chunk|
					data = chunk.unpack('C*')
					p = 0
					pe = data.length

					%% write exec;
				end
      end
    end
  end
end
# vim: ts=2 sw=2:
# encoding: utf-8

=begin
%%{
machine bel;

  include 'common.rl';
  include 'define.rl';
  include 'set.rl';
  include 'statement.rl';
  include 'term.rl';

  document_main :=
    (
		  SP* <: (
			  DEFINE_KW SP+ ANNOTATION_KW @call_define_annotation |
			  DEFINE_KW SP+ NAMESPACE_KW @call_define_namespace |
			  SET_KW @call_set |
			  COMMENT ^NL+ >s $n NL %comment |
			  UNSET_KW @call_unset |
			  ^('\r' | '\n' | '#' | 'D' | 'S' | 'U') >{fpc -= 1;} >statement_init >call_statement |
			  NL @newline
			)
    )+;
}%%
=end

require_relative 'language'
require_relative 'namespace'
require_relative 'evidence_model'
require_relative 'nonblocking_io_wrapper'

module BEL
  module Script

    class << self
      def parse(content, namespaces = {})
        return nil unless content

        parser =
          if content.is_a? String
					  if !content.end_with?("\n")
							content << "\n"
						end
            Parser.new(content, namespaces)
          elsif content.respond_to? :read
            IOParser.new(content, namespaces)
          else
            nil
          end

        unless parser
          fail ArgumentError, "content: expected string or io-like"
        end

        if block_given?
          parser.each do |obj|
            yield obj
          end
        else
          if parser.respond_to? :lazy
            parser.lazy
          else
            parser
          end
        end
      end
    end

    private

    class Parser
      include Enumerable

      def initialize(content, namespaces = {})
        @content = content
        @namespaces =
          case namespaces
          when BEL::Namespace::ResourceIndex
            Hash[namespaces.namespaces.map { |ns| [ns.prefix, ns] }]
          when Hash
            namespaces
          end
        @annotations = {}
        @statement_group = nil
        %% write data;
      end

      def each
        eof = :ignored
        buffer = []
        stack = []
        data = @content.unpack('C*')

        %% write init;
        %% write exec;
      end
    end

    class IOParser
      include Enumerable

      MAX_LENGTH = 1024 * 128 # 128K

      def initialize(io, namespaces = {})
        @io         = NonblockingIOWrapper.new(io, MAX_LENGTH)
        @namespaces =
          case namespaces
          when BEL::Namespace::ResourceIndex
            Hash[namespaces.namespaces.map { |ns| [ns.prefix, ns] }]
          when Hash
            namespaces
          end
        @annotations = {}
        @statement_group = nil
        %% write data;
      end

      def each
        pe = :ignored
        eof = :ignored
        buffer = []
        stack = []

        %% write init;

        leftover = []
        my_ts = nil
        my_te = nil

				@io.each do |chunk|
					data = leftover + chunk.unpack('C*')
					p = 0
					pe = data.length
					%% write exec;
					if my_ts
						leftover = data[my_ts..-1]
						my_te = my_te - my_ts if my_te
						my_ts = 0
					else
						leftover = []
					end
				end
      end
    end
  end
end

if __FILE__ == $0
  namespaces = BEL::Namespace::DEFAULT_NAMESPACES.map { |ns|
    [ns.prefix, ns]
  }.to_h

  if ARGV[0]
    File.open(ARGV[0], 'r:UTF-8') do |f|
      BEL::Script.parse(f, namespaces).each do |obj|
        puts obj
      end
    end
  else
    BEL::Script.parse($stdin, namespaces).each do |obj|
      puts obj
    end
  end
end
# vim: ts=2 sw=2:
# encoding: utf-8


cfg: Use log_* instead of fmt.

cfg: Use log_* instead of fmt.

/* cfg.rl - configure file parser for ndhs
 *
 * Copyright 2016-2020 Nicholas J. Kain <njkain at gmail dot com>
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are met:
 *
 * - Redistributions of source code must retain the above copyright notice,
 *   this list of conditions and the following disclaimer.
 *
 * - Redistributions in binary form must reproduce the above copyright notice,
 *   this list of conditions and the following disclaimer in the documentation
 *   and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 * POSSIBILITY OF SUCH DAMAGE.
 */

#include <string>
#include <cstdio>
#include <nk/scopeguard.hpp>
#include <nk/from_string.hpp>
#include "dhcp_state.hpp"
extern "C" {
#include "nk/log.h"
}
extern void set_user_runas(size_t linenum, std::string &&username);
extern void set_chroot_path(size_t linenum, std::string &&path);

#define MAX_LINE 2048

extern void create_dns_search_blob();

struct cfg_parse_state {
    cfg_parse_state() : st(nullptr), cs(0), last_addr(addr_type::null), default_lifetime(7200),
                        default_preference(0), parse_error(false) {}
    void newline() {
        duid.clear();
        iaid.clear();
        macaddr.clear();
        ipaddr.clear();
        ipaddr2.clear();
        last_addr = addr_type::null;
        parse_error = false;
    }
    const char *st;
    int cs;

    std::string duid;
    std::string iaid;
    std::string macaddr;
    std::string ipaddr;
    std::string ipaddr2;
    addr_type last_addr;
    std::string interface;
    uint32_t default_lifetime;
    uint8_t default_preference;
    bool parse_error;
};

static inline std::string lc_string(const char *s, size_t slen)
{
    auto r = std::string(s, slen);
    for (auto &i: r) i = tolower(i);
    return r;
}

%%{
    machine cfg_line_m;
    access cps.;

    action St { cps.st = p; }

    action DuidEn { cps.duid = lc_string(cps.st, p - cps.st); }
    action IaidEn { cps.iaid = lc_string(cps.st, p - cps.st); }
    action MacAddrEn { cps.macaddr = lc_string(cps.st, p - cps.st); }
    action V4AddrEn {
        cps.ipaddr = lc_string(cps.st, p - cps.st);
        cps.last_addr = addr_type::v4;
    }
    action V6AddrEn {
        cps.ipaddr = lc_string(cps.st, p - cps.st);
        cps.last_addr = addr_type::v6;
    }
    action Bind4En { emplace_bind(linenum, std::string(cps.st, p - cps.st), true); }
    action Bind6En { emplace_bind(linenum, std::string(cps.st, p - cps.st), false); }
    action UserEn { set_user_runas(linenum, std::string(cps.st, p - cps.st)); }
    action ChrootEn { set_chroot_path(linenum, std::string(cps.st, p - cps.st)); }
    action DefLifeEn {
        if (auto t = nk::from_string<uint32_t>(cps.st, p - cps.st)) cps.default_lifetime = *t; else {
            cps.parse_error = true;
            fbreak;
        }
    }
    action DefPrefEn {
        if (auto t = nk::from_string<uint8_t>(cps.st, p - cps.st)) cps.default_preference = *t; else {
            log_warning("default_preference on line %zu out of range [0,255]: %s",
                        linenum, std::string(cps.st, p - cps.st));
            cps.parse_error = true;
            fbreak;
        }
    }
    action InterfaceEn {
        cps.interface = std::string(cps.st, p - cps.st);
        emplace_interface(linenum, cps.interface, cps.default_preference);
    }
    action DnsServerEn {
        emplace_dns_server(linenum, cps.interface, cps.ipaddr, cps.last_addr);
    }
    action DnsSearchEn {
        emplace_dns_search(linenum, cps.interface, std::string(cps.st, p - cps.st));
    }
    action NtpServerEn {
        emplace_ntp_server(linenum, cps.interface, cps.ipaddr, cps.last_addr);
    }
    action SubnetEn {
        emplace_subnet(linenum, cps.interface, cps.ipaddr);
    }
    action GatewayEn {
        emplace_gateway(linenum, cps.interface, cps.ipaddr);
    }
    action BroadcastEn {
        emplace_broadcast(linenum, cps.interface, cps.ipaddr);
    }
    action DynRangePreEn {
        cps.ipaddr2 = std::move(cps.ipaddr);
    }
    action DynRangeEn {
        emplace_dynamic_range(linenum, cps.interface, cps.ipaddr2, cps.ipaddr,
                              cps.default_lifetime);
    }
    action DynamicV6En {
        emplace_dynamic_v6(linenum, cps.interface);
    }
    action V4EntryEn {
        emplace_dhcp_state(linenum, cps.interface, cps.macaddr, cps.ipaddr,
                           cps.default_lifetime);
    }
    action V6EntryEn {
        if (auto iaid = nk::from_string<uint32_t>(cps.iaid)) {
            emplace_dhcp_state(linenum, cps.interface, std::move(cps.duid),
                               *iaid, cps.ipaddr, cps.default_lifetime);
        } else {
            cps.parse_error = true;
            fbreak;
        }
    }

    duid = (xdigit+ | (xdigit{2} ('-' xdigit{2})*)+) >St %DuidEn;
    iaid = digit+ >St %IaidEn;
    macaddr = ((xdigit{2} ':'){5} xdigit{2}) >St %MacAddrEn;
    v4_addr = (digit{1,3} | '.')+ >St %V4AddrEn;
    v6_addr = (xdigit{1,4} | ':')+ >St %V6AddrEn;

    comment = space* ('//' any*)?;
    tcomment = (space+ '//' any*)?;
    bind4 = space* 'bind4' (space+ alnum+ >St %Bind4En)+ tcomment;
    bind6 = space* 'bind6' (space+ alnum+ >St %Bind6En)+ tcomment;
    user = space* 'user' space+ graph+ >St %UserEn tcomment;
    chroot = space* 'chroot' space+ graph+ >St %ChrootEn tcomment;
    default_lifetime = space* 'default_lifetime' space+ digit+ >St %DefLifeEn tcomment;
    default_preference = space* 'default_preference' space+ digit+ >St %DefPrefEn tcomment;
    interface = space* 'interface' space+ alnum+ >St %InterfaceEn tcomment;
    dns_server = space* 'dns_server' (space+ (v4_addr | v6_addr) %DnsServerEn)+ tcomment;
    dns_search = space* 'dns_search' (space+ graph+ >St %DnsSearchEn)+ tcomment;
    ntp_server = space* 'ntp_server' (space+ (v4_addr | v6_addr) %NtpServerEn)+ tcomment;
    subnet = space* 'subnet' space+ v4_addr %SubnetEn tcomment;
    gateway = space* 'gateway' space+ v4_addr %GatewayEn tcomment;
    broadcast = space* 'broadcast' space+ v4_addr %BroadcastEn tcomment;
    dynamic_range = space* 'dynamic_range' space+ v4_addr %DynRangePreEn space+ v4_addr %DynRangeEn tcomment;
    dynamic_v6 = space* 'dynamic_v6' %DynamicV6En tcomment;
    v4_entry = space* 'v4' space+ macaddr space+ v4_addr tcomment;
    v6_entry = space* 'v6' space+ duid space+ iaid space+ v6_addr tcomment;

    main := comment | bind4 | bind6 | user | chroot | default_lifetime | default_preference
          | interface | dns_server | dns_search | ntp_server | subnet | gateway | broadcast
          | dynamic_range | dynamic_v6 | v6_entry %V6EntryEn | v4_entry %V4EntryEn;
}%%

%% write data;

static int do_parse_cfg_line(cfg_parse_state &cps, const char *p, size_t plen,
                             const size_t linenum)
{
    const char *pe = p + plen;
    const char *eof = pe;

    %% write init;
    %% write exec;

    if (cps.parse_error) return -1;
    if (cps.cs >= cfg_line_m_first_final)
        return 1;
    if (cps.cs == cfg_line_m_error)
        return -1;
    return -2;
}

void parse_config(const std::string &path)
{
    char buf[MAX_LINE];
    auto f = fopen(path.c_str(), "r");
    if (!f) {
        log_warning("%s: failed to open config file \"%s\" for read: %s",
                    __func__, path.c_str(), strerror(errno));
        return;
    }
    SCOPE_EXIT{ fclose(f); };
    size_t linenum = 0;
    cfg_parse_state ps;
    while (!feof(f)) {
        if (!fgets(buf, sizeof buf, f)) {
            if (!feof(f))
                log_warning("%s: io error fetching line of '%s'", __func__, path.c_str());
            break;
        }
        auto llen = strlen(buf);
        if (llen == 0)
            continue;
        if (buf[llen-1] == '\n')
            buf[--llen] = 0;
        ++linenum;
        ps.newline();
        const auto r = do_parse_cfg_line(ps, buf, llen, linenum);
        if (r < 0) {
            if (r == -2)
                log_warning("%s: Incomplete configuration at line %zu; ignoring",
                            __func__, linenum);
            else
                log_warning("%s: Malformed configuration at line %zu; ignoring.",
                            __func__, linenum);
            continue;
        }
    }
    create_blobs();
}

/* cfg.rl - configure file parser for ndhs
 *
 * Copyright 2016-2017 Nicholas J. Kain <njkain at gmail dot com>
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are met:
 *
 * - Redistributions of source code must retain the above copyright notice,
 *   this list of conditions and the following disclaimer.
 *
 * - Redistributions in binary form must reproduce the above copyright notice,
 *   this list of conditions and the following disclaimer in the documentation
 *   and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 * POSSIBILITY OF SUCH DAMAGE.
 */

#include <string>
#include <cstdio>
#include <nk/scopeguard.hpp>
#include <fmt/format.h>
#include <nk/from_string.hpp>
#include "dhcp_state.hpp"
extern void set_user_runas(size_t linenum, std::string &&username);
extern void set_chroot_path(size_t linenum, std::string &&path);

#define MAX_LINE 2048

extern void create_dns_search_blob();

struct cfg_parse_state {
    cfg_parse_state() : st(nullptr), cs(0), last_addr(addr_type::null), default_lifetime(7200),
                        default_preference(0), parse_error(false) {}
    void newline() {
        duid.clear();
        iaid.clear();
        macaddr.clear();
        ipaddr.clear();
        ipaddr2.clear();
        last_addr = addr_type::null;
        parse_error = false;
    }
    const char *st;
    int cs;

    std::string duid;
    std::string iaid;
    std::string macaddr;
    std::string ipaddr;
    std::string ipaddr2;
    addr_type last_addr;
    std::string interface;
    uint32_t default_lifetime;
    uint8_t default_preference;
    bool parse_error;
};

static inline std::string lc_string(const char *s, size_t slen)
{
    auto r = std::string(s, slen);
    for (auto &i: r) i = tolower(i);
    return r;
}

%%{
    machine cfg_line_m;
    access cps.;

    action St { cps.st = p; }

    action DuidEn { cps.duid = lc_string(cps.st, p - cps.st); }
    action IaidEn { cps.iaid = lc_string(cps.st, p - cps.st); }
    action MacAddrEn { cps.macaddr = lc_string(cps.st, p - cps.st); }
    action V4AddrEn {
        cps.ipaddr = lc_string(cps.st, p - cps.st);
        cps.last_addr = addr_type::v4;
    }
    action V6AddrEn {
        cps.ipaddr = lc_string(cps.st, p - cps.st);
        cps.last_addr = addr_type::v6;
    }
    action Bind4En { emplace_bind(linenum, std::string(cps.st, p - cps.st), true); }
    action Bind6En { emplace_bind(linenum, std::string(cps.st, p - cps.st), false); }
    action UserEn { set_user_runas(linenum, std::string(cps.st, p - cps.st)); }
    action ChrootEn { set_chroot_path(linenum, std::string(cps.st, p - cps.st)); }
    action DefLifeEn {
        if (auto t = nk::from_string<uint32_t>(cps.st, p - cps.st)) cps.default_lifetime = *t; else {
            cps.parse_error = true;
            fbreak;
        }
    }
    action DefPrefEn {
        if (auto t = nk::from_string<uint8_t>(cps.st, p - cps.st)) cps.default_preference = *t; else {
            fmt::print(stderr, "default_preference on line {} out of range [0,255]: {}\n",
                       linenum, std::string(cps.st, p - cps.st));
            cps.parse_error = true;
            fbreak;
        }
    }
    action InterfaceEn {
        cps.interface = std::string(cps.st, p - cps.st);
        emplace_interface(linenum, cps.interface, cps.default_preference);
    }
    action DnsServerEn {
        emplace_dns_server(linenum, cps.interface, cps.ipaddr, cps.last_addr);
    }
    action DnsSearchEn {
        emplace_dns_search(linenum, cps.interface, std::string(cps.st, p - cps.st));
    }
    action NtpServerEn {
        emplace_ntp_server(linenum, cps.interface, cps.ipaddr, cps.last_addr);
    }
    action SubnetEn {
        emplace_subnet(linenum, cps.interface, cps.ipaddr);
    }
    action GatewayEn {
        emplace_gateway(linenum, cps.interface, cps.ipaddr);
    }
    action BroadcastEn {
        emplace_broadcast(linenum, cps.interface, cps.ipaddr);
    }
    action DynRangePreEn {
        cps.ipaddr2 = std::move(cps.ipaddr);
    }
    action DynRangeEn {
        emplace_dynamic_range(linenum, cps.interface, cps.ipaddr2, cps.ipaddr,
                              cps.default_lifetime);
    }
    action DynamicV6En {
        emplace_dynamic_v6(linenum, cps.interface);
    }
    action V4EntryEn {
        emplace_dhcp_state(linenum, cps.interface, cps.macaddr, cps.ipaddr,
                           cps.default_lifetime);
    }
    action V6EntryEn {
        if (auto iaid = nk::from_string<uint32_t>(cps.iaid)) {
            emplace_dhcp_state(linenum, cps.interface, std::move(cps.duid),
                               *iaid, cps.ipaddr, cps.default_lifetime);
        } else {
            cps.parse_error = true;
            fbreak;
        }
    }

    duid = (xdigit+ | (xdigit{2} ('-' xdigit{2})*)+) >St %DuidEn;
    iaid = digit+ >St %IaidEn;
    macaddr = ((xdigit{2} ':'){5} xdigit{2}) >St %MacAddrEn;
    v4_addr = (digit{1,3} | '.')+ >St %V4AddrEn;
    v6_addr = (xdigit{1,4} | ':')+ >St %V6AddrEn;

    comment = space* ('//' any*)?;
    tcomment = (space+ '//' any*)?;
    bind4 = space* 'bind4' (space+ alnum+ >St %Bind4En)+ tcomment;
    bind6 = space* 'bind6' (space+ alnum+ >St %Bind6En)+ tcomment;
    user = space* 'user' space+ graph+ >St %UserEn tcomment;
    chroot = space* 'chroot' space+ graph+ >St %ChrootEn tcomment;
    default_lifetime = space* 'default_lifetime' space+ digit+ >St %DefLifeEn tcomment;
    default_preference = space* 'default_preference' space+ digit+ >St %DefPrefEn tcomment;
    interface = space* 'interface' space+ alnum+ >St %InterfaceEn tcomment;
    dns_server = space* 'dns_server' (space+ (v4_addr | v6_addr) %DnsServerEn)+ tcomment;
    dns_search = space* 'dns_search' (space+ graph+ >St %DnsSearchEn)+ tcomment;
    ntp_server = space* 'ntp_server' (space+ (v4_addr | v6_addr) %NtpServerEn)+ tcomment;
    subnet = space* 'subnet' space+ v4_addr %SubnetEn tcomment;
    gateway = space* 'gateway' space+ v4_addr %GatewayEn tcomment;
    broadcast = space* 'broadcast' space+ v4_addr %BroadcastEn tcomment;
    dynamic_range = space* 'dynamic_range' space+ v4_addr %DynRangePreEn space+ v4_addr %DynRangeEn tcomment;
    dynamic_v6 = space* 'dynamic_v6' %DynamicV6En tcomment;
    v4_entry = space* 'v4' space+ macaddr space+ v4_addr tcomment;
    v6_entry = space* 'v6' space+ duid space+ iaid space+ v6_addr tcomment;

    main := comment | bind4 | bind6 | user | chroot | default_lifetime | default_preference
          | interface | dns_server | dns_search | ntp_server | subnet | gateway | broadcast
          | dynamic_range | dynamic_v6 | v6_entry %V6EntryEn | v4_entry %V4EntryEn;
}%%

%% write data;

static int do_parse_cfg_line(cfg_parse_state &cps, const char *p, size_t plen,
                             const size_t linenum)
{
    const char *pe = p + plen;
    const char *eof = pe;

    %% write init;
    %% write exec;

    if (cps.parse_error) return -1;
    if (cps.cs >= cfg_line_m_first_final)
        return 1;
    if (cps.cs == cfg_line_m_error)
        return -1;
    return -2;
}

void parse_config(const std::string &path)
{
    char buf[MAX_LINE];
    auto f = fopen(path.c_str(), "r");
    if (!f) {
        fmt::print(stderr, "{}: failed to open config file \"{}\" for read: {}\n",
                   __func__, path, strerror(errno));
        return;
    }
    SCOPE_EXIT{ fclose(f); };
    size_t linenum = 0;
    cfg_parse_state ps;
    while (!feof(f)) {
        if (!fgets(buf, sizeof buf, f)) {
            if (!feof(f))
                fmt::print(stderr, "{}: io error fetching line of '{}'\n", __func__, path);
            break;
        }
        auto llen = strlen(buf);
        if (llen == 0)
            continue;
        if (buf[llen-1] == '\n')
            buf[--llen] = 0;
        ++linenum;
        ps.newline();
        const auto r = do_parse_cfg_line(ps, buf, llen, linenum);
        if (r < 0) {
            if (r == -2)
                fmt::print(stderr, "{}: Incomplete configuration at line {}; ignoring\n",
                           __func__, linenum);
            else
                fmt::print(stderr, "{}: Malformed configuration at line {}; ignoring.\n",
                           __func__, linenum);
            continue;
        }
    }
    create_blobs();
}


Removed the buffering crap from the lexer.

%%machine lexer; # %

module Oga
  module XML
    ##
    # Low level lexer that supports both XML and HTML (using an extra option).
    # To lex HTML input set the `:html` option to `true` when creating an
    # instance of the lexer:
    #
    #     lexer = Oga::XML::Lexer.new(:html => true)
    #
    # @!attribute [r] html
    #  @return [TrueClass|FalseClass]
    #
    # @!attribute [r] tokens
    #  @return [Array]
    #
    class Lexer
      %% write data;

      # % fix highlight

      attr_reader :html

      ##
      # Names of the HTML void elements that should be handled when HTML lexing
      # is enabled.
      #
      # @return [Set]
      #
      HTML_VOID_ELEMENTS = Set.new([
        'area',
        'base',
        'br',
        'col',
        'command',
        'embed',
        'hr',
        'img',
        'input',
        'keygen',
        'link',
        'meta',
        'param',
        'source',
        'track',
        'wbr'
      ])

      ##
      # @param [String] data The data to lex.
      #
      # @param [Hash] options
      #
      # @option options [Symbol] :html When set to `true` the lexer will treat
      # the input as HTML instead of SGML/XML. This makes it possible to lex
      # HTML void elements such as `<link href="">`.
      #
      def initialize(data, options = {})
        options.each do |key, value|
          instance_variable_set("@#{key}", value) if respond_to?(key)
        end

        @data = data

        reset
      end

      ##
      # Resets the internal state of the lexer. Typically you don't need to
      # call this method yourself as its called by #lex after lexing a given
      # String.
      #
      def reset
        @line     = 1
        @elements = []

        @buffer_start_position = nil
      end

      ##
      # Gathers all the tokens for the input and returns them as an Array.
      #
      # This method resets the internal state of the lexer after consuming the
      # input.
      #
      # @param [String] data The string to consume.
      # @return [Array]
      # @see #advance
      #
      def lex
        tokens = []

        advance do |token|
          tokens << token
        end

        reset

        return tokens
      end

      ##
      # Advances through the input and generates the corresponding tokens. Each
      # token is yielded to the supplied block.
      #
      # Each token is an Array in the following format:
      #
      #     [TYPE, VALUE]
      #
      # The type is a symbol, the value is either nil or a String.
      #
      # This method stores the supplied block in `@block` and resets it after
      # the lexer loop has finished.
      #
      # This method does *not* reset the internal state of the lexer.
      #
      #
      # @param [String] data The String to consume.
      # @return [Array]
      #
      def advance(&block)
        @block = block

        data  = @data
        ts    = nil
        te    = nil
        stack = []
        top   = 0
        cs    = self.class.lexer_start
        act   = 0
        eof   = @data.bytesize
        p     = 0
        pe    = eof

        _lexer_eof_trans          = self.class.send(:_lexer_eof_trans)
        _lexer_from_state_actions = self.class.send(:_lexer_from_state_actions)
        _lexer_index_offsets      = self.class.send(:_lexer_index_offsets)
        _lexer_indicies           = self.class.send(:_lexer_indicies)
        _lexer_key_spans          = self.class.send(:_lexer_key_spans)
        _lexer_to_state_actions   = self.class.send(:_lexer_to_state_actions)
        _lexer_trans_actions      = self.class.send(:_lexer_trans_actions)
        _lexer_trans_keys         = self.class.send(:_lexer_trans_keys)
        _lexer_trans_targs        = self.class.send(:_lexer_trans_targs)

        %% write exec;

        # % fix highlight
      ensure
        @block = nil
      end

      ##
      # @return [TrueClass|FalseClass]
      #
      def html?
        return !!html
      end

      private

      ##
      # @param [Fixnum] amount The amount of lines to advance.
      #
      def advance_line(amount = 1)
        @line += amount
      end

      ##
      # Emits a token who's value is based on the supplied start/stop position.
      #
      # @param [Symbol] type The token type.
      # @param [Fixnum] start
      # @param [Fixnum] stop
      #
      # @see #text
      # @see #add_token
      #
      def emit(type, start, stop)
        value = text(start, stop)

        add_token(type, value)
      end

      ##
      # Returns the text of the current buffer based on the supplied start and
      # stop position.
      #
      # @param [Fixnum] start
      # @param [Fixnum] stop
      # @return [String]
      #
      def text(start, stop)
        return @data.byteslice(start, stop - start)
      end

      ##
      # Adds a token with the given type and value to the list.
      #
      # @param [Symbol] type The token type.
      # @param [String] value The token value.
      #
      def add_token(type, value = nil)
        token = [type, value, @line]

        @block.call(token)
      end

      ##
      # Enables buffering starting at the given position.
      #
      # @param [Fixnum] position The start position of the buffer.
      #
      def start_buffer(position)
        @buffer_start_position = position
      end

      ##
      # Emits a text token.
      #
      # @param [Fixnum] start
      # @param [Fixnum] stop
      #
      def emit_text(start, stop)
        content = text(start, stop)

        unless content.empty?
          add_token(:T_TEXT, content)

          lines = content.count("\n")

          advance_line(lines) if lines > 0
        end
      end

      ##
      # Returns the name of the element we're currently in.
      #
      # @return [String]
      #
      def current_element
        return @elements.last
      end

      %%{
        getkey (data.getbyte(p) || 0);

        newline    = '\n' | '\r\n';
        whitespace = [ \t];
        identifier = [a-zA-Z0-9\-_:]+;

        # Strings
        #
        # Strings in HTML can either be single or double quoted. If a string
        # starts with one of these quotes it must be closed with the same type
        # of quote.
        dquote = '"';
        squote = "'";

        # Machine for processing double quoted strings.
        string_dquote := |*
           ^dquote+ => {
             emit(:T_STRING, ts, te)
           };

           dquote => { fret; };
        *|;

        # Machine for processing single quoted strings.
        string_squote := |*
          ^squote+ => {
            emit(:T_STRING, ts, te)
          };

          squote => { fret; };
        *|;

        # DOCTYPES
        #
        # http://www.w3.org/TR/html-markup/syntax.html#doctype-syntax
        #
        # These rules support the 3 flavours of doctypes:
        #
        # 1. Normal doctypes, as introduced in the HTML5 specification.
        # 2. Deprecated doctypes, the more verbose ones used prior to HTML5.
        # 3. Legacy doctypes
        #
        doctype_start = '<!DOCTYPE'i whitespace+;

        action start_doctype {
          add_token(:T_DOCTYPE_START)
          fcall doctype;
        }

        # Machine for processing doctypes. Doctype values such as the public
        # and system IDs are treated as T_STRING tokens.
        doctype := |*
          'PUBLIC' | 'SYSTEM' => { emit(:T_DOCTYPE_TYPE, ts, te) };

          # Lex the public/system IDs as regular strings.
          dquote => { fcall string_dquote; };
          squote => { fcall string_squote; };

          # Whitespace inside doctypes is ignored since there's no point in
          # including it.
          whitespace;

          identifier => { emit(:T_DOCTYPE_NAME, ts, te) };

          '>' => {
            add_token(:T_DOCTYPE_END)
            fret;
          };
        *|;

        # CDATA
        #
        # http://www.w3.org/TR/html-markup/syntax.html#cdata-sections
        #
        # CDATA tags are broken up into 3 parts: the start, the content and the
        # end tag.
        #
        # In HTML CDATA tags have no meaning/are not supported. Oga does
        # support them but treats their contents as plain text.
        #
        cdata_start = '<![CDATA[';
        cdata_end   = ']]>';

        action start_cdata {
          add_token(:T_CDATA_START)

          fcall cdata;
        }

        # Machine that for processing the contents of CDATA tags. Everything
        # inside a CDATA tag is treated as plain text.
        cdata := |*
          any* cdata_end => {
            emit_text(ts, te - 3)
            add_token(:T_CDATA_END)

            fret;
          };
        *|;

        # Comments
        #
        # http://www.w3.org/TR/html-markup/syntax.html#comments
        #
        # Comments are lexed into 3 parts: the start tag, the content and the
        # end tag.
        #
        # Unlike the W3 specification these rules *do* allow character
        # sequences such as `--` and `->`. Putting extra checks in for these
        # sequences would actually make the rules/actions more complex.
        #
        comment_start = '<!--';
        comment_end   = '-->';

        action start_comment {
          add_token(:T_COMMENT_START)

          fcall comment;
        }

        # Machine used for processing the contents of a comment. Everything
        # inside a comment is treated as plain text (similar to CDATA tags).
        comment := |*
          any* comment_end => {
            emit_text(ts, te - 3)
            add_token(:T_COMMENT_END)

            fret;
          };
        *|;

        # XML declaration tags
        #
        # http://www.w3.org/TR/REC-xml/#sec-prolog-dtd
        #
        xml_decl_start = '<?xml';
        xml_decl_end   = '?>';

        action start_xml_decl {
          add_token(:T_XML_DECL_START)

          fcall xml_decl;
        }

        # Machine that processes the contents of an XML declaration tag.
        xml_decl := |*
          xml_decl_end => {
            add_token(:T_XML_DECL_END)

            fret;
          };

          # Attributes and their values (e.g. version="1.0").
          identifier => { emit(:T_ATTR, ts, te) };

          dquote => { fcall string_dquote; };
          squote => { fcall string_squote; };

          any;
        *|;

        # Elements
        #
        # http://www.w3.org/TR/html-markup/syntax.html#syntax-elements
        #

        # Action that creates the tokens for the opening tag, name and
        # namespace (if any). Remaining work is delegated to a dedicated
        # machine.
        action start_element {
          add_token(:T_ELEM_START)

          # Add the element name. If the name includes a namespace we'll break
          # the name up into two separate tokens.
          name = text(ts + 1, te)

          if name.include?(':')
            ns, name = name.split(':')

            add_token(:T_ELEM_NS, ns)
          end

          @elements << name if html?

          add_token(:T_ELEM_NAME, name)

          fcall element_head;
        }

        element_start = '<' identifier;

        # Machine used for processing the characters inside a element head. An
        # element head is everything between `<NAME` (where NAME is the element
        # name) and `>`.
        #
        # For example, in `<p foo="bar">` the element head is ` foo="bar"`.
        #
        element_head := |*
          whitespace | '=';

          newline => { advance_line };

          # Attribute names.
          identifier => { emit(:T_ATTR, ts, te) };

          # Attribute values.
          dquote => { fcall string_dquote; };
          squote => { fcall string_squote; };

          # The closing character of the open tag.
          ('>' | '/') => {
            fhold;
            fret;
          };
        *|;

        main := |*
          element_start  => start_element;
          doctype_start  => start_doctype;
          cdata_start    => start_cdata;
          comment_start  => start_comment;
          xml_decl_start => start_xml_decl;

          # Enter the body of the tag. If HTML mode is enabled and the current
          # element is a void element we'll close it and bail out.
          '>' => {
            if html? and HTML_VOID_ELEMENTS.include?(current_element)
              add_token(:T_ELEM_END, nil)
              @elements.pop
            end
          };

          # Regular closing tags.
          '</' identifier '>' => {
            add_token(:T_ELEM_END, nil)

            @elements.pop if html?
          };

          # Self closing elements that are not handled by the HTML mode.
          '/>' => {
            add_token(:T_ELEM_END, nil)

            @elements.pop if html?
          };

          # Note that this rule should be declared at the very bottom as it
          # will otherwise take precedence over the other rules.
          ^('<' | '>')+ => {
            emit_text(ts, te)
          };
        *|;
      }%%
    end # Lexer
  end # XML
end # Oga

%%machine lexer; # %

module Oga
  module XML
    ##
    # Low level lexer that supports both XML and HTML (using an extra option).
    # To lex HTML input set the `:html` option to `true` when creating an
    # instance of the lexer:
    #
    #     lexer = Oga::XML::Lexer.new(:html => true)
    #
    # @!attribute [r] html
    #  @return [TrueClass|FalseClass]
    #
    # @!attribute [r] tokens
    #  @return [Array]
    #
    class Lexer
      %% write data;

      # % fix highlight

      attr_reader :html

      ##
      # Names of the HTML void elements that should be handled when HTML lexing
      # is enabled.
      #
      # @return [Set]
      #
      HTML_VOID_ELEMENTS = Set.new([
        'area',
        'base',
        'br',
        'col',
        'command',
        'embed',
        'hr',
        'img',
        'input',
        'keygen',
        'link',
        'meta',
        'param',
        'source',
        'track',
        'wbr'
      ])

      ##
      # @param [String] data The data to lex.
      #
      # @param [Hash] options
      #
      # @option options [Symbol] :html When set to `true` the lexer will treat
      # the input as HTML instead of SGML/XML. This makes it possible to lex
      # HTML void elements such as `<link href="">`.
      #
      def initialize(data, options = {})
        options.each do |key, value|
          instance_variable_set("@#{key}", value) if respond_to?(key)
        end

        @data = data

        reset
      end

      ##
      # Resets the internal state of the lexer. Typically you don't need to
      # call this method yourself as its called by #lex after lexing a given
      # String.
      #
      def reset
        @line     = 1
        @elements = []

        @buffer_start_position = nil
      end

      ##
      # Gathers all the tokens for the input and returns them as an Array.
      #
      # This method resets the internal state of the lexer after consuming the
      # input.
      #
      # @param [String] data The string to consume.
      # @return [Array]
      # @see #advance
      #
      def lex
        tokens = []

        advance do |token|
          tokens << token
        end

        reset

        return tokens
      end

      ##
      # Advances through the input and generates the corresponding tokens. Each
      # token is yielded to the supplied block.
      #
      # Each token is an Array in the following format:
      #
      #     [TYPE, VALUE]
      #
      # The type is a symbol, the value is either nil or a String.
      #
      # This method stores the supplied block in `@block` and resets it after
      # the lexer loop has finished.
      #
      # This method does *not* reset the internal state of the lexer.
      #
      #
      # @param [String] data The String to consume.
      # @return [Array]
      #
      def advance(&block)
        @block = block

        data  = @data
        ts    = nil
        te    = nil
        stack = []
        top   = 0
        cs    = self.class.lexer_start
        act   = 0
        eof   = @data.bytesize
        p     = 0
        pe    = eof

        _lexer_eof_trans          = self.class.send(:_lexer_eof_trans)
        _lexer_from_state_actions = self.class.send(:_lexer_from_state_actions)
        _lexer_index_offsets      = self.class.send(:_lexer_index_offsets)
        _lexer_indicies           = self.class.send(:_lexer_indicies)
        _lexer_key_spans          = self.class.send(:_lexer_key_spans)
        _lexer_to_state_actions   = self.class.send(:_lexer_to_state_actions)
        _lexer_trans_actions      = self.class.send(:_lexer_trans_actions)
        _lexer_trans_keys         = self.class.send(:_lexer_trans_keys)
        _lexer_trans_targs        = self.class.send(:_lexer_trans_targs)

        %% write exec;

        # % fix highlight
      ensure
        @block = nil
      end

      ##
      # @return [TrueClass|FalseClass]
      #
      def html?
        return !!html
      end

      private

      ##
      # @param [Fixnum] amount The amount of lines to advance.
      #
      def advance_line(amount = 1)
        @line += amount
      end

      ##
      # Emits a token who's value is based on the supplied start/stop position.
      #
      # @param [Symbol] type The token type.
      # @param [Fixnum] start
      # @param [Fixnum] stop
      #
      # @see #text
      # @see #add_token
      #
      def emit(type, start, stop)
        value = text(start, stop)

        add_token(type, value)
      end

      ##
      # Returns the text of the current buffer based on the supplied start and
      # stop position.
      #
      # @param [Fixnum] start
      # @param [Fixnum] stop
      # @return [String]
      #
      def text(start, stop)
        return @data.byteslice(start, stop - start)
      end

      ##
      # Adds a token with the given type and value to the list.
      #
      # @param [Symbol] type The token type.
      # @param [String] value The token value.
      #
      def add_token(type, value = nil)
        token = [type, value, @line]

        @block.call(token)
      end

      ##
      # Enables buffering starting at the given position.
      #
      # @param [Fixnum] position The start position of the buffer.
      #
      def start_buffer(position)
        @buffer_start_position = position
      end

      ##
      # Emits the current buffer if we have any. The current line number is
      # advanced based on the amount of newlines in the buffer.
      #
      # @param [Fixnum] position The end position of the buffer.
      # @param [Symbol] type The type of node to emit.
      #
      def emit_buffer(position, type = :T_TEXT)
        return unless @buffer_start_position

        content = text(@buffer_start_position, position)

        unless content.empty?
          add_token(type, content)

          lines = content.count("\n")

          advance_line(lines) if lines > 0
        end

        @buffer_start_position = nil
      end

      ##
      # Returns the name of the element we're currently in.
      #
      # @return [String]
      #
      def current_element
        return @elements.last
      end

      %%{
        getkey (data.getbyte(p) || 0);

        newline    = '\n' | '\r\n';
        whitespace = [ \t];
        identifier = [a-zA-Z0-9\-_:]+;

        # Strings
        #
        # Strings in HTML can either be single or double quoted. If a string
        # starts with one of these quotes it must be closed with the same type
        # of quote.
        dquote = '"';
        squote = "'";

        action start_string_dquote {
          start_buffer(te)

          fcall string_dquote;
        }

        action start_string_squote {
          start_buffer(te)

          fcall string_squote;
        }

        # Machine for processing double quoted strings.
        string_dquote := |*
          dquote => {
            emit_buffer(ts, :T_STRING)
            fret;
          };

          any;
        *|;

        # Machine for processing single quoted strings.
        string_squote := |*
          squote => {
            emit_buffer(ts, :T_STRING)
            fret;
          };

          any;
        *|;

        # DOCTYPES
        #
        # http://www.w3.org/TR/html-markup/syntax.html#doctype-syntax
        #
        # These rules support the 3 flavours of doctypes:
        #
        # 1. Normal doctypes, as introduced in the HTML5 specification.
        # 2. Deprecated doctypes, the more verbose ones used prior to HTML5.
        # 3. Legacy doctypes
        #
        doctype_start = '<!DOCTYPE'i whitespace+;

        action start_doctype {
          emit_buffer(ts)
          add_token(:T_DOCTYPE_START)
          fcall doctype;
        }

        # Machine for processing doctypes. Doctype values such as the public
        # and system IDs are treated as T_STRING tokens.
        doctype := |*
          'PUBLIC' | 'SYSTEM' => { emit(:T_DOCTYPE_TYPE, ts, te) };

          # Lex the public/system IDs as regular strings.
          dquote => start_string_dquote;
          squote => start_string_squote;

          # Whitespace inside doctypes is ignored since there's no point in
          # including it.
          whitespace;

          identifier => { emit(:T_DOCTYPE_NAME, ts, te) };

          '>' => {
            add_token(:T_DOCTYPE_END)
            fret;
          };
        *|;

        # CDATA
        #
        # http://www.w3.org/TR/html-markup/syntax.html#cdata-sections
        #
        # CDATA tags are broken up into 3 parts: the start, the content and the
        # end tag.
        #
        # In HTML CDATA tags have no meaning/are not supported. Oga does
        # support them but treats their contents as plain text.
        #
        cdata_start = '<![CDATA[';
        cdata_end   = ']]>';

        action start_cdata {
          emit_buffer(ts)
          add_token(:T_CDATA_START)

          start_buffer(te)

          fcall cdata;
        }

        # Machine that for processing the contents of CDATA tags. Everything
        # inside a CDATA tag is treated as plain text.
        cdata := |*
          cdata_end => {
            emit_buffer(ts)
            add_token(:T_CDATA_END)

            fret;
          };

          any;
        *|;

        # Comments
        #
        # http://www.w3.org/TR/html-markup/syntax.html#comments
        #
        # Comments are lexed into 3 parts: the start tag, the content and the
        # end tag.
        #
        # Unlike the W3 specification these rules *do* allow character
        # sequences such as `--` and `->`. Putting extra checks in for these
        # sequences would actually make the rules/actions more complex.
        #
        comment_start = '<!--';
        comment_end   = '-->';

        action start_comment {
          emit_buffer(ts)
          add_token(:T_COMMENT_START)

          start_buffer(te)

          fcall comment;
        }

        # Machine used for processing the contents of a comment. Everything
        # inside a comment is treated as plain text (similar to CDATA tags).
        comment := |*
          comment_end => {
            emit_buffer(ts)
            add_token(:T_COMMENT_END)

            fret;
          };

          any;
        *|;

        # XML declaration tags
        #
        # http://www.w3.org/TR/REC-xml/#sec-prolog-dtd
        #
        xml_decl_start = '<?xml';
        xml_decl_end   = '?>';

        action start_xml_decl {
          emit_buffer(ts)
          add_token(:T_XML_DECL_START)

          start_buffer(te)

          fcall xml_decl;
        }

        # Machine that processes the contents of an XML declaration tag.
        xml_decl := |*
          xml_decl_end => {
            emit_buffer(ts)
            add_token(:T_XML_DECL_END)

            fret;
          };

          # Attributes and their values (e.g. version="1.0").
          identifier => { emit(:T_ATTR, ts, te) };

          dquote => start_string_dquote;
          squote => start_string_squote;

          any;
        *|;

        # Elements
        #
        # http://www.w3.org/TR/html-markup/syntax.html#syntax-elements
        #

        # Action that creates the tokens for the opening tag, name and
        # namespace (if any). Remaining work is delegated to a dedicated
        # machine.
        action start_element {
          emit_buffer(ts)
          add_token(:T_ELEM_START)

          # Add the element name. If the name includes a namespace we'll break
          # the name up into two separate tokens.
          name = text(ts + 1, te)

          if name.include?(':')
            ns, name = name.split(':')

            add_token(:T_ELEM_NS, ns)
          end

          @elements << name if html?

          add_token(:T_ELEM_NAME, name)

          fcall element_head;
        }

        element_start = '<' identifier;

        # Machine used for processing the characters inside a element head. An
        # element head is everything between `<NAME` (where NAME is the element
        # name) and `>`.
        #
        # For example, in `<p foo="bar">` the element head is ` foo="bar"`.
        #
        element_head := |*
          whitespace | '=';

          newline => { advance_line };

          # Attribute names.
          identifier => { emit(:T_ATTR, ts, te) };

          # Attribute values.
          dquote => start_string_dquote;
          squote => start_string_squote;

          # The closing character of the open tag.
          ('>' | '/') => {
            fhold;
            fret;
          };
        *|;

        main := |*
          element_start  => start_element;
          doctype_start  => start_doctype;
          cdata_start    => start_cdata;
          comment_start  => start_comment;
          xml_decl_start => start_xml_decl;

          # Enter the body of the tag. If HTML mode is enabled and the current
          # element is a void element we'll close it and bail out.
          '>' => {
            if html? and HTML_VOID_ELEMENTS.include?(current_element)
              add_token(:T_ELEM_END, nil)
              @elements.pop
            end
          };

          # Regular closing tags.
          '</' identifier '>' => {
            emit_buffer(ts)
            add_token(:T_ELEM_END, nil)

            @elements.pop if html?
          };

          # Self closing elements that are not handled by the HTML mode.
          '/>' => {
            add_token(:T_ELEM_END, nil)

            @elements.pop if html?
          };

          # Note that this rule should be declared at the very bottom as it
          # will otherwise take precedence over the other rules.
          any => {
            # First character, start buffering (unless we already are buffering).
            start_buffer(ts) unless @buffer_start_position

            # EOF, emit the text buffer.
            if te == eof
              emit_buffer(te)
            end
          };
        *|;
      }%%
    end # Lexer
  end # XML
end # Oga