Fixed priority for groups and split the assertions

%%{
  machine re_scanner;

  wild                  = '.';
  backslash             = '\\';
  alternation           = '|';
  beginning_of_line     = '^';
  end_of_line           = '$';

  range_open            = '{';
  range_close           = '}';
  curlies               = range_open | range_close;

  group_open            = '(';
  group_close           = ')';
  parantheses           = group_open | group_close;

  set_open              = '[';
  set_close             = ']';
  brackets              = set_open | set_close;

  posix_class_name      = 'alnum' | 'alpha' | 'blank' |
                          'cntrl' | 'digit' | 'graph' |
                          'lower' | 'print' | 'punct' |
                          'space' | 'upper' | 'xdigit' |
                          'word'  | 'ascii';

  property_name         = 'Alnum' | 'Alpha' | 'Any'   | 'Ascii' | 'Blank' |
                          'Cntrl' | 'Digit' | 'Graph' | 'Lower' | 'Print' |
                          'Punct' | 'Space' | 'Upper' | 'Word' | 'Xdigit';

  category_letter       = 'L' . [ultmo]?;
  category_mark         = 'M' . [nce]?;
  category_number       = 'N' . [dlo]?;
  category_punctuation  = 'P' . [cdseifo]?;
  category_symbol       = 'S' . [mcko]?;
  category_separator    = 'Z' . [slp]?;
  category_codepoint    = 'C' . [cfson]?;
  general_category      = category_letter | category_mark |
                          category_number | category_punctuation |
                          category_symbol | category_separator |
                          category_codepoint;

  posix_class           = '[:' . posix_class_name . ':]';

  char_type             = [dDhHsSwW];

  line_anchor           = beginning_of_line | end_of_line;
  anchor_char           = [AbBzZG];
  property_char         = [pP];

  escaped_char          = [aefnrtv];
  octal_sequence        = '0' . [0-7]{2};

  hex_sequence          = 'x' . xdigit{2};
  wide_hex_sequence     = 'x' . '{7' . xdigit{1,7} . '}';
  control_sequence      = ('c' | 'C-') . xdigit{1,2};
  meta_sequence         = 'M-' . xdigit{1,2};
  meta_control_sequence = 'M-\\C-' . xdigit{1,2};

  zero_or_one           = '?' | '??' | '?+';
  zero_or_more          = '*' | '*?' | '*+';
  one_or_more           = '+' | '+?' | '++';

  quantifier_greedy     = '?'  | '*'  | '+';
  quantifier_reluctant  = '??' | '*?' | '+?';
  quantifier_possessive = '?+' | '*+' | '++';
  quantifier_mode       = '?'  | '+';

  quantifier_range      = range_open . (digit+)? . ','? . (digit+)? .
                          range_close . quantifier_mode?;

  quantifiers           = quantifier_greedy | quantifier_reluctant |
                          quantifier_possessive | quantifier_range;

  group_comment         = '?#' . [^)]+ . group_close;

  group_atomic          = '?>';
  group_passive         = '?:';

  assertion_lookahead   = '?=';
  assertion_nlookahead  = '?!';
  assertion_lookbehind  = '?<=';
  assertion_nlookbehind = '?<!';

  group_options         = '?' . ([mix]{1,3})? . '-' . ([mix]{1,3})? . ':';

  group_name            = alpha . alnum+;
  group_named           = '?<' . group_name . '>';

  group_type            = group_atomic | group_passive | group_named;

  assertion_type        = assertion_lookahead  | assertion_nlookahead |
                          assertion_lookbehind | assertion_nlookbehind;

  # characters the 'break' a literal
  meta_char             = wild | backslash | alternation |
                          curlies | parantheses | brackets |
                          line_anchor | quantifier_greedy;

  # character set scanner, continues consuming characters until it meets the
  # closing bracket of the set.
  # --------------------------------------------------------------------------
  character_set := |*
    ']' {
      self.emit(:character_set, :close, data[ts..te-1].pack('c*'), ts, te)
      fret;
    };

    '^' {
      self.emit(:character_set, :negate, data[ts..te-1].pack('c*'), ts, te)
    };

    alnum . '-' . alnum { # TODO: add properties
      self.emit(:character_set, :range, data[ts..te-1].pack('c*'), ts, te)
    };

    '&&' {
      self.emit(:character_set, :intersection, data[ts..te-1].pack('c*'), ts, te)
    };

    '\\' . [\]\-\,b] {
      self.emit(:character_set, :member, data[ts..te-1].pack('c*'), ts, te)
    };

    posix_class {
      case text = data[ts..te-1].pack('c*')
      when '[:alnum:]';  self.emit(:character_set, :set_class_alnum,  text, ts, te)
      when '[:alpha:]';  self.emit(:character_set, :set_class_alpha,  text, ts, te)
      when '[:blank:]';  self.emit(:character_set, :set_class_blank,  text, ts, te)
      when '[:cntrl:]';  self.emit(:character_set, :set_class_cntrl,  text, ts, te)
      when '[:digit:]';  self.emit(:character_set, :set_class_digit,  text, ts, te)
      when '[:graph:]';  self.emit(:character_set, :set_class_graph,  text, ts, te)
      when '[:lower:]';  self.emit(:character_set, :set_class_lower,  text, ts, te)
      when '[:print:]';  self.emit(:character_set, :set_class_print,  text, ts, te)
      when '[:punct:]';  self.emit(:character_set, :set_class_punct,  text, ts, te)
      when '[:space:]';  self.emit(:character_set, :set_class_space,  text, ts, te)
      when '[:upper:]';  self.emit(:character_set, :set_class_upper,  text, ts, te)
      when '[:xdigit:]'; self.emit(:character_set, :set_class_xdigit, text, ts, te)
      when '[:word:]';   self.emit(:character_set, :set_class_word,   text, ts, te)
      when '[:ascii:]';  self.emit(:character_set, :set_class_ascii,  text, ts, te)
      else raise "Unsupported character posixe class at #{text} (char #{ts})"
      end
    };

    any {
      self.emit(:character_set, :member, data[ts..te-1].pack('c*'), ts, te)
    };
  *|;

  # escape sequence scanner
  # --------------------------------------------------------------------------
  escape_sequence := |*
    hex_sequence {
      self.emit(:escape_sequence, :hex, data[ts..te-1].pack('c*'), ts, te)
      fret;
    };

    wide_hex_sequence {
      self.emit(:escape_sequence, :hex_wide, data[ts..te-1].pack('c*'), ts, te)
      fret;
    };

    control_sequence {
      self.emit(:escape_sequence, :control, data[ts..te-1].pack('c*'), ts, te)
      fret;
    };

    meta_sequence {
      self.emit(:escape_sequence, :meta, data[ts..te-1].pack('c*'), ts, te)
      fret;
    };

    meta_control_sequence {
      self.emit(:escape_sequence, :meta_control, data[ts..te-1].pack('c*'), ts, te)
      fret;
    };

    # TODO: extract into a separate machine... use in sets
    property_char . '{' . (property_name | general_category) . '}' > (escaped_alpha, 2) {
      text = data[ts..te-1].pack('c*')
      type = text[0,1] == 'p' ? :property: :inverted_property

      case name = data[ts+2..te-2].pack('c*')

      # Named
      when 'Alnum';   self.emit(type, :alnum,  text, ts, te)
      when 'Alpha';   self.emit(type, :alpha,  text, ts, te)
      when 'Any';     self.emit(type, :any,    text, ts, te)
      when 'Ascii';   self.emit(type, :ascii,  text, ts, te)
      when 'Blank';   self.emit(type, :blank,  text, ts, te)
      when 'Cntrl';   self.emit(type, :cntrl,  text, ts, te)
      when 'Digit';   self.emit(type, :digit,  text, ts, te)
      when 'Graph';   self.emit(type, :graph,  text, ts, te)
      when 'Lower';   self.emit(type, :lower,  text, ts, te)
      when 'Print';   self.emit(type, :print,  text, ts, te)
      when 'Punct';   self.emit(type, :punct,  text, ts, te)
      when 'Space';   self.emit(type, :space,  text, ts, te)
      when 'Upper';   self.emit(type, :upper,  text, ts, te)
      when 'Word';    self.emit(type, :word,   text, ts, te)
      when 'Xdigit';  self.emit(type, :xdigit, text, ts, te)

      # Letters
      when 'Lu'; self.emit(type, :letter_uppercase, text, ts, te)
      when 'Ll'; self.emit(type, :letter_lowercase, text, ts, te)
      when 'Lt'; self.emit(type, :letter_titlecase, text, ts, te)
      when 'Lm'; self.emit(type, :letter_modifier,  text, ts, te)
      when 'Lo'; self.emit(type, :letter_other,     text, ts, te)

      # Marks
      when 'Mn'; self.emit(type, :mark_nonspacing,  text, ts, te)
      when 'Mc'; self.emit(type, :mark_spacing,     text, ts, te)
      when 'Me'; self.emit(type, :mark_enclosing,   text, ts, te)

      # Numbers
      when 'Nd'; self.emit(type, :number_decimal, text, ts, te)
      when 'Nl'; self.emit(type, :number_letter, text, ts, te)
      when 'No'; self.emit(type, :number_other, text, ts, te)

      # Punctuation
      when 'Pc'; self.emit(type, :punct_connector, text, ts, te)
      when 'Pd'; self.emit(type, :punct_dash, text, ts, te)
      when 'Ps'; self.emit(type, :punct_open, text, ts, te)
      when 'Pe'; self.emit(type, :punct_close, text, ts, te)
      when 'Pi'; self.emit(type, :punct_initial, text, ts, te)
      when 'Pf'; self.emit(type, :punct_final, text, ts, te)
      when 'Po'; self.emit(type, :punct_other, text, ts, te)

      # Symbols
      when 'Sm'; self.emit(type, :symbol_math, text, ts, te)
      when 'Sc'; self.emit(type, :symbol_currency, text, ts, te)
      when 'Sk'; self.emit(type, :symbol_modifier, text, ts, te)
      when 'So'; self.emit(type, :symbol_other, text, ts, te)

      # Separators
      when 'Zs'; self.emit(type, :separator_space, text, ts, te)
      when 'Zl'; self.emit(type, :separator_line, text, ts, te)
      when 'Zp'; self.emit(type, :separator_paragraph, text, ts, te)

      # Codepoints
      when 'Cc'; self.emit(type, :code_control, text, ts, te)
      when 'Cf'; self.emit(type, :code_format, text, ts, te)
      when 'Cs'; self.emit(type, :code_surrogate, text, ts, te)
      when 'Co'; self.emit(type, :code_private, text, ts, te)
      when 'Cn'; self.emit(type, :code_unassigned, text, ts, te)
      end
    };

    any > (escaped_alpha, 1)  {
      self.emit(:escape_sequence, :literal, data[ts..te-1].pack('c*'), ts, te)
      fret;
    };
  *|;

  # Main scanner
  main := |*

    # Anchors
    alternation {
      self.emit(:meta, :alternation, data[ts..te-1].pack('c*'), ts, te)
    };

    # Character types
    #   .         any (except new line)
    #   \d, \D    digit, non-digit
    #   \h, \H    hex, non-hex
    #   \s, \S    whitespace, non-whitespace
    #   \w, \W    word, non-word
    # ------------------------------------------------------------------------
    wild {
      self.emit(:character_type, :any, data[ts..te-1].pack('c*'), ts, te)
    };

    backslash . char_type > (backslashed, 2) {
      case text = data[ts..te-1].pack('c*')
      when '\\d'; self.emit(:character_type, :digit,      text, ts, te)
      when '\\D'; self.emit(:character_type, :non_digit,  text, ts, te)
      when '\\h'; self.emit(:character_type, :hex,        text, ts, te)
      when '\\H'; self.emit(:character_type, :non_hex,    text, ts, te)
      when '\\s'; self.emit(:character_type, :space,      text, ts, te)
      when '\\S'; self.emit(:character_type, :non_space,  text, ts, te)
      when '\\w'; self.emit(:character_type, :word,       text, ts, te)
      when '\\W'; self.emit(:character_type, :non_word,   text, ts, te)
      else raise "Unsupported character type at #{text} (char #{ts})"
      end
    };

    # Anchors
    beginning_of_line {
      self.emit(:anchor, :beginning_of_line, data[ts..te-1].pack('c*'), ts, te)
    };

    end_of_line {
      self.emit(:anchor, :end_of_line, data[ts..te-1].pack('c*'), ts, te)
    };

    backslash . anchor_char > (backslashed, 3) {
      case text = data[ts..te-1].pack('c*')
      when '\\A'; self.emit(:anchor, :bos,                text, ts, te)
      when '\\z'; self.emit(:anchor, :eos,                text, ts, te)
      when '\\Z'; self.emit(:anchor, :eos_or_before_eol,  text, ts, te)
      when '\\b'; self.emit(:anchor, :word_boundary,      text, ts, te)
      when '\\B'; self.emit(:anchor, :non_word_boundary,  text, ts, te)
      else raise "Unsupported anchor at #{text} (char #{ts})"
      end
    };

    # Escaped sequences
    backslash > (backslashed, 1) {
      fcall escape_sequence;
    };

    # Character sets
    set_open {
      self.emit(:character_set, :open, data[ts..te-1].pack('c*'), ts, te)
      fcall character_set;
    };

    # (?#...) comments: parsed as a single expression, without introducing a
    # new nesting level. Comments may not include parentheses, escaped or not.
    group_open . group_comment {
      self.emit(:group, :comment, data[ts..te-1].pack('c*'), ts, te)
    };

    # (?mix-mix...) expression options:
    #   (?imx-imx)          option on/off
    #                         i: ignore case
    #                         m: multi-line (dot(.) match newline)
    #                         x: extended form
    #
    #   (?imx-imx:subexp)   option on/off for subexp
    group_open . group_options > (grouped, 1) {
      self.emit(:group, :options, data[ts..te-1].pack('c*'), ts, te)
    };

    # Assertions
    #   (?=subexp)          look-ahead
    #   (?!subexp)          negative look-ahead
    #   (?<=subexp)         look-behind
    #   (?<!subexp)         negative look-behind
    # ------------------------------------------------------------------------
    group_open . assertion_type? > (grouped, 2) {
      case text =  data[ts..te-1].pack('c*')
      when '(?=';  self.emit(:assertion, :lookahead,    text, ts, te)
      when '(?!';  self.emit(:assertion, :nlookahead,   text, ts, te)
      when '(?<='; self.emit(:assertion, :lookbehind,   text, ts, te)
      when '(?<!'; self.emit(:assertion, :nlookbehind,  text, ts, te)
      end
    };

    # Groups
    #   (?:subexp)          passive (non-captured) group
    #   (?>subexp)          atomic group, don't backtrack in subexp.
    #   (?<name>subexp)     named group (single quotes are no supported, yet)
    #   (subexp)            captured group
    # ------------------------------------------------------------------------
    group_open . group_type? > (grouped, 3) {
      case text =  data[ts..te-1].pack('c*')
      when '(?:';  self.emit(:group, :passive,      text, ts, te)
      when '(?>';  self.emit(:group, :atomic,       text, ts, te)
      when '(?<[a-zA-Z]\w+>'
        self.emit(:group, :named, text, ts, te)
      else
        self.emit(:group, :capture, text, ts, te)
      end
    };

    group_close {
      self.emit(:group, :close, data[ts..te-1].pack('c*'), ts, te)
    };

    # Quantifiers
    # ------------------------------------------------------------------------
    zero_or_one {
      case text =  data[ts..te-1].pack('c*')
      when '?' ;  self.emit(:quantifier, :zero_or_one,            text, ts, te)
      when '??';  self.emit(:quantifier, :zero_or_one_reluctant,  text, ts, te)
      when '?+';  self.emit(:quantifier, :zero_or_one_possessive, text, ts, te)
      end
    };

    zero_or_more {
      case text =  data[ts..te-1].pack('c*')
      when '*' ;  self.emit(:quantifier, :zero_or_more,            text, ts, te)
      when '*?';  self.emit(:quantifier, :zero_or_more_reluctant,  text, ts, te)
      when '*+';  self.emit(:quantifier, :zero_or_more_possessive, text, ts, te)
      end
    };

    one_or_more {
      case text =  data[ts..te-1].pack('c*')
      when '+' ;  self.emit(:quantifier, :one_or_more,            text, ts, te)
      when '+?';  self.emit(:quantifier, :one_or_more_reluctant,  text, ts, te)
      when '++';  self.emit(:quantifier, :one_or_more_possessive, text, ts, te)
      end
    };

    # Intervals: min, max, and exact notations
    # ------------------------------------------------------------------------
    range_open . (digit+)? . ','? . (digit+)? . range_close . quantifier_mode? {
      self.emit(:quantifier, :interval, data[ts..te-1].pack('c*'), ts, te)
    };

    # Literal: anything, except meta characters. This includes 2, 3, and 4
    # unicode byte sequences.
    # ------------------------------------------------------------------------
    (any - meta_char)+ {
      self.emit(:literal, :literal, data[ts..te-1].pack('c*'), ts, te)
    };

  *|;
}%%

%%{
  machine re_scanner;

  wild                  = '.';
  backslash             = '\\';
  alternation           = '|';
  beginning_of_line     = '^';
  end_of_line           = '$';

  range_open            = '{';
  range_close           = '}';
  curlies               = range_open | range_close;

  group_open            = '(';
  group_close           = ')';
  parantheses           = group_open | group_close;

  set_open              = '[';
  set_close             = ']';
  brackets              = set_open | set_close;

  posix_class_name      = 'alnum' | 'alpha' | 'blank' |
                          'cntrl' | 'digit' | 'graph' |
                          'lower' | 'print' | 'punct' |
                          'space' | 'upper' | 'xdigit' |
                          'word'  | 'ascii';

  property_name         = 'Alnum' | 'Alpha' | 'Any'   | 'Ascii' | 'Blank' |
                          'Cntrl' | 'Digit' | 'Graph' | 'Lower' | 'Print' |
                          'Punct' | 'Space' | 'Upper' | 'Word' | 'Xdigit';

  category_letter       = 'L' . [ultmo]?;
  category_mark         = 'M' . [nce]?;
  category_number       = 'N' . [dlo]?;
  category_punctuation  = 'P' . [cdseifo]?;
  category_symbol       = 'S' . [mcko]?;
  category_separator    = 'Z' . [slp]?;
  category_codepoint    = 'C' . [cfson]?;
  general_category      = category_letter | category_mark |
                          category_number | category_punctuation |
                          category_symbol | category_separator |
                          category_codepoint;

  posix_class           = '[:' . posix_class_name . ':]';

  char_type             = [dDhHsSwW];

  line_anchor           = beginning_of_line | end_of_line;
  anchor_char           = [AbBzZG];
  property_char         = [pP];

  escaped_char          = [aefnrtv];
  octal_sequence        = '0' . [0-7]{2};

  hex_sequence          = 'x' . xdigit{2};
  wide_hex_sequence     = 'x' . '{7' . xdigit{1,7} . '}';
  control_sequence      = ('c' | 'C-') . xdigit{1,2};
  meta_sequence         = 'M-' . xdigit{1,2};
  meta_control_sequence = 'M-\\C-' . xdigit{1,2};

  zero_or_one           = '?' | '??' | '?+';
  zero_or_more          = '*' | '*?' | '*+';
  one_or_more           = '+' | '+?' | '++';

  quantifier_greedy     = '?'  | '*'  | '+';
  quantifier_reluctant  = '??' | '*?' | '+?';
  quantifier_possessive = '?+' | '*+' | '++';
  quantifier_mode       = '?'  | '+';

  quantifier_range      = range_open . (digit+)? . ','? . (digit+)? .
                          range_close . quantifier_mode?;

  quantifiers           = quantifier_greedy | quantifier_reluctant |
                          quantifier_possessive | quantifier_range;

  group_comment         = '?#' . [^)]+ . group_close;

  group_atomic          = '?>';
  group_passive         = '?:';

  assertion_lookahead   = '?=';
  assertion_nlookahead  = '?!';
  assertion_lookbehind  = '?<=';
  assertion_nlookbehind = '?<!';

  group_options         = '?' . ([mix]{1,3})? . '-' . ([mix]{1,3})? . ':';

  group_name            = alpha . alnum+;
  group_named           = '?<' . group_name . '>';

  group_type            = group_atomic | group_passive | group_named;

  assertion_type        = assertion_lookahead  | assertion_nlookahead |
                          assertion_lookbehind | assertion_nlookbehind;

  extended_group        = group_type | assertion_type;

  # characters the 'break' a literal
  meta_char             = wild | backslash | alternation |
                          curlies | parantheses | brackets |
                          line_anchor | quantifier_greedy;

  # character set scanner, continues consuming characters until it meets the
  # closing bracket of the set.
  # --------------------------------------------------------------------------
  character_set := |*
    ']' {
      self.emit(:character_set, :close, data[ts..te-1].pack('c*'), ts, te)
      fret;
    };

    '^' {
      self.emit(:character_set, :negate, data[ts..te-1].pack('c*'), ts, te)
    };

    alnum . '-' . alnum { # TODO: add properties
      self.emit(:character_set, :range, data[ts..te-1].pack('c*'), ts, te)
    };

    '&&' {
      self.emit(:character_set, :intersection, data[ts..te-1].pack('c*'), ts, te)
    };

    '\\' . [\]\-\,b] {
      self.emit(:character_set, :member, data[ts..te-1].pack('c*'), ts, te)
    };

    posix_class {
      case text = data[ts..te-1].pack('c*')
      when '[:alnum:]';  self.emit(:character_set, :set_class_alnum,  text, ts, te)
      when '[:alpha:]';  self.emit(:character_set, :set_class_alpha,  text, ts, te)
      when '[:blank:]';  self.emit(:character_set, :set_class_blank,  text, ts, te)
      when '[:cntrl:]';  self.emit(:character_set, :set_class_cntrl,  text, ts, te)
      when '[:digit:]';  self.emit(:character_set, :set_class_digit,  text, ts, te)
      when '[:graph:]';  self.emit(:character_set, :set_class_graph,  text, ts, te)
      when '[:lower:]';  self.emit(:character_set, :set_class_lower,  text, ts, te)
      when '[:print:]';  self.emit(:character_set, :set_class_print,  text, ts, te)
      when '[:punct:]';  self.emit(:character_set, :set_class_punct,  text, ts, te)
      when '[:space:]';  self.emit(:character_set, :set_class_space,  text, ts, te)
      when '[:upper:]';  self.emit(:character_set, :set_class_upper,  text, ts, te)
      when '[:xdigit:]'; self.emit(:character_set, :set_class_xdigit, text, ts, te)
      when '[:word:]';   self.emit(:character_set, :set_class_word,   text, ts, te)
      when '[:ascii:]';  self.emit(:character_set, :set_class_ascii,  text, ts, te)
      else raise "Unsupported character posixe class at #{text} (char #{ts})"
      end
    };

    any {
      self.emit(:character_set, :member, data[ts..te-1].pack('c*'), ts, te)
    };
  *|;

  # escape sequence scanner
  # --------------------------------------------------------------------------
  escape_sequence := |*
    hex_sequence {
      self.emit(:escape_sequence, :hex, data[ts..te-1].pack('c*'), ts, te)
      fret;
    };

    wide_hex_sequence {
      self.emit(:escape_sequence, :hex_wide, data[ts..te-1].pack('c*'), ts, te)
      fret;
    };

    control_sequence {
      self.emit(:escape_sequence, :control, data[ts..te-1].pack('c*'), ts, te)
      fret;
    };

    meta_sequence {
      self.emit(:escape_sequence, :meta, data[ts..te-1].pack('c*'), ts, te)
      fret;
    };

    meta_control_sequence {
      self.emit(:escape_sequence, :meta_control, data[ts..te-1].pack('c*'), ts, te)
      fret;
    };

    # TODO: extract into a separate machine... use in sets
    property_char . '{' . (property_name | general_category) . '}' > (escaped_alpha, 2) {
      text = data[ts..te-1].pack('c*')
      type = text[0,1] == 'p' ? :property: :inverted_property

      case name = data[ts+2..te-2].pack('c*')

      # Named
      when 'Alnum';   self.emit(type, :alnum,  text, ts, te)
      when 'Alpha';   self.emit(type, :alpha,  text, ts, te)
      when 'Any';     self.emit(type, :any,    text, ts, te)
      when 'Ascii';   self.emit(type, :ascii,  text, ts, te)
      when 'Blank';   self.emit(type, :blank,  text, ts, te)
      when 'Cntrl';   self.emit(type, :cntrl,  text, ts, te)
      when 'Digit';   self.emit(type, :digit,  text, ts, te)
      when 'Graph';   self.emit(type, :graph,  text, ts, te)
      when 'Lower';   self.emit(type, :lower,  text, ts, te)
      when 'Print';   self.emit(type, :print,  text, ts, te)
      when 'Punct';   self.emit(type, :punct,  text, ts, te)
      when 'Space';   self.emit(type, :space,  text, ts, te)
      when 'Upper';   self.emit(type, :upper,  text, ts, te)
      when 'Word';    self.emit(type, :word,   text, ts, te)
      when 'Xdigit';  self.emit(type, :xdigit, text, ts, te)

      # Letters
      when 'Lu'; self.emit(type, :letter_uppercase, text, ts, te)
      when 'Ll'; self.emit(type, :letter_lowercase, text, ts, te)
      when 'Lt'; self.emit(type, :letter_titlecase, text, ts, te)
      when 'Lm'; self.emit(type, :letter_modifier,  text, ts, te)
      when 'Lo'; self.emit(type, :letter_other,     text, ts, te)

      # Marks
      when 'Mn'; self.emit(type, :mark_nonspacing,  text, ts, te)
      when 'Mc'; self.emit(type, :mark_spacing,     text, ts, te)
      when 'Me'; self.emit(type, :mark_enclosing,   text, ts, te)

      # Numbers
      when 'Nd'; self.emit(type, :number_decimal, text, ts, te)
      when 'Nl'; self.emit(type, :number_letter, text, ts, te)
      when 'No'; self.emit(type, :number_other, text, ts, te)

      # Punctuation
      when 'Pc'; self.emit(type, :punct_connector, text, ts, te)
      when 'Pd'; self.emit(type, :punct_dash, text, ts, te)
      when 'Ps'; self.emit(type, :punct_open, text, ts, te)
      when 'Pe'; self.emit(type, :punct_close, text, ts, te)
      when 'Pi'; self.emit(type, :punct_initial, text, ts, te)
      when 'Pf'; self.emit(type, :punct_final, text, ts, te)
      when 'Po'; self.emit(type, :punct_other, text, ts, te)

      # Symbols
      when 'Sm'; self.emit(type, :symbol_math, text, ts, te)
      when 'Sc'; self.emit(type, :symbol_currency, text, ts, te)
      when 'Sk'; self.emit(type, :symbol_modifier, text, ts, te)
      when 'So'; self.emit(type, :symbol_other, text, ts, te)

      # Separators
      when 'Zs'; self.emit(type, :separator_space, text, ts, te)
      when 'Zl'; self.emit(type, :separator_line, text, ts, te)
      when 'Zp'; self.emit(type, :separator_paragraph, text, ts, te)

      # Codepoints
      when 'Cc'; self.emit(type, :code_control, text, ts, te)
      when 'Cf'; self.emit(type, :code_format, text, ts, te)
      when 'Cs'; self.emit(type, :code_surrogate, text, ts, te)
      when 'Co'; self.emit(type, :code_private, text, ts, te)
      when 'Cn'; self.emit(type, :code_unassigned, text, ts, te)
      end
    };

    any > (escaped_alpha, 1)  {
      self.emit(:escape_sequence, :literal, data[ts..te-1].pack('c*'), ts, te)
      fret;
    };
  *|;

  # Main scanner
  main := |*

    # Anchors
    alternation {
      self.emit(:meta, :alternation, data[ts..te-1].pack('c*'), ts, te)
    };

    # Character types
    #   .         any (except new line)
    #   \d, \D    digit, non-digit
    #   \h, \H    hex, non-hex
    #   \s, \S    whitespace, non-whitespace
    #   \w, \W    word, non-word
    # ------------------------------------------------------------------------
    wild {
      self.emit(:character_type, :any, data[ts..te-1].pack('c*'), ts, te)
    };

    backslash . char_type > (backslashed, 2) {
      case text = data[ts..te-1].pack('c*')
      when '\\d'; self.emit(:character_type, :digit,      text, ts, te)
      when '\\D'; self.emit(:character_type, :non_digit,  text, ts, te)
      when '\\h'; self.emit(:character_type, :hex,        text, ts, te)
      when '\\H'; self.emit(:character_type, :non_hex,    text, ts, te)
      when '\\s'; self.emit(:character_type, :space,      text, ts, te)
      when '\\S'; self.emit(:character_type, :non_space,  text, ts, te)
      when '\\w'; self.emit(:character_type, :word,       text, ts, te)
      when '\\W'; self.emit(:character_type, :non_word,   text, ts, te)
      else raise "Unsupported character type at #{text} (char #{ts})"
      end
    };

    # Anchors
    beginning_of_line {
      self.emit(:anchor, :beginning_of_line, data[ts..te-1].pack('c*'), ts, te)
    };

    end_of_line {
      self.emit(:anchor, :end_of_line, data[ts..te-1].pack('c*'), ts, te)
    };

    backslash . anchor_char > (backslashed, 3) {
      case text = data[ts..te-1].pack('c*')
      when '\\A'; self.emit(:anchor, :bos,                text, ts, te)
      when '\\z'; self.emit(:anchor, :eos,                text, ts, te)
      when '\\Z'; self.emit(:anchor, :eos_or_before_eol,  text, ts, te)
      when '\\b'; self.emit(:anchor, :word_boundary,      text, ts, te)
      when '\\B'; self.emit(:anchor, :non_word_boundary,  text, ts, te)
      else raise "Unsupported anchor at #{text} (char #{ts})"
      end
    };

    # Escaped sequences
    backslash > (backslashed, 1) {
      fcall escape_sequence;
    };

    # Character sets
    set_open {
      self.emit(:character_set, :open, data[ts..te-1].pack('c*'), ts, te)
      fcall character_set;
    };

    # (?#...) comments: parsed as a single expression, without introducing a
    # new nesting level. Comments may not include parentheses, escaped or not.
    group_open . group_comment {
      self.emit(:group, :comment, data[ts..te-1].pack('c*'), ts, te)
    };

    # (?mix-mix...) expression options:
    #   (?imx-imx)          option on/off
    #                         i: ignore case
    #                         m: multi-line (dot(.) match newline)
    #                         x: extended form
    #
    #   (?imx-imx:subexp)   option on/off for subexp
    group_open . group_options > (grouped, 2) {
      self.emit(:group, :options, data[ts..te-1].pack('c*'), ts, te)
    };

    # Extended Groups
    #   (subexp)            captured group
    #   (?:subexp)          passive (non-captured) group
    #   (?>subexp)          atomic group, don't backtrack in subexp.
    #   (?=subexp)          look-ahead
    #   (?!subexp)          negative look-ahead
    #   (?<=subexp)         look-behind
    #   (?<!subexp)         negative look-behind
    #   (?<name>subexp)     named group (single quotes are no supported, yet)
    # ------------------------------------------------------------------------
    group_open . extended_group? > (grouped, 1) {
      case text =  data[ts..te-1].pack('c*')
      when '(?:';  self.emit(:group, :passive,      text, ts, te)
      when '(?>';  self.emit(:group, :atomic,       text, ts, te)
      when '(?=';  self.emit(:group, :lookahead,    text, ts, te)
      when '(?!';  self.emit(:group, :nlookahead,   text, ts, te)
      when '(?<='; self.emit(:group, :lookbehind,   text, ts, te)
      when '(?<!'; self.emit(:group, :nlookbehind,  text, ts, te)
      when '(?<[a-zA-Z]\w+>'
        self.emit(:group, :named, text, ts, te)
      else
        self.emit(:group, :capture, text, ts, te)
      end
    };

    group_close {
      self.emit(:group, :close, data[ts..te-1].pack('c*'), ts, te)
    };

    # Quantifiers
    # ------------------------------------------------------------------------
    zero_or_one {
      case text =  data[ts..te-1].pack('c*')
      when '?' ;  self.emit(:quantifier, :zero_or_one,            text, ts, te)
      when '??';  self.emit(:quantifier, :zero_or_one_reluctant,  text, ts, te)
      when '?+';  self.emit(:quantifier, :zero_or_one_possessive, text, ts, te)
      end
    };

    zero_or_more {
      case text =  data[ts..te-1].pack('c*')
      when '*' ;  self.emit(:quantifier, :zero_or_more,            text, ts, te)
      when '*?';  self.emit(:quantifier, :zero_or_more_reluctant,  text, ts, te)
      when '*+';  self.emit(:quantifier, :zero_or_more_possessive, text, ts, te)
      end
    };

    one_or_more {
      case text =  data[ts..te-1].pack('c*')
      when '+' ;  self.emit(:quantifier, :one_or_more,            text, ts, te)
      when '+?';  self.emit(:quantifier, :one_or_more_reluctant,  text, ts, te)
      when '++';  self.emit(:quantifier, :one_or_more_possessive, text, ts, te)
      end
    };

    # Intervals: min, max, and exact notations
    # ------------------------------------------------------------------------
    range_open . (digit+)? . ','? . (digit+)? . range_close . quantifier_mode? {
      self.emit(:quantifier, :interval, data[ts..te-1].pack('c*'), ts, te)
    };

    # Literal: anything, except meta characters. This includes 2, 3, and 4
    # unicode byte sequences.
    # ------------------------------------------------------------------------
    (any - meta_char)+ {
      self.emit(:literal, :literal, data[ts..te-1].pack('c*'), ts, te)
    };

  *|;
}%%


Re-indent parser

Re-indent parser

#import "MustacheParser.h"
#import <stdio.h>
#import <assert.h>
#import <stdlib.h>
#import <ctype.h>
#import <string.h>

#define LEN(AT, FPC) (FPC - buffer - AT)
#define MARK(M,FPC) (M = (FPC) - buffer)
#define PTR_TO(F) (buffer + F)

/** Machine **/
%%{

	machine mustache_parser;

	# Character classes
	open  = '{' :> '{';
	close = '}' :> '}';
	identifier = ( alnum | [?!_] | '-')+;
	simple_type = [=!<>&];
	section_type = [#^/];

	action mark { MARK(mark, fpc); }

	action start_identifier {
		MARK(identifier_start, fpc);
		printf("Start of %c tag\n", tag_type);
	}

	action got_identifier {
		printf("Tag: ");
		fwrite(PTR_TO(identifier_start), sizeof(char), LEN(identifier_start, fpc), stdout);
		printf("\n");
		[self.delegate parser:self foundTag:PTR_TO(identifier_start) ofLength:LEN(identifier_start, fpc) withSigil:tag_type];
	}

	action write_static {
		// Write out all the static text up to this tag
		printf("Static text: ");
		fwrite(PTR_TO(mark), sizeof(char), LEN(mark, fpc), stdout);
		printf("\n");
		[self.delegate parser:self foundText:PTR_TO(mark) ofLength:LEN(mark, fpc)];
	}

	action set_type {
		tag_type = fc;
	}

	action init_type {
		tag_type = ' ';
	}

	var = (
		open
		simple_type? >init_type $set_type <: # Need to add error for unknown type
		space*
		identifier >start_identifier %got_identifier
		space*
		close %mark # Need to skip whitespace for section tags
	) >write_static;

	section = (
		open
		section_type $set_type# Need to add error for unknown type
		space*
		identifier >start_identifier %got_identifier
		space*
		close
		space* %mark # Skip trailing whitespace
	) >write_static;

	# Special case for triple mustache
	unescaped = (
		open
		'{' $set_type
		space*
		identifier >start_identifier %got_identifier
		space*
		'}'
		close %mark
	) >write_static;

	text = (any+ -- open);

	body = (
		var
		| unescaped
		| section
		| text
	);

	main := body* %eof(write_static);

}%%

/** Data **/
%% write data;

@implementation MustacheParser

@synthesize delegate;

//int http_parser_init(http_parser *parser)  {
- (id)initWithDelegate:(id <MustacheParserDelegate>)parserDelegate
{
	if((self = [super init]) != nil)
	{
		mark = 0; // Mark the start of the text
		%% write init;
		delegate = parserDelegate;
	}

	return self;
}

/** exec **/
//size_t http_parser_execute(http_parser *parser, const char *buffer, size_t len, size_t off)  {
- (size_t)execute:(const char *)buffer length:(size_t)len offset:(size_t)off
{
	const char *p, *pe, *eof;
	//int cs = parser->cs;
	char tag_type = ' ';

	assert(off <= len && "offset past end of buffer");

	p = buffer+off;
	pe = buffer+len;
	eof = pe;

	assert(*pe == '\0' && "pointer does not end on NUL");
	assert(pe - p == len - off && "pointers aren't same distance");

	%% write exec;

	//parser->cs = cs;
	nread += p - (buffer + off);
	//
	//  assert(p <= pe && "buffer overflow after parsing execute");
	assert(nread <= len && "nread longer than length");
	//  assert(body_start <= len && "body starts after buffer end");
	//  assert(mark < len && "mark is after buffer end");
	//  assert(field_len <= len && "field has length longer than whole buffer");
	//  assert(field_start < len && "field starts after buffer end");

	//  if(body_start) {
	//    /* final \r\n combo encountered so stop right here */
	//    //%%write eof;
	//    nread++;
	//  }

	//return nread;
	return 0;
}

- (int)finish {
	if ([self hasError]) {
		return -1;
	} else if ([self isFinished]) {
		return 1;
	} else {
		return 0;
	}
}

- (BOOL)hasError
{
	return cs == mustache_parser_error ? YES : NO;
}

- (BOOL)isFinished
{
	return cs == mustache_parser_first_final ? YES : NO;
}

- (size_t)bytesRead
{
	return nread;
}

@end


credit where credit is due

credit where credit is due

%%{
  machine irc_parser;

  NUL = '\0';
  BELL = '\a';
  CR = '\r';
  LF = '\n';
  SPACE = ' '+;

  crlf = CR LF;
  comma = ',';
  nonwhite = ascii -- (SPACE | NUL | CR | LF);
  special = ('_' | '-' | '[' | ']' | '\\' | '`' | '^' | '{' | '}' | '~');

  # unicode sequences taken from Wincent.com 's wikitext
  # http://git.wincent.com/wikitext.git/blob/HEAD:/ext/wikitext_ragel.rl#l459
  unicode = (0x01..0x1f | 0x7f)                         |
            (0xc2..0xdf 0x80..0xbf)                     |
            (0xe0..0xef 0x80..0xbf 0x80..0xbf)          |
            (0xf0..0xf4 0x80..0xbf 0x80..0xbf 0x80..0xbf);

  chstring = ascii -- (SPACE | BELL | NUL | CR | LF | comma);
  mask = ('#' | '$') chstring;
  nick = alpha (alpha | digit | special)*;
  host = (alpha | digit | '-' | '.')+;
  servername = host;
  channel = ('#' | '&') chstring;
  user = nonwhite+;
  to_ = channel | (user '@' servername) | nick | mask;
  target = to_ (',' to_)*;

  action mark { mark = p }
  action prefix { prefix = data[mark..(p-1)] }
  action command { command = data[mark..(p-1)] }
  action params { params << data[mark..(p-1)] }

  trailing = ( ( ascii | unicode ) -- (NUL | CR | LF) )* >mark %params;
  middle = ( (nonwhite - ':') nonwhite* ) >mark %params;
  params = SPACE? (SPACE middle)* (SPACE ':' trailing)?;
  command = (alpha+ | digit digit digit) >mark %command;
  prefix = (servername | nick ('!' user)? ('@' host)?) >mark %prefix;
  message = (':' prefix SPACE)? command params crlf;

  main := message;
}%%

module IRCParser
  class Parser
    class Error < RuntimeError
      attr_accessor :source, :prefix, :identifier, :params

      def initialize(from, source, prefix, identifier, params)
        @from, @source, @prefix, @identifier, @params = from, source, prefix, identifier, params
      end

      def to_s
        "While #{@from}: #{source.inspect}"
      end
      alias_method :message, :to_s
    end

    %% write data;

    def self.run(message)
      data = message.unpack("c*") if message.is_a?(String)

      prefix = nil
      command = nil
      params = []

      %% write init;
      %% write exec;

      if cs >= irc_parser_first_final
        prefix = prefix.pack("c*") if prefix
        command = command.pack("c*") if command
        params = params.map { |a| a.pack("c*") } if params
        return prefix, command, *params
      else
        raise IRCParser::Parser::Error.new("parsing", message, prefix, command, params)
      end
    end

  end
end

%%{
  machine irc_parser;

  NUL = '\0';
  BELL = '\a';
  CR = '\r';
  LF = '\n';
  SPACE = ' '+;

  crlf = CR LF;
  comma = ',';
  nonwhite = ascii -- (SPACE | NUL | CR | LF);
  special = ('_' | '-' | '[' | ']' | '\\' | '`' | '^' | '{' | '}' | '~');

  unicode = (0x01..0x1f | 0x7f)                         |
            (0xc2..0xdf 0x80..0xbf)                     |
            (0xe0..0xef 0x80..0xbf 0x80..0xbf)          |
            (0xf0..0xf4 0x80..0xbf 0x80..0xbf 0x80..0xbf);

  chstring = ascii -- (SPACE | BELL | NUL | CR | LF | comma);
  mask = ('#' | '$') chstring;
  nick = alpha (alpha | digit | special)*;
  host = (alpha | digit | '-' | '.')+;
  servername = host;
  channel = ('#' | '&') chstring;
  user = nonwhite+;
  to_ = channel | (user '@' servername) | nick | mask;
  target = to_ (',' to_)*;

  action mark { mark = p }
  action prefix { prefix = data[mark..(p-1)] }
  action command { command = data[mark..(p-1)] }
  action params { params << data[mark..(p-1)] }

  trailing = ( ( ascii | unicode ) -- (NUL | CR | LF) )* >mark %params;
  middle = ( (nonwhite - ':') nonwhite* ) >mark %params;
  params = SPACE? (SPACE middle)* (SPACE ':' trailing)?;
  command = (alpha+ | digit digit digit) >mark %command;
  prefix = (servername | nick ('!' user)? ('@' host)?) >mark %prefix;
  message = (':' prefix SPACE)? command params crlf;

  main := message;
}%%

module IRCParser
  class Parser
    class Error < RuntimeError
      attr_accessor :source, :prefix, :identifier, :params

      def initialize(from, source, prefix, identifier, params)
        @from, @source, @prefix, @identifier, @params = from, source, prefix, identifier, params
      end

      def to_s
        "While #{@from}: #{source.inspect}"
      end
      alias_method :message, :to_s
    end

    %% write data;

    def self.run(message)
      data = message.unpack("c*") if message.is_a?(String)

      prefix = nil
      command = nil
      params = []

      %% write init;
      %% write exec;

      if cs >= irc_parser_first_final
        prefix = prefix.pack("c*") if prefix
        command = command.pack("c*") if command
        params = params.map { |a| a.pack("c*") } if params
        return prefix, command, *params
      else
        raise IRCParser::Parser::Error.new("parsing", message, prefix, command, params)
      end
    end

  end
end


Fixed error porting MRI uu_decode.

Fixed error porting MRI uu_decode.


went on commenting mtoc

went on commenting mtoc

Conflicts:

	3rd_party/mtoc/confscanner.rl

#include "confscanner.h"

extern "C" {
#include <fnmatch.h>
}

using std::cerr;
using std::cout;
using std::endl;
using std::string;
using std::vector;
using std::map;
using std::set;
using std::ifstream;

%%{
  machine ConfFileScanner;
  write data;

  action st_tok { /*cerr << "st_tok" << line << "\n";*/ tmp_p = p; }

  action string_tok {
    tmp_string.assign(tmp_p, p-tmp_p);
  }

  action group_add
  {
//    cerr << "in group add: " << endl;
    if(match_at_level_[level_])
    {
//      cerr << "do it" << endl;
      tmp_string.assign(tmp_p, p-tmp_p);
      // clear groupset_ first, if we did not use '+='
      if(!arg_to_be_added_)
        groupset_.clear();

      groupset_.insert(tmp_string);
    }
  }

  action set_docu_block
  {
    tmp_string.assign(tmp_p, p-tmp_p);
    if(tmp_string == "brief")
      cblock_ = &docuheader_;
    else if(tmp_string == "doc")
      cblock_ = &docubody_;
    else if(tmp_string == "extra")
      cblock_ = &docuextra_;
  }

  action set_docu_list
  {
    tmp_string.assign(tmp_p, p-tmp_p);
    //cerr << "docu_list: " << tmp_string << "\n";
    if(tmp_string == "params")
    {
      //cerr << "paramlist\n";
      clist_ = &param_list_;
    }
    else if(tmp_string == "return")
      clist_ = &return_list_;
  }

  action set_docu_list_map
  {
    tmp_string.assign(tmp_p, p-tmp_p);
    if(tmp_string == "fields")
      clistmap_ = &field_docu_;
  }

  # add line of a block to the cblock structure which can be one of
  # docublock, docuheader, docuextra or clist[] or clistmap[][]
  action add_block_line
  {
    if(match_at_level_[level_])
    {
      tmp_string.assign(tmp_p, p-tmp_p);
      string :: size_type found = tmp_string.find_first_not_of(" \t");
      if(found == string::npos || string(tmp_string.substr(found)) != "\"\"")
      {
        found = tmp_string.rfind("\"");
        if(found != string::npos && string(tmp_string.substr(found-1)) == "\"\"")
        {
          //cerr << "a1: " << tmp_string.substr(0, found-1) + '\n';
          (*cblock_).push_back(tmp_string.substr(0, found-1) + '\n');
          //cerr << "after a1: " << param_list_["detailed_data"][0];
        }
        else
        {
          //cerr << "a2: " << tmp_string + '\n';
          (*cblock_).push_back(tmp_string + '\n');
        }
      }
      opt = true;
    }
  }

  # set the cblock pointer to clist_[value]
  action set_block_from_list
  {
    if(match_at_level_[level_])
    {
      tmp_string.assign(tmp_p, p-tmp_p);
      //cerr << "ts:" << tmp_string << ":ets\n";//<< " " << tmp_p << "p: " << p << "\n";
      cblock_ = &((*clist_)[tmp_string]);
    }
  }

  # set the cblock pointer to clist_[value1][value2]
  action set_block_from_listmap
  {
    string s(tmp_p, p-tmp_p);
    cblock_ = &((*clistmap_)[tmp_string][s]);
  }

  action reg_glob
  {
    tmp_string.assign(tmp_p, p-tmp_p);
    //cerr << "glob: " << tmp_string << endl;
    // TODO: use globlist_map
    globlist_stack_.back().push_back(tmp_string);
  }

  # end of file
  EOF = 0;

  # end of line (CRLF, LF)
  EOL = '\r'? . '\n';

  # any not end of file character
  default = ^0;

  # identifier
  IDENT = [A-Z]+;

  # matlab identifier
  MIDENT = [A-Za-z][A-Za-z0-9_]*;

  # glob expression
  GLOB = (default - [\n \t;\{])+;

  # swallow till the end of line
  garbling := (default - '\n')* . EOL @{line++;fret;};

  add_param_docu_block = ('brief'|'doc'|'extra');
  add_param_docu_list = ('params'|'return');
  add_param_docu_list_map = ('fields');

  # matlab white space or comment
  MWSOC = (
           [ \t]
           | EOL @{line++;}
           | (('#'. any) -'##') @{ /*cerr << "l: " << line << "\n";*/ fhold;fcall garbling; }
          );
#  MWSOC = ( [ \t\n]);

  # equal sign or += sign
  EQ = ('+'? @{arg_to_be_added_ = true;} . '=');

  # end of a rule sign
  ENDRULE = ';' @{arg_to_be_added_ = false;};

  # add line in a documentation block
  docu_block_line :=
    (
     # default action (just copy text to tmp_p)
     ( ( default - ["\n] )+ >{ if(opt) { tmp_p = p; } } )
     |
     # end line in documenation block (calls add_block_line)
     EOL @(add_block_line) @{line++;}
     |
     # end of documentation block (calls add_block_line)
     '"""' @(add_block_line) @{fret;}
     |
     # ignore quotes (less than 3 in a row)
     ('"'{1,2} . (default - ["\n])) @{ fhold; opt=false; }
    )**;

  # begin documentation block
  docu_block = ('"""'
                @{
                opt=true;
                if(!arg_to_be_added_)
                  (*cblock_).clear();
                fcall docu_block_line;
                }
               );

  # documentation list item (something like key => value)
  docu_list_item =
    (
     ( MIDENT
         >{tmp_p = p;}
         %(set_block_from_list)
     ) . MWSOC*
     . '=>' . MWSOC* . docu_block
    );

  docu_list = (
    (
     '('
     . ( MWSOC* . docu_list_item . MWSOC* . ',')*
     . MWSOC* . docu_list_item . MWSOC* . ')'
    )
    |
    docu_list_item
  );

  docu_list_map_item =
    (
     (MIDENT
       >{tmp_p = p;/* cerr << "dlmi\n";*/} %(string_tok)
     )
     . '.'
     .(MIDENT
       >{tmp_p = p;/* cerr << "dlmi2\n";*/} %(set_block_from_listmap)
      )
     . MWSOC*
     . '=>' . MWSOC* . docu_block
    );

  docu_list_map = (
    (
     '('
     . ( MWSOC* . docu_list_map_item . MWSOC* . ',')*
     . MWSOC* . docu_list_map_item . MWSOC* . ')'
    )
    |
    docu_list_map_item
  );

  globlist =
    (GLOB >{/*cerr << "glob_list";*/ tmp_p = p;} %(reg_glob))
    . (MWSOC* . [ \t]+ . MWSOC*
        . (GLOB >{/*cerr << "glob list2";*/ tmp_p = p;} %(reg_glob)))*;

  grouplist = (
    ((MIDENT >(st_tok)
             %(group_add))
     . ',')*
    .(MIDENT >(st_tok)
      %(group_add))
    );

  rules := (
    # glob list
    ('glob' .MWSOC* . '=' . MWSOC* . globlist . MWSOC* . '{')
      @{ check_glob_level_up(); fcall rules; }
    |
    (MWSOC)
    |
    # groups rule
    ('groups'. MWSOC* . EQ
     . MWSOC* . grouplist . MWSOC* . ENDRULE)
    |
    # add rule
    ('add(' %{/*cerr << "add:" << '\n';*/ tmp_p = p;}
     . (
        # add documentation block (brief, docu, extra)
        ((add_param_docu_block
            %(set_docu_block)
         )
         . ')' . MWSOC* . EQ
         . MWSOC* . docu_block . MWSOC* . ENDRULE )
        |
        # add documentation list (documentation block for a parameter or a
        # parameter or a return value)
        ((add_param_docu_list
            %(set_docu_list)
         )
         . ')' . MWSOC* . EQ
         . MWSOC* :> docu_list . MWSOC* . ENDRULE )
        |
        # add a map of documentation lists (documentation blocks for structure
        # fields of parameters or return values)
        ((add_param_docu_list_map
            %(set_docu_list_map)
         )
         . ')' . MWSOC* . EQ
         . MWSOC* :> docu_list_map . MWSOC* ENDRULE )
       )
    )
    |
    # go level down (end of a glob match block)
    ('}') @{ go_level_down(); fret; }
    )*;

  # main parser (first entry point) This block comprises variable definitions.
  main := (
           # ignored characters
           MWSOC
           | (
              # variable definitions (not yet implemented)
              (IDENT
                 >(st_tok)
                 %(string_tok)
              ). MWSOC* . ':=' . MWSOC*
              . (GLOB >st_tok @{/*TODO*/})
              . (MWSOC* . (GLOB >st_tok @{/*TODO*/}))*
              . MWSOC*
              . ';'
             )
           |
           # end variable definitions
           '##' . [ \t]* . EOL @{ line++; /*cerr<<"-> rules\n";*/ fgoto rules; }
          )**;
}%%

void ConfFileScanner :: go_level_down()
{
  globlist_stack_.pop_back();
  globlist_stack_.back().clear();
  level_--;
}

bool ConfFileScanner :: check_for_match(int l, const char * str,
                                        bool match_path_sep)
{
  typedef GlobList :: iterator iterator;
  GlobList & gl = globlist_stack_[l];
  /*  cerr << "globlist at level" << l << "\n  ";
  for(unsigned int i = 0; i < gl.size(); ++i)
  {
    cerr << gl[i] << " ";
  }
  cerr << endl;*/
  iterator endit = gl.end();
  int flags = (match_path_sep? FNM_PATHNAME : 0);
  // cerr << "checking for match at level " << l <<" of string " << str << ":" << endl;
  for( iterator it = gl.begin(); it != endit; it++ )
  {
    // cerr << "with pattern: " << (*it).c_str() << endl;
    if(fnmatch((*it).c_str(), str, flags) == 0)
    {
      // cerr << "SUCCESS!" << endl;
      return true;
    }
  }
  return false;
}

bool ConfFileScanner :: check_glob_rec(int l, const string & s)
{
  string str;
  string :: size_type found;
  if(l == level_+1)
    return true;

  found = s.find("/"); // try to match dir in path
  while(found != string :: npos)
  {
    str = s.substr(0, found);
//    cerr << str << endl;
    if(check_for_match(l, str.c_str())
        && check_glob_rec(l+1, s.substr(found+1)))
      return true;

    found = s.find("/", found+1); // try to match more dirs
  }
  if(l == level_) // try also to match the entire string
  {
    if(check_for_match(l, s.c_str()))
      return true;
  }

  return false;
}

void ConfFileScanner :: check_glob_level_up()
{
  match_at_level_[level_+1] = check_glob_rec(0, filename_);
/*  cerr << "going one level up, match at this level is: "
       << (match_at_level_[level_+1] ? "true" : "false");
  cerr << endl;*/

  globlist_stack_.push_back(GlobList());
  level_++;
}

ConfFileScanner
:: ConfFileScanner(const std::string & filename)
 : line(1), have(0), top(0), opt(true),
   filename_(filename),
   confistream_(set_conffile()),
   level_(0),
   arg_to_be_added_(false)
{
  globlist_stack_.push_back(GlobList());
  match_at_level_[0] = true;
}

const char * ConfFileScanner ::
set_conffile()
{
  conffile_ = "doxygen/mtoc.conf";
  return conffile_.c_str();
}

int ConfFileScanner :: execute()
{
  //cerr << "execute\n";
  std::ios::sync_with_stdio(false);

  %% write init;

  /* Do the first read. */
  bool done = false;
  while ( !done )
  {
    char *p = buf + have;
    char *tmp_p = p;// *tmp_p2 = p;
    string tmp_string;
    int space = BUFSIZE - have;

    if ( space == 0 )
    {
      /* We filled up the buffer trying to scan a token. */
      cerr << "OUT OF BUFFER SPACE" << endl;
      exit(1);
    }

    confistream_.read( p, space );
    int len = confistream_.gcount();
    char *pe = p + len;
    char *rpe = pe;
    char *eof = 0;

    /* If we see eof then append the EOF char. */
    if ( confistream_.eof() )
    {
      eof = pe;
      done = true;
    }
    else
    {
      /* Find the last semicolon by searching backwards. This
       * is where we will stop processing on this iteration. */
      while ( *pe!= ';' && pe > p )
        pe--;
    }

    %% write exec;

    /* Check if we failed. */
    if ( cs == ConfFileScanner_error )
    {
      /* Machine failed before finding a token. */
      cerr << "in conffile: PARSE ERROR in line " << line << endl;
      exit(1);
    }

    have = rpe - pe;
    /* cerr << "memmove by " << have << "bytes\n";*/
    memmove( buf, pe, have );
  }

  return 0;
}
/* vim: set et sw=2 ft=ragel: */

#include "confscanner.h"

extern "C" {
#include <fnmatch.h>
}

using std::cerr;
using std::cout;
using std::endl;
using std::string;
using std::vector;
using std::map;
using std::set;
using std::ifstream;

%%{
  machine ConfFileScanner;
  write data;

  action st_tok { /*cerr << "st_tok" << line << "\n";*/ tmp_p = p; }

  action string_tok {
    tmp_string.assign(tmp_p, p-tmp_p);
  }

  action group_add
  {
//    cerr << "in group add: " << endl;
    if(match_at_level_[level_])
    {
//      cerr << "do it" << endl;
      tmp_string.assign(tmp_p, p-tmp_p);
      // clear groupset_ first, if we did not use '+='
      if(!arg_to_be_added_)
        groupset_.clear();

      groupset_.insert(tmp_string);
    }
  }

  action set_docu_block
  {
    tmp_string.assign(tmp_p, p-tmp_p);
    if(tmp_string == "brief")
      cblock_ = &docuheader_;
    else if(tmp_string == "doc")
      cblock_ = &docubody_;
    else if(tmp_string == "extra")
      cblock_ = &docuextra_;
  }

  action set_docu_list
  {
    tmp_string.assign(tmp_p, p-tmp_p);
    //cerr << "docu_list: " << tmp_string << "\n";
    if(tmp_string == "params")
    {
      //cerr << "paramlist\n";
      clist_ = &param_list_;
    }
    else if(tmp_string == "return")
      clist_ = &return_list_;
  }

  action set_docu_list_map
  {
    tmp_string.assign(tmp_p, p-tmp_p);
    if(tmp_string == "fields")
      clistmap_ = &field_docu_;
  }

  # add line of a block to the cblock structure which can be one of
  # docublock, docuheader, docuextra or clist[] or clistmap[][]
  action add_block_line
  {
    if(match_at_level_[level_])
    {
      tmp_string.assign(tmp_p, p-tmp_p);
      string :: size_type found = tmp_string.find_first_not_of(" \t");
      if(found == string::npos || string(tmp_string.substr(found)) != "\"\"")
      {
        found = tmp_string.rfind("\"");
        if(found != string::npos && string(tmp_string.substr(found-1)) == "\"\"")
        {
          //cerr << "a1: " << tmp_string.substr(0, found-1) + '\n';
          (*cblock_).push_back(tmp_string.substr(0, found-1) + '\n');
          //cerr << "after a1: " << param_list_["detailed_data"][0];
        }
        else
        {
          //cerr << "a2: " << tmp_string + '\n';
          (*cblock_).push_back(tmp_string + '\n');
        }
      }
      opt = true;
    }
  }

  # set the cblock pointer to clist_[value]
  action set_block_from_list
  {
    if(match_at_level_[level_])
    {
      tmp_string.assign(tmp_p, p-tmp_p);
      //cerr << "ts:" << tmp_string << ":ets\n";//<< " " << tmp_p << "p: " << p << "\n";
      cblock_ = &((*clist_)[tmp_string]);
    }
  }

  # set the cblock pointer to clist_[value1][value2]
  action set_block_from_listmap
  {
    string s(tmp_p, p-tmp_p);
    cblock_ = &((*clistmap_)[tmp_string][s]);
  }

  action reg_glob
  {
    tmp_string.assign(tmp_p, p-tmp_p);
    //cerr << "glob: " << tmp_string << endl;
    // TODO: use globlist_map
    globlist_stack_.back().push_back(tmp_string);
  }

  # end of file
  EOF = 0;

  # end of line (CRLF, LF)
  EOL = '\r'? . '\n';

  # any not end of file character
  default = ^0;

  # identifier
  IDENT = [A-Z]+;

  # matlab identifier
  MIDENT = [A-Za-z][A-Za-z0-9_]*;

  # glob expression
  GLOB = (default - [\n \t;\{])+;

  # swallow till the end of line
  garbling := (default - '\n')* . EOL @{line++;fret;};

  add_param_docu_block = ('brief'|'doc'|'extra');
  add_param_docu_list = ('params'|'return');
  add_param_docu_list_map = ('fields');

  # matlab white space or comment
  MWSOC = (
           [ \t]
           | EOL @{line++;}
           | (('#'. any) -'##') @{ /*cerr << "l: " << line << "\n";*/ fhold;fcall garbling; }
          );
#  MWSOC = ( [ \t\n]);

  # equal sign or += sign
  EQ = ('+'? @{arg_to_be_added_ = true;} . '=');

  # end of a rule sign
  ENDRULE = ';' @{arg_to_be_added_ = false;};

  # line in documentation block
  docu_block_line :=
    (
     ( ( default - ["\n] )+ >{ if(opt) { tmp_p = p; } } )
     |
     EOL @(add_block_line) @{line++;}
     |
     '"""' @(add_block_line) @{fret;}
     |
     ('"'{1,2} . (default - ["\n])) @{ fhold; opt=false; }
    )**;

  # begin documentation block
  docu_block = ('"""'
                @{
                opt=true;
                if(!arg_to_be_added_)
                  (*cblock_).clear();
                fcall docu_block_line;
                }
               );

  # documentation list item (something like key => value)
  docu_list_item =
    (
     ( MIDENT
         >{tmp_p = p;}
         %(set_block_from_list)
     ) . MWSOC*
     . '=>' . MWSOC* . docu_block
    );

  docu_list = (
    (
     '('
     . ( MWSOC* . docu_list_item . MWSOC* . ',')*
     . MWSOC* . docu_list_item . MWSOC* . ')'
    )
    |
    docu_list_item
  );

  docu_list_map_item =
    (
     (MIDENT
       >{tmp_p = p;/* cerr << "dlmi\n";*/} %(string_tok)
     )
     . '.'
     .(MIDENT
       >{tmp_p = p;/* cerr << "dlmi2\n";*/} %(set_block_from_listmap)
      )
     . MWSOC*
     . '=>' . MWSOC* . docu_block
    );

  docu_list_map = (
    (
     '('
     . ( MWSOC* . docu_list_map_item . MWSOC* . ',')*
     . MWSOC* . docu_list_map_item . MWSOC* . ')'
    )
    |
    docu_list_map_item
  );

  globlist =
    (GLOB >{/*cerr << "glob_list";*/ tmp_p = p;} %(reg_glob))
    . (MWSOC* . [ \t]+ . MWSOC*
        . (GLOB >{/*cerr << "glob list2";*/ tmp_p = p;} %(reg_glob)))*;

  grouplist = (
    ((MIDENT >(st_tok)
             %(group_add))
     . ',')*
    .(MIDENT >(st_tok)
      %(group_add))
    );

  rules := (
    # glob list
    ('glob' .MWSOC* . '=' . MWSOC* . globlist . MWSOC* . '{')
      @{ check_glob_level_up(); fcall rules; }
    |
    (MWSOC)
    |
    # groups rule
    ('groups'. MWSOC* . EQ
     . MWSOC* . grouplist . MWSOC* . ENDRULE)
    |
    # add rule
    ('add(' %{/*cerr << "add:" << '\n';*/ tmp_p = p;}
     . (
        ((add_param_docu_block
            %(set_docu_block)
         )
         . ')' . MWSOC* . EQ
         . MWSOC* . docu_block . MWSOC* . ENDRULE )
        |
        ((add_param_docu_list
            %(set_docu_list)
         )
         . ')' . MWSOC* . EQ
         . MWSOC* :> docu_list . MWSOC* . ENDRULE )
        |
        ((add_param_docu_list_map
            %(set_docu_list_map)
         )
         . ')' . MWSOC* . EQ
         . MWSOC* :> docu_list_map . MWSOC* ENDRULE )
       )
    )
    |
    # go level down
    ('}') @{ go_level_down(); fret; }
    )*;

  main := (
           MWSOC
           | (
              (IDENT
                 >(st_tok)
                 %(string_tok)
              ). MWSOC* . ':=' . MWSOC*
              . (GLOB >st_tok @{/*TODO*/})
              . (MWSOC* . (GLOB >st_tok @{/*TODO*/}))*
              . MWSOC*
              . ';'
             )
           |
           '##' . [ \t]* . EOL @{ line++; /*cerr<<"-> rules\n";*/ fgoto rules; }
          )**;
}%%

void ConfFileScanner :: go_level_down()
{
  globlist_stack_.pop_back();
  globlist_stack_.back().clear();
  level_--;
}

bool ConfFileScanner :: check_for_match(int l, const char * str,
                                        bool match_path_sep)
{
  typedef GlobList :: iterator iterator;
  GlobList & gl = globlist_stack_[l];
  /*  cerr << "globlist at level" << l << "\n  ";
  for(unsigned int i = 0; i < gl.size(); ++i)
  {
    cerr << gl[i] << " ";
  }
  cerr << endl;*/
  iterator endit = gl.end();
  int flags = (match_path_sep? FNM_PATHNAME : 0);
  // cerr << "checking for match at level " << l <<" of string " << str << ":" << endl;
  for( iterator it = gl.begin(); it != endit; it++ )
  {
    // cerr << "with pattern: " << (*it).c_str() << endl;
    if(fnmatch((*it).c_str(), str, flags) == 0)
    {
      // cerr << "SUCCESS!" << endl;
      return true;
    }
  }
  return false;
}

bool ConfFileScanner :: check_glob_rec(int l, const string & s)
{
  string str;
  string :: size_type found;
  if(l == level_+1)
    return true;

  found = s.find("/"); // try to match dir in path
  while(found != string :: npos)
  {
    str = s.substr(0, found);
//    cerr << str << endl;
    if(check_for_match(l, str.c_str())
        && check_glob_rec(l+1, s.substr(found+1)))
      return true;

    found = s.find("/", found+1); // try to match more dirs
  }
  if(l == level_) // try also to match the entire string
  {
    if(check_for_match(l, s.c_str()))
      return true;
  }

  return false;
}

void ConfFileScanner :: check_glob_level_up()
{
  match_at_level_[level_+1] = check_glob_rec(0, filename_);
/*  cerr << "going one level up, match at this level is: "
       << (match_at_level_[level_+1] ? "true" : "false");
  cerr << endl;*/

  globlist_stack_.push_back(GlobList());
  level_++;
}

ConfFileScanner
:: ConfFileScanner(const std::string & filename)
 : line(1), have(0), top(0), opt(true),
   filename_(filename),
   confistream_(set_conffile()),
   level_(0),
   arg_to_be_added_(false)
{
  globlist_stack_.push_back(GlobList());
  match_at_level_[0] = true;
}

const char * ConfFileScanner ::
set_conffile()
{
  conffile_ = "doxygen/mtoc.conf";
  return conffile_.c_str();
}

int ConfFileScanner :: execute()
{
  //cerr << "execute\n";
  std::ios::sync_with_stdio(false);

  %% write init;

  /* Do the first read. */
  bool done = false;
  while ( !done )
  {
    char *p = buf + have;
    char *tmp_p = p;// *tmp_p2 = p;
    string tmp_string;
    int space = BUFSIZE - have;

    if ( space == 0 )
    {
      /* We filled up the buffer trying to scan a token. */
      cerr << "OUT OF BUFFER SPACE" << endl;
      exit(1);
    }

    confistream_.read( p, space );
    int len = confistream_.gcount();
    char *pe = p + len;
    char *rpe = pe;
    char *eof = 0;

    /* If we see eof then append the EOF char. */
    if ( confistream_.eof() )
    {
      eof = pe;
      done = true;
    }
    else
    {
      /* Find the last semicolon by searching backwards. This
       * is where we will stop processing on this iteration. */
      while ( *pe!= ';' && pe > p )
        pe--;
    }

    %% write exec;

    /* Check if we failed. */
    if ( cs == ConfFileScanner_error )
    {
      /* Machine failed before finding a token. */
      cerr << "in conffile: PARSE ERROR in line " << line << endl;
      exit(1);
    }

    have = rpe - pe;
    /* cerr << "memmove by " << have << "bytes\n";*/
    memmove( buf, pe, have );
  }

  return 0;
}
/* vim: set et sw=2 ft=ragel: */


Remove non-HTML `pubdate` attribute from `<time>`

extends base

macro post-content(post, type, hasTrail)
	if type === 'text'
		if hasTrail
			yield
		else
			!"#{rewriteHTML(post.body)}"
	elif type === 'answer'
		blockquote class: "post-question"
			header
				h4
					strong class: "asker"
						if post.asking_url
							a "#{post.asking_name}" href: "#{rewriteLinkString(post.asking_url)}"
						else
							"#{post.asking_name}"

					" asked:"

			p "#{post.question}"

		!"#{rewriteHTML(post.answer)}"
	elif type === 'photo'
		for photo of post.photos
			% const displaySize = photo.original_size;
			% const mediaLink = rewriteLink(url.parse(displaySize.url, false, true));

			if mediaLink.embeddable
				figure class: "post-photo"
					img
						src: "#{url.format(mediaLink)}"
						alt: "#{photo.caption}"
						width: "#{displaySize.width}"
						height: "#{displaySize.height}"

					if photo.caption
						figcaption
							"#{photo.caption}"
			else
				a
					href: "#{url.format(mediaLink)}"
					rel: "noreferrer"
					"#{photo.caption || 'View image'}"

		if hasTrail
			yield
		else
			div class: "post-caption"
				!"#{rewriteHTML(post.caption)}"
	elif type === 'video'
		% const thumbnailLink = post.thumbnail_url && rewriteLink(url.parse(post.thumbnail_url, false, true));

		% if (post.video_url)
			figure class: "post-video"
				video
					src: "#{post.video_url}"
					type: "video/mp4"
					preload: "none"
					controls:
					poster: "#{post.thumbnail_url}"
		% else if (post.permalink_url)
			a
				href: "#{post.permalink_url}"
				rel: "noreferrer"
				"View video"
		% else
			p class: "error"
				"Video unavailable."

		if hasTrail
			yield
		else
			div class: "post-caption"
				!"#{rewriteHTML(post.caption)}"
	elif type === 'audio'
		figure
			audio
				class: "post-audio"
				src: "#{rewriteLinkString(post.audio_url)}"
				type: "audio/mp3"
				controls:

			if !hasTrail
				figcaption class: "post-caption"
					!"#{rewriteHTML(post.caption)}"

		if hasTrail
			yield
	elif type === 'link'
		div class: "post-caption"
			!"#{rewriteHTML(post.description)}"
	elif type === 'quote'
		blockquote class: "post-quote"
			!"#{rewriteHTML(post.text)}"

			footer
				cite !"— #{rewriteHTML(post.source)}"
	elif type === 'chat'
		dl class: "post-dialogue"
			for retort of post.dialogue
				dt "#{retort.label}"
				dd "#{retort.phrase}"
	else
		% throw new Error('Unexpected type: ' + type);

append html
	do
		const rewriteHTML = html => clean.rewriteHTML(html, data.domain);
		const rewriteLink = l => clean.rewriteLink(l, data.domain);
		const rewriteLinkString = l => clean.rewriteLinkString(l, data.domain);

append head
	title
		if data.tag
			"##{data.tag} · "
		"#{data.blog.title || data.name} · splash"

	style
		!"#{stylesheet}"

append body
	if data.blog.is_nsfw
		div id: "nsfw-indicator"
			span "NSFW"

	header id: "header"
		a href: "/blog/#{data.name}" rel: "index"
			img id: "avatar"
				src: "https://api.tumblr.com/v2/blog/#{data.name}/avatar/96"
				alt: "#{data.name}’s avatar"
				width: "96"
				height: "96"

		h1 id: "title"
			a href: "/blog/#{data.name}/" rel: "index"
				"#{data.blog.title || data.name}"

		h2 id: "description"
			!"#{rewriteHTML(data.blog.description)}"

	main id: "posts"
		if data.tag
			div id: "tag-indicator"
				"Browsing posts tagged "
				svg width: "24" height: "16"
					g fill: "white"
						rect x: "8" y: "0" width: "16" height: "16" rx: "2"
						rect x: "0" y: "0" width: "11.314" height: "11.314" rx: "2" transform: "translate(9 0) rotate(45)"
						circle cx: "8" cy: "8" r: "3" fill: "#2c4762"
				" "
				strong "#{data.tag}"

		for post of data.posts
			% const date = new Date(post.timestamp * 1000);

			article class: "post"
				if post.title
					header h2 class: "post-title"
						if post.url
							a href: "#{rewriteLinkString(post.url)}" "#{post.title}"
						else
							"#{post.title}"
				elif post.url
					header h2 class: "post-title"
						a href: "#{rewriteLinkString(post.url)}" "#{post.url}"

				% let {trail = []} = post;
				% const root = post.reblogged_root_id;

				do
					if (root !== undefined && trail.length !== 0 && root !== trail[0].post.id) {
						trail = [{
							blog: {
								name: post.reblogged_root_name,
							},
							post: {id: root},
							content: null,
						}].concat(trail);
					}

				%
					post-content(post, post.type, trail.length > 1)
						for reblog of trail
							div class: "trail"
								h3 class: "trail-blog"
									a href: "#{clean.blogPath(reblog.blog.name, `/post/${reblog.post.id}/`)}"
										"#{reblog.blog.name}"

								if reblog.content !== null
									div class: "trail-content"
										!"#{rewriteHTML(reblog.content)}"

				footer class: "post-info"
					div
						a href: "#{rewriteLinkString(post.post_url)}" rel: "bookmark"
							time class: "post-date" datetime: "#{date.toISOString()}" title: "#{date.toString()}"
								"#{relativeDate(date)}"

						" " span class: "post-alternates"
							"("
							a href: "#{post.post_url}"
								"original"
							", "
							a href: "data:text/plain;charset=utf-8,#{encodeURIComponent(inspect(post, { depth: null }))}"
								"data"
							")"

					ul class: "post-tags"
						for tag of post.tags
							li a rel: "tag" href: "/blog/#{data.name}/tagged/#{encodeURIComponent(tag)}" "##{tag}"

					div class: "post-stats"
						if post.note_count
							span class: "note-count" "#{pluralize(post.note_count, '%d note', '%d notes')}"

					if post.notes
						ul class: "post-notes"
							for note of post.notes
								li
									if note.post_id
										% const reblogBlog = url.parse(note.blog_url).hostname;
										a href: "#{clean.blogPath(reblogBlog, `/post/${note.post_id}/`)}" "#{note.type}"
									else
										"#{note.type}"
									" from "
									a href: "#{rewriteLinkString(note.blog_url)}" "#{note.blog_name}"

	% const {offset, limit} = data;
	% const prev = offset > limit ? offset - limit : offset > 0 ? '' : null;
	% const next = offset + data.posts.length < data.total_posts ? offset + data.posts.length : null;

	footer id: "footer"
		nav id: "page-navigation"
			if prev !== null
				a
					class: "prev"
					rel: "prev"
					href: "#{url.format(update(data.pageUri, { search: null, query: { offset: prev }, hash: '#posts' }))}"
					accesskey: "p" "← Previous page"
				" "
			if next !== null
				a
					class: "next"
					rel: "next"
					href: "#{url.format(update(data.pageUri, { search: null, query: { offset: next }, hash: '#posts' }))}"
					accesskey: "n" "Next page →"

extends base

macro post-content(post, type, hasTrail)
	if type === 'text'
		if hasTrail
			yield
		else
			!"#{rewriteHTML(post.body)}"
	elif type === 'answer'
		blockquote class: "post-question"
			header
				h4
					strong class: "asker"
						if post.asking_url
							a "#{post.asking_name}" href: "#{rewriteLinkString(post.asking_url)}"
						else
							"#{post.asking_name}"

					" asked:"

			p "#{post.question}"

		!"#{rewriteHTML(post.answer)}"
	elif type === 'photo'
		for photo of post.photos
			% const displaySize = photo.original_size;
			% const mediaLink = rewriteLink(url.parse(displaySize.url, false, true));

			if mediaLink.embeddable
				figure class: "post-photo"
					img
						src: "#{url.format(mediaLink)}"
						alt: "#{photo.caption}"
						width: "#{displaySize.width}"
						height: "#{displaySize.height}"

					if photo.caption
						figcaption
							"#{photo.caption}"
			else
				a
					href: "#{url.format(mediaLink)}"
					rel: "noreferrer"
					"#{photo.caption || 'View image'}"

		if hasTrail
			yield
		else
			div class: "post-caption"
				!"#{rewriteHTML(post.caption)}"
	elif type === 'video'
		% const thumbnailLink = post.thumbnail_url && rewriteLink(url.parse(post.thumbnail_url, false, true));

		% if (post.video_url)
			figure class: "post-video"
				video
					src: "#{post.video_url}"
					type: "video/mp4"
					preload: "none"
					controls:
					poster: "#{post.thumbnail_url}"
		% else if (post.permalink_url)
			a
				href: "#{post.permalink_url}"
				rel: "noreferrer"
				"View video"
		% else
			p class: "error"
				"Video unavailable."

		if hasTrail
			yield
		else
			div class: "post-caption"
				!"#{rewriteHTML(post.caption)}"
	elif type === 'audio'
		figure
			audio
				class: "post-audio"
				src: "#{rewriteLinkString(post.audio_url)}"
				type: "audio/mp3"
				controls:

			if !hasTrail
				figcaption class: "post-caption"
					!"#{rewriteHTML(post.caption)}"

		if hasTrail
			yield
	elif type === 'link'
		div class: "post-caption"
			!"#{rewriteHTML(post.description)}"
	elif type === 'quote'
		blockquote class: "post-quote"
			!"#{rewriteHTML(post.text)}"

			footer
				cite !"— #{rewriteHTML(post.source)}"
	elif type === 'chat'
		dl class: "post-dialogue"
			for retort of post.dialogue
				dt "#{retort.label}"
				dd "#{retort.phrase}"
	else
		% throw new Error('Unexpected type: ' + type);

append html
	do
		const rewriteHTML = html => clean.rewriteHTML(html, data.domain);
		const rewriteLink = l => clean.rewriteLink(l, data.domain);
		const rewriteLinkString = l => clean.rewriteLinkString(l, data.domain);

append head
	title
		if data.tag
			"##{data.tag} · "
		"#{data.blog.title || data.name} · splash"

	style
		!"#{stylesheet}"

append body
	if data.blog.is_nsfw
		div id: "nsfw-indicator"
			span "NSFW"

	header id: "header"
		a href: "/blog/#{data.name}" rel: "index"
			img id: "avatar"
				src: "https://api.tumblr.com/v2/blog/#{data.name}/avatar/96"
				alt: "#{data.name}’s avatar"
				width: "96"
				height: "96"

		h1 id: "title"
			a href: "/blog/#{data.name}/" rel: "index"
				"#{data.blog.title || data.name}"

		h2 id: "description"
			!"#{rewriteHTML(data.blog.description)}"

	main id: "posts"
		if data.tag
			div id: "tag-indicator"
				"Browsing posts tagged "
				svg width: "24" height: "16"
					g fill: "white"
						rect x: "8" y: "0" width: "16" height: "16" rx: "2"
						rect x: "0" y: "0" width: "11.314" height: "11.314" rx: "2" transform: "translate(9 0) rotate(45)"
						circle cx: "8" cy: "8" r: "3" fill: "#2c4762"
				" "
				strong "#{data.tag}"

		for post of data.posts
			% const date = new Date(post.timestamp * 1000);

			article class: "post"
				if post.title
					header h2 class: "post-title"
						if post.url
							a href: "#{rewriteLinkString(post.url)}" "#{post.title}"
						else
							"#{post.title}"
				elif post.url
					header h2 class: "post-title"
						a href: "#{rewriteLinkString(post.url)}" "#{post.url}"

				% let {trail = []} = post;
				% const root = post.reblogged_root_id;

				do
					if (root !== undefined && trail.length !== 0 && root !== trail[0].post.id) {
						trail = [{
							blog: {
								name: post.reblogged_root_name,
							},
							post: {id: root},
							content: null,
						}].concat(trail);
					}

				%
					post-content(post, post.type, trail.length > 1)
						for reblog of trail
							div class: "trail"
								h3 class: "trail-blog"
									a href: "#{clean.blogPath(reblog.blog.name, `/post/${reblog.post.id}/`)}"
										"#{reblog.blog.name}"

								if reblog.content !== null
									div class: "trail-content"
										!"#{rewriteHTML(reblog.content)}"

				footer class: "post-info"
					div
						a href: "#{rewriteLinkString(post.post_url)}" rel: "bookmark"
							time class: "post-date" datetime: "#{date.toISOString()}" title: "#{date.toString()}" pubdate:
								"#{relativeDate(date)}"

						" " span class: "post-alternates"
							"("
							a href: "#{post.post_url}"
								"original"
							", "
							a href: "data:text/plain;charset=utf-8,#{encodeURIComponent(inspect(post, { depth: null }))}"
								"data"
							")"

					ul class: "post-tags"
						for tag of post.tags
							li a rel: "tag" href: "/blog/#{data.name}/tagged/#{encodeURIComponent(tag)}" "##{tag}"

					div class: "post-stats"
						if post.note_count
							span class: "note-count" "#{pluralize(post.note_count, '%d note', '%d notes')}"

					if post.notes
						ul class: "post-notes"
							for note of post.notes
								li
									if note.post_id
										% const reblogBlog = url.parse(note.blog_url).hostname;
										a href: "#{clean.blogPath(reblogBlog, `/post/${note.post_id}/`)}" "#{note.type}"
									else
										"#{note.type}"
									" from "
									a href: "#{rewriteLinkString(note.blog_url)}" "#{note.blog_name}"

	% const {offset, limit} = data;
	% const prev = offset > limit ? offset - limit : offset > 0 ? '' : null;
	% const next = offset + data.posts.length < data.total_posts ? offset + data.posts.length : null;

	footer id: "footer"
		nav id: "page-navigation"
			if prev !== null
				a
					class: "prev"
					rel: "prev"
					href: "#{url.format(update(data.pageUri, { search: null, query: { offset: prev }, hash: '#posts' }))}"
					accesskey: "p" "← Previous page"
				" "
			if next !== null
				a
					class: "next"
					rel: "next"
					href: "#{url.format(update(data.pageUri, { search: null, query: { offset: next }, hash: '#posts' }))}"
					accesskey: "n" "Next page →"


track namespace prefix (pfx) as instance variable

The namespace prefix is being forgotten when landing between IO chunks
because it is tracked as a local variable that is re-initialized. To
correct this we assign as an instance variable to maintain state between
parsing of IO chunks.

=begin
%%{
machine bel;

  action call_term {fcall term;}
  action term {
    yield @term
  }
  action term_init {
    @term_stack = []
  }
  action term_fx {
    fx = @name.to_sym
    @term_stack.push(BEL::Model::Term.new(fx, []))
    @pfx = nil
    @pbuf = []
  }
  action term_arg {
    val = @pbuf.map(&:chr).join()
    if not val.empty?
      if val.start_with? '"' and val.end_with? '"'
        val = val.strip()[1...-1]
      end

      ns =
			  if @pfx
				  @namespaces[@pfx.to_sym] ||= BEL::Namespace::NamespaceDefinition.new(@pfx, nil, nil)
			  else
				  nil
			  end

      param = BEL::Model::Parameter.new(ns, val)
      @term_stack.last << param

      yield param
    end
    @pbuf = []
    @pfx = nil
  }
  action term_pop {
    @term = @term_stack.pop
    if not @term_stack.empty?
      @term_stack.last << @term
    end
  }
  action pbuf  { @pbuf << fc }
  action pns {
    @pfx = @pbuf.map(&:chr).join()
    @pbuf = []
  }

  include 'common.rl';

  term :=
    (
      SP* (IDENT $pbuf ':')? @pns SP* (STRING $pbuf | IDENT $pbuf) %term_arg |
      SP* FUNCTION SP* '(' @term_fx @call_term
    )
    (
      SP* ',' SP*
      (
        (IDENT $pbuf ':')? @pns SP* (STRING $pbuf | IDENT $pbuf) %term_arg |
        FUNCTION SP* '(' @term_fx @call_term
      )
    )* SP* ')' >term_pop >term @{n = 0} @return;

  term_main :=
    (
      FUNCTION SP* '(' @term_init @term_fx @call_term NL @term
    )+;
}%%
=end

require 'observer'
require_relative 'language'

module BEL
  module Script
    class Parser
      include Observable

      def initialize
        @annotations = {}
        @statement_group = nil
        %% write data;
      end

      def parse(content)
        eof = :ignored
        buffer = []
        stack = []
        data = content.unpack('C*')

        if block_given?
          observer = Observer.new(&Proc.new)
          self.add_observer(observer)
        end

        %% write init;
        %% write exec;

        if block_given?
          self.delete_observer(observer)
        end
      end
    end

    private

    class Observer
      include Observable

      def initialize(&block)
        @block = block
      end

      def update(obj)
        @block.call(obj)
      end
    end
  end
end
# vim: ts=2 sw=2:
# encoding: utf-8

=begin
%%{
machine bel;

  action call_term {fcall term;}
  action term {
    yield @term
  }
  action term_init {
    @term_stack = []
  }
  action term_fx {
    fx = @name.to_sym
    @term_stack.push(BEL::Model::Term.new(fx, []))
    pfx = nil
    @pbuf = []
  }
  action term_arg {
    val = @pbuf.map(&:chr).join()
    if not val.empty?
      if val.start_with? '"' and val.end_with? '"'
        val = val.strip()[1...-1]
      end

      ns =
			  if pfx
				  @namespaces[pfx.to_sym] ||= BEL::Namespace::NamespaceDefinition.new(pfx, nil, nil)
			  else
				  nil
			  end

      param = BEL::Model::Parameter.new(ns, val)
      @term_stack.last << param

      yield param
    end
    @pbuf = []
    pfx = nil
  }
  action term_pop {
    @term = @term_stack.pop
    if not @term_stack.empty?
      @term_stack.last << @term
    end
  }
  action pbuf  { @pbuf << fc }
  action pns {
    pfx = @pbuf.map(&:chr).join()
    @pbuf = []
  }

  include 'common.rl';

  term :=
    (
      SP* (IDENT $pbuf ':')? @pns SP* (STRING $pbuf | IDENT $pbuf) %term_arg |
      SP* FUNCTION SP* '(' @term_fx @call_term
    )
    (
      SP* ',' SP*
      (
        (IDENT $pbuf ':')? @pns SP* (STRING $pbuf | IDENT $pbuf) %term_arg |
        FUNCTION SP* '(' @term_fx @call_term
      )
    )* SP* ')' >term_pop >term @{n = 0} @return;

  term_main :=
    (
      FUNCTION SP* '(' @term_init @term_fx @call_term NL @term
    )+;
}%%
=end

require 'observer'
require_relative 'language'

module BEL
  module Script
    class Parser
      include Observable

      def initialize
        @annotations = {}
        @statement_group = nil
        %% write data;
      end

      def parse(content)
        eof = :ignored
        buffer = []
        stack = []
        data = content.unpack('C*')

        if block_given?
          observer = Observer.new(&Proc.new)
          self.add_observer(observer)
        end

        %% write init;
        %% write exec;

        if block_given?
          self.delete_observer(observer)
        end
      end
    end

    private

    class Observer
      include Observable

      def initialize(&block)
        @block = block
      end

      def update(obj)
        @block.call(obj)
      end
    end
  end
end

# intended for direct testing
if __FILE__ == $0
  require 'bel'

  if ARGV[0]
    content = (File.exists? ARGV[0]) ? File.open(ARGV[0], 'r:UTF-8').read : ARGV[0]
  else
    content = $stdin.read
  end

  class DefaultObserver
    def update(obj)
      puts obj
    end
  end

  parser = BEL::Script::Parser.new
  parser.add_observer(DefaultObserver.new)
  parser.parse(content)
end
# vim: ts=2 sw=2:
# encoding: utf-8