* ext/superredcloth_scan/superredcloth_scan.rl: scale back strikethroughs.  they now only work if the opening dash is preceded by a space and the closing dash is followed by a space.  they must be placed up against the word to strike-out.

/*
 * superredcloth_scan.rl
 *
 * $Author: why $
 * $Date$
 *
 * Copyright (C) 2007 why the lucky stiff
 */
#include <ruby.h>
#include "superredcloth.h"

static VALUE superredcloth_transform2(VALUE str, int top);
static VALUE super_ParseError, super_RedCloth;

#define INLINE(T)    rb_str_append(block, rb_funcall(super_RedCloth, rb_intern(#T), 1, regs))
#define BLOCK(T) \
  { \
    VALUE sblock = rb_funcall(block, rb_intern("strip"), 0); \
    if (RSTRING(sblock)->len > 0) \
    { \
      rb_str_append(html, rb_funcall(super_RedCloth, rb_intern(#T), 1, block)); \
    } \
    block = rb_str_new2(""); \
  }
#define FORMAT(A, T) \
  { \
    rb_hash_aset(regs, ID2SYM(rb_intern(#A)), superredcloth_transform2(rb_hash_aref(regs, ID2SYM(rb_intern(#A))), 1)); \
    rb_str_append(block, rb_funcall(super_RedCloth, rb_intern(#T), 1, regs)); \
  }
#define FORMAT_BLOCK(A, T) \
  { \
    /* rb_p(regs); */ \
    VALUE btype = rb_hash_aref(regs, ID2SYM(rb_intern(#T))); \
    rb_hash_aset(regs, ID2SYM(rb_intern(#A)), superredcloth_transform2(rb_hash_aref(regs, ID2SYM(rb_intern(#A))), 1)); \
    StringValue(btype); \
    rb_str_append(html, rb_funcall(super_RedCloth, rb_intern(RSTRING(btype)->ptr), 1, regs)); \
    block = rb_str_new2(""); \
  }
#define STORE(T)  \
  if (p > reg && reg >= tokstart) { \
    while (reg < p && ( *reg == '\r' || *reg == '\n' ) ) { reg++; } \
    while (p > reg && ( *(p - 1) == '\r' || *(p - 1) == '\n' ) ) { p--; } \
  } \
  if (p > reg && reg >= tokstart) { \
    VALUE str = rb_str_new(reg, p-reg); \
    rb_hash_aset(regs, ID2SYM(rb_intern(#T)), str); \
    /* printf("STORE(" #T ") %s\n", RSTRING(str)->ptr); */ \
  } else { \
    rb_hash_aset(regs, ID2SYM(rb_intern(#T)), Qnil); \
  }
#define STORE_URL(T) \
  if (p > reg && reg >= tokstart) { \
    char punct = 1; \
    while (p > reg && punct == 1) { \
      switch (*(p - 1)) { \
        case '!': case '"': case '#': case '$': case '%': case ']': case '[': case '&': case '\'': \
        case '*': case '+': case ',': case '-': case '.': case ')': case '(': case ':':  \
        case ';': case '=': case '?': case '@': case '\\': case '^': case '_': \
        case '`': case '|': case '~': p--; break; \
        default: punct = 0; \
      } \
    } \
    tokend = p; \
  } \
  STORE(T)

%%{
  machine superredcloth_scan;

  action A { reg = p; }
  action X { regs = rb_hash_new(); reg = NULL; }
  action cat {
    rb_str_cat(block, tokstart, tokend-tokstart);
  }
  action esc {
    if (tokend > tokstart) {
      char *t = tokstart, *t2 = tokstart, *ch = NULL;
      while (t2 < tokend) {
        ch = NULL;
        switch (*t2)
        {
          case '&':  ch = "&";    break;
          case '>':  ch = ">";     break;
          case '<':  ch = "<";     break;
          case '"':  ch = """;   break;
          case '\n': ch = "<br />\n"; break;
          case '\'': ch = "’";  break;
        }

        if (ch != NULL)
        {
          if (t2 > t)
            rb_str_cat(block, t, t2-t);
          rb_str_cat2(block, ch);
          t = t2 + 1;
        }

        t2++;
      }
      if (t2 > t)
        rb_str_cat(block, t, t2-t);
    }
  }
  action ignore { rb_str_append(block, rb_funcall(super_RedCloth, rb_intern("ignore"), 1, regs)); }
  action notextile { BLOCK(para); rb_str_append(html, rb_funcall(super_RedCloth, rb_intern("ignore"), 1, regs)); }

  # minor character groups
  CRLF = ( '\r'? '\n' ) ;
  A_HLGN = ( "<>" | "<" | ">" | "=" | [()]+ ) ;
  A_LIMIT = ( "<" | "=" | ">" ) ;
  A_VLGN = ( "-" | "^" | "~" ) ;
  C_CLAS = ( "(" [^)]+ >A %{ STORE(class) } ")" ) ;
  C_LNGE = ( "[" [^\]]+ >A %{ STORE(lang) } "]" ) ;
  C_STYL = ( "{" [^}]+ >A %{ STORE(style) } "}" ) ;
  S_CSPN = ( "\\" [0-9]+ ) ;
  S_RSPN = ( "/" [0-9]+ ) ;
  A = ( ( A_HLGN | A_VLGN )* ) >A %{ STORE(align) } ;
  A2 = ( A_LIMIT? ) >A %{ STORE(align) } ;
  S = ( S_CSPN S_RSPN  | S_RSPN S_CSPN? ) >A %{ STORE(span) } ;
  C = ( C_CLAS | C_STYL | C_LNGE )* ;
  PUNCT = ( "!" | '"' | "#" | "$" | "%" | "&" | "'" | "*" | "+" | "," | "-" | "." | "/" | ":" | ";" | "=" | "?" | "@" | "\\" | "^" | "_" | "`" | "|" | "~" | "[" | "(" | "<" ) ;
  dotspace = [. ] ;

  # URI tokens (lifted from Mongrel)
  CTL = (cntrl | 127);
  safe = ("$" | "-" | "_" | ".");
  extra = ("!" | "*" | "'" | "(" | ")" | ",");
  reserved = (";" | "/" | "?" | ":" | "@" | "&" | "=" | "+");
  unsafe = (CTL | " " | "\"" | "#" | "%" | "<" | ">");
  national = any -- (alpha | digit | reserved | extra | safe | unsafe);
  unreserved = (alpha | digit | safe | extra | national);
  escape = ("%" xdigit xdigit);
  uchar = (unreserved | escape);
  pchar = (uchar | ":" | "@" | "&" | "=" | "+");
  scheme = ( alpha | digit | "+" | "-" | "." )+ ;
  absolute_uri = (scheme ":" (uchar | reserved )*);
  safepath = (pchar* (alpha | digit | safe) pchar*) ;
  path = (safepath ( "/" pchar* )*) ;
  query = ( uchar | reserved )* ;
  param = ( pchar | "/" )* ;
  params = (param ( ";" param )*) ;
  rel_path = (path (";" params)?) ("?" query)?;
  absolute_path = ("/"+ rel_path?);
  target = ("#" pchar*) ;
  uri = (target | absolute_uri target? | absolute_path target? | rel_path target?) ;

  default = ^0 ;
  trailing = PUNCT - ("'" | '"') ;
  chars = (default - space)+ ;
  phrase = chars -- trailing ;
  EOF = 0 ;

  # html tags (from Hpricot)
  NameChar = [\-A-Za-z0-9._:?] ;
  Name = [A-Za-z_:] NameChar* ;
  NameAttr = NameChar+ ;
  Q1Attr = [^']* ;
  Q2Attr = [^"]* ;
  UnqAttr = ( space | [^ \t\n<>"'] [^ \t\n<>]* ) ;
  Nmtoken = NameChar+ ;
  Attr =  NameAttr space* "=" space* ('"' Q2Attr '"' | "'" Q1Attr "'" | UnqAttr space+ ) space* ;
  AttrEnd = ( NameAttr space* "=" space* UnqAttr? | Nmtoken ) ;
  AttrSet = ( Attr | Nmtoken space+ ) ;
  start_tag = "<" Name space+ AttrSet* (AttrEnd)? ">" | "<" Name ">";
  empty_tag = "<" Name space+ AttrSet* (AttrEnd)? "/>" | "<" Name "/>" ;
  end_tag = "</" Name space* ">" ;
  html_comment = "<!--" (default+) :> "-->";

  # common
  title = ( '(' [^)]+ >A %{ STORE(title) } ')' ) ;
  word = ( alnum | safe | " " ) ;

  # links
  link_says = ( word+ ) >A %{ STORE(name) } ;
  link_is = ( C dotspace? link_says title? ) ;
  link = ( '"' link_is '":' %A uri ) >X ;

  # images
  image_src = ( uri ) >A %{ STORE(src) } ;
  image_is = ( A2 C dotspace? image_src :> title? ) ;
  image_link = ( ":" uri ) ;
  image = ( "!" image_is "!" %A image_link? ) >X ;

  action T { STORE(text); }

  # footnotes
  footno = "[" >X %A digit+ %T "]" ;

  # markup
  mspace = ( ( " " | "\t" | CRLF )+ ) -- CRLF{2} ;
  mtext = ( chars (mspace chars)* ) ;
  code = "@" >X C mtext >A %T :> "@" ;
  strong = "*" >X C mtext >A %T :> "*" ;
  b = "**" >X C mtext >A %T :> "**" ;
  em = "_" >X C mtext >A %T :> "_" ;
  i = "__" >X C mtext >A %T :> "__" ;
  del = " -" >X C ( mtext -- "-" ) >A %T :> "- " ;
  ins = "+" >X C mtext >A %T :> "+" ;
  sup = "^" >X C mtext >A %T :> "^" ;
  sub = "~" >X C mtext >A %T :> "~" ;
  span = "%" >X C mtext >A %T :> "%" ;
  cite = "??" >X C mtext >A %T :> "??" ;
  ignore = "==" >X %A mtext %T :> "==" ;
  snip = "```" >X %A mtext %T :> "```" ;
  quote1 = "'" >X %A mtext %T :> "'" ;
  quote2 = '"' >X %A mtext %T :> '"' ;
  apos = "'" ;
  notextile = "<notextile>" >X %A ( default+ ) %T :> "</notextile>" ;

  # glyphs
  ellipsis = ( " "? >A %T "..." ) >X ;
  emdash = ( " "? "--" " "? ) >X ;
  arrow = ( " "? "->" " "? ) >X ;
  endash = ( " "? "-" " "? ) >X ;
  acronym = ( [A-Z] >A [A-Z0-9]{2,} %T "(" [^)]+ >A %{ STORE(title) } ")" ) >X ;
  dim = ( digit+ >A %{ STORE(x) } " x " digit+ >A %{ STORE(y) } ) >X ;
  tm = [Tt] [Mm] ;
  trademark = " "? ( "[" tm "]" | "(" tm ")" ) ;
  reg = [Rr] ;
  registered = " "? ( "[" reg "]" | "(" reg ")" ) ;
  cee = [Cc] ;
  copyright = " "? ( "[" cee "]" | "(" cee ")" ) ;

  # blocks
  btext = ( ( default+ ) -- CRLF{2} ) ( CRLF{2} )? ;
  btype = ( "p" | "h1" | "h2" | "h3" | "h4" | "h5" | "h6" | "bq" ) >A %{ STORE(type) } ;
  block = ( btype A C ". " %A btext ) >X ;
  ftype = ( "fn" >A %{ STORE(type) } digit+ >A %{ STORE(id) } ) ;
  fblock = ( ftype A C ". " %A btext ) >X ;

  main := |*

    image { if ( *reg == ':') { reg += 1; STORE_URL(href); } INLINE(image); };

    link { STORE_URL(href); INLINE(link); };

    code { FORMAT(text, code); };
    strong { FORMAT(text, strong); };
    b { FORMAT(text, b); };
    em { FORMAT(text, em); };
    i { FORMAT(text, i); };
    del { FORMAT(text, del); };
    ins { FORMAT(text, ins); };
    sup { FORMAT(text, sup); };
    sub { FORMAT(text, sub); };
    span { FORMAT(text, span); };
    cite { FORMAT(text, cite); };
    ignore => ignore;
    snip { FORMAT(text, snip); };
    quote1 { FORMAT(text, quote1); };
    quote2 { FORMAT(text, quote2); };
    notextile => notextile;

    ellipsis { INLINE(ellipsis); };
    emdash { INLINE(emdash); };
    endash { INLINE(endash); };
    arrow { INLINE(arrow); };
    acronym { INLINE(acronym); };
    dim { INLINE(dim); };
    trademark { INLINE(trademark); };
    registered { INLINE(registered); };
    copyright { INLINE(copyright); };

    footno { FORMAT(text, footno); };
    fblock { STORE(text); FORMAT_BLOCK(text, type); };

    block { STORE(text); FORMAT_BLOCK(text, type); };
    CRLF{2,} { BLOCK(para); };

    start_tag => cat;
    end_tag => cat;
    empty_tag => cat;
    html_comment => cat;

    phrase => esc;
    PUNCT => esc;
    space => esc;

    EOF;

  *|;
}%%

%% write data nofinal;

static VALUE
superredcloth_transform(p, pe, top)
  char *p, *pe;
  int top;
{
  int cs, act;
  char *tokstart, *tokend, *reg;
  VALUE html = rb_str_new2("");
  VALUE block = rb_str_new2("");
  VALUE regs = Qnil;

  %% write init;

  %% write exec;

  if (RSTRING(block)->len > 0)
  {
    if (top == 1)
    {
      BLOCK(para);
    }
    else
    {
      rb_str_append(html, block);
    }
  }

  return html;
}

static VALUE
superredcloth_transform2(str, top)
  VALUE str;
  int top;
{
  StringValue(str);
  return superredcloth_transform(RSTRING(str)->ptr, RSTRING(str)->ptr + RSTRING(str)->len + 1, 0);
}

static VALUE
superredcloth_to_html(self)
  VALUE self;
{
  char *pe, *p;
  int len = 0;

  StringValue(self);
  return superredcloth_transform(RSTRING(self)->ptr, RSTRING(self)->ptr + RSTRING(self)->len + 1, 1);
}

void Init_superredcloth_scan()
{
  super_RedCloth = rb_define_class("SuperRedCloth", rb_cString);
  rb_define_method(super_RedCloth, "to_html", superredcloth_to_html, 0);
  super_ParseError = rb_define_class_under(super_RedCloth, "ParseError", rb_eException);
}

/*
 * superredcloth_scan.rl
 *
 * $Author: why $
 * $Date$
 *
 * Copyright (C) 2007 why the lucky stiff
 */
#include <ruby.h>
#include "superredcloth.h"

static VALUE superredcloth_transform2(VALUE str, int top);
static VALUE super_ParseError, super_RedCloth;

#define INLINE(T)    rb_str_append(block, rb_funcall(super_RedCloth, rb_intern(#T), 1, regs))
#define BLOCK(T) \
  { \
    VALUE sblock = rb_funcall(block, rb_intern("strip"), 0); \
    if (RSTRING(sblock)->len > 0) \
    { \
      rb_str_append(html, rb_funcall(super_RedCloth, rb_intern(#T), 1, block)); \
    } \
    block = rb_str_new2(""); \
  }
#define FORMAT(A, T) \
  { \
    rb_hash_aset(regs, ID2SYM(rb_intern(#A)), superredcloth_transform2(rb_hash_aref(regs, ID2SYM(rb_intern(#A))), 1)); \
    rb_str_append(block, rb_funcall(super_RedCloth, rb_intern(#T), 1, regs)); \
  }
#define FORMAT_BLOCK(A, T) \
  { \
    /* rb_p(regs); */ \
    VALUE btype = rb_hash_aref(regs, ID2SYM(rb_intern(#T))); \
    rb_hash_aset(regs, ID2SYM(rb_intern(#A)), superredcloth_transform2(rb_hash_aref(regs, ID2SYM(rb_intern(#A))), 1)); \
    StringValue(btype); \
    rb_str_append(html, rb_funcall(super_RedCloth, rb_intern(RSTRING(btype)->ptr), 1, regs)); \
    block = rb_str_new2(""); \
  }
#define STORE(T)  \
  if (p > reg && reg >= tokstart) { \
    while (reg < p && ( *reg == '\r' || *reg == '\n' ) ) { reg++; } \
    while (p > reg && ( *(p - 1) == '\r' || *(p - 1) == '\n' ) ) { p--; } \
  } \
  if (p > reg && reg >= tokstart) { \
    VALUE str = rb_str_new(reg, p-reg); \
    rb_hash_aset(regs, ID2SYM(rb_intern(#T)), str); \
    /* printf("STORE(" #T ") %s\n", RSTRING(str)->ptr); */ \
  } else { \
    rb_hash_aset(regs, ID2SYM(rb_intern(#T)), Qnil); \
  }
#define STORE_URL(T) \
  if (p > reg && reg >= tokstart) { \
    char punct = 1; \
    while (p > reg && punct == 1) { \
      switch (*(p - 1)) { \
        case '!': case '"': case '#': case '$': case '%': case ']': case '[': case '&': case '\'': \
        case '*': case '+': case ',': case '-': case '.': case ')': case '(': case ':':  \
        case ';': case '=': case '?': case '@': case '\\': case '^': case '_': \
        case '`': case '|': case '~': p--; break; \
        default: punct = 0; \
      } \
    } \
    tokend = p; \
  } \
  STORE(T)

%%{
  machine superredcloth_scan;

  action A { reg = p; }
  action X { regs = rb_hash_new(); reg = NULL; }
  action cat {
    rb_str_cat(block, tokstart, tokend-tokstart);
  }
  action esc {
    if (tokend > tokstart) {
      char *t = tokstart, *t2 = tokstart, *ch = NULL;
      while (t2 < tokend) {
        ch = NULL;
        switch (*t2)
        {
          case '&':  ch = "&";    break;
          case '>':  ch = ">";     break;
          case '<':  ch = "<";     break;
          case '"':  ch = """;   break;
          case '\n': ch = "<br />\n"; break;
          case '\'': ch = "’";  break;
        }

        if (ch != NULL)
        {
          if (t2 > t)
            rb_str_cat(block, t, t2-t);
          rb_str_cat2(block, ch);
          t = t2 + 1;
        }

        t2++;
      }
      if (t2 > t)
        rb_str_cat(block, t, t2-t);
    }
  }
  action ignore { rb_str_append(block, rb_funcall(super_RedCloth, rb_intern("ignore"), 1, regs)); }
  action notextile { BLOCK(para); rb_str_append(html, rb_funcall(super_RedCloth, rb_intern("ignore"), 1, regs)); }

  # minor character groups
  CRLF = ( '\r'? '\n' ) ;
  A_HLGN = ( "<>" | "<" | ">" | "=" | [()]+ ) ;
  A_LIMIT = ( "<" | "=" | ">" ) ;
  A_VLGN = ( "-" | "^" | "~" ) ;
  C_CLAS = ( "(" [^)]+ >A %{ STORE(class) } ")" ) ;
  C_LNGE = ( "[" [^\]]+ >A %{ STORE(lang) } "]" ) ;
  C_STYL = ( "{" [^}]+ >A %{ STORE(style) } "}" ) ;
  S_CSPN = ( "\\" [0-9]+ ) ;
  S_RSPN = ( "/" [0-9]+ ) ;
  A = ( ( A_HLGN | A_VLGN )* ) >A %{ STORE(align) } ;
  A2 = ( A_LIMIT? ) >A %{ STORE(align) } ;
  S = ( S_CSPN S_RSPN  | S_RSPN S_CSPN? ) >A %{ STORE(span) } ;
  C = ( C_CLAS | C_STYL | C_LNGE )* ;
  PUNCT = ( "!" | '"' | "#" | "$" | "%" | "&" | "'" | "*" | "+" | "," | "-" | "." | "/" | ":" | ";" | "=" | "?" | "@" | "\\" | "^" | "_" | "`" | "|" | "~" | "[" | "(" | "<" ) ;
  dotspace = [. ] ;

  # URI tokens (lifted from Mongrel)
  CTL = (cntrl | 127);
  safe = ("$" | "-" | "_" | ".");
  extra = ("!" | "*" | "'" | "(" | ")" | ",");
  reserved = (";" | "/" | "?" | ":" | "@" | "&" | "=" | "+");
  unsafe = (CTL | " " | "\"" | "#" | "%" | "<" | ">");
  national = any -- (alpha | digit | reserved | extra | safe | unsafe);
  unreserved = (alpha | digit | safe | extra | national);
  escape = ("%" xdigit xdigit);
  uchar = (unreserved | escape);
  pchar = (uchar | ":" | "@" | "&" | "=" | "+");
  scheme = ( alpha | digit | "+" | "-" | "." )+ ;
  absolute_uri = (scheme ":" (uchar | reserved )*);
  safepath = (pchar* (alpha | digit | safe) pchar*) ;
  path = (safepath ( "/" pchar* )*) ;
  query = ( uchar | reserved )* ;
  param = ( pchar | "/" )* ;
  params = (param ( ";" param )*) ;
  rel_path = (path (";" params)?) ("?" query)?;
  absolute_path = ("/"+ rel_path?);
  target = ("#" pchar*) ;
  uri = (target | absolute_uri target? | absolute_path target? | rel_path target?) ;

  default = ^0 ;
  trailing = PUNCT - ("'" | '"') ;
  chars = (default - space)+ ;
  phrase = chars -- trailing ;
  EOF = 0 ;

  # html tags (from Hpricot)
  NameChar = [\-A-Za-z0-9._:?] ;
  Name = [A-Za-z_:] NameChar* ;
  NameAttr = NameChar+ ;
  Q1Attr = [^']* ;
  Q2Attr = [^"]* ;
  UnqAttr = ( space | [^ \t\n<>"'] [^ \t\n<>]* ) ;
  Nmtoken = NameChar+ ;
  Attr =  NameAttr space* "=" space* ('"' Q2Attr '"' | "'" Q1Attr "'" | UnqAttr space+ ) space* ;
  AttrEnd = ( NameAttr space* "=" space* UnqAttr? | Nmtoken ) ;
  AttrSet = ( Attr | Nmtoken space+ ) ;
  start_tag = "<" Name space+ AttrSet* (AttrEnd)? ">" | "<" Name ">";
  empty_tag = "<" Name space+ AttrSet* (AttrEnd)? "/>" | "<" Name "/>" ;
  end_tag = "</" Name space* ">" ;
  html_comment = "<!--" (default+) :> "-->";

  # common
  title = ( '(' [^)]+ >A %{ STORE(title) } ')' ) ;
  word = ( alnum | safe | " " ) ;

  # links
  link_says = ( word+ ) >A %{ STORE(name) } ;
  link_is = ( C dotspace? link_says title? ) ;
  link = ( '"' link_is '":' %A uri ) >X ;

  # images
  image_src = ( uri ) >A %{ STORE(src) } ;
  image_is = ( A2 C dotspace? image_src :> title? ) ;
  image_link = ( ":" uri ) ;
  image = ( "!" image_is "!" %A image_link? ) >X ;

  action T { STORE(text); }

  # footnotes
  footno = "[" >X %A digit+ %T "]" ;

  # markup
  mspace = ( ( " " | "\t" | CRLF )+ ) -- CRLF{2} ;
  mtext = ( chars (mspace chars)* ) ;
  code = "@" >X C mtext >A %T :> "@" ;
  strong = "*" >X C mtext >A %T :> "*" ;
  b = "**" >X C mtext >A %T :> "**" ;
  em = "_" >X C mtext >A %T :> "_" ;
  i = "__" >X C mtext >A %T :> "__" ;
  del = "-" >X C mtext >A %T :> "-" ;
  ins = "+" >X C mtext >A %T :> "+" ;
  sup = "^" >X C mtext >A %T :> "^" ;
  sub = "~" >X C mtext >A %T :> "~" ;
  span = "%" >X C mtext >A %T :> "%" ;
  cite = "??" >X C mtext >A %T :> "??" ;
  ignore = "==" >X %A mtext %T :> "==" ;
  snip = "```" >X %A mtext %T :> "```" ;
  quote1 = "'" >X %A mtext %T :> "'" ;
  quote2 = '"' >X %A mtext %T :> '"' ;
  apos = "'" ;
  notextile = "<notextile>" >X %A ( default+ ) %T :> "</notextile>" ;

  # glyphs
  ellipsis = ( " "? >A %T "..." ) >X ;
  emdash = ( " "? "--" " "? ) >X ;
  arrow = ( " "? "->" " "? ) >X ;
  endash = ( " "? "-" " "? ) >X ;
  acronym = ( [A-Z] >A [A-Z0-9]{2,} %T "(" [^)]+ >A %{ STORE(title) } ")" ) >X ;
  dim = ( digit+ >A %{ STORE(x) } " x " digit+ >A %{ STORE(y) } ) >X ;
  tm = [Tt] [Mm] ;
  trademark = " "? ( "[" tm "]" | "(" tm ")" ) ;
  reg = [Rr] ;
  registered = " "? ( "[" reg "]" | "(" reg ")" ) ;
  cee = [Cc] ;
  copyright = " "? ( "[" cee "]" | "(" cee ")" ) ;

  # blocks
  btext = ( ( default+ ) -- CRLF{2} ) ( CRLF{2} )? ;
  btype = ( "p" | "h1" | "h2" | "h3" | "h4" | "h5" | "h6" | "bq" ) >A %{ STORE(type) } ;
  block = ( btype A C ". " %A btext ) >X ;
  ftype = ( "fn" >A %{ STORE(type) } digit+ >A %{ STORE(id) } ) ;
  fblock = ( ftype A C ". " %A btext ) >X ;

  main := |*

    image { if ( *reg == ':') { reg += 1; STORE_URL(href); } INLINE(image); };

    link { STORE_URL(href); INLINE(link); };

    code { FORMAT(text, code); };
    strong { FORMAT(text, strong); };
    b { FORMAT(text, b); };
    em { FORMAT(text, em); };
    i { FORMAT(text, i); };
    del { FORMAT(text, del); };
    ins { FORMAT(text, ins); };
    sup { FORMAT(text, sup); };
    sub { FORMAT(text, sub); };
    span { FORMAT(text, span); };
    cite { FORMAT(text, cite); };
    ignore => ignore;
    snip { FORMAT(text, snip); };
    quote1 { FORMAT(text, quote1); };
    quote2 { FORMAT(text, quote2); };
    notextile => notextile;

    ellipsis { INLINE(ellipsis); };
    emdash { INLINE(emdash); };
    endash { INLINE(endash); };
    arrow { INLINE(arrow); };
    acronym { INLINE(acronym); };
    dim { INLINE(dim); };
    trademark { INLINE(trademark); };
    registered { INLINE(registered); };
    copyright { INLINE(copyright); };

    footno { FORMAT(text, footno); };
    fblock { STORE(text); FORMAT_BLOCK(text, type); };

    block { STORE(text); FORMAT_BLOCK(text, type); };
    CRLF{2,} { BLOCK(para); };

    start_tag => cat;
    end_tag => cat;
    empty_tag => cat;
    html_comment => cat;

    phrase => esc;
    PUNCT => esc;
    space => esc;

    EOF;

  *|;
}%%

%% write data nofinal;

static VALUE
superredcloth_transform(p, pe, top)
  char *p, *pe;
  int top;
{
  int cs, act;
  char *tokstart, *tokend, *reg;
  VALUE html = rb_str_new2("");
  VALUE block = rb_str_new2("");
  VALUE regs = Qnil;

  %% write init;

  %% write exec;

  if (RSTRING(block)->len > 0)
  {
    if (top == 1)
    {
      BLOCK(para);
    }
    else
    {
      rb_str_append(html, block);
    }
  }

  return html;
}

static VALUE
superredcloth_transform2(str, top)
  VALUE str;
  int top;
{
  StringValue(str);
  return superredcloth_transform(RSTRING(str)->ptr, RSTRING(str)->ptr + RSTRING(str)->len + 1, 0);
}

static VALUE
superredcloth_to_html(self)
  VALUE self;
{
  char *pe, *p;
  int len = 0;

  StringValue(self);
  return superredcloth_transform(RSTRING(self)->ptr, RSTRING(self)->ptr + RSTRING(self)->len + 1, 1);
}

void Init_superredcloth_scan()
{
  super_RedCloth = rb_define_class("SuperRedCloth", rb_cString);
  rb_define_method(super_RedCloth, "to_html", superredcloth_to_html, 0);
  super_ParseError = rb_define_class_under(super_RedCloth, "ParseError", rb_eException);
}


nicer error messages when schema parsing fails

#include <inttypes.h>
#include <stdlib.h>
#include <stdio.h>
#include <string.h>
#include <ctype.h>

#include "qstruct_compiler.h"
#include "internal.h"

%%{
  machine qstruct;
  write data;
}%%

struct qstruct_definition *parse_qstructs(char *schema, size_t schema_size, char *err_buf, size_t err_buf_size) {
  char *p = schema, *pe = schema + schema_size, *eof = 0;
  int cs = -1;

  int curr_line = 1;
  ssize_t i;
  int err = 0;

  struct qstruct_definition *def = NULL, *new_def;
  struct qstruct_item curr_item;
  ssize_t curr_item_index;

  struct qstruct_item *new_items;
  ssize_t items_allocated;
  ssize_t largest_item;
  ssize_t packing_result;

  char err_ctx_buf[256];
  char err_desc_buf[512];
  char *err_ctx_start, *err_ctx_end;

  // the ragel state machine will initialise these variables, assignments silence compiler warnings
  curr_item_index = 0;
  memset(&curr_item, '\0', sizeof(curr_item));
  items_allocated = 0;
  largest_item = -1;

  #define PARSE_ERROR(...) do { \
    snprintf(err_desc_buf, sizeof(err_desc_buf), __VA_ARGS__); \
    for(err_ctx_start=p; err_ctx_start>schema && *err_ctx_start != '\n' && (p-err_ctx_start) < 20; err_ctx_start--) {} \
    while (isspace(*err_ctx_start) && err_ctx_start < p) err_ctx_start++; \
    for(err_ctx_end=p; err_ctx_end<(pe-1) && *err_ctx_end != '\n' && (err_ctx_end-p) < 20; err_ctx_end++) {} \
    memcpy(err_ctx_buf, err_ctx_start, err_ctx_end - err_ctx_start); \
    *(err_ctx_buf + (err_ctx_end - err_ctx_start)) = '\0'; \
    snprintf(err_buf, err_buf_size, "\n------------------------------------------------------------\nQstruct schema parse error (line %d, character %d)\n\n  %s\n  %*s^\n  %*s|--%s\n\n------------------------------------------------------------\n", curr_line, (int)(p-schema), err_ctx_buf, (int)(p-err_ctx_start), " ", (int)(p-err_ctx_start), " ", err_desc_buf); \
    err = 1; \
    goto bail; \
  } while(0);

  %%{
    action init_qstruct {
      new_def = malloc(sizeof(struct qstruct_definition));
      if (new_def == NULL)
        PARSE_ERROR("out of memory");

      new_def->next = def;
      def = new_def;
      new_def = NULL;

      items_allocated = 64;
      def->items = malloc(items_allocated * sizeof(struct qstruct_item));
      if (def->items == NULL)
        PARSE_ERROR("out of memory");

      largest_item = -1;
      for (i=0; i<items_allocated; i++) def->items[i].occupied = 0;
    }

    action handle_item {
      if (curr_item_index < 0)
        PARSE_ERROR("id value overflow"); // FIXME: prolly need to check upper bound

      if (curr_item_index >= items_allocated) {
        new_items = realloc(def->items, curr_item_index*2 * sizeof(struct qstruct_item));
        if (new_items == NULL)
          PARSE_ERROR("out of memory");

        def->items = new_items;
        new_items = NULL;

        for(i=items_allocated; i<curr_item_index*2; i++) def->items[i].occupied = 0;
        items_allocated = curr_item_index*2;
      }

      if ((curr_item.type & 0xFFFF) == QSTRUCT_TYPE_BOOL && (curr_item.type & (QSTRUCT_TYPE_MOD_ARRAY_FIX | QSTRUCT_TYPE_MOD_ARRAY_DYN)))
        PARSE_ERROR("bools can't be arrays");

      if ((curr_item.type & 0xFFFF) == QSTRUCT_TYPE_STRING && (curr_item.type & QSTRUCT_TYPE_MOD_ARRAY_FIX))
        PARSE_ERROR("strings can't be fixed-size arrays");

      if ((curr_item.type & 0xFFFF) == QSTRUCT_TYPE_BLOB && (curr_item.type & QSTRUCT_TYPE_MOD_ARRAY_FIX))
        PARSE_ERROR("blobs can't be fixed-size arrays");

      if (def->items[curr_item_index].occupied)
        PARSE_ERROR("duplicated index %ld", curr_item_index);

      def->items[curr_item_index].name = curr_item.name;
      def->items[curr_item_index].name_len = curr_item.name_len;
      def->items[curr_item_index].type = curr_item.type;
      def->items[curr_item_index].fixed_array_size = curr_item.fixed_array_size;
      def->items[curr_item_index].occupied = 1;

      if (curr_item_index > largest_item) largest_item = curr_item_index;
    }

    action handle_qstruct {
      for(i=0; i<largest_item; i++) {
        if (!def->items[i].occupied)
          PARSE_ERROR("missing item %ld", i);
      }

      def[0].num_items = largest_item+1;

      packing_result = calculate_qstruct_packing(def);
      if (packing_result < 0)
        PARSE_ERROR("memory error in packing");

      def[0].body_size = (size_t) packing_result - QSTRUCT_HEADER_SIZE;
    }

    newline = '\n' @{curr_line++;};
    any_count_line = any | newline;
    whitespace_char = any_count_line - 0x21..0x7e;

    alnum_u = alnum | '_';
    alpha_u = alpha | '_';
    identifier = alpha_u alnum_u*;
    identifier_with_package = alpha_u alnum_u* ('::' alpha_u alnum_u*)*;
    integer = digit+;

    ws = (
           whitespace_char |
           ( '#' [^\n]* newline ) |
           ( '/*' ( ( any_count_line )* - ( any_count_line* '*/' any_count_line* ) ) '*/' )
         );

    type = 'string' %{ curr_item.type = QSTRUCT_TYPE_STRING; } |
           'blob' %{ curr_item.type = QSTRUCT_TYPE_BLOB; } |
           'bool' %{ curr_item.type = QSTRUCT_TYPE_BOOL; } |
           'float' %{ curr_item.type = QSTRUCT_TYPE_FLOAT; } |
           'double' %{ curr_item.type = QSTRUCT_TYPE_DOUBLE; } |
           'int8' %{ curr_item.type = QSTRUCT_TYPE_INT8; } |
           'uint8' %{ curr_item.type = QSTRUCT_TYPE_INT8 | QSTRUCT_TYPE_MOD_UNSIGNED; } |
           'int16' %{ curr_item.type = QSTRUCT_TYPE_INT16; } |
           'uint16' %{ curr_item.type = QSTRUCT_TYPE_INT16 | QSTRUCT_TYPE_MOD_UNSIGNED; } |
           'int32' %{ curr_item.type = QSTRUCT_TYPE_INT32; } |
           'uint32' %{ curr_item.type = QSTRUCT_TYPE_INT32 | QSTRUCT_TYPE_MOD_UNSIGNED; } |
           'int64' %{ curr_item.type = QSTRUCT_TYPE_INT64; } |
           'uint64' %{ curr_item.type = QSTRUCT_TYPE_INT64 | QSTRUCT_TYPE_MOD_UNSIGNED; }
      ;

    array_spec = ('['
                    integer >{ curr_item.fixed_array_size = 0; }
                            @{ curr_item.fixed_array_size = curr_item.fixed_array_size * 10 + (fc - '0'); }
                  ']' >{ curr_item.type |= QSTRUCT_TYPE_MOD_ARRAY_FIX; }

                   |

                   '[' ws* ']' >{ curr_item.type |= QSTRUCT_TYPE_MOD_ARRAY_DYN; }
                 );

    item = identifier >{ curr_item.name = p; curr_item.fixed_array_size = 1; }
                      %{ curr_item.name_len = p - curr_item.name; }
           ws+
           '@' integer >{ curr_item_index = 0; }
                       @{ curr_item_index = curr_item_index * 10 + (fc - '0'); }
           ws+
           type $!{ PARSE_ERROR("unrecognized type"); }
           array_spec?
           ws* ';';

    qstruct = ws*
              'qstruct' >init_qstruct
              ws+
              identifier_with_package >{ def[0].name = p; }
                                      %{ def[0].name_len = p - def[0].name; }
              ws*
             '{'
               ws* (item @handle_item ws*)*
             '}' @handle_qstruct;

    ########################

    main := qstruct* ws*;

    write init;
    write exec;
  }%%

  if (cs < qstruct_first_final)
    PARSE_ERROR("general parse error");

  bail:

  if (err) {
    free_qstruct_definitions(def);
    return NULL;
  }

  return def;
}

void free_qstruct_definitions(struct qstruct_definition *def) {
  struct qstruct_definition *temp;

  while (def) {
    if (def->items) free(def->items);
    temp = def->next;
    free(def);
    def = temp;
  }
}

#include <inttypes.h>
#include <stdlib.h>
#include <stdio.h>
#include <string.h>

#include "qstruct_compiler.h"
#include "internal.h"

%%{
  machine qstruct;
  write data;
}%%

#define PARSE_ERROR(...) do { \
  snprintf(err_buf, err_buf_size, __VA_ARGS__); \
  err = 1; \
  goto bail; \
} while(0);

struct qstruct_definition *parse_qstructs(char *schema, size_t schema_size, char *err_buf, size_t err_buf_size) {
  char *p = schema, *pe = schema + schema_size, *eof = 0;
  int cs = -1;

  int curr_line = 1;
  ssize_t i;
  int err = 0;

  struct qstruct_definition *def = NULL, *new_def;
  struct qstruct_item curr_item;
  ssize_t curr_item_index;

  struct qstruct_item *new_items;
  ssize_t items_allocated;
  ssize_t largest_item;
  ssize_t packing_result;

  // the ragel state machine will initialise these variables, assignments silence compiler warnings
  curr_item_index = 0;
  memset(&curr_item, '\0', sizeof(curr_item));
  items_allocated = 0;
  largest_item = -1;

  %%{
    action init_qstruct {
      new_def = malloc(sizeof(struct qstruct_definition));
      if (new_def == NULL)
        PARSE_ERROR("out of memory");

      new_def->next = def;
      def = new_def;
      new_def = NULL;

      items_allocated = 64;
      def->items = malloc(items_allocated * sizeof(struct qstruct_item));
      if (def->items == NULL)
        PARSE_ERROR("out of memory");

      largest_item = -1;
      for (i=0; i<items_allocated; i++) def->items[i].occupied = 0;
    }

    action handle_item {
      if (curr_item_index < 0)
        PARSE_ERROR("id value overflow"); // FIXME: prolly need to check upper bound

      if (curr_item_index >= items_allocated) {
        new_items = realloc(def->items, curr_item_index*2 * sizeof(struct qstruct_item));
        if (new_items == NULL)
          PARSE_ERROR("out of memory");

        def->items = new_items;
        new_items = NULL;

        for(i=items_allocated; i<curr_item_index*2; i++) def->items[i].occupied = 0;
        items_allocated = curr_item_index*2;
      }

      if ((curr_item.type & 0xFFFF) == QSTRUCT_TYPE_BOOL && (curr_item.type & (QSTRUCT_TYPE_MOD_ARRAY_FIX | QSTRUCT_TYPE_MOD_ARRAY_DYN)))
        PARSE_ERROR("bools can't be arrays (line %d)", curr_line);

      if ((curr_item.type & 0xFFFF) == QSTRUCT_TYPE_STRING && (curr_item.type & QSTRUCT_TYPE_MOD_ARRAY_FIX))
        PARSE_ERROR("strings can't be fixed-size arrays (line %d)", curr_line);

      if ((curr_item.type & 0xFFFF) == QSTRUCT_TYPE_BLOB && (curr_item.type & QSTRUCT_TYPE_MOD_ARRAY_FIX))
        PARSE_ERROR("blobs can't be fixed-size arrays (line %d)", curr_line);

      if (def->items[curr_item_index].occupied)
        PARSE_ERROR("duplicated index %ld (line %d)", curr_item_index, curr_line);

      def->items[curr_item_index].name = curr_item.name;
      def->items[curr_item_index].name_len = curr_item.name_len;
      def->items[curr_item_index].type = curr_item.type;
      def->items[curr_item_index].fixed_array_size = curr_item.fixed_array_size;
      def->items[curr_item_index].occupied = 1;

      if (curr_item_index > largest_item) largest_item = curr_item_index;
    }

    action handle_qstruct {
      for(i=0; i<largest_item; i++) {
        if (!def->items[i].occupied)
          PARSE_ERROR("missing item %ld (line %d)", i, curr_line);
      }

      def[0].num_items = largest_item+1;

      packing_result = calculate_qstruct_packing(def);
      if (packing_result < 0)
        PARSE_ERROR("memory error in packing (line %d)", curr_line);

      def[0].body_size = (size_t) packing_result - QSTRUCT_HEADER_SIZE;
    }

    newline = '\n' @{curr_line++;};
    any_count_line = any | newline;
    whitespace_char = any_count_line - 0x21..0x7e;

    alnum_u = alnum | '_';
    alpha_u = alpha | '_';
    identifier = alpha_u alnum_u*;
    identifier_with_package = alpha_u alnum_u* ('::' alpha_u alnum_u*)*;
    integer = digit+;

    ws = (
           whitespace_char |
           ( '#' [^\n]* newline ) |
           ( '/*' ( ( any_count_line )* - ( any_count_line* '*/' any_count_line* ) ) '*/' )
         );

    type = 'string' %{ curr_item.type = QSTRUCT_TYPE_STRING; } |
           'blob' %{ curr_item.type = QSTRUCT_TYPE_BLOB; } |
           'bool' %{ curr_item.type = QSTRUCT_TYPE_BOOL; } |
           'float' %{ curr_item.type = QSTRUCT_TYPE_FLOAT; } |
           'double' %{ curr_item.type = QSTRUCT_TYPE_DOUBLE; } |
           'int8' %{ curr_item.type = QSTRUCT_TYPE_INT8; } |
           'uint8' %{ curr_item.type = QSTRUCT_TYPE_INT8 | QSTRUCT_TYPE_MOD_UNSIGNED; } |
           'int16' %{ curr_item.type = QSTRUCT_TYPE_INT16; } |
           'uint16' %{ curr_item.type = QSTRUCT_TYPE_INT16 | QSTRUCT_TYPE_MOD_UNSIGNED; } |
           'int32' %{ curr_item.type = QSTRUCT_TYPE_INT32; } |
           'uint32' %{ curr_item.type = QSTRUCT_TYPE_INT32 | QSTRUCT_TYPE_MOD_UNSIGNED; } |
           'int64' %{ curr_item.type = QSTRUCT_TYPE_INT64; } |
           'uint64' %{ curr_item.type = QSTRUCT_TYPE_INT64 | QSTRUCT_TYPE_MOD_UNSIGNED; }
      ;

    array_spec = ('['
                    integer >{ curr_item.fixed_array_size = 0; }
                            @{ curr_item.fixed_array_size = curr_item.fixed_array_size * 10 + (fc - '0'); }
                  ']' >{ curr_item.type |= QSTRUCT_TYPE_MOD_ARRAY_FIX; }

                   |

                   '[' ws* ']' >{ curr_item.type |= QSTRUCT_TYPE_MOD_ARRAY_DYN; }
                 );

    item = identifier >{ curr_item.name = p; curr_item.fixed_array_size = 1; }
                      %{ curr_item.name_len = p - curr_item.name; }
           ws+
           '@' integer >{ curr_item_index = 0; }
                       @{ curr_item_index = curr_item_index * 10 + (fc - '0'); }
           ws+
           type $!{ PARSE_ERROR("unrecognized type (line %d)", curr_line); }
           array_spec?
           ws* ';';

    qstruct = ws*
              'qstruct' >init_qstruct
              ws+
              identifier_with_package >{ def[0].name = p; }
                                      %{ def[0].name_len = p - def[0].name; }
              ws*
             '{'
               ws* (item @handle_item ws*)*
             '}' @handle_qstruct;

    ########################

    main := qstruct* ws*;

    write init;
    write exec;
  }%%

  if (cs < qstruct_first_final)
    PARSE_ERROR("general parse error (line %d)", curr_line);

  bail:

  if (err) {
    free_qstruct_definitions(def);
    return NULL;
  }

  return def;
}

void free_qstruct_definitions(struct qstruct_definition *def) {
  struct qstruct_definition *temp;

  while (def) {
    if (def->items) free(def->items);
    temp = def->next;
    free(def);
    def = temp;
  }
}


Ouput buffer contents when an unexpected character is encountered

/*
 Copyright (c) 2017 Tuukka Norri
 This code is licensed under MIT license (see LICENSE for details).
 */

#include <vcf2multialign/util.hh>
#include <vcf2multialign/vcf_reader.hh>

%% machine vcf_parser;
%% write data;

// We would like to post-process strings that have been read and then pass
// them to the variant object with a simple function call at the call site.
// Solution: Templates can have member function pointers as parameters but of a specific type.
#define HANDLE_STRING_END(FN, ...)	caller <decltype(FN)>::handle_string_end <FN>(*this, ##__VA_ARGS__)
#define HANDLE_INTEGER_END(FN, ...)	caller <decltype(FN)>::handle_integer_end <FN>(*this, ##__VA_ARGS__)

namespace vcf2multialign { namespace detail {

	template <typename t_enum>
	constexpr typename std::underlying_type <t_enum>::type to_underlying(t_enum e)
	{
		return static_cast <typename std::underlying_type <t_enum>::type>(e);
	}

	// Split a t_string into string_views.
	template <bool t_emplace_back, typename t_string, typename t_invalid_pos>
	size_t read_fields(
		t_string const &string,
		t_invalid_pos const invalid_pos,
		char const *string_start,
		char const *sep,
		size_t const count,
		std::vector <std::string_view> &res
	)
	{
		char const *string_ptr(string_start);

		if (t_emplace_back)
			res.clear();
		else
			res.resize(count);

		typename t_string::size_type sep_pos(0);
		size_t i(0);
		bool should_break(false);
		while (i < count)
		{
			sep_pos = string.find(sep, sep_pos);
			if (invalid_pos == sep_pos)
			{
				should_break = true;
				sep_pos = string.size();
			}

			if (t_emplace_back)
				res.emplace_back(string_ptr, sep_pos - (string_ptr - string_start));
			else
			{
				std::string_view sv(string_ptr, sep_pos - (string_ptr - string_start));
				res[i] = std::move(sv);
			}

			++i;
			++sep_pos;
			string_ptr = string_start + sep_pos;

			if (should_break)
				break;
		}
		return i;
	}

	// Split a string into string_views.
	template <bool t_emplace_back>
	size_t read_fields(
		std::string const &string,
		char const *sep,
		size_t const count,
		std::vector <std::string_view> &res
	)
	{
		return read_fields <t_emplace_back>(string, std::string::npos, string.c_str(), sep, count, res);
	}

	template <bool t_emplace_back>
	size_t read_fields(
		std::string_view const &sv,
		char const *sep,
		size_t const count,
		std::vector <std::string_view> &res
	)
	{
		return read_fields <t_emplace_back>(sv, std::string_view::npos, static_cast <char const *>(sv.data()), sep, count, res);
	}
}}

namespace vcf2multialign {

	template <typename t_fnt>
	struct vcf_reader::caller
	{
		template <t_fnt t_fn, typename ... t_args>
		static void handle_string_end(vcf_reader &r, t_args ... args)
		{
			std::string_view sv(r.m_start, r.m_fsm.p - r.m_start);
			(r.m_current_variant.*(t_fn))(sv, std::forward <t_args>(args)...);
		}

		template <t_fnt t_fn, typename ... t_args>
		static void handle_integer_end(vcf_reader &r, t_args ... args)
		{
			(r.m_current_variant.*(t_fn))(r.m_integer, std::forward <t_args>(args)...);
		}
	};

	template <int t_continue, int t_break>
	int vcf_reader::check_max_field(vcf_field const field, int const target, callback_fn const &cb)
	{
		if (field <= m_max_parsed_field)
			return target;

		skip_to_next_nl();
		if (!cb(m_current_variant))
			return t_break;

		return t_continue;
	}

	void vcf_reader::report_unexpected_character(char const *current_character, int const current_state)
	{
		std::cerr
		<< "Unexpected character '" << *current_character << "' at " << m_lineno << ':' << (current_character - m_line_start)
		<< ", state " << current_state << '.' << std::endl;

		std::cerr
		<< "** Buffer contents from the start:" << std::endl;
		std::copy(m_buffer.cbegin(), m_buffer.cend(), std::ostream_iterator <char>(std::cerr));

		std::string_view buffer_end(m_fsm.p, m_fsm.pe - m_fsm.p);
		std::cerr
		<< "** Buffer contents starting from m_fsm.p:" << std::endl
		<< buffer_end << std::endl;

		abort();
	}

	void vcf_reader::skip_to_next_nl()
	{
		std::string_view sv(m_fsm.p, m_fsm.pe - m_fsm.p);
		auto const pos(sv.find('\n'));
		always_assert(std::string_view::npos != pos, "Unable to find the next newline");
		m_fsm.p += pos;
	}

	// Seek to the beginning of the records.
	void vcf_reader::reset()
	{
		m_stream->clear();
		m_stream->seekg(m_first_variant_offset);
		m_lineno = m_last_header_lineno;
		m_len = 0;
		m_pos = 0;
	}

	// Return the 1-based number of the given sample.
	size_t vcf_reader::sample_no(std::string const &sample_name) const
	{
		auto const it(m_sample_names.find(sample_name));
		if (it == m_sample_names.cend())
			return 0;
		return it->second;
	}

	void vcf_reader::read_header()
	{
		// For now, just skip lines that begin with "##".
		std::string line;
		while (std::getline(*m_stream, line))
		{
			++m_lineno;
			if (! ('#' == line[0] && '#' == line[1]))
				break;

			// FIXME: header handling goes here.
		}

		// Check for column headers.
		{
			auto const prefix(std::string("#CHROM"));
			auto const res(mismatch(line.cbegin(), line.cend(), prefix.cbegin(), prefix.cend()));
			if (prefix.cend() != res.second)
			{
				std::cerr << "Expected the next line to start with '#CHROM', instead got: '" << line << "'" << std::endl;
				exit(1);
			}
		}

		// Read sample names.
		std::vector <std::string_view> header_names;
		detail::read_fields <true>(line, "\t", -1, header_names);
		std::size_t i(1);
		for (auto it(header_names.cbegin() + detail::to_underlying(vcf_field::ALL)), end(header_names.cend()); it != end; ++it)
		{
			std::string str(*it);
			auto const res(m_sample_names.emplace(std::move(str), i));
			always_assert(res.second, "Duplicate sample name");
			++i;
		}

		// stream now points to the first variant.
		m_first_variant_offset = m_stream->tellg();
		m_last_header_lineno = m_lineno;

		// Instantiate a variant.
		transient_variant var(sample_count());
		using std::swap;
		swap(m_current_variant, var);
	}

	void vcf_reader::fill_buffer()
	{
		// Copy the remainder to the beginning.
		if (m_pos + 1 < m_len)
		{
			char *data_start(m_buffer.data());
			char const *start(data_start + m_pos + 1);
			char const *end(data_start + m_len);
			std::copy(start, end, data_start);
			m_len -= m_pos + 1;
		}
		else
		{
			m_len = 0;
		}

		// Read until there's at least one newline in the buffer.
		while (true)
		{
			char *data_start(m_buffer.data());
			char *data(data_start + m_len);

			std::size_t space(m_buffer.size() - m_len);
			m_stream->read(data, space);
			std::size_t const read_len(m_stream->gcount());
			m_len += read_len;

			if (m_stream->eof())
			{
				m_pos = m_len;
				m_fsm.p = data_start;
				m_fsm.pe = data_start + m_len;
				m_fsm.eof = m_fsm.pe;
				return;
			}

			// Try to find the last newline in the new part.
			std::string_view sv(data, read_len);
			m_pos = sv.rfind('\n');
			if (std::string_view::npos != m_pos)
			{
				m_pos += (data - data_start);
				m_fsm.p = data_start;
				m_fsm.pe = m_fsm.p + m_pos + 1;
				return;
			}

			m_buffer.resize(2 * m_buffer.size());
		}
	}

	bool vcf_reader::parse(callback_fn &&callback)
	{
		return parse(callback);
	}

	bool vcf_reader::parse(callback_fn const &cb)
	{
		typedef variant_tpl <std::string_view> vc;
		bool retval(true);
		int cs(0);
		// FIXME: add throwing EOF actions?
		%%{
			variable p		m_fsm.p;
			variable pe		m_fsm.pe;
			variable eof	m_fsm.eof;

			action start_string {
				m_start = fpc;
			}

			action start_integer {
				m_integer = 0;
			}

			action update_integer {
				// Shift and add the current number.
				m_integer *= 10;
				m_integer += fc - '0';
			}

			action end_sample_field {
				// Check that the current field is followed by another field, another sample or a new record.
				switch (fc)
				{
					case ':':
						fgoto sample_rec_f;

					case '\t':
					{
						always_assert(m_format.size() == m_format_idx, "Not all fields present in the sample");
						m_format_idx = 0;
						fgoto sample_rec_f;
					}

					case '\n':
					{
						always_assert(m_format.size() == m_format_idx, "Not all fields present in the sample");

						if(!cb(m_current_variant))
						{
							fhold;
							fbreak;
						}

						fgoto main;
					}

					default:
						report_unexpected_character(fpc, fcurs);
				}
			}

			action error {
				report_unexpected_character(fpc, fcurs);
			}

			tab			= '\t';

			id_string	= '<' alnum+ '>';

			chrom_id	= (alnum+)
				>(start_string)
				%{ HANDLE_STRING_END(&vc::set_chrom_id); };

			pos			= (digit+)
				>(start_integer)
				$(update_integer)
				%{ HANDLE_INTEGER_END(&vc::set_pos); };

			id_part		= (([.] | alnum)+)
				>(start_string)
				%{ HANDLE_STRING_END(&vc::set_id, m_idx++); };
			id_rec		= (id_part (';' id_part)*) >{ m_idx = 0; };

			ref			= ([ACGTN]+)
				>(start_string)
				%{ HANDLE_STRING_END(&vc::set_ref); };

			# FIXME: add breakends.
			simple_alt	= ([ACGTN]+);
			complex_alt	= ([*]+) %{ m_alt_is_complex = true; };
			alt_part	= (simple_alt | complex_alt)
				>(start_string)
				%{ HANDLE_STRING_END(&vc::set_alt, m_idx++, m_alt_is_complex); };
			alt			= (alt_part (',' alt_part)*) >{ m_idx = 0; };

			qual		= (digit+)
				>(start_integer)
				$(update_integer)
				%{ HANDLE_INTEGER_END(&vc::set_qual); };

			# FIXME: add actions.
			filter_pass	= 'PASS';
			filter_part	= alnum+;
			filter		= (filter_pass | (filter_part (';' filter_part)*));

			# FIXME: add actions.
			info_key	= (alnum | '_') +;
			info_str	= (alnum | [_.]) +;
			info_val	= info_str (',' info_str)*;
			info_part	= info_key ('=' info_val)?;
			info		= (info_part (';' info_part)*);

			# FIXME: handle other formats.
			format_gt	= ('GT') @{ m_format.emplace_back(format_field::GT); };
			format_dp	= ('DP') @{ m_format.emplace_back(format_field::DP); };
			format_gq	= ('GQ') @{ m_format.emplace_back(format_field::GQ); };
			format_ps	= ('PS') @{ m_format.emplace_back(format_field::PS); };
			format_pq	= ('PQ') @{ m_format.emplace_back(format_field::PQ); };
			format_mq	= ('MQ') @{ m_format.emplace_back(format_field::MQ); };
			format_part	= format_gt | format_dp | format_gq | format_ps | format_pq | format_mq;
			format		= ((format_part (':' format_part)*) >{ m_format.clear(); });

			# Genotype field in sample.
			sample_gt_null_allele	= '.'
				%{
					m_integer = NULL_ALLELE;
					HANDLE_INTEGER_END(&vc::set_gt, m_sample_idx, m_idx++, m_gt_is_phased);
				};

			sample_gt_allele_idx	= (digit+)
				>(start_integer)
				$(update_integer)
				%{
					HANDLE_INTEGER_END(&vc::set_gt, m_sample_idx, m_idx++, m_gt_is_phased);
					m_gt_is_phased = false;
				};

			sample_gt_part			= (sample_gt_null_allele | sample_gt_allele_idx);
			sample_gt_p				= sample_gt_part ('|' ${ m_gt_is_phased = true; } sample_gt_part)+;
			sample_gt_up			= sample_gt_part ('/' sample_gt_part)+;
			sample_gt				= (sample_gt_part | sample_gt_p | sample_gt_up)
				>{
					m_idx = 0;
				};

			# FIXME: add actions.
			sample_dp		= (digit+);
			sample_gq		= (digit+);
			sample_ps		= (digit+);
			sample_pq		= (digit+);
			sample_mq		= (digit+);

			sep				= '\t';		# Field separator
			ssep			= [\t\n:];	# Sample separator

			# Handle a newline and continue.
			# The newline gets eaten before its checked, though, so use any instead.
			main_nl := any @{ fhold; fgoto main; };
			break_nl := any @{ fhold; fbreak; };

			# Line start.
			# Apparently main has to be able to read a character, so use fhold.
			# Allow EOF, though, with '?'.
			main := any?
				${
					fhold;

					m_idx = 0;
					m_format_idx = 0;
					m_sample_idx = 0;
					m_alt_is_complex = false;
					++m_lineno;
					m_current_variant.reset();
					m_current_variant.set_lineno(m_lineno);
					m_line_start = fpc;

					fgoto *check_max_field <fentry(main_nl), fentry(break_nl)>(vcf_field::CHROM, fentry(chrom_id_f), cb);
				}
				$err(error)
				$eof{ retval = false; };

			# #CHROM
			chrom_id_f :=
				(chrom_id sep)
				@{
					fgoto *check_max_field <fentry(main_nl), fentry(break_nl)>(vcf_field::POS, fentry(pos_f), cb);
				}
				$err(error);

			# POS
			pos_f :=
				(pos sep)
				@{
					fgoto *check_max_field <fentry(main_nl), fentry(break_nl)>(vcf_field::ID, fentry(id_f), cb);
				}
				$err(error);

			# ID
			id_f :=
				(id_rec sep)
				@{
					fgoto *check_max_field <fentry(main_nl), fentry(break_nl)>(vcf_field::REF, fentry(ref_f), cb);
				}
				$err(error);

			# REF
			ref_f :=
				(ref sep)
				@{
					fgoto *check_max_field <fentry(main_nl), fentry(break_nl)>(vcf_field::ALT, fentry(alt_f), cb);
				}
				$err(error);

			# ALT
			alt_f :=
				(alt sep)
				@{
					fgoto *check_max_field <fentry(main_nl), fentry(break_nl)>(vcf_field::QUAL, fentry(qual_f), cb);
				}
				$err(error);

			# QUAL
			qual_f :=
				(qual sep)
				@{
					fgoto *check_max_field <fentry(main_nl), fentry(break_nl)>(vcf_field::FILTER, fentry(filter_f), cb);
				}
				$err(error);

			# FILTER
			filter_f :=
				(filter sep)
				@{
					fgoto *check_max_field <fentry(main_nl), fentry(break_nl)>(vcf_field::INFO, fentry(info_f), cb);
				}
				$err(error);

			# INFO
			info_f :=
				(info sep)
				@{
					fgoto *check_max_field <fentry(main_nl), fentry(break_nl)>(vcf_field::FORMAT, fentry(format_f), cb);
				}
				$err(error);

			# FORMAT
			# FIXME: try to skip parsing the field and compare to the existing format?
			format_f :=
				(format sep)
				@{
					fgoto *check_max_field <fentry(main_nl), fentry(break_nl)>(vcf_field::ALL, fentry(sample_rec_f), cb);
				}
				$err(error);

			# Sample fields
			sample_gt_f := ((sample_gt) ssep @(end_sample_field)) $err(error);
			sample_dp_f := ((sample_dp) ssep @(end_sample_field)) $err(error);
			sample_gq_f := ((sample_gq) ssep @(end_sample_field)) $err(error);
			sample_ps_f := ((sample_ps) ssep @(end_sample_field)) $err(error);
			sample_pq_f := ((sample_pq) ssep @(end_sample_field)) $err(error);
			sample_mq_f := ((sample_mq) ssep @(end_sample_field)) $err(error);

			# Sample record
			sample_rec_f := "" >to{
				always_assert(m_format_idx < m_format.size(), "Format does not match the sample");

				// Sample index 0 is reserved for the reference.
				++m_sample_idx;

				// Parse according to the format field.
				auto const idx(m_format_idx);
				++m_format_idx;
				auto const field(m_format[idx]);
				switch (field)
				{
					case format_field::GT:
						fgoto sample_gt_f;

					case format_field::DP:
						fgoto sample_dp_f;

					case format_field::GQ:
						fgoto sample_gq_f;

					case format_field::PS:
						fgoto sample_ps_f;

					case format_field::PQ:
						fgoto sample_pq_f;

					case format_field::MQ:
						fgoto sample_mq_f;

					default:
						fail("Unexpected format value");
				}
			};

			dummy := any
				@{
					// Dummy action,
					// make sure that all the machines are instantiated.
					if (false)
					{
						fgoto chrom_id_f;
						fgoto pos_f;
						fgoto id_f;
						fgoto ref_f;
						fgoto alt_f;
						fgoto qual_f;
						fgoto filter_f;
						fgoto info_f;
						fgoto format_f;
						fgoto sample_rec_f;

						fgoto sample_gt_f;
						fgoto sample_dp_f;
						fgoto sample_gq_f;
						fgoto sample_ps_f;
						fgoto sample_pq_f;
						fgoto sample_mq_f;
					}
				};

			write init;
			write exec;
		}%%

		return retval;
	}
}

/*
 Copyright (c) 2017 Tuukka Norri
 This code is licensed under MIT license (see LICENSE for details).
 */

#include <vcf2multialign/util.hh>
#include <vcf2multialign/vcf_reader.hh>

%% machine vcf_parser;
%% write data;

// We would like to post-process strings that have been read and then pass
// them to the variant object with a simple function call at the call site.
// Solution: Templates can have member function pointers as parameters but of a specific type.
#define HANDLE_STRING_END(FN, ...)	caller <decltype(FN)>::handle_string_end <FN>(*this, ##__VA_ARGS__)
#define HANDLE_INTEGER_END(FN, ...)	caller <decltype(FN)>::handle_integer_end <FN>(*this, ##__VA_ARGS__)

namespace vcf2multialign { namespace detail {

	template <typename t_enum>
	constexpr typename std::underlying_type <t_enum>::type to_underlying(t_enum e)
	{
		return static_cast <typename std::underlying_type <t_enum>::type>(e);
	}

	// Split a t_string into string_views.
	template <bool t_emplace_back, typename t_string, typename t_invalid_pos>
	size_t read_fields(
		t_string const &string,
		t_invalid_pos const invalid_pos,
		char const *string_start,
		char const *sep,
		size_t const count,
		std::vector <std::string_view> &res
	)
	{
		char const *string_ptr(string_start);

		if (t_emplace_back)
			res.clear();
		else
			res.resize(count);

		typename t_string::size_type sep_pos(0);
		size_t i(0);
		bool should_break(false);
		while (i < count)
		{
			sep_pos = string.find(sep, sep_pos);
			if (invalid_pos == sep_pos)
			{
				should_break = true;
				sep_pos = string.size();
			}

			if (t_emplace_back)
				res.emplace_back(string_ptr, sep_pos - (string_ptr - string_start));
			else
			{
				std::string_view sv(string_ptr, sep_pos - (string_ptr - string_start));
				res[i] = std::move(sv);
			}

			++i;
			++sep_pos;
			string_ptr = string_start + sep_pos;

			if (should_break)
				break;
		}
		return i;
	}

	// Split a string into string_views.
	template <bool t_emplace_back>
	size_t read_fields(
		std::string const &string,
		char const *sep,
		size_t const count,
		std::vector <std::string_view> &res
	)
	{
		return read_fields <t_emplace_back>(string, std::string::npos, string.c_str(), sep, count, res);
	}

	template <bool t_emplace_back>
	size_t read_fields(
		std::string_view const &sv,
		char const *sep,
		size_t const count,
		std::vector <std::string_view> &res
	)
	{
		return read_fields <t_emplace_back>(sv, std::string_view::npos, static_cast <char const *>(sv.data()), sep, count, res);
	}
}}

namespace vcf2multialign {

	template <typename t_fnt>
	struct vcf_reader::caller
	{
		template <t_fnt t_fn, typename ... t_args>
		static void handle_string_end(vcf_reader &r, t_args ... args)
		{
			std::string_view sv(r.m_start, r.m_fsm.p - r.m_start);
			(r.m_current_variant.*(t_fn))(sv, std::forward <t_args>(args)...);
		}

		template <t_fnt t_fn, typename ... t_args>
		static void handle_integer_end(vcf_reader &r, t_args ... args)
		{
			(r.m_current_variant.*(t_fn))(r.m_integer, std::forward <t_args>(args)...);
		}
	};

	template <int t_continue, int t_break>
	int vcf_reader::check_max_field(vcf_field const field, int const target, callback_fn const &cb)
	{
		if (field <= m_max_parsed_field)
			return target;

		skip_to_next_nl();
		if (!cb(m_current_variant))
			return t_break;

		return t_continue;
	}

	void vcf_reader::report_unexpected_character(char const *current_character, int const current_state)
	{
		std::cerr
		<< "Unexpected character '" << *current_character << "' at " << m_lineno << ':' << (current_character - m_line_start)
		<< ", state " << current_state << '.' << std::endl;
		abort();
	}

	void vcf_reader::skip_to_next_nl()
	{
		std::string_view sv(m_fsm.p, m_fsm.pe - m_fsm.p);
		auto const pos(sv.find('\n'));
		always_assert(std::string_view::npos != pos, "Unable to find the next newline");
		m_fsm.p += pos;
	}

	// Seek to the beginning of the records.
	void vcf_reader::reset()
	{
		m_stream->clear();
		m_stream->seekg(m_first_variant_offset);
		m_lineno = m_last_header_lineno;
		m_len = 0;
		m_pos = 0;
	}

	// Return the 1-based number of the given sample.
	size_t vcf_reader::sample_no(std::string const &sample_name) const
	{
		auto const it(m_sample_names.find(sample_name));
		if (it == m_sample_names.cend())
			return 0;
		return it->second;
	}

	void vcf_reader::read_header()
	{
		// For now, just skip lines that begin with "##".
		std::string line;
		while (std::getline(*m_stream, line))
		{
			++m_lineno;
			if (! ('#' == line[0] && '#' == line[1]))
				break;

			// FIXME: header handling goes here.
		}

		// Check for column headers.
		{
			auto const prefix(std::string("#CHROM"));
			auto const res(mismatch(line.cbegin(), line.cend(), prefix.cbegin(), prefix.cend()));
			if (prefix.cend() != res.second)
			{
				std::cerr << "Expected the next line to start with '#CHROM', instead got: '" << line << "'" << std::endl;
				exit(1);
			}
		}

		// Read sample names.
		std::vector <std::string_view> header_names;
		detail::read_fields <true>(line, "\t", -1, header_names);
		std::size_t i(1);
		for (auto it(header_names.cbegin() + detail::to_underlying(vcf_field::ALL)), end(header_names.cend()); it != end; ++it)
		{
			std::string str(*it);
			auto const res(m_sample_names.emplace(std::move(str), i));
			always_assert(res.second, "Duplicate sample name");
			++i;
		}

		// stream now points to the first variant.
		m_first_variant_offset = m_stream->tellg();
		m_last_header_lineno = m_lineno;

		// Instantiate a variant.
		transient_variant var(sample_count());
		using std::swap;
		swap(m_current_variant, var);
	}

	void vcf_reader::fill_buffer()
	{
		// Copy the remainder to the beginning.
		if (m_pos + 1 < m_len)
		{
			char *data_start(m_buffer.data());
			char const *start(data_start + m_pos + 1);
			char const *end(data_start + m_len);
			std::copy(start, end, data_start);
			m_len -= m_pos + 1;
		}
		else
		{
			m_len = 0;
		}

		// Read until there's at least one newline in the buffer.
		while (true)
		{
			char *data_start(m_buffer.data());
			char *data(data_start + m_len);

			std::size_t space(m_buffer.size() - m_len);
			m_stream->read(data, space);
			std::size_t const read_len(m_stream->gcount());
			m_len += read_len;

			if (m_stream->eof())
			{
				m_pos = m_len;
				m_fsm.p = data_start;
				m_fsm.pe = data_start + m_len;
				m_fsm.eof = m_fsm.pe;
				return;
			}

			// Try to find the last newline in the new part.
			std::string_view sv(data, read_len);
			m_pos = sv.rfind('\n');
			if (std::string_view::npos != m_pos)
			{
				m_pos += (data - data_start);
				m_fsm.p = data_start;
				m_fsm.pe = m_fsm.p + m_pos + 1;
				return;
			}

			m_buffer.resize(2 * m_buffer.size());
		}
	}

	bool vcf_reader::parse(callback_fn &&callback)
	{
		return parse(callback);
	}

	bool vcf_reader::parse(callback_fn const &cb)
	{
		typedef variant_tpl <std::string_view> vc;
		bool retval(true);
		int cs(0);
		// FIXME: add throwing EOF actions?
		%%{
			variable p		m_fsm.p;
			variable pe		m_fsm.pe;
			variable eof	m_fsm.eof;

			action start_string {
				m_start = fpc;
			}

			action start_integer {
				m_integer = 0;
			}

			action update_integer {
				// Shift and add the current number.
				m_integer *= 10;
				m_integer += fc - '0';
			}

			action end_sample_field {
				// Check that the current field is followed by another field, another sample or a new record.
				switch (fc)
				{
					case ':':
						fgoto sample_rec_f;

					case '\t':
					{
						always_assert(m_format.size() == m_format_idx, "Not all fields present in the sample");
						m_format_idx = 0;
						fgoto sample_rec_f;
					}

					case '\n':
					{
						always_assert(m_format.size() == m_format_idx, "Not all fields present in the sample");

						if(!cb(m_current_variant))
						{
							fhold;
							fbreak;
						}

						fgoto main;
					}

					default:
						report_unexpected_character(fpc, fcurs);
				}
			}

			action error {
				report_unexpected_character(fpc, fcurs);
			}

			tab			= '\t';

			id_string	= '<' alnum+ '>';

			chrom_id	= (alnum+)
				>(start_string)
				%{ HANDLE_STRING_END(&vc::set_chrom_id); };

			pos			= (digit+)
				>(start_integer)
				$(update_integer)
				%{ HANDLE_INTEGER_END(&vc::set_pos); };

			id_part		= (([.] | alnum)+)
				>(start_string)
				%{ HANDLE_STRING_END(&vc::set_id, m_idx++); };
			id_rec		= (id_part (';' id_part)*) >{ m_idx = 0; };

			ref			= ([ACGTN]+)
				>(start_string)
				%{ HANDLE_STRING_END(&vc::set_ref); };

			# FIXME: add breakends.
			simple_alt	= ([ACGTN]+);
			complex_alt	= ([*]+) %{ m_alt_is_complex = true; };
			alt_part	= (simple_alt | complex_alt)
				>(start_string)
				%{ HANDLE_STRING_END(&vc::set_alt, m_idx++, m_alt_is_complex); };
			alt			= (alt_part (',' alt_part)*) >{ m_idx = 0; };

			qual		= (digit+)
				>(start_integer)
				$(update_integer)
				%{ HANDLE_INTEGER_END(&vc::set_qual); };

			# FIXME: add actions.
			filter_pass	= 'PASS';
			filter_part	= alnum+;
			filter		= (filter_pass | (filter_part (';' filter_part)*));

			# FIXME: add actions.
			info_key	= (alnum | '_') +;
			info_str	= (alnum | [_.]) +;
			info_val	= info_str (',' info_str)*;
			info_part	= info_key ('=' info_val)?;
			info		= (info_part (';' info_part)*);

			# FIXME: handle other formats.
			format_gt	= ('GT') @{ m_format.emplace_back(format_field::GT); };
			format_dp	= ('DP') @{ m_format.emplace_back(format_field::DP); };
			format_gq	= ('GQ') @{ m_format.emplace_back(format_field::GQ); };
			format_ps	= ('PS') @{ m_format.emplace_back(format_field::PS); };
			format_pq	= ('PQ') @{ m_format.emplace_back(format_field::PQ); };
			format_mq	= ('MQ') @{ m_format.emplace_back(format_field::MQ); };
			format_part	= format_gt | format_dp | format_gq | format_ps | format_pq | format_mq;
			format		= ((format_part (':' format_part)*) >{ m_format.clear(); });

			# Genotype field in sample.
			sample_gt_null_allele	= '.'
				%{
					m_integer = NULL_ALLELE;
					HANDLE_INTEGER_END(&vc::set_gt, m_sample_idx, m_idx++, m_gt_is_phased);
				};

			sample_gt_allele_idx	= (digit+)
				>(start_integer)
				$(update_integer)
				%{
					HANDLE_INTEGER_END(&vc::set_gt, m_sample_idx, m_idx++, m_gt_is_phased);
					m_gt_is_phased = false;
				};

			sample_gt_part			= (sample_gt_null_allele | sample_gt_allele_idx);
			sample_gt_p				= sample_gt_part ('|' ${ m_gt_is_phased = true; } sample_gt_part)+;
			sample_gt_up			= sample_gt_part ('/' sample_gt_part)+;
			sample_gt				= (sample_gt_part | sample_gt_p | sample_gt_up)
				>{
					m_idx = 0;
				};

			# FIXME: add actions.
			sample_dp		= (digit+);
			sample_gq		= (digit+);
			sample_ps		= (digit+);
			sample_pq		= (digit+);
			sample_mq		= (digit+);

			sep				= '\t';		# Field separator
			ssep			= [\t\n:];	# Sample separator

			# Handle a newline and continue.
			# The newline gets eaten before its checked, though, so use any instead.
			main_nl := any @{ fhold; fgoto main; };
			break_nl := any @{ fhold; fbreak; };

			# Line start.
			# Apparently main has to be able to read a character, so use fhold.
			# Allow EOF, though, with '?'.
			main := any?
				${
					fhold;

					m_idx = 0;
					m_format_idx = 0;
					m_sample_idx = 0;
					m_alt_is_complex = false;
					++m_lineno;
					m_current_variant.reset();
					m_current_variant.set_lineno(m_lineno);
					m_line_start = fpc;

					fgoto *check_max_field <fentry(main_nl), fentry(break_nl)>(vcf_field::CHROM, fentry(chrom_id_f), cb);
				}
				$err(error)
				$eof{ retval = false; };

			# #CHROM
			chrom_id_f :=
				(chrom_id sep)
				@{
					fgoto *check_max_field <fentry(main_nl), fentry(break_nl)>(vcf_field::POS, fentry(pos_f), cb);
				}
				$err(error);

			# POS
			pos_f :=
				(pos sep)
				@{
					fgoto *check_max_field <fentry(main_nl), fentry(break_nl)>(vcf_field::ID, fentry(id_f), cb);
				}
				$err(error);

			# ID
			id_f :=
				(id_rec sep)
				@{
					fgoto *check_max_field <fentry(main_nl), fentry(break_nl)>(vcf_field::REF, fentry(ref_f), cb);
				}
				$err(error);

			# REF
			ref_f :=
				(ref sep)
				@{
					fgoto *check_max_field <fentry(main_nl), fentry(break_nl)>(vcf_field::ALT, fentry(alt_f), cb);
				}
				$err(error);

			# ALT
			alt_f :=
				(alt sep)
				@{
					fgoto *check_max_field <fentry(main_nl), fentry(break_nl)>(vcf_field::QUAL, fentry(qual_f), cb);
				}
				$err(error);

			# QUAL
			qual_f :=
				(qual sep)
				@{
					fgoto *check_max_field <fentry(main_nl), fentry(break_nl)>(vcf_field::FILTER, fentry(filter_f), cb);
				}
				$err(error);

			# FILTER
			filter_f :=
				(filter sep)
				@{
					fgoto *check_max_field <fentry(main_nl), fentry(break_nl)>(vcf_field::INFO, fentry(info_f), cb);
				}
				$err(error);

			# INFO
			info_f :=
				(info sep)
				@{
					fgoto *check_max_field <fentry(main_nl), fentry(break_nl)>(vcf_field::FORMAT, fentry(format_f), cb);
				}
				$err(error);

			# FORMAT
			# FIXME: try to skip parsing the field and compare to the existing format?
			format_f :=
				(format sep)
				@{
					fgoto *check_max_field <fentry(main_nl), fentry(break_nl)>(vcf_field::ALL, fentry(sample_rec_f), cb);
				}
				$err(error);

			# Sample fields
			sample_gt_f := ((sample_gt) ssep @(end_sample_field)) $err(error);
			sample_dp_f := ((sample_dp) ssep @(end_sample_field)) $err(error);
			sample_gq_f := ((sample_gq) ssep @(end_sample_field)) $err(error);
			sample_ps_f := ((sample_ps) ssep @(end_sample_field)) $err(error);
			sample_pq_f := ((sample_pq) ssep @(end_sample_field)) $err(error);
			sample_mq_f := ((sample_mq) ssep @(end_sample_field)) $err(error);

			# Sample record
			sample_rec_f := "" >to{
				always_assert(m_format_idx < m_format.size(), "Format does not match the sample");

				// Sample index 0 is reserved for the reference.
				++m_sample_idx;

				// Parse according to the format field.
				auto const idx(m_format_idx);
				++m_format_idx;
				auto const field(m_format[idx]);
				switch (field)
				{
					case format_field::GT:
						fgoto sample_gt_f;

					case format_field::DP:
						fgoto sample_dp_f;

					case format_field::GQ:
						fgoto sample_gq_f;

					case format_field::PS:
						fgoto sample_ps_f;

					case format_field::PQ:
						fgoto sample_pq_f;

					case format_field::MQ:
						fgoto sample_mq_f;

					default:
						fail("Unexpected format value");
				}
			};

			dummy := any
				@{
					// Dummy action,
					// make sure that all the machines are instantiated.
					if (false)
					{
						fgoto chrom_id_f;
						fgoto pos_f;
						fgoto id_f;
						fgoto ref_f;
						fgoto alt_f;
						fgoto qual_f;
						fgoto filter_f;
						fgoto info_f;
						fgoto format_f;
						fgoto sample_rec_f;

						fgoto sample_gt_f;
						fgoto sample_dp_f;
						fgoto sample_gq_f;
						fgoto sample_ps_f;
						fgoto sample_pq_f;
						fgoto sample_mq_f;
					}
				};

			write init;
			write exec;
		}%%

		return retval;
	}
}


Successfully extract static text and tags

However static text is duped

#import "MustacheParserMachine.h"
#import <stdio.h>
#import <assert.h>
#import <stdlib.h>
#import <ctype.h>
#import <string.h>

#define LEN(AT, FPC) (FPC - buffer - AT)
#define MARK(M,FPC) (M = (FPC) - buffer)
#define PTR_TO(F) (buffer + F)

/** Machine **/
%%{

	machine mustache_parser;

	# Line endings
	newline = ( "\r" | "\r\n" | "\n" );

	# Character classes
	white = ( " " | "\t" )*;
	open  = "{{";
	close = "}}";

	# Action
    # # After these types of tags, all whitespace will be skipped.
    # SKIP_WHITESPACE = [ '#', '^', '/' ]
    #
    # # The content allowed in a tag name.
    # ALLOWED_CONTENT = /(\w|[?!\/-])*/
    #
    # # These types of tags allow any content,
    # # the rest only allow ALLOWED_CONTENT.
    # ANY_CONTENT = [ '!', '=' ]

	identifier = ( alnum | [?!/] | '-')*;
	# [a-zA-Z_0-9?!/]*;
	# /(\w|[?!\/-])*/;

	# Tags
	#type  = [#^/=<>&{];

	# var     = ( open white identifier white close );
	#
	# enum    = ( open '#' white identifier white close );
	#
	# inverted = ( open '^' white identifier white close );
	#
	# end_enum = ( open '/' white identifier white close );
	#
	# comment = ( open '!' (any* -- close) close );

	#body = ( comment | var | enum | nl );

	action mark { MARK(mark, fpc); }

	action start_identifier {
		MARK(identifier_start, fpc);
	}

	action got_identifier {
		// NSLog(@"mark %d, len: %d\n", mark, LEN(mark, fpc));

		printf("Tag: ");
		fwrite(PTR_TO(identifier_start), sizeof(char), LEN(identifier_start, fpc), stdout);
		printf("\n");
	}

	action write_static {
		printf("Static text: ");
		fwrite(ts, sizeof(char), te - ts, stdout);
		printf("\n");
	}

	action start_tag {
		// Write out all the static text up to this tag
		printf("Static text: ");
		fwrite(PTR_TO(mark), sizeof(char), LEN(mark, fpc), stdout);
		printf("\n");
	}

	#tag_type = [&/<>{^];

	tag = (
		open >start_tag
		white
		identifier >start_identifier %got_identifier
		white
		close %mark
	);

	text = (any -- open)+;

	main := ( tag | text )*;

}%%

/** Data **/
%% write data;

@implementation MustacheParserMachine

//int http_parser_init(http_parser *parser)  {
- (id)initWithDelegate:(id)parser_delegate
{
	if((self = [super init]) != nil)
	{
		mark = 0; // Mark the start of the text
		%% write init;
		//delegate = [parser_delegate retain];
	}

	return self;
}

/** exec **/
//size_t http_parser_execute(http_parser *parser, const char *buffer, size_t len, size_t off)  {
- (size_t)execute:(const char *)buffer length:(size_t)len offset:(size_t)off
{
  const char *p, *pe, *eof;
  //int cs = parser->cs;

  assert(off <= len && "offset past end of buffer");

  p = buffer+off;
  pe = buffer+len;
  eof = pe;

  assert(*pe == '\0' && "pointer does not end on NUL");
  assert(pe - p == len - off && "pointers aren't same distance");

  %% write exec;

  //parser->cs = cs;
  nread += p - (buffer + off);
//
//  assert(p <= pe && "buffer overflow after parsing execute");
  assert(nread <= len && "nread longer than length");
//  assert(body_start <= len && "body starts after buffer end");
//  assert(mark < len && "mark is after buffer end");
//  assert(field_len <= len && "field has length longer than whole buffer");
//  assert(field_start < len && "field starts after buffer end");

//  if(body_start) {
//    /* final \r\n combo encountered so stop right here */
//    //%%write eof;
//    nread++;
//  }

  //return nread;
  return 0;
}

- (int)finish {
 if ([self hasError]) {
   return -1;
 } else if ([self isFinished]) {
   return 1;
 } else {
   return 0;
 }
}

- (BOOL)hasError
{
 return cs == mustache_parser_error ? YES : NO;
}

- (BOOL)isFinished
{
 return cs == mustache_parser_first_final ? YES : NO;
}

- (size_t)bytesRead
{
	return nread;
}

//- (void)dealloc
//{
//	[delegate release];
//	[super dealloc];
//}

@end

#import "MustacheParserMachine.h"
#import <stdio.h>
#import <assert.h>
#import <stdlib.h>
#import <ctype.h>
#import <string.h>

#define LEN(AT, FPC) (FPC - buffer - AT)
#define MARK(M,FPC) (M = (FPC) - buffer)
#define PTR_TO(F) (buffer + F)

/** Machine **/
%%{

	machine mustache_parser;

	# Line endings
	newline = ( "\r" | "\r\n" | "\n" );

	# Character classes
	white = ( " " | "\t" )*;
	open  = "{{";
	close = "}}";

	# Action
    # # After these types of tags, all whitespace will be skipped.
    # SKIP_WHITESPACE = [ '#', '^', '/' ]
    #
    # # The content allowed in a tag name.
    # ALLOWED_CONTENT = /(\w|[?!\/-])*/
    #
    # # These types of tags allow any content,
    # # the rest only allow ALLOWED_CONTENT.
    # ANY_CONTENT = [ '!', '=' ]

	identifier = ( alnum | [?!/] | '-')*;
	# [a-zA-Z_0-9?!/]*;
	# /(\w|[?!\/-])*/;

	# Tags
	#type  = [#^/=<>&{];

	# var     = ( open white identifier white close );
	#
	# enum    = ( open '#' white identifier white close );
	#
	# inverted = ( open '^' white identifier white close );
	#
	# end_enum = ( open '/' white identifier white close );
	#
	# comment = ( open '!' (any* -- close) close );

	#body = ( comment | var | enum | nl );

	action mark { MARK(mark, fpc); }

	action start_identifier {
		MARK(identifier_start, fpc);
	}

	action got_identifier {
		// NSLog(@"mark %d, len: %d\n", mark, LEN(mark, fpc));

		printf("Tag: ");
		fwrite(PTR_TO(identifier_start), sizeof(char), LEN(identifier_start, fpc), stdout);
		printf("\n");
	}

	action write_static {
		printf("Static text: ");
		fwrite(ts, sizeof(char), te - ts, stdout);
		printf("\n");
	}

	tag_type = [&/<>{^];

	tag = ( open white >start_identifier identifier %got_identifier white close );

	text = (any -- open)+;

	main := |*
		tag;
		text => write_static;
	*|;

	# Actions

		# action mark {MARK(mark, fpc); }

}%%

/** Data **/
%% write data;

@implementation MustacheParserMachine

//int http_parser_init(http_parser *parser)  {
- (id)initWithDelegate:(id)parser_delegate
{
	if((self = [super init]) != nil)
	{

		%% write init;
		//delegate = [parser_delegate retain];
	}

	return self;
}

/** exec **/
//size_t http_parser_execute(http_parser *parser, const char *buffer, size_t len, size_t off)  {
- (size_t)execute:(const char *)buffer length:(size_t)len offset:(size_t)off
{
  const char *p, *pe, *eof;
  //int cs = parser->cs;

  assert(off <= len && "offset past end of buffer");

  p = buffer+off;
  pe = buffer+len;
  eof = pe;

  assert(*pe == '\0' && "pointer does not end on NUL");
  assert(pe - p == len - off && "pointers aren't same distance");

  %% write exec;

  //parser->cs = cs;
  nread += p - (buffer + off);
//
//  assert(p <= pe && "buffer overflow after parsing execute");
  assert(nread <= len && "nread longer than length");
//  assert(body_start <= len && "body starts after buffer end");
//  assert(mark < len && "mark is after buffer end");
//  assert(field_len <= len && "field has length longer than whole buffer");
//  assert(field_start < len && "field starts after buffer end");

//  if(body_start) {
//    /* final \r\n combo encountered so stop right here */
//    //%%write eof;
//    nread++;
//  }

  //return nread;
  return 0;
}

- (int)finish {
 if ([self hasError]) {
   return -1;
 } else if ([self isFinished]) {
   return 1;
 } else {
   return 0;
 }
}

- (BOOL)hasError
{
 return cs == mustache_parser_error ? YES : NO;
}

- (BOOL)isFinished
{
 return cs == mustache_parser_first_final ? YES : NO;
}

- (size_t)bytesRead
{
	return nread;
}

//- (void)dealloc
//{
//	[delegate release];
//	[super dealloc];
//}

@end