make this compatible with other ragel -On levels

Signed-off-by: JT Archie <4f9753a5161b47f990c4f321986d40dee5fdf803@gmail.com>

package syslog

import (
  "fmt"
  "time"
)

%%{
  machine syslog_rfc5424;
  write data;
}%%

func bytesRef(a []byte) []byte {
  if len(a) == 1 && a[0] == '-' {
    return nil
  }
  return a
}

func atoi(a []byte) int {
  var x int
  for _, c := range a {
    x = x * 10 + int(c - '0')
  }
  return x
}

func atoi2(a []byte) int {
  return int(a[1] - '0') + int(a[0] - '0') * 10
}

func atoi4(a []byte) int {
  return int(a[3] - '0') +
  int(a[2] - '0') * 10 +
  int(a[1] - '0') * 100 +
  int(a[0] - '0') * 1000
}

func Parse(data []byte) (*Log, error) {
  var (
    paramName []byte
    nanosecond int
  )

  log := &Log{}
  var location *time.Location

  // set defaults for state machine parsing
  cs, p, pe, eof := 0, 0, len(data), len(data)

  // use to keep track start of value
  mark := 0

  // taken directly from https://tools.ietf.org/html/rfc5424#page-8
  %%{
    action mark      { mark = p }
    action version   { log.version = atoi(data[mark:p]) }
    action priority  { log.priority = atoi(data[mark:p]) }
    action hostname  { log.hostname = bytesRef(data[mark:p]) }
    action appname   { log.appname = bytesRef(data[mark:p]) }
    action procid    { log.procID = bytesRef(data[mark:p]) }
    action msgid     { log.msgID = bytesRef(data[mark:p]) }
    action sdid      {
      log.data = &structureData{
        id: data[mark:p],
        properties: make([]Property, 0, 5),
      }
    }
    action paramname  { paramName = data[mark:p] }
    action paramvalue { log.data.properties = append(log.data.properties, Property{paramName,data[mark:p]}) }

    action timestamp {
      location = time.UTC
      if data[mark+19] == '.' {
        offset := 1

        if data[p-1] != 'Z' {
          offset = 6
          dir := 1
          if data[p-6] == '-' {
            dir = -1
          }

          location = time.FixedZone(
            "",
            dir * (atoi2(data[p-5:p-3]) * 3600 + atoi(data[p-2:p]) * 60),
          )
        }
        nbytes := ( p - offset - 1 ) - ( mark + 19 )
        for i := mark + 20; i < p-offset; i++ {
          nanosecond = nanosecond*10 + int(data[i]-'0')
        }
        for i := 0; i < 9-nbytes; i++ {
          nanosecond *= 10
        }
      }

      log.timestamp = time.Date(
        atoi4(data[mark:mark+4]),
        time.Month(atoi2(data[mark+5:mark+7])),
        atoi2(data[mark+8:mark+10]),
        atoi2(data[mark+11:mark+13]),
        atoi2(data[mark+14:mark+16]),
        atoi2(data[mark+17:mark+19]),
        nanosecond,
        location,
      ).UTC()
    }
    action message { log.message = data[mark:p] }

    include syslog_rfc5424 "syslog.rl";
    write init;
    write exec;
  }%%

  if cs < syslog_rfc5424_first_final {
    return nil, fmt.Errorf("error in msg at pos %d: %s", p, data)
  }

  return log, nil
}

package syslog

import (
  "fmt"
  "time"
)

%%{
  machine syslog_rfc5424;
  write data;
}%%

func bytesRef(a []byte) []byte {
  if len(a) == 1 && a[0] == '-' {
    return nil
  }
  return a
}

func atoi(a []byte) int {
  var x int
  for _, c := range a {
    x = x * 10 + int(c - '0')
  }
  return x
}

func atoi2(a []byte) int {
  return int(a[1] - '0') + int(a[0] - '0') * 10
}

func atoi4(a []byte) int {
  return int(a[3] - '0') +
  int(a[2] - '0') * 10 +
  int(a[1] - '0') * 100 +
  int(a[0] - '0') * 1000
}

func Parse(data []byte) (*Log, error) {
  var (
    paramName []byte
    nanosecond int
  )

  log := &Log{}

  // set defaults for state machine parsing
  cs, p, pe, eof := 0, 0, len(data), len(data)

  // use to keep track start of value
  mark := 0

  // taken directly from https://tools.ietf.org/html/rfc5424#page-8
  %%{
    action mark      { mark = p }
    action version   { log.version = atoi(data[mark:p]) }
    action priority  { log.priority = atoi(data[mark:p]) }
    action hostname  { log.hostname = bytesRef(data[mark:p]) }
    action appname   { log.appname = bytesRef(data[mark:p]) }
    action procid    { log.procID = bytesRef(data[mark:p]) }
    action msgid     { log.msgID = bytesRef(data[mark:p]) }
    action sdid      {
      log.data = &structureData{
        id: data[mark:p],
        properties: make([]Property, 0, 5),
      }
    }
    action paramname  { paramName = data[mark:p] }
    action paramvalue { log.data.properties = append(log.data.properties, Property{paramName,data[mark:p]}) }

    action timestamp {
      location := time.UTC
      if data[mark+19] == '.' {
        offset := 1

        if data[p-1] != 'Z' {
          offset = 6
          dir := 1
          if data[p-6] == '-' {
            dir = -1
          }

          location = time.FixedZone(
            "",
            dir * (atoi2(data[p-5:p-3]) * 3600 + atoi(data[p-2:p]) * 60),
          )
        }
        nbytes := ( p - offset - 1 ) - ( mark + 19 )
        for i := mark + 20; i < p-offset; i++ {
          nanosecond = nanosecond*10 + int(data[i]-'0')
        }
        for i := 0; i < 9-nbytes; i++ {
          nanosecond *= 10
        }
      }

      log.timestamp = time.Date(
        atoi4(data[mark:mark+4]),
        time.Month(atoi2(data[mark+5:mark+7])),
        atoi2(data[mark+8:mark+10]),
        atoi2(data[mark+11:mark+13]),
        atoi2(data[mark+14:mark+16]),
        atoi2(data[mark+17:mark+19]),
        nanosecond,
        location,
      ).UTC()
    }
    action message { log.message = data[mark:p] }

    include syslog_rfc5424 "syslog.rl";
    write init;
    write exec;
  }%%

  if cs < syslog_rfc5424_first_final {
    return nil, fmt.Errorf("error in msg at pos %d: %s", p, data)
  }

  return log, nil
}


GCNodeParser cleanup

GCNodeParser cleanup

/**

Ragel state machine for GEDCOM nodes based on the 5.5 documentation.

gedcom_line :=
level + delim + [xref_id + delim +] tag + [delim + line_value +] terminator
level + delim + optional_xref_id + tag + delim + optional_line_value + terminator

*/

#import "GCNodeParser.h"
#import "GCNodeParserDelegate.h"

#import "GCNode.h"

#import "GedcomErrors.h"

#define DEBUGLINECOUNT 84300

%%{
    machine nodeParser;

	action log {
		//if (lineCount > DEBUGLINECOUNT) NSLog(@"%p log: (%X) '%c'", fpc, fc, fc);
	}

	action tag {
		tag = fpc - data;
        //if (lineCount > DEBUGLINECOUNT) NSLog(@"%p     TAG: %ld", fpc, tag);
	}

	action number {
		long len = (fpc - data) - tag;
		currentNumber = [[[NSString alloc] initWithBytes:fpc-len length:len encoding:NSUTF8StringEncoding] intValue];
		//if (lineCount > DEBUGLINECOUNT) NSLog(@"%p num: %d", fpc, currentNumber);
	}

	action string {
		long len = (fpc - data) - tag;
		currentString = [[NSString alloc] initWithBytes:fpc-len length:len encoding:NSUTF8StringEncoding];
		//if (lineCount > DEBUGLINECOUNT) NSLog(@"%p string: %@", fpc, currentString);
	}

	action saveLevel {
        level = currentNumber;
	}

    action saveXref {
        xref = currentString;
    }

    action saveTagCode {
        code = currentString;
    }

    action saveValue {
        value = [currentString stringByReplacingOccurrencesOfString:@"@@" withString:@"@"]; // unescape at-sign
    }

	action saveNode {
        lineCount++;

        // CONT/CONC nodes are continuations of values from the previous node, so fold them up here:
        // TODO: refactor (use dict?)
        if ([code isEqualToString:@"CONT"]) {
            if (!currentNode.gedValue) {
                currentNode.gedValue = value;
            } else {
                currentNode.gedValue = [NSString stringWithFormat:@"%@%@%@", currentNode.gedValue, @"\n", value ? value : @""];
            }
        } else if ([code isEqualToString:@"CONC"]) {
            if (!currentNode.gedValue) {
                currentNode.gedValue = value;
            } else {
                currentNode.gedValue = [NSString stringWithFormat:@"%@%@%@", currentNode.gedValue, concSeparator, value ? value : @""];
            }
        } else {
            if (xref) {
                node = [GCNode nodeWithTag:code
                                      xref:xref];
            } else {
                node = [GCNode nodeWithTag:code
                                     value:value];
            }

            GCNode *parent = nil;

            if (level == 0) { //root
                if (currentRoot) {
                    nodeCount++;
                    if (_delegate && [_delegate respondsToSelector:@selector(parser:didParseNode:)]) {
                        [_delegate parser:self didParseNode:currentRoot];
                    }
                }
            } else { //find correct parent
                parent = currentNode;
                for (int i = currentLevel; i >= level; i--) {
                    parent = [parent parent];
                }
            }

            if (parent) {
                [parent.mutableSubNodes addObject:node];
            } else {
                currentRoot = node;
            }

            currentLevel = level;
            currentNode = node;
        }

        level = -1;
        xref = nil;
        code = nil;
        value = nil;
	}

	action finish {
        if (currentRoot) {
            nodeCount++;
            if (_delegate && [_delegate respondsToSelector:@selector(parser:didParseNode:)]) {
                [_delegate parser:self didParseNode:currentRoot];
            }
        }
		didFinish = YES;
	}

    delim                   = ' ';
    terminator              = ( '\r' | '\n' )+;

    keyword                 = ( alpha ( alnum | '_' )+ );

    level                   = ( digit+ >tag %number ) %saveLevel;
    xref                    = ( ( '@' keyword '@' ) >tag %string ) %saveXref;
    tagCode                 = ( ( '_'? keyword ) >tag %string ) %saveTagCode;
    value                   = ( ( extend - terminator )+ >tag %string ) %saveValue;

    contents                = ( ( xref delim tagCode ) | ( tagCode delim value ) | ( tagCode delim? ) );

    node                    = ( level delim contents terminator ) %saveNode;

    main                   := ( node+ ) %/finish;

}%%

%% write data;

@implementation GCNodeParser

__strong static id _sharedNodeParser = nil;

+ (void)initialize
{
    _sharedNodeParser = [[self alloc] init];
}

+ (id)sharedNodeParser
{
    return _sharedNodeParser;
}

- (BOOL)parseString:(id)gedString error:(NSError *__autoreleasing *)error
{
    @synchronized(self) {
        if ([gedString isKindOfClass:[NSAttributedString class]]) {
            gedString = [gedString string];
        }

        NSParameterAssert([gedString isKindOfClass:[NSString class]]);

        gedString = [gedString stringByAppendingString:@"\n"];

        NSUInteger nodeCount = 0;
        NSUInteger lineCount = 0;
        int currentLevel = 0;
        GCNode *currentNode = nil;
        GCNode *currentRoot = nil;

        long tag = 0;
        int currentNumber = 0;
        NSString *currentString = nil;

        int level = -1;
        NSString *xref = nil;
        NSString *code = nil;
        NSString *value = nil;
        GCNode* node = nil;

        BOOL didFinish = NO;

        int cs = 0;
        const char *data = [gedString UTF8String];
        const char *p = data;
        const char *pe = p + [gedString lengthOfBytesUsingEncoding:NSUTF8StringEncoding];
        const char *eof = pe;

#ifdef DEBUGLEVEL
        clock_t start, end;
        double elapsed;
        start = clock();
        //NSLog(@"Began parsing gedcom.");
#endif

        %% write init;
        %% write exec;

#ifdef DEBUGLEVEL
        end = clock();
        elapsed = ((double) (end - start)) / CLOCKS_PER_SEC;
        NSLog(@"parsed %ld nodes in %ld lines - Time: %f seconds", nodeCount, lineCount, elapsed);
#endif

        if (_delegate && [_delegate respondsToSelector:@selector(parser:didParseNodesWithCount:)]) {
            [_delegate parser:self didParseNodesWithCount:nodeCount];
        }

        if (!didFinish) {
            if (error != NULL) {
                *error = [NSError errorWithDomain:GCErrorDomain
                                             code:GCNodeParsingError
                                         userInfo:@{NSLocalizedDescriptionKey: [NSString stringWithFormat:@"Parser stopped at node #%ld (line %ld)", nodeCount, lineCount], NSAffectedObjectsErrorKey: currentNode}];
            }
        }

        return didFinish;
    }
}

@end

@interface _GCNodeParserConvenienceDelegate : NSObject <GCNodeParserDelegate>

@property NSArray *nodes;

@end

@implementation _GCNodeParserConvenienceDelegate {
    NSMutableArray *_nodes;
}

- (void)parser:(GCNodeParser *)parser didParseNode:(GCNode *)node
{
    //NSLog(@"%p got node: %@", self, node);
    if (!_nodes) {
        _nodes = [NSMutableArray array];
    }
    [_nodes addObject:node];
}

- (void)parser:(GCNodeParser *)parser didParseNodesWithCount:(NSUInteger)nodeCount
{
    NSParameterAssert(nodeCount == [_nodes count]);
}

@end

@implementation GCNodeParser (GCConvenienceMethods)

+ (NSArray*)arrayOfNodesFromString:(id)gedString
{
    _GCNodeParserConvenienceDelegate *delegate = [[_GCNodeParserConvenienceDelegate alloc] init];

    GCNodeParser *parser = [[GCNodeParser alloc] init];
    parser.delegate = delegate;

    NSError *error = nil;
    BOOL succeeded = [parser parseString:gedString error:&error];

    if (succeeded) {
        return delegate.nodes;
    } else {
        NSLog(@"error: %@", error);
        return nil;
    }
}

@end

/**

Ragel state machine for GEDCOM nodes based on the 5.5 documentation.

gedcom_line :=
level + delim + [xref_id + delim +] tag + [delim + line_value +] terminator
level + delim + optional_xref_id + tag + delim + optional_line_value + terminator

*/

#import "GCNodeParser.h"
#import "GCNodeParserDelegate.h"

#import "GCNode.h"

#import "GedcomErrors.h"

#define DEBUGLINECOUNT 84300

%%{
    machine nodeParser;

	action log {
		//if (lineCount > DEBUGLINECOUNT) NSLog(@"%p log: (%X) '%c'", fpc, fc, fc);
	}

	action tag {
		tag = fpc - data;
        //if (lineCount > DEBUGLINECOUNT) NSLog(@"%p     TAG: %ld", fpc, tag);
	}

	action number {
		long len = (fpc - data) - tag;
		currentNumber = [[[NSString alloc] initWithBytes:fpc-len length:len encoding:NSUTF8StringEncoding] intValue];
		//if (lineCount > DEBUGLINECOUNT) NSLog(@"%p num: %d", fpc, currentNumber);
	}

	action string {
		long len = (fpc - data) - tag;
		currentString = [[NSString alloc] initWithBytes:fpc-len length:len encoding:NSUTF8StringEncoding];
		//if (lineCount > DEBUGLINECOUNT) NSLog(@"%p string: %@", fpc, currentString);
	}

	action saveLevel {
        level = currentNumber;
	}

    action saveXref {
        xref = currentString;
    }

    action saveTagCode {
        code = currentString;
    }

    action saveValue {
        value = [currentString stringByReplacingOccurrencesOfString:@"@@" withString:@"@"]; // unescape at-sign
    }

	action saveNode {
        lineCount++;

        // CONT/CONC nodes are continuations of values from the previous node, so fold them up here:
        // TODO: refactor (use dict?)
        if ([code isEqualToString:@"CONT"]) {
            if (!currentNode.gedValue) {
                currentNode.gedValue = value;
            } else {
                currentNode.gedValue = [NSString stringWithFormat:@"%@%@%@", currentNode.gedValue, @"\n", value ? value : @""];
            }
        } else if ([code isEqualToString:@"CONC"]) {
            if (!currentNode.gedValue) {
                currentNode.gedValue = value;
            } else {
                currentNode.gedValue = [NSString stringWithFormat:@"%@%@%@", currentNode.gedValue, concSeparator, value ? value : @""];
            }
        } else {
            if (xref) {
                node = [GCNode nodeWithTag:code
                                      xref:xref];
            } else {
                node = [GCNode nodeWithTag:code
                                     value:value];
            }

            GCNode *parent = nil;

            if (level == 0) { //root
                if (currentRoot) {
                    nodeCount++;
                    if (_delegate && [_delegate respondsToSelector:@selector(parser:didParseNode:)]) {
                        [_delegate parser:self didParseNode:currentRoot];
                    }
                }
            } else { //find correct parent
                parent = currentNode;
                for (int i = currentLevel; i >= level; i--) {
                    parent = [parent parent];
                }
            }

            if (parent) {
                [parent.mutableSubNodes addObject:node];
            } else {
                currentRoot = node;
            }

            currentLevel = level;
            currentNode = node;
        }

        level = -1;
        xref = nil;
        code = nil;
        value = nil;
	}

	action finish {
		didFinish = YES;
	}

    delim                   = ' ';
    terminator              = ( '\r' | '\n' )+;

    keyword                 = ( alpha ( alnum | '_' )+ );

    level                   = ( digit+ >tag %number ) %saveLevel;
    xref                    = ( ( '@' keyword '@' ) >tag %string ) %saveXref;
    tagCode                 = ( ( '_'? keyword ) >tag %string ) %saveTagCode;
    value                   = ( ( extend - terminator )+ >tag %string ) %saveValue;

    contents                = ( ( xref delim tagCode ) | ( tagCode delim value ) | ( tagCode delim? ) );

    node                    = ( level delim contents terminator ) %saveNode;

    main                   := ( node+ ) %/finish;

}%%

%% write data;

@implementation GCNodeParser

__strong static id _sharedNodeParser = nil;

+ (void)initialize
{
    _sharedNodeParser = [[self alloc] init];
}

+ (id)sharedNodeParser
{
    return _sharedNodeParser;
}

- (BOOL)parseString:(id)gedString error:(NSError *__autoreleasing *)error
{
    @synchronized(self) {
        if ([gedString isKindOfClass:[NSAttributedString class]]) {
            gedString = [gedString string];
        }

        NSParameterAssert([gedString isKindOfClass:[NSString class]]);

        gedString = [gedString stringByAppendingString:@"\n"];

        NSUInteger nodeCount = 0;
        NSUInteger lineCount = 0;
        int currentLevel = 0;
        GCNode *currentNode = nil;
        GCNode *currentRoot = nil;

#ifdef DEBUGLEVEL
        clock_t start, end;
        double elapsed;
        start = clock();
        //NSLog(@"Began parsing gedcom.");
#endif

        long tag = 0;
        int currentNumber = 0;
        NSString *currentString = nil;
        BOOL didFinish = NO;

        int level = -1;
        NSString *xref = nil;
        NSString *code = nil;
        NSString *value = nil;
        GCNode* node = nil;

        int cs = 0;
        const char *data = [gedString UTF8String];
        const char *p = data;
        const char *pe = p + [gedString lengthOfBytesUsingEncoding:NSUTF8StringEncoding];
        const char *eof = pe;

        %% write init;
        %% write exec;

        // TODO fix order so this is unneeded.
        if (currentRoot) {
            nodeCount++;
            if (_delegate && [_delegate respondsToSelector:@selector(parser:didParseNode:)]) {
                [_delegate parser:self didParseNode:currentRoot];
            }
        }

#ifdef DEBUGLEVEL
        end = clock();
        elapsed = ((double) (end - start)) / CLOCKS_PER_SEC;
        NSLog(@"parsed %ld nodes in %ld lines - Time: %f seconds", nodeCount, lineCount, elapsed);
#endif

        if (_delegate && [_delegate respondsToSelector:@selector(parser:didParseNodesWithCount:)]) {
            [_delegate parser:self didParseNodesWithCount:nodeCount];
        }

        if (!didFinish) {
            if (error != NULL) {
                *error = [NSError errorWithDomain:GCErrorDomain
                                             code:GCNodeParsingError
                                         userInfo:@{NSLocalizedDescriptionKey: [NSString stringWithFormat:@"Parser stopped at node #%ld (line %ld)", nodeCount, lineCount], NSAffectedObjectsErrorKey: currentNode}];
            }
        }

        return didFinish;
    }
}

@end

@interface _GCNodeParserConvenienceDelegate : NSObject <GCNodeParserDelegate>

@property NSArray *nodes;

@end

@implementation _GCNodeParserConvenienceDelegate {
    NSMutableArray *_nodes;
}

- (void)parser:(GCNodeParser *)parser didParseNode:(GCNode *)node
{
    //NSLog(@"%p got node: %@", self, node);
    if (!_nodes) {
        _nodes = [NSMutableArray array];
    }
    [_nodes addObject:node];
}

- (void)parser:(GCNodeParser *)parser didParseNodesWithCount:(NSUInteger)nodeCount
{
    NSParameterAssert(nodeCount == [_nodes count]);
}

@end

@implementation GCNodeParser (GCConvenienceMethods)

+ (NSArray*)arrayOfNodesFromString:(id)gedString
{
    _GCNodeParserConvenienceDelegate *delegate = [[_GCNodeParserConvenienceDelegate alloc] init];

    GCNodeParser *parser = [[GCNodeParser alloc] init];
    parser.delegate = delegate;

    NSError *error = nil;
    BOOL succeeded = [parser parseString:gedString error:&error];

    if (succeeded) {
        return delegate.nodes;
    } else {
        NSLog(@"error: %@", error);
        return nil;
    }
}

@end


Create Hash from key-values

Create Hash from key-values

# Parser for VCF-header info and format fields

module VcfHeader

  module RagelKeyValues

=begin
%%{

  machine simple_lexer;

  action mark { ts=p }
  action endquoted {
    emit.call(:value,data,ts,p)
  }

  squote = "'";
  dquote = '"';
  not_squote_or_escape = [^'\\];
  not_dquote_or_escape = [^"\\];
  escaped_something = /\\./;
  ss = space* squote ( not_squote_or_escape | escaped_something )* >mark %endquoted squote;
  dd = space* dquote ( not_dquote_or_escape | escaped_something )* >mark %endquoted dquote;

  integer     = ('+'|'-')?digit+;
  float       = ('+'|'-')?digit+'.'digit+;
  assignment  = '=';
  identifier  = (alpha alnum+);
  str         = (ss|dd)* ;
  boolean     = '.';
  key_word    = ( ('ID'|'Number'|'Type'|'Description') >mark %{ emit.call(:key_word,data,ts,p) } );
  value       = ( (integer|float|boolean|identifier|str) >mark %{ emit.call(:value,data,ts,p) } );

  key_value = ( key_word '=' value ) ;

  main := ('##FORMAT'|'##INFO') '=' (('<'|',') key_value )*;
}%%
=end

%% write data;
# %% this just fixes our syntax highlighting...

def self.run_lexer(data, options = {})
  data = data.unpack("c*") if(data.is_a?(String))
  eof = data.length
  values = []
  stack = []

  emit = lambda { |type, data, ts, p|
    # Print the type and text of the last read token
    # p ts,p
    puts "#{type}: #{data[ts...p].pack('c*')}" if options[:debug]==true
    values << [type,data[ts...p].pack('c*')]
  }

  %% write init;
  %% write exec;

  res = {}
  values.each_slice(2) do | a,b |
    # p '*',a,b
    res[a[1]] = b[1]
    # p h[:value] if h[:name]==:identifier or h[:name]==:value or h[:name]==:string
  end
  p res
end

  end
end

if __FILE__ == $0

lines = <<LINES
##FORMAT=<ID=GT,Number=1,Type=String,Description="Genotype">
##FORMAT=<ID=DP,Number=1,Type=Integer,Description="Total read depth">
##FORMAT=<ID=DP4,Number=4,Type=Integer,Description="# high-quality ref-forward bases, ref-reverse, alt-forward and alt-reverse bases">
##INFO=<ID=PM,Number=0,Type=Flag,Description="Variant is Precious(Clinical,Pubmed Cited)">
##INFO=<ID=VP,Number=1,Type=String,Description="Variation Property.  Documentation is at ftp://ftp.ncbi.nlm.nih.gov/snp/specs/dbSNP_BitField_latest.pdf">
##INFO=<ID=GENEINFO,Number=1,Type=String,Description="Pairs each of gene symbol:gene id.  The gene symbol and id are delimited by a colon (:), and each pair is delimited by a vertical bar (|)">
##INFO=<ID=CLNHGVS,Number=.,Type=String,Description="Variant names from HGVS. The order of these variants corresponds to the order of the info in the other clinical  INFO tags.">
##INFO=<ID=CLNHGVS1,Number=.,Type=String,Description="Variant names from \\"HGVS\\". The order of these 'variants' corresponds to the order of the info in the other clinical  INFO tags.">
LINES

lines.strip.split("\n").each { |s|
  print s,"\n"
  VcfHeader::RagelKeyValues.run_lexer(s, debug: false)
}

end

# Parser for VCF-header info and format fields

module VcfHeader

  module RagelKeyValues

=begin
%%{

  machine simple_lexer;

  action mark { ts=p }
  action endquoted {
    emit.call(:value,data,ts,p)
  }

  squote = "'";
  dquote = '"';
  not_squote_or_escape = [^'\\];
  not_dquote_or_escape = [^"\\];
  escaped_something = /\\./;
  ss = space* squote ( not_squote_or_escape | escaped_something )* >mark %endquoted squote;
  dd = space* dquote ( not_dquote_or_escape | escaped_something )* >mark %endquoted dquote;

  integer     = ('+'|'-')?digit+;
  float       = ('+'|'-')?digit+'.'digit+;
  assignment  = '=';
  identifier  = (alpha alnum+);
  str         = (ss|dd)* ;
  boolean     = '.';
  key_word    = ( ('ID'|'Number'|'Type'|'Description') >mark %{ emit.call(:key_word,data,ts,p) } );
  value       = ( (integer|float|boolean|identifier|str) >mark %{ emit.call(:value,data,ts,p) } );

  key_value = ( key_word '=' value ) ;

  main := ('##FORMAT'|'##INFO') '=' (('<'|',') key_value )*;
}%%
=end

%% write data;
# %% this just fixes our syntax highlighting...

def self.run_lexer(data, options = {})
  data = data.unpack("c*") if(data.is_a?(String))
  eof = data.length
  values = []
  stack = []

  emit = lambda { |type, data, ts, p|
    # Print the type and text of the last read token
    # p ts,p
    puts "#{type}: #{data[ts...p].pack('c*')}" if options[:debug]==true
    values << [type,data[ts...p].pack('c*')]
  }

  %% write init;
  %% write exec;

  values.each do | v |
    p v
    # p h[:value] if h[:name]==:identifier or h[:name]==:value or h[:name]==:string
  end
end

  end
end

if __FILE__ == $0

lines = <<LINES
##FORMAT=<ID=GT,Number=1,Type=String,Description="Genotype">
##FORMAT=<ID=DP,Number=1,Type=Integer,Description="Total read depth">
##FORMAT=<ID=DP4,Number=4,Type=Integer,Description="# high-quality ref-forward bases, ref-reverse, alt-forward and alt-reverse bases">
##INFO=<ID=PM,Number=0,Type=Flag,Description="Variant is Precious(Clinical,Pubmed Cited)">
##INFO=<ID=VP,Number=1,Type=String,Description="Variation Property.  Documentation is at ftp://ftp.ncbi.nlm.nih.gov/snp/specs/dbSNP_BitField_latest.pdf">
##INFO=<ID=GENEINFO,Number=1,Type=String,Description="Pairs each of gene symbol:gene id.  The gene symbol and id are delimited by a colon (:), and each pair is delimited by a vertical bar (|)">
##INFO=<ID=CLNHGVS,Number=.,Type=String,Description="Variant names from HGVS. The order of these variants corresponds to the order of the info in the other clinical  INFO tags.">
##INFO=<ID=CLNHGVS1,Number=.,Type=String,Description="Variant names from \\"HGVS\\". The order of these 'variants' corresponds to the order of the info in the other clinical  INFO tags.">
LINES

lines.strip.split("\n").each { |s|
  p s
  VcfHeader::RagelKeyValues.run_lexer(s, debug: false)
}

end


Fix formatting

Fix formatting

/*
 Copyright (c) 2017 Tuukka Norri
 This code is licensed under MIT license (see LICENSE for details).
 */

#include <vcf2multialign/util.hh>
#include <vcf2multialign/vcf_reader.hh>

%% machine vcf_parser;
%% write data;

// We would like to post-process strings that have been read and then pass
// them to the variant object with a simple function call at the call site.
// Solution: Templates can have member function pointers as parameters but of a specific type.
#define HANDLE_STRING_END(FN, ...)	caller <decltype(FN)>::handle_string_end <FN>(*this, ##__VA_ARGS__)
#define HANDLE_INTEGER_END(FN, ...)	caller <decltype(FN)>::handle_integer_end <FN>(*this, ##__VA_ARGS__)

namespace vcf2multialign { namespace detail {

	template <typename t_enum>
	constexpr typename std::underlying_type <t_enum>::type to_underlying(t_enum e)
	{
		return static_cast <typename std::underlying_type <t_enum>::type>(e);
	}

	// Split a t_string into string_views.
	template <bool t_emplace_back, typename t_string, typename t_invalid_pos>
	size_t read_fields(
		t_string const &string,
		t_invalid_pos const invalid_pos,
		char const *string_start,
		char const *sep,
		size_t const count,
		std::vector <std::string_view> &res
	)
	{
		char const *string_ptr(string_start);

		if (t_emplace_back)
			res.clear();
		else
			res.resize(count);

		typename t_string::size_type sep_pos(0);
		size_t i(0);
		bool should_break(false);
		while (i < count)
		{
			sep_pos = string.find(sep, sep_pos);
			if (invalid_pos == sep_pos)
			{
				should_break = true;
				sep_pos = string.size();
			}

			if (t_emplace_back)
				res.emplace_back(string_ptr, sep_pos - (string_ptr - string_start));
			else
			{
				std::string_view sv(string_ptr, sep_pos - (string_ptr - string_start));
				res[i] = std::move(sv);
			}

			++i;
			++sep_pos;
			string_ptr = string_start + sep_pos;

			if (should_break)
				break;
		}
		return i;
	}

	// Split a string into string_views.
	template <bool t_emplace_back>
	size_t read_fields(
		std::string const &string,
		char const *sep,
		size_t const count,
		std::vector <std::string_view> &res
	)
	{
		return read_fields <t_emplace_back>(string, std::string::npos, string.c_str(), sep, count, res);
	}

	template <bool t_emplace_back>
	size_t read_fields(
		std::string_view const &sv,
		char const *sep,
		size_t const count,
		std::vector <std::string_view> &res
	)
	{
		return read_fields <t_emplace_back>(sv, std::string_view::npos, static_cast <char const *>(sv.data()), sep, count, res);
	}
}}

namespace vcf2multialign {

	template <typename t_fnt>
	struct vcf_reader::caller
	{
		template <t_fnt t_fn, typename ... t_args>
		static void handle_string_end(vcf_reader &r, t_args ... args)
		{
			std::string_view sv(r.m_start, r.m_fsm.p - r.m_start);
			(r.m_current_variant.*(t_fn))(sv, std::forward <t_args>(args)...);
		}

		template <t_fnt t_fn, typename ... t_args>
		static void handle_integer_end(vcf_reader &r, t_args ... args)
		{
			(r.m_current_variant.*(t_fn))(r.m_integer, std::forward <t_args>(args)...);
		}
	};

	template <int t_continue, int t_break>
	int vcf_reader::check_max_field(vcf_field const field, int const target, callback_fn const &cb)
	{
		if (field <= m_max_parsed_field)
			return target;

		skip_to_next_nl();
		if (!cb(m_current_variant))
			return t_break;

		return t_continue;
	}

	void vcf_reader::report_unexpected_character(char const *current_character, int const current_state)
	{
		std::cerr
		<< "Unexpected character '" << *current_character << "' at " << m_lineno << ':' << (current_character - m_line_start)
		<< ", state " << current_state << '.' << std::endl;

		std::cerr
		<< "** Buffer contents from the start:" << std::endl;
		std::copy(m_buffer.cbegin(), m_buffer.cend(), std::ostream_iterator <char>(std::cerr));
		std::cerr << std::endl;

		std::string_view buffer_end(m_fsm.p, m_fsm.pe - m_fsm.p);
		std::cerr
		<< "** Buffer contents starting from m_fsm.p:" << std::endl
		<< buffer_end << std::endl;

		abort();
	}

	void vcf_reader::skip_to_next_nl()
	{
		std::string_view sv(m_fsm.p, m_fsm.pe - m_fsm.p);
		auto const pos(sv.find('\n'));
		always_assert(std::string_view::npos != pos, "Unable to find the next newline");
		m_fsm.p += pos;
	}

	// Seek to the beginning of the records.
	void vcf_reader::reset()
	{
		m_stream->clear();
		m_stream->seekg(m_first_variant_offset);
		m_lineno = m_last_header_lineno;
		m_len = 0;
		m_pos = 0;
		m_fsm.eof = nullptr;
	}

	// Return the 1-based number of the given sample.
	size_t vcf_reader::sample_no(std::string const &sample_name) const
	{
		auto const it(m_sample_names.find(sample_name));
		if (it == m_sample_names.cend())
			return 0;
		return it->second;
	}

	void vcf_reader::read_header()
	{
		// For now, just skip lines that begin with "##".
		std::string line;
		while (std::getline(*m_stream, line))
		{
			++m_lineno;
			if (! ('#' == line[0] && '#' == line[1]))
				break;

			// FIXME: header handling goes here.
		}

		// Check for column headers.
		{
			auto const prefix(std::string("#CHROM"));
			auto const res(mismatch(line.cbegin(), line.cend(), prefix.cbegin(), prefix.cend()));
			if (prefix.cend() != res.second)
			{
				std::cerr << "Expected the next line to start with '#CHROM', instead got: '" << line << "'" << std::endl;
				exit(1);
			}
		}

		// Read sample names.
		std::vector <std::string_view> header_names;
		detail::read_fields <true>(line, "\t", -1, header_names);
		std::size_t i(1);
		for (auto it(header_names.cbegin() + detail::to_underlying(vcf_field::ALL)), end(header_names.cend()); it != end; ++it)
		{
			std::string str(*it);
			auto const res(m_sample_names.emplace(std::move(str), i));
			always_assert(res.second, "Duplicate sample name");
			++i;
		}

		// stream now points to the first variant.
		m_first_variant_offset = m_stream->tellg();
		m_last_header_lineno = m_lineno;

		// Instantiate a variant.
		transient_variant var(sample_count());
		using std::swap;
		swap(m_current_variant, var);
	}

	void vcf_reader::fill_buffer()
	{
		// Copy the remainder to the beginning.
		if (m_pos + 1 < m_len)
		{
			char *data_start(m_buffer.data());
			char const *start(data_start + m_pos + 1);
			char const *end(data_start + m_len);
			std::copy(start, end, data_start);
			m_len -= m_pos + 1;
		}
		else
		{
			m_len = 0;
		}

		// Read until there's at least one newline in the buffer.
		while (true)
		{
			char *data_start(m_buffer.data());
			char *data(data_start + m_len);

			std::size_t space(m_buffer.size() - m_len);
			m_stream->read(data, space);
			std::size_t const read_len(m_stream->gcount());
			m_len += read_len;

			if (m_stream->eof())
			{
				m_pos = m_len;
				m_fsm.p = data_start;
				m_fsm.pe = data_start + m_len;
				m_fsm.eof = m_fsm.pe;
				return;
			}

			// Try to find the last newline in the new part.
			std::string_view sv(data, read_len);
			m_pos = sv.rfind('\n');
			if (std::string_view::npos != m_pos)
			{
				m_pos += (data - data_start);
				m_fsm.p = data_start;
				m_fsm.pe = m_fsm.p + m_pos + 1;
				return;
			}

			m_buffer.resize(2 * m_buffer.size());
		}
	}

	bool vcf_reader::parse(callback_fn &&callback)
	{
		return parse(callback);
	}

	bool vcf_reader::parse(callback_fn const &cb)
	{
		typedef variant_tpl <std::string_view> vc;
		bool retval(true);
		int cs(0);
		// FIXME: add throwing EOF actions?
		%%{
			variable p		m_fsm.p;
			variable pe		m_fsm.pe;
			variable eof	m_fsm.eof;

			action start_string {
				m_start = fpc;
			}

			action start_integer {
				m_integer = 0;
			}

			action update_integer {
				// Shift and add the current number.
				m_integer *= 10;
				m_integer += fc - '0';
			}

			action set_unknown_quality_value {
				m_integer = std::numeric_limits <decltype(m_integer)>::max();
			}

			action end_sample_field {
				// Check that the current field is followed by another field, another sample or a new record.
				switch (fc)
				{
					case ':':
						fgoto sample_rec_f;

					case '\t':
					{
						always_assert(m_format.size() == m_format_idx, "Not all fields present in the sample");
						m_format_idx = 0;
						fgoto sample_rec_f;
					}

					case '\n':
					{
						always_assert(m_format.size() == m_format_idx, "Not all fields present in the sample");

						if (!cb(m_current_variant))
						{
							fhold;
							fbreak;
						}

						fgoto main;
					}

					default:
						report_unexpected_character(fpc, fcurs);
				}
			}

			action error {
				report_unexpected_character(fpc, fcurs);
			}

			tab			= '\t';

			id_string	= '<' alnum+ '>';

			chrom_id	= (alnum+)
				>(start_string)
				%{ HANDLE_STRING_END(&vc::set_chrom_id); };

			pos			= (digit+)
				>(start_integer)
				$(update_integer)
				%{ HANDLE_INTEGER_END(&vc::set_pos); };

			id_part		= (([.] | alnum)+)
				>(start_string)
				%{ HANDLE_STRING_END(&vc::set_id, m_idx++); };
			id_rec		= (id_part (';' id_part)*) >{ m_idx = 0; };

			ref			= ([ACGTN]+)
				>(start_string)
				%{ HANDLE_STRING_END(&vc::set_ref); };

			# FIXME: add breakends.
			simple_alt	= ([ACGTN]+);
			complex_alt	= ([*]+) %{ m_alt_is_complex = true; };
			alt_part	= (simple_alt | complex_alt)
				>(start_string)
				%{ HANDLE_STRING_END(&vc::set_alt, m_idx++, m_alt_is_complex); };
			alt			= (alt_part (',' alt_part)*) >{ m_idx = 0; };

			qual_numeric	= (digit+)
				>(start_integer)
				$(update_integer)
				%{ HANDLE_INTEGER_END(&vc::set_qual); };
			qual_unknown	= '.'
				$(set_unknown_quality_value)
				%{ HANDLE_INTEGER_END(&vc::set_qual); };

			qual			= qual_numeric | qual_unknown;

			# FIXME: add actions.
			filter_pass	= 'PASS';
			filter_part	= alnum+;
			filter		= (filter_pass | (filter_part (';' filter_part)*));

			# FIXME: add actions.
			info_key		= (alnum | '_') +;
			info_str		= (alnum | [+-_.]) +;
			info_val		= info_str (',' info_str)*;
			info_part		= info_key ('=' info_val)?;
			info_missing	= '.';
			info			= info_missing | (info_part (';' info_part)*);

			# FIXME: handle other formats.
			format_gt	= ('GT') @{ m_format.emplace_back(format_field::GT); };
			format_dp	= ('DP') @{ m_format.emplace_back(format_field::DP); };
			format_gq	= ('GQ') @{ m_format.emplace_back(format_field::GQ); };
			format_ps	= ('PS') @{ m_format.emplace_back(format_field::PS); };
			format_pq	= ('PQ') @{ m_format.emplace_back(format_field::PQ); };
			format_mq	= ('MQ') @{ m_format.emplace_back(format_field::MQ); };
			format_part	= format_gt | format_dp | format_gq | format_ps | format_pq | format_mq;
			format		= ((format_part (':' format_part)*) >{ m_format.clear(); });

			# Genotype field in sample.
			sample_gt_null_allele	= '.'
				%{
					m_integer = NULL_ALLELE;
					HANDLE_INTEGER_END(&vc::set_gt, m_sample_idx, m_idx++, m_gt_is_phased);
				};

			sample_gt_allele_idx	= (digit+)
				>(start_integer)
				$(update_integer)
				%{
					HANDLE_INTEGER_END(&vc::set_gt, m_sample_idx, m_idx++, m_gt_is_phased);
					m_gt_is_phased = false;
				};

			sample_gt_part			= (sample_gt_null_allele | sample_gt_allele_idx);
			sample_gt_p				= sample_gt_part ('|' ${ m_gt_is_phased = true; } sample_gt_part)+;
			sample_gt_up			= sample_gt_part ('/' sample_gt_part)+;
			sample_gt				= (sample_gt_part | sample_gt_p | sample_gt_up)
				>{
					m_idx = 0;
				};

			# FIXME: add actions.
			sample_dp		= (digit+);
			sample_gq		= (digit+);
			sample_ps		= (digit+);
			sample_pq		= (digit+);
			sample_mq		= (digit+);

			sep				= '\t';		# Field separator
			ssep			= [\t\n:];	# Sample separator

			# Handle a newline and continue.
			# The newline gets eaten before its checked, though, so use any instead.
			main_nl := any @{ fhold; fgoto main; };
			break_nl := any @{ fhold; fbreak; };

			# Line start.
			# Apparently main has to be able to read a character, so use fhold.
			# Allow EOF, though, with '?'.
			main := any?
				${
					fhold;

					m_idx = 0;
					m_format_idx = 0;
					m_sample_idx = 0;
					m_alt_is_complex = false;
					++m_lineno;
					m_current_variant.reset();
					m_current_variant.set_lineno(m_lineno);
					m_line_start = fpc;

					fgoto *check_max_field <fentry(main_nl), fentry(break_nl)>(vcf_field::CHROM, fentry(chrom_id_f), cb);
				}
				$err(error)
				$eof{ retval = false; };

			# #CHROM
			chrom_id_f :=
				(chrom_id sep)
				@{
					fgoto *check_max_field <fentry(main_nl), fentry(break_nl)>(vcf_field::POS, fentry(pos_f), cb);
				}
				$err(error);

			# POS
			pos_f :=
				(pos sep)
				@{
					fgoto *check_max_field <fentry(main_nl), fentry(break_nl)>(vcf_field::ID, fentry(id_f), cb);
				}
				$err(error);

			# ID
			id_f :=
				(id_rec sep)
				@{
					fgoto *check_max_field <fentry(main_nl), fentry(break_nl)>(vcf_field::REF, fentry(ref_f), cb);
				}
				$err(error);

			# REF
			ref_f :=
				(ref sep)
				@{
					fgoto *check_max_field <fentry(main_nl), fentry(break_nl)>(vcf_field::ALT, fentry(alt_f), cb);
				}
				$err(error);

			# ALT
			alt_f :=
				(alt sep)
				@{
					fgoto *check_max_field <fentry(main_nl), fentry(break_nl)>(vcf_field::QUAL, fentry(qual_f), cb);
				}
				$err(error);

			# QUAL
			qual_f :=
				(qual sep)
				@{
					fgoto *check_max_field <fentry(main_nl), fentry(break_nl)>(vcf_field::FILTER, fentry(filter_f), cb);
				}
				$err(error);

			# FILTER
			filter_f :=
				(filter sep)
				@{
					fgoto *check_max_field <fentry(main_nl), fentry(break_nl)>(vcf_field::INFO, fentry(info_f), cb);
				}
				$err(error);

			# INFO
			info_f :=
				(info sep)
				@{
					fgoto *check_max_field <fentry(main_nl), fentry(break_nl)>(vcf_field::FORMAT, fentry(format_f), cb);
				}
				$err(error);

			# FORMAT
			# FIXME: try to skip parsing the field and compare to the existing format?
			format_f :=
				(format sep)
				@{
					fgoto *check_max_field <fentry(main_nl), fentry(break_nl)>(vcf_field::ALL, fentry(sample_rec_f), cb);
				}
				$err(error);

			# Sample fields
			sample_gt_f := ((sample_gt) ssep @(end_sample_field)) $err(error);
			sample_dp_f := ((sample_dp) ssep @(end_sample_field)) $err(error);
			sample_gq_f := ((sample_gq) ssep @(end_sample_field)) $err(error);
			sample_ps_f := ((sample_ps) ssep @(end_sample_field)) $err(error);
			sample_pq_f := ((sample_pq) ssep @(end_sample_field)) $err(error);
			sample_mq_f := ((sample_mq) ssep @(end_sample_field)) $err(error);

			# Sample record
			sample_rec_f := "" >to{
				always_assert(m_format_idx < m_format.size(), "Format does not match the sample");

				// Sample index 0 is reserved for the reference.
				++m_sample_idx;

				// Parse according to the format field.
				auto const idx(m_format_idx);
				++m_format_idx;
				auto const field(m_format[idx]);
				switch (field)
				{
					case format_field::GT:
						fgoto sample_gt_f;

					case format_field::DP:
						fgoto sample_dp_f;

					case format_field::GQ:
						fgoto sample_gq_f;

					case format_field::PS:
						fgoto sample_ps_f;

					case format_field::PQ:
						fgoto sample_pq_f;

					case format_field::MQ:
						fgoto sample_mq_f;

					default:
						fail("Unexpected format value");
				}
			};

			dummy := any
				@{
					// Dummy action,
					// make sure that all the machines are instantiated.
					if (false)
					{
						fgoto chrom_id_f;
						fgoto pos_f;
						fgoto id_f;
						fgoto ref_f;
						fgoto alt_f;
						fgoto qual_f;
						fgoto filter_f;
						fgoto info_f;
						fgoto format_f;
						fgoto sample_rec_f;

						fgoto sample_gt_f;
						fgoto sample_dp_f;
						fgoto sample_gq_f;
						fgoto sample_ps_f;
						fgoto sample_pq_f;
						fgoto sample_mq_f;
					}
				};

			write init;
			write exec;
		}%%

		return retval;
	}
}

/*
 Copyright (c) 2017 Tuukka Norri
 This code is licensed under MIT license (see LICENSE for details).
 */

#include <vcf2multialign/util.hh>
#include <vcf2multialign/vcf_reader.hh>

%% machine vcf_parser;
%% write data;

// We would like to post-process strings that have been read and then pass
// them to the variant object with a simple function call at the call site.
// Solution: Templates can have member function pointers as parameters but of a specific type.
#define HANDLE_STRING_END(FN, ...)	caller <decltype(FN)>::handle_string_end <FN>(*this, ##__VA_ARGS__)
#define HANDLE_INTEGER_END(FN, ...)	caller <decltype(FN)>::handle_integer_end <FN>(*this, ##__VA_ARGS__)

namespace vcf2multialign { namespace detail {

	template <typename t_enum>
	constexpr typename std::underlying_type <t_enum>::type to_underlying(t_enum e)
	{
		return static_cast <typename std::underlying_type <t_enum>::type>(e);
	}

	// Split a t_string into string_views.
	template <bool t_emplace_back, typename t_string, typename t_invalid_pos>
	size_t read_fields(
		t_string const &string,
		t_invalid_pos const invalid_pos,
		char const *string_start,
		char const *sep,
		size_t const count,
		std::vector <std::string_view> &res
	)
	{
		char const *string_ptr(string_start);

		if (t_emplace_back)
			res.clear();
		else
			res.resize(count);

		typename t_string::size_type sep_pos(0);
		size_t i(0);
		bool should_break(false);
		while (i < count)
		{
			sep_pos = string.find(sep, sep_pos);
			if (invalid_pos == sep_pos)
			{
				should_break = true;
				sep_pos = string.size();
			}

			if (t_emplace_back)
				res.emplace_back(string_ptr, sep_pos - (string_ptr - string_start));
			else
			{
				std::string_view sv(string_ptr, sep_pos - (string_ptr - string_start));
				res[i] = std::move(sv);
			}

			++i;
			++sep_pos;
			string_ptr = string_start + sep_pos;

			if (should_break)
				break;
		}
		return i;
	}

	// Split a string into string_views.
	template <bool t_emplace_back>
	size_t read_fields(
		std::string const &string,
		char const *sep,
		size_t const count,
		std::vector <std::string_view> &res
	)
	{
		return read_fields <t_emplace_back>(string, std::string::npos, string.c_str(), sep, count, res);
	}

	template <bool t_emplace_back>
	size_t read_fields(
		std::string_view const &sv,
		char const *sep,
		size_t const count,
		std::vector <std::string_view> &res
	)
	{
		return read_fields <t_emplace_back>(sv, std::string_view::npos, static_cast <char const *>(sv.data()), sep, count, res);
	}
}}

namespace vcf2multialign {

	template <typename t_fnt>
	struct vcf_reader::caller
	{
		template <t_fnt t_fn, typename ... t_args>
		static void handle_string_end(vcf_reader &r, t_args ... args)
		{
			std::string_view sv(r.m_start, r.m_fsm.p - r.m_start);
			(r.m_current_variant.*(t_fn))(sv, std::forward <t_args>(args)...);
		}

		template <t_fnt t_fn, typename ... t_args>
		static void handle_integer_end(vcf_reader &r, t_args ... args)
		{
			(r.m_current_variant.*(t_fn))(r.m_integer, std::forward <t_args>(args)...);
		}
	};

	template <int t_continue, int t_break>
	int vcf_reader::check_max_field(vcf_field const field, int const target, callback_fn const &cb)
	{
		if (field <= m_max_parsed_field)
			return target;

		skip_to_next_nl();
		if (!cb(m_current_variant))
			return t_break;

		return t_continue;
	}

	void vcf_reader::report_unexpected_character(char const *current_character, int const current_state)
	{
		std::cerr
		<< "Unexpected character '" << *current_character << "' at " << m_lineno << ':' << (current_character - m_line_start)
		<< ", state " << current_state << '.' << std::endl;

		std::cerr
		<< "** Buffer contents from the start:" << std::endl;
		std::copy(m_buffer.cbegin(), m_buffer.cend(), std::ostream_iterator <char>(std::cerr));
		std::cerr << std::endl;

		std::string_view buffer_end(m_fsm.p, m_fsm.pe - m_fsm.p);
		std::cerr
		<< "** Buffer contents starting from m_fsm.p:" << std::endl
		<< buffer_end << std::endl;

		abort();
	}

	void vcf_reader::skip_to_next_nl()
	{
		std::string_view sv(m_fsm.p, m_fsm.pe - m_fsm.p);
		auto const pos(sv.find('\n'));
		always_assert(std::string_view::npos != pos, "Unable to find the next newline");
		m_fsm.p += pos;
	}

	// Seek to the beginning of the records.
	void vcf_reader::reset()
	{
		m_stream->clear();
		m_stream->seekg(m_first_variant_offset);
		m_lineno = m_last_header_lineno;
		m_len = 0;
		m_pos = 0;
		m_fsm.eof = nullptr;
	}

	// Return the 1-based number of the given sample.
	size_t vcf_reader::sample_no(std::string const &sample_name) const
	{
		auto const it(m_sample_names.find(sample_name));
		if (it == m_sample_names.cend())
			return 0;
		return it->second;
	}

	void vcf_reader::read_header()
	{
		// For now, just skip lines that begin with "##".
		std::string line;
		while (std::getline(*m_stream, line))
		{
			++m_lineno;
			if (! ('#' == line[0] && '#' == line[1]))
				break;

			// FIXME: header handling goes here.
		}

		// Check for column headers.
		{
			auto const prefix(std::string("#CHROM"));
			auto const res(mismatch(line.cbegin(), line.cend(), prefix.cbegin(), prefix.cend()));
			if (prefix.cend() != res.second)
			{
				std::cerr << "Expected the next line to start with '#CHROM', instead got: '" << line << "'" << std::endl;
				exit(1);
			}
		}

		// Read sample names.
		std::vector <std::string_view> header_names;
		detail::read_fields <true>(line, "\t", -1, header_names);
		std::size_t i(1);
		for (auto it(header_names.cbegin() + detail::to_underlying(vcf_field::ALL)), end(header_names.cend()); it != end; ++it)
		{
			std::string str(*it);
			auto const res(m_sample_names.emplace(std::move(str), i));
			always_assert(res.second, "Duplicate sample name");
			++i;
		}

		// stream now points to the first variant.
		m_first_variant_offset = m_stream->tellg();
		m_last_header_lineno = m_lineno;

		// Instantiate a variant.
		transient_variant var(sample_count());
		using std::swap;
		swap(m_current_variant, var);
	}

	void vcf_reader::fill_buffer()
	{
		// Copy the remainder to the beginning.
		if (m_pos + 1 < m_len)
		{
			char *data_start(m_buffer.data());
			char const *start(data_start + m_pos + 1);
			char const *end(data_start + m_len);
			std::copy(start, end, data_start);
			m_len -= m_pos + 1;
		}
		else
		{
			m_len = 0;
		}

		// Read until there's at least one newline in the buffer.
		while (true)
		{
			char *data_start(m_buffer.data());
			char *data(data_start + m_len);

			std::size_t space(m_buffer.size() - m_len);
			m_stream->read(data, space);
			std::size_t const read_len(m_stream->gcount());
			m_len += read_len;

			if (m_stream->eof())
			{
				m_pos = m_len;
				m_fsm.p = data_start;
				m_fsm.pe = data_start + m_len;
				m_fsm.eof = m_fsm.pe;
				return;
			}

			// Try to find the last newline in the new part.
			std::string_view sv(data, read_len);
			m_pos = sv.rfind('\n');
			if (std::string_view::npos != m_pos)
			{
				m_pos += (data - data_start);
				m_fsm.p = data_start;
				m_fsm.pe = m_fsm.p + m_pos + 1;
				return;
			}

			m_buffer.resize(2 * m_buffer.size());
		}
	}

	bool vcf_reader::parse(callback_fn &&callback)
	{
		return parse(callback);
	}

	bool vcf_reader::parse(callback_fn const &cb)
	{
		typedef variant_tpl <std::string_view> vc;
		bool retval(true);
		int cs(0);
		// FIXME: add throwing EOF actions?
		%%{
			variable p		m_fsm.p;
			variable pe		m_fsm.pe;
			variable eof	m_fsm.eof;

			action start_string {
				m_start = fpc;
			}

			action start_integer {
				m_integer = 0;
			}

			action update_integer {
				// Shift and add the current number.
				m_integer *= 10;
				m_integer += fc - '0';
			}

			action set_unknown_quality_value {
				m_integer = std::numeric_limits <decltype(m_integer)>::max();
			}

			action end_sample_field {
				// Check that the current field is followed by another field, another sample or a new record.
				switch (fc)
				{
					case ':':
						fgoto sample_rec_f;

					case '\t':
					{
						always_assert(m_format.size() == m_format_idx, "Not all fields present in the sample");
						m_format_idx = 0;
						fgoto sample_rec_f;
					}

					case '\n':
					{
						always_assert(m_format.size() == m_format_idx, "Not all fields present in the sample");

						if(!cb(m_current_variant))
						{
							fhold;
							fbreak;
						}

						fgoto main;
					}

					default:
						report_unexpected_character(fpc, fcurs);
				}
			}

			action error {
				report_unexpected_character(fpc, fcurs);
			}

			tab			= '\t';

			id_string	= '<' alnum+ '>';

			chrom_id	= (alnum+)
				>(start_string)
				%{ HANDLE_STRING_END(&vc::set_chrom_id); };

			pos			= (digit+)
				>(start_integer)
				$(update_integer)
				%{ HANDLE_INTEGER_END(&vc::set_pos); };

			id_part		= (([.] | alnum)+)
				>(start_string)
				%{ HANDLE_STRING_END(&vc::set_id, m_idx++); };
			id_rec		= (id_part (';' id_part)*) >{ m_idx = 0; };

			ref			= ([ACGTN]+)
				>(start_string)
				%{ HANDLE_STRING_END(&vc::set_ref); };

			# FIXME: add breakends.
			simple_alt	= ([ACGTN]+);
			complex_alt	= ([*]+) %{ m_alt_is_complex = true; };
			alt_part	= (simple_alt | complex_alt)
				>(start_string)
				%{ HANDLE_STRING_END(&vc::set_alt, m_idx++, m_alt_is_complex); };
			alt			= (alt_part (',' alt_part)*) >{ m_idx = 0; };

			qual_numeric	= (digit+)
				>(start_integer)
				$(update_integer)
				%{ HANDLE_INTEGER_END(&vc::set_qual); };
			qual_unknown	= '.'
				$(set_unknown_quality_value)
				%{ HANDLE_INTEGER_END(&vc::set_qual); };

			qual			= qual_numeric | qual_unknown;

			# FIXME: add actions.
			filter_pass	= 'PASS';
			filter_part	= alnum+;
			filter		= (filter_pass | (filter_part (';' filter_part)*));

			# FIXME: add actions.
			info_key		= (alnum | '_') +;
			info_str		= (alnum | [+-_.]) +;
			info_val		= info_str (',' info_str)*;
			info_part		= info_key ('=' info_val)?;
			info_missing	= '.';
			info			= info_missing | (info_part (';' info_part)*);

			# FIXME: handle other formats.
			format_gt	= ('GT') @{ m_format.emplace_back(format_field::GT); };
			format_dp	= ('DP') @{ m_format.emplace_back(format_field::DP); };
			format_gq	= ('GQ') @{ m_format.emplace_back(format_field::GQ); };
			format_ps	= ('PS') @{ m_format.emplace_back(format_field::PS); };
			format_pq	= ('PQ') @{ m_format.emplace_back(format_field::PQ); };
			format_mq	= ('MQ') @{ m_format.emplace_back(format_field::MQ); };
			format_part	= format_gt | format_dp | format_gq | format_ps | format_pq | format_mq;
			format		= ((format_part (':' format_part)*) >{ m_format.clear(); });

			# Genotype field in sample.
			sample_gt_null_allele	= '.'
				%{
					m_integer = NULL_ALLELE;
					HANDLE_INTEGER_END(&vc::set_gt, m_sample_idx, m_idx++, m_gt_is_phased);
				};

			sample_gt_allele_idx	= (digit+)
				>(start_integer)
				$(update_integer)
				%{
					HANDLE_INTEGER_END(&vc::set_gt, m_sample_idx, m_idx++, m_gt_is_phased);
					m_gt_is_phased = false;
				};

			sample_gt_part			= (sample_gt_null_allele | sample_gt_allele_idx);
			sample_gt_p				= sample_gt_part ('|' ${ m_gt_is_phased = true; } sample_gt_part)+;
			sample_gt_up			= sample_gt_part ('/' sample_gt_part)+;
			sample_gt				= (sample_gt_part | sample_gt_p | sample_gt_up)
				>{
					m_idx = 0;
				};

			# FIXME: add actions.
			sample_dp		= (digit+);
			sample_gq		= (digit+);
			sample_ps		= (digit+);
			sample_pq		= (digit+);
			sample_mq		= (digit+);

			sep				= '\t';		# Field separator
			ssep			= [\t\n:];	# Sample separator

			# Handle a newline and continue.
			# The newline gets eaten before its checked, though, so use any instead.
			main_nl := any @{ fhold; fgoto main; };
			break_nl := any @{ fhold; fbreak; };

			# Line start.
			# Apparently main has to be able to read a character, so use fhold.
			# Allow EOF, though, with '?'.
			main := any?
				${
					fhold;

					m_idx = 0;
					m_format_idx = 0;
					m_sample_idx = 0;
					m_alt_is_complex = false;
					++m_lineno;
					m_current_variant.reset();
					m_current_variant.set_lineno(m_lineno);
					m_line_start = fpc;

					fgoto *check_max_field <fentry(main_nl), fentry(break_nl)>(vcf_field::CHROM, fentry(chrom_id_f), cb);
				}
				$err(error)
				$eof{ retval = false; };

			# #CHROM
			chrom_id_f :=
				(chrom_id sep)
				@{
					fgoto *check_max_field <fentry(main_nl), fentry(break_nl)>(vcf_field::POS, fentry(pos_f), cb);
				}
				$err(error);

			# POS
			pos_f :=
				(pos sep)
				@{
					fgoto *check_max_field <fentry(main_nl), fentry(break_nl)>(vcf_field::ID, fentry(id_f), cb);
				}
				$err(error);

			# ID
			id_f :=
				(id_rec sep)
				@{
					fgoto *check_max_field <fentry(main_nl), fentry(break_nl)>(vcf_field::REF, fentry(ref_f), cb);
				}
				$err(error);

			# REF
			ref_f :=
				(ref sep)
				@{
					fgoto *check_max_field <fentry(main_nl), fentry(break_nl)>(vcf_field::ALT, fentry(alt_f), cb);
				}
				$err(error);

			# ALT
			alt_f :=
				(alt sep)
				@{
					fgoto *check_max_field <fentry(main_nl), fentry(break_nl)>(vcf_field::QUAL, fentry(qual_f), cb);
				}
				$err(error);

			# QUAL
			qual_f :=
				(qual sep)
				@{
					fgoto *check_max_field <fentry(main_nl), fentry(break_nl)>(vcf_field::FILTER, fentry(filter_f), cb);
				}
				$err(error);

			# FILTER
			filter_f :=
				(filter sep)
				@{
					fgoto *check_max_field <fentry(main_nl), fentry(break_nl)>(vcf_field::INFO, fentry(info_f), cb);
				}
				$err(error);

			# INFO
			info_f :=
				(info sep)
				@{
					fgoto *check_max_field <fentry(main_nl), fentry(break_nl)>(vcf_field::FORMAT, fentry(format_f), cb);
				}
				$err(error);

			# FORMAT
			# FIXME: try to skip parsing the field and compare to the existing format?
			format_f :=
				(format sep)
				@{
					fgoto *check_max_field <fentry(main_nl), fentry(break_nl)>(vcf_field::ALL, fentry(sample_rec_f), cb);
				}
				$err(error);

			# Sample fields
			sample_gt_f := ((sample_gt) ssep @(end_sample_field)) $err(error);
			sample_dp_f := ((sample_dp) ssep @(end_sample_field)) $err(error);
			sample_gq_f := ((sample_gq) ssep @(end_sample_field)) $err(error);
			sample_ps_f := ((sample_ps) ssep @(end_sample_field)) $err(error);
			sample_pq_f := ((sample_pq) ssep @(end_sample_field)) $err(error);
			sample_mq_f := ((sample_mq) ssep @(end_sample_field)) $err(error);

			# Sample record
			sample_rec_f := "" >to{
				always_assert(m_format_idx < m_format.size(), "Format does not match the sample");

				// Sample index 0 is reserved for the reference.
				++m_sample_idx;

				// Parse according to the format field.
				auto const idx(m_format_idx);
				++m_format_idx;
				auto const field(m_format[idx]);
				switch (field)
				{
					case format_field::GT:
						fgoto sample_gt_f;

					case format_field::DP:
						fgoto sample_dp_f;

					case format_field::GQ:
						fgoto sample_gq_f;

					case format_field::PS:
						fgoto sample_ps_f;

					case format_field::PQ:
						fgoto sample_pq_f;

					case format_field::MQ:
						fgoto sample_mq_f;

					default:
						fail("Unexpected format value");
				}
			};

			dummy := any
				@{
					// Dummy action,
					// make sure that all the machines are instantiated.
					if (false)
					{
						fgoto chrom_id_f;
						fgoto pos_f;
						fgoto id_f;
						fgoto ref_f;
						fgoto alt_f;
						fgoto qual_f;
						fgoto filter_f;
						fgoto info_f;
						fgoto format_f;
						fgoto sample_rec_f;

						fgoto sample_gt_f;
						fgoto sample_dp_f;
						fgoto sample_gq_f;
						fgoto sample_ps_f;
						fgoto sample_pq_f;
						fgoto sample_mq_f;
					}
				};

			write init;
			write exec;
		}%%

		return retval;
	}
}


slightly more readable?

slightly more readable?

git-svn-id: 42af35d80111c0a63d26505f5f279a5822402eca@651 19e92222-5c0b-0410-8929-a290d50e31e9

/**
 * Copyright (c) 2005 Zed A. Shaw
 * You can redistribute it and/or modify it under the same terms as Ruby.
 */
#include "http11_parser.h"
#include <stdio.h>
#include <assert.h>
#include <stdlib.h>
#include <ctype.h>
#include <string.h>

#define LEN(AT, FPC) (FPC - buffer - parser->AT)
#define MARK(M,FPC) (parser->M = (FPC) - buffer)
#define PTR_TO(F) (buffer + parser->F)

/** machine **/
%%{
  machine http_parser;

  action mark {MARK(mark, fpc); }

  action start_field { MARK(field_start, fpc); }
  action write_field {
    parser->field_len = LEN(field_start, fpc);
  }

  action start_value { MARK(mark, fpc); }
  action write_value {
    if(parser->http_field != NULL) {
      parser->http_field(parser->data, PTR_TO(field_start), parser->field_len, PTR_TO(mark), LEN(mark, fpc));
    }
  }
  action request_method {
    if(parser->request_method != NULL)
      parser->request_method(parser->data, PTR_TO(mark), LEN(mark, fpc));
  }
  action request_uri {
    if(parser->request_uri != NULL)
      parser->request_uri(parser->data, PTR_TO(mark), LEN(mark, fpc));
  }
  action fragment {
    if(parser->fragment != NULL)
      parser->fragment(parser->data, PTR_TO(mark), LEN(mark, fpc));
  }

  action start_query {MARK(query_start, fpc); }
  action query_string {
    if(parser->query_string != NULL)
      parser->query_string(parser->data, PTR_TO(query_start), LEN(query_start, fpc));
  }

  action http_version {
    if(parser->http_version != NULL)
      parser->http_version(parser->data, PTR_TO(mark), LEN(mark, fpc));
  }

  action request_path {
    if(parser->request_path != NULL)
      parser->request_path(parser->data, PTR_TO(mark), LEN(mark,fpc));
  }

  action done {
    parser->body_start = fpc - buffer + 1;
    if(parser->header_done != NULL)
      parser->header_done(parser->data, fpc + 1, pe - fpc - 1);
    fbreak;
  }

#### HTTP PROTOCOL GRAMMAR
# line endings
  CRLF = "\r\n";

# character types
  CTL = (cntrl | 127);
  safe = ("$" | "-" | "_" | ".");
  extra = ("!" | "*" | "'" | "(" | ")" | ",");
  reserved = (";" | "/" | "?" | ":" | "@" | "&" | "=" | "+");
  unsafe = (CTL | " " | "\"" | "#" | "%" | "<" | ">");
  national = any -- (alpha | digit | reserved | extra | safe | unsafe);
  unreserved = (alpha | digit | safe | extra | national);
  escape = ("%" xdigit xdigit);
  uchar = (unreserved | escape);
  pchar = (uchar | ":" | "@" | "&" | "=" | "+");
  tspecials = ("(" | ")" | "<" | ">" | "@" | "," | ";" | ":" | "\\" | "\"" | "/" | "[" | "]" | "?" | "=" | "{" | "}" | " " | "\t");

# elements
  token = (ascii -- (CTL | tspecials));

# URI schemes and absolute paths
  scheme = ( alpha | digit | "+" | "-" | "." )* ;
  absolute_uri = (scheme ":" (uchar | reserved )*);

  path = ( pchar+ ( "/" pchar* )* ) ;
  query = ( uchar | reserved )* %query_string ;
  param = ( pchar | "/" )* ;
  params = ( param ( ";" param )* ) ;
  rel_path = ( path? %request_path (";" params)? ) ("?" %start_query query)?;
  absolute_path = ( "/"+ rel_path );

  Request_URI = ( "*" | absolute_uri | absolute_path ) >mark %request_uri;
  Fragment = ( uchar | reserved )* >mark %fragment;
  Method = ( upper | digit | safe ){1,20} >mark %request_method;

  http_number = ( digit+ "." digit+ ) ;
  HTTP_Version = ( "HTTP/" http_number ) >mark %http_version ;
  Request_Line = ( Method " " Request_URI ("#" Fragment){0,1} " " HTTP_Version CRLF ) ;

  field_name = ( token -- ":" )+ >start_field %write_field;

  field_value = any* >start_value %write_value;

  message_header = field_name ":" " "* field_value :> CRLF;

  Request = Request_Line ( message_header )* ( CRLF @done );

main := Request;
}%%

/** Data **/
%% write data;

int http_parser_init(http_parser *parser)  {
  int cs = 0;
  %% write init;
  parser->cs = cs;
  parser->body_start = 0;
  parser->content_len = 0;
  parser->mark = 0;
  parser->nread = 0;
  parser->field_len = 0;
  parser->field_start = 0;

  return(1);
}

/** exec **/
size_t http_parser_execute(http_parser *parser, const char *buffer, size_t len, size_t off)  {
  const char *p, *pe;
  int cs = parser->cs;

  assert(off <= len && "offset past end of buffer");

  p = buffer+off;
  pe = buffer+len;

  assert(*pe == '\0' && "pointer does not end on NUL");
  assert(pe - p == len - off && "pointers aren't same distance");

  %% write exec;

  parser->cs = cs;
  parser->nread += p - (buffer + off);

  assert(p <= pe && "buffer overflow after parsing execute");
  assert(parser->nread <= len && "nread longer than length");
  assert(parser->body_start <= len && "body starts after buffer end");
  assert(parser->mark < len && "mark is after buffer end");
  assert(parser->field_len <= len && "field has length longer than whole buffer");
  assert(parser->field_start < len && "field starts after buffer end");

  if(parser->body_start) {
    /* final \r\n combo encountered so stop right here */
    %%write eof;
    parser->nread++;
  }

  return(parser->nread);
}

int http_parser_finish(http_parser *parser)
{
  int cs = parser->cs;

  %%write eof;

  parser->cs = cs;

  if (http_parser_has_error(parser) ) {
    return -1;
  } else if (http_parser_is_finished(parser) ) {
    return 1;
  } else {
    return 0;
  }
}

int http_parser_has_error(http_parser *parser) {
  return parser->cs == http_parser_error;
}

int http_parser_is_finished(http_parser *parser) {
  return parser->cs == http_parser_first_final;
}

/**
 * Copyright (c) 2005 Zed A. Shaw
 * You can redistribute it and/or modify it under the same terms as Ruby.
 */
#include "http11_parser.h"
#include <stdio.h>
#include <assert.h>
#include <stdlib.h>
#include <ctype.h>
#include <string.h>

#define LEN(AT, FPC) (FPC - buffer - parser->AT)
#define MARK(M,FPC) (parser->M = (FPC) - buffer)
#define PTR_TO(F) (buffer + parser->F)

/** machine **/
%%{
  machine http_parser;

  action mark {MARK(mark, fpc); }

  action start_field { MARK(field_start, fpc); }
  action write_field {
    parser->field_len = LEN(field_start, fpc);
  }

  action start_value { MARK(mark, fpc); }
  action write_value {
    if(parser->http_field != NULL) {
      parser->http_field(parser->data, PTR_TO(field_start), parser->field_len, PTR_TO(mark), LEN(mark, fpc));
    }
  }
  action request_method {
    if(parser->request_method != NULL)
      parser->request_method(parser->data, PTR_TO(mark), LEN(mark, fpc));
  }
  action request_uri {
    if(parser->request_uri != NULL)
      parser->request_uri(parser->data, PTR_TO(mark), LEN(mark, fpc));
  }
  action fragment {
    if(parser->fragment != NULL)
      parser->fragment(parser->data, PTR_TO(mark), LEN(mark, fpc));
  }

  action start_query {MARK(query_start, fpc); }
  action query_string {
    if(parser->query_string != NULL)
      parser->query_string(parser->data, PTR_TO(query_start), LEN(query_start, fpc));
  }

  action http_version {
    if(parser->http_version != NULL)
      parser->http_version(parser->data, PTR_TO(mark), LEN(mark, fpc));
  }

  action request_path {
    if(parser->request_path != NULL)
      parser->request_path(parser->data, PTR_TO(mark), LEN(mark,fpc));
  }

  action done {
    parser->body_start = fpc - buffer + 1;
    if(parser->header_done != NULL)
      parser->header_done(parser->data, fpc + 1, pe - fpc - 1);
    fbreak;
  }

#### HTTP PROTOCOL GRAMMAR
# line endings
  CRLF = "\r\n";

# character types
  CTL = (cntrl | 127);
  safe = ("$" | "-" | "_" | ".");
  extra = ("!" | "*" | "'" | "(" | ")" | ",");
  reserved = (";" | "/" | "?" | ":" | "@" | "&" | "=" | "+");
  unsafe = (CTL | " " | "\"" | "#" | "%" | "<" | ">");
  national = any -- (alpha | digit | reserved | extra | safe | unsafe);
  unreserved = (alpha | digit | safe | extra | national);
  escape = ("%" xdigit xdigit);
  uchar = (unreserved | escape);
  pchar = (uchar | ":" | "@" | "&" | "=" | "+");
  tspecials = ("(" | ")" | "<" | ">" | "@" | "," | ";" | ":" | "\\" | "\"" | "/" | "[" | "]" | "?" | "=" | "{" | "}" | " " | "\t");

# elements
  token = (ascii -- (CTL | tspecials));

# URI schemes and absolute paths
  scheme = ( alpha | digit | "+" | "-" | "." )* ;
  absolute_uri = (scheme ":" (uchar | reserved )*);

  path = (pchar+ ( "/" pchar* )*) ;
  query = ( uchar | reserved )* %query_string ;
  param = ( pchar | "/" )* ;
  params = (param ( ";" param )*) ;
  rel_path = (path? %request_path (";" params)?) ("?" %start_query query)?;
  absolute_path = ("/"+ rel_path);

  Request_URI = ("*" | absolute_uri | absolute_path) >mark %request_uri;
  Fragment = ( uchar | reserved )* >mark %fragment;
  Method = (upper | digit | safe){1,20} >mark %request_method;

  http_number = (digit+ "." digit+) ;
  HTTP_Version = ("HTTP/" http_number) >mark %http_version ;
  Request_Line = (Method " " Request_URI ("#" Fragment ){0,1} " " HTTP_Version CRLF) ;

  field_name = (token -- ":")+ >start_field %write_field;

  field_value = any* >start_value %write_value;

  message_header = field_name ":" " "* field_value :> CRLF;

  Request = Request_Line (message_header)* ( CRLF @done);

main := Request;
}%%

/** Data **/
%% write data;

int http_parser_init(http_parser *parser)  {
  int cs = 0;
  %% write init;
  parser->cs = cs;
  parser->body_start = 0;
  parser->content_len = 0;
  parser->mark = 0;
  parser->nread = 0;
  parser->field_len = 0;
  parser->field_start = 0;

  return(1);
}

/** exec **/
size_t http_parser_execute(http_parser *parser, const char *buffer, size_t len, size_t off)  {
  const char *p, *pe;
  int cs = parser->cs;

  assert(off <= len && "offset past end of buffer");

  p = buffer+off;
  pe = buffer+len;

  assert(*pe == '\0' && "pointer does not end on NUL");
  assert(pe - p == len - off && "pointers aren't same distance");

  %% write exec;

  parser->cs = cs;
  parser->nread += p - (buffer + off);

  assert(p <= pe && "buffer overflow after parsing execute");
  assert(parser->nread <= len && "nread longer than length");
  assert(parser->body_start <= len && "body starts after buffer end");
  assert(parser->mark < len && "mark is after buffer end");
  assert(parser->field_len <= len && "field has length longer than whole buffer");
  assert(parser->field_start < len && "field starts after buffer end");

  if(parser->body_start) {
    /* final \r\n combo encountered so stop right here */
    %%write eof;
    parser->nread++;
  }

  return(parser->nread);
}

int http_parser_finish(http_parser *parser)
{
  int cs = parser->cs;

  %%write eof;

  parser->cs = cs;

  if (http_parser_has_error(parser) ) {
    return -1;
  } else if (http_parser_is_finished(parser) ) {
    return 1;
  } else {
    return 0;
  }
}

int http_parser_has_error(http_parser *parser) {
  return parser->cs == http_parser_error;
}

int http_parser_is_finished(http_parser *parser) {
  return parser->cs == http_parser_first_final;
}


Faster ragel

Faster ragel

#include "../common.h"

#define P(C) fputs(C, stdout);
#define INDENT fputs("   ", stdout);
#define ON echo = 1;
#define OFF echo = 0;

int echo = 0;

%%{
  machine csv2json;

  action dump {
    if(echo) {
      fprintf(stdout, "%c", fc);
    }
  }

  ipVal     = ([0-9]{1,3} '.' [0-9]{1,3} '.' [0-9]{1,3} '.' [0-9]{1,3}) > { P("\"") } % { P("\"") };
  stringVal = [^,\n]* > { P("\"") } % { P("\"") };
  numVal    = [0-9]+;
  row       = numVal    > { ON; INDENT; P("\"id\"         : "); } % { OFF; P(",\n"); } ','
              stringVal > { ON; INDENT; P("\"first_name\" : "); } % { OFF; P(",\n"); } ','
              stringVal > { ON; INDENT; P("\"last_name\"  : "); } % { OFF; P(",\n"); } ','
              stringVal > { ON; INDENT; P("\"email\"      : "); } % { OFF; P(",\n"); } ','
              stringVal > { ON; INDENT; P("\"country\"    : "); } % { OFF; P(",\n"); } ','
              ipVal     > { ON; INDENT; P("\"ip\"         : "); } % { OFF; P("\n"); };
  csv2json  = row > { P("{\n") } % { P("}\n") } '\n';

  main := csv2json $ dump;
}%%

%% write data;

int main(int argc, char **argv) {

  PRE;

  while(fgets(buffer, sizeof(buffer), stdin)) {
    INIT_LINE;
    %% write init;
    %% write exec;

    if(p != pe) {
      FAIL;
    }
  }

  POST;

  return 0;
}

#include "../common.h"

#define P(C) fputs(C, stdout);
#define INDENT fputs("   ", stdout);
#define ON echo = 1;
#define OFF echo = 0;

int echo = 0;

%%{
  machine csv2json;

  action dump {
    if(echo) {
      fprintf(stdout, "%.*s", 1, &fc);
    }
  }

  ipVal     = ([0-9]{1,3} '.' [0-9]{1,3} '.' [0-9]{1,3} '.' [0-9]{1,3}) > { P("\"") } % { P("\"") };
  stringVal = [^,\n]* > { P("\"") } % { P("\"") };
  numVal    = [0-9]+;
  row       = numVal    > { ON; INDENT; P("\"id\"         : "); } % { OFF; P(",\n"); } ','
              stringVal > { ON; INDENT; P("\"first_name\" : "); } % { OFF; P(",\n"); } ','
              stringVal > { ON; INDENT; P("\"last_name\"  : "); } % { OFF; P(",\n"); } ','
              stringVal > { ON; INDENT; P("\"email\"      : "); } % { OFF; P(",\n"); } ','
              stringVal > { ON; INDENT; P("\"country\"    : "); } % { OFF; P(",\n"); } ','
              ipVal     > { ON; INDENT; P("\"ip\"         : "); } % { OFF; P("\n"); };
  csv2json  = row > { P("{\n") } % { P("}\n") } '\n';

  main := csv2json $ dump;
}%%

%% write data;

int main(int argc, char **argv) {

  PRE;

  while(fgets(buffer, sizeof(buffer), stdin)) {
    INIT_LINE;
    %% write init;
    %% write exec;

    if(p != pe) {
      FAIL;
    }
  }

  POST;

  return 0;
}