GCNodeParser: clean up CONT/CONC code

GCNodeParser: clean up CONT/CONC code

/**

Ragel state machine for GEDCOM nodes based on the 5.5 documentation.

gedcom_line :=
level + delim + [xref_id + delim +] tag + [delim + line_value +] terminator
level + delim + optional_xref_id + tag + delim + optional_line_value + terminator

*/

#import "GCNodeParser.h"
#import "GCNodeParserDelegate.h"

#import "GCNode_internal.h"

#import "GedcomErrors.h"

#define DEBUGLINECOUNT 84300

%%{
    machine nodeParser;

	action log {
		//if (lineCount > DEBUGLINECOUNT) NSLog(@"%p log: (%X) '%c'", fpc, fc, fc);
	}

	action tag {
		tag = fpc - data;
        //if (lineCount > DEBUGLINECOUNT) NSLog(@"%p     TAG: %ld", fpc, tag);
	}

	action number {
		long len = (fpc - data) - tag;
		currentNumber = [[[NSString alloc] initWithBytes:fpc-len length:len encoding:NSUTF8StringEncoding] intValue];
		//if (lineCount > DEBUGLINECOUNT) NSLog(@"%p num: %d", fpc, currentNumber);
	}

	action string {
		long len = (fpc - data) - tag;
		currentString = [[NSString alloc] initWithBytes:fpc-len length:len encoding:NSUTF8StringEncoding];
		//if (lineCount > DEBUGLINECOUNT) NSLog(@"%p string: %@", fpc, currentString);
	}

	action saveLevel {
        level = currentNumber;
	}

    action saveXref {
        xref = currentString;
    }

    action saveTagCode {
        code = currentString;
    }

    action saveValue {
        value = [currentString stringByReplacingOccurrencesOfString:@"@@" withString:@"@"]; // unescape at-sign
    }

	action saveNode {
        lineCount++;

        // CONT/CONC nodes are continuations of values from the previous node, so fold them up here:
        if (_contConcMap[code]) {
            if (!currentNode.gedValue) {
                currentNode.gedValue = value;
            } else {
                currentNode.gedValue = [NSString stringWithFormat:@"%@%@%@", currentNode.gedValue, _contConcMap[code], value ? value : @""];
            }
        } else {
            if (xref) {
                node = [GCNode nodeWithTag:code
                                      xref:xref];
            } else {
                node = [GCNode nodeWithTag:code
                                     value:value];
            }

            GCNode *parent = nil;

            if (level == 0) { //root
                if (currentRoot) {
                    nodeCount++;
                    if (_delegate && [_delegate respondsToSelector:@selector(parser:didParseNode:)]) {
                        [_delegate parser:self didParseNode:currentRoot];
                    }
                }
            } else { //find correct parent
                parent = currentNode;
                for (int i = currentLevel; i >= level; i--) {
                    parent = [parent parent];
                }
            }

            if (parent) {
                [parent _addSubNode:node];
            } else {
                currentRoot = node;
            }

            currentLevel = level;
            currentNode = node;
        }

        level = -1;
        xref = nil;
        code = nil;
        value = nil;
	}

	action finish {
        if (currentRoot) {
            nodeCount++;
            if (_delegate && [_delegate respondsToSelector:@selector(parser:didParseNode:)]) {
                [_delegate parser:self didParseNode:currentRoot];
            }
        }
		didFinish = YES;
	}

    delim                   = ' ';
    terminator              = ( '\r' | '\n' )+;

    keyword                 = ( alpha ( alnum | '_' )+ );

    level                   = ( digit+ >tag %number ) %saveLevel;
    xref                    = ( ( '@' keyword '@' ) >tag %string ) %saveXref;
    tagCode                 = ( ( '_'? keyword ) >tag %string ) %saveTagCode;
    value                   = ( ( extend - terminator )+ >tag %string ) %saveValue;

    contents                = ( ( xref delim tagCode ) | ( tagCode delim value ) | ( tagCode delim? ) );

    node                    = ( level delim contents terminator ) %saveNode;

    main                   := ( node+ ) %/finish;

}%%

%% write data;

@implementation GCNodeParser

__strong static id _sharedNodeParser = nil;

+ (void)initialize
{
    _sharedNodeParser = [[self alloc] init];
}

+ (id)sharedNodeParser
{
    return _sharedNodeParser;
}

- (BOOL)parseString:(id)gedString error:(NSError *__autoreleasing *)error
{
    static dispatch_once_t onceToken;
    static NSDictionary *_contConcMap = nil;
    dispatch_once(&onceToken, ^{
        _contConcMap = @{
            @"CONT": @"\n",
            @"CONC": concSeparator,
        };
    });

    @synchronized(self) {
        if ([gedString isKindOfClass:[NSAttributedString class]]) {
            gedString = [gedString string];
        }

        NSParameterAssert([gedString isKindOfClass:[NSString class]]);

        gedString = [gedString stringByAppendingString:@"\n"];

        NSUInteger nodeCount = 0;
        NSUInteger lineCount = 0;
        int currentLevel = 0;
        GCNode *currentNode = nil;
        GCNode *currentRoot = nil;

        long tag = 0;
        int currentNumber = 0;
        NSString *currentString = nil;

        int level = -1;
        NSString *xref = nil;
        NSString *code = nil;
        NSString *value = nil;
        GCNode* node = nil;

        BOOL didFinish = NO;

        int cs = 0;
        const char *data = [gedString UTF8String];
        const char *p = data;
        const char *pe = p + [gedString lengthOfBytesUsingEncoding:NSUTF8StringEncoding];
        const char *eof = pe;

        if (_delegate && [_delegate respondsToSelector:@selector(parser:willParseCharacterCount:)]) {
            [_delegate parser:self willParseCharacterCount:[gedString length]];
        }

#ifdef DEBUGLEVEL
        NSDate *start = [NSDate date];
#endif

        %% write init;
        %% write exec;

#ifdef DEBUGLEVEL
        NSTimeInterval timeInterval = fabs([start timeIntervalSinceNow]);
        NSLog(@"parsed %ld nodes in %ld lines - Time: %f seconds", nodeCount, lineCount, timeInterval);
#endif

        if (_delegate && [_delegate respondsToSelector:@selector(parser:didParseNodesWithCount:)]) {
            [_delegate parser:self didParseNodesWithCount:nodeCount];
        }

        if (!didFinish) {
            if (error != NULL) {
                NSBundle *frameworkBundle = [NSBundle bundleForClass:[self class]];

                NSString *formatString = [frameworkBundle localizedStringForKey:@"Parser stopped at node #%ld (line %ld)"
                                                                          value:@"Parser stopped at node #%ld (line %ld)"
                                                                          table:@"Errors"];
                NSDictionary *userInfo = @{
                                           NSLocalizedDescriptionKey: [NSString stringWithFormat:formatString, nodeCount, lineCount],
                                           NSAffectedObjectsErrorKey: currentNode
                                           };

                *error = [NSError errorWithDomain:GCErrorDomain
                                             code:GCNodeParsingError
                                         userInfo:userInfo];
            }
        }

        return didFinish;
    }
}

@end

@interface _GCNodeParserConvenienceDelegate : NSObject <GCNodeParserDelegate>

@property NSArray *nodes;

@end

@implementation _GCNodeParserConvenienceDelegate {
    NSMutableArray *_nodes;
}

- (void)parser:(GCNodeParser *)parser didParseNode:(GCNode *)node
{
    //NSLog(@"%p got node: %@", self, node);
    if (!_nodes) {
        _nodes = [NSMutableArray array];
    }
    [_nodes addObject:node];
}

- (void)parser:(GCNodeParser *)parser didParseNodesWithCount:(NSUInteger)nodeCount
{
    NSParameterAssert(nodeCount == [_nodes count]);
}

@end

@implementation GCNodeParser (GCConvenienceMethods)

+ (NSArray*)arrayOfNodesFromString:(id)gedString
{
    _GCNodeParserConvenienceDelegate *delegate = [[_GCNodeParserConvenienceDelegate alloc] init];

    GCNodeParser *parser = [[GCNodeParser alloc] init];
    parser.delegate = delegate;

    NSError *error = nil;
    BOOL succeeded = [parser parseString:gedString error:&error];

    if (succeeded) {
        return delegate.nodes;
    } else {
        NSLog(@"error: %@", error);
        return nil;
    }
}

@end

/**

Ragel state machine for GEDCOM nodes based on the 5.5 documentation.

gedcom_line :=
level + delim + [xref_id + delim +] tag + [delim + line_value +] terminator
level + delim + optional_xref_id + tag + delim + optional_line_value + terminator

*/

#import "GCNodeParser.h"
#import "GCNodeParserDelegate.h"

#import "GCNode_internal.h"

#import "GedcomErrors.h"

#define DEBUGLINECOUNT 84300

%%{
    machine nodeParser;

	action log {
		//if (lineCount > DEBUGLINECOUNT) NSLog(@"%p log: (%X) '%c'", fpc, fc, fc);
	}

	action tag {
		tag = fpc - data;
        //if (lineCount > DEBUGLINECOUNT) NSLog(@"%p     TAG: %ld", fpc, tag);
	}

	action number {
		long len = (fpc - data) - tag;
		currentNumber = [[[NSString alloc] initWithBytes:fpc-len length:len encoding:NSUTF8StringEncoding] intValue];
		//if (lineCount > DEBUGLINECOUNT) NSLog(@"%p num: %d", fpc, currentNumber);
	}

	action string {
		long len = (fpc - data) - tag;
		currentString = [[NSString alloc] initWithBytes:fpc-len length:len encoding:NSUTF8StringEncoding];
		//if (lineCount > DEBUGLINECOUNT) NSLog(@"%p string: %@", fpc, currentString);
	}

	action saveLevel {
        level = currentNumber;
	}

    action saveXref {
        xref = currentString;
    }

    action saveTagCode {
        code = currentString;
    }

    action saveValue {
        value = [currentString stringByReplacingOccurrencesOfString:@"@@" withString:@"@"]; // unescape at-sign
    }

	action saveNode {
        lineCount++;

        // CONT/CONC nodes are continuations of values from the previous node, so fold them up here:
        // TODO: refactor (use dict?)
        if ([code isEqualToString:@"CONT"]) {
            if (!currentNode.gedValue) {
                currentNode.gedValue = value;
            } else {
                currentNode.gedValue = [NSString stringWithFormat:@"%@%@%@", currentNode.gedValue, @"\n", value ? value : @""];
            }
        } else if ([code isEqualToString:@"CONC"]) {
            if (!currentNode.gedValue) {
                currentNode.gedValue = value;
            } else {
                currentNode.gedValue = [NSString stringWithFormat:@"%@%@%@", currentNode.gedValue, concSeparator, value ? value : @""];
            }
        } else {
            if (xref) {
                node = [GCNode nodeWithTag:code
                                      xref:xref];
            } else {
                node = [GCNode nodeWithTag:code
                                     value:value];
            }

            GCNode *parent = nil;

            if (level == 0) { //root
                if (currentRoot) {
                    nodeCount++;
                    if (_delegate && [_delegate respondsToSelector:@selector(parser:didParseNode:)]) {
                        [_delegate parser:self didParseNode:currentRoot];
                    }
                }
            } else { //find correct parent
                parent = currentNode;
                for (int i = currentLevel; i >= level; i--) {
                    parent = [parent parent];
                }
            }

            if (parent) {
                [parent _addSubNode:node];
            } else {
                currentRoot = node;
            }

            currentLevel = level;
            currentNode = node;
        }

        level = -1;
        xref = nil;
        code = nil;
        value = nil;
	}

	action finish {
        if (currentRoot) {
            nodeCount++;
            if (_delegate && [_delegate respondsToSelector:@selector(parser:didParseNode:)]) {
                [_delegate parser:self didParseNode:currentRoot];
            }
        }
		didFinish = YES;
	}

    delim                   = ' ';
    terminator              = ( '\r' | '\n' )+;

    keyword                 = ( alpha ( alnum | '_' )+ );

    level                   = ( digit+ >tag %number ) %saveLevel;
    xref                    = ( ( '@' keyword '@' ) >tag %string ) %saveXref;
    tagCode                 = ( ( '_'? keyword ) >tag %string ) %saveTagCode;
    value                   = ( ( extend - terminator )+ >tag %string ) %saveValue;

    contents                = ( ( xref delim tagCode ) | ( tagCode delim value ) | ( tagCode delim? ) );

    node                    = ( level delim contents terminator ) %saveNode;

    main                   := ( node+ ) %/finish;

}%%

%% write data;

@implementation GCNodeParser

__strong static id _sharedNodeParser = nil;

+ (void)initialize
{
    _sharedNodeParser = [[self alloc] init];
}

+ (id)sharedNodeParser
{
    return _sharedNodeParser;
}

- (BOOL)parseString:(id)gedString error:(NSError *__autoreleasing *)error
{
    @synchronized(self) {
        if ([gedString isKindOfClass:[NSAttributedString class]]) {
            gedString = [gedString string];
        }

        NSParameterAssert([gedString isKindOfClass:[NSString class]]);

        gedString = [gedString stringByAppendingString:@"\n"];

        NSUInteger nodeCount = 0;
        NSUInteger lineCount = 0;
        int currentLevel = 0;
        GCNode *currentNode = nil;
        GCNode *currentRoot = nil;

        long tag = 0;
        int currentNumber = 0;
        NSString *currentString = nil;

        int level = -1;
        NSString *xref = nil;
        NSString *code = nil;
        NSString *value = nil;
        GCNode* node = nil;

        BOOL didFinish = NO;

        int cs = 0;
        const char *data = [gedString UTF8String];
        const char *p = data;
        const char *pe = p + [gedString lengthOfBytesUsingEncoding:NSUTF8StringEncoding];
        const char *eof = pe;

        if (_delegate && [_delegate respondsToSelector:@selector(parser:willParseCharacterCount:)]) {
            [_delegate parser:self willParseCharacterCount:[gedString length]];
        }

#ifdef DEBUGLEVEL
        NSDate *start = [NSDate date];
#endif

        %% write init;
        %% write exec;

#ifdef DEBUGLEVEL
        NSTimeInterval timeInterval = fabs([start timeIntervalSinceNow]);
        NSLog(@"parsed %ld nodes in %ld lines - Time: %f seconds", nodeCount, lineCount, timeInterval);
#endif

        if (_delegate && [_delegate respondsToSelector:@selector(parser:didParseNodesWithCount:)]) {
            [_delegate parser:self didParseNodesWithCount:nodeCount];
        }

        if (!didFinish) {
            if (error != NULL) {
                NSBundle *frameworkBundle = [NSBundle bundleForClass:[self class]];

                NSString *formatString = [frameworkBundle localizedStringForKey:@"Parser stopped at node #%ld (line %ld)"
                                                                          value:@"Parser stopped at node #%ld (line %ld)"
                                                                          table:@"Errors"];
                NSDictionary *userInfo = @{
                                           NSLocalizedDescriptionKey: [NSString stringWithFormat:formatString, nodeCount, lineCount],
                                           NSAffectedObjectsErrorKey: currentNode
                                           };

                *error = [NSError errorWithDomain:GCErrorDomain
                                             code:GCNodeParsingError
                                         userInfo:userInfo];
            }
        }

        return didFinish;
    }
}

@end

@interface _GCNodeParserConvenienceDelegate : NSObject <GCNodeParserDelegate>

@property NSArray *nodes;

@end

@implementation _GCNodeParserConvenienceDelegate {
    NSMutableArray *_nodes;
}

- (void)parser:(GCNodeParser *)parser didParseNode:(GCNode *)node
{
    //NSLog(@"%p got node: %@", self, node);
    if (!_nodes) {
        _nodes = [NSMutableArray array];
    }
    [_nodes addObject:node];
}

- (void)parser:(GCNodeParser *)parser didParseNodesWithCount:(NSUInteger)nodeCount
{
    NSParameterAssert(nodeCount == [_nodes count]);
}

@end

@implementation GCNodeParser (GCConvenienceMethods)

+ (NSArray*)arrayOfNodesFromString:(id)gedString
{
    _GCNodeParserConvenienceDelegate *delegate = [[_GCNodeParserConvenienceDelegate alloc] init];

    GCNodeParser *parser = [[GCNodeParser alloc] init];
    parser.delegate = delegate;

    NSError *error = nil;
    BOOL succeeded = [parser parseString:gedString error:&error];

    if (succeeded) {
        return delegate.nodes;
    } else {
        NSLog(@"error: %@", error);
        return nil;
    }
}

@end


Ragel works for all test cases and does embedded quotes!

# Parser for VCF-header info and format fields

=begin
%%{

  machine simple_lexer;

  action mark { ts=p }
  action Start {
    ts = p
  }
  action Stop {
    quoted_text = data[ts...p].pack('c*')
    # do something with the quoted text!
    emit('string',data,ts,p)
  }

  squote = "'";
  dquote = '"';
  not_squote_or_escape = [^'\\];
  not_dquote_or_escape = [^"\\];
  escaped_something = /\\./;
  ss = space* squote ( not_squote_or_escape | escaped_something )* >Start %Stop squote;
  dd = space* dquote ( not_dquote_or_escape | escaped_something )* >Start %Stop dquote;

  # main := (ss | dd)*;

  integer     = ('+'|'-')?[0-9]+             >mark %{ emit("integer",data,ts,p)  };
  float       = ('+'|'-')?[0-9]+'.'[0-9]+    >mark %{ emit("float",data,ts,p) };
  assignment  = '=';
  identifier  = ([a-zA-Z][a-zA-Z_0-9]+)         >mark %{ emit("identifier",data,ts,p) };
  str         = (ss|dd)* ;
  boolean     = '.'                          >mark %{ emit("bool",data,ts,p) };
  key_word    = ( ('ID'|'Number'|'Type'|'Description') >mark %{ print "*k:"; emit("key_word",data,ts,p) } );
  value       = ( (integer|float|boolean|identifier|str) >mark %{ print "*v:" ; emit("value",data,ts,p) } );

  key_value = ( key_word '=' value ) ;

  main := ( ('<'|',') key_value )*;
}%%
=end

%% write data;
# %% this just fixes our syntax highlighting...

def emit type, data, ts, p
  # Print the type and text of the last read token
  # p ts,p
  puts "#{type}: #{data[ts...p].pack('c*')}"
end

def xemit(token_name, data, target_array, ts, te)
  target_array << {:name => token_name.to_sym, :value => data.pack("c*")[ts...te] }
end

def run_lexer(data)
  p data
  data = data.unpack("c*") if(data.is_a?(String))
  eof = data.length
  token_array = []
  stack = []

  %% write init;
  %% write exec;

  p token_array
  token_array.each do | h |
    p h
    # p h[:value] if h[:name]==:identifier or h[:name]==:value or h[:name]==:string
  end
end

# run_lexer("value = -2.00")

lines = <<LINES
##FORMAT=<ID=GT,Number=1,Type=String,Description="Genotype">
##FORMAT=<ID=DP,Number=1,Type=Integer,Description="Total read depth">
##FORMAT=<ID=DP4,Number=4,Type=Integer,Description="# high-quality ref-forward bases, ref-reverse, alt-forward and alt-reverse bases">
##INFO=<ID=PM,Number=0,Type=Flag,Description="Variant is Precious(Clinical,Pubmed Cited)">
##INFO=<ID=VP,Number=1,Type=String,Description="Variation Property.  Documentation is at ftp://ftp.ncbi.nlm.nih.gov/snp/specs/dbSNP_BitField_latest.pdf">
##INFO=<ID=GENEINFO,Number=1,Type=String,Description="Pairs each of gene symbol:gene id.  The gene symbol and id are delimited by a colon (:), and each pair is delimited by a vertical bar (|)">
##INFO=<ID=CLNHGVS,Number=.,Type=String,Description="Variant names from HGVS. The order of these variants corresponds to the order of the info in the other clinical  INFO tags.">
##INFO=<ID=CLNHGVS1,Number=.,Type=String,Description="Variant names from \"HGVS\". The order of these 'variants' corresponds to the order of the info in the other clinical  INFO tags.">
LINES

lines.strip.split("\n").each { |s| run_lexer(s.sub(/^##(FORMAT|INFO)=/,'')) }

cmd=ARGV.shift
run_lexer(cmd)

# Parser for VCF-header info and format fields

=begin
%%{

  machine simple_lexer;

  action mark { ts=p }
  action Start {
    ts = p
  }
  action Stop {
    quoted_text = data[ts...p].pack('c*')
    # do something with the quoted text!
    emit('string',data,ts,p)
  }

  squote = "'";
  dquote = '"';
  not_squote_or_escape = [^'\\];
  not_dquote_or_escape = [^"\\];
  escaped_something = /\\./;
  ss = space* squote ( not_squote_or_escape | escaped_something )* >Start %Stop squote;
  dd = space* dquote ( not_dquote_or_escape | escaped_something )* >Start %Stop dquote;

  # main := (ss | dd)*;

  integer     = ('+'|'-')?[0-9]+             >mark %{ emit("integer",data,ts,p)  };
  float       = ('+'|'-')?[0-9]+'.'[0-9]+    >mark %{ emit("float",data,ts,p) };
  assignment  = '=';
  identifier  = ([a-zA-Z][a-zA-Z_0-9]+)         >mark %{ emit("identifier",data,ts,p) };
  str         = (ss|dd)* ;
  boolean     = '.'                          >mark %{ emit("bool",data,ts,p) };
  key_word    = ( ('ID'|'Number'|'Type'|'Description') >mark %{ print "*k:"; emit("key_word",data,ts,p) } );
  value       = ( (integer|float|boolean|identifier|str) >mark %{ print "*v:" ; emit("value",data,ts,p) } );

  key_value = ( key_word '=' value ) ;

  main := ( ('<'|',') key_value )*;
}%%
=end

%% write data;
# %% this just fixes our syntax highlighting...

def emit type, data, ts, p
  # Print the type and text of the last read token
  # p ts,p
  puts "#{type}: #{data[ts...p].pack('c*')}"
end

def xemit(token_name, data, target_array, ts, te)
  target_array << {:name => token_name.to_sym, :value => data.pack("c*")[ts...te] }
end

def run_lexer(data)
  p data
  data = data.unpack("c*") if(data.is_a?(String))
  eof = data.length
  token_array = []
  stack = []

  %% write init;
  %% write exec;

  p token_array
  token_array.each do | h |
    p h
    # p h[:value] if h[:name]==:identifier or h[:name]==:value or h[:name]==:string
  end
end

# run_lexer("value = -2.00")

lines = <<LINES
##FORMAT=<ID=GT,Number=1,Type=String,Description="Genotype">
##FORMAT=<ID=DP,Number=1,Type=Integer,Description="Total read depth">
##FORMAT=<ID=DP4,Number=4,Type=Integer,Description="# high-quality ref-forward bases, ref-reverse, alt-forward and alt-reverse bases">
##INFO=<ID=PM,Number=0,Type=Flag,Description="Variant is Precious(Clinical,Pubmed Cited)">
##INFO=<ID=VP,Number=1,Type=String,Description="Variation Property.  Documentation is at ftp://ftp.ncbi.nlm.nih.gov/snp/specs/dbSNP_BitField_latest.pdf">
##INFO=<ID=GENEINFO,Number=1,Type=String,Description="Pairs each of gene symbol:gene id.  The gene symbol and id are delimited by a colon (:), and each pair is delimited by a vertical bar (|)">
##INFO=<ID=CLNHGVS,Number=.,Type=String,Description="Variant names from HGVS. The order of these variants corresponds to the order of the info in the other clinical  INFO tags.">
LINES

lines.strip.split("\n").each { |s| run_lexer(s.sub(/^##(FORMAT|INFO)=/,'')) }

cmd=ARGV.shift
run_lexer(cmd)


ragel

ragel

# Parser for VCF-header info and format fields

=begin
%%{

  machine simple_lexer;

  integer     = ('+'|'-')?[0-9]+;
  float       = ('+'|'-')?[0-9]+'.'[0-9]+;
  assignment  = '=';
  identifier  = [a-zA-Z][a-zA-Z_]+;
  string      = ["][^"]*["];
  boolean     = '.';
  id_kw       = 'ID';

  action call_pairs { fcall pairs ; }

  # keyword id pair parser
  # pairs := ( id_kw );
  pairs := |*
    id_kw => {
               emit(:identifier, data, token_array, ts, te)
             };
  *|;

  # parser
  main := '<'  @ call_pairs ;
}%%
=end

%% write data;
# %% this just fixes our syntax highlighting...

def emit(token_name, data, target_array, ts, te)
  target_array << {:name => token_name.to_sym, :value => data.pack("c*")[ts...te] }
end

def run_lexer(data)
  p data
  data = data.unpack("c*") if(data.is_a?(String))
  eof = data.length
  token_array = []
  stack = []

  %% write init;
  %% write exec;

  p token_array
  token_array.each do | h |
    p h
    # p h[:value] if h[:name]==:identifier or h[:name]==:value or h[:name]==:string
  end
end

# run_lexer("value = -2.00")

lines = <<LINES
##FORMAT=<ID=GT,Number=1,Type=String,Description="Genotype">
##FORMAT=<ID=DP,Number=1,Type=Integer,Description="Total read depth">
##FORMAT=<ID=DP4,Number=4,Type=Integer,Description="# high-quality ref-forward bases, ref-reverse, alt-forward and alt-reverse bases">
##INFO=<ID=PM,Number=0,Type=Flag,Description="Variant is Precious(Clinical,Pubmed Cited)">
##INFO=<ID=VP,Number=1,Type=String,Description="Variation Property.  Documentation is at ftp://ftp.ncbi.nlm.nih.gov/snp/specs/dbSNP_BitField_latest.pdf">
##INFO=<ID=GENEINFO,Number=1,Type=String,Description="Pairs each of gene symbol:gene id.  The gene symbol and id are delimited by a colon (:), and each pair is delimited by a vertical bar (|)">
##INFO=<ID=CLNHGVS,Number=.,Type=String,Description="Variant names from HGVS. The order of these variants corresponds to the order of the info in the other clinical  INFO tags.">
LINES

lines.strip.split("\n").each { |s| run_lexer(s.sub(/^##(FORMAT|INFO)=/,'')) }

cmd=ARGV.shift
run_lexer(cmd)

  # key := (

    # integer => {
    #   emit(:integer_literal, data, token_array, ts, te)
    # };

    # float => {
    #   emit(:float_literal, data, token_array, ts, te)
    # };

    # assignment => {
    #   emit(:assignment_operator, data, token_array, ts, te)
    # };

    # boolean => {
    #   emit(:boolean, data, token_array, ts, te)
    # };

    # identifier => {
    #   emit(:identifier, data, token_array, ts, te)
    # };

    # string => {
    #   emit(:string, data, token_array, ts, te)
    # };

    # space|',';

  # );

# Parser for VCF-header info and format fields

=begin
%%{

  machine simple_lexer;

  integer     = ('+'|'-')?[0-9]+;
  float       = ('+'|'-')?[0-9]+'.'[0-9]+;
  assignment  = '=';
  identifier  = [a-zA-Z][a-zA-Z_]+;
  string      = ["][^"]*["];
  boolean     = '.';

  main := |*

    integer => {
      emit(:integer_literal, data, token_array, ts, te)
    };

    float => {
      emit(:float_literal, data, token_array, ts, te)
    };

    assignment => {
      emit(:assignment_operator, data, token_array, ts, te)
    };

    boolean => {
      emit(:boolean, data, token_array, ts, te)
    };

    identifier => {
      emit(:identifier, data, token_array, ts, te)
    };

    string => {
      emit(:string, data, token_array, ts, te)
    };

    space|',';

  *|;

}%%
=end

%% write data;
# %% this just fixes our syntax highlighting...

def emit(token_name, data, target_array, ts, te)
  target_array << {:name => token_name.to_sym, :value => data[ts...te].pack("c*") }
end

def run_lexer(data)
  p data
  data = data.unpack("c*") if(data.is_a?(String))
  eof = data.length
  token_array = []

  %% write init;
  %% write exec;

  p token_array
  token_array.each do | h |
    # p h
    p h[:value] if h[:name]==:identifier or h[:name]==:value or h[:name]==:string
  end
end

# run_lexer("value = -2.00")

lines = <<LINES
##FORMAT=<ID=GT,Number=1,Type=String,Description="Genotype">
##FORMAT=<ID=DP,Number=1,Type=Integer,Description="Total read depth">
##FORMAT=<ID=DP4,Number=4,Type=Integer,Description="# high-quality ref-forward bases, ref-reverse, alt-forward and alt-reverse bases">
##INFO=<ID=PM,Number=0,Type=Flag,Description="Variant is Precious(Clinical,Pubmed Cited)">
##INFO=<ID=VP,Number=1,Type=String,Description="Variation Property.  Documentation is at ftp://ftp.ncbi.nlm.nih.gov/snp/specs/dbSNP_BitField_latest.pdf">
##INFO=<ID=GENEINFO,Number=1,Type=String,Description="Pairs each of gene symbol:gene id.  The gene symbol and id are delimited by a colon (:), and each pair is delimited by a vertical bar (|)">
##INFO=<ID=CLNHGVS,Number=.,Type=String,Description="Variant names from HGVS. The order of these variants corresponds to the order of the info in the other clinical  INFO tags.">
LINES

lines.split("\n").each { |s| run_lexer(s.sub(/^##(FORMAT|INFO)=</,'')) }

cmd=ARGV.shift
run_lexer(cmd)