Ragel works for all test cases

Ragel works for all test cases

# Parser for VCF-header info and format fields

=begin
%%{

  machine simple_lexer;

  action mark { ts=p }
  action Start {
    ts = p
  }
  action Stop {
    quoted_text = data[ts...p].pack('c*')
    # do something with the quoted text!
    emit('string',data,ts,p)
  }

  squote = "'";
  dquote = '"';
  not_squote_or_escape = [^'\\];
  not_dquote_or_escape = [^"\\];
  escaped_something = /\\./;
  ss = space* squote ( not_squote_or_escape | escaped_something )* >Start %Stop squote;
  dd = space* dquote ( not_dquote_or_escape | escaped_something )* >Start %Stop dquote;

  # main := (ss | dd)*;

  integer     = ('+'|'-')?[0-9]+             >mark %{ emit("integer",data,ts,p)  };
  float       = ('+'|'-')?[0-9]+'.'[0-9]+    >mark %{ emit("float",data,ts,p) };
  assignment  = '=';
  identifier  = ([a-zA-Z][a-zA-Z_0-9]+)         >mark %{ emit("identifier",data,ts,p) };
  str         = (ss|dd)* ;
  boolean     = '.'                          >mark %{ emit("bool",data,ts,p) };
  key_word    = ( ('ID'|'Number'|'Type'|'Description') >mark %{ print "*k:"; emit("key_word",data,ts,p) } );
  value       = ( (integer|float|boolean|identifier|str) >mark %{ print "*v:" ; emit("value",data,ts,p) } );

  key_value = ( key_word '=' value ) ;

  main := ( ('<'|',') key_value )*;
}%%
=end

%% write data;
# %% this just fixes our syntax highlighting...

def emit type, data, ts, p
  # Print the type and text of the last read token
  # p ts,p
  puts "#{type}: #{data[ts...p].pack('c*')}"
end

def xemit(token_name, data, target_array, ts, te)
  target_array << {:name => token_name.to_sym, :value => data.pack("c*")[ts...te] }
end

def run_lexer(data)
  p data
  data = data.unpack("c*") if(data.is_a?(String))
  eof = data.length
  token_array = []
  stack = []

  %% write init;
  %% write exec;

  p token_array
  token_array.each do | h |
    p h
    # p h[:value] if h[:name]==:identifier or h[:name]==:value or h[:name]==:string
  end
end

# run_lexer("value = -2.00")

lines = <<LINES
##FORMAT=<ID=GT,Number=1,Type=String,Description="Genotype">
##FORMAT=<ID=DP,Number=1,Type=Integer,Description="Total read depth">
##FORMAT=<ID=DP4,Number=4,Type=Integer,Description="# high-quality ref-forward bases, ref-reverse, alt-forward and alt-reverse bases">
##INFO=<ID=PM,Number=0,Type=Flag,Description="Variant is Precious(Clinical,Pubmed Cited)">
##INFO=<ID=VP,Number=1,Type=String,Description="Variation Property.  Documentation is at ftp://ftp.ncbi.nlm.nih.gov/snp/specs/dbSNP_BitField_latest.pdf">
##INFO=<ID=GENEINFO,Number=1,Type=String,Description="Pairs each of gene symbol:gene id.  The gene symbol and id are delimited by a colon (:), and each pair is delimited by a vertical bar (|)">
##INFO=<ID=CLNHGVS,Number=.,Type=String,Description="Variant names from HGVS. The order of these variants corresponds to the order of the info in the other clinical  INFO tags.">
LINES

lines.strip.split("\n").each { |s| run_lexer(s.sub(/^##(FORMAT|INFO)=/,'')) }

cmd=ARGV.shift
run_lexer(cmd)

# Parser for VCF-header info and format fields

=begin
%%{

  machine simple_lexer;

  action mark { ts=p }
  action Start {
    ts = p
  }
  action Stop {
    quoted_text = data[ts...p].pack('c*')
    # do something with the quoted text!
    emit('string',data,ts,p)
  }

  squote = "'";
  dquote = '"';
  not_squote_or_escape = [^'\\];
  not_dquote_or_escape = [^"\\];
  escaped_something = /\\./;
  ss = space* squote ( not_squote_or_escape | escaped_something )* >Start %Stop squote;
  dd = space* dquote ( not_dquote_or_escape | escaped_something )* >Start %Stop dquote;

  # main := (ss | dd)*;

  integer     = ('+'|'-')?[0-9]+             >mark %{ emit("integer",data,ts,p)  };
  float       = ('+'|'-')?[0-9]+'.'[0-9]+    >mark %{ emit("float",data,ts,p) };
  assignment  = '=';
  identifier  = ([a-zA-Z][a-zA-Z_]+)         >mark %{ emit("identifier",data,ts,p) };
  str         = (ss|dd)* ;
  boolean     = '.'                          >mark %{ emit("bool",data,ts,p) };
  key_word    = ( ('ID'|'Number'|'Type'|'Description') >mark %{ print "*k:"; emit("key_word",data,ts,p) } );
  value       = ( (integer|float|boolean|identifier|str) >mark %{ print "*v:" ; emit("value",data,ts,p) } );

  key_value = ( key_word '=' value ) ;

  main := ( ('<'|',') key_value )*;
}%%
=end

%% write data;
# %% this just fixes our syntax highlighting...

def emit type, data, ts, p
  # Print the type and text of the last read token
  # p ts,p
  puts "#{type}: #{data[ts...p].pack('c*')}"
end

def xemit(token_name, data, target_array, ts, te)
  target_array << {:name => token_name.to_sym, :value => data.pack("c*")[ts...te] }
end

def run_lexer(data)
  p data
  data = data.unpack("c*") if(data.is_a?(String))
  eof = data.length
  token_array = []
  stack = []

  %% write init;
  %% write exec;

  p token_array
  token_array.each do | h |
    p h
    # p h[:value] if h[:name]==:identifier or h[:name]==:value or h[:name]==:string
  end
end

# run_lexer("value = -2.00")

lines = <<LINES
##FORMAT=<ID=GT,Number=1,Type=String,Description="Genotype">
##FORMAT=<ID=DP,Number=1,Type=Integer,Description="Total read depth">
##FORMAT=<ID=DP4,Number=4,Type=Integer,Description="# high-quality ref-forward bases, ref-reverse, alt-forward and alt-reverse bases">
##INFO=<ID=PM,Number=0,Type=Flag,Description="Variant is Precious(Clinical,Pubmed Cited)">
##INFO=<ID=VP,Number=1,Type=String,Description="Variation Property.  Documentation is at ftp://ftp.ncbi.nlm.nih.gov/snp/specs/dbSNP_BitField_latest.pdf">
##INFO=<ID=GENEINFO,Number=1,Type=String,Description="Pairs each of gene symbol:gene id.  The gene symbol and id are delimited by a colon (:), and each pair is delimited by a vertical bar (|)">
##INFO=<ID=CLNHGVS,Number=.,Type=String,Description="Variant names from HGVS. The order of these variants corresponds to the order of the info in the other clinical  INFO tags.">
LINES

lines.strip.split("\n").each { |s| run_lexer(s.sub(/^##(FORMAT|INFO)=/,'')) }

cmd=ARGV.shift
run_lexer(cmd)


Type checking

Type checking

# Parser for VCF-header info and format fields

module VcfHeader

  module RagelKeyValues

=begin
%%{

  machine simple_lexer;

  action mark { ts=p }
  action endquoted {
    emit.call(:value,data,ts,p)
  }

  action kw {
    emit.call(:key_word,data,ts,p)
  }

  squote = "'";
  dquote = '"';
  not_squote_or_escape = [^'\\];
  not_dquote_or_escape = [^"\\];
  escaped_something = /\\./;
  ss = space* squote ( not_squote_or_escape | escaped_something )* >mark %endquoted squote;
  dd = space* dquote ( not_dquote_or_escape | escaped_something )* >mark %endquoted dquote;

  integer     = ('+'|'-')?digit+;
  float       = ('+'|'-')?digit+'.'digit+;
  assignment  = '=';
  identifier  = (alpha alnum+);
  str         = (ss|dd)* ;
  boolean     = '.';
  key_word    = ( ('ID'|'Number'|'Type'|'Description') >mark %{ emit.call(:key_word,data,ts,p) } );
  any_value   = ( (integer|float|boolean|identifier|str) >mark %{ emit.call(:value,data,ts,p) } );
  id_value   = ( identifier >mark %{ emit.call(:value,data,ts,p) } );

  number_value = ( ( integer|boolean|'A' ) >mark %{ emit.call(:value,data,ts,p) } );

  id        = ( 'ID=' id_value ) @kw;
  key_value = ( (id | (key_word '=' any_value) ) >mark @!{ error_code="key-value" };

  main := ( '##' ('FILTER'|'FORMAT'|'INFO'|'ALT') '=') (('<'|',') key_value )* ;
}%%
=end

%% write data;
# %% this just fixes our syntax highlighting...

def self.run_lexer(buf, options = {})
  data = buf.unpack("c*") if(buf.is_a?(String))
  eof = data.length
  values = []
  stack = []

  emit = lambda { |type, data, ts, p|
    # Print the type and text of the last read token
    # p ts,p
    puts "#{type}: #{data[ts...p].pack('c*')}" if options[:debug]==true
    values << [type,data[ts...p].pack('c*')]
  }

  error_code = nil

  %% write init;
  %% write exec;

  raise "ERROR: "+error_code+" in "+buf if error_code

  res = {}
  # p values
  values.each_slice(2) do | a,b |
    # p '*',a,b
    res[a[1]] = b[1]
    # p h[:value] if h[:name]==:identifier or h[:name]==:value or h[:name]==:string
  end
  p res
end

  end
end

if __FILE__ == $0

lines = <<LINES
##FORMAT=<ID=GT,Number=1,Type=String,Description="Genotype">
##FORMAT=<ID=DP,Number=1,Type=Integer,Description="Total read depth">
##FORMAT=<ID=DP4,Number=4,Type=Integer,Description="# high-quality ref-forward bases, ref-reverse, alt-forward and alt-reverse bases">
##INFO=<ID=PM,Number=0,Type=Flag,Description="Variant is Precious(Clinical,Pubmed Cited)">
##INFO=<ID=VP,Number=1,Type=String,Description="Variation Property.  Documentation is at ftp://ftp.ncbi.nlm.nih.gov/snp/specs/dbSNP_BitField_latest.pdf">
##INFO=<ID=GENEINFO,Number=1,Type=String,Description="Pairs each of gene symbol:gene id.  The gene symbol and id are delimited by a colon (:), and each pair is delimited by a vertical bar (|)">
##INFO=<ID=CLNHGVS,Number=.,Type=String,Description="Variant names from HGVS. The order of these variants corresponds to the order of the info in the other clinical  INFO tags.">
##INFO=<ID="CLNHGVS1",Number=.,Type=String,Description="Variant names from \\"HGVS\\". The order of these 'variants' corresponds to the order of the info in the other clinical  INFO tags.">
LINES

lines.strip.split("\n").each { |s|
  print s,"\n"
  VcfHeader::RagelKeyValues.run_lexer(s, debug: false)
}

end

# Parser for VCF-header info and format fields

module VcfHeader

  module RagelKeyValues

=begin
%%{

  machine simple_lexer;

  action mark { ts=p }
  action endquoted {
    emit.call(:value,data,ts,p)
  }

  squote = "'";
  dquote = '"';
  not_squote_or_escape = [^'\\];
  not_dquote_or_escape = [^"\\];
  escaped_something = /\\./;
  ss = space* squote ( not_squote_or_escape | escaped_something )* >mark %endquoted squote;
  dd = space* dquote ( not_dquote_or_escape | escaped_something )* >mark %endquoted dquote;

  integer     = ('+'|'-')?digit+;
  float       = ('+'|'-')?digit+'.'digit+;
  assignment  = '=';
  identifier  = (alpha alnum+);
  str         = (ss|dd)* ;
  boolean     = '.';
  key_word    = ( ('Number'|'Type'|'Description') >mark %{ emit.call(:key_word,data,ts,p) } );
  value       = ( (integer|float|boolean|identifier|str) >mark %{ emit.call(:value,data,ts,p) } );

  key_value = ( ('ID=' identifier) | (key_word '=' value) ) @!{ error_code="key-value" };

  main := ( '##' ('FILTER'|'FORMAT'|'INFO'|'ALT') '=') (('<'|',') key_value )* ;
}%%
=end

%% write data;
# %% this just fixes our syntax highlighting...

def self.run_lexer(buf, options = {})
  data = buf.unpack("c*") if(buf.is_a?(String))
  eof = data.length
  values = []
  stack = []

  emit = lambda { |type, data, ts, p|
    # Print the type and text of the last read token
    # p ts,p
    puts "#{type}: #{data[ts...p].pack('c*')}" if options[:debug]==true
    values << [type,data[ts...p].pack('c*')]
  }

  error_code = nil

  %% write init;
  %% write exec;

  raise "ERROR: "+error_code+" in "+buf if error_code

  res = {}
  values.each_slice(2) do | a,b |
    # p '*',a,b
    res[a[1]] = b[1]
    # p h[:value] if h[:name]==:identifier or h[:name]==:value or h[:name]==:string
  end
  p res
end

  end
end

if __FILE__ == $0

lines = <<LINES
##FORMAT=<ID=GT,Number=1,Type=String,Description="Genotype">
##FORMAT=<ID=DP,Number=1,Type=Integer,Description="Total read depth">
##FORMAT=<ID=DP4,Number=4,Type=Integer,Description="# high-quality ref-forward bases, ref-reverse, alt-forward and alt-reverse bases">
##INFO=<ID=PM,Number=0,Type=Flag,Description="Variant is Precious(Clinical,Pubmed Cited)">
##INFO=<ID=VP,Number=1,Type=String,Description="Variation Property.  Documentation is at ftp://ftp.ncbi.nlm.nih.gov/snp/specs/dbSNP_BitField_latest.pdf">
##INFO=<ID=GENEINFO,Number=1,Type=String,Description="Pairs each of gene symbol:gene id.  The gene symbol and id are delimited by a colon (:), and each pair is delimited by a vertical bar (|)">
##INFO=<ID=CLNHGVS,Number=.,Type=String,Description="Variant names from HGVS. The order of these variants corresponds to the order of the info in the other clinical  INFO tags.">
##INFO=<ID="CLNHGVS1",Number=.,Type=String,Description="Variant names from \\"HGVS\\". The order of these 'variants' corresponds to the order of the info in the other clinical  INFO tags.">
LINES

lines.strip.split("\n").each { |s|
  print s,"\n"
  VcfHeader::RagelKeyValues.run_lexer(s, debug: false)
}

end


Trying to minimize wasted space.

Trying to minimize wasted space.

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/meqif/lolbot.git/'


Explicitly initialise to remove compile warning

IP temporary variables shouldn't be used before initialisation due to
the parsing rules, but explicitly assigning them dummy values makes GCC
stop complaining.

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/gim/libgim.git/'


parser: some cleaning up. parser: now with 2% more readability parser: corrected bug concerning redirection operator order and another one regarding correct string termination.

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/meqif/redshell.git/'


added a test case for fnbreak, works with -B0, needs proper impl for others

/*
 * @LANG: c
 *
 * Tests fnext in combination with fbreak.
 */

#include <string.h>
#include <stdio.h>

char comm;
int top;
int stack [32];

%%{
    machine fnext;
	action break {fnbreak;}

	main := 'h'   @{ /*h*/  fnext e; fnbreak; };
	e    := 'e'   @{ /*e*/  fnext l; }   @{ fnbreak; };
	l    := 'll'  @{ /*ll*/ fnext o; }   ${ fnbreak; };
	o    := |* 'o' { /*o*/  fnext nl; fnbreak; }; *|;
	nl   := '\n'  @{ /*nl*/ fnbreak; printf("ACCEPT\n"); };
}%%

int cs;
char *ts, *te;
int act;

%% write data;

void init()
{
	%% write init;
}

void exec( char *data, int len )
{
	char *p = data;
	char *pe = data + len;

	while ( cs != fnext_error && p < pe ) {
		printf( "%c\n", *p );
		%% write exec;
	}
}

void finish( )
{
	if ( cs >= fnext_first_final )
		printf( "ACCEPT\n" );
	else
		printf( "FAIL\n" );
}

char *inp[] = {
	"hello\n"
};

int inplen = 1;

int main( )
{
	int i;
	for ( i = 0; i < inplen; i++ ) {
		init();
		exec( inp[i], strlen(inp[i]) );
		finish();
	}
	return 0;
}

#ifdef _____OUTPUT_____
h
e
l
l
o

ACCEPT
ACCEPT
#endif

error: pathspec 'test/fnext3.rl' did not match any file(s) known to git


Unneeded namespace qualification for json::parse


cond error, not currently giving the right results

The error action is getting run on non-error transitions. The condition
implementation is not yet complete (still use of ctList.head in place of proper
traversals).

/*
 * @LANG: c++
 */

#include <iostream>
#include <string.h>
using std::cout;
using std::endl;

%%{
	machine foo;

	action c {c}
	main :=
		( ( 'a' when c )+ ) $err{ cout << "  bail" << endl; }
		'\n';
}%%

%% write data noerror;

void test( int c, const char *str )
{
	int cs = foo_start;
	const char *p = str;
	const char *pe = str + strlen( str );
	const char *eof = pe;

	cout << "run:" << endl;
	%% write exec;
	if ( cs >= foo_first_final )
		cout << "  success" << endl;
	else
		cout << "  failure" << endl;
	cout << endl;
}

int main()
{
	test( 1, "aa\n" );
	test( 1, "ab\n" );
	test( 0, "aa\n" );
	test( 0, "ab\n" );
	return 0;
}

#ifdef _____OUTPUT_____
run:
  bail
  bail
  success

run:
  bail
  bail
  failure

run:
  bail
  failure

run:
  bail
  failure

#endif

error: pathspec 'test/conderr.rl' did not match any file(s) known to git


Remove alignment and class from definition lists.  It increases the size of the compiled program from 2 to 3.5MB and increases compile time dramatically!

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/github/redcloth.git/'


parser: using a ragel state machine for parsing redirection.


* add missing file

* add missing file

(%
 % rowl - generation 1
 % Copyright (C) 2010 nineties
 %
 % $Id: headergen.rl 2010-04-07 07:58:38 nineties $
 %);

(% automatic header file generation %);

include (stddef, code);
export (headergen);

(% p0: output channel, p1: header filename %);
print_header: (p0, p1) {
    fputs(p0, "(% ");
    fputs(p0, p1);
    fputs(p0, " generated by rlc1 %)\n"); (% TODO: output date %);
};

gen_ident_decl: (p0, p1) {
    if (p1[0] != NODE_IDENTIFIER) {
        fputs(stderr, "ERROR: invalid export declaration\n");
        exit(1);
    };

    fputs(p0, "external ");
    fputs(p0, get_ident_name(p1));
    fputs(p0, "! ");
    put_type(p0, p1[1]);
};

export_item: (p0, p1) {
    allocate(1);
    x0 = p1[1]; (% exported item %);
    if (x0[0] == NODE_DECL) {
        gen_ident_decl(p0, x0[2]);
        return;
    };
    if (x0[0] == NODE_TYPEDECL) {
        put_item(p0, x0);
        return;
    };
    fputs(stderr, "ERROR: not reachable\n");
    exit(1);
};

(% p0: header filename, p1: program %);
headergen: (p0, p1) {
    allocate(3);
    x0 = p1[1]; (% item list %);
    x1 = open_out(p0); (% output channel %);
    print_header(x1, p0);
    while (x0 != NULL) {
        x2 = ls_value(x0);
        if (x2[0] == NODE_EXPORT) {
            fputc(x1, '\n');
            export_item(x1, x2);
            fputs(x1, ";\n");
        };
        x0 = ls_next(x0);
    };
    close_out(x1);
};

error: pathspec 'rlc1/headergen.rl' did not match any file(s) known to git


clean up packing back to string

clean up packing back to string

module Gherkin
  class Feature
    %%{
      machine feature;

      action accumulate_content {
        @content_start ||= p
      }

      action keyword_found {
        @keyword_start = p
      }

      action clear_content {
        @content_start = nil
      }

      action store_feature_content {
        con = data[@content_start...@keyword_start].pack("U*")
        con.strip!
        @listener.feature(con)
      }

      action store_scenario_content {
        con = data[@content_start...@keyword_start].pack("U*")
        con.strip!
        @listener.scenario(con)
      }

      action store_step_content {
        con = data[@content_start...@keyword_start].pack("U*")
        con.strip!
        @listener.given(con)
      }

      FEATURE = 'Feature:';
      SCENARIO = 'Scenario:';
      GIVEN = 'Given' | 'When';
      AND = 'And';
      THEN = 'Then';

      EOL = '\r'? '\n';

      KEYWORDS = (FEATURE | SCENARIO | GIVEN);

      feature = (
        start: (
          FEATURE space* ->feature_content
        ),

        feature_content: (
          any+ >accumulate_content ->feature_content |
          EOL+ space* SCENARIO >keyword_found @store_feature_content %clear_content ->scenario_content
        ),

        scenario_content: (
          any+ >accumulate_content ->scenario_content |
          EOL+ space* GIVEN >keyword_found @store_scenario_content %clear_content ->step_content
        ),

        step_content: (
          any+ >accumulate_content ->step_content |
          EOL+ >keyword_found %store_step_content -> final
        )
      );

      main := feature;
    }%%

    def initialize
      %% write data;
    end

    def scan(data, listener)
      @listener = listener
      data = data.unpack("U*") if data.is_a?(String)
      eof = data.size
      %% write init;
      %% write exec;
    end
  end
end

module Gherkin
  class Feature
    %%{
      machine feature;

      action accumulate_content {
        @content_start ||= p
      }

      action keyword_found {
        @keyword_start = p
      }

      action clear_content {
        @content_start = nil
      }

      action store_feature_content {
        con = data[@content_start...@keyword_start].map { |x| [x].pack("U*")}.join
        con.strip!
        @listener.feature(con)
      }

      action store_scenario_content {
        con = data[@content_start...@keyword_start].map { |x| [x].pack("U*")}.join
        con.strip!
        @listener.scenario(con)
      }

      action store_step_content {
        con = data[@content_start...@keyword_start].map { |x| [x].pack("U*")}.join
        con.strip!
        @listener.given(con)
      }

      FEATURE = 'Feature:';
      SCENARIO = 'Scenario:';
      GIVEN = 'Given' | 'When';
      AND = 'And';
      THEN = 'Then';

      EOL = '\r'? '\n';

      KEYWORDS = (FEATURE | SCENARIO | GIVEN);

      feature = (
        start: (
          FEATURE space* ->feature_content
        ),

        feature_content: (
          any+ >accumulate_content ->feature_content |
          EOL+ space* SCENARIO >keyword_found @store_feature_content %clear_content ->scenario_content
        ),

        scenario_content: (
          any+ >accumulate_content ->scenario_content |
          EOL+ space* GIVEN >keyword_found @store_scenario_content %clear_content ->step_content
        ),

        step_content: (
          any+ >accumulate_content ->step_content |
          EOL+ >keyword_found %store_step_content -> final
        )
      );

      main := feature;
    }%%

    def initialize
      %% write data;
    end

    def scan(data, listener)
      @listener = listener
      data = data.unpack("U*") if data.is_a?(String)
      eof = data.size
      %% write init;
      %% write exec;
    end
  end
end


Added missing ifdef DEBUG.

Added missing ifdef DEBUG.


Add link to hello_world_ruby.rl to start testing implicit output format detection based on filename

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/excepttheweasel/guard-ragel.git/'


Added missing free.

Added missing free.


Updated state machine that fixes the duplicate registrations problem.

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/msteinert/mongrel2.git/'


Fix a compilation failure introduced by the previous patch.

Complete the renamve of all iov functions.

/*
 * Copyright (C) 2010, 2011 Mail.RU
 * Copyright (C) 2010, 2011 Yuriy Vostrikov
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS ``AS IS'' AND
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 * SUCH DAMAGE.
 */

%%{
	machine memcached;
	write data;
}%%

static int __attribute__((noinline))
memcached_dispatch()
{
	int cs;
	u8 *p, *pe;
	u8 *fstart;
	struct tbuf *keys = tbuf_alloc(fiber->gc_pool);
	void *key;
	bool append, show_cas;
	int incr_sign;
	u64 cas, incr;
	u32 flags, exptime, bytes;
	bool noreply = false;
	u8 *data = NULL;
	bool done = false;
	int r;
	size_t saved_iov_cnt = fiber->iov_cnt;
	uintptr_t flush_delay = 0;
	size_t keys_count = 0;

	p = fiber->rbuf->data;
	pe = fiber->rbuf->data + fiber->rbuf->len;

	say_debug("memcached_dispatch '%.*s'", MIN((int)(pe - p), 40) , p);

	%%{
		action set {
			key = read_field(keys);
			STORE;
		}

		action add {
			key = read_field(keys);
			struct box_tuple *tuple = find(key);
			if (tuple != NULL && !expired(tuple))
				iov_add("NOT_STORED\r\n", 12);
			else
				STORE;
		}

		action replace {
			key = read_field(keys);
			struct box_tuple *tuple = find(key);
			if (tuple == NULL || expired(tuple))
				iov_add("NOT_STORED\r\n", 12);
			else
				STORE;
		}

		action cas {
			key = read_field(keys);
			struct box_tuple *tuple = find(key);
			if (tuple == NULL || expired(tuple))
				iov_add("NOT_FOUND\r\n", 11);
			else if (meta(tuple)->cas != cas)
				iov_add("EXISTS\r\n", 8);
			else
				STORE;
		}

		action append_prepend {
			struct tbuf *b;
			void *value;
			u32 value_len;

			key = read_field(keys);
			struct box_tuple *tuple = find(key);
			if (tuple == NULL || tuple->flags & GHOST) {
				iov_add("NOT_STORED\r\n", 12);
			} else {
				value = tuple_field(tuple, 3);
				value_len = load_varint32(&value);
				b = tbuf_alloc(fiber->gc_pool);
				if (append) {
					tbuf_append(b, value, value_len);
					tbuf_append(b, data, bytes);
				} else {
					tbuf_append(b, data, bytes);
					tbuf_append(b, value, value_len);
				}

				bytes += value_len;
				data = b->data;
				STORE;
			}
		}

		action incr_decr {
			struct meta *m;
			struct tbuf *b;
			void *field;
			u32 value_len;
			u64 value;

			key = read_field(keys);
			struct box_tuple *tuple = find(key);
			if (tuple == NULL || tuple->flags & GHOST || expired(tuple)) {
				iov_add("NOT_FOUND\r\n", 11);
			} else {
				m = meta(tuple);
				field = tuple_field(tuple, 3);
				value_len = load_varint32(&field);

				if (is_numeric(field, value_len)) {
					value = natoq(field, field + value_len);

					if (incr_sign > 0) {
						value += incr;
					} else {
						if (incr > value)
							value = 0;
						else
							value -= incr;
					}

					exptime = m->exptime;
					flags = m->flags;

					b = tbuf_alloc(fiber->gc_pool);
					tbuf_printf(b, "%"PRIu64, value);
					data = b->data;
					bytes = b->len;

					stats.cmd_set++;
					@try {
						store(key, exptime, flags, bytes, data);
						stats.total_items++;
						iov_add(b->data, b->len);
						iov_add("\r\n", 2);
					}
					@catch (ClientError *e) {
						iov_add("SERVER_ERROR ", 13);
						iov_add(e->errmsg, strlen(e->errmsg));
						iov_add("\r\n", 2);
					}
				} else {
					iov_add("CLIENT_ERROR cannot increment or decrement non-numeric value\r\n", 62);
				}
			}

		}

		action delete {
			key = read_field(keys);
			struct box_tuple *tuple = find(key);
			if (tuple == NULL || tuple->flags & GHOST || expired(tuple)) {
				iov_add("NOT_FOUND\r\n", 11);
			} else {
				@try {
					delete(key);
					iov_add("DELETED\r\n", 9);
				}
				@catch (ClientError *e) {
					iov_add("SERVER_ERROR ", 13);
					iov_add(e->errmsg, strlen(e->errmsg));
					iov_add("\r\n", 2);
				}
			}
		}

		action get {
			struct box_txn *txn = txn_alloc(BOX_QUIET);
			txn->op = SELECT;
			fiber_register_cleanup((void *)txn_cleanup, txn);
			stat_collect(stat_base, MEMC_GET, 1);
			stats.cmd_get++;
			say_debug("ensuring space for %"PRI_SZ" keys", keys_count);
			iov_ensure(keys_count * 5 + 1);
			while (keys_count-- > 0) {
				struct box_tuple *tuple;
				struct meta *m;
				void *field;
				void *value;
				void *suffix;
				u32 key_len;
				u32 value_len;
				u32 suffix_len;
				u32 _l;

				key = read_field(keys);
				tuple = find(key);
				key_len = load_varint32(&key);

				if (tuple == NULL || tuple->flags & GHOST) {
					stat_collect(stat_base, MEMC_GET_MISS, 1);
					stats.get_misses++;
					continue;
				}

				field = tuple->data;

				/* skip key */
				_l = load_varint32(&field);
				field += _l;

				/* metainfo */
				_l = load_varint32(&field);
				m = field;
				field += _l;

				/* suffix */
				suffix_len = load_varint32(&field);
				suffix = field;
				field += suffix_len;

				/* value */
				value_len = load_varint32(&field);
				value = field;

				if (m->exptime > 0 && m->exptime < ev_now()) {
					stats.get_misses++;
					stat_collect(stat_base, MEMC_GET_MISS, 1);
					continue;
				} else {
					stats.get_hits++;
					stat_collect(stat_base, MEMC_GET_HIT, 1);
				}

				tuple_txn_ref(txn, tuple);

				if (show_cas) {
					struct tbuf *b = tbuf_alloc(fiber->gc_pool);
					tbuf_printf(b, "VALUE %.*s %"PRIu32" %"PRIu32" %"PRIu64"\r\n", key_len, (u8 *)key, m->flags, value_len, m->cas);
					iov_add_unsafe(b->data, b->len);
					stats.bytes_written += b->len;
				} else {
					iov_add_unsafe("VALUE ", 6);
					iov_add_unsafe(key, key_len);
					iov_add_unsafe(suffix, suffix_len);
				}
				iov_add_unsafe(value, value_len);
				iov_add_unsafe("\r\n", 2);
				stats.bytes_written += value_len + 2;
			}
			iov_add_unsafe("END\r\n", 5);
			stats.bytes_written += 5;
		}

		action flush_all {
			if (flush_delay > 0) {
				struct fiber *f = fiber_create("flush_all", -1, -1, flush_all, (void *)flush_delay);
				if (f)
					fiber_call(f);
			} else
				flush_all((void *)0);
			iov_add("OK\r\n", 4);
		}

		action stats {
			print_stats();
		}

		action quit {
			return 0;
		}

		action fstart { fstart = p; }
		action key_start {
			fstart = p;
			for (; p < pe && *p != ' ' && *p != '\r' && *p != '\n'; p++);
			if ( *p == ' ' || *p == '\r' || *p == '\n') {
				write_varint32(keys, p - fstart);
				tbuf_append(keys, fstart, p - fstart);
				keys_count++;
				p--;
			} else
				p = fstart;
 		}

		printable = [^ \t\r\n];
		key = printable >key_start ;

		action exptime {
			exptime = natoq(fstart, p);
			if (exptime > 0 && exptime <= 60*60*24*30)
				exptime = exptime + ev_now();
		}
		exptime = digit+ >fstart %exptime;

		flags = digit+ >fstart %{flags = natoq(fstart, p);};
		bytes = digit+ >fstart %{bytes = natoq(fstart, p);};
		cas_value = digit+ >fstart %{cas = natoq(fstart, p);};
		incr_value = digit+ >fstart %{incr = natoq(fstart, p);};
		flush_delay = digit+ >fstart %{flush_delay = natoq(fstart, p);};

		action read_data {
			size_t parsed = p - (u8 *)fiber->rbuf->data;
			while (fiber->rbuf->len - parsed < bytes + 2) {
				if ((r = fiber_bread(fiber->rbuf, bytes + 2 - (pe - p))) <= 0) {
					say_debug("read returned %i, closing connection", r);
					return 0;
				}
			}

			p = fiber->rbuf->data + parsed;
			pe = fiber->rbuf->data + fiber->rbuf->len;

			data = p;

			if (strncmp((char *)(p + bytes), "\r\n", 2) == 0) {
				p += bytes + 2;
			} else {
				goto exit;
			}
		}

		action done {
			done = true;
			stats.bytes_read += p - (u8 *)fiber->rbuf->data;
			tbuf_peek(fiber->rbuf, p - (u8 *)fiber->rbuf->data);
		}

		eol = ("\r\n" | "\n") @{ p++; };
		spc = " "+;
		noreply = (spc "noreply" %{ noreply = true; })?;
		store_command_body = spc key spc flags spc exptime spc bytes noreply eol;

		set = ("set" store_command_body) @read_data @done @set;
		add = ("add" store_command_body) @read_data @done @add;
		replace = ("replace" store_command_body) @read_data @done @replace;
		append  = ("append"  %{append = true; } store_command_body) @read_data @done @append_prepend;
		prepend = ("prepend" %{append = false;} store_command_body) @read_data @done @append_prepend;
		cas = ("cas" spc key spc flags spc exptime spc bytes spc cas_value noreply spc?) eol @read_data @done @cas;

		get = "get" %{show_cas = false;} spc key (spc key)* spc? eol @done @get;
		gets = "gets" %{show_cas = true;} spc key (spc key)* spc? eol @done @get;
		delete = "delete" spc key (spc exptime)? noreply spc? eol @done @delete;
		incr = "incr" %{incr_sign = 1; } spc key spc incr_value noreply spc? eol @done @incr_decr;
		decr = "decr" %{incr_sign = -1;} spc key spc incr_value noreply spc? eol @done @incr_decr;

		stats = "stats" eol @done @stats;
		flush_all = "flush_all" (spc flush_delay)? noreply spc? eol @done @flush_all;
		quit = "quit" eol @done @quit;

	        main := set | cas | add | replace | append | prepend | get | gets | delete | incr | decr | stats | flush_all | quit;
	        #main := set;
		write init;
		write exec;
	}%%

	if (!done) {
		say_debug("parse failed after: `%.*s'", (int)(pe - p), p);
		if (pe - p > (1 << 20)) {
		exit:
			say_warn("memcached proto error");
			iov_add("ERROR\r\n", 7);
			stats.bytes_written += 7;
			return -1;
		}
		char *r;
		if ((r = memmem(p, pe - p, "\r\n", 2)) != NULL) {
			tbuf_peek(fiber->rbuf, r + 2 - (char *)fiber->rbuf->data);
			iov_add("CLIENT_ERROR bad command line format\r\n", 38);
			return 1;
		}
		return 0;
	}

	if (noreply) {
		fiber->iov_cnt = saved_iov_cnt;
		fiber->iov->len = saved_iov_cnt * sizeof(struct iovec);
	}

	return 1;
}

/*
 * Local Variables:
 * mode: c
 * End:
 * vim: syntax=objc
 */

/*
 * Copyright (C) 2010, 2011 Mail.RU
 * Copyright (C) 2010, 2011 Yuriy Vostrikov
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS ``AS IS'' AND
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 * SUCH DAMAGE.
 */

%%{
	machine memcached;
	write data;
}%%

static int __attribute__((noinline))
memcached_dispatch()
{
	int cs;
	u8 *p, *pe;
	u8 *fstart;
	struct tbuf *keys = tbuf_alloc(fiber->gc_pool);
	void *key;
	bool append, show_cas;
	int incr_sign;
	u64 cas, incr;
	u32 flags, exptime, bytes;
	bool noreply = false;
	u8 *data = NULL;
	bool done = false;
	int r;
	size_t saved_iov_cnt = fiber->iov_cnt;
	uintptr_t flush_delay = 0;
	size_t keys_count = 0;

	p = fiber->rbuf->data;
	pe = fiber->rbuf->data + fiber->rbuf->len;

	say_debug("memcached_dispatch '%.*s'", MIN((int)(pe - p), 40) , p);

	%%{
		action set {
			key = read_field(keys);
			STORE;
		}

		action add {
			key = read_field(keys);
			struct box_tuple *tuple = find(key);
			if (tuple != NULL && !expired(tuple))
				add_iov("NOT_STORED\r\n", 12);
			else
				STORE;
		}

		action replace {
			key = read_field(keys);
			struct box_tuple *tuple = find(key);
			if (tuple == NULL || expired(tuple))
				add_iov("NOT_STORED\r\n", 12);
			else
				STORE;
		}

		action cas {
			key = read_field(keys);
			struct box_tuple *tuple = find(key);
			if (tuple == NULL || expired(tuple))
				add_iov("NOT_FOUND\r\n", 11);
			else if (meta(tuple)->cas != cas)
				add_iov("EXISTS\r\n", 8);
			else
				STORE;
		}

		action append_prepend {
			struct tbuf *b;
			void *value;
			u32 value_len;

			key = read_field(keys);
			struct box_tuple *tuple = find(key);
			if (tuple == NULL || tuple->flags & GHOST) {
				add_iov("NOT_STORED\r\n", 12);
			} else {
				value = tuple_field(tuple, 3);
				value_len = load_varint32(&value);
				b = tbuf_alloc(fiber->gc_pool);
				if (append) {
					tbuf_append(b, value, value_len);
					tbuf_append(b, data, bytes);
				} else {
					tbuf_append(b, data, bytes);
					tbuf_append(b, value, value_len);
				}

				bytes += value_len;
				data = b->data;
				STORE;
			}
		}

		action incr_decr {
			struct meta *m;
			struct tbuf *b;
			void *field;
			u32 value_len;
			u64 value;

			key = read_field(keys);
			struct box_tuple *tuple = find(key);
			if (tuple == NULL || tuple->flags & GHOST || expired(tuple)) {
				add_iov("NOT_FOUND\r\n", 11);
			} else {
				m = meta(tuple);
				field = tuple_field(tuple, 3);
				value_len = load_varint32(&field);

				if (is_numeric(field, value_len)) {
					value = natoq(field, field + value_len);

					if (incr_sign > 0) {
						value += incr;
					} else {
						if (incr > value)
							value = 0;
						else
							value -= incr;
					}

					exptime = m->exptime;
					flags = m->flags;

					b = tbuf_alloc(fiber->gc_pool);
					tbuf_printf(b, "%"PRIu64, value);
					data = b->data;
					bytes = b->len;

					stats.cmd_set++;
					@try {
						store(key, exptime, flags, bytes, data);
						stats.total_items++;
						add_iov(b->data, b->len);
						add_iov("\r\n", 2);
					}
					@catch (ClientError *e) {
						add_iov("SERVER_ERROR ", 13);
						add_iov(e->errmsg, strlen(e->errmsg));
						add_iov("\r\n", 2);
					}
				} else {
					add_iov("CLIENT_ERROR cannot increment or decrement non-numeric value\r\n", 62);
				}
			}

		}

		action delete {
			key = read_field(keys);
			struct box_tuple *tuple = find(key);
			if (tuple == NULL || tuple->flags & GHOST || expired(tuple)) {
				add_iov("NOT_FOUND\r\n", 11);
			} else {
				@try {
					delete(key);
					add_iov("DELETED\r\n", 9);
				}
				@catch (ClientError *e) {
					add_iov("SERVER_ERROR ", 13);
					add_iov(e->errmsg, strlen(e->errmsg));
					add_iov("\r\n", 2);
				}
			}
		}

		action get {
			struct box_txn *txn = txn_alloc(BOX_QUIET);
			txn->op = SELECT;
			fiber_register_cleanup((void *)txn_cleanup, txn);
			stat_collect(stat_base, MEMC_GET, 1);
			stats.cmd_get++;
			say_debug("ensuring space for %"PRI_SZ" keys", keys_count);
			iov_ensure(keys_count * 5 + 1);
			while (keys_count-- > 0) {
				struct box_tuple *tuple;
				struct meta *m;
				void *field;
				void *value;
				void *suffix;
				u32 key_len;
				u32 value_len;
				u32 suffix_len;
				u32 _l;

				key = read_field(keys);
				tuple = find(key);
				key_len = load_varint32(&key);

				if (tuple == NULL || tuple->flags & GHOST) {
					stat_collect(stat_base, MEMC_GET_MISS, 1);
					stats.get_misses++;
					continue;
				}

				field = tuple->data;

				/* skip key */
				_l = load_varint32(&field);
				field += _l;

				/* metainfo */
				_l = load_varint32(&field);
				m = field;
				field += _l;

				/* suffix */
				suffix_len = load_varint32(&field);
				suffix = field;
				field += suffix_len;

				/* value */
				value_len = load_varint32(&field);
				value = field;

				if (m->exptime > 0 && m->exptime < ev_now()) {
					stats.get_misses++;
					stat_collect(stat_base, MEMC_GET_MISS, 1);
					continue;
				} else {
					stats.get_hits++;
					stat_collect(stat_base, MEMC_GET_HIT, 1);
				}

				tuple_txn_ref(txn, tuple);

				if (show_cas) {
					struct tbuf *b = tbuf_alloc(fiber->gc_pool);
					tbuf_printf(b, "VALUE %.*s %"PRIu32" %"PRIu32" %"PRIu64"\r\n", key_len, (u8 *)key, m->flags, value_len, m->cas);
					iov_add_unsafe(b->data, b->len);
					stats.bytes_written += b->len;
				} else {
					iov_add_unsafe("VALUE ", 6);
					iov_add_unsafe(key, key_len);
					iov_add_unsafe(suffix, suffix_len);
				}
				iov_add_unsafe(value, value_len);
				iov_add_unsafe("\r\n", 2);
				stats.bytes_written += value_len + 2;
			}
			iov_add_unsafe("END\r\n", 5);
			stats.bytes_written += 5;
		}

		action flush_all {
			if (flush_delay > 0) {
				struct fiber *f = fiber_create("flush_all", -1, -1, flush_all, (void *)flush_delay);
				if (f)
					fiber_call(f);
			} else
				flush_all((void *)0);
			add_iov("OK\r\n", 4);
		}

		action stats {
			print_stats();
		}

		action quit {
			return 0;
		}

		action fstart { fstart = p; }
		action key_start {
			fstart = p;
			for (; p < pe && *p != ' ' && *p != '\r' && *p != '\n'; p++);
			if ( *p == ' ' || *p == '\r' || *p == '\n') {
				write_varint32(keys, p - fstart);
				tbuf_append(keys, fstart, p - fstart);
				keys_count++;
				p--;
			} else
				p = fstart;
 		}

		printable = [^ \t\r\n];
		key = printable >key_start ;

		action exptime {
			exptime = natoq(fstart, p);
			if (exptime > 0 && exptime <= 60*60*24*30)
				exptime = exptime + ev_now();
		}
		exptime = digit+ >fstart %exptime;

		flags = digit+ >fstart %{flags = natoq(fstart, p);};
		bytes = digit+ >fstart %{bytes = natoq(fstart, p);};
		cas_value = digit+ >fstart %{cas = natoq(fstart, p);};
		incr_value = digit+ >fstart %{incr = natoq(fstart, p);};
		flush_delay = digit+ >fstart %{flush_delay = natoq(fstart, p);};

		action read_data {
			size_t parsed = p - (u8 *)fiber->rbuf->data;
			while (fiber->rbuf->len - parsed < bytes + 2) {
				if ((r = fiber_bread(fiber->rbuf, bytes + 2 - (pe - p))) <= 0) {
					say_debug("read returned %i, closing connection", r);
					return 0;
				}
			}

			p = fiber->rbuf->data + parsed;
			pe = fiber->rbuf->data + fiber->rbuf->len;

			data = p;

			if (strncmp((char *)(p + bytes), "\r\n", 2) == 0) {
				p += bytes + 2;
			} else {
				goto exit;
			}
		}

		action done {
			done = true;
			stats.bytes_read += p - (u8 *)fiber->rbuf->data;
			tbuf_peek(fiber->rbuf, p - (u8 *)fiber->rbuf->data);
		}

		eol = ("\r\n" | "\n") @{ p++; };
		spc = " "+;
		noreply = (spc "noreply" %{ noreply = true; })?;
		store_command_body = spc key spc flags spc exptime spc bytes noreply eol;

		set = ("set" store_command_body) @read_data @done @set;
		add = ("add" store_command_body) @read_data @done @add;
		replace = ("replace" store_command_body) @read_data @done @replace;
		append  = ("append"  %{append = true; } store_command_body) @read_data @done @append_prepend;
		prepend = ("prepend" %{append = false;} store_command_body) @read_data @done @append_prepend;
		cas = ("cas" spc key spc flags spc exptime spc bytes spc cas_value noreply spc?) eol @read_data @done @cas;

		get = "get" %{show_cas = false;} spc key (spc key)* spc? eol @done @get;
		gets = "gets" %{show_cas = true;} spc key (spc key)* spc? eol @done @get;
		delete = "delete" spc key (spc exptime)? noreply spc? eol @done @delete;
		incr = "incr" %{incr_sign = 1; } spc key spc incr_value noreply spc? eol @done @incr_decr;
		decr = "decr" %{incr_sign = -1;} spc key spc incr_value noreply spc? eol @done @incr_decr;

		stats = "stats" eol @done @stats;
		flush_all = "flush_all" (spc flush_delay)? noreply spc? eol @done @flush_all;
		quit = "quit" eol @done @quit;

	        main := set | cas | add | replace | append | prepend | get | gets | delete | incr | decr | stats | flush_all | quit;
	        #main := set;
		write init;
		write exec;
	}%%

	if (!done) {
		say_debug("parse failed after: `%.*s'", (int)(pe - p), p);
		if (pe - p > (1 << 20)) {
		exit:
			say_warn("memcached proto error");
			add_iov("ERROR\r\n", 7);
			stats.bytes_written += 7;
			return -1;
		}
		char *r;
		if ((r = memmem(p, pe - p, "\r\n", 2)) != NULL) {
			tbuf_peek(fiber->rbuf, r + 2 - (char *)fiber->rbuf->data);
			add_iov("CLIENT_ERROR bad command line format\r\n", 38);
			return 1;
		}
		return 0;
	}

	if (noreply) {
		fiber->iov_cnt = saved_iov_cnt;
		fiber->iov->len = saved_iov_cnt * sizeof(struct iovec);
	}

	return 1;
}

/*
 * Local Variables:
 * mode: c
 * End:
 * vim: syntax=objc
 */


ip: express equality in terms of words

ip: express equality in terms of words


Add convenience wrapper json::write function


ragel csharp example

ragel csharp example


add utf8 option in uri

add utf8 option in uri

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/imgix/siphon.git/'


don't match comma in rule name

don't match comma in rule name


omg saved

omg saved


json: give invalid character offset in error msg


Use nullptr rather than NULL

Use nullptr rather than NULL


Fix relative symlink

Fix relative symlink


Removed unneeded debug prints.

Removed unneeded debug prints.


Allow empty arrays as values in grammar

Allow empty arrays as values in grammar

require "logstash/namespace"

%%{
  machine logstash_config;

  action mark {
    @tokenstack.push(p)
    #puts "Mark: #{self.line(string, p)}##{self.column(string, p)}"
  }

  action stack_numeric {
    startpos = @tokenstack.pop
    endpos = p
    token = string[startpos ... endpos]
    #puts "numeric: #{token}"
    #puts "numeric?: #{string[startpos,50]}"
    #puts [startpos, endpos].join(",")
    # TODO(sissel): Don't do 'to_i' here. Type coersion is the job of the
    # plugin and the validator.
    @stack << token.to_i
  }

  action stack_string {
    startpos = @tokenstack.pop
    endpos = p
    token = string[startpos ... endpos]
    #puts "string: #{token}"
    @stack << token
  }

  action stack_quoted_string {
    startpos = @tokenstack.pop
    endpos = p
    token = string[startpos + 1 ... endpos - 1] # Skip quotations

    # Parse escapes.
    token.gsub(/\\./) { |m| m[1,1] }
    #puts "quotedstring: #{token}"
    @stack << token
  }

  action array_init {
    @array = []
    @stack << :array_init
  }

  action array_push {
    while @stack.last != :array_init
      @array.unshift @stack.pop
    end
    @stack.pop # pop :array_init

    @stack << @array
  }

  action parameter_init {
    # nothing
  }

  action parameter {
    value = @stack.pop
    name = @stack.pop
    #puts "parameter: #{name} => #{value}"
    if value.is_a?(Array)
      @parameters[name] += value
    else
      @parameters[name] << value
    end
  }

  action plugin {
    @components ||= []
    name = @stack.pop
    #@components << { :name => name, :parameters => @parameters }
    @components << { name => @parameters }
    @parameters = Hash.new { |h,k| h[k] = [] }
  }

  action component_init {
    @components = []
    @parameters = Hash.new { |h,k| h[k] = [] }
  }

  action component {
    name = @stack.pop
    @config ||= Hash.new { |h,k| h[k] = [] }
    @config[name] += @components
    #puts "Config component: #{name}"
  }

  #%{ e = @tokenstack.pop; puts "Comment: #{string[e ... p]}" };
  comment = "#" (any - [\n])* >mark ;
  ws = ([ \t\n] | comment)** ;
  #ws = ([ \t\n])** ;

  # TODO(sissel): Support floating point values?
  numeric = ( ("+" | "-")?  [0-9] :>> [0-9]** ) >mark %stack_numeric;
  quoted_string = (
    ( "\"" ( ( (any - [\\"\n]) | "\\" any )* ) "\"" )
    | ( "'" ( ( (any - [\\'\n]) | "\\" any )* ) "'" )
  ) >mark %stack_quoted_string ;
  naked_string = ( [A-Za-z_] :>> [A-Za-z0-9_]* ) >mark %stack_string ;
  string = ( quoted_string | naked_string ) ;

  # TODO(sissel): allow use of this.
  regexp_literal = ( "/" ( ( (any - [\\'\n]) | "\\" any )* ) "/" )  ;

  array = (
    ( "[" ( ws | "" ) "]" )
    | ( "[" ws ( string | numeric ) ws ("," ws (string | numeric ) ws)* "]" )
  ) >array_init %array_push;
  # TODO(sissel): Implement hash syntax { key => value, ... }
  # TODO(sissel): hashes should support arrays as values.

  parameter_value = ( numeric | string | array );
  parameter = ( string ws "=>" ws parameter_value ) %parameter ;
  parameters = ( parameter ( ws parameter )** ) >parameter_init ;

  # Statement:
  # component {
  #   plugin_name {
  #     bar => ...
  #     baz => ...
  #   }
  #   ...
  # }

  plugin = (
    (
      naked_string ws "{" ws
        parameters
      ws "}"
    ) | (
      naked_string ws "{" ws "}"
    )
  ) %plugin ;

  component = (
    naked_string ws "{"
      >component_init
      ( ws plugin )**
    ws "}"
  ) %component ;

  config = (ws component? )** ;

  main := config %{ puts "END" }
          $err {
            # Compute line and column of the cursor (p)
            $stderr.puts "Error at line #{self.line(string, p)}, column #{self.column(string, p)}: #{string[p .. -1].inspect}"
            # TODO(sissel): Note what we were expecting?
          } ;
}%%

class LogStash::Config::Grammar
  attr_accessor :eof
  attr_accessor :config

  def initialize
    # BEGIN RAGEL DATA
    %% write data;
    # END RAGEL DATA

    @tokenstack = Array.new
    @stack = Array.new

    @types = Hash.new { |h,k| h[k] = [] }
    @edges = []
  end

  def parse(string)
    # TODO(sissel): Due to a bug in my parser, we need one trailing whitespace
    # at the end of the string. I'll fix this later.
    string += "\n"

    data = string.unpack("c*")

    # BEGIN RAGEL INIT
    %% write init;
    # END RAGEL INIT

    begin
      # BEGIN RAGEL EXEC
      %% write exec;
      # END RAGEL EXEC
    rescue => e
      # Compute line and column of the cursor (p)
      raise e
    end

    if cs < self.logstash_config_first_final
      $stderr.puts "Error at line #{self.line(string, p)}, column #{self.column(string, p)}: #{string[p .. -1].inspect}"
      raise "Invalid Configuration. Check syntax of config file."
    end
    return cs
  end # def parse

  def line(str, pos)
    return str[0 .. pos].count("\n") + 1
  end

  def column(str, pos)
    return str[0 .. pos].split("\n").last.length
  end
end # class LogStash::Config::Grammar

#def parse(string)
  #cfgparser = LogStash::Config::Grammar.new
  #result = cfgparser.parse(string)
  #puts "result %s" % result
  #ap cfgparser.config
#end

#parse(File.open(ARGV[0]).read)

require "logstash/namespace"

%%{
  machine logstash_config;

  action mark {
    @tokenstack.push(p)
    #puts "Mark: #{self.line(string, p)}##{self.column(string, p)}"
  }

  action stack_numeric {
    startpos = @tokenstack.pop
    endpos = p
    token = string[startpos ... endpos]
    #puts "numeric: #{token}"
    #puts "numeric?: #{string[startpos,50]}"
    #puts [startpos, endpos].join(",")
    # TODO(sissel): Don't do 'to_i' here. Type coersion is the job of the
    # plugin and the validator.
    @stack << token.to_i
  }

  action stack_string {
    startpos = @tokenstack.pop
    endpos = p
    token = string[startpos ... endpos]
    #puts "string: #{token}"
    @stack << token
  }

  action stack_quoted_string {
    startpos = @tokenstack.pop
    endpos = p
    token = string[startpos + 1 ... endpos - 1] # Skip quotations

    # Parse escapes.
    token.gsub(/\\./) { |m| m[1,1] }
    #puts "quotedstring: #{token}"
    @stack << token
  }

  action array_init {
    @array = []
    @stack << :array_init
  }

  action array_push {
    while @stack.last != :array_init
      @array.unshift @stack.pop
    end
    @stack.pop # pop :array_init

    @stack << @array
  }

  action parameter_init {
    # nothing
  }

  action parameter {
    value = @stack.pop
    name = @stack.pop
    #puts "parameter: #{name} => #{value}"
    if value.is_a?(Array)
      @parameters[name] += value
    else
      @parameters[name] << value
    end
  }

  action plugin {
    @components ||= []
    name = @stack.pop
    #@components << { :name => name, :parameters => @parameters }
    @components << { name => @parameters }
    @parameters = Hash.new { |h,k| h[k] = [] }
  }

  action component_init {
    @components = []
    @parameters = Hash.new { |h,k| h[k] = [] }
  }

  action component {
    name = @stack.pop
    @config ||= Hash.new { |h,k| h[k] = [] }
    @config[name] += @components
    #puts "Config component: #{name}"
  }

  #%{ e = @tokenstack.pop; puts "Comment: #{string[e ... p]}" };
  comment = "#" (any - [\n])* >mark ;
  ws = ([ \t\n] | comment)** ;
  #ws = ([ \t\n])** ;

  # TODO(sissel): Support floating point values?
  numeric = ( ("+" | "-")?  [0-9] :>> [0-9]** ) >mark %stack_numeric;
  quoted_string = (
    ( "\"" ( ( (any - [\\"\n]) | "\\" any )* ) "\"" )
    | ( "'" ( ( (any - [\\'\n]) | "\\" any )* ) "'" )
  ) >mark %stack_quoted_string ;
  naked_string = ( [A-Za-z_] :>> [A-Za-z0-9_]* ) >mark %stack_string ;
  string = ( quoted_string | naked_string ) ;

  # TODO(sissel): allow use of this.
  regexp_literal = ( "/" ( ( (any - [\\'\n]) | "\\" any )* ) "/" )  ;

  array = ( "[" ws ( string | numeric ) ws ("," ws (string | numeric ) ws)* "]" ) >array_init %array_push;
  # TODO(sissel): Implement hash syntax { key => value, ... }
  # TODO(sissel): hashes should support arrays as values.

  parameter_value = ( numeric | string | array );
  parameter = ( string ws "=>" ws parameter_value ) %parameter ;
  parameters = ( parameter ( ws parameter )** ) >parameter_init ;

  # Statement:
  # component {
  #   plugin_name {
  #     bar => ...
  #     baz => ...
  #   }
  #   ...
  # }

  plugin = (
    (
      naked_string ws "{" ws
        parameters
      ws "}"
    ) | (
      naked_string ws "{" ws "}"
    )
  ) %plugin ;

  component = (
    naked_string ws "{"
      >component_init
      ( ws plugin )**
    ws "}"
  ) %component ;

  config = (ws component? )** ;

  main := config %{ puts "END" }
          $err {
            # Compute line and column of the cursor (p)
            $stderr.puts "Error at line #{self.line(string, p)}, column #{self.column(string, p)}: #{string[p .. -1].inspect}"
            # TODO(sissel): Note what we were expecting?
          } ;
}%%

class LogStash::Config::Grammar
  attr_accessor :eof
  attr_accessor :config

  def initialize
    # BEGIN RAGEL DATA
    %% write data;
    # END RAGEL DATA

    @tokenstack = Array.new
    @stack = Array.new

    @types = Hash.new { |h,k| h[k] = [] }
    @edges = []
  end

  def parse(string)
    # TODO(sissel): Due to a bug in my parser, we need one trailing whitespace
    # at the end of the string. I'll fix this later.
    string += "\n"

    data = string.unpack("c*")

    # BEGIN RAGEL INIT
    %% write init;
    # END RAGEL INIT

    begin
      # BEGIN RAGEL EXEC
      %% write exec;
      # END RAGEL EXEC
    rescue => e
      # Compute line and column of the cursor (p)
      raise e
    end

    if cs < self.logstash_config_first_final
      $stderr.puts "Error at line #{self.line(string, p)}, column #{self.column(string, p)}: #{string[p .. -1].inspect}"
      raise "Invalid Configuration. Check syntax of config file."
    end
    return cs
  end # def parse

  def line(str, pos)
    return str[0 .. pos].count("\n") + 1
  end

  def column(str, pos)
    return str[0 .. pos].split("\n").last.length
  end
end # class LogStash::Config::Grammar

#def parse(string)
  #cfgparser = LogStash::Config::Grammar.new
  #result = cfgparser.parse(string)
  #puts "result %s" % result
  #ap cfgparser.config
#end

#parse(File.open(ARGV[0]).read)


http: eases several targets for Python clients

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/jfsmig/oio-sds-cpp.git/'


Test fnext in combination with fbreak.

Test fnext in combination with fbreak.

/*
 * @LANG: c
 *
 * Tests fnext in combination with fbreak.
 */

#include <string.h>
#include <stdio.h>

char comm;
int top;
int stack [32];

%%{
    machine fnext;
	action break {fbreak;}

	main := 'h'   @{ /*h*/  fnext e; fbreak; };
	e    := 'e'   @{ /*e*/  fnext l; }   @{ fbreak; };
	l    := 'll'  @{ /*ll*/ fnext o; }   ${ fbreak; };
	o    := |* 'o' { /*o*/  fnext nl; fbreak; }; *|;
	nl   := '\n'  @{ /*nl*/ fbreak; printf("ACCEPT\n"); };
}%%

int cs;
char *ts, *te;
int act;

%% write data;

void init()
{
	%% write init;
}

void exec( char *data, int len )
{
	char *p = data;
	char *pe = data + len;

	while ( cs != fnext_error && p < pe ) {
		printf( "%c\n", *p );
		%% write exec;
	}
}

void finish( )
{
	if ( cs >= fnext_first_final )
		printf( "ACCEPT\n" );
	else
		printf( "FAIL\n" );
}

char *inp[] = {
	"hello\n"
};

int inplen = 1;

int main( )
{
	int i;
	for ( i = 0; i < inplen; i++ ) {
		init();
		exec( inp[i], strlen(inp[i]) );
		finish();
	}
	return 0;
}

#ifdef _____OUTPUT_____
h
e
l
l
o

ACCEPT
#endif

error: pathspec 'test/fnext1.rl' did not match any file(s) known to git


Small clean up. Removed magic constant.

Small clean up. Removed magic constant.


Remove useless debug check

Remove useless debug check


Noting down desired file index.

Noting down desired file index.


add test.rl example

add test.rl example

var is_child uint
var Key item
var Num_worlds uint
var Num_trails uint
var _HIDE uint

begin:
    if is_child:
        inc Num_trails 1
        set _HIDE 0
    else:
        set _HIDE 1

if is_child:
    if $second_field $second_field='world':
        inc Num_worlds 1
else:
    fork $first_field:
        send Key $first_field
        send is_child 1

error: pathspec 'test.rl' did not match any file(s) known to git


* add missing file

* add missing file


added test case for action params/args

added test case for action params/args

refs #66

/*
 * @LANG: c++
 *
 * This is a conds test modified to exercise actions arguments.
 */

#include <iostream>
#include <string.h>
using std::cout;
using std::endl;

%%{
	machine foo;

	action c(v) { $v }

	action c1 {i}
	action c2 {j}

	action one {"one"}
	action pr(str) { cout << "  " $str << endl;}

	main := (
			[a-z] |
			('\n' when c(c1) @pr( one ) )
		)*
		('\n' when c({j}) @pr( {"two"} ) );
}%%

%% write data noerror;

void test( int i, int j, const char *str )
{
	int cs = foo_start;
	const char *p = str;
	const char *pe = str + strlen( str );

	cout << "run:" << endl;
	%% write exec;
	if ( cs >= foo_first_final )
		cout << "  success" << endl;
	else
		cout << "  failure" << endl;
	cout << endl;
}

int main()
{
	test( 0, 0, "hi\n\n" );
	test( 1, 0, "hi\n\n" );
	test( 0, 1, "hi\n" );
	test( 0, 1, "hi\n\n" );
	test( 1, 1, "hi\n" );
	test( 1, 1, "hi\n\n" );
	test( 1, 1, "hi\n\nx" );
	return 0;
}

##### OUTPUT #####
run:
  failure

run:
  one
  one
  failure

run:
  two
  success

run:
  two
  failure

run:
  one
  two
  success

run:
  one
  two
  one
  two
  success

run:
  one
  two
  one
  two
  failure

error: pathspec 'test/args1.rl' did not match any file(s) known to git


Convert ruby conditional action from if to ternary so if isn't nested.


parser: small cleaning.

parser: small cleaning.


Instead of JSON use MSG to be generic and future proof.


ip: style

ip: style


parser: simplified a little bit.

parser: simplified a little bit.


[Minor] Avoid gmime in parsing time

[Minor] Avoid gmime in parsing time

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/AlexeySa/rspamd.git/'


Some cleanups.

Some cleanups.


[Minor] Fix destroy->free transition

[Minor] Fix destroy->free transition


parser: added command connection mask parsing.


Moved string concatenations somewhere safer.


Added PING/PONG support.

Added PING/PONG support.


Ragel

Ragel

# Parser for VCF-header info and format fields

=begin
%%{

  machine simple_lexer;

  integer     = ('+'|'-')?[0-9]+;
  float       = ('+'|'-')?[0-9]+'.'[0-9]+;
  assignment  = '=';
  identifier  = [a-zA-Z][a-zA-Z_]+;
  string      = ["][^"]*["];
  boolean     = '.';

  main := |*

    integer => {
      emit(:integer_literal, data, token_array, ts, te)
    };

    float => {
      emit(:float_literal, data, token_array, ts, te)
    };

    assignment => {
      emit(:assignment_operator, data, token_array, ts, te)
    };

    boolean => {
      emit(:boolean, data, token_array, ts, te)
    };

    identifier => {
      emit(:identifier, data, token_array, ts, te)
    };

    string => {
      emit(:string, data, token_array, ts, te)
    };

    space|',';

  *|;

}%%
=end

%% write data;
# %% this just fixes our syntax highlighting...

def emit(token_name, data, target_array, ts, te)
  target_array << {:name => token_name.to_sym, :value => data[ts...te].pack("c*") }
end

def run_lexer(data)
  p data
  data = data.unpack("c*") if(data.is_a?(String))
  eof = data.length
  token_array = []

  %% write init;
  %% write exec;

  p token_array
  token_array.each do | h |
    # p h
    p h[:value] if h[:name]==:identifier or h[:name]==:value or h[:name]==:string
  end
end

# run_lexer("value = -2.00")

lines = <<LINES
##FORMAT=<ID=GT,Number=1,Type=String,Description="Genotype">
##FORMAT=<ID=DP,Number=1,Type=Integer,Description="Total read depth">
##FORMAT=<ID=DP4,Number=4,Type=Integer,Description="# high-quality ref-forward bases, ref-reverse, alt-forward and alt-reverse bases">
##INFO=<ID=PM,Number=0,Type=Flag,Description="Variant is Precious(Clinical,Pubmed Cited)">
##INFO=<ID=VP,Number=1,Type=String,Description="Variation Property.  Documentation is at ftp://ftp.ncbi.nlm.nih.gov/snp/specs/dbSNP_BitField_latest.pdf">
##INFO=<ID=GENEINFO,Number=1,Type=String,Description="Pairs each of gene symbol:gene id.  The gene symbol and id are delimited by a colon (:), and each pair is delimited by a vertical bar (|)">
##INFO=<ID=CLNHGVS,Number=.,Type=String,Description="Variant names from HGVS. The order of these variants corresponds to the order of the info in the other clinical  INFO tags.">
LINES

lines.split("\n").each { |s| run_lexer(s.sub(/^##(FORMAT|INFO)=</,'')) }

cmd=ARGV.shift
run_lexer(cmd)

error: pathspec 'ragel/gen_vcfheaderline_parser.rl' did not match any file(s) known to git


[Fix] Add spaces instead of newlines to the normalized content

%%{
  machine newlines_strip;

  action Double_CRLF {
    if (!crlf_added && p > c) {
      (*newlines_count)++;
      g_byte_array_append (data, (const guint8 *)" ", 1);
      c = p;
    }

    crlf_added = TRUE;
    c = p;
  }

  action WSP {
    g_byte_array_append (data, (const guint8 *)" ", 1);
    c = p;
  }

  action Text_Start {
    crlf_added = FALSE;
    c = p;
  }

  action Text_End {
    if (p > c) {
      g_byte_array_append (data, (const guint8 *)c, p - c);
      last_c = *(p - 1);
    }

    c = p;
  }

  action Line_CRLF {
    if (!crlf_added) {
      if (is_html || g_ascii_ispunct (last_c)) {
         g_byte_array_append (data, (const guint8 *)" ", 1);
         crlf_added = TRUE;
      }
    }

    (*newlines_count)++;
    g_ptr_array_add (newlines, (gpointer)p);
    c = p;
  }

  WSP   = " " | "\t" | "\v";
  CRLF  = ("\r" . "\n") | ( "\r" ) | ("\n");
  DOUBLE_CRLF = (CRLF <: (WSP* CRLF)+) %Double_CRLF;
  ANY_CRLF = CRLF | DOUBLE_CRLF;
  LINE = (([^\r\n]+) >Text_Start %Text_End);
  TEXT  = ANY_CRLF* . (LINE <: ANY_CRLF %Line_CRLF)+ | LINE | ANY_CRLF %Line_CRLF;

  main := TEXT;
}%%

#include <glib.h>

%% write data;

void
rspamd_strip_newlines_parse (const gchar *begin, const gchar *pe,
    GByteArray *data, gboolean is_html, guint *newlines_count,
    GPtrArray *newlines)
{
  const gchar *c, *p, *eof;
  gint last_c = -1;
  gint cs = 0;
  gboolean crlf_added = FALSE;

  c = begin;
  p = begin;
  eof = pe;

  %% write init;
  %% write exec;

  if (p > c) {
     g_byte_array_append (data, (const guint8 *)c, p - c);
  }
}

%%{
  machine newlines_strip;

  action Double_CRLF {
    if (!crlf_added && p > c) {
      (*newlines_count)++;
      g_byte_array_append (data, (const guint8 *)"\n", 1);
      c = p;
    }

    crlf_added = TRUE;
    c = p;
  }

  action WSP {
    g_byte_array_append (data, (const guint8 *)" ", 1);
    c = p;
  }

  action Text_Start {
    crlf_added = FALSE;
    c = p;
  }

  action Text_End {
    if (p > c) {
      g_byte_array_append (data, (const guint8 *)c, p - c);
      last_c = *(p - 1);
    }

    c = p;
  }

  action Line_CRLF {
    if (!crlf_added) {
      if (is_html || g_ascii_ispunct (last_c)) {
         g_byte_array_append (data, (const guint8 *)"\n", 1);
         crlf_added = TRUE;
      }
    }

    (*newlines_count)++;
    g_ptr_array_add (newlines, (gpointer)p);
    c = p;
  }

  WSP   = " " | "\t" | "\v";
  CRLF  = ("\r" . "\n") | ( "\r" ) | ("\n");
  DOUBLE_CRLF = (CRLF <: (WSP* CRLF)+) %Double_CRLF;
  ANY_CRLF = CRLF | DOUBLE_CRLF;
  LINE = (([^\r\n]+) >Text_Start %Text_End);
  TEXT  = ANY_CRLF* . (LINE <: ANY_CRLF %Line_CRLF)+ | LINE | ANY_CRLF %Line_CRLF;

  main := TEXT;
}%%

#include <glib.h>

%% write data;

void
rspamd_strip_newlines_parse (const gchar *begin, const gchar *pe,
    GByteArray *data, gboolean is_html, guint *newlines_count,
    GPtrArray *newlines)
{
  const gchar *c, *p, *eof;
  gint last_c = -1;
  gint cs = 0;
  gboolean crlf_added = FALSE;

  c = begin;
  p = begin;
  eof = pe;

  %% write init;
  %% write exec;

  if (p > c) {
     g_byte_array_append (data, (const guint8 *)c, p - c);
  }
}


Removed unused C lexer macros.

Removed unused C lexer macros.

#include "lexer.h"

/*
The following two macros allow the Ragel grammar to use generic function calls
without relying on the setup of the C or Java lexer. Using these macros we can
also pass along `self` to the callback functions without having to hard-code
this in to the Ragel grammar.

In the C lexer we don't need the `data` variable (since this is pulled in based
on `ts` and `te`) so the macro ignores this argument.
*/

#define callback(name, data, encoding, start, stop) \
    liboga_xml_lexer_callback(self, name, encoding, start, stop);

#define callback_simple(name) \
    liboga_xml_lexer_callback_simple(self, name);

#define advance_line(amount) \
    rb_funcall(self, rb_intern("advance_line"), 1, INT2NUM(amount));

%%machine c_lexer;

/**
 * Calls a method defined in the Ruby side of the lexer. The String value is
 * created based on the values of `ts` and `te` and uses the encoding specified
 * in `encoding`.
 *
 * @example
 *  rb_encoding *encoding = rb_enc_get(...);
 *  liboga_xml_lexer_callback(self, "on_string", encoding, ts, te);
 */
void liboga_xml_lexer_callback(
    VALUE self,
    VALUE name,
    rb_encoding *encoding,
    const char *ts,
    const char *te
)
{
    VALUE value = rb_enc_str_new(ts, te - ts, encoding);

    rb_funcall(self, name, 1, value);
}

/**
 * Calls a method defined in the Ruby side of the lexer without passing it any
 * arguments.
 *
 * @example
 *  liboga_xml_lexer_callback_simple(self, "on_cdata_start");
 */
void liboga_xml_lexer_callback_simple(VALUE self, VALUE name)
{
    rb_funcall(self, name, 0);
}

%% write data;

/**
 * Lexes the String specifies as the method argument. Token values have the
 * same encoding as the input value.
 *
 * This method keeps track of an internal state using the instance variables
 * `@act` and `@cs`.
 */
VALUE oga_xml_lexer_advance(VALUE self, VALUE data_block)
{
    OgaLexerState *state;

    /* Make sure that all data passed back to Ruby has the proper encoding. */
    rb_encoding *encoding = rb_enc_get(data_block);

    char *data_str_val = StringValueCStr(data_block);

    Data_Get_Struct(self, OgaLexerState, state);

    const char *p    = data_str_val;
    const char *pe   = data_str_val + strlen(data_str_val);
    const char *eof  = pe;
    const char *ts   = 0;
    const char *te   = 0;
    const char *mark = 0;

    int lines = state->lines;

    ID id_advance_line        = rb_intern("advance_line");
    ID id_on_attribute        = rb_intern("on_attribute");
    ID id_on_attribute_ns     = rb_intern("on_attribute_ns");
    ID id_on_cdata            = rb_intern("on_cdata");
    ID id_on_comment          = rb_intern("on_comment");
    ID id_on_doctype_end      = rb_intern("on_doctype_end");
    ID id_on_doctype_inline   = rb_intern("on_doctype_inline");
    ID id_on_doctype_name     = rb_intern("on_doctype_name");
    ID id_on_doctype_start    = rb_intern("on_doctype_start");
    ID id_on_doctype_type     = rb_intern("on_doctype_type");
    ID id_on_element_end      = rb_intern("on_element_end");
    ID id_on_element_name     = rb_intern("on_element_name");
    ID id_on_element_ns       = rb_intern("on_element_ns");
    ID id_on_element_open_end = rb_intern("on_element_open_end");
    ID id_on_element_start    = rb_intern("on_element_start");
    ID id_on_proc_ins_end     = rb_intern("on_proc_ins_end");
    ID id_on_proc_ins_name    = rb_intern("on_proc_ins_name");
    ID id_on_proc_ins_start   = rb_intern("on_proc_ins_start");
    ID id_on_string_body      = rb_intern("on_string_body");
    ID id_on_string_dquote    = rb_intern("on_string_dquote");
    ID id_on_string_squote    = rb_intern("on_string_squote");
    ID id_on_text             = rb_intern("on_text");
    ID id_on_xml_decl_end     = rb_intern("on_xml_decl_end");
    ID id_on_xml_decl_start   = rb_intern("on_xml_decl_start");

    %% write exec;

    state->lines = lines;

    return Qnil;
}

/**
 * Resets the internal state of the lexer.
 */
VALUE oga_xml_lexer_reset(VALUE self)
{
    OgaLexerState *state;

    Data_Get_Struct(self, OgaLexerState, state);

    state->act   = 0;
    state->cs    = c_lexer_start;
    state->lines = 0;
    state->top   = 0;

    return Qnil;
}

/**
 * Frees the associated lexer state struct.
 */
void oga_xml_lexer_free(void *state)
{
    free((OgaLexerState *) state);
}

/**
 * Allocates and wraps the C lexer state struct. This state is used to keep
 * track of the current position, line numbers, etc.
 */
VALUE oga_xml_lexer_allocate(VALUE klass)
{
    OgaLexerState *state = malloc(sizeof(OgaLexerState));

    return Data_Wrap_Struct(klass, NULL, oga_xml_lexer_free, state);
}

%%{
    include base_lexer "base_lexer.rl";

    variable top state->top;
    variable stack state->stack;
    variable act state->act;
    variable cs state->cs;
}%%

void Init_liboga_xml_lexer()
{
    VALUE mOga   = rb_const_get(rb_cObject, rb_intern("Oga"));
    VALUE mXML   = rb_const_get(mOga, rb_intern("XML"));
    VALUE cLexer = rb_define_class_under(mXML, "Lexer", rb_cObject);

    rb_define_method(cLexer, "advance_native", oga_xml_lexer_advance, 1);
    rb_define_method(cLexer, "reset_native", oga_xml_lexer_reset, 0);

    rb_define_alloc_func(cLexer, oga_xml_lexer_allocate);
}

#include "lexer.h"

/*
The following two macros allow the Ragel grammar to use generic function calls
without relying on the setup of the C or Java lexer. Using these macros we can
also pass along `self` to the callback functions without having to hard-code
this in to the Ragel grammar.

In the C lexer we don't need the `data` variable (since this is pulled in based
on `ts` and `te`) so the macro ignores this argument.
*/

#define callback(name, data, encoding, start, stop) \
    liboga_xml_lexer_callback(self, name, encoding, start, stop);

#define callback_simple(name) \
    liboga_xml_lexer_callback_simple(self, name);

#define oga_ivar_get(owner, name) \
    rb_ivar_get(owner, rb_intern(name))

#define oga_ivar_set(owner, name, value) \
    rb_ivar_set(owner, rb_intern(name), value)

#define advance_line(amount) \
    rb_funcall(self, rb_intern("advance_line"), 1, INT2NUM(amount));

%%machine c_lexer;

/**
 * Calls a method defined in the Ruby side of the lexer. The String value is
 * created based on the values of `ts` and `te` and uses the encoding specified
 * in `encoding`.
 *
 * @example
 *  rb_encoding *encoding = rb_enc_get(...);
 *  liboga_xml_lexer_callback(self, "on_string", encoding, ts, te);
 */
void liboga_xml_lexer_callback(
    VALUE self,
    VALUE name,
    rb_encoding *encoding,
    const char *ts,
    const char *te
)
{
    VALUE value = rb_enc_str_new(ts, te - ts, encoding);

    rb_funcall(self, name, 1, value);
}

/**
 * Calls a method defined in the Ruby side of the lexer without passing it any
 * arguments.
 *
 * @example
 *  liboga_xml_lexer_callback_simple(self, "on_cdata_start");
 */
void liboga_xml_lexer_callback_simple(VALUE self, VALUE name)
{
    rb_funcall(self, name, 0);
}

%% write data;

/**
 * Lexes the String specifies as the method argument. Token values have the
 * same encoding as the input value.
 *
 * This method keeps track of an internal state using the instance variables
 * `@act` and `@cs`.
 */
VALUE oga_xml_lexer_advance(VALUE self, VALUE data_block)
{
    OgaLexerState *state;

    /* Make sure that all data passed back to Ruby has the proper encoding. */
    rb_encoding *encoding = rb_enc_get(data_block);

    char *data_str_val = StringValueCStr(data_block);

    Data_Get_Struct(self, OgaLexerState, state);

    const char *p    = data_str_val;
    const char *pe   = data_str_val + strlen(data_str_val);
    const char *eof  = pe;
    const char *ts   = 0;
    const char *te   = 0;
    const char *mark = 0;

    int lines = state->lines;

    ID id_advance_line        = rb_intern("advance_line");
    ID id_on_attribute        = rb_intern("on_attribute");
    ID id_on_attribute_ns     = rb_intern("on_attribute_ns");
    ID id_on_cdata            = rb_intern("on_cdata");
    ID id_on_comment          = rb_intern("on_comment");
    ID id_on_doctype_end      = rb_intern("on_doctype_end");
    ID id_on_doctype_inline   = rb_intern("on_doctype_inline");
    ID id_on_doctype_name     = rb_intern("on_doctype_name");
    ID id_on_doctype_start    = rb_intern("on_doctype_start");
    ID id_on_doctype_type     = rb_intern("on_doctype_type");
    ID id_on_element_end      = rb_intern("on_element_end");
    ID id_on_element_name     = rb_intern("on_element_name");
    ID id_on_element_ns       = rb_intern("on_element_ns");
    ID id_on_element_open_end = rb_intern("on_element_open_end");
    ID id_on_element_start    = rb_intern("on_element_start");
    ID id_on_proc_ins_end     = rb_intern("on_proc_ins_end");
    ID id_on_proc_ins_name    = rb_intern("on_proc_ins_name");
    ID id_on_proc_ins_start   = rb_intern("on_proc_ins_start");
    ID id_on_string_body      = rb_intern("on_string_body");
    ID id_on_string_dquote    = rb_intern("on_string_dquote");
    ID id_on_string_squote    = rb_intern("on_string_squote");
    ID id_on_text             = rb_intern("on_text");
    ID id_on_xml_decl_end     = rb_intern("on_xml_decl_end");
    ID id_on_xml_decl_start   = rb_intern("on_xml_decl_start");

    %% write exec;

    state->lines = lines;

    return Qnil;
}

/**
 * Resets the internal state of the lexer.
 */
VALUE oga_xml_lexer_reset(VALUE self)
{
    OgaLexerState *state;

    Data_Get_Struct(self, OgaLexerState, state);

    state->act   = 0;
    state->cs    = c_lexer_start;
    state->lines = 0;
    state->top   = 0;

    return Qnil;
}

/**
 * Frees the associated lexer state struct.
 */
void oga_xml_lexer_free(void *state)
{
    free((OgaLexerState *) state);
}

/**
 * Allocates and wraps the C lexer state struct. This state is used to keep
 * track of the current position, line numbers, etc.
 */
VALUE oga_xml_lexer_allocate(VALUE klass)
{
    OgaLexerState *state = malloc(sizeof(OgaLexerState));

    return Data_Wrap_Struct(klass, NULL, oga_xml_lexer_free, state);
}

%%{
    include base_lexer "base_lexer.rl";

    variable top state->top;
    variable stack state->stack;
    variable act state->act;
    variable cs state->cs;
}%%

void Init_liboga_xml_lexer()
{
    VALUE mOga   = rb_const_get(rb_cObject, rb_intern("Oga"));
    VALUE mXML   = rb_const_get(mOga, rb_intern("XML"));
    VALUE cLexer = rb_define_class_under(mXML, "Lexer", rb_cObject);

    rb_define_method(cLexer, "advance_native", oga_xml_lexer_advance, 1);
    rb_define_method(cLexer, "reset_native", oga_xml_lexer_reset, 0);

    rb_define_alloc_func(cLexer, oga_xml_lexer_allocate);
}


nfa-repetition operator test case

nfa-repetition operator test case

/*
 * @LANG: c
 * @PROHIBIT_GENFLAGS: -T0 -T1 -F0 -G0 -G1 -G2
 */

#include <stddef.h>  /* NULL */
#include <stdint.h>  /* uint64_t */
#include <stdlib.h>  /* malloc(3) free(3) */
#include <stdbool.h> /* bool */
#include <string.h>
#include <stdio.h>

struct nfa_bp_rec
{
	long state;
	const unsigned char *p;
	int pop;
};

struct nfa_bp_rec nfa_bp[1024];
long nfa_len = 0;

long c;

struct nfa_state_rec
{
	long c;
};

struct nfa_state_rec nfa_s[1024];

void nfa_push()
{
	printf( "push\n" );
	nfa_s[nfa_len].c = c;
}

void nfa_pop()
{
	printf( "pop\n" );
	c = nfa_s[nfa_len].c;
}

%%{
	machine match_any;
	alphtype unsigned char;

	action eol { p+1 == eof }

	eol = '' %when eol;

	action ini6
	{
		printf( "ini_6\n" );
		c = 0;
	}

	action min6
	{
		printf( "min_6\n" );
		if ( ++c < 2 )
			fgoto *match_any_error;
	}

	action max6
	{
		printf( "max_6\n" );
		if ( ++c == 3 )
			fgoto *match_any_error;
	}

	main :=
		( :( ( 'a' @{printf("b\n");} ),
				ini6, min6, max6, {nfa_push();}, {nfa_pop();} ): ' ' ) {2}
		eol
		any @{printf("----- MATCH\n");}
	;

	write data;
}%%

int test( const char *data )
{
	int cs;
	const unsigned char *p = (const unsigned char *)data;
	const unsigned char *pe = p + strlen(data) + 1;
	const unsigned char *eof = pe;

	%%write init;
	%%write exec;

	return 0;
}

int main()
{
	test( "aaa aaa " );
	return 0;
}

##### OUTPUT #####
push
pop
ini_6
b
push
push
push
pop
min_6
pop
max_6
b
push
push
push
pop
min_6
pop
max_6
b
push
push
push
pop
min_6
push
pop
ini_6
b
push
push
push
pop
min_6
pop
max_6
b
push
push
push
pop
min_6
pop
max_6
b
push
push
push
pop
min_6
----- MATCH
pop
max_6
pop
pop
pop
pop
max_6
pop
pop
pop

error: pathspec 'test/genrep1.rl' did not match any file(s) known to git


Whoops, forgot to add email.rl

Whoops, forgot to add email.rl

git-svn-id: 26a6d2a58b162bc133329b2f36847a121ee54040@1001 06fd6eb0-0002-0410-a719-e5602cce40bc

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/dimelo/ferret-dimelo.git/'


Use C++ style comments for sections

Use C++ style comments for sections

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/gim/libgim.git/'


version: silence potential use before assign


dont delete it

dont delete it

package dns

// Parse RRs
// With the thankful help of gdnsd and the Go examples for Ragel.
//

import (
    "os"
    "io"
    "net"
    "strconv"
)

const _RDATAMAX = 7
const _IOBUF = 65365

// Save up tokens, after we've seen the entire rdata
// we can use this.
type token struct {
        T []string      // text
        N []int         // number
        ti int          // text counter
        ni int          // number counter
}

func newToken() *token {
        to := new(token)
        to.T = make([]string, _RDATAMAX)
        to.N = make([]int, _RDATAMAX)
        to.ni, to.ti = 0, 0
        return to
}

// Only push functions are provided. Reading is done, by directly
// accessing the members (T and N). See types.rl.
func (to *token) pushInt(s string) {
        i, err := strconv.Atoi(s)
        if err != nil {
                panic("Failure to parse to int: " + s)
        }
        to.N[to.ni] = i
        to.ni++
        if to.ni > _RDATAMAX {
                panic("Too much rdata (int)")
        }
}

func (to *token) pushString(s string) {
        to.T[to.ti] = s
        to.ti++
        if to.ti > _RDATAMAX {
                panic("Too much rdata (string)")
        }
}

func (to *token) reset() {
        to.ni, to.ti = 0, 0
}

func rdata_aaaa(hdr RR_Header, tok *token) RR {
        rr := new(RR_AAAA)
        rr.Hdr = hdr
        rr.Hdr.Rrtype = TypeAAAA
        rr.AAAA = net.ParseIP(tok.T[0])
        println("Calling aaaa", tok.T[0])
        return rr
}

func rdata_a(hdr RR_Header, tok *token) RR {
        rr := new(RR_A)
        rr.Hdr = hdr
        rr.Hdr.Rrtype = TypeA
        rr.A = net.ParseIP(tok.T[0])
        return rr
}

func rdata_ns(hdr RR_Header, tok *token) RR {
        rr := new(RR_NS)
        rr.Hdr = hdr
        rr.Hdr.Rrtype = TypeNS
        rr.Ns = tok.T[0]
        return rr
}
func rdata_cname(hdr RR_Header, tok *token) RR {
        rr := new(RR_CNAME)
        rr.Hdr = hdr
        rr.Hdr.Rrtype = TypeCNAME
        rr.Cname = tok.T[0]
        return rr
}
func rdata_soa(hdr RR_Header, tok *token) RR {
        rr := new(RR_SOA)
        rr.Hdr = hdr
        rr.Hdr.Rrtype = TypeSOA
        rr.Ns = tok.T[0]
        rr.Mbox = tok.T[1]
        rr.Serial = uint32(tok.N[0])
        rr.Refresh = uint32(tok.N[1])
        rr.Retry = uint32(tok.N[2])
        rr.Expire = uint32(tok.N[3])
        rr.Minttl = uint32(tok.N[4])
        return rr
}
func rdata_mx(hdr RR_Header, tok *token) RR {
        rr := new(RR_MX)
        rr.Hdr = hdr;
        rr.Hdr.Rrtype = TypeMX
        rr.Pref = uint16(tok.N[0])
        rr.Mx = tok.T[0]
        return rr
}
func rdata_ds(hdr RR_Header, tok *token) RR {
        rr := new(RR_DS)
        rr.Hdr = hdr;
        rr.Hdr.Rrtype = TypeDS
        rr.KeyTag = uint16(tok.N[0])
        rr.Algorithm = uint8(tok.N[1])
        rr.DigestType = uint8(tok.N[2])
        rr.Digest = tok.T[0]
        return rr
}
func rdata_dnskey(hdr RR_Header, tok *token) RR {
        rr := new(RR_DNSKEY)
        rr.Hdr = hdr;
        rr.Hdr.Rrtype = TypeDNSKEY
        rr.Flags = uint16(tok.N[0])
        rr.Protocol = uint8(tok.N[1])
        rr.Algorithm = uint8(tok.N[2])
        rr.PublicKey = tok.T[0]
        return rr
}
func rdata_rrsig(hdr RR_Header, tok *token) RR {
        rr := new(RR_RRSIG)
        rr.Hdr = hdr;
        rr.Hdr.Rrtype = TypeRRSIG
        rr.TypeCovered = uint16(tok.N[0])
        rr.Algorithm = uint8(tok.N[1])
        rr.Labels = uint8(tok.N[2])
        rr.OrigTtl = uint32(tok.N[3])
        rr.Expiration = uint32(tok.N[4])
        rr.Inception = uint32(tok.N[5])
        rr.KeyTag = uint16(tok.N[6])
        rr.SignerName = tok.T[0]
        rr.Signature = tok.T[1]
        return rr
}

func set(r RR, z *Zone, tok *token) {
    z.Push(r)
    tok.reset()
    println("Resetting")
}

%%{
        machine z;
        write data;
}%%

// SetString
// All the NewReader stuff is expensive...
// only works for short io.Readers as we put the whole thing
// in a string -- needs to be extended for large files (sliding window).
func Zparse(q io.Reader) (z *Zone, err os.Error) {
        buf := make([]byte, _IOBUF)
        n, err := q.Read(buf)
        if err != nil {
            return nil, err
        }
        buf = buf[:n]
        z = new(Zone)

        data := string(buf)
        cs, p, pe := 0, 0, len(data)
        ts, te, act := 0, 0, 0
//        top := 0
//        stack := make([]int, 100)
        eof := len(data)
        // keep Go happy - need to fix this ofcourse
        ts = ts; te = te; act = act

        brace := false
        lines := 0
        mark := 0
        hdr := new(RR_Header)
        tok := newToken()
        var rr RR
        rr = rr

        %%{
                action mark       { mark = p }
                action qname      { hdr.Name = data[mark:p] }
                action qclass     { hdr.Class = Str_class[data[mark:p]] }
                action defTtl     { /* ... */ }
                action setTtl     { ttl, _ :=  strconv.Atoi(data[mark:p]); hdr.Ttl = uint32(ttl) }
                action number     { tok.pushInt(data[mark:p]) }
                action text       { tok.pushString(data[mark:p]) }
                action openBrace  { if brace { println("Brace already open")} ; brace = true }
                action closeBrace { if !brace { println("Brace already closed")}; brace = false }
                action brace      { brace }
                action linecount  { lines++ }

                # Newlines
                nl = [\n]+ $linecount;

                # Comments, entire line. Shorter comments are handled in the
                # 'bl' definition below.
                comment = ';' [^\n]*;

                bl = (
                    [ \t]+
                    | '(' $openBrace
                    | ')' $closeBrace
                    | (comment? nl)+ when brace
                )+ %mark;

#                chars       = [^; \t"\n\\)(];
                ws          = [ \t]+;
                qclass      = ('IN'i|'CS'i|'CH'i|'HS'i|'ANY'i|'NONE'i) %qclass;
                qname       = [a-zA-Z0-9_\-.]+ %qname;
                t           = [a-zA-Z0-9_\-.:]+ $1 %0 %text;
                tb          = [a-zA-Z0-9_\-.: ]+ $1 %0 %text;
                n           = [0-9]+ $1 %0 %number;
                ttl         = digit+ >mark;

                lhs = qname? bl %defTtl (
                      (ttl %setTtl bl (qclass bl)?)
                    | (qclass bl (ttl %setTtl bl)?)
                )?;

                main := |*
                    lhs 'A'i        bl t nl      => { rr = rdata_a(*hdr, tok); set(rr, z, tok); };
                    lhs 'NS'i       bl t nl      => { rr = rdata_ns(*hdr, tok); set(rr, z, tok); };
                    lhs 'CNAME'i    bl t nl      => { rr = rdata_cname(*hdr, tok); set(rr, z, tok); };
                    lhs 'AAAA'i     bl t nl      => { rr = rdata_aaaa(*hdr, tok); set(rr, z, tok); };
                    lhs 'MX'i       bl n bl t nl => { rr = rdata_mx(*hdr, tok); set(rr, z, tok); };
                    lhs 'SOA'i      bl t bl t bl n bl n bl n bl n bl n nl => { rr = rdata_soa(*hdr, tok); set(rr, z, tok); };
                *|;
#                main := (rr? bl? ((comment? nl) when !brace))*;

                write init;
                write exec;
        }%%

        if eof > -1 {
                if cs < z_first_final {
                        // No clue what I'm doing what so ever
                        if p == pe {
                                println("unexpected eof")
                                return z, nil
                        } else {
                                println("error at position ", p, "\"",data[mark:p],"\"")
                                return z, nil
                        }
                }
        }
        return z, nil
}

error: pathspec 'zparse.rl' did not match any file(s) known to git


[Minor] Further simplifications

[Minor] Further simplifications