clang5 ocaml test case working

clang5 ocaml test case working

(*
 * @LANG: ocaml
 *)

let id x = x
let fail fmt = Printf.ksprintf failwith fmt
let pr fmt = Printf.ksprintf print_endline fmt

let failed fmt = Printf.ksprintf (fun s -> prerr_endline s; exit 1) fmt
let test' show f x y = if f x <> y then failed "FAILED: test %S" (show x)
let case = ref 0
let test f x y = incr case; if f x <> y then failed "FAILED: case %d" !case
let error f x = match try Some (f x) with _ -> None with Some _ -> failed "FAILED: fail %S" x | None -> ()

%%{
	machine clang;

	newline = '\n' @{ incr curlin };
	any_count_line = any | newline;

	# Consume a C comment.
	c_comment := any_count_line* :>> '*/' @{fgoto main;};

	main := |*

	# Alpha numberic characters or underscore.
	alnum_u = alnum | '_';

	# Alpha charactres or underscore.
	alpha_u = alpha | '_';

	# Symbols. Upon entering clear the buffer. On all transitions
	# buffer a character. Upon leaving dump the symbol.
	( punct - [_'"] ) {
		pr "symbol(%i): %c" !curlin data.[!ts];
	};

	# Identifier. Upon entering clear the buffer. On all transitions
	# buffer a character. Upon leaving, dump the identifier.
	alpha_u alnum_u* {
		pr "ident(%i): %s" !curlin (String.sub data !ts (!te - !ts));
	};

	# Single Quote.
	sliteralChar = [^'\\] | newline | ( '\\' . any_count_line );
	'\'' . sliteralChar* . '\'' {
		pr "single_lit(%i): %s" !curlin (String.sub data !ts (!te - !ts));
	};

	# Double Quote.
	dliteralChar = [^"\\] | newline | ( '\\' any_count_line );
	'"' . dliteralChar* . '"' {
		pr "double_lit(%i): %s" !curlin (String.sub data !ts (!te - !ts));
	};

	# Whitespace is standard ws, newlines and control codes.
	any_count_line - 0x21..0x7e;

	# Describe both c style comments and c++ style comments. The
	# priority bump on tne terminator of the comments brings us
	# out of the extend* which matches everything.
	'//' [^\n]* newline;

	'/*' { fgoto c_comment; };

	# Match an integer. We don't bother clearing the buf or filling it.
	# The float machine overlaps with int and it will do it.
	digit+ {
		pr "int(%i): %s" !curlin (String.sub data !ts (!te - !ts));
	};

	# Match a float. Upon entering the machine clear the buf, buffer
	# characters on every trans and dump the float upon leaving.
	digit+ '.' digit+ {
		pr "float(%i): %s" !curlin (String.sub data !ts (!te - !ts));
	};

	# Match a hex. Upon entering the hex part, clear the buf, buffer characters
	# on every trans and dump the hex on leaving transitions.
	'0x' xdigit+ {
		pr "hex(%i): %s" !curlin (String.sub data !ts (!te - !ts));
	};

	*|;
}%%

%% write data;

let clang data =
	let cs = ref 0 in
	let p = ref 0 in
	let pe = ref (String.length data) in
	let ts = ref 0 in
	let te = ref 0 in
	let eof = pe in
	let curlin = ref 1 in

	%%{
		write init;
		write exec;
	}%%

	if !cs < clang_first_final then
		fail "atoi: cs %d < %d" !cs clang_first_final;
;;

let () =
	clang "999 0xaAFF99 99.99 /!\n!/ 'lksdj' //\n\"\n\nliteral\n\n\n\"0x00aba foobardd.ddsf 0x0.9\n";
	clang "wordwithnum00asdf\n000wordfollowsnum,makes new symbol\n\nfinishing early /! unfinished ...\n";
  ()
;;

(* _____OUTPUT_____
int(1): 999
hex(1): 0xaAFF99
float(1): 99.99
symbol(1): /
symbol(1): !
symbol(2): !
symbol(2): /
single_lit(2): 'lksdj'
double_lit(8): "

literal

"
hex(8): 0x00aba
ident(8): foobardd
symbol(8): .
ident(8): ddsf
hex(8): 0x0
symbol(8): .
int(8): 9
ident(1): wordwithnum00asdf
int(2): 000
ident(2): wordfollowsnum
symbol(2): ,
ident(2): makes
ident(2): new
ident(2): symbol
ident(4): finishing
ident(4): early
symbol(4): /
symbol(4): !
ident(4): unfinished
symbol(4): .
symbol(4): .
symbol(4): .
_____OUTPUT_____ *)

(*
 * @LANG: ocaml
 *)

let id x = x
let fail fmt = Printf.ksprintf failwith fmt
let pr fmt = Printf.ksprintf print_endline fmt

let failed fmt = Printf.ksprintf (fun s -> prerr_endline s; exit 1) fmt
let test' show f x y = if f x <> y then failed "FAILED: test %S" (show x)
let case = ref 0
let test f x y = incr case; if f x <> y then failed "FAILED: case %d" !case
let error f x = match try Some (f x) with _ -> None with Some _ -> failed "FAILED: fail %S" x | None -> ()

%%{
	machine clang;

	newline = '\n' @{ incr curlin };
	any_count_line = any | newline;

	# Consume a C comment.
	c_comment := any_count_line* :>> '*/' @{fgoto main;};

	main := |*

	# Alpha numberic characters or underscore.
	alnum_u = alnum | '_';

	# Alpha charactres or underscore.
	alpha_u = alpha | '_';

	# Symbols. Upon entering clear the buffer. On all transitions
	# buffer a character. Upon leaving dump the symbol.
	( punct - [_'"] ) {
		T.pr "symbol(%i): %c" !curlin data.[!ts];
	};

	# Identifier. Upon entering clear the buffer. On all transitions
	# buffer a character. Upon leaving, dump the identifier.
	alpha_u alnum_u* {
		T.pr "ident(%i): %s" !curlin (String.sub data !ts (!te - !ts));
	};

	# Single Quote.
	sliteralChar = [^'\\] | newline | ( '\\' . any_count_line );
	'\'' . sliteralChar* . '\'' {
		T.pr "single_lit(%i): %s" !curlin (String.sub data !ts (!te - !ts));
	};

	# Double Quote.
	dliteralChar = [^"\\] | newline | ( '\\' any_count_line );
	'"' . dliteralChar* . '"' {
		T.pr "double_lit(%i): %s" !curlin (String.sub data !ts (!te - !ts));
	};

	# Whitespace is standard ws, newlines and control codes.
	any_count_line - 0x21..0x7e;

	# Describe both c style comments and c++ style comments. The
	# priority bump on tne terminator of the comments brings us
	# out of the extend* which matches everything.
	'//' [^\n]* newline;

	'/*' { fgoto c_comment; };

	# Match an integer. We don't bother clearing the buf or filling it.
	# The float machine overlaps with int and it will do it.
	digit+ {
		T.pr "int(%i): %s" !curlin (String.sub data !ts (!te - !ts));
	};

	# Match a float. Upon entering the machine clear the buf, buffer
	# characters on every trans and dump the float upon leaving.
	digit+ '.' digit+ {
		T.pr "float(%i): %s" !curlin (String.sub data !ts (!te - !ts));
	};

	# Match a hex. Upon entering the hex part, clear the buf, buffer characters
	# on every trans and dump the hex on leaving transitions.
	'0x' xdigit+ {
		T.pr "hex(%i): %s" !curlin (String.sub data !ts (!te - !ts));
	};

	*|;
}%%

%% write data nofinal;

let fail fmt = Printf.ksprintf failwith fmt

let () =
	let data = String.create 2048 in
	let cs = ref 0 and act = ref 0 and have = ref 0 and curlin = ref 1 in
	let ts = ref 0 and te = ref 0 in
	let fin = ref false in

	%% write init;

	while not !fin do
		let p = ref !have and eof = ref (-1) in
		let space = String.length data - !have in

		if space = 0 then
			(* We've used up the entire buffer storing an already-parsed token
			 * prefix that must be preserved. *)
			fail "OUT OF BUFFER SPACE";

    let len = Unix.read Unix.stdin data !p space in
		let pe = ref (!p + len) in

		(* Check if this is the end of file. *)
		if len < space then (eof := !pe; fin := true);

		%% write exec;

		if !cs = clang_error then
			fail "PARSE ERROR";

		if !ts = -1 then
			have := 0
		else
    begin
			(* There is a prefix to preserve, shift it over. *)
			have := !pe - !ts;
			String.blit data !ts data 0 !have;
			te := !te - !ts;
			ts := 0;
		end
	done

(* _____OUTPUT_____
7
666
-666
666
123456789
123456789
0
_____OUTPUT_____ *)


Back to scanner

Back to scanner

%%{

  machine lexer;

  variable data @data;
  variable p    @p;
  variable pe   @pe;
  variable eof  @eof;
  access @;

  action mark {
    @start = @p
  }

  number_integer = '-'? [0-9][1-9]*;
  number_frac = '.' [0-9]+;
  number_e = ('e' | 'E') ('+' | '-' | '');
  number_exp = number_e [0-9]+;
  number = number_integer number_frac? number_exp?;

  string_double = '"' (
      any -- '"' |
      '\\"'
    )* '"';
  identifier = [A-Za-z_$][A-Za-z0-9_$]*;
  string_single = "'" [A-Za-z0-9_$\-]+;
  keywords = 'undefined' | 'null' | 'true' | 'false';
  unops = '!' | '++' | '--' | '~' | 'new' | 'return' |
    'typeof';
  binops = '+' | '-' | '*' | '/' | '&' | '|' | '^' | '<<' | '>>' |
    '>>>' | '==' | '!=' | '===' | '!==' | '>' | '>=' | '<' | '<=' |
    '&&' | '||' | 'instanceof' | 'or' | 'and';

  istring_part  = ( any -- '"' | '\\"' );
  istring_start = '"' ( istring_part* ) '#{';
  istring_mid   = '}' ( istring_part* ) '#{';
  istring_end   = '}' ( istring_part* ) '"';

  main := |*
    number        => { emit :number      };
    string_double => { emit :dstring     };
    string_single => { emit :sstring     };
    istring_start => { emit :istart      };
    istring_mid   => { emit :imid        };
    istring_end   => { emit :iend        };
    'class'       => { emit :class       };
    'module'      => { emit :module      };
    'if'          => { emit :if          };
    'unless'      => { emit :unless      };
    'elsif'       => { emit :elsif       };
    'else'        => { emit :else        };
    unops         => { emit :unop        };
    binops        => { emit :binop       };
    keywords      => { emit :keyword     };
    identifier    => { emit :identifier  };
    '->'          => { emit :arrow       };
    '='           => { emit :equal       };
    '{'           => { emit :lbrack      };
    '('           => { emit :lparen      };
    '['           => { emit :lbrace      };
    '}'           => { emit :rbrack      };
    ')'           => { emit :rparen      };
    ']'           => { emit :rbrace      };
    ':'           => { emit :colon       };
    '.'           => { emit :prop        };
    ','           => { emit :comma       };
    '\n'          => { line.call         };
    space         => {                   };
    any           => { error             };
  *|;
}%%

module Liquidscript
  class Scanner

    # A lexer, built from ragel.
    #
    # @private
    class Lexer

      attr_reader :tokens

      def initialize
        %% write data;
        # %% # fix
        @tokens = []
        clean!
      end

      def clean!
        @p = nil
        @pe = nil
        @te = nil
        @ts = nil
        @act = nil
        @eof = nil
        @top = nil
        @line = { :start => 0, :num => 0 }
        @data = nil
        @stack = nil
        @start = nil
      end

      def emit(type)
        @tokens << if @start
          Token.new(type, @data[@start..(@p - 1)],
            @line[:num], @p - @line[:start])
        else
          Token.new(type, nil, @line[:num], @p - @line[:start])
        end

        @start = nil
      end

      def error
        raise SyntaxError, "Unexpected #{@data[@start..(@p-1)].pack('c*')}"
      end

      def perform(data)
        @data = data.unpack("c*") if data.is_a? String
        @eof = data.length

        @tokens = []

        line = proc do
          @line[:start] = @ts
          @line[:num] += 1
          emit :newline
        end

        %% write init;
        %% write exec;

        clean!

        @tokens
      end
    end
  end
end

%%{

  machine lexer;

  variable data @data;
  variable p    @p;
  variable pe   @pe;
  variable eof  @eof;
  access @;

  action mark {
    @start = @p
  }

  number_integer = '-'? [0-9][1-9]*;
  number_frac = '.' [0-9]+;
  number_e = ('e' | 'E') ('+' | '-' | '');
  number_exp = number_e [0-9]+;
  number = number_integer number_frac? number_exp?;

  string_double = '"' (
      any -- '"' |
      '\\"'
    )* '"';
  identifier = [A-Za-z_$][A-Za-z0-9_$]*;
  string_single = "'" [A-Za-z0-9_$\-]+;
  keywords = 'undefined' | 'null' | 'true' | 'false';
  unops = '!' | '++' | '--' | '~' | 'new' | 'return' |
    'typeof';
  binops = '+' | '-' | '*' | '/' | '&' | '|' | '^' | '<<' | '>>' |
    '>>>' | '==' | '!=' | '===' | '!==' | '>' | '>=' | '<' | '<=' |
    '&&' | '||' | 'instanceof' | 'or' | 'and';

  istring_part   = ( any -- '"' | '\\"' );
  istring_start  = '"' ( istring_part* ) '#{';
  istring_middle = '}' ( istring_part* ) '#{';
  istring_end    = '}' ( istring_part* ) '"';

  body = (
           ( number         >mark %{ emit :number     } ) |
           ( string_double  >mark %{ emit :dstring    } ) |
           ( string_single  >mark %{ emit :sstring    } ) |
           ( istring_start  >mark %{ emit :istart     } ) |
           ( istring_middle >mark %{ emit :imiddle    } ) |
           ( istring_end    >mark %{ emit :iend       } ) |
           (       'class'        %{ emit :class      } ) |
           (       'module'       %{ emit :module     } ) |
           (       'if'           %{ emit :if         } ) |
           (       'unless'       %{ emit :unless     } ) |
           (       'elsif'        %{ emit :elsif      } ) |
           (       'else'         %{ emit :else       } ) |
           ( unops          >mark %{ emit :unop       } ) |
           ( binops         >mark %{ emit :binop      } ) |
           ( keywords       >mark %{ emit :keyword    } ) |
           ( identifier     >mark %{ emit :identifier } ) |
           (       '->'           %{ emit :arrow      } ) |
           (       '='            %{ emit :equal      } ) |
           (       '{'            %{ emit :lbrack     } ) |
           (       '('            %{ emit :lparen     } ) |
           (       '['            %{ emit :lbrace     } ) |
           (       '}'            %{ emit :rbrack     } ) |
           (       ')'            %{ emit :rparen     } ) |
           (       ']'            %{ emit :rbrace     } ) |
           (       ':'            %{ emit :colon      } ) |
           (       '.'            %{ emit :prop       } ) |
           (       ','            %{ emit :comma      } ) |
           (       '\n'           %{ line.call        } )
    );

  loop = ( body** );

  main := loop;
}%%

module Liquidscript
  class Scanner

    # A lexer, built from ragel.
    #
    # @private
    class Lexer

      attr_reader :tokens

      def initialize
        %% write data;
        # %% # fix
        @tokens = []
        clean!
      end

      def clean!
        @p = nil
        @pe = nil
        @te = nil
        @ts = nil
        @act = nil
        @eof = nil
        @top = nil
        @line = { :start => 0, :num => 0 }
        @data = nil
        @stack = nil
        @start = nil
      end

      def emit(type)
        @tokens << if @start
          Token.new(type, @data[@start..(@p - 1)],
            @line[:num], @p - @line[:start])
        else
          Token.new(type, nil, @line[:num], @p - @line[:start])
        end

        @start = nil
      end

      def error
        raise SyntaxError, "Unexpected #{@data[@start..(@p-1)].pack('c*')}"
      end

      def perform(data)
        @data = data.unpack("c*") if data.is_a? String
        @eof = data.length

        @tokens = []

        line = proc do
          @line[:start] = @ts
          @line[:num] += 1
          emit :newline
        end

        %% write init;
        %% write exec;

        clean!

        @tokens
      end
    end
  end
end


Disable emoticons to fix tokenisation of ':</p>'

/* -*- Mode: C++; indent-tabs-mode: nil -*- */

%%{
  machine tokenizer;

  action ignore {}
  action word { word_(WORD, dest, s); }
  action punct { punct_(PUNCTUATION, dest, s); }
  action end { end_(PUNCTUATION, dest, s); }
  action contraction { split_(WORD, CONTRACTION, dest, s); }
  action catchall {
    switch(errors){
      case ERROR_SKIP:
        break;
      case ERROR_CALL:
        error_(dest, s);
        break;
      case ERROR_THROW:
        die_(msg << "stuck on character " << (int)*p
                 << " at offset " << (p - s.offset));
        break;
      default:
        die_(msg << "illegal value " << errors << " for bad byte error handling");
    }
  }

  main := |*
    single_quote => { single_quote_(dest, s, eof); };
    double_quote => { double_quote_(dest, s, eof); };

    open_single_quote => { open_single_quote_(dest, s); };
    close_single_quote => { close_single_quote_(dest, s); };

    open_double_quote => { open_double_quote_(dest, s); };
    close_double_quote => { close_double_quote_(dest, s); };

    full_stop => { terminator_(dest, s, "."); };
    question_mark => { terminator_(dest, s, "?"); };
    exclamation_mark => { terminator_(dest, s, "!"); };
    ellipsis => { terminator_(dest, s, "..."); };

    dash => { dash_or_item_(dest, s); };
#    [1-9][0-9]* "." => { number_or_item_(dest, s); };

    (space - newline | other_ws)+ | newline => ignore;
    newline{2,} => { sep_text_paragraph_(dest, s); };

    open_p_tag => { begin_html_paragraph_(dest, s); };
    close_p_tag => { end_html_paragraph_(dest, s); };
    sep_p_tag => { sep_html_paragraph_(dest, s); };

    open_ul_tag | open_ol_tag => { begin_html_list_(dest, s); };
    close_ul_tag | close_ol_tag => { end_html_list_(dest, s); };

    open_li_tag => { begin_html_item_(dest, s); };
    close_li_tag => { end_html_item_(dest, s); };

    html_tag | html_comment | script_tag | style_tag => ignore;

    neg => contraction;
    neg_error => contraction;
    letter+ cont_suffix => contraction;

    (letter+ "."? possessive) - abbrev_decade => { split_(WORD, POSSESSIVE, dest, s); };

    (numbers units) - abbrev_decade => { split_(NUMBER, UNIT, dest, s); };
    time_ambiguous meridian => { split_(NUMBER, UNIT, dest, s); };
    meridian_token | date_time => word;

    (integer | float) "-" alpha+ ("-" alpha+)* => word;

    "and/or" | "AND/OR" => word;

    cont_misc | acron | title => word;
    symbols => punct;
    end_punct => end;
#    emoticon => punct;
    date_abbrev | state | addr => word;

    org | abbrev | lines | dollars | numbers | date_time => word;
    uri | host_name | email_address | twitter_username | hash_tag => word;
    default => word;

    any => catchall;
  *|;

}%%

/* -*- Mode: C++; indent-tabs-mode: nil -*- */

%%{
  machine tokenizer;

  action ignore {}
  action word { word_(WORD, dest, s); }
  action punct { punct_(PUNCTUATION, dest, s); }
  action end { end_(PUNCTUATION, dest, s); }
  action contraction { split_(WORD, CONTRACTION, dest, s); }
  action catchall {
    switch(errors){
      case ERROR_SKIP:
        break;
      case ERROR_CALL:
        error_(dest, s);
        break;
      case ERROR_THROW:
        die_(msg << "stuck on character " << (int)*p
                 << " at offset " << (p - s.offset));
        break;
      default:
        die_(msg << "illegal value " << errors << " for bad byte error handling");
    }
  }

  main := |*
    single_quote => { single_quote_(dest, s, eof); };
    double_quote => { double_quote_(dest, s, eof); };

    open_single_quote => { open_single_quote_(dest, s); };
    close_single_quote => { close_single_quote_(dest, s); };

    open_double_quote => { open_double_quote_(dest, s); };
    close_double_quote => { close_double_quote_(dest, s); };

    full_stop => { terminator_(dest, s, "."); };
    question_mark => { terminator_(dest, s, "?"); };
    exclamation_mark => { terminator_(dest, s, "!"); };
    ellipsis => { terminator_(dest, s, "..."); };

    dash => { dash_or_item_(dest, s); };
#    [1-9][0-9]* "." => { number_or_item_(dest, s); };

    (space - newline | other_ws)+ | newline => ignore;
    newline{2,} => { sep_text_paragraph_(dest, s); };

    open_p_tag => { begin_html_paragraph_(dest, s); };
    close_p_tag => { end_html_paragraph_(dest, s); };
    sep_p_tag => { sep_html_paragraph_(dest, s); };

    open_ul_tag | open_ol_tag => { begin_html_list_(dest, s); };
    close_ul_tag | close_ol_tag => { end_html_list_(dest, s); };

    open_li_tag => { begin_html_item_(dest, s); };
    close_li_tag => { end_html_item_(dest, s); };

    html_tag | html_comment | script_tag | style_tag => ignore;

    neg => contraction;
    neg_error => contraction;
    letter+ cont_suffix => contraction;

    (letter+ "."? possessive) - abbrev_decade => { split_(WORD, POSSESSIVE, dest, s); };

    (numbers units) - abbrev_decade => { split_(NUMBER, UNIT, dest, s); };
    time_ambiguous meridian => { split_(NUMBER, UNIT, dest, s); };
    meridian_token | date_time => word;

    (integer | float) "-" alpha+ ("-" alpha+)* => word;

    "and/or" | "AND/OR" => word;

    cont_misc | acron | title => word;
    symbols => punct;
    end_punct => end;
    emoticon => punct;
    date_abbrev | state | addr => word;

    org | abbrev | lines | dollars | numbers | date_time => word;
    uri | host_name | email_address | twitter_username | hash_tag => word;
    default => word;

    any => catchall;
  *|;

}%%


went on commenting mtoc

went on commenting mtoc

Conflicts:

	3rd_party/mtoc/confscanner.rl

#include "confscanner.h"

extern "C" {
#include <fnmatch.h>
}

using std::cerr;
using std::cout;
using std::endl;
using std::string;
using std::vector;
using std::map;
using std::set;
using std::ifstream;

%%{
  machine ConfFileScanner;
  write data;

  action st_tok { /*cerr << "st_tok" << line << "\n";*/ tmp_p = p; }

  action string_tok {
    tmp_string.assign(tmp_p, p-tmp_p);
  }

  action group_add
  {
//    cerr << "in group add: " << endl;
    if(match_at_level_[level_])
    {
//      cerr << "do it" << endl;
      tmp_string.assign(tmp_p, p-tmp_p);
      // clear groupset_ first, if we did not use '+='
      if(!arg_to_be_added_)
        groupset_.clear();

      groupset_.insert(tmp_string);
    }
  }

  action set_docu_block
  {
    tmp_string.assign(tmp_p, p-tmp_p);
    if(tmp_string == "brief")
      cblock_ = &docuheader_;
    else if(tmp_string == "doc")
      cblock_ = &docubody_;
    else if(tmp_string == "extra")
      cblock_ = &docuextra_;
  }

  action set_docu_list
  {
    tmp_string.assign(tmp_p, p-tmp_p);
    //cerr << "docu_list: " << tmp_string << "\n";
    if(tmp_string == "params")
    {
      //cerr << "paramlist\n";
      clist_ = &param_list_;
    }
    else if(tmp_string == "return")
      clist_ = &return_list_;
  }

  action set_docu_list_map
  {
    tmp_string.assign(tmp_p, p-tmp_p);
    if(tmp_string == "fields")
      clistmap_ = &field_docu_;
  }

  # add line of a block to the cblock structure which can be one of
  # docublock, docuheader, docuextra or clist[] or clistmap[][]
  action add_block_line
  {
    if(match_at_level_[level_])
    {
      tmp_string.assign(tmp_p, p-tmp_p);
      string :: size_type found = tmp_string.find_first_not_of(" \t");
      if(found == string::npos || string(tmp_string.substr(found)) != "\"\"")
      {
        found = tmp_string.rfind("\"");
        if(found != string::npos && string(tmp_string.substr(found-1)) == "\"\"")
        {
          //cerr << "a1: " << tmp_string.substr(0, found-1) + '\n';
          (*cblock_).push_back(tmp_string.substr(0, found-1) + '\n');
          //cerr << "after a1: " << param_list_["detailed_data"][0];
        }
        else
        {
          //cerr << "a2: " << tmp_string + '\n';
          (*cblock_).push_back(tmp_string + '\n');
        }
      }
      opt = true;
    }
  }

  # set the cblock pointer to clist_[value]
  action set_block_from_list
  {
    if(match_at_level_[level_])
    {
      tmp_string.assign(tmp_p, p-tmp_p);
      //cerr << "ts:" << tmp_string << ":ets\n";//<< " " << tmp_p << "p: " << p << "\n";
      cblock_ = &((*clist_)[tmp_string]);
    }
  }

  # set the cblock pointer to clist_[value1][value2]
  action set_block_from_listmap
  {
    string s(tmp_p, p-tmp_p);
    cblock_ = &((*clistmap_)[tmp_string][s]);
  }

  action reg_glob
  {
    tmp_string.assign(tmp_p, p-tmp_p);
    //cerr << "glob: " << tmp_string << endl;
    // TODO: use globlist_map
    globlist_stack_.back().push_back(tmp_string);
  }

  # end of file
  EOF = 0;

  # end of line (CRLF, LF)
  EOL = '\r'? . '\n';

  # any not end of file character
  default = ^0;

  # identifier
  IDENT = [A-Z]+;

  # matlab identifier
  MIDENT = [A-Za-z][A-Za-z0-9_]*;

  # glob expression
  GLOB = (default - [\n \t;\{])+;

  # swallow till the end of line
  garbling := (default - '\n')* . EOL @{line++;fret;};

  add_param_docu_block = ('brief'|'doc'|'extra');
  add_param_docu_list = ('params'|'return');
  add_param_docu_list_map = ('fields');

  # matlab white space or comment
  MWSOC = (
           [ \t]
           | EOL @{line++;}
           | (('#'. any) -'##') @{ /*cerr << "l: " << line << "\n";*/ fhold;fcall garbling; }
          );
#  MWSOC = ( [ \t\n]);

  # equal sign or += sign
  EQ = ('+'? @{arg_to_be_added_ = true;} . '=');

  # end of a rule sign
  ENDRULE = ';' @{arg_to_be_added_ = false;};

  # add line in a documentation block
  docu_block_line :=
    (
     # default action (just copy text to tmp_p)
     ( ( default - ["\n] )+ >{ if(opt) { tmp_p = p; } } )
     |
     # end line in documenation block (calls add_block_line)
     EOL @(add_block_line) @{line++;}
     |
     # end of documentation block (calls add_block_line)
     '"""' @(add_block_line) @{fret;}
     |
     # ignore quotes (less than 3 in a row)
     ('"'{1,2} . (default - ["\n])) @{ fhold; opt=false; }
    )**;

  # begin documentation block
  docu_block = ('"""'
                @{
                opt=true;
                if(!arg_to_be_added_)
                  (*cblock_).clear();
                fcall docu_block_line;
                }
               );

  # documentation list item (something like key => value)
  docu_list_item =
    (
     ( MIDENT
         >{tmp_p = p;}
         %(set_block_from_list)
     ) . MWSOC*
     . '=>' . MWSOC* . docu_block
    );

  docu_list = (
    (
     '('
     . ( MWSOC* . docu_list_item . MWSOC* . ',')*
     . MWSOC* . docu_list_item . MWSOC* . ')'
    )
    |
    docu_list_item
  );

  docu_list_map_item =
    (
     (MIDENT
       >{tmp_p = p;/* cerr << "dlmi\n";*/} %(string_tok)
     )
     . '.'
     .(MIDENT
       >{tmp_p = p;/* cerr << "dlmi2\n";*/} %(set_block_from_listmap)
      )
     . MWSOC*
     . '=>' . MWSOC* . docu_block
    );

  docu_list_map = (
    (
     '('
     . ( MWSOC* . docu_list_map_item . MWSOC* . ',')*
     . MWSOC* . docu_list_map_item . MWSOC* . ')'
    )
    |
    docu_list_map_item
  );

  globlist =
    (GLOB >{/*cerr << "glob_list";*/ tmp_p = p;} %(reg_glob))
    . (MWSOC* . [ \t]+ . MWSOC*
        . (GLOB >{/*cerr << "glob list2";*/ tmp_p = p;} %(reg_glob)))*;

  grouplist = (
    ((MIDENT >(st_tok)
             %(group_add))
     . ',')*
    .(MIDENT >(st_tok)
      %(group_add))
    );

  rules := (
    # glob list
    ('glob' .MWSOC* . '=' . MWSOC* . globlist . MWSOC* . '{')
      @{ check_glob_level_up(); fcall rules; }
    |
    (MWSOC)
    |
    # groups rule
    ('groups'. MWSOC* . EQ
     . MWSOC* . grouplist . MWSOC* . ENDRULE)
    |
    # add rule
    ('add(' %{/*cerr << "add:" << '\n';*/ tmp_p = p;}
     . (
        # add documentation block (brief, docu, extra)
        ((add_param_docu_block
            %(set_docu_block)
         )
         . ')' . MWSOC* . EQ
         . MWSOC* . docu_block . MWSOC* . ENDRULE )
        |
        # add documentation list (documentation block for a parameter or a
        # parameter or a return value)
        ((add_param_docu_list
            %(set_docu_list)
         )
         . ')' . MWSOC* . EQ
         . MWSOC* :> docu_list . MWSOC* . ENDRULE )
        |
        # add a map of documentation lists (documentation blocks for structure
        # fields of parameters or return values)
        ((add_param_docu_list_map
            %(set_docu_list_map)
         )
         . ')' . MWSOC* . EQ
         . MWSOC* :> docu_list_map . MWSOC* ENDRULE )
       )
    )
    |
    # go level down (end of a glob match block)
    ('}') @{ go_level_down(); fret; }
    )*;

  # main parser (first entry point) This block comprises variable definitions.
  main := (
           # ignored characters
           MWSOC
           | (
              # variable definitions (not yet implemented)
              (IDENT
                 >(st_tok)
                 %(string_tok)
              ). MWSOC* . ':=' . MWSOC*
              . (GLOB >st_tok @{/*TODO*/})
              . (MWSOC* . (GLOB >st_tok @{/*TODO*/}))*
              . MWSOC*
              . ';'
             )
           |
           # end variable definitions
           '##' . [ \t]* . EOL @{ line++; /*cerr<<"-> rules\n";*/ fgoto rules; }
          )**;
}%%

void ConfFileScanner :: go_level_down()
{
  globlist_stack_.pop_back();
  globlist_stack_.back().clear();
  level_--;
}

bool ConfFileScanner :: check_for_match(int l, const char * str,
                                        bool match_path_sep)
{
  typedef GlobList :: iterator iterator;
  GlobList & gl = globlist_stack_[l];
  /*  cerr << "globlist at level" << l << "\n  ";
  for(unsigned int i = 0; i < gl.size(); ++i)
  {
    cerr << gl[i] << " ";
  }
  cerr << endl;*/
  iterator endit = gl.end();
  int flags = (match_path_sep? FNM_PATHNAME : 0);
  // cerr << "checking for match at level " << l <<" of string " << str << ":" << endl;
  for( iterator it = gl.begin(); it != endit; it++ )
  {
    // cerr << "with pattern: " << (*it).c_str() << endl;
    if(fnmatch((*it).c_str(), str, flags) == 0)
    {
      // cerr << "SUCCESS!" << endl;
      return true;
    }
  }
  return false;
}

bool ConfFileScanner :: check_glob_rec(int l, const string & s)
{
  string str;
  string :: size_type found;
  if(l == level_+1)
    return true;

  found = s.find("/"); // try to match dir in path
  while(found != string :: npos)
  {
    str = s.substr(0, found);
//    cerr << str << endl;
    if(check_for_match(l, str.c_str())
        && check_glob_rec(l+1, s.substr(found+1)))
      return true;

    found = s.find("/", found+1); // try to match more dirs
  }
  if(l == level_) // try also to match the entire string
  {
    if(check_for_match(l, s.c_str()))
      return true;
  }

  return false;
}

void ConfFileScanner :: check_glob_level_up()
{
  match_at_level_[level_+1] = check_glob_rec(0, filename_);
/*  cerr << "going one level up, match at this level is: "
       << (match_at_level_[level_+1] ? "true" : "false");
  cerr << endl;*/

  globlist_stack_.push_back(GlobList());
  level_++;
}

ConfFileScanner
:: ConfFileScanner(const std::string & filename)
 : line(1), have(0), top(0), opt(true),
   filename_(filename),
   confistream_(set_conffile()),
   level_(0),
   arg_to_be_added_(false)
{
  globlist_stack_.push_back(GlobList());
  match_at_level_[0] = true;
}

const char * ConfFileScanner ::
set_conffile()
{
  conffile_ = "doxygen/mtoc.conf";
  return conffile_.c_str();
}

int ConfFileScanner :: execute()
{
  //cerr << "execute\n";
  std::ios::sync_with_stdio(false);

  %% write init;

  /* Do the first read. */
  bool done = false;
  while ( !done )
  {
    char *p = buf + have;
    char *tmp_p = p;// *tmp_p2 = p;
    string tmp_string;
    int space = BUFSIZE - have;

    if ( space == 0 )
    {
      /* We filled up the buffer trying to scan a token. */
      cerr << "OUT OF BUFFER SPACE" << endl;
      exit(1);
    }

    confistream_.read( p, space );
    int len = confistream_.gcount();
    char *pe = p + len;
    char *rpe = pe;
    char *eof = 0;

    /* If we see eof then append the EOF char. */
    if ( confistream_.eof() )
    {
      eof = pe;
      done = true;
    }
    else
    {
      /* Find the last semicolon by searching backwards. This
       * is where we will stop processing on this iteration. */
      while ( *pe!= ';' && pe > p )
        pe--;
    }

    %% write exec;

    /* Check if we failed. */
    if ( cs == ConfFileScanner_error )
    {
      /* Machine failed before finding a token. */
      cerr << "in conffile: PARSE ERROR in line " << line << endl;
      exit(1);
    }

    have = rpe - pe;
    /* cerr << "memmove by " << have << "bytes\n";*/
    memmove( buf, pe, have );
  }

  return 0;
}
/* vim: set et sw=2 ft=ragel: */

#include "confscanner.h"

extern "C" {
#include <fnmatch.h>
}

using std::cerr;
using std::cout;
using std::endl;
using std::string;
using std::vector;
using std::map;
using std::set;
using std::ifstream;

%%{
  machine ConfFileScanner;
  write data;

  action st_tok { /*cerr << "st_tok" << line << "\n";*/ tmp_p = p; }

  action string_tok {
    tmp_string.assign(tmp_p, p-tmp_p);
  }

  action group_add
  {
//    cerr << "in group add: " << endl;
    if(match_at_level_[level_])
    {
//      cerr << "do it" << endl;
      tmp_string.assign(tmp_p, p-tmp_p);
      // clear groupset_ first, if we did not use '+='
      if(!arg_to_be_added_)
        groupset_.clear();

      groupset_.insert(tmp_string);
    }
  }

  action set_docu_block
  {
    tmp_string.assign(tmp_p, p-tmp_p);
    if(tmp_string == "brief")
      cblock_ = &docuheader_;
    else if(tmp_string == "doc")
      cblock_ = &docubody_;
    else if(tmp_string == "extra")
      cblock_ = &docuextra_;
  }

  action set_docu_list
  {
    tmp_string.assign(tmp_p, p-tmp_p);
    //cerr << "docu_list: " << tmp_string << "\n";
    if(tmp_string == "params")
    {
      //cerr << "paramlist\n";
      clist_ = &param_list_;
    }
    else if(tmp_string == "return")
      clist_ = &return_list_;
  }

  action set_docu_list_map
  {
    tmp_string.assign(tmp_p, p-tmp_p);
    if(tmp_string == "fields")
      clistmap_ = &field_docu_;
  }

  # add line of a block to the cblock structure which can be one of
  # docublock, docuheader, docuextra or clist[] or clistmap[][]
  action add_block_line
  {
    if(match_at_level_[level_])
    {
      tmp_string.assign(tmp_p, p-tmp_p);
      string :: size_type found = tmp_string.find_first_not_of(" \t");
      if(found == string::npos || string(tmp_string.substr(found)) != "\"\"")
      {
        found = tmp_string.rfind("\"");
        if(found != string::npos && string(tmp_string.substr(found-1)) == "\"\"")
        {
          //cerr << "a1: " << tmp_string.substr(0, found-1) + '\n';
          (*cblock_).push_back(tmp_string.substr(0, found-1) + '\n');
          //cerr << "after a1: " << param_list_["detailed_data"][0];
        }
        else
        {
          //cerr << "a2: " << tmp_string + '\n';
          (*cblock_).push_back(tmp_string + '\n');
        }
      }
      opt = true;
    }
  }

  # set the cblock pointer to clist_[value]
  action set_block_from_list
  {
    if(match_at_level_[level_])
    {
      tmp_string.assign(tmp_p, p-tmp_p);
      //cerr << "ts:" << tmp_string << ":ets\n";//<< " " << tmp_p << "p: " << p << "\n";
      cblock_ = &((*clist_)[tmp_string]);
    }
  }

  # set the cblock pointer to clist_[value1][value2]
  action set_block_from_listmap
  {
    string s(tmp_p, p-tmp_p);
    cblock_ = &((*clistmap_)[tmp_string][s]);
  }

  action reg_glob
  {
    tmp_string.assign(tmp_p, p-tmp_p);
    //cerr << "glob: " << tmp_string << endl;
    // TODO: use globlist_map
    globlist_stack_.back().push_back(tmp_string);
  }

  # end of file
  EOF = 0;

  # end of line (CRLF, LF)
  EOL = '\r'? . '\n';

  # any not end of file character
  default = ^0;

  # identifier
  IDENT = [A-Z]+;

  # matlab identifier
  MIDENT = [A-Za-z][A-Za-z0-9_]*;

  # glob expression
  GLOB = (default - [\n \t;\{])+;

  # swallow till the end of line
  garbling := (default - '\n')* . EOL @{line++;fret;};

  add_param_docu_block = ('brief'|'doc'|'extra');
  add_param_docu_list = ('params'|'return');
  add_param_docu_list_map = ('fields');

  # matlab white space or comment
  MWSOC = (
           [ \t]
           | EOL @{line++;}
           | (('#'. any) -'##') @{ /*cerr << "l: " << line << "\n";*/ fhold;fcall garbling; }
          );
#  MWSOC = ( [ \t\n]);

  # equal sign or += sign
  EQ = ('+'? @{arg_to_be_added_ = true;} . '=');

  # end of a rule sign
  ENDRULE = ';' @{arg_to_be_added_ = false;};

  # line in documentation block
  docu_block_line :=
    (
     ( ( default - ["\n] )+ >{ if(opt) { tmp_p = p; } } )
     |
     EOL @(add_block_line) @{line++;}
     |
     '"""' @(add_block_line) @{fret;}
     |
     ('"'{1,2} . (default - ["\n])) @{ fhold; opt=false; }
    )**;

  # begin documentation block
  docu_block = ('"""'
                @{
                opt=true;
                if(!arg_to_be_added_)
                  (*cblock_).clear();
                fcall docu_block_line;
                }
               );

  # documentation list item (something like key => value)
  docu_list_item =
    (
     ( MIDENT
         >{tmp_p = p;}
         %(set_block_from_list)
     ) . MWSOC*
     . '=>' . MWSOC* . docu_block
    );

  docu_list = (
    (
     '('
     . ( MWSOC* . docu_list_item . MWSOC* . ',')*
     . MWSOC* . docu_list_item . MWSOC* . ')'
    )
    |
    docu_list_item
  );

  docu_list_map_item =
    (
     (MIDENT
       >{tmp_p = p;/* cerr << "dlmi\n";*/} %(string_tok)
     )
     . '.'
     .(MIDENT
       >{tmp_p = p;/* cerr << "dlmi2\n";*/} %(set_block_from_listmap)
      )
     . MWSOC*
     . '=>' . MWSOC* . docu_block
    );

  docu_list_map = (
    (
     '('
     . ( MWSOC* . docu_list_map_item . MWSOC* . ',')*
     . MWSOC* . docu_list_map_item . MWSOC* . ')'
    )
    |
    docu_list_map_item
  );

  globlist =
    (GLOB >{/*cerr << "glob_list";*/ tmp_p = p;} %(reg_glob))
    . (MWSOC* . [ \t]+ . MWSOC*
        . (GLOB >{/*cerr << "glob list2";*/ tmp_p = p;} %(reg_glob)))*;

  grouplist = (
    ((MIDENT >(st_tok)
             %(group_add))
     . ',')*
    .(MIDENT >(st_tok)
      %(group_add))
    );

  rules := (
    # glob list
    ('glob' .MWSOC* . '=' . MWSOC* . globlist . MWSOC* . '{')
      @{ check_glob_level_up(); fcall rules; }
    |
    (MWSOC)
    |
    # groups rule
    ('groups'. MWSOC* . EQ
     . MWSOC* . grouplist . MWSOC* . ENDRULE)
    |
    # add rule
    ('add(' %{/*cerr << "add:" << '\n';*/ tmp_p = p;}
     . (
        ((add_param_docu_block
            %(set_docu_block)
         )
         . ')' . MWSOC* . EQ
         . MWSOC* . docu_block . MWSOC* . ENDRULE )
        |
        ((add_param_docu_list
            %(set_docu_list)
         )
         . ')' . MWSOC* . EQ
         . MWSOC* :> docu_list . MWSOC* . ENDRULE )
        |
        ((add_param_docu_list_map
            %(set_docu_list_map)
         )
         . ')' . MWSOC* . EQ
         . MWSOC* :> docu_list_map . MWSOC* ENDRULE )
       )
    )
    |
    # go level down
    ('}') @{ go_level_down(); fret; }
    )*;

  main := (
           MWSOC
           | (
              (IDENT
                 >(st_tok)
                 %(string_tok)
              ). MWSOC* . ':=' . MWSOC*
              . (GLOB >st_tok @{/*TODO*/})
              . (MWSOC* . (GLOB >st_tok @{/*TODO*/}))*
              . MWSOC*
              . ';'
             )
           |
           '##' . [ \t]* . EOL @{ line++; /*cerr<<"-> rules\n";*/ fgoto rules; }
          )**;
}%%

void ConfFileScanner :: go_level_down()
{
  globlist_stack_.pop_back();
  globlist_stack_.back().clear();
  level_--;
}

bool ConfFileScanner :: check_for_match(int l, const char * str,
                                        bool match_path_sep)
{
  typedef GlobList :: iterator iterator;
  GlobList & gl = globlist_stack_[l];
  /*  cerr << "globlist at level" << l << "\n  ";
  for(unsigned int i = 0; i < gl.size(); ++i)
  {
    cerr << gl[i] << " ";
  }
  cerr << endl;*/
  iterator endit = gl.end();
  int flags = (match_path_sep? FNM_PATHNAME : 0);
  // cerr << "checking for match at level " << l <<" of string " << str << ":" << endl;
  for( iterator it = gl.begin(); it != endit; it++ )
  {
    // cerr << "with pattern: " << (*it).c_str() << endl;
    if(fnmatch((*it).c_str(), str, flags) == 0)
    {
      // cerr << "SUCCESS!" << endl;
      return true;
    }
  }
  return false;
}

bool ConfFileScanner :: check_glob_rec(int l, const string & s)
{
  string str;
  string :: size_type found;
  if(l == level_+1)
    return true;

  found = s.find("/"); // try to match dir in path
  while(found != string :: npos)
  {
    str = s.substr(0, found);
//    cerr << str << endl;
    if(check_for_match(l, str.c_str())
        && check_glob_rec(l+1, s.substr(found+1)))
      return true;

    found = s.find("/", found+1); // try to match more dirs
  }
  if(l == level_) // try also to match the entire string
  {
    if(check_for_match(l, s.c_str()))
      return true;
  }

  return false;
}

void ConfFileScanner :: check_glob_level_up()
{
  match_at_level_[level_+1] = check_glob_rec(0, filename_);
/*  cerr << "going one level up, match at this level is: "
       << (match_at_level_[level_+1] ? "true" : "false");
  cerr << endl;*/

  globlist_stack_.push_back(GlobList());
  level_++;
}

ConfFileScanner
:: ConfFileScanner(const std::string & filename)
 : line(1), have(0), top(0), opt(true),
   filename_(filename),
   confistream_(set_conffile()),
   level_(0),
   arg_to_be_added_(false)
{
  globlist_stack_.push_back(GlobList());
  match_at_level_[0] = true;
}

const char * ConfFileScanner ::
set_conffile()
{
  conffile_ = "doxygen/mtoc.conf";
  return conffile_.c_str();
}

int ConfFileScanner :: execute()
{
  //cerr << "execute\n";
  std::ios::sync_with_stdio(false);

  %% write init;

  /* Do the first read. */
  bool done = false;
  while ( !done )
  {
    char *p = buf + have;
    char *tmp_p = p;// *tmp_p2 = p;
    string tmp_string;
    int space = BUFSIZE - have;

    if ( space == 0 )
    {
      /* We filled up the buffer trying to scan a token. */
      cerr << "OUT OF BUFFER SPACE" << endl;
      exit(1);
    }

    confistream_.read( p, space );
    int len = confistream_.gcount();
    char *pe = p + len;
    char *rpe = pe;
    char *eof = 0;

    /* If we see eof then append the EOF char. */
    if ( confistream_.eof() )
    {
      eof = pe;
      done = true;
    }
    else
    {
      /* Find the last semicolon by searching backwards. This
       * is where we will stop processing on this iteration. */
      while ( *pe!= ';' && pe > p )
        pe--;
    }

    %% write exec;

    /* Check if we failed. */
    if ( cs == ConfFileScanner_error )
    {
      /* Machine failed before finding a token. */
      cerr << "in conffile: PARSE ERROR in line " << line << endl;
      exit(1);
    }

    have = rpe - pe;
    /* cerr << "memmove by " << have << "bytes\n";*/
    memmove( buf, pe, have );
  }

  return 0;
}
/* vim: set et sw=2 ft=ragel: */


Remove usage.rl

Remove usage.rl


Use alpha for upper & lower case chars.

Use alpha for upper & lower case chars.

Sourcify.require_rb('proc', 'scanner', 'extensions')

module Sourcify
  module Proc
    module Scanner #:nodoc:all

%%{
  machine proc_scanner;

  kw_class  = 'class';
  kw_module = 'module';
  kw_def    = 'def';
  kw_begin  = 'begin';
  kw_case   = 'case';
  kw_if     = 'if';
  kw_unless = 'unless';
  kw_do     = 'do';
  kw_then   = 'then';
  kw_for    = 'for';
  kw_while  = 'while';
  kw_until  = 'until';
  kw_end    = 'end';

  const   = upper . (alnum | '_')* . ('::' . alpha . (alnum | '_')*)*;
  var     = (lower | '_') . (alnum | '_')*;
  symbol  = ':' . (var | const);
  assoc   = '=>';
  assgn   = '=';
  comma   = ',';
  label   = (var | const) . ':';

  lparen  = '(';
  rparen  = ')';
  lbrace  = '{';
  rbrace  = '}';
  smcolon = ';';
  newline = '\n';
  ospaces = space* -- newline;
  mspaces = space+ -- newline;

  ## MACHINE >> New statement
  new_statement := |*

    kw_for | kw_while | kw_until => {
      push(:kw_do_alias2, ts, te)
      increment_counter(:do_end, 0..1)
      fgoto main;
    };

    (
      kw_class | kw_module | kw_def | kw_begin | kw_case | kw_if | kw_unless |
      kw_class . ospaces . '<<' . ospaces . ^newline+
    ) => {
      push(:kw_do_alias1, ts, te)
      increment_counter(:do_end, 1)
      fgoto main;
    };

    ospaces => { push(:space, ts, te) };
    any => { fhold; fgoto main; };

  *|;

  ## MACHINE >> One-liner comment
  per_line_comment := |*
    ^newline* => {
      push(:comment, ts.pred, te)
      fgoto main;
    };
  *|;

  ## MACHINE >> Block comment
  block_comment := |*
    any* . newline . '=end' . ospaces . ^newline* => {
      unless push_comment(ts, te)
        fgoto main;
      end
    };
  *|;

  ## MACHINE >> Heredoc
  heredoc := |*
    ^newline* . newline . ospaces . ^newline+ => {
      unless push_heredoc(ts, te)
        fgoto main;
      end
    };
  *|;

  ## MACHINE >> Double quote string
  double_quote_str := |*
    delimiter = [\~\`\!\@\#\$\%\^\&\*\)\-\_\+\=\}\]\:\;\'\"\>\,\.\?\/\|\\];
    (^delimiter* . delimiter) -- ('\\' . delimiter) => {
      unless push_dstring(ts, te)
        fgoto main;
      end
    };
  *|;

  ## MACHINE >> Main
  main := |*

    ## == Start/end of do..end block

    kw_do => {
      push(:kw_do, ts, te)
      increment_counter(:do_end)
      fgoto new_statement;
    };

    kw_end => {
      push(:kw_end, ts, te)
      decrement_counter(:do_end)
    };

    ## == Start/end of {...} block

    lbrace => {
      push(:lbrace, ts, te)
      increment_counter(:brace)
    };

    rbrace => {
      push(:rbrace, ts, te)
      decrement_counter(:brace)
    };

    assoc => {
      push(:assoc, ts, te)
      fix_counter_false_start(:brace)
    };

    label => {
      push_label(ts, te)
      fix_counter_false_start(:brace)
    };

    ## == New statement

    newline => {
      push(:newline, ts, te)
      increment_lineno
      fgoto new_statement;
    };

    smcolon | lparen | assgn | kw_then | comma => {
      push(:newline_alias, ts, te)
      fgoto new_statement;
    };

    ## == Comment

    '#' => {
      fgoto per_line_comment;
    };

    newline . '=begin' . ospaces . (ospaces . ^newline+)* . newline => {
      push_comment(ts, te)
      increment_lineno
      fgoto block_comment;
    };

    ## == Misc

    var     => { push(:var, ts, te) };
    const   => { push(:const, ts, te) };
    symbol  => { push(:symbol, ts, te) };
    mspaces => { push(:space, ts, te) };
    any     => { push(:any, ts, te) };

    ## == Heredoc

    ('<<' | '<<-') . ["']? . (const | var) . ["']? . newline => {
      push_heredoc(ts, te)
      increment_lineno
      fgoto heredoc;
    };

    ## == Single quote strings

    sqs1  = "'" . ([^\']* | ([^\'\\]*[\\][\'][^\']*)*) . '\\\\'* . "'";
    sqs2  = '~' . ([^\~]* | ([^\~\\]*[\\][\~][^\~]*)*) . '\\\\'* . '~';
    sqs3  = '`' . ([^\`]* | ([^\`\\]*[\\][\`][^\`]*)*) . '\\\\'* . '`';
    sqs4  = '!' . ([^\!]* | ([^\!\\]*[\\][\!][^\!]*)*) . '\\\\'* . '!';
    sqs5  = '@' . ([^\@]* | ([^\@\\]*[\\][\@][^\@]*)*) . '\\\\'* . '@';
    sqs6  = '#' . ([^\#]* | ([^\#\\]*[\\][\#][^\#]*)*) . '\\\\'* . '#';
    sqs7  = '$' . ([^\$]* | ([^\$\\]*[\\][\$][^\$]*)*) . '\\\\'* . '$';
    sqs8  = '%' . ([^\%]* | ([^\%\\]*[\\][\%][^\%]*)*) . '\\\\'* . '%';
    sqs9  = '^' . ([^\^]* | ([^\^\\]*[\\][\^][^\^]*)*) . '\\\\'* . '^';
    sqs10 = '&' . ([^\&]* | ([^\&\\]*[\\][\&][^\&]*)*) . '\\\\'* . '&';
    sqs11 = '*' . ([^\*]* | ([^\*\\]*[\\][\*][^\*]*)*) . '\\\\'* . '*';
    sqs12 = '-' . ([^\-]* | ([^\-\\]*[\\][\-][^\-]*)*) . '\\\\'* . '-';
    sqs13 = '_' . ([^\_]* | ([^\_\\]*[\\][\_][^\_]*)*) . '\\\\'* . '_';
    sqs14 = '+' . ([^\+]* | ([^\+\\]*[\\][\+][^\+]*)*) . '\\\\'* . '+';
    sqs15 = '=' . ([^\=]* | ([^\=\\]*[\\][\=][^\=]*)*) . '\\\\'* . '=';
    sqs16 = '<' . ([^\>]* | ([^\>\\]*[\\][\>][^\>]*)*) . '\\\\'* . '>';
    sqs17 = '|' . ([^\|]* | ([^\|\\]*[\\][\|][^\|]*)*) . '\\\\'* . '|';
    sqs18 = ':' . ([^\:]* | ([^\:\\]*[\\][\:][^\:]*)*) . '\\\\'* . ':';
    sqs19 = ';' . ([^\;]* | ([^\;\\]*[\\][\;][^\;]*)*) . '\\\\'* . ';';
    sqs20 = '"' . ([^\"]* | ([^\"\\]*[\\][\"][^\"]*)*) . '\\\\'* . '"';
    sqs21 = ',' . ([^\,]* | ([^\,\\]*[\\][\,][^\,]*)*) . '\\\\'* . ',';
    sqs22 = '.' . ([^\.]* | ([^\.\\]*[\\][\.][^\.]*)*) . '\\\\'* . '.';
    sqs23 = '?' . ([^\?]* | ([^\?\\]*[\\][\?][^\?]*)*) . '\\\\'* . '?';
    sqs24 = '/' . ([^\/]* | ([^\/\\]*[\\][\/][^\/]*)*) . '\\\\'* . '/';
    sqs25 = '{' . ([^\}]* | ([^\}\\]*[\\][\}][^\}]*)*) . '\\\\'* . '}';
    sqs26 = '[' . ([^\]]* | ([^\]\\]*[\\][\]][^\]]*)*) . '\\\\'* . ']';
    sqs27 = '(' . ([^\)]* | ([^\)\\]*[\\][\)][^\)]*)*) . '\\\\'* . ')';
    sqs28 = '\\' . ([^\\]* | ([^\)]*[\\][\\][^\\]*)*) . '\\';

    sqm = ('%q' | '%w'); (
      sqs1        | sqm . sqs1  | sqm . sqs2  | sqm . sqs3  | sqm . sqs4  |
      sqm . sqs5  | sqm . sqs6  | sqm . sqs7  | sqm . sqs8  | sqm . sqs9  |
      sqm . sqs10 | sqm . sqs11 | sqm . sqs12 | sqm . sqs13 | sqm . sqs14 |
      sqm . sqs15 | sqm . sqs16 | sqm . sqs17 | sqm . sqs18 | sqm . sqs19 |
      sqm . sqs20 | sqm . sqs21 | sqm . sqs22 | sqm . sqs23 | sqm . sqs24 |
      sqm . sqs25 | sqm . sqs26 | sqm . sqs27 | sqm . sqs28
    ) => {
      push(:sstring, ts, te)
    };

    ## == Double quote strings

    dqs1  = '"' . (([^\"]* | ([^\"\\]*[\\][\"][^\"]*)*) -- '#{') . '\\\\'* . ('"' | '#{');
    dqs2  = '`' . (([^\`]* | ([^\`\\]*[\\][\`][^\`]*)*) -- '#{') . '\\\\'* . ('`' | '#{');
    dqs3  = '/' . (([^\/]* | ([^\/\\]*[\\][\/][^\/]*)*) -- '#{') . '\\\\'* . ('/' | '#{');
    dqs4  = "'" . (([^\']* | ([^\'\\]*[\\][\'][^\']*)*) -- '#{') . '\\\\'* . ("'" | '#{');
    dqs5  = '~' . (([^\~]* | ([^\~\\]*[\\][\~][^\~]*)*) -- '#{') . '\\\\'* . ('~' | '#{');
    dqs6  = '!' . (([^\!]* | ([^\!\\]*[\\][\!][^\!]*)*) -- '#{') . '\\\\'* . ('!' | '#{');
    dqs7  = '@' . (([^\@]* | ([^\@\\]*[\\][\@][^\@]*)*) -- '#{') . '\\\\'* . ('@' | '#{');
    dqs8  = '#' . (([^\#]* | ([^\#\\]*[\\][\#][^\#]*)*) -- '#{') . '\\\\'* . ('#' | '#{');
    dqs9  = '$' . (([^\$]* | ([^\$\\]*[\\][\$][^\$]*)*) -- '#{') . '\\\\'* . ('$' | '#{');
    dqs10 = '%' . (([^\%]* | ([^\%\\]*[\\][\%][^\%]*)*) -- '#{') . '\\\\'* . ('%' | '#{');
    dqs11 = '^' . (([^\^]* | ([^\^\\]*[\\][\^][^\^]*)*) -- '#{') . '\\\\'* . ('^' | '#{');
    dqs12 = '&' . (([^\&]* | ([^\&\\]*[\\][\&][^\&]*)*) -- '#{') . '\\\\'* . ('&' | '#{');
    dqs13 = '*' . (([^\*]* | ([^\*\\]*[\\][\*][^\*]*)*) -- '#{') . '\\\\'* . ('*' | '#{');
    dqs14 = '-' . (([^\-]* | ([^\-\\]*[\\][\-][^\-]*)*) -- '#{') . '\\\\'* . ('-' | '#{');
    dqs15 = '_' . (([^\_]* | ([^\_\\]*[\\][\_][^\_]*)*) -- '#{') . '\\\\'* . ('_' | '#{');
    dqs16 = '+' . (([^\+]* | ([^\+\\]*[\\][\+][^\+]*)*) -- '#{') . '\\\\'* . ('+' | '#{');
    dqs17 = '=' . (([^\=]* | ([^\=\\]*[\\][\=][^\=]*)*) -- '#{') . '\\\\'* . ('=' | '#{');
    dqs18 = '<' . (([^\>]* | ([^\>\\]*[\\][\>][^\>]*)*) -- '#{') . '\\\\'* . ('>' | '#{');
    dqs19 = '|' . (([^\|]* | ([^\|\\]*[\\][\|][^\|]*)*) -- '#{') . '\\\\'* . ('|' | '#{');
    dqs20 = ':' . (([^\:]* | ([^\:\\]*[\\][\:][^\:]*)*) -- '#{') . '\\\\'* . (':' | '#{');
    dqs21 = ';' . (([^\;]* | ([^\;\\]*[\\][\;][^\;]*)*) -- '#{') . '\\\\'* . (';' | '#{');
    dqs22 = ',' . (([^\,]* | ([^\,\\]*[\\][\,][^\,]*)*) -- '#{') . '\\\\'* . (',' | '#{');
    dqs23 = '.' . (([^\.]* | ([^\.\\]*[\\][\.][^\.]*)*) -- '#{') . '\\\\'* . ('.' | '#{');
    dqs24 = '?' . (([^\?]* | ([^\?\\]*[\\][\?][^\?]*)*) -- '#{') . '\\\\'* . ('?' | '#{');
    dqs25 = '{' . (([^\}]* | ([^\}\\]*[\\][\}][^\}]*)*) -- '#{') . '\\\\'* . ('}' | '#{');
    dqs26 = '[' . (([^\]]* | ([^\]\\]*[\\][\]][^\]]*)*) -- '#{') . '\\\\'* . (']' | '#{');
    dqs27 = '(' . (([^\)]* | ([^\)\\]*[\\][\)][^\)]*)*) -- '#{') . '\\\\'* . (')' | '#{');
    dqs28 = '\\' . (([^\\]* | ([^\\]*[\\][\\][^\\]*)*) -- '#{') . ('\\' | '#{');

    dqm = ('%Q' | '%W' | '%x' | '%r' | '%'); (
      dqs1        | dqs2        | dqs3        | dqm . dqs1  | dqm . dqs2  |
      dqm . dqs3  | dqm . dqs4  | dqm . dqs5  | dqm . dqs6  | dqm . dqs7  |
      dqm . dqs8  | dqm . dqs9  | dqm . dqs10 | dqm . dqs11 | dqm . dqs12 |
      dqm . dqs13 | dqm . dqs14 | dqm . dqs15 | dqm . dqs16 | dqm . dqs17 |
      dqm . dqs18 | dqm . dqs19 | dqm . dqs20 | dqm . dqs21 | dqm . dqs22 |
      dqm . dqs23 | dqm . dqs24 | dqm . dqs25 | dqm . dqs26 | dqm . dqs27 |
      dqm . dqs28
    ) => {
      if push_dstring(ts, te)
        fgoto double_quote_str;
      end
    };

  *|;

}%%
%% write data;

      extend Scanner::Extensions

      def self.execute!
        data = @data
        eof = data.length
        %% write init;
        %% write exec;
      end

    end
  end
end

Sourcify.require_rb('proc', 'scanner', 'extensions')

module Sourcify
  module Proc
    module Scanner #:nodoc:all

%%{
  machine proc_scanner;

  kw_class  = 'class';
  kw_module = 'module';
  kw_def    = 'def';
  kw_begin  = 'begin';
  kw_case   = 'case';
  kw_if     = 'if';
  kw_unless = 'unless';
  kw_do     = 'do';
  kw_then   = 'then';
  kw_for    = 'for';
  kw_while  = 'while';
  kw_until  = 'until';
  kw_end    = 'end';

  const   = upper . (alnum | '_')* . ('::' . (upper | lower) . (alnum | '_')*)*;
  var     = (lower | '_') . (alnum | '_')*;
  symbol  = ':' . (var | const);
  assoc   = '=>';
  assgn   = '=';
  comma   = ',';
  label   = (var | const) . ':';

  lparen  = '(';
  rparen  = ')';
  lbrace  = '{';
  rbrace  = '}';
  smcolon = ';';
  newline = '\n';
  ospaces = space* -- newline;
  mspaces = space+ -- newline;

  ## MACHINE >> New statement
  new_statement := |*

    kw_for | kw_while | kw_until => {
      push(:kw_do_alias2, ts, te)
      increment_counter(:do_end, 0..1)
      fgoto main;
    };

    (
      kw_class | kw_module | kw_def | kw_begin | kw_case | kw_if | kw_unless |
      kw_class . ospaces . '<<' . ospaces . ^newline+
    ) => {
      push(:kw_do_alias1, ts, te)
      increment_counter(:do_end, 1)
      fgoto main;
    };

    ospaces => { push(:space, ts, te) };
    any => { fhold; fgoto main; };

  *|;

  ## MACHINE >> One-liner comment
  per_line_comment := |*
    ^newline* => {
      push(:comment, ts.pred, te)
      fgoto main;
    };
  *|;

  ## MACHINE >> Block comment
  block_comment := |*
    any* . newline . '=end' . ospaces . ^newline* => {
      unless push_comment(ts, te)
        fgoto main;
      end
    };
  *|;

  ## MACHINE >> Heredoc
  heredoc := |*
    ^newline* . newline . ospaces . ^newline+ => {
      unless push_heredoc(ts, te)
        fgoto main;
      end
    };
  *|;

  ## MACHINE >> Double quote string
  double_quote_str := |*
    delimiter = [\~\`\!\@\#\$\%\^\&\*\)\-\_\+\=\}\]\:\;\'\"\>\,\.\?\/\|\\];
    (^delimiter* . delimiter) -- ('\\' . delimiter) => {
      unless push_dstring(ts, te)
        fgoto main;
      end
    };
  *|;

  ## MACHINE >> Main
  main := |*

    ## == Start/end of do..end block

    kw_do => {
      push(:kw_do, ts, te)
      increment_counter(:do_end)
      fgoto new_statement;
    };

    kw_end => {
      push(:kw_end, ts, te)
      decrement_counter(:do_end)
    };

    ## == Start/end of {...} block

    lbrace => {
      push(:lbrace, ts, te)
      increment_counter(:brace)
    };

    rbrace => {
      push(:rbrace, ts, te)
      decrement_counter(:brace)
    };

    assoc => {
      push(:assoc, ts, te)
      fix_counter_false_start(:brace)
    };

    label => {
      push_label(ts, te)
      fix_counter_false_start(:brace)
    };

    ## == New statement

    newline => {
      push(:newline, ts, te)
      increment_lineno
      fgoto new_statement;
    };

    smcolon | lparen | assgn | kw_then | comma => {
      push(:newline_alias, ts, te)
      fgoto new_statement;
    };

    ## == Comment

    '#' => {
      fgoto per_line_comment;
    };

    newline . '=begin' . ospaces . (ospaces . ^newline+)* . newline => {
      push_comment(ts, te)
      increment_lineno
      fgoto block_comment;
    };

    ## == Misc

    var     => { push(:var, ts, te) };
    const   => { push(:const, ts, te) };
    symbol  => { push(:symbol, ts, te) };
    mspaces => { push(:space, ts, te) };
    any     => { push(:any, ts, te) };

    ## == Heredoc

    ('<<' | '<<-') . ["']? . (const | var) . ["']? . newline => {
      push_heredoc(ts, te)
      increment_lineno
      fgoto heredoc;
    };

    ## == Single quote strings

    sqs1  = "'" . ([^\']* | ([^\'\\]*[\\][\'][^\']*)*) . '\\\\'* . "'";
    sqs2  = '~' . ([^\~]* | ([^\~\\]*[\\][\~][^\~]*)*) . '\\\\'* . '~';
    sqs3  = '`' . ([^\`]* | ([^\`\\]*[\\][\`][^\`]*)*) . '\\\\'* . '`';
    sqs4  = '!' . ([^\!]* | ([^\!\\]*[\\][\!][^\!]*)*) . '\\\\'* . '!';
    sqs5  = '@' . ([^\@]* | ([^\@\\]*[\\][\@][^\@]*)*) . '\\\\'* . '@';
    sqs6  = '#' . ([^\#]* | ([^\#\\]*[\\][\#][^\#]*)*) . '\\\\'* . '#';
    sqs7  = '$' . ([^\$]* | ([^\$\\]*[\\][\$][^\$]*)*) . '\\\\'* . '$';
    sqs8  = '%' . ([^\%]* | ([^\%\\]*[\\][\%][^\%]*)*) . '\\\\'* . '%';
    sqs9  = '^' . ([^\^]* | ([^\^\\]*[\\][\^][^\^]*)*) . '\\\\'* . '^';
    sqs10 = '&' . ([^\&]* | ([^\&\\]*[\\][\&][^\&]*)*) . '\\\\'* . '&';
    sqs11 = '*' . ([^\*]* | ([^\*\\]*[\\][\*][^\*]*)*) . '\\\\'* . '*';
    sqs12 = '-' . ([^\-]* | ([^\-\\]*[\\][\-][^\-]*)*) . '\\\\'* . '-';
    sqs13 = '_' . ([^\_]* | ([^\_\\]*[\\][\_][^\_]*)*) . '\\\\'* . '_';
    sqs14 = '+' . ([^\+]* | ([^\+\\]*[\\][\+][^\+]*)*) . '\\\\'* . '+';
    sqs15 = '=' . ([^\=]* | ([^\=\\]*[\\][\=][^\=]*)*) . '\\\\'* . '=';
    sqs16 = '<' . ([^\>]* | ([^\>\\]*[\\][\>][^\>]*)*) . '\\\\'* . '>';
    sqs17 = '|' . ([^\|]* | ([^\|\\]*[\\][\|][^\|]*)*) . '\\\\'* . '|';
    sqs18 = ':' . ([^\:]* | ([^\:\\]*[\\][\:][^\:]*)*) . '\\\\'* . ':';
    sqs19 = ';' . ([^\;]* | ([^\;\\]*[\\][\;][^\;]*)*) . '\\\\'* . ';';
    sqs20 = '"' . ([^\"]* | ([^\"\\]*[\\][\"][^\"]*)*) . '\\\\'* . '"';
    sqs21 = ',' . ([^\,]* | ([^\,\\]*[\\][\,][^\,]*)*) . '\\\\'* . ',';
    sqs22 = '.' . ([^\.]* | ([^\.\\]*[\\][\.][^\.]*)*) . '\\\\'* . '.';
    sqs23 = '?' . ([^\?]* | ([^\?\\]*[\\][\?][^\?]*)*) . '\\\\'* . '?';
    sqs24 = '/' . ([^\/]* | ([^\/\\]*[\\][\/][^\/]*)*) . '\\\\'* . '/';
    sqs25 = '{' . ([^\}]* | ([^\}\\]*[\\][\}][^\}]*)*) . '\\\\'* . '}';
    sqs26 = '[' . ([^\]]* | ([^\]\\]*[\\][\]][^\]]*)*) . '\\\\'* . ']';
    sqs27 = '(' . ([^\)]* | ([^\)\\]*[\\][\)][^\)]*)*) . '\\\\'* . ')';
    sqs28 = '\\' . ([^\\]* | ([^\)]*[\\][\\][^\\]*)*) . '\\';

    sqm = ('%q' | '%w'); (
      sqs1        | sqm . sqs1  | sqm . sqs2  | sqm . sqs3  | sqm . sqs4  |
      sqm . sqs5  | sqm . sqs6  | sqm . sqs7  | sqm . sqs8  | sqm . sqs9  |
      sqm . sqs10 | sqm . sqs11 | sqm . sqs12 | sqm . sqs13 | sqm . sqs14 |
      sqm . sqs15 | sqm . sqs16 | sqm . sqs17 | sqm . sqs18 | sqm . sqs19 |
      sqm . sqs20 | sqm . sqs21 | sqm . sqs22 | sqm . sqs23 | sqm . sqs24 |
      sqm . sqs25 | sqm . sqs26 | sqm . sqs27 | sqm . sqs28
    ) => {
      push(:sstring, ts, te)
    };

    ## == Double quote strings

    dqs1  = '"' . (([^\"]* | ([^\"\\]*[\\][\"][^\"]*)*) -- '#{') . '\\\\'* . ('"' | '#{');
    dqs2  = '`' . (([^\`]* | ([^\`\\]*[\\][\`][^\`]*)*) -- '#{') . '\\\\'* . ('`' | '#{');
    dqs3  = '/' . (([^\/]* | ([^\/\\]*[\\][\/][^\/]*)*) -- '#{') . '\\\\'* . ('/' | '#{');
    dqs4  = "'" . (([^\']* | ([^\'\\]*[\\][\'][^\']*)*) -- '#{') . '\\\\'* . ("'" | '#{');
    dqs5  = '~' . (([^\~]* | ([^\~\\]*[\\][\~][^\~]*)*) -- '#{') . '\\\\'* . ('~' | '#{');
    dqs6  = '!' . (([^\!]* | ([^\!\\]*[\\][\!][^\!]*)*) -- '#{') . '\\\\'* . ('!' | '#{');
    dqs7  = '@' . (([^\@]* | ([^\@\\]*[\\][\@][^\@]*)*) -- '#{') . '\\\\'* . ('@' | '#{');
    dqs8  = '#' . (([^\#]* | ([^\#\\]*[\\][\#][^\#]*)*) -- '#{') . '\\\\'* . ('#' | '#{');
    dqs9  = '$' . (([^\$]* | ([^\$\\]*[\\][\$][^\$]*)*) -- '#{') . '\\\\'* . ('$' | '#{');
    dqs10 = '%' . (([^\%]* | ([^\%\\]*[\\][\%][^\%]*)*) -- '#{') . '\\\\'* . ('%' | '#{');
    dqs11 = '^' . (([^\^]* | ([^\^\\]*[\\][\^][^\^]*)*) -- '#{') . '\\\\'* . ('^' | '#{');
    dqs12 = '&' . (([^\&]* | ([^\&\\]*[\\][\&][^\&]*)*) -- '#{') . '\\\\'* . ('&' | '#{');
    dqs13 = '*' . (([^\*]* | ([^\*\\]*[\\][\*][^\*]*)*) -- '#{') . '\\\\'* . ('*' | '#{');
    dqs14 = '-' . (([^\-]* | ([^\-\\]*[\\][\-][^\-]*)*) -- '#{') . '\\\\'* . ('-' | '#{');
    dqs15 = '_' . (([^\_]* | ([^\_\\]*[\\][\_][^\_]*)*) -- '#{') . '\\\\'* . ('_' | '#{');
    dqs16 = '+' . (([^\+]* | ([^\+\\]*[\\][\+][^\+]*)*) -- '#{') . '\\\\'* . ('+' | '#{');
    dqs17 = '=' . (([^\=]* | ([^\=\\]*[\\][\=][^\=]*)*) -- '#{') . '\\\\'* . ('=' | '#{');
    dqs18 = '<' . (([^\>]* | ([^\>\\]*[\\][\>][^\>]*)*) -- '#{') . '\\\\'* . ('>' | '#{');
    dqs19 = '|' . (([^\|]* | ([^\|\\]*[\\][\|][^\|]*)*) -- '#{') . '\\\\'* . ('|' | '#{');
    dqs20 = ':' . (([^\:]* | ([^\:\\]*[\\][\:][^\:]*)*) -- '#{') . '\\\\'* . (':' | '#{');
    dqs21 = ';' . (([^\;]* | ([^\;\\]*[\\][\;][^\;]*)*) -- '#{') . '\\\\'* . (';' | '#{');
    dqs22 = ',' . (([^\,]* | ([^\,\\]*[\\][\,][^\,]*)*) -- '#{') . '\\\\'* . (',' | '#{');
    dqs23 = '.' . (([^\.]* | ([^\.\\]*[\\][\.][^\.]*)*) -- '#{') . '\\\\'* . ('.' | '#{');
    dqs24 = '?' . (([^\?]* | ([^\?\\]*[\\][\?][^\?]*)*) -- '#{') . '\\\\'* . ('?' | '#{');
    dqs25 = '{' . (([^\}]* | ([^\}\\]*[\\][\}][^\}]*)*) -- '#{') . '\\\\'* . ('}' | '#{');
    dqs26 = '[' . (([^\]]* | ([^\]\\]*[\\][\]][^\]]*)*) -- '#{') . '\\\\'* . (']' | '#{');
    dqs27 = '(' . (([^\)]* | ([^\)\\]*[\\][\)][^\)]*)*) -- '#{') . '\\\\'* . (')' | '#{');
    dqs28 = '\\' . (([^\\]* | ([^\\]*[\\][\\][^\\]*)*) -- '#{') . ('\\' | '#{');

    dqm = ('%Q' | '%W' | '%x' | '%r' | '%'); (
      dqs1        | dqs2        | dqs3        | dqm . dqs1  | dqm . dqs2  |
      dqm . dqs3  | dqm . dqs4  | dqm . dqs5  | dqm . dqs6  | dqm . dqs7  |
      dqm . dqs8  | dqm . dqs9  | dqm . dqs10 | dqm . dqs11 | dqm . dqs12 |
      dqm . dqs13 | dqm . dqs14 | dqm . dqs15 | dqm . dqs16 | dqm . dqs17 |
      dqm . dqs18 | dqm . dqs19 | dqm . dqs20 | dqm . dqs21 | dqm . dqs22 |
      dqm . dqs23 | dqm . dqs24 | dqm . dqs25 | dqm . dqs26 | dqm . dqs27 |
      dqm . dqs28
    ) => {
      if push_dstring(ts, te)
        fgoto double_quote_str;
      end
    };

  *|;

}%%
%% write data;

      extend Scanner::Extensions

      def self.execute!
        data = @data
        eof = data.length
        %% write init;
        %% write exec;
      end

    end
  end
end