XPath idents can't start with a star.

XPath idents can't start with a star.

That is, names such as "*foo" are not valid. This was most likely a typo in the
first place.

%%machine xpath_lexer; # %

module Oga
  module XPath
    ##
    # Lexer for turning XPath expressions into a set of tokens. Tokens are
    # returned as arrays with every array having two values:
    #
    # 1. The token type as a symbol
    # 2. The token value or nil if there is no value
    #
    # Basic usage of this lexer is as following:
    #
    #     lexer  = Oga::XPath::Lexer.new('//foo/bar')
    #     tokens = lexer.lex
    #
    # Alternatively you can stream tokens instead of returning them as a whole:
    #
    #     lexer = Oga::XPath::Lexer.new('//foo/bar')
    #
    #     lexer.advance do |type, value|
    #
    #     end
    #
    # Unlike the XML lexer the XPath lexer does not support IO instances, it can
    # only lex strings.
    #
    # ## Thread Safety
    #
    # This class keeps track of an internal state. As a result it's not safe to
    # share a single instance between multiple threads. However, you're free to
    # use separate instances per thread as there is no global (= class level)
    # shared state.
    #
    class Lexer
      %% write data;

      # % fix highlight

      ##
      # Maps certain XPath axes written in their short form to their long form
      # equivalents.
      #
      # @return [Hash]
      #
      AXIS_MAPPING = {
        '@'  => 'attribute',
        '//' => 'descendant-or-self',
        '..' => 'parent',
        '.'  => 'self'
      }

      ##
      # Axes that require a separate `node()` call to be emitted.
      #
      # @return [Array]
      #
      AXIS_EMIT_NODE = %w{descendant-or-self parent self}

      ##
      # Axes that require an extra T_SLASH token to be emitted.
      #
      # @return [Array]
      #
      AXIS_EMIT_EXTRA_SLASH = %w{descendant-or-self}

      ##
      # @param [String] data The data to lex.
      #
      def initialize(data)
        @data = data
      end

      ##
      # Gathers all the tokens for the input and returns them as an Array.
      #
      # @see [#advance]
      # @return [Array]
      #
      def lex
        tokens = []

        advance do |type, value|
          tokens << [type, value]
        end

        return tokens
      end

      ##
      # Advances through the input and generates the corresponding tokens. Each
      # token is yielded to the supplied block.
      #
      # Each token is an Array in the following format:
      #
      #     [TYPE, VALUE]
      #
      # The type is a symbol, the value is either nil or a String.
      #
      # This method stores the supplied block in `@block` and resets it after
      # the lexer loop has finished.
      #
      # @see [#add_token]
      #
      def advance(&block)
        @block = block

        data  = @data # saves ivar lookups while lexing.
        ts    = nil
        te    = nil
        stack = []
        top   = 0
        cs    = self.class.xpath_lexer_start
        act   = 0
        eof   = @data.bytesize
        p     = 0
        pe    = eof

        _xpath_lexer_eof_trans          = self.class.send(:_xpath_lexer_eof_trans)
        _xpath_lexer_from_state_actions = self.class.send(:_xpath_lexer_from_state_actions)
        _xpath_lexer_index_offsets      = self.class.send(:_xpath_lexer_index_offsets)
        _xpath_lexer_indicies           = self.class.send(:_xpath_lexer_indicies)
        _xpath_lexer_key_spans          = self.class.send(:_xpath_lexer_key_spans)
        _xpath_lexer_to_state_actions   = self.class.send(:_xpath_lexer_to_state_actions)
        _xpath_lexer_trans_actions      = self.class.send(:_xpath_lexer_trans_actions)
        _xpath_lexer_trans_keys         = self.class.send(:_xpath_lexer_trans_keys)
        _xpath_lexer_trans_targs        = self.class.send(:_xpath_lexer_trans_targs)

        %% write exec;

        # % fix highlight
      ensure
        @block = nil
      end

      private

      ##
      # Emits a token of which the value is based on the supplied start/stop
      # position.
      #
      # @param [Symbol] type The token type.
      # @param [Fixnum] start
      # @param [Fixnum] stop
      #
      # @see [#text]
      # @see [#add_token]
      #
      def emit(type, start, stop)
        value = slice_input(start, stop)

        add_token(type, value)
      end

      ##
      # Returns the text between the specified start and stop position.
      #
      # @param [Fixnum] start
      # @param [Fixnum] stop
      # @return [String]
      #
      def slice_input(start, stop)
        return @data.byteslice(start, stop - start)
      end

      ##
      # Yields a new token to the supplied block.
      #
      # @param [Symbol] type The token type.
      # @param [String] value The token value.
      #
      # @yieldparam [Symbol] type
      # @yieldparam [String|NilClass] value
      #
      def add_token(type, value = nil)
        @block.call(type, value)
      end

      %%{
        getkey (data.getbyte(p) || 0);

        whitespace = [\n\t ];

        slash  = '/' @{ add_token(:T_SLASH) };
        lparen = '(' @{ add_token(:T_LPAREN) };
        rparen = ')' @{ add_token(:T_RPAREN) };
        comma  = ',' @{ add_token(:T_COMMA) };
        colon  = ':' @{ add_token(:T_COLON) };
        lbrack = '[' @{ add_token(:T_LBRACK) };
        rbrack = ']' @{ add_token(:T_RBRACK) };

        # Identifiers
        #
        # Identifiers are used for element names, namespaces, attribute names,
        # etc. Identifiers have to start with a letter.

        identifier = '*' | [a-zA-Z_]+ [a-zA-Z\-_0-9]*;

        action emit_identifier {
          emit(:T_IDENT, ts, te)
        }

        # Numbers
        #
        # XPath expressions can contain both integers and floats. The W3
        # specification treats these both as the same type of number. Oga
        # instead lexes them separately so that we can convert the values to
        # the corresponding Ruby types (Fixnum and Float).

        integer = ('-' | '+')* digit+;
        float   = ('-' | '+')* digit+ ('.' digit+)*;

        action emit_integer {
          value = slice_input(ts, te).to_i

          add_token(:T_INT, value)
        }

        action emit_float {
          value = slice_input(ts, te).to_f

          add_token(:T_FLOAT, value)
        }

        # Strings
        #
        # Strings can be single or double quoted. They are mainly used for
        # attribute values.
        #
        dquote = '"';
        squote = "'";

        string_dquote = (dquote ^dquote* dquote);
        string_squote = (squote ^squote* squote);

        string = string_dquote | string_squote;

        action emit_string {
          emit(:T_STRING, ts + 1, te - 1)
        }

        # Full Axes
        #
        # XPath axes in their full syntax.
        #
        axis_full = ('ancestor'
          | 'ancestor-or-self'
          | 'attribute'
          | 'child'
          | 'descendant'
          | 'descendant-or-self'
          | 'following'
          | 'following-sibling'
          | 'namespace'
          | 'parent'
          | 'preceding'
          | 'preceding-sibling'
          | 'self') '::';

        action emit_axis_full {
          emit(:T_AXIS, ts, te - 2)
        }

        # Short Axes
        #
        # XPath axes in their abbreviated form. When lexing these are mapped to
        # their full forms so that the parser doesn't have to take care of
        # this.
        #
        axis_short = '@' | '//' | '..' | '.';

        action emit_axis_short {
          value = AXIS_MAPPING[slice_input(ts, te)]

          add_token(:T_AXIS, value)

          # Short axes that use node() as their default, implicit test. This is
          # added on lexer level to make it easier to handle these cases on
          # parser/evaluator level.
          if AXIS_EMIT_NODE.include?(value)
            add_token(:T_TYPE_TEST, 'node')

            if AXIS_EMIT_EXTRA_SLASH.include?(value) and te != eof
              add_token(:T_SLASH)
            end
          end
        }

        # Operators
        #
        # Operators can only be used inside predicates due to "div" and "mod"
        # conflicting with the patterns used for matching identifiers (=
        # element names and the likes).

        op_pipe = '|'  %{ add_token(:T_PIPE) };
        op_plus = '+'  %{ add_token(:T_ADD) };
        op_eq   = '='  %{ add_token(:T_EQ) };
        op_neq  = '!=' %{ add_token(:T_NEQ) };
        op_lt   = '<'  %{ add_token(:T_LT) };
        op_gt   = '>'  %{ add_token(:T_GT) };
        op_lte  = '<=' %{ add_token(:T_LTE) };
        op_gte  = '>=' %{ add_token(:T_GTE) };

        # These operators require whitespace around them in order to be lexed
        # as operators. This is due to "-" being allowed in node names and "*"
        # also being used as a whildcard.
        #
        # THINK: relying on whitespace is a rather fragile solution, even
        # though the W3 actually recommends this for the "-" operator. Perhaps
        # there's a better way of doing this.

        op_and = ' and ' %{ add_token(:T_AND) };
        op_or  = ' or '  %{ add_token(:T_OR) };
        op_div = ' div ' %{ add_token(:T_DIV) };
        op_mod = ' mod ' %{ add_token(:T_MOD) };
        op_mul = ' * '   %{ add_token(:T_MUL) };
        op_sub = ' - '   %{ add_token(:T_SUB) };

        operator = op_pipe
          | op_and
          | op_or
          | op_plus
          | op_div
          | op_mod
          | op_eq
          | op_neq
          | op_lt
          | op_gt
          | op_lte
          | op_gte
          | op_mul
          | op_sub
          ;

        # Node type tests
        #
        # While these look like functions they are actually node type tests. For
        # example, comment() matches all comment nodes.
        #
        # See http://www.w3.org/TR/xpath/#NT-NodeType for more information.

        type_test = (
          'comment' |
          'text' |
          'processing-instruction' |
          'node'
        ) '()';

        action emit_type_test {
          emit(:T_TYPE_TEST, ts, te - 2)
        }

        # Variables
        #
        # XPath 1.0 allows the use of variables in expressions. Oddly enough you
        # can not assign variables in an expression, you can only refer to them.
        # This means that libraries themselves have to expose an interface for
        # setting variables.
        #
        var = '$' identifier;

        action emit_variable {
          emit(:T_VAR, ts + 1, te)
        }

        main := |*
          operator;
          whitespace | slash | lparen | rparen | comma | colon | lbrack | rbrack;

          type_test  => emit_type_test;
          var        => emit_variable;
          string     => emit_string;
          integer    => emit_integer;
          float      => emit_float;
          axis_full  => emit_axis_full;
          axis_short => emit_axis_short;
          identifier => emit_identifier;
        *|;
      }%%
    end # Lexer
  end # XPath
end # Oga

%%machine xpath_lexer; # %

module Oga
  module XPath
    ##
    # Lexer for turning XPath expressions into a set of tokens. Tokens are
    # returned as arrays with every array having two values:
    #
    # 1. The token type as a symbol
    # 2. The token value or nil if there is no value
    #
    # Basic usage of this lexer is as following:
    #
    #     lexer  = Oga::XPath::Lexer.new('//foo/bar')
    #     tokens = lexer.lex
    #
    # Alternatively you can stream tokens instead of returning them as a whole:
    #
    #     lexer = Oga::XPath::Lexer.new('//foo/bar')
    #
    #     lexer.advance do |type, value|
    #
    #     end
    #
    # Unlike the XML lexer the XPath lexer does not support IO instances, it can
    # only lex strings.
    #
    # ## Thread Safety
    #
    # This class keeps track of an internal state. As a result it's not safe to
    # share a single instance between multiple threads. However, you're free to
    # use separate instances per thread as there is no global (= class level)
    # shared state.
    #
    class Lexer
      %% write data;

      # % fix highlight

      ##
      # Maps certain XPath axes written in their short form to their long form
      # equivalents.
      #
      # @return [Hash]
      #
      AXIS_MAPPING = {
        '@'  => 'attribute',
        '//' => 'descendant-or-self',
        '..' => 'parent',
        '.'  => 'self'
      }

      ##
      # Axes that require a separate `node()` call to be emitted.
      #
      # @return [Array]
      #
      AXIS_EMIT_NODE = %w{descendant-or-self parent self}

      ##
      # Axes that require an extra T_SLASH token to be emitted.
      #
      # @return [Array]
      #
      AXIS_EMIT_EXTRA_SLASH = %w{descendant-or-self}

      ##
      # @param [String] data The data to lex.
      #
      def initialize(data)
        @data = data
      end

      ##
      # Gathers all the tokens for the input and returns them as an Array.
      #
      # @see [#advance]
      # @return [Array]
      #
      def lex
        tokens = []

        advance do |type, value|
          tokens << [type, value]
        end

        return tokens
      end

      ##
      # Advances through the input and generates the corresponding tokens. Each
      # token is yielded to the supplied block.
      #
      # Each token is an Array in the following format:
      #
      #     [TYPE, VALUE]
      #
      # The type is a symbol, the value is either nil or a String.
      #
      # This method stores the supplied block in `@block` and resets it after
      # the lexer loop has finished.
      #
      # @see [#add_token]
      #
      def advance(&block)
        @block = block

        data  = @data # saves ivar lookups while lexing.
        ts    = nil
        te    = nil
        stack = []
        top   = 0
        cs    = self.class.xpath_lexer_start
        act   = 0
        eof   = @data.bytesize
        p     = 0
        pe    = eof

        _xpath_lexer_eof_trans          = self.class.send(:_xpath_lexer_eof_trans)
        _xpath_lexer_from_state_actions = self.class.send(:_xpath_lexer_from_state_actions)
        _xpath_lexer_index_offsets      = self.class.send(:_xpath_lexer_index_offsets)
        _xpath_lexer_indicies           = self.class.send(:_xpath_lexer_indicies)
        _xpath_lexer_key_spans          = self.class.send(:_xpath_lexer_key_spans)
        _xpath_lexer_to_state_actions   = self.class.send(:_xpath_lexer_to_state_actions)
        _xpath_lexer_trans_actions      = self.class.send(:_xpath_lexer_trans_actions)
        _xpath_lexer_trans_keys         = self.class.send(:_xpath_lexer_trans_keys)
        _xpath_lexer_trans_targs        = self.class.send(:_xpath_lexer_trans_targs)

        %% write exec;

        # % fix highlight
      ensure
        @block = nil
      end

      private

      ##
      # Emits a token of which the value is based on the supplied start/stop
      # position.
      #
      # @param [Symbol] type The token type.
      # @param [Fixnum] start
      # @param [Fixnum] stop
      #
      # @see [#text]
      # @see [#add_token]
      #
      def emit(type, start, stop)
        value = slice_input(start, stop)

        add_token(type, value)
      end

      ##
      # Returns the text between the specified start and stop position.
      #
      # @param [Fixnum] start
      # @param [Fixnum] stop
      # @return [String]
      #
      def slice_input(start, stop)
        return @data.byteslice(start, stop - start)
      end

      ##
      # Yields a new token to the supplied block.
      #
      # @param [Symbol] type The token type.
      # @param [String] value The token value.
      #
      # @yieldparam [Symbol] type
      # @yieldparam [String|NilClass] value
      #
      def add_token(type, value = nil)
        @block.call(type, value)
      end

      %%{
        getkey (data.getbyte(p) || 0);

        whitespace = [\n\t ];

        slash  = '/' @{ add_token(:T_SLASH) };
        lparen = '(' @{ add_token(:T_LPAREN) };
        rparen = ')' @{ add_token(:T_RPAREN) };
        comma  = ',' @{ add_token(:T_COMMA) };
        colon  = ':' @{ add_token(:T_COLON) };
        lbrack = '[' @{ add_token(:T_LBRACK) };
        rbrack = ']' @{ add_token(:T_RBRACK) };

        # Identifiers
        #
        # Identifiers are used for element names, namespaces, attribute names,
        # etc. Identifiers have to start with a letter.

        identifier = '*' | [a-zA-Z*_]+ [a-zA-Z\-_0-9]*;

        action emit_identifier {
          emit(:T_IDENT, ts, te)
        }

        # Numbers
        #
        # XPath expressions can contain both integers and floats. The W3
        # specification treats these both as the same type of number. Oga
        # instead lexes them separately so that we can convert the values to
        # the corresponding Ruby types (Fixnum and Float).

        integer = ('-' | '+')* digit+;
        float   = ('-' | '+')* digit+ ('.' digit+)*;

        action emit_integer {
          value = slice_input(ts, te).to_i

          add_token(:T_INT, value)
        }

        action emit_float {
          value = slice_input(ts, te).to_f

          add_token(:T_FLOAT, value)
        }

        # Strings
        #
        # Strings can be single or double quoted. They are mainly used for
        # attribute values.
        #
        dquote = '"';
        squote = "'";

        string_dquote = (dquote ^dquote* dquote);
        string_squote = (squote ^squote* squote);

        string = string_dquote | string_squote;

        action emit_string {
          emit(:T_STRING, ts + 1, te - 1)
        }

        # Full Axes
        #
        # XPath axes in their full syntax.
        #
        axis_full = ('ancestor'
          | 'ancestor-or-self'
          | 'attribute'
          | 'child'
          | 'descendant'
          | 'descendant-or-self'
          | 'following'
          | 'following-sibling'
          | 'namespace'
          | 'parent'
          | 'preceding'
          | 'preceding-sibling'
          | 'self') '::';

        action emit_axis_full {
          emit(:T_AXIS, ts, te - 2)
        }

        # Short Axes
        #
        # XPath axes in their abbreviated form. When lexing these are mapped to
        # their full forms so that the parser doesn't have to take care of
        # this.
        #
        axis_short = '@' | '//' | '..' | '.';

        action emit_axis_short {
          value = AXIS_MAPPING[slice_input(ts, te)]

          add_token(:T_AXIS, value)

          # Short axes that use node() as their default, implicit test. This is
          # added on lexer level to make it easier to handle these cases on
          # parser/evaluator level.
          if AXIS_EMIT_NODE.include?(value)
            add_token(:T_TYPE_TEST, 'node')

            if AXIS_EMIT_EXTRA_SLASH.include?(value) and te != eof
              add_token(:T_SLASH)
            end
          end
        }

        # Operators
        #
        # Operators can only be used inside predicates due to "div" and "mod"
        # conflicting with the patterns used for matching identifiers (=
        # element names and the likes).

        op_pipe = '|'  %{ add_token(:T_PIPE) };
        op_plus = '+'  %{ add_token(:T_ADD) };
        op_eq   = '='  %{ add_token(:T_EQ) };
        op_neq  = '!=' %{ add_token(:T_NEQ) };
        op_lt   = '<'  %{ add_token(:T_LT) };
        op_gt   = '>'  %{ add_token(:T_GT) };
        op_lte  = '<=' %{ add_token(:T_LTE) };
        op_gte  = '>=' %{ add_token(:T_GTE) };

        # These operators require whitespace around them in order to be lexed
        # as operators. This is due to "-" being allowed in node names and "*"
        # also being used as a whildcard.
        #
        # THINK: relying on whitespace is a rather fragile solution, even
        # though the W3 actually recommends this for the "-" operator. Perhaps
        # there's a better way of doing this.

        op_and = ' and ' %{ add_token(:T_AND) };
        op_or  = ' or '  %{ add_token(:T_OR) };
        op_div = ' div ' %{ add_token(:T_DIV) };
        op_mod = ' mod ' %{ add_token(:T_MOD) };
        op_mul = ' * '   %{ add_token(:T_MUL) };
        op_sub = ' - '   %{ add_token(:T_SUB) };

        operator = op_pipe
          | op_and
          | op_or
          | op_plus
          | op_div
          | op_mod
          | op_eq
          | op_neq
          | op_lt
          | op_gt
          | op_lte
          | op_gte
          | op_mul
          | op_sub
          ;

        # Node type tests
        #
        # While these look like functions they are actually node type tests. For
        # example, comment() matches all comment nodes.
        #
        # See http://www.w3.org/TR/xpath/#NT-NodeType for more information.

        type_test = (
          'comment' |
          'text' |
          'processing-instruction' |
          'node'
        ) '()';

        action emit_type_test {
          emit(:T_TYPE_TEST, ts, te - 2)
        }

        # Variables
        #
        # XPath 1.0 allows the use of variables in expressions. Oddly enough you
        # can not assign variables in an expression, you can only refer to them.
        # This means that libraries themselves have to expose an interface for
        # setting variables.
        #
        var = '$' identifier;

        action emit_variable {
          emit(:T_VAR, ts + 1, te)
        }

        main := |*
          operator;
          whitespace | slash | lparen | rparen | comma | colon | lbrack | rbrack;

          type_test  => emit_type_test;
          var        => emit_variable;
          string     => emit_string;
          integer    => emit_integer;
          float      => emit_float;
          axis_full  => emit_axis_full;
          axis_short => emit_axis_short;
          identifier => emit_identifier;
        *|;
      }%%
    end # Lexer
  end # XPath
end # Oga


switched nesting to a vector with checks for matching nesting symbols

/*
vim: syntax=ragel
*/

#include <stdlib.h>
#include <stdio.h>
#include <string.h>

#include <map>
#include <string>
#include <vector>
#include <sstream>
#include <iostream>
#include <algorithm>
#include <exception>
#include <stdexcept>

#include "lexer.h"
#include "ascii.h"

using std::cerr;
using std::cout;
using std::cin;
using std::endl;
using std::copy;

#define T(t,i,v) {i,v},
static std::map<long, const char *> TokenMap = {
    ASCII
    TOKENS
};
#undef T

inline void write(const char *data) {
    cout << data;
}

inline void write(const char c) {
    cout << c;
}

inline void write(const char *data, int len) {
    cout.write(data, len);
}

class SotaLexerException : public std::exception {
public:
    virtual char const * what() {
        return "Sota Lexer Exception";
    }
};

%%{
    machine sota;

    whitespace      = ' '+;
    tab             = '\t';
    hash            = '#';
    singlequote     = "'";
    doublequote     = '"';
    assign          = '=';
    comma           = ',';
    newline         = "\n\r"|'\n'|'\r';
    number          = digit+ ('.' digit+)?;
    syntax          = '"'|"'"|'.'|'('|')'|'['|']'|'{'|'}'|';';
    symbol          = (any - (assign|comma|hash|whitespace|newline|syntax))+;
    counter         = (any | newline @{AddNewline(fpc);})*;

    commenter := |*
        ("##" (any - newline)* newline) & counter => {
            Token(TokenKind::Comment);
            fgoto body;
        };

        ("#!" (any - newline)* newline) & counter => {
            Token(TokenKind::Comment, 0, 1);
            fgoto body;
        };

        ('#' (any - ('!'|'#')) (any - '#')* '#') & counter => {
            Token(TokenKind::Comment);
            fgoto body;
        };
    *|;

    string := |*
        ('"' (any - '"')* '"') & counter => {
            Token(TokenKind::String);
            fgoto body;
        };

        ("'" (any - "'")* "'") & counter => {
            Token(TokenKind::String);
            fgoto body;
        };
    *|;

    body := |*

        whitespace => {
        };

        ' '* newline & counter => {
        };

        hash => {
            fhold;
            fgoto commenter;
        };

        singlequote => {
            fhold;
            fgoto string;
        };

        doublequote => {
            fhold;
            fgoto string;
        };

        assign => {
            Token(fc);
        };

        comma => {
            Token(fc);
        };

        tab => {
            printf("TAB ERROR\n");
            exit(-1);
        };

        '{'|'['|'(' => {
            nesting.push_back(*ts);
            Token(fc);
        };

        '}'|']'|')' => {
            int back = nesting.back();
            switch(*ts) {
                case '}':
                    if (back != '{')
                        throw SotaLexerException();
                    break;
                case ']':
                    if (back != '[')
                        throw SotaLexerException();
                    break;
                case ')':
                    if (back != '(')
                        throw SotaLexerException();
                    break;
            }
            nesting.pop_back();
            Token(fc);
        };

        number => {
            Token(TokenKind::Number);
        };

        symbol => {
            Token(TokenKind::Symbol);
        };

        syntax => {
            Token(fc);
        };

    *|;

    write data nofinal;
}%%

class Lexer {
    char const* const source;
    char const* const pe;
    char const* const eof;
    char const* p;
    char const* ts;
    char const* te;
    int stack[1];
    int cs;
    int act;
    int top;
    std::vector<int> nesting;
    std::vector<const char *> newlines;
    std::vector<CToken> tokens;

public:
    Lexer(char const* const source)
        : source(source)
        , pe(source + strlen(source))
        , eof(source + strlen(source))
        , p(source)
        , nesting(0) {
        %% write init;
        //pretend newline before start of file
        newlines.push_back(source-1);
    }

    ~Lexer() {
        nesting.clear();
        newlines.clear();
        tokens.clear();
    }

    void AddNewline(const char *pchar) {
        if (pchar > newlines.back())
            newlines.push_back(pchar);
    }

    const char * Newline() {
        for (unsigned i = newlines.size(); i-- > 0;) {
            if (ts > newlines[i])
                return newlines[i];
        }
        return 0;
    }

    long Start() {
        return ts - source;
    }

    long End() {
        return te - source;
    }

    long Line() {
        for (unsigned i = newlines.size(); i-- > 0;) {
            if (ts > newlines[i])
                return i + 1;
        }
        return 0;
    }

    long Pos() {
        const char *nl = Newline();
        if (nl)
            return ts - nl;
        return 0;
    }

    void Token(long kind, long ltrim=0, long rtrim=0, long skip=0) {
        tokens.push_back({
            kind,
            Start() + ltrim,
            End() - rtrim,
            Line(),
            Pos(),
            skip});
    }

    long Scan(CToken **tokens) {
        %% write exec;
        *tokens = (struct CToken *)malloc(this->tokens.size() * sizeof(struct CToken));
        copy(this->tokens.begin(), this->tokens.end(), *tokens);
        return this->tokens.size();
    }

};

extern "C" long scan(const char *source, struct CToken **tokens) {
    std::ios::sync_with_stdio(false);
    Lexer lexer(source);
    return lexer.Scan(tokens);
}

/*
vim: syntax=ragel
*/

#include <stdlib.h>
#include <stdio.h>
#include <string.h>

#include <map>
#include <string>
#include <vector>
#include <sstream>
#include <iostream>
#include <algorithm>
#include <exception>
#include <stdexcept>

#include "lexer.h"
#include "ascii.h"

using std::cerr;
using std::cout;
using std::cin;
using std::endl;
using std::copy;

#define T(t,i,v) {i,v},
static std::map<long, const char *> TokenMap = {
    ASCII
    TOKENS
};
#undef T

inline void write(const char *data) {
    cout << data;
}

inline void write(const char c) {
    cout << c;
}

inline void write(const char *data, int len) {
    cout.write(data, len);
}

%%{
    machine sota;

    whitespace      = ' '+;
    tab             = '\t';
    hash            = '#';
    singlequote     = "'";
    doublequote     = '"';
    assign          = '=';
    comma           = ',';
    newline         = "\n\r"|'\n'|'\r';
    number          = digit+ ('.' digit+)?;
    syntax          = '"'|"'"|'.'|'('|')'|'['|']'|'{'|'}'|';';
    symbol          = (any - (assign|comma|hash|whitespace|newline|syntax))+;
    counter         = (any | newline @{AddNewline(fpc);})*;

    commenter := |*
        ("##" (any - newline)* newline) & counter => {
            Token(TokenKind::Comment);
            fgoto body;
        };

        ("#!" (any - newline)* newline) & counter => {
            Token(TokenKind::Comment, 0, 1);
            fgoto body;
        };

        ('#' (any - ('!'|'#')) (any - '#')* '#') & counter => {
            Token(TokenKind::Comment);
            fgoto body;
        };
    *|;

    string := |*
        ('"' (any - '"')* '"') & counter => {
            Token(TokenKind::String);
            fgoto body;
        };

        ("'" (any - "'")* "'") & counter => {
            Token(TokenKind::String);
            fgoto body;
        };
    *|;

    body := |*

        whitespace => {
        };

        ' '* newline & counter => {
        };

        hash => {
            fhold;
            fgoto commenter;
        };

        singlequote => {
            fhold;
            fgoto string;
        };

        doublequote => {
            fhold;
            fgoto string;
        };

        assign => {
            Token(fc);
        };

        comma => {
            Token(fc);
        };

        tab => {
            printf("TAB ERROR\n");
            exit(-1);
        };

        '{'|'['|'(' => {
            Token(fc);
            ++nesting;
        };

        '}'|']'|')' => {
            Token(fc);
            --nesting;
        };

        number => {
            Token(TokenKind::Number);
        };

        symbol => {
            Token(TokenKind::Symbol);
        };

        syntax => {
            Token(fc);
        };

    *|;

    write data nofinal;
}%%

class Lexer {
    char const* const source;
    char const* const pe;
    char const* const eof;
    char const* p;
    char const* ts;
    char const* te;
    int stack[1];
    int cs;
    int act;
    int top;
    int nesting;
    std::vector<const char *> newlines;
    std::vector<CToken> tokens;

public:
    Lexer(char const* const source)
        : source(source)
        , pe(source + strlen(source))
        , eof(source + strlen(source))
        , p(source)
        , nesting(0) {
        %% write init;
        //pretend newline before start of file
        newlines.push_back(source-1);
    }

    ~Lexer() {
        newlines.clear();
        tokens.clear();
    }

    void AddNewline(const char *pchar) {
        if (pchar > newlines.back())
            newlines.push_back(pchar);
    }

    const char * Newline() {
        for (unsigned i = newlines.size(); i-- > 0;) {
            if (ts > newlines[i])
                return newlines[i];
        }
        return 0;
    }

    long Start() {
        return ts - source;
    }

    long End() {
        return te - source;
    }

    long Line() {
        for (unsigned i = newlines.size(); i-- > 0;) {
            if (ts > newlines[i])
                return i + 1;
        }
        return 0;
    }

    long Pos() {
        const char *nl = Newline();
        if (nl)
            return ts - nl;
        return 0;
    }

    void Token(long kind, long ltrim=0, long rtrim=0, long skip=0) {
        tokens.push_back({
            kind,
            Start() + ltrim,
            End() - rtrim,
            Line(),
            Pos(),
            skip});
    }

    long Scan(CToken **tokens) {
        %% write exec;
        *tokens = (struct CToken *)malloc(this->tokens.size() * sizeof(struct CToken));
        copy(this->tokens.begin(), this->tokens.end(), *tokens);
        return this->tokens.size();
    }

};

extern "C" long scan(const char *source, struct CToken **tokens) {
    std::ios::sync_with_stdio(false);
    Lexer lexer(source);
    return lexer.Scan(tokens);
}


expand common definitions

expand common definitions

=begin
%%{
machine bel;

  NL          = '\n';
  SP          = ' ' | '\t';
  WS          = space;
  EQL         = '=';
  NUMBER_SIGN = '#';
  SQ		  = "'";
  DQ          = '"';
  ESCAPED     = /\\./;
  NOT_SQESC	  = [^'\\];
  NOT_DQESC	  = [^"\\];
  COMMA_DELIM = SP* ',' SP*;

  KW_SET = /SET/i;
  KW_DOCUMENT = /DOCUMENT/i;
  KW_AUTHORS = /Authors/i;
  KW_CONTACT_INFO = /ContactInfo/i;
  KW_COPYRIGHT = /Copyright/i;
  KW_DESCRIPTION = /Description/i;
  KW_LICENSES = /Licenses/i;
  KW_NAME = /Name/i;
  KW_VERSION = /Version/i;

  DOCUMENT_PROPERTY =
    KW_AUTHORS |
    KW_CONTACT_INFO |
    KW_COPYRIGHT |
    KW_DESCRIPTION |
    KW_LICENSES |
    KW_NAME |
    KW_VERSION
    ;
}%%
=end
# vim: ts=2 sw=2 ft=ragel:
# encoding: utf-8

=begin
%%{
machine bel;

  NL          = '\n';
  SP          = ' ' | '\t';
  WS          = space;
  EQL         = '=';
  NUMBER_SIGN = '#';
  SQ		  = "'";
  DQ          = '"';
  ESCAPED     = /\\./;
  NOT_SQESC	  = [^'\\];
  NOT_DQESC	  = [^"\\];
  COMMA_DELIM = SP* ',' SP*;

}%%
=end
# vim: ts=2 sw=2 ft=ragel:
# encoding: utf-8


nicer output for return values

nicer output for return values

#include "config.h"

#include <string>
#include <fstream>
#include <iostream>
#include <cstdlib>

using std::cin;
using std::cout;
using std::cerr;
using std::ifstream;
using std::ofstream;
using std::string;
using std::ios_base;
using std::endl;

%%{
  machine PostProcess;
  write data;

  # end of file character
  EOF = 0;

  # all but end of file character
  default = ^0;

  # end of line
  EOL = ('\r'? . '\n') @{line++;};

  # matlab identifier
  IDENT = [A-Z\\a-z\-_][A-Z\\a-z0-9\-_]*;

  # matlab ifdentifier without underscore characters
  IDENT_WO_US = [A-Z\\a-z\-][A-Z\\a-z0-9\-]*;

  ANY_TAG = '<' . [^>]* . '>';

  action echo {
    fout << fpc;
    cout << fpc;
  }

  action st_tok { tmp_p = p; }

  action echo_tok { fout.write(tmp_p, p - tmp_p); cout.write(tmp_p, p-tmp_p); }

  rettype:= |*

    # matlab is typeless, so discard the type
    ('matlabtypesubstitute') => {fout << " ";};

    # replace all "::" by "."
    ('::' . [A-Z\\a-z\-_]) => { fout << '.' << *(te-1); };

    # a word
    (any - [\n <>\&:,\t])+ => { fout.write(ts, te-ts); };

    # word separators
    ([\n <>\&:\t]) => {fout << *ts;};

    # comma or > end the type
    (',') => { fhold; fret; };
    ('>') => { p -=4; fret; };
  *|;

  # reconstruct return values
  retval:= |*
    ('<' . ('::')?) => { fcall rettype; };

    # matlab identifier (1 return value)
#    ('<' . (default - [,&])*) => { cerr.write(ts+4, te - ts-4); cerr << std::endl; fout.write(ts+4, te - ts-4); };

    (',' . (default - '\&')*) => { fout << " "; fout.write(ts+1, te-ts-1); fout << ""; };

    # end of return value
    ('>') => {
                  if(only_retval) { fout << " ="; }
                  fret;
                };

    # white spaces
    ([ \t\n]*) => { fout.write(ts, te - ts); };

    # typebreak
    ('
') => {};

    # other tags
    #(ANY_TAG) => { fout.write(ts, te-ts); };

  *|;

  retvals := |*

    ('<') => {};

    ('mlhsInnerSubst') => { fcall retval; };

    ('>') => { fout << "] ="; fgoto main; };

    # white spaces and commata
    ([ \t\n]*) => { fout.write(ts, te - ts); };
    ('
' . [ \t\n]* . ',') => { fout << ",
\n"; };
    (',') => { fout << ", "; };

    # other tags
    (ANY_TAG) => { fout.write(ts, te-ts); };

  *|;

  # function name
  mtocsubst:= |*
    '_';

    (IDENT_WO_US);

    '_tsbus_cotm' => { fgoto main; };
  *|;

  main:= |*
   # list of return values
   ('mlhsSubst') => { fout << "function ["; only_retval = false; fgoto retvals; };

   # one return value
   ('mlhsInnerSubst') => { fout << "function "; only_retval = true; fcall retval; fout << " ="; };

   # no return values
   ('noret::substitute') => {fout << "function ";};

   # function name
   ('mtoc_subst_') => { fgoto mtocsubst; };

   # matlab is typeless, so discard the type
   ('matlabtypesubstitute') => {fout << " ";};

   # remove leading "::" (global namespace identifier)
   ([(,>] . '::') => { fout.write(ts, 1); };
   ('<' . '::') => { fout.write(ts, 4); };

   # replace all "::" by "."
   ('::' . [A-Z\\a-z\-_]) => { fout << '.' << *(te-1); };

   # a word
   (any - [\n <>()[\]{}\&:,;_\t])+ => { fout.write(ts, te-ts); };

   # word separators
   ([\n <>()[\]{}\t:;,_\&]) => {fout << *ts;};
   *|;
}%%

class PostProcess
{

public:
  // constructor
  PostProcess(const string & filename) :
    filename_(filename),
    line(1),
    ts(0), te(0), have(0),
    top(0), only_retval(false)
  { }

  // run postprocessor
  int execute()
  {
    std::ios::sync_with_stdio(false);

    %% write init;

    ifstream is;
    try
    {
      is.open(filename_.c_str());
    }
    catch (std::ifstream::failure e)
    {
      cerr << "Exception opening/reading file";
      exit(-1);
    }

    is.seekg(0, ios_base::end);
    int length = is.tellg();
    is.seekg(0, ios_base::beg);

    char * buf = new char[(int)(1.1*length)];
    char * p = buf;
//    char * tmp_p = p;

    is.read(buf, length);
    is.close();

    ofstream fout;
    try
    {
      fout.open(filename_.c_str(), ios_base::trunc);
    }
    catch (std::ofstream::failure e)
    {
      cerr << "Exception opening/writing file";
      exit(-1);
    }

    int len = is.gcount();
    char *pe = p + len;
    char *eof = pe;

    %% write exec;

    /* Check if we failed. */
    if ( cs == PostProcess_error )
    {
      /* Machine failed before finding a token. */
      cerr << filename_ << ": PARSE ERROR in line " << line << endl;
      cerr.write(p, 100);
      exit(-1);
    }

    fout.close();
    delete buf;

    return 0;
  }

private:
  string filename_;
  int          line            , col;
  char        *ts              , *te;
  int          act             , have;
  int          cs;
  int          top;
  int          stack[5];
  bool         only_retval;

};

void usage()
{
  cout << "postprocess Version " << MTOCPP_VERSION_MAJOR << "."
    << MTOCPP_VERSION_MINOR << endl;
  cout << "Usage: ./postprocess filename" << endl;
}

int main(int argc, char ** argv)
{
  string filename;
  if(argc >= 2)
  {
    if (std::string("--help") == std::string(argv[1]))
    {
      usage();
      return 0;
    }
    filename = argv[1];
  }
  else
  {
    cerr << "wrong number of arguments!" << endl;
    exit(-2);
  }

  PostProcess scanner(filename);
  scanner.execute();
  return 0;
}

/* vim: set et sw=2 ft=ragel: */