Fix transition positioning

Fix transition positioning

%%{

  machine lexer;

  variable data @data;
  variable p    @p;
  variable pe   @pe;
  variable eof  @eof;
  access @;

  action mark {
    @start = p
  }

  number_integer = '-'? [0-9][1-9]*;
  number_frac = '.' [0-9]+;
  number_e = ('e' | 'E') ('+' | '-' | '');
  number_exp = number_e [0-9]+;
  number = number_integer number_frac? number_exp?;

  string_double = '"' (
      any -- '"' |
      '\\"'
    )* '"';
  identifier = [A-Za-z_$][A-Za-z0-9_$]*;
  string_single = "'" [A-Za-z0-9_$\-]+;
  keywords = 'undefined' | 'null' | 'true' | 'false';
  unops = '!' | '++' | '--' | '~' | 'new' | 'return' |
    'typeof';
  binops = '+' | '-' | '*' | '/' | '&' | '|' | '^' | '<<' | '>>' |
    '>>>' | '==' | '!=' | '===' | '!==' | '>' | '>=' | '<' | '<=' |
    '&&' | '||' | 'instanceof' | 'or' | 'and';

  istring_part   = ( any -- '"' | '\\"' );
  istring_start  = '"' ( istring_part* ) '#{';
  istring_middle = '}' ( istring_part* ) '#{';
  istring_end    = '}' ( istring_part* ) '"';

  body = (
           ( number         >mark %{ emit :number     } ) |
           ( string_double  >mark %{ emit :dstring    } ) |
           ( string_single  >mark %{ emit :sstring    } ) |
           ( istring_start  >mark %{ emit :istart     } ) |
           ( istring_middle >mark %{ emit :imiddle    } ) |
           ( istring_end    >mark %{ emit :iend       } ) |
           (       'class'        %{ emit :class      } ) |
           (       'module'       %{ emit :module     } ) |
           (       'if'           %{ emit :if         } ) |
           (       'unless'       %{ emit :unless     } ) |
           (       'elsif'        %{ emit :elsif      } ) |
           (       'else'         %{ emit :else       } ) |
           ( unops          >mark %{ emit :unop       } ) |
           ( binops         >mark %{ emit :binop      } ) |
           ( keywords       >mark %{ emit :keyword    } ) |
           ( identifier     >mark %{ emit :identifier } ) |
           (       '->'           %{ emit :arrow      } ) |
           (       '='            %{ emit :equal      } ) |
           (       '{'            %{ emit :lbrack     } ) |
           (       '('            %{ emit :lparen     } ) |
           (       '['            %{ emit :lbrace     } ) |
           (       '}'            %{ emit :rbrack     } ) |
           (       ')'            %{ emit :rparen     } ) |
           (       ']'            %{ emit :rbrace     } ) |
           (       ':'            %{ emit :colon      } ) |
           (       '.'            %{ emit :prop       } ) |
           (       ','            %{ emit :comma      } ) |
           (       '\n'           %{ line.call        } )
    );

  loop = ( body** );

  main := loop;
}%%

module Liquidscript
  class Scanner

    # A lexer, built from ragel.
    #
    # @private
    class Lexer

      attr_reader :tokens

      def initialize
        %% write data;
        # %% # fix
        @tokens = []
        clean!
      end

      def clean!
        @p = nil
        @pe = nil
        @te = nil
        @ts = nil
        @act = nil
        @eof = nil
        @top = nil
        @line = { :start => 0, :num => 0 }
        @data = nil
        @stack = nil
        @start = nil
      end

      def emit(type)
        @tokens << if @start
          Token.new(type, @data[@start..(@p - 1)],
            @line[:num], @p - @line[:start])
        else
          Token.new(type, nil, @line[:num], @p - @line[:start])
        end

        @start = nil
      end

      def error
        raise SyntaxError, "Unexpected #{@data[@start..(@p-1)].pack('c*')}"
      end

      def perform(data)
        @data = data.unpack("c*") if data.is_a? String
        @eof = data.length

        @tokens = []

        line = proc do
          @line[:start] = @ts
          @line[:num] += 1
          emit :newline
        end

        %% write init;
        %% write exec;

        clean!

        @tokens
      end
    end
  end
end

%%{

  machine lexer;

  variable data @data;
  variable p    @p;
  variable pe   @pe;
  variable eof  @eof;
  access @;

  action mark {
    @start = p
  }

  number_integer = '-'? [0-9][1-9]*;
  number_frac = '.' [0-9]+;
  number_e = ('e' | 'E') ('+' | '-' | '');
  number_exp = number_e [0-9]+;
  number = number_integer number_frac? number_exp?;

  string_double = '"' (
      any -- '"' |
      '\\"'
    )* '"';
  identifier = [A-Za-z_$][A-Za-z0-9_$]*;
  string_single = "'" [A-Za-z0-9_$\-]+;
  keywords = 'undefined' | 'null' | 'true' | 'false';
  unops = '!' | '++' | '--' | '~' | 'new' | 'return' |
    'typeof';
  binops = '+' | '-' | '*' | '/' | '&' | '|' | '^' | '<<' | '>>' |
    '>>>' | '==' | '!=' | '===' | '!==' | '>' | '>=' | '<' | '<=' |
    '&&' | '||' | 'instanceof' | 'or' | 'and';

  istring_part   = ( any -- '"' | '\\"' );
  istring_start  = '"' ( istring_part* ) '#{';
  istring_middle = '}' ( istring_part* ) '#{';
  istring_end    = '}' ( istring_part* ) '"';

  body = (
           ( >mark number         %{ emit :number     } ) |
           ( >mark string_double  %{ emit :dstring    } ) |
           ( >mark string_single  %{ emit :sstring    } ) |
           ( >mark istring_start  %{ emit :istart     } ) |
           ( >mark istring_middle %{ emit :imiddle    } ) |
           ( >mark istring_end    %{ emit :iend       } ) |
           (       'class'        %{ emit :class      } ) |
           (       'module'       %{ emit :module     } ) |
           (       'if'           %{ emit :if         } ) |
           (       'unless'       %{ emit :unless     } ) |
           (       'elsif'        %{ emit :elsif      } ) |
           (       'else'         %{ emit :else       } ) |
           ( >mark unops          %{ emit :unop       } ) |
           ( >mark binops         %{ emit :binop      } ) |
           ( >mark keywords       %{ emit :keyword    } ) |
           ( >mark identifier     %{ emit :identifier } ) |
           (       '->'           %{ emit :arrow      } ) |
           (       '='            %{ emit :equal      } ) |
           (       '{'            %{ emit :lbrack     } ) |
           (       '('            %{ emit :lparen     } ) |
           (       '['            %{ emit :lbrace     } ) |
           (       '}'            %{ emit :rbrack     } ) |
           (       ')'            %{ emit :rparen     } ) |
           (       ']'            %{ emit :rbrace     } ) |
           (       ':'            %{ emit :colon      } ) |
           (       '.'            %{ emit :prop       } ) |
           (       ','            %{ emit :comma      } ) |
           (       '\n'           %{ line.call        } )
    );

  loop = ( body** );

  main := loop;
}%%

module Liquidscript
  class Scanner

    # A lexer, built from ragel.
    #
    # @private
    class Lexer

      attr_reader :tokens

      def initialize
        %% write data;
        # %% # fix
        @tokens = []
        clean!
      end

      def clean!
        @p = nil
        @pe = nil
        @te = nil
        @ts = nil
        @act = nil
        @eof = nil
        @top = nil
        @line = { :start => 0, :num => 0 }
        @data = nil
        @stack = nil
        @start = nil
      end

      def emit(type)
        @tokens << if @start
          Token.new(type, @data[@start..(@p - 1)],
            @line[:num], @p - @line[:start])
        else
          Token.new(type, nil, @line[:num], @p - @line[:start])
        end

        @start = nil
      end

      def error
        raise SyntaxError, "Unexpected #{@data[@start..(@p-1)].pack('c*')}"
      end

      def perform(data)
        @data = data.unpack("c*") if data.is_a? String
        @eof = data.length

        @tokens = []

        line = proc do
          @line[:start] = @ts
          @line[:num] += 1
          emit :newline
        end

        %% write init;
        %% write exec;

        clean!

        @tokens
      end
    end
  end
end


base64_decode must return an ASCII-8BIT String*

/* This file was generated by Ragel. Your edits will be lost.
 *
 * This is a state machine implementation of String#unpack.
 * See http://github.com/rubinius/rapa.
 *
 * vim: filetype=cpp
 */

#include <sstream>

#include "vm/config.h"

#include "vm.hpp"
#include "object_utils.hpp"
#include "on_stack.hpp"
#include "objectmemory.hpp"

#include "builtin/array.hpp"
#include "builtin/bytearray.hpp"
#include "builtin/encoding.hpp"
#include "builtin/exception.hpp"
#include "builtin/fixnum.hpp"
#include "builtin/float.hpp"
#include "builtin/string.hpp"

namespace rubinius {

  namespace unpack19 {
    void inline increment(native_int& index, native_int n, native_int limit) {
      if(index + n < limit) {
        index += n;
      } else {
        index = limit;
      }
    }

    uint16_t swap_2bytes(uint16_t x) {
      return ((((x)&0xff)<<8) | (((x)>>8)&0xff));
    }

    uint32_t swap_4bytes(uint32_t x) {
      return ((((x)&0xff)<<24)
             |(((x)>>24)&0xff)
             |(((x)&0x0000ff00)<<8)
             |(((x)&0x00ff0000)>>8));
    }

    uint64_t swap_8bytes(uint64_t x) {
      return ((((x)&0x00000000000000ffLL)<<56)
             |(((x)&0xff00000000000000LL)>>56)
             |(((x)&0x000000000000ff00LL)<<40)
             |(((x)&0x00ff000000000000LL)>>40)
             |(((x)&0x0000000000ff0000LL)<<24)
             |(((x)&0x0000ff0000000000LL)>>24)
             |(((x)&0x00000000ff000000LL)<<8)
             |(((x)&0x000000ff00000000LL)>>8));
    }

    float swap_float(const uint8_t* str) {
      uint32_t x;
      float y;

      memcpy(&x, str, sizeof(uint32_t));
      x = swap_4bytes(x);
      memcpy(&y, &x, sizeof(float));

      return y;
    }

    double swap_double(const uint8_t* str) {
      uint64_t x;
      double y;

      memcpy(&x, str, sizeof(uint64_t));
      x = swap_8bytes(x);
      memcpy(&y, &x, sizeof(double));

      return y;
    }

    inline int hex2num(char c) {
      switch (c) {
      case '0': case '1': case '2': case '3': case '4':
      case '5': case '6': case '7': case '8': case '9':
        return c - '0';
      case 'a': case 'b': case 'c':
      case 'd': case 'e': case 'f':
        return c - 'a' + 10;
      case 'A': case 'B': case 'C':
      case 'D': case 'E': case 'F':
        return c - 'A' + 10;
      default:
        return -1;
      }
    }

    String* quotable_printable(STATE, const char*& bytes,
                               const char* bytes_end, native_int remainder)
    {
      if(remainder == 0) {
        return String::create(state, 0, 0);
      }

      String* str = String::create(state, 0, remainder);
      uint8_t *buf = str->byte_address();

      while(bytes < bytes_end) {
        if(*bytes == '=') {
          if(++bytes == bytes_end)
            break;

          if(bytes+1 < bytes_end && bytes[0] == '\r' && bytes[1] == '\n')
            bytes++;

          if(*bytes != '\n') {
            int c1, c2;

            if((c1 = hex2num(*bytes)) == -1)
              break;
            if(++bytes == bytes_end)
              break;
            if((c2 = hex2num(*bytes)) == -1)
              break;
            *buf++ = c1 << 4 | c2;
          }
        } else {
          *buf++ = *bytes;
        }
        bytes++;
      }

      *buf = 0;
      str->num_bytes(state, Fixnum::from(buf - str->byte_address()));

      return str;
    }

    String* base64_decode(STATE, const char*& bytes,
                          const char* bytes_end, native_int remainder)
    {
      if(remainder == 0) {
        String* str = String::create(state, 0, 0);

        str->encoding(state, Encoding::ascii8bit_encoding(state));

        return str;
      }

      static bool initialized = false;
      static signed char b64_xtable[256];

      if(!initialized) {
        initialized = true;

        for(int i = 0; i < 256; i++) {
          b64_xtable[i] = -1;
        }

        for(int i = 0; i < 64; i++) {
          static const char table[] =
            "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
          b64_xtable[(int)(table[i])] = i;
        }
      }

      native_int num_bytes = (bytes_end - bytes) * 3 / 4;
      String* str = String::create(state, 0, num_bytes);
      uint8_t *buf = str->byte_address();

      int a = -1, b = -1, c = 0, d = 0;
      while(bytes < bytes_end) {
        a = b = c = d = -1;

        while((a = b64_xtable[(int)(*bytes)]) == -1 && bytes < bytes_end)
          bytes++;
        if(bytes >= bytes_end)
          break;
        bytes++;

        while((b = (int)b64_xtable[(int)(*bytes)]) == -1 && bytes < bytes_end)
          bytes++;
        if(bytes >= bytes_end)
          break;
        bytes++;

        while((c = (int)b64_xtable[(int)(*bytes)]) == -1 && bytes < bytes_end) {
          if(*bytes == '=')
            break;
          bytes++;
        }
        if(*bytes == '=' || bytes >= bytes_end)
          break;
        bytes++;

        while((d = (int)b64_xtable[(int)(*bytes)]) == -1 && bytes < bytes_end) {
          if(*bytes == '=')
            break;
          bytes++;
        }
        if(*bytes == '=' || bytes >= bytes_end)
          break;
        bytes++;

        *buf++ = a << 2 | b >> 4;
        *buf++ = b << 4 | c >> 2;
        *buf++ = c << 6 | d;
      }

      if(a != -1 && b != -1) {
        if(c == -1 && *bytes == '=') {
          *buf++ = a << 2 | b >> 4;
        } else if(c != -1 && *bytes == '=') {
          *buf++ = a << 2 | b >> 4;
          *buf++ = b << 4 | c >> 2;
        }
      }

      *buf = 0;
      str->encoding(state, Encoding::ascii8bit_encoding(state));
      str->num_bytes(state, Fixnum::from(buf - str->byte_address()));
      return str;
    }

    String* uu_decode(STATE, const char*& bytes,
                      const char* bytes_end, native_int remainder)
    {
      if(remainder == 0) {
        return String::create(state, 0, 0);
      }

      native_int length = 0, num_bytes = (bytes_end - bytes) * 3 / 4;
      String* str = String::create(state, 0, num_bytes);
      uint8_t *buf = str->byte_address();

      while(bytes < bytes_end && *bytes > ' ' && *bytes < 'a') {
        native_int line = (*bytes++ - ' ') & 0x3f;
        length += line;
        if(length > num_bytes) {
          line -= length - num_bytes;
          length = num_bytes;
        }

        while(line > 0) {
          char values[4];
          int l = line > 3 ? 3 : line;

          for(int i = 0; i < 4; i++) {
            if(bytes < bytes_end && *bytes >= ' ') {
              values[i] = (*bytes++ - ' ') & 0x3f;
            } else {
              values[i] = 0;
            }
          }

          switch(l) {
          case 3:
            buf[2] = values[2] << 6 | values[3];
          case 2:
            buf[1] = values[1] << 4 | values[2] >> 2;
          case 1:
            buf[0] = values[0] << 2 | values[1] >> 4;
          }

          buf += l;
          line -= l;
        }

        if(*bytes == '\r') bytes++;
        if(*bytes == '\n') {
          bytes++;
        } else if(bytes < bytes_end && (bytes+1 == bytes_end || bytes[1] == '\n')) {
          // possible checksum byte
          bytes += 2;
        }
      }

      buf[length] = 0;
      str->num_bytes(state, Fixnum::from(length));
      return str;
    }

    static const int32_t utf8_limits[] = {
      0x0,        /* 1 */
      0x80,       /* 2 */
      0x800,      /* 3 */
      0x10000,    /* 4 */
      0x200000,   /* 5 */
      0x4000000,  /* 6 */
      0x80000000, /* 7 */
    };

#define MALFORMED_UTF8_ERROR_SIZE 60

    void utf8_decode(STATE, Array* array,
                     const char* bytes, const char* bytes_end,
                     native_int count, native_int& index)
    {
      int length;

      for(; count > 0 && bytes < bytes_end; count--) {
        native_int remainder = bytes_end - bytes;
        int32_t c = *bytes++ & 0xff, value = c;
        int n = 0;
        length = 1;

        if(value & 0x80) {
          if(!(value & 0x40)) {
            Exception::argument_error(state, "malformed UTF-8 character");
          }

          if(!(value & 0x20)) {
            n = 2;
            value &= 0x1f;
          } else if(!(value & 0x10)) {
            n = 3;
            value &= 0x0f;
          } else if(!(value & 0x08)) {
            n = 4;
            value &= 0x07;
          } else if(!(value & 0x04)) {
            n = 5;
            value &= 0x03;
          } else if(!(value & 0x02)) {
            n = 6;
            value &= 0x01;
          } else {
            Exception::argument_error(state, "malformed UTF-8 character");
          }

          if(n > remainder) {
            char error_msg[MALFORMED_UTF8_ERROR_SIZE];
            snprintf(error_msg, MALFORMED_UTF8_ERROR_SIZE,
                    "malformed UTF-8 character (expected %d bytes, given %d bytes)",
                    n, (int)remainder);
            Exception::argument_error(state, error_msg);
          }

          length = n--;
          if(n != 0) {
            while(n--) {
              c = *bytes++ & 0xff;
              if((c & 0xc0) != 0x80) {
                Exception::argument_error(state, "malformed UTF-8 character");
              } else {
                c &= 0x3f;
                value = value << 6 | c;
              }
            }
          }

          if(value < utf8_limits[length-1]) {
            Exception::argument_error(state, "redundant UTF-8 sequence");
          }
        }

        array->append(state, Integer::from(state, value));
        index += length;
      }
    }

    void ber_decode(STATE, Array* array,
                     const char*& bytes, const char* bytes_end,
                     native_int count, native_int& index)
    {
      static unsigned long mask = 0xfeUL << ((sizeof(unsigned long) - 1) * 8);
      static Fixnum* base = Fixnum::from(128);
      unsigned long value = 0;

      while(count > 0 && bytes < bytes_end) {
        value <<= 7;
        value |= (*bytes & 0x7f);
        if(!(*bytes++ & 0x80)) {
          array->append(state, Integer::from(state, value));
          count--;
          value = 0;
        } else if(value & mask) {
          Integer* result = Integer::from(state, value);

          while(bytes < bytes_end) {
            if(result->fixnum_p()) {
              result = as<Fixnum>(result)->mul(state, base);
            } else {
              result = as<Bignum>(result)->mul(state, base);
            }

            Fixnum* b = Fixnum::from(*bytes & 0x7f);
            if(result->fixnum_p()) {
              result = as<Fixnum>(result)->add(state, b);
            } else {
              result = as<Bignum>(result)->add(state, b);
            }

            if(!(*bytes++ & 0x80)) {
              if(result->fixnum_p()) {
                array->append(state, result);
              } else {
                array->append(state, Bignum::normalize(state, as<Bignum>(result)));
              }
              count--;
              value = 0;
              break;
            }
          }
        }
      }
    }

    String* bit_high(STATE, const char*& bytes, native_int count) {
      String* str = String::create(state, 0, count);
      uint8_t *buf = str->byte_address();
      int bits = 0;

      for(native_int i = 0; i < count; i++) {
        if(i & 7) {
          bits <<= 1;
        } else {
          bits = *bytes++;
        }

        buf[i] = (bits & 128) ? '1' : '0';
      }

      return str;
    }

    String* bit_low(STATE, const char*& bytes, native_int count) {
      String* str = String::create(state, 0, count);
      uint8_t *buf = str->byte_address();
      int bits = 0;

      for(native_int i = 0; i < count; i++) {
        if(i & 7) {
          bits >>= 1;
        } else {
          bits = *bytes++;
        }

        buf[i] = (bits & 1) ? '1' : '0';
      }

      return str;
    }

    static const char hexdigits[] = "0123456789abcdef0123456789ABCDEFx";

    String* hex_high(STATE, const char*& bytes, native_int count) {
      String* str = String::create(state, 0, count);
      uint8_t *buf = str->byte_address();
      int bits = 0;

      for(native_int i = 0; i < count; i++) {
        if(i & 1) {
          bits <<= 4;
        } else {
          bits = *bytes++;
        }

        buf[i] = unpack19::hexdigits[(bits >> 4) & 15];
      }

      return str;
    }

    String* hex_low(STATE, const char*& bytes, native_int count) {
      String* str = String::create(state, 0, count);
      uint8_t *buf = str->byte_address();
      int bits = 0;

      for(native_int i = 0; i < count; i++) {
        if(i & 1) {
          bits >>= 4;
        } else {
          bits = *bytes++;
        }

        buf[i] = unpack19::hexdigits[bits & 15];
      }

      return str;
    }

    ByteArray* prepare_directives(STATE, String* directives,
                                  const char** p, const char** pe)
    {
      native_int size = directives->byte_size();
      ByteArray* ba = ByteArray::create_pinned(state, size);
      char* b = reinterpret_cast<char*>(ba->raw_bytes());
      char* d = reinterpret_cast<char*>(directives->byte_address());
      int i = 0, j = 0;

      while(i < size) {
        switch(d[i]) {
        case 0:
        case ' ':
        case '\t':
        case '\n':
        case '\v':
        case '\f':
        case '\r':
          i++;
          break;
        case '#':
          while(++i < size && d[i] != '\n')
            ; // ignore
          if(d[i] == '\n') i++;
          break;
        default:
          b[j++] = d[i++];
          break;
        }
      }

      *p = const_cast<const char*>(b);
      *pe = *p + j;

      return ba;
    }
  }

#define unpack_elements(create, bits)                     \
  for(; index < stop; index += width) {                   \
    const uint8_t* bytes = self->byte_address() + index;  \
    array->append(state, create(bits(bytes)));            \
    if(count > 0) count--;                                \
  }

#define UNPACK_ELEMENTS unpack_elements
#define unpack_integer(b)         unpack_elements(new_integer, b)
#define unpack_float_elements(b)  unpack_elements(new_float, b)

#define FIXNUM(b)         (Fixnum::from(b))
#define INTEGER(b)        (Integer::from(state, b))

#define new_integer(b)    (Integer::from(state, b))
#define new_float(b)      (Float::create(state, b))

#define sbyte(p)          (*(int8_t*)(p))
#define ubyte(p)          (*(uint8_t*)(p))

#define s2bytes(p)        (*(int16_t*)(p))
#define u2bytes(p)        (*(uint16_t*)(p))

#define s4bytes(p)        (*(int32_t*)(p))
#define u4bytes(p)        (*(uint32_t*)(p))

#define s8bytes(p)        (*(int64_t*)(p))
#define u8bytes(p)        (*(uint64_t*)(p))

#define float_bits(p)     (*(float*)(p))
#define double_bits(p)    (*(double*)(p))

#ifdef RBX_LITTLE_ENDIAN

# define s2bytes_le(p)            (s2bytes(p))
# define u2bytes_le(p)            (u2bytes(p))
# define s4bytes_le(p)            (s4bytes(p))
# define u4bytes_le(p)            (u4bytes(p))
# define s8bytes_le(p)            (s8bytes(p))
# define u8bytes_le(p)            (u8bytes(p))

# define s2bytes_be(p)            ((int16_t)(unpack19::swap_2bytes(u2bytes(p))))
# define u2bytes_be(p)            ((uint16_t)(unpack19::swap_2bytes(u2bytes(p))))
# define s4bytes_be(p)            ((int32_t)(unpack19::swap_4bytes(u4bytes(p))))
# define u4bytes_be(p)            ((uint32_t)(unpack19::swap_4bytes(u4bytes(p))))
# define s8bytes_be(p)            ((int64_t)(unpack19::swap_8bytes(u8bytes(p))))
# define u8bytes_be(p)            ((uint64_t)(unpack19::swap_8bytes(u8bytes(p))))

# define unpack_double            unpack_double_le
# define unpack_float             unpack_float_le

# define unpack_double_le         unpack_float_elements(double_bits)
# define unpack_float_le          unpack_float_elements(float_bits)

# define unpack_double_be         unpack_float_elements(unpack19::swap_double)
# define unpack_float_be          unpack_float_elements(unpack19::swap_float)

#else // Big endian

# define s2bytes_le(p)            ((int16_t)(unpack19::swap_2bytes(u2bytes(p))))
# define u2bytes_le(p)            ((uint16_t)(unpack19::swap_2bytes(u2bytes(p))))
# define s4bytes_le(p)            ((int32_t)(unpack19::swap_4bytes(u4bytes(p))))
# define u4bytes_le(p)            ((uint32_t)(unpack19::swap_4bytes(u4bytes(p))))
# define s8bytes_le(p)            ((int64_t)(unpack19::swap_8bytes(u8bytes(p))))
# define u8bytes_le(p)            ((uint64_t)(unpack19::swap_8bytes(u8bytes(p))))

# define s2bytes_be(p)            (s2bytes(p))
# define u2bytes_be(p)            (u2bytes(p))
# define s4bytes_be(p)            (s4bytes(p))
# define u4bytes_be(p)            (u4bytes(p))
# define s8bytes_be(p)            (s8bytes(p))
# define u8bytes_be(p)            (u8bytes(p))

# define unpack_double            unpack_double_be
# define unpack_float             unpack_float_be

# define unpack_double_le         unpack_float_elements(unpack19::swap_double)
# define unpack_float_le          unpack_float_elements(unpack19::swap_float)

# define unpack_double_be         unpack_float_elements(double_bits)
# define unpack_float_be          unpack_float_elements(float_bits)

#endif

  Array* String::unpack19(STATE, String* directives) {
    // Ragel-specific variables
    const char* p;
    const char* pe;
    ByteArray* d = unpack19::prepare_directives(state, directives, &p, &pe);
    const char *eof = pe;
    int cs;

    // pack-specific variables
    String* self = this;
    Array* array = Array::create(state, 0);
    OnStack<3> sv(state, self, array, d);
    const char* bytes = 0;
    const char* bytes_end = 0;

    native_int bytes_size = self->byte_size();
    native_int index = 0;
    native_int stop = 0;
    native_int width = 0;
    native_int count = 0;
    native_int remainder = 0;
    bool rest = false;

%%{
  machine unpack;

  include "unpack.rl";

}%%

    if(en_main) {
      // do nothing
    }

    return force_as<Array>(Primitives::failure());
  }
}

/* This file was generated by Ragel. Your edits will be lost.
 *
 * This is a state machine implementation of String#unpack.
 * See http://github.com/rubinius/rapa.
 *
 * vim: filetype=cpp
 */

#include <sstream>

#include "vm/config.h"

#include "vm.hpp"
#include "object_utils.hpp"
#include "on_stack.hpp"
#include "objectmemory.hpp"

#include "builtin/array.hpp"
#include "builtin/bytearray.hpp"
#include "builtin/exception.hpp"
#include "builtin/fixnum.hpp"
#include "builtin/float.hpp"
#include "builtin/string.hpp"

namespace rubinius {

  namespace unpack19 {
    void inline increment(native_int& index, native_int n, native_int limit) {
      if(index + n < limit) {
        index += n;
      } else {
        index = limit;
      }
    }

    uint16_t swap_2bytes(uint16_t x) {
      return ((((x)&0xff)<<8) | (((x)>>8)&0xff));
    }

    uint32_t swap_4bytes(uint32_t x) {
      return ((((x)&0xff)<<24)
             |(((x)>>24)&0xff)
             |(((x)&0x0000ff00)<<8)
             |(((x)&0x00ff0000)>>8));
    }

    uint64_t swap_8bytes(uint64_t x) {
      return ((((x)&0x00000000000000ffLL)<<56)
             |(((x)&0xff00000000000000LL)>>56)
             |(((x)&0x000000000000ff00LL)<<40)
             |(((x)&0x00ff000000000000LL)>>40)
             |(((x)&0x0000000000ff0000LL)<<24)
             |(((x)&0x0000ff0000000000LL)>>24)
             |(((x)&0x00000000ff000000LL)<<8)
             |(((x)&0x000000ff00000000LL)>>8));
    }

    float swap_float(const uint8_t* str) {
      uint32_t x;
      float y;

      memcpy(&x, str, sizeof(uint32_t));
      x = swap_4bytes(x);
      memcpy(&y, &x, sizeof(float));

      return y;
    }

    double swap_double(const uint8_t* str) {
      uint64_t x;
      double y;

      memcpy(&x, str, sizeof(uint64_t));
      x = swap_8bytes(x);
      memcpy(&y, &x, sizeof(double));

      return y;
    }

    inline int hex2num(char c) {
      switch (c) {
      case '0': case '1': case '2': case '3': case '4':
      case '5': case '6': case '7': case '8': case '9':
        return c - '0';
      case 'a': case 'b': case 'c':
      case 'd': case 'e': case 'f':
        return c - 'a' + 10;
      case 'A': case 'B': case 'C':
      case 'D': case 'E': case 'F':
        return c - 'A' + 10;
      default:
        return -1;
      }
    }

    String* quotable_printable(STATE, const char*& bytes,
                               const char* bytes_end, native_int remainder)
    {
      if(remainder == 0) {
        return String::create(state, 0, 0);
      }

      String* str = String::create(state, 0, remainder);
      uint8_t *buf = str->byte_address();

      while(bytes < bytes_end) {
        if(*bytes == '=') {
          if(++bytes == bytes_end)
            break;

          if(bytes+1 < bytes_end && bytes[0] == '\r' && bytes[1] == '\n')
            bytes++;

          if(*bytes != '\n') {
            int c1, c2;

            if((c1 = hex2num(*bytes)) == -1)
              break;
            if(++bytes == bytes_end)
              break;
            if((c2 = hex2num(*bytes)) == -1)
              break;
            *buf++ = c1 << 4 | c2;
          }
        } else {
          *buf++ = *bytes;
        }
        bytes++;
      }

      *buf = 0;
      str->num_bytes(state, Fixnum::from(buf - str->byte_address()));

      return str;
    }

    String* base64_decode(STATE, const char*& bytes,
                          const char* bytes_end, native_int remainder)
    {
      if(remainder == 0) {
        return String::create(state, 0, 0);
      }

      static bool initialized = false;
      static signed char b64_xtable[256];

      if(!initialized) {
        initialized = true;

        for(int i = 0; i < 256; i++) {
          b64_xtable[i] = -1;
        }

        for(int i = 0; i < 64; i++) {
          static const char table[] =
            "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
          b64_xtable[(int)(table[i])] = i;
        }
      }

      native_int num_bytes = (bytes_end - bytes) * 3 / 4;
      String* str = String::create(state, 0, num_bytes);
      uint8_t *buf = str->byte_address();

      int a = -1, b = -1, c = 0, d = 0;
      while(bytes < bytes_end) {
        a = b = c = d = -1;

        while((a = b64_xtable[(int)(*bytes)]) == -1 && bytes < bytes_end)
          bytes++;
        if(bytes >= bytes_end)
          break;
        bytes++;

        while((b = (int)b64_xtable[(int)(*bytes)]) == -1 && bytes < bytes_end)
          bytes++;
        if(bytes >= bytes_end)
          break;
        bytes++;

        while((c = (int)b64_xtable[(int)(*bytes)]) == -1 && bytes < bytes_end) {
          if(*bytes == '=')
            break;
          bytes++;
        }
        if(*bytes == '=' || bytes >= bytes_end)
          break;
        bytes++;

        while((d = (int)b64_xtable[(int)(*bytes)]) == -1 && bytes < bytes_end) {
          if(*bytes == '=')
            break;
          bytes++;
        }
        if(*bytes == '=' || bytes >= bytes_end)
          break;
        bytes++;

        *buf++ = a << 2 | b >> 4;
        *buf++ = b << 4 | c >> 2;
        *buf++ = c << 6 | d;
      }

      if(a != -1 && b != -1) {
        if(c == -1 && *bytes == '=') {
          *buf++ = a << 2 | b >> 4;
        } else if(c != -1 && *bytes == '=') {
          *buf++ = a << 2 | b >> 4;
          *buf++ = b << 4 | c >> 2;
        }
      }

      *buf = 0;
      str->num_bytes(state, Fixnum::from(buf - str->byte_address()));
      return str;
    }

    String* uu_decode(STATE, const char*& bytes,
                      const char* bytes_end, native_int remainder)
    {
      if(remainder == 0) {
        return String::create(state, 0, 0);
      }

      native_int length = 0, num_bytes = (bytes_end - bytes) * 3 / 4;
      String* str = String::create(state, 0, num_bytes);
      uint8_t *buf = str->byte_address();

      while(bytes < bytes_end && *bytes > ' ' && *bytes < 'a') {
        native_int line = (*bytes++ - ' ') & 0x3f;
        length += line;
        if(length > num_bytes) {
          line -= length - num_bytes;
          length = num_bytes;
        }

        while(line > 0) {
          char values[4];
          int l = line > 3 ? 3 : line;

          for(int i = 0; i < 4; i++) {
            if(bytes < bytes_end && *bytes >= ' ') {
              values[i] = (*bytes++ - ' ') & 0x3f;
            } else {
              values[i] = 0;
            }
          }

          switch(l) {
          case 3:
            buf[2] = values[2] << 6 | values[3];
          case 2:
            buf[1] = values[1] << 4 | values[2] >> 2;
          case 1:
            buf[0] = values[0] << 2 | values[1] >> 4;
          }

          buf += l;
          line -= l;
        }

        if(*bytes == '\r') bytes++;
        if(*bytes == '\n') {
          bytes++;
        } else if(bytes < bytes_end && (bytes+1 == bytes_end || bytes[1] == '\n')) {
          // possible checksum byte
          bytes += 2;
        }
      }

      buf[length] = 0;
      str->num_bytes(state, Fixnum::from(length));
      return str;
    }

    static const int32_t utf8_limits[] = {
      0x0,        /* 1 */
      0x80,       /* 2 */
      0x800,      /* 3 */
      0x10000,    /* 4 */
      0x200000,   /* 5 */
      0x4000000,  /* 6 */
      0x80000000, /* 7 */
    };

#define MALFORMED_UTF8_ERROR_SIZE 60

    void utf8_decode(STATE, Array* array,
                     const char* bytes, const char* bytes_end,
                     native_int count, native_int& index)
    {
      int length;

      for(; count > 0 && bytes < bytes_end; count--) {
        native_int remainder = bytes_end - bytes;
        int32_t c = *bytes++ & 0xff, value = c;
        int n = 0;
        length = 1;

        if(value & 0x80) {
          if(!(value & 0x40)) {
            Exception::argument_error(state, "malformed UTF-8 character");
          }

          if(!(value & 0x20)) {
            n = 2;
            value &= 0x1f;
          } else if(!(value & 0x10)) {
            n = 3;
            value &= 0x0f;
          } else if(!(value & 0x08)) {
            n = 4;
            value &= 0x07;
          } else if(!(value & 0x04)) {
            n = 5;
            value &= 0x03;
          } else if(!(value & 0x02)) {
            n = 6;
            value &= 0x01;
          } else {
            Exception::argument_error(state, "malformed UTF-8 character");
          }

          if(n > remainder) {
            char error_msg[MALFORMED_UTF8_ERROR_SIZE];
            snprintf(error_msg, MALFORMED_UTF8_ERROR_SIZE,
                    "malformed UTF-8 character (expected %d bytes, given %d bytes)",
                    n, (int)remainder);
            Exception::argument_error(state, error_msg);
          }

          length = n--;
          if(n != 0) {
            while(n--) {
              c = *bytes++ & 0xff;
              if((c & 0xc0) != 0x80) {
                Exception::argument_error(state, "malformed UTF-8 character");
              } else {
                c &= 0x3f;
                value = value << 6 | c;
              }
            }
          }

          if(value < utf8_limits[length-1]) {
            Exception::argument_error(state, "redundant UTF-8 sequence");
          }
        }

        array->append(state, Integer::from(state, value));
        index += length;
      }
    }

    void ber_decode(STATE, Array* array,
                     const char*& bytes, const char* bytes_end,
                     native_int count, native_int& index)
    {
      static unsigned long mask = 0xfeUL << ((sizeof(unsigned long) - 1) * 8);
      static Fixnum* base = Fixnum::from(128);
      unsigned long value = 0;

      while(count > 0 && bytes < bytes_end) {
        value <<= 7;
        value |= (*bytes & 0x7f);
        if(!(*bytes++ & 0x80)) {
          array->append(state, Integer::from(state, value));
          count--;
          value = 0;
        } else if(value & mask) {
          Integer* result = Integer::from(state, value);

          while(bytes < bytes_end) {
            if(result->fixnum_p()) {
              result = as<Fixnum>(result)->mul(state, base);
            } else {
              result = as<Bignum>(result)->mul(state, base);
            }

            Fixnum* b = Fixnum::from(*bytes & 0x7f);
            if(result->fixnum_p()) {
              result = as<Fixnum>(result)->add(state, b);
            } else {
              result = as<Bignum>(result)->add(state, b);
            }

            if(!(*bytes++ & 0x80)) {
              if(result->fixnum_p()) {
                array->append(state, result);
              } else {
                array->append(state, Bignum::normalize(state, as<Bignum>(result)));
              }
              count--;
              value = 0;
              break;
            }
          }
        }
      }
    }

    String* bit_high(STATE, const char*& bytes, native_int count) {
      String* str = String::create(state, 0, count);
      uint8_t *buf = str->byte_address();
      int bits = 0;

      for(native_int i = 0; i < count; i++) {
        if(i & 7) {
          bits <<= 1;
        } else {
          bits = *bytes++;
        }

        buf[i] = (bits & 128) ? '1' : '0';
      }

      return str;
    }

    String* bit_low(STATE, const char*& bytes, native_int count) {
      String* str = String::create(state, 0, count);
      uint8_t *buf = str->byte_address();
      int bits = 0;

      for(native_int i = 0; i < count; i++) {
        if(i & 7) {
          bits >>= 1;
        } else {
          bits = *bytes++;
        }

        buf[i] = (bits & 1) ? '1' : '0';
      }

      return str;
    }

    static const char hexdigits[] = "0123456789abcdef0123456789ABCDEFx";

    String* hex_high(STATE, const char*& bytes, native_int count) {
      String* str = String::create(state, 0, count);
      uint8_t *buf = str->byte_address();
      int bits = 0;

      for(native_int i = 0; i < count; i++) {
        if(i & 1) {
          bits <<= 4;
        } else {
          bits = *bytes++;
        }

        buf[i] = unpack19::hexdigits[(bits >> 4) & 15];
      }

      return str;
    }

    String* hex_low(STATE, const char*& bytes, native_int count) {
      String* str = String::create(state, 0, count);
      uint8_t *buf = str->byte_address();
      int bits = 0;

      for(native_int i = 0; i < count; i++) {
        if(i & 1) {
          bits >>= 4;
        } else {
          bits = *bytes++;
        }

        buf[i] = unpack19::hexdigits[bits & 15];
      }

      return str;
    }

    ByteArray* prepare_directives(STATE, String* directives,
                                  const char** p, const char** pe)
    {
      native_int size = directives->byte_size();
      ByteArray* ba = ByteArray::create_pinned(state, size);
      char* b = reinterpret_cast<char*>(ba->raw_bytes());
      char* d = reinterpret_cast<char*>(directives->byte_address());
      int i = 0, j = 0;

      while(i < size) {
        switch(d[i]) {
        case 0:
        case ' ':
        case '\t':
        case '\n':
        case '\v':
        case '\f':
        case '\r':
          i++;
          break;
        case '#':
          while(++i < size && d[i] != '\n')
            ; // ignore
          if(d[i] == '\n') i++;
          break;
        default:
          b[j++] = d[i++];
          break;
        }
      }

      *p = const_cast<const char*>(b);
      *pe = *p + j;

      return ba;
    }
  }

#define unpack_elements(create, bits)                     \
  for(; index < stop; index += width) {                   \
    const uint8_t* bytes = self->byte_address() + index;  \
    array->append(state, create(bits(bytes)));            \
    if(count > 0) count--;                                \
  }

#define UNPACK_ELEMENTS unpack_elements
#define unpack_integer(b)         unpack_elements(new_integer, b)
#define unpack_float_elements(b)  unpack_elements(new_float, b)

#define FIXNUM(b)         (Fixnum::from(b))
#define INTEGER(b)        (Integer::from(state, b))

#define new_integer(b)    (Integer::from(state, b))
#define new_float(b)      (Float::create(state, b))

#define sbyte(p)          (*(int8_t*)(p))
#define ubyte(p)          (*(uint8_t*)(p))

#define s2bytes(p)        (*(int16_t*)(p))
#define u2bytes(p)        (*(uint16_t*)(p))

#define s4bytes(p)        (*(int32_t*)(p))
#define u4bytes(p)        (*(uint32_t*)(p))

#define s8bytes(p)        (*(int64_t*)(p))
#define u8bytes(p)        (*(uint64_t*)(p))

#define float_bits(p)     (*(float*)(p))
#define double_bits(p)    (*(double*)(p))

#ifdef RBX_LITTLE_ENDIAN

# define s2bytes_le(p)            (s2bytes(p))
# define u2bytes_le(p)            (u2bytes(p))
# define s4bytes_le(p)            (s4bytes(p))
# define u4bytes_le(p)            (u4bytes(p))
# define s8bytes_le(p)            (s8bytes(p))
# define u8bytes_le(p)            (u8bytes(p))

# define s2bytes_be(p)            ((int16_t)(unpack19::swap_2bytes(u2bytes(p))))
# define u2bytes_be(p)            ((uint16_t)(unpack19::swap_2bytes(u2bytes(p))))
# define s4bytes_be(p)            ((int32_t)(unpack19::swap_4bytes(u4bytes(p))))
# define u4bytes_be(p)            ((uint32_t)(unpack19::swap_4bytes(u4bytes(p))))
# define s8bytes_be(p)            ((int64_t)(unpack19::swap_8bytes(u8bytes(p))))
# define u8bytes_be(p)            ((uint64_t)(unpack19::swap_8bytes(u8bytes(p))))

# define unpack_double            unpack_double_le
# define unpack_float             unpack_float_le

# define unpack_double_le         unpack_float_elements(double_bits)
# define unpack_float_le          unpack_float_elements(float_bits)

# define unpack_double_be         unpack_float_elements(unpack19::swap_double)
# define unpack_float_be          unpack_float_elements(unpack19::swap_float)

#else // Big endian

# define s2bytes_le(p)            ((int16_t)(unpack19::swap_2bytes(u2bytes(p))))
# define u2bytes_le(p)            ((uint16_t)(unpack19::swap_2bytes(u2bytes(p))))
# define s4bytes_le(p)            ((int32_t)(unpack19::swap_4bytes(u4bytes(p))))
# define u4bytes_le(p)            ((uint32_t)(unpack19::swap_4bytes(u4bytes(p))))
# define s8bytes_le(p)            ((int64_t)(unpack19::swap_8bytes(u8bytes(p))))
# define u8bytes_le(p)            ((uint64_t)(unpack19::swap_8bytes(u8bytes(p))))

# define s2bytes_be(p)            (s2bytes(p))
# define u2bytes_be(p)            (u2bytes(p))
# define s4bytes_be(p)            (s4bytes(p))
# define u4bytes_be(p)            (u4bytes(p))
# define s8bytes_be(p)            (s8bytes(p))
# define u8bytes_be(p)            (u8bytes(p))

# define unpack_double            unpack_double_be
# define unpack_float             unpack_float_be

# define unpack_double_le         unpack_float_elements(unpack19::swap_double)
# define unpack_float_le          unpack_float_elements(unpack19::swap_float)

# define unpack_double_be         unpack_float_elements(double_bits)
# define unpack_float_be          unpack_float_elements(float_bits)

#endif

  Array* String::unpack19(STATE, String* directives) {
    // Ragel-specific variables
    const char* p;
    const char* pe;
    ByteArray* d = unpack19::prepare_directives(state, directives, &p, &pe);
    const char *eof = pe;
    int cs;

    // pack-specific variables
    String* self = this;
    Array* array = Array::create(state, 0);
    OnStack<3> sv(state, self, array, d);
    const char* bytes = 0;
    const char* bytes_end = 0;

    native_int bytes_size = self->byte_size();
    native_int index = 0;
    native_int stop = 0;
    native_int width = 0;
    native_int count = 0;
    native_int remainder = 0;
    bool rest = false;

%%{
  machine unpack;

  include "unpack.rl";

}%%

    if(en_main) {
      // do nothing
    }

    return force_as<Array>(Primitives::failure());
  }
}


Ragel: rename

Ragel: rename

# Parser for VCF-header info and format fields

module VcfHeader

  module RagelKeyValues

=begin
%%{

  machine simple_lexer;

  action mark { ts=p }
  action endquoted {
    quoted_text = data[ts...p].pack('c*')
    # do something with the quoted text!
    emit.call(:string,data,ts,p)
  }

  squote = "'";
  dquote = '"';
  not_squote_or_escape = [^'\\];
  not_dquote_or_escape = [^"\\];
  escaped_something = /\\./;
  ss = space* squote ( not_squote_or_escape | escaped_something )* >mark %endquoted squote;
  dd = space* dquote ( not_dquote_or_escape | escaped_something )* >mark %endquoted dquote;

  # main := (ss | dd)*;

  integer     = ('+'|'-')?[0-9]+             >mark %{ emit.call(:integer,data,ts,p)  };
  float       = ('+'|'-')?[0-9]+'.'[0-9]+    >mark %{ emit.call(:float,data,ts,p) };
  assignment  = '=';
  identifier  = ([a-zA-Z][a-zA-Z_0-9]+)         >mark %{ emit.call(:identifier,data,ts,p) };
  str         = (ss|dd)* ;
  boolean     = '.'                          >mark %{ emit.call("bool",data,ts,p) };
  key_word    = ( ('ID'|'Number'|'Type'|'Description') >mark %{ print "*k:"; emit.call("key_word",data,ts,p) } );
  value       = ( (integer|float|boolean|identifier|str) >mark %{ print "*v:" ; emit.call("value",data,ts,p) } );

  key_value = ( key_word '=' value ) ;

  main := ('##FORMAT'|'##INFO') '=' (('<'|',') key_value )*;
}%%
=end

%% write data;
# %% this just fixes our syntax highlighting...

def self.run_lexer(data)
  p data
  data = data.unpack("c*") if(data.is_a?(String))
  eof = data.length
  token_array = []
  stack = []

  emit = lambda { |type, data, ts, p|
    # Print the type and text of the last read token
    # p ts,p
    puts "#{type}: #{data[ts...p].pack('c*')}"
  }

  %% write init;
  %% write exec;

  token_array.each do | h |
    p h
    # p h[:value] if h[:name]==:identifier or h[:name]==:value or h[:name]==:string
  end
end

  end
end

if __FILE__ == $0

lines = <<LINES
##FORMAT=<ID=GT,Number=1,Type=String,Description="Genotype">
##FORMAT=<ID=DP,Number=1,Type=Integer,Description="Total read depth">
##FORMAT=<ID=DP4,Number=4,Type=Integer,Description="# high-quality ref-forward bases, ref-reverse, alt-forward and alt-reverse bases">
##INFO=<ID=PM,Number=0,Type=Flag,Description="Variant is Precious(Clinical,Pubmed Cited)">
##INFO=<ID=VP,Number=1,Type=String,Description="Variation Property.  Documentation is at ftp://ftp.ncbi.nlm.nih.gov/snp/specs/dbSNP_BitField_latest.pdf">
##INFO=<ID=GENEINFO,Number=1,Type=String,Description="Pairs each of gene symbol:gene id.  The gene symbol and id are delimited by a colon (:), and each pair is delimited by a vertical bar (|)">
##INFO=<ID=CLNHGVS,Number=.,Type=String,Description="Variant names from HGVS. The order of these variants corresponds to the order of the info in the other clinical  INFO tags.">
##INFO=<ID=CLNHGVS1,Number=.,Type=String,Description="Variant names from \\"HGVS\\". The order of these 'variants' corresponds to the order of the info in the other clinical  INFO tags.">
LINES

lines.strip.split("\n").each { |s| VcfHeader::RagelKeyValues.run_lexer(s) }

end

# Parser for VCF-header info and format fields

module VcfHeader

  module RagelKeyValues

=begin
%%{

  machine simple_lexer;

  action mark { ts=p }
  action endquoted {
    quoted_text = data[ts...p].pack('c*')
    # do something with the quoted text!
    emit.call('string',data,ts,p)
  }

  squote = "'";
  dquote = '"';
  not_squote_or_escape = [^'\\];
  not_dquote_or_escape = [^"\\];
  escaped_something = /\\./;
  ss = space* squote ( not_squote_or_escape | escaped_something )* >mark %endquoted squote;
  dd = space* dquote ( not_dquote_or_escape | escaped_something )* >mark %endquoted dquote;

  # main := (ss | dd)*;

  integer     = ('+'|'-')?[0-9]+             >mark %{ emit.call("integer",data,ts,p)  };
  float       = ('+'|'-')?[0-9]+'.'[0-9]+    >mark %{ emit.call("float",data,ts,p) };
  assignment  = '=';
  identifier  = ([a-zA-Z][a-zA-Z_0-9]+)         >mark %{ emit.call("identifier",data,ts,p) };
  str         = (ss|dd)* ;
  boolean     = '.'                          >mark %{ emit.call("bool",data,ts,p) };
  key_word    = ( ('ID'|'Number'|'Type'|'Description') >mark %{ print "*k:"; emit.call("key_word",data,ts,p) } );
  value       = ( (integer|float|boolean|identifier|str) >mark %{ print "*v:" ; emit.call("value",data,ts,p) } );

  key_value = ( key_word '=' value ) ;

  main := ('##FORMAT'|'##INFO') '=' (('<'|',') key_value )*;
}%%
=end

%% write data;
# %% this just fixes our syntax highlighting...

def self.run_lexer(data)
  p data
  data = data.unpack("c*") if(data.is_a?(String))
  eof = data.length
  token_array = []
  stack = []

  emit = lambda { |type, data, ts, p|
    # Print the type and text of the last read token
    # p ts,p
    puts "#{type}: #{data[ts...p].pack('c*')}"
  }

  %% write init;
  %% write exec;

  token_array.each do | h |
    p h
    # p h[:value] if h[:name]==:identifier or h[:name]==:value or h[:name]==:string
  end
end

  end
end

if __FILE__ == $0

lines = <<LINES
##FORMAT=<ID=GT,Number=1,Type=String,Description="Genotype">
##FORMAT=<ID=DP,Number=1,Type=Integer,Description="Total read depth">
##FORMAT=<ID=DP4,Number=4,Type=Integer,Description="# high-quality ref-forward bases, ref-reverse, alt-forward and alt-reverse bases">
##INFO=<ID=PM,Number=0,Type=Flag,Description="Variant is Precious(Clinical,Pubmed Cited)">
##INFO=<ID=VP,Number=1,Type=String,Description="Variation Property.  Documentation is at ftp://ftp.ncbi.nlm.nih.gov/snp/specs/dbSNP_BitField_latest.pdf">
##INFO=<ID=GENEINFO,Number=1,Type=String,Description="Pairs each of gene symbol:gene id.  The gene symbol and id are delimited by a colon (:), and each pair is delimited by a vertical bar (|)">
##INFO=<ID=CLNHGVS,Number=.,Type=String,Description="Variant names from HGVS. The order of these variants corresponds to the order of the info in the other clinical  INFO tags.">
##INFO=<ID=CLNHGVS1,Number=.,Type=String,Description="Variant names from \\"HGVS\\". The order of these 'variants' corresponds to the order of the info in the other clinical  INFO tags.">
LINES

lines.strip.split("\n").each { |s| VcfHeader::RagelKeyValues.run_lexer(s) }

end


handle EOF on one ident branch

handle EOF on one ident branch

# begin: ragel
=begin
%%{
  machine bel;

  include 'common.rl';

  action start_identifier {
    $stderr.puts 'IDENTIFIER start_identifier'
    @identifier_started = true
    p_start = p;
  }

  action end_identifier {
    $stderr.puts 'IDENTIFIER end_identifier'
    p_end = p
    chars = data[p_start...p_end]
    completed = !chars.empty?
    ast_node = identifier(utf8_string(chars), complete: completed)
    @buffers[:ident] = ast_node
  }

  action yield_identifier {
    $stderr.puts 'IDENTIFIER yield_identifier'
    yield @buffers[:ident]
  }

  action an_ident_eof {
    $stderr.puts 'IDENTIFIER an_ident_eof'
    if @identifier_started
      p_end = p
      chars = data[p_start...p_end]
      completed = !chars.empty?
      ast_node = identifier(utf8_string(chars), complete: completed)
      @buffers[:ident] = ast_node
    end
  }

  ID_CHARS = [a-zA-Z0-9_]+;
  ident =
    ID_CHARS
    >start_identifier
    %end_identifier
    ;

  maybe_ident =
    ident?
    ;

  an_ident =
    ident
    $eof(an_ident_eof)
    ;

  ident_node :=
    ident
    NL?
    %yield_identifier
    ;
}%%
=end
# end: ragel

require_relative '../ast/node'
require_relative '../mixin/buffer'
require_relative '../nonblocking_io_wrapper'

module BELParser
  module Parsers
    module Common
      module Identifier

        class << self

          MAX_LENGTH = 1024 * 128 # 128K

          def parse(content)
            return nil unless content

            Parser.new(content).each do |obj|
              yield obj
            end
          end
        end

        private

        class Parser
          include Enumerable
          include BELParser::Parsers::Buffer
          include BELParser::Parsers::AST::Sexp

          def initialize(content)
            @content = content
      # begin: ragel
            %% write data;
      # end: ragel
          end

          def each
            @buffers    = {}
            @incomplete = {}
            data        = @content.unpack('C*')
            p           = 0
            p_start     = 0
            p_end       = 0
            pe          = data.length
            eof         = data.length
            @identifier_started = false

      # begin: ragel
            %% write init;
            %% write exec;
      # end: ragel
          end
        end
      end
    end
  end
end

if __FILE__ == $0
  $stdin.each_line do |line|
    BELParser::Parsers::Common::Identifier.parse(line) { |obj|
      puts obj.inspect
    }
  end
end

# vim: ft=ruby ts=2 sw=2:
# encoding: utf-8

# begin: ragel
=begin
%%{
  machine bel;

  include 'common.rl';

  action start_identifier {
    $stderr.puts 'IDENTIFIER start_identifier'
    p_start = p;
  }

  action end_identifier {
    $stderr.puts 'IDENTIFIER end_identifier'
    p_end = p
    chars = data[p_start...p_end]
    completed = !chars.empty?
    ast_node = identifier(utf8_string(chars), complete: completed)
    @buffers[:ident] = ast_node
  }

  action yield_identifier {
    $stderr.puts 'IDENTIFIER yield_identifier'
    yield @buffers[:ident]
  }

  ID_CHARS = [a-zA-Z0-9_]+;
  ident =
    ID_CHARS
    >start_identifier
    %end_identifier
    ;

  maybe_ident =
    ident?
    ;

  an_ident =
    ident
    ;

  ident_node :=
    ident
    NL?
    %yield_identifier
    ;
}%%
=end
# end: ragel

require_relative '../ast/node'
require_relative '../mixin/buffer'
require_relative '../nonblocking_io_wrapper'

module BELParser
  module Parsers
    module Common
      module Identifier

        class << self

          MAX_LENGTH = 1024 * 128 # 128K

          def parse(content)
            return nil unless content

            Parser.new(content).each do |obj|
              yield obj
            end
          end
        end

        private

        class Parser
          include Enumerable
          include BELParser::Parsers::Buffer
          include BELParser::Parsers::AST::Sexp

          def initialize(content)
            @content = content
      # begin: ragel
            %% write data;
      # end: ragel
          end

          def each
            @buffers    = {}
            @incomplete = {}
            data        = @content.unpack('C*')
            p           = 0
            p_start     = 0
            p_end       = 0
            pe          = data.length
            eof         = data.length

      # begin: ragel
            %% write init;
            %% write exec;
      # end: ragel
          end
        end
      end
    end
  end
end

if __FILE__ == $0
  $stdin.each_line do |line|
    BELParser::Parsers::Common::Identifier.parse(line) { |obj|
      puts obj.inspect
    }
  end
end

# vim: ft=ruby ts=2 sw=2:
# encoding: utf-8


Cleaned up the XML lexer documentation.

Cleaned up the XML lexer documentation.

%%machine base_lexer;

%%{
    ##
    # Base grammar for the XML lexer.
    #
    # This grammar is shared between the C and Java extensions. As a result of
    # this you should **not** include language specific code in Ragel
    # actions/callbacks.
    #
    # To call back in to Ruby you can use one of the following two functions:
    #
    # * callback
    # * callback_simple
    #
    # The first function takes 5 arguments:
    #
    # * The name of the Ruby method to call.
    # * The input data.
    # * The encoding of the input data.
    # * The start of the current buffer.
    # * The end of the current buffer.
    #
    # The function callback_simple only takes one argument: the name of the
    # method to call. This function should be used for callbacks that don't
    # require any values.
    #
    # When you call a method in Ruby make sure that said method is defined as
    # an instance method in the `Oga::XML::Lexer` class.

    newline    = '\n' | '\r\n';
    whitespace = [ \t];
    identifier = [a-zA-Z0-9\-_]+;
    attribute  = [a-zA-Z0-9\-_:]+;

    cdata_start = '<![CDATA[';
    cdata_end   = ']]>';

    comment_start = '<!--';
    comment_end   = '-->';

    # Strings
    #
    # Strings in HTML can either be single or double quoted. If a string
    # starts with one of these quotes it must be closed with the same type
    # of quote.
    dquote = '"';
    squote = "'";

    # Machine for processing double quoted strings.
    string_dquote := |*
        ^dquote+ => {
            callback("on_string", data, encoding, ts, te);
        };

        dquote => { fret; };
    *|;

    # Machine for processing single quoted strings.
    string_squote := |*
        ^squote+ => {
            callback("on_string", data, encoding, ts, te);
        };

        squote => { fret; };
    *|;

    # DOCTYPES
    #
    # http://www.w3.org/TR/html-markup/syntax.html#doctype-syntax
    #
    # These rules support the 3 flavours of doctypes:
    #
    # 1. Normal doctypes, as introduced in the HTML5 specification.
    # 2. Deprecated doctypes, the more verbose ones used prior to HTML5.
    # 3. Legacy doctypes
    #
    doctype_start = '<!DOCTYPE'i whitespace+;

    action start_doctype {
        callback_simple("on_doctype_start");
        fcall doctype;
    }

    # Machine for processing doctypes. Doctype values such as the public
    # and system IDs are treated as T_STRING tokens.
    doctype := |*
        'PUBLIC' | 'SYSTEM' => {
            callback("on_doctype_type", data, encoding, ts, te);
        };

        # Consumes everything between the [ and ]. Due to the use of :> the ]
        # is not consumed by any+.
        '[' any+ :> ']' => {
            callback("on_doctype_inline", data, encoding, ts + 1, te - 1);
        };

        # Lex the public/system IDs as regular strings.
        dquote => { fcall string_dquote; };
        squote => { fcall string_squote; };

        # Whitespace inside doctypes is ignored since there's no point in
        # including it.
        whitespace;

        identifier => {
            callback("on_doctype_name", data, encoding, ts, te);
        };

        '>' => {
            callback_simple("on_doctype_end");
            fret;
        };
    *|;

    # XML declaration tags
    #
    # http://www.w3.org/TR/REC-xml/#sec-prolog-dtd
    #
    xml_decl_start = '<?xml';
    xml_decl_end   = '?>';

    action start_xml_decl {
        callback_simple("on_xml_decl_start");
        fcall xml_decl;
    }

    # Machine that processes the contents of an XML declaration tag.
    xml_decl := |*
        xml_decl_end => {
            callback_simple("on_xml_decl_end");
            fret;
        };

        # Attributes and their values (e.g. version="1.0").
        attribute => {
            callback("on_attribute", data, encoding, ts, te);
        };

        dquote => { fcall string_dquote; };
        squote => { fcall string_squote; };

        any;
    *|;

    # Elements
    #
    # http://www.w3.org/TR/html-markup/syntax.html#syntax-elements
    #

    # Action that creates the tokens for the opening tag, name and
    # namespace (if any). Remaining work is delegated to a dedicated
    # machine.
    action start_element {
        fhold;
        fcall element_head;
    }

    # Machine used for lexing the name/namespace of an element.
    element_name := |*
        identifier ':' => {
            callback("on_element_ns", data, encoding, ts, te - 1);
        };

        identifier => {
            callback("on_element_name", data, encoding, ts, te);
            fret;
        };
    *|;

    # Machine used for processing the characters inside a element head. An
    # element head is everything between `<NAME` (where NAME is the element
    # name) and `>`.
    #
    # For example, in `<p foo="bar">` the element head is ` foo="bar"`.
    #
    element_head := |*
        whitespace | '=';

        '<' => {
            callback_simple("on_element_start");
            fcall element_name;
        };

        newline => {
            callback_simple("on_newline");
        };

        # Attribute names.
        attribute => {
            callback("on_attribute", data, encoding, ts, te);
        };

        # Attribute values.
        dquote => { fcall string_dquote; };
        squote => { fcall string_squote; };

        # The closing character of the open tag.
        ('>' | '/') => {
            fhold;
            fret;
        };
    *|;

    main := |*
        '<'            => start_element;
        doctype_start  => start_doctype;
        xml_decl_start => start_xml_decl;

        # Comments
        #
        # http://www.w3.org/TR/html-markup/syntax.html#comments
        #
        # Unlike the W3 specification these rules *do* allow character
        # sequences such as `--` and `->`. Putting extra checks in for these
        # sequences would actually make the rules/actions more complex.
        #
        comment_start any* comment_end => {
            callback("on_comment", data, encoding, ts + 4, te - 3);
        };

        # CDATA
        #
        # http://www.w3.org/TR/html-markup/syntax.html#cdata-sections
        #
        # In HTML CDATA tags have no meaning/are not supported. Oga does
        # support them but treats their contents as plain text.
        #
        cdata_start any* cdata_end => {
            callback("on_cdata", data, encoding, ts + 9, te - 3);
        };

        # Enter the body of the tag. If HTML mode is enabled and the current
        # element is a void element we'll close it and bail out.
        '>' => {
            callback_simple("on_element_open_end");
        };

        # Regular closing tags.
        '</' identifier '>' => {
            callback_simple("on_element_end");
        };

        # Self closing elements that are not handled by the HTML mode.
        '/>' => {
            callback_simple("on_element_end");
        };

        # Note that this rule should be declared at the very bottom as it
        # will otherwise take precedence over the other rules.
        ^('<' | '>')+ => {
            callback("on_text", data, encoding, ts, te);
        };
    *|;
}%%

%%machine base_lexer;

%%{
    ##
    # Base grammar for the XML lexer.
    #
    # This grammar is shared between the C and Java extensions. As a result of
    # this you should **not** include language specific code in Ragel
    # actions/callbacks.
    #
    # To call back in to Ruby you can use one of the following two functions:
    #
    # * callback
    # * callback_simple
    #
    # The first function takes 5 arguments:
    #
    # * The name of the Ruby method to call.
    # * The input data.
    # * The encoding of the input data.
    # * The start of the current buffer.
    # * The end of the current buffer.
    #
    # The function callback_simple only takes one argument: the name of the
    # method to call. This function should be used for callbacks that don't
    # require any values.
    #
    # When you call a method in Ruby make sure that said method is defined as
    # an instance method in the `Oga::XML::Lexer` class.

    newline    = '\n' | '\r\n';
    whitespace = [ \t];
    identifier = [a-zA-Z0-9\-_]+;
    attribute  = [a-zA-Z0-9\-_:]+;

    # Strings
    #
    # Strings in HTML can either be single or double quoted. If a string
    # starts with one of these quotes it must be closed with the same type
    # of quote.
    dquote = '"';
    squote = "'";

    # Machine for processing double quoted strings.
    string_dquote := |*
        ^dquote+ => {
            callback("on_string", data, encoding, ts, te);
        };

        dquote => { fret; };
    *|;

    # Machine for processing single quoted strings.
    string_squote := |*
        ^squote+ => {
            callback("on_string", data, encoding, ts, te);
        };

        squote => { fret; };
    *|;

    # DOCTYPES
    #
    # http://www.w3.org/TR/html-markup/syntax.html#doctype-syntax
    #
    # These rules support the 3 flavours of doctypes:
    #
    # 1. Normal doctypes, as introduced in the HTML5 specification.
    # 2. Deprecated doctypes, the more verbose ones used prior to HTML5.
    # 3. Legacy doctypes
    #
    doctype_start = '<!DOCTYPE'i whitespace+;

    action start_doctype {
        callback_simple("on_doctype_start");
        fcall doctype;
    }

    # Machine for processing doctypes. Doctype values such as the public
    # and system IDs are treated as T_STRING tokens.
    doctype := |*
        'PUBLIC' | 'SYSTEM' => {
            callback("on_doctype_type", data, encoding, ts, te);
        };

        # Consumes everything between the [ and ]. Due to the use of :> the ]
        # is not consumed by any+.
        '[' any+ :> ']' => {
            callback("on_doctype_inline", data, encoding, ts + 1, te - 1);
        };

        # Lex the public/system IDs as regular strings.
        dquote => { fcall string_dquote; };
        squote => { fcall string_squote; };

        # Whitespace inside doctypes is ignored since there's no point in
        # including it.
        whitespace;

        identifier => {
            callback("on_doctype_name", data, encoding, ts, te);
        };

        '>' => {
            callback_simple("on_doctype_end");
            fret;
        };
    *|;

    # CDATA
    #
    # http://www.w3.org/TR/html-markup/syntax.html#cdata-sections
    #
    # CDATA tags are broken up into 3 parts: the start, the content and the
    # end tag.
    #
    # In HTML CDATA tags have no meaning/are not supported. Oga does
    # support them but treats their contents as plain text.
    #
    cdata_start = '<![CDATA[';
    cdata_end   = ']]>';

    # Comments
    #
    # http://www.w3.org/TR/html-markup/syntax.html#comments
    #
    # Comments are lexed into 3 parts: the start tag, the content and the
    # end tag.
    #
    # Unlike the W3 specification these rules *do* allow character
    # sequences such as `--` and `->`. Putting extra checks in for these
    # sequences would actually make the rules/actions more complex.
    #
    comment_start = '<!--';
    comment_end   = '-->';

    # XML declaration tags
    #
    # http://www.w3.org/TR/REC-xml/#sec-prolog-dtd
    #
    xml_decl_start = '<?xml';
    xml_decl_end   = '?>';

    action start_xml_decl {
        callback_simple("on_xml_decl_start");
        fcall xml_decl;
    }

    # Machine that processes the contents of an XML declaration tag.
    xml_decl := |*
        xml_decl_end => {
            callback_simple("on_xml_decl_end");
            fret;
        };

        # Attributes and their values (e.g. version="1.0").
        attribute => {
            callback("on_attribute", data, encoding, ts, te);
        };

        dquote => { fcall string_dquote; };
        squote => { fcall string_squote; };

        any;
    *|;

    # Elements
    #
    # http://www.w3.org/TR/html-markup/syntax.html#syntax-elements
    #

    # Action that creates the tokens for the opening tag, name and
    # namespace (if any). Remaining work is delegated to a dedicated
    # machine.
    action start_element {
        fhold;
        fcall element_head;
    }

    # Machine used for lexing the name/namespace of an element.
    element_name := |*
        identifier ':' => {
            callback("on_element_ns", data, encoding, ts, te - 1);
        };

        identifier => {
            callback("on_element_name", data, encoding, ts, te);
            fret;
        };
    *|;

    # Machine used for processing the characters inside a element head. An
    # element head is everything between `<NAME` (where NAME is the element
    # name) and `>`.
    #
    # For example, in `<p foo="bar">` the element head is ` foo="bar"`.
    #
    element_head := |*
        whitespace | '=';

        '<' => {
            callback_simple("on_element_start");
            fcall element_name;
        };

        newline => {
            callback_simple("on_newline");
        };

        # Attribute names.
        attribute => {
            callback("on_attribute", data, encoding, ts, te);
        };

        # Attribute values.
        dquote => { fcall string_dquote; };
        squote => { fcall string_squote; };

        # The closing character of the open tag.
        ('>' | '/') => {
            fhold;
            fret;
        };
    *|;

    main := |*
        '<'            => start_element;
        doctype_start  => start_doctype;
        cdata_start    => start_cdata;
        xml_decl_start => start_xml_decl;

        comment_start any* comment_end => {
            callback("on_comment", data, encoding, ts + 4, te - 3);
        };

        cdata_start any* cdata_end => {
            callback("on_cdata", data, encoding, ts + 9, te - 3);
        };

        # Enter the body of the tag. If HTML mode is enabled and the current
        # element is a void element we'll close it and bail out.
        '>' => {
            callback_simple("on_element_open_end");
        };

        # Regular closing tags.
        '</' identifier '>' => {
            callback_simple("on_element_end");
        };

        # Self closing elements that are not handled by the HTML mode.
        '/>' => {
            callback_simple("on_element_end");
        };

        # Note that this rule should be declared at the very bottom as it
        # will otherwise take precedence over the other rules.
        ^('<' | '>')+ => {
            callback("on_text", data, encoding, ts, te);
        };
    *|;
}%%