Fix stos special instruction handling

Fix stos special instruction handling

/*
 * Copyright (c) 2011 The Native Client Authors. All rights reserved.
 * Use of this source code is governed by a BSD-style license that can be
 * found in the LICENSE file.
 */

#include <assert.h>
#include <elf.h>
#include <inttypes.h>
#include <stddef.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include "validator-x86_64.h"

#undef TRUE
#define TRUE    1

#undef FALSE
#define FALSE   0

#include "validator-x86_64-instruction-consts.c"

#define check_jump_dest \
    if ((jump_dest & bundle_mask) != bundle_mask) { \
      if (jump_dest >= size) { \
        printf("direct jump out of range: %zx\n", jump_dest); \
        result = 1; \
        goto error_detected; \
      } else { \
        BitmapSetBit(jump_dests, jump_dest + 1); \
      } \
    } \
    operand0 = JMP_TO; \
    base = REG_RIP; \
    index = REG_NONE;

%%{
  machine x86_64_decoder;
  alphtype unsigned char;

  action check_access {
    if ((base == REG_RIP) || (base == REG_R15) ||
	((base == REG_RSP) && (restricted_register != REG_RSP)) ||
	((base == REG_RBP) && (restricted_register != REG_RBP))) {
      if ((index == restricted_register) ||
	  ((index == REG_RDI) &&
	   (restricted_register == kSandboxedRsiRestrictedRdi))) {
	BitmapClearBit(valid_targets, begin - data);
      } else if ((index != REG_NONE) && (index != REG_RIZ)) {
	fprintf(stderr,"Improper sandboxing in instruction %zx", begin - data);
	exit(1);
      }
    } else if ((index == REG_RIP) || (index == REG_R15) ||
	       ((index == REG_RSP) && (restricted_register != REG_RSP)) ||
	       ((index == REG_RBP) && (restricted_register != REG_RBP))) {
      if ((base == restricted_register) ||
	  ((base == REG_RDI) &&
	   (restricted_register == kSandboxedRsiRestrictedRdi))) {
	BitmapClearBit(valid_targets, begin - data);
      } else if ((base != REG_NONE) && (base != REG_RIZ)) {
	fprintf(stderr,"Improper sandboxing in instruction @%zx", begin - data);
	exit(1);
      }
    } else {
      fprintf(stderr,"Improper sandboxing in instruction @%zx", begin - data);
    }
  }

  action rel8_operand {
    int8_t offset = (uint8_t) (p[0]);
    size_t jump_dest = offset + (p - data);
    check_jump_dest;
  }
  action rel16_operand {
    assert(FALSE);
  }
  action rel32_operand {
    int32_t offset =
	   (uint32_t) (p[-3] + 256U * (p[-2] + 256U * (p[-1] + 256U * (p[0]))));
    size_t jump_dest = offset + (p - data);
    check_jump_dest;
  }

  include decode_x86_64 "validator-x86_64-instruction.rl";

  action process_normal_instruction {
    /* Restricted %rsp or %rbp must be processed by appropriate nacl-special
       instruction, not with regular instruction.  */
    if (restricted_register == REG_RSP) {
      printf("Incorrectly modified register %%rsp at the %zx\n", p - data);
      exit(1);
    } else if (restricted_register == REG_RBP) {
      printf("Incorrectly modified register %%rbp at the %zx\n", p - data);
      exit(1);
    }
    /* If Sandboxed Rsi is destroyed then we must note that.  */
    if (restricted_register == kSandboxedRsi) {
      for (i = 0; i < operands_count; ++i) {
	if ((operands[i].write == TRUE) &&
	    (operands[i].name == REG_RSI) &&
	    ((operands[i].type == OperandSandboxRestricted) ||
	     (operands[i].type == OperandSandboxUnrestricted))) {
	  restricted_register = kNoRestrictedReg;
	  break;
	}
      }
    }
    if (restricted_register == kSandboxedRsi) {
      for (i = 0; i < operands_count; ++i) {
	if ((operands[i].write == TRUE) &&
	    (operands[i].name == REG_RDI) &&
	    (operands[i].type == OperandSandboxRestricted)) {
	  sandboxed_rsi_restricted_rdi = begin;
	  restricted_register = kSandboxedRsiRestrictedRdi;
	}
      }
    }
    if (restricted_register != kSandboxedRsiRestrictedRdi) {
      restricted_register = kNoRestrictedReg;
      for (i = 0; i < operands_count; ++i) {
	if (operands[i].write && operands[i].name <= REG_R15) {
	  if (operands[i].type == OperandSandboxRestricted) {
	    if (operands[i].name == REG_R15) {
	      printf("Incorrectly modified register %%r15 at the %zx\n",
								      p - data);
	      exit(1);
	    } else {
	      restricted_register = operands[i].name;
	    }
	  } else if (operands[i].type == OperandSandboxUnrestricted) {
	    if (operands[i].name == REG_RBP) {
	      printf("Incorrectly modified register %%rbp at the %zx\n",
								      p - data);
	      exit(1);
	    } else if (operands[i].name == REG_RSP) {
	      printf("Incorrectly modified register %%rsp at the %zx\n",
								      p - data);
	      exit(1);
	    } else if (operands[i].name == REG_R15) {
	      printf("Incorrectly modified register %%r15 at the %zx\n",
								      p - data);
	      exit(1);
	    }
	  } else if (operands[i].type == OperandNoSandboxEffect) {
	    if (operands[i].name == REG_R15) {
	      printf("Incorrectly modified register %%r15 at the %zx\n",
								      p - data);
	      exit(1);
	    }
	  }
        }
      }
    }
  }

  # Remove special instructions which are only allowed in special cases.
  normal_instruction = (one_instruction - (
    (0x48 0x89 0xe5)		 | # mov %rsp,%rbp
    (0x48 0x89 0xec)		 | # mov %rbp,%rsp
    (0x48 0x81 0xe4 any{4})	 | # and $XXX,%rsp
    (0x48 0x83 0xe4 any)	 | # and $XXX,%rsp
    (0x4c 0x01 0xfd)		 | # add %r15,%rbp
    (0x49 0x8d 0x2c 0x2f)	 | # lea (%r15,%rbp,1),%rbp
    (0x4a 0x8d 0x6c 0x3d any)	 | # lea 0x0(%rbp,%r15,1),%rbp
    (0x48 0x81 0xe5 any{4})	 | # and $XXX,%rsp
    (0x48 0x83 0xe5 any)	 | # and $XXX,%rsp
    (0x4c 0x01 0xfc)		 | # add %r15,%rsp
    (0x4a 0x8d 0x24 0x3c)	 | # lea (%rsp,%r15,1),%rsp
    (0x49 0x8d 0x34 0x37)	 | # lea (%r15,%rsi,1),%rsi
    (0x49 0x8d 0x3c 0x3f)	   # lea (%r15,%rdi,1),%rdi
  )) @process_normal_instruction;

  special_instruction =
    (0x48 0x89 0xe5)			   | # mov %rsp,%rbp
    (0x48 0x81 0xe4 any{3} (0x80 .. 0xff)) | # and $XXX,%rsp
    (0x48 0x83 0xe4 (0x80 .. 0xff))	     # and $XXX,%rsp
    @{ if (restricted_register == REG_RSP) {
	 printf("Incorrectly modified register %%rsp at the %zx\n", p - data);
	 exit(1);
       }
       restricted_register = kNoRestrictedReg;
    } |
    (0x48 0x89 0xec)			   | # mov %rbp,%rsp
    (0x48 0x81 0xe5 any{3} (0x80 .. 0xff)) | # and $XXX,%rsp
    (0x48 0x83 0xe5 (0x80 .. 0xff))	     # and $XXX,%rsp
    @{ if (restricted_register == REG_RBP) {
	 printf("Incorrectly modified register %%rbp at the %zx\n", p - data);
	 exit(1);
       }
       restricted_register = kNoRestrictedReg;
    } |
    (0x4c 0x01 0xfd	       | # add %r15,%rbp
     0x49 0x8d 0x2c 0x2f       | # lea (%r15,%rbp,1),%rbp
     0x4a 0x8d 0x6c 0x3d 0x00)	 # lea 0x0(%rbp,%r15,1),%rbp
    @{ if (restricted_register != REG_RBP) {
	 printf("Incorrectly sandboxed %%rbp at the %zx\n", p - data);
	 exit(1);
       }
       restricted_register = kNoRestrictedReg;
       BitmapClearBit(valid_targets, (begin - data));
    } |
    (0x4c 0x01 0xfc	  | # add %r15,%rsp
     0x4a 0x8d 0x24 0x3c)   # lea (%rsp,%r15,1),%rsp
    @{ if (restricted_register != REG_RSP) {
	 printf("Incorrectly sandboxed %%rsp at the %zx\n", p - data);
	 exit(1);
       }
       restricted_register = kNoRestrictedReg;
       BitmapClearBit(valid_targets, (begin - data));
    } |
    (0x83 0xe0 0xe0 0x4c 0x01 0xf8 0xff (0xd0|0xe0) | # naclcall/jmp %eax, %r15
     0x83 0xe1 0xe0 0x4c 0x01 0xf9 0xff (0xd1|0xe1) | # naclcall/jmp %ecx, %r15
     0x83 0xe2 0xe0 0x4c 0x01 0xfa 0xff (0xd2|0xe2) | # naclcall/jmp %edx, %r15
     0x83 0xe3 0xe0 0x4c 0x01 0xfb 0xff (0xd3|0xe3) | # naclcall/jmp %ebx, %r15
     0x83 0xe4 0xe0 0x4c 0x01 0xfc 0xff (0xd4|0xe4) | # naclcall/jmp %esp, %r15
     0x83 0xe5 0xe0 0x4c 0x01 0xfd 0xff (0xd5|0xe5) | # naclcall/jmp %ebp, %r15
     0x83 0xe6 0xe0 0x4c 0x01 0xfe 0xff (0xd6|0xe6) | # naclcall/jmp %esi, %r15
     0x83 0xe7 0xe0 0x4c 0x01 0xff 0xff (0xd7|0xe7))  # naclcall/jmp %edi, %r15
    @{ if (restricted_register == REG_RSP) {
	 printf("Incorrectly modified register %%rsp at the %zx\n", p - data);
	 exit(1);
       } else if (restricted_register == REG_RBP) {
	 printf("Incorrectly modified register %%rbp at the %zx\n", p - data);
	 exit(1);
       }
       BitmapClearBit(valid_targets, (p - data) - 4);
       BitmapClearBit(valid_targets, (p - data) - 1);
       restricted_register = kNoRestrictedReg;
    } |
    (0x41 0x83 0xe0 0xe0 0x4d 0x01 0xf8 0x41 0xff (0xd0|0xe0) | # naclcall/jmp
     0x41 0x83 0xe1 0xe0 0x4d 0x01 0xf9 0x41 0xff (0xd1|0xe1) | #    %r8d, %r15
     0x41 0x83 0xe2 0xe0 0x4d 0x01 0xfa 0x41 0xff (0xd2|0xe2) | # ⋮
     0x41 0x83 0xe3 0xe0 0x4d 0x01 0xfb 0x41 0xff (0xd3|0xe3) | # ⋮
     0x41 0x83 0xe4 0xe0 0x4d 0x01 0xfc 0x41 0xff (0xd4|0xe4) | # ⋮
     0x41 0x83 0xe5 0xe0 0x4d 0x01 0xfd 0x41 0xff (0xd5|0xe5) | # naclcall/jmp
     0x41 0x83 0xe6 0xe0 0x4d 0x01 0xfe 0x41 0xff (0xd6|0xe6))  #   %r14d, %r15
    @{ if (restricted_register == REG_RSP) {
	 printf("Incorrectly modified register %%rsp at the %zx\n", p - data);
	 exit(1);
       } else if (restricted_register == REG_RBP) {
	 printf("Incorrectly modified register %%rbp at the %zx\n", p - data);
	 exit(1);
       }
       BitmapClearBit(valid_targets, (p - data) - 5);
       BitmapClearBit(valid_targets, (p - data) - 2);
       restricted_register = kNoRestrictedReg;
    } |
    (0x49 0x8d 0x34 0x37) # lea (%r15,%rsi,1),%rsi
    @{ if (restricted_register == REG_RSI) {
	 sandboxed_rsi = begin;
	 restricted_register = kSandboxedRsi;
       } else {
	 restricted_register = kNoRestrictedReg;
       }
    } |
    (0x49 0x8d 0x3c 0x3f) # lea (%r15,%rdi,1),%rdi
    @{ if (restricted_register == REG_RDI) {
	 sandboxed_rdi = begin;
	 restricted_register = kSandboxedRdi;
       } else if (restricted_register == kSandboxedRsiRestrictedRdi) {
	 sandboxed_rdi = begin;
	 restricted_register = kSandboxedRsiSandboxedRdi;
       } else {
         restricted_register = kNoRestrictedReg;
       }
    } |
    (0xac		       | # lods   %ds:(%rsi),%al
     (data16|REXW_NONE)? 0xad)	 # lods   %ds:(%rsi),%ax/%eax/%rax
    @{ if (restricted_register != kSandboxedRsi) {
	 printf("Incorrectly sandboxed %%rdi at the %zx\n", p - data);
	 exit(1);
       }
       restricted_register = kNoRestrictedReg;
       BitmapClearBit(valid_targets, (begin - data));
       BitmapClearBit(valid_targets, (sandboxed_rdi - data));
    } |
    (0xae		       | # scas   %ds:(%rsi),%al
     (data16|REXW_NONE)? 0xaf  | # scas   %ds:(%rsi),%ax/%eax/%rax
     (rep? 0xaa)	       | # stos   %al,%es:(%rdi)
     (data16 |
      rep data16 |
      data16 rep) 0xab	       | # stos   %ax,%es:(%rdi)
      rep? REXW_NONE? 0xab)	 # stos   %eax/%rax,%es:(%rdi)
    @{ if (restricted_register != kSandboxedRdi &&
	   restricted_register != kSandboxedRsiSandboxedRdi) {
	 printf("Incorrectly sandboxed %%rdi at the %zx\n", p - data);
	 exit(1);
       }
       restricted_register = kNoRestrictedReg;
       BitmapClearBit(valid_targets, (begin - data));
       BitmapClearBit(valid_targets, (sandboxed_rdi - data));
    } |
    (condrep? 0xa6	      | # cmpsb	   %es:(%rdi),%ds:(%rsi)
     (data16 |
      condrep data16 |
      data16 condrep) 0xa7    | # cmpsw	   %es:(%rdi),%ds:(%rsi)
     condrep? REXW_NONE? 0xa7 | # cmps[lq] %es:(%rdi),%ds:(%rsi)
     rep? 0xa4		      | # movsb	   %es:(%rdi),%ds:(%rsi)
     (data16 |
      rep data16 |
      data16 rep) 0xa5	      | # movsw	   %es:(%rdi),%ds:(%rsi)
     rep? REXW_NONE? 0xa5)	# movs[lq] %es:(%rdi),%ds:(%rsi)
    @{ if (restricted_register != kSandboxedRsiSandboxedRdi) {
	 printf("Incorrectly sandboxed %%rsi or %%rdi at the %zx\n", p - data);
	 exit(1);
       }
       restricted_register = kNoRestrictedReg;
       BitmapClearBit(valid_targets, (begin - data));
       BitmapClearBit(valid_targets, (sandboxed_rsi - data));
       BitmapClearBit(valid_targets, (sandboxed_rsi_restricted_rdi - data));
       BitmapClearBit(valid_targets, (sandboxed_rdi - data));
    };

  main := ((normal_instruction | special_instruction) >{
	begin = p;
	BitmapSetBit(valid_targets, p - data);
	rex_prefix = FALSE;
	vex_prefix2 = 0xe0;
	vex_prefix3 = 0x00;
     })*
    $!{ process_error(p, userdata);
	result = 1;
	goto error_detected;
    };

}%%

%% write data;

/* Ignore this information for now.  */
#define data16_prefix if (0) result
#define lock_prefix if (0) result
#define repz_prefix if (0) result
#define repnz_prefix if (0) result
#define branch_not_taken if (0) result
#define branch_taken if (0) result
#define imm if (0) begin
#define imm_operand if (0) result
#define imm2 if (0) begin
#define imm2_operand if (0) result
#define scale if (0) result
#define disp if (0) begin
#define disp_type if (0) result
#define operand0 operands[0].name
#define operand1 operands[1].name
#define operand2 operands[2].name
#define operand3 operands[3].name
#define operand4 operands[4].name
#define operand0_type operands[0].type
#define operand1_type operands[1].type
#define operand2_type operands[2].type
#define operand3_type operands[3].type
#define operand4_type operands[4].type
/* It's not important for us.  */
#define operand0_read if (0) result
#define operand1_read if (0) result
#define operand2_read if (0) result
#define operand3_read if (0) result
#define operand4_read if (0) result
#define operand0_write operands[0].write
#define operand1_write operands[1].write
#define operand2_write operands[2].write
#define operand3_write operands[3].write
#define operand4_write operands[4].write

enum {
  REX_B = 1,
  REX_X = 2,
  REX_R = 4,
  REX_W = 8
};

enum disp_mode {
  DISPNONE,
  DISP8,
  DISP16,
  DISP32,
  DISP64,
};

enum imm_mode {
  IMMNONE,
  IMM2,
  IMM8,
  IMM16,
  IMM32,
  IMM64
};

static const uint8_t one = 1;

static const int kBitsPerByte = 8;

static inline uint8_t *BitmapAllocate(uint32_t indexes) {
  uint32_t byte_count = (indexes + kBitsPerByte - 1) / kBitsPerByte;
  uint8_t *bitmap = malloc(byte_count);
  if (bitmap != NULL) {
    memset(bitmap, 0, byte_count);
  }
  return bitmap;
}

static inline int BitmapIsBitSet(uint8_t *bitmap, uint32_t index) {
  return (bitmap[index / kBitsPerByte] & (1 << (index % kBitsPerByte))) != 0;
}

static inline void BitmapSetBit(uint8_t *bitmap, uint32_t index) {
  bitmap[index / kBitsPerByte] |= 1 << (index % kBitsPerByte);
}

static inline void BitmapClearBit(uint8_t *bitmap, uint32_t index) {
  bitmap[index / kBitsPerByte] &= ~(1 << (index % kBitsPerByte));
}

int CheckJumpTargets(uint8_t *valid_targets, uint8_t *jump_dests,
                     size_t size) {
  size_t i;
  for (i = 0; i < size / 32; i++) {
    uint32_t jump_dest_mask = ((uint32_t *) jump_dests)[i];
    uint32_t valid_target_mask = ((uint32_t *) valid_targets)[i];
    if ((jump_dest_mask & ~valid_target_mask) != 0) {
      printf("bad jump to around %x\n", (unsigned)(i * 32));
      return 1;
    }
  }
  return 0;
}

int ValidateChunk(const uint8_t *data, size_t size,
		  process_error_func process_error, void *userdata) {
  const size_t bundle_size = 32;
  const size_t bundle_mask = bundle_size - 1;

  uint8_t *valid_targets = BitmapAllocate(size);
  uint8_t *jump_dests = BitmapAllocate(size);

  const uint8_t *p = data;
  const uint8_t *begin;

  uint8_t rex_prefix, vex_prefix2, vex_prefix3;
  struct Operand {
    unsigned int name	:5;
    unsigned int type	:2;
    bool	 write	:1;
  } operands[5];
  enum register_name base, index;
  uint8_t operands_count, i;
  int result = 0;
  /*
   * These are borders of the appropriate instructions.  Initialize them to make
   * compiler happy: they are never used uninitialized even without explicit
   * initialization but GCC is not sophysicated enough to prove that.
   */
  const uint8_t *sandboxed_rsi = 0;
  const uint8_t *sandboxed_rsi_restricted_rdi = 0;
  const uint8_t *sandboxed_rdi = 0;

  assert(size % bundle_size == 0);

  while (p < data + size) {
    const uint8_t *pe = p + bundle_size;
    const uint8_t *eof = pe;
    int cs;
    enum {
      kNoRestrictedReg = 32,
      kSandboxedRsi,
      kSandboxedRdi,
      kSandboxedRsiRestrictedRdi,
      kSandboxedRsiSandboxedRdi
    }; uint8_t restricted_register = kNoRestrictedReg;

    %% write init;
    %% write exec;

    if (restricted_register == REG_RBP) {
      printf("Incorrectly sandboxed %%rbp at the %d%zx\n", *data, p - data);
      exit(1);
    } else if (restricted_register == REG_RSP) {
      printf("Incorrectly sandboxed %%rbp at the %zx\n", p - data);
      exit(1);
    }
  }

  if (CheckJumpTargets(valid_targets, jump_dests, size)) {
    result = 1;
    goto error_detected;
  }

error_detected:
  return result;
}

/*
 * Copyright (c) 2011 The Native Client Authors. All rights reserved.
 * Use of this source code is governed by a BSD-style license that can be
 * found in the LICENSE file.
 */

#include <assert.h>
#include <elf.h>
#include <inttypes.h>
#include <stddef.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include "validator-x86_64.h"

#undef TRUE
#define TRUE    1

#undef FALSE
#define FALSE   0

#include "validator-x86_64-instruction-consts.c"

#define check_jump_dest \
    if ((jump_dest & bundle_mask) != bundle_mask) { \
      if (jump_dest >= size) { \
        printf("direct jump out of range: %zx\n", jump_dest); \
        result = 1; \
        goto error_detected; \
      } else { \
        BitmapSetBit(jump_dests, jump_dest + 1); \
      } \
    } \
    operand0 = JMP_TO; \
    base = REG_RIP; \
    index = REG_NONE;

%%{
  machine x86_64_decoder;
  alphtype unsigned char;

  action check_access {
    if ((base == REG_RIP) || (base == REG_R15) ||
	((base == REG_RSP) && (restricted_register != REG_RSP)) ||
	((base == REG_RBP) && (restricted_register != REG_RBP))) {
      if ((index == restricted_register) ||
	  ((index == REG_RDI) &&
	   (restricted_register == kSandboxedRsiRestrictedRdi))) {
	BitmapClearBit(valid_targets, begin - data);
      } else if ((index != REG_NONE) && (index != REG_RIZ)) {
	fprintf(stderr,"Improper sandboxing in instruction %zx", begin - data);
	exit(1);
      }
    } else if ((index == REG_RIP) || (index == REG_R15) ||
	       ((index == REG_RSP) && (restricted_register != REG_RSP)) ||
	       ((index == REG_RBP) && (restricted_register != REG_RBP))) {
      if ((base == restricted_register) ||
	  ((base == REG_RDI) &&
	   (restricted_register == kSandboxedRsiRestrictedRdi))) {
	BitmapClearBit(valid_targets, begin - data);
      } else if ((base != REG_NONE) && (base != REG_RIZ)) {
	fprintf(stderr,"Improper sandboxing in instruction @%zx", begin - data);
	exit(1);
      }
    } else {
      fprintf(stderr,"Improper sandboxing in instruction @%zx", begin - data);
    }
  }

  action rel8_operand {
    int8_t offset = (uint8_t) (p[0]);
    size_t jump_dest = offset + (p - data);
    check_jump_dest;
  }
  action rel16_operand {
    assert(FALSE);
  }
  action rel32_operand {
    int32_t offset =
	   (uint32_t) (p[-3] + 256U * (p[-2] + 256U * (p[-1] + 256U * (p[0]))));
    size_t jump_dest = offset + (p - data);
    check_jump_dest;
  }

  include decode_x86_64 "validator-x86_64-instruction.rl";

  action process_normal_instruction {
    /* Restricted %rsp or %rbp must be processed by appropriate nacl-special
       instruction, not with regular instruction.  */
    if (restricted_register == REG_RSP) {
      printf("Incorrectly modified register %%rsp at the %zx\n", p - data);
      exit(1);
    } else if (restricted_register == REG_RBP) {
      printf("Incorrectly modified register %%rbp at the %zx\n", p - data);
      exit(1);
    }
    /* If Sandboxed Rsi is destroyed then we must note that.  */
    if (restricted_register == kSandboxedRsi) {
      for (i = 0; i < operands_count; ++i) {
	if ((operands[i].write == TRUE) &&
	    (operands[i].name == REG_RSI) &&
	    ((operands[i].type == OperandSandboxRestricted) ||
	     (operands[i].type == OperandSandboxUnrestricted))) {
	  restricted_register = kNoRestrictedReg;
	  break;
	}
      }
    }
    if (restricted_register == kSandboxedRsi) {
      for (i = 0; i < operands_count; ++i) {
	if ((operands[i].write == TRUE) &&
	    (operands[i].name == REG_RDI) &&
	    (operands[i].type == OperandSandboxRestricted)) {
	  sandboxed_rsi_restricted_rdi = begin;
	  restricted_register = kSandboxedRsiRestrictedRdi;
	}
      }
    }
    if (restricted_register != kSandboxedRsiRestrictedRdi) {
      restricted_register = kNoRestrictedReg;
      for (i = 0; i < operands_count; ++i) {
	if (operands[i].write && operands[i].name <= REG_R15) {
	  if (operands[i].type == OperandSandboxRestricted) {
	    if (operands[i].name == REG_R15) {
	      printf("Incorrectly modified register %%r15 at the %zx\n",
								      p - data);
	      exit(1);
	    } else {
	      restricted_register = operands[i].name;
	    }
	  } else if (operands[i].type == OperandSandboxUnrestricted) {
	    if (operands[i].name == REG_RBP) {
	      printf("Incorrectly modified register %%rbp at the %zx\n",
								      p - data);
	      exit(1);
	    } else if (operands[i].name == REG_RSP) {
	      printf("Incorrectly modified register %%rsp at the %zx\n",
								      p - data);
	      exit(1);
	    } else if (operands[i].name == REG_R15) {
	      printf("Incorrectly modified register %%r15 at the %zx\n",
								      p - data);
	      exit(1);
	    }
	  } else if (operands[i].type == OperandNoSandboxEffect) {
	    if (operands[i].name == REG_R15) {
	      printf("Incorrectly modified register %%r15 at the %zx\n",
								      p - data);
	      exit(1);
	    }
	  }
        }
      }
    }
  }

  # Remove special instructions which are only allowed in special cases.
  normal_instruction = (one_instruction - (
    (0x48 0x89 0xe5)		 | # mov %rsp,%rbp
    (0x48 0x89 0xec)		 | # mov %rbp,%rsp
    (0x48 0x81 0xe4 any{4})	 | # and $XXX,%rsp
    (0x48 0x83 0xe4 any)	 | # and $XXX,%rsp
    (0x4c 0x01 0xfd)		 | # add %r15,%rbp
    (0x49 0x8d 0x2c 0x2f)	 | # lea (%r15,%rbp,1),%rbp
    (0x4a 0x8d 0x6c 0x3d any)	 | # lea 0x0(%rbp,%r15,1),%rbp
    (0x48 0x81 0xe5 any{4})	 | # and $XXX,%rsp
    (0x48 0x83 0xe5 any)	 | # and $XXX,%rsp
    (0x4c 0x01 0xfc)		 | # add %r15,%rsp
    (0x4a 0x8d 0x24 0x3c)	 | # lea (%rsp,%r15,1),%rsp
    (0x49 0x8d 0x34 0x37)	 | # lea (%r15,%rsi,1),%rsi
    (0x49 0x8d 0x3c 0x3f)	   # lea (%r15,%rdi,1),%rdi
  )) @process_normal_instruction;

  special_instruction =
    (0x48 0x89 0xe5)			   | # mov %rsp,%rbp
    (0x48 0x81 0xe4 any{3} (0x80 .. 0xff)) | # and $XXX,%rsp
    (0x48 0x83 0xe4 (0x80 .. 0xff))	     # and $XXX,%rsp
    @{ if (restricted_register == REG_RSP) {
	 printf("Incorrectly modified register %%rsp at the %zx\n", p - data);
	 exit(1);
       }
       restricted_register = kNoRestrictedReg;
    } |
    (0x48 0x89 0xec)			   | # mov %rbp,%rsp
    (0x48 0x81 0xe5 any{3} (0x80 .. 0xff)) | # and $XXX,%rsp
    (0x48 0x83 0xe5 (0x80 .. 0xff))	     # and $XXX,%rsp
    @{ if (restricted_register == REG_RBP) {
	 printf("Incorrectly modified register %%rbp at the %zx\n", p - data);
	 exit(1);
       }
       restricted_register = kNoRestrictedReg;
    } |
    (0x4c 0x01 0xfd	       | # add %r15,%rbp
     0x49 0x8d 0x2c 0x2f       | # lea (%r15,%rbp,1),%rbp
     0x4a 0x8d 0x6c 0x3d 0x00)	 # lea 0x0(%rbp,%r15,1),%rbp
    @{ if (restricted_register != REG_RBP) {
	 printf("Incorrectly sandboxed %%rbp at the %zx\n", p - data);
	 exit(1);
       }
       restricted_register = kNoRestrictedReg;
       BitmapClearBit(valid_targets, (begin - data));
    } |
    (0x4c 0x01 0xfc	  | # add %r15,%rsp
     0x4a 0x8d 0x24 0x3c)   # lea (%rsp,%r15,1),%rsp
    @{ if (restricted_register != REG_RSP) {
	 printf("Incorrectly sandboxed %%rsp at the %zx\n", p - data);
	 exit(1);
       }
       restricted_register = kNoRestrictedReg;
       BitmapClearBit(valid_targets, (begin - data));
    } |
    (0x83 0xe0 0xe0 0x4c 0x01 0xf8 0xff (0xd0|0xe0) | # naclcall/jmp %eax, %r15
     0x83 0xe1 0xe0 0x4c 0x01 0xf9 0xff (0xd1|0xe1) | # naclcall/jmp %ecx, %r15
     0x83 0xe2 0xe0 0x4c 0x01 0xfa 0xff (0xd2|0xe2) | # naclcall/jmp %edx, %r15
     0x83 0xe3 0xe0 0x4c 0x01 0xfb 0xff (0xd3|0xe3) | # naclcall/jmp %ebx, %r15
     0x83 0xe4 0xe0 0x4c 0x01 0xfc 0xff (0xd4|0xe4) | # naclcall/jmp %esp, %r15
     0x83 0xe5 0xe0 0x4c 0x01 0xfd 0xff (0xd5|0xe5) | # naclcall/jmp %ebp, %r15
     0x83 0xe6 0xe0 0x4c 0x01 0xfe 0xff (0xd6|0xe6) | # naclcall/jmp %esi, %r15
     0x83 0xe7 0xe0 0x4c 0x01 0xff 0xff (0xd7|0xe7))  # naclcall/jmp %edi, %r15
    @{ if (restricted_register == REG_RSP) {
	 printf("Incorrectly modified register %%rsp at the %zx\n", p - data);
	 exit(1);
       } else if (restricted_register == REG_RBP) {
	 printf("Incorrectly modified register %%rbp at the %zx\n", p - data);
	 exit(1);
       }
       BitmapClearBit(valid_targets, (p - data) - 4);
       BitmapClearBit(valid_targets, (p - data) - 1);
       restricted_register = kNoRestrictedReg;
    } |
    (0x41 0x83 0xe0 0xe0 0x4d 0x01 0xf8 0x41 0xff (0xd0|0xe0) | # naclcall/jmp
     0x41 0x83 0xe1 0xe0 0x4d 0x01 0xf9 0x41 0xff (0xd1|0xe1) | #    %r8d, %r15
     0x41 0x83 0xe2 0xe0 0x4d 0x01 0xfa 0x41 0xff (0xd2|0xe2) | # ⋮
     0x41 0x83 0xe3 0xe0 0x4d 0x01 0xfb 0x41 0xff (0xd3|0xe3) | # ⋮
     0x41 0x83 0xe4 0xe0 0x4d 0x01 0xfc 0x41 0xff (0xd4|0xe4) | # ⋮
     0x41 0x83 0xe5 0xe0 0x4d 0x01 0xfd 0x41 0xff (0xd5|0xe5) | # naclcall/jmp
     0x41 0x83 0xe6 0xe0 0x4d 0x01 0xfe 0x41 0xff (0xd6|0xe6))  #   %r14d, %r15
    @{ if (restricted_register == REG_RSP) {
	 printf("Incorrectly modified register %%rsp at the %zx\n", p - data);
	 exit(1);
       } else if (restricted_register == REG_RBP) {
	 printf("Incorrectly modified register %%rbp at the %zx\n", p - data);
	 exit(1);
       }
       BitmapClearBit(valid_targets, (p - data) - 5);
       BitmapClearBit(valid_targets, (p - data) - 2);
       restricted_register = kNoRestrictedReg;
    } |
    (0x49 0x8d 0x34 0x37) # lea (%r15,%rsi,1),%rsi
    @{ if (restricted_register == REG_RSI) {
	 sandboxed_rsi = begin;
	 restricted_register = kSandboxedRsi;
       } else {
	 restricted_register = kNoRestrictedReg;
       }
    } |
    (0x49 0x8d 0x3c 0x3f) # lea (%r15,%rdi,1),%rdi
    @{ if (restricted_register == REG_RDI) {
	 sandboxed_rdi = begin;
	 restricted_register = kSandboxedRdi;
       } else if (restricted_register == kSandboxedRsiRestrictedRdi) {
	 sandboxed_rdi = begin;
	 restricted_register = kSandboxedRsiSandboxedRdi;
       } else {
         restricted_register = kNoRestrictedReg;
       }
    } |
    (0xac		       | # lods   %ds:(%rsi),%al
     (data16|REXW_NONE)? 0xad)	 # lods   %ds:(%rsi),%ax/%eax/%rax
    @{ if (restricted_register != kSandboxedRsi) {
	 printf("Incorrectly sandboxed %%rdi at the %zx\n", p - data);
	 exit(1);
       }
       restricted_register = kNoRestrictedReg;
       BitmapClearBit(valid_targets, (begin - data));
       BitmapClearBit(valid_targets, (sandboxed_rdi - data));
    } |
    (0xae		       | # scas   %ds:(%rsi),%al
     (data16|REXW_NONE)? 0xaf  | # scas   %ds:(%rsi),%ax/%eax/%rax
     0xaa		       | # stos   %al,%es:(%rdi)
     (data16|REXW_NONE)? 0xab)	 # stos   %ax/%eax/%rax,%es:(%rdi)
    @{ if (restricted_register != kSandboxedRdi &&
	   restricted_register != kSandboxedRsiSandboxedRdi) {
	 printf("Incorrectly sandboxed %%rdi at the %zx\n", p - data);
	 exit(1);
       }
       restricted_register = kNoRestrictedReg;
       BitmapClearBit(valid_targets, (begin - data));
       BitmapClearBit(valid_targets, (sandboxed_rdi - data));
    } |
    (condrep? 0xa6	      | # cmpsb	   %es:(%rdi),%ds:(%rsi)
     (data16 |
      condrep data16 |
      data16 condrep) 0xa7    | # cmpsw	   %es:(%rdi),%ds:(%rsi)
     condrep? REXW_NONE? 0xa7 | # cmps[lq] %es:(%rdi),%ds:(%rsi)
     rep? 0xa4		      | # movsb	   %es:(%rdi),%ds:(%rsi)
     (data16 |
      rep data16 |
      data16 rep) 0xa5	      | # movsw	   %es:(%rdi),%ds:(%rsi)
     rep? REXW_NONE? 0xa5)	# movs[lq] %es:(%rdi),%ds:(%rsi)
    @{ if (restricted_register != kSandboxedRsiSandboxedRdi) {
	 printf("Incorrectly sandboxed %%rsi or %%rdi at the %zx\n", p - data);
	 exit(1);
       }
       restricted_register = kNoRestrictedReg;
       BitmapClearBit(valid_targets, (begin - data));
       BitmapClearBit(valid_targets, (sandboxed_rsi - data));
       BitmapClearBit(valid_targets, (sandboxed_rsi_restricted_rdi - data));
       BitmapClearBit(valid_targets, (sandboxed_rdi - data));
    };

  main := ((normal_instruction | special_instruction) >{
	begin = p;
	BitmapSetBit(valid_targets, p - data);
	rex_prefix = FALSE;
	vex_prefix2 = 0xe0;
	vex_prefix3 = 0x00;
     })*
    $!{ process_error(p, userdata);
	result = 1;
	goto error_detected;
    };

}%%

%% write data;

/* Ignore this information for now.  */
#define data16_prefix if (0) result
#define lock_prefix if (0) result
#define repz_prefix if (0) result
#define repnz_prefix if (0) result
#define branch_not_taken if (0) result
#define branch_taken if (0) result
#define imm if (0) begin
#define imm_operand if (0) result
#define imm2 if (0) begin
#define imm2_operand if (0) result
#define scale if (0) result
#define disp if (0) begin
#define disp_type if (0) result
#define operand0 operands[0].name
#define operand1 operands[1].name
#define operand2 operands[2].name
#define operand3 operands[3].name
#define operand4 operands[4].name
#define operand0_type operands[0].type
#define operand1_type operands[1].type
#define operand2_type operands[2].type
#define operand3_type operands[3].type
#define operand4_type operands[4].type
/* It's not important for us.  */
#define operand0_read if (0) result
#define operand1_read if (0) result
#define operand2_read if (0) result
#define operand3_read if (0) result
#define operand4_read if (0) result
#define operand0_write operands[0].write
#define operand1_write operands[1].write
#define operand2_write operands[2].write
#define operand3_write operands[3].write
#define operand4_write operands[4].write

enum {
  REX_B = 1,
  REX_X = 2,
  REX_R = 4,
  REX_W = 8
};

enum disp_mode {
  DISPNONE,
  DISP8,
  DISP16,
  DISP32,
  DISP64,
};

enum imm_mode {
  IMMNONE,
  IMM2,
  IMM8,
  IMM16,
  IMM32,
  IMM64
};

static const uint8_t one = 1;

static const int kBitsPerByte = 8;

static inline uint8_t *BitmapAllocate(uint32_t indexes) {
  uint32_t byte_count = (indexes + kBitsPerByte - 1) / kBitsPerByte;
  uint8_t *bitmap = malloc(byte_count);
  if (bitmap != NULL) {
    memset(bitmap, 0, byte_count);
  }
  return bitmap;
}

static inline int BitmapIsBitSet(uint8_t *bitmap, uint32_t index) {
  return (bitmap[index / kBitsPerByte] & (1 << (index % kBitsPerByte))) != 0;
}

static inline void BitmapSetBit(uint8_t *bitmap, uint32_t index) {
  bitmap[index / kBitsPerByte] |= 1 << (index % kBitsPerByte);
}

static inline void BitmapClearBit(uint8_t *bitmap, uint32_t index) {
  bitmap[index / kBitsPerByte] &= ~(1 << (index % kBitsPerByte));
}

int CheckJumpTargets(uint8_t *valid_targets, uint8_t *jump_dests,
                     size_t size) {
  size_t i;
  for (i = 0; i < size / 32; i++) {
    uint32_t jump_dest_mask = ((uint32_t *) jump_dests)[i];
    uint32_t valid_target_mask = ((uint32_t *) valid_targets)[i];
    if ((jump_dest_mask & ~valid_target_mask) != 0) {
      printf("bad jump to around %x\n", (unsigned)(i * 32));
      return 1;
    }
  }
  return 0;
}

int ValidateChunk(const uint8_t *data, size_t size,
		  process_error_func process_error, void *userdata) {
  const size_t bundle_size = 32;
  const size_t bundle_mask = bundle_size - 1;

  uint8_t *valid_targets = BitmapAllocate(size);
  uint8_t *jump_dests = BitmapAllocate(size);

  const uint8_t *p = data;
  const uint8_t *begin;

  uint8_t rex_prefix, vex_prefix2, vex_prefix3;
  struct Operand {
    unsigned int name	:5;
    unsigned int type	:2;
    bool	 write	:1;
  } operands[5];
  enum register_name base, index;
  uint8_t operands_count, i;
  int result = 0;
  /*
   * These are borders of the appropriate instructions.  Initialize them to make
   * compiler happy: they are never used uninitialized even without explicit
   * initialization but GCC is not sophysicated enough to prove that.
   */
  const uint8_t *sandboxed_rsi = 0;
  const uint8_t *sandboxed_rsi_restricted_rdi = 0;
  const uint8_t *sandboxed_rdi = 0;

  assert(size % bundle_size == 0);

  while (p < data + size) {
    const uint8_t *pe = p + bundle_size;
    const uint8_t *eof = pe;
    int cs;
    enum {
      kNoRestrictedReg = 32,
      kSandboxedRsi,
      kSandboxedRdi,
      kSandboxedRsiRestrictedRdi,
      kSandboxedRsiSandboxedRdi
    }; uint8_t restricted_register = kNoRestrictedReg;

    %% write init;
    %% write exec;

    if (restricted_register == REG_RBP) {
      printf("Incorrectly sandboxed %%rbp at the %d%zx\n", *data, p - data);
      exit(1);
    } else if (restricted_register == REG_RSP) {
      printf("Incorrectly sandboxed %%rbp at the %zx\n", p - data);
      exit(1);
    }
  }

  if (CheckJumpTargets(valid_targets, jump_dests, size)) {
    result = 1;
    goto error_detected;
  }

error_detected:
  return result;
}


Ragel variable of element closing tags.

Ragel variable of element closing tags.

%%machine base_lexer;

%%{
    ##
    # Base grammar for the XML lexer.
    #
    # This grammar is shared between the C and Java extensions. As a result of
    # this you should **not** include language specific code in Ragel
    # actions/callbacks.
    #
    # To call back in to Ruby you can use one of the following two functions:
    #
    # * callback
    # * callback_simple
    #
    # The first function takes 5 arguments:
    #
    # * The name of the Ruby method to call.
    # * The input data.
    # * The encoding of the input data.
    # * The start of the current buffer.
    # * The end of the current buffer.
    #
    # The function callback_simple only takes one argument: the name of the
    # method to call. This function should be used for callbacks that don't
    # require any values.
    #
    # When you call a method in Ruby make sure that said method is defined as
    # an instance method in the `Oga::XML::Lexer` class.
    #
    # ## Machine Transitions
    #
    # To transition from one machine to another always use `fnext` instead of
    # `fcall` and `fret`. This removes the need for the code to keep track of a
    # stack.
    #

    newline    = '\n' | '\r\n';
    whitespace = [ \t];
    identifier = [a-zA-Z0-9\-_]+;

    # Comments
    #
    # http://www.w3.org/TR/html-markup/syntax.html#comments
    #
    # Unlike the W3 specification these rules *do* allow character sequences
    # such as `--` and `->`. Putting extra checks in for these sequences would
    # actually make the rules/actions more complex.
    #

    comment_start = '<!--';
    comment_end   = '-->';
    comment       = comment_start (any* -- comment_end) comment_end;

    action start_comment {
        callback("on_comment", data, encoding, ts + 4, te - 3);
    }

    # CDATA
    #
    # http://www.w3.org/TR/html-markup/syntax.html#cdata-sections
    #
    # In HTML CDATA tags have no meaning/are not supported. Oga does
    # support them but treats their contents as plain text.
    #

    cdata_start = '<![CDATA[';
    cdata_end   = ']]>';
    cdata       = cdata_start (any* -- cdata_end) cdata_end;

    action start_cdata {
        callback("on_cdata", data, encoding, ts + 9, te - 3);
    }

    # Processing Instructions
    #
    # http://www.w3.org/TR/xpath/#section-Processing-Instruction-Nodes
    # http://en.wikipedia.org/wiki/Processing_Instruction
    #
    # These are tags meant to be used by parsers/libraries for custom behaviour.
    # One example are the tags used by PHP: <?php and ?>. Note that the XML
    # declaration tags (<?xml ?>) are not considered to be a processing
    # instruction.
    #

    proc_ins_start = '<?' identifier;
    proc_ins_end   = '?>';

    action start_proc_ins {
        callback_simple("on_proc_ins_start");
        callback("on_proc_ins_name", data, encoding, ts + 2, te);

        mark = te;

        fnext proc_ins_body;
    }

    proc_ins_body := |*
        proc_ins_end => {
            callback("on_text", data, encoding, mark, ts);
            callback_simple("on_proc_ins_end");

            fnext main;
        };

        any;
    *|;

    # Strings
    #
    # Strings in HTML can either be single or double quoted. If a string
    # starts with one of these quotes it must be closed with the same type
    # of quote.
    #
    dquote = '"';
    squote = "'";

    string_dquote = (dquote ^dquote+ dquote);
    string_squote = (squote ^squote+ squote);

    string = string_dquote | string_squote;

    action emit_string {
        callback("on_string", data, encoding, ts + 1, te - 1);
    }

    # DOCTYPES
    #
    # http://www.w3.org/TR/html-markup/syntax.html#doctype-syntax
    #
    # These rules support the 3 flavours of doctypes:
    #
    # 1. Normal doctypes, as introduced in the HTML5 specification.
    # 2. Deprecated doctypes, the more verbose ones used prior to HTML5.
    # 3. Legacy doctypes
    #
    doctype_start = '<!DOCTYPE'i whitespace+;

    action start_doctype {
        callback_simple("on_doctype_start");
        fnext doctype;
    }

    # Machine for processing doctypes. Doctype values such as the public
    # and system IDs are treated as T_STRING tokens.
    doctype := |*
        'PUBLIC' | 'SYSTEM' => {
            callback("on_doctype_type", data, encoding, ts, te);
        };

        # Consumes everything between the [ and ]. Due to the use of :> the ]
        # is not consumed by any+.
        '[' any+ :> ']' => {
            callback("on_doctype_inline", data, encoding, ts + 1, te - 1);
        };

        # Lex the public/system IDs as regular strings.
        string => emit_string;

        # Whitespace inside doctypes is ignored since there's no point in
        # including it.
        whitespace;

        identifier => {
            callback("on_doctype_name", data, encoding, ts, te);
        };

        '>' => {
            callback_simple("on_doctype_end");
            fnext main;
        };
    *|;

    # XML declaration tags
    #
    # http://www.w3.org/TR/REC-xml/#sec-prolog-dtd
    #
    xml_decl_start = '<?xml';
    xml_decl_end   = '?>';

    action start_xml_decl {
        callback_simple("on_xml_decl_start");
        fnext xml_decl;
    }

    # Machine that processes the contents of an XML declaration tag.
    xml_decl := |*
        xml_decl_end => {
            callback_simple("on_xml_decl_end");
            fnext main;
        };

        # Attributes and their values (e.g. version="1.0").
        identifier => {
            callback("on_attribute", data, encoding, ts, te);
        };

        string => emit_string;

        any;
    *|;

    # Elements
    #
    # http://www.w3.org/TR/html-markup/syntax.html#syntax-elements
    #
    # Lexing of elements is broken up into different machines that handle the
    # name/namespace, contents of the open tag and the body of an element. The
    # body of an element is lexed using the `main` machine.
    #

    element_end = '</' identifier (':' identifier)* '>';

    action start_element {
        callback_simple("on_element_start");
        fnext element_name;
    }

    # Machine used for lexing the name/namespace of an element.
    element_name := |*
        identifier ':' => {
            callback("on_element_ns", data, encoding, ts, te - 1);
        };

        identifier => {
            callback("on_element_name", data, encoding, ts, te);
            fnext element_head;
        };
    *|;

    # Machine used for processing the contents of an element's starting tag.
    # This includes the name, namespace and attributes.
    element_head := |*
        whitespace | '=';

        newline => {
            callback_simple("advance_line");
        };

        # Attribute names and namespaces.
        identifier ':' => {
            callback("on_attribute_ns", data, encoding, ts, te - 1);
        };

        identifier => {
            callback("on_attribute", data, encoding, ts, te);
        };

        # Attribute values.
        string => emit_string;

        # We're done with the open tag of the element.
        '>' => {
            callback_simple("on_element_open_end");
            fnext main;
        };

        # Self closing tags.
        '/>' => {
            callback_simple("on_element_end");
            fnext main;
        };
    *|;

    # The main machine aka the entry point of Ragel.
    main := |*
        doctype_start  => start_doctype;
        xml_decl_start => start_xml_decl;
        comment        => start_comment;
        cdata          => start_cdata;
        proc_ins_start => start_proc_ins;

        # The start of an element.
        '<' => start_element;

        # Regular closing tags.
        element_end => {
            callback_simple("on_element_end");
        };

        # Treat everything else, except for "<", as regular text. The "<" sign
        # is used for tags so we can't emit text nodes for these characters.
        any+ -- '<' => {
            callback("on_text", data, encoding, ts, te);
        };
    *|;
}%%

%%machine base_lexer;

%%{
    ##
    # Base grammar for the XML lexer.
    #
    # This grammar is shared between the C and Java extensions. As a result of
    # this you should **not** include language specific code in Ragel
    # actions/callbacks.
    #
    # To call back in to Ruby you can use one of the following two functions:
    #
    # * callback
    # * callback_simple
    #
    # The first function takes 5 arguments:
    #
    # * The name of the Ruby method to call.
    # * The input data.
    # * The encoding of the input data.
    # * The start of the current buffer.
    # * The end of the current buffer.
    #
    # The function callback_simple only takes one argument: the name of the
    # method to call. This function should be used for callbacks that don't
    # require any values.
    #
    # When you call a method in Ruby make sure that said method is defined as
    # an instance method in the `Oga::XML::Lexer` class.
    #
    # ## Machine Transitions
    #
    # To transition from one machine to another always use `fnext` instead of
    # `fcall` and `fret`. This removes the need for the code to keep track of a
    # stack.
    #

    newline    = '\n' | '\r\n';
    whitespace = [ \t];
    identifier = [a-zA-Z0-9\-_]+;

    # Comments
    #
    # http://www.w3.org/TR/html-markup/syntax.html#comments
    #
    # Unlike the W3 specification these rules *do* allow character sequences
    # such as `--` and `->`. Putting extra checks in for these sequences would
    # actually make the rules/actions more complex.
    #

    comment_start = '<!--';
    comment_end   = '-->';
    comment       = comment_start (any* -- comment_end) comment_end;

    action start_comment {
        callback("on_comment", data, encoding, ts + 4, te - 3);
    }

    # CDATA
    #
    # http://www.w3.org/TR/html-markup/syntax.html#cdata-sections
    #
    # In HTML CDATA tags have no meaning/are not supported. Oga does
    # support them but treats their contents as plain text.
    #

    cdata_start = '<![CDATA[';
    cdata_end   = ']]>';
    cdata       = cdata_start (any* -- cdata_end) cdata_end;

    action start_cdata {
        callback("on_cdata", data, encoding, ts + 9, te - 3);
    }

    # Processing Instructions
    #
    # http://www.w3.org/TR/xpath/#section-Processing-Instruction-Nodes
    # http://en.wikipedia.org/wiki/Processing_Instruction
    #
    # These are tags meant to be used by parsers/libraries for custom behaviour.
    # One example are the tags used by PHP: <?php and ?>. Note that the XML
    # declaration tags (<?xml ?>) are not considered to be a processing
    # instruction.
    #

    proc_ins_start = '<?' identifier;
    proc_ins_end   = '?>';

    action start_proc_ins {
        callback_simple("on_proc_ins_start");
        callback("on_proc_ins_name", data, encoding, ts + 2, te);

        mark = te;

        fnext proc_ins_body;
    }

    proc_ins_body := |*
        proc_ins_end => {
            callback("on_text", data, encoding, mark, ts);
            callback_simple("on_proc_ins_end");

            fnext main;
        };

        any;
    *|;

    # Strings
    #
    # Strings in HTML can either be single or double quoted. If a string
    # starts with one of these quotes it must be closed with the same type
    # of quote.
    #
    dquote = '"';
    squote = "'";

    string_dquote = (dquote ^dquote+ dquote);
    string_squote = (squote ^squote+ squote);

    string = string_dquote | string_squote;

    action emit_string {
        callback("on_string", data, encoding, ts + 1, te - 1);
    }

    # DOCTYPES
    #
    # http://www.w3.org/TR/html-markup/syntax.html#doctype-syntax
    #
    # These rules support the 3 flavours of doctypes:
    #
    # 1. Normal doctypes, as introduced in the HTML5 specification.
    # 2. Deprecated doctypes, the more verbose ones used prior to HTML5.
    # 3. Legacy doctypes
    #
    doctype_start = '<!DOCTYPE'i whitespace+;

    action start_doctype {
        callback_simple("on_doctype_start");
        fnext doctype;
    }

    # Machine for processing doctypes. Doctype values such as the public
    # and system IDs are treated as T_STRING tokens.
    doctype := |*
        'PUBLIC' | 'SYSTEM' => {
            callback("on_doctype_type", data, encoding, ts, te);
        };

        # Consumes everything between the [ and ]. Due to the use of :> the ]
        # is not consumed by any+.
        '[' any+ :> ']' => {
            callback("on_doctype_inline", data, encoding, ts + 1, te - 1);
        };

        # Lex the public/system IDs as regular strings.
        string => emit_string;

        # Whitespace inside doctypes is ignored since there's no point in
        # including it.
        whitespace;

        identifier => {
            callback("on_doctype_name", data, encoding, ts, te);
        };

        '>' => {
            callback_simple("on_doctype_end");
            fnext main;
        };
    *|;

    # XML declaration tags
    #
    # http://www.w3.org/TR/REC-xml/#sec-prolog-dtd
    #
    xml_decl_start = '<?xml';
    xml_decl_end   = '?>';

    action start_xml_decl {
        callback_simple("on_xml_decl_start");
        fnext xml_decl;
    }

    # Machine that processes the contents of an XML declaration tag.
    xml_decl := |*
        xml_decl_end => {
            callback_simple("on_xml_decl_end");
            fnext main;
        };

        # Attributes and their values (e.g. version="1.0").
        identifier => {
            callback("on_attribute", data, encoding, ts, te);
        };

        string => emit_string;

        any;
    *|;

    # Elements
    #
    # http://www.w3.org/TR/html-markup/syntax.html#syntax-elements
    #
    # Lexing of elements is broken up into different machines that handle the
    # name/namespace, contents of the open tag and the body of an element. The
    # body of an element is lexed using the `main` machine.
    #
    action start_element {
        callback_simple("on_element_start");
        fnext element_name;
    }

    # Machine used for lexing the name/namespace of an element.
    element_name := |*
        identifier ':' => {
            callback("on_element_ns", data, encoding, ts, te - 1);
        };

        identifier => {
            callback("on_element_name", data, encoding, ts, te);
            fnext element_head;
        };
    *|;

    # Machine used for processing the contents of an element's starting tag.
    # This includes the name, namespace and attributes.
    element_head := |*
        whitespace | '=';

        newline => {
            callback_simple("advance_line");
        };

        # Attribute names and namespaces.
        identifier ':' => {
            callback("on_attribute_ns", data, encoding, ts, te - 1);
        };

        identifier => {
            callback("on_attribute", data, encoding, ts, te);
        };

        # Attribute values.
        string => emit_string;

        # We're done with the open tag of the element.
        '>' => {
            callback_simple("on_element_open_end");
            fnext main;
        };

        # Self closing tags.
        '/>' => {
            callback_simple("on_element_end");
            fnext main;
        };
    *|;

    # The main machine aka the entry point of Ragel.
    main := |*
        doctype_start  => start_doctype;
        xml_decl_start => start_xml_decl;
        comment        => start_comment;
        cdata          => start_cdata;
        proc_ins_start => start_proc_ins;

        # The start of an element.
        '<' => start_element;

        # Regular closing tags.
        '</' identifier (':' identifier)* '>' => {
            callback_simple("on_element_end");
        };

        # Treat everything else, except for "<", as regular text. The "<" sign
        # is used for tags so we can't emit text nodes for these characters.
        ^'<'+ => {
            callback("on_text", data, encoding, ts, te);
        };
    *|;
}%%


Added "any" general properties

Added "any" general properties

%%{
  machine re_scanner;

  wild                  = '.';
  backslash             = '\\';
  alternation           = '|';
  beginning_of_line     = '^';
  end_of_line           = '$';

  range_open            = '{';
  range_close           = '}';
  curlies               = range_open | range_close;

  group_open            = '(';
  group_close           = ')';
  parantheses           = group_open | group_close;

  set_open              = '[';
  set_close             = ']';
  brackets              = set_open | set_close;

  posix_class_name      = 'alnum' | 'alpha' | 'blank' |
                          'cntrl' | 'digit' | 'graph' |
                          'lower' | 'print' | 'punct' |
                          'space' | 'upper' | 'xdigit' |
                          'word'  | 'ascii';

  property_name         = 'Alnum' | 'Alpha' | 'Any'   | 'Ascii' | 'Blank' |
                          'Cntrl' | 'Digit' | 'Graph' | 'Lower' | 'Print' |
                          'Punct' | 'Space' | 'Upper' | 'Word' | 'Xdigit';

  category_letter       = 'L' . [ultmo]?;
  category_mark         = 'M' . [nce]?;
  category_number       = 'N' . [dlo]?;
  category_punctuation  = 'P' . [cdseifo]?;
  category_symbol       = 'S' . [mcko]?;
  category_separator    = 'Z' . [slp]?;
  category_codepoint    = 'C' . [cfson]?;
  general_category      = category_letter | category_mark |
                          category_number | category_punctuation |
                          category_symbol | category_separator |
                          category_codepoint;

  posix_class           = '[:' . posix_class_name . ':]';

  char_type             = [dDhHsSwW];

  line_anchor           = beginning_of_line | end_of_line;
  anchor_char           = [AbBzZG];
  property_char         = [pP];

  escaped_char          = [aefnrtv];
  octal_sequence        = '0' . [0-7]{2};

  hex_sequence          = 'x' . xdigit{2};
  wide_hex_sequence     = 'x' . '{7' . xdigit{1,7} . '}';
  control_sequence      = ('c' | 'C-') . xdigit{1,2};
  meta_sequence         = 'M-' . xdigit{1,2};
  meta_control_sequence = 'M-\\C-' . xdigit{1,2};

  zero_or_one           = '?' | '??' | '?+';
  zero_or_more          = '*' | '*?' | '*+';
  one_or_more           = '+' | '+?' | '++';

  quantifier_greedy     = '?'  | '*'  | '+';
  quantifier_reluctant  = '??' | '*?' | '+?';
  quantifier_possessive = '?+' | '*+' | '++';
  quantifier_mode       = '?'  | '+';

  quantifier_range      = range_open . (digit+)? . ','? . (digit+)? .
                          range_close . quantifier_mode?;

  quantifiers           = quantifier_greedy | quantifier_reluctant |
                          quantifier_possessive | quantifier_range;

  group_comment         = '?#' . [^)]+ . group_close;

  group_atomic          = '?>';
  group_passive         = '?:';

  assertion_lookahead   = '?=';
  assertion_nlookahead  = '?!';
  assertion_lookbehind  = '?<=';
  assertion_nlookbehind = '?<!';

  group_options         = '?' . ([mix]{1,3})? . '-' . ([mix]{1,3})? . ':';

  group_name            = alpha . alnum+;
  group_named           = '?<' . group_name . '>';

  group_type            = group_atomic | group_passive | group_named;

  assertion_type        = assertion_lookahead  | assertion_nlookahead |
                          assertion_lookbehind | assertion_nlookbehind;

  # characters the 'break' a literal
  meta_char             = wild | backslash | alternation |
                          curlies | parantheses | brackets |
                          line_anchor | quantifier_greedy;

  # character set scanner, continues consuming characters until it meets the
  # closing bracket of the set.
  # --------------------------------------------------------------------------
  character_set := |*
    ']' {
      self.emit(:character_set, :close, data[ts..te-1].pack('c*'), ts, te)
      fret;
    };

    '^' {
      self.emit(:character_set, :negate, data[ts..te-1].pack('c*'), ts, te)
    };

    alnum . '-' . alnum { # TODO: add properties
      self.emit(:character_set, :range, data[ts..te-1].pack('c*'), ts, te)
    };

    '&&' {
      self.emit(:character_set, :intersection, data[ts..te-1].pack('c*'), ts, te)
    };

    '\\' . [\]\-\,b] {
      self.emit(:character_set, :member, data[ts..te-1].pack('c*'), ts, te)
    };

    posix_class {
      case text = data[ts..te-1].pack('c*')
      when '[:alnum:]';  self.emit(:character_set, :set_class_alnum,  text, ts, te)
      when '[:alpha:]';  self.emit(:character_set, :set_class_alpha,  text, ts, te)
      when '[:blank:]';  self.emit(:character_set, :set_class_blank,  text, ts, te)
      when '[:cntrl:]';  self.emit(:character_set, :set_class_cntrl,  text, ts, te)
      when '[:digit:]';  self.emit(:character_set, :set_class_digit,  text, ts, te)
      when '[:graph:]';  self.emit(:character_set, :set_class_graph,  text, ts, te)
      when '[:lower:]';  self.emit(:character_set, :set_class_lower,  text, ts, te)
      when '[:print:]';  self.emit(:character_set, :set_class_print,  text, ts, te)
      when '[:punct:]';  self.emit(:character_set, :set_class_punct,  text, ts, te)
      when '[:space:]';  self.emit(:character_set, :set_class_space,  text, ts, te)
      when '[:upper:]';  self.emit(:character_set, :set_class_upper,  text, ts, te)
      when '[:xdigit:]'; self.emit(:character_set, :set_class_xdigit, text, ts, te)
      when '[:word:]';   self.emit(:character_set, :set_class_word,   text, ts, te)
      when '[:ascii:]';  self.emit(:character_set, :set_class_ascii,  text, ts, te)
      else raise "Unsupported character posixe class at #{text} (char #{ts})"
      end
    };

    any {
      self.emit(:character_set, :member, data[ts..te-1].pack('c*'), ts, te)
    };
  *|;

  # escape sequence scanner
  # --------------------------------------------------------------------------
  escape_sequence := |*
    hex_sequence {
      self.emit(:escape_sequence, :hex, data[ts..te-1].pack('c*'), ts, te)
      fret;
    };

    wide_hex_sequence {
      self.emit(:escape_sequence, :hex_wide, data[ts..te-1].pack('c*'), ts, te)
      fret;
    };

    control_sequence {
      self.emit(:escape_sequence, :control, data[ts..te-1].pack('c*'), ts, te)
      fret;
    };

    meta_sequence {
      self.emit(:escape_sequence, :meta, data[ts..te-1].pack('c*'), ts, te)
      fret;
    };

    meta_control_sequence {
      self.emit(:escape_sequence, :meta_control, data[ts..te-1].pack('c*'), ts, te)
      fret;
    };

    # TODO: extract into a separate machine... use in sets
    property_char . '{' . (property_name | general_category) . '}' > (escaped_alpha, 2) {
      text = data[ts..te-1].pack('c*')
      type = text[0,1] == 'p' ? :property: :inverted_property

      case name = data[ts+2..te-2].pack('c*')

      # Named
      when 'Alnum';   self.emit(type, :alnum,  text, ts, te)
      when 'Alpha';   self.emit(type, :alpha,  text, ts, te)
      when 'Any';     self.emit(type, :any,    text, ts, te)
      when 'Ascii';   self.emit(type, :ascii,  text, ts, te)
      when 'Blank';   self.emit(type, :blank,  text, ts, te)
      when 'Cntrl';   self.emit(type, :cntrl,  text, ts, te)
      when 'Digit';   self.emit(type, :digit,  text, ts, te)
      when 'Graph';   self.emit(type, :graph,  text, ts, te)
      when 'Lower';   self.emit(type, :lower,  text, ts, te)
      when 'Print';   self.emit(type, :print,  text, ts, te)
      when 'Punct';   self.emit(type, :punct,  text, ts, te)
      when 'Space';   self.emit(type, :space,  text, ts, te)
      when 'Upper';   self.emit(type, :upper,  text, ts, te)
      when 'Word';    self.emit(type, :word,   text, ts, te)
      when 'Xdigit';  self.emit(type, :xdigit, text, ts, te)

      # Letters
      when 'L';  self.emit(type, :letter_any,       text, ts, te)
      when 'Lu'; self.emit(type, :letter_uppercase, text, ts, te)
      when 'Ll'; self.emit(type, :letter_lowercase, text, ts, te)
      when 'Lt'; self.emit(type, :letter_titlecase, text, ts, te)
      when 'Lm'; self.emit(type, :letter_modifier,  text, ts, te)
      when 'Lo'; self.emit(type, :letter_other,     text, ts, te)

      # Marks
      when 'M';  self.emit(type, :mark_any,         text, ts, te)
      when 'Mn'; self.emit(type, :mark_nonspacing,  text, ts, te)
      when 'Mc'; self.emit(type, :mark_spacing,     text, ts, te)
      when 'Me'; self.emit(type, :mark_enclosing,   text, ts, te)

      # Numbers
      when 'N';  self.emit(type, :number_any, text, ts, te)
      when 'Nd'; self.emit(type, :number_decimal, text, ts, te)
      when 'Nl'; self.emit(type, :number_letter, text, ts, te)
      when 'No'; self.emit(type, :number_other, text, ts, te)

      # Punctuation
      when 'P';  self.emit(type, :punct_any, text, ts, te)
      when 'Pc'; self.emit(type, :punct_connector, text, ts, te)
      when 'Pd'; self.emit(type, :punct_dash, text, ts, te)
      when 'Ps'; self.emit(type, :punct_open, text, ts, te)
      when 'Pe'; self.emit(type, :punct_close, text, ts, te)
      when 'Pi'; self.emit(type, :punct_initial, text, ts, te)
      when 'Pf'; self.emit(type, :punct_final, text, ts, te)
      when 'Po'; self.emit(type, :punct_other, text, ts, te)

      # Symbols
      when 'S';  self.emit(type, :symbol_any, text, ts, te)
      when 'Sm'; self.emit(type, :symbol_math, text, ts, te)
      when 'Sc'; self.emit(type, :symbol_currency, text, ts, te)
      when 'Sk'; self.emit(type, :symbol_modifier, text, ts, te)
      when 'So'; self.emit(type, :symbol_other, text, ts, te)

      # Separators
      when 'Z';  self.emit(type, :separator_any, text, ts, te)
      when 'Zs'; self.emit(type, :separator_space, text, ts, te)
      when 'Zl'; self.emit(type, :separator_line, text, ts, te)
      when 'Zp'; self.emit(type, :separator_paragraph, text, ts, te)

      # Codepoints
      when 'C';  self.emit(type, :code_any, text, ts, te)
      when 'Cc'; self.emit(type, :code_control, text, ts, te)
      when 'Cf'; self.emit(type, :code_format, text, ts, te)
      when 'Cs'; self.emit(type, :code_surrogate, text, ts, te)
      when 'Co'; self.emit(type, :code_private, text, ts, te)
      when 'Cn'; self.emit(type, :code_unassigned, text, ts, te)
      end
    };

    any > (escaped_alpha, 1)  {
      self.emit(:escape_sequence, :literal, data[ts..te-1].pack('c*'), ts, te)
      fret;
    };
  *|;

  # Main scanner
  main := |*

    # Anchors
    alternation {
      self.emit(:meta, :alternation, data[ts..te-1].pack('c*'), ts, te)
    };

    # Character types
    #   .         any (except new line)
    #   \d, \D    digit, non-digit
    #   \h, \H    hex, non-hex
    #   \s, \S    whitespace, non-whitespace
    #   \w, \W    word, non-word
    # ------------------------------------------------------------------------
    wild {
      self.emit(:character_type, :any, data[ts..te-1].pack('c*'), ts, te)
    };

    backslash . char_type > (backslashed, 2) {
      case text = data[ts..te-1].pack('c*')
      when '\\d'; self.emit(:character_type, :digit,      text, ts, te)
      when '\\D'; self.emit(:character_type, :non_digit,  text, ts, te)
      when '\\h'; self.emit(:character_type, :hex,        text, ts, te)
      when '\\H'; self.emit(:character_type, :non_hex,    text, ts, te)
      when '\\s'; self.emit(:character_type, :space,      text, ts, te)
      when '\\S'; self.emit(:character_type, :non_space,  text, ts, te)
      when '\\w'; self.emit(:character_type, :word,       text, ts, te)
      when '\\W'; self.emit(:character_type, :non_word,   text, ts, te)
      else raise "Unsupported character type at #{text} (char #{ts})"
      end
    };

    # Anchors
    beginning_of_line {
      self.emit(:anchor, :beginning_of_line, data[ts..te-1].pack('c*'), ts, te)
    };

    end_of_line {
      self.emit(:anchor, :end_of_line, data[ts..te-1].pack('c*'), ts, te)
    };

    backslash . anchor_char > (backslashed, 3) {
      case text = data[ts..te-1].pack('c*')
      when '\\A'; self.emit(:anchor, :bos,                text, ts, te)
      when '\\z'; self.emit(:anchor, :eos,                text, ts, te)
      when '\\Z'; self.emit(:anchor, :eos_or_before_eol,  text, ts, te)
      when '\\b'; self.emit(:anchor, :word_boundary,      text, ts, te)
      when '\\B'; self.emit(:anchor, :non_word_boundary,  text, ts, te)
      else raise "Unsupported anchor at #{text} (char #{ts})"
      end
    };

    # Escaped sequences
    backslash > (backslashed, 1) {
      fcall escape_sequence;
    };

    # Character sets
    set_open {
      self.emit(:character_set, :open, data[ts..te-1].pack('c*'), ts, te)
      fcall character_set;
    };

    # (?#...) comments: parsed as a single expression, without introducing a
    # new nesting level. Comments may not include parentheses, escaped or not.
    group_open . group_comment {
      self.emit(:group, :comment, data[ts..te-1].pack('c*'), ts, te)
    };

    # (?mix-mix...) expression options:
    #   (?imx-imx)          option on/off
    #                         i: ignore case
    #                         m: multi-line (dot(.) match newline)
    #                         x: extended form
    #
    #   (?imx-imx:subexp)   option on/off for subexp
    group_open . group_options > (grouped, 1) {
      self.emit(:group, :options, data[ts..te-1].pack('c*'), ts, te)
    };

    # Assertions
    #   (?=subexp)          look-ahead
    #   (?!subexp)          negative look-ahead
    #   (?<=subexp)         look-behind
    #   (?<!subexp)         negative look-behind
    # ------------------------------------------------------------------------
    group_open . assertion_type? > (grouped, 2) {
      case text =  data[ts..te-1].pack('c*')
      when '(?=';  self.emit(:assertion, :lookahead,    text, ts, te)
      when '(?!';  self.emit(:assertion, :nlookahead,   text, ts, te)
      when '(?<='; self.emit(:assertion, :lookbehind,   text, ts, te)
      when '(?<!'; self.emit(:assertion, :nlookbehind,  text, ts, te)
      end
    };

    # Groups
    #   (?:subexp)          passive (non-captured) group
    #   (?>subexp)          atomic group, don't backtrack in subexp.
    #   (?<name>subexp)     named group (single quotes are no supported, yet)
    #   (subexp)            captured group
    # ------------------------------------------------------------------------
    group_open . group_type? > (grouped, 3) {
      case text =  data[ts..te-1].pack('c*')
      when '(?:';  self.emit(:group, :passive,      text, ts, te)
      when '(?>';  self.emit(:group, :atomic,       text, ts, te)
      when '(?<[a-zA-Z]\w+>'
        self.emit(:group, :named, text, ts, te)
      else
        self.emit(:group, :capture, text, ts, te)
      end
    };

    group_close {
      self.emit(:group, :close, data[ts..te-1].pack('c*'), ts, te)
    };

    # Quantifiers
    # ------------------------------------------------------------------------
    zero_or_one {
      case text =  data[ts..te-1].pack('c*')
      when '?' ;  self.emit(:quantifier, :zero_or_one,            text, ts, te)
      when '??';  self.emit(:quantifier, :zero_or_one_reluctant,  text, ts, te)
      when '?+';  self.emit(:quantifier, :zero_or_one_possessive, text, ts, te)
      end
    };

    zero_or_more {
      case text =  data[ts..te-1].pack('c*')
      when '*' ;  self.emit(:quantifier, :zero_or_more,            text, ts, te)
      when '*?';  self.emit(:quantifier, :zero_or_more_reluctant,  text, ts, te)
      when '*+';  self.emit(:quantifier, :zero_or_more_possessive, text, ts, te)
      end
    };

    one_or_more {
      case text =  data[ts..te-1].pack('c*')
      when '+' ;  self.emit(:quantifier, :one_or_more,            text, ts, te)
      when '+?';  self.emit(:quantifier, :one_or_more_reluctant,  text, ts, te)
      when '++';  self.emit(:quantifier, :one_or_more_possessive, text, ts, te)
      end
    };

    # Intervals: min, max, and exact notations
    # ------------------------------------------------------------------------
    range_open . (digit+)? . ','? . (digit+)? . range_close . quantifier_mode? {
      self.emit(:quantifier, :interval, data[ts..te-1].pack('c*'), ts, te)
    };

    # Literal: anything, except meta characters. This includes 2, 3, and 4
    # unicode byte sequences.
    # ------------------------------------------------------------------------
    (any - meta_char)+ {
      self.emit(:literal, :literal, data[ts..te-1].pack('c*'), ts, te)
    };

  *|;
}%%

%%{
  machine re_scanner;

  wild                  = '.';
  backslash             = '\\';
  alternation           = '|';
  beginning_of_line     = '^';
  end_of_line           = '$';

  range_open            = '{';
  range_close           = '}';
  curlies               = range_open | range_close;

  group_open            = '(';
  group_close           = ')';
  parantheses           = group_open | group_close;

  set_open              = '[';
  set_close             = ']';
  brackets              = set_open | set_close;

  posix_class_name      = 'alnum' | 'alpha' | 'blank' |
                          'cntrl' | 'digit' | 'graph' |
                          'lower' | 'print' | 'punct' |
                          'space' | 'upper' | 'xdigit' |
                          'word'  | 'ascii';

  property_name         = 'Alnum' | 'Alpha' | 'Any'   | 'Ascii' | 'Blank' |
                          'Cntrl' | 'Digit' | 'Graph' | 'Lower' | 'Print' |
                          'Punct' | 'Space' | 'Upper' | 'Word' | 'Xdigit';

  category_letter       = 'L' . [ultmo]?;
  category_mark         = 'M' . [nce]?;
  category_number       = 'N' . [dlo]?;
  category_punctuation  = 'P' . [cdseifo]?;
  category_symbol       = 'S' . [mcko]?;
  category_separator    = 'Z' . [slp]?;
  category_codepoint    = 'C' . [cfson]?;
  general_category      = category_letter | category_mark |
                          category_number | category_punctuation |
                          category_symbol | category_separator |
                          category_codepoint;

  posix_class           = '[:' . posix_class_name . ':]';

  char_type             = [dDhHsSwW];

  line_anchor           = beginning_of_line | end_of_line;
  anchor_char           = [AbBzZG];
  property_char         = [pP];

  escaped_char          = [aefnrtv];
  octal_sequence        = '0' . [0-7]{2};

  hex_sequence          = 'x' . xdigit{2};
  wide_hex_sequence     = 'x' . '{7' . xdigit{1,7} . '}';
  control_sequence      = ('c' | 'C-') . xdigit{1,2};
  meta_sequence         = 'M-' . xdigit{1,2};
  meta_control_sequence = 'M-\\C-' . xdigit{1,2};

  zero_or_one           = '?' | '??' | '?+';
  zero_or_more          = '*' | '*?' | '*+';
  one_or_more           = '+' | '+?' | '++';

  quantifier_greedy     = '?'  | '*'  | '+';
  quantifier_reluctant  = '??' | '*?' | '+?';
  quantifier_possessive = '?+' | '*+' | '++';
  quantifier_mode       = '?'  | '+';

  quantifier_range      = range_open . (digit+)? . ','? . (digit+)? .
                          range_close . quantifier_mode?;

  quantifiers           = quantifier_greedy | quantifier_reluctant |
                          quantifier_possessive | quantifier_range;

  group_comment         = '?#' . [^)]+ . group_close;

  group_atomic          = '?>';
  group_passive         = '?:';

  assertion_lookahead   = '?=';
  assertion_nlookahead  = '?!';
  assertion_lookbehind  = '?<=';
  assertion_nlookbehind = '?<!';

  group_options         = '?' . ([mix]{1,3})? . '-' . ([mix]{1,3})? . ':';

  group_name            = alpha . alnum+;
  group_named           = '?<' . group_name . '>';

  group_type            = group_atomic | group_passive | group_named;

  assertion_type        = assertion_lookahead  | assertion_nlookahead |
                          assertion_lookbehind | assertion_nlookbehind;

  # characters the 'break' a literal
  meta_char             = wild | backslash | alternation |
                          curlies | parantheses | brackets |
                          line_anchor | quantifier_greedy;

  # character set scanner, continues consuming characters until it meets the
  # closing bracket of the set.
  # --------------------------------------------------------------------------
  character_set := |*
    ']' {
      self.emit(:character_set, :close, data[ts..te-1].pack('c*'), ts, te)
      fret;
    };

    '^' {
      self.emit(:character_set, :negate, data[ts..te-1].pack('c*'), ts, te)
    };

    alnum . '-' . alnum { # TODO: add properties
      self.emit(:character_set, :range, data[ts..te-1].pack('c*'), ts, te)
    };

    '&&' {
      self.emit(:character_set, :intersection, data[ts..te-1].pack('c*'), ts, te)
    };

    '\\' . [\]\-\,b] {
      self.emit(:character_set, :member, data[ts..te-1].pack('c*'), ts, te)
    };

    posix_class {
      case text = data[ts..te-1].pack('c*')
      when '[:alnum:]';  self.emit(:character_set, :set_class_alnum,  text, ts, te)
      when '[:alpha:]';  self.emit(:character_set, :set_class_alpha,  text, ts, te)
      when '[:blank:]';  self.emit(:character_set, :set_class_blank,  text, ts, te)
      when '[:cntrl:]';  self.emit(:character_set, :set_class_cntrl,  text, ts, te)
      when '[:digit:]';  self.emit(:character_set, :set_class_digit,  text, ts, te)
      when '[:graph:]';  self.emit(:character_set, :set_class_graph,  text, ts, te)
      when '[:lower:]';  self.emit(:character_set, :set_class_lower,  text, ts, te)
      when '[:print:]';  self.emit(:character_set, :set_class_print,  text, ts, te)
      when '[:punct:]';  self.emit(:character_set, :set_class_punct,  text, ts, te)
      when '[:space:]';  self.emit(:character_set, :set_class_space,  text, ts, te)
      when '[:upper:]';  self.emit(:character_set, :set_class_upper,  text, ts, te)
      when '[:xdigit:]'; self.emit(:character_set, :set_class_xdigit, text, ts, te)
      when '[:word:]';   self.emit(:character_set, :set_class_word,   text, ts, te)
      when '[:ascii:]';  self.emit(:character_set, :set_class_ascii,  text, ts, te)
      else raise "Unsupported character posixe class at #{text} (char #{ts})"
      end
    };

    any {
      self.emit(:character_set, :member, data[ts..te-1].pack('c*'), ts, te)
    };
  *|;

  # escape sequence scanner
  # --------------------------------------------------------------------------
  escape_sequence := |*
    hex_sequence {
      self.emit(:escape_sequence, :hex, data[ts..te-1].pack('c*'), ts, te)
      fret;
    };

    wide_hex_sequence {
      self.emit(:escape_sequence, :hex_wide, data[ts..te-1].pack('c*'), ts, te)
      fret;
    };

    control_sequence {
      self.emit(:escape_sequence, :control, data[ts..te-1].pack('c*'), ts, te)
      fret;
    };

    meta_sequence {
      self.emit(:escape_sequence, :meta, data[ts..te-1].pack('c*'), ts, te)
      fret;
    };

    meta_control_sequence {
      self.emit(:escape_sequence, :meta_control, data[ts..te-1].pack('c*'), ts, te)
      fret;
    };

    # TODO: extract into a separate machine... use in sets
    property_char . '{' . (property_name | general_category) . '}' > (escaped_alpha, 2) {
      text = data[ts..te-1].pack('c*')
      type = text[0,1] == 'p' ? :property: :inverted_property

      case name = data[ts+2..te-2].pack('c*')

      # Named
      when 'Alnum';   self.emit(type, :alnum,  text, ts, te)
      when 'Alpha';   self.emit(type, :alpha,  text, ts, te)
      when 'Any';     self.emit(type, :any,    text, ts, te)
      when 'Ascii';   self.emit(type, :ascii,  text, ts, te)
      when 'Blank';   self.emit(type, :blank,  text, ts, te)
      when 'Cntrl';   self.emit(type, :cntrl,  text, ts, te)
      when 'Digit';   self.emit(type, :digit,  text, ts, te)
      when 'Graph';   self.emit(type, :graph,  text, ts, te)
      when 'Lower';   self.emit(type, :lower,  text, ts, te)
      when 'Print';   self.emit(type, :print,  text, ts, te)
      when 'Punct';   self.emit(type, :punct,  text, ts, te)
      when 'Space';   self.emit(type, :space,  text, ts, te)
      when 'Upper';   self.emit(type, :upper,  text, ts, te)
      when 'Word';    self.emit(type, :word,   text, ts, te)
      when 'Xdigit';  self.emit(type, :xdigit, text, ts, te)

      # Letters
      when 'Lu'; self.emit(type, :letter_uppercase, text, ts, te)
      when 'Ll'; self.emit(type, :letter_lowercase, text, ts, te)
      when 'Lt'; self.emit(type, :letter_titlecase, text, ts, te)
      when 'Lm'; self.emit(type, :letter_modifier,  text, ts, te)
      when 'Lo'; self.emit(type, :letter_other,     text, ts, te)

      # Marks
      when 'Mn'; self.emit(type, :mark_nonspacing,  text, ts, te)
      when 'Mc'; self.emit(type, :mark_spacing,     text, ts, te)
      when 'Me'; self.emit(type, :mark_enclosing,   text, ts, te)

      # Numbers
      when 'Nd'; self.emit(type, :number_decimal, text, ts, te)
      when 'Nl'; self.emit(type, :number_letter, text, ts, te)
      when 'No'; self.emit(type, :number_other, text, ts, te)

      # Punctuation
      when 'Pc'; self.emit(type, :punct_connector, text, ts, te)
      when 'Pd'; self.emit(type, :punct_dash, text, ts, te)
      when 'Ps'; self.emit(type, :punct_open, text, ts, te)
      when 'Pe'; self.emit(type, :punct_close, text, ts, te)
      when 'Pi'; self.emit(type, :punct_initial, text, ts, te)
      when 'Pf'; self.emit(type, :punct_final, text, ts, te)
      when 'Po'; self.emit(type, :punct_other, text, ts, te)

      # Symbols
      when 'Sm'; self.emit(type, :symbol_math, text, ts, te)
      when 'Sc'; self.emit(type, :symbol_currency, text, ts, te)
      when 'Sk'; self.emit(type, :symbol_modifier, text, ts, te)
      when 'So'; self.emit(type, :symbol_other, text, ts, te)

      # Separators
      when 'Zs'; self.emit(type, :separator_space, text, ts, te)
      when 'Zl'; self.emit(type, :separator_line, text, ts, te)
      when 'Zp'; self.emit(type, :separator_paragraph, text, ts, te)

      # Codepoints
      when 'Cc'; self.emit(type, :code_control, text, ts, te)
      when 'Cf'; self.emit(type, :code_format, text, ts, te)
      when 'Cs'; self.emit(type, :code_surrogate, text, ts, te)
      when 'Co'; self.emit(type, :code_private, text, ts, te)
      when 'Cn'; self.emit(type, :code_unassigned, text, ts, te)
      end
    };

    any > (escaped_alpha, 1)  {
      self.emit(:escape_sequence, :literal, data[ts..te-1].pack('c*'), ts, te)
      fret;
    };
  *|;

  # Main scanner
  main := |*

    # Anchors
    alternation {
      self.emit(:meta, :alternation, data[ts..te-1].pack('c*'), ts, te)
    };

    # Character types
    #   .         any (except new line)
    #   \d, \D    digit, non-digit
    #   \h, \H    hex, non-hex
    #   \s, \S    whitespace, non-whitespace
    #   \w, \W    word, non-word
    # ------------------------------------------------------------------------
    wild {
      self.emit(:character_type, :any, data[ts..te-1].pack('c*'), ts, te)
    };

    backslash . char_type > (backslashed, 2) {
      case text = data[ts..te-1].pack('c*')
      when '\\d'; self.emit(:character_type, :digit,      text, ts, te)
      when '\\D'; self.emit(:character_type, :non_digit,  text, ts, te)
      when '\\h'; self.emit(:character_type, :hex,        text, ts, te)
      when '\\H'; self.emit(:character_type, :non_hex,    text, ts, te)
      when '\\s'; self.emit(:character_type, :space,      text, ts, te)
      when '\\S'; self.emit(:character_type, :non_space,  text, ts, te)
      when '\\w'; self.emit(:character_type, :word,       text, ts, te)
      when '\\W'; self.emit(:character_type, :non_word,   text, ts, te)
      else raise "Unsupported character type at #{text} (char #{ts})"
      end
    };

    # Anchors
    beginning_of_line {
      self.emit(:anchor, :beginning_of_line, data[ts..te-1].pack('c*'), ts, te)
    };

    end_of_line {
      self.emit(:anchor, :end_of_line, data[ts..te-1].pack('c*'), ts, te)
    };

    backslash . anchor_char > (backslashed, 3) {
      case text = data[ts..te-1].pack('c*')
      when '\\A'; self.emit(:anchor, :bos,                text, ts, te)
      when '\\z'; self.emit(:anchor, :eos,                text, ts, te)
      when '\\Z'; self.emit(:anchor, :eos_or_before_eol,  text, ts, te)
      when '\\b'; self.emit(:anchor, :word_boundary,      text, ts, te)
      when '\\B'; self.emit(:anchor, :non_word_boundary,  text, ts, te)
      else raise "Unsupported anchor at #{text} (char #{ts})"
      end
    };

    # Escaped sequences
    backslash > (backslashed, 1) {
      fcall escape_sequence;
    };

    # Character sets
    set_open {
      self.emit(:character_set, :open, data[ts..te-1].pack('c*'), ts, te)
      fcall character_set;
    };

    # (?#...) comments: parsed as a single expression, without introducing a
    # new nesting level. Comments may not include parentheses, escaped or not.
    group_open . group_comment {
      self.emit(:group, :comment, data[ts..te-1].pack('c*'), ts, te)
    };

    # (?mix-mix...) expression options:
    #   (?imx-imx)          option on/off
    #                         i: ignore case
    #                         m: multi-line (dot(.) match newline)
    #                         x: extended form
    #
    #   (?imx-imx:subexp)   option on/off for subexp
    group_open . group_options > (grouped, 1) {
      self.emit(:group, :options, data[ts..te-1].pack('c*'), ts, te)
    };

    # Assertions
    #   (?=subexp)          look-ahead
    #   (?!subexp)          negative look-ahead
    #   (?<=subexp)         look-behind
    #   (?<!subexp)         negative look-behind
    # ------------------------------------------------------------------------
    group_open . assertion_type? > (grouped, 2) {
      case text =  data[ts..te-1].pack('c*')
      when '(?=';  self.emit(:assertion, :lookahead,    text, ts, te)
      when '(?!';  self.emit(:assertion, :nlookahead,   text, ts, te)
      when '(?<='; self.emit(:assertion, :lookbehind,   text, ts, te)
      when '(?<!'; self.emit(:assertion, :nlookbehind,  text, ts, te)
      end
    };

    # Groups
    #   (?:subexp)          passive (non-captured) group
    #   (?>subexp)          atomic group, don't backtrack in subexp.
    #   (?<name>subexp)     named group (single quotes are no supported, yet)
    #   (subexp)            captured group
    # ------------------------------------------------------------------------
    group_open . group_type? > (grouped, 3) {
      case text =  data[ts..te-1].pack('c*')
      when '(?:';  self.emit(:group, :passive,      text, ts, te)
      when '(?>';  self.emit(:group, :atomic,       text, ts, te)
      when '(?<[a-zA-Z]\w+>'
        self.emit(:group, :named, text, ts, te)
      else
        self.emit(:group, :capture, text, ts, te)
      end
    };

    group_close {
      self.emit(:group, :close, data[ts..te-1].pack('c*'), ts, te)
    };

    # Quantifiers
    # ------------------------------------------------------------------------
    zero_or_one {
      case text =  data[ts..te-1].pack('c*')
      when '?' ;  self.emit(:quantifier, :zero_or_one,            text, ts, te)
      when '??';  self.emit(:quantifier, :zero_or_one_reluctant,  text, ts, te)
      when '?+';  self.emit(:quantifier, :zero_or_one_possessive, text, ts, te)
      end
    };

    zero_or_more {
      case text =  data[ts..te-1].pack('c*')
      when '*' ;  self.emit(:quantifier, :zero_or_more,            text, ts, te)
      when '*?';  self.emit(:quantifier, :zero_or_more_reluctant,  text, ts, te)
      when '*+';  self.emit(:quantifier, :zero_or_more_possessive, text, ts, te)
      end
    };

    one_or_more {
      case text =  data[ts..te-1].pack('c*')
      when '+' ;  self.emit(:quantifier, :one_or_more,            text, ts, te)
      when '+?';  self.emit(:quantifier, :one_or_more_reluctant,  text, ts, te)
      when '++';  self.emit(:quantifier, :one_or_more_possessive, text, ts, te)
      end
    };

    # Intervals: min, max, and exact notations
    # ------------------------------------------------------------------------
    range_open . (digit+)? . ','? . (digit+)? . range_close . quantifier_mode? {
      self.emit(:quantifier, :interval, data[ts..te-1].pack('c*'), ts, te)
    };

    # Literal: anything, except meta characters. This includes 2, 3, and 4
    # unicode byte sequences.
    # ------------------------------------------------------------------------
    (any - meta_char)+ {
      self.emit(:literal, :literal, data[ts..te-1].pack('c*'), ts, te)
    };

  *|;
}%%


DRY up attributes machine.

DRY up attributes machine.

/*
 * redcloth_attributes.rl
 *
 * Copyright (C) 2008 Jason Garber
 */
import java.io.IOException;

import org.jruby.Ruby;
import org.jruby.RubyArray;
import org.jruby.RubyClass;
import org.jruby.RubyHash;
import org.jruby.RubyModule;
import org.jruby.RubyNumeric;
import org.jruby.RubyObject;
import org.jruby.RubyString;
import org.jruby.RubySymbol;
import org.jruby.anno.JRubyMethod;
import org.jruby.runtime.Block;
import org.jruby.runtime.CallbackFactory;
import org.jruby.runtime.builtin.IRubyObject;
import org.jruby.exceptions.RaiseException;
import org.jruby.runtime.load.BasicLibraryService;

import org.jruby.util.ByteList;

public class RedclothAttributes extends RedclothScanService.Base {

%%{

  machine redcloth_attributes;
  include redcloth_common "redcloth_common.java.rl";
  include redcloth_attributes "redcloth_attributes.rl";

}%%

%% write data nofinal;

  public void SET_ATTRIBUTES() {
    SET_ATTRIBUTE("class_buf", "class");
    SET_ATTRIBUTE("id_buf", "id");
    SET_ATTRIBUTE("lang_buf", "lang");
    SET_ATTRIBUTE("style_buf", "style");
  }

  public void SET_ATTRIBUTE(String B, String A) {
    buf = ((RubyHash)regs).aref(runtime.newSymbol(B));
    if(!buf.isNil()) {
      ((RubyHash)regs).aset(runtime.newSymbol(A), buf);
    }
  }

  private int machine;
  private IRubyObject buf;

  public RedclothAttributes(int machine, IRubyObject self, byte[] data, int p, int pe) {
    this.runtime = self.getRuntime();
    this.self = self;

    // This is GROSS but necessary for EOF matching
    this.data = new byte[pe+1];
    System.arraycopy(data, p, this.data, 0, pe);
    this.data[pe] = 0;

    this.p = 0;
    this.pe = pe+1;
    this.eof = this.pe;
    this.orig_p = 0;
    this.orig_pe = this.pe;

    this.regs = RubyHash.newHash(runtime);
    this.buf = runtime.getNil();
    this.machine = machine;
  }

  public IRubyObject parse() {
    %% write init;

    cs = machine;

    %% write exec;

    return regs;
  }

  public static IRubyObject attributes(IRubyObject self, IRubyObject str) {
    ByteList bl = str.convertToString().getByteList();
    int cs = redcloth_attributes_en_inline;
    return new RedclothAttributes(cs, self, bl.bytes, bl.begin, bl.realSize).parse();
  }

  public static IRubyObject link_attributes(IRubyObject self, IRubyObject str) {
    ByteList bl = str.convertToString().getByteList();
    int cs = redcloth_attributes_en_link_says;
    return new RedclothAttributes(cs, self, bl.bytes, bl.begin, bl.realSize).parse();
  }
}

/*
 * redcloth_attributes.rl
 *
 * Copyright (C) 2008 Jason Garber
 */
import java.io.IOException;

import org.jruby.Ruby;
import org.jruby.RubyArray;
import org.jruby.RubyClass;
import org.jruby.RubyHash;
import org.jruby.RubyModule;
import org.jruby.RubyNumeric;
import org.jruby.RubyObject;
import org.jruby.RubyString;
import org.jruby.RubySymbol;
import org.jruby.anno.JRubyMethod;
import org.jruby.runtime.Block;
import org.jruby.runtime.CallbackFactory;
import org.jruby.runtime.builtin.IRubyObject;
import org.jruby.exceptions.RaiseException;
import org.jruby.runtime.load.BasicLibraryService;

import org.jruby.util.ByteList;

public class RedclothAttributes extends RedclothScanService.Base {

%%{

  machine redcloth_attributes;
  include redcloth_common "redcloth_common.java.rl";

  C2_CLAS = ( "(" ( [^)#]+ >A %{ STORE("class_buf"); } )? ("#" [^)]+ >A %{STORE("id_buf");} )? ")" ) ;
  C2_LNGE = ( "[" [^\]]+ >A %{ STORE("lang_buf"); } "]" ) ;
  C2_STYL = ( "{" [^}]+ >A %{ STORE("style_buf"); } "}" ) ;
  C2 = ( C2_CLAS | C2_STYL | C2_LNGE )+ ;

  mtext_with_attributes = ( C2 mtext >A %T ) >X ;

  inline := |*

    mtext_with_attributes { SET_ATTRIBUTES(); } ;

  *|;

  link_text_with_attributes = C2 "."* " "* ( mtext+ ) >A %{ STORE("name"); } ;
  link_text_without_attributes = ( mtext+ ) >B %{ STORE_B("name_without_attributes"); } ;

  link_says := |*

    link_text_with_attributes { SET_ATTRIBUTES(); } ;
    link_text_without_attributes { SET_ATTRIBUTE("name_without_attributes", "name"); } ;

  *|;

}%%

%% write data nofinal;

  public void SET_ATTRIBUTES() {
    SET_ATTRIBUTE("class_buf", "class");
    SET_ATTRIBUTE("id_buf", "id");
    SET_ATTRIBUTE("lang_buf", "lang");
    SET_ATTRIBUTE("style_buf", "style");
  }

  public void SET_ATTRIBUTE(String B, String A) {
    buf = ((RubyHash)regs).aref(runtime.newSymbol(B));
    if(!buf.isNil()) {
      ((RubyHash)regs).aset(runtime.newSymbol(A), buf);
    }
  }

  private int machine;
  private IRubyObject buf;

  public RedclothAttributes(int machine, IRubyObject self, byte[] data, int p, int pe) {
    this.runtime = self.getRuntime();
    this.self = self;

    // This is GROSS but necessary for EOF matching
    this.data = new byte[pe+1];
    System.arraycopy(data, p, this.data, 0, pe);
    this.data[pe] = 0;

    this.p = 0;
    this.pe = pe+1;
    this.eof = this.pe;
    this.orig_p = 0;
    this.orig_pe = this.pe;

    this.regs = RubyHash.newHash(runtime);
    this.buf = runtime.getNil();
    this.machine = machine;
  }

  public IRubyObject parse() {
    %% write init;

    cs = machine;

    %% write exec;

    return regs;
  }

  public static IRubyObject attributes(IRubyObject self, IRubyObject str) {
    ByteList bl = str.convertToString().getByteList();
    int cs = redcloth_attributes_en_inline;
    return new RedclothAttributes(cs, self, bl.bytes, bl.begin, bl.realSize).parse();
  }

  public static IRubyObject link_attributes(IRubyObject self, IRubyObject str) {
    ByteList bl = str.convertToString().getByteList();
    int cs = redcloth_attributes_en_link_says;
    return new RedclothAttributes(cs, self, bl.bytes, bl.begin, bl.realSize).parse();
  }
}