Work around gcc strict-aliasing warning.

/* This file was generated by Ragel. Your edits will be lost.
 *
 * This is a state machine implementation of Array#pack.
 *
 * vim: filetype=cpp
 */

#include "vm/config.h"

#include "vm.hpp"
#include "object_utils.hpp"
#include "on_stack.hpp"

#include "builtin/array.hpp"
#include "builtin/exception.hpp"
#include "builtin/float.hpp"
#include "builtin/module.hpp"
#include "builtin/object.hpp"
#include "builtin/string.hpp"

namespace rubinius {
  namespace pack {
    static Object* integer(STATE, CallFrame* call_frame, Object* obj) {
      Array* args = Array::create(state, 1);
      args->set(state, 0, obj);

      return G(rubinius)->send(state, call_frame, state->symbol("pack_to_int"), args);
    }

    static String* string_or_nil(STATE, CallFrame* call_frame, Object* obj) {
      Array* args = Array::create(state, 1);
      args->set(state, 0, obj);

      Object* result = G(rubinius)->send(state, call_frame,
            state->symbol("pack_to_str_or_nil"), args);

      if(!result) return 0;
      return as<String>(result);
    }

    static Object* float_t(STATE, CallFrame* call_frame, Object* obj) {
      Array* args = Array::create(state, 1);
      args->set(state, 0, obj);

      return G(rubinius)->send(state, call_frame, state->symbol("pack_to_float"), args);
    }

    inline uint16_t swap16(uint16_t x) {
      return (((x & 0x00ff)<<8) | ((x & 0xff00)>>8));
    }

    inline uint32_t swap32(uint32_t x) {
      return (((x & 0x000000ff) << 24)
             |((x & 0xff000000) >> 24)
             |((x & 0x0000ff00) << 8)
             |((x & 0x00ff0000) >> 8));
    }

    inline uint64_t swap64(uint64_t x) {
      return (((x & 0x00000000000000ffLL) << 56)
             |((x & 0xff00000000000000LL) >> 56)
             |((x & 0x000000000000ff00LL) << 40)
             |((x & 0x00ff000000000000LL) >> 40)
             |((x & 0x0000000000ff0000LL) << 24)
             |((x & 0x0000ff0000000000LL) >> 24)
             |((x & 0x00000000ff000000LL) << 8)
             |((x & 0x000000ff00000000LL) >> 8));
    }

    inline static void swapf(std::string& str, float value) {
      uint32_t x;

      memcpy(&x, &value, sizeof(float));
      x = swap32(x);

      str.append((const char*)&x, sizeof(uint32_t));
    }

    inline static void swapd(std::string& str, double value) {
      uint64_t x;

      memcpy(&x, &value, sizeof(double));
      x = swap64(x);

      str.append((const char*)&x, sizeof(uint64_t));
    }

    inline static void double_element(std::string& str, double value) {
      str.append((const char*)&value, sizeof(double));
    }

    inline static void float_element(std::string& str, float value) {
      str.append((const char*)&value, sizeof(float));
    }
  }

#define BITS_LONG   (RBX_SIZEOF_LONG * 8)
#define BITS_64     (64)

#define CONVERT_INTEGER(T, v, m, b, n)    \
  if((n)->fixnum_p()) {                   \
    v = (T)STRIP_FIXNUM_TAG(n);           \
  } else {                                \
    Bignum* big = as<Bignum>(n);          \
    big->verify_size(state, b);           \
    v = big->m();                         \
  }

#define CONVERT_TO_INT(n)   CONVERT_INTEGER(int, int_value, to_int, BITS_LONG, n)
#define CONVERT_TO_LONG(n)  CONVERT_INTEGER(long long, long_value, to_long_long, BITS_64, n)

#define PACK_INT_ELEMENTS(mask)   PACK_ELEMENTS(Integer, pack::integer, INT, mask)
#define PACK_LONG_ELEMENTS(mask)  PACK_ELEMENTS(Integer, pack::integer, LONG, mask)

#define pack_float_elements(format)   pack_elements(Float, pack::float_t, format)

#define pack_double_le                pack_float_elements(pack_double_element_le)
#define pack_double_be                pack_float_elements(pack_double_element_be)

#define pack_float_le                 pack_float_elements(pack_float_element_le)
#define pack_float_be                 pack_float_elements(pack_float_element_be)

#define pack_elements(T, coerce, format)        \
  for(; index < stop; index++) {                \
    Object* item = self->get(state, index);     \
    T* value = try_as<T>(item);                 \
    if(!value) {                                \
      item = coerce(state, call_frame, item);   \
      if(!item) return 0;                       \
      value = as<T>(item);                      \
    }                                           \
    format(value);                              \
  }

#define PACK_ELEMENTS(T, coerce, size, format)  \
  for(; index < stop; index++) {                \
    Object* item = self->get(state, index);     \
    T* value = try_as<T>(item);                 \
    if(!value) {                                \
      item = coerce(state, call_frame, item);   \
      if(!item) return 0;                       \
      value = as<T>(item);                      \
    }                                           \
    CONVERT_TO_ ## size(value);                 \
    format;                                     \
  }

#define PACK_STRING_ELEMENT(coerce)  {                      \
  Object* item = self->get(state, index);                   \
  String* value = try_as<String>(item);                     \
  if(!value) {                                              \
    value = coerce(state, call_frame, item);                \
    if(!value) return 0;                                    \
  }                                                         \
  if(RTEST(value->tainted_p(state))) tainted = true;        \
  size_t size = value->size();                              \
  if(rest) count = size;                                    \
  if(count <= size) {                                       \
    str.append((const char*)value->byte_address(), count);  \
    count = 0;                                              \
  } else {                                                  \
    str.append((const char*)value->byte_address(), size);   \
    count = count - size;                                   \
  }                                                         \
  index++;                                                  \
}

#define BYTE1(x)        (((x) & 0x00000000000000ff))
#define BYTE2(x)        (((x) & 0x000000000000ff00) >> 8)
#define BYTE3(x)        (((x) & 0x0000000000ff0000) >> 16)
#define BYTE4(x)        (((x) & 0x00000000ff000000) >> 24)

#define BYTE5(x)        (((x) & 0x000000ff00000000LL) >> 32)
#define BYTE6(x)        (((x) & 0x0000ff0000000000LL) >> 40)
#define BYTE7(x)        (((x) & 0x00ff000000000000LL) >> 48)
#define BYTE8(x)        (((x) & 0xff00000000000000LL) >> 56)

#ifdef RBX_LITTLE_ENDIAN
# define MASK_16BITS     LE_MASK_16BITS
# define MASK_32BITS     LE_MASK_32BITS
# define MASK_64BITS     LE_MASK_64BITS

# define pack_double_element_le(v)  (pack::double_element(str, (v)->val))
# define pack_double_element_be(v)  (pack::swapd(str, (v)->val))
# define pack_double                pack_double_le

# define pack_float_element_le(v)   (pack::float_element(str, (v)->val))
# define pack_float_element_be(v)   (pack::swapf(str, (v)->val))
# define pack_float                 pack_float_le
#else
# define MASK_16BITS     BE_MASK_16BITS
# define MASK_32BITS     BE_MASK_32BITS
# define MASK_64BITS     BE_MASK_64BITS

# define pack_double_element_le(v)  (pack::swapd(str, (v)->val))
# define pack_double_element_be(v)  (pack::double_element(str, (v)->val))
# define pack_double                pack_double_be

# define pack_float_element_le(v)   (pack::swapf(str, (v)->val))
# define pack_float_element_be(v)   (pack::float_element(str, (v)->val))
# define pack_float                 pack_float_be
#endif

#define LE_MASK_64BITS              \
  str.push_back(BYTE1(long_value)); \
  str.push_back(BYTE2(long_value)); \
  str.push_back(BYTE3(long_value)); \
  str.push_back(BYTE4(long_value)); \
  str.push_back(BYTE5(long_value)); \
  str.push_back(BYTE6(long_value)); \
  str.push_back(BYTE7(long_value)); \
  str.push_back(BYTE8(long_value)); \

#define BE_MASK_64BITS              \
  str.push_back(BYTE8(long_value)); \
  str.push_back(BYTE7(long_value)); \
  str.push_back(BYTE6(long_value)); \
  str.push_back(BYTE5(long_value)); \
  str.push_back(BYTE4(long_value)); \
  str.push_back(BYTE3(long_value)); \
  str.push_back(BYTE2(long_value)); \
  str.push_back(BYTE1(long_value)); \

#define LE_MASK_32BITS             \
  str.push_back(BYTE1(int_value)); \
  str.push_back(BYTE2(int_value)); \
  str.push_back(BYTE3(int_value)); \
  str.push_back(BYTE4(int_value)); \

#define BE_MASK_32BITS             \
  str.push_back(BYTE4(int_value)); \
  str.push_back(BYTE3(int_value)); \
  str.push_back(BYTE2(int_value)); \
  str.push_back(BYTE1(int_value)); \

#define LE_MASK_16BITS             \
  str.push_back(BYTE1(int_value)); \
  str.push_back(BYTE2(int_value)); \

#define BE_MASK_16BITS             \
  str.push_back(BYTE2(int_value)); \
  str.push_back(BYTE1(int_value)); \

#define MASK_BYTE                  \
  str.push_back(BYTE1(int_value));

  String* Array::pack(STATE, String* directives, CallFrame* call_frame) {
    // Ragel-specific variables
    std::string d(directives->c_str(), directives->size());
    const char *p  = d.c_str();
    const char *pe = p + d.size();
    const char *eof = pe;
    int cs;

    // pack-specific variables
    Array* self = this;
    OnStack<1> sv(state, self);

    size_t index = 0;
    size_t count = 0;
    size_t stop = 0;
    bool rest = false;
    bool platform = false;
    bool tainted = false;

    int int_value = 0;
    long long long_value = 0;
    std::string str("");

    // Use information we have to reduce repeated allocation.
    str.reserve(size() * 4);

%%{

  machine pack;

  include "pack.rl";

}%%

    if(pack_first_final && pack_error && pack_en_main) {
      // do nothing
    }

    return force_as<String>(Primitives::failure());
  }
}

/* This file was generated by Ragel. Your edits will be lost.
 *
 * This is a state machine implementation of Array#pack.
 *
 * vim: filetype=cpp
 */

#include "vm/config.h"

#include "vm.hpp"
#include "object_utils.hpp"
#include "on_stack.hpp"

#include "builtin/array.hpp"
#include "builtin/exception.hpp"
#include "builtin/float.hpp"
#include "builtin/module.hpp"
#include "builtin/object.hpp"
#include "builtin/string.hpp"

namespace rubinius {
  namespace pack {
    static Object* integer(STATE, CallFrame* call_frame, Object* obj) {
      Array* args = Array::create(state, 1);
      args->set(state, 0, obj);

      return G(rubinius)->send(state, call_frame, state->symbol("pack_to_int"), args);
    }

    static String* string_or_nil(STATE, CallFrame* call_frame, Object* obj) {
      Array* args = Array::create(state, 1);
      args->set(state, 0, obj);

      Object* result = G(rubinius)->send(state, call_frame,
            state->symbol("pack_to_str_or_nil"), args);

      if(!result) return 0;
      return as<String>(result);
    }

    static Object* float_t(STATE, CallFrame* call_frame, Object* obj) {
      Array* args = Array::create(state, 1);
      args->set(state, 0, obj);

      return G(rubinius)->send(state, call_frame, state->symbol("pack_to_float"), args);
    }

    inline uint16_t swap16(uint16_t x) {
      return (((x & 0x00ff)<<8) | ((x & 0xff00)>>8));
    }

    inline uint32_t swap32(uint32_t x) {
      return (((x & 0x000000ff) << 24)
             |((x & 0xff000000) >> 24)
             |((x & 0x0000ff00) << 8)
             |((x & 0x00ff0000) >> 8));
    }

    inline uint64_t swap64(uint64_t x) {
      return (((x & 0x00000000000000ffLL) << 56)
             |((x & 0xff00000000000000LL) >> 56)
             |((x & 0x000000000000ff00LL) << 40)
             |((x & 0x00ff000000000000LL) >> 40)
             |((x & 0x0000000000ff0000LL) << 24)
             |((x & 0x0000ff0000000000LL) >> 24)
             |((x & 0x00000000ff000000LL) << 8)
             |((x & 0x000000ff00000000LL) >> 8));
    }

    inline static void swapf(std::string& str, float value) {
      uint32_t x = swap32(*(uint32_t*)(&value));

      str.append((const char*)&x, sizeof(uint32_t));
    }

    inline static void swapd(std::string& str, double value) {
      uint64_t x = swap64(*(uint64_t*)(&value));

      str.append((const char*)&x, sizeof(uint64_t));
    }

    inline static void double_element(std::string& str, double value) {
      str.append((const char*)&value, sizeof(double));
    }

    inline static void float_element(std::string& str, float value) {
      str.append((const char*)&value, sizeof(float));
    }
  }

#define BITS_LONG   (RBX_SIZEOF_LONG * 8)
#define BITS_64     (64)

#define CONVERT_INTEGER(T, v, m, b, n)    \
  if((n)->fixnum_p()) {                   \
    v = (T)STRIP_FIXNUM_TAG(n);           \
  } else {                                \
    Bignum* big = as<Bignum>(n);          \
    big->verify_size(state, b);           \
    v = big->m();                         \
  }

#define CONVERT_TO_INT(n)   CONVERT_INTEGER(int, int_value, to_int, BITS_LONG, n)
#define CONVERT_TO_LONG(n)  CONVERT_INTEGER(long long, long_value, to_long_long, BITS_64, n)

#define PACK_INT_ELEMENTS(mask)   PACK_ELEMENTS(Integer, pack::integer, INT, mask)
#define PACK_LONG_ELEMENTS(mask)  PACK_ELEMENTS(Integer, pack::integer, LONG, mask)

#define pack_float_elements(format)   pack_elements(Float, pack::float_t, format)

#define pack_double_le                pack_float_elements(pack_double_element_le)
#define pack_double_be                pack_float_elements(pack_double_element_be)

#define pack_float_le                 pack_float_elements(pack_float_element_le)
#define pack_float_be                 pack_float_elements(pack_float_element_be)

#define pack_elements(T, coerce, format)        \
  for(; index < stop; index++) {                \
    Object* item = self->get(state, index);     \
    T* value = try_as<T>(item);                 \
    if(!value) {                                \
      item = coerce(state, call_frame, item);   \
      if(!item) return 0;                       \
      value = as<T>(item);                      \
    }                                           \
    format(value);                              \
  }

#define PACK_ELEMENTS(T, coerce, size, format)  \
  for(; index < stop; index++) {                \
    Object* item = self->get(state, index);     \
    T* value = try_as<T>(item);                 \
    if(!value) {                                \
      item = coerce(state, call_frame, item);   \
      if(!item) return 0;                       \
      value = as<T>(item);                      \
    }                                           \
    CONVERT_TO_ ## size(value);                 \
    format;                                     \
  }

#define PACK_STRING_ELEMENT(coerce)  {                      \
  Object* item = self->get(state, index);                   \
  String* value = try_as<String>(item);                     \
  if(!value) {                                              \
    value = coerce(state, call_frame, item);                \
    if(!value) return 0;                                    \
  }                                                         \
  if(RTEST(value->tainted_p(state))) tainted = true;        \
  size_t size = value->size();                              \
  if(rest) count = size;                                    \
  if(count <= size) {                                       \
    str.append((const char*)value->byte_address(), count);  \
    count = 0;                                              \
  } else {                                                  \
    str.append((const char*)value->byte_address(), size);   \
    count = count - size;                                   \
  }                                                         \
  index++;                                                  \
}

#define BYTE1(x)        (((x) & 0x00000000000000ff))
#define BYTE2(x)        (((x) & 0x000000000000ff00) >> 8)
#define BYTE3(x)        (((x) & 0x0000000000ff0000) >> 16)
#define BYTE4(x)        (((x) & 0x00000000ff000000) >> 24)

#define BYTE5(x)        (((x) & 0x000000ff00000000LL) >> 32)
#define BYTE6(x)        (((x) & 0x0000ff0000000000LL) >> 40)
#define BYTE7(x)        (((x) & 0x00ff000000000000LL) >> 48)
#define BYTE8(x)        (((x) & 0xff00000000000000LL) >> 56)

#ifdef RBX_LITTLE_ENDIAN
# define MASK_16BITS     LE_MASK_16BITS
# define MASK_32BITS     LE_MASK_32BITS
# define MASK_64BITS     LE_MASK_64BITS

# define pack_double_element_le(v)  (pack::double_element(str, (v)->val))
# define pack_double_element_be(v)  (pack::swapd(str, (v)->val))
# define pack_double                pack_double_le

# define pack_float_element_le(v)   (pack::float_element(str, (v)->val))
# define pack_float_element_be(v)   (pack::swapf(str, (v)->val))
# define pack_float                 pack_float_le
#else
# define MASK_16BITS     BE_MASK_16BITS
# define MASK_32BITS     BE_MASK_32BITS
# define MASK_64BITS     BE_MASK_64BITS

# define pack_double_element_le(v)  (pack::swapd(str, (v)->val))
# define pack_double_element_be(v)  (pack::double_element(str, (v)->val))
# define pack_double                pack_double_be

# define pack_float_element_le(v)   (pack::swapf(str, (v)->val))
# define pack_float_element_be(v)   (pack::float_element(str, (v)->val))
# define pack_float                 pack_float_be
#endif

#define LE_MASK_64BITS              \
  str.push_back(BYTE1(long_value)); \
  str.push_back(BYTE2(long_value)); \
  str.push_back(BYTE3(long_value)); \
  str.push_back(BYTE4(long_value)); \
  str.push_back(BYTE5(long_value)); \
  str.push_back(BYTE6(long_value)); \
  str.push_back(BYTE7(long_value)); \
  str.push_back(BYTE8(long_value)); \

#define BE_MASK_64BITS              \
  str.push_back(BYTE8(long_value)); \
  str.push_back(BYTE7(long_value)); \
  str.push_back(BYTE6(long_value)); \
  str.push_back(BYTE5(long_value)); \
  str.push_back(BYTE4(long_value)); \
  str.push_back(BYTE3(long_value)); \
  str.push_back(BYTE2(long_value)); \
  str.push_back(BYTE1(long_value)); \

#define LE_MASK_32BITS             \
  str.push_back(BYTE1(int_value)); \
  str.push_back(BYTE2(int_value)); \
  str.push_back(BYTE3(int_value)); \
  str.push_back(BYTE4(int_value)); \

#define BE_MASK_32BITS             \
  str.push_back(BYTE4(int_value)); \
  str.push_back(BYTE3(int_value)); \
  str.push_back(BYTE2(int_value)); \
  str.push_back(BYTE1(int_value)); \

#define LE_MASK_16BITS             \
  str.push_back(BYTE1(int_value)); \
  str.push_back(BYTE2(int_value)); \

#define BE_MASK_16BITS             \
  str.push_back(BYTE2(int_value)); \
  str.push_back(BYTE1(int_value)); \

#define MASK_BYTE                  \
  str.push_back(BYTE1(int_value));

  String* Array::pack(STATE, String* directives, CallFrame* call_frame) {
    // Ragel-specific variables
    std::string d(directives->c_str(), directives->size());
    const char *p  = d.c_str();
    const char *pe = p + d.size();
    const char *eof = pe;
    int cs;

    // pack-specific variables
    Array* self = this;
    OnStack<1> sv(state, self);

    size_t index = 0;
    size_t count = 0;
    size_t stop = 0;
    bool rest = false;
    bool platform = false;
    bool tainted = false;

    int int_value = 0;
    long long long_value = 0;
    std::string str("");

    // Use information we have to reduce repeated allocation.
    str.reserve(size() * 4);

%%{

  machine pack;

  include "pack.rl";

}%%

    if(pack_first_final && pack_error && pack_en_main) {
      // do nothing
    }

    return force_as<String>(Primitives::failure());
  }
}


Added missing write init.

Added missing write init.

#include <stdio.h>
#include <stdlib.h>
#include <string.h>

#define BUFSIZE 4096

typedef struct _Scanner {
	/* Scanner state. */
    int cs;
    int act;
    int have;
    int curline;
    char *tokstart;
    char *tokend;
    char *p;
    char *pe;
	FILE *file;
	int done;

	/* Token data */
	char *data;
	int len;
    int value;

	char buf[BUFSIZE];
} Scanner;

%%{
	machine Scanner;
	write data;
}%%

void scan_init( Scanner *s, FILE *file )
{
	memset (s, '\0', sizeof(Scanner));
	s->curline = 1;
	s->file = file;
	%% write init;
}

#define TK_NO_TOKEN (-1)
#define TK_ERR 128
#define TK_EOF 129
#define TK_Identifier 130
#define TK_Number 131

#define ret_tok( _tok ) token = _tok; s->data = s->tokstart

int scan( Scanner *s )
{
	char *p = s->p;
	char *pe = s->pe;
	int token = TK_NO_TOKEN;
	int space, readlen;

	while ( 1 ) {
		if ( p == pe ) {
			printf("scanner: need more data\n");

			if ( s->tokstart == 0 )
				s->have = 0;
			else {
				/* There is data that needs to be shifted over. */
				printf("scanner: buffer broken mid token\n");
				s->have = pe - s->tokstart;
				memmove( s->buf, s->tokstart, s->have );
				s->tokend -= (s->tokstart-s->buf);
				s->tokstart = s->buf;
			}

			p = s->buf + s->have;
			space = BUFSIZE - s->have;

			if ( space == 0 ) {
				/* We filled up the buffer trying to scan a token. */
				printf("scanner: out of buffer space\n");
				return TK_ERR;
			}

			if ( s->done ) {
				printf("scanner: end of file\n");
				p[0] = 0;
				readlen = 1;
			}
			else {
				readlen = fread( p, 1, space, s->file );
				if ( readlen < space )
					s->done = 1;
			}

			pe = p + readlen;
		}

		%%{
			machine Scanner;
			access s->;

			main := |*

			# Identifiers
			( [a-zA-Z_] [a-zA-Z0-9_]* ) =>
				{ ret_tok( TK_Identifier ); fbreak; };

			# Whitespace
			[ \t\n];

			'"' ( [^\\"] | '\\' any ) * '"';

			# Number
			digit+ =>
				{ ret_tok( TK_Number ); fbreak; };

			# EOF
			0 =>
				{ ret_tok( TK_EOF ); fbreak; };

			# Anything else
			any =>
				{ ret_tok( *p ); fbreak; };

			*|;

			write exec;
		}%%

		if ( s->cs == Scanner_error )
			return TK_ERR;

		if ( token != TK_NO_TOKEN ) {
			/* Save p and pe. fbreak does not advance p. */
			s->p = p + 1;
			s->pe = pe;
			s->len = s->p - s->data;
			return token;
		}
	}
}

int main (int argc, char** argv)
{
	Scanner ss;
	int tok;

	scan_init(&ss, stdin);

	while ( 1 ) {
		tok = scan (&ss);
		if ( tok == TK_EOF ) {
			printf ("parser: EOF\n");
			break;
		}
		else if ( tok == TK_ERR ) {
			printf ("parser: ERR\n");
			break;
		}
		else {
			printf ("parser: %d \"", tok);
			fwrite ( ss.data, 1, ss.len, stdout );
			printf ("\"\n" );
		}
	}

	return 0;
}

#include <stdio.h>
#include <stdlib.h>
#include <string.h>

#define BUFSIZE 4096

typedef struct _Scanner {
	/* Scanner state. */
    int cs;
    int act;
    int have;
    int curline;
    char *tokstart;
    char *tokend;
    char *p;
    char *pe;
	FILE *file;
	int done;

	/* Token data */
	char *data;
	int len;
    int value;

	char buf[BUFSIZE];
} Scanner;

void scan_init( Scanner *s, FILE *file )
{
	memset (s, '\0', sizeof(Scanner));
	s->curline = 1;
	s->file = file;
}

#define TK_NO_TOKEN (-1)
#define TK_ERR 128
#define TK_EOF 129
#define TK_Identifier 130
#define TK_Number 131

%%{
	machine Scanner;
	write data;
}%%

#define ret_tok( _tok ) token = _tok; s->data = s->tokstart

int scan( Scanner *s )
{
	char *p = s->p;
	char *pe = s->pe;
	int token = TK_NO_TOKEN;
	int space, readlen;

	while ( 1 ) {
		if ( p == pe ) {
			printf("scanner: need more data\n");

			if ( s->tokstart == 0 )
				s->have = 0;
			else {
				/* There is data that needs to be shifted over. */
				printf("scanner: buffer broken mid token\n");
				s->have = pe - s->tokstart;
				memmove( s->buf, s->tokstart, s->have );
				s->tokend -= (s->tokstart-s->buf);
				s->tokstart = s->buf;
			}

			p = s->buf + s->have;
			space = BUFSIZE - s->have;

			if ( space == 0 ) {
				/* We filled up the buffer trying to scan a token. */
				printf("scanner: out of buffer space\n");
				return TK_ERR;
			}

			if ( s->done ) {
				printf("scanner: end of file\n");
				p[0] = 0;
				readlen = 1;
			}
			else {
				readlen = fread( p, 1, space, s->file );
				if ( readlen < space )
					s->done = 1;
			}

			pe = p + readlen;
		}

		%%{
			machine Scanner;
			access s->;

			main := |*

			# Identifiers
			( [a-zA-Z_] [a-zA-Z0-9_]* ) =>
				{ ret_tok( TK_Identifier ); fbreak; };

			# Whitespace
			[ \t\n];

			# Number
			digit+ =>
				{ ret_tok( TK_Number ); fbreak; };

			# EOF
			0 =>
				{ ret_tok( TK_EOF ); fbreak; };

			# Anything else
			any =>
				{ ret_tok( *p ); fbreak; };

			*|;

			write exec;
		}%%

		if ( s->cs == Scanner_error )
			return TK_ERR;

		if ( token != TK_NO_TOKEN ) {
			/* Save p and pe. fbreak does not advance p. */
			s->p = p + 1;
			s->pe = pe;
			s->len = s->p - s->data;
			return token;
		}
	}
}

int main (int argc, char** argv)
{
	Scanner ss;
	int tok;

	scan_init(&ss, stdin);

	while ( 1 ) {
		tok = scan (&ss);
		if ( tok == TK_EOF ) {
			printf ("parser: EOF\n");
			break;
		}
		else if ( tok == TK_ERR ) {
			printf ("parser: ERR\n");
			break;
		}
		else {
			printf ("parser: %d \"", tok);
			fwrite ( ss.data, 1, ss.len, stdout );
			printf ("\"\n" );
		}
	}

	return 0;
}


Indentation fix in the lexer.

Indentation fix in the lexer.

%%machine lexer; # %

module Oga
  module XML
    ##
    # Low level lexer that supports both XML and HTML (using an extra option).
    # To lex HTML input set the `:html` option to `true` when creating an
    # instance of the lexer:
    #
    #     lexer = Oga::XML::Lexer.new(:html => true)
    #
    # @!attribute [r] html
    #  @return [TrueClass|FalseClass]
    #
    # @!attribute [r] tokens
    #  @return [Array]
    #
    class Lexer
      %% write data;

      # % fix highlight

      attr_reader :html

      ##
      # Names of the HTML void elements that should be handled when HTML lexing
      # is enabled.
      #
      # @return [Set]
      #
      HTML_VOID_ELEMENTS = Set.new([
        'area',
        'base',
        'br',
        'col',
        'command',
        'embed',
        'hr',
        'img',
        'input',
        'keygen',
        'link',
        'meta',
        'param',
        'source',
        'track',
        'wbr'
      ])

      ##
      # @param [String] data The data to lex.
      #
      # @param [Hash] options
      #
      # @option options [Symbol] :html When set to `true` the lexer will treat
      # the input as HTML instead of SGML/XML. This makes it possible to lex
      # HTML void elements such as `<link href="">`.
      #
      def initialize(data, options = {})
        options.each do |key, value|
          instance_variable_set("@#{key}", value) if respond_to?(key)
        end

        @data = data

        reset
      end

      ##
      # Resets the internal state of the lexer. Typically you don't need to
      # call this method yourself as its called by #lex after lexing a given
      # String.
      #
      def reset
        @line     = 1
        @elements = []

        @buffer_start_position = nil
      end

      ##
      # Gathers all the tokens for the input and returns them as an Array.
      #
      # This method resets the internal state of the lexer after consuming the
      # input.
      #
      # @param [String] data The string to consume.
      # @return [Array]
      # @see #advance
      #
      def lex
        tokens = []

        advance do |token|
          tokens << token
        end

        reset

        return tokens
      end

      ##
      # Advances through the input and generates the corresponding tokens. Each
      # token is yielded to the supplied block.
      #
      # Each token is an Array in the following format:
      #
      #     [TYPE, VALUE]
      #
      # The type is a symbol, the value is either nil or a String.
      #
      # This method stores the supplied block in `@block` and resets it after
      # the lexer loop has finished.
      #
      # This method does *not* reset the internal state of the lexer.
      #
      #
      # @param [String] data The String to consume.
      # @return [Array]
      #
      def advance(&block)
        @block = block

        data  = @data
        ts    = nil
        te    = nil
        stack = []
        top   = 0
        cs    = self.class.lexer_start
        act   = 0
        eof   = @data.bytesize
        p     = 0
        pe    = eof

        _lexer_eof_trans          = self.class.send(:_lexer_eof_trans)
        _lexer_from_state_actions = self.class.send(:_lexer_from_state_actions)
        _lexer_index_offsets      = self.class.send(:_lexer_index_offsets)
        _lexer_indicies           = self.class.send(:_lexer_indicies)
        _lexer_key_spans          = self.class.send(:_lexer_key_spans)
        _lexer_to_state_actions   = self.class.send(:_lexer_to_state_actions)
        _lexer_trans_actions      = self.class.send(:_lexer_trans_actions)
        _lexer_trans_keys         = self.class.send(:_lexer_trans_keys)
        _lexer_trans_targs        = self.class.send(:_lexer_trans_targs)

        %% write exec;

        # % fix highlight
      ensure
        @block = nil
      end

      ##
      # @return [TrueClass|FalseClass]
      #
      def html?
        return !!html
      end

      private

      ##
      # @param [Fixnum] amount The amount of lines to advance.
      #
      def advance_line(amount = 1)
        @line += amount
      end

      ##
      # Emits a token who's value is based on the supplied start/stop position.
      #
      # @param [Symbol] type The token type.
      # @param [Fixnum] start
      # @param [Fixnum] stop
      #
      # @see #text
      # @see #add_token
      #
      def emit(type, start, stop)
        value = text(start, stop)

        add_token(type, value)
      end

      ##
      # Returns the text of the current buffer based on the supplied start and
      # stop position.
      #
      # @param [Fixnum] start
      # @param [Fixnum] stop
      # @return [String]
      #
      def text(start, stop)
        return @data.byteslice(start, stop - start)
      end

      ##
      # Adds a token with the given type and value to the list.
      #
      # @param [Symbol] type The token type.
      # @param [String] value The token value.
      #
      def add_token(type, value = nil)
        token = [type, value, @line]

        @block.call(token)
      end

      ##
      # Enables buffering starting at the given position.
      #
      # @param [Fixnum] position The start position of the buffer.
      #
      def start_buffer(position)
        @buffer_start_position = position
      end

      ##
      # Emits a text token.
      #
      # @param [Fixnum] start
      # @param [Fixnum] stop
      #
      def emit_text(start, stop)
        content = text(start, stop)

        unless content.empty?
          add_token(:T_TEXT, content)

          lines = content.count("\n")

          advance_line(lines) if lines > 0
        end
      end

      ##
      # Returns the name of the element we're currently in.
      #
      # @return [String]
      #
      def current_element
        return @elements.last
      end

      %%{
        getkey (data.getbyte(p) || 0);

        newline    = '\n' | '\r\n';
        whitespace = [ \t];
        identifier = [a-zA-Z0-9\-_:]+;

        # Strings
        #
        # Strings in HTML can either be single or double quoted. If a string
        # starts with one of these quotes it must be closed with the same type
        # of quote.
        dquote = '"';
        squote = "'";

        # Machine for processing double quoted strings.
        string_dquote := |*
          ^dquote+ => {
            emit(:T_STRING, ts, te)
          };

          dquote => { fret; };
        *|;

        # Machine for processing single quoted strings.
        string_squote := |*
          ^squote+ => {
            emit(:T_STRING, ts, te)
          };

          squote => { fret; };
        *|;

        # DOCTYPES
        #
        # http://www.w3.org/TR/html-markup/syntax.html#doctype-syntax
        #
        # These rules support the 3 flavours of doctypes:
        #
        # 1. Normal doctypes, as introduced in the HTML5 specification.
        # 2. Deprecated doctypes, the more verbose ones used prior to HTML5.
        # 3. Legacy doctypes
        #
        doctype_start = '<!DOCTYPE'i whitespace+;

        action start_doctype {
          add_token(:T_DOCTYPE_START)
          fcall doctype;
        }

        # Machine for processing doctypes. Doctype values such as the public
        # and system IDs are treated as T_STRING tokens.
        doctype := |*
          'PUBLIC' | 'SYSTEM' => { emit(:T_DOCTYPE_TYPE, ts, te) };

          # Lex the public/system IDs as regular strings.
          dquote => { fcall string_dquote; };
          squote => { fcall string_squote; };

          # Whitespace inside doctypes is ignored since there's no point in
          # including it.
          whitespace;

          identifier => { emit(:T_DOCTYPE_NAME, ts, te) };

          '>' => {
            add_token(:T_DOCTYPE_END)
            fret;
          };
        *|;

        # CDATA
        #
        # http://www.w3.org/TR/html-markup/syntax.html#cdata-sections
        #
        # CDATA tags are broken up into 3 parts: the start, the content and the
        # end tag.
        #
        # In HTML CDATA tags have no meaning/are not supported. Oga does
        # support them but treats their contents as plain text.
        #
        cdata_start = '<![CDATA[';
        cdata_end   = ']]>';

        action start_cdata {
          add_token(:T_CDATA_START)

          fcall cdata;
        }

        # Machine that for processing the contents of CDATA tags. Everything
        # inside a CDATA tag is treated as plain text.
        cdata := |*
          any* cdata_end => {
            emit_text(ts, te - 3)
            add_token(:T_CDATA_END)

            fret;
          };
        *|;

        # Comments
        #
        # http://www.w3.org/TR/html-markup/syntax.html#comments
        #
        # Comments are lexed into 3 parts: the start tag, the content and the
        # end tag.
        #
        # Unlike the W3 specification these rules *do* allow character
        # sequences such as `--` and `->`. Putting extra checks in for these
        # sequences would actually make the rules/actions more complex.
        #
        comment_start = '<!--';
        comment_end   = '-->';

        action start_comment {
          add_token(:T_COMMENT_START)

          fcall comment;
        }

        # Machine used for processing the contents of a comment. Everything
        # inside a comment is treated as plain text (similar to CDATA tags).
        comment := |*
          any* comment_end => {
            emit_text(ts, te - 3)
            add_token(:T_COMMENT_END)

            fret;
          };
        *|;

        # XML declaration tags
        #
        # http://www.w3.org/TR/REC-xml/#sec-prolog-dtd
        #
        xml_decl_start = '<?xml';
        xml_decl_end   = '?>';

        action start_xml_decl {
          add_token(:T_XML_DECL_START)

          fcall xml_decl;
        }

        # Machine that processes the contents of an XML declaration tag.
        xml_decl := |*
          xml_decl_end => {
            add_token(:T_XML_DECL_END)

            fret;
          };

          # Attributes and their values (e.g. version="1.0").
          identifier => { emit(:T_ATTR, ts, te) };

          dquote => { fcall string_dquote; };
          squote => { fcall string_squote; };

          any;
        *|;

        # Elements
        #
        # http://www.w3.org/TR/html-markup/syntax.html#syntax-elements
        #

        # Action that creates the tokens for the opening tag, name and
        # namespace (if any). Remaining work is delegated to a dedicated
        # machine.
        action start_element {
          add_token(:T_ELEM_START)

          # Add the element name. If the name includes a namespace we'll break
          # the name up into two separate tokens.
          name = text(ts + 1, te)

          if name.include?(':')
            ns, name = name.split(':')

            add_token(:T_ELEM_NS, ns)
          end

          @elements << name if html?

          add_token(:T_ELEM_NAME, name)

          fcall element_head;
        }

        element_start = '<' identifier;

        # Machine used for processing the characters inside a element head. An
        # element head is everything between `<NAME` (where NAME is the element
        # name) and `>`.
        #
        # For example, in `<p foo="bar">` the element head is ` foo="bar"`.
        #
        element_head := |*
          whitespace | '=';

          newline => { advance_line };

          # Attribute names.
          identifier => { emit(:T_ATTR, ts, te) };

          # Attribute values.
          dquote => { fcall string_dquote; };
          squote => { fcall string_squote; };

          # The closing character of the open tag.
          ('>' | '/') => {
            fhold;
            fret;
          };
        *|;

        main := |*
          element_start  => start_element;
          doctype_start  => start_doctype;
          cdata_start    => start_cdata;
          comment_start  => start_comment;
          xml_decl_start => start_xml_decl;

          # Enter the body of the tag. If HTML mode is enabled and the current
          # element is a void element we'll close it and bail out.
          '>' => {
            if html? and HTML_VOID_ELEMENTS.include?(current_element)
              add_token(:T_ELEM_END, nil)
              @elements.pop
            end
          };

          # Regular closing tags.
          '</' identifier '>' => {
            add_token(:T_ELEM_END, nil)

            @elements.pop if html?
          };

          # Self closing elements that are not handled by the HTML mode.
          '/>' => {
            add_token(:T_ELEM_END, nil)

            @elements.pop if html?
          };

          # Note that this rule should be declared at the very bottom as it
          # will otherwise take precedence over the other rules.
          ^('<' | '>')+ => {
            emit_text(ts, te)
          };
        *|;
      }%%
    end # Lexer
  end # XML
end # Oga


No need to pack and unpack when dealing with binary blob strings

%%{
  machine Message;

  variable data @_data;
  variable p    @_p;

  # data, p, pe, eof, cs, top, stack, ts, te and act

  # Actions.
  action Mark { mark }

  # Common constants.
  NULL = "\0";
  LF = "\n";
  CR = "\r";
  EOL = CR? LF;

  # Message components.
  command = "CONNECT";

  message := (command > Mark @ { @message.command = consume }) EOL EOL NULL;
}%%

module Stompede
  module Stomp
    %% write data;

    class << self
      # @param [String] data
      # @return [Stomp::Message]
      def parse(data)
        @message = Stomp::Message.new

        @_data = data.force_encoding("BINARY")
        @_p = 0
        pe = data.length
        cs = self.Message_start

        %% write exec;

        @message
      end

      def mark
        @mark = @_p
      end

      def consume
        @_data[@mark..@_p].force_encoding("UTF-8")
      end
    end
  end
end

%%{
  machine Message;

  variable data @_data;
  variable p    @_p;

  # data, p, pe, eof, cs, top, stack, ts, te and act

  # Actions.
  action Mark { mark }

  # Common constants.
  NULL = "\0";
  LF = "\n";
  CR = "\r";
  EOL = CR? LF;

  # Message components.
  command = "CONNECT";

  message := (command > Mark @ { @message.command = consume }) EOL EOL NULL;
}%%

module Stompede
  module Stomp
    %% write data;

    class << self
      # @param [String] data
      # @return [Stomp::Message]
      def parse(data)
        @message = Stomp::Message.new

        @_data = data.unpack("c*")
        @_p = 0
        pe = data.length
        cs = self.Message_start

        %% write exec;

        @message
      end

      def mark
        @mark = @_p
      end

      def consume
        @_data[@mark..@_p].pack("c*")
      end
    end
  end
end