Updated accessor for Float values.

Updated accessor for Float values.

/* This file was generated by Ragel. Your edits will be lost.
 *
 * This is a state machine implementation of Array#pack.
 * See http://github.com/rubinius/rapa.
 *
 * vim: filetype=cpp
 */

#include <stdint.h>
#include <sstream>

#include "machine/config.h"

#include "alloc.hpp"
#include "memory.hpp"
#include "object_utils.hpp"
#include "on_stack.hpp"
#include "state.hpp"
#include "vm.hpp"

#include "builtin/array.hpp"
#include "builtin/byte_array.hpp"
#include "builtin/class.hpp"
#include "builtin/encoding.hpp"
#include "builtin/exception.hpp"
#include "builtin/float.hpp"
#include "builtin/module.hpp"
#include "builtin/object.hpp"
#include "builtin/string.hpp"

#ifdef RBX_WINDOWS
#include <malloc.h>
#endif

namespace rubinius {
  namespace pack {
    inline Object* to_int(STATE, Object* obj) {
      Array* args = Array::create(state, 1);
      args->set(state, 0, obj);

      return G(rubinius)->send(state, state->symbol("pack_to_int"), args);
    }

#define BITS_LONG   (RBX_SIZEOF_LONG * 8)

    inline long check_long(STATE, Integer* obj) {
      if((obj)->fixnum_p()) {
        return force_as<Fixnum>(obj)->to_long();
      } else {
        Bignum* big = as<Bignum>(obj);
        big->verify_size(state, BITS_LONG);
        return big->to_long();
      }
    }

#define BITS_64     (64)

    inline long long check_long_long(STATE, Integer* obj) {
      if((obj)->fixnum_p()) {
        return force_as<Fixnum>(obj)->to_long_long();
      } else {
        Bignum* big = as<Bignum>(obj);
        big->verify_size(state, BITS_64);
        return big->to_long_long();
      }
    }

    inline Object* to_f(STATE, Object* obj) {
      Array* args = Array::create(state, 1);
      args->set(state, 0, obj);

      return G(rubinius)->send(state, state->symbol("pack_to_float"), args);
    }

    inline String* encoding_string(STATE, Object* obj, const char* coerce_name)
    {
      String* s = try_as<String>(obj);
      if(s) return s;

      Array* args = Array::create(state, 1);
      args->set(state, 0, obj);

      std::string coerce_method("pack_");
      coerce_method += coerce_name;
      Object* result = G(rubinius)->send(state, state->symbol(coerce_method.c_str()), args);

      if(!result) return 0;
      return as<String>(result);
    }

    inline uint16_t swap_2bytes(uint16_t x) {
      return (((x & 0x00ff)<<8) | ((x & 0xff00)>>8));
    }

    inline uint32_t swap_4bytes(uint32_t x) {
      return (((x & 0x000000ff) << 24)
             |((x & 0xff000000) >> 24)
             |((x & 0x0000ff00) << 8)
             |((x & 0x00ff0000) >> 8));
    }

    inline uint64_t swap_8bytes(uint64_t x) {
      return (((x & 0x00000000000000ffLL) << 56)
             |((x & 0xff00000000000000LL) >> 56)
             |((x & 0x000000000000ff00LL) << 40)
             |((x & 0x00ff000000000000LL) >> 40)
             |((x & 0x0000000000ff0000LL) << 24)
             |((x & 0x0000ff0000000000LL) >> 24)
             |((x & 0x00000000ff000000LL) << 8)
             |((x & 0x000000ff00000000LL) >> 8));
    }

    inline void swap_float(std::string& str, float value) {
      uint32_t x;

      memcpy(&x, &value, sizeof(float));
      x = swap_4bytes(x);

      str.append((const char*)&x, sizeof(uint32_t));
    }

    inline void swap_double(std::string& str, double value) {
      uint64_t x;

      memcpy(&x, &value, sizeof(double));
      x = swap_8bytes(x);

      str.append((const char*)&x, sizeof(uint64_t));
    }

    inline void double_element(std::string& str, double value) {
      str.append((const char*)&value, sizeof(double));
    }

    inline void float_element(std::string& str, float value) {
      str.append((const char*)&value, sizeof(float));
    }

    inline void short_element(std::string& str, int16_t value) {
      str.append((const char*)&value, sizeof(int16_t));
    }

    inline void int_element(std::string& str, int32_t value) {
      str.append((const char*)&value, sizeof(int32_t));
    }

    inline void long_element(std::string& str, int64_t value) {
      str.append((const char*)&value, sizeof(int64_t));
    }

    inline int32_t int32_element(STATE, Integer* value) {
      if(value->fixnum_p()) {
        long l = as<Fixnum>(value)->to_long();
        if(l > INT32_MAX || l < INT32_MIN) {
          Exception::raise_range_error(state, "Fixnum value out of range of int32");
        }
        return l;
      } else {
        Bignum* big = as<Bignum>(value);
        big->verify_size(state, 32);
        return big->to_int();
      }
    }

#define QUOTABLE_PRINTABLE_BUFSIZE 1024

    static const char hex_table[] = "0123456789ABCDEF";
    void quotable_printable(String* s, std::string& str, int count) {
      char buf[QUOTABLE_PRINTABLE_BUFSIZE];

      uint8_t* b = s->byte_address();
      uint8_t* e = b + s->byte_size();
      int i = 0, n = 0, prev = -1;

      for(; b < e; b++) {
        if((*b > 126) || (*b < 32 && *b != '\n' && *b != '\t') || (*b == '=')) {
          buf[i++] = '=';
          buf[i++] = hex_table[*b >> 4];
          buf[i++] = hex_table[*b & 0x0f];
          n += 3;
          prev = -1;
        } else if(*b == '\n') {
          if(prev == ' ' || prev == '\t') {
            buf[i++] = '=';
            buf[i++] = *b;
          }
          buf[i++] = *b;
          n = 0;
          prev = *b;
        } else {
          buf[i++] = *b;
          n++;
          prev = *b;
        }

        if(n > count) {
          buf[i++] = '=';
          buf[i++] = '\n';
          n = 0;
          prev = '\n';
        }

        if(i > QUOTABLE_PRINTABLE_BUFSIZE - 5) {
          str.append(buf, i);
          i = 0;
        }
      }

      if(n > 0) {
        buf[i++] = '=';
        buf[i++] = '\n';
      }

      if(i > 0) {
        str.append(buf, i);
      }
    }

    static const char uu_table[] =
      "`!\"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_";
    static const char b64_table[] =
      "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";

#define b64_uu_byte1(t, b)      t[077 & (*b >> 2)]
#define b64_uu_byte2(t, b, c)   t[077 & (((*b << 4) & 060) | ((c >> 4) & 017))]
#define b64_uu_byte3(t, b, c)   t[077 & (((b[1] << 2) & 074) | ((c >> 6) & 03))];
#define b64_uu_byte4(t, b)      t[077 & b[2]];

    void b64_uu_encode(String* s, std::string& str,
                       native_int count, native_int count_flag,
                       const char* table, int padding, bool encode_size)
    {
      char *buf = ALLOCA_N(char, count * 4 / 3 + 6);
      native_int i, chars, line, total = s->byte_size();
      uint8_t* b = s->byte_address();

      for(i = 0; total > 0; i = 0, total -= line) {
        line = total > count ? count : total;

        if(encode_size) buf[i++] = line + ' ';

        for(chars = line; chars >= 3; chars -= 3, b += 3) {
          buf[i++] = b64_uu_byte1(table, b);
          buf[i++] = b64_uu_byte2(table, b, b[1]);
          buf[i++] = b64_uu_byte3(table, b, b[2]);
          buf[i++] = b64_uu_byte4(table, b);
        }

        if(chars == 2) {
          buf[i++] = b64_uu_byte1(table, b);
          buf[i++] = b64_uu_byte2(table, b, b[1]);
          buf[i++] = b64_uu_byte3(table, b, '\0');
          buf[i++] = padding;
        } else if(chars == 1) {
          buf[i++] = b64_uu_byte1(table, b);
          buf[i++] = b64_uu_byte2(table, b, '\0');
          buf[i++] = padding;
          buf[i++] = padding;
        }

        b += chars;
        if(encode_size || (!encode_size && count_flag > 0)) {
          buf[i++] = '\n';
        }
        str.append(buf, i);
      }
    }

    void utf8_encode(STATE, std::string& str, Integer* value) {
      int32_t v = int32_element(state, value);

      if(!(v & ~0x7f)) {
        str.push_back(v);
      } else if(!(v & ~0x7ff)) {
        str.push_back(((v >> 6) & 0xff) | 0xc0);
        str.push_back((v & 0x3f) | 0x80);
      } else if(!(v & ~0xffff)) {
        str.push_back(((v >> 12) & 0xff) | 0xe0);
        str.push_back(((v >> 6)  & 0x3f) | 0x80);
        str.push_back((v & 0x3f) | 0x80);
      } else if(!(v & ~0x1fffff)) {
        str.push_back(((v >> 18) & 0xff) | 0xf0);
        str.push_back(((v >> 12) & 0x3f) | 0x80);
        str.push_back(((v >> 6)  & 0x3f) | 0x80);
        str.push_back((v & 0x3f) | 0x80);
      } else if(!(v & ~0x3ffffff)) {
        str.push_back(((v >> 24) & 0xff) | 0xf8);
        str.push_back(((v >> 18) & 0x3f) | 0x80);
        str.push_back(((v >> 12) & 0x3f) | 0x80);
        str.push_back(((v >> 6)  & 0x3f) | 0x80);
        str.push_back((v & 0x3f) | 0x80);
      } else if(!(v & ~0x7fffffff)) {
        str.push_back(((v >> 30) & 0xff) | 0xfc);
        str.push_back(((v >> 24) & 0x3f) | 0x80);
        str.push_back(((v >> 18) & 0x3f) | 0x80);
        str.push_back(((v >> 12) & 0x3f) | 0x80);
        str.push_back(((v >> 6)  & 0x3f) | 0x80);
        str.push_back((v & 0x3f) | 0x80);
      } else {
        Exception::raise_range_error(state, "pack('U') value out of range");
      }
    }

    void ber_encode(STATE, std::string& str, Integer* value) {
      if(!value->positive_p()) {
        Exception::raise_argument_error(state, "cannot BER compress a negative number");
      }

      std::string buf;

      if(try_as<Bignum>(value)) {
        Fixnum* base = Fixnum::from(128);
        while(try_as<Bignum>(value)) {
          Array* ary;
          if(value->fixnum_p()) {
            ary = as<Fixnum>(value)->divmod(state, base);
          } else {
            ary = as<Bignum>(value)->divmod(state, base);
          }
          buf.push_back(as<Fixnum>(ary->get(state, 1))->to_native() | 0x80);
          value = as<Integer>(ary->get(state, 0));
        }
      }

      long v = value->to_long();

      while(v) {
        buf.push_back((v & 0x7f) | 0x80);
        v >>= 7;
      }

      if(buf.size() > 0) {
        char* a = const_cast<char*>(buf.c_str());
        char* b = a + buf.size() - 1;

        // clear continue bit
        *a &= 0x7f;

        // reverse string
        while(a < b) {
          int k = *a;
          *a++ = *b;
          *b-- = k;
        }

        str.append(buf.c_str(), buf.size());
      } else {
        str.push_back(0);
      }
    }

    inline native_int bit_extra(String* s, bool rest, native_int& count) {
      native_int extra = 0;

      if(rest) {
        count = s->byte_size();
      } else {
        native_int size = s->byte_size();
        if(count > size) {
          extra = (count - size + 1) / 2;
          count = size;
        }
      }

      return extra;
    }

    void bit_high(String* s, std::string& str, native_int count) {
      uint8_t* b = s->byte_address();
      int byte = 0;

      for(native_int i = 0; i++ < count; b++) {
        byte |= *b & 1;
        if(i & 7) {
          byte <<= 1;
        } else {
          str.push_back(byte & 0xff);
          byte = 0;
        }
      }

      if(count & 7) {
        byte <<= 7 - (count & 7);
        str.push_back(byte & 0xff);
      }
    }

    void bit_low(String* s, std::string& str, native_int count) {
      uint8_t* b = s->byte_address();
      int byte = 0;

      for(native_int i = 0; i++ < count; b++) {
        if(*b & 1)
          byte |= 128;

        if(i & 7) {
          byte >>= 1;
        } else {
          str.push_back(byte & 0xff);
          byte = 0;
        }
      }

      if(count & 7) {
        byte >>= 7 - (count & 7);
        str.push_back(byte & 0xff);
      }
    }

    inline native_int hex_extra(String* s, bool rest, native_int& count) {
      native_int extra = 0;

      if(rest) {
        count = s->byte_size();
      } else {
        native_int size = s->byte_size();
        if(count > size) {
          extra = (count + 1) / 2 - (size + 1) / 2;
          count = size;
        }
      }

      return extra;
    }

    void hex_high(String* s, std::string& str, native_int count) {
      uint8_t* b = s->byte_address();
      int byte = 0;

      for(native_int i = 0; i++ < count; b++) {
        if(ISALPHA(*b)) {
          byte |= ((*b & 15) + 9) & 15;
        } else {
          byte |= *b & 15;
        }

        if(i & 1) {
          byte <<= 4;
        } else {
          str.push_back(byte & 0xff);
          byte = 0;
        }
      }

      if(count & 1) {
        str.push_back(byte & 0xff);
      }
    }

    void hex_low(String* s, std::string& str, native_int count) {
      uint8_t* b = s->byte_address();
      int byte = 0;

      for(native_int i = 0; i++ < count; b++) {
        if(ISALPHA(*b)) {
          byte |= (((*b & 15) + 9) & 15) << 4;
        } else {
          byte |= (*b & 15) << 4;
        }

        if(i & 1) {
          byte >>= 4;
        } else {
          str.push_back(byte & 0xff);
          byte = 0;
        }
      }

      if(count & 1) {
        str.push_back(byte & 0xff);
      }
    }

    ByteArray* prepare_directives(STATE, String* directives,
                                  const char** p, const char** pe)
    {
      native_int size = directives->byte_size();
      ByteArray* ba = ByteArray::create_pinned(state, size + 1);
      char* b = reinterpret_cast<char*>(ba->raw_bytes());
      char* d = reinterpret_cast<char*>(directives->byte_address());
      int i = 0, j = 0;

      while(i < size) {
        switch(d[i]) {
        case 0:
        case ' ':
        case '\t':
        case '\n':
        case '\v':
        case '\f':
        case '\r':
          i++;
          break;
        case '#':
          while(++i < size && d[i] != '\n')
            ; // ignore
          if(d[i] == '\n') i++;
          break;
        default:
          b[j++] = d[i++];
          break;
        }
      }

      *p = const_cast<const char*>(b);
      *pe = *p + j;

      return ba;
    }

    void exceeds_length_of_string(STATE, native_int count) {
      std::ostringstream msg;
      msg << "X" << count << " exceeds length of string";
      Exception::raise_argument_error(state, msg.str().c_str());
    }

    void non_native_error(STATE, const char c) {
      std::ostringstream msg;
      msg << "'" << c << "' allowed only after types sSiIlL";
      Exception::raise_argument_error(state, msg.str().c_str());
    }
  }

// Pack Float elements
#define pack_float_elements(format)   pack_elements(Float, pack::to_f, format)

#define pack_double_le                pack_float_elements(pack_double_element_le)
#define pack_double_be                pack_float_elements(pack_double_element_be)

#define pack_float_le                 pack_float_elements(pack_float_element_le)
#define pack_float_be                 pack_float_elements(pack_float_element_be)

// Pack Integer elements
#define pack_integer_elements(format) pack_elements(Integer, pack::to_int, format)

#define pack_byte_element(v)          str.push_back(pack::check_long(state, v))
#define pack_byte                     pack_integer_elements(pack_byte_element)

#define pack_short_le                 pack_integer_elements(pack_short_element_le)
#define pack_short_be                 pack_integer_elements(pack_short_element_be)

#define pack_int_le                   pack_integer_elements(pack_int_element_le)
#define pack_int_be                   pack_integer_elements(pack_int_element_be)

#define pack_long_le                  pack_integer_elements(pack_long_element_le)
#define pack_long_be                  pack_integer_elements(pack_long_element_be)

// Pack UTF-8 elements
#define pack_utf8_element(v)          pack::utf8_encode(state, str, v)
#define pack_utf8                     pack_elements(Integer, pack::to_int, pack_utf8_element)

// Pack BER-compressed integers
#define pack_ber_element(v)           pack::ber_encode(state, str, v)
#define pack_ber                      pack_elements(Integer, pack::to_int, pack_ber_element)

// Wraps the logic for iterating over a number of elements,
// coercing them to the correct class and formatting them
// for the output string.
#define pack_elements(T, coerce, format)        \
  for(; index < stop; index++) {                \
    Object* item = self->get(state, index);     \
    T* value = try_as<T>(item);                 \
    if(!value) {                                \
      item = coerce(state, item);   \
      if(!item) return 0;                       \
      value = as<T>(item);                      \
    }                                           \
    format(value);                              \
  }

// Macros that depend on endianness
#ifdef RBX_LITTLE_ENDIAN

# define pack_double_element_le(v)  (pack::double_element(str, (v)->value()))
# define pack_double_element_be(v)  (pack::swap_double(str, (v)->value()))
# define pack_double                pack_double_le

# define pack_float_element_le(v)   (pack::float_element(str, (v)->value()))
# define pack_float_element_be(v)   (pack::swap_float(str, (v)->value()))
# define pack_float                 pack_float_le

# define pack_short_element_le(v)   (pack::short_element(str, pack::check_long(state, v)))
# define pack_short_element_be(v)   (pack::short_element(str, \
                                        pack::swap_2bytes(pack::check_long(state, v))))
# define pack_short                 pack_short_le

# define pack_int_element_le(v)     (pack::int_element(str, pack::check_long(state, v)))
# define pack_int_element_be(v)     (pack::int_element(str, \
                                        pack::swap_4bytes(pack::check_long(state, v))))
# define pack_int                   pack_int_le

# define pack_long_element_le(v)    (pack::long_element(str, pack::check_long_long(state, v)))
# define pack_long_element_be(v)    (pack::long_element(str, \
                                        pack::swap_8bytes(pack::check_long_long(state, v))))
# define pack_long                  pack_long_le

#else // Big endian

# define pack_double_element_le(v)  (pack::swap_double(str, (v)->value()))
# define pack_double_element_be(v)  (pack::double_element(str, (v)->value()))
# define pack_double                pack_double_be

# define pack_float_element_le(v)   (pack::swap_float(str, (v)->value()))
# define pack_float_element_be(v)   (pack::float_element(str, (v)->value()))
# define pack_float                 pack_float_be

# define pack_short_element_le(v)   (pack::short_element(str, \
                                        pack::swap_2bytes(pack::check_long(state, v))))
# define pack_short_element_be(v)   (pack::short_element(str, pack::check_long(state, v)))
# define pack_short                 pack_short_be

# define pack_int_element_le(v)     (pack::int_element(str, \
                                        pack::swap_4bytes(pack::check_long(state, v))))
# define pack_int_element_be(v)     (pack::int_element(str, pack::check_long(state, v)))
# define pack_int                   pack_int_be

# define pack_long_element_le(v)    (pack::long_element(str, \
                                        pack::swap_8bytes(pack::check_long_long(state, v))))
# define pack_long_element_be(v)    (pack::long_element(str, pack::check_long_long(state, v)))
# define pack_long                  pack_long_be

#endif

  String* Array::pack(STATE, String* directives) {
    // Ragel-specific variables
    const char* p;
    const char* pe;
    ByteArray* d = pack::prepare_directives(state, directives, &p, &pe);
    const char *eof = pe;
    int cs;

    // pack-specific variables
    Array* self = this;
    OnStack<2> sv(state, self, d);

    native_int array_size = self->size();
    native_int index = 0;
    native_int count UNUSED = 0;
    native_int count_flag = -1;
    native_int stop = 0;
    bool rest UNUSED = false;
    bool tainted UNUSED = false;
    bool untrusted UNUSED = false;
    bool ascii_encoding = false;
    bool utf8_encoding = false;
    bool string_encoding = false;

    String* string_value = 0;
    std::string str("");

    // Use information we have to reduce repeated allocation.
    str.reserve(array_size * 4);

    if(directives->byte_size() == 0) ascii_encoding = true;
    if(CBOOL(directives->tainted_p(state))) tainted = true;
    if(CBOOL(directives->untrusted_p(state))) untrusted = true;
%%{

  machine pack;

  include "pack.rl";

}%%

    if(en_main) {
      // do nothing
    }

    return force_as<String>(Primitives::failure());
  }
}

/* This file was generated by Ragel. Your edits will be lost.
 *
 * This is a state machine implementation of Array#pack.
 * See http://github.com/rubinius/rapa.
 *
 * vim: filetype=cpp
 */

#include <stdint.h>
#include <sstream>

#include "machine/config.h"

#include "alloc.hpp"
#include "memory.hpp"
#include "object_utils.hpp"
#include "on_stack.hpp"
#include "state.hpp"
#include "vm.hpp"

#include "builtin/array.hpp"
#include "builtin/byte_array.hpp"
#include "builtin/class.hpp"
#include "builtin/encoding.hpp"
#include "builtin/exception.hpp"
#include "builtin/float.hpp"
#include "builtin/module.hpp"
#include "builtin/object.hpp"
#include "builtin/string.hpp"

#ifdef RBX_WINDOWS
#include <malloc.h>
#endif

namespace rubinius {
  namespace pack {
    inline Object* to_int(STATE, Object* obj) {
      Array* args = Array::create(state, 1);
      args->set(state, 0, obj);

      return G(rubinius)->send(state, state->symbol("pack_to_int"), args);
    }

#define BITS_LONG   (RBX_SIZEOF_LONG * 8)

    inline long check_long(STATE, Integer* obj) {
      if((obj)->fixnum_p()) {
        return force_as<Fixnum>(obj)->to_long();
      } else {
        Bignum* big = as<Bignum>(obj);
        big->verify_size(state, BITS_LONG);
        return big->to_long();
      }
    }

#define BITS_64     (64)

    inline long long check_long_long(STATE, Integer* obj) {
      if((obj)->fixnum_p()) {
        return force_as<Fixnum>(obj)->to_long_long();
      } else {
        Bignum* big = as<Bignum>(obj);
        big->verify_size(state, BITS_64);
        return big->to_long_long();
      }
    }

    inline Object* to_f(STATE, Object* obj) {
      Array* args = Array::create(state, 1);
      args->set(state, 0, obj);

      return G(rubinius)->send(state, state->symbol("pack_to_float"), args);
    }

    inline String* encoding_string(STATE, Object* obj, const char* coerce_name)
    {
      String* s = try_as<String>(obj);
      if(s) return s;

      Array* args = Array::create(state, 1);
      args->set(state, 0, obj);

      std::string coerce_method("pack_");
      coerce_method += coerce_name;
      Object* result = G(rubinius)->send(state, state->symbol(coerce_method.c_str()), args);

      if(!result) return 0;
      return as<String>(result);
    }

    inline uint16_t swap_2bytes(uint16_t x) {
      return (((x & 0x00ff)<<8) | ((x & 0xff00)>>8));
    }

    inline uint32_t swap_4bytes(uint32_t x) {
      return (((x & 0x000000ff) << 24)
             |((x & 0xff000000) >> 24)
             |((x & 0x0000ff00) << 8)
             |((x & 0x00ff0000) >> 8));
    }

    inline uint64_t swap_8bytes(uint64_t x) {
      return (((x & 0x00000000000000ffLL) << 56)
             |((x & 0xff00000000000000LL) >> 56)
             |((x & 0x000000000000ff00LL) << 40)
             |((x & 0x00ff000000000000LL) >> 40)
             |((x & 0x0000000000ff0000LL) << 24)
             |((x & 0x0000ff0000000000LL) >> 24)
             |((x & 0x00000000ff000000LL) << 8)
             |((x & 0x000000ff00000000LL) >> 8));
    }

    inline void swap_float(std::string& str, float value) {
      uint32_t x;

      memcpy(&x, &value, sizeof(float));
      x = swap_4bytes(x);

      str.append((const char*)&x, sizeof(uint32_t));
    }

    inline void swap_double(std::string& str, double value) {
      uint64_t x;

      memcpy(&x, &value, sizeof(double));
      x = swap_8bytes(x);

      str.append((const char*)&x, sizeof(uint64_t));
    }

    inline void double_element(std::string& str, double value) {
      str.append((const char*)&value, sizeof(double));
    }

    inline void float_element(std::string& str, float value) {
      str.append((const char*)&value, sizeof(float));
    }

    inline void short_element(std::string& str, int16_t value) {
      str.append((const char*)&value, sizeof(int16_t));
    }

    inline void int_element(std::string& str, int32_t value) {
      str.append((const char*)&value, sizeof(int32_t));
    }

    inline void long_element(std::string& str, int64_t value) {
      str.append((const char*)&value, sizeof(int64_t));
    }

    inline int32_t int32_element(STATE, Integer* value) {
      if(value->fixnum_p()) {
        long l = as<Fixnum>(value)->to_long();
        if(l > INT32_MAX || l < INT32_MIN) {
          Exception::raise_range_error(state, "Fixnum value out of range of int32");
        }
        return l;
      } else {
        Bignum* big = as<Bignum>(value);
        big->verify_size(state, 32);
        return big->to_int();
      }
    }

#define QUOTABLE_PRINTABLE_BUFSIZE 1024

    static const char hex_table[] = "0123456789ABCDEF";
    void quotable_printable(String* s, std::string& str, int count) {
      char buf[QUOTABLE_PRINTABLE_BUFSIZE];

      uint8_t* b = s->byte_address();
      uint8_t* e = b + s->byte_size();
      int i = 0, n = 0, prev = -1;

      for(; b < e; b++) {
        if((*b > 126) || (*b < 32 && *b != '\n' && *b != '\t') || (*b == '=')) {
          buf[i++] = '=';
          buf[i++] = hex_table[*b >> 4];
          buf[i++] = hex_table[*b & 0x0f];
          n += 3;
          prev = -1;
        } else if(*b == '\n') {
          if(prev == ' ' || prev == '\t') {
            buf[i++] = '=';
            buf[i++] = *b;
          }
          buf[i++] = *b;
          n = 0;
          prev = *b;
        } else {
          buf[i++] = *b;
          n++;
          prev = *b;
        }

        if(n > count) {
          buf[i++] = '=';
          buf[i++] = '\n';
          n = 0;
          prev = '\n';
        }

        if(i > QUOTABLE_PRINTABLE_BUFSIZE - 5) {
          str.append(buf, i);
          i = 0;
        }
      }

      if(n > 0) {
        buf[i++] = '=';
        buf[i++] = '\n';
      }

      if(i > 0) {
        str.append(buf, i);
      }
    }

    static const char uu_table[] =
      "`!\"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_";
    static const char b64_table[] =
      "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";

#define b64_uu_byte1(t, b)      t[077 & (*b >> 2)]
#define b64_uu_byte2(t, b, c)   t[077 & (((*b << 4) & 060) | ((c >> 4) & 017))]
#define b64_uu_byte3(t, b, c)   t[077 & (((b[1] << 2) & 074) | ((c >> 6) & 03))];
#define b64_uu_byte4(t, b)      t[077 & b[2]];

    void b64_uu_encode(String* s, std::string& str,
                       native_int count, native_int count_flag,
                       const char* table, int padding, bool encode_size)
    {
      char *buf = ALLOCA_N(char, count * 4 / 3 + 6);
      native_int i, chars, line, total = s->byte_size();
      uint8_t* b = s->byte_address();

      for(i = 0; total > 0; i = 0, total -= line) {
        line = total > count ? count : total;

        if(encode_size) buf[i++] = line + ' ';

        for(chars = line; chars >= 3; chars -= 3, b += 3) {
          buf[i++] = b64_uu_byte1(table, b);
          buf[i++] = b64_uu_byte2(table, b, b[1]);
          buf[i++] = b64_uu_byte3(table, b, b[2]);
          buf[i++] = b64_uu_byte4(table, b);
        }

        if(chars == 2) {
          buf[i++] = b64_uu_byte1(table, b);
          buf[i++] = b64_uu_byte2(table, b, b[1]);
          buf[i++] = b64_uu_byte3(table, b, '\0');
          buf[i++] = padding;
        } else if(chars == 1) {
          buf[i++] = b64_uu_byte1(table, b);
          buf[i++] = b64_uu_byte2(table, b, '\0');
          buf[i++] = padding;
          buf[i++] = padding;
        }

        b += chars;
        if(encode_size || (!encode_size && count_flag > 0)) {
          buf[i++] = '\n';
        }
        str.append(buf, i);
      }
    }

    void utf8_encode(STATE, std::string& str, Integer* value) {
      int32_t v = int32_element(state, value);

      if(!(v & ~0x7f)) {
        str.push_back(v);
      } else if(!(v & ~0x7ff)) {
        str.push_back(((v >> 6) & 0xff) | 0xc0);
        str.push_back((v & 0x3f) | 0x80);
      } else if(!(v & ~0xffff)) {
        str.push_back(((v >> 12) & 0xff) | 0xe0);
        str.push_back(((v >> 6)  & 0x3f) | 0x80);
        str.push_back((v & 0x3f) | 0x80);
      } else if(!(v & ~0x1fffff)) {
        str.push_back(((v >> 18) & 0xff) | 0xf0);
        str.push_back(((v >> 12) & 0x3f) | 0x80);
        str.push_back(((v >> 6)  & 0x3f) | 0x80);
        str.push_back((v & 0x3f) | 0x80);
      } else if(!(v & ~0x3ffffff)) {
        str.push_back(((v >> 24) & 0xff) | 0xf8);
        str.push_back(((v >> 18) & 0x3f) | 0x80);
        str.push_back(((v >> 12) & 0x3f) | 0x80);
        str.push_back(((v >> 6)  & 0x3f) | 0x80);
        str.push_back((v & 0x3f) | 0x80);
      } else if(!(v & ~0x7fffffff)) {
        str.push_back(((v >> 30) & 0xff) | 0xfc);
        str.push_back(((v >> 24) & 0x3f) | 0x80);
        str.push_back(((v >> 18) & 0x3f) | 0x80);
        str.push_back(((v >> 12) & 0x3f) | 0x80);
        str.push_back(((v >> 6)  & 0x3f) | 0x80);
        str.push_back((v & 0x3f) | 0x80);
      } else {
        Exception::raise_range_error(state, "pack('U') value out of range");
      }
    }

    void ber_encode(STATE, std::string& str, Integer* value) {
      if(!value->positive_p()) {
        Exception::raise_argument_error(state, "cannot BER compress a negative number");
      }

      std::string buf;

      if(try_as<Bignum>(value)) {
        Fixnum* base = Fixnum::from(128);
        while(try_as<Bignum>(value)) {
          Array* ary;
          if(value->fixnum_p()) {
            ary = as<Fixnum>(value)->divmod(state, base);
          } else {
            ary = as<Bignum>(value)->divmod(state, base);
          }
          buf.push_back(as<Fixnum>(ary->get(state, 1))->to_native() | 0x80);
          value = as<Integer>(ary->get(state, 0));
        }
      }

      long v = value->to_long();

      while(v) {
        buf.push_back((v & 0x7f) | 0x80);
        v >>= 7;
      }

      if(buf.size() > 0) {
        char* a = const_cast<char*>(buf.c_str());
        char* b = a + buf.size() - 1;

        // clear continue bit
        *a &= 0x7f;

        // reverse string
        while(a < b) {
          int k = *a;
          *a++ = *b;
          *b-- = k;
        }

        str.append(buf.c_str(), buf.size());
      } else {
        str.push_back(0);
      }
    }

    inline native_int bit_extra(String* s, bool rest, native_int& count) {
      native_int extra = 0;

      if(rest) {
        count = s->byte_size();
      } else {
        native_int size = s->byte_size();
        if(count > size) {
          extra = (count - size + 1) / 2;
          count = size;
        }
      }

      return extra;
    }

    void bit_high(String* s, std::string& str, native_int count) {
      uint8_t* b = s->byte_address();
      int byte = 0;

      for(native_int i = 0; i++ < count; b++) {
        byte |= *b & 1;
        if(i & 7) {
          byte <<= 1;
        } else {
          str.push_back(byte & 0xff);
          byte = 0;
        }
      }

      if(count & 7) {
        byte <<= 7 - (count & 7);
        str.push_back(byte & 0xff);
      }
    }

    void bit_low(String* s, std::string& str, native_int count) {
      uint8_t* b = s->byte_address();
      int byte = 0;

      for(native_int i = 0; i++ < count; b++) {
        if(*b & 1)
          byte |= 128;

        if(i & 7) {
          byte >>= 1;
        } else {
          str.push_back(byte & 0xff);
          byte = 0;
        }
      }

      if(count & 7) {
        byte >>= 7 - (count & 7);
        str.push_back(byte & 0xff);
      }
    }

    inline native_int hex_extra(String* s, bool rest, native_int& count) {
      native_int extra = 0;

      if(rest) {
        count = s->byte_size();
      } else {
        native_int size = s->byte_size();
        if(count > size) {
          extra = (count + 1) / 2 - (size + 1) / 2;
          count = size;
        }
      }

      return extra;
    }

    void hex_high(String* s, std::string& str, native_int count) {
      uint8_t* b = s->byte_address();
      int byte = 0;

      for(native_int i = 0; i++ < count; b++) {
        if(ISALPHA(*b)) {
          byte |= ((*b & 15) + 9) & 15;
        } else {
          byte |= *b & 15;
        }

        if(i & 1) {
          byte <<= 4;
        } else {
          str.push_back(byte & 0xff);
          byte = 0;
        }
      }

      if(count & 1) {
        str.push_back(byte & 0xff);
      }
    }

    void hex_low(String* s, std::string& str, native_int count) {
      uint8_t* b = s->byte_address();
      int byte = 0;

      for(native_int i = 0; i++ < count; b++) {
        if(ISALPHA(*b)) {
          byte |= (((*b & 15) + 9) & 15) << 4;
        } else {
          byte |= (*b & 15) << 4;
        }

        if(i & 1) {
          byte >>= 4;
        } else {
          str.push_back(byte & 0xff);
          byte = 0;
        }
      }

      if(count & 1) {
        str.push_back(byte & 0xff);
      }
    }

    ByteArray* prepare_directives(STATE, String* directives,
                                  const char** p, const char** pe)
    {
      native_int size = directives->byte_size();
      ByteArray* ba = ByteArray::create_pinned(state, size + 1);
      char* b = reinterpret_cast<char*>(ba->raw_bytes());
      char* d = reinterpret_cast<char*>(directives->byte_address());
      int i = 0, j = 0;

      while(i < size) {
        switch(d[i]) {
        case 0:
        case ' ':
        case '\t':
        case '\n':
        case '\v':
        case '\f':
        case '\r':
          i++;
          break;
        case '#':
          while(++i < size && d[i] != '\n')
            ; // ignore
          if(d[i] == '\n') i++;
          break;
        default:
          b[j++] = d[i++];
          break;
        }
      }

      *p = const_cast<const char*>(b);
      *pe = *p + j;

      return ba;
    }

    void exceeds_length_of_string(STATE, native_int count) {
      std::ostringstream msg;
      msg << "X" << count << " exceeds length of string";
      Exception::raise_argument_error(state, msg.str().c_str());
    }

    void non_native_error(STATE, const char c) {
      std::ostringstream msg;
      msg << "'" << c << "' allowed only after types sSiIlL";
      Exception::raise_argument_error(state, msg.str().c_str());
    }
  }

// Pack Float elements
#define pack_float_elements(format)   pack_elements(Float, pack::to_f, format)

#define pack_double_le                pack_float_elements(pack_double_element_le)
#define pack_double_be                pack_float_elements(pack_double_element_be)

#define pack_float_le                 pack_float_elements(pack_float_element_le)
#define pack_float_be                 pack_float_elements(pack_float_element_be)

// Pack Integer elements
#define pack_integer_elements(format) pack_elements(Integer, pack::to_int, format)

#define pack_byte_element(v)          str.push_back(pack::check_long(state, v))
#define pack_byte                     pack_integer_elements(pack_byte_element)

#define pack_short_le                 pack_integer_elements(pack_short_element_le)
#define pack_short_be                 pack_integer_elements(pack_short_element_be)

#define pack_int_le                   pack_integer_elements(pack_int_element_le)
#define pack_int_be                   pack_integer_elements(pack_int_element_be)

#define pack_long_le                  pack_integer_elements(pack_long_element_le)
#define pack_long_be                  pack_integer_elements(pack_long_element_be)

// Pack UTF-8 elements
#define pack_utf8_element(v)          pack::utf8_encode(state, str, v)
#define pack_utf8                     pack_elements(Integer, pack::to_int, pack_utf8_element)

// Pack BER-compressed integers
#define pack_ber_element(v)           pack::ber_encode(state, str, v)
#define pack_ber                      pack_elements(Integer, pack::to_int, pack_ber_element)

// Wraps the logic for iterating over a number of elements,
// coercing them to the correct class and formatting them
// for the output string.
#define pack_elements(T, coerce, format)        \
  for(; index < stop; index++) {                \
    Object* item = self->get(state, index);     \
    T* value = try_as<T>(item);                 \
    if(!value) {                                \
      item = coerce(state, item);   \
      if(!item) return 0;                       \
      value = as<T>(item);                      \
    }                                           \
    format(value);                              \
  }

// Macros that depend on endianness
#ifdef RBX_LITTLE_ENDIAN

# define pack_double_element_le(v)  (pack::double_element(str, (v)->val))
# define pack_double_element_be(v)  (pack::swap_double(str, (v)->val))
# define pack_double                pack_double_le

# define pack_float_element_le(v)   (pack::float_element(str, (v)->val))
# define pack_float_element_be(v)   (pack::swap_float(str, (v)->val))
# define pack_float                 pack_float_le

# define pack_short_element_le(v)   (pack::short_element(str, pack::check_long(state, v)))
# define pack_short_element_be(v)   (pack::short_element(str, \
                                        pack::swap_2bytes(pack::check_long(state, v))))
# define pack_short                 pack_short_le

# define pack_int_element_le(v)     (pack::int_element(str, pack::check_long(state, v)))
# define pack_int_element_be(v)     (pack::int_element(str, \
                                        pack::swap_4bytes(pack::check_long(state, v))))
# define pack_int                   pack_int_le

# define pack_long_element_le(v)    (pack::long_element(str, pack::check_long_long(state, v)))
# define pack_long_element_be(v)    (pack::long_element(str, \
                                        pack::swap_8bytes(pack::check_long_long(state, v))))
# define pack_long                  pack_long_le

#else // Big endian

# define pack_double_element_le(v)  (pack::swap_double(str, (v)->val))
# define pack_double_element_be(v)  (pack::double_element(str, (v)->val))
# define pack_double                pack_double_be

# define pack_float_element_le(v)   (pack::swap_float(str, (v)->val))
# define pack_float_element_be(v)   (pack::float_element(str, (v)->val))
# define pack_float                 pack_float_be

# define pack_short_element_le(v)   (pack::short_element(str, \
                                        pack::swap_2bytes(pack::check_long(state, v))))
# define pack_short_element_be(v)   (pack::short_element(str, pack::check_long(state, v)))
# define pack_short                 pack_short_be

# define pack_int_element_le(v)     (pack::int_element(str, \
                                        pack::swap_4bytes(pack::check_long(state, v))))
# define pack_int_element_be(v)     (pack::int_element(str, pack::check_long(state, v)))
# define pack_int                   pack_int_be

# define pack_long_element_le(v)    (pack::long_element(str, \
                                        pack::swap_8bytes(pack::check_long_long(state, v))))
# define pack_long_element_be(v)    (pack::long_element(str, pack::check_long_long(state, v)))
# define pack_long                  pack_long_be

#endif

  String* Array::pack(STATE, String* directives) {
    // Ragel-specific variables
    const char* p;
    const char* pe;
    ByteArray* d = pack::prepare_directives(state, directives, &p, &pe);
    const char *eof = pe;
    int cs;

    // pack-specific variables
    Array* self = this;
    OnStack<2> sv(state, self, d);

    native_int array_size = self->size();
    native_int index = 0;
    native_int count UNUSED = 0;
    native_int count_flag = -1;
    native_int stop = 0;
    bool rest UNUSED = false;
    bool tainted UNUSED = false;
    bool untrusted UNUSED = false;
    bool ascii_encoding = false;
    bool utf8_encoding = false;
    bool string_encoding = false;

    String* string_value = 0;
    std::string str("");

    // Use information we have to reduce repeated allocation.
    str.reserve(array_size * 4);

    if(directives->byte_size() == 0) ascii_encoding = true;
    if(CBOOL(directives->tainted_p(state))) tainted = true;
    if(CBOOL(directives->untrusted_p(state))) untrusted = true;
%%{

  machine pack;

  include "pack.rl";

}%%

    if(en_main) {
      // do nothing
    }

    return force_as<String>(Primitives::failure());
  }
}


some helpful printfs on error (for now)

some helpful printfs on error (for now)

#include <stdlib.h>
#include <string.h>
#include <stdio.h>
#include <ruby/ruby.h>
#include "tnetstrings.h"

%%{
  machine tnetstrings;
  access parser->;

  action mark_ends {
    // this marks the start of the payload -
    // move ahead 2 for the colon and the type spec.
    parser->payload = p + 2;

    // this is a bit hacky, but I think it's the only way to do it.
    // only now do we know where the end is, since we've parsed the payload_size.
    // This will cause EOF actions to be triggered at the end of the payload.
    // TODO: eventually, have an option to read this from a stream,
    // which is arguably the whole point.
    eof = pe = parser->payload + parser->payload_size;

    // assert(pe < str + size, "Holy crap we overfloweded the buffer");
  }

  action collect_payload_size {
    parser->payload_size *= 10;
    parser->payload_size += CTOI(fc);
  }

  # -*- boolean (!) -*- #
  action wrap_true  { parser->result = TNETS_WRAP_TRUE;  }
  action wrap_false { parser->result = TNETS_WRAP_FALSE; }

  # -*- null (~) -*- #
  action wrap_null  { parser->result = Qnil;   }

  # -*- number (#) -*- #
  action neg {
    parser->seen_neg = 1;
  }

  action collect_num {
    parser->num_builder *= 10;
    parser->num_builder += CTOI(fc);
  }

  action wrap_num {
    if (parser->seen_neg) parser->num_builder *= -1;
    parser->result = TNETS_WRAP_NUM(parser->num_builder);
  }

  # -*- string (,) -*- #
  action wrap_str {
    parser->result = TNETS_WRAP_STR(p+1, parser->payload_size);
  }

  action parse_dict { }
  action parse_arr {
    VALUE arr = TNETS_NEW_ARR;
    tnets_parser* sub_parser = tnets_parser_new();
    char* sub_data = parser->payload;
    int sub_size = parser->payload_size;
    do {
      tnets_parser_chunk(sub_parser, sub_data, sub_size);
      TNETS_ARR_PUSH(arr, sub_parser->result);

      sub_data += (sub_parser->payload_size + 2 + 1);
      sub_size -= (sub_parser->payload_size + 2 + 1);
      // reset current state and such
      tnets_parser_clear(sub_parser);

    } while(sub_size > 0);
    free(sub_parser);

    parser->result = arr;
  }

  action error {
    printf("error!\n"); // TODO: real error handling
    printf("p: [%s]\n", p);
    printf("pe: [%s]\n", pe);
    printf("str: [%s]\n", str);
  }

  # parses the size spec at the start of the payload
  tnets_size = digit+ $collect_payload_size;
  colon = ':' @mark_ends;

  # primitives
  tnets_num  = '#' ('-' @neg)? (digit+ $collect_num) %/wrap_num;
  tnets_str  = ',' @wrap_str;
  # NB: any string that is not "true" will return "false".
  tnets_bool = '!' ('true' %/wrap_true | !'true' %/wrap_false);
  tnets_null = '~' %/wrap_null;

  # recursive structures
  tnets_dict = '}' @parse_dict;
  tnets_arr  = ']' @parse_arr;

  main := tnets_size colon (
    tnets_dict |
    tnets_arr  |
    tnets_num  |
    tnets_str  |
    tnets_bool |
    tnets_null
  ) @err(error);
}%%

%% write data;

int tnets_parser_init(tnets_parser *parser) {
  %% write init;
}

int tnets_parser_clear(tnets_parser *parser) {
  tnets_parser_init(parser);
  parser->num_builder = 0;
  parser->seen_neg = 0;
  parser->result = TNETS_WRAP_NULL;
  parser->payload = NULL;
  parser->payload_size = 0;
}

int tnets_parser_chunk(tnets_parser *parser, char *str, int size) {
  char *p = str;
  // pe gets re-set once we've parsed the length.
  // set to 10 characters here because that's the maximum
  // length of the length spec.
  char *pe = p + 11;
  char *eof;

  %% write exec;

printf("%s\n", RSTRING_PTR(rb_funcall(parser->result, rb_intern("inspect"), 0)));
  return 0;
}

tnets_parser *tnets_parser_new(void) {
  tnets_parser *parser = calloc(sizeof(tnets_parser), 1);
  tnets_parser_init(parser);
  return parser;
}

tnets_parser *parse_str(char *str, int size) {
  tnets_parser *parser = tnets_parser_new();
  tnets_parser_chunk(parser, str, size);
  return parser;
}

VALUE rb_parse_tnets(VALUE self, VALUE rbstr) {
  char *tnets = RSTRING_PTR(rbstr);
  int len = RSTRING_LEN(rbstr);

  tnets_parser *parser = parse_str(RSTRING_PTR(rbstr), len);

  VALUE result = parser->result;
  free(parser);

  return result;
}

void Init_tnetstrings() {
  VALUE module = rb_define_module("TnetstringsCMethods");

  rb_define_method(module, "c_parse", rb_parse_tnets, 1);
}

#include <stdlib.h>
#include <string.h>
#include <stdio.h>
#include <ruby/ruby.h>
#include "tnetstrings.h"

%%{
  machine tnetstrings;
  access parser->;

  action mark_ends {
    // this marks the start of the payload -
    // move ahead 2 for the colon and the type spec.
    parser->payload = p + 2;

    // this is a bit hacky, but I think it's the only way to do it.
    // only now do we know where the end is, since we've parsed the payload_size.
    // This will cause EOF actions to be triggered at the end of the payload.
    // TODO: eventually, have an option to read this from a stream,
    // which is arguably the whole point.
    eof = pe = parser->payload + parser->payload_size;

    // assert(pe < str + size, "Holy crap we overfloweded the buffer");
  }

  action collect_payload_size {
    parser->payload_size *= 10;
    parser->payload_size += CTOI(fc);
  }

  # -*- boolean (!) -*- #
  action wrap_true  { parser->result = TNETS_WRAP_TRUE;  }
  action wrap_false { parser->result = TNETS_WRAP_FALSE; }

  # -*- null (~) -*- #
  action wrap_null  { parser->result = Qnil;   }

  # -*- number (#) -*- #
  action neg {
    parser->seen_neg = 1;
  }

  action collect_num {
    parser->num_builder *= 10;
    parser->num_builder += CTOI(fc);
  }

  action wrap_num {
    if (parser->seen_neg) parser->num_builder *= -1;
    parser->result = TNETS_WRAP_NUM(parser->num_builder);
  }

  # -*- string (,) -*- #
  action wrap_str {
    parser->result = TNETS_WRAP_STR(p+1, parser->payload_size);
  }

  action parse_dict { }
  action parse_arr {
    VALUE arr = TNETS_NEW_ARR;
    tnets_parser* sub_parser = tnets_parser_new();
    char* sub_data = parser->payload;
    int sub_size = parser->payload_size;
    do {
      tnets_parser_chunk(sub_parser, sub_data, sub_size);
      TNETS_ARR_PUSH(arr, sub_parser->result);

      sub_data += (sub_parser->payload_size + 2 + 1);
      sub_size -= (sub_parser->payload_size + 2 + 1);
      // reset current state and such
      tnets_parser_clear(sub_parser);

    } while(sub_size > 0);
    free(sub_parser);

    parser->result = arr;
  }

  action error {
    printf("error!\n"); // TODO: real error handling
  }

  # parses the size spec at the start of the payload
  tnets_size = digit+ $collect_payload_size;
  colon = ':' @mark_ends;

  # primitives
  tnets_num  = '#' ('-' @neg)? (digit+ $collect_num) %/wrap_num;
  tnets_str  = ',' @wrap_str;
  # NB: any string that is not "true" will return "false".
  tnets_bool = '!' ('true' %/wrap_true | !'true' %/wrap_false);
  tnets_null = '~' %/wrap_null;

  # recursive structures
  tnets_dict = '}' @parse_dict;
  tnets_arr  = ']' @parse_arr;

  main := tnets_size colon (
    tnets_dict |
    tnets_arr  |
    tnets_num  |
    tnets_str  |
    tnets_bool |
    tnets_null
  ) @err(error);
}%%

%% write data;

int tnets_parser_init(tnets_parser *parser) {
  %% write init;
}

int tnets_parser_clear(tnets_parser *parser) {
  tnets_parser_init(parser);
  parser->num_builder = 0;
  parser->seen_neg = 0;
  parser->result = TNETS_WRAP_NULL;
  parser->payload = NULL;
  parser->payload_size = 0;
}

int tnets_parser_chunk(tnets_parser *parser, char *str, int size) {
  char *p = str;
  // pe gets re-set once we've parsed the length.
  // set to 10 characters here because that's the maximum
  // length of the length spec.
  char *pe = p + 11;
  char *eof;

  %% write exec;

printf("%s\n", RSTRING_PTR(rb_funcall(parser->result, rb_intern("inspect"), 0)));
  return 0;
}

tnets_parser *tnets_parser_new(void) {
  tnets_parser *parser = calloc(sizeof(tnets_parser), 1);
  tnets_parser_init(parser);
  return parser;
}

tnets_parser *parse_str(char *str, int size) {
  tnets_parser *parser = tnets_parser_new();
  tnets_parser_chunk(parser, str, size);
  return parser;
}

VALUE rb_parse_tnets(VALUE self, VALUE rbstr) {
  char *tnets = RSTRING_PTR(rbstr);
  int len = RSTRING_LEN(rbstr);

  tnets_parser *parser = parse_str(RSTRING_PTR(rbstr), len);

  VALUE result = parser->result;
  free(parser);

  return result;
}

void Init_tnetstrings() {
  VALUE module = rb_define_module("TnetstringsCMethods");

  rb_define_method(module, "c_parse", rb_parse_tnets, 1);
}


added documentation to lexer

added documentation to lexer

#ifndef HEXT_LEXER_H
#define HEXT_LEXER_H

#include <string>
#include <vector>
#include <sstream>
#include <stdexcept>
#include <utility>

#include "hext/token.h"
#include "hext/lexer-util.h"

namespace hext {

namespace ragel {
  %%{
    machine hext;
    include "hext.rl";
    write data;
  }%%
} // namespace ragel

/// lexer contains the ragel state machine and shields the user from its
/// details.
class lexer
{
public:
  class lex_error : public std::runtime_error
  {
  public:
    explicit lex_error(const std::string& msg);
  };

  /// Setup ragel
  lexer(const char * begin, const char * end);

  /// Throws lexer::lex_error on invalid input (by calling lexer::throw_error)
  std::vector<token> lex();

private:
  /// Print diagnostics and throw lexer::lex_error.
  void throw_error() const;

  /// A pointer to the beginning of the overall input is needed to provide
  /// good error diagnostics.
  const char * p_begin;

  /// ragel's runtime state
  const char * p;
  const char * pe;
  const char * eof;
  int cs;
};

} // namespace hext

#endif // HEXT_LEXER_H

#ifndef HEXT_LEXER_H
#define HEXT_LEXER_H

#include <string>
#include <vector>
#include <sstream>
#include <stdexcept>
#include <utility>

#include "hext/token.h"
#include "hext/lexer-util.h"

namespace hext {

namespace ragel {
  %%{
    machine hext;
    include "hext.rl";
    write data;
  }%%
} // namespace ragel

class lexer
{
public:
  class lex_error : public std::runtime_error
  {
  public:
    explicit lex_error(const std::string& msg);
  };

  lexer(const char * begin, const char * end);

  std::vector<token> lex();

private:
  void throw_error() const;

  const char * p_begin;
  const char * p;
  const char * pe;
  const char * eof;
  int cs;
};

} // namespace hext

#endif // HEXT_LEXER_H


handle EOF on one list branch

handle EOF on one list branch

# begin: ragel
=begin
%%{
  machine bel;

  include 'common.rl';
  include 'identifier.rl';
  include 'string.rl';

  action start_list {
    $stderr.puts "LIST start_list"
    @list_opened = true
    @incomplete[:list] = list()
  }

  action stop_list {
    $stderr.puts "LIST stop_list"
    @list_closed = true
  }

  action add_string {
    $stderr.puts "LIST add_string"
    string = @buffers.delete(:string)
    item = list_item(string, complete: string.complete)
    @incomplete[:list] <<= item
  }

  action add_ident {
    $stderr.puts "LIST add_ident"
    ident = @buffers.delete(:ident)
    item = list_item(ident, complete: ident.complete)
    @incomplete[:list] <<= item
  }

  action list_end {
    $stderr.puts "LIST list_end"
    if @list_opened && @list_closed
      list = @incomplete.delete(:list)
      list.complete = true
    elsif !@list_closed
      list = @incomplete.delete(:list)
      list.complete = false
    end
    @buffers[:list] = list
  }

  action a_list_eof {
    $stderr.puts "LIST a_list_eof"
    list = @incomplete.delete(:list)
    string = @buffers.delete(:string)
    unless string.nil?
      item = list_item(string, complete: string.complete)
      list <<= item
    end
    ident = @buffers.delete(:ident)
    unless ident.nil?
      item = list_item(ident, complete: ident.complete)
      list <<= item
    end
    if @list_opened && @list_closed
      list.complete = true
    else
      list.complete = false
    end
    @buffers[:list] = list
  }

  action list_node_eof {
    $stderr.puts "LIST list_node_eof"
    list = @incomplete.delete(:list)
    string = @buffers.delete(:string)
    item = list_item(string, complete: string.complete)
    list <<= item
    list.complete = false
    yield list
  }

  action yield_list {
    $stderr.puts "LIST yield_list"
    yield @buffers[:list]
  }

  START_LIST = '{' SP* >start_list;
  END_LIST = '}' %stop_list;

  string_item =
    a_string
    %add_string
    ;

  ident_item =
    an_ident
    %add_ident
    ;

  item =
    string_item |
    ident_item
    ;

  list_item_0 =
    item
    ;

  list_item_n =
    COMMA_DELIM
    item
    ;

  items =
    list_item_0
    list_item_n*
    SP*
    ;

  a_list =
    START_LIST
    items
    END_LIST
    %list_end
    @eof(a_list_eof)
    ;

  list_node :=
    START_LIST
    items?
    @eof(list_node_eof)
    END_LIST?
    @eof(list_node_eof)
    NL?
    %list_end
    %yield_list
    ;
}%%
=end
# end: ragel

require_relative '../ast/node'
require_relative '../mixin/buffer'
require_relative '../nonblocking_io_wrapper'

module BELParser
  module Parsers
    module Common
      module List

        class << self

          MAX_LENGTH = 1024 * 128 # 128K

          def parse(content)
            return nil unless content

            Parser.new(content).each do |obj|
              yield obj
            end
          end
        end

        private

        class Parser
          include Enumerable
          include BELParser::Parsers::Buffer
          include BELParser::Parsers::AST::Sexp

          def initialize(content)
            @content = content
            $stderr.puts "\n---\ncontent: '" + @content + "'"
      # begin: ragel
            %% write data;
      # end: ragel
          end

          def each
            @buffers      = {}
            @incomplete   = {}
            @list_opened  = false
            @list_closed  = false
            data          = @content.unpack('C*')
            p             = 0
            pe            = data.length
            eof           = data.length

      # begin: ragel
            %% write init;
            %% write exec;
      # end: ragel
          end
        end
      end
    end
  end
end

if __FILE__ == $0
  $stdin.each_line do |line|
    BELParser::Parsers::Common::List.parse(line) { |obj|
      puts obj.inspect
    }
  end
end

# vim: ft=ruby ts=2 sw=2:
# encoding: utf-8

# begin: ragel
=begin
%%{
  machine bel;

  include 'common.rl';
  include 'identifier.rl';
  include 'string.rl';

  action start_list {
    $stderr.puts "LIST start_list"
    @list_opened = true
    @incomplete[:list] = list()
  }

  action stop_list {
    $stderr.puts "LIST stop_list"
    @list_closed = true
  }

  action add_string {
    $stderr.puts "LIST add_string"
    string = @buffers.delete(:string)
    item = list_item(string, complete: string.complete)
    @incomplete[:list] <<= item
  }

  action add_ident {
    $stderr.puts "LIST add_ident"
    ident = @buffers.delete(:ident)
    item = list_item(ident, complete: ident.complete)
    @incomplete[:list] <<= item
  }

  action list_end {
    $stderr.puts "LIST list_end"
    if @list_opened && @list_closed
      list = @incomplete.delete(:list)
      list.complete = true
    elsif !@list_closed
      list = @incomplete.delete(:list)
      list.complete = false
    end
    @buffers[:list] = list
  }

  action list_node_eof {
    $stderr.puts "LIST list_node_eof"
    list = @incomplete.delete(:list)
    string = @buffers.delete(:string)
    item = list_item(string, complete: string.complete)
    list <<= item
    list.complete = false
    yield list
  }

  action yield_list {
    $stderr.puts "LIST yield_list"
    yield @buffers[:list]
  }

  START_LIST = '{' SP* >start_list;
  END_LIST = '}' %stop_list;

  string_item =
    a_string
    %add_string
    ;

  ident_item =
    an_ident
    %add_ident
    ;

  item =
    string_item |
    ident_item
    ;

  list_item_0 =
    item
    ;

  list_item_n =
    COMMA_DELIM
    item
    ;

  items =
    list_item_0
    list_item_n*
    SP*
    ;

  a_list =
    START_LIST
    items
    END_LIST
    %list_end
    ;

  list_node :=
    START_LIST
    items?
    @eof(list_node_eof)
    END_LIST?
    @eof(list_node_eof)
    NL?
    %list_end
    %yield_list
    ;
}%%
=end
# end: ragel

require_relative '../ast/node'
require_relative '../mixin/buffer'
require_relative '../nonblocking_io_wrapper'

module BELParser
  module Parsers
    module Common
      module List

        class << self

          MAX_LENGTH = 1024 * 128 # 128K

          def parse(content)
            return nil unless content

            Parser.new(content).each do |obj|
              yield obj
            end
          end
        end

        private

        class Parser
          include Enumerable
          include BELParser::Parsers::Buffer
          include BELParser::Parsers::AST::Sexp

          def initialize(content)
            @content = content
            $stderr.puts "\n---\ncontent: '" + @content + "'"
      # begin: ragel
            %% write data;
      # end: ragel
          end

          def each
            @buffers      = {}
            @incomplete   = {}
            @list_opened  = false
            @list_closed  = false
            data          = @content.unpack('C*')
            p             = 0
            pe            = data.length
            eof           = data.length

      # begin: ragel
            %% write init;
            %% write exec;
      # end: ragel
          end
        end
      end
    end
  end
end

if __FILE__ == $0
  $stdin.each_line do |line|
    BELParser::Parsers::Common::List.parse(line) { |obj|
      puts obj.inspect
    }
  end
end

# vim: ft=ruby ts=2 sw=2:
# encoding: utf-8


Fixed constants starting with an underbar

#include "parse.h"
#include <stdio.h>
#include <string.h>
#include <stdlib.h>
#include <assert.h>
#import <Tranquil/CodeGen/TQNodeBlock.h>

#define PushStr(dblQuot, rstr) \
    [strings addObject:[NSDictionary dictionaryWithObjectsAndKeys: \
        [NSNumber numberWithBool:dblQuot], @"dblQuot", \
        [NSNumber numberWithBool:rstr], @"isRstr", \
        [NSMutableString string], @"value", nil]];
#define PopStr() ({ id last = [Str() retain]; [strings removeLastObject]; [last autorelease]; })
#define Str() [[strings lastObject] objectForKey:@"value"]
#define IsDblQuot() [[[strings lastObject] objectForKey:@"dblQuot"] boolValue]
#define IsRstr() [[[strings lastObject] objectForKey:@"isRstr"] boolValue]

typedef struct {
    int currentLine;
    BOOL atBeginningOfExpr;
    TQNodeRootBlock *root;
    NSError *syntaxError;
} TQParserState;

#define NSStr(lTrim, rTrim) (ts ? [[[NSMutableString alloc] initWithBytes:ts+(lTrim) length:te-ts-(lTrim)-(rTrim) encoding:NSUTF8StringEncoding] autorelease] : nil)

#define CopyCStr() ((unsigned char *)strndup((char *)ts, te-ts))

#define _EmitToken(tokenId, val) do { \
    int tokenId_ = tokenId; \
    id val_ = val; \
    /*NSLog(@"emitting %d = '%@' on line: %d", tokenId, val_, parserState.currentLine); */\
    Parse(parser, tokenId_, [TQToken withId:tokenId value:val_ line:parserState.currentLine], &parserState); \
    parserState.atBeginningOfExpr = NO; \
} while(0);

#define EmitToken(tokenId) EmitStringToken((tokenId), 0, 0)

#define EmitStringToken(tokenId, lTrim, rTrim) _EmitToken(tokenId, NSStr(lTrim, rTrim))

#define EmitConstStringToken(tokenId, lTrim, rTrim) do { \
    BOOL hasQuotes = *(ts+1) == '"'; \
    EmitStringToken((tokenId), lTrim+1+hasQuotes, rTrim+hasQuotes); \
} while(0)

#define EmitIntToken(tokenId, base, prefixLen) do { \
    unsigned char *str = CopyCStr(); \
    long long numVal = strtoll((char *)str + (prefixLen), NULL, (base)); \
    _EmitToken(tokenId, [NSNumber numberWithLongLong:numVal]); \
    free(str); \
} while(0)

#define EmitFloatToken(tokenId) do { \
    unsigned char *str = CopyCStr(); \
    _EmitToken((tokenId), [NSNumber numberWithDouble:atof((char *)str)]); \
    free(str); \
} while(0)

#define IncrementLine() (++parserState.currentLine)

#define BacktrackTerm() do { \
    if(*p == '}') \
        --p; \
    for(unsigned char *cursor = ts; cursor != te; ++cursor) { \
        if(*cursor == '\n') \
            IncrementLine(); \
    } \
} while(0)

#define ExprBeg() parserState.atBeginningOfExpr = YES

@interface TQToken : NSObject
@property(readwrite, assign) NSUInteger id, lineNumber;
@property(readwrite, strong) id value;
+ (TQToken *)withId:(int)id value:(id)value line:(int)line;
@end
@implementation TQToken
+ (TQToken *)withId:(int)id value:(id)value line:(int)line
{
    TQToken *ret = [self new];
    ret.id = id;
    ret.value = value;
    ret.lineNumber = line;
    return [ret autorelease];
}
@end

%%{
    machine Tranquil;
    alphtype unsigned char;
    include WChar "utf8.rl";

    int         = udigit+;
    float       = (udigit+)? '.' udigit+;
    hex         = "0x"i xdigit+;
    bin         = "0b"i [0-1]+;
    oct         = "0o"i [0-7]+;
    anybase     = udigit+ "r"i ([0-9]|[a-zA-Z])+;

    lGuillemet  = 0xC2 0xAB; # «
    rGuillemet  = 0xC2 0xBB; # »
    specialChar = 0xC2 | '"' | "'" | '.' | ',' | ';' | ':' | '|' | '#' | '@' | '~' | '`' | '{' | '}' | '[' | ']' | '(' | ')' | '+' | '-' | '*' | '/' | '%' | '=' | '<' | '>' | '^' | '\\' | '\r' | '\n' | space;

    char        = (any - specialChar) | (0xC2 ^(0xAB|0xBB));
    constant    = '_'* uupper char*;
    identifier  = (char -- (uupper|udigit)) char*;
    comment     = '\\' [^\n]*;
    nl          = ((space|comment)* '\n' space*);
    whitespace  = (nl|space|comment);
    whitespaceNoNl  = (" "|"\t"|comment);
    term        = whitespace* (nl|"}");

    simpleStr   = ('"' (any - '"')* '"') | ("'" (any - "'")* "'");
    constStr    = "#" (simpleStr | (char | [:@#~+\-*/%=<>^])+);
    selector    = char* ":";

    regexCont   = (ualnum | ascii) - '/' - '\\';

string := |*
    '\n'                             => { [Str() appendString:@"\n"]; IncrementLine();                   };
    '\\n'                            => { [Str() appendString:@"\n"];                                    };
    '\\t'                            => { [Str() appendString:@"\t"];                                    };
    '\\r'                            => { [Str() appendString:@"\r"];                                    };
    '\\"'                            => { [Str() appendString:@"\""];                                    };
    "\\'"                            => { [Str() appendString:@"'"];                                     };
    "\\\\"                           => { [Str() appendString:@"\\"];                                    };
    "\\" lGuillemet                  => { [Str() appendString:@"«"];                                     };
    '\\x' %{ temp1 = p; } [0-9a-zA-Z]* => {
        long long code = strtoll((char*)temp1, NULL, 16);
        [Str() appendFormat:@"%c", (int)code];
    };
    "\\" any                         => { TQAssert(NO, @"Invalid escape %@", NSStr(0,0));                };

    lGuillemet                       => { _EmitToken(IsRstr() ? MSTR : LSTR, Str()); fret;               };
    '"' term => {
        if(IsDblQuot()) {
            _EmitToken(IsRstr() ? RSTRNL : STRNL, PopStr());
            BacktrackTerm();
            fret;
        } else
            [Str() appendString:NSStr(0,0)];
    };
    '"' => {
        if(IsDblQuot()) {
            _EmitToken(IsRstr() ? RSTR : STR, PopStr());
            fret;
        } else
            [Str() appendString:NSStr(0,0)];
    };
    "'" term => {
        if(!IsDblQuot()) {
            _EmitToken(IsRstr() ? RSTRNL : STRNL, PopStr());
            BacktrackTerm();
            fret;
        } else
            [Str() appendString:NSStr(0,0)];
    };
    "'" => {
        if(!IsDblQuot()) {
            _EmitToken(IsRstr() ? RSTR : STR, PopStr());
            fret;
        } else
            [Str() appendString:NSStr(0,0)];
    };

    any => { [Str() appendString:NSStr(0, 0)];                                                           };
*|;

regex := |*
    '\n'                             => { [Str() appendString:@"\n"]; IncrementLine();                   };
    "\\/"                            => { [Str() appendString:@"\\/"];                                   };
    "\\"                             => { [Str() appendString:@"\\"];                                    };
    regexCont                        => { [Str() appendString:NSStr(0, 0)];                              };
    "/" [im]* term                   => { [Str() appendString:@"/"]; _EmitToken(REGEXNL, PopStr());
                                          BacktrackTerm(); fret;                                         };
    "/" [im]*                        => { [Str() appendString:@"/"]; _EmitToken(REGEX, PopStr()); fret;  };
*|;

main := |*
# Symbols
 # Forward slash is context sensitive, at the beginning of an expression it means the start of a regular expression
    "/"                              => {
        if(!parserState.atBeginningOfExpr) {
            if(p != eof && *(p+1) == '=') {
                EmitToken(ASSIGNDIV);
                p += 2;
            } else
                EmitToken(FSLASH);
            ExprBeg();
        } else {
            PushStr(false, false);
            [Str() appendString:@"/"];
            fcall regex;
        }
    };
    "{"                              => { EmitToken(LBRACE);     ExprBeg();                               };
    "}" term                         => { EmitToken(RBRACENL);   ExprBeg(); BacktrackTerm();              };
    "}"                              => { EmitToken(RBRACE);                                              };
    "["                              => { EmitToken(LBRACKET);   ExprBeg();                               };
    "]" term                         => { EmitToken(RBRACKETNL); ExprBeg(); BacktrackTerm();              };
    "]"                              => { EmitToken(RBRACKET);                                            };
    "("                              => { EmitToken(LPAREN);     ExprBeg();                               };
    ")" term                         => { EmitToken(RPARENNL);   ExprBeg(); BacktrackTerm();              };
    ")"                              => { EmitToken(RPAREN);                                              };
    ","                              => { EmitToken(COMMA);      ExprBeg();                               };
    "`" term                         => { EmitToken(BACKTICKNL); BacktrackTerm();                         };
    "`"                              => { EmitToken(BACKTICK);                                            };
    ";"                              => { EmitToken(SEMICOLON);                                           };
    "="                              => { EmitToken(ASSIGN);     ExprBeg();                               };
    "+="                             => { EmitToken(ASSIGNADD);  ExprBeg();                               };
    "-="                             => { EmitToken(ASSIGNSUB);  ExprBeg();                               };
    "*="                             => { EmitToken(ASSIGNMUL);  ExprBeg();                               };
    "||="                            => { EmitToken(ASSIGNOR);   ExprBeg();                               };
    "+"                              => { EmitToken(PLUS);       ExprBeg();                               };
    "-"                              => { EmitToken(MINUS);      ExprBeg();                               };
    "--"                             => { EmitToken(DECR);                                                };
    "--" term                        => { EmitToken(DECRNL);     ExprBeg();BacktrackTerm();               };
    "++"                             => { EmitToken(INCR);                                                };
    "++" term                        => { EmitToken(INCRNL);     ExprBeg();BacktrackTerm();               };
    "*"                              => { EmitToken(ASTERISK);   ExprBeg();                               };
    "%"                              => { EmitToken(PERCENT);    ExprBeg();                               };
    "~"                              => { EmitToken(TILDE);      ExprBeg();                               };
    "@"                              => { EmitToken(ATMARK);                                              };
    "|"                              => { EmitToken(PIPE);       ExprBeg();                               };
    "^"                              => { EmitToken(CARET);      ExprBeg();                               };
    "?"                              => { EmitToken(TERNIF);     ExprBeg();                               };
    "!"                              => { EmitToken(TERNELSE);   ExprBeg();                               };
    "=="                             => { EmitToken(EQUAL);      ExprBeg();                               };
    "!="                             => { EmitToken(INEQUAL);    ExprBeg();                               };
    "~="                             => { EmitToken(INEQUAL);    ExprBeg();                               };
    "<"                              => { EmitToken(LESSER);     ExprBeg();                               };
    ">"                              => { EmitToken(GREATER);    ExprBeg();                               };
    "<="                             => { EmitToken(LEQUAL);     ExprBeg();                               };
    ">="                             => { EmitToken(GEQUAL);     ExprBeg();                               };
    "=>"                             => { EmitToken(DICTSEP);    ExprBeg();                               };

# Message selectors
    selector                         => { EmitStringToken(SELPART, 0, 1); ExprBeg();                      };

# Keywords
    "if"      term?                  => { EmitToken(IF);        ExprBeg(); BacktrackTerm();               };
    "unless"  term?                  => { EmitToken(UNLESS);    ExprBeg(); BacktrackTerm();               };
    "then"    term?                  => { EmitToken(THEN);      ExprBeg(); BacktrackTerm();               };
    "do"    term?                    => { EmitToken(DO);        ExprBeg(); BacktrackTerm();               };
    "else"    term?                  => { EmitToken(ELSE);      ExprBeg(); BacktrackTerm();               };
    "and"|"&&"term?                  => { EmitToken(AND);       ExprBeg(); BacktrackTerm();               };
    "or"|"||" term?                  => { EmitToken(OR);        ExprBeg(); BacktrackTerm();               };
    "while"   term?                  => { EmitToken(WHILE);     ExprBeg(); BacktrackTerm();               };
    "until"   term?                  => { EmitToken(UNTIL);     ExprBeg(); BacktrackTerm();               };
    "import"  term?                  => { EmitToken(IMPORT);    ExprBeg(); BacktrackTerm();               };
    "async"   term?                  => { EmitToken(ASYNC);     ExprBeg(); BacktrackTerm();               };
    "wait"    term                   => { EmitToken(WAITNL);    ExprBeg(); BacktrackTerm();               };
    "wait"                           => { EmitToken(WAIT);                                                };
    "lock"                           => { EmitToken(LOCK);                                                };
    "whenFinished"                   => { EmitToken(WHENFINISHED);                                        };
    "collect"                        => { EmitToken(COLLECT);                                             };
    "break"   term                   => { EmitToken(BREAKNL);   ExprBeg(); BacktrackTerm();               };
    "break"                          => { EmitToken(BREAK);                                               };
    "skip"    term                   => { EmitToken(SKIPNL);    ExprBeg(); BacktrackTerm();               };
    "skip"                           => { EmitToken(SKIP);                                                };
    "self"    term                   => { EmitToken(SELFNL);    ExprBeg(); BacktrackTerm();               };
    "super"   term                   => { EmitToken(SUPERNL);   ExprBeg(); BacktrackTerm();               };
    "..."     term                   => { EmitToken(VAARGNL);   ExprBeg(); BacktrackTerm();               };
    "nil"     term                   => { EmitToken(NILNL);     ExprBeg(); BacktrackTerm();               };
    "valid"   term                   => { EmitToken(VALIDNL);   ExprBeg(); BacktrackTerm();               };
    "yes"     term                   => { EmitToken(YESTOKNL);  ExprBeg(); BacktrackTerm();               };
    "no"      term                   => { EmitToken(NOTOKNL);   ExprBeg(); BacktrackTerm();               };
    "nothing" term                   => { EmitToken(NOTHINGNL); ExprBeg(); BacktrackTerm();               };
    "self"                           => { EmitToken(SELF);                                                };
    "super"                          => { EmitToken(SUPER);                                               };
    "..."                            => { EmitToken(VAARG);                                               };
    "nil"                            => { EmitToken(NIL);                                                 };
    "valid"                          => { EmitToken(VALID);                                               };
    "yes"                            => { EmitToken(YESTOK);                                              };
    "no"                             => { EmitToken(NOTOK);                                               };
    "nothing"                        => { EmitToken(NOTHING);                                             };

# Identifiers
    constant   %{temp1 = p;} term    => { EmitStringToken(CONSTNL, 0, te-temp1); ExprBeg(); BacktrackTerm(); };
    constant                         => { EmitStringToken(CONST,   0, 0);                                 };

    identifier %{temp1 = p;} term    => { EmitStringToken(IDENTNL, 0, te-temp1); ExprBeg(); BacktrackTerm(); };
    identifier                       => { EmitStringToken(IDENT,   0, 0);                                 };

# Literals
    int   term                       => { EmitIntToken(NUMBERNL, 10, 0); ExprBeg(); BacktrackTerm();      };
    float term                       => { EmitFloatToken(NUMBERNL);      ExprBeg(); BacktrackTerm();      };
    bin   term                       => { EmitIntToken(NUMBERNL, 2,  2); ExprBeg(); BacktrackTerm();      };
    oct   term                       => { EmitIntToken(NUMBERNL, 8,  2); ExprBeg(); BacktrackTerm();      };
    hex   term                       => { EmitIntToken(NUMBERNL, 16, 2); ExprBeg(); BacktrackTerm();      };
    anybase term                     => {
        // Find the base
        NSString *str = NSStr(0,0);
        NSString *baseStr = [str substringToIndex:[str rangeOfString:@"r"].location];
        EmitIntToken(NUMBERNL, atoi([baseStr UTF8String]), [baseStr length]+1); ExprBeg(); BacktrackTerm();
    };

    int                              => { EmitIntToken(NUMBER, 10, 0);                                    };
    float                            => { EmitFloatToken(NUMBER);                                         };
    bin                              => { EmitIntToken(NUMBER, 2,  2);                                    };
    oct                              => { EmitIntToken(NUMBER, 8,  2);                                    };
    hex                              => { EmitIntToken(NUMBER, 16, 2);                                    };

    constStr %{temp1 = p;} term      => { EmitConstStringToken(CONSTSTRNL, 0, te-temp1); ExprBeg(); BacktrackTerm(); };
    constStr                         => { EmitConstStringToken(CONSTSTR, 0, 0);                           };

    '"'                              => { PushStr(YES, NO); fcall string;                                 };
    "'"                              => { PushStr(NO, NO);  fcall string;                                 };
    rGuillemet                       => {
        BOOL dbl = IsDblQuot();
        PopStr();
        PushStr(dbl, YES);
        fcall string;
    };

    "."                              => { EmitToken(PERIOD);                                              };
    nl                               => { IncrementLine();                                                };
    space;
*|;
}%%

%% write data nofinal;

#include "parse.mm"

extern "C" TQNode *TQParseString(NSString *str, NSError **aoErr)
{
    if(![str hasSuffix:@"\n"])
        str = [str stringByAppendingString:@"\n"];

    int cs, act;
    unsigned char *ts, *te = 0;

    // Lexer setup
    unsigned char *p = (unsigned char *)[str UTF8String];
    unsigned char *pe = p + strlen((char*)p);
    unsigned char *eof = pe;
    int top;
    int stack[1024];
    NSMutableArray *strings = [NSMutableArray array];

    unsigned char *temp1, *temp2;

    // Parser setup
    void *parser = ParseAlloc(malloc);
    TQParserState parserState = {
        1, YES, [TQNodeRootBlock new], nil
    };

    @try {
        %% write init;
        %% write exec;

        if(p != pe)
            fprintf(stderr, "invalid character '%c'\n", p[0]);

        _EmitToken(0, @"EOF");
        return [parserState.root autorelease];
    } @catch(NSException *e) {
        if(aoErr)
            *aoErr = parserState.syntaxError;
        return nil;
    } @finally {
        ParseFree(parser, free);
    }
    return nil;
}

#include "parse.h"
#include <stdio.h>
#include <string.h>
#include <stdlib.h>
#include <assert.h>
#import <Tranquil/CodeGen/TQNodeBlock.h>

#define PushStr(dblQuot, rstr) \
    [strings addObject:[NSDictionary dictionaryWithObjectsAndKeys: \
        [NSNumber numberWithBool:dblQuot], @"dblQuot", \
        [NSNumber numberWithBool:rstr], @"isRstr", \
        [NSMutableString string], @"value", nil]];
#define PopStr() ({ id last = [Str() retain]; [strings removeLastObject]; [last autorelease]; })
#define Str() [[strings lastObject] objectForKey:@"value"]
#define IsDblQuot() [[[strings lastObject] objectForKey:@"dblQuot"] boolValue]
#define IsRstr() [[[strings lastObject] objectForKey:@"isRstr"] boolValue]

typedef struct {
    int currentLine;
    BOOL atBeginningOfExpr;
    TQNodeRootBlock *root;
    NSError *syntaxError;
} TQParserState;

#define NSStr(lTrim, rTrim) (ts ? [[[NSMutableString alloc] initWithBytes:ts+(lTrim) length:te-ts-(lTrim)-(rTrim) encoding:NSUTF8StringEncoding] autorelease] : nil)

#define CopyCStr() ((unsigned char *)strndup((char *)ts, te-ts))

#define _EmitToken(tokenId, val) do { \
    int tokenId_ = tokenId; \
    id val_ = val; \
    /*NSLog(@"emitting %d = '%@' on line: %d", tokenId, val_, parserState.currentLine); */\
    Parse(parser, tokenId_, [TQToken withId:tokenId value:val_ line:parserState.currentLine], &parserState); \
    parserState.atBeginningOfExpr = NO; \
} while(0);

#define EmitToken(tokenId) EmitStringToken((tokenId), 0, 0)

#define EmitStringToken(tokenId, lTrim, rTrim) _EmitToken(tokenId, NSStr(lTrim, rTrim))

#define EmitConstStringToken(tokenId, lTrim, rTrim) do { \
    BOOL hasQuotes = *(ts+1) == '"'; \
    EmitStringToken((tokenId), lTrim+1+hasQuotes, rTrim+hasQuotes); \
} while(0)

#define EmitIntToken(tokenId, base, prefixLen) do { \
    unsigned char *str = CopyCStr(); \
    long long numVal = strtoll((char *)str + (prefixLen), NULL, (base)); \
    _EmitToken(tokenId, [NSNumber numberWithLongLong:numVal]); \
    free(str); \
} while(0)

#define EmitFloatToken(tokenId) do { \
    unsigned char *str = CopyCStr(); \
    _EmitToken((tokenId), [NSNumber numberWithDouble:atof((char *)str)]); \
    free(str); \
} while(0)

#define IncrementLine() (++parserState.currentLine)

#define BacktrackTerm() do { \
    if(*p == '}') \
        --p; \
    for(unsigned char *cursor = ts; cursor != te; ++cursor) { \
        if(*cursor == '\n') \
            IncrementLine(); \
    } \
} while(0)

#define ExprBeg() parserState.atBeginningOfExpr = YES

@interface TQToken : NSObject
@property(readwrite, assign) NSUInteger id, lineNumber;
@property(readwrite, strong) id value;
+ (TQToken *)withId:(int)id value:(id)value line:(int)line;
@end
@implementation TQToken
+ (TQToken *)withId:(int)id value:(id)value line:(int)line
{
    TQToken *ret = [self new];
    ret.id = id;
    ret.value = value;
    ret.lineNumber = line;
    return [ret autorelease];
}
@end

%%{
    machine Tranquil;
    alphtype unsigned char;
    include WChar "utf8.rl";

    int         = udigit+;
    float       = (udigit+)? '.' udigit+;
    hex         = "0x"i xdigit+;
    bin         = "0b"i [0-1]+;
    oct         = "0o"i [0-7]+;
    anybase     = udigit+ "r"i ([0-9]|[a-zA-Z])+;

    lGuillemet  = 0xC2 0xAB; # «
    rGuillemet  = 0xC2 0xBB; # »
    specialChar = 0xC2 | '"' | "'" | '.' | ',' | ';' | ':' | '|' | '#' | '@' | '~' | '`' | '{' | '}' | '[' | ']' | '(' | ')' | '+' | '-' | '*' | '/' | '%' | '=' | '<' | '>' | '^' | '\\' | '\r' | '\n' | space;

    char        = (any - specialChar) | (0xC2 ^(0xAB|0xBB));
    constant    = uupper char*;
    identifier  = (char -- (uupper|udigit)) char*;
    comment     = '\\' [^\n]*;
    nl          = ((space|comment)* '\n' space*);
    whitespace  = (nl|space|comment);
    whitespaceNoNl  = (" "|"\t"|comment);
    term        = whitespace* (nl|"}");

    simpleStr   = ('"' (any - '"')* '"') | ("'" (any - "'")* "'");
    constStr    = "#" (simpleStr | (char | [:@#~+\-*/%=<>^])+);
    selector    = char* ":";

    regexCont   = (ualnum | ascii) - '/' - '\\';

string := |*
    '\n'                             => { [Str() appendString:@"\n"]; IncrementLine();                   };
    '\\n'                            => { [Str() appendString:@"\n"];                                    };
    '\\t'                            => { [Str() appendString:@"\t"];                                    };
    '\\r'                            => { [Str() appendString:@"\r"];                                    };
    '\\"'                            => { [Str() appendString:@"\""];                                    };
    "\\'"                            => { [Str() appendString:@"'"];                                     };
    "\\\\"                           => { [Str() appendString:@"\\"];                                    };
    "\\" lGuillemet                  => { [Str() appendString:@"«"];                                     };
    '\\x' %{ temp1 = p; } [0-9a-zA-Z]* => {
        long long code = strtoll((char*)temp1, NULL, 16);
        [Str() appendFormat:@"%c", (int)code];
    };
    "\\" any                         => { TQAssert(NO, @"Invalid escape %@", NSStr(0,0));                };

    lGuillemet                       => { _EmitToken(IsRstr() ? MSTR : LSTR, Str()); fret;               };
    '"' term => {
        if(IsDblQuot()) {
            _EmitToken(IsRstr() ? RSTRNL : STRNL, PopStr());
            BacktrackTerm();
            fret;
        } else
            [Str() appendString:NSStr(0,0)];
    };
    '"' => {
        if(IsDblQuot()) {
            _EmitToken(IsRstr() ? RSTR : STR, PopStr());
            fret;
        } else
            [Str() appendString:NSStr(0,0)];
    };
    "'" term => {
        if(!IsDblQuot()) {
            _EmitToken(IsRstr() ? RSTRNL : STRNL, PopStr());
            BacktrackTerm();
            fret;
        } else
            [Str() appendString:NSStr(0,0)];
    };
    "'" => {
        if(!IsDblQuot()) {
            _EmitToken(IsRstr() ? RSTR : STR, PopStr());
            fret;
        } else
            [Str() appendString:NSStr(0,0)];
    };

    any => { [Str() appendString:NSStr(0, 0)];                                                           };
*|;

regex := |*
    '\n'                             => { [Str() appendString:@"\n"]; IncrementLine();                   };
    "\\/"                            => { [Str() appendString:@"\\/"];                                   };
    "\\"                             => { [Str() appendString:@"\\"];                                    };
    regexCont                        => { [Str() appendString:NSStr(0, 0)];                              };
    "/" [im]* term                   => { [Str() appendString:@"/"]; _EmitToken(REGEXNL, PopStr());
                                          BacktrackTerm(); fret;                                         };
    "/" [im]*                        => { [Str() appendString:@"/"]; _EmitToken(REGEX, PopStr()); fret;  };
*|;

main := |*
# Symbols
 # Forward slash is context sensitive, at the beginning of an expression it means the start of a regular expression
    "/"                              => {
        if(!parserState.atBeginningOfExpr) {
            if(p != eof && *(p+1) == '=') {
                EmitToken(ASSIGNDIV);
                p += 2;
            } else
                EmitToken(FSLASH);
            ExprBeg();
        } else {
            PushStr(false, false);
            [Str() appendString:@"/"];
            fcall regex;
        }
    };
    "{"                              => { EmitToken(LBRACE);     ExprBeg();                               };
    "}" term                         => { EmitToken(RBRACENL);   ExprBeg(); BacktrackTerm();              };
    "}"                              => { EmitToken(RBRACE);                                              };
    "["                              => { EmitToken(LBRACKET);   ExprBeg();                               };
    "]" term                         => { EmitToken(RBRACKETNL); ExprBeg(); BacktrackTerm();              };
    "]"                              => { EmitToken(RBRACKET);                                            };
    "("                              => { EmitToken(LPAREN);     ExprBeg();                               };
    ")" term                         => { EmitToken(RPARENNL);   ExprBeg(); BacktrackTerm();              };
    ")"                              => { EmitToken(RPAREN);                                              };
    ","                              => { EmitToken(COMMA);      ExprBeg();                               };
    "`" term                         => { EmitToken(BACKTICKNL); BacktrackTerm();                         };
    "`"                              => { EmitToken(BACKTICK);                                            };
    ";"                              => { EmitToken(SEMICOLON);                                           };
    "="                              => { EmitToken(ASSIGN);     ExprBeg();                               };
    "+="                             => { EmitToken(ASSIGNADD);  ExprBeg();                               };
    "-="                             => { EmitToken(ASSIGNSUB);  ExprBeg();                               };
    "*="                             => { EmitToken(ASSIGNMUL);  ExprBeg();                               };
    "||="                            => { EmitToken(ASSIGNOR);   ExprBeg();                               };
    "+"                              => { EmitToken(PLUS);       ExprBeg();                               };
    "-"                              => { EmitToken(MINUS);      ExprBeg();                               };
    "--"                             => { EmitToken(DECR);                                                };
    "--" term                        => { EmitToken(DECRNL);     ExprBeg();BacktrackTerm();               };
    "++"                             => { EmitToken(INCR);                                                };
    "++" term                        => { EmitToken(INCRNL);     ExprBeg();BacktrackTerm();               };
    "*"                              => { EmitToken(ASTERISK);   ExprBeg();                               };
    "%"                              => { EmitToken(PERCENT);    ExprBeg();                               };
    "~"                              => { EmitToken(TILDE);      ExprBeg();                               };
    "@"                              => { EmitToken(ATMARK);                                              };
    "|"                              => { EmitToken(PIPE);       ExprBeg();                               };
    "^"                              => { EmitToken(CARET);      ExprBeg();                               };
    "?"                              => { EmitToken(TERNIF);     ExprBeg();                               };
    "!"                              => { EmitToken(TERNELSE);   ExprBeg();                               };
    "=="                             => { EmitToken(EQUAL);      ExprBeg();                               };
    "!="                             => { EmitToken(INEQUAL);    ExprBeg();                               };
    "~="                             => { EmitToken(INEQUAL);    ExprBeg();                               };
    "<"                              => { EmitToken(LESSER);     ExprBeg();                               };
    ">"                              => { EmitToken(GREATER);    ExprBeg();                               };
    "<="                             => { EmitToken(LEQUAL);     ExprBeg();                               };
    ">="                             => { EmitToken(GEQUAL);     ExprBeg();                               };
    "=>"                             => { EmitToken(DICTSEP);    ExprBeg();                               };

# Message selectors
    selector                         => { EmitStringToken(SELPART, 0, 1); ExprBeg();                      };

# Keywords
    "if"      term?                  => { EmitToken(IF);        ExprBeg(); BacktrackTerm();               };
    "unless"  term?                  => { EmitToken(UNLESS);    ExprBeg(); BacktrackTerm();               };
    "then"    term?                  => { EmitToken(THEN);      ExprBeg(); BacktrackTerm();               };
    "do"    term?                    => { EmitToken(DO);        ExprBeg(); BacktrackTerm();               };
    "else"    term?                  => { EmitToken(ELSE);      ExprBeg(); BacktrackTerm();               };
    "and"|"&&"term?                  => { EmitToken(AND);       ExprBeg(); BacktrackTerm();               };
    "or"|"||" term?                  => { EmitToken(OR);        ExprBeg(); BacktrackTerm();               };
    "while"   term?                  => { EmitToken(WHILE);     ExprBeg(); BacktrackTerm();               };
    "until"   term?                  => { EmitToken(UNTIL);     ExprBeg(); BacktrackTerm();               };
    "import"  term?                  => { EmitToken(IMPORT);    ExprBeg(); BacktrackTerm();               };
    "async"   term?                  => { EmitToken(ASYNC);     ExprBeg(); BacktrackTerm();               };
    "wait"    term                   => { EmitToken(WAITNL);    ExprBeg(); BacktrackTerm();               };
    "wait"                           => { EmitToken(WAIT);                                                };
    "lock"                           => { EmitToken(LOCK);                                                };
    "whenFinished"                   => { EmitToken(WHENFINISHED);                                        };
    "collect"                        => { EmitToken(COLLECT);                                             };
    "break"   term                   => { EmitToken(BREAKNL);   ExprBeg(); BacktrackTerm();               };
    "break"                          => { EmitToken(BREAK);                                               };
    "skip"    term                   => { EmitToken(SKIPNL);    ExprBeg(); BacktrackTerm();               };
    "skip"                           => { EmitToken(SKIP);                                                };
    "self"    term                   => { EmitToken(SELFNL);    ExprBeg(); BacktrackTerm();               };
    "super"   term                   => { EmitToken(SUPERNL);   ExprBeg(); BacktrackTerm();               };
    "..."     term                   => { EmitToken(VAARGNL);   ExprBeg(); BacktrackTerm();               };
    "nil"     term                   => { EmitToken(NILNL);     ExprBeg(); BacktrackTerm();               };
    "valid"   term                   => { EmitToken(VALIDNL);   ExprBeg(); BacktrackTerm();               };
    "yes"     term                   => { EmitToken(YESTOKNL);  ExprBeg(); BacktrackTerm();               };
    "no"      term                   => { EmitToken(NOTOKNL);   ExprBeg(); BacktrackTerm();               };
    "nothing" term                   => { EmitToken(NOTHINGNL); ExprBeg(); BacktrackTerm();               };
    "self"                           => { EmitToken(SELF);                                                };
    "super"                          => { EmitToken(SUPER);                                               };
    "..."                            => { EmitToken(VAARG);                                               };
    "nil"                            => { EmitToken(NIL);                                                 };
    "valid"                          => { EmitToken(VALID);                                               };
    "yes"                            => { EmitToken(YESTOK);                                              };
    "no"                             => { EmitToken(NOTOK);                                               };
    "nothing"                        => { EmitToken(NOTHING);                                             };

# Identifiers
    identifier %{temp1 = p;} term    => { EmitStringToken(IDENTNL, 0, te-temp1); ExprBeg(); BacktrackTerm(); };
    identifier                       => { EmitStringToken(IDENT,   0, 0);                                 };

    constant   %{temp1 = p;} term    => { EmitStringToken(CONSTNL, 0, te-temp1); ExprBeg(); BacktrackTerm(); };
    constant                         => { EmitStringToken(CONST,   0, 0);                                 };

# Literals
    int   term                       => { EmitIntToken(NUMBERNL, 10, 0); ExprBeg(); BacktrackTerm();      };
    float term                       => { EmitFloatToken(NUMBERNL);      ExprBeg(); BacktrackTerm();      };
    bin   term                       => { EmitIntToken(NUMBERNL, 2,  2); ExprBeg(); BacktrackTerm();      };
    oct   term                       => { EmitIntToken(NUMBERNL, 8,  2); ExprBeg(); BacktrackTerm();      };
    hex   term                       => { EmitIntToken(NUMBERNL, 16, 2); ExprBeg(); BacktrackTerm();      };
    anybase term                     => {
        // Find the base
        NSString *str = NSStr(0,0);
        NSString *baseStr = [str substringToIndex:[str rangeOfString:@"r"].location];
        EmitIntToken(NUMBERNL, atoi([baseStr UTF8String]), [baseStr length]+1); ExprBeg(); BacktrackTerm();
    };

    int                              => { EmitIntToken(NUMBER, 10, 0);                                    };
    float                            => { EmitFloatToken(NUMBER);                                         };
    bin                              => { EmitIntToken(NUMBER, 2,  2);                                    };
    oct                              => { EmitIntToken(NUMBER, 8,  2);                                    };
    hex                              => { EmitIntToken(NUMBER, 16, 2);                                    };

    constStr %{temp1 = p;} term      => { EmitConstStringToken(CONSTSTRNL, 0, te-temp1); ExprBeg(); BacktrackTerm(); };
    constStr                         => { EmitConstStringToken(CONSTSTR, 0, 0);                           };

    '"'                              => { PushStr(YES, NO); fcall string;                                 };
    "'"                              => { PushStr(NO, NO);  fcall string;                                 };
    rGuillemet                       => {
        BOOL dbl = IsDblQuot();
        PopStr();
        PushStr(dbl, YES);
        fcall string;
    };

    "."                              => { EmitToken(PERIOD);                                              };
    nl                               => { IncrementLine();                                                };
    space;
*|;
}%%

%% write data nofinal;

#include "parse.mm"

extern "C" TQNode *TQParseString(NSString *str, NSError **aoErr)
{
    if(![str hasSuffix:@"\n"])
        str = [str stringByAppendingString:@"\n"];

    int cs, act;
    unsigned char *ts, *te = 0;

    // Lexer setup
    unsigned char *p = (unsigned char *)[str UTF8String];
    unsigned char *pe = p + strlen((char*)p);
    unsigned char *eof = pe;
    int top;
    int stack[1024];
    NSMutableArray *strings = [NSMutableArray array];

    unsigned char *temp1, *temp2;

    // Parser setup
    void *parser = ParseAlloc(malloc);
    TQParserState parserState = {
        1, YES, [TQNodeRootBlock new], nil
    };

    @try {
        %% write init;
        %% write exec;

        if(p != pe)
            fprintf(stderr, "invalid character '%c'\n", p[0]);

        _EmitToken(0, @"EOF");
        return [parserState.root autorelease];
    } @catch(NSException *e) {
        if(aoErr)
            *aoErr = parserState.syntaxError;
        return nil;
    } @finally {
        ParseFree(parser, free);
    }
    return nil;
}