Header file for alloca on Windows.

Header file for alloca on Windows.

/* This file was generated by Ragel. Your edits will be lost.
 *
 * This is a state machine implementation of Array#pack.
 * See http://github.com/brixen/rapa.
 *
 * vim: filetype=cpp
 */

#include <stdint.h>
#include <sstream>

#include "vm/config.h"

#include "vm.hpp"
#include "object_utils.hpp"
#include "on_stack.hpp"
#include "objectmemory.hpp"

#include "builtin/array.hpp"
#include "builtin/exception.hpp"
#include "builtin/float.hpp"
#include "builtin/module.hpp"
#include "builtin/object.hpp"
#include "builtin/string.hpp"

#ifdef RBX_WINDOWS
#include <malloc.h>
#endif

namespace rubinius {
  namespace pack {
    inline Object* to_int(STATE, CallFrame* call_frame, Object* obj) {
      Array* args = Array::create(state, 1);
      args->set(state, 0, obj);

      return G(rubinius)->send(state, call_frame, state->symbol("pack_to_int"), args);
    }

#define BITS_LONG   (RBX_SIZEOF_LONG * 8)

    inline long check_long(STATE, Integer* obj) {
      if((obj)->fixnum_p()) {
        return force_as<Fixnum>(obj)->to_long();
      } else {
        Bignum* big = as<Bignum>(obj);
        big->verify_size(state, BITS_LONG);
        return big->to_long();
      }
    }

#define BITS_64     (64)

    inline long long check_long_long(STATE, Integer* obj) {
      if((obj)->fixnum_p()) {
        return force_as<Fixnum>(obj)->to_long_long();
      } else {
        Bignum* big = as<Bignum>(obj);
        big->verify_size(state, BITS_64);
        return big->to_long_long();
      }
    }

    inline Object* to_f(STATE, CallFrame* call_frame, Object* obj) {
      Array* args = Array::create(state, 1);
      args->set(state, 0, obj);

      return G(rubinius)->send(state, call_frame, state->symbol("pack_to_float"), args);
    }

    inline String* encoding_string(STATE, CallFrame* call_frame, Object* obj,
                                          const char* coerce_name)
    {
      String* s = try_as<String>(obj);
      if(s) return s;

      Array* args = Array::create(state, 1);
      args->set(state, 0, obj);

      std::string coerce_method("pack_");
      coerce_method += coerce_name;
      Object* result = G(rubinius)->send(state, call_frame,
            state->symbol(coerce_method.c_str()), args);

      if(!result) return 0;
      return as<String>(result);
    }

    inline uint16_t swap_2bytes(uint16_t x) {
      return (((x & 0x00ff)<<8) | ((x & 0xff00)>>8));
    }

    inline uint32_t swap_4bytes(uint32_t x) {
      return (((x & 0x000000ff) << 24)
             |((x & 0xff000000) >> 24)
             |((x & 0x0000ff00) << 8)
             |((x & 0x00ff0000) >> 8));
    }

    inline uint64_t swap_8bytes(uint64_t x) {
      return (((x & 0x00000000000000ffLL) << 56)
             |((x & 0xff00000000000000LL) >> 56)
             |((x & 0x000000000000ff00LL) << 40)
             |((x & 0x00ff000000000000LL) >> 40)
             |((x & 0x0000000000ff0000LL) << 24)
             |((x & 0x0000ff0000000000LL) >> 24)
             |((x & 0x00000000ff000000LL) << 8)
             |((x & 0x000000ff00000000LL) >> 8));
    }

    inline void swap_float(std::string& str, float value) {
      uint32_t x;

      memcpy(&x, &value, sizeof(float));
      x = swap_4bytes(x);

      str.append((const char*)&x, sizeof(uint32_t));
    }

    inline void swap_double(std::string& str, double value) {
      uint64_t x;

      memcpy(&x, &value, sizeof(double));
      x = swap_8bytes(x);

      str.append((const char*)&x, sizeof(uint64_t));
    }

    inline void double_element(std::string& str, double value) {
      str.append((const char*)&value, sizeof(double));
    }

    inline void float_element(std::string& str, float value) {
      str.append((const char*)&value, sizeof(float));
    }

    inline void short_element(std::string& str, int16_t value) {
      str.append((const char*)&value, sizeof(int16_t));
    }

    inline void int_element(std::string& str, int32_t value) {
      str.append((const char*)&value, sizeof(int32_t));
    }

    inline void long_element(std::string& str, int64_t value) {
      str.append((const char*)&value, sizeof(int64_t));
    }

    inline int32_t int32_element(STATE, Integer* value) {
      if(value->fixnum_p()) {
        long l = as<Fixnum>(value)->to_long();
        if(l > INT32_MAX || l < INT32_MIN) {
          Exception::range_error(state, "Fixnum value out of range of int32");
        }
        return l;
      } else {
        Bignum* big = as<Bignum>(value);
        big->verify_size(state, 32);
        return big->to_int();
      }
    }

#define QUOTABLE_PRINTABLE_BUFSIZE 1024

    void quotable_printable(String* s, std::string& str, int count) {
      static char hex_table[] = "0123456789ABCDEF";
      char buf[QUOTABLE_PRINTABLE_BUFSIZE];

      uint8_t* b = s->byte_address();
      uint8_t* e = b + s->size();
      int i = 0, n = 0, prev = -1;

      for(; b < e; b++) {
        if((*b > 126) || (*b < 32 && *b != '\n' && *b != '\t') || (*b == '=')) {
          buf[i++] = '=';
          buf[i++] = hex_table[*b >> 4];
          buf[i++] = hex_table[*b & 0x0f];
          n += 3;
          prev = -1;
        } else if(*b == '\n') {
          if(prev == ' ' || prev == '\t') {
            buf[i++] = '=';
            buf[i++] = *b;
          }
          buf[i++] = *b;
          n = 0;
          prev = *b;
        } else {
          buf[i++] = *b;
          n++;
          prev = *b;
        }

        if(n > count) {
          buf[i++] = '=';
          buf[i++] = '\n';
          n = 0;
          prev = '\n';
        }

        if(i > QUOTABLE_PRINTABLE_BUFSIZE - 5) {
          str.append(buf, i);
          i = 0;
        }
      }

      if(n > 0) {
        buf[i++] = '=';
        buf[i++] = '\n';
      }

      if(i > 0) {
        str.append(buf, i);
      }
    }

    static const char uu_table[] =
      "`!\"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_";
    static const char b64_table[] =
      "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";

#define b64_uu_byte1(t, b)      t[077 & (*b >> 2)]
#define b64_uu_byte2(t, b, c)   t[077 & (((*b << 4) & 060) | ((c >> 4) & 017))]
#define b64_uu_byte3(t, b, c)   t[077 & (((b[1] << 2) & 074) | ((c >> 6) & 03))];
#define b64_uu_byte4(t, b)      t[077 & b[2]];

    void b64_uu_encode(String* s, std::string& str, size_t count,
                              const char* table, int padding, bool encode_size)
    {
      char *buf = ALLOCA_N(char, count * 4 / 3 + 6);
      size_t i, chars, line, total = s->size();
      uint8_t* b = s->byte_address();

      for(i = 0; total > 0; i = 0, total -= line) {
        line = total > count ? count : total;

        if(encode_size) buf[i++] = line + ' ';

        for(chars = line; chars >= 3; chars -= 3, b += 3) {
          buf[i++] = b64_uu_byte1(table, b);
          buf[i++] = b64_uu_byte2(table, b, b[1]);
          buf[i++] = b64_uu_byte3(table, b, b[2]);
          buf[i++] = b64_uu_byte4(table, b);
        }

        if(chars == 2) {
          buf[i++] = b64_uu_byte1(table, b);
          buf[i++] = b64_uu_byte2(table, b, b[1]);
          buf[i++] = b64_uu_byte3(table, b, '\0');
          buf[i++] = padding;
        } else if(chars == 1) {
          buf[i++] = b64_uu_byte1(table, b);
          buf[i++] = b64_uu_byte2(table, b, '\0');
          buf[i++] = padding;
          buf[i++] = padding;
        }

        b += chars;
        buf[i++] = '\n';
        str.append(buf, i);
      }
    }

    void utf8_encode(STATE, std::string& str, Integer* value) {
      int32_t v = int32_element(state, value);

      if(!(v & ~0x7f)) {
        str.push_back(v);
      } else if(!(v & ~0x7ff)) {
        str.push_back(((v >> 6) & 0xff) | 0xc0);
        str.push_back((v & 0x3f) | 0x80);
      } else if(!(v & ~0xffff)) {
        str.push_back(((v >> 12) & 0xff) | 0xe0);
        str.push_back(((v >> 6)  & 0x3f) | 0x80);
        str.push_back((v & 0x3f) | 0x80);
      } else if(!(v & ~0x1fffff)) {
        str.push_back(((v >> 18) & 0xff) | 0xf0);
        str.push_back(((v >> 12) & 0x3f) | 0x80);
        str.push_back(((v >> 6)  & 0x3f) | 0x80);
        str.push_back((v & 0x3f) | 0x80);
      } else if(!(v & ~0x3ffffff)) {
        str.push_back(((v >> 24) & 0xff) | 0xf8);
        str.push_back(((v >> 18) & 0x3f) | 0x80);
        str.push_back(((v >> 12) & 0x3f) | 0x80);
        str.push_back(((v >> 6)  & 0x3f) | 0x80);
        str.push_back((v & 0x3f) | 0x80);
      } else if(!(v & ~0x7fffffff)) {
        str.push_back(((v >> 30) & 0xff) | 0xfc);
        str.push_back(((v >> 24) & 0x3f) | 0x80);
        str.push_back(((v >> 18) & 0x3f) | 0x80);
        str.push_back(((v >> 12) & 0x3f) | 0x80);
        str.push_back(((v >> 6)  & 0x3f) | 0x80);
        str.push_back((v & 0x3f) | 0x80);
      } else {
        Exception::range_error(state, "pack('U') value out of range");
      }
    }

    void ber_encode(STATE, std::string& str, Integer* value) {
      if(!value->positive_p()) {
        Exception::argument_error(state, "cannot BER compress a negative number");
      }

      std::string buf;

      if(try_as<Bignum>(value)) {
        static Fixnum* base = Fixnum::from(128);
        while(try_as<Bignum>(value)) {
          Array* ary;
          if(value->fixnum_p()) {
            ary = as<Fixnum>(value)->divmod(state, base);
          } else {
            ary = as<Bignum>(value)->divmod(state, base);
          }
          buf.push_back(as<Fixnum>(ary->get(state, 1))->to_native() | 0x80);
          value = as<Integer>(ary->get(state, 0));
        }
      }

      long v = value->to_long();

      while(v) {
        buf.push_back((v & 0x7f) | 0x80);
        v >>= 7;
      }

      if(buf.size() > 0) {
        char* a = const_cast<char*>(buf.c_str());
        char* b = a + buf.size() - 1;

        // clear continue bit
        *a &= 0x7f;

        // reverse string
        while(a < b) {
          int k = *a;
          *a++ = *b;
          *b-- = k;
        }

        str.append(buf.c_str(), buf.size());
      } else {
        str.push_back(0);
      }
    }

    inline size_t bit_extra(String* s, bool rest, size_t& count) {
      size_t extra = 0;

      if(rest) {
        count = s->size();
      } else {
        size_t size = s->size();
        if(count > size) {
          extra = (count - size + 1) / 2;
          count = size;
        }
      }

      return extra;
    }

    void bit_high(String* s, std::string& str, size_t count) {
      uint8_t* b = s->byte_address();
      int byte = 0;

      for(size_t i = 0; i++ < count; b++) {
        byte |= *b & 1;
        if(i & 7) {
          byte <<= 1;
        } else {
          str.push_back(byte & 0xff);
          byte = 0;
        }
      }

      if (count & 7) {
        byte <<= 7 - (count & 7);
        str.push_back(byte & 0xff);
      }
    }

    void bit_low(String* s, std::string& str, size_t count) {
      uint8_t* b = s->byte_address();
      int byte = 0;

      for(size_t i = 0; i++ < count; b++) {
        if(*b & 1)
          byte |= 128;

        if(i & 7) {
          byte >>= 1;
        } else {
          str.push_back(byte & 0xff);
          byte = 0;
        }
      }

      if(count & 7) {
        byte >>= 7 - (count & 7);
        str.push_back(byte & 0xff);
      }
    }

    inline size_t hex_extra(String* s, bool rest, size_t& count) {
      size_t extra = 0;

      if(rest) {
        count = s->size();
      } else {
        size_t size = s->size();
        if(count > size) {
          extra = (count + 1) / 2 - (size + 1) / 2;
          count = size;
        }
      }

      return extra;
    }

    void hex_high(String* s, std::string& str, size_t count) {
      uint8_t* b = s->byte_address();
      int byte = 0;

      for(size_t i = 0; i++ < count; b++) {
        if(ISALPHA(*b)) {
          byte |= ((*b & 15) + 9) & 15;
        } else {
          byte |= *b & 15;
        }

        if(i & 1) {
          byte <<= 4;
        } else {
          str.push_back(byte & 0xff);
          byte = 0;
        }
      }

      if(count & 1) {
        str.push_back(byte & 0xff);
      }
    }

    void hex_low(String* s, std::string& str, size_t count) {
      uint8_t* b = s->byte_address();
      int byte = 0;

      for(size_t i = 0; i++ < count; b++) {
        if(ISALPHA(*b)) {
          byte |= (((*b & 15) + 9) & 15) << 4;
        } else {
          byte |= (*b & 15) << 4;
        }

        if(i & 1) {
          byte >>= 4;
        } else {
          str.push_back(byte & 0xff);
          byte = 0;
        }
      }

      if(count & 1) {
        str.push_back(byte & 0xff);
      }
    }
  }

// Pack Float elements
#define pack_float_elements(format)   pack_elements(Float, pack::to_f, format)

#define pack_double_le                pack_float_elements(pack_double_element_le)
#define pack_double_be                pack_float_elements(pack_double_element_be)

#define pack_float_le                 pack_float_elements(pack_float_element_le)
#define pack_float_be                 pack_float_elements(pack_float_element_be)

// Pack Integer elements
#define pack_integer_elements(format) pack_elements(Integer, pack::to_int, format)

#define pack_byte_element(v)          str.push_back(pack::check_long(state, v))
#define pack_byte                     pack_integer_elements(pack_byte_element)

#define pack_short_le                 pack_integer_elements(pack_short_element_le)
#define pack_short_be                 pack_integer_elements(pack_short_element_be)

#define pack_int_le                   pack_integer_elements(pack_int_element_le)
#define pack_int_be                   pack_integer_elements(pack_int_element_be)

#define pack_long_le                  pack_integer_elements(pack_long_element_le)
#define pack_long_be                  pack_integer_elements(pack_long_element_be)

// Pack UTF-8 elements
#define pack_utf8_element(v)          pack::utf8_encode(state, str, v)
#define pack_utf8                     pack_elements(Integer, pack::to_int, pack_utf8_element)

// Pack BER-compressed integers
#define pack_ber_element(v)           pack::ber_encode(state, str, v)
#define pack_ber                      pack_elements(Integer, pack::to_int, pack_ber_element)

// Wraps the logic for iterating over a number of elements,
// coercing them to the correct class and formatting them
// for the output string.
#define pack_elements(T, coerce, format)        \
  for(; index < stop; index++) {                \
    Object* item = self->get(state, index);     \
    T* value = try_as<T>(item);                 \
    if(!value) {                                \
      item = coerce(state, call_frame, item);   \
      if(!item) return 0;                       \
      value = as<T>(item);                      \
    }                                           \
    format(value);                              \
  }

// Macros that depend on endianness
#ifdef RBX_LITTLE_ENDIAN

# define pack_double_element_le(v)  (pack::double_element(str, (v)->val))
# define pack_double_element_be(v)  (pack::swap_double(str, (v)->val))
# define pack_double                pack_double_le

# define pack_float_element_le(v)   (pack::float_element(str, (v)->val))
# define pack_float_element_be(v)   (pack::swap_float(str, (v)->val))
# define pack_float                 pack_float_le

# define pack_short_element_le(v)   (pack::short_element(str, pack::check_long(state, v)))
# define pack_short_element_be(v)   (pack::short_element(str, \
                                        pack::swap_2bytes(pack::check_long(state, v))))
# define pack_short                 pack_short_le

# define pack_int_element_le(v)     (pack::int_element(str, pack::check_long(state, v)))
# define pack_int_element_be(v)     (pack::int_element(str, \
                                        pack::swap_4bytes(pack::check_long(state, v))))
# define pack_int                   pack_int_le

# define pack_long_element_le(v)    (pack::long_element(str, pack::check_long_long(state, v)))
# define pack_long_element_be(v)    (pack::long_element(str, \
                                        pack::swap_8bytes(pack::check_long_long(state, v))))
# define pack_long                  pack_long_le

#else // Big endian

# define pack_double_element_le(v)  (pack::swap_double(str, (v)->val))
# define pack_double_element_be(v)  (pack::double_element(str, (v)->val))
# define pack_double                pack_double_be

# define pack_float_element_le(v)   (pack::swap_float(str, (v)->val))
# define pack_float_element_be(v)   (pack::float_element(str, (v)->val))
# define pack_float                 pack_float_be

# define pack_short_element_le(v)   (pack::short_element(str, \
                                        pack::swap_2bytes(pack::check_long(state, v))))
# define pack_short_element_be(v)   (pack::short_element(str, pack::check_long(state, v)))
# define pack_short                 pack_short_be

# define pack_int_element_le(v)     (pack::int_element(str, \
                                        pack::swap_4bytes(pack::check_long(state, v))))
# define pack_int_element_be(v)     (pack::int_element(str, pack::check_long(state, v)))
# define pack_int                   pack_int_be

# define pack_long_element_le(v)    (pack::long_element(str, \
                                        pack::swap_8bytes(pack::check_long_long(state, v))))
# define pack_long_element_be(v)    (pack::long_element(str, pack::check_long_long(state, v)))
# define pack_long                  pack_long_be

#endif

  String* Array::pack(STATE, String* directives, CallFrame* call_frame) {
    // Ragel-specific variables
    std::string d(directives->c_str(), directives->size());
    const char *p  = d.c_str();
    const char *pe = p + d.size();
    const char *eof = pe;
    int cs;

    // pack-specific variables
    Array* self = this;
    OnStack<1> sv(state, self);

    size_t array_size = self->size();
    size_t index = 0;
    size_t count = 0;
    size_t stop = 0;
    bool rest = false;
    bool platform = false;
    bool tainted = false;

    String* string_value = 0;
    std::string str("");

    // Use information we have to reduce repeated allocation.
    str.reserve(array_size * 4);

    if(RTEST(directives->tainted_p(state))) tainted = true;
%%{

  machine pack;

  include "pack.rl";

}%%

    if(en_main) {
      // do nothing
    }

    return force_as<String>(Primitives::failure());
  }
}

/* This file was generated by Ragel. Your edits will be lost.
 *
 * This is a state machine implementation of Array#pack.
 * See http://github.com/brixen/rapa.
 *
 * vim: filetype=cpp
 */

#include <stdint.h>
#include <sstream>

#include "vm/config.h"

#include "vm.hpp"
#include "object_utils.hpp"
#include "on_stack.hpp"
#include "objectmemory.hpp"

#include "builtin/array.hpp"
#include "builtin/exception.hpp"
#include "builtin/float.hpp"
#include "builtin/module.hpp"
#include "builtin/object.hpp"
#include "builtin/string.hpp"

namespace rubinius {
  namespace pack {
    inline Object* to_int(STATE, CallFrame* call_frame, Object* obj) {
      Array* args = Array::create(state, 1);
      args->set(state, 0, obj);

      return G(rubinius)->send(state, call_frame, state->symbol("pack_to_int"), args);
    }

#define BITS_LONG   (RBX_SIZEOF_LONG * 8)

    inline long check_long(STATE, Integer* obj) {
      if((obj)->fixnum_p()) {
        return force_as<Fixnum>(obj)->to_long();
      } else {
        Bignum* big = as<Bignum>(obj);
        big->verify_size(state, BITS_LONG);
        return big->to_long();
      }
    }

#define BITS_64     (64)

    inline long long check_long_long(STATE, Integer* obj) {
      if((obj)->fixnum_p()) {
        return force_as<Fixnum>(obj)->to_long_long();
      } else {
        Bignum* big = as<Bignum>(obj);
        big->verify_size(state, BITS_64);
        return big->to_long_long();
      }
    }

    inline Object* to_f(STATE, CallFrame* call_frame, Object* obj) {
      Array* args = Array::create(state, 1);
      args->set(state, 0, obj);

      return G(rubinius)->send(state, call_frame, state->symbol("pack_to_float"), args);
    }

    inline String* encoding_string(STATE, CallFrame* call_frame, Object* obj,
                                          const char* coerce_name)
    {
      String* s = try_as<String>(obj);
      if(s) return s;

      Array* args = Array::create(state, 1);
      args->set(state, 0, obj);

      std::string coerce_method("pack_");
      coerce_method += coerce_name;
      Object* result = G(rubinius)->send(state, call_frame,
            state->symbol(coerce_method.c_str()), args);

      if(!result) return 0;
      return as<String>(result);
    }

    inline uint16_t swap_2bytes(uint16_t x) {
      return (((x & 0x00ff)<<8) | ((x & 0xff00)>>8));
    }

    inline uint32_t swap_4bytes(uint32_t x) {
      return (((x & 0x000000ff) << 24)
             |((x & 0xff000000) >> 24)
             |((x & 0x0000ff00) << 8)
             |((x & 0x00ff0000) >> 8));
    }

    inline uint64_t swap_8bytes(uint64_t x) {
      return (((x & 0x00000000000000ffLL) << 56)
             |((x & 0xff00000000000000LL) >> 56)
             |((x & 0x000000000000ff00LL) << 40)
             |((x & 0x00ff000000000000LL) >> 40)
             |((x & 0x0000000000ff0000LL) << 24)
             |((x & 0x0000ff0000000000LL) >> 24)
             |((x & 0x00000000ff000000LL) << 8)
             |((x & 0x000000ff00000000LL) >> 8));
    }

    inline void swap_float(std::string& str, float value) {
      uint32_t x;

      memcpy(&x, &value, sizeof(float));
      x = swap_4bytes(x);

      str.append((const char*)&x, sizeof(uint32_t));
    }

    inline void swap_double(std::string& str, double value) {
      uint64_t x;

      memcpy(&x, &value, sizeof(double));
      x = swap_8bytes(x);

      str.append((const char*)&x, sizeof(uint64_t));
    }

    inline void double_element(std::string& str, double value) {
      str.append((const char*)&value, sizeof(double));
    }

    inline void float_element(std::string& str, float value) {
      str.append((const char*)&value, sizeof(float));
    }

    inline void short_element(std::string& str, int16_t value) {
      str.append((const char*)&value, sizeof(int16_t));
    }

    inline void int_element(std::string& str, int32_t value) {
      str.append((const char*)&value, sizeof(int32_t));
    }

    inline void long_element(std::string& str, int64_t value) {
      str.append((const char*)&value, sizeof(int64_t));
    }

    inline int32_t int32_element(STATE, Integer* value) {
      if(value->fixnum_p()) {
        long l = as<Fixnum>(value)->to_long();
        if(l > INT32_MAX || l < INT32_MIN) {
          Exception::range_error(state, "Fixnum value out of range of int32");
        }
        return l;
      } else {
        Bignum* big = as<Bignum>(value);
        big->verify_size(state, 32);
        return big->to_int();
      }
    }

#define QUOTABLE_PRINTABLE_BUFSIZE 1024

    void quotable_printable(String* s, std::string& str, int count) {
      static char hex_table[] = "0123456789ABCDEF";
      char buf[QUOTABLE_PRINTABLE_BUFSIZE];

      uint8_t* b = s->byte_address();
      uint8_t* e = b + s->size();
      int i = 0, n = 0, prev = -1;

      for(; b < e; b++) {
        if((*b > 126) || (*b < 32 && *b != '\n' && *b != '\t') || (*b == '=')) {
          buf[i++] = '=';
          buf[i++] = hex_table[*b >> 4];
          buf[i++] = hex_table[*b & 0x0f];
          n += 3;
          prev = -1;
        } else if(*b == '\n') {
          if(prev == ' ' || prev == '\t') {
            buf[i++] = '=';
            buf[i++] = *b;
          }
          buf[i++] = *b;
          n = 0;
          prev = *b;
        } else {
          buf[i++] = *b;
          n++;
          prev = *b;
        }

        if(n > count) {
          buf[i++] = '=';
          buf[i++] = '\n';
          n = 0;
          prev = '\n';
        }

        if(i > QUOTABLE_PRINTABLE_BUFSIZE - 5) {
          str.append(buf, i);
          i = 0;
        }
      }

      if(n > 0) {
        buf[i++] = '=';
        buf[i++] = '\n';
      }

      if(i > 0) {
        str.append(buf, i);
      }
    }

    static const char uu_table[] =
      "`!\"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_";
    static const char b64_table[] =
      "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";

#define b64_uu_byte1(t, b)      t[077 & (*b >> 2)]
#define b64_uu_byte2(t, b, c)   t[077 & (((*b << 4) & 060) | ((c >> 4) & 017))]
#define b64_uu_byte3(t, b, c)   t[077 & (((b[1] << 2) & 074) | ((c >> 6) & 03))];
#define b64_uu_byte4(t, b)      t[077 & b[2]];

    void b64_uu_encode(String* s, std::string& str, size_t count,
                              const char* table, int padding, bool encode_size)
    {
      char *buf = ALLOCA_N(char, count * 4 / 3 + 6);
      size_t i, chars, line, total = s->size();
      uint8_t* b = s->byte_address();

      for(i = 0; total > 0; i = 0, total -= line) {
        line = total > count ? count : total;

        if(encode_size) buf[i++] = line + ' ';

        for(chars = line; chars >= 3; chars -= 3, b += 3) {
          buf[i++] = b64_uu_byte1(table, b);
          buf[i++] = b64_uu_byte2(table, b, b[1]);
          buf[i++] = b64_uu_byte3(table, b, b[2]);
          buf[i++] = b64_uu_byte4(table, b);
        }

        if(chars == 2) {
          buf[i++] = b64_uu_byte1(table, b);
          buf[i++] = b64_uu_byte2(table, b, b[1]);
          buf[i++] = b64_uu_byte3(table, b, '\0');
          buf[i++] = padding;
        } else if(chars == 1) {
          buf[i++] = b64_uu_byte1(table, b);
          buf[i++] = b64_uu_byte2(table, b, '\0');
          buf[i++] = padding;
          buf[i++] = padding;
        }

        b += chars;
        buf[i++] = '\n';
        str.append(buf, i);
      }
    }

    void utf8_encode(STATE, std::string& str, Integer* value) {
      int32_t v = int32_element(state, value);

      if(!(v & ~0x7f)) {
        str.push_back(v);
      } else if(!(v & ~0x7ff)) {
        str.push_back(((v >> 6) & 0xff) | 0xc0);
        str.push_back((v & 0x3f) | 0x80);
      } else if(!(v & ~0xffff)) {
        str.push_back(((v >> 12) & 0xff) | 0xe0);
        str.push_back(((v >> 6)  & 0x3f) | 0x80);
        str.push_back((v & 0x3f) | 0x80);
      } else if(!(v & ~0x1fffff)) {
        str.push_back(((v >> 18) & 0xff) | 0xf0);
        str.push_back(((v >> 12) & 0x3f) | 0x80);
        str.push_back(((v >> 6)  & 0x3f) | 0x80);
        str.push_back((v & 0x3f) | 0x80);
      } else if(!(v & ~0x3ffffff)) {
        str.push_back(((v >> 24) & 0xff) | 0xf8);
        str.push_back(((v >> 18) & 0x3f) | 0x80);
        str.push_back(((v >> 12) & 0x3f) | 0x80);
        str.push_back(((v >> 6)  & 0x3f) | 0x80);
        str.push_back((v & 0x3f) | 0x80);
      } else if(!(v & ~0x7fffffff)) {
        str.push_back(((v >> 30) & 0xff) | 0xfc);
        str.push_back(((v >> 24) & 0x3f) | 0x80);
        str.push_back(((v >> 18) & 0x3f) | 0x80);
        str.push_back(((v >> 12) & 0x3f) | 0x80);
        str.push_back(((v >> 6)  & 0x3f) | 0x80);
        str.push_back((v & 0x3f) | 0x80);
      } else {
        Exception::range_error(state, "pack('U') value out of range");
      }
    }

    void ber_encode(STATE, std::string& str, Integer* value) {
      if(!value->positive_p()) {
        Exception::argument_error(state, "cannot BER compress a negative number");
      }

      std::string buf;

      if(try_as<Bignum>(value)) {
        static Fixnum* base = Fixnum::from(128);
        while(try_as<Bignum>(value)) {
          Array* ary;
          if(value->fixnum_p()) {
            ary = as<Fixnum>(value)->divmod(state, base);
          } else {
            ary = as<Bignum>(value)->divmod(state, base);
          }
          buf.push_back(as<Fixnum>(ary->get(state, 1))->to_native() | 0x80);
          value = as<Integer>(ary->get(state, 0));
        }
      }

      long v = value->to_long();

      while(v) {
        buf.push_back((v & 0x7f) | 0x80);
        v >>= 7;
      }

      if(buf.size() > 0) {
        char* a = const_cast<char*>(buf.c_str());
        char* b = a + buf.size() - 1;

        // clear continue bit
        *a &= 0x7f;

        // reverse string
        while(a < b) {
          int k = *a;
          *a++ = *b;
          *b-- = k;
        }

        str.append(buf.c_str(), buf.size());
      } else {
        str.push_back(0);
      }
    }

    inline size_t bit_extra(String* s, bool rest, size_t& count) {
      size_t extra = 0;

      if(rest) {
        count = s->size();
      } else {
        size_t size = s->size();
        if(count > size) {
          extra = (count - size + 1) / 2;
          count = size;
        }
      }

      return extra;
    }

    void bit_high(String* s, std::string& str, size_t count) {
      uint8_t* b = s->byte_address();
      int byte = 0;

      for(size_t i = 0; i++ < count; b++) {
        byte |= *b & 1;
        if(i & 7) {
          byte <<= 1;
        } else {
          str.push_back(byte & 0xff);
          byte = 0;
        }
      }

      if (count & 7) {
        byte <<= 7 - (count & 7);
        str.push_back(byte & 0xff);
      }
    }

    void bit_low(String* s, std::string& str, size_t count) {
      uint8_t* b = s->byte_address();
      int byte = 0;

      for(size_t i = 0; i++ < count; b++) {
        if(*b & 1)
          byte |= 128;

        if(i & 7) {
          byte >>= 1;
        } else {
          str.push_back(byte & 0xff);
          byte = 0;
        }
      }

      if(count & 7) {
        byte >>= 7 - (count & 7);
        str.push_back(byte & 0xff);
      }
    }

    inline size_t hex_extra(String* s, bool rest, size_t& count) {
      size_t extra = 0;

      if(rest) {
        count = s->size();
      } else {
        size_t size = s->size();
        if(count > size) {
          extra = (count + 1) / 2 - (size + 1) / 2;
          count = size;
        }
      }

      return extra;
    }

    void hex_high(String* s, std::string& str, size_t count) {
      uint8_t* b = s->byte_address();
      int byte = 0;

      for(size_t i = 0; i++ < count; b++) {
        if(ISALPHA(*b)) {
          byte |= ((*b & 15) + 9) & 15;
        } else {
          byte |= *b & 15;
        }

        if(i & 1) {
          byte <<= 4;
        } else {
          str.push_back(byte & 0xff);
          byte = 0;
        }
      }

      if(count & 1) {
        str.push_back(byte & 0xff);
      }
    }

    void hex_low(String* s, std::string& str, size_t count) {
      uint8_t* b = s->byte_address();
      int byte = 0;

      for(size_t i = 0; i++ < count; b++) {
        if(ISALPHA(*b)) {
          byte |= (((*b & 15) + 9) & 15) << 4;
        } else {
          byte |= (*b & 15) << 4;
        }

        if(i & 1) {
          byte >>= 4;
        } else {
          str.push_back(byte & 0xff);
          byte = 0;
        }
      }

      if(count & 1) {
        str.push_back(byte & 0xff);
      }
    }
  }

// Pack Float elements
#define pack_float_elements(format)   pack_elements(Float, pack::to_f, format)

#define pack_double_le                pack_float_elements(pack_double_element_le)
#define pack_double_be                pack_float_elements(pack_double_element_be)

#define pack_float_le                 pack_float_elements(pack_float_element_le)
#define pack_float_be                 pack_float_elements(pack_float_element_be)

// Pack Integer elements
#define pack_integer_elements(format) pack_elements(Integer, pack::to_int, format)

#define pack_byte_element(v)          str.push_back(pack::check_long(state, v))
#define pack_byte                     pack_integer_elements(pack_byte_element)

#define pack_short_le                 pack_integer_elements(pack_short_element_le)
#define pack_short_be                 pack_integer_elements(pack_short_element_be)

#define pack_int_le                   pack_integer_elements(pack_int_element_le)
#define pack_int_be                   pack_integer_elements(pack_int_element_be)

#define pack_long_le                  pack_integer_elements(pack_long_element_le)
#define pack_long_be                  pack_integer_elements(pack_long_element_be)

// Pack UTF-8 elements
#define pack_utf8_element(v)          pack::utf8_encode(state, str, v)
#define pack_utf8                     pack_elements(Integer, pack::to_int, pack_utf8_element)

// Pack BER-compressed integers
#define pack_ber_element(v)           pack::ber_encode(state, str, v)
#define pack_ber                      pack_elements(Integer, pack::to_int, pack_ber_element)

// Wraps the logic for iterating over a number of elements,
// coercing them to the correct class and formatting them
// for the output string.
#define pack_elements(T, coerce, format)        \
  for(; index < stop; index++) {                \
    Object* item = self->get(state, index);     \
    T* value = try_as<T>(item);                 \
    if(!value) {                                \
      item = coerce(state, call_frame, item);   \
      if(!item) return 0;                       \
      value = as<T>(item);                      \
    }                                           \
    format(value);                              \
  }

// Macros that depend on endianness
#ifdef RBX_LITTLE_ENDIAN

# define pack_double_element_le(v)  (pack::double_element(str, (v)->val))
# define pack_double_element_be(v)  (pack::swap_double(str, (v)->val))
# define pack_double                pack_double_le

# define pack_float_element_le(v)   (pack::float_element(str, (v)->val))
# define pack_float_element_be(v)   (pack::swap_float(str, (v)->val))
# define pack_float                 pack_float_le

# define pack_short_element_le(v)   (pack::short_element(str, pack::check_long(state, v)))
# define pack_short_element_be(v)   (pack::short_element(str, \
                                        pack::swap_2bytes(pack::check_long(state, v))))
# define pack_short                 pack_short_le

# define pack_int_element_le(v)     (pack::int_element(str, pack::check_long(state, v)))
# define pack_int_element_be(v)     (pack::int_element(str, \
                                        pack::swap_4bytes(pack::check_long(state, v))))
# define pack_int                   pack_int_le

# define pack_long_element_le(v)    (pack::long_element(str, pack::check_long_long(state, v)))
# define pack_long_element_be(v)    (pack::long_element(str, \
                                        pack::swap_8bytes(pack::check_long_long(state, v))))
# define pack_long                  pack_long_le

#else // Big endian

# define pack_double_element_le(v)  (pack::swap_double(str, (v)->val))
# define pack_double_element_be(v)  (pack::double_element(str, (v)->val))
# define pack_double                pack_double_be

# define pack_float_element_le(v)   (pack::swap_float(str, (v)->val))
# define pack_float_element_be(v)   (pack::float_element(str, (v)->val))
# define pack_float                 pack_float_be

# define pack_short_element_le(v)   (pack::short_element(str, \
                                        pack::swap_2bytes(pack::check_long(state, v))))
# define pack_short_element_be(v)   (pack::short_element(str, pack::check_long(state, v)))
# define pack_short                 pack_short_be

# define pack_int_element_le(v)     (pack::int_element(str, \
                                        pack::swap_4bytes(pack::check_long(state, v))))
# define pack_int_element_be(v)     (pack::int_element(str, pack::check_long(state, v)))
# define pack_int                   pack_int_be

# define pack_long_element_le(v)    (pack::long_element(str, \
                                        pack::swap_8bytes(pack::check_long_long(state, v))))
# define pack_long_element_be(v)    (pack::long_element(str, pack::check_long_long(state, v)))
# define pack_long                  pack_long_be

#endif

  String* Array::pack(STATE, String* directives, CallFrame* call_frame) {
    // Ragel-specific variables
    std::string d(directives->c_str(), directives->size());
    const char *p  = d.c_str();
    const char *pe = p + d.size();
    const char *eof = pe;
    int cs;

    // pack-specific variables
    Array* self = this;
    OnStack<1> sv(state, self);

    size_t array_size = self->size();
    size_t index = 0;
    size_t count = 0;
    size_t stop = 0;
    bool rest = false;
    bool platform = false;
    bool tainted = false;

    String* string_value = 0;
    std::string str("");

    // Use information we have to reduce repeated allocation.
    str.reserve(array_size * 4);

    if(RTEST(directives->tainted_p(state))) tainted = true;
%%{

  machine pack;

  include "pack.rl";

}%%

    if(en_main) {
      // do nothing
    }

    return force_as<String>(Primitives::failure());
  }
}


Remove unnecessary regular expression machines from Ragel machine

I just learned that the "i" (case-insensitive) modifier can be applied
equally to standard concatenation literals as it can to regular
expressions. Simplify the grammar (fewer escape sequences) and make life
easier on the syntax highlighter by replacing the regular expression
machines with concatentation literals. The generated code is identical
before and after.

Signed-off-by: Wincent Colaiuta <c76ac2e9cd86441713c7dfae92c451f3e6d17fdc@wincent.com>

// Copyright 2008 Wincent Colaiuta
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program.  If not, see <http://www.gnu.org/licenses/>.

//----------------------------------------------------------------------//
// NOTE: wikitext_ragel.c is generated from wikitext_ragel.rl, so       //
//       if you make changes to the former they will be overwritten.    //
//       You should perform all your edits in wikitext_ragel.rl.        //
//----------------------------------------------------------------------//

#include "wikitext_ragel.h"
#include "wikitext.h"
#include <stdio.h>

#define EMIT(t)     do { out->type = t; out->stop = p + 1; out->column_stop += (out->stop - out->start); } while (0)
#define MARK()      do { mark = p; } while (0)
#define REWIND()    do { p = mark; } while (0)
#define AT_END()    (p + 1 == pe)
#define DISTANCE()  (p + 1 - ts)
#define NEXT_CHAR() (*(p + 1))

%%{
    machine wikitext;

    action mark
    {
        MARK();
    }

    action non_printable_ascii
    {
        out->code_point = *p & 0x7f;
    }

    action two_byte_utf8_sequence
    {
        out->code_point = ((uint32_t)(*(p - 1)) & 0x1f) << 6 |
            (*p & 0x3f);
    }

    action three_byte_utf8_sequence
    {
        out->code_point = ((uint32_t)(*(p - 2)) & 0x0f) << 12 |
            ((uint32_t)(*(p - 1)) & 0x3f) << 6 |
            (*p & 0x3f);
    }

    action four_byte_utf8_sequence
    {
        out->code_point = ((uint32_t)(*(p - 3)) & 0x07) << 18 |
            ((uint32_t)(*(p - 2)) & 0x3f) << 12 |
            ((uint32_t)(*(p - 1)) & 0x3f) << 6 |
            (*p & 0x3f);
    }

    # simple approximation for matching email addresses; not quite RFC 2822!
    user                = (alnum | [_.] | '-')+ ;
    tld                 = alpha{2,5} ;
    domain              = (alnum+ '.')+ tld ;
    mail                = user '@' domain ;

    uri_chars           = (alnum | [@$&'(\*\+=%_~/#] | '-')+ ;
    special_uri_chars   = ([:!),;\.\?])+ ;
    uri                 = ('mailto:'i mail) |
                          (('http'i [sS]? '://' | 'ftp://'i | 'svn://'i) uri_chars (special_uri_chars uri_chars)*) ;

    main := |*

        '<nowiki>'i
        {
            EMIT(NO_WIKI_START);
            fbreak;
        };

        '</nowiki>'i
        {
            EMIT(NO_WIKI_END);
            fbreak;
        };

        "'"{1,5}
        {
            if (DISTANCE() == 5)
                EMIT(STRONG_EM);
            else if (DISTANCE() == 4)
            {
                p--;
                EMIT(STRONG_EM);
            }
            else if (DISTANCE() == 3)
                EMIT(STRONG);
            else if (DISTANCE() == 2)
                EMIT(EM);
            else
                EMIT(PRINTABLE);
            fbreak;
        };

        ''i
        {
            EMIT(STRONG_START);
            fbreak;
        };

        ''i
        {
            EMIT(STRONG_END);
            fbreak;
        };

        ''i
        {
            EMIT(EM_START);
            fbreak;
        };

        ''i
        {
            EMIT(EM_END);
            fbreak;
        };

        '`'
        {
            EMIT(TT);
            fbreak;
        };

        '<tt>'i
        {
            EMIT(TT_START);
            fbreak;
        };

        '</tt>'i
        {
            EMIT(TT_END);
            fbreak;
        };

        # consider adding

 HTML tags as well later on
        '>' @mark ' '?
        {
            if (out->column_start == 1 || last_token_type == BLOCKQUOTE)
                EMIT(BLOCKQUOTE);
            else
            {
                REWIND();
                EMIT(GREATER);
            }
            fbreak;
        };

        # shorthand for
 and

        # consider adding real
 and
 HTML tags later on
        ' ' @mark ' '*
        {
            if (out->column_start == 1 || last_token_type == BLOCKQUOTE)
            {
                REWIND();
                EMIT(PRE);
            }
            else
                EMIT(SPACE);
            fbreak;
        };

        '#'
        {
            if (out->column_start == 1 || last_token_type == OL || last_token_type == UL || last_token_type == BLOCKQUOTE)
                EMIT(OL);
            else
                EMIT(PRINTABLE);
            fbreak;
        };

        '*'
        {
            if (out->column_start == 1 || last_token_type == OL || last_token_type == UL || last_token_type == BLOCKQUOTE)
                EMIT(UL);
            else
                EMIT(PRINTABLE);
            fbreak;
        };

        '='+ @mark ' '*
        {
            if (out->column_start == 1 || last_token_type == BLOCKQUOTE)
            {
                REWIND();
                if (DISTANCE() == 1)
                    EMIT(H1_START);
                else if (DISTANCE() == 2)
                    EMIT(H2_START);
                else if (DISTANCE() == 3)
                    EMIT(H3_START);
                else if (DISTANCE() == 4)
                    EMIT(H4_START);
                else if (DISTANCE() == 5)
                    EMIT(H5_START);
                else if (DISTANCE() == 6)
                    EMIT(H6_START);
                else if (DISTANCE() > 6)
                {
                    p = ts + 6;
                    EMIT(H6_START);
                }
            }
            else if (AT_END() || NEXT_CHAR() == '\n' || NEXT_CHAR() == '\r')
            {
                REWIND();
                if (DISTANCE() == 1)
                    EMIT(H1_END);
                else if (DISTANCE() == 2)
                    EMIT(H2_END);
                else if (DISTANCE() == 3)
                    EMIT(H3_END);
                else if (DISTANCE() == 4)
                    EMIT(H4_END);
                else if (DISTANCE() == 5)
                    EMIT(H5_END);
                else if (DISTANCE() == 6)
                    EMIT(H6_END);
                else if (DISTANCE() > 6)
                {
                    p -= 6; // will scan the H6 on the next scan
                    EMIT(PRINTABLE);
                }
            }
            else
            {
                REWIND();
                EMIT(PRINTABLE);
            }
            fbreak;
        };

        uri
        {
            EMIT(URI);
            fbreak;
        };

        mail
        {
            EMIT(MAIL);
            fbreak;
        };

        '[['
        {
            EMIT(LINK_START);
            fbreak;
        };

        ']]'
        {
            EMIT(LINK_END);
            fbreak;
        };

        '|'
        {
            EMIT(SEPARATOR);
            fbreak;
        };

        '['
        {
            EMIT(EXT_LINK_START);
            fbreak;
        };

        ']'
        {
            EMIT(EXT_LINK_END);
            fbreak;
        };

        '"'
        {
            EMIT(QUOT_ENTITY);
            fbreak;
        };

        '&'
        {
            EMIT(AMP_ENTITY);
            fbreak;
        };

        '&' alpha+ digit* ';'
        {
            EMIT(NAMED_ENTITY);
            fbreak;
        };

        '&#' [xX] xdigit+ ';'
        {
            EMIT(HEX_ENTITY);
            fbreak;
        };

        '&#' digit+ ';'
        {
            EMIT(DECIMAL_ENTITY);
            fbreak;
        };

        '"'
        {
            EMIT(QUOT);
            fbreak;
        };

        '&'
        {
            EMIT(AMP);
            fbreak;
        };

        '<'
        {
            EMIT(LESS);
            fbreak;
        };

        '>'
        {
            EMIT(GREATER);
            fbreak;
        };

        ("\r"? "\n") | "\r"
        {
            EMIT(CRLF);
            out->column_stop = 1;
            out->line_stop++;
            fbreak;
        };

        # all the printable ASCII characters (0x20 to 0x7e) excluding those explicitly covered elsewhere:
        # skip space (0x20), quote (0x22), hash (0x23), ampersand (0x26), apostrophe (0x27),
        # asterisk (0x2a), less than (0x3c), equals (0x3d), greater than (0x3e), left bracket 0x5b,
        # right bracket 0x5d, backtick (0x60), and vertical bar (0x7c)
        (0x21 | 0x24..0x25 | 0x28..0x29 | 0x2b..0x3b | 0x3f..0x5a | 0x5c | 0x5e..0x5f | 0x61..0x7b | 0x7d..0x7e)+
        {
            EMIT(PRINTABLE);
            fbreak;
        };

        # here is where we handle the UTF-8 and everything else
        #
        #     one_byte_sequence   = byte begins with zero;
        #     two_byte_sequence   = first byte begins with 110 (0xc0..0xdf), next with 10 (0x80..9xbf);
        #     three_byte_sequence = first byte begins with 1110 (0xe0..0xef), next two with 10 (0x80..9xbf);
        #     four_byte_sequence  = first byte begins with 11110 (0xf0..0xf7), next three with 10 (0x80..9xbf);
        #
        #     within the ranges specified, we also exclude these illegal sequences:
        #       1100000x (c0 c1)    overlong encoding, lead byte of 2 byte seq but code point <= 127
        #       11110101 (f5)       restricted by RFC 3629 lead byte of 4-byte sequence for codepoint above 10ffff
        #       1111011x (f6, f7)   restricted by RFC 3629 lead byte of 4-byte sequence for codepoint above 10ffff
        (0x01..0x1f | 0x7f)                             @non_printable_ascii        |
        (0xc2..0xdf 0x80..0xbf)                         @two_byte_utf8_sequence     |
        (0xe0..0xef 0x80..0xbf 0x80..0xbf)              @three_byte_utf8_sequence   |
        (0xf0..0xf4 0x80..0xbf 0x80..0xbf 0x80..0xbf)   @four_byte_utf8_sequence
        {
            EMIT(DEFAULT);
            out->column_stop = out->column_start + 1;
            fbreak;
        };

    *|;

    write data;
}%%

// for now we use the scanner as a tokenizer that returns one token at a time, just like ANTLR
// ultimately we could look at embedding all of the transformation inside the scanner itself (combined scanner/parser)
// pass in the last token because that's useful for the scanner to know
// p data pointer (required by Ragel machine); overriden with contents of last_token if supplied
// pe data end pointer (required by Ragel machine)
void next_token(token_t *out, token_t *last_token, char *p, char *pe)
{
    int last_token_type = NO_TOKEN;
    if (last_token)
    {
        last_token_type     = last_token->type;
        p = last_token->stop;
        out->line_start     = out->line_stop    = last_token->line_stop;
        out->column_start   = out->column_stop  = last_token->column_stop;
    }
    else
    {
        out->line_start     = 1;
        out->column_start   = 1;
        out->line_stop      = 1;
        out->column_stop    = 1;
    }
    out->type       = NO_TOKEN;
    out->code_point = 0;
    out->start      = p;
    if (p == pe)
    {
        // all done, have reached end of input
        out->stop  = p;
        out->type  = END_OF_FILE;
        return;
    }

    char    *mark;      // for manual backtracking
    char    *eof = pe;  // required for backtracking (longest match determination)
    int     cs;         // current state (standard Ragel)
    char    *ts;        // token start (scanner)
    char    *te;        // token end (scanner)
    int     act;        // identity of last patterned matched (scanner)
    %% write init;
    %% write exec;
    if (cs == wikitext_error)
        rb_raise(eWikitextParserError, "failed before finding a token");
    else if (out->type == NO_TOKEN)
        rb_raise(eWikitextParserError, "failed to produce a token");
}

// Copyright 2008 Wincent Colaiuta
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program.  If not, see <http://www.gnu.org/licenses/>.

//----------------------------------------------------------------------//
// NOTE: wikitext_ragel.c is generated from wikitext_ragel.rl, so       //
//       if you make changes to the former they will be overwritten.    //
//       You should perform all your edits in wikitext_ragel.rl.        //
//----------------------------------------------------------------------//

#include "wikitext_ragel.h"
#include "wikitext.h"
#include <stdio.h>

#define EMIT(t)     do { out->type = t; out->stop = p + 1; out->column_stop += (out->stop - out->start); } while (0)
#define MARK()      do { mark = p; } while (0)
#define REWIND()    do { p = mark; } while (0)
#define AT_END()    (p + 1 == pe)
#define DISTANCE()  (p + 1 - ts)
#define NEXT_CHAR() (*(p + 1))

%%{
    machine wikitext;

    action mark
    {
        MARK();
    }

    action non_printable_ascii
    {
        out->code_point = *p & 0x7f;
    }

    action two_byte_utf8_sequence
    {
        out->code_point = ((uint32_t)(*(p - 1)) & 0x1f) << 6 |
            (*p & 0x3f);
    }

    action three_byte_utf8_sequence
    {
        out->code_point = ((uint32_t)(*(p - 2)) & 0x0f) << 12 |
            ((uint32_t)(*(p - 1)) & 0x3f) << 6 |
            (*p & 0x3f);
    }

    action four_byte_utf8_sequence
    {
        out->code_point = ((uint32_t)(*(p - 3)) & 0x07) << 18 |
            ((uint32_t)(*(p - 2)) & 0x3f) << 12 |
            ((uint32_t)(*(p - 1)) & 0x3f) << 6 |
            (*p & 0x3f);
    }

    # simple approximation for matching email addresses; not quite RFC 2822!
    user                = (alnum | [_.] | '-')+ ;
    tld                 = alpha{2,5} ;
    domain              = (alnum+ '.')+ tld ;
    mail                = user '@' domain ;

    uri_chars           = (alnum | [@$&'(\*\+=%_~/#] | '-')+ ;
    special_uri_chars   = ([:!),;\.\?])+ ;
    uri                 = ('mailto:'i mail) |
                          (('http'i [sS]? '://' | 'ftp://'i | 'svn://'i) uri_chars (special_uri_chars uri_chars)*) ;

    main := |*

        /<nowiki>/i
        {
            EMIT(NO_WIKI_START);
            fbreak;
        };

        /<\/nowiki>/i   # this comment because the Ragel TextMate bundle miscolorizes without it /
        {
            EMIT(NO_WIKI_END);
            fbreak;
        };

        "'"{1,5}
        {
            if (DISTANCE() == 5)
                EMIT(STRONG_EM);
            else if (DISTANCE() == 4)
            {
                p--;
                EMIT(STRONG_EM);
            }
            else if (DISTANCE() == 3)
                EMIT(STRONG);
            else if (DISTANCE() == 2)
                EMIT(EM);
            else
                EMIT(PRINTABLE);
            fbreak;
        };

        //i
        {
            EMIT(STRONG_START);
            fbreak;
        };

        /<\/strong>/i   # this comment because the Ragel TextMate bundle miscolorizes without it /
        {
            EMIT(STRONG_END);
            fbreak;
        };

        //i
        {
            EMIT(EM_START);
            fbreak;
        };

        /<\/em>/i   # this comment because the Ragel TextMate bundle miscolorizes without it /
        {
            EMIT(EM_END);
            fbreak;
        };

        '`'
        {
            EMIT(TT);
            fbreak;
        };

        /<tt>/i
        {
            EMIT(TT_START);
            fbreak;
        };

        /<\/tt>/i   # this comment because the Ragel TextMate bundle miscolorizes without it /
        {
            EMIT(TT_END);
            fbreak;
        };

        # consider adding

 HTML tags as well later on
        '>' @mark ' '?
        {
            if (out->column_start == 1 || last_token_type == BLOCKQUOTE)
                EMIT(BLOCKQUOTE);
            else
            {
                REWIND();
                EMIT(GREATER);
            }
            fbreak;
        };

        # shorthand for
 and

        # consider adding real
 and
 HTML tags later on
        ' ' @mark ' '*
        {
            if (out->column_start == 1 || last_token_type == BLOCKQUOTE)
            {
                REWIND();
                EMIT(PRE);
            }
            else
                EMIT(SPACE);
            fbreak;
        };

        '#'
        {
            if (out->column_start == 1 || last_token_type == OL || last_token_type == UL || last_token_type == BLOCKQUOTE)
                EMIT(OL);
            else
                EMIT(PRINTABLE);
            fbreak;
        };

        '*'
        {
            if (out->column_start == 1 || last_token_type == OL || last_token_type == UL || last_token_type == BLOCKQUOTE)
                EMIT(UL);
            else
                EMIT(PRINTABLE);
            fbreak;
        };

        '='+ @mark ' '*
        {
            if (out->column_start == 1 || last_token_type == BLOCKQUOTE)
            {
                REWIND();
                if (DISTANCE() == 1)
                    EMIT(H1_START);
                else if (DISTANCE() == 2)
                    EMIT(H2_START);
                else if (DISTANCE() == 3)
                    EMIT(H3_START);
                else if (DISTANCE() == 4)
                    EMIT(H4_START);
                else if (DISTANCE() == 5)
                    EMIT(H5_START);
                else if (DISTANCE() == 6)
                    EMIT(H6_START);
                else if (DISTANCE() > 6)
                {
                    p = ts + 6;
                    EMIT(H6_START);
                }
            }
            else if (AT_END() || NEXT_CHAR() == '\n' || NEXT_CHAR() == '\r')
            {
                REWIND();
                if (DISTANCE() == 1)
                    EMIT(H1_END);
                else if (DISTANCE() == 2)
                    EMIT(H2_END);
                else if (DISTANCE() == 3)
                    EMIT(H3_END);
                else if (DISTANCE() == 4)
                    EMIT(H4_END);
                else if (DISTANCE() == 5)
                    EMIT(H5_END);
                else if (DISTANCE() == 6)
                    EMIT(H6_END);
                else if (DISTANCE() > 6)
                {
                    p -= 6; // will scan the H6 on the next scan
                    EMIT(PRINTABLE);
                }
            }
            else
            {
                REWIND();
                EMIT(PRINTABLE);
            }
            fbreak;
        };

        uri
        {
            EMIT(URI);
            fbreak;
        };

        mail
        {
            EMIT(MAIL);
            fbreak;
        };

        '[['
        {
            EMIT(LINK_START);
            fbreak;
        };

        ']]'
        {
            EMIT(LINK_END);
            fbreak;
        };

        '|'
        {
            EMIT(SEPARATOR);
            fbreak;
        };

        '['
        {
            EMIT(EXT_LINK_START);
            fbreak;
        };

        ']'
        {
            EMIT(EXT_LINK_END);
            fbreak;
        };

        '"'
        {
            EMIT(QUOT_ENTITY);
            fbreak;
        };

        '&'
        {
            EMIT(AMP_ENTITY);
            fbreak;
        };

        '&' alpha+ digit* ';'
        {
            EMIT(NAMED_ENTITY);
            fbreak;
        };

        '&#' [xX] xdigit+ ';'
        {
            EMIT(HEX_ENTITY);
            fbreak;
        };

        '&#' digit+ ';'
        {
            EMIT(DECIMAL_ENTITY);
            fbreak;
        };

        '"'
        {
            EMIT(QUOT);
            fbreak;
        };

        '&'
        {
            EMIT(AMP);
            fbreak;
        };

        '<'
        {
            EMIT(LESS);
            fbreak;
        };

        '>'
        {
            EMIT(GREATER);
            fbreak;
        };

        ("\r"? "\n") | "\r"
        {
            EMIT(CRLF);
            out->column_stop = 1;
            out->line_stop++;
            fbreak;
        };

        # all the printable ASCII characters (0x20 to 0x7e) excluding those explicitly covered elsewhere:
        # skip space (0x20), quote (0x22), hash (0x23), ampersand (0x26), apostrophe (0x27),
        # asterisk (0x2a), less than (0x3c), equals (0x3d), greater than (0x3e), left bracket 0x5b,
        # right bracket 0x5d, backtick (0x60), and vertical bar (0x7c)
        (0x21 | 0x24..0x25 | 0x28..0x29 | 0x2b..0x3b | 0x3f..0x5a | 0x5c | 0x5e..0x5f | 0x61..0x7b | 0x7d..0x7e)+
        {
            EMIT(PRINTABLE);
            fbreak;
        };

        # here is where we handle the UTF-8 and everything else
        #
        #     one_byte_sequence   = byte begins with zero;
        #     two_byte_sequence   = first byte begins with 110 (0xc0..0xdf), next with 10 (0x80..9xbf);
        #     three_byte_sequence = first byte begins with 1110 (0xe0..0xef), next two with 10 (0x80..9xbf);
        #     four_byte_sequence  = first byte begins with 11110 (0xf0..0xf7), next three with 10 (0x80..9xbf);
        #
        #     within the ranges specified, we also exclude these illegal sequences:
        #       1100000x (c0 c1)    overlong encoding, lead byte of 2 byte seq but code point <= 127
        #       11110101 (f5)       restricted by RFC 3629 lead byte of 4-byte sequence for codepoint above 10ffff
        #       1111011x (f6, f7)   restricted by RFC 3629 lead byte of 4-byte sequence for codepoint above 10ffff
        (0x01..0x1f | 0x7f)                             @non_printable_ascii        |
        (0xc2..0xdf 0x80..0xbf)                         @two_byte_utf8_sequence     |
        (0xe0..0xef 0x80..0xbf 0x80..0xbf)              @three_byte_utf8_sequence   |
        (0xf0..0xf4 0x80..0xbf 0x80..0xbf 0x80..0xbf)   @four_byte_utf8_sequence
        {
            EMIT(DEFAULT);
            out->column_stop = out->column_start + 1;
            fbreak;
        };

    *|;

    write data;
}%%

// for now we use the scanner as a tokenizer that returns one token at a time, just like ANTLR
// ultimately we could look at embedding all of the transformation inside the scanner itself (combined scanner/parser)
// pass in the last token because that's useful for the scanner to know
// p data pointer (required by Ragel machine); overriden with contents of last_token if supplied
// pe data end pointer (required by Ragel machine)
void next_token(token_t *out, token_t *last_token, char *p, char *pe)
{
    int last_token_type = NO_TOKEN;
    if (last_token)
    {
        last_token_type     = last_token->type;
        p = last_token->stop;
        out->line_start     = out->line_stop    = last_token->line_stop;
        out->column_start   = out->column_stop  = last_token->column_stop;
    }
    else
    {
        out->line_start     = 1;
        out->column_start   = 1;
        out->line_stop      = 1;
        out->column_stop    = 1;
    }
    out->type       = NO_TOKEN;
    out->code_point = 0;
    out->start      = p;
    if (p == pe)
    {
        // all done, have reached end of input
        out->stop  = p;
        out->type  = END_OF_FILE;
        return;
    }

    char    *mark;      // for manual backtracking
    char    *eof = pe;  // required for backtracking (longest match determination)
    int     cs;         // current state (standard Ragel)
    char    *ts;        // token start (scanner)
    char    *te;        // token end (scanner)
    int     act;        // identity of last patterned matched (scanner)
    %% write init;
    %% write exec;
    if (cs == wikitext_error)
        rb_raise(eWikitextParserError, "failed before finding a token");
    else if (out->type == NO_TOKEN)
        rb_raise(eWikitextParserError, "failed to produce a token");
}


Save in the hash table also the headers that are recognised.

This allow simpler access to them when copying them for responding.

/* -*- c -*- */

%% machine rtsp_request;

static int ragel_parse_request(RTSP_Request *req, char *msg) {
    int cs;
    char *p = msg, *pe = p + strlen(msg), *eof, *s;

    /* Variables used for adding headers to the hash table */
    char *hdr, *hdr_val;
    size_t hdr_size, hdr_val_size;

    /* We want to express clearly which versions we support, so that we
     * can return right away if an unsupported one is found.
     */

    %%{
        SP = ' ';
        CRLF = "\r\n";

        action set_s {
            s = p;
        }

        action end_method {
            req->method = g_strndup(s, p-s);
        }

        Supported_Method =
            "DESCRIBE" @ { req->method_id = RTSP_ID_DESCRIBE; } |
            "OPTIONS" @ { req->method_id = RTSP_ID_OPTIONS; } |
            "PAUSE" @ { req->method_id = RTSP_ID_PAUSE; } |
            "PLAY" @ { req->method_id = RTSP_ID_PLAY; } |
            "SETUP" @ { req->method_id = RTSP_ID_SETUP; } |
            "SET_PARAMETER" @ { req->method_id = RTSP_ID_SET_PARAMETER; } |
            "TEARDOWN" @ { req->method_id = RTSP_ID_TEARDOWN; };

        action method_not_implemented {
            return RTSP_NotImplemented;
        }

        Other_Method = (alpha+ - Supported_Method) %method_not_implemented;
        Method = (Supported_Method | Other_Method )
            > set_s % end_method;

        action version_not_supported {
            return RTSP_VersionNotSupported;
        }

        Supported_RTSP_Version = "RTSP/1.0";
        RTSP_Version = (alpha . '/' . [0-9] '.' [0-9]) -
            Supported_RTSP_Version % version_not_supported;

        action end_object {
            req->object = g_strndup(s, p-s);
        }

        Request_Line = (Supported_Method | Method) . SP
            (print*) > set_s % end_object . SP
            (Supported_RTSP_Version | RTSP_Version) . CRLF;

        action hdr_start {
            hdr = p;
        }

        action hdr_end {
            hdr_size = p-hdr;
        }

        action hdr_val_start {
            hdr_val = p;
        }

        action hdr_val_end {
            hdr_val_size = p-hdr_val;
        }

        Header = (alpha|'-')+ > hdr_start % hdr_end
            . ':' . SP . print+ > hdr_val_start % hdr_val_end . CRLF;

        action cseq_header {
            {
                /* Discard two bytes for \r\n */
                char *tmp = g_strndup(s, p-s-2);
                req->cseq = strtol(s, NULL, 0);
                g_free(tmp);
            }
        }

        Cseq_Header = "CSeq: " . (digit+ >set_s) . CRLF %cseq_header;

        action session_header {
            {
                /* Discard two bytes for \r\n */
                char *tmp = g_strndup(s, p-s-2);
                req->session_id = g_ascii_strtoull(s, NULL, 0);
                g_free(tmp);
            }
        }

        Session_Header = "Session: " . (digit+ >set_s) . CRLF %session_header;

        Headers = Cseq_Header | Session_Header | Header;

        action save_header {
            {
                g_hash_table_insert(req->headers,
                                    g_strndup(hdr, hdr_size),
                                    g_strndup(hdr_val, hdr_val_size));
            }
        }

        main := Request_Line . ( Headers @ save_header )+;

        write data;
        write init;
        write exec;
    }%%

    return RTSP_Ok;
}

/* -*- c -*- */

%% machine rtsp_request;

static int ragel_parse_request(RTSP_Request *req, char *msg) {
    int cs;
    char *p = msg, *pe = p + strlen(msg), *eof, *s;

    /* Variables used for adding headers to the hash table */
    char *hdr, *hdr_val;
    size_t hdr_size, hdr_val_size;

    /* We want to express clearly which versions we support, so that we
     * can return right away if an unsupported one is found.
     */

    %%{
        SP = ' ';
        CRLF = "\r\n";

        action set_s {
            s = p;
        }

        action end_method {
            req->method = g_strndup(s, p-s);
        }

        Supported_Method =
            "DESCRIBE" @ { req->method_id = RTSP_ID_DESCRIBE; } |
            "OPTIONS" @ { req->method_id = RTSP_ID_OPTIONS; } |
            "PAUSE" @ { req->method_id = RTSP_ID_PAUSE; } |
            "PLAY" @ { req->method_id = RTSP_ID_PLAY; } |
            "SETUP" @ { req->method_id = RTSP_ID_SETUP; } |
            "SET_PARAMETER" @ { req->method_id = RTSP_ID_SET_PARAMETER; } |
            "TEARDOWN" @ { req->method_id = RTSP_ID_TEARDOWN; };

        action method_not_implemented {
            return RTSP_NotImplemented;
        }

        Other_Method = (alpha+ - Supported_Method) %method_not_implemented;
        Method = (Supported_Method | Other_Method )
            > set_s % end_method;

        action version_not_supported {
            return RTSP_VersionNotSupported;
        }

        Supported_RTSP_Version = "RTSP/1.0";
        RTSP_Version = (alpha . '/' . [0-9] '.' [0-9]) -
            Supported_RTSP_Version % version_not_supported;

        action end_object {
            req->object = g_strndup(s, p-s);
        }

        Request_Line = (Supported_Method | Method) . SP
            (print*) > set_s % end_object . SP
            (Supported_RTSP_Version | RTSP_Version) . CRLF;

        action hdr_start {
            hdr = p;
        }

        action hdr_end {
            hdr_size = p-hdr;
        }

        action hdr_val_start {
            hdr_val = p;
        }

        action hdr_val_end {
            hdr_val_size = p-hdr_val;
        }

        Header = (alpha|'-')+ > hdr_start % hdr_end
            . ':' . SP . print+ > hdr_val_start % hdr_val_end . CRLF;

        action cseq_header {
            {
                /* Discard two bytes for \r\n */
                char *tmp = g_strndup(s, p-s-2);
                req->cseq = strtol(s, NULL, 0);
                g_free(tmp);
            }
        }

        Cseq_Header = "CSeq: " . (digit+ >set_s) . CRLF %cseq_header;

        action session_header {
            {
                /* Discard two bytes for \r\n */
                char *tmp = g_strndup(s, p-s-2);
                req->session_id = g_ascii_strtoull(s, NULL, 0);
                g_free(tmp);
            }
        }

        Session_Header = "Session: " . (digit+ >set_s) . CRLF %session_header;

        Known_Headers = Cseq_Header | Session_Header;
        Other_Headers = Header - Known_Headers;

        action save_header {
            {
                g_hash_table_insert(req->headers,
                                    g_strndup(hdr, hdr_size),
                                    g_strndup(hdr_val, hdr_val_size));
            }
        }

        main := Request_Line . ( Known_Headers | Other_Headers @ save_header )+;

        write data;
        write init;
        write exec;
    }%%

    return RTSP_Ok;
}