Move callback return value checking into macro

/* Copyright (c) 2008, 2009 Ryan Dahl (ry@tinyclouds.org)
 * Based on Zed Shaw's Mongrel, copyright (c) Zed A. Shaw
 *
 * All rights reserved.
 *
 * Permission is hereby granted, free of charge, to any person obtaining
 * a copy of this software and associated documentation files (the
 * "Software"), to deal in the Software without restriction, including
 * without limitation the rights to use, copy, modify, merge, publish,
 * distribute, sublicense, and/or sell copies of the Software, and to
 * permit persons to whom the Software is furnished to do so, subject to
 * the following conditions:
 *
 * The above copyright notice and this permission notice shall be
 * included in all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
 * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
 * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
 * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 */
#include "http_parser.h"
#include <limits.h>
#include <assert.h>

/* parser->flags */
#define EATING      0x01
#define ERROR       0x02
#define CHUNKED     0x04
#define EAT_FOREVER 0x10

static int unhex[] = {-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1
                     ,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1
                     ,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1
                     , 0, 1, 2, 3, 4, 5, 6, 7, 8, 9,-1,-1,-1,-1,-1,-1
                     ,-1,10,11,12,13,14,15,-1,-1,-1,-1,-1,-1,-1,-1,-1
                     ,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1
                     ,-1,10,11,12,13,14,15,-1,-1,-1,-1,-1,-1,-1,-1,-1
                     ,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1
                     };

#undef MIN
#define MIN(a,b) ((a) < (b) ? (a) : (b))

#undef NULL
#define NULL ((void*)(0))

#define MAX_FIELD_SIZE (80*1024)

#define REMAINING (unsigned long)(pe - p)
#define CALLBACK(FOR)                                                \
do {                                                                 \
  if (parser->FOR##_mark) {                                          \
    parser->FOR##_size += p - parser->FOR##_mark;                    \
    if (parser->FOR##_size > MAX_FIELD_SIZE) {                       \
      parser->flags |= ERROR;                                        \
      return 0;                                                      \
    }                                                                \
    if (parser->on_##FOR) {                                          \
      callback_return_value = parser->on_##FOR(parser,               \
        parser->FOR##_mark,                                          \
        p - parser->FOR##_mark);                                     \
    }                                                                \
    if (callback_return_value != 0) {                                \
      parser->flags |= ERROR;                                        \
      return 0;                                                      \
    }                                                                \
  }                                                                  \
} while(0)

#define RESET_PARSER(parser)                                         \
  parser->chunk_size = 0;                                            \
  parser->flags = 0;                                                 \
  parser->header_field_mark = NULL;                                  \
  parser->header_value_mark = NULL;                                  \
  parser->query_string_mark = NULL;                                  \
  parser->path_mark = NULL;                                          \
  parser->uri_mark = NULL;                                           \
  parser->fragment_mark = NULL;                                      \
  parser->status_code = 0;                                           \
  parser->method = 0;                                                \
  parser->version = HTTP_VERSION_OTHER;                              \
  parser->keep_alive = -1;                                           \
  parser->content_length = -1;                                       \
  parser->body_read = 0

#define END_REQUEST                                                  \
do {                                                                 \
    if (parser->on_message_complete) {                               \
      callback_return_value =                                        \
        parser->on_message_complete(parser);                         \
    }                                                                \
    RESET_PARSER(parser);                                            \
} while (0)

#define SKIP_BODY(nskip)                                             \
do {                                                                 \
  tmp = (nskip);                                                     \
  if (parser->on_body && tmp > 0) {                                  \
    callback_return_value = parser->on_body(parser, p, tmp);         \
  }                                                                  \
  if (callback_return_value == 0) {                                  \
    p += tmp;                                                        \
    parser->body_read += tmp;                                        \
    parser->chunk_size -= tmp;                                       \
    if (0 == parser->chunk_size) {                                   \
      parser->flags &= ~EATING;                                      \
      if (!(parser->flags & CHUNKED)) {                              \
        END_REQUEST;                                                 \
      }                                                              \
    } else {                                                         \
      parser->flags |= EATING;                                       \
    }                                                                \
  }                                                                  \
} while (0)

%%{
  machine http_parser;

  action mark_header_field {
    parser->header_field_mark = p;
    parser->header_field_size = 0;
  }

  action mark_header_value {
    parser->header_value_mark = p;
    parser->header_value_size = 0;
  }

  action mark_fragment {
    parser->fragment_mark = p;
    parser->fragment_size = 0;
  }

  action mark_query_string {
    parser->query_string_mark = p;
    parser->query_string_size = 0;
  }

  action mark_request_path {
    parser->path_mark = p;
    parser->path_size = 0;
  }

  action mark_request_uri {
    parser->uri_mark = p;
    parser->uri_size = 0;
  }

  action header_field {
    CALLBACK(header_field);
    parser->header_field_mark = NULL;
    parser->header_field_size = 0;
  }

  action header_value {
    CALLBACK(header_value);
    parser->header_value_mark = NULL;
    parser->header_value_size = 0;
  }

  action request_uri {
    CALLBACK(uri);
    parser->uri_mark = NULL;
    parser->uri_size = 0;
  }

  action fragment {
    CALLBACK(fragment);
    parser->fragment_mark = NULL;
    parser->fragment_size = 0;
  }

  action query_string {
    CALLBACK(query_string);
    parser->query_string_mark = NULL;
    parser->query_string_size = 0;
  }

  action request_path {
    CALLBACK(path);
    parser->path_mark = NULL;
    parser->path_size = 0;
  }

  action headers_complete {
    if(parser->on_headers_complete) {
      callback_return_value = parser->on_headers_complete(parser);
      if (callback_return_value != 0) {
        parser->flags |= ERROR;
        return 0;
      }
    }
  }

  action begin_message {
    if(parser->on_message_begin) {
      callback_return_value = parser->on_message_begin(parser);
      if (callback_return_value != 0) {
        parser->flags |= ERROR;
        return 0;
      }
    }
  }

  action content_length {
    if (parser->content_length == -1) parser->content_length = 0;
    if (parser->content_length > INT_MAX) {
      parser->flags |= ERROR;
      return 0;
    }
    parser->content_length *= 10;
    parser->content_length += *p - '0';
  }

  action status_code {
    parser->status_code *= 10;
    parser->status_code += *p - '0';
  }

  action use_chunked_encoding { parser->flags |= CHUNKED;  }

  action set_keep_alive { parser->keep_alive = 1; }
  action set_not_keep_alive { parser->keep_alive = 0; }

  action version_11 { parser->version = HTTP_VERSION_11; }
  action version_10 { parser->version = HTTP_VERSION_10; }
  action version_09 { parser->version = HTTP_VERSION_09; }

  action add_to_chunk_size {
    parser->chunk_size *= 16;
    parser->chunk_size += unhex[(int)*p];
  }

  action skip_chunk_data {
    SKIP_BODY(MIN(parser->chunk_size, REMAINING));
    if (callback_return_value != 0) {
      parser->flags |= ERROR;
      return 0;
    }

    fhold;
    if (parser->chunk_size > REMAINING) {
      fbreak;
    } else {
      fgoto chunk_end;
    }
  }

  action end_chunked_body {
    END_REQUEST;
    if (parser->type == HTTP_REQUEST) {
      fnext Requests;
    } else {
      fnext Responses;
    }
  }

  action body_logic {
    if (parser->flags & CHUNKED) {
      fnext ChunkedBody;
    } else {
      /* this is pretty stupid. i'd prefer to combine this with
       * skip_chunk_data */
      if (parser->content_length < 0) {
        /* If we didn't get a content length; if not keep-alive
         * just read body until EOF */
        if (!http_parser_should_keep_alive(parser)) {
          parser->flags |= EAT_FOREVER;
          parser->chunk_size = REMAINING;
        } else {
          /* Otherwise, if keep-alive, then assume the message
           * has no body. */
          parser->chunk_size = parser->content_length = 0;
        }
      } else {
        parser->chunk_size = parser->content_length;
      }
      p += 1;

      SKIP_BODY(MIN(REMAINING, parser->chunk_size));

      if (callback_return_value != 0) {
        parser->flags |= ERROR;
        return 0;
      }

      fhold;
      if(parser->chunk_size > REMAINING) {
        fbreak;
      }
    }
  }

  CRLF = "\r\n";

# character types
  CTL = (cntrl | 127);
  safe = ("$" | "-" | "_" | ".");
  extra = ("!" | "*" | "'" | "(" | ")" | ",");
  reserved = (";" | "/" | "?" | ":" | "@" | "&" | "=" | "+");
  unsafe = (CTL | " " | "\"" | "#" | "%" | "<" | ">");
  national = any -- (alpha | digit | reserved | extra | safe | unsafe);
  unreserved = (alpha | digit | safe | extra | national);
  escape = ("%" xdigit xdigit);
  uchar = (unreserved | escape);
  pchar = (uchar | ":" | "@" | "&" | "=" | "+");
  tspecials = ("(" | ")" | "<" | ">" | "@" | "," | ";" | ":" | "\\" | "\""
              | "/" | "[" | "]" | "?" | "=" | "{" | "}" | " " | "\t");

# elements
  token = (ascii -- (CTL | tspecials));
  quote = "\"";
#  qdtext = token -- "\"";
#  quoted_pair = "\" ascii;
#  quoted_string = "\"" (qdtext | quoted_pair )* "\"";

#  headers

  Method = ( "COPY"      %{ parser->method = HTTP_COPY;      }
           | "DELETE"    %{ parser->method = HTTP_DELETE;    }
           | "GET"       %{ parser->method = HTTP_GET;       }
           | "HEAD"      %{ parser->method = HTTP_HEAD;      }
           | "LOCK"      %{ parser->method = HTTP_LOCK;      }
           | "MKCOL"     %{ parser->method = HTTP_MKCOL;     }
           | "MOVE"      %{ parser->method = HTTP_MOVE;      }
           | "OPTIONS"   %{ parser->method = HTTP_OPTIONS;   }
           | "POST"      %{ parser->method = HTTP_POST;      }
           | "PROPFIND"  %{ parser->method = HTTP_PROPFIND;  }
           | "PROPPATCH" %{ parser->method = HTTP_PROPPATCH; }
           | "PUT"       %{ parser->method = HTTP_PUT;       }
           | "TRACE"     %{ parser->method = HTTP_TRACE;     }
           | "UNLOCK"    %{ parser->method = HTTP_UNLOCK;    }
           ); # Not allowing extension methods

  HTTP_Version = "HTTP/" ( "1.1" %version_11
                         | "1.0" %version_10
                         | "0.9" %version_09
                         | (digit "." digit)
                         );

  scheme = ( alpha | digit | "+" | "-" | "." )* ;
  absolute_uri = (scheme ":" (uchar | reserved )*);
  path = ( pchar+ ( "/" pchar* )* ) ;
  query = ( uchar | reserved )* >mark_query_string %query_string ;
  param = ( pchar | "/" )* ;
  params = ( param ( ";" param )* ) ;
  rel_path = ( path? (";" params)? ) ;
  absolute_path = ( "/"+ rel_path ) >mark_request_path %request_path ("?" query)?;
  Request_URI = ( "*" | absolute_uri | absolute_path ) >mark_request_uri %request_uri;
  Fragment = ( uchar | reserved )* >mark_fragment %fragment;

  field_name = ( token -- ":" )+;
  Field_Name = field_name >mark_header_field %header_field;

  field_value = ((any - " ") any*)?;
  Field_Value = field_value >mark_header_value %header_value;

  hsep = ":" " "*;
  header = (field_name hsep field_value) :> CRLF;
  Header = ( ("Content-Length"i hsep digit+ $content_length)
           | ("Connection"i hsep
               ( "Keep-Alive"i %set_keep_alive
               | "close"i %set_not_keep_alive
               )
             )
           | ("Transfer-Encoding" hsep "chunked"i %use_chunked_encoding)
           | (Field_Name hsep Field_Value)
           ) :> CRLF;

  Headers = (Header)* :> CRLF @headers_complete;

  Request_Line = ( Method " " Request_URI ("#" Fragment)? " " HTTP_Version CRLF ) ;

  StatusCode = (digit digit digit) $status_code;
  ReasonPhrase =  ascii* -- ("\r" | "\n");
  StatusLine = HTTP_Version  " " StatusCode (" " ReasonPhrase)? CRLF;

# chunked message
  trailing_headers = header*;
  #chunk_ext_val   = token | quoted_string;
  chunk_ext_val = token*;
  chunk_ext_name = token*;
  chunk_extension = ( ";" " "* chunk_ext_name ("=" chunk_ext_val)? )*;
  last_chunk = "0"+ ( chunk_extension | " "+) CRLF;
  chunk_size = (xdigit* [1-9a-fA-F] xdigit* ) $add_to_chunk_size;
  chunk_end  = CRLF;
  chunk_body = any >skip_chunk_data;
  chunk_begin = chunk_size ( chunk_extension | " "+ ) CRLF;
  chunk = chunk_begin chunk_body chunk_end;
  ChunkedBody := chunk* last_chunk trailing_headers CRLF @end_chunked_body;

  Request = (Request_Line Headers) >begin_message @body_logic;
  Response = (StatusLine Headers) >begin_message @body_logic;

  Requests := Request*;
  Responses := Response*;

  main := any >{
    fhold;
    if (parser->type == HTTP_REQUEST) {
      fgoto Requests;
    } else {
      fgoto Responses;
    }
  };

}%%

%% write data;

void
http_parser_init (http_parser *parser, enum http_parser_type type)
{
  int cs = 0;
  %% write init;
  parser->cs = cs;
  parser->type = type;

  parser->on_message_begin = NULL;
  parser->on_path = NULL;
  parser->on_query_string = NULL;
  parser->on_uri = NULL;
  parser->on_fragment = NULL;
  parser->on_header_field = NULL;
  parser->on_header_value = NULL;
  parser->on_headers_complete = NULL;
  parser->on_body = NULL;
  parser->on_message_complete = NULL;

  RESET_PARSER(parser);
}

/** exec **/
size_t
http_parser_execute (http_parser *parser, const char *buffer, size_t len)
{
  size_t tmp; // REMOVE ME this is extremely hacky
  int callback_return_value = 0;
  const char *p, *pe, *eof;
  int cs = parser->cs;

  p = buffer;
  pe = buffer+len;
  eof = len ? NULL : pe;

  if (parser->flags & EAT_FOREVER) {
    if (len == 0) {
      if (parser->on_message_complete) {
        callback_return_value = parser->on_message_complete(parser);
        if (callback_return_value != 0) parser->flags |= ERROR;
      }
    } else {
      if (parser->on_body) {
        callback_return_value = parser->on_body(parser, p, len);
        if (callback_return_value != 0) parser->flags |= ERROR;
      }
    }
    return len;
  }

  if (0 < parser->chunk_size && (parser->flags & EATING)) {
    /* eat body */
    SKIP_BODY(MIN(len, parser->chunk_size));
    if (callback_return_value != 0) {
      parser->flags |= ERROR;
      return 0;
    }
  }

  if (parser->header_field_mark)   parser->header_field_mark   = buffer;
  if (parser->header_value_mark)   parser->header_value_mark   = buffer;
  if (parser->fragment_mark)       parser->fragment_mark       = buffer;
  if (parser->query_string_mark)   parser->query_string_mark   = buffer;
  if (parser->path_mark)           parser->path_mark           = buffer;
  if (parser->uri_mark)            parser->uri_mark            = buffer;

  %% write exec;

  parser->cs = cs;

  CALLBACK(header_field);
  CALLBACK(header_value);
  CALLBACK(fragment);
  CALLBACK(query_string);
  CALLBACK(path);
  CALLBACK(uri);

  assert(p <= pe && "buffer overflow after parsing execute");
  return(p - buffer);
}

int
http_parser_has_error (http_parser *parser)
{
  if (parser->flags & ERROR) return 1;
  return parser->cs == http_parser_error;
}

/* Copyright (c) 2008, 2009 Ryan Dahl (ry@tinyclouds.org)
 * Based on Zed Shaw's Mongrel, copyright (c) Zed A. Shaw
 *
 * All rights reserved.
 *
 * Permission is hereby granted, free of charge, to any person obtaining
 * a copy of this software and associated documentation files (the
 * "Software"), to deal in the Software without restriction, including
 * without limitation the rights to use, copy, modify, merge, publish,
 * distribute, sublicense, and/or sell copies of the Software, and to
 * permit persons to whom the Software is furnished to do so, subject to
 * the following conditions:
 *
 * The above copyright notice and this permission notice shall be
 * included in all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
 * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
 * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
 * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 */
#include "http_parser.h"
#include <limits.h>
#include <assert.h>

/* parser->flags */
#define EATING      0x01
#define ERROR       0x02
#define CHUNKED     0x04
#define EAT_FOREVER 0x10

static int unhex[] = {-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1
                     ,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1
                     ,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1
                     , 0, 1, 2, 3, 4, 5, 6, 7, 8, 9,-1,-1,-1,-1,-1,-1
                     ,-1,10,11,12,13,14,15,-1,-1,-1,-1,-1,-1,-1,-1,-1
                     ,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1
                     ,-1,10,11,12,13,14,15,-1,-1,-1,-1,-1,-1,-1,-1,-1
                     ,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1
                     };

#undef MIN
#define MIN(a,b) ((a) < (b) ? (a) : (b))

#undef NULL
#define NULL ((void*)(0))

#define MAX_FIELD_SIZE (80*1024)

#define REMAINING (unsigned long)(pe - p)
#define CALLBACK(FOR)                                                \
do {                                                                 \
  if (parser->FOR##_mark) {                                          \
    parser->FOR##_size += p - parser->FOR##_mark;                    \
    if (parser->FOR##_size > MAX_FIELD_SIZE) {                       \
      parser->flags |= ERROR;                                        \
      return 0;                                                      \
    }                                                                \
    if (parser->on_##FOR) {                                          \
      callback_return_value = parser->on_##FOR(parser,               \
        parser->FOR##_mark,                                          \
        p - parser->FOR##_mark);                                     \
    }                                                                \
  }                                                                  \
} while(0)

#define RESET_PARSER(parser)                                         \
  parser->chunk_size = 0;                                            \
  parser->flags = 0;                                                 \
  parser->header_field_mark = NULL;                                  \
  parser->header_value_mark = NULL;                                  \
  parser->query_string_mark = NULL;                                  \
  parser->path_mark = NULL;                                          \
  parser->uri_mark = NULL;                                           \
  parser->fragment_mark = NULL;                                      \
  parser->status_code = 0;                                           \
  parser->method = 0;                                                \
  parser->version = HTTP_VERSION_OTHER;                              \
  parser->keep_alive = -1;                                           \
  parser->content_length = -1;                                       \
  parser->body_read = 0

#define END_REQUEST                                                  \
do {                                                                 \
    if (parser->on_message_complete) {                               \
      callback_return_value =                                        \
        parser->on_message_complete(parser);                         \
    }                                                                \
    RESET_PARSER(parser);                                            \
} while (0)

#define SKIP_BODY(nskip)                                             \
do {                                                                 \
  tmp = (nskip);                                                     \
  if (parser->on_body && tmp > 0) {                                  \
    callback_return_value = parser->on_body(parser, p, tmp);         \
  }                                                                  \
  if (callback_return_value == 0) {                                  \
    p += tmp;                                                        \
    parser->body_read += tmp;                                        \
    parser->chunk_size -= tmp;                                       \
    if (0 == parser->chunk_size) {                                   \
      parser->flags &= ~EATING;                                      \
      if (!(parser->flags & CHUNKED)) {                              \
        END_REQUEST;                                                 \
      }                                                              \
    } else {                                                         \
      parser->flags |= EATING;                                       \
    }                                                                \
  }                                                                  \
} while (0)

%%{
  machine http_parser;

  action mark_header_field {
    parser->header_field_mark = p;
    parser->header_field_size = 0;
  }

  action mark_header_value {
    parser->header_value_mark = p;
    parser->header_value_size = 0;
  }

  action mark_fragment {
    parser->fragment_mark = p;
    parser->fragment_size = 0;
  }

  action mark_query_string {
    parser->query_string_mark = p;
    parser->query_string_size = 0;
  }

  action mark_request_path {
    parser->path_mark = p;
    parser->path_size = 0;
  }

  action mark_request_uri {
    parser->uri_mark = p;
    parser->uri_size = 0;
  }

  action header_field {
    CALLBACK(header_field);
    if (callback_return_value != 0) {
      parser->flags |= ERROR;
      return 0;
    }
    parser->header_field_mark = NULL;
    parser->header_field_size = 0;
  }

  action header_value {
    CALLBACK(header_value);
    if (callback_return_value != 0) {
      parser->flags |= ERROR;
      return 0;
    }
    parser->header_value_mark = NULL;
    parser->header_value_size = 0;
  }

  action request_uri {
    CALLBACK(uri);
    if (callback_return_value != 0) {
      parser->flags |= ERROR;
      return 0;
    }
    parser->uri_mark = NULL;
    parser->uri_size = 0;
  }

  action fragment {
    CALLBACK(fragment);
    if (callback_return_value != 0) {
      parser->flags |= ERROR;
      return 0;
    }
    parser->fragment_mark = NULL;
    parser->fragment_size = 0;
  }

  action query_string {
    CALLBACK(query_string);
    if (callback_return_value != 0) {
      parser->flags |= ERROR;
      return 0;
    }
    parser->query_string_mark = NULL;
    parser->query_string_size = 0;
  }

  action request_path {
    CALLBACK(path);
    if (callback_return_value != 0) {
      parser->flags |= ERROR;
      return 0;
    }
    parser->path_mark = NULL;
    parser->path_size = 0;
  }

  action headers_complete {
    if(parser->on_headers_complete) {
      callback_return_value = parser->on_headers_complete(parser);
      if (callback_return_value != 0) {
        parser->flags |= ERROR;
        return 0;
      }
    }
  }

  action begin_message {
    if(parser->on_message_begin) {
      callback_return_value = parser->on_message_begin(parser);
      if (callback_return_value != 0) {
        parser->flags |= ERROR;
        return 0;
      }
    }
  }

  action content_length {
    if (parser->content_length == -1) parser->content_length = 0;
    if (parser->content_length > INT_MAX) {
      parser->flags |= ERROR;
      return 0;
    }
    parser->content_length *= 10;
    parser->content_length += *p - '0';
  }

  action status_code {
    parser->status_code *= 10;
    parser->status_code += *p - '0';
  }

  action use_chunked_encoding { parser->flags |= CHUNKED;  }

  action set_keep_alive { parser->keep_alive = 1; }
  action set_not_keep_alive { parser->keep_alive = 0; }

  action version_11 { parser->version = HTTP_VERSION_11; }
  action version_10 { parser->version = HTTP_VERSION_10; }
  action version_09 { parser->version = HTTP_VERSION_09; }

  action add_to_chunk_size {
    parser->chunk_size *= 16;
    parser->chunk_size += unhex[(int)*p];
  }

  action skip_chunk_data {
    SKIP_BODY(MIN(parser->chunk_size, REMAINING));
    if (callback_return_value != 0) {
      parser->flags |= ERROR;
      return 0;
    }

    fhold;
    if (parser->chunk_size > REMAINING) {
      fbreak;
    } else {
      fgoto chunk_end;
    }
  }

  action end_chunked_body {
    END_REQUEST;
    if (parser->type == HTTP_REQUEST) {
      fnext Requests;
    } else {
      fnext Responses;
    }
  }

  action body_logic {
    if (parser->flags & CHUNKED) {
      fnext ChunkedBody;
    } else {
      /* this is pretty stupid. i'd prefer to combine this with
       * skip_chunk_data */
      if (parser->content_length < 0) {
        /* If we didn't get a content length; if not keep-alive
         * just read body until EOF */
        if (!http_parser_should_keep_alive(parser)) {
          parser->flags |= EAT_FOREVER;
          parser->chunk_size = REMAINING;
        } else {
          /* Otherwise, if keep-alive, then assume the message
           * has no body. */
          parser->chunk_size = parser->content_length = 0;
        }
      } else {
        parser->chunk_size = parser->content_length;
      }
      p += 1;

      SKIP_BODY(MIN(REMAINING, parser->chunk_size));

      if (callback_return_value != 0) {
        parser->flags |= ERROR;
        return 0;
      }

      fhold;
      if(parser->chunk_size > REMAINING) {
        fbreak;
      }
    }
  }

  CRLF = "\r\n";

# character types
  CTL = (cntrl | 127);
  safe = ("$" | "-" | "_" | ".");
  extra = ("!" | "*" | "'" | "(" | ")" | ",");
  reserved = (";" | "/" | "?" | ":" | "@" | "&" | "=" | "+");
  unsafe = (CTL | " " | "\"" | "#" | "%" | "<" | ">");
  national = any -- (alpha | digit | reserved | extra | safe | unsafe);
  unreserved = (alpha | digit | safe | extra | national);
  escape = ("%" xdigit xdigit);
  uchar = (unreserved | escape);
  pchar = (uchar | ":" | "@" | "&" | "=" | "+");
  tspecials = ("(" | ")" | "<" | ">" | "@" | "," | ";" | ":" | "\\" | "\""
              | "/" | "[" | "]" | "?" | "=" | "{" | "}" | " " | "\t");

# elements
  token = (ascii -- (CTL | tspecials));
  quote = "\"";
#  qdtext = token -- "\"";
#  quoted_pair = "\" ascii;
#  quoted_string = "\"" (qdtext | quoted_pair )* "\"";

#  headers

  Method = ( "COPY"      %{ parser->method = HTTP_COPY;      }
           | "DELETE"    %{ parser->method = HTTP_DELETE;    }
           | "GET"       %{ parser->method = HTTP_GET;       }
           | "HEAD"      %{ parser->method = HTTP_HEAD;      }
           | "LOCK"      %{ parser->method = HTTP_LOCK;      }
           | "MKCOL"     %{ parser->method = HTTP_MKCOL;     }
           | "MOVE"      %{ parser->method = HTTP_MOVE;      }
           | "OPTIONS"   %{ parser->method = HTTP_OPTIONS;   }
           | "POST"      %{ parser->method = HTTP_POST;      }
           | "PROPFIND"  %{ parser->method = HTTP_PROPFIND;  }
           | "PROPPATCH" %{ parser->method = HTTP_PROPPATCH; }
           | "PUT"       %{ parser->method = HTTP_PUT;       }
           | "TRACE"     %{ parser->method = HTTP_TRACE;     }
           | "UNLOCK"    %{ parser->method = HTTP_UNLOCK;    }
           ); # Not allowing extension methods

  HTTP_Version = "HTTP/" ( "1.1" %version_11
                         | "1.0" %version_10
                         | "0.9" %version_09
                         | (digit "." digit)
                         );

  scheme = ( alpha | digit | "+" | "-" | "." )* ;
  absolute_uri = (scheme ":" (uchar | reserved )*);
  path = ( pchar+ ( "/" pchar* )* ) ;
  query = ( uchar | reserved )* >mark_query_string %query_string ;
  param = ( pchar | "/" )* ;
  params = ( param ( ";" param )* ) ;
  rel_path = ( path? (";" params)? ) ;
  absolute_path = ( "/"+ rel_path ) >mark_request_path %request_path ("?" query)?;
  Request_URI = ( "*" | absolute_uri | absolute_path ) >mark_request_uri %request_uri;
  Fragment = ( uchar | reserved )* >mark_fragment %fragment;

  field_name = ( token -- ":" )+;
  Field_Name = field_name >mark_header_field %header_field;

  field_value = ((any - " ") any*)?;
  Field_Value = field_value >mark_header_value %header_value;

  hsep = ":" " "*;
  header = (field_name hsep field_value) :> CRLF;
  Header = ( ("Content-Length"i hsep digit+ $content_length)
           | ("Connection"i hsep
               ( "Keep-Alive"i %set_keep_alive
               | "close"i %set_not_keep_alive
               )
             )
           | ("Transfer-Encoding" hsep "chunked"i %use_chunked_encoding)
           | (Field_Name hsep Field_Value)
           ) :> CRLF;

  Headers = (Header)* :> CRLF @headers_complete;

  Request_Line = ( Method " " Request_URI ("#" Fragment)? " " HTTP_Version CRLF ) ;

  StatusCode = (digit digit digit) $status_code;
  ReasonPhrase =  ascii* -- ("\r" | "\n");
  StatusLine = HTTP_Version  " " StatusCode (" " ReasonPhrase)? CRLF;

# chunked message
  trailing_headers = header*;
  #chunk_ext_val   = token | quoted_string;
  chunk_ext_val = token*;
  chunk_ext_name = token*;
  chunk_extension = ( ";" " "* chunk_ext_name ("=" chunk_ext_val)? )*;
  last_chunk = "0"+ ( chunk_extension | " "+) CRLF;
  chunk_size = (xdigit* [1-9a-fA-F] xdigit* ) $add_to_chunk_size;
  chunk_end  = CRLF;
  chunk_body = any >skip_chunk_data;
  chunk_begin = chunk_size ( chunk_extension | " "+ ) CRLF;
  chunk = chunk_begin chunk_body chunk_end;
  ChunkedBody := chunk* last_chunk trailing_headers CRLF @end_chunked_body;

  Request = (Request_Line Headers) >begin_message @body_logic;
  Response = (StatusLine Headers) >begin_message @body_logic;

  Requests := Request*;
  Responses := Response*;

  main := any >{
    fhold;
    if (parser->type == HTTP_REQUEST) {
      fgoto Requests;
    } else {
      fgoto Responses;
    }
  };

}%%

%% write data;

void
http_parser_init (http_parser *parser, enum http_parser_type type)
{
  int cs = 0;
  %% write init;
  parser->cs = cs;
  parser->type = type;

  parser->on_message_begin = NULL;
  parser->on_path = NULL;
  parser->on_query_string = NULL;
  parser->on_uri = NULL;
  parser->on_fragment = NULL;
  parser->on_header_field = NULL;
  parser->on_header_value = NULL;
  parser->on_headers_complete = NULL;
  parser->on_body = NULL;
  parser->on_message_complete = NULL;

  RESET_PARSER(parser);
}

/** exec **/
size_t
http_parser_execute (http_parser *parser, const char *buffer, size_t len)
{
  size_t tmp; // REMOVE ME this is extremely hacky
  int callback_return_value = 0;
  const char *p, *pe, *eof;
  int cs = parser->cs;

  p = buffer;
  pe = buffer+len;
  eof = len ? NULL : pe;

  if (parser->flags & EAT_FOREVER) {
    if (len == 0) {
      if (parser->on_message_complete) {
        callback_return_value = parser->on_message_complete(parser);
        if (callback_return_value != 0) parser->flags |= ERROR;
      }
    } else {
      if (parser->on_body) {
        callback_return_value = parser->on_body(parser, p, len);
        if (callback_return_value != 0) parser->flags |= ERROR;
      }
    }
    return len;
  }

  if (0 < parser->chunk_size && (parser->flags & EATING)) {
    /* eat body */
    SKIP_BODY(MIN(len, parser->chunk_size));
    if (callback_return_value != 0) {
      parser->flags |= ERROR;
      return 0;
    }
  }

  if (parser->header_field_mark)   parser->header_field_mark   = buffer;
  if (parser->header_value_mark)   parser->header_value_mark   = buffer;
  if (parser->fragment_mark)       parser->fragment_mark       = buffer;
  if (parser->query_string_mark)   parser->query_string_mark   = buffer;
  if (parser->path_mark)           parser->path_mark           = buffer;
  if (parser->uri_mark)            parser->uri_mark            = buffer;

  %% write exec;

  parser->cs = cs;

  CALLBACK(header_field);
  CALLBACK(header_value);
  CALLBACK(fragment);
  CALLBACK(query_string);
  CALLBACK(path);
  CALLBACK(uri);

  assert(p <= pe && "buffer overflow after parsing execute");
  return(p - buffer);
}

int
http_parser_has_error (http_parser *parser)
{
  if (parser->flags & ERROR) return 1;
  return parser->cs == http_parser_error;
}


Apply macro substitutions.

Apply macro substitutions.

// Experimental parser for macros
//
// TODO:
//
// - Eat whitespace between opening/closing quotes and the actual query.
// - Add error handling.
// - If everything works, remove main.

#include <algorithm>
#include <fstream>
#include <iostream>
#include <iterator>
#include <map>
#include <sstream>
#include <vector>

#include <cstring>

%%{
	machine macros;
	write data;
}%%

namespace {

struct Substitution
{
	size_t begin;
	size_t n;
	std::string macro;
};

}

typedef std::map<std::string, std::string> Macros;
typedef std::vector<Substitution> Substitutions;

Macros parseMacros(char const *data)
{
	int cs = 0;
	char const *p = data;
	char const *pe = p + std::strlen(data);
	char const *eof = p + std::strlen(data);

	char const *strStart = p;

	char const *substStart = 0;

	Macros macros;

	Substitutions substitutions;

	std::string lastKey;

	std::ostringstream buf;
%%{
	action str_char {
		buf << fc;
	}

	action key {
		lastKey = buf.str();
		buf.str("");
	}

	action queryStart {
		strStart = p;
	}

	action queryEnd {
		// This action is executed when the query is closed ("""), we have to
		// three to avoid including the quotes.
		std::string query(strStart, p - 3);

		// Apply substitutions that were found in the macro.
		for (Substitutions::const_reverse_iterator iter = substitutions.rbegin();
			iter != substitutions.rend(); ++iter)
		{
			Macros::const_iterator mIter = macros.find(iter->macro);
			if (mIter == macros.end())
			{
				std::cerr << "Unknown macro: " << iter->macro << std::endl;
				continue;
			}

			query.replace(iter->begin, iter->n, mIter->second);
		}

		substitutions.clear();
		macros[lastKey] = query;
	}

	action substBegin {
		substStart = p;
	}

	action substEnd {
		std::string macro(substStart, p);

		// substStart is the position of the macro name. Decrement by one to
		// get the position of the percentage sign.
		size_t begin = substStart - strStart - 1;

		// Add two, to account for both percentage signs.
		size_t n = p - substStart + 2;

		Substitution subst = {begin, n, macro};
		substitutions.push_back(subst);
	}

	separator = "\"\"\"";
	whitespace = [\n\r\t ]+;

	key = ([A-Za-z0-9_]+) $ str_char % key;

	substitution = ('%' % substBegin) (any - '%')+ ('%' > substEnd);
	query = (substitution | (any - '%')+)* -- separator;
	queryVal = (separator % queryStart) query (separator % queryEnd);

	main := (whitespace* key whitespace* '=' whitespace* queryVal)*;

	write init;
	write exec;
}%%

	return macros;
}

int main(int argc, char const *argv[])
{
	if (argc != 2)
		return 1;

	std::ifstream in(argv[1]);
	if (!in)
		return 1;

	in >> std::noskipws;

	std::ostringstream iss;
	std::copy(std::istream_iterator<char>(in),
		std::istream_iterator<char>(),
		std::ostream_iterator<char>(iss));

	std::string data = iss.str();
	//std::string test = "\"\"\"test\"\"\"";

	//parseMacros(test.c_str());

	Macros macros = parseMacros(data.c_str());
	for (Macros::const_iterator iter = macros.begin(); iter != macros.end(); ++iter)
		std::cout << iter->first << ": " << iter->second << std::endl;
}

// Experimental parser for macros
//
// TODO:
//
// - Store macros and keys nicely in a map
// - Expand queries, marked with percent signs.
// - If everything works, remove main.

#include <algorithm>
#include <fstream>
#include <iostream>
#include <iterator>
#include <map>
#include <sstream>
#include <cstring>

%%{
	machine macros;
	write data;
}%%

typedef std::map<std::string, std::string> Macros;

Macros parseMacros(char const *data)
{
	int cs = 0;
	char const *p = data;
	char const *pe = p + std::strlen(data);
	char const *eof = p + std::strlen(data);

	char const *strStart = p;

	char const *substStart = 0;

	Macros macros;

	std::string lastKey;

	std::ostringstream buf;
%%{
	action str_char {
		buf << fc;
	}

	action key {
		lastKey = buf.str();
		buf.str("");
	}

	action queryStart {
		strStart = p;
	}

	action queryEnd {
		std::string query(strStart, p - 3);
		macros[lastKey] = query;
	}

	action substBegin {
		substStart = p;
	}

	action substEnd {
		std::string subst(substStart, p);
		std::cerr << "Found substitution: " << subst << std::endl;
	}

	separator = "\"\"\"";
	whitespace = [\n\r\t ]+;

	key = ([A-Za-z0-9_]+) $ str_char % key;

	substitution = ('%' % substBegin) (any - '%')+ ('%' > substEnd);
	query = (substitution | (any - '%')+)* -- separator;
	queryVal = (separator % queryStart) query (separator % queryEnd);

	main := (whitespace* key whitespace* '=' whitespace* queryVal)*;

	write init;
	write exec;
}%%

	return macros;
}

int main(int argc, char const *argv[])
{
	if (argc != 2)
		return 1;

	std::ifstream in(argv[1]);
	if (!in)
		return 1;

	in >> std::noskipws;

	std::ostringstream iss;
	std::copy(std::istream_iterator<char>(in),
		std::istream_iterator<char>(),
		std::ostream_iterator<char>(iss));

	std::string data = iss.str();
	//std::string test = "\"\"\"test\"\"\"";

	//parseMacros(test.c_str());

	Macros macros = parseMacros(data.c_str());
	for (Macros::const_iterator iter = macros.begin(); iter != macros.end(); ++iter)
		std::cout << iter->first << ": " << iter->second << std::endl;
}


Updated Rapa source to cbc06042.

Updated Rapa source to cbc06042.

/* This file was generated by Ragel. Your edits will be lost.
 *
 * This is a state machine implementation of Array#pack.
 * See http://github.com/brixen/rapa.
 *
 * vim: filetype=cpp
 */

#include <stdint.h>
#include <sstream>

#include "vm/config.h"

#include "vm.hpp"
#include "object_utils.hpp"
#include "on_stack.hpp"
#include "objectmemory.hpp"

#include "builtin/array.hpp"
#include "builtin/exception.hpp"
#include "builtin/float.hpp"
#include "builtin/module.hpp"
#include "builtin/object.hpp"
#include "builtin/string.hpp"

#ifdef RBX_WINDOWS
#include <malloc.h>
#endif

namespace rubinius {
  namespace pack {
    inline Object* to_int(STATE, CallFrame* call_frame, Object* obj) {
      Array* args = Array::create(state, 1);
      args->set(state, 0, obj);

      return G(rubinius)->send(state, call_frame, state->symbol("pack_to_int"), args);
    }

#define BITS_LONG   (RBX_SIZEOF_LONG * 8)

    inline long check_long(STATE, Integer* obj) {
      if((obj)->fixnum_p()) {
        return force_as<Fixnum>(obj)->to_long();
      } else {
        Bignum* big = as<Bignum>(obj);
        big->verify_size(state, BITS_LONG);
        return big->to_long();
      }
    }

#define BITS_64     (64)

    inline long long check_long_long(STATE, Integer* obj) {
      if((obj)->fixnum_p()) {
        return force_as<Fixnum>(obj)->to_long_long();
      } else {
        Bignum* big = as<Bignum>(obj);
        big->verify_size(state, BITS_64);
        return big->to_long_long();
      }
    }

    inline Object* to_f(STATE, CallFrame* call_frame, Object* obj) {
      Array* args = Array::create(state, 1);
      args->set(state, 0, obj);

      return G(rubinius)->send(state, call_frame, state->symbol("pack_to_float"), args);
    }

    inline String* encoding_string(STATE, CallFrame* call_frame, Object* obj,
                                          const char* coerce_name)
    {
      String* s = try_as<String>(obj);
      if(s) return s;

      Array* args = Array::create(state, 1);
      args->set(state, 0, obj);

      std::string coerce_method("pack_");
      coerce_method += coerce_name;
      Object* result = G(rubinius)->send(state, call_frame,
            state->symbol(coerce_method.c_str()), args);

      if(!result) return 0;
      return as<String>(result);
    }

    inline uint16_t swap_2bytes(uint16_t x) {
      return (((x & 0x00ff)<<8) | ((x & 0xff00)>>8));
    }

    inline uint32_t swap_4bytes(uint32_t x) {
      return (((x & 0x000000ff) << 24)
             |((x & 0xff000000) >> 24)
             |((x & 0x0000ff00) << 8)
             |((x & 0x00ff0000) >> 8));
    }

    inline uint64_t swap_8bytes(uint64_t x) {
      return (((x & 0x00000000000000ffLL) << 56)
             |((x & 0xff00000000000000LL) >> 56)
             |((x & 0x000000000000ff00LL) << 40)
             |((x & 0x00ff000000000000LL) >> 40)
             |((x & 0x0000000000ff0000LL) << 24)
             |((x & 0x0000ff0000000000LL) >> 24)
             |((x & 0x00000000ff000000LL) << 8)
             |((x & 0x000000ff00000000LL) >> 8));
    }

    inline void swap_float(std::string& str, float value) {
      uint32_t x;

      memcpy(&x, &value, sizeof(float));
      x = swap_4bytes(x);

      str.append((const char*)&x, sizeof(uint32_t));
    }

    inline void swap_double(std::string& str, double value) {
      uint64_t x;

      memcpy(&x, &value, sizeof(double));
      x = swap_8bytes(x);

      str.append((const char*)&x, sizeof(uint64_t));
    }

    inline void double_element(std::string& str, double value) {
      str.append((const char*)&value, sizeof(double));
    }

    inline void float_element(std::string& str, float value) {
      str.append((const char*)&value, sizeof(float));
    }

    inline void short_element(std::string& str, int16_t value) {
      str.append((const char*)&value, sizeof(int16_t));
    }

    inline void int_element(std::string& str, int32_t value) {
      str.append((const char*)&value, sizeof(int32_t));
    }

    inline void long_element(std::string& str, int64_t value) {
      str.append((const char*)&value, sizeof(int64_t));
    }

    inline int32_t int32_element(STATE, Integer* value) {
      if(value->fixnum_p()) {
        long l = as<Fixnum>(value)->to_long();
        if(l > INT32_MAX || l < INT32_MIN) {
          Exception::range_error(state, "Fixnum value out of range of int32");
        }
        return l;
      } else {
        Bignum* big = as<Bignum>(value);
        big->verify_size(state, 32);
        return big->to_int();
      }
    }

#define QUOTABLE_PRINTABLE_BUFSIZE 1024

    void quotable_printable(String* s, std::string& str, int count) {
      static char hex_table[] = "0123456789ABCDEF";
      char buf[QUOTABLE_PRINTABLE_BUFSIZE];

      uint8_t* b = s->byte_address();
      uint8_t* e = b + s->size();
      int i = 0, n = 0, prev = -1;

      for(; b < e; b++) {
        if((*b > 126) || (*b < 32 && *b != '\n' && *b != '\t') || (*b == '=')) {
          buf[i++] = '=';
          buf[i++] = hex_table[*b >> 4];
          buf[i++] = hex_table[*b & 0x0f];
          n += 3;
          prev = -1;
        } else if(*b == '\n') {
          if(prev == ' ' || prev == '\t') {
            buf[i++] = '=';
            buf[i++] = *b;
          }
          buf[i++] = *b;
          n = 0;
          prev = *b;
        } else {
          buf[i++] = *b;
          n++;
          prev = *b;
        }

        if(n > count) {
          buf[i++] = '=';
          buf[i++] = '\n';
          n = 0;
          prev = '\n';
        }

        if(i > QUOTABLE_PRINTABLE_BUFSIZE - 5) {
          str.append(buf, i);
          i = 0;
        }
      }

      if(n > 0) {
        buf[i++] = '=';
        buf[i++] = '\n';
      }

      if(i > 0) {
        str.append(buf, i);
      }
    }

    static const char uu_table[] =
      "`!\"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_";
    static const char b64_table[] =
      "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";

#define b64_uu_byte1(t, b)      t[077 & (*b >> 2)]
#define b64_uu_byte2(t, b, c)   t[077 & (((*b << 4) & 060) | ((c >> 4) & 017))]
#define b64_uu_byte3(t, b, c)   t[077 & (((b[1] << 2) & 074) | ((c >> 6) & 03))];
#define b64_uu_byte4(t, b)      t[077 & b[2]];

    void b64_uu_encode(String* s, std::string& str, size_t count,
                              const char* table, int padding, bool encode_size)
    {
      char *buf = ALLOCA_N(char, count * 4 / 3 + 6);
      size_t i, chars, line, total = s->size();
      uint8_t* b = s->byte_address();

      for(i = 0; total > 0; i = 0, total -= line) {
        line = total > count ? count : total;

        if(encode_size) buf[i++] = line + ' ';

        for(chars = line; chars >= 3; chars -= 3, b += 3) {
          buf[i++] = b64_uu_byte1(table, b);
          buf[i++] = b64_uu_byte2(table, b, b[1]);
          buf[i++] = b64_uu_byte3(table, b, b[2]);
          buf[i++] = b64_uu_byte4(table, b);
        }

        if(chars == 2) {
          buf[i++] = b64_uu_byte1(table, b);
          buf[i++] = b64_uu_byte2(table, b, b[1]);
          buf[i++] = b64_uu_byte3(table, b, '\0');
          buf[i++] = padding;
        } else if(chars == 1) {
          buf[i++] = b64_uu_byte1(table, b);
          buf[i++] = b64_uu_byte2(table, b, '\0');
          buf[i++] = padding;
          buf[i++] = padding;
        }

        b += chars;
        buf[i++] = '\n';
        str.append(buf, i);
      }
    }

    void utf8_encode(STATE, std::string& str, Integer* value) {
      int32_t v = int32_element(state, value);

      if(!(v & ~0x7f)) {
        str.push_back(v);
      } else if(!(v & ~0x7ff)) {
        str.push_back(((v >> 6) & 0xff) | 0xc0);
        str.push_back((v & 0x3f) | 0x80);
      } else if(!(v & ~0xffff)) {
        str.push_back(((v >> 12) & 0xff) | 0xe0);
        str.push_back(((v >> 6)  & 0x3f) | 0x80);
        str.push_back((v & 0x3f) | 0x80);
      } else if(!(v & ~0x1fffff)) {
        str.push_back(((v >> 18) & 0xff) | 0xf0);
        str.push_back(((v >> 12) & 0x3f) | 0x80);
        str.push_back(((v >> 6)  & 0x3f) | 0x80);
        str.push_back((v & 0x3f) | 0x80);
      } else if(!(v & ~0x3ffffff)) {
        str.push_back(((v >> 24) & 0xff) | 0xf8);
        str.push_back(((v >> 18) & 0x3f) | 0x80);
        str.push_back(((v >> 12) & 0x3f) | 0x80);
        str.push_back(((v >> 6)  & 0x3f) | 0x80);
        str.push_back((v & 0x3f) | 0x80);
      } else if(!(v & ~0x7fffffff)) {
        str.push_back(((v >> 30) & 0xff) | 0xfc);
        str.push_back(((v >> 24) & 0x3f) | 0x80);
        str.push_back(((v >> 18) & 0x3f) | 0x80);
        str.push_back(((v >> 12) & 0x3f) | 0x80);
        str.push_back(((v >> 6)  & 0x3f) | 0x80);
        str.push_back((v & 0x3f) | 0x80);
      } else {
        Exception::range_error(state, "pack('U') value out of range");
      }
    }

    void ber_encode(STATE, std::string& str, Integer* value) {
      if(!value->positive_p()) {
        Exception::argument_error(state, "cannot BER compress a negative number");
      }

      std::string buf;

      if(try_as<Bignum>(value)) {
        static Fixnum* base = Fixnum::from(128);
        while(try_as<Bignum>(value)) {
          Array* ary;
          if(value->fixnum_p()) {
            ary = as<Fixnum>(value)->divmod(state, base);
          } else {
            ary = as<Bignum>(value)->divmod(state, base);
          }
          buf.push_back(as<Fixnum>(ary->get(state, 1))->to_native() | 0x80);
          value = as<Integer>(ary->get(state, 0));
        }
      }

      long v = value->to_long();

      while(v) {
        buf.push_back((v & 0x7f) | 0x80);
        v >>= 7;
      }

      if(buf.size() > 0) {
        char* a = const_cast<char*>(buf.c_str());
        char* b = a + buf.size() - 1;

        // clear continue bit
        *a &= 0x7f;

        // reverse string
        while(a < b) {
          int k = *a;
          *a++ = *b;
          *b-- = k;
        }

        str.append(buf.c_str(), buf.size());
      } else {
        str.push_back(0);
      }
    }

    inline size_t bit_extra(String* s, bool rest, size_t& count) {
      size_t extra = 0;

      if(rest) {
        count = s->size();
      } else {
        size_t size = s->size();
        if(count > size) {
          extra = (count - size + 1) / 2;
          count = size;
        }
      }

      return extra;
    }

    void bit_high(String* s, std::string& str, size_t count) {
      uint8_t* b = s->byte_address();
      int byte = 0;

      for(size_t i = 0; i++ < count; b++) {
        byte |= *b & 1;
        if(i & 7) {
          byte <<= 1;
        } else {
          str.push_back(byte & 0xff);
          byte = 0;
        }
      }

      if (count & 7) {
        byte <<= 7 - (count & 7);
        str.push_back(byte & 0xff);
      }
    }

    void bit_low(String* s, std::string& str, size_t count) {
      uint8_t* b = s->byte_address();
      int byte = 0;

      for(size_t i = 0; i++ < count; b++) {
        if(*b & 1)
          byte |= 128;

        if(i & 7) {
          byte >>= 1;
        } else {
          str.push_back(byte & 0xff);
          byte = 0;
        }
      }

      if(count & 7) {
        byte >>= 7 - (count & 7);
        str.push_back(byte & 0xff);
      }
    }

    inline size_t hex_extra(String* s, bool rest, size_t& count) {
      size_t extra = 0;

      if(rest) {
        count = s->size();
      } else {
        size_t size = s->size();
        if(count > size) {
          extra = (count + 1) / 2 - (size + 1) / 2;
          count = size;
        }
      }

      return extra;
    }

    void hex_high(String* s, std::string& str, size_t count) {
      uint8_t* b = s->byte_address();
      int byte = 0;

      for(size_t i = 0; i++ < count; b++) {
        if(ISALPHA(*b)) {
          byte |= ((*b & 15) + 9) & 15;
        } else {
          byte |= *b & 15;
        }

        if(i & 1) {
          byte <<= 4;
        } else {
          str.push_back(byte & 0xff);
          byte = 0;
        }
      }

      if(count & 1) {
        str.push_back(byte & 0xff);
      }
    }

    void hex_low(String* s, std::string& str, size_t count) {
      uint8_t* b = s->byte_address();
      int byte = 0;

      for(size_t i = 0; i++ < count; b++) {
        if(ISALPHA(*b)) {
          byte |= (((*b & 15) + 9) & 15) << 4;
        } else {
          byte |= (*b & 15) << 4;
        }

        if(i & 1) {
          byte >>= 4;
        } else {
          str.push_back(byte & 0xff);
          byte = 0;
        }
      }

      if(count & 1) {
        str.push_back(byte & 0xff);
      }
    }
  }

// Pack Float elements
#define pack_float_elements(format)   pack_elements(Float, pack::to_f, format)

#define pack_double_le                pack_float_elements(pack_double_element_le)
#define pack_double_be                pack_float_elements(pack_double_element_be)

#define pack_float_le                 pack_float_elements(pack_float_element_le)
#define pack_float_be                 pack_float_elements(pack_float_element_be)

// Pack Integer elements
#define pack_integer_elements(format) pack_elements(Integer, pack::to_int, format)

#define pack_byte_element(v)          str.push_back(pack::check_long(state, v))
#define pack_byte                     pack_integer_elements(pack_byte_element)

#define pack_short_le                 pack_integer_elements(pack_short_element_le)
#define pack_short_be                 pack_integer_elements(pack_short_element_be)

#define pack_int_le                   pack_integer_elements(pack_int_element_le)
#define pack_int_be                   pack_integer_elements(pack_int_element_be)

#define pack_long_le                  pack_integer_elements(pack_long_element_le)
#define pack_long_be                  pack_integer_elements(pack_long_element_be)

// Pack UTF-8 elements
#define pack_utf8_element(v)          pack::utf8_encode(state, str, v)
#define pack_utf8                     pack_elements(Integer, pack::to_int, pack_utf8_element)

// Pack BER-compressed integers
#define pack_ber_element(v)           pack::ber_encode(state, str, v)
#define pack_ber                      pack_elements(Integer, pack::to_int, pack_ber_element)

// Wraps the logic for iterating over a number of elements,
// coercing them to the correct class and formatting them
// for the output string.
#define pack_elements(T, coerce, format)        \
  for(; index < stop; index++) {                \
    Object* item = self->get(state, index);     \
    T* value = try_as<T>(item);                 \
    if(!value) {                                \
      item = coerce(state, call_frame, item);   \
      if(!item) return 0;                       \
      value = as<T>(item);                      \
    }                                           \
    format(value);                              \
  }

// Macros that depend on endianness
#ifdef RBX_LITTLE_ENDIAN

# define pack_double_element_le(v)  (pack::double_element(str, (v)->val))
# define pack_double_element_be(v)  (pack::swap_double(str, (v)->val))
# define pack_double                pack_double_le

# define pack_float_element_le(v)   (pack::float_element(str, (v)->val))
# define pack_float_element_be(v)   (pack::swap_float(str, (v)->val))
# define pack_float                 pack_float_le

# define pack_short_element_le(v)   (pack::short_element(str, pack::check_long(state, v)))
# define pack_short_element_be(v)   (pack::short_element(str, \
                                        pack::swap_2bytes(pack::check_long(state, v))))
# define pack_short                 pack_short_le

# define pack_int_element_le(v)     (pack::int_element(str, pack::check_long(state, v)))
# define pack_int_element_be(v)     (pack::int_element(str, \
                                        pack::swap_4bytes(pack::check_long(state, v))))
# define pack_int                   pack_int_le

# define pack_long_element_le(v)    (pack::long_element(str, pack::check_long_long(state, v)))
# define pack_long_element_be(v)    (pack::long_element(str, \
                                        pack::swap_8bytes(pack::check_long_long(state, v))))
# define pack_long                  pack_long_le

#else // Big endian

# define pack_double_element_le(v)  (pack::swap_double(str, (v)->val))
# define pack_double_element_be(v)  (pack::double_element(str, (v)->val))
# define pack_double                pack_double_be

# define pack_float_element_le(v)   (pack::swap_float(str, (v)->val))
# define pack_float_element_be(v)   (pack::float_element(str, (v)->val))
# define pack_float                 pack_float_be

# define pack_short_element_le(v)   (pack::short_element(str, \
                                        pack::swap_2bytes(pack::check_long(state, v))))
# define pack_short_element_be(v)   (pack::short_element(str, pack::check_long(state, v)))
# define pack_short                 pack_short_be

# define pack_int_element_le(v)     (pack::int_element(str, \
                                        pack::swap_4bytes(pack::check_long(state, v))))
# define pack_int_element_be(v)     (pack::int_element(str, pack::check_long(state, v)))
# define pack_int                   pack_int_be

# define pack_long_element_le(v)    (pack::long_element(str, \
                                        pack::swap_8bytes(pack::check_long_long(state, v))))
# define pack_long_element_be(v)    (pack::long_element(str, pack::check_long_long(state, v)))
# define pack_long                  pack_long_be

#endif

  String* Array::pack(STATE, String* directives, CallFrame* call_frame) {
    // Ragel-specific variables
    std::string d(directives->c_str(), directives->size());
    const char *p  = d.c_str();
    const char *pe = p + d.size();
    const char *eof = pe;
    int cs;

    // pack-specific variables
    Array* self = this;
    OnStack<1> sv(state, self);

    size_t array_size = self->size();
    size_t index = 0;
    size_t count = 0;
    size_t stop = 0;
    bool rest = false;
    bool platform = false;
    bool tainted = false;

    String* string_value = 0;
    std::string str("");

    // Use information we have to reduce repeated allocation.
    str.reserve(array_size * 4);

    if(RTEST(directives->tainted_p(state))) tainted = true;
%%{

  machine pack;

  include "pack.rl";

}%%

    if(en_main) {
      // do nothing
    }

    return force_as<String>(Primitives::failure());
  }
}

/* This file was generated by Ragel. Your edits will be lost.
 *
 * This is a state machine implementation of Array#pack.
 * See http://github.com/brixen/rapa.
 *
 * vim: filetype=cpp
 */

#include <stdint.h>
#include <sstream>

#include "vm/config.h"

#include "vm.hpp"
#include "object_utils.hpp"
#include "on_stack.hpp"
#include "objectmemory.hpp"

#include "builtin/array.hpp"
#include "builtin/exception.hpp"
#include "builtin/float.hpp"
#include "builtin/module.hpp"
#include "builtin/object.hpp"
#include "builtin/string.hpp"

namespace rubinius {
  namespace pack {
    inline Object* to_int(STATE, CallFrame* call_frame, Object* obj) {
      Array* args = Array::create(state, 1);
      args->set(state, 0, obj);

      return G(rubinius)->send(state, call_frame, state->symbol("pack_to_int"), args);
    }

#define BITS_LONG   (RBX_SIZEOF_LONG * 8)

    inline long check_long(STATE, Integer* obj) {
      if((obj)->fixnum_p()) {
        return force_as<Fixnum>(obj)->to_long();
      } else {
        Bignum* big = as<Bignum>(obj);
        big->verify_size(state, BITS_LONG);
        return big->to_long();
      }
    }

#define BITS_64     (64)

    inline long long check_long_long(STATE, Integer* obj) {
      if((obj)->fixnum_p()) {
        return force_as<Fixnum>(obj)->to_long_long();
      } else {
        Bignum* big = as<Bignum>(obj);
        big->verify_size(state, BITS_64);
        return big->to_long_long();
      }
    }

    inline Object* to_f(STATE, CallFrame* call_frame, Object* obj) {
      Array* args = Array::create(state, 1);
      args->set(state, 0, obj);

      return G(rubinius)->send(state, call_frame, state->symbol("pack_to_float"), args);
    }

    inline String* encoding_string(STATE, CallFrame* call_frame, Object* obj,
                                          const char* coerce_name)
    {
      String* s = try_as<String>(obj);
      if(s) return s;

      Array* args = Array::create(state, 1);
      args->set(state, 0, obj);

      std::string coerce_method("pack_");
      coerce_method += coerce_name;
      Object* result = G(rubinius)->send(state, call_frame,
            state->symbol(coerce_method.c_str()), args);

      if(!result) return 0;
      return as<String>(result);
    }

    inline uint16_t swap_2bytes(uint16_t x) {
      return (((x & 0x00ff)<<8) | ((x & 0xff00)>>8));
    }

    inline uint32_t swap_4bytes(uint32_t x) {
      return (((x & 0x000000ff) << 24)
             |((x & 0xff000000) >> 24)
             |((x & 0x0000ff00) << 8)
             |((x & 0x00ff0000) >> 8));
    }

    inline uint64_t swap_8bytes(uint64_t x) {
      return (((x & 0x00000000000000ffLL) << 56)
             |((x & 0xff00000000000000LL) >> 56)
             |((x & 0x000000000000ff00LL) << 40)
             |((x & 0x00ff000000000000LL) >> 40)
             |((x & 0x0000000000ff0000LL) << 24)
             |((x & 0x0000ff0000000000LL) >> 24)
             |((x & 0x00000000ff000000LL) << 8)
             |((x & 0x000000ff00000000LL) >> 8));
    }

    inline void swap_float(std::string& str, float value) {
      uint32_t x;

      memcpy(&x, &value, sizeof(float));
      x = swap_4bytes(x);

      str.append((const char*)&x, sizeof(uint32_t));
    }

    inline void swap_double(std::string& str, double value) {
      uint64_t x;

      memcpy(&x, &value, sizeof(double));
      x = swap_8bytes(x);

      str.append((const char*)&x, sizeof(uint64_t));
    }

    inline void double_element(std::string& str, double value) {
      str.append((const char*)&value, sizeof(double));
    }

    inline void float_element(std::string& str, float value) {
      str.append((const char*)&value, sizeof(float));
    }

    inline void short_element(std::string& str, int16_t value) {
      str.append((const char*)&value, sizeof(int16_t));
    }

    inline void int_element(std::string& str, int32_t value) {
      str.append((const char*)&value, sizeof(int32_t));
    }

    inline void long_element(std::string& str, int64_t value) {
      str.append((const char*)&value, sizeof(int64_t));
    }

    inline int32_t int32_element(STATE, Integer* value) {
      if(value->fixnum_p()) {
        long l = as<Fixnum>(value)->to_long();
        if(l > INT32_MAX || l < INT32_MIN) {
          Exception::range_error(state, "Fixnum value out of range of int32");
        }
        return l;
      } else {
        Bignum* big = as<Bignum>(value);
        big->verify_size(state, 32);
        return big->to_int();
      }
    }

#define QUOTABLE_PRINTABLE_BUFSIZE 1024

    void quotable_printable(String* s, std::string& str, int count) {
      static char hex_table[] = "0123456789ABCDEF";
      char buf[QUOTABLE_PRINTABLE_BUFSIZE];

      uint8_t* b = s->byte_address();
      uint8_t* e = b + s->size();
      int i = 0, n = 0, prev = -1;

      for(; b < e; b++) {
        if((*b > 126) || (*b < 32 && *b != '\n' && *b != '\t') || (*b == '=')) {
          buf[i++] = '=';
          buf[i++] = hex_table[*b >> 4];
          buf[i++] = hex_table[*b & 0x0f];
          n += 3;
          prev = -1;
        } else if(*b == '\n') {
          if(prev == ' ' || prev == '\t') {
            buf[i++] = '=';
            buf[i++] = *b;
          }
          buf[i++] = *b;
          n = 0;
          prev = *b;
        } else {
          buf[i++] = *b;
          n++;
          prev = *b;
        }

        if(n > count) {
          buf[i++] = '=';
          buf[i++] = '\n';
          n = 0;
          prev = '\n';
        }

        if(i > QUOTABLE_PRINTABLE_BUFSIZE - 5) {
          str.append(buf, i);
          i = 0;
        }
      }

      if(n > 0) {
        buf[i++] = '=';
        buf[i++] = '\n';
      }

      if(i > 0) {
        str.append(buf, i);
      }
    }

    static const char uu_table[] =
      "`!\"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_";
    static const char b64_table[] =
      "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";

#define b64_uu_byte1(t, b)      t[077 & (*b >> 2)]
#define b64_uu_byte2(t, b, c)   t[077 & (((*b << 4) & 060) | ((c >> 4) & 017))]
#define b64_uu_byte3(t, b, c)   t[077 & (((b[1] << 2) & 074) | ((c >> 6) & 03))];
#define b64_uu_byte4(t, b)      t[077 & b[2]];

    void b64_uu_encode(String* s, std::string& str, size_t count,
                              const char* table, int padding, bool encode_size)
    {
      char *buf = ALLOCA_N(char, count * 4 / 3 + 6);
      size_t i, chars, line, total = s->size();
      uint8_t* b = s->byte_address();

      for(i = 0; total > 0; i = 0, total -= line) {
        line = total > count ? count : total;

        if(encode_size) buf[i++] = line + ' ';

        for(chars = line; chars >= 3; chars -= 3, b += 3) {
          buf[i++] = b64_uu_byte1(table, b);
          buf[i++] = b64_uu_byte2(table, b, b[1]);
          buf[i++] = b64_uu_byte3(table, b, b[2]);
          buf[i++] = b64_uu_byte4(table, b);
        }

        if(chars == 2) {
          buf[i++] = b64_uu_byte1(table, b);
          buf[i++] = b64_uu_byte2(table, b, b[1]);
          buf[i++] = b64_uu_byte3(table, b, '\0');
          buf[i++] = padding;
        } else if(chars == 1) {
          buf[i++] = b64_uu_byte1(table, b);
          buf[i++] = b64_uu_byte2(table, b, '\0');
          buf[i++] = padding;
          buf[i++] = padding;
        }

        b += chars;
        buf[i++] = '\n';
        str.append(buf, i);
      }
    }

    void utf8_encode(STATE, std::string& str, Integer* value) {
      int32_t v = int32_element(state, value);

      if(!(v & ~0x7f)) {
        str.push_back(v);
      } else if(!(v & ~0x7ff)) {
        str.push_back(((v >> 6) & 0xff) | 0xc0);
        str.push_back((v & 0x3f) | 0x80);
      } else if(!(v & ~0xffff)) {
        str.push_back(((v >> 12) & 0xff) | 0xe0);
        str.push_back(((v >> 6)  & 0x3f) | 0x80);
        str.push_back((v & 0x3f) | 0x80);
      } else if(!(v & ~0x1fffff)) {
        str.push_back(((v >> 18) & 0xff) | 0xf0);
        str.push_back(((v >> 12) & 0x3f) | 0x80);
        str.push_back(((v >> 6)  & 0x3f) | 0x80);
        str.push_back((v & 0x3f) | 0x80);
      } else if(!(v & ~0x3ffffff)) {
        str.push_back(((v >> 24) & 0xff) | 0xf8);
        str.push_back(((v >> 18) & 0x3f) | 0x80);
        str.push_back(((v >> 12) & 0x3f) | 0x80);
        str.push_back(((v >> 6)  & 0x3f) | 0x80);
        str.push_back((v & 0x3f) | 0x80);
      } else if(!(v & ~0x7fffffff)) {
        str.push_back(((v >> 30) & 0xff) | 0xfc);
        str.push_back(((v >> 24) & 0x3f) | 0x80);
        str.push_back(((v >> 18) & 0x3f) | 0x80);
        str.push_back(((v >> 12) & 0x3f) | 0x80);
        str.push_back(((v >> 6)  & 0x3f) | 0x80);
        str.push_back((v & 0x3f) | 0x80);
      } else {
        Exception::range_error(state, "pack('U') value out of range");
      }
    }

    void ber_encode(STATE, std::string& str, Integer* value) {
      if(!value->positive_p()) {
        Exception::argument_error(state, "cannot BER compress a negative number");
      }

      std::string buf;

      if(try_as<Bignum>(value)) {
        static Fixnum* base = Fixnum::from(128);
        while(try_as<Bignum>(value)) {
          Array* ary;
          if(value->fixnum_p()) {
            ary = as<Fixnum>(value)->divmod(state, base);
          } else {
            ary = as<Bignum>(value)->divmod(state, base);
          }
          buf.push_back(as<Fixnum>(ary->get(state, 1))->to_native() | 0x80);
          value = as<Integer>(ary->get(state, 0));
        }
      }

      long v = value->to_long();

      while(v) {
        buf.push_back((v & 0x7f) | 0x80);
        v >>= 7;
      }

      if(buf.size() > 0) {
        char* a = const_cast<char*>(buf.c_str());
        char* b = a + buf.size() - 1;

        // clear continue bit
        *a &= 0x7f;

        // reverse string
        while(a < b) {
          int k = *a;
          *a++ = *b;
          *b-- = k;
        }

        str.append(buf.c_str(), buf.size());
      } else {
        str.push_back(0);
      }
    }

    inline size_t bit_extra(String* s, bool rest, size_t& count) {
      size_t extra = 0;

      if(rest) {
        count = s->size();
      } else {
        size_t size = s->size();
        if(count > size) {
          extra = (count - size + 1) / 2;
          count = size;
        }
      }

      return extra;
    }

    void bit_high(String* s, std::string& str, size_t count) {
      uint8_t* b = s->byte_address();
      int byte = 0;

      for(size_t i = 0; i++ < count; b++) {
        byte |= *b & 1;
        if(i & 7) {
          byte <<= 1;
        } else {
          str.push_back(byte & 0xff);
          byte = 0;
        }
      }

      if (count & 7) {
        byte <<= 7 - (count & 7);
        str.push_back(byte & 0xff);
      }
    }

    void bit_low(String* s, std::string& str, size_t count) {
      uint8_t* b = s->byte_address();
      int byte = 0;

      for(size_t i = 0; i++ < count; b++) {
        if(*b & 1)
          byte |= 128;

        if(i & 7) {
          byte >>= 1;
        } else {
          str.push_back(byte & 0xff);
          byte = 0;
        }
      }

      if(count & 7) {
        byte >>= 7 - (count & 7);
        str.push_back(byte & 0xff);
      }
    }

    inline size_t hex_extra(String* s, bool rest, size_t& count) {
      size_t extra = 0;

      if(rest) {
        count = s->size();
      } else {
        size_t size = s->size();
        if(count > size) {
          extra = (count + 1) / 2 - (size + 1) / 2;
          count = size;
        }
      }

      return extra;
    }

    void hex_high(String* s, std::string& str, size_t count) {
      uint8_t* b = s->byte_address();
      int byte = 0;

      for(size_t i = 0; i++ < count; b++) {
        if(ISALPHA(*b)) {
          byte |= ((*b & 15) + 9) & 15;
        } else {
          byte |= *b & 15;
        }

        if(i & 1) {
          byte <<= 4;
        } else {
          str.push_back(byte & 0xff);
          byte = 0;
        }
      }

      if(count & 1) {
        str.push_back(byte & 0xff);
      }
    }

    void hex_low(String* s, std::string& str, size_t count) {
      uint8_t* b = s->byte_address();
      int byte = 0;

      for(size_t i = 0; i++ < count; b++) {
        if(ISALPHA(*b)) {
          byte |= (((*b & 15) + 9) & 15) << 4;
        } else {
          byte |= (*b & 15) << 4;
        }

        if(i & 1) {
          byte >>= 4;
        } else {
          str.push_back(byte & 0xff);
          byte = 0;
        }
      }

      if(count & 1) {
        str.push_back(byte & 0xff);
      }
    }
  }

// Pack Float elements
#define pack_float_elements(format)   pack_elements(Float, pack::to_f, format)

#define pack_double_le                pack_float_elements(pack_double_element_le)
#define pack_double_be                pack_float_elements(pack_double_element_be)

#define pack_float_le                 pack_float_elements(pack_float_element_le)
#define pack_float_be                 pack_float_elements(pack_float_element_be)

// Pack Integer elements
#define pack_integer_elements(format) pack_elements(Integer, pack::to_int, format)

#define pack_byte_element(v)          str.push_back(pack::check_long(state, v))
#define pack_byte                     pack_integer_elements(pack_byte_element)

#define pack_short_le                 pack_integer_elements(pack_short_element_le)
#define pack_short_be                 pack_integer_elements(pack_short_element_be)

#define pack_int_le                   pack_integer_elements(pack_int_element_le)
#define pack_int_be                   pack_integer_elements(pack_int_element_be)

#define pack_long_le                  pack_integer_elements(pack_long_element_le)
#define pack_long_be                  pack_integer_elements(pack_long_element_be)

// Pack UTF-8 elements
#define pack_utf8_element(v)          pack::utf8_encode(state, str, v)
#define pack_utf8                     pack_elements(Integer, pack::to_int, pack_utf8_element)

// Pack BER-compressed integers
#define pack_ber_element(v)           pack::ber_encode(state, str, v)
#define pack_ber                      pack_elements(Integer, pack::to_int, pack_ber_element)

// Wraps the logic for iterating over a number of elements,
// coercing them to the correct class and formatting them
// for the output string.
#define pack_elements(T, coerce, format)        \
  for(; index < stop; index++) {                \
    Object* item = self->get(state, index);     \
    T* value = try_as<T>(item);                 \
    if(!value) {                                \
      item = coerce(state, call_frame, item);   \
      if(!item) return 0;                       \
      value = as<T>(item);                      \
    }                                           \
    format(value);                              \
  }

// Macros that depend on endianness
#ifdef RBX_LITTLE_ENDIAN

# define pack_double_element_le(v)  (pack::double_element(str, (v)->val))
# define pack_double_element_be(v)  (pack::swap_double(str, (v)->val))
# define pack_double                pack_double_le

# define pack_float_element_le(v)   (pack::float_element(str, (v)->val))
# define pack_float_element_be(v)   (pack::swap_float(str, (v)->val))
# define pack_float                 pack_float_le

# define pack_short_element_le(v)   (pack::short_element(str, pack::check_long(state, v)))
# define pack_short_element_be(v)   (pack::short_element(str, \
                                        pack::swap_2bytes(pack::check_long(state, v))))
# define pack_short                 pack_short_le

# define pack_int_element_le(v)     (pack::int_element(str, pack::check_long(state, v)))
# define pack_int_element_be(v)     (pack::int_element(str, \
                                        pack::swap_4bytes(pack::check_long(state, v))))
# define pack_int                   pack_int_le

# define pack_long_element_le(v)    (pack::long_element(str, pack::check_long_long(state, v)))
# define pack_long_element_be(v)    (pack::long_element(str, \
                                        pack::swap_8bytes(pack::check_long_long(state, v))))
# define pack_long                  pack_long_le

#else // Big endian

# define pack_double_element_le(v)  (pack::swap_double(str, (v)->val))
# define pack_double_element_be(v)  (pack::double_element(str, (v)->val))
# define pack_double                pack_double_be

# define pack_float_element_le(v)   (pack::swap_float(str, (v)->val))
# define pack_float_element_be(v)   (pack::float_element(str, (v)->val))
# define pack_float                 pack_float_be

# define pack_short_element_le(v)   (pack::short_element(str, \
                                        pack::swap_2bytes(pack::check_long(state, v))))
# define pack_short_element_be(v)   (pack::short_element(str, pack::check_long(state, v)))
# define pack_short                 pack_short_be

# define pack_int_element_le(v)     (pack::int_element(str, \
                                        pack::swap_4bytes(pack::check_long(state, v))))
# define pack_int_element_be(v)     (pack::int_element(str, pack::check_long(state, v)))
# define pack_int                   pack_int_be

# define pack_long_element_le(v)    (pack::long_element(str, \
                                        pack::swap_8bytes(pack::check_long_long(state, v))))
# define pack_long_element_be(v)    (pack::long_element(str, pack::check_long_long(state, v)))
# define pack_long                  pack_long_be

#endif

  String* Array::pack(STATE, String* directives, CallFrame* call_frame) {
    // Ragel-specific variables
    std::string d(directives->c_str(), directives->size());
    const char *p  = d.c_str();
    const char *pe = p + d.size();
    const char *eof = pe;
    int cs;

    // pack-specific variables
    Array* self = this;
    OnStack<1> sv(state, self);

    size_t array_size = self->size();
    size_t index = 0;
    size_t count = 0;
    size_t stop = 0;
    bool rest = false;
    bool platform = false;
    bool tainted = false;

    String* string_value = 0;
    std::string str("");

    // Use information we have to reduce repeated allocation.
    str.reserve(array_size * 4);

    if(RTEST(directives->tainted_p(state))) tainted = true;
%%{

  machine pack;

  include "pack.rl";

}%%

    if(en_main) {
      // do nothing
    }

    return force_as<String>(Primitives::failure());
  }
}


don't set the end pointer out farther than the specified length

#include <stdlib.h>
#include <string.h>
#include <stdio.h>
#include <ruby/ruby.h>
#include "tnetstrings.h"

%%{
  machine tnetstrings;
  access parser->;

  action mark_ends {
    // this marks the start of the payload -
    // move ahead 2 for the colon and the type spec.
    parser->payload = p + 2;

    // this is a bit hacky, but I think it's the only way to do it.
    // only now do we know where the end is, since we've parsed the payload_size.
    // This will cause EOF actions to be triggered at the end of the payload.
    // TODO: eventually, have an option to read this from a stream,
    // which is arguably the whole point.
    eof = pe = parser->payload + parser->payload_size;

    // assert(pe < str + size, "Holy crap we overfloweded the buffer");
  }

  action collect_payload_size {
    parser->payload_size *= 10;
    parser->payload_size += CTOI(fc);
  }

  # -*- boolean (!) -*- #
  action wrap_true  { parser->result = TNETS_WRAP_TRUE;  }
  action wrap_false { parser->result = TNETS_WRAP_FALSE; }

  # -*- null (~) -*- #
  action wrap_null  { parser->result = Qnil;   }

  # -*- number (#) -*- #
  action neg {
    parser->seen_neg = 1;
  }

  action collect_num {
    parser->num_builder *= 10;
    parser->num_builder += CTOI(fc);
  }

  action wrap_num {
    if (parser->seen_neg) parser->num_builder *= -1;
    parser->result = TNETS_WRAP_NUM(parser->num_builder);
  }

  # -*- string (,) -*- #
  action wrap_str {
    parser->result = TNETS_WRAP_STR(p+1, parser->payload_size);
  }

  action wrap_dict_key {
    parser->result = TNETS_WRAP_DICT_KEY(p+1, parser->payload_size);
  }

  action parse_dict {
    TNETS_T dict = TNETS_NEW_DICT;
    TNETS_T key;
    tnets_parser* sub_parser = tnets_parser_new();
    char* sub_data = parser->payload;
    int sub_size = parser->payload_size;

    // TODO: seriously DRY this up, if only with #defines
    do {
      // XXX HACK.  Ask the ragel guys for a better way to do this.
      // This sets the "current state" to the entry point for the
      // dict_key machine, which only accepts strings.
      sub_parser->cs = tnetstrings_en_dict_key;
      tnets_parser_chunk(sub_parser, sub_data, sub_size);
      key = sub_parser->result;

      sub_data += (sub_parser->payload_size + 2 + 1);
      sub_size -= (sub_parser->payload_size + 2 + 1);
      tnets_parser_clear(sub_parser);

      tnets_parser_chunk(sub_parser, sub_data, sub_size);
      TNETS_DICT_ADD(dict, key, sub_parser->result);

      sub_data += (sub_parser->payload_size + 2 + 1);
      sub_size -= (sub_parser->payload_size + 2 + 1);
      tnets_parser_clear(sub_parser);
    } while(sub_size > 0);

    free(sub_parser);

    parser->result = dict;
  }

  action parse_arr {
    TNETS_T arr = TNETS_NEW_ARR;
    tnets_parser* sub_parser = tnets_parser_new();
    char* sub_data = parser->payload;
    int sub_size = parser->payload_size;
    do {
      tnets_parser_chunk(sub_parser, sub_data, sub_size);
      TNETS_ARR_PUSH(arr, sub_parser->result);

      sub_data += (sub_parser->payload_size + 2 + 1);
      sub_size -= (sub_parser->payload_size + 2 + 1);
      // reset current state and such
      tnets_parser_clear(sub_parser);

    } while(sub_size > 0);
    free(sub_parser);

    parser->result = arr;
  }

  action error {
    printf("error!\n"); // TODO: real error handling
    printf("p: [%s]\n", p);
    printf("pe: [%s]\n", pe);
    printf("str: [%s]\n", str);
  }

  # parses the size spec at the start of the payload
  tnets_size = digit+ $collect_payload_size;
  colon = ':' @mark_ends;

  # primitives
  tnets_num  = '#' ('-' @neg)? (digit+ $collect_num) %/wrap_num;
  tnets_str  = ',' @wrap_str;
  tnets_dict_key = ',' @wrap_dict_key;
  # NB: any string that is not "true" will return "false".
  tnets_bool = '!' ('true' %/wrap_true | !'true' %/wrap_false);
  tnets_null = '~' %/wrap_null;

  # recursive structures
  tnets_dict = '}' @parse_dict;
  tnets_arr  = ']' @parse_arr;

  main := (
    tnets_size colon (
      tnets_dict |
      tnets_arr  |
      tnets_num  |
      tnets_str  |
      tnets_bool |
      tnets_null
    )
  ) @err(error);

  dict_key := (tnets_size colon tnets_dict_key) @err(error);
}%%

%% write data;

int tnets_parser_init(tnets_parser *parser) {
  %% write init;
}

int tnets_parser_clear(tnets_parser *parser) {
  tnets_parser_init(parser);
  parser->num_builder = 0;
  parser->seen_neg = 0;
  parser->result = TNETS_WRAP_NULL;
  parser->payload = NULL;
  parser->payload_size = 0;
}

int tnets_parser_chunk(tnets_parser *parser, char *str, int size) {
  char *p = str;
  // pe gets re-set once we've parsed the length.
  // set to 10 characters here because that's the maximum
  // length of the length spec.  Also add one for the colon.
  char *pe = p + ( (size < 11) ? size : 11 );
  char *eof;

  %% write exec;

  return 0;
}

tnets_parser *tnets_parser_new(void) {
  tnets_parser *parser = calloc(sizeof(tnets_parser), 1);
  tnets_parser_init(parser);
  return parser;
}

tnets_parser *parse_str(char *str, int size) {
  tnets_parser *parser = tnets_parser_new();
  tnets_parser_chunk(parser, str, size);
  return parser;
}

VALUE rb_parse_tnets(VALUE self, VALUE rbstr) {
  char *tnets = RSTRING_PTR(rbstr);
  int len = RSTRING_LEN(rbstr);

  tnets_parser *parser = parse_str(RSTRING_PTR(rbstr), len);

  VALUE result = parser->result;
  free(parser);

  return result;
}

void Init_tnetstrings() {
  VALUE module = rb_define_module("TnetstringsCMethods");

  rb_define_method(module, "c_parse", rb_parse_tnets, 1);
}

#include <stdlib.h>
#include <string.h>
#include <stdio.h>
#include <ruby/ruby.h>
#include "tnetstrings.h"

%%{
  machine tnetstrings;
  access parser->;

  action mark_ends {
    // this marks the start of the payload -
    // move ahead 2 for the colon and the type spec.
    parser->payload = p + 2;

    // this is a bit hacky, but I think it's the only way to do it.
    // only now do we know where the end is, since we've parsed the payload_size.
    // This will cause EOF actions to be triggered at the end of the payload.
    // TODO: eventually, have an option to read this from a stream,
    // which is arguably the whole point.
    eof = pe = parser->payload + parser->payload_size;

    // assert(pe < str + size, "Holy crap we overfloweded the buffer");
  }

  action collect_payload_size {
    parser->payload_size *= 10;
    parser->payload_size += CTOI(fc);
  }

  # -*- boolean (!) -*- #
  action wrap_true  { parser->result = TNETS_WRAP_TRUE;  }
  action wrap_false { parser->result = TNETS_WRAP_FALSE; }

  # -*- null (~) -*- #
  action wrap_null  { parser->result = Qnil;   }

  # -*- number (#) -*- #
  action neg {
    parser->seen_neg = 1;
  }

  action collect_num {
    parser->num_builder *= 10;
    parser->num_builder += CTOI(fc);
  }

  action wrap_num {
    if (parser->seen_neg) parser->num_builder *= -1;
    parser->result = TNETS_WRAP_NUM(parser->num_builder);
  }

  # -*- string (,) -*- #
  action wrap_str {
    parser->result = TNETS_WRAP_STR(p+1, parser->payload_size);
  }

  action wrap_dict_key {
    parser->result = TNETS_WRAP_DICT_KEY(p+1, parser->payload_size);
  }

  action parse_dict {
    TNETS_T dict = TNETS_NEW_DICT;
    TNETS_T key;
    tnets_parser* sub_parser = tnets_parser_new();
    char* sub_data = parser->payload;
    int sub_size = parser->payload_size;

    // TODO: seriously DRY this up, if only with #defines
    do {
      // XXX HACK.  Ask the ragel guys for a better way to do this.
      // This sets the "current state" to the entry point for the
      // dict_key machine, which only accepts strings.
      sub_parser->cs = tnetstrings_en_dict_key;
      tnets_parser_chunk(sub_parser, sub_data, sub_size);
      key = sub_parser->result;

      sub_data += (sub_parser->payload_size + 2 + 1);
      sub_size -= (sub_parser->payload_size + 2 + 1);
      tnets_parser_clear(sub_parser);

      tnets_parser_chunk(sub_parser, sub_data, sub_size);
      TNETS_DICT_ADD(dict, key, sub_parser->result);

      sub_data += (sub_parser->payload_size + 2 + 1);
      sub_size -= (sub_parser->payload_size + 2 + 1);
      tnets_parser_clear(sub_parser);
    } while(sub_size > 0);

    free(sub_parser);

    parser->result = dict;
  }

  action parse_arr {
    TNETS_T arr = TNETS_NEW_ARR;
    tnets_parser* sub_parser = tnets_parser_new();
    char* sub_data = parser->payload;
    int sub_size = parser->payload_size;
    do {
      tnets_parser_chunk(sub_parser, sub_data, sub_size);
      TNETS_ARR_PUSH(arr, sub_parser->result);

      sub_data += (sub_parser->payload_size + 2 + 1);
      sub_size -= (sub_parser->payload_size + 2 + 1);
      // reset current state and such
      tnets_parser_clear(sub_parser);

    } while(sub_size > 0);
    free(sub_parser);

    parser->result = arr;
  }

  action error {
    printf("error!\n"); // TODO: real error handling
    printf("p: [%s]\n", p);
    printf("pe: [%s]\n", pe);
    printf("str: [%s]\n", str);
  }

  # parses the size spec at the start of the payload
  tnets_size = digit+ $collect_payload_size;
  colon = ':' @mark_ends;

  # primitives
  tnets_num  = '#' ('-' @neg)? (digit+ $collect_num) %/wrap_num;
  tnets_str  = ',' @wrap_str;
  tnets_dict_key = ',' @wrap_dict_key;
  # NB: any string that is not "true" will return "false".
  tnets_bool = '!' ('true' %/wrap_true | !'true' %/wrap_false);
  tnets_null = '~' %/wrap_null;

  # recursive structures
  tnets_dict = '}' @parse_dict;
  tnets_arr  = ']' @parse_arr;

  main := (
    tnets_size colon (
      tnets_dict |
      tnets_arr  |
      tnets_num  |
      tnets_str  |
      tnets_bool |
      tnets_null
    )
  ) @err(error);

  dict_key := (tnets_size colon tnets_dict_key) @err(error);
}%%

%% write data;

int tnets_parser_init(tnets_parser *parser) {
  %% write init;
}

int tnets_parser_clear(tnets_parser *parser) {
  tnets_parser_init(parser);
  parser->num_builder = 0;
  parser->seen_neg = 0;
  parser->result = TNETS_WRAP_NULL;
  parser->payload = NULL;
  parser->payload_size = 0;
}

int tnets_parser_chunk(tnets_parser *parser, char *str, int size) {
  char *p = str;
  // pe gets re-set once we've parsed the length.
  // set to 10 characters here because that's the maximum
  // length of the length spec.
  char *pe = p + 11;
  char *eof;

  %% write exec;

  return 0;
}

tnets_parser *tnets_parser_new(void) {
  tnets_parser *parser = calloc(sizeof(tnets_parser), 1);
  tnets_parser_init(parser);
  return parser;
}

tnets_parser *parse_str(char *str, int size) {
  tnets_parser *parser = tnets_parser_new();
  tnets_parser_chunk(parser, str, size);
  return parser;
}

VALUE rb_parse_tnets(VALUE self, VALUE rbstr) {
  char *tnets = RSTRING_PTR(rbstr);
  int len = RSTRING_LEN(rbstr);

  tnets_parser *parser = parse_str(RSTRING_PTR(rbstr), len);

  VALUE result = parser->result;
  free(parser);

  return result;
}

void Init_tnetstrings() {
  VALUE module = rb_define_module("TnetstringsCMethods");

  rb_define_method(module, "c_parse", rb_parse_tnets, 1);
}


added , and : as syntax symbols

added , and : as syntax symbols

/*
vim: syntax=ragel
*/

#include <stdlib.h>
#include <stdio.h>
#include <string.h>

#include <map>
#include <string>
#include <vector>
#include <sstream>
#include <iostream>
#include <algorithm>
#include <exception>
#include <stdexcept>

#include "lexer.h"
#include "ascii.h"

using std::cerr;
using std::cout;
using std::cin;
using std::endl;
using std::copy;

#define T(t,i,v) {i,v},
static std::map<long, const char *> TokenMap = {
    ASCII
    TOKENS
};
#undef T

inline void write(const char *data) {
    cout << data;
}

inline void write(const char c) {
    cout << c;
}

inline void write(const char *data, int len) {
    cout.write(data, len);
}

%%{
    machine sast;

    whitespace      = ' '+;
    tab             = '\t';
    hash            = '#';
    doublequote     = '"';
    newline         = "\n\r"|'\n'|'\r';
    number          = digit+ ('.' digit+)?;
    syntax          = '"'|"'"|'.'|','|'('|')'|'['|']'|'{'|'}'|':'|';';
    symbol          = (any - ('#'|whitespace|newline|syntax))+;
    counter         = (any | newline @{AddNewline(fpc);})*;

    commenter := |*
        ("##" (any - newline)* newline) & counter => {
            Token(TokenKind::Comment);
            fgoto body;
        };

        ('#' (any - '#')* '#') & counter => {
            Token(TokenKind::Comment);
            fgoto body;
        };
    *|;

    string := |*
        ('"' (any - '"')* '"') & counter => {
            Token(TokenKind::String, 1);
            fgoto body;
        };
    *|;

    body := |*

        whitespace => {
        };

        ' '* newline & counter => {
        };

        hash => {
            fhold;
            fgoto commenter;
        };

        doublequote => {
            fhold;
            fgoto string;
        };

        tab => {
            printf("TAB ERROR\n");
            exit(-1);
        };

        '{'|'['|'(' => {
            Token(fc);
            ++nesting;
        };

        '}'|']'|')' => {
            Token(fc);
            --nesting;
        };

        number => {
            Token(TokenKind::Number);
        };

        "'" => {
            Token(TokenKind::Symbol);
        };

        symbol => {
            Token(TokenKind::Symbol);
        };

        syntax => {
            Token(fc);
        };

    *|;

    write data nofinal;
}%%

class SotaLexer {
    char const* const source;
    char const* const pe;
    char const* const eof;
    char const* p;
    char const* ts;
    char const* te;
    int stack[1];
    int cs;
    int act;
    int top;
    int nesting;
    std::vector<const char *> newlines;
    std::vector<CToken> tokens;

public:
    SotaLexer(char const* const source)
        : source(source)
        , pe(source + strlen(source))
        , eof(source + strlen(source))
        , p(source)
        , nesting(0) {
        %% write init;
        //pretend newline before start of file
        newlines.push_back(source-1);
    }

    ~SotaLexer() {
        newlines.clear();
        tokens.clear();
    }

    void AddNewline(const char *pchar) {
        if (pchar > newlines.back())
            newlines.push_back(pchar);
        else
            printf("UNEXPECTED BEHAVIOR!!!\n");
    }

    const char * Newline(const char *pchar) {
        for (unsigned i = newlines.size(); i-- > 0;) {
            if (pchar > newlines[i])
                return newlines[i];
        }
        return 0;
    }

    long Line(const char *pchar) {
        for (unsigned i = newlines.size(); i-- > 0;) {
            if (pchar > newlines[i])
                return i + 1;
        }
        return 0;
    }

    long Pos(const char *pchar) {
        const char *nl = Newline(pchar);
        if (nl)
            return pchar - nl;
        return 0;
    }

    void Token(long kind, long trim=0) {
        tokens.push_back({
            ts - source + trim,
            te - source - trim,
            kind,
            Line(ts),
            Pos(ts)});
    }

    void Token(long start, long end, long kind, long line, long pos, long skip) {
        tokens.push_back({
            start,
            end,
            kind,
            line,
            pos,
            skip});
    }

    long Scan(CToken **tokens) {
        %% write exec;
        *tokens = (struct CToken *)malloc(this->tokens.size() * sizeof(struct CToken));
        copy(this->tokens.begin(), this->tokens.end(), *tokens);
        return this->tokens.size();
    }

};

extern "C" long scan(const char *source, struct CToken **tokens) {
    std::ios::sync_with_stdio(false);
    SotaLexer lexer(source);
    return lexer.Scan(tokens);
}

/*
vim: syntax=ragel
*/

#include <stdlib.h>
#include <stdio.h>
#include <string.h>

#include <map>
#include <string>
#include <vector>
#include <sstream>
#include <iostream>
#include <algorithm>
#include <exception>
#include <stdexcept>

#include "lexer.h"
#include "ascii.h"

using std::cerr;
using std::cout;
using std::cin;
using std::endl;
using std::copy;

#define T(t,i,v) {i,v},
static std::map<long, const char *> TokenMap = {
    ASCII
    TOKENS
};
#undef T

inline void write(const char *data) {
    cout << data;
}

inline void write(const char c) {
    cout << c;
}

inline void write(const char *data, int len) {
    cout.write(data, len);
}

%%{
    machine sast;

    whitespace      = ' '+;
    tab             = '\t';
    hash            = '#';
    doublequote     = '"';
    newline         = "\n\r"|'\n'|'\r';
    number          = digit+ ('.' digit+)?;
    syntax          = '"'|"'"|'.'|'('|')'|'['|']'|'{'|'}'|';';
    symbol          = (any - ('#'|whitespace|newline|syntax))+;
    counter         = (any | newline @{AddNewline(fpc);})*;

    commenter := |*
        ("##" (any - newline)* newline) & counter => {
            Token(TokenKind::Comment);
            fgoto body;
        };

        ('#' (any - '#')* '#') & counter => {
            Token(TokenKind::Comment);
            fgoto body;
        };
    *|;

    string := |*
        ('"' (any - '"')* '"') & counter => {
            Token(TokenKind::String, 1);
            fgoto body;
        };
    *|;

    body := |*

        whitespace => {
        };

        ' '* newline & counter => {
        };

        hash => {
            fhold;
            fgoto commenter;
        };

        doublequote => {
            fhold;
            fgoto string;
        };

        tab => {
            printf("TAB ERROR\n");
            exit(-1);
        };

        '{'|'['|'(' => {
            Token(fc);
            ++nesting;
        };

        '}'|']'|')' => {
            Token(fc);
            --nesting;
        };

        number => {
            Token(TokenKind::Number);
        };

        "'" => {
            Token(TokenKind::Symbol);
        };

        symbol => {
            Token(TokenKind::Symbol);
        };

        syntax => {
            Token(fc);
        };

    *|;

    write data nofinal;
}%%

class SotaLexer {
    char const* const source;
    char const* const pe;
    char const* const eof;
    char const* p;
    char const* ts;
    char const* te;
    int stack[1];
    int cs;
    int act;
    int top;
    int nesting;
    std::vector<const char *> newlines;
    std::vector<CToken> tokens;

public:
    SotaLexer(char const* const source)
        : source(source)
        , pe(source + strlen(source))
        , eof(source + strlen(source))
        , p(source)
        , nesting(0) {
        %% write init;
        //pretend newline before start of file
        newlines.push_back(source-1);
    }

    ~SotaLexer() {
        newlines.clear();
        tokens.clear();
    }

    void AddNewline(const char *pchar) {
        if (pchar > newlines.back())
            newlines.push_back(pchar);
        else
            printf("UNEXPECTED BEHAVIOR!!!\n");
    }

    const char * Newline(const char *pchar) {
        for (unsigned i = newlines.size(); i-- > 0;) {
            if (pchar > newlines[i])
                return newlines[i];
        }
        return 0;
    }

    long Line(const char *pchar) {
        for (unsigned i = newlines.size(); i-- > 0;) {
            if (pchar > newlines[i])
                return i + 1;
        }
        return 0;
    }

    long Pos(const char *pchar) {
        const char *nl = Newline(pchar);
        if (nl)
            return pchar - nl;
        return 0;
    }

    void Token(long kind, long trim=0) {
        tokens.push_back({
            ts - source + trim,
            te - source - trim,
            kind,
            Line(ts),
            Pos(ts)});
    }

    void Token(long start, long end, long kind, long line, long pos, long skip) {
        tokens.push_back({
            start,
            end,
            kind,
            line,
            pos,
            skip});
    }

    long Scan(CToken **tokens) {
        %% write exec;
        *tokens = (struct CToken *)malloc(this->tokens.size() * sizeof(struct CToken));
        copy(this->tokens.begin(), this->tokens.end(), *tokens);
        return this->tokens.size();
    }

};

extern "C" long scan(const char *source, struct CToken **tokens) {
    std::ios::sync_with_stdio(false);
    SotaLexer lexer(source);
    return lexer.Scan(tokens);
}