Check for integer overflow in content-length.

Also only allow single digits in the HTTP version.
Need multiple digits? Convince me.

/* Copyright (c) 2008, 2009 Ryan Dahl (ry@tinyclouds.org)
 * Based on Zed Shaw's Mongrel, copyright (c) Zed A. Shaw
 *
 * All rights reserved.
 *
 * Permission is hereby granted, free of charge, to any person obtaining
 * a copy of this software and associated documentation files (the
 * "Software"), to deal in the Software without restriction, including
 * without limitation the rights to use, copy, modify, merge, publish,
 * distribute, sublicense, and/or sell copies of the Software, and to
 * permit persons to whom the Software is furnished to do so, subject to
 * the following conditions:
 *
 * The above copyright notice and this permission notice shall be
 * included in all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
 * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
 * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
 * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 */
#include "http_parser.h"
#include <limits.h>
#ifndef NDEBUG
# include <assert.h>
#endif

static int unhex[] = {-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1
                     ,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1
                     ,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1
                     , 0, 1, 2, 3, 4, 5, 6, 7, 8, 9,-1,-1,-1,-1,-1,-1
                     ,-1,10,11,12,13,14,15,-1,-1,-1,-1,-1,-1,-1,-1,-1
                     ,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1
                     ,-1,10,11,12,13,14,15,-1,-1,-1,-1,-1,-1,-1,-1,-1
                     ,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1
                     };
#define TRUE 1
#define FALSE 0
#define MIN(a,b) (a < b ? a : b)
#define NULL (void*)(0)

#define MAX_FIELD_SIZE 80*1024

#define REMAINING (pe - p)
#define CALLBACK(FOR)                                                \
do {                                                                 \
  if (parser->FOR##_mark) {                                          \
    parser->FOR##_size += p - parser->FOR##_mark;                    \
    if (parser->FOR##_size > MAX_FIELD_SIZE) {                       \
      parser->buffer_overflow = TRUE;                                \
      return 0;                                                      \
    }                                                                \
    if (parser->on_##FOR) {                                          \
      callback_return_value = parser->on_##FOR(parser,               \
        parser->FOR##_mark,                                          \
        p - parser->FOR##_mark);                                     \
    }                                                                \
  }                                                                  \
} while(0)

#define RESET_PARSER(parser)                                         \
    parser->chunk_size = 0;                                          \
    parser->eating = 0;                                              \
    parser->header_field_mark = NULL;                                \
    parser->header_value_mark = NULL;                                \
    parser->query_string_mark = NULL;                                \
    parser->path_mark = NULL;                                        \
    parser->uri_mark = NULL;                                         \
    parser->fragment_mark = NULL;                                    \
    parser->status_code = 0;                                         \
    parser->method = 0;                                              \
    parser->transfer_encoding = HTTP_IDENTITY;                       \
    parser->version_major = 0;                                       \
    parser->version_minor = 0;                                       \
    parser->keep_alive = -1;                                         \
    parser->content_length = 0;                                      \
    parser->body_read = 0;

#define END_REQUEST                                                  \
do {                                                                 \
    if (parser->on_message_complete) {                               \
      callback_return_value =                                        \
        parser->on_message_complete(parser);                         \
    }                                                                \
    RESET_PARSER(parser);                                            \
} while (0)

#define SKIP_BODY(nskip)                                             \
do {                                                                 \
  tmp = (nskip);                                                     \
  if (parser->on_body && tmp > 0) {                                  \
    callback_return_value = parser->on_body(parser, p, tmp);         \
  }                                                                  \
  if (callback_return_value == 0) {                                  \
    p += tmp;                                                        \
    parser->body_read += tmp;                                        \
    parser->chunk_size -= tmp;                                       \
    if (0 == parser->chunk_size) {                                   \
      parser->eating = FALSE;                                        \
      if (parser->transfer_encoding == HTTP_IDENTITY) {              \
        END_REQUEST;                                                 \
      }                                                              \
    } else {                                                         \
      parser->eating = TRUE;                                         \
    }                                                                \
  }                                                                  \
} while (0)

%%{
  machine http_parser;

  action mark_header_field {
    parser->header_field_mark = p;
    parser->header_field_size = 0;
  }

  action mark_header_value {
    parser->header_value_mark = p;
    parser->header_value_size = 0;
  }

  action mark_fragment {
    parser->fragment_mark = p;
    parser->fragment_size = 0;
  }

  action mark_query_string {
    parser->query_string_mark = p;
    parser->query_string_size = 0;
  }

  action mark_request_path {
    parser->path_mark = p;
    parser->path_size = 0;
  }

  action mark_request_uri {
    parser->uri_mark = p;
    parser->uri_size = 0;
  }

  action header_field {
    CALLBACK(header_field);
    if (callback_return_value != 0) fbreak;
    parser->header_field_mark = NULL;
    parser->header_field_size = 0;
  }

  action header_value {
    CALLBACK(header_value);
    if (callback_return_value != 0) fbreak;
    parser->header_value_mark = NULL;
    parser->header_value_size = 0;
  }

  action request_uri {
    CALLBACK(uri);
    if (callback_return_value != 0) fbreak;
    parser->uri_mark = NULL;
    parser->uri_size = 0;
  }

  action fragment {
    CALLBACK(fragment);
    if (callback_return_value != 0) fbreak;
    parser->fragment_mark = NULL;
    parser->fragment_size = 0;
  }

  action query_string {
    CALLBACK(query_string);
    if (callback_return_value != 0) fbreak;
    parser->query_string_mark = NULL;
    parser->query_string_size = 0;
  }

  action request_path {
    CALLBACK(path);
    if (callback_return_value != 0) fbreak;
    parser->path_mark = NULL;
    parser->path_size = 0;
  }

  action headers_complete {
    if(parser->on_headers_complete) {
      callback_return_value = parser->on_headers_complete(parser);
      if (callback_return_value != 0) fbreak;
    }
  }

  action begin_message {
    if(parser->on_message_begin) {
      callback_return_value = parser->on_message_begin(parser);
      if (callback_return_value != 0) fbreak;
    }
  }

  action content_length {
    if (parser->content_length > INT_MAX) {
      parser->buffer_overflow = TRUE;
      return 0;
    }
    parser->content_length *= 10;
    parser->content_length += *p - '0';
  }

  action status_code {
    parser->status_code *= 10;
    parser->status_code += *p - '0';
  }

  action use_identity_encoding { parser->transfer_encoding = HTTP_IDENTITY; }
  action use_chunked_encoding  { parser->transfer_encoding = HTTP_CHUNKED;  }

  action set_keep_alive { parser->keep_alive = TRUE; }
  action set_not_keep_alive { parser->keep_alive = FALSE; }

  action version_major {
    parser->version_major *= 10;
    parser->version_major += *p - '0';
  }

  action version_minor {
    parser->version_minor *= 10;
    parser->version_minor += *p - '0';
  }

  action add_to_chunk_size {
    parser->chunk_size *= 16;
    parser->chunk_size += unhex[(int)*p];
  }

  action skip_chunk_data {
    SKIP_BODY(MIN(parser->chunk_size, REMAINING));
    if (callback_return_value != 0) fbreak;

    fhold;
    if (parser->chunk_size > REMAINING) {
      fbreak;
    } else {
      fgoto chunk_end;
    }
  }

  action end_chunked_body {
    END_REQUEST;
    if (parser->type == HTTP_REQUEST) {
      fnext Requests;
    } else {
      fnext Responses;
    }
  }

  action body_logic {
    if (parser->transfer_encoding == HTTP_CHUNKED) {
      fnext ChunkedBody;
    } else {
      /* this is pretty stupid. i'd prefer to combine this with skip_chunk_data */
      parser->chunk_size = parser->content_length;
      p += 1;

      SKIP_BODY(MIN(REMAINING, parser->content_length));
      if (callback_return_value != 0) fbreak;

      fhold;
      if(parser->chunk_size > REMAINING) {
        fbreak;
      }
    }
  }

  CRLF = "\r\n";

# character types
  CTL = (cntrl | 127);
  safe = ("$" | "-" | "_" | ".");
  extra = ("!" | "*" | "'" | "(" | ")" | ",");
  reserved = (";" | "/" | "?" | ":" | "@" | "&" | "=" | "+");
  unsafe = (CTL | " " | "\"" | "#" | "%" | "<" | ">");
  national = any -- (alpha | digit | reserved | extra | safe | unsafe);
  unreserved = (alpha | digit | safe | extra | national);
  escape = ("%" xdigit xdigit);
  uchar = (unreserved | escape);
  pchar = (uchar | ":" | "@" | "&" | "=" | "+");
  tspecials = ("(" | ")" | "<" | ">" | "@" | "," | ";" | ":" | "\\" | "\""
              | "/" | "[" | "]" | "?" | "=" | "{" | "}" | " " | "\t");

# elements
  token = (ascii -- (CTL | tspecials));
  quote = "\"";
#  qdtext = token -- "\"";
#  quoted_pair = "\" ascii;
#  quoted_string = "\"" (qdtext | quoted_pair )* "\"";

#  headers

  Method = ( "COPY"      %{ parser->method = HTTP_COPY;      }
           | "DELETE"    %{ parser->method = HTTP_DELETE;    }
           | "GET"       %{ parser->method = HTTP_GET;       }
           | "HEAD"      %{ parser->method = HTTP_HEAD;      }
           | "LOCK"      %{ parser->method = HTTP_LOCK;      }
           | "MKCOL"     %{ parser->method = HTTP_MKCOL;     }
           | "MOVE"      %{ parser->method = HTTP_MOVE;      }
           | "OPTIONS"   %{ parser->method = HTTP_OPTIONS;   }
           | "POST"      %{ parser->method = HTTP_POST;      }
           | "PROPFIND"  %{ parser->method = HTTP_PROPFIND;  }
           | "PROPPATCH" %{ parser->method = HTTP_PROPPATCH; }
           | "PUT"       %{ parser->method = HTTP_PUT;       }
           | "TRACE"     %{ parser->method = HTTP_TRACE;     }
           | "UNLOCK"    %{ parser->method = HTTP_UNLOCK;    }
           ); # Not allowing extension methods

  HTTP_Version = "HTTP/" digit $version_major "." digit $version_minor;

  scheme = ( alpha | digit | "+" | "-" | "." )* ;
  absolute_uri = (scheme ":" (uchar | reserved )*);
  path = ( pchar+ ( "/" pchar* )* ) ;
  query = ( uchar | reserved )* >mark_query_string %query_string ;
  param = ( pchar | "/" )* ;
  params = ( param ( ";" param )* ) ;
  rel_path = ( path? (";" params)? ) ;
  absolute_path = ( "/"+ rel_path ) >mark_request_path %request_path ("?" query)?;
  Request_URI = ( "*" | absolute_uri | absolute_path ) >mark_request_uri %request_uri;
  Fragment = ( uchar | reserved )* >mark_fragment %fragment;

  field_name = ( token -- ":" )+;
  Field_Name = field_name >mark_header_field %header_field;

  field_value = ((any - " ") any*)?;
  Field_Value = field_value >mark_header_value %header_value;

  hsep = ":" " "*;
  header = (field_name hsep field_value) :> CRLF;
  Header = ( ("Content-Length"i hsep digit+ $content_length)
           | ("Connection"i hsep
               ( "Keep-Alive"i %set_keep_alive
               | "close"i %set_not_keep_alive
               )
             )
           | ("Transfer-Encoding"i %use_chunked_encoding hsep "identity" %use_identity_encoding)
           | (Field_Name hsep Field_Value)
           ) :> CRLF;

  Headers = (Header)* :> CRLF @headers_complete;

  Request_Line = ( Method " " Request_URI ("#" Fragment)? " " HTTP_Version CRLF ) ;

  StatusCode = (digit digit digit) $status_code;
  ReasonPhrase =  ascii+ -- ("\r" | "\n");
  StatusLine = HTTP_Version  " " StatusCode " " ReasonPhrase CRLF;

# chunked message
  trailing_headers = header*;
  #chunk_ext_val   = token | quoted_string;
  chunk_ext_val = token*;
  chunk_ext_name = token*;
  chunk_extension = ( ";" " "* chunk_ext_name ("=" chunk_ext_val)? )*;
  last_chunk = "0"+ chunk_extension CRLF;
  chunk_size = (xdigit* [1-9a-fA-F] xdigit*) $add_to_chunk_size;
  chunk_end  = CRLF;
  chunk_body = any >skip_chunk_data;
  chunk_begin = chunk_size chunk_extension CRLF;
  chunk = chunk_begin chunk_body chunk_end;
  ChunkedBody := chunk* last_chunk trailing_headers CRLF @end_chunked_body;

  Request = (Request_Line Headers) >begin_message @body_logic;
  Response = (StatusLine Headers) >begin_message @body_logic;

  Requests := Request*;
  Responses := Response*;

  main := any >{
    fhold;
    if (parser->type == HTTP_REQUEST) {
      fgoto Requests;
    } else {
      fgoto Responses;
    }
  };

}%%

%% write data;

void
http_parser_init (http_parser *parser, enum http_parser_type type)
{
  int cs = 0;
  %% write init;
  parser->cs = cs;
  parser->type = type;
  parser->buffer_overflow = 0;

  parser->on_message_begin = NULL;
  parser->on_path = NULL;
  parser->on_query_string = NULL;
  parser->on_uri = NULL;
  parser->on_header_field = NULL;
  parser->on_header_value = NULL;
  parser->on_headers_complete = NULL;
  parser->on_body = NULL;
  parser->on_message_complete = NULL;

  RESET_PARSER(parser);
}

/** exec **/
size_t
http_parser_execute (http_parser *parser, const char *buffer, size_t len)
{
  size_t tmp; // REMOVE ME this is extremely hacky
  int callback_return_value = 0;
  const char *p, *pe;
  int cs = parser->cs;

  p = buffer;
  pe = buffer+len;

  if (0 < parser->chunk_size && parser->eating) {
    /* eat body */
    SKIP_BODY(MIN(len, parser->chunk_size));
    if (callback_return_value != 0) goto out;
  }

  if (parser->header_field_mark)   parser->header_field_mark   = buffer;
  if (parser->header_value_mark)   parser->header_value_mark   = buffer;
  if (parser->fragment_mark)       parser->fragment_mark       = buffer;
  if (parser->query_string_mark)   parser->query_string_mark   = buffer;
  if (parser->path_mark)           parser->path_mark           = buffer;
  if (parser->uri_mark)            parser->uri_mark            = buffer;

  %% write exec;

  parser->cs = cs;

  CALLBACK(header_field);
  CALLBACK(header_value);
  CALLBACK(fragment);
  CALLBACK(query_string);
  CALLBACK(path);
  CALLBACK(uri);

out:
  assert(p <= pe && "buffer overflow after parsing execute");
  return(p - buffer);
}

int
http_parser_has_error (http_parser *parser)
{
  if (parser->buffer_overflow) return TRUE;
  return parser->cs == http_parser_error;
}

int
http_parser_should_keep_alive (http_parser *parser)
{
  if (parser->keep_alive == -1)
    if (parser->version_major == 1)
      return (parser->version_minor != 0);
    else if (parser->version_major == 0)
      return FALSE;
    else
      return TRUE;
  else
    return parser->keep_alive;
}

/* Copyright (c) 2008, 2009 Ryan Dahl (ry@tinyclouds.org)
 * Based on Zed Shaw's Mongrel, copyright (c) Zed A. Shaw
 *
 * All rights reserved.
 *
 * Permission is hereby granted, free of charge, to any person obtaining
 * a copy of this software and associated documentation files (the
 * "Software"), to deal in the Software without restriction, including
 * without limitation the rights to use, copy, modify, merge, publish,
 * distribute, sublicense, and/or sell copies of the Software, and to
 * permit persons to whom the Software is furnished to do so, subject to
 * the following conditions:
 *
 * The above copyright notice and this permission notice shall be
 * included in all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
 * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
 * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
 * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 */
#include "http_parser.h"
#ifndef NDEBUG
# include <assert.h>
#endif

static int unhex[] = {-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1
                     ,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1
                     ,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1
                     , 0, 1, 2, 3, 4, 5, 6, 7, 8, 9,-1,-1,-1,-1,-1,-1
                     ,-1,10,11,12,13,14,15,-1,-1,-1,-1,-1,-1,-1,-1,-1
                     ,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1
                     ,-1,10,11,12,13,14,15,-1,-1,-1,-1,-1,-1,-1,-1,-1
                     ,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1
                     };
#define TRUE 1
#define FALSE 0
#define MIN(a,b) (a < b ? a : b)
#define NULL (void*)(0)

#define MAX_FIELD_SIZE 80*1024

#define REMAINING (pe - p)
#define CALLBACK(FOR)                                                \
do {                                                                 \
  if (parser->FOR##_mark) {                                          \
    parser->FOR##_size += p - parser->FOR##_mark;                    \
    if (parser->FOR##_size > MAX_FIELD_SIZE) {                       \
      parser->buffer_overflow = TRUE;                                \
      return 0;                                                      \
    }                                                                \
    if (parser->on_##FOR) {                                          \
      callback_return_value = parser->on_##FOR(parser,               \
        parser->FOR##_mark,                                          \
        p - parser->FOR##_mark);                                     \
    }                                                                \
  }                                                                  \
} while(0)

#define RESET_PARSER(parser)                                         \
    parser->chunk_size = 0;                                          \
    parser->eating = 0;                                              \
    parser->header_field_mark = NULL;                                \
    parser->header_value_mark = NULL;                                \
    parser->query_string_mark = NULL;                                \
    parser->path_mark = NULL;                                        \
    parser->uri_mark = NULL;                                         \
    parser->fragment_mark = NULL;                                    \
    parser->status_code = 0;                                         \
    parser->method = 0;                                              \
    parser->transfer_encoding = HTTP_IDENTITY;                       \
    parser->version_major = 0;                                       \
    parser->version_minor = 0;                                       \
    parser->keep_alive = -1;                                         \
    parser->content_length = 0;                                      \
    parser->body_read = 0;

#define END_REQUEST                                                  \
do {                                                                 \
    if (parser->on_message_complete) {                               \
      callback_return_value =                                        \
        parser->on_message_complete(parser);                         \
    }                                                                \
    RESET_PARSER(parser);                                            \
} while (0)

#define SKIP_BODY(nskip)                                             \
do {                                                                 \
  tmp = (nskip);                                                     \
  if (parser->on_body && tmp > 0) {                                  \
    callback_return_value = parser->on_body(parser, p, tmp);         \
  }                                                                  \
  if (callback_return_value == 0) {                                  \
    p += tmp;                                                        \
    parser->body_read += tmp;                                        \
    parser->chunk_size -= tmp;                                       \
    if (0 == parser->chunk_size) {                                   \
      parser->eating = FALSE;                                        \
      if (parser->transfer_encoding == HTTP_IDENTITY) {              \
        END_REQUEST;                                                 \
      }                                                              \
    } else {                                                         \
      parser->eating = TRUE;                                         \
    }                                                                \
  }                                                                  \
} while (0)

%%{
  machine http_parser;

  action mark_header_field {
    parser->header_field_mark = p;
    parser->header_field_size = 0;
  }

  action mark_header_value {
    parser->header_value_mark = p;
    parser->header_value_size = 0;
  }

  action mark_fragment {
    parser->fragment_mark = p;
    parser->fragment_size = 0;
  }

  action mark_query_string {
    parser->query_string_mark = p;
    parser->query_string_size = 0;
  }

  action mark_request_path {
    parser->path_mark = p;
    parser->path_size = 0;
  }

  action mark_request_uri {
    parser->uri_mark = p;
    parser->uri_size = 0;
  }

  action header_field {
    CALLBACK(header_field);
    if (callback_return_value != 0) fbreak;
    parser->header_field_mark = NULL;
    parser->header_field_size = 0;
  }

  action header_value {
    CALLBACK(header_value);
    if (callback_return_value != 0) fbreak;
    parser->header_value_mark = NULL;
    parser->header_value_size = 0;
  }

  action request_uri {
    CALLBACK(uri);
    if (callback_return_value != 0) fbreak;
    parser->uri_mark = NULL;
    parser->uri_size = 0;
  }

  action fragment {
    CALLBACK(fragment);
    if (callback_return_value != 0) fbreak;
    parser->fragment_mark = NULL;
    parser->fragment_size = 0;
  }

  action query_string {
    CALLBACK(query_string);
    if (callback_return_value != 0) fbreak;
    parser->query_string_mark = NULL;
    parser->query_string_size = 0;
  }

  action request_path {
    CALLBACK(path);
    if (callback_return_value != 0) fbreak;
    parser->path_mark = NULL;
    parser->path_size = 0;
  }

  action headers_complete {
    if(parser->on_headers_complete) {
      callback_return_value = parser->on_headers_complete(parser);
      if (callback_return_value != 0) fbreak;
    }
  }

  action begin_message {
    if(parser->on_message_begin) {
      callback_return_value = parser->on_message_begin(parser);
      if (callback_return_value != 0) fbreak;
    }
  }

  action content_length {
    parser->content_length *= 10;
    parser->content_length += *p - '0';
  }

  action status_code {
    parser->status_code *= 10;
    parser->status_code += *p - '0';
  }

  action use_identity_encoding { parser->transfer_encoding = HTTP_IDENTITY; }
  action use_chunked_encoding  { parser->transfer_encoding = HTTP_CHUNKED;  }

  action set_keep_alive { parser->keep_alive = TRUE; }
  action set_not_keep_alive { parser->keep_alive = FALSE; }

  action version_major {
    parser->version_major *= 10;
    parser->version_major += *p - '0';
  }

  action version_minor {
    parser->version_minor *= 10;
    parser->version_minor += *p - '0';
  }

  action add_to_chunk_size {
    parser->chunk_size *= 16;
    parser->chunk_size += unhex[(int)*p];
  }

  action skip_chunk_data {
    SKIP_BODY(MIN(parser->chunk_size, REMAINING));
    if (callback_return_value != 0) fbreak;

    fhold;
    if (parser->chunk_size > REMAINING) {
      fbreak;
    } else {
      fgoto chunk_end;
    }
  }

  action end_chunked_body {
    END_REQUEST;
    if (parser->type == HTTP_REQUEST) {
      fnext Requests;
    } else {
      fnext Responses;
    }
  }

  action body_logic {
    if (parser->transfer_encoding == HTTP_CHUNKED) {
      fnext ChunkedBody;
    } else {
      /* this is pretty stupid. i'd prefer to combine this with skip_chunk_data */
      parser->chunk_size = parser->content_length;
      p += 1;

      SKIP_BODY(MIN(REMAINING, parser->content_length));
      if (callback_return_value != 0) fbreak;

      fhold;
      if(parser->chunk_size > REMAINING) {
        fbreak;
      }
    }
  }

  CRLF = "\r\n";

# character types
  CTL = (cntrl | 127);
  safe = ("$" | "-" | "_" | ".");
  extra = ("!" | "*" | "'" | "(" | ")" | ",");
  reserved = (";" | "/" | "?" | ":" | "@" | "&" | "=" | "+");
  unsafe = (CTL | " " | "\"" | "#" | "%" | "<" | ">");
  national = any -- (alpha | digit | reserved | extra | safe | unsafe);
  unreserved = (alpha | digit | safe | extra | national);
  escape = ("%" xdigit xdigit);
  uchar = (unreserved | escape);
  pchar = (uchar | ":" | "@" | "&" | "=" | "+");
  tspecials = ("(" | ")" | "<" | ">" | "@" | "," | ";" | ":" | "\\" | "\""
              | "/" | "[" | "]" | "?" | "=" | "{" | "}" | " " | "\t");

# elements
  token = (ascii -- (CTL | tspecials));
  quote = "\"";
#  qdtext = token -- "\"";
#  quoted_pair = "\" ascii;
#  quoted_string = "\"" (qdtext | quoted_pair )* "\"";

#  headers

  Method = ( "COPY"      %{ parser->method = HTTP_COPY;      }
           | "DELETE"    %{ parser->method = HTTP_DELETE;    }
           | "GET"       %{ parser->method = HTTP_GET;       }
           | "HEAD"      %{ parser->method = HTTP_HEAD;      }
           | "LOCK"      %{ parser->method = HTTP_LOCK;      }
           | "MKCOL"     %{ parser->method = HTTP_MKCOL;     }
           | "MOVE"      %{ parser->method = HTTP_MOVE;      }
           | "OPTIONS"   %{ parser->method = HTTP_OPTIONS;   }
           | "POST"      %{ parser->method = HTTP_POST;      }
           | "PROPFIND"  %{ parser->method = HTTP_PROPFIND;  }
           | "PROPPATCH" %{ parser->method = HTTP_PROPPATCH; }
           | "PUT"       %{ parser->method = HTTP_PUT;       }
           | "TRACE"     %{ parser->method = HTTP_TRACE;     }
           | "UNLOCK"    %{ parser->method = HTTP_UNLOCK;    }
           ); # Not allowing extension methods

  HTTP_Version = "HTTP/" digit+ $version_major "." digit+ $version_minor;

  scheme = ( alpha | digit | "+" | "-" | "." )* ;
  absolute_uri = (scheme ":" (uchar | reserved )*);
  path = ( pchar+ ( "/" pchar* )* ) ;
  query = ( uchar | reserved )* >mark_query_string %query_string ;
  param = ( pchar | "/" )* ;
  params = ( param ( ";" param )* ) ;
  rel_path = ( path? (";" params)? ) ;
  absolute_path = ( "/"+ rel_path ) >mark_request_path %request_path ("?" query)?;
  Request_URI = ( "*" | absolute_uri | absolute_path ) >mark_request_uri %request_uri;
  Fragment = ( uchar | reserved )* >mark_fragment %fragment;

  field_name = ( token -- ":" )+;
  Field_Name = field_name >mark_header_field %header_field;

  field_value = ((any - " ") any*)?;
  Field_Value = field_value >mark_header_value %header_value;

  hsep = ":" " "*;
  header = (field_name hsep field_value) :> CRLF;
  Header = ( ("Content-Length"i hsep digit+ $content_length)
           | ("Connection"i hsep
               ( "Keep-Alive"i %set_keep_alive
               | "close"i %set_not_keep_alive
               )
             )
           | ("Transfer-Encoding"i %use_chunked_encoding hsep "identity" %use_identity_encoding)
           | (Field_Name hsep Field_Value)
           ) :> CRLF;

  Headers = (Header)* :> CRLF @headers_complete;

  Request_Line = ( Method " " Request_URI ("#" Fragment)? " " HTTP_Version CRLF ) ;

  StatusCode = (digit digit digit) $status_code;
  ReasonPhrase =  ascii+ -- ("\r" | "\n");
  StatusLine = HTTP_Version  " " StatusCode " " ReasonPhrase CRLF;

# chunked message
  trailing_headers = header*;
  #chunk_ext_val   = token | quoted_string;
  chunk_ext_val = token*;
  chunk_ext_name = token*;
  chunk_extension = ( ";" " "* chunk_ext_name ("=" chunk_ext_val)? )*;
  last_chunk = "0"+ chunk_extension CRLF;
  chunk_size = (xdigit* [1-9a-fA-F] xdigit*) $add_to_chunk_size;
  chunk_end  = CRLF;
  chunk_body = any >skip_chunk_data;
  chunk_begin = chunk_size chunk_extension CRLF;
  chunk = chunk_begin chunk_body chunk_end;
  ChunkedBody := chunk* last_chunk trailing_headers CRLF @end_chunked_body;

  Request = (Request_Line Headers) >begin_message @body_logic;
  Response = (StatusLine Headers) >begin_message @body_logic;

  Requests := Request*;
  Responses := Response*;

  main := any >{
    fhold;
    if (parser->type == HTTP_REQUEST) {
      fgoto Requests;
    } else {
      fgoto Responses;
    }
  };

}%%

%% write data;

void
http_parser_init (http_parser *parser, enum http_parser_type type)
{
  int cs = 0;
  %% write init;
  parser->cs = cs;
  parser->type = type;
  parser->buffer_overflow = 0;

  parser->on_message_begin = NULL;
  parser->on_path = NULL;
  parser->on_query_string = NULL;
  parser->on_uri = NULL;
  parser->on_header_field = NULL;
  parser->on_header_value = NULL;
  parser->on_headers_complete = NULL;
  parser->on_body = NULL;
  parser->on_message_complete = NULL;

  RESET_PARSER(parser);
}

/** exec **/
size_t
http_parser_execute (http_parser *parser, const char *buffer, size_t len)
{
  size_t tmp; // REMOVE ME this is extremely hacky
  int callback_return_value = 0;
  const char *p, *pe;
  int cs = parser->cs;

  p = buffer;
  pe = buffer+len;

  if (0 < parser->chunk_size && parser->eating) {
    /* eat body */
    SKIP_BODY(MIN(len, parser->chunk_size));
    if (callback_return_value != 0) goto out;
  }

  if (parser->header_field_mark)   parser->header_field_mark   = buffer;
  if (parser->header_value_mark)   parser->header_value_mark   = buffer;
  if (parser->fragment_mark)       parser->fragment_mark       = buffer;
  if (parser->query_string_mark)   parser->query_string_mark   = buffer;
  if (parser->path_mark)           parser->path_mark           = buffer;
  if (parser->uri_mark)            parser->uri_mark            = buffer;

  %% write exec;

  parser->cs = cs;

  CALLBACK(header_field);
  CALLBACK(header_value);
  CALLBACK(fragment);
  CALLBACK(query_string);
  CALLBACK(path);
  CALLBACK(uri);

out:
  assert(p <= pe && "buffer overflow after parsing execute");
  return(p - buffer);
}

int
http_parser_has_error (http_parser *parser)
{
  if (parser->buffer_overflow) return TRUE;
  return parser->cs == http_parser_error;
}

int
http_parser_should_keep_alive (http_parser *parser)
{
  if (parser->keep_alive == -1)
    if (parser->version_major == 1)
      return (parser->version_minor != 0);
    else if (parser->version_major == 0)
      return FALSE;
    else
      return TRUE;
  else
    return parser->keep_alive;
}


Add header parsing to JavaParser

Add header parsing to JavaParser

package stompede.stomp;

import org.jruby.Ruby;
import org.jruby.RubyModule;
import org.jruby.RubyClass;
import org.jruby.RubyObject;
import org.jruby.RubyFixnum;
import org.jruby.RubyString;
import org.jruby.exceptions.RaiseException;

import org.jruby.runtime.ThreadContext;
import org.jruby.runtime.builtin.IRubyObject;
import org.jruby.runtime.Block;

import org.jruby.anno.JRubyClass;
import org.jruby.anno.JRubyMethod;

%%{
  machine message;

  action mark {
    mark = p;
  }

  action mark_message {
    mark_message = context.runtime.getClassFromPath("Stompede::Stomp::Message").callMethod("new", context.nil, context.nil);
    /*mark_message_size = 0;*/
  }

  action write_command {
    mark_message.callMethod(context, "write_command", RubyString.newString(context.runtime, data, mark, p - mark));
    mark = -1;
  }

  action mark_key {
    mark_key = RubyString.newString(context.runtime, data, mark, p - mark);
    mark = -1;
  }

  action write_header {
    IRubyObject args[] = { mark_key, RubyString.newString(context.runtime, data, mark, p - mark) };
    mark_message.callMethod(context, "write_header", args);
    mark_key = null;
    mark = -1;
  }

  action finish_headers {
    /*VALUE length = rb_funcall(mark_message, g_content_length, 0);*/
    /*if ( ! NIL_P(length)) {*/
    /*  mark_content_length = NUM2LONG(length);*/
    /*} else {*/
    /*  mark_content_length = -1;*/
    /*}*/
  }

  action write_body {
    /*rb_funcall(mark_message, g_write_body, 1, MARK_STR_NEW());*/
    /*mark = NULL;*/
  }

  action consume_null {
    false
    /*((mark_content_length != -1) && (MARK_LEN < mark_content_length))*/
  }

  action consume_octet {
    true
    /*((mark_content_length == -1) || (MARK_LEN < mark_content_length))*/
  }

  action check_message_size {
    /*mark_message_size += 1;*/
    /*if (mark_message_size > max_message_size) {*/
    /*  rb_raise(eMessageSizeExceeded, "");*/
    /*}*/
  }

  action finish_message {
    block.yield(context, mark_message);
    mark_message = null;
  }

  include message_common "parser_common.rl";
}%%

@JRubyClass(name="JavaParser", parent="Object")
public class JavaParser extends RubyObject {
  %% write data noprefix;

  private class State {
    public int cs = JavaParser.start;
    public byte[] chunk;
    public int mark = -1;
    public RubyString mark_key;
    public IRubyObject mark_message;
    public int mark_message_size = -1;
    public int mark_content_length = -1;
  }

  private RaiseException exception;
  private long maxMessageSize;
  private State state;

  public JavaParser(Ruby runtime, RubyClass klass) {
    super(runtime, klass);
    state = new State();
  }

  @JRubyMethod
  public IRubyObject initialize(ThreadContext context) {
    RubyModule mStomp = context.runtime.getClassFromPath("Stompede::Stomp");
    return initialize(context, mStomp.callMethod("max_message_size"));
  }

  @JRubyMethod(argTypes = {RubyFixnum.class})
  public IRubyObject initialize(ThreadContext context, IRubyObject maxMessageSize) {
    this.maxMessageSize = ((RubyFixnum) maxMessageSize).getLongValue();
    return context.nil;
  }

  @JRubyMethod(argTypes = {RubyString.class})
  public IRubyObject parse(ThreadContext context, IRubyObject chunk, Block block) {
    byte data[] = ((RubyString) chunk).getBytes();

    int p = 0;
    int pe = data.length;

    int cs = state.cs;
    int mark = state.mark;
    RubyString mark_key = state.mark_key;
    IRubyObject mark_message = state.mark_message;
    int mark_message_size = state.mark_message_size;
    int mark_content_length = state.mark_content_length;

    %% write exec;

    state.cs = cs;
    state.mark = mark;
    state.mark_key = mark_key;
    state.mark_message = mark_message;
    state.mark_message_size = mark_message_size;
    state.mark_content_length = mark_content_length;

    if (cs == error) {
      // RaiseException error = context.runtime.getClassFromPath("Stompede::Stomp").callMethod("build_parse_error");
      // throw error;
    }

    return context.nil;
  }
}

package stompede.stomp;

import org.jruby.Ruby;
import org.jruby.RubyModule;
import org.jruby.RubyClass;
import org.jruby.RubyObject;
import org.jruby.RubyFixnum;
import org.jruby.RubyString;
import org.jruby.exceptions.RaiseException;

import org.jruby.runtime.ThreadContext;
import org.jruby.runtime.builtin.IRubyObject;
import org.jruby.runtime.Block;

import org.jruby.anno.JRubyClass;
import org.jruby.anno.JRubyMethod;

%%{
  machine message;

  action mark {
    mark = p;
  }

  action mark_message {
    mark_message = context.runtime.getClassFromPath("Stompede::Stomp::Message").callMethod("new", context.nil, context.nil);
    /*mark_message_size = 0;*/
  }

  action write_command {
    mark_message.callMethod(context, "write_command", RubyString.newString(context.runtime, data, mark, p - mark));
    mark = -1;
  }

  action mark_key {
    /*mark_key = MARK_STR_NEW();*/
    /*mark = NULL;*/
  }

  action write_header {
    /*rb_funcall(mark_message, g_write_header, 2, mark_key, MARK_STR_NEW());*/
    /*mark_key = Qnil;*/
    /*mark = NULL;*/
  }

  action finish_headers {
    /*VALUE length = rb_funcall(mark_message, g_content_length, 0);*/
    /*if ( ! NIL_P(length)) {*/
    /*  mark_content_length = NUM2LONG(length);*/
    /*} else {*/
    /*  mark_content_length = -1;*/
    /*}*/
  }

  action write_body {
    /*rb_funcall(mark_message, g_write_body, 1, MARK_STR_NEW());*/
    /*mark = NULL;*/
  }

  action consume_null {
    false
    /*((mark_content_length != -1) && (MARK_LEN < mark_content_length))*/
  }

  action consume_octet {
    true
    /*((mark_content_length == -1) || (MARK_LEN < mark_content_length))*/
  }

  action check_message_size {
    /*mark_message_size += 1;*/
    /*if (mark_message_size > max_message_size) {*/
    /*  rb_raise(eMessageSizeExceeded, "");*/
    /*}*/
  }

  action finish_message {
    block.yield(context, mark_message);
    mark_message = null;
  }

  include message_common "parser_common.rl";
}%%

@JRubyClass(name="JavaParser", parent="Object")
public class JavaParser extends RubyObject {
  %% write data noprefix;

  private class State {
    public int cs = JavaParser.start;
    public byte[] chunk;
    public int mark = -1;
    public RubyString mark_key;
    public IRubyObject mark_message;
    public int mark_message_size = -1;
    public int mark_content_length = -1;
  }

  private RaiseException exception;
  private long maxMessageSize;
  private State state;

  public JavaParser(Ruby runtime, RubyClass klass) {
    super(runtime, klass);
    state = new State();
  }

  @JRubyMethod
  public IRubyObject initialize(ThreadContext context) {
    RubyModule mStomp = context.runtime.getClassFromPath("Stompede::Stomp");
    return initialize(context, mStomp.callMethod("max_message_size"));
  }

  @JRubyMethod(argTypes = {RubyFixnum.class})
  public IRubyObject initialize(ThreadContext context, IRubyObject maxMessageSize) {
    this.maxMessageSize = ((RubyFixnum) maxMessageSize).getLongValue();
    return context.nil;
  }

  @JRubyMethod(argTypes = {RubyString.class})
  public IRubyObject parse(ThreadContext context, IRubyObject chunk, Block block) {
    byte data[] = ((RubyString) chunk).getBytes();

    int p = 0;
    int pe = data.length;

    int cs = state.cs;
    int mark = state.mark;
    RubyString mark_key = state.mark_key;
    IRubyObject mark_message = state.mark_message;
    int mark_message_size = state.mark_message_size;
    int mark_content_length = state.mark_content_length;

    %% write exec;

    state.cs = cs;
    state.mark = mark;
    state.mark_key = mark_key;
    state.mark_message = mark_message;
    state.mark_message_size = mark_message_size;
    state.mark_content_length = mark_content_length;

    if (cs == error) {
      // RaiseException error = context.runtime.getClassFromPath("Stompede::Stomp").callMethod("build_parse_error");
      // throw error;
    }

    return context.nil;
  }
}


minor code cleanup

minor code cleanup

Allocate scanners for included files on the heap. Consistent with the main
line.

/*
 *  Copyright 2006-2012 Adrian Thurston <thurston@complang.org>
 */

/*  This file is part of Colm.
 *
 *  Colm is free software; you can redistribute it and/or modify
 *  it under the terms of the GNU General Public License as published by
 *  the Free Software Foundation; either version 2 of the License, or
 *  (at your option) any later version.
 *
 *  Colm is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *  GNU General Public License for more details.
 *
 *  You should have received a copy of the GNU General Public License
 *  along with Colm; if not, write to the Free Software
 *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 */

#include <iostream>
#include <fstream>
#include <string.h>

#include "global.h"
#include "lmscan.h"
#include "lmparse.h"
#include "parsedata.h"
#include "avltree.h"
#include "vector.h"

//#define PRINT_TOKENS

using std::ifstream;
using std::istream;
using std::ostream;
using std::cout;
using std::cerr;
using std::endl;

%%{
	machine section_parse;
	alphtype int;
	write data;
}%%

void ColmScanner::sectionParseInit()
{
	%% write init;
}

ostream &ColmScanner::scan_error()
{
	/* Maintain the error count. */
	gblErrorCount += 1;
	cerr << fileName << ":" << line << ":" << column << ": ";
	return cerr;
}

bool ColmScanner::recursiveInclude( const char *inclFileName )
{
	for ( IncludeStack::Iter si = includeStack; si.lte(); si++ ) {
		if ( strcmp( si->fileName, inclFileName ) == 0 )
			return true;
	}
	return false;
}

void ColmScanner::updateCol()
{
	char *from = lastnl;
	if ( from == 0 )
		from = ts;
	//cerr << "adding " << te - from << " to column" << endl;
	column += te - from;
	lastnl = 0;
}

void ColmScanner::token( int type, char c )
{
	token( type, &c, &c + 1 );
}

void ColmScanner::token( int type )
{
	token( type, 0, 0 );
}

bool isAbsolutePath( const char *path )
{
	return path[0] == '/';
}

ifstream *ColmScanner::tryOpenInclude( char **pathChecks, long &found )
{
	char **check = pathChecks;
	ifstream *inFile = new ifstream;

	while ( *check != 0 ) {
		inFile->open( *check );
		if ( inFile->is_open() ) {
			found = check - pathChecks;
			return inFile;
		}
		check += 1;
	}

	found = -1;
	delete inFile;
	return 0;
}

char **ColmScanner::makeIncludePathChecks( const char *thisFileName, const char *fileName )
{
	char **checks = 0;
	long nextCheck = 0;
	char *data = strdup(fileName);
	long length = strlen(fileName);

	/* Absolute path? */
	if ( isAbsolutePath( data ) ) {
		checks = new char*[2];
		checks[nextCheck++] = data;
	}
	else {
		/* Search from the the location of the current file. */
		checks = new char *[2 + includePaths.length()];
		const char *lastSlash = strrchr( thisFileName, '/' );
		if ( lastSlash == 0 )
			checks[nextCheck++] = data;
		else {
			long givenPathLen = (lastSlash - thisFileName) + 1;
			long checklen = givenPathLen + length;
			char *check = new char[checklen+1];
			memcpy( check, thisFileName, givenPathLen );
			memcpy( check+givenPathLen, data, length );
			check[checklen] = 0;
			checks[nextCheck++] = check;
		}

		/* Search from the include paths given on the command line. */
		for ( ArgsVector::Iter incp = includePaths; incp.lte(); incp++ ) {
			long pathLen = strlen( *incp );
			long checkLen = pathLen + 1 + length;
			char *check = new char[checkLen+1];
			memcpy( check, *incp, pathLen );
			check[pathLen] = '/';
			memcpy( check+pathLen+1, data, length );
			check[checkLen] = 0;
			checks[nextCheck++] = check;
		}
	}

	checks[nextCheck] = 0;
	return checks;
}

%%{
	machine section_parse;
	import "lmparse.h";

	action clear_words { word = lit = 0; word_len = lit_len = 0; }
	action store_lit { lit = tokdata; lit_len = toklen; }

	action mach_err { scan_error() << "bad machine statement" << endl; }
	action incl_err { scan_error() << "bad include statement" << endl; }
	action write_err { scan_error() << "bad write statement" << endl; }

	action handle_include
	{
		String src( lit, lit_len );
		String fileName;
		bool unused;

		/* Need a location. */
		InputLoc here;
		here.fileName = fileName;
		here.line = line;
		here.col = column;

		prepareLitString( fileName, unused, src, here );
		char **checks = makeIncludePathChecks( this->fileName, fileName );

		/* Open the input file for reading. */
		long found = 0;
		ifstream *inFile = tryOpenInclude( checks, found );
		if ( inFile == 0 ) {
			scan_error() << "include: could not open " <<
					fileName << " for reading" << endl;
		}
		else {
			/* Only proceed with the include if it was found. */
			if ( recursiveInclude( checks[found] ) )
				scan_error() << "include: this is a recursive include operation" << endl;

			/* Check for a recursive include structure. Add the current file/section
			 * name then check if what we are including is already in the stack. */
			includeStack.append( IncludeStackItem( checks[found] ) );

			ColmScanner *scanner = new ColmScanner( fileName, *inFile, output, parser, includeDepth+1 );
			scanner->scan();
			delete inFile;

			/* Remove the last element (len-1) */
			includeStack.remove( -1 );

			delete scanner;
		}
	}

	include_target =
		TK_Literal >clear_words @store_lit;

	include_stmt =
		( KW_Include include_target ) @handle_include
		<>err incl_err <>eof incl_err;

	action handle_token
	{
//	cout << Parser_lelNames[type] << " ";
//	if ( start != 0 ) {
//		cout.write( start, end-start );
//	}
//	cout << endl;

		InputLoc loc;

		#ifdef PRINT_TOKENS
		cerr << "scanner:" << line << ":" << column <<
				": sending token to the parser " << Parser_lelNames[*p];
		cerr << " " << toklen;
		if ( tokdata != 0 )
			cerr << " " << tokdata;
		cerr << endl;
		#endif

		loc.fileName = fileName;
		loc.line = line;
		loc.col = column;

		if ( tokdata != 0 && tokdata[toklen-1] == '\n' )
			loc.line -= 1;

		parser->token( loc, type, tokdata, toklen );
	}

	# Catch everything else.
	everything_else = ^( KW_Include ) @handle_token;

	main := (
		include_stmt |
		everything_else
	)*;
}%%

void ColmScanner::token( int type, char *start, char *end )
{
	char *tokdata = 0;
	int toklen = 0;
	int *p = &type;
	int *pe = &type + 1;
	int *eof = 0;

	if ( start != 0 ) {
		toklen = end-start;
		tokdata = new char[toklen+1];
		memcpy( tokdata, start, toklen );
		tokdata[toklen] = 0;
	}

	%%{
		machine section_parse;
		write exec;
	}%%

	updateCol();
}

void ColmScanner::endSection( )
{
	/* Execute the eof actions for the section parser. */
	/* Probably use: token( -1 ); */
}

%%{
	machine rlscan;

	# This is sent by the driver code.
	EOF = 0;

	action inc_nl {
		lastnl = p;
		column = 0;
		line++;
	}
	NL = '\n' @inc_nl;

	# Identifiers, numbers, commetns, and other common things.
	ident = ( alpha | '_' ) ( alpha |digit |'_' )*;
	number = digit+;
	hex_number = '0x' [0-9a-fA-F]+;

	# These literal forms are common to C-like host code and ragel.
	s_literal = "'" ([^'\\] | NL | '\\' (any | NL))* "'";
	d_literal = '"' ([^"\\] | NL | '\\' (any | NL))* '"';

	whitespace = [ \t] | NL;
	pound_comment = '#' [^\n]* NL;

	or_literal := |*
		# Escape sequences in OR expressions.
		'\\0' => { token( TK_ReChar, '\0' ); };
		'\\a' => { token( TK_ReChar, '\a' ); };
		'\\b' => { token( TK_ReChar, '\b' ); };
		'\\t' => { token( TK_ReChar, '\t' ); };
		'\\n' => { token( TK_ReChar, '\n' ); };
		'\\v' => { token( TK_ReChar, '\v' ); };
		'\\f' => { token( TK_ReChar, '\f' ); };
		'\\r' => { token( TK_ReChar, '\r' ); };
		'\\\n' => { updateCol(); };
		'\\' any => { token( TK_ReChar, ts+1, te ); };

		# Range dash in an OR expression.
		'-' => { token( TK_Dash, 0, 0 ); };

		# Terminate an OR expression.
		']'	=> { token( TK_SqClose ); fret; };

		EOF => {
			scan_error() << "unterminated OR literal" << endl;
		};

		# Characters in an OR expression.
		[^\]] => { token( TK_ReChar, ts, te ); };

	*|;

	regular_type := |*
		# Identifiers.
		ident => { token( TK_Word, ts, te ); } ;

		# Numbers
		number => { token( TK_UInt, ts, te ); };
		hex_number => { token( TK_Hex, ts, te ); };

		# Literals, with optionals.
		( s_literal | d_literal ) [i]?
			=> { token( TK_Literal, ts, te ); };

		'[' => { token( TK_SqOpen ); fcall or_literal; };
		'[^' => { token( TK_SqOpenNeg ); fcall or_literal; };

		'/' => { token( '/'); fret; };

		# Ignore.
		pound_comment => { updateCol(); };

		'..' => { token( TK_DotDot ); };
		'**' => { token( TK_StarStar ); };
		'--' => { token( TK_DashDash ); };

		':>'  => { token( TK_ColonGt ); };
		':>>' => { token( TK_ColonGtGt ); };
		'<:'  => { token( TK_LtColon ); };

		# Whitespace other than newline.
		[ \t\r]+ => { updateCol(); };

		# If we are in a single line machine then newline may end the spec.
		NL => { updateCol(); };

		# Consume eof.
		EOF;

		any => { token( *ts ); } ;
	*|;

	literal_pattern := |*
		'\\' '0' { litBuf.append( '\0' ); };
		'\\' 'a' { litBuf.append( '\a' ); };
		'\\' 'b' { litBuf.append( '\b' ); };
		'\\' 't' { litBuf.append( '\t' ); };
		'\\' 'n' { litBuf.append( '\n' ); };
		'\\' 'v' { litBuf.append( '\v' ); };
		'\\' 'f' { litBuf.append( '\f' ); };
		'\\' 'r' { litBuf.append( '\r' ); };

		'\\' any {
			litBuf.append( ts[1] );
		};
		'"' => {
			if ( litBuf.length > 0 ) {
				token( TK_LitPat, litBuf.data, litBuf.data+litBuf.length );
				litBuf.clear();
			}
			token( '"' );
			fret;
		};
		NL => {
			litBuf.append( '\n' );
			token( TK_LitPat, litBuf.data, litBuf.data+litBuf.length );
			litBuf.clear();
			token( '"' );
			fret;
		};
		'[' => {
			if ( litBuf.length > 0 ) {
				token( TK_LitPat, litBuf.data, litBuf.data+litBuf.length );
				litBuf.clear();
			}
			token( '[' );
			fcall main;
		};
		any => {
			litBuf.append( *ts );
		};
	*|;

	# Parser definitions.
	main := |*
		'lex' => { token( KW_Lex ); };
		'commit' => { token( KW_Commit ); };
		'token' => { token( KW_Token ); };
		'literal' => { token( KW_Literal ); };
		'rl' => { token( KW_Rl ); };
		'def' => { token( KW_Def ); };
		'ignore' => { token( KW_Ignore ); };
		'construct' => { token( KW_Construct ); };
		'cons' => { token( KW_Construct ); };
		'new' => { token( KW_New ); };
		'if' => { token( KW_If ); };
		'reject' => { token( KW_Reject ); };
		'while' => { token( KW_While ); };
		'else' => { token( KW_Else ); };
		'elsif' => { token( KW_Elsif ); };
		'match' => { token( KW_Match ); };
		'for' => { token( KW_For ); };
		'iter' => { token( KW_Iter ); };
		'prints' => { token( KW_PrintStream ); };
		'print' => { token( KW_Print ); };
		'print_xml_ac' => { token( KW_PrintXMLAC ); };
		'print_xml' => { token( KW_PrintXML ); };
		'namespace' => { token( KW_Namespace ); };
		'lex' => { token( KW_Lex ); };
		'map' => { token( KW_Map ); };
		'list' => { token( KW_List ); };
		'vector' => { token( KW_Vector ); };
		'accum' => { token( KW_Accum ); };
		'parser' => { token( KW_Accum ); };
		'return' => { token( KW_Return ); };
		'break' => { token( KW_Break ); };
		'yield' => { token( KW_Yield ); };
		'typeid' => { token( KW_TypeId ); };
		'make_token' => { token( KW_MakeToken ); };
		'make_tree' => { token( KW_MakeTree ); };
		'reducefirst' => { token( KW_ReduceFirst ); };
		'for' => { token( KW_For ); };
		'in' => { token( KW_In ); };
		'nil' => { token( KW_Nil ); };
		'true' => { token( KW_True ); };
		'false' => { token( KW_False ); };
		'parse' => { token( KW_Parse ); };
		'parse_stop' => { token( KW_ParseStop ); };
		'global' => { token( KW_Global ); };
		'export' => { token( KW_Export ); };
		'ptr' => { token( KW_Ptr ); };
		'ref' => { token( KW_Ref ); };
		'deref' => { token( KW_Deref ); };
		'require' => { token( KW_Require ); };
		'preeof' => { token( KW_Preeof ); };
		'left' => { token( KW_Left ); };
		'right' => { token( KW_Right ); };
		'nonassoc' => { token( KW_Nonassoc ); };
		'prec' => { token( KW_Prec ); };
		'include' => { token( KW_Include ); };
		'context' => { token( KW_Context ); };
		'alias' => { token( KW_Alias ); };
		'send' => { token( KW_Send ); };
		'ni' => { token( KW_Ni ); };
		'ci' => { token( KW_Ci ); };

		# Identifiers.
		ident => { token( TK_Word, ts, te ); } ;

		number => { token( TK_Number, ts, te ); };

		'/' => {
			token( '/' );
			if ( parser->enterRl )
				fcall regular_type;
		};

		"~" [^\n]* NL => {
			token( '"' );
			token( TK_LitPat, ts+1, te );
			token( '"' );
		};

		"'" ([^'\\\n] | '\\' (any | NL))* ( "'" | NL ) => {
			token( TK_Literal, ts, te );
		};

		'"' => {
			token( '"' );
			litBuf.clear();
			fcall literal_pattern;
		};
		'[' => {
			token( '[' );
			fcall main;
		};

		']' => {
			token( ']' );
			if ( top > 0 )
				fret;
		};

		# Ignore.
		pound_comment => { updateCol(); };

		'=>' => { token( TK_DoubleArrow ); };
		'==' => { token( TK_DoubleEql ); };
		'!=' => { token( TK_NotEql ); };
		'::' => { token( TK_DoubleColon ); };
		'<=' => { token( TK_LessEql ); };
		'>=' => { token( TK_GrtrEql ); };
		'->' => { token( TK_RightArrow ); };
		'&&' => { token( TK_AmpAmp ); };
		'||' => { token( TK_BarBar ); };
		'<<' => { token( TK_LtLt ); };

		('+' | '-' | '*' | '/' | '(' | ')' | '@' | '$' | '^' ) => { token( *ts ); };

		# Whitespace other than newline.
		[ \t\r]+ => { updateCol(); };
		NL => { updateCol(); };

		# Consume eof.
		EOF;

		any => { token( *ts ); } ;
	*|;
}%%

%% write data;

void ColmScanner::scan()
{
	int bufsize = 8;
	char *buf = new char[bufsize];
	const char last_char = 0;
	int cs, act, have = 0;
	int top, stack[32];
	bool execute = true;

	sectionParseInit();
	%% write init;

	while ( execute ) {
		char *p = buf + have;
		int space = bufsize - have;

		if ( space == 0 ) {
			/* We filled up the buffer trying to scan a token. Grow it. */
			bufsize = bufsize * 2;
			char *newbuf = new char[bufsize];

			/* Recompute p and space. */
			p = newbuf + have;
			space = bufsize - have;

			/* Patch up pointers possibly in use. */
			if ( ts != 0 )
				ts = newbuf + ( ts - buf );
			te = newbuf + ( te - buf );

			/* Copy the new buffer in. */
			memcpy( newbuf, buf, have );
			delete[] buf;
			buf = newbuf;
		}

		input.read( p, space );
		int len = input.gcount();

		/* If we see eof then append the EOF char. */
	 	if ( len == 0 ) {
			p[0] = last_char, len = 1;
			execute = false;
		}

		char *pe = p + len;
		char *eof = 0;
		%% write exec;

		/* Check if we failed. */
		if ( cs == rlscan_error ) {
			/* Machine failed before finding a token. I'm not yet sure if this
			 * is reachable. */
			scan_error() << "colm scanner error (metalanguage)" << endl;
			exit(1);
		}

		/* Decide if we need to preserve anything. */
		char *preserve = ts;

		/* Now set up the prefix. */
		if ( preserve == 0 )
			have = 0;
		else {
			/* There is data that needs to be shifted over. */
			have = pe - preserve;
			memmove( buf, preserve, have );
			unsigned int shiftback = preserve - buf;
			if ( ts != 0 )
				ts -= shiftback;
			te -= shiftback;

			preserve = buf;
		}
	}
	delete[] buf;
}

void ColmScanner::eof()
{
	InputLoc loc;
	loc.fileName = "<EOF>";
	loc.line = line;
	loc.col = 1;
	parser->token( loc, ColmParser_tk_eof, 0, 0 );
}

/*
 *  Copyright 2006-2012 Adrian Thurston <thurston@complang.org>
 */

/*  This file is part of Colm.
 *
 *  Colm is free software; you can redistribute it and/or modify
 *  it under the terms of the GNU General Public License as published by
 *  the Free Software Foundation; either version 2 of the License, or
 *  (at your option) any later version.
 *
 *  Colm is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *  GNU General Public License for more details.
 *
 *  You should have received a copy of the GNU General Public License
 *  along with Colm; if not, write to the Free Software
 *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 */

#include <iostream>
#include <fstream>
#include <string.h>

#include "global.h"
#include "lmscan.h"
#include "lmparse.h"
#include "parsedata.h"
#include "avltree.h"
#include "vector.h"

//#define PRINT_TOKENS

using std::ifstream;
using std::istream;
using std::ostream;
using std::cout;
using std::cerr;
using std::endl;

%%{
	machine section_parse;
	alphtype int;
	write data;
}%%

void ColmScanner::sectionParseInit()
{
	%% write init;
}

ostream &ColmScanner::scan_error()
{
	/* Maintain the error count. */
	gblErrorCount += 1;
	cerr << fileName << ":" << line << ":" << column << ": ";
	return cerr;
}

bool ColmScanner::recursiveInclude( const char *inclFileName )
{
	for ( IncludeStack::Iter si = includeStack; si.lte(); si++ ) {
		if ( strcmp( si->fileName, inclFileName ) == 0 )
			return true;
	}
	return false;
}

void ColmScanner::updateCol()
{
	char *from = lastnl;
	if ( from == 0 )
		from = ts;
	//cerr << "adding " << te - from << " to column" << endl;
	column += te - from;
	lastnl = 0;
}

void ColmScanner::token( int type, char c )
{
	token( type, &c, &c + 1 );
}

void ColmScanner::token( int type )
{
	token( type, 0, 0 );
}

bool isAbsolutePath( const char *path )
{
	return path[0] == '/';
}

ifstream *ColmScanner::tryOpenInclude( char **pathChecks, long &found )
{
	char **check = pathChecks;
	ifstream *inFile = new ifstream;

	while ( *check != 0 ) {
		inFile->open( *check );
		if ( inFile->is_open() ) {
			found = check - pathChecks;
			return inFile;
		}
		check += 1;
	}

	found = -1;
	delete inFile;
	return 0;
}

char **ColmScanner::makeIncludePathChecks( const char *thisFileName, const char *fileName )
{
	char **checks = 0;
	long nextCheck = 0;
	char *data = strdup(fileName);
	long length = strlen(fileName);

	/* Absolute path? */
	if ( isAbsolutePath( data ) ) {
		checks = new char*[2];
		checks[nextCheck++] = data;
	}
	else {
		/* Search from the the location of the current file. */
		checks = new char *[2 + includePaths.length()];
		const char *lastSlash = strrchr( thisFileName, '/' );
		if ( lastSlash == 0 )
			checks[nextCheck++] = data;
		else {
			long givenPathLen = (lastSlash - thisFileName) + 1;
			long checklen = givenPathLen + length;
			char *check = new char[checklen+1];
			memcpy( check, thisFileName, givenPathLen );
			memcpy( check+givenPathLen, data, length );
			check[checklen] = 0;
			checks[nextCheck++] = check;
		}

		/* Search from the include paths given on the command line. */
		for ( ArgsVector::Iter incp = includePaths; incp.lte(); incp++ ) {
			long pathLen = strlen( *incp );
			long checkLen = pathLen + 1 + length;
			char *check = new char[checkLen+1];
			memcpy( check, *incp, pathLen );
			check[pathLen] = '/';
			memcpy( check+pathLen+1, data, length );
			check[checkLen] = 0;
			checks[nextCheck++] = check;
		}
	}

	checks[nextCheck] = 0;
	return checks;
}

%%{
	machine section_parse;
	import "lmparse.h";

	action clear_words { word = lit = 0; word_len = lit_len = 0; }
	action store_lit { lit = tokdata; lit_len = toklen; }

	action mach_err { scan_error() << "bad machine statement" << endl; }
	action incl_err { scan_error() << "bad include statement" << endl; }
	action write_err { scan_error() << "bad write statement" << endl; }

	action handle_include
	{
		String src( lit, lit_len );
		String fileName;
		bool unused;

		/* Need a location. */
		InputLoc here;
		here.fileName = fileName;
		here.line = line;
		here.col = column;

		prepareLitString( fileName, unused, src, here );
		char **checks = makeIncludePathChecks( this->fileName, fileName );

		/* Open the input file for reading. */
		long found = 0;
		ifstream *inFile = tryOpenInclude( checks, found );
		if ( inFile == 0 ) {
			scan_error() << "include: could not open " <<
					fileName << " for reading" << endl;
		}
		else {
			/* Only proceed with the include if it was found. */
			if ( recursiveInclude( checks[found] ) )
				scan_error() << "include: this is a recursive include operation" << endl;

			/* Check for a recursive include structure. Add the current file/section
			 * name then check if what we are including is already in the stack. */
			includeStack.append( IncludeStackItem( checks[found] ) );

			ColmScanner scanner( fileName, *inFile, output, parser, includeDepth+1 );
			scanner.scan();
			delete inFile;

			/* Remove the last element (len-1) */
			includeStack.remove( -1 );
		}
	}

	include_target =
		TK_Literal >clear_words @store_lit;

	include_stmt =
		( KW_Include include_target ) @handle_include
		<>err incl_err <>eof incl_err;

	action handle_token
	{
//	cout << Parser_lelNames[type] << " ";
//	if ( start != 0 ) {
//		cout.write( start, end-start );
//	}
//	cout << endl;

		InputLoc loc;

		#ifdef PRINT_TOKENS
		cerr << "scanner:" << line << ":" << column <<
				": sending token to the parser " << Parser_lelNames[*p];
		cerr << " " << toklen;
		if ( tokdata != 0 )
			cerr << " " << tokdata;
		cerr << endl;
		#endif

		loc.fileName = fileName;
		loc.line = line;
		loc.col = column;

		if ( tokdata != 0 && tokdata[toklen-1] == '\n' )
			loc.line -= 1;

		parser->token( loc, type, tokdata, toklen );
	}

	# Catch everything else.
	everything_else = ^( KW_Include ) @handle_token;

	main := (
		include_stmt |
		everything_else
	)*;
}%%

void ColmScanner::token( int type, char *start, char *end )
{
	char *tokdata = 0;
	int toklen = 0;
	int *p = &type;
	int *pe = &type + 1;
	int *eof = 0;

	if ( start != 0 ) {
		toklen = end-start;
		tokdata = new char[toklen+1];
		memcpy( tokdata, start, toklen );
		tokdata[toklen] = 0;
	}

	%%{
		machine section_parse;
		write exec;
	}%%

	updateCol();
}

void ColmScanner::endSection( )
{
	/* Execute the eof actions for the section parser. */
	/* Probably use: token( -1 ); */
}

%%{
	machine rlscan;

	# This is sent by the driver code.
	EOF = 0;

	action inc_nl {
		lastnl = p;
		column = 0;
		line++;
	}
	NL = '\n' @inc_nl;

	# Identifiers, numbers, commetns, and other common things.
	ident = ( alpha | '_' ) ( alpha |digit |'_' )*;
	number = digit+;
	hex_number = '0x' [0-9a-fA-F]+;

	# These literal forms are common to C-like host code and ragel.
	s_literal = "'" ([^'\\] | NL | '\\' (any | NL))* "'";
	d_literal = '"' ([^"\\] | NL | '\\' (any | NL))* '"';

	whitespace = [ \t] | NL;
	pound_comment = '#' [^\n]* NL;

	or_literal := |*
		# Escape sequences in OR expressions.
		'\\0' => { token( TK_ReChar, '\0' ); };
		'\\a' => { token( TK_ReChar, '\a' ); };
		'\\b' => { token( TK_ReChar, '\b' ); };
		'\\t' => { token( TK_ReChar, '\t' ); };
		'\\n' => { token( TK_ReChar, '\n' ); };
		'\\v' => { token( TK_ReChar, '\v' ); };
		'\\f' => { token( TK_ReChar, '\f' ); };
		'\\r' => { token( TK_ReChar, '\r' ); };
		'\\\n' => { updateCol(); };
		'\\' any => { token( TK_ReChar, ts+1, te ); };

		# Range dash in an OR expression.
		'-' => { token( TK_Dash, 0, 0 ); };

		# Terminate an OR expression.
		']'	=> { token( TK_SqClose ); fret; };

		EOF => {
			scan_error() << "unterminated OR literal" << endl;
		};

		# Characters in an OR expression.
		[^\]] => { token( TK_ReChar, ts, te ); };

	*|;

	regular_type := |*
		# Identifiers.
		ident => { token( TK_Word, ts, te ); } ;

		# Numbers
		number => { token( TK_UInt, ts, te ); };
		hex_number => { token( TK_Hex, ts, te ); };

		# Literals, with optionals.
		( s_literal | d_literal ) [i]?
			=> { token( TK_Literal, ts, te ); };

		'[' => { token( TK_SqOpen ); fcall or_literal; };
		'[^' => { token( TK_SqOpenNeg ); fcall or_literal; };

		'/' => { token( '/'); fret; };

		# Ignore.
		pound_comment => { updateCol(); };

		'..' => { token( TK_DotDot ); };
		'**' => { token( TK_StarStar ); };
		'--' => { token( TK_DashDash ); };

		':>'  => { token( TK_ColonGt ); };
		':>>' => { token( TK_ColonGtGt ); };
		'<:'  => { token( TK_LtColon ); };

		# Whitespace other than newline.
		[ \t\r]+ => { updateCol(); };

		# If we are in a single line machine then newline may end the spec.
		NL => { updateCol(); };

		# Consume eof.
		EOF;

		any => { token( *ts ); } ;
	*|;

	literal_pattern := |*
		'\\' '0' { litBuf.append( '\0' ); };
		'\\' 'a' { litBuf.append( '\a' ); };
		'\\' 'b' { litBuf.append( '\b' ); };
		'\\' 't' { litBuf.append( '\t' ); };
		'\\' 'n' { litBuf.append( '\n' ); };
		'\\' 'v' { litBuf.append( '\v' ); };
		'\\' 'f' { litBuf.append( '\f' ); };
		'\\' 'r' { litBuf.append( '\r' ); };

		'\\' any {
			litBuf.append( ts[1] );
		};
		'"' => {
			if ( litBuf.length > 0 ) {
				token( TK_LitPat, litBuf.data, litBuf.data+litBuf.length );
				litBuf.clear();
			}
			token( '"' );
			fret;
		};
		NL => {
			litBuf.append( '\n' );
			token( TK_LitPat, litBuf.data, litBuf.data+litBuf.length );
			litBuf.clear();
			token( '"' );
			fret;
		};
		'[' => {
			if ( litBuf.length > 0 ) {
				token( TK_LitPat, litBuf.data, litBuf.data+litBuf.length );
				litBuf.clear();
			}
			token( '[' );
			fcall main;
		};
		any => {
			litBuf.append( *ts );
		};
	*|;

	# Parser definitions.
	main := |*
		'lex' => { token( KW_Lex ); };
		'commit' => { token( KW_Commit ); };
		'token' => { token( KW_Token ); };
		'literal' => { token( KW_Literal ); };
		'rl' => { token( KW_Rl ); };
		'def' => { token( KW_Def ); };
		'ignore' => { token( KW_Ignore ); };
		'construct' => { token( KW_Construct ); };
		'cons' => { token( KW_Construct ); };
		'new' => { token( KW_New ); };
		'if' => { token( KW_If ); };
		'reject' => { token( KW_Reject ); };
		'while' => { token( KW_While ); };
		'else' => { token( KW_Else ); };
		'elsif' => { token( KW_Elsif ); };
		'match' => { token( KW_Match ); };
		'for' => { token( KW_For ); };
		'iter' => { token( KW_Iter ); };
		'prints' => { token( KW_PrintStream ); };
		'print' => { token( KW_Print ); };
		'print_xml_ac' => { token( KW_PrintXMLAC ); };
		'print_xml' => { token( KW_PrintXML ); };
		'namespace' => { token( KW_Namespace ); };
		'lex' => { token( KW_Lex ); };
		'map' => { token( KW_Map ); };
		'list' => { token( KW_List ); };
		'vector' => { token( KW_Vector ); };
		'accum' => { token( KW_Accum ); };
		'parser' => { token( KW_Accum ); };
		'return' => { token( KW_Return ); };
		'break' => { token( KW_Break ); };
		'yield' => { token( KW_Yield ); };
		'typeid' => { token( KW_TypeId ); };
		'make_token' => { token( KW_MakeToken ); };
		'make_tree' => { token( KW_MakeTree ); };
		'reducefirst' => { token( KW_ReduceFirst ); };
		'for' => { token( KW_For ); };
		'in' => { token( KW_In ); };
		'nil' => { token( KW_Nil ); };
		'true' => { token( KW_True ); };
		'false' => { token( KW_False ); };
		'parse' => { token( KW_Parse ); };
		'parse_stop' => { token( KW_ParseStop ); };
		'global' => { token( KW_Global ); };
		'export' => { token( KW_Export ); };
		'ptr' => { token( KW_Ptr ); };
		'ref' => { token( KW_Ref ); };
		'deref' => { token( KW_Deref ); };
		'require' => { token( KW_Require ); };
		'preeof' => { token( KW_Preeof ); };
		'left' => { token( KW_Left ); };
		'right' => { token( KW_Right ); };
		'nonassoc' => { token( KW_Nonassoc ); };
		'prec' => { token( KW_Prec ); };
		'include' => { token( KW_Include ); };
		'context' => { token( KW_Context ); };
		'alias' => { token( KW_Alias ); };
		'send' => { token( KW_Send ); };
		'ni' => { token( KW_Ni ); };
		'ci' => { token( KW_Ci ); };

		# Identifiers.
		ident => { token( TK_Word, ts, te ); } ;

		number => { token( TK_Number, ts, te ); };

		'/' => {
			token( '/' );
			if ( parser->enterRl )
				fcall regular_type;
		};

		"~" [^\n]* NL => {
			token( '"' );
			token( TK_LitPat, ts+1, te );
			token( '"' );
		};

		"'" ([^'\\\n] | '\\' (any | NL))* ( "'" | NL ) => {
			token( TK_Literal, ts, te );
		};

		'"' => {
			token( '"' );
			litBuf.clear();
			fcall literal_pattern;
		};
		'[' => {
			token( '[' );
			fcall main;
		};

		']' => {
			token( ']' );
			if ( top > 0 )
				fret;
		};

		# Ignore.
		pound_comment => { updateCol(); };

		'=>' => { token( TK_DoubleArrow ); };
		'==' => { token( TK_DoubleEql ); };
		'!=' => { token( TK_NotEql ); };
		'::' => { token( TK_DoubleColon ); };
		'<=' => { token( TK_LessEql ); };
		'>=' => { token( TK_GrtrEql ); };
		'->' => { token( TK_RightArrow ); };
		'&&' => { token( TK_AmpAmp ); };
		'||' => { token( TK_BarBar ); };
		'<<' => { token( TK_LtLt ); };

		('+' | '-' | '*' | '/' | '(' | ')' | '@' | '$' | '^' ) => { token( *ts ); };

		# Whitespace other than newline.
		[ \t\r]+ => { updateCol(); };
		NL => { updateCol(); };

		# Consume eof.
		EOF;

		any => { token( *ts ); } ;
	*|;
}%%

%% write data;

void ColmScanner::scan()
{
	int bufsize = 8;
	char *buf = new char[bufsize];
	const char last_char = 0;
	int cs, act, have = 0;
	int top, stack[32];
	bool execute = true;

	sectionParseInit();
	%% write init;

	while ( execute ) {
		char *p = buf + have;
		int space = bufsize - have;

		if ( space == 0 ) {
			/* We filled up the buffer trying to scan a token. Grow it. */
			bufsize = bufsize * 2;
			char *newbuf = new char[bufsize];

			/* Recompute p and space. */
			p = newbuf + have;
			space = bufsize - have;

			/* Patch up pointers possibly in use. */
			if ( ts != 0 )
				ts = newbuf + ( ts - buf );
			te = newbuf + ( te - buf );

			/* Copy the new buffer in. */
			memcpy( newbuf, buf, have );
			delete[] buf;
			buf = newbuf;
		}

		input.read( p, space );
		int len = input.gcount();

		/* If we see eof then append the EOF char. */
	 	if ( len == 0 ) {
			p[0] = last_char, len = 1;
			execute = false;
		}

		char *pe = p + len;
		char *eof = 0;
		%% write exec;

		/* Check if we failed. */
		if ( cs == rlscan_error ) {
			/* Machine failed before finding a token. I'm not yet sure if this
			 * is reachable. */
			scan_error() << "colm scanner error (metalanguage)" << endl;
			exit(1);
		}

		/* Decide if we need to preserve anything. */
		char *preserve = ts;

		/* Now set up the prefix. */
		if ( preserve == 0 )
			have = 0;
		else {
			/* There is data that needs to be shifted over. */
			have = pe - preserve;
			memmove( buf, preserve, have );
			unsigned int shiftback = preserve - buf;
			if ( ts != 0 )
				ts -= shiftback;
			te -= shiftback;

			preserve = buf;
		}
	}
	delete[] buf;
}

void ColmScanner::eof()
{
	InputLoc loc;
	loc.fileName = "<EOF>";
	loc.line = line;
	loc.col = 1;
	parser->token( loc, ColmParser_tk_eof, 0, 0 );
}


Make sure C strings always end with a NULL.

Haven't bumped into any problems just yet. However, in theory all sorts of evil
could happen here. Which is part of the problem of C: so much shit is undefined
behaviour that you can take a single step and fall in 15 holes at the same time.
In theory, because nobody bothered to actually specify it properly.

#include "lexer.h"

/*
The following two macros allow the Ragel grammar to use generic function calls
without relying on the setup of the C or Java lexer. Using these macros we can
also pass along `self` to the callback functions without having to hard-code
this in to the Ragel grammar.

In the C lexer we don't need the `data` variable (since this is pulled in based
on `ts` and `te`) so the macro ignores this argument.
*/

#define callback(name, data, encoding, start, stop) \
    liboga_xml_lexer_callback(self, name, encoding, start, stop);

#define callback_simple(name) \
    liboga_xml_lexer_callback_simple(self, name);

#define oga_ivar_get(owner, name) \
    rb_ivar_get(owner, rb_intern(name))

#define oga_ivar_set(owner, name, value) \
    rb_ivar_set(owner, rb_intern(name), value)

#define advance_line(amount) \
    rb_funcall(self, rb_intern("advance_line"), 1, INT2NUM(amount));

%%machine c_lexer;

/**
 * Calls a method defined in the Ruby side of the lexer. The String value is
 * created based on the values of `ts` and `te` and uses the encoding specified
 * in `encoding`.
 *
 * @example
 *  rb_encoding *encoding = rb_enc_get(...);
 *  liboga_xml_lexer_callback(self, "on_string", encoding, ts, te);
 */
void liboga_xml_lexer_callback(
    VALUE self,
    const char *name,
    rb_encoding *encoding,
    const char *ts,
    const char *te
)
{
    VALUE value  = rb_enc_str_new(ts, te - ts, encoding);
    VALUE method = rb_intern(name);

    rb_funcall(self, method, 1, value);
}

/**
 * Calls a method defined in the Ruby side of the lexer without passing it any
 * arguments.
 *
 * @example
 *  liboga_xml_lexer_callback_simple(self, "on_cdata_start");
 */
void liboga_xml_lexer_callback_simple(VALUE self, const char *name)
{
    VALUE method = rb_intern(name);

    rb_funcall(self, method, 0);
}

%% write data;

/**
 * Lexes the String specifies as the method argument. Token values have the
 * same encoding as the input value.
 *
 * This method keeps track of an internal state using the instance variables
 * `@act` and `@cs`.
 */
VALUE oga_xml_lexer_advance(VALUE self, VALUE data_block)
{
    /* Make sure that all data passed back to Ruby has the proper encoding. */
    rb_encoding *encoding = rb_enc_get(data_block);

    char *data_str_val = StringValueCStr(data_block);

    const char *p    = data_str_val;
    const char *pe   = data_str_val + strlen(data_str_val);
    const char *eof  = pe;
    const char *ts   = 0;
    const char *te   = 0;
    const char *mark = 0;

    int act   = NUM2INT(oga_ivar_get(self, "@act"));
    int cs    = NUM2INT(oga_ivar_get(self, "@cs"));
    int lines = 0;

    %% write exec;

    oga_ivar_set(self, "@act", INT2NUM(act));
    oga_ivar_set(self, "@cs", INT2NUM(cs));

    return Qnil;
}

/**
 * Resets the internal state of the lexer.
 */
VALUE oga_xml_lexer_reset(VALUE self)
{
    oga_ivar_set(self, "@act", INT2NUM(0));
    oga_ivar_set(self, "@cs", INT2NUM(c_lexer_start));

    return Qnil;
}

%%{
    include base_lexer "base_lexer.rl";
}%%

void Init_liboga_xml_lexer()
{
    VALUE mOga   = rb_const_get(rb_cObject, rb_intern("Oga"));
    VALUE mXML   = rb_const_get(mOga, rb_intern("XML"));
    VALUE cLexer = rb_define_class_under(mXML, "Lexer", rb_cObject);

    rb_define_method(cLexer, "advance_native", oga_xml_lexer_advance, 1);
    rb_define_method(cLexer, "reset_native", oga_xml_lexer_reset, 0);
}

#include "lexer.h"

/*
The following two macros allow the Ragel grammar to use generic function calls
without relying on the setup of the C or Java lexer. Using these macros we can
also pass along `self` to the callback functions without having to hard-code
this in to the Ragel grammar.

In the C lexer we don't need the `data` variable (since this is pulled in based
on `ts` and `te`) so the macro ignores this argument.
*/

#define callback(name, data, encoding, start, stop) \
    liboga_xml_lexer_callback(self, name, encoding, start, stop);

#define callback_simple(name) \
    liboga_xml_lexer_callback_simple(self, name);

#define oga_ivar_get(owner, name) \
    rb_ivar_get(owner, rb_intern(name))

#define oga_ivar_set(owner, name, value) \
    rb_ivar_set(owner, rb_intern(name), value)

#define advance_line(amount) \
    rb_funcall(self, rb_intern("advance_line"), 1, INT2NUM(amount));

%%machine c_lexer;

/**
 * Calls a method defined in the Ruby side of the lexer. The String value is
 * created based on the values of `ts` and `te` and uses the encoding specified
 * in `encoding`.
 *
 * @example
 *  rb_encoding *encoding = rb_enc_get(...);
 *  liboga_xml_lexer_callback(self, "on_string", encoding, ts, te);
 */
void liboga_xml_lexer_callback(
    VALUE self,
    const char *name,
    rb_encoding *encoding,
    const char *ts,
    const char *te
)
{
    VALUE value  = rb_enc_str_new(ts, te - ts, encoding);
    VALUE method = rb_intern(name);

    rb_funcall(self, method, 1, value);
}

/**
 * Calls a method defined in the Ruby side of the lexer without passing it any
 * arguments.
 *
 * @example
 *  liboga_xml_lexer_callback_simple(self, "on_cdata_start");
 */
void liboga_xml_lexer_callback_simple(VALUE self, const char *name)
{
    VALUE method = rb_intern(name);

    rb_funcall(self, method, 0);
}

%% write data;

/**
 * Lexes the String specifies as the method argument. Token values have the
 * same encoding as the input value.
 *
 * This method keeps track of an internal state using the instance variables
 * `@act` and `@cs`.
 */
VALUE oga_xml_lexer_advance(VALUE self, VALUE data_block)
{
    /* Make sure that all data passed back to Ruby has the proper encoding. */
    rb_encoding *encoding = rb_enc_get(data_block);

    char *data_str_val = StringValuePtr(data_block);

    const char *p    = data_str_val;
    const char *pe   = data_str_val + strlen(data_str_val);
    const char *eof  = pe;
    const char *ts   = 0;
    const char *te   = 0;
    const char *mark = 0;

    int act   = NUM2INT(oga_ivar_get(self, "@act"));
    int cs    = NUM2INT(oga_ivar_get(self, "@cs"));
    int lines = 0;

    %% write exec;

    oga_ivar_set(self, "@act", INT2NUM(act));
    oga_ivar_set(self, "@cs", INT2NUM(cs));

    return Qnil;
}

/**
 * Resets the internal state of the lexer.
 */
VALUE oga_xml_lexer_reset(VALUE self)
{
    oga_ivar_set(self, "@act", INT2NUM(0));
    oga_ivar_set(self, "@cs", INT2NUM(c_lexer_start));

    return Qnil;
}

%%{
    include base_lexer "base_lexer.rl";
}%%

void Init_liboga_xml_lexer()
{
    VALUE mOga   = rb_const_get(rb_cObject, rb_intern("Oga"));
    VALUE mXML   = rb_const_get(mOga, rb_intern("XML"));
    VALUE cLexer = rb_define_class_under(mXML, "Lexer", rb_cObject);

    rb_define_method(cLexer, "advance_native", oga_xml_lexer_advance, 1);
    rb_define_method(cLexer, "reset_native", oga_xml_lexer_reset, 0);
}


Removed extra space in c/lexer.rl

Removed extra space in c/lexer.rl

#include "lexer.h"

/*
The following two macros allow the Ragel grammar to use generic function calls
without relying on the setup of the C or Java lexer. Using these macros we can
also pass along `self` to the callback functions without having to hard-code
this in to the Ragel grammar.

In the C lexer we don't need the `data` variable (since this is pulled in based
on `ts` and `te`) so the macro ignores this argument.
*/

#define callback(name, data, encoding, start, stop) \
    liboga_xml_lexer_callback(self, name, encoding, start, stop);

#define callback_simple(name) \
    liboga_xml_lexer_callback_simple(self, name);

#define oga_ivar_get(owner, name) \
    rb_ivar_get(owner, rb_intern(name))

#define oga_ivar_set(owner, name, value) \
    rb_ivar_set(owner, rb_intern(name), value)

#define advance_line(amount) \
    rb_funcall(self, rb_intern("advance_line"), 1, INT2NUM(amount));

%%machine c_lexer;

/**
 * Calls a method defined in the Ruby side of the lexer. The String value is
 * created based on the values of `ts` and `te` and uses the encoding specified
 * in `encoding`.
 *
 * @example
 *  rb_encoding *encoding = rb_enc_get(...);
 *  liboga_xml_lexer_callback(self, "on_string", encoding, ts, te);
 */
void liboga_xml_lexer_callback(
    VALUE self,
    VALUE name,
    rb_encoding *encoding,
    const char *ts,
    const char *te
)
{
    VALUE value = rb_enc_str_new(ts, te - ts, encoding);

    rb_funcall(self, name, 1, value);
}

/**
 * Calls a method defined in the Ruby side of the lexer without passing it any
 * arguments.
 *
 * @example
 *  liboga_xml_lexer_callback_simple(self, "on_cdata_start");
 */
void liboga_xml_lexer_callback_simple(VALUE self, VALUE name)
{
    rb_funcall(self, name, 0);
}

%% write data;

/**
 * Lexes the String specifies as the method argument. Token values have the
 * same encoding as the input value.
 *
 * This method keeps track of an internal state using the instance variables
 * `@act` and `@cs`.
 */
VALUE oga_xml_lexer_advance(VALUE self, VALUE data_block)
{
    OgaLexerState *state;

    /* Make sure that all data passed back to Ruby has the proper encoding. */
    rb_encoding *encoding = rb_enc_get(data_block);

    char *data_str_val = StringValueCStr(data_block);

    Data_Get_Struct(self, OgaLexerState, state);

    const char *p    = data_str_val;
    const char *pe   = data_str_val + strlen(data_str_val);
    const char *eof  = pe;
    const char *ts   = 0;
    const char *te   = 0;
    const char *mark = 0;

    int lines = state->lines;

    VALUE id_advance_line        = rb_intern("advance_line");
    VALUE id_on_attribute        = rb_intern("on_attribute");
    VALUE id_on_attribute_ns     = rb_intern("on_attribute_ns");
    VALUE id_on_cdata            = rb_intern("on_cdata");
    VALUE id_on_comment          = rb_intern("on_comment");
    VALUE id_on_doctype_end      = rb_intern("on_doctype_end");
    VALUE id_on_doctype_inline   = rb_intern("on_doctype_inline");
    VALUE id_on_doctype_name     = rb_intern("on_doctype_name");
    VALUE id_on_doctype_start    = rb_intern("on_doctype_start");
    VALUE id_on_doctype_type     = rb_intern("on_doctype_type");
    VALUE id_on_element_end      = rb_intern("on_element_end");
    VALUE id_on_element_name     = rb_intern("on_element_name");
    VALUE id_on_element_ns       = rb_intern("on_element_ns");
    VALUE id_on_element_open_end = rb_intern("on_element_open_end");
    VALUE id_on_element_start    = rb_intern("on_element_start");
    VALUE id_on_proc_ins_end     = rb_intern("on_proc_ins_end");
    VALUE id_on_proc_ins_name    = rb_intern("on_proc_ins_name");
    VALUE id_on_proc_ins_start   = rb_intern("on_proc_ins_start");
    VALUE id_on_string_body      = rb_intern("on_string_body");
    VALUE id_on_string_dquote    = rb_intern("on_string_dquote");
    VALUE id_on_string_squote    = rb_intern("on_string_squote");
    VALUE id_on_text             = rb_intern("on_text");
    VALUE id_on_xml_decl_end     = rb_intern("on_xml_decl_end");
    VALUE id_on_xml_decl_start   = rb_intern("on_xml_decl_start");

    %% write exec;

    state->lines = lines;

    return Qnil;
}

/**
 * Resets the internal state of the lexer.
 */
VALUE oga_xml_lexer_reset(VALUE self)
{
    OgaLexerState *state;

    Data_Get_Struct(self, OgaLexerState, state);

    state->act   = 0;
    state->cs    = c_lexer_start;
    state->lines = 0;
    state->top   = 0;

    return Qnil;
}

/**
 * Frees the associated lexer state struct.
 */
void oga_xml_lexer_free(void *state)
{
    free((OgaLexerState *) state);
}

/**
 * Allocates and wraps the C lexer state struct. This state is used to keep
 * track of the current position, line numbers, etc.
 */
VALUE oga_xml_lexer_allocate(VALUE klass)
{
    OgaLexerState *state = malloc(sizeof(OgaLexerState));

    return Data_Wrap_Struct(klass, NULL, oga_xml_lexer_free, state);
}

%%{
    include base_lexer "base_lexer.rl";

    variable top state->top;
    variable stack state->stack;
    variable act state->act;
    variable cs state->cs;
}%%

void Init_liboga_xml_lexer()
{
    VALUE mOga   = rb_const_get(rb_cObject, rb_intern("Oga"));
    VALUE mXML   = rb_const_get(mOga, rb_intern("XML"));
    VALUE cLexer = rb_define_class_under(mXML, "Lexer", rb_cObject);

    rb_define_method(cLexer, "advance_native", oga_xml_lexer_advance, 1);
    rb_define_method(cLexer, "reset_native", oga_xml_lexer_reset, 0);

    rb_define_alloc_func(cLexer, oga_xml_lexer_allocate);
}


[jnc_ct] opt: avoid escape-decode when there are no backslashes in string

//..............................................................................
//
//  This file is part of the Jancy toolkit.
//
//  Jancy is distributed under the MIT license.
//  For details see accompanying license.txt file,
//  the public copy of which is also available at:
//  http://tibbo.com/downloads/archive/jancy/license.txt
//
//..............................................................................

// warning C4065: switch statement contains 'default' but no 'case' labels

#pragma warning (disable: 4065)

namespace jnc {
namespace ct {

//..............................................................................

%%{

machine jnc;
write data;

#. . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . .
#
# prepush / postpop (for fcall/fret)
#

prepush
{
	stack = prePush ();
}

postpop
{
	postPop ();
}

#. . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . .
#
# standard definitions
#

dec    = [0-9];
hex    = [0-9a-fA-F];
oct    = [0-7];
bin    = [01];
id     = [_a-zA-Z] [_a-zA-Z0-9]*;
ws     = [ \t\r]+;
nl     = '\n' @{ newLine (p + 1); };
lc_nl  = '\\' '\r'? nl;
esc    = '\\' [^\n];

lit_dq     = '"' ([^"\n\\] | esc)* (["\\] | nl);
lit_sq     = "'" ([^'\n\\] | esc)* (['\\] | nl);
raw_lit_dq = '"' [^"\n]* ('"' | nl);
raw_lit_sq = "'" [^'\n]* ("'" | nl);

#. . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . .
#
# multi-line literal machine
#

lit_ml := |*

(nl ws?)? '"""'
		  { createMlLiteralToken (); fgoto main; };
nl        ;
any       ;

*|;

#. . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . .
#
# formatting literal machines
#

lit_fmt := |*

esc       ;
'$!'      { createFmtLastErrorDescriptionTokens (); };
'$' id    { createFmtSimpleIdentifierTokens (); };
'%' dec+  { createFmtIndexTokens (); };
[$%] '('  { createFmtLiteralToken (TokenKind_FmtLiteral, *ts == '%'); m_parenthesesLevelStack.append (1); fcall main; };
'"' | nl  { createFmtLiteralToken (TokenKind_Literal); fret; };
any       ;

*|;

fmt_spec := |*

';' [^")\n]* { createFmtSpecifierToken (); fret; };
any          { ASSERT (false); fret; };

*|;

#. . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . .
#
# main machine
#

main := |*

#. . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . .

# global declarations & pragmas

'import'         { createToken (TokenKind_Import); };
'namespace'      { createToken (TokenKind_Namespace); };
'extension'      { createToken (TokenKind_Extension); };
'dynamiclib'     { createToken (TokenKind_DynamicLib); };
'using'          { createToken (TokenKind_Using); };
'friend'         { createToken (TokenKind_Friend); };
'public'         { createToken (TokenKind_Public); };
'protected'      { createToken (TokenKind_Protected); };
'alignment'      { createToken (TokenKind_Alignment); };
'setas'          { createToken (TokenKind_SetAs); };

#. . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . .

# storage specifiers

'typedef'        { createToken (TokenKind_Typedef); };
'alias'          { createToken (TokenKind_Alias); };
'static'         { createToken (TokenKind_Static); };
'threadlocal'    { createToken (TokenKind_ThreadLocal); };
'abstract'       { createToken (TokenKind_Abstract); };
'virtual'        { createToken (TokenKind_Virtual); };
'override'       { createToken (TokenKind_Override); };
'mutable'        { createToken (TokenKind_Mutable); };
'disposable'     { createToken (TokenKind_Disposable); };

#. . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . .

# type modifiers

'unsigned'       { createToken (TokenKind_Unsigned); };
'bigendian'      { createToken (TokenKind_BigEndian); };
'const'          { createToken (TokenKind_Const); };
'readonly'       { createToken (TokenKind_ReadOnly); };
'cmut'           { createToken (TokenKind_CMut); };
'volatile'       { createToken (TokenKind_Volatile); };
'weak'           { createToken (TokenKind_Weak); };
'thin'           { createToken (TokenKind_Thin); };
'safe'           { createToken (TokenKind_Safe); };
'unsafe'         { createToken (TokenKind_Unsafe); };
'errorcode'      { createToken (TokenKind_ErrorCode); };
'cdecl'          { createToken (TokenKind_Cdecl); };
'stdcall'        { createToken (TokenKind_Stdcall); };
'jnccall'        { createToken (TokenKind_Jnccall); };
'thiscall'       { createToken (TokenKind_Thiscall); };
'array'          { createToken (TokenKind_Array); };
'function'       { createToken (TokenKind_Function); };
'property'       { createToken (TokenKind_Property); };
'bindable'       { createToken (TokenKind_Bindable); };
'autoget'        { createToken (TokenKind_AutoGet); };
'indexed'        { createToken (TokenKind_Indexed); };
'multicast'      { createToken (TokenKind_Multicast); };
'event'          { createToken (TokenKind_Event); };
'reactor'        { createToken (TokenKind_Reactor); };
'async'          { createToken (TokenKind_Async); };

#. . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . .

# type specifiers

'anydata'        { createToken (TokenKind_AnyData); };
'void'           { createToken (TokenKind_Void); };
'variant'        { createToken (TokenKind_Variant); };
'bool'           { createToken (TokenKind_Bool); };
'int'            { createToken (TokenKind_Int); };
'intptr'         { createToken (TokenKind_IntPtr); };
'char'           { createToken (TokenKind_Char); };
'short'          { createToken (TokenKind_Short); };
'long'           { createToken (TokenKind_Long); };
'float'          { createToken (TokenKind_Float); };
'double'         { createToken (TokenKind_Double); };

#. . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . .

# named type specifiers

'enum'           { createToken (TokenKind_Enum); };
'struct'         { createToken (TokenKind_Struct); };
'union'          { createToken (TokenKind_Union); };
'class'          { createToken (TokenKind_Class); };
'opaque'         { createToken (TokenKind_Opaque); };
'exposed'        { createToken (TokenKind_Exposed); };
'bitflag'        { createToken (TokenKind_BitFlag); };

#. . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . .

# special member methods

'get'            { createToken (TokenKind_Get); };
'set'            { createToken (TokenKind_Set); };
'preconstruct'   { createToken (TokenKind_PreConstruct); };
'construct'      { createToken (TokenKind_Construct); };
'destruct'       { createToken (TokenKind_Destruct); };
'operator'       { createToken (TokenKind_Operator); };
'postfix'        { createToken (TokenKind_Postfix); };

#. . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . .

# operators

'new'            { createToken (TokenKind_New); };
'sizeof'         { createToken (TokenKind_SizeOf); };
'countof'        { createToken (TokenKind_CountOf); };
'offsetof'       { createToken (TokenKind_OffsetOf); };
'typeof'         { createToken (TokenKind_TypeOf); };
'bindingof'      { createToken (TokenKind_BindingOf); };
'dynamic'        { createToken (TokenKind_Dynamic); };

#. . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . .

# statements

'if'             { createToken (TokenKind_If); };
'else'           { createToken (TokenKind_Else); };
'for'            { createToken (TokenKind_For); };
'while'          { createToken (TokenKind_While); };
'do'             { createToken (TokenKind_Do); };
'break'          { createKeywordTokenEx (TokenKind_Break, 1); };
'break' [1-9]    { createKeywordTokenEx (TokenKind_Break, te [-1] - '0'); };
'continue'       { createKeywordTokenEx (TokenKind_Continue, 1); };
'continue' [1-9] { createKeywordTokenEx (TokenKind_Continue, te [-1] - '0'); };
'return'         { createToken (TokenKind_Return); };
'switch'         { createToken (TokenKind_Switch); };
'reswitch'       { createToken (TokenKind_ReSwitch); };
'case'           { createToken (TokenKind_Case); };
'default'        { createToken (TokenKind_Default); };
'once'           { createToken (TokenKind_Once); };
'onevent'        { createToken (TokenKind_OnEvent); };
'try'            { createToken (TokenKind_Try); };
'throw'          { createToken (TokenKind_Throw); };
'catch'          { createToken (TokenKind_Catch); };
'finally'        { createToken (TokenKind_Finally); };
'nestedscope'    { createToken (TokenKind_NestedScope); };
'assert'         { createToken (TokenKind_Assert); };
'await'          { createToken (TokenKind_Await); };

#. . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . .

# pre-defined values

'basetype'       { createKeywordTokenEx (TokenKind_BaseType, 1); };
'basetype' [1-9] { createKeywordTokenEx (TokenKind_BaseType, te [-1] - '0'); };
'this'           { createToken (TokenKind_This); };
'true'           { createToken (TokenKind_True); };
'false'          { createToken (TokenKind_False); };
'null'           { createToken (TokenKind_Null); };

#. . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . .

# symbol tokens

'++'             { createToken (TokenKind_Inc); };
'--'             { createToken (TokenKind_Dec); };
'->'             { createToken (TokenKind_Ptr); };
'=>'             { createToken (TokenKind_Imply); };
'<<'             { createToken (TokenKind_Shl); };
'>>'             { createToken (TokenKind_Shr); };
'&&'             { createToken (TokenKind_LogAnd); };
'||'             { createToken (TokenKind_LogOr); };
'=='             { createToken (TokenKind_Eq); };
'!='             { createToken (TokenKind_Ne); };
'<='             { createToken (TokenKind_Le); };
'>='             { createToken (TokenKind_Ge); };
':='             { createToken (TokenKind_RefAssign); };
'+='             { createToken (TokenKind_AddAssign); };
'-='             { createToken (TokenKind_SubAssign); };
'*='             { createToken (TokenKind_MulAssign); };
'/='             { createToken (TokenKind_DivAssign); };
'%='             { createToken (TokenKind_ModAssign); };
'<<='            { createToken (TokenKind_ShlAssign); };
'>>='            { createToken (TokenKind_ShrAssign); };
'&='             { createToken (TokenKind_AndAssign); };
'^='             { createToken (TokenKind_XorAssign); };
'|='             { createToken (TokenKind_OrAssign); };
'@='             { createToken (TokenKind_AtAssign); };
'...'            { createToken (TokenKind_Ellipsis); };

#. . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . .

# literals

[$fF] '"'        { preCreateFmtLiteralToken (); fcall lit_fmt; };
'"""' (ws? '\r'? nl)?
				 { preCreateMlLiteralToken (); fgoto lit_ml; };

'0' [xX] '"""'   { preCreateMlLiteralToken (16); fgoto lit_ml; };
'0' [oO] '"""'   { preCreateMlLiteralToken (8); fgoto lit_ml; };
'0' [bB] '"""'   { preCreateMlLiteralToken (2); fgoto lit_ml; };
'0' [nNdD] '"""' { preCreateMlLiteralToken (10); fgoto lit_ml; };

#. . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . .

# lexer-time constants

'__FILE__'       { createSourceFileToken (); };
'__DIR__'        { createSourceDirToken (); };

#. . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . .

# common tokens

id               { createStringToken (TokenKind_Identifier); };
lit_sq           { createCharToken (TokenKind_Integer, 1, 1, true); };
lit_dq           { createStringToken (TokenKind_Literal, 1, 1, true); };
[rR] raw_lit_sq  { createCharToken (TokenKind_Integer, 2, 1, false); };
[rR] raw_lit_dq  { createStringToken (TokenKind_Literal, 2, 1, false); };
'0' oct+         { createIntegerToken (8); };
dec+             { createIntegerToken (10); };
'0' [xX] hex+    { createIntegerToken (16, 2); };
'0' [oO] oct+    { createIntegerToken (8, 2); };
'0' [bB] bin+    { createIntegerToken (2, 2); };
'0' [nNdD] dec+  { createIntegerToken (10, 2); };

'0' [xX] raw_lit_dq
				 { createBinLiteralToken (16); };
'0' [oO] raw_lit_dq
				 { createBinLiteralToken (8); };
'0' [bB] raw_lit_dq
				 { createBinLiteralToken (2); };
'0' [nNdD] raw_lit_dq
				 { createBinLiteralToken (10); };
dec+ ('.' dec*) | ([eE] [+\-]? dec+)
				 { createFpToken (); };

'///' [^\n]*     { createDoxyCommentToken (TokenKind_DoxyComment1); };
'//!' [^\n]*     { createDoxyCommentToken (TokenKind_DoxyComment2); };
'/**' (any | nl)* :>> '*/'
				 { createDoxyCommentToken (TokenKind_DoxyComment3); };
'/*!' (any | nl)* :>> '*/'
				 { createDoxyCommentToken (TokenKind_DoxyComment4); };

'//' [^\n]*      ;
'/*' (any | nl)* :>> '*/'
				 ;

'('              { onLeftParentheses (); };
')'              { if (!onRightParentheses ()) fret; };
';'              { if (!onSemicolon ()) fcall fmt_spec; };

ws | nl          ;
print            { createToken (ts [0]); };
any              { createErrorToken (ts [0]); };

*|;

}%%

//..............................................................................

void
Lexer::init ()
{
	%% write init;
}

void
Lexer::exec ()
{
	%% write exec;
}

//..............................................................................

} // namespace ct
} // namespace jnc