Added guard macro for linux libdispatch

Added guard macro for linux libdispatch

/*! @file turtle.rl
 *
 *  Base file for the `Ragel's` preprocessor. It builds up on the lexer.h abstract
 *  interface.
 *
 *  @author Tomas Michalek <tomas.michalek.st@vsb.cz>
 *  @date   2015
 *
 *  @TODO column guard
 *  @FIXME Not all the linebreaks are being counted, causing misaligned lines
 */

/* vim: set ts=8 sw=4 tw=0 noet : set foldmethod=marker */

// HEADERS {{{ --------------------------------------------------------------------
#include "config.h"
#include <unistd.h>             /* sleep() */
#include <errno.h>              /* Included for 'stderr' */
#include <ctype.h>              /* isalnum() ... */
#include <iso646.h>             /* verbose || and && */
#include <string.h>             /* C buffer manipulation functions */
#include <assert.h>             /* Asserts in the code */
#include <dispatch/dispatch.h>  /* Clang GCD dispatch_* functions */

#include <ardp/lexer.h>         /* Generic lexer */
#include <ardp/lexer.turtle.h>  /* Turtle specific lexer constants */

#include <ardp/util.h>
#include <ardp/color.h>
#include <ardp/string.h>

//#include "config.h"

//------------------------------------------------------------------------------}}}

#define BUFSIZE ( 0x10000 )

/*!
 * Global variable holding the shared lexer;
 */
static struct lexer *_Nullable shared_lexer;

/*! Holds the column reference point */
static uint8_t* column;

/*!
 * Converst constrained string buffer to unsigned integer.
 *
 * @param[in] src String to probe.
 * @param[in] len Length to probe.
 *
 * @return UTF-8 codepoint.
 */
static uint32_t hex(const unsigned char *src, unsigned int len) {
    uint32_t i = 0;
    for (uint8_t j = 0; j < len; j++) {
        i *= 16;
        uint8_t c = src[j];
        if (isdigit(c)) {
            i += c - '0';
        } else if (isxdigit(c)) {
                i += c - ((isupper(c) ? 'A' : 'a')) + 10;
        }
    }
    return i;
}

/*!
 * Shorthand to speedup token input.
 */
#define var( s ) ( shared_lexer->env.s )

/* clang-format off */
/* ragel-machine() {{{*/
%%{
    machine turtle;
    include rdf_turtle_tokens "../grammars/rdf.turtle.tokens.rl";

    alphtype unsigned char;
    access shared_lexer->env.;

    action u8_create {
        if (shared_lexer->string != NULL)
                string_dealloc(shared_lexer->string);
        shared_lexer->string = string_new();
    }

    action u8_finish {
            if (shared_lexer->string) {
                string_finish(shared_lexer->string);
            }
    }

    action u8_push {
            if (shared_lexer->string) {
                string_append_char(&shared_lexer->string, fc);
            }
    }

    action u8_check {
                if (shared_lexer->string) {
                        if (shared_lexer->string[string_strlen(shared_lexer->string)] == '.') {
                                struct string_header *x = string_hdr(shared_lexer->string);
                                shared_lexer->string[x->length--] = '\0';
                        }
                }
    }

    action mark {
        mark = p;
    }

    action unescape {
        uint32_t codepoint = hex(mark, p - mark);
        mark = NULL;
        if (likely(shared_lexer->string))
                if (!string_append_utf8(&shared_lexer->string, codepoint))
                        shared_lexer->finished = 1;
    }

    # TERMINALS

    UCHAR     = '\\' . ( ('u' xdigit{4} >mark %unescape) | ('U' xdigit{8} >mark %unescape) )
              ;

    ECHAR     = '\\'. (
                        ('t' ${
                                if (shared_lexer->string)
                                        string_append_char(&shared_lexer->string, '\t');
                              }
                        )
                        |
                        ('b' ${
                                if (shared_lexer->string)
                                        string_append_char(&shared_lexer->string, '\b');
                              }
                        )
                        |
                        ('n' ${
                                if (shared_lexer->string)
                                        string_append_char(&shared_lexer->string, '\n');
                              }
                        )
                        |
                        ('r' ${
                                if (shared_lexer->string)
                                        string_append_char(&shared_lexer->string, '\r');
                              }
                        )
                        |
                        ('f' ${
                                if (shared_lexer->string)
                                        string_append_char(&shared_lexer->string, '\f');
                              }
                        )
                        |
                        ('"' ${
                                if (shared_lexer->string)
                                        string_append_char(&shared_lexer->string, '\"');
                              }
                        )
                        |
                        ('\'' ${
                                if (shared_lexer->string)
                                        string_append_char(&shared_lexer->string, '\'');
                              }
                        )
                        |
                        ('\\' ${
                                if (shared_lexer->string)
                                        string_append_char(&shared_lexer->string, '\\');
                              }
                        )
              )
              ;

    # FIXME: This is not as specification specifies the IRIREF, is it enought ?
    IRI_VALUE = (^([<>{}\"`^\\] | 0x00 .. 0x20) $u8_push | UCHAR )* >u8_create %u8_finish;

    #IRIREF    = '<' (PN_CHARS | '.' | ':' | '/' | '\\' | '#' | '@' | '%' | '&' | UCHAR)* '>';
    IRIREF = '<' IRI_VALUE :>> '>' %! { ardp_fprintf(stderr, kARDPColorCyan, "Lexer error on line: %lld, reason: IRI value contains forbidden characters.\n", shared_lexer->line); };

    STRING_VALUE_QUOTE             = ( ^(0x22 | 0x5C | 0xA | 0xD) $u8_push | ECHAR | UCHAR )*;
    STRING_VALUE_SINGLE_QUOTE      = ( ^(0x27 | 0x5C | 0xA | 0xD) $u8_push | ECHAR | UCHAR )*;

    STRING_VALUE_LONG_QUOTE        = ( ('"' | '""')? (^('"' | '\\' | "'''") $u8_push | ECHAR | UCHAR) )*;
    STRING_VALUE_LONG_SINGLE_QUOTE = ( ("'" | "''")? (^("'" | '\\' | '"""') $u8_push | ECHAR | UCHAR) )*;

    STR_Q    = ( STRING_VALUE_QUOTE        >u8_create %u8_finish );
    STR_DQ   = ( STRING_VALUE_SINGLE_QUOTE >u8_create %u8_finish );

    STR_LQ   = ( STRING_VALUE_LONG_QUOTE        >u8_create %u8_finish );
    STR_LDQ  = ( STRING_VALUE_LONG_SINGLE_QUOTE >u8_create %u8_finish );

    STRING_LITERAL_QUOTE        = '"' STR_Q :>> '"'  %! { ardp_fprintf(stderr, kARDPColorCyan, "Lexer error on line:%lld STRING value contains forbidden characters.\n", shared_lexer->line); };
    STRING_LITERAL_SINGLE_QUOTE = "'" STR_DQ :>> "'"  %! { ardp_fprintf(stderr, kARDPColorCyan, "Lexer error on line:%lld: String value contains forbidden characters.\n", shared_lexer->line); };

    STRING_LITERAL_LONG_QUOTE        = '"""' . STR_LQ :>  '"""';
    STRING_LITERAL_LONG_SINGLE_QUOTE = "'''" . STR_LDQ :> "'''";

    BLANK_NODE_LABEL = '_:' >u8_create
                        (PN_CHARS_U $u8_push | digit $u8_push )
                              (
                                (((PN_CHARS | '.')*  (PN_CHARS)) $u8_push)
                              )?

                        %u8_finish;

    QNAME     = (PN_PREFIX? ':' PN_LOCAL?) >u8_create %u8_finish $u8_push;

    #
    ## Scanner (Lexer with backtracking)
    #

    main := |*

        PREFIX        => { lexer_emit_token_const(PREFIX); };
        SPARQL_PREFIX => { lexer_emit_token_const(SPARQL_PREFIX); };
        BASE          => { lexer_emit_token_const(BASE); };
        SPARQL_BASE   => { lexer_emit_token_const(SPARQL_BASE); };

        BLANK_NODE_LABEL  { lexer_emit_u8_token(BLANK_LITERAL); };
        QNAME             { lexer_emit_u8_token(QNAME); };
        IRIREF            { lexer_emit_u8_token(IRIREF); };

        INTEGER { lexer_emit_token(INTEGER_LITERAL, var(ts), var(te) - var(ts)); };
        DECIMAL { lexer_emit_token(DECIMAL_LITERAL, var(ts), var(te) - var(ts)); };
        DOUBLE  { lexer_emit_token( DOUBLE_LITERAL, var(ts), var(te) - var(ts)); };
        BOOLEAN { lexer_emit_token(BOOLEAN_LITERAL, var(ts), var(te) - var(ts)); };

        STRING_LITERAL_LONG_QUOTE        => { lexer_emit_u8_token(STRING_LITERAL); };
        STRING_LITERAL_LONG_SINGLE_QUOTE => { lexer_emit_u8_token(STRING_LITERAL); };

        STRING_LITERAL_QUOTE        { lexer_emit_u8_token(STRING_LITERAL); };
        STRING_LITERAL_SINGLE_QUOTE { lexer_emit_u8_token(STRING_LITERAL); };

        HAT => {
              lexer_emit_token_const(HAT);
        };
        LANGTAG => {
              // '@'. VALUE
              lexer_emit_token(LANGTAG, var(ts)+1, var(te) - (var(ts)+1));
        };

        A         => { lexer_emit_token_const(A); };
        DOT       => { lexer_emit_token_const(DOT); };
        COMMA     => { lexer_emit_token_const(COMMA); };
        SEMICOLON => { lexer_emit_token_const(SEMICOLON); };

        L_SQUARE  => { lexer_emit_token_const(L_SQUARE); };
        R_SQUARE  => { lexer_emit_token_const(R_SQUARE); };
        L_ROUND   => { lexer_emit_token_const(L_ROUND);  };
        R_ROUND   => { lexer_emit_token_const(R_ROUND);  };
        L_CURLY   => { lexer_emit_token_const(L_CURLY);  };
        R_CURLY   => { lexer_emit_token_const(R_CURLY);  };

        EOL => {
              dispatch_sync(shared_lexer->event_queue, ^{
                  shared_lexer->line++;
              });
        };

        COMMENT {
              dispatch_sync(shared_lexer->event_queue, ^{
                  shared_lexer->line++;
              });
        };

        #WS*;
        any; #invalids to be skiped
    *|;

    write data;
}%%
/* }}} */
/* clang-format on */

/*--------------------------------------------------------------------------------*/
// LOCAL FUNCTIONS, HELPERS, SHORTHANDS
/*--------------------------------------------------------------------------------*/
/* log() {{{ */
static void log(int level, const char* message)
{
    if (shared_lexer->log.level >= level )
        if (shared_lexer->log.eprintf) {
                shared_lexer->log.eprintf(level, message);
        }
}
/*}}}*/
/* lexer_emit_u8_token() {{{*/
static void lexer_emit_u8_token(enum turtle_token_type type)
{
        if (likely(shared_lexer->cb.stoken)) {
                /*__block*/ utf8 s;
                if (shared_lexer->string)
                         s = string_copy(shared_lexer->string);
                else
                         s = NULL;
                dispatch_async( shared_lexer->event_queue, ^{
                        shared_lexer->cb.stoken(type, s, shared_lexer->line, 0);
                        log (DEBUG, "Emitted token (UTF8)");
                });
        }
}
/*}}}*/
/* lexer_emit_token() {{{ */
static void lexer_emit_token( enum turtle_token_type type, uint8_t *_Nullable str, size_t len )
{
        if (likely(shared_lexer->cb.stoken)) {
                // dispatch_barrier_async(shared_lexer->lexer_queue, ^{
                        utf8 s = string_create_n(str,len,len);

                        dispatch_async( shared_lexer->event_queue, ^{
                                shared_lexer->cb.stoken( type, s, shared_lexer->line, 0);
                                log( DEBUG, "Token emmitted" );
                        });
                // });
        }
}
/*}}}*/
/* lexer_emit_token_const() {{{ */
static void lexer_emit_token_const( enum turtle_token_type type )
{
        if ( likely(shared_lexer->cb.stoken) ) {
            dispatch_async( shared_lexer->event_queue, ^{
                shared_lexer->cb.stoken( type, NULL, shared_lexer->line, 0 );
                log( DEBUG, "Token emmitted");
            });
        }
}
/*}}}*/
/* ardp_lexer_turtle_debug() {{{ */
void ardp_lexer_turtle_debug( void )
{
        const char states[][15] = {"created", "initialized", "ready", "unknown"};
        fprintf(stderr, "Shared lexer %s created.\n", shared_lexer ? "is" : "isn't" );
        fprintf(stderr, "It's state is: %s\n", states[shared_lexer->state] );
}
/*}}}*/

/* dispatch_queue_isempty() {{{ */
bool dispatch_queue_isempty(dispatch_queue_t queue)
{
    dispatch_group_t group = dispatch_group_create();

    dispatch_group_enter(group);
    dispatch_async(queue, ^{
        dispatch_group_leave(group);
    });

    int64_t maxWaitTime = 5 * NSEC_PER_SEC;
    bool isReady = dispatch_group_wait(group, maxWaitTime) == 0;

    dispatch_async(dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_LOW, 0), ^{
        dispatch_group_wait(group, DISPATCH_TIME_FOREVER);
        dispatch_release(group);
    });

    return isReady;
}
/*}}}*/

/*--------------------------------------------------------------------------------*/
// EXTERN FUNCTIONS
/*--------------------------------------------------------------------------------*/
/* ardp_lexer_create() {{{ */
int ardp_lexer_create(void)
{
        if ( unlikely(shared_lexer isnt NULL) ) {
            log( WARNING, "The lexer did already exist. Destroying it." );
            ardp_lexer_destroy();
        }

        shared_lexer = calloc(1, sizeof(struct lexer)); // malloc(sizeof(struct lexer));

        if ( unlikely(shared_lexer is NULL) ) {
            log( ERROR, "Didn't allocate the shared lexer. (MALLOC ERROR)");
            //return ARDP_ERROR_LEXER_MALLOC;
            return ARDP_FAILURE;
        }

        shared_lexer->lexer_queue =
            dispatch_queue_create( "eu.cre8iv.ardp.Lexer", 0  /*DISPATCH_QUEUE_SERIAL*/ );

#ifdef ARDP_ON_OSX
        shared_lexer->event_queue =
            dispatch_queue_create( "eu.cre8iv.ardp.Events", DISPATCH_QUEUE_SERIAL );
#else
        shared_lexer->event_queue =
                dispatch_queue_create("eu.cre8iv.ardp.Events", NULL);
#endif
        shared_lexer->state = ARDP_LEXER_TURTLE_STATUS_CREATED;
        shared_lexer->finished = 1;

        log ( INFO, "Created the shared lexer." );
        return ARDP_SUCCESS;
}
/*}}}*/
/* ardp_lexer_defaults() {{{ */
int ardp_lexer_defaults(void)
{
    switch(ardp_lexer_state()) {
    case ARDP_LEXER_TURTLE_STATUS_CREATED:
    {
        struct lexer *this = shared_lexer;
                      this->line  = 1;
                      this->state = ARDP_LEXER_TURTLE_STATUS_PREINITIALIZED;

        column = NULL;

        log(INFO, "Defaults for loaded");
        return ARDP_SUCCESS;
    }
    default:
        log(ERROR, "Invalid state of the lexer. Defaults not loaded!");
        return ARDP_FAILURE;
    }
}
/*}}}*/
/* ardp_lexer_init() {{{*/
int ardp_lexer_init( struct ardp_lexer_config *_Nullable cfg)
{
    switch( ardp_lexer_state() ) {
    case ARDP_LEXER_TURTLE_STATUS_PREINITIALIZED:
    case ARDP_LEXER_TURTLE_STATUS_READY:
        /* clang-format off */
        %% write init;
        /* clang-format on */
        shared_lexer->cb.token    = cfg->cb.token;
        shared_lexer->cb.stoken   = cfg->cb.stoken;
        shared_lexer->log.eprintf = cfg->logging.eprintf;
        shared_lexer->log.level   = cfg->logging.level;

        shared_lexer->state = ARDP_LEXER_TURTLE_STATUS_READY;
        log( INFO, "Shared lexer initialized." );
        return ARDP_SUCCESS;

    default:
        log( ERROR, "Wrong state. Initialization aborted." );
        return ARDP_FAILURE;
    }
}
/*}}}*/
/* ardp_lexer_destroy() {{{ */
void ardp_lexer_destroy()
{
        if ( shared_lexer isnt NULL ) {
                if (shared_lexer->string)
                        string_dealloc(shared_lexer->string);
                free( shared_lexer );
                log(INFO, "Freed the shared lexer.");
        }
}

int ardp_lexer_state()
{
        if ( shared_lexer is NULL ) {
                return ARDP_LEXER_TURTLE_STATUS_UNKNOWN;
        } else {
                __block int state;
                state = shared_lexer->state;
                return state;
        }
}
/*}}}*/
/* ardp_lexer_is_ready() {{{ */
bool ardp_lexer_is_ready()
{
        return ( ardp_lexer_state() is ARDP_LEXER_TURTLE_STATUS_READY );
}
/*}}}*/

/* ardp_lexer_process_block() {{{ */
int ardp_lexer_process_block( uint8_t *_Nullable v,
                              size_t             len,
                              uint8_t *_Nullable mark,
                              bool               is_eof)
{
    if ( not ardp_lexer_is_ready() ) {
        log (ERROR, "Shared lexer is note yet ready to process the data.");
        return ARDP_FAILURE;
    }

    /*__block*/ uint8_t *_Nullable p  = v; /* Indirection to allow the GCD */
    const   uint8_t *_Nullable pe = p + len; /* Points to fist byte beyond the data */
    const   uint8_t *_Nullable eof= (is_eof) ? pe : NULL; /* Indicated end of all data*/

    /* clang-format off */
        %% write exec;
    /* clang-format on */

    return ARDP_SUCCESS;
}
/*}}}*/
/* ardp_lexer_process_block() OLD using block {{{*/
void ardp_lexer_turtle_process_block( uint8_t *_Nullable v,
                                      size_t             len,
                                      uint8_t *_Nullable mark,
                                      bool               is_eof,
                                      completation_block handler )
{
        if ( !ardp_lexer_is_ready() ) {
                handler( ARDP_FAILURE );
                return;
        }

        __block uint8_t *_Nullable p = v; // redirection to allow GCD

        const uint8_t *_Nullable pe;
        const uint8_t *_Nullable eof;

        pe  = p + len; /* pe points to 1 byte beyond the end of this block of data */
        eof = ( is_eof ) ? pe : ( ( uint8_t * )0 ); /* Indicates the end of all data, 0 if
                                                       not in this block */

        /* clang-format off */
          %% write exec;
        /* clang-format on */
        handler( ARDP_SUCCESS );
}
/*}}}*/
/* 'ardp_lexer_process_reader() {{{ */

int ardp_lexer_process_reader( lexer_reader reader, void *_Nullable reader_args)
{
    if (reader == NULL)
        return 1;

    int status = 0;

    if ( !ardp_lexer_is_ready() )
        return 2;

    shared_lexer->finished = 0;

    uint8_t *buf = malloc(sizeof(*buf)*BUFSIZE);

    size_t   have = 0;
    uint8_t *mark = NULL;
    bool     eof  = false;

    uint8_t  nbufsize = 1;

      while (!shared_lexer->finished) {
            uint8_t *p = buf + have;
            size_t space = BUFSIZE - have;

            if (space == 0) {
                    /* Finished space while scanning token without the data; Grow buffer*/
                    uint8_t* newbuf = malloc(sizeof(*buf)*(++nbufsize * BUFSIZE));
                    p = newbuf + have;
                    space = nbufsize - have;

                    /* patch up the pointers */
                    if (shared_lexer->env.ts != 0)
                        shared_lexer->env.ts = newbuf + (shared_lexer->env.ts - buf);

                    shared_lexer->env.te = newbuf + (shared_lexer->env.te - buf);
                    memcpy( newbuf, buf, have );
                    free(buf);
                    buf = newbuf;
            }

            int len = reader(p, space, reader_args);
            if ( len < space ) {
                    eof = true;
                    shared_lexer->finished = 1;
            }

            status = ardp_lexer_process_block( p, len, mark, eof );

            if (shared_lexer->env.cs == turtle_error ) {
                    /* Machine failed before finding a token.
                     *  I'm not yet sure if this
                           * is reachable. */
                     status = ARDP_LEXER_GENERIC_ERROR;
                     if (buf)
                        free(buf);
                     return status;
            }

            /* Decide if we need to preserve anything. */
            uint8_t *preserve = shared_lexer->env.ts;
            if ( preserve == 0 ) {
                have = 0;
            } else {
                /* There is data that needs to be shifted over. */
                have = (p+len) - preserve;
                memmove( buf, preserve, have );
                unsigned int shiftback = preserve - buf;
                if ( shared_lexer->env.ts != 0 )
                    shared_lexer->env.ts -= shiftback;
                shared_lexer->env.te -= shiftback;

                preserve = buf;
            }
      }

        if (buf)
          free(buf);

        //dispatch_queue_t *q = &shared_lexer->lexer_queue;
        //dispatch_queue_t *e = &shared_lexer->event_queue;

        // Optimalization to free CPU operations
        //while(!dispatch_queue_isempty(*q) || !dispatch_queue_isempty(*e))
        //        usleep(25);

        return status;
}

int ardp_lexer_process_reader_BUG( lexer_reader reader, void *_Nullable reader_args)
{
        if ( reader == NULL )
                return 1;

        int status = 0;

        if ( !ardp_lexer_is_ready() )
                return 2;

        shared_lexer->finished = 0;

        uint8_t buf[BUFSIZE];

        size_t   have = 0;
        uint8_t *mark = NULL;
        bool     eof  = false;

        while (!shared_lexer->finished) {
               uint8_t *p = buf + have;

               size_t space = BUFSIZE - have;
               if (space <= 0) {
                       status = ARDP_LEXER_OMEM; /* Out of the buffer space */
                       break;                    /* goto exit; */
               }

               int len = reader(p, space, reader_args);
               if (len < space) {
                       eof = true;
                       shared_lexer->finished = 1;
               }

               status = ardp_lexer_process_block( p, len, mark, eof );
               if (shared_lexer->env.cs == turtle_error ) {
                        status = ARDP_LEXER_GENERIC_ERROR;
                        break;
               } else if (mark) {
                        have = ( p + len ) - mark;
                        memmove( buf, mark, have );
                        mark = buf;
               } else {
                        have = 0;
               }
        }

        dispatch_queue_t *q = &shared_lexer->lexer_queue;
        dispatch_queue_t *e = &shared_lexer->event_queue;

        // Optimalization to free CPU operations
        while(!dispatch_queue_isempty(*q) || !dispatch_queue_isempty(*e))
                usleep(25);

        return status;
}
/*}}}*/
/* ardp_lexer_process_reader_old()  {{{ */
void ardp_lexer_process_reader_old( lexer_reader reader,
                                       void *_Nullable reader_args,
                                       completation_block handler )
{
        if ( ardp_lexer_is_ready() ) {
                handler( ARDP_LEXER_ERROR );
                return;
        }

        if ( reader is NULL ) {
                handler( ARDP_LEXER_READER_NOT_READY );
                return;
        }

        __block uint8_t *p;

        uint8_t buf[BUFSIZE];
        size_t have   = 0;
        uint8_t *mark = NULL; //nullptr;
        bool eof      = false;

        int status = 0;

        while ( not shared_lexer->finished ) {
                p = buf + have;

                size_t space = BUFSIZE - have;
                if ( space <= 0 ) {
                        /* Out of buffer space */
                        status = ARDP_LEXER_OMEM;
                        break; /* goto exit */
                }

                int len = reader( p, space, reader_args );

                if ( len < space ) {
                        eof                    = true;
                        shared_lexer->finished = 1;
                }

                ardp_lexer_turtle_process_block( p, len, mark, eof, ^( int success ) {
                  if ( success isnt ARDP_SUCCESS )
                          printf( "Internal lexer error - recoverable\n" );
                } );

                if ( shared_lexer->env.cs is turtle_error ) {
                        status = ARDP_LEXER_GENERIC_ERROR;
                        break; /* goto exit */
                } else if ( mark ) {
                        have = ( p + len ) - mark;
                        memmove( buf, mark, have );
                        mark = buf;
                } else {
                        have = 0;
                }
        }
        handler( status );
}
/*}}}*/

/*! @file turtle.rl
 *
 *  Base file for the `Ragel's` preprocessor. It builds up on the lexer.h abstract
 *  interface.
 *
 *  @author Tomas Michalek <tomas.michalek.st@vsb.cz>
 *  @date   2015
 *
 *  @TODO column guard
 *  @FIXME Not all the linebreaks are being counted, causing misaligned lines
 */

/* vim: set ts=8 sw=4 tw=0 noet : set foldmethod=marker */

// HEADERS {{{ --------------------------------------------------------------------

#include <unistd.h>             /* sleep() */
#include <errno.h>              /* Included for 'stderr' */
#include <ctype.h>              /* isalnum() ... */
#include <iso646.h>             /* verbose || and && */
#include <string.h>             /* C buffer manipulation functions */
#include <assert.h>             /* Asserts in the code */
#include <dispatch/dispatch.h>  /* Clang GCD dispatch_* functions */

#include <ardp/lexer.h>         /* Generic lexer */
#include <ardp/lexer.turtle.h>  /* Turtle specific lexer constants */

#include <ardp/util.h>
#include <ardp/color.h>
#include <ardp/string.h>

//#include "config.h"

//------------------------------------------------------------------------------}}}

#define BUFSIZE ( 0x10000 )

/*!
 * Global variable holding the shared lexer;
 */
static struct lexer *_Nullable shared_lexer;

/*! Holds the column reference point */
static uint8_t* column;

/*!
 * Converst constrained string buffer to unsigned integer.
 *
 * @param[in] src String to probe.
 * @param[in] len Length to probe.
 *
 * @return UTF-8 codepoint.
 */
static uint32_t hex(const unsigned char *src, unsigned int len) {
    uint32_t i = 0;
    for (uint8_t j = 0; j < len; j++) {
        i *= 16;
        uint8_t c = src[j];
        if (isdigit(c)) {
            i += c - '0';
        } else if (isxdigit(c)) {
                i += c - ((isupper(c) ? 'A' : 'a')) + 10;
        }
    }
    return i;
}

/*!
 * Shorthand to speedup token input.
 */
#define var( s ) ( shared_lexer->env.s )

/* clang-format off */
/* ragel-machine() {{{*/
%%{
    machine turtle;
    include rdf_turtle_tokens "../grammars/rdf.turtle.tokens.rl";

    alphtype unsigned char;
    access shared_lexer->env.;

    action u8_create {
        if (shared_lexer->string != NULL)
                string_dealloc(shared_lexer->string);
        shared_lexer->string = string_new();
    }

    action u8_finish {
            if (shared_lexer->string) {
                string_finish(shared_lexer->string);
            }
    }

    action u8_push {
            if (shared_lexer->string) {
                string_append_char(&shared_lexer->string, fc);
            }
    }

    action u8_check {
                if (shared_lexer->string) {
                        if (shared_lexer->string[string_strlen(shared_lexer->string)] == '.') {
                                struct string_header *x = string_hdr(shared_lexer->string);
                                shared_lexer->string[x->length--] = '\0';
                        }
                }
    }

    action mark {
        mark = p;
    }

    action unescape {
        uint32_t codepoint = hex(mark, p - mark);
        mark = NULL;
        if (likely(shared_lexer->string))
                if (!string_append_utf8(&shared_lexer->string, codepoint))
                        shared_lexer->finished = 1;
    }

    # TERMINALS

    UCHAR     = '\\' . ( ('u' xdigit{4} >mark %unescape) | ('U' xdigit{8} >mark %unescape) )
              ;

    ECHAR     = '\\'. (
                        ('t' ${
                                if (shared_lexer->string)
                                        string_append_char(&shared_lexer->string, '\t');
                              }
                        )
                        |
                        ('b' ${
                                if (shared_lexer->string)
                                        string_append_char(&shared_lexer->string, '\b');
                              }
                        )
                        |
                        ('n' ${
                                if (shared_lexer->string)
                                        string_append_char(&shared_lexer->string, '\n');
                              }
                        )
                        |
                        ('r' ${
                                if (shared_lexer->string)
                                        string_append_char(&shared_lexer->string, '\r');
                              }
                        )
                        |
                        ('f' ${
                                if (shared_lexer->string)
                                        string_append_char(&shared_lexer->string, '\f');
                              }
                        )
                        |
                        ('"' ${
                                if (shared_lexer->string)
                                        string_append_char(&shared_lexer->string, '\"');
                              }
                        )
                        |
                        ('\'' ${
                                if (shared_lexer->string)
                                        string_append_char(&shared_lexer->string, '\'');
                              }
                        )
                        |
                        ('\\' ${
                                if (shared_lexer->string)
                                        string_append_char(&shared_lexer->string, '\\');
                              }
                        )
              )
              ;

    # FIXME: This is not as specification specifies the IRIREF, is it enought ?
    IRI_VALUE = (^([<>{}\"`^\\] | 0x00 .. 0x20) $u8_push | UCHAR )* >u8_create %u8_finish;

    #IRIREF    = '<' (PN_CHARS | '.' | ':' | '/' | '\\' | '#' | '@' | '%' | '&' | UCHAR)* '>';
    IRIREF = '<' IRI_VALUE :>> '>' %! { ardp_fprintf(stderr, kARDPColorCyan, "Lexer error on line: %lld, reason: IRI value contains forbidden characters.\n", shared_lexer->line); };

    STRING_VALUE_QUOTE             = ( ^(0x22 | 0x5C | 0xA | 0xD) $u8_push | ECHAR | UCHAR )*;
    STRING_VALUE_SINGLE_QUOTE      = ( ^(0x27 | 0x5C | 0xA | 0xD) $u8_push | ECHAR | UCHAR )*;

    STRING_VALUE_LONG_QUOTE        = ( ('"' | '""')? (^('"' | '\\' | "'''") $u8_push | ECHAR | UCHAR) )*;
    STRING_VALUE_LONG_SINGLE_QUOTE = ( ("'" | "''")? (^("'" | '\\' | '"""') $u8_push | ECHAR | UCHAR) )*;

    STR_Q    = ( STRING_VALUE_QUOTE        >u8_create %u8_finish );
    STR_DQ   = ( STRING_VALUE_SINGLE_QUOTE >u8_create %u8_finish );

    STR_LQ   = ( STRING_VALUE_LONG_QUOTE        >u8_create %u8_finish );
    STR_LDQ  = ( STRING_VALUE_LONG_SINGLE_QUOTE >u8_create %u8_finish );

    STRING_LITERAL_QUOTE        = '"' STR_Q :>> '"'  %! { ardp_fprintf(stderr, kARDPColorCyan, "Lexer error on line:%lld STRING value contains forbidden characters.\n", shared_lexer->line); };
    STRING_LITERAL_SINGLE_QUOTE = "'" STR_DQ :>> "'"  %! { ardp_fprintf(stderr, kARDPColorCyan, "Lexer error on line:%lld: String value contains forbidden characters.\n", shared_lexer->line); };

    STRING_LITERAL_LONG_QUOTE        = '"""' . STR_LQ :>  '"""';
    STRING_LITERAL_LONG_SINGLE_QUOTE = "'''" . STR_LDQ :> "'''";

    BLANK_NODE_LABEL = '_:' >u8_create
                        (PN_CHARS_U $u8_push | digit $u8_push )
                              (
                                (((PN_CHARS | '.')*  (PN_CHARS)) $u8_push)
                              )?

                        %u8_finish;

    QNAME     = (PN_PREFIX? ':' PN_LOCAL?) >u8_create %u8_finish $u8_push;

    #
    ## Scanner (Lexer with backtracking)
    #

    main := |*

        PREFIX        => { lexer_emit_token_const(PREFIX); };
        SPARQL_PREFIX => { lexer_emit_token_const(SPARQL_PREFIX); };
        BASE          => { lexer_emit_token_const(BASE); };
        SPARQL_BASE   => { lexer_emit_token_const(SPARQL_BASE); };

        BLANK_NODE_LABEL  { lexer_emit_u8_token(BLANK_LITERAL); };
        QNAME             { lexer_emit_u8_token(QNAME); };
        IRIREF            { lexer_emit_u8_token(IRIREF); };

        INTEGER { lexer_emit_token(INTEGER_LITERAL, var(ts), var(te) - var(ts)); };
        DECIMAL { lexer_emit_token(DECIMAL_LITERAL, var(ts), var(te) - var(ts)); };
        DOUBLE  { lexer_emit_token( DOUBLE_LITERAL, var(ts), var(te) - var(ts)); };
        BOOLEAN { lexer_emit_token(BOOLEAN_LITERAL, var(ts), var(te) - var(ts)); };

        STRING_LITERAL_LONG_QUOTE        => { lexer_emit_u8_token(STRING_LITERAL); };
        STRING_LITERAL_LONG_SINGLE_QUOTE => { lexer_emit_u8_token(STRING_LITERAL); };

        STRING_LITERAL_QUOTE        { lexer_emit_u8_token(STRING_LITERAL); };
        STRING_LITERAL_SINGLE_QUOTE { lexer_emit_u8_token(STRING_LITERAL); };

        HAT => {
              lexer_emit_token_const(HAT);
        };
        LANGTAG => {
              // '@'. VALUE
              lexer_emit_token(LANGTAG, var(ts)+1, var(te) - (var(ts)+1));
        };

        A         => { lexer_emit_token_const(A); };
        DOT       => { lexer_emit_token_const(DOT); };
        COMMA     => { lexer_emit_token_const(COMMA); };
        SEMICOLON => { lexer_emit_token_const(SEMICOLON); };

        L_SQUARE  => { lexer_emit_token_const(L_SQUARE); };
        R_SQUARE  => { lexer_emit_token_const(R_SQUARE); };
        L_ROUND   => { lexer_emit_token_const(L_ROUND);  };
        R_ROUND   => { lexer_emit_token_const(R_ROUND);  };
        L_CURLY   => { lexer_emit_token_const(L_CURLY);  };
        R_CURLY   => { lexer_emit_token_const(R_CURLY);  };

        EOL => {
              dispatch_sync(shared_lexer->event_queue, ^{
                  shared_lexer->line++;
              });
        };

        COMMENT {
              dispatch_sync(shared_lexer->event_queue, ^{
                  shared_lexer->line++;
              });
        };

        #WS*;
        any; #invalids to be skiped
    *|;

    write data;
}%%
/* }}} */
/* clang-format on */

/*--------------------------------------------------------------------------------*/
// LOCAL FUNCTIONS, HELPERS, SHORTHANDS
/*--------------------------------------------------------------------------------*/
/* log() {{{ */
static void log(int level, const char* message)
{
    if (shared_lexer->log.level >= level )
        if (shared_lexer->log.eprintf) {
                shared_lexer->log.eprintf(level, message);
        }
}
/*}}}*/
/* lexer_emit_u8_token() {{{*/
static void lexer_emit_u8_token(enum turtle_token_type type)
{
        if (likely(shared_lexer->cb.stoken)) {
                /*__block*/ utf8 s;
                if (shared_lexer->string)
                         s = string_copy(shared_lexer->string);
                else
                         s = NULL;
                dispatch_async( shared_lexer->event_queue, ^{
                        shared_lexer->cb.stoken(type, s, shared_lexer->line, 0);
                        log (DEBUG, "Emitted token (UTF8)");
                });
        }
}
/*}}}*/
/* lexer_emit_token() {{{ */
static void lexer_emit_token( enum turtle_token_type type, uint8_t *_Nullable str, size_t len )
{
        if (likely(shared_lexer->cb.stoken)) {
                // dispatch_barrier_async(shared_lexer->lexer_queue, ^{
                        utf8 s = string_create_n(str,len,len);

                        dispatch_async( shared_lexer->event_queue, ^{
                                shared_lexer->cb.stoken( type, s, shared_lexer->line, 0);
                                log( DEBUG, "Token emmitted" );
                        });
                // });
        }
}
/*}}}*/
/* lexer_emit_token_const() {{{ */
static void lexer_emit_token_const( enum turtle_token_type type )
{
        if ( likely(shared_lexer->cb.stoken) ) {
            dispatch_async( shared_lexer->event_queue, ^{
                shared_lexer->cb.stoken( type, NULL, shared_lexer->line, 0 );
                log( DEBUG, "Token emmitted");
            });
        }
}
/*}}}*/
/* ardp_lexer_turtle_debug() {{{ */
void ardp_lexer_turtle_debug( void )
{
        const char states[][15] = {"created", "initialized", "ready", "unknown"};
        fprintf(stderr, "Shared lexer %s created.\n", shared_lexer ? "is" : "isn't" );
        fprintf(stderr, "It's state is: %s\n", states[shared_lexer->state] );
}
/*}}}*/

/* dispatch_queue_isempty() {{{ */
bool dispatch_queue_isempty(dispatch_queue_t queue)
{
    dispatch_group_t group = dispatch_group_create();

    dispatch_group_enter(group);
    dispatch_async(queue, ^{
        dispatch_group_leave(group);
    });

    int64_t maxWaitTime = 5 * NSEC_PER_SEC;
    bool isReady = dispatch_group_wait(group, maxWaitTime) == 0;

    dispatch_async(dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_LOW, 0), ^{
        dispatch_group_wait(group, DISPATCH_TIME_FOREVER);
        dispatch_release(group);
    });

    return isReady;
}
/*}}}*/

/*--------------------------------------------------------------------------------*/
// EXTERN FUNCTIONS
/*--------------------------------------------------------------------------------*/
/* ardp_lexer_create() {{{ */
int ardp_lexer_create(void)
{
        if ( unlikely(shared_lexer isnt NULL) ) {
            log( WARNING, "The lexer did already exist. Destroying it." );
            ardp_lexer_destroy();
        }

        shared_lexer = calloc(1, sizeof(struct lexer)); // malloc(sizeof(struct lexer));

        if ( unlikely(shared_lexer is NULL) ) {
            log( ERROR, "Didn't allocate the shared lexer. (MALLOC ERROR)");
            //return ARDP_ERROR_LEXER_MALLOC;
            return ARDP_FAILURE;
        }

        shared_lexer->lexer_queue =
            dispatch_queue_create( "eu.cre8iv.ardp.Lexer", 0  /*DISPATCH_QUEUE_SERIAL*/ );

#ifdef ARDP_ON_OSX
        shared_lexer->event_queue =
            dispatch_queue_create( "eu.cre8iv.ardp.Events", DISPATCH_QUEUE_SERIAL );
#else
        shared_lexer->event_queue =
                dispatch_queue_create("eu.cre8iv.ardp.Events", NULL);
#endif
        shared_lexer->state = ARDP_LEXER_TURTLE_STATUS_CREATED;
        shared_lexer->finished = 1;

        log ( INFO, "Created the shared lexer." );
        return ARDP_SUCCESS;
}
/*}}}*/
/* ardp_lexer_defaults() {{{ */
int ardp_lexer_defaults(void)
{
    switch(ardp_lexer_state()) {
    case ARDP_LEXER_TURTLE_STATUS_CREATED:
    {
        struct lexer *this = shared_lexer;
                      this->line  = 1;
                      this->state = ARDP_LEXER_TURTLE_STATUS_PREINITIALIZED;

        column = NULL;

        log(INFO, "Defaults for loaded");
        return ARDP_SUCCESS;
    }
    default:
        log(ERROR, "Invalid state of the lexer. Defaults not loaded!");
        return ARDP_FAILURE;
    }
}
/*}}}*/
/* ardp_lexer_init() {{{*/
int ardp_lexer_init( struct ardp_lexer_config *_Nullable cfg)
{
    switch( ardp_lexer_state() ) {
    case ARDP_LEXER_TURTLE_STATUS_PREINITIALIZED:
    case ARDP_LEXER_TURTLE_STATUS_READY:
        /* clang-format off */
        %% write init;
        /* clang-format on */
        shared_lexer->cb.token    = cfg->cb.token;
        shared_lexer->cb.stoken   = cfg->cb.stoken;
        shared_lexer->log.eprintf = cfg->logging.eprintf;
        shared_lexer->log.level   = cfg->logging.level;

        shared_lexer->state = ARDP_LEXER_TURTLE_STATUS_READY;
        log( INFO, "Shared lexer initialized." );
        return ARDP_SUCCESS;

    default:
        log( ERROR, "Wrong state. Initialization aborted." );
        return ARDP_FAILURE;
    }
}
/*}}}*/
/* ardp_lexer_destroy() {{{ */
void ardp_lexer_destroy()
{
        if ( shared_lexer isnt NULL ) {
                if (shared_lexer->string)
                        string_dealloc(shared_lexer->string);
                free( shared_lexer );
                log(INFO, "Freed the shared lexer.");
        }
}

int ardp_lexer_state()
{
        if ( shared_lexer is NULL ) {
                return ARDP_LEXER_TURTLE_STATUS_UNKNOWN;
        } else {
                __block int state;
                state = shared_lexer->state;
                return state;
        }
}
/*}}}*/
/* ardp_lexer_is_ready() {{{ */
bool ardp_lexer_is_ready()
{
        return ( ardp_lexer_state() is ARDP_LEXER_TURTLE_STATUS_READY );
}
/*}}}*/

/* ardp_lexer_process_block() {{{ */
int ardp_lexer_process_block( uint8_t *_Nullable v,
                              size_t             len,
                              uint8_t *_Nullable mark,
                              bool               is_eof)
{
    if ( not ardp_lexer_is_ready() ) {
        log (ERROR, "Shared lexer is note yet ready to process the data.");
        return ARDP_FAILURE;
    }

    /*__block*/ uint8_t *_Nullable p  = v; /* Indirection to allow the GCD */
    const   uint8_t *_Nullable pe = p + len; /* Points to fist byte beyond the data */
    const   uint8_t *_Nullable eof= (is_eof) ? pe : NULL; /* Indicated end of all data*/

    /* clang-format off */
        %% write exec;
    /* clang-format on */

    return ARDP_SUCCESS;
}
/*}}}*/
/* ardp_lexer_process_block() OLD using block {{{*/
void ardp_lexer_turtle_process_block( uint8_t *_Nullable v,
                                      size_t             len,
                                      uint8_t *_Nullable mark,
                                      bool               is_eof,
                                      completation_block handler )
{
        if ( !ardp_lexer_is_ready() ) {
                handler( ARDP_FAILURE );
                return;
        }

        __block uint8_t *_Nullable p = v; // redirection to allow GCD

        const uint8_t *_Nullable pe;
        const uint8_t *_Nullable eof;

        pe  = p + len; /* pe points to 1 byte beyond the end of this block of data */
        eof = ( is_eof ) ? pe : ( ( uint8_t * )0 ); /* Indicates the end of all data, 0 if
                                                       not in this block */

        /* clang-format off */
          %% write exec;
        /* clang-format on */
        handler( ARDP_SUCCESS );
}
/*}}}*/
/* 'ardp_lexer_process_reader() {{{ */

int ardp_lexer_process_reader( lexer_reader reader, void *_Nullable reader_args)
{
    if (reader == NULL)
        return 1;

    int status = 0;

    if ( !ardp_lexer_is_ready() )
        return 2;

    shared_lexer->finished = 0;

    uint8_t *buf = malloc(sizeof(*buf)*BUFSIZE);

    size_t   have = 0;
    uint8_t *mark = NULL;
    bool     eof  = false;

    uint8_t  nbufsize = 1;

      while (!shared_lexer->finished) {
            uint8_t *p = buf + have;
            size_t space = BUFSIZE - have;

            if (space == 0) {
                    /* Finished space while scanning token without the data; Grow buffer*/
                    uint8_t* newbuf = malloc(sizeof(*buf)*(++nbufsize * BUFSIZE));
                    p = newbuf + have;
                    space = nbufsize - have;

                    /* patch up the pointers */
                    if (shared_lexer->env.ts != 0)
                        shared_lexer->env.ts = newbuf + (shared_lexer->env.ts - buf);

                    shared_lexer->env.te = newbuf + (shared_lexer->env.te - buf);
                    memcpy( newbuf, buf, have );
                    free(buf);
                    buf = newbuf;
            }

            int len = reader(p, space, reader_args);
            if ( len < space ) {
                    eof = true;
                    shared_lexer->finished = 1;
            }

            status = ardp_lexer_process_block( p, len, mark, eof );

            if (shared_lexer->env.cs == turtle_error ) {
                    /* Machine failed before finding a token.
                     *  I'm not yet sure if this
                           * is reachable. */
                     status = ARDP_LEXER_GENERIC_ERROR;
                     if (buf)
                        free(buf);
                     return status;
            }

            /* Decide if we need to preserve anything. */
            uint8_t *preserve = shared_lexer->env.ts;
            if ( preserve == 0 ) {
                have = 0;
            } else {
                /* There is data that needs to be shifted over. */
                have = (p+len) - preserve;
                memmove( buf, preserve, have );
                unsigned int shiftback = preserve - buf;
                if ( shared_lexer->env.ts != 0 )
                    shared_lexer->env.ts -= shiftback;
                shared_lexer->env.te -= shiftback;

                preserve = buf;
            }
      }

        if (buf)
          free(buf);

        //dispatch_queue_t *q = &shared_lexer->lexer_queue;
        //dispatch_queue_t *e = &shared_lexer->event_queue;

        // Optimalization to free CPU operations
        //while(!dispatch_queue_isempty(*q) || !dispatch_queue_isempty(*e))
        //        usleep(25);

        return status;
}

int ardp_lexer_process_reader_BUG( lexer_reader reader, void *_Nullable reader_args)
{
        if ( reader == NULL )
                return 1;

        int status = 0;

        if ( !ardp_lexer_is_ready() )
                return 2;

        shared_lexer->finished = 0;

        uint8_t buf[BUFSIZE];

        size_t   have = 0;
        uint8_t *mark = NULL;
        bool     eof  = false;

        while (!shared_lexer->finished) {
               uint8_t *p = buf + have;

               size_t space = BUFSIZE - have;
               if (space <= 0) {
                       status = ARDP_LEXER_OMEM; /* Out of the buffer space */
                       break;                    /* goto exit; */
               }

               int len = reader(p, space, reader_args);
               if (len < space) {
                       eof = true;
                       shared_lexer->finished = 1;
               }

               status = ardp_lexer_process_block( p, len, mark, eof );
               if (shared_lexer->env.cs == turtle_error ) {
                        status = ARDP_LEXER_GENERIC_ERROR;
                        break;
               } else if (mark) {
                        have = ( p + len ) - mark;
                        memmove( buf, mark, have );
                        mark = buf;
               } else {
                        have = 0;
               }
        }

        dispatch_queue_t *q = &shared_lexer->lexer_queue;
        dispatch_queue_t *e = &shared_lexer->event_queue;

        // Optimalization to free CPU operations
        while(!dispatch_queue_isempty(*q) || !dispatch_queue_isempty(*e))
                usleep(25);

        return status;
}
/*}}}*/
/* ardp_lexer_process_reader_old()  {{{ */
void ardp_lexer_process_reader_old( lexer_reader reader,
                                       void *_Nullable reader_args,
                                       completation_block handler )
{
        if ( ardp_lexer_is_ready() ) {
                handler( ARDP_LEXER_ERROR );
                return;
        }

        if ( reader is NULL ) {
                handler( ARDP_LEXER_READER_NOT_READY );
                return;
        }

        __block uint8_t *p;

        uint8_t buf[BUFSIZE];
        size_t have   = 0;
        uint8_t *mark = NULL; //nullptr;
        bool eof      = false;

        int status = 0;

        while ( not shared_lexer->finished ) {
                p = buf + have;

                size_t space = BUFSIZE - have;
                if ( space <= 0 ) {
                        /* Out of buffer space */
                        status = ARDP_LEXER_OMEM;
                        break; /* goto exit */
                }

                int len = reader( p, space, reader_args );

                if ( len < space ) {
                        eof                    = true;
                        shared_lexer->finished = 1;
                }

                ardp_lexer_turtle_process_block( p, len, mark, eof, ^( int success ) {
                  if ( success isnt ARDP_SUCCESS )
                          printf( "Internal lexer error - recoverable\n" );
                } );

                if ( shared_lexer->env.cs is turtle_error ) {
                        status = ARDP_LEXER_GENERIC_ERROR;
                        break; /* goto exit */
                } else if ( mark ) {
                        have = ( p + len ) - mark;
                        memmove( buf, mark, have );
                        mark = buf;
                } else {
                        have = 0;
                }
        }
        handler( status );
}
/*}}}*/


Fix prototype.

Fix prototype.

/* -*-c-*- */
/* This file is part of liberis
 *
 * Copyright (C) 2008 by LScube team <team@streaming.polito.it>
 * See AUTHORS for more details
 *
 * liberis is free software: you can redistribute it and/or modify it
 * under the terms of the GNU Lesser General Public License as
 * published by the Free Software Foundation, either version 3 of the
 * License, or (at your option) any later version.
 *
 * liberis is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with liberis.  If not, see
 * <http://www.gnu.org/licenses/>.
 */

#include <config.h>
#include <string.h>
#include "uri.h"

%%{
    machine uri_parser;
	include common "common.rl";

    action collapse_path {
        if ( ! g_queue_is_empty(stack) ) {
            GString *uri_path = g_string_new("");
            g_queue_foreach(stack, append_to_string, uri_path);
            uri->path = g_string_free(uri_path, FALSE);
        }
    }

	reg_name = ( unreserved | pct_encoded | sub_delims )*;
	host = IP_literal | IPv4address | reg_name;
	authority = ( userinfo >mark
                     %{uri->userinfo = g_strndup(mark, fpc-mark);}
                     "@" )?
                    host >mark
                     %{uri->host = g_strndup(mark, fpc-mark);}
                    ( ":" digit* >mark
                     %{uri->port = g_strndup(mark, fpc-mark);})?;
	pchar = unreserved | pct_encoded | sub_delims | ":" | "@";
	segment = ( pchar+ - ".." - "." ) >mark
                        %{mark = g_strndup(mark, fpc-mark);
                          g_queue_push_tail(stack, mark);}|
                  ".." %{g_queue_pop_tail(stack);}|
                    zlen | ".";
	path_abempty = ( "/" segment )*;
	segment_nz = ( pchar+ - ".." ) | "..";
	path_absolute = "/" ( segment_nz ( "/" segment )* )?;
	path_rootless = segment_nz ( "/" segment )*;
	path_empty = empty;
	hier_part = ( "//" authority path_abempty ) | path_absolute | path_rootless | path_empty;
	query = ( pchar | "/" | "?" )*;
	fragment = ( pchar | "/" | "?" )*;
	URI = scheme >mark
               %{uri->scheme = g_strndup(mark, fpc-mark);}
              ":" hier_part ( "?" query >mark
               %{uri->query = g_strndup(mark, fpc-mark);}
                )? ( "#" fragment >mark
               %{uri->fragment = g_strndup(mark, fpc-mark);} )?
               %(collapse_path);
	segment_nz_nc = ( unreserved | pct_encoded | sub_delims | "@" )+;
	path_noscheme = segment_nz_nc ( "/" segment )*;
	relative_part = ( "//" authority path_abempty ) | path_absolute | path_noscheme | path_empty;
	relative_ref = relative_part ( "?" query )? ( "#" fragment )?;
	URI_reference = URI | relative_ref;
	absolute_URI = scheme ":" hier_part ( "?" query )?;
	path = path_abempty | path_absolute | path_noscheme | path_rootless | path_empty;
	gen_delims = ":" | "/" | "?" | "#" | "[" | "]" | "@";
	reserved = gen_delims | sub_delims;

    # instantiate machine rules
    main:= URI 0;
}%%

static void append_to_string(gpointer data, gpointer user_data)
{
    GString *path = (GString*)user_data;
    char *segment = (char *)data;
    g_string_append_printf(path, "/%s", segment);
}

/**
 * @brief Parse and split an URI string into an URI structure
 *
 * @param uri_string The URI string to parse and split
 *
 * @return A pointer to a newly-allocated URI structure.
 *
 * This function parses an URI, as defined by RFC1738, and splits it
 * into its components in the given structure.
 */
URI *uri_parse(const char *uri_string)
{
    const char *p = uri_string, *pe = p + strlen(p) + 1, *mark;
    GQueue *stack = g_queue_new();
    URI *uri = g_slice_new0(URI);
    int cs;

    %% write data;
    %% write init;
    %% write exec;

    g_queue_free(stack);

    if (cs < uri_parser_first_final) {
        uri_free(uri);
        return NULL;
    }

    return uri;
}

/* -*-c-*- */
/* This file is part of liberis
 *
 * Copyright (C) 2008 by LScube team <team@streaming.polito.it>
 * See AUTHORS for more details
 *
 * liberis is free software: you can redistribute it and/or modify it
 * under the terms of the GNU Lesser General Public License as
 * published by the Free Software Foundation, either version 3 of the
 * License, or (at your option) any later version.
 *
 * liberis is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with liberis.  If not, see
 * <http://www.gnu.org/licenses/>.
 */

#include <config.h>
#include <string.h>
#include "uri.h"

%%{
    machine uri_parser;
	include common "common.rl";

    action collapse_path {
        if ( ! g_queue_is_empty(stack) ) {
            GString *uri_path = g_string_new("");
            g_queue_foreach(stack, (GFunc)append_to_string, uri_path);
            uri->path = g_string_free(uri_path, FALSE);
        }
    }

	reg_name = ( unreserved | pct_encoded | sub_delims )*;
	host = IP_literal | IPv4address | reg_name;
	authority = ( userinfo >mark
                     %{uri->userinfo = g_strndup(mark, fpc-mark);}
                     "@" )?
                    host >mark
                     %{uri->host = g_strndup(mark, fpc-mark);}
                    ( ":" digit* >mark
                     %{uri->port = g_strndup(mark, fpc-mark);})?;
	pchar = unreserved | pct_encoded | sub_delims | ":" | "@";
	segment = ( pchar+ - ".." - "." ) >mark
                        %{mark = g_strndup(mark, fpc-mark);
                          g_queue_push_tail(stack, mark);}|
                  ".." %{g_queue_pop_tail(stack);}|
                    zlen | ".";
	path_abempty = ( "/" segment )*;
	segment_nz = ( pchar+ - ".." ) | "..";
	path_absolute = "/" ( segment_nz ( "/" segment )* )?;
	path_rootless = segment_nz ( "/" segment )*;
	path_empty = empty;
	hier_part = ( "//" authority path_abempty ) | path_absolute | path_rootless | path_empty;
	query = ( pchar | "/" | "?" )*;
	fragment = ( pchar | "/" | "?" )*;
	URI = scheme >mark
               %{uri->scheme = g_strndup(mark, fpc-mark);}
              ":" hier_part ( "?" query >mark
               %{uri->query = g_strndup(mark, fpc-mark);}
                )? ( "#" fragment >mark
               %{uri->fragment = g_strndup(mark, fpc-mark);} )?
               %(collapse_path);
	segment_nz_nc = ( unreserved | pct_encoded | sub_delims | "@" )+;
	path_noscheme = segment_nz_nc ( "/" segment )*;
	relative_part = ( "//" authority path_abempty ) | path_absolute | path_noscheme | path_empty;
	relative_ref = relative_part ( "?" query )? ( "#" fragment )?;
	URI_reference = URI | relative_ref;
	absolute_URI = scheme ":" hier_part ( "?" query )?;
	path = path_abempty | path_absolute | path_noscheme | path_rootless | path_empty;
	gen_delims = ":" | "/" | "?" | "#" | "[" | "]" | "@";
	reserved = gen_delims | sub_delims;

    # instantiate machine rules
    main:= URI 0;
}%%

static void append_to_string(gpointer *data, gpointer *user_data)
{
    GString *path = (GString*)user_data;
    char *segment = (char *)data;
    g_string_append_printf(path, "/%s", segment);
}

/**
 * @brief Parse and split an URI string into an URI structure
 *
 * @param uri_string The URI string to parse and split
 *
 * @return A pointer to a newly-allocated URI structure.
 *
 * This function parses an URI, as defined by RFC1738, and splits it
 * into its components in the given structure.
 */
URI *uri_parse(const char *uri_string)
{
    const char *p = uri_string, *pe = p + strlen(p) + 1, *mark;
    GQueue *stack = g_queue_new();
    URI *uri = g_slice_new0(URI);
    int cs;

    %% write data;
    %% write init;
    %% write exec;

    g_queue_free(stack);

    if (cs < uri_parser_first_final) {
        uri_free(uri);
        return NULL;
    }

    return uri;
}


warning elimination

warning elimination

/*
 * @LANG: c++
 * @CFLAGS: -I../aapl
 */

#include <iostream>
#include <string.h>

using std::cin;
using std::cout;
using std::cerr;
using std::endl;

%%{
	machine mailbox;

	action prn_char { cout << *p; }
	action prn_space { cout << ' '; }
	action prn_word { cout.write(ws, p-ws); cout << ' '; }
	action prn_addr1 { cout << "| "; cout.write(ws+1, p-ws-2); }
	action prn_addr2 { cout << "| "; cout.write(ws, p-ws); }
	action prn_tab { cout << '\t'; }
	action prn_nl { cout << '\n'; }
	action prn_separator { cout << "------\n"; }
	action prn_from { cout << "FROM\n"; }
	action prn_to { cout << "TO\n"; }
	action prn_subj { cout << "SUBJECT\n"; }

	action start_word { ws = p; }
	action start_headers { /*preserve = p;*/ }
	action end_headers { /*preserve = 0;*/ }

	day = upper lower{2};
	month = upper lower{2};
	year = digit{4};
	time = digit{2} ':' digit{2}
		( ':' digit{2} )?;
	letterZone = upper{3};
	numZone = [+\-] digit{4};
	zone = letterZone | numZone;
	dayNum = ( digit | ' ' ) digit;

	date = day ' ' month ' '
		dayNum ' ' time ' '
		(
			year |
			year ' ' zone |
			zone ' ' year
		);

	fromLine = 'From ' [^\n]* ' '
		date '\n' @start_headers;

	headerChar = print - [ :];
	headersToPrint = 'From' |
		'To' | 'Subject';
	headersToConsume =
		headerChar+ - headersToPrint;

	action init_hlen {hlen = 0;}
	action hlen {hlen++ < 50}

	consumeHeaderBody =
		':' @init_hlen
		(
			[^\n] |
			( '\n' [ \t] )
		)* when hlen
		'\n';

	consumeHeader =
		headersToConsume consumeHeaderBody;

	addrWS = ( [ \t]+ | '\n' [ \t]+ );
	addrComment = '(' [^)]* ')';
	addrWord = [^"'@,<>() \t\n]+;
	addrAddr1 = '<' [^>]* '>';
	addrAddr2 = addrWord '@' addrWord;
	addrString =
		'"' [^"]* '"' |
		"'" [^']* "'";

	addrItem = (
			addrAddr1 %prn_addr1 |
			addrAddr2 %prn_addr2 |
			addrWord %prn_word |
			addrString %prn_word
		) >start_word;

	address = (
			addrWS |
			addrComment |
			addrItem
		)** >prn_tab;

	addrHeader = (
			'From' %prn_from |
			'To' %prn_to
		) ':' @init_hlen
		( address ( ',' @prn_nl address )* ) when hlen
		'\n' %prn_nl;

	subjectHeader =
		'Subject:' @prn_subj @prn_tab @init_hlen
		(
			' '* <:
			(
				[^\n] @prn_char  |
				( '\n' [ \t]+ ) %prn_space
			)**
		) when hlen
		'\n' %prn_nl;

	header = consumeHeader |
		addrHeader | subjectHeader;

	messageLine =
		( [^\n]* when hlen '\n' @init_hlen ) - fromLine;

	main := (
			fromLine %prn_separator
			header*
			'\n' @end_headers @init_hlen
			messageLine*
		)*;
 }%%

%% write data;

#define BUFSIZE 8192

void test( const char *buf )
{
	int cs, len = strlen( buf );
	const char /**preserve = 0,*/ *ws = 0;
	int hlen = 0;

	%% write init;
	const char *p = buf;
	const char *pe = p + len;
	%% write exec;

	if ( cs < mailbox_first_final ) {
		cout << endl << endl;
		cout << "DID NOT FINISH IN A FINAL STATE" << endl;
	}
}

int main()
{
	test(
		"From user@host.com Wed Nov 28 13:30:05 2001\n"
		"From: \"Adrian D. Thurston\" <thurston@complang.org>\n"
		"Subject:   the squirrel has landed\n"
		"\n"
		"Message goes here. \n"
		"From (trick from line).\n"
		"From: not really a header\n"
		"\n"
		"From user2@host2.com Wed Nov 28 13:30:05 2001\n"
		"To: \"(kill 1)\" Edgar Allen Poe <ep@net.com> (da man)\n"
		"Subject:   (no subject) this is a really long subject which should fail the length constraint \n"
		"Other: 0123456789\n"
		"\n"
		"Message goes here. \n"
		"\n"
	);
	test(
		"From user@host.com Wed Nov 28 13:30:05 2001\n"
		"To: \"(kill 2)\" some guy <sg@net.com>\n"
		"From: \"Adrian D. Thurston this name is far too long\" <thurston@complang.org>\n"
		"Subject:   the squirrel has landed\n"
		"\n"
		"From user2@host2.com Wed Nov 28 13:30:05 2001\n"
		"To: Edgar Allen Poe <ep@net.com> (da man)\n"
		"Subject:   (no subject) \n"
		"\n"
	);
	test(
		"From user@host.com Wed Nov 28 13:30:05 2001\n"
		"To: \"(kill 3)\" some guy <sg@net.com>\n"
		"From: \"Adrian D. Thurston This name is fore sure absolutely too long\" <t@cs.ca>\n"
		"Subject:   the squirrel has landed\n"
		"\n"
	);
	test(
		"From user@host.com Wed Nov 28 13:30:05 2001\n"
		"From: \"Adrian D. Thurston \" <t@cs.ca>\n"
		"Subject:   (kill 4) the squirrel has landed\n"
		"Other: This is another header field, not interpreted, that is too long\n"
		"\n"
	);
	test(
		"From user@host.com Wed Nov 28 13:30:05 2001\n"
		"From: \"Adrian D. Thurston \" <t@cs.ca>\n"
		"Subject:   (kill 5)the squirrel has landed\n"
		"\n"
		"This message line is okay.\n"
		"But this message line is far too long and will cause an error.\n"
	);
	return 0;
}

#ifdef _____OUTPUT_____
------
FROM
	"Adrian D. Thurston" | thurston@complang.org
SUBJECT
	the squirrel has landed
------
TO
	"(kill 1)" Edgar Allen Poe | ep@net.com
SUBJECT
	(no subject) this is a really long subject whic

DID NOT FINISH IN A FINAL STATE
------
TO
	"(kill 2)" some guy | sg@net.com
FROM
	"Adrian D. Thurston this name is far too long"

DID NOT FINISH IN A FINAL STATE
------
TO
	"(kill 3)" some guy | sg@net.com
FROM

DID NOT FINISH IN A FINAL STATE
------
FROM
	"Adrian D. Thurston " | t@cs.ca
SUBJECT
	(kill 4) the squirrel has landed

DID NOT FINISH IN A FINAL STATE
------
FROM
	"Adrian D. Thurston " | t@cs.ca
SUBJECT
	(kill 5)the squirrel has landed

DID NOT FINISH IN A FINAL STATE
#endif

/*
 * @LANG: c++
 * @CFLAGS: -I../aapl
 */

#include <iostream>
#include <string.h>

using std::cin;
using std::cout;
using std::cerr;
using std::endl;

%%{
	machine mailbox;

	action prn_char { cout << *p; }
	action prn_space { cout << ' '; }
	action prn_word { cout.write(ws, p-ws); cout << ' '; }
	action prn_addr1 { cout << "| "; cout.write(ws+1, p-ws-2); }
	action prn_addr2 { cout << "| "; cout.write(ws, p-ws); }
	action prn_tab { cout << '\t'; }
	action prn_nl { cout << '\n'; }
	action prn_separator { cout << "------\n"; }
	action prn_from { cout << "FROM\n"; }
	action prn_to { cout << "TO\n"; }
	action prn_subj { cout << "SUBJECT\n"; }

	action start_word { ws = p; }
	action start_headers { preserve = p; }
	action end_headers {preserve = 0;}

	day = upper lower{2};
	month = upper lower{2};
	year = digit{4};
	time = digit{2} ':' digit{2}
		( ':' digit{2} )?;
	letterZone = upper{3};
	numZone = [+\-] digit{4};
	zone = letterZone | numZone;
	dayNum = ( digit | ' ' ) digit;

	date = day ' ' month ' '
		dayNum ' ' time ' '
		(
			year |
			year ' ' zone |
			zone ' ' year
		);

	fromLine = 'From ' [^\n]* ' '
		date '\n' @start_headers;

	headerChar = print - [ :];
	headersToPrint = 'From' |
		'To' | 'Subject';
	headersToConsume =
		headerChar+ - headersToPrint;

	action init_hlen {hlen = 0;}
	action hlen {hlen++ < 50}

	consumeHeaderBody =
		':' @init_hlen
		(
			[^\n] |
			( '\n' [ \t] )
		)* when hlen
		'\n';

	consumeHeader =
		headersToConsume consumeHeaderBody;

	addrWS = ( [ \t]+ | '\n' [ \t]+ );
	addrComment = '(' [^)]* ')';
	addrWord = [^"'@,<>() \t\n]+;
	addrAddr1 = '<' [^>]* '>';
	addrAddr2 = addrWord '@' addrWord;
	addrString =
		'"' [^"]* '"' |
		"'" [^']* "'";

	addrItem = (
			addrAddr1 %prn_addr1 |
			addrAddr2 %prn_addr2 |
			addrWord %prn_word |
			addrString %prn_word
		) >start_word;

	address = (
			addrWS |
			addrComment |
			addrItem
		)** >prn_tab;

	addrHeader = (
			'From' %prn_from |
			'To' %prn_to
		) ':' @init_hlen
		( address ( ',' @prn_nl address )* ) when hlen
		'\n' %prn_nl;

	subjectHeader =
		'Subject:' @prn_subj @prn_tab @init_hlen
		(
			' '* <:
			(
				[^\n] @prn_char  |
				( '\n' [ \t]+ ) %prn_space
			)**
		) when hlen
		'\n' %prn_nl;

	header = consumeHeader |
		addrHeader | subjectHeader;

	messageLine =
		( [^\n]* when hlen '\n' @init_hlen ) - fromLine;

	main := (
			fromLine %prn_separator
			header*
			'\n' @end_headers @init_hlen
			messageLine*
		)*;
 }%%

%% write data;

#define BUFSIZE 8192

void test( const char *buf )
{
	int cs, len = strlen( buf );
	const char *preserve = 0, *ws = 0;
	int hlen = 0;

	%% write init;
	const char *p = buf;
	const char *pe = p + len;
	%% write exec;

	if ( cs < mailbox_first_final ) {
		cout << endl << endl;
		cout << "DID NOT FINISH IN A FINAL STATE" << endl;
	}
}

int main()
{
	test(
		"From user@host.com Wed Nov 28 13:30:05 2001\n"
		"From: \"Adrian D. Thurston\" <thurston@complang.org>\n"
		"Subject:   the squirrel has landed\n"
		"\n"
		"Message goes here. \n"
		"From (trick from line).\n"
		"From: not really a header\n"
		"\n"
		"From user2@host2.com Wed Nov 28 13:30:05 2001\n"
		"To: \"(kill 1)\" Edgar Allen Poe <ep@net.com> (da man)\n"
		"Subject:   (no subject) this is a really long subject which should fail the length constraint \n"
		"Other: 0123456789\n"
		"\n"
		"Message goes here. \n"
		"\n"
	);
	test(
		"From user@host.com Wed Nov 28 13:30:05 2001\n"
		"To: \"(kill 2)\" some guy <sg@net.com>\n"
		"From: \"Adrian D. Thurston this name is far too long\" <thurston@complang.org>\n"
		"Subject:   the squirrel has landed\n"
		"\n"
		"From user2@host2.com Wed Nov 28 13:30:05 2001\n"
		"To: Edgar Allen Poe <ep@net.com> (da man)\n"
		"Subject:   (no subject) \n"
		"\n"
	);
	test(
		"From user@host.com Wed Nov 28 13:30:05 2001\n"
		"To: \"(kill 3)\" some guy <sg@net.com>\n"
		"From: \"Adrian D. Thurston This name is fore sure absolutely too long\" <t@cs.ca>\n"
		"Subject:   the squirrel has landed\n"
		"\n"
	);
	test(
		"From user@host.com Wed Nov 28 13:30:05 2001\n"
		"From: \"Adrian D. Thurston \" <t@cs.ca>\n"
		"Subject:   (kill 4) the squirrel has landed\n"
		"Other: This is another header field, not interpreted, that is too long\n"
		"\n"
	);
	test(
		"From user@host.com Wed Nov 28 13:30:05 2001\n"
		"From: \"Adrian D. Thurston \" <t@cs.ca>\n"
		"Subject:   (kill 5)the squirrel has landed\n"
		"\n"
		"This message line is okay.\n"
		"But this message line is far too long and will cause an error.\n"
	);
	return 0;
}

#ifdef _____OUTPUT_____
------
FROM
	"Adrian D. Thurston" | thurston@complang.org
SUBJECT
	the squirrel has landed
------
TO
	"(kill 1)" Edgar Allen Poe | ep@net.com
SUBJECT
	(no subject) this is a really long subject whic

DID NOT FINISH IN A FINAL STATE
------
TO
	"(kill 2)" some guy | sg@net.com
FROM
	"Adrian D. Thurston this name is far too long"

DID NOT FINISH IN A FINAL STATE
------
TO
	"(kill 3)" some guy | sg@net.com
FROM

DID NOT FINISH IN A FINAL STATE
------
FROM
	"Adrian D. Thurston " | t@cs.ca
SUBJECT
	(kill 4) the squirrel has landed

DID NOT FINISH IN A FINAL STATE
------
FROM
	"Adrian D. Thurston " | t@cs.ca
SUBJECT
	(kill 5)the squirrel has landed

DID NOT FINISH IN A FINAL STATE
#endif


SSA: Ignore lines with the wrong number of fields.

/*
 *  SubParsing.c
 *  SSARender2
 *
 *  Created by Alexander Strange on 7/25/07.
 *  Copyright 2007 __MyCompanyName__. All rights reserved.
 *
 */

#import "SubParsing.h"
#import "SubUtilities.h"
#import "SubContext.h"

%%machine SSAfile;
%%write data;

@implementation SubRenderSpan
+(SubRenderSpan*)startingSpanForDiv:(SubRenderDiv*)div delegate:(SubRenderer*)delegate
{
	SubRenderSpan *span = [[SubRenderSpan alloc] init];
	span->offset = 0;
	span->ex = [delegate spanExtraFromRenderDiv:div];
	span->delegate = delegate;
	return [span autorelease];
}

-(SubRenderSpan*)cloneWithDelegate:(SubRenderer*)delegate_
{
	SubRenderSpan *span = [[SubRenderSpan alloc] init];
	span->offset = offset;
	span->ex = [delegate_ cloneSpanExtra:self];
	span->delegate = delegate_;
	return [span autorelease];
}

-(void)dealloc
{
	[delegate releaseSpanExtra:ex];
	[super dealloc];
}
@end

@implementation SubRenderDiv
-(NSString*)description
{
	int i, sc = [spans count];
	NSMutableString *tmp = [NSMutableString stringWithFormat:@"div %d spans:",sc];
	for (i = 0; i < sc; i++) {[tmp appendFormat:@" %d",((SubRenderSpan*)[spans objectAtIndex:i])->offset];}
	[tmp appendFormat:@" %d", [text length]];
	return tmp;
}

-(SubRenderDiv*)init
{
	if (self = [super init]) {
		text = nil;
		styleLine = nil;
		marginL = marginR = marginV = layer = 0;
		spans = nil;

		posX = posY = -1;
		alignH = kSubAlignmentMiddle; alignV = kSubAlignmentBottom;

		is_shape = NO;
		render_complexity = 0;
	}

	return self;
}

-(SubRenderDiv*)nextDivWithDelegate:(SubRenderer*)delegate
{
	SubRenderDiv *div = [[[SubRenderDiv alloc] init] autorelease];

	div->text    = [[NSMutableString string] retain];
  div->styleLine = [styleLine retain];
	div->marginL = marginL;
	div->marginR = marginR;
	div->marginV = marginV;
	div->layer   = layer;

	div->spans   = [[NSMutableArray arrayWithObject:[[spans objectAtIndex:[spans count]-1] cloneWithDelegate:delegate]] retain];

	div->posX    = posX;
	div->posY    = posY;
	div->alignH  = alignH;
	div->alignV  = alignV;
  div->wrapStyle = wrapStyle;

    div->is_shape = NO;
	div->render_complexity = render_complexity;

	return div;
}

-(void)dealloc
{
	[text release];
	[styleLine release];
	[spans release];
	[super dealloc];
}
@end

extern BOOL IsScriptASS(NSDictionary *headers);

static NSArray *SplitByFormat(NSString *format, NSArray *lines)
{
	NSArray *formarray = STSplitStringIgnoringWhitespace(format,@",");
	int i, numlines = [lines count], numfields = [formarray count];
	NSMutableArray *ar = [NSMutableArray arrayWithCapacity:numlines];

	for (i = 0; i < numlines; i++) {
		NSString *s = [lines objectAtIndex:i];
		NSArray *splitline = STSplitStringWithCount(s, @",", numfields);

		if ([splitline count] != numfields) continue;
		[ar addObject:[NSDictionary dictionaryWithObjects:splitline
												  forKeys:formarray]];
	}

	return ar;
}

void SubParseSSAFile(const unichar *ssa, size_t len, NSDictionary **headers, NSArray **styles, NSArray **subs)
{
	const unichar *p = ssa, *pe = ssa + len, *strbegin = p;
	int cs=0;

	NSMutableDictionary *hd = [NSMutableDictionary dictionary];
	NSMutableArray *stylearr = [NSMutableArray array], *eventarr = [NSMutableArray array], *cur_array=NULL;
	NSCharacterSet *wcs = [NSCharacterSet whitespaceCharacterSet];
	NSString *str=NULL, *styleformat=NULL, *eventformat=NULL;

#define send() [NSString stringWithCharacters:strbegin length:p-strbegin]

	%%{
		alphtype unsigned short;

		action sstart {strbegin = p;}
		action setheaderval {[hd setObject:send() forKey:str];}
		action savestr {str = send();}
		action csvlineend {[cur_array addObject:[send() stringByTrimmingCharactersInSet:wcs]];}
		action setupevents {
			cur_array=eventarr;
			eventformat = IsScriptASS(hd) ?
				@"Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text":
				@"Marked, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text";
		}

		nl = ("\n" | "\r" | "\r\n");
		str = any*;
		ws = space | 0xa0;
		bom = 0xfeff;

		hline = ((";" str) | (([^;] str) >sstart %savestr :> (":" ws* str >sstart %setheaderval)?))? :> nl;

		header = "[Script Info]" nl hline*;

		format = "Format:" ws* %sstart str %savestr :> nl;

		action assformat {styleformat = @"Name, Fontname, Fontsize, PrimaryColour, SecondaryColour, OutlineColour, BackColour, Bold, Italic, Underline, StrikeOut, ScaleX, ScaleY, Spacing, Angle, BorderStyle, Outline, Shadow, Alignment, MarginL, MarginR, MarginV, Encoding";}
		action ssaformat {styleformat = @"Name, Fontname, Fontsize, PrimaryColour, SecondaryColour, TertiaryColour, BackColour, Bold, Italic, BorderStyle, Outline, Shadow, Alignment, MarginL, MarginR, MarginV, AlphaLevel, Encoding";}

		stylename = "[V4 Styles]" %ssaformat | "[V4+ Styles]" %assformat;

		sline = (("Style:" ws* %sstart str %csvlineend) | str) :> nl;

		styles = stylename % {cur_array=stylearr;} nl (format %{styleformat=str;})? <: (sline*);

		event_txt = (("Dialogue:" ws* %sstart str %csvlineend) | str);
		event = event_txt :> nl;

		lines = "[Events]" %setupevents nl (format %{eventformat=str;})? <: (event*);

		main := bom? header :> styles :> lines?;
	}%%

	%%write init;
	%%write exec;
	%%write eof;

	*headers = hd;
	if (styles) *styles = SplitByFormat(styleformat, stylearr);
	if (subs) *subs = SplitByFormat(eventformat, eventarr);
}

%%machine SSAtag;
%%write data;

static NSMutableString *FilterSlashEscapes(NSMutableString *s)
{
	[s replaceOccurrencesOfString:@"\\n" withString:@"\n" options:NSCaseInsensitiveSearch range:NSMakeRange(0, [s length])];
	unichar nbsp = 0xA0;

	[s replaceOccurrencesOfString:@"\\h" withString:[NSString stringWithCharacters:&nbsp length:1] options:0 range: NSMakeRange(0,[s length])];
	return s;
}

NSArray *SubParsePacket(NSString *packet, SubContext *context, SubRenderer *delegate, unichar *linebuf)
{
	packet = STStandardizeStringNewlines(packet);
	NSArray *lines = (context->scriptType == kSubTypeSRT) ? [NSArray arrayWithObject:[packet substringToIndex:[packet length]-1]] : [packet componentsSeparatedByString:@"\n"];
	size_t line_count = [lines count];
	NSMutableArray *divs = [NSMutableArray arrayWithCapacity:line_count];
	int i;

	for (i = 0; i < line_count; i++) {
		NSString *inputText = [lines objectAtIndex:(context->collisions == kSubCollisionsReverse) ? (line_count - i - 1) : i];
		SubRenderDiv *div = [[[SubRenderDiv alloc] init] autorelease];

		div->text = [[NSMutableString string] retain];
		div->spans = [[NSMutableArray array] retain];
		div->wrapStyle = context->wrapStyle;

		if (context->scriptType == kSubTypeSRT) {
			div->styleLine = [context->defaultStyle retain];
			div->marginL = div->styleLine->marginL;
			div->marginR = div->styleLine->marginR;
			div->marginV = div->styleLine->marginV;
			div->layer = 0;
			div->wrapStyle = kSubLineWrapTopWider;
		} else {
			NSArray *fields = STSplitStringWithCount(inputText, @",", 9);
			if ([fields count] < 9) continue;
			div->layer = [[fields objectAtIndex:1] intValue];
			div->styleLine = [[context styleForName:[fields objectAtIndex:2]] retain];
			div->marginL = [[fields objectAtIndex:4] intValue];
			div->marginR = [[fields objectAtIndex:5] intValue];
			div->marginV = [[fields objectAtIndex:6] intValue];
			inputText = [fields objectAtIndex:8];
			if ([inputText length] == 0) continue;

			if (div->marginL == 0) div->marginL = div->styleLine->marginL;
			if (div->marginR == 0) div->marginR = div->styleLine->marginR;
			if (div->marginV == 0) div->marginV = div->styleLine->marginV;

			div->wrapStyle = context->wrapStyle;
		}

		div->alignH = div->styleLine->alignH;
		div->alignV = div->styleLine->alignV;

		size_t linelen = [inputText length];
		[inputText getCharacters:linebuf];
		linebuf[linelen] = 0;

#undef send
#define send() [NSString stringWithCharacters:outputbegin length:p-outputbegin]
#define psend() [NSString stringWithCharacters:parambegin length:p-parambegin]
#define tag(tagt, p) [delegate spanChangedTag:tag_##tagt span:current_span div:div param:&(p)]

		{
			unichar *p = linebuf, *pe = linebuf + linelen, *outputbegin = p, *parambegin=p, *last_cmd_start=p;
			const unichar *pb = p;
			int cs = 0;
			SubRenderSpan *current_span = [SubRenderSpan startingSpanForDiv:div delegate:delegate];
			unsigned chars_deleted = 0; int intnum = 0; float floatnum = 0;
			NSString *strval=NULL;
			unsigned curX, curY;
			BOOL reached_end = NO, startNewLayout = NO;

			%%{
				action bold {tag(b, intnum);}
				action italic {tag(i, intnum);}
				action underline {tag(u, intnum);}
				action strikeout {tag(s, intnum);}
				action outlinesize {tag(bord, floatnum);}
				action shadowdist {tag(shad, floatnum);}
				action bluredge {tag(be, intnum);}
				action fontname {tag(fn, strval);}
				action fontsize {tag(fs, floatnum);}
				action scalex {tag(fscx, floatnum);}
				action scaley {tag(fscy, floatnum);}
				action tracking {tag(fsp, floatnum);}
				action frz {tag(frz, floatnum);}
				action frx {tag(frx, floatnum);}
				action fry {tag(fry, floatnum);}
				action primaryc {tag(1c, intnum);}
				action secondaryc {tag(2c, intnum);}
				action outlinec {tag(3c, intnum);}
				action shadowc {tag(4c, intnum);}
				action alpha {tag(alpha, intnum);}
				action primarya {tag(1a, intnum);}
				action secondarya {tag(2a, intnum);}
				action outlinea {tag(3a, intnum);}
				action shadowa {tag(4a, intnum);}
				action stylerevert {tag(r, strval);}

				action paramset {parambegin=p;}
				action setintnum {intnum = [psend() intValue];}
				action sethexnum {intnum = strtoul([psend() UTF8String], NULL, 16);}
				action setfloatnum {floatnum = [psend() floatValue];}
				action setstringval {strval = psend();}
				action setxypos {curX=curY=-1; sscanf([psend() UTF8String], "(%d,%d)", &curX, &curY);}

				action ssaalign {
					if (outputbegin == pb) ParseASSAlignment(SSA2ASSAlignment(intnum), &div->alignH, &div->alignV);
				}

				action align {
					if (outputbegin == pb) ParseASSAlignment(intnum, &div->alignH, &div->alignV);
				}

				action wrapstyle {
					if (!startNewLayout) {
						startNewLayout = YES;

						if ([div->text length] > 0) {[divs addObject:div]; div = [div nextDivWithDelegate:delegate];}
					}

					div->wrapStyle = intnum;
				}

				action position {
					if (!startNewLayout) {
						startNewLayout = YES;

						if ([div->text length] > 0) {[divs addObject:div]; div = [div nextDivWithDelegate:delegate];}
					}

					div->posX = curX;
					div->posY = curY;
				}

				intnum = ("-"? [0-9]+) >paramset %setintnum;
				flag = [01] >paramset %setintnum;
				floatnum = ([0-9]+ ("." [0-9]*)?) >paramset %setfloatnum;
				string = ([^\\}]*) >paramset %setstringval;
				color = ("H"|"&"){,2} (xdigit+) >paramset %sethexnum "&"?;
				parens = "(" [^)]* ")";
				xypos = ("(" [0-9]+ "," [0-9]+ ")") >paramset %setxypos;

				cmd = "\\" (
							"b" intnum %bold
							|"i" flag %italic
							|"u" flag %underline
							|"s" flag %strikeout
							|"bord" floatnum %outlinesize
							|"shad" floatnum %shadowdist
							|"be" flag %bluredge
							|"fn" string %fontname
							|"fs" floatnum %fontsize
							|"fscx" floatnum %scalex
							|"fscy" floatnum %scaley
							|"fsp" floatnum %tracking
							|("fr" "z"? floatnum %frz)
							|"frx" floatnum %frx
							|"fry" floatnum %fry
							|"fe" intnum
							|("1"? "c" color %primaryc)
							|"2c" color %secondaryc
							|"3c" color %outlinec
							|"4c" color %shadowc
							|"alpha" color %alpha
							|"1a" color %primarya
							|"2a" color %secondarya
							|"3a" color %outlinea
							|"4a" color %shadowa
							|"a" intnum %ssaalign
							|"an" intnum %align
							|([kK] [fo]? intnum)
							|"q" intnum %wrapstyle
							|"r" string %stylerevert
							|"pos" xypos %position
							|"t" parens
							|"org" parens
							|("fad" "e"? parens)
							|"clip" parens
							|"p" floatnum
							|"pbo" floatnum
					   );

				cmd_list = "{" (cmd* | any*) :> "}";

				action backslash_handler {
					[div->text appendString:send()];
					unichar c = *(p+1), o=c;

					if (c) {
						switch (c) {
							case 'N': case 'n':
								o = '\n';
								break;
							case 'h':
								o = 0xA0; //non-breaking space
								break;
							default:
								o = c;
						}
					}

					[div->text appendFormat:@"%C",o];

					chars_deleted++;

					outputbegin = p+2;
				}

				action enter_tag {
					if (p > outputbegin) [div->text appendString:send()];

					if (p != pb) {
						[div->spans addObject:current_span];
						current_span = [current_span cloneWithDelegate:delegate];
					}

					last_cmd_start = p;
					if (p == pe) reached_end = YES;
				}

				action exit_tag {
					p++;
					chars_deleted += (p - last_cmd_start);

					current_span->offset = (p - pb) - chars_deleted;
					outputbegin = p;

					if (startNewLayout) {
						startNewLayout = NO;
						chars_deleted = outputbegin - pb;
					}

					p--;
				}

				special = ("\\" any) >backslash_handler | cmd_list >enter_tag @exit_tag;
				sub_text_char = [^\\{];
				sub_text = sub_text_char*;

				main := ((sub_text | special)* "\\"?) %/enter_tag;
			}%%

			%%write init;
			%%write exec;
			%%write eof;

			if (!reached_end) NSLog(@"parse error: %@",inputText);
			if (linebuf[linelen-1] == '\\') [div->text appendString:@"\\"];
			[divs addObject:div];
		}

	}

	return divs;
}

/*
 *  SubParsing.c
 *  SSARender2
 *
 *  Created by Alexander Strange on 7/25/07.
 *  Copyright 2007 __MyCompanyName__. All rights reserved.
 *
 */

#import "SubParsing.h"
#import "SubUtilities.h"
#import "SubContext.h"

%%machine SSAfile;
%%write data;

@implementation SubRenderSpan
+(SubRenderSpan*)startingSpanForDiv:(SubRenderDiv*)div delegate:(SubRenderer*)delegate
{
	SubRenderSpan *span = [[SubRenderSpan alloc] init];
	span->offset = 0;
	span->ex = [delegate spanExtraFromRenderDiv:div];
	span->delegate = delegate;
	return [span autorelease];
}

-(SubRenderSpan*)cloneWithDelegate:(SubRenderer*)delegate_
{
	SubRenderSpan *span = [[SubRenderSpan alloc] init];
	span->offset = offset;
	span->ex = [delegate_ cloneSpanExtra:self];
	span->delegate = delegate_;
	return [span autorelease];
}

-(void)dealloc
{
	[delegate releaseSpanExtra:ex];
	[super dealloc];
}
@end

@implementation SubRenderDiv
-(NSString*)description
{
	int i, sc = [spans count];
	NSMutableString *tmp = [NSMutableString stringWithFormat:@"div %d spans:",sc];
	for (i = 0; i < sc; i++) {[tmp appendFormat:@" %d",((SubRenderSpan*)[spans objectAtIndex:i])->offset];}
	[tmp appendFormat:@" %d", [text length]];
	return tmp;
}

-(SubRenderDiv*)init
{
	if (self = [super init]) {
		text = nil;
		styleLine = nil;
		marginL = marginR = marginV = layer = 0;
		spans = nil;

		posX = posY = -1;
		alignH = kSubAlignmentMiddle; alignV = kSubAlignmentBottom;

		is_shape = NO;
		render_complexity = 0;
	}

	return self;
}

-(SubRenderDiv*)nextDivWithDelegate:(SubRenderer*)delegate
{
	SubRenderDiv *div = [[[SubRenderDiv alloc] init] autorelease];

	div->text    = [[NSMutableString string] retain];
  div->styleLine = [styleLine retain];
	div->marginL = marginL;
	div->marginR = marginR;
	div->marginV = marginV;
	div->layer   = layer;

	div->spans   = [[NSMutableArray arrayWithObject:[[spans objectAtIndex:[spans count]-1] cloneWithDelegate:delegate]] retain];

	div->posX    = posX;
	div->posY    = posY;
	div->alignH  = alignH;
	div->alignV  = alignV;
  div->wrapStyle = wrapStyle;

    div->is_shape = NO;
	div->render_complexity = render_complexity;

	return div;
}

-(void)dealloc
{
	[text release];
	[styleLine release];
	[spans release];
	[super dealloc];
}
@end

extern BOOL IsScriptASS(NSDictionary *headers);

static NSArray *SplitByFormat(NSString *format, NSArray *lines)
{
	NSArray *formarray = STSplitStringIgnoringWhitespace(format,@",");
	int i, numlines = [lines count], numfields = [formarray count];
	NSMutableArray *ar = [NSMutableArray arrayWithCapacity:numlines];

	for (i = 0; i < numlines; i++) {
		NSString *s = [lines objectAtIndex:i];
		NSArray *splitline = STSplitStringWithCount(s, @",", numfields);

		[ar addObject:[NSDictionary dictionaryWithObjects:splitline
												  forKeys:formarray]];
	}

	return ar;
}

void SubParseSSAFile(const unichar *ssa, size_t len, NSDictionary **headers, NSArray **styles, NSArray **subs)
{
	const unichar *p = ssa, *pe = ssa + len, *strbegin = p;
	int cs=0;

	NSMutableDictionary *hd = [NSMutableDictionary dictionary];
	NSMutableArray *stylearr = [NSMutableArray array], *eventarr = [NSMutableArray array], *cur_array=NULL;
	NSCharacterSet *wcs = [NSCharacterSet whitespaceCharacterSet];
	NSString *str=NULL, *styleformat=NULL, *eventformat=NULL;

#define send() [NSString stringWithCharacters:strbegin length:p-strbegin]

	%%{
		alphtype unsigned short;

		action sstart {strbegin = p;}
		action setheaderval {[hd setObject:send() forKey:str];}
		action savestr {str = send();}
		action csvlineend {[cur_array addObject:[send() stringByTrimmingCharactersInSet:wcs]];}
		action setupevents {
			cur_array=eventarr;
			eventformat = IsScriptASS(hd) ?
				@"Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text":
				@"Marked, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text";
		}

		nl = ("\n" | "\r" | "\r\n");
		str = any*;
		ws = space | 0xa0;
		bom = 0xfeff;

		hline = ((";" str) | (([^;] str) >sstart %savestr :> (":" ws* str >sstart %setheaderval)?))? :> nl;

		header = "[Script Info]" nl hline*;

		format = "Format:" ws* %sstart str %savestr :> nl;

		action assformat {styleformat = @"Name, Fontname, Fontsize, PrimaryColour, SecondaryColour, OutlineColour, BackColour, Bold, Italic, Underline, StrikeOut, ScaleX, ScaleY, Spacing, Angle, BorderStyle, Outline, Shadow, Alignment, MarginL, MarginR, MarginV, Encoding";}
		action ssaformat {styleformat = @"Name, Fontname, Fontsize, PrimaryColour, SecondaryColour, TertiaryColour, BackColour, Bold, Italic, BorderStyle, Outline, Shadow, Alignment, MarginL, MarginR, MarginV, AlphaLevel, Encoding";}

		stylename = "[V4 Styles]" %ssaformat | "[V4+ Styles]" %assformat;

		sline = (("Style:" ws* %sstart str %csvlineend) | str) :> nl;

		styles = stylename % {cur_array=stylearr;} nl (format %{styleformat=str;})? <: (sline*);

		event_txt = (("Dialogue:" ws* %sstart str %csvlineend) | str);
		event = event_txt :> nl;

		lines = "[Events]" %setupevents nl (format %{eventformat=str;})? <: (event*);

		main := bom? header :> styles :> lines?;
	}%%

	%%write init;
	%%write exec;
	%%write eof;

	*headers = hd;
	if (styles) *styles = SplitByFormat(styleformat, stylearr);
	if (subs) *subs = SplitByFormat(eventformat, eventarr);
}

%%machine SSAtag;
%%write data;

static NSMutableString *FilterSlashEscapes(NSMutableString *s)
{
	[s replaceOccurrencesOfString:@"\\n" withString:@"\n" options:NSCaseInsensitiveSearch range:NSMakeRange(0, [s length])];
	unichar nbsp = 0xA0;

	[s replaceOccurrencesOfString:@"\\h" withString:[NSString stringWithCharacters:&nbsp length:1] options:0 range: NSMakeRange(0,[s length])];
	return s;
}

NSArray *SubParsePacket(NSString *packet, SubContext *context, SubRenderer *delegate, unichar *linebuf)
{
	packet = STStandardizeStringNewlines(packet);
	NSArray *lines = (context->scriptType == kSubTypeSRT) ? [NSArray arrayWithObject:[packet substringToIndex:[packet length]-1]] : [packet componentsSeparatedByString:@"\n"];
	size_t line_count = [lines count];
	NSMutableArray *divs = [NSMutableArray arrayWithCapacity:line_count];
	int i;

	for (i = 0; i < line_count; i++) {
		NSString *inputText = [lines objectAtIndex:(context->collisions == kSubCollisionsReverse) ? (line_count - i - 1) : i];
		SubRenderDiv *div = [[[SubRenderDiv alloc] init] autorelease];

		div->text = [[NSMutableString string] retain];
		div->spans = [[NSMutableArray array] retain];
		div->wrapStyle = context->wrapStyle;

		if (context->scriptType == kSubTypeSRT) {
			div->styleLine = [context->defaultStyle retain];
			div->marginL = div->styleLine->marginL;
			div->marginR = div->styleLine->marginR;
			div->marginV = div->styleLine->marginV;
			div->layer = 0;
			div->wrapStyle = kSubLineWrapTopWider;
		} else {
			NSArray *fields = STSplitStringWithCount(inputText, @",", 9);
			if ([fields count] < 9) continue;
			div->layer = [[fields objectAtIndex:1] intValue];
			div->styleLine = [[context styleForName:[fields objectAtIndex:2]] retain];
			div->marginL = [[fields objectAtIndex:4] intValue];
			div->marginR = [[fields objectAtIndex:5] intValue];
			div->marginV = [[fields objectAtIndex:6] intValue];
			inputText = [fields objectAtIndex:8];
			if ([inputText length] == 0) continue;

			if (div->marginL == 0) div->marginL = div->styleLine->marginL;
			if (div->marginR == 0) div->marginR = div->styleLine->marginR;
			if (div->marginV == 0) div->marginV = div->styleLine->marginV;

			div->wrapStyle = context->wrapStyle;
		}

		div->alignH = div->styleLine->alignH;
		div->alignV = div->styleLine->alignV;

		size_t linelen = [inputText length];
		[inputText getCharacters:linebuf];
		linebuf[linelen] = 0;

#undef send
#define send() [NSString stringWithCharacters:outputbegin length:p-outputbegin]
#define psend() [NSString stringWithCharacters:parambegin length:p-parambegin]
#define tag(tagt, p) [delegate spanChangedTag:tag_##tagt span:current_span div:div param:&(p)]

		{
			unichar *p = linebuf, *pe = linebuf + linelen, *outputbegin = p, *parambegin=p, *last_cmd_start=p;
			const unichar *pb = p;
			int cs = 0;
			SubRenderSpan *current_span = [SubRenderSpan startingSpanForDiv:div delegate:delegate];
			unsigned chars_deleted = 0; int intnum = 0; float floatnum = 0;
			NSString *strval=NULL;
			unsigned curX, curY;
			BOOL reached_end = NO, startNewLayout = NO;

			%%{
				action bold {tag(b, intnum);}
				action italic {tag(i, intnum);}
				action underline {tag(u, intnum);}
				action strikeout {tag(s, intnum);}
				action outlinesize {tag(bord, floatnum);}
				action shadowdist {tag(shad, floatnum);}
				action bluredge {tag(be, intnum);}
				action fontname {tag(fn, strval);}
				action fontsize {tag(fs, floatnum);}
				action scalex {tag(fscx, floatnum);}
				action scaley {tag(fscy, floatnum);}
				action tracking {tag(fsp, floatnum);}
				action frz {tag(frz, floatnum);}
				action frx {tag(frx, floatnum);}
				action fry {tag(fry, floatnum);}
				action primaryc {tag(1c, intnum);}
				action secondaryc {tag(2c, intnum);}
				action outlinec {tag(3c, intnum);}
				action shadowc {tag(4c, intnum);}
				action alpha {tag(alpha, intnum);}
				action primarya {tag(1a, intnum);}
				action secondarya {tag(2a, intnum);}
				action outlinea {tag(3a, intnum);}
				action shadowa {tag(4a, intnum);}
				action stylerevert {tag(r, strval);}

				action paramset {parambegin=p;}
				action setintnum {intnum = [psend() intValue];}
				action sethexnum {intnum = strtoul([psend() UTF8String], NULL, 16);}
				action setfloatnum {floatnum = [psend() floatValue];}
				action setstringval {strval = psend();}
				action setxypos {curX=curY=-1; sscanf([psend() UTF8String], "(%d,%d)", &curX, &curY);}

				action ssaalign {
					if (outputbegin == pb) ParseASSAlignment(SSA2ASSAlignment(intnum), &div->alignH, &div->alignV);
				}

				action align {
					if (outputbegin == pb) ParseASSAlignment(intnum, &div->alignH, &div->alignV);
				}

				action wrapstyle {
					if (!startNewLayout) {
						startNewLayout = YES;

						if ([div->text length] > 0) {[divs addObject:div]; div = [div nextDivWithDelegate:delegate];}
					}

					div->wrapStyle = intnum;
				}

				action position {
					if (!startNewLayout) {
						startNewLayout = YES;

						if ([div->text length] > 0) {[divs addObject:div]; div = [div nextDivWithDelegate:delegate];}
					}

					div->posX = curX;
					div->posY = curY;
				}

				intnum = ("-"? [0-9]+) >paramset %setintnum;
				flag = [01] >paramset %setintnum;
				floatnum = ([0-9]+ ("." [0-9]*)?) >paramset %setfloatnum;
				string = ([^\\}]*) >paramset %setstringval;
				color = ("H"|"&"){,2} (xdigit+) >paramset %sethexnum "&"?;
				parens = "(" [^)]* ")";
				xypos = ("(" [0-9]+ "," [0-9]+ ")") >paramset %setxypos;

				cmd = "\\" (
							"b" intnum %bold
							|"i" flag %italic
							|"u" flag %underline
							|"s" flag %strikeout
							|"bord" floatnum %outlinesize
							|"shad" floatnum %shadowdist
							|"be" flag %bluredge
							|"fn" string %fontname
							|"fs" floatnum %fontsize
							|"fscx" floatnum %scalex
							|"fscy" floatnum %scaley
							|"fsp" floatnum %tracking
							|("fr" "z"? floatnum %frz)
							|"frx" floatnum %frx
							|"fry" floatnum %fry
							|"fe" intnum
							|("1"? "c" color %primaryc)
							|"2c" color %secondaryc
							|"3c" color %outlinec
							|"4c" color %shadowc
							|"alpha" color %alpha
							|"1a" color %primarya
							|"2a" color %secondarya
							|"3a" color %outlinea
							|"4a" color %shadowa
							|"a" intnum %ssaalign
							|"an" intnum %align
							|([kK] [fo]? intnum)
							|"q" intnum %wrapstyle
							|"r" string %stylerevert
							|"pos" xypos %position
							|"t" parens
							|"org" parens
							|("fad" "e"? parens)
							|"clip" parens
							|"p" floatnum
							|"pbo" floatnum
					   );

				cmd_list = "{" (cmd* | any*) :> "}";

				action backslash_handler {
					[div->text appendString:send()];
					unichar c = *(p+1), o=c;

					if (c) {
						switch (c) {
							case 'N': case 'n':
								o = '\n';
								break;
							case 'h':
								o = 0xA0; //non-breaking space
								break;
							default:
								o = c;
						}
					}

					[div->text appendFormat:@"%C",o];

					chars_deleted++;

					outputbegin = p+2;
				}

				action enter_tag {
					if (p > outputbegin) [div->text appendString:send()];

					if (p != pb) {
						[div->spans addObject:current_span];
						current_span = [current_span cloneWithDelegate:delegate];
					}

					last_cmd_start = p;
					if (p == pe) reached_end = YES;
				}

				action exit_tag {
					p++;
					chars_deleted += (p - last_cmd_start);

					current_span->offset = (p - pb) - chars_deleted;
					outputbegin = p;

					if (startNewLayout) {
						startNewLayout = NO;
						chars_deleted = outputbegin - pb;
					}

					p--;
				}

				special = ("\\" any) >backslash_handler | cmd_list >enter_tag @exit_tag;
				sub_text_char = [^\\{];
				sub_text = sub_text_char*;

				main := ((sub_text | special)* "\\"?) %/enter_tag;
			}%%

			%%write init;
			%%write exec;
			%%write eof;

			if (!reached_end) NSLog(@"parse error: %@",inputText);
			if (linebuf[linelen-1] == '\\') [div->text appendString:@"\\"];
			[divs addObject:div];
		}

	}

	return divs;
}