Remove debug instruction.

Remove debug instruction.

/* -*- c -*- */

%% machine ragel_transport_header;

static gboolean ragel_parse_transport_header(RTSP_buffer *rtsp,
                                             RTP_transport *rtp_t,
                                             char *header) {
    struct ParsedTransport transport;
    int cs;
    char *p = header, *pe = p + strlen(p) +1, *eof;
    uint32_t portval; uint16_t chanval;

    %%{
        action start_port {
            portval = 0;
        }

        action count_port {
            portval = (portval*10) + (fc - '0');
        }

        action check_port {
            if ( portval > G_MAXUINT16 )
                portval = G_MAXUINT16;
        }

        action start_channel {
            chanval = 0;
        }

        action count_channel {
            chanval = (chanval*10) + (fc - '0');
        }

        action check_channel {
            if ( chanval > G_MAXUINT8 )
                chanval = G_MAXUINT8;
        }

        Port = digit{,5} >start_port @count_port %check_port;
        Channel = digit{,3} >start_channel @count_channel %check_channel;

        Unicast = ";unicast" %{transport.mode = TransportUnicast;};
        Multicast = ";multicast" %{transport.mode = TransportMulticast;};

        TransportParamName = ( alnum | '_' )+;
        TransportParamValue = (print - ('=' | ';' ))+;

        TransportParam = ";" . TransportParamName . ( '=' . TransportParamValue )?;

        ClientPort = ";client_port=" .
            Port%{transport.parameters.UDP.Unicast.port_rtp = portval;} .
            ( "-" . Port%{transport.parameters.UDP.Unicast.port_rtcp = portval;} );

        UnicastUDPParams = Unicast . ( ClientPort | TransportParam )+;
        MulticastUDPParams = Multicast . TransportParam+;

        UDPParams = ( UnicastUDPParams | MulticastUDPParams );

        Interleaved = ";interleaved=" .
            Channel%{transport.parameters.TCP.ich_rtp = chanval;} .
            ( "-" . Channel%{transport.parameters.TCP.ich_rtcp = chanval;} );

        TCPParams = ( Interleaved | TransportParam)+;

        Streams = ";streams" .
            Channel%{transport.parameters.SCTP.ch_rtp = chanval;} .
            ( "-" . Channel%{transport.parameters.SCTP.ch_rtcp = chanval;} );

        SCTPParams = TransportParam+;

        TransportUDP = ("/UDP")? %{transport.protocol = TransportUDP; }
            . UDPParams;
        TransportTCP = "/TCP" %{transport.protocol = TransportTCP; }
            . TCPParams;
        TransportSCTP = "/SCTP" %{transport.protocol = TransportSCTP; }
            . SCTPParams;

        action check_transport {
            /* If the transport is valid, our work is done */
            if ( check_parsed_transport(rtsp, rtp_t, &transport) )
                return true;
        }

        TransportSpec = "RTP/AVP" . ( TransportUDP|TransportTCP|TransportSCTP )
            > { memset(&transport, 0, sizeof(transport)); }
            % check_transport;

        main := TransportSpec . ( ',' . TransportSpec )* . 0;

        write data;
        write init;
        write exec;
    }%%

    return false;
}

/* -*- c -*- */

%% machine ragel_transport_header;

static gboolean ragel_parse_transport_header(RTSP_buffer *rtsp,
                                             RTP_transport *rtp_t,
                                             char *header) {
    struct ParsedTransport transport;
    int cs;
    char *p = header, *pe = p + strlen(p) +1, *eof;
    uint32_t portval; uint16_t chanval;

    %%{
        action debug {
            fprintf(stderr, "Diego debugs: %s\n", p);
        }

        action start_port {
            portval = 0;
        }

        action count_port {
            portval = (portval*10) + (fc - '0');
        }

        action check_port {
            if ( portval > G_MAXUINT16 )
                portval = G_MAXUINT16;
        }

        action start_channel {
            chanval = 0;
        }

        action count_channel {
            chanval = (chanval*10) + (fc - '0');
        }

        action check_channel {
            if ( chanval > G_MAXUINT8 )
                chanval = G_MAXUINT8;
        }

        Port = digit{,5} >start_port @count_port %check_port;
        Channel = digit{,3} >start_channel @count_channel %check_channel;

        Unicast = ";unicast" %{transport.mode = TransportUnicast;};
        Multicast = ";multicast" %{transport.mode = TransportMulticast;};

        TransportParamName = ( alnum | '_' )+;
        TransportParamValue = (print - ('=' | ';' ))+;

        TransportParam = ";" . TransportParamName . ( '=' . TransportParamValue )?;

        ClientPort = ";client_port=" .
            Port%{transport.parameters.UDP.Unicast.port_rtp = portval;} .
            ( "-" . Port%{transport.parameters.UDP.Unicast.port_rtcp = portval;} );

        UnicastUDPParams = Unicast . ( ClientPort | TransportParam )+;
        MulticastUDPParams = Multicast . TransportParam+;

        UDPParams = ( UnicastUDPParams | MulticastUDPParams );

        Interleaved = ";interleaved=" .
            Channel%{transport.parameters.TCP.ich_rtp = chanval;} .
            ( "-" . Channel%{transport.parameters.TCP.ich_rtcp = chanval;} );

        TCPParams = ( Interleaved | TransportParam)+;

        Streams = ";streams" .
            Channel%{transport.parameters.SCTP.ch_rtp = chanval;} .
            ( "-" . Channel%{transport.parameters.SCTP.ch_rtcp = chanval;} );

        SCTPParams = TransportParam+;

        TransportUDP = ("/UDP")? %{transport.protocol = TransportUDP; }
            . UDPParams;
        TransportTCP = "/TCP" %{transport.protocol = TransportTCP; }
            . TCPParams;
        TransportSCTP = "/SCTP" %{transport.protocol = TransportSCTP; }
            . SCTPParams;

        action check_transport {
            /* If the transport is valid, our work is done */
            if ( check_parsed_transport(rtsp, rtp_t, &transport) )
                return true;
        }

        TransportSpec = "RTP/AVP" . ( TransportUDP|TransportTCP|TransportSCTP )
            > { memset(&transport, 0, sizeof(transport)); }
            % check_transport;

        main := TransportSpec . ( ',' . TransportSpec )* . 0;

        write data;
        write init;
        write exec;
    }%%

    return false;
}


added comment explaining clang's -Wweak-vtables warning for lex_error

#ifndef HEXT_LEXER_H
#define HEXT_LEXER_H

#include <string>
#include <vector>
#include <sstream>
#include <stdexcept>
#include <utility>

#include "hext/token.h"
#include "hext/lexer-util.h"

namespace hext {

namespace ragel {
  %%{
    machine hext;
    include "hext.rl";
    write data;
  }%%
} // namespace ragel

/// lexer contains the ragel state machine and shields the user from its
/// details.
class lexer
{
public:
  /// Clang warns (-Wweak-vtables) that a vtable for lex_error may be placed
  /// in every translation unit, because lex_error doesn't have any
  /// 'out-of-line virtual method definitions', where it would normally put
  /// the vtable. But http://stackoverflow.com/a/23749273 suggests that this
  /// is a non-issue; the linker will clean it up.
  class lex_error : public std::runtime_error
  {
  public:
    explicit lex_error(const std::string& msg);
  };

  /// Setup ragel
  lexer(const char * begin, const char * end);

  /// Throws lexer::lex_error on invalid input (by calling lexer::throw_error)
  std::vector<token> lex();

private:
  /// Print diagnostics and throw lexer::lex_error.
  void throw_error() const;

  /// A pointer to the beginning of the overall input is needed to provide
  /// good error diagnostics.
  const char * p_begin;

  /// ragel's runtime state
  const char * p;
  const char * pe;
  const char * eof;
  int cs;
};

} // namespace hext

#endif // HEXT_LEXER_H

#ifndef HEXT_LEXER_H
#define HEXT_LEXER_H

#include <string>
#include <vector>
#include <sstream>
#include <stdexcept>
#include <utility>

#include "hext/token.h"
#include "hext/lexer-util.h"

namespace hext {

namespace ragel {
  %%{
    machine hext;
    include "hext.rl";
    write data;
  }%%
} // namespace ragel

/// lexer contains the ragel state machine and shields the user from its
/// details.
class lexer
{
public:
  class lex_error : public std::runtime_error
  {
  public:
    explicit lex_error(const std::string& msg);
  };

  /// Setup ragel
  lexer(const char * begin, const char * end);

  /// Throws lexer::lex_error on invalid input (by calling lexer::throw_error)
  std::vector<token> lex();

private:
  /// Print diagnostics and throw lexer::lex_error.
  void throw_error() const;

  /// A pointer to the beginning of the overall input is needed to provide
  /// good error diagnostics.
  const char * p_begin;

  /// ragel's runtime state
  const char * p;
  const char * pe;
  const char * eof;
  int cs;
};

} // namespace hext

#endif // HEXT_LEXER_H


Beginning table cleanup

Beginning table cleanup

%%{
  machine table_common;

  EOL = '\r'? '\n' @inc_line_number;
  cell_content = ^('|' | EOL);

  # table is one or more rows
  # a row is space* then any number of cells followed by a '| space* EOL'
  # a cell is a '|' followed by anything that is not a '|'

  cell = cell_content+ >begin_content '|' >store_cell_content | '|' >no_content;

  table_row = space* '|' >initialize cell+ space* %/store_row %/store_table space* :>> (EOL+ space*);

  unclosed_table_row = space* '|' >initialize cell* space* cell_content+ %/bad_table_row %/store_table :>> (EOL+ space*);

  Table = (table_row %store_row | unclosed_table_row >set_bad_table_row_line %bad_table_row)+ %store_table;

  main := Table;
}%%

%%{
  machine table_common;

  EOL = '\r'? '\n' @inc_line_number;
  BAR = '|';
  cell_content = ^('|' | EOL);

  cell = cell_content+ >begin_content BAR >store_cell_content | BAR >no_content;
  table_row = space* BAR >initialize cell+ space* %/store_row %/store_table space* :>> (EOL+ space*);
  unclosed_table_row = space* BAR >initialize cell* space* ^(BAR|EOL)+ %/bad_table_row %/store_table :>> (EOL+ space*);
  table = (table_row %store_row | unclosed_table_row >set_bad_table_row_line %bad_table_row)+ %store_table;
  main := table;
}%%


Perform manual lookahead for H1_END and friends

Ragel is much closer to the metal than ANTLR; in order to check if
we've found the last token before a newline or the end-of-file we
have to do manually lookahead using pointers.

Signed-off-by: Wincent Colaiuta <c76ac2e9cd86441713c7dfae92c451f3e6d17fdc@wincent.com>

// Copyright 2008 Wincent Colaiuta
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program.  If not, see <http://www.gnu.org/licenses/>.

//----------------------------------------------------------------------//
// NOTE: wikitext_ragel.c is generated from wikitext_ragel.rl, so       //
//       if you make changes to the former they will be overwritten.    //
//       You should perform all your edits in wikitext_ragel.rl.        //
//----------------------------------------------------------------------//

#include "wikitext_ragel.h"
#include <stdio.h>

#define EMIT(t)     do { token.type = t; token.stop = p + 1; token.column_stop += (token.stop - token.start); } while (0)
#define MARK()      do { mark = p; } while (0)
#define REWIND()    do { p = mark; } while (0)

%%{
    machine wikitext;

    action mark
    {
        MARK();
    }

    action non_printable_ascii
    {
        token.code_point = *p & 0x7f;
    }

    action two_byte_utf8_sequence
    {
        token.code_point = ((uint32_t)(*(p - 1)) & 0x1f) << 6 |
            (*p & 0x3f);
    }

    action three_byte_utf8_sequence
    {
        token.code_point = ((uint32_t)(*(p - 2)) & 0x0f) << 12 |
            ((uint32_t)(*(p - 1)) & 0x3f) << 6 |
            (*p & 0x3f);
    }

    action four_byte_utf8_sequence
    {
        token.code_point = ((uint32_t)(*(p - 3)) & 0x07) << 18 |
            ((uint32_t)(*(p - 2)) & 0x3f) << 12 |
            ((uint32_t)(*(p - 1)) & 0x3f) << 6 |
            (*p & 0x3f);
    }
#    uri

    main := |*

        /<nowiki>/i
        {
            EMIT(NO_WIKI_START);
            fbreak;
        };

        /<\/nowiki>/i   # this comment because the Ragel TextMate bundle miscolorizes without it /
        {
            EMIT(NO_WIKI_END);
            fbreak;
        };

        "'"{5}
        {
            EMIT(STRONG_EM);
            fbreak;
        };

        "'"{3}
        {
            EMIT(STRONG);
            fbreak;
        };

        "'"{2}
        {
            EMIT(EM);
            fbreak;
        };

        '`'
        {
            EMIT(TT);
            fbreak;
        };

        /<tt>/i
        {
            EMIT(TT_START);
            fbreak;
        };

        /<\/tt>/i   # this comment because the Ragel TextMate bundle miscolorizes without it /
        {
            EMIT(TT_END);
            fbreak;
        };

        # consider adding

 HTML tags as well later on
        '>' %mark ' '?
        {
            if (token.column_start == 1 || last_token_type == BLOCKQUOTE)
                EMIT(BLOCKQUOTE);
            else
            {
                REWIND();
                EMIT(GREATER);
            }
            fbreak;
        };

        # shorthand for
 and

        # consider adding real
 and
 HTML tags later on
        ' '
        {
            if (token.column_start == 1)
                EMIT(PRE);
            else
                EMIT(SPACE);
            fbreak;
        };

        '#'
        {
            if (token.column_start == 1 || last_token_type == OL || last_token_type == UL || last_token_type == BLOCKQUOTE)
                EMIT(OL);
            else
                EMIT(PRINTABLE);
            fbreak;
        };

        '*'
        {
            if (token.column_start == 1 || last_token_type == OL || last_token_type == UL || last_token_type == BLOCKQUOTE)
                EMIT(UL);
            else
                EMIT(PRINTABLE);
            fbreak;
        };

        '='{6} %mark ' '*
        {
            if (token.column_start == 1 || last_token_type == BLOCKQUOTE)
            {
                REWIND();
                EMIT(H6_START);
            }
            else if (p > mark && (p == pe || *(p + 1) == '\n' || *(p + 1) == '\r'))
            {
                REWIND();
                EMIT(H6_END);
            }
            else
                EMIT(PRINTABLE);
            fbreak;
        };

        '='{5} %mark ' '*
        {
            if (token.column_start == 1 || last_token_type == BLOCKQUOTE)
            {
                REWIND();
                EMIT(H5_START);
            }
            else if (p > mark && (p == pe || *(p + 1) == '\n' || *(p + 1) == '\r'))
            {
                REWIND();
                EMIT(H6_END);
            }
                EMIT(PRINTABLE);
            fbreak;
        };

        '='{4} %mark ' '*
        {
            if (token.column_start == 1 || last_token_type == BLOCKQUOTE)
            {
                REWIND();
                EMIT(H4_START);
            }
            else if (p > mark && (p == pe || *(p + 1) == '\n' || *(p + 1) == '\r'))
            {
                REWIND();
                EMIT(H4_END);
            }
            else
                EMIT(PRINTABLE);
            fbreak;
        };

        '='{3} %mark ' '*
        {
            if (token.column_start == 1 || last_token_type == BLOCKQUOTE)
            {
                REWIND();
                EMIT(H3_START);
            }
            else if (p > mark && (p == pe || *(p + 1) == '\n' || *(p + 1) == '\r'))
            {
                REWIND();
                EMIT(H3_END);
            }
            else
                EMIT(PRINTABLE);
            fbreak;
        };

        '='{2} %mark ' '*
        {
            if (token.column_start == 1 || last_token_type == BLOCKQUOTE)
            {
                REWIND();
                EMIT(H2_START);
            }
            else if (p > mark && (p == pe || *(p + 1) == '\n' || *(p + 1) == '\r'))
            {
                REWIND();
                EMIT(H2_END);
            }
            else
                EMIT(PRINTABLE);
            fbreak;
        };

        '=' %mark ' '*
        {
            if (token.column_start == 1 || last_token_type == BLOCKQUOTE)
            {
                REWIND();
                EMIT(H1_START);
            }
            else if (p > mark && (p == pe || *(p + 1) == '\n' || *(p + 1) == '\r'))
            {
                REWIND();
                EMIT(H1_END);
            }
            else
                EMIT(PRINTABLE);
            fbreak;
        };

        '[['
        {
            EMIT(LINK_START);
            fbreak;
        };

        ']]'
        {
            EMIT(LINK_END);
            fbreak;
        };

        '|'
        {
            EMIT(SEPARATOR);
            fbreak;
        };

        '['
        {
            EMIT(EXT_LINK_START);
            fbreak;
        };

        ']'
        {
            EMIT(EXT_LINK_END);
            fbreak;
        };

        '"'
        {
            EMIT(QUOT_ENTITY);
            fbreak;
        };

        '&'
        {
            EMIT(AMP_ENTITY);
            fbreak;
        };

        '&' alpha+ digit* ';'
        {
            EMIT(NAMED_ENTITY);
            fbreak;
        };

        '&#' [xX] xdigit+ ';'
        {
            EMIT(HEX_ENTITY);
            fbreak;
        };

        '&#' digit+ ';'
        {
            EMIT(DECIMAL_ENTITY);
            fbreak;
        };

        '"'
        {
            EMIT(QUOT);
            fbreak;
        };

        '&'
        {
            EMIT(AMP);
            fbreak;
        };

        '<'
        {
            EMIT(LESS);
            fbreak;
        };

        '>'
        {
            EMIT(GREATER);
            fbreak;
        };

        ("\r"? "\n") | "\r"
        {
            EMIT(CRLF);
            token.column_stop = 1;
            token.line_stop++;
            fbreak;
        };

        # all the printable ASCII characters (0x20 to 0x7e) excluding those explicitly covered elsewhere:
        # skip space (0x20), quote (0x22), ampersand (0x26), less than (0x3c), greater than (0x3e),
        # left bracket 0x5b, right bracket 0x5d, backtick (0x60), and vertical bar (0x7c)
        (0x21 | 0x23..0x25 | 0x27..0x3b | 0x3d | 0x3f..0x5a | 0x5c | 0x5e..0x5f | 0x61..0x7b | 0x7e)+
        {
            EMIT(PRINTABLE);
            fbreak;
        };

        # here is where we handle the UTF-8 and everything else
        #
        #     one_byte_sequence   = byte begins with zero;
        #     two_byte_sequence   = first byte begins with 110 (0xc0..0xdf), next with 10 (0x80..9xbf);
        #     three_byte_sequence = first byte begins with 1110 (0xe0..0xef), next two with 10 (0x80..9xbf);
        #     four_byte_sequence  = first byte begins with 11110 (0xf0..0xf7), next three with 10 (0x80..9xbf);
        #
        #     within the ranges specified, we also exclude these illegal sequences:
        #       1100000x (c0 c1)    overlong encoding, lead byte of 2 byte seq but code point <= 127
        #       11110101 (f5)       restricted by RFC 3629 lead byte of 4-byte sequence for codepoint above 10ffff
        #       1111011x (f6, f7)   restricted by RFC 3629 lead byte of 4-byte sequence for codepoint above 10ffff
        (0x01..0x1f | 0x7f)                             %non_printable_ascii        |
        (0xc2..0xdf 0x80..0xbf)                         %two_byte_utf8_sequence     |
        (0xe0..0xef 0x80..0xbf 0x80..0xbf)              %three_byte_utf8_sequence   |
        (0xf0..0xf4 0x80..0xbf 0x80..0xbf 0x80..0xbf)   %four_byte_utf8_sequence
        {
            EMIT(DEFAULT);
            token.column_stop = token.column_start + 1;
            fbreak;
        };

    *|;

    write data;
}%%

// for now we use the scanner as a tokenizer that returns one token at a time, just like ANTLR
// ultimately we could look at embedding all of the transformation inside the scanner itself (combined scanner/parser)
// pass in the last token because that's useful for the scanner to know
// p data pointer (required by Ragel machine); overriden with contents of last_token if supplied
// pe data end pointer (required by Ragel machine)
token_t next_token(token_t *last_token, char *p, char *pe)
{
    // the token object to be returned
    token_t token;
    token.type              = NO_TOKEN;
    int     last_token_type = NO_TOKEN;
    if (last_token)
    {
        last_token_type     = last_token->type;
        p = last_token->stop;
        token.line_start    = token.line_stop   = last_token->line_stop;
        token.column_start  = token.column_stop = last_token->column_stop;
    }
    else
    {
        token.line_start    = 1;
        token.column_start  = 1;
        token.line_stop     = 1;
        token.column_stop   = 1;
    }
    token.start = p;
    if (p == pe)
    {
        // all done, have reached end of input
        token.stop  = p;
        token.type  = END_OF_FILE;
        return token;
    }

    char    *mark;      // for manual backtracking
    char    *eof = pe;  // required for backtracking (longest match determination)
    int     cs;         // current state (standard Ragel)
    char    *ts;        // token start (scanner)
    char    *te;        // token end (scanner)
    int     act;        // identity of last patterned matched (scanner)
    %% write init;
    %% write exec;
    if (cs == wikitext_error)
        rb_raise(eWikitextError, "failed before finding a token");
    else if (token.type == NO_TOKEN)
        rb_raise(eWikitextError, "failed to produce a token");
    return token;
}

// Copyright 2008 Wincent Colaiuta
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program.  If not, see <http://www.gnu.org/licenses/>.

//----------------------------------------------------------------------//
// NOTE: wikitext_ragel.c is generated from wikitext_ragel.rl, so       //
//       if you make changes to the former they will be overwritten.    //
//       You should perform all your edits in wikitext_ragel.rl.        //
//----------------------------------------------------------------------//

#include "wikitext_ragel.h"
#include <stdio.h>

#define EMIT(t)     do { token.type = t; token.stop = p + 1; token.column_stop += (token.stop - token.start); } while (0)
#define MARK()      do { mark = p; } while (0)
#define REWIND()    do { p = mark; } while (0)

%%{
    machine wikitext;

    action mark
    {
        MARK();
    }

    action non_printable_ascii
    {
        token.code_point = *p & 0x7f;
    }

    action two_byte_utf8_sequence
    {
        token.code_point = ((uint32_t)(*(p - 1)) & 0x1f) << 6 |
            (*p & 0x3f);
    }

    action three_byte_utf8_sequence
    {
        token.code_point = ((uint32_t)(*(p - 2)) & 0x0f) << 12 |
            ((uint32_t)(*(p - 1)) & 0x3f) << 6 |
            (*p & 0x3f);
    }

    action four_byte_utf8_sequence
    {
        token.code_point = ((uint32_t)(*(p - 3)) & 0x07) << 18 |
            ((uint32_t)(*(p - 2)) & 0x3f) << 12 |
            ((uint32_t)(*(p - 1)) & 0x3f) << 6 |
            (*p & 0x3f);
    }
#    uri

    main := |*

        /<nowiki>/i
        {
            EMIT(NO_WIKI_START);
            fbreak;
        };

        /<\/nowiki>/i   # this comment because the Ragel TextMate bundle miscolorizes without it /
        {
            EMIT(NO_WIKI_END);
            fbreak;
        };

        "'"{5}
        {
            EMIT(STRONG_EM);
            fbreak;
        };

        "'"{3}
        {
            EMIT(STRONG);
            fbreak;
        };

        "'"{2}
        {
            EMIT(EM);
            fbreak;
        };

        '`'
        {
            EMIT(TT);
            fbreak;
        };

        /<tt>/i
        {
            EMIT(TT_START);
            fbreak;
        };

        /<\/tt>/i   # this comment because the Ragel TextMate bundle miscolorizes without it /
        {
            EMIT(TT_END);
            fbreak;
        };

        # consider adding

 HTML tags as well later on
        '>' %mark ' '?
        {
            if (token.column_start == 1 || last_token_type == BLOCKQUOTE)
                EMIT(BLOCKQUOTE);
            else
            {
                REWIND();
                EMIT(GREATER);
            }
            fbreak;
        };

        # shorthand for
 and

        # consider adding real
 and
 HTML tags later on
        ' '
        {
            if (token.column_start == 1)
                EMIT(PRE);
            else
                EMIT(SPACE);
            fbreak;
        };

        '#'
        {
            if (token.column_start == 1 || last_token_type == OL || last_token_type == UL || last_token_type == BLOCKQUOTE)
                EMIT(OL);
            else
                EMIT(PRINTABLE);
            fbreak;
        };

        '*'
        {
            if (token.column_start == 1 || last_token_type == OL || last_token_type == UL || last_token_type == BLOCKQUOTE)
                EMIT(UL);
            else
                EMIT(PRINTABLE);
            fbreak;
        };

        # not sure how to test for EOF
        #'='{6} %mark (' '* ( '\n' | '\r' | EOF ))?
        '='{6} %mark (' '* ( '\n' | '\r'))?
        {
            if (token.column_start == 1 || last_token_type == BLOCKQUOTE)
            {
                REWIND();
                EMIT(H6_START);
            }
            else if (p > mark)
            {
                REWIND();
                EMIT(H6_END);
            }
            else
                EMIT(PRINTABLE);
            fbreak;
        };

        '='{5} %mark (' '* ( '\n' | '\r'))?
        {
            if (token.column_start == 1 || last_token_type == BLOCKQUOTE)
            {
                REWIND();
                EMIT(H5_START);
            }
            else if (p > mark)
            {
                REWIND();
                EMIT(H6_END);
            }
                EMIT(PRINTABLE);
            fbreak;
        };

        '='{4} %mark (' '* ( '\n' | '\r'))?
        {
            if (token.column_start == 1 || last_token_type == BLOCKQUOTE)
            {
                REWIND();
                EMIT(H4_START);
            }
            else if (p > mark)
            {
                REWIND();
                EMIT(H4_END);
            }
            else
                EMIT(PRINTABLE);
            fbreak;
        };

        '='{3} %mark (' '* ( '\n' | '\r'))?
        {
            if (token.column_start == 1 || last_token_type == BLOCKQUOTE)
            {
                REWIND();
                EMIT(H3_START);
            }
            else if (p > mark)
            {
                REWIND();
                EMIT(H3_END);
            }
            else
                EMIT(PRINTABLE);
            fbreak;
        };

        '='{2} %mark (' '* ( '\n' | '\r'))?
        {
            if (token.column_start == 1 || last_token_type == BLOCKQUOTE)
            {
                REWIND();
                EMIT(H2_START);
            }
            else if (p > mark)
            {
                REWIND();
                EMIT(H2_END);
            }
            else
                EMIT(PRINTABLE);
            fbreak;
        };

        '=' %mark (' '* ( '\n' | '\r'))?
        {
            if (token.column_start == 1 || last_token_type == BLOCKQUOTE)
            {
                REWIND();
                EMIT(H1_START);
            }
            else if (p > mark)
            {
                REWIND();
                EMIT(H1_END);
            }
            else
                EMIT(PRINTABLE);
            fbreak;
        };

        '[['
        {
            EMIT(LINK_START);
            fbreak;
        };

        ']]'
        {
            EMIT(LINK_END);
            fbreak;
        };

        '|'
        {
            EMIT(SEPARATOR);
            fbreak;
        };

        '['
        {
            EMIT(EXT_LINK_START);
            fbreak;
        };

        ']'
        {
            EMIT(EXT_LINK_END);
            fbreak;
        };

        '"'
        {
            EMIT(QUOT_ENTITY);
            fbreak;
        };

        '&'
        {
            EMIT(AMP_ENTITY);
            fbreak;
        };

        '&' alpha+ digit* ';'
        {
            EMIT(NAMED_ENTITY);
            fbreak;
        };

        '&#' [xX] xdigit+ ';'
        {
            EMIT(HEX_ENTITY);
            fbreak;
        };

        '&#' digit+ ';'
        {
            EMIT(DECIMAL_ENTITY);
            fbreak;
        };

        '"'
        {
            EMIT(QUOT);
            fbreak;
        };

        '&'
        {
            EMIT(AMP);
            fbreak;
        };

        '<'
        {
            EMIT(LESS);
            fbreak;
        };

        '>'
        {
            EMIT(GREATER);
            fbreak;
        };

        ("\r"? "\n") | "\r"
        {
            EMIT(CRLF);
            token.column_stop = 1;
            token.line_stop++;
            fbreak;
        };

        # all the printable ASCII characters (0x20 to 0x7e) excluding those explicitly covered elsewhere:
        # skip space (0x20), quote (0x22), ampersand (0x26), less than (0x3c), greater than (0x3e),
        # left bracket 0x5b, right bracket 0x5d, backtick (0x60), and vertical bar (0x7c)
        (0x21 | 0x23..0x25 | 0x27..0x3b | 0x3d | 0x3f..0x5a | 0x5c | 0x5e..0x5f | 0x61..0x7b | 0x7e)+
        {
            EMIT(PRINTABLE);
            fbreak;
        };

        # here is where we handle the UTF-8 and everything else
        #
        #     one_byte_sequence   = byte begins with zero;
        #     two_byte_sequence   = first byte begins with 110 (0xc0..0xdf), next with 10 (0x80..9xbf);
        #     three_byte_sequence = first byte begins with 1110 (0xe0..0xef), next two with 10 (0x80..9xbf);
        #     four_byte_sequence  = first byte begins with 11110 (0xf0..0xf7), next three with 10 (0x80..9xbf);
        #
        #     within the ranges specified, we also exclude these illegal sequences:
        #       1100000x (c0 c1)    overlong encoding, lead byte of 2 byte seq but code point <= 127
        #       11110101 (f5)       restricted by RFC 3629 lead byte of 4-byte sequence for codepoint above 10ffff
        #       1111011x (f6, f7)   restricted by RFC 3629 lead byte of 4-byte sequence for codepoint above 10ffff
        (0x01..0x1f | 0x7f)                             %non_printable_ascii        |
        (0xc2..0xdf 0x80..0xbf)                         %two_byte_utf8_sequence     |
        (0xe0..0xef 0x80..0xbf 0x80..0xbf)              %three_byte_utf8_sequence   |
        (0xf0..0xf4 0x80..0xbf 0x80..0xbf 0x80..0xbf)   %four_byte_utf8_sequence
        {
            EMIT(DEFAULT);
            token.column_stop = token.column_start + 1;
            fbreak;
        };

    *|;

    write data;
}%%

// for now we use the scanner as a tokenizer that returns one token at a time, just like ANTLR
// ultimately we could look at embedding all of the transformation inside the scanner itself (combined scanner/parser)
// pass in the last token because that's useful for the scanner to know
// p data pointer (required by Ragel machine); overriden with contents of last_token if supplied
// pe data end pointer (required by Ragel machine)
token_t next_token(token_t *last_token, char *p, char *pe)
{
    // the token object to be returned
    token_t token;
    token.type              = NO_TOKEN;
    int     last_token_type = NO_TOKEN;
    if (last_token)
    {
        last_token_type     = last_token->type;
        p = last_token->stop;
        token.line_start    = token.line_stop   = last_token->line_stop;
        token.column_start  = token.column_stop = last_token->column_stop;
    }
    else
    {
        token.line_start    = 1;
        token.column_start  = 1;
        token.line_stop     = 1;
        token.column_stop   = 1;
    }
    token.start = p;
    if (p == pe)
    {
        // all done, have reached end of input
        token.stop  = p;
        token.type  = END_OF_FILE;
        return token;
    }

    char    *mark;      // for manual backtracking
    char    *eof = pe;  // required for backtracking (longest match determination)
    int     cs;         // current state (standard Ragel)
    char    *ts;        // token start (scanner)
    char    *te;        // token end (scanner)
    int     act;        // identity of last patterned matched (scanner)
    %% write init;
    %% write exec;
    if (cs == wikitext_error)
        rb_raise(eWikitextError, "failed before finding a token");
    else if (token.type == NO_TOKEN)
        rb_raise(eWikitextError, "failed to produce a token");
    return token;
}


Fix multi_token parsing in ascii_client

Fix multi_token parsing in ascii_client

Summary:
Spaces inside multi_token and ' '* after cause non-determinism.
We don't need to parse spaces after separately anyway.

Test Plan: unit tests

Reviewed By: stepan@fb.com

Subscribers: alikhtarov

FB internal diff: D1496655

@override-unit-failures

/* -*- mode: c; -*- */
/* ascii_client.c is built from ascii_client.rl using ragel */

#include <ctype.h>

#include "mcrouter/lib/fbi/debug.h"
#include "mcrouter/lib/mc/_protocol.h"
#include "mcrouter/lib/mc/parser.h"
#include "mcrouter/lib/mc/msg.h"
#include "mcrouter/lib/mc/util.h"

// In libmc.client, only read the first N stats for simplicity of
// implementation. In the future it might make sense to allow an
// arbitrary number of stats, but for now that makes the code far
// easier
#define MAX_NUM_STATS 20

// Are we in the middle of a key being skipped?
#define skipping_key(parser) ((parser)->in_skipped_key &&       \
                              !(parser)->record_skip_key)

%%{

# input file for ragel (http://www.complang.org/ragel/)

# this machine name, which must come first, is used as a prefix for variables
machine ascii;

# named actions. All the actions (named and unnamed) are executed in the
# context of _on_ascii_rx() where "%% write exec;" is called.
action start_token {
  if (!skipping_key(parser)) {
    parser->partial_token = 1;
    parser->te = parser->tbuf;
    ts = p;
  }
}
action finish_token {
  if (parser->partial_token && !skipping_key(parser)) {
    FBI_ASSERT(ts);
    parser->partial_token = 0;
    if (parser->te && parser->te > parser->tbuf) {
      /* we have something stuffed in our cheeks,
         let's put them together and update ts. */
      FBI_ASSERT(ts <= p);
      size_t n = p - ts;
      if(mc_parser_ensure_tbuf(parser, n) != 0) {
        if (parser->in_key) {
          parser->bad_key = 1;
        } else {
          parser->error = parser_malformed_request;
          fgoto error_handling;
        }
      } else {
        strncpy(parser->te, ts, n);
        parser->te += n;
        *parser->te = '\0';
        tlen = parser->te - parser->tbuf;
        ts = parser->tbuf;
      }
    } else {
      tlen = p - ts;
      if (tlen >= MC_TOKEN_MAX_LEN) {
        if (parser->in_key) {
          parser->bad_key = 1;
        } else {
          parser->error = parser_malformed_request;
          fgoto error_handling;
        }
      }
    }
  }
}

action record_key {
  if (!skipping_key(parser)) {
    if (tlen > MC_KEY_MAX_LEN_ASCII) {
      parser->bad_key = 1;
    }
    if (parser->bad_key) {
      parser->msg->result = mc_res_bad_key;
    } else {
      if (mc_msg_grow(&parser->msg, tlen + 1,
                      (void**) &parser->msg->key.str) != 0) {
        parser->error = parser_out_of_memory;
        fgoto error_handling;
      }
      parser->msg->key.len = tlen;
      strncpy(parser->msg->key.str, ts, tlen);
      FBI_ASSERT(!memcmp(parser->msg->key.str, ts, tlen));
      parser->msg->key.str[tlen] = '\0';
    }
  }
  parser->in_skipped_key = 0;
  parser->in_key = 0;
  parser->bad_key = 0;
}

action fin {
  FBI_ASSERT(parser->msg != NULL);

  // end of msg, fake eof
  pe = eof = p + 1;
  parser->parser_state = parser_msg_header;
  /* Note: we need to reset parser->msg first,
     in case mc_parser_reset is called from the callback */
  mc_msg_t* msg = parser->msg;
  parser->msg = NULL;
  parser->msg_ready(parser->context, 0, msg);
  nmsgs++;

  mc_parser_cleanup_tbuf(parser);
}

nl = '\r'? '\n';
fin = nl @fin;

# server-reported errors. We should only ever see SERVER_ERROR. If we see
# CLIENT_ERROR then libmc probably didn't do enough error checking before
# sending to memcache in the first place. ERROR is absolutely a bug in libmc.
error_handling := '' >{
  if (parser->msg != NULL) {
    mc_msg_decref(parser->msg);
  }
  parser->msg = NULL;
  parser->te = NULL;
  parser->in_key = 0;
  parser->bad_key = 0;
  fbreak;
};

error_code = ' ' digit+ ' '
  @{
    parser->msg->err_code = strtoll(ts, NULL, 10);
  };

action capture_error_value {
  // Since the token we are parsing includes the \r\n, we need
  // to invoke this code upon entering the DFA's final state.
  // Only the '\r' is stored in the token buffer, so we only
  // need to remove one character.
  tlen--;
  if (mc_msg_grow(&parser->msg, tlen + 1,
                  (void**) &parser->msg->value.str) != 0) {
    parser->error = parser_out_of_memory;
    fgoto error_handling;
  }
  parser->msg->value.len = tlen;
  memcpy(parser->msg->value.str, ts, tlen);
  parser->msg->value.str[tlen] = '\0';
}

# server error apparently needs to be able to read binary data to support
# TAO, which uses SERVER_ERROR to send the fbid of objects back to the
# client during set requests.  There is no size field, so we need to
# keep parsing the response string until we hit \r\n.  Server error
# 307 is returned by a server that is overloaded or is shutting down;
# it is treated specially so that it can cross the application/transport
# boundary.  (mc_res_busy is an artifact of our use of TCP/IP, of our
# desired to keep the connection open while not accepting requests.
# If there was a reliable request-based protocol it would be less
# necessary.)
server_error = 'SERVER_ERROR' error_code? ' '? (/.*\r\n/) >start_token
  @finish_token
  @capture_error_value
  @{
    if (parser->msg->err_code == SERVER_ERROR_BUSY) {
      parser->msg->result = mc_res_busy;
    } else {
      parser->msg->result = mc_res_remote_error;
    }
  }
  @fin;

# client error message may contain useful stuff, so we also want to save
# it for the caller
client_error = 'CLIENT_ERROR' error_code? ' '? (/.*\r\n/) >start_token
  @finish_token
  @capture_error_value
  @{
    parser->msg->result = mc_res_client_error;
  }
  @fin;

command_error = 'ERROR'
  @{
    parser->msg->result = mc_res_local_error;
    parser->msg->value.str = "unexpected ERROR reply";
    parser->msg->value.len = strlen(parser->msg->value.str);
    dbg_error("%s", parser->msg->value.str);
    FBI_ASSERT(0);
  } fin;

error = server_error | client_error | command_error;

# Get request is ready to be passed to the caller
action get_req_is_ready {
  FBI_ASSERT(parser->msg != NULL);

  mc_op_t op = parser->msg->op;
  /* Note: we need to reset parser->msg first,
     in case mc_parser_reset is called from the callback */
  mc_msg_t* msg = parser->msg;
  parser->msg = mc_msg_new(0);
  parser->msg_ready(parser->context, 0, msg);
  if (parser->msg == NULL) {
    parser->error = parser_out_of_memory;
    fgoto error_handling;
  }

  parser->msg->op = op;
}

# building blocks
stored = 'STORED' @{ parser->msg->result = mc_res_stored; };
stale_stored = 'STALE_STORED' @{
  parser->msg->result = mc_res_stalestored;
} fin;
not_stored = 'NOT_STORED' @{ parser->msg->result = mc_res_notstored; };
exists = 'EXISTS' @{ parser->msg->result = mc_res_exists; };
not_found = 'NOT_FOUND' @{ parser->msg->result = mc_res_notfound; };
deleted = 'DELETED' @{ parser->msg->result = mc_res_deleted; };

storage_reply = stored | stale_stored | not_stored | exists | not_found | deleted;
storage = storage_reply fin;
arithmetic_reply = (not_found |
  digit+ >start_token %finish_token %{
    parser->msg->delta = strtoll(ts, NULL, 10);
    parser->msg->result = mc_res_stored;
  } ' '* fin);

key_skip = (any+ -- (cntrl | space))
  >{ parser->in_skipped_key = 1; parser->in_key = 1; }
  >start_token %finish_token
  %record_key;

key = (any+ -- (cntrl | space))
  >{ parser->in_skipped_key = 0; parser->in_key = 1; }
  >start_token %finish_token
  %record_key;

# This is used for stats and exec commands which can have keys with spaces.
multi_token = ' '* (print+ -- ( '\r' | '\n' ))
  >start_token %finish_token %{
  while (tlen > 0 && isspace(ts[tlen-1])) {
    tlen--;
  }
  if (mc_msg_grow(&parser->msg, tlen+1, (void**) &parser->msg->key.str) != 0) {
    parser->error = parser_out_of_memory;
    fgoto error_handling;
  }
  parser->msg->key.len = tlen;
  strncpy(parser->msg->key.str, ts, tlen);
  parser->msg->key.str[tlen] = '\0';
};

lease_token = ('-'? digit+) >start_token %finish_token %{
  parser->msg->lease_id = strtol(ts, NULL, 10);
};
casid = digit+ >start_token %finish_token %{
  parser->msg->cas = strtoll(ts, NULL, 10);
};
flags = digit+ >start_token %finish_token %{
  parser->msg->flags = strtoll(ts, NULL, 10);
};
exptime = digit+ >start_token %finish_token %{
  parser->msg->exptime = strtol(ts, NULL, 10);
};
delta = digit+ >start_token %finish_token %{
  parser->msg->delta = strtoll(ts, NULL, 10);
};
number = digit+ >start_token %finish_token %{
  parser->msg->number = strtoll(ts, NULL, 10);
};
noreply = 'noreply' %{
  parser->msg->noreply = true;
};

value_bytes = digit+ >start_token %finish_token %{
  /* prepare for receiving data */
  size_t len = parser->resid = strtol(ts, NULL, 10);
  parser->off = 0;

  if (mc_msg_grow(&parser->msg, len + 1,
                  (void**) &parser->msg->value.str) != 0) {
    parser->error = parser_out_of_memory;
    fgoto error_handling;
  }
  parser->msg->value.str[len] = '\0';
  parser->msg->value.len = len;
};
cas_unique = digit+ >start_token %finish_token %{
  parser->msg->cas = strtoull(ts, NULL, 10);
};
dec_octet = digit{1,3};
ipv4_addr = dec_octet "." dec_octet "." dec_octet "." dec_octet;
ipv6_addr = (":" | xdigit)+ ipv4_addr?;
ip_addr = (ipv4_addr | ipv6_addr) >start_token %finish_token %{
  ts[tlen]='\0';
  parser->msg->ipv = 0;
  if (strchr(ts, ':') == NULL) {
    if (inet_pton(AF_INET, ts, &parser->msg->ip_addr) > 0)
      parser->msg->ipv = 4;
  } else {
    if (inet_pton(AF_INET6, ts, &parser->msg->ip_addr) > 0)
      parser->msg->ipv = 6;
  }
};
age_unknown = 'unknown' %{ parser->msg->number = -1;};
action rx_data {
  /* the binary data of a specified isn't really regular, so we drop out of
     the state machine and read it, then come back in to finish the
     "\r\nEND\r\n". */
  parser->parser_state = parser_body;
  fbreak;
}

# replies

VALUE = 'VALUE' % { parser->msg->result = mc_res_found; };

hit = VALUE ' '+ key_skip ' '+ flags ' '+ value_bytes (' '+ cas_unique)? nl
  @rx_data nl;

get_reply = (hit? >{ parser->msg->result = mc_res_notfound; } 'END' fin);
lease_get = ((hit |
  ('LVALUE' ' '+ key_skip ' '+ lease_token ' '+ flags ' '+ value_bytes (' '+ cas_unique)? nl
   @rx_data nl
   @{ parser->msg->result = mc_res_notfound; }))
  'END' fin);
META = 'META' % {parser->msg->result = mc_res_found;};
mhit = META ' '+ key_skip ' '+ 'age:' ' '* (number | age_unknown) ';' ' '*
  'exptime:' ' '* exptime ';' ' '* 'from:' ' '* (ip_addr|'unknown') ';' ' '*
  'is_transient:' ' '* flags ' '* nl;
metaget_reply = (mhit? >{ parser->msg->result = mc_res_notfound; } 'END' fin);
reply_set = storage;
reply_add = storage;
reply_replace = storage;
reply_lease_set = storage;
reply_append = storage;
reply_prepend = storage;
reply_cas = storage;
delete_reply = deleted | not_found;

stat_name = (any+ -- (cntrl | space)) >start_token %finish_token
%{
  if (parser->msg->stats == NULL) {
    FBI_ASSERT(parser->msg->number == 0);
    size_t stats_size = sizeof(nstring_t) * 2 * MAX_NUM_STATS;
    if (mc_msg_grow(&parser->msg, stats_size, (void**) &parser->msg->stats)) {
      parser->error = parser_out_of_memory;
      fgoto error_handling;
    }
    memset(parser->msg->stats, 0, stats_size);
  }

  if (parser->msg->number < MAX_NUM_STATS) {
    parser->msg->number++;

    int idx = (parser->msg->number * 2) - 2;
    if (mc_msg_grow(&parser->msg, tlen + 1,
                    (void**) &parser->msg->stats[idx].str) != 0) {
      parser->error = parser_out_of_memory;
      fgoto error_handling;
    }
    parser->msg->stats[idx].len = tlen;
    parser->msg->stats[idx].str[tlen] = '\0';
    strncpy(parser->msg->stats[idx].str, ts, tlen);
  }
};

stat_value = (any++ -- cntrl) > start_token %finish_token
%{
  if (parser->msg->number <= MAX_NUM_STATS) {
    int idx = (parser->msg->number * 2) - 1;
    if (mc_msg_grow(&parser->msg, tlen + 1,
                    (void**) &parser->msg->stats[idx].str) != 0) {
      parser->error = parser_out_of_memory;
      fgoto error_handling;
    }

    parser->msg->stats[idx].len = tlen;
    parser->msg->stats[idx].str[tlen] = '\0';
    strncpy(parser->msg->stats[idx].str, ts, tlen);
  }
};

stats_reply = (('STAT ' @{ parser->msg->result = mc_res_ok; }
                stat_name ' '+ stat_value nl)* 'END' fin);
ok = ('OK' @{ parser->msg->result = mc_res_ok; } fin);
version_reply = ('VERSION ' (any* -- ('\r' | '\n')) >start_token %finish_token
  %{
    // use new_msg because msg stays live if realloc fails and can leak
    mc_msg_t *new_msg = mc_msg_realloc(parser->msg,
                                       parser->msg->_extra_size + tlen + 1);
    if (new_msg == NULL) {
      parser->error = parser_out_of_memory;
      fgoto error_handling;
    }
    parser->msg = new_msg;
    parser->msg->result = mc_res_ok;
    char* version = parser->msg->value.str = (char *) &parser->msg[1];
    parser->msg->value.len = tlen;
    strncpy(version, ts, tlen);
    version[tlen] = '\0';
  }
  fin);

reply := get_reply | lease_get | metaget_reply |
        reply_set | reply_add | reply_replace | reply_lease_set |
        reply_append | reply_prepend | reply_cas | delete_reply |
        arithmetic_reply |
        stats_reply | ok | version_reply | error;

# requests

set = 'set' @ { parser->msg->op = mc_op_set; };
add = 'add' @ { parser->msg->op = mc_op_add; };
replace = 'replace' @ { parser->msg->op = mc_op_replace; };
leaseset = 'lease-set' @ { parser->msg->op = mc_op_lease_set; };
append = 'append' @ { parser->msg->op = mc_op_append; };
prepend = 'prepend' @ { parser->msg->op = mc_op_prepend; };

set_op = set | add | replace | append | prepend;

delete_op = 'delete' @ { parser->msg->op = mc_op_delete; };

get_op = 'get' @ { parser->msg->op = mc_op_get; } |
         'gets' @ {parser->msg->op = mc_op_gets; } |
         'lease-get' @ { parser->msg->op = mc_op_lease_get; } |
         'metaget' @ { parser->msg->op = mc_op_metaget; } ;

arithmetic_op = 'incr' @ { parser->msg->op = mc_op_incr; } |
                'decr' @ { parser->msg->op = mc_op_decr; };

version_op = 'version' @ { parser->msg->op = mc_op_version; };

quit_op = 'quit' @ { parser->msg->op = mc_op_quit;  };

stats_op = 'stats' @ { parser->msg->op = mc_op_stats; };

exec_op = ('exec' | 'admin') @ { parser->msg->op = mc_op_exec; };

shutdown_op = 'shutdown' @ { parser->msg->op = mc_op_shutdown; };

flush_all_op = 'flush_all' @ { parser->msg->op = mc_op_flushall; };
flush_regex_op = 'flush_regex' @ { parser->msg->op = mc_op_flushre; };

cas = 'cas' %{
  parser->msg->op = mc_op_cas;
} ' '+ key ' '+ flags ' '+ exptime ' '+ value_bytes ' '+ casid ' '* nl
  @rx_data fin;

store = (set_op ' '+ key | leaseset ' '+ key ' '+ lease_token) ' '+ flags ' '+ exptime ' '+ value_bytes (' '+ noreply)? ' '* nl
  @rx_data fin;

get = get_op (' '+ key %get_req_is_ready)+ ' '* fin >{
  parser->msg->op = mc_op_end;
};

arithmetic = arithmetic_op ' '+ key ' '+ delta (' '+ noreply)? ' '* fin;

delete = delete_op ' '+ key (' '+ exptime)? (' '+ noreply)? ' '* fin;

quit = quit_op ' '* fin;

version = version_op ' '* fin;

stats = stats_op (' '+ multi_token)? fin;

exec = exec_op ' '+ multi_token fin;

shutdown = shutdown_op (' '+ number)? ' '* fin;

flush_all = flush_all_op (' '+ number)? ' '* fin;

flush_regex = flush_regex_op ' ' key ' '* fin;

request := get | store | cas | delete | arithmetic
  | quit | version | stats | shutdown | flush_all | flush_regex | exec;

request_reply := get | store | cas | delete | arithmetic
  | quit | version | stats | shutdown | flush_all | flush_regex | exec |
  get_reply | lease_get | metaget_reply |
  reply_set | reply_add | reply_replace | reply_lease_set |
  reply_append | reply_prepend | reply_cas | delete_reply |
  arithmetic_reply |
  stats_reply | ok | version_reply | error;

write data;
}%%

int get_reply_ragel_start_state() {
  return ascii_en_reply;
}

int get_request_ragel_start_state() {
  return ascii_en_request;
}

int get_request_reply_ragel_start_state() {
  return ascii_en_request_reply;
}

/** ascii reply parsing
    This is the real workhorse. It consumes buf, resuming the partial reply if
    one exists. As each reply is fully received, we call mcc_req_complete().
    If there is a parse error or remote error, we handle that as well. If
    there is a parse error or unrecoverable remote error, we call
    mcc_on_down() and the rest of the buffer is ignored.
    @return 0 on error */
int _on_ascii_rx(mc_parser_t* parser, char* buf, size_t nbuf) {
  FBI_ASSERT(parser);
  FBI_ASSERT(buf);
  dbg_fentry("parser=%p nbuf=%lu", parser, nbuf);

  int success = 0;

  // ragel variables
  int cs = ascii_error; // current state
  char* p = buf;
  char* pe = buf + nbuf;
  char* eof = NULL;
  %% write init nocs;

  char* ts = NULL; // token start
  size_t tlen = 0; // token length

  int nmsgs = 0;

  /* each iteration of this loop processes one msg, or in the case of value
     msgs the header of the msg, the body of the msg, or the tail of
     the msg. If we reach the end of buf before the end of a msg (whether
     in data or header/tail), we have a partial read and will resume when the
     next packet arrives. */
  while (p < pe) {
    /* sanity check: if memcached has a bug, or we somehow lost a msg, log
       the whatever's left and disconnect */
    if (parser->parser_state == parser_idle) {
      dbg_error("Unexpected msg '%s'", p);
      parser->parse_error(parser->context, parser->error);
      break;
    }

    FBI_ASSERT(parser->parser_state == parser_msg_header ||
           parser->parser_state == parser_body ||
           parser->parser_state == parser_partial);

    /* prepare to resume. If we're not in the middle of a msg, these will
       not be used */
    cs = parser->ragel_state;

    if (parser->parser_state == parser_msg_header) {
      if (parser->msg != NULL) {
        dbg_error("parser->msg already exists at start of new message. "
                  "Probable memory leak.");
      }

      // new msg
      parser->msg = mc_msg_new(0);
      if (parser->msg == NULL) {
        parser->error = parser_out_of_memory;
        parser->resid = parser->off = 0;
        parser->parse_error(parser->context, parser->error);
        success = 0;
        goto epilogue;
      }

      cs = parser->ragel_start_state;
      parser->parser_state = parser_partial;
    }
    FBI_ASSERT(parser->msg);

    dbg_low("resid=%lu msg=%p parser_state=%s cs=%d",
            pe - p, parser->msg,
            parser_state_to_string(parser->parser_state), cs);

    if (parser->parser_state == parser_body) {
      FBI_ASSERT(pe >= p);
      size_t n = MIN(parser->resid, (size_t)(pe - p));
      memcpy(&parser->msg->value.str[parser->off], p, n);
      p += n;
      parser->off += n;
      FBI_ASSERT(parser->resid >= n);
      parser->resid -= n;
      if (parser->resid == 0) {
        FBI_ASSERT(parser->off == parser->msg->value.len);
        parser->parser_state = parser_partial;
      }
    } else {
      if (parser->partial_token) {
        // we're in the middle of a token
        ts = p;
      } else {
        ts = NULL;
      }

      /* this is where ragel emits the state machine code */
      %% write exec;

      /* if msg == NULL, we finished the msg */
      if (parser->msg) {
        parser->ragel_state = cs;
      } else {
        /* sentinel values */
        parser->ragel_state = ascii_error;
      }

      if (ts && parser->partial_token) {
        // we're in the middle of a token, stuff it in our cheeks
        int n = p - ts;
        if(mc_parser_ensure_tbuf(parser, n) == 0) {
          strncpy(parser->te, ts, n);
          parser->te += n;
        } else {
          if (parser->in_key) {
            parser->bad_key = 1;
          } else {
            // fake fgoto error here
            parser->error = parser_malformed_request;
            cs = ascii_error;
          }
        }
      }

      if (cs == ascii_error) {
        if (parser->msg != NULL) {
          mc_msg_decref(parser->msg);
          parser->msg = NULL;
        }
        parser->resid = parser->off = 0;
        parser->parse_error(parser->context, parser->error);
        goto epilogue;
      }
    }

    /* We may have adjusted pe and eof to simulate EOF at the end of a msg.
       Restore them for continued parsing of additional msgs. */
    pe = buf + nbuf;
    eof = NULL;
  }

  success = 1;

epilogue:
  dbg_fexit("%d msgs finished. parser_state=%s cs=%d",
            nmsgs, parser_state_to_string(parser->parser_state), cs);
  return success;
}

%%{
# vim: ft=ragel
}%%

/* -*- mode: c; -*- */
/* ascii_client.c is built from ascii_client.rl using ragel */

#include <ctype.h>

#include "mcrouter/lib/fbi/debug.h"
#include "mcrouter/lib/mc/_protocol.h"
#include "mcrouter/lib/mc/parser.h"
#include "mcrouter/lib/mc/msg.h"
#include "mcrouter/lib/mc/util.h"

// In libmc.client, only read the first N stats for simplicity of
// implementation. In the future it might make sense to allow an
// arbitrary number of stats, but for now that makes the code far
// easier
#define MAX_NUM_STATS 20

// Are we in the middle of a key being skipped?
#define skipping_key(parser) ((parser)->in_skipped_key &&       \
                              !(parser)->record_skip_key)

%%{

# input file for ragel (http://www.complang.org/ragel/)

# this machine name, which must come first, is used as a prefix for variables
machine ascii;

# named actions. All the actions (named and unnamed) are executed in the
# context of _on_ascii_rx() where "%% write exec;" is called.
action start_token {
  if (!skipping_key(parser)) {
    parser->partial_token = 1;
    parser->te = parser->tbuf;
    ts = p;
  }
}
action finish_token {
  if (parser->partial_token && !skipping_key(parser)) {
    FBI_ASSERT(ts);
    parser->partial_token = 0;
    if (parser->te && parser->te > parser->tbuf) {
      /* we have something stuffed in our cheeks,
         let's put them together and update ts. */
      FBI_ASSERT(ts <= p);
      size_t n = p - ts;
      if(mc_parser_ensure_tbuf(parser, n) != 0) {
        if (parser->in_key) {
          parser->bad_key = 1;
        } else {
          parser->error = parser_malformed_request;
          fgoto error_handling;
        }
      } else {
        strncpy(parser->te, ts, n);
        parser->te += n;
        *parser->te = '\0';
        tlen = parser->te - parser->tbuf;
        ts = parser->tbuf;
      }
    } else {
      tlen = p - ts;
      if (tlen >= MC_TOKEN_MAX_LEN) {
        if (parser->in_key) {
          parser->bad_key = 1;
        } else {
          parser->error = parser_malformed_request;
          fgoto error_handling;
        }
      }
    }
  }
}

action record_key {
  if (!skipping_key(parser)) {
    if (tlen > MC_KEY_MAX_LEN_ASCII) {
      parser->bad_key = 1;
    }
    if (parser->bad_key) {
      parser->msg->result = mc_res_bad_key;
    } else {
      if (mc_msg_grow(&parser->msg, tlen + 1,
                      (void**) &parser->msg->key.str) != 0) {
        parser->error = parser_out_of_memory;
        fgoto error_handling;
      }
      parser->msg->key.len = tlen;
      strncpy(parser->msg->key.str, ts, tlen);
      FBI_ASSERT(!memcmp(parser->msg->key.str, ts, tlen));
      parser->msg->key.str[tlen] = '\0';
    }
  }
  parser->in_skipped_key = 0;
  parser->in_key = 0;
  parser->bad_key = 0;
}

action fin {
  FBI_ASSERT(parser->msg != NULL);

  // end of msg, fake eof
  pe = eof = p + 1;
  parser->parser_state = parser_msg_header;
  /* Note: we need to reset parser->msg first,
     in case mc_parser_reset is called from the callback */
  mc_msg_t* msg = parser->msg;
  parser->msg = NULL;
  parser->msg_ready(parser->context, 0, msg);
  nmsgs++;

  mc_parser_cleanup_tbuf(parser);
}

nl = '\r'? '\n';
fin = nl @fin;

# server-reported errors. We should only ever see SERVER_ERROR. If we see
# CLIENT_ERROR then libmc probably didn't do enough error checking before
# sending to memcache in the first place. ERROR is absolutely a bug in libmc.
error_handling := '' >{
  if (parser->msg != NULL) {
    mc_msg_decref(parser->msg);
  }
  parser->msg = NULL;
  parser->te = NULL;
  parser->in_key = 0;
  parser->bad_key = 0;
  fbreak;
};

error_code = ' ' digit+ ' '
  @{
    parser->msg->err_code = strtoll(ts, NULL, 10);
  };

action capture_error_value {
  // Since the token we are parsing includes the \r\n, we need
  // to invoke this code upon entering the DFA's final state.
  // Only the '\r' is stored in the token buffer, so we only
  // need to remove one character.
  tlen--;
  if (mc_msg_grow(&parser->msg, tlen + 1,
                  (void**) &parser->msg->value.str) != 0) {
    parser->error = parser_out_of_memory;
    fgoto error_handling;
  }
  parser->msg->value.len = tlen;
  memcpy(parser->msg->value.str, ts, tlen);
  parser->msg->value.str[tlen] = '\0';
}

# server error apparently needs to be able to read binary data to support
# TAO, which uses SERVER_ERROR to send the fbid of objects back to the
# client during set requests.  There is no size field, so we need to
# keep parsing the response string until we hit \r\n.  Server error
# 307 is returned by a server that is overloaded or is shutting down;
# it is treated specially so that it can cross the application/transport
# boundary.  (mc_res_busy is an artifact of our use of TCP/IP, of our
# desired to keep the connection open while not accepting requests.
# If there was a reliable request-based protocol it would be less
# necessary.)
server_error = 'SERVER_ERROR' error_code? ' '? (/.*\r\n/) >start_token
  @finish_token
  @capture_error_value
  @{
    if (parser->msg->err_code == SERVER_ERROR_BUSY) {
      parser->msg->result = mc_res_busy;
    } else {
      parser->msg->result = mc_res_remote_error;
    }
  }
  @fin;

# client error message may contain useful stuff, so we also want to save
# it for the caller
client_error = 'CLIENT_ERROR' error_code? ' '? (/.*\r\n/) >start_token
  @finish_token
  @capture_error_value
  @{
    parser->msg->result = mc_res_client_error;
  }
  @fin;

command_error = 'ERROR'
  @{
    parser->msg->result = mc_res_local_error;
    parser->msg->value.str = "unexpected ERROR reply";
    parser->msg->value.len = strlen(parser->msg->value.str);
    dbg_error("%s", parser->msg->value.str);
    FBI_ASSERT(0);
  } fin;

error = server_error | client_error | command_error;

# Get request is ready to be passed to the caller
action get_req_is_ready {
  FBI_ASSERT(parser->msg != NULL);

  mc_op_t op = parser->msg->op;
  /* Note: we need to reset parser->msg first,
     in case mc_parser_reset is called from the callback */
  mc_msg_t* msg = parser->msg;
  parser->msg = mc_msg_new(0);
  parser->msg_ready(parser->context, 0, msg);
  if (parser->msg == NULL) {
    parser->error = parser_out_of_memory;
    fgoto error_handling;
  }

  parser->msg->op = op;
}

# building blocks
stored = 'STORED' @{ parser->msg->result = mc_res_stored; };
stale_stored = 'STALE_STORED' @{
  parser->msg->result = mc_res_stalestored;
} fin;
not_stored = 'NOT_STORED' @{ parser->msg->result = mc_res_notstored; };
exists = 'EXISTS' @{ parser->msg->result = mc_res_exists; };
not_found = 'NOT_FOUND' @{ parser->msg->result = mc_res_notfound; };
deleted = 'DELETED' @{ parser->msg->result = mc_res_deleted; };

storage_reply = stored | stale_stored | not_stored | exists | not_found | deleted;
storage = storage_reply fin;
arithmetic_reply = (not_found |
  digit+ >start_token %finish_token %{
    parser->msg->delta = strtoll(ts, NULL, 10);
    parser->msg->result = mc_res_stored;
  } ' '* fin);

key_skip = (any+ -- (cntrl | space))
  >{ parser->in_skipped_key = 1; parser->in_key = 1; }
  >start_token %finish_token
  %record_key;

key = (any+ -- (cntrl | space))
  >{ parser->in_skipped_key = 0; parser->in_key = 1; }
  >start_token %finish_token
  %record_key;

# This is used for stats and exec commands which can have keys with spaces.
multi_token = ' '* (print+ -- ( '\r' | '\n' ))
  >start_token %finish_token %{
  while (tlen > 0 && isspace(ts[tlen-1])) {
    tlen--;
  }
  if (mc_msg_grow(&parser->msg, tlen+1, (void**) &parser->msg->key.str) != 0) {
    parser->error = parser_out_of_memory;
    fgoto error_handling;
  }
  parser->msg->key.len = tlen;
  strncpy(parser->msg->key.str, ts, tlen);
  parser->msg->key.str[tlen] = '\0';
};

lease_token = ('-'? digit+) >start_token %finish_token %{
  parser->msg->lease_id = strtol(ts, NULL, 10);
};
casid = digit+ >start_token %finish_token %{
  parser->msg->cas = strtoll(ts, NULL, 10);
};
flags = digit+ >start_token %finish_token %{
  parser->msg->flags = strtoll(ts, NULL, 10);
};
exptime = digit+ >start_token %finish_token %{
  parser->msg->exptime = strtol(ts, NULL, 10);
};
delta = digit+ >start_token %finish_token %{
  parser->msg->delta = strtoll(ts, NULL, 10);
};
number = digit+ >start_token %finish_token %{
  parser->msg->number = strtoll(ts, NULL, 10);
};
noreply = 'noreply' %{
  parser->msg->noreply = true;
};

value_bytes = digit+ >start_token %finish_token %{
  /* prepare for receiving data */
  size_t len = parser->resid = strtol(ts, NULL, 10);
  parser->off = 0;

  if (mc_msg_grow(&parser->msg, len + 1,
                  (void**) &parser->msg->value.str) != 0) {
    parser->error = parser_out_of_memory;
    fgoto error_handling;
  }
  parser->msg->value.str[len] = '\0';
  parser->msg->value.len = len;
};
cas_unique = digit+ >start_token %finish_token %{
  parser->msg->cas = strtoull(ts, NULL, 10);
};
dec_octet = digit{1,3};
ipv4_addr = dec_octet "." dec_octet "." dec_octet "." dec_octet;
ipv6_addr = (":" | xdigit)+ ipv4_addr?;
ip_addr = (ipv4_addr | ipv6_addr) >start_token %finish_token %{
  ts[tlen]='\0';
  parser->msg->ipv = 0;
  if (strchr(ts, ':') == NULL) {
    if (inet_pton(AF_INET, ts, &parser->msg->ip_addr) > 0)
      parser->msg->ipv = 4;
  } else {
    if (inet_pton(AF_INET6, ts, &parser->msg->ip_addr) > 0)
      parser->msg->ipv = 6;
  }
};
age_unknown = 'unknown' %{ parser->msg->number = -1;};
action rx_data {
  /* the binary data of a specified isn't really regular, so we drop out of
     the state machine and read it, then come back in to finish the
     "\r\nEND\r\n". */
  parser->parser_state = parser_body;
  fbreak;
}

# replies

VALUE = 'VALUE' % { parser->msg->result = mc_res_found; };

hit = VALUE ' '+ key_skip ' '+ flags ' '+ value_bytes (' '+ cas_unique)? nl
  @rx_data nl;

get_reply = (hit? >{ parser->msg->result = mc_res_notfound; } 'END' fin);
lease_get = ((hit |
  ('LVALUE' ' '+ key_skip ' '+ lease_token ' '+ flags ' '+ value_bytes (' '+ cas_unique)? nl
   @rx_data nl
   @{ parser->msg->result = mc_res_notfound; }))
  'END' fin);
META = 'META' % {parser->msg->result = mc_res_found;};
mhit = META ' '+ key_skip ' '+ 'age:' ' '* (number | age_unknown) ';' ' '*
  'exptime:' ' '* exptime ';' ' '* 'from:' ' '* (ip_addr|'unknown') ';' ' '*
  'is_transient:' ' '* flags ' '* nl;
metaget_reply = (mhit? >{ parser->msg->result = mc_res_notfound; } 'END' fin);
reply_set = storage;
reply_add = storage;
reply_replace = storage;
reply_lease_set = storage;
reply_append = storage;
reply_prepend = storage;
reply_cas = storage;
delete_reply = deleted | not_found;

stat_name = (any+ -- (cntrl | space)) >start_token %finish_token
%{
  if (parser->msg->stats == NULL) {
    FBI_ASSERT(parser->msg->number == 0);
    size_t stats_size = sizeof(nstring_t) * 2 * MAX_NUM_STATS;
    if (mc_msg_grow(&parser->msg, stats_size, (void**) &parser->msg->stats)) {
      parser->error = parser_out_of_memory;
      fgoto error_handling;
    }
    memset(parser->msg->stats, 0, stats_size);
  }

  if (parser->msg->number < MAX_NUM_STATS) {
    parser->msg->number++;

    int idx = (parser->msg->number * 2) - 2;
    if (mc_msg_grow(&parser->msg, tlen + 1,
                    (void**) &parser->msg->stats[idx].str) != 0) {
      parser->error = parser_out_of_memory;
      fgoto error_handling;
    }
    parser->msg->stats[idx].len = tlen;
    parser->msg->stats[idx].str[tlen] = '\0';
    strncpy(parser->msg->stats[idx].str, ts, tlen);
  }
};

stat_value = (any++ -- cntrl) > start_token %finish_token
%{
  if (parser->msg->number <= MAX_NUM_STATS) {
    int idx = (parser->msg->number * 2) - 1;
    if (mc_msg_grow(&parser->msg, tlen + 1,
                    (void**) &parser->msg->stats[idx].str) != 0) {
      parser->error = parser_out_of_memory;
      fgoto error_handling;
    }

    parser->msg->stats[idx].len = tlen;
    parser->msg->stats[idx].str[tlen] = '\0';
    strncpy(parser->msg->stats[idx].str, ts, tlen);
  }
};

stats_reply = (('STAT ' @{ parser->msg->result = mc_res_ok; }
                stat_name ' '+ stat_value nl)* 'END' fin);
ok = ('OK' @{ parser->msg->result = mc_res_ok; } fin);
version_reply = ('VERSION ' (any* -- ('\r' | '\n')) >start_token %finish_token
  %{
    // use new_msg because msg stays live if realloc fails and can leak
    mc_msg_t *new_msg = mc_msg_realloc(parser->msg,
                                       parser->msg->_extra_size + tlen + 1);
    if (new_msg == NULL) {
      parser->error = parser_out_of_memory;
      fgoto error_handling;
    }
    parser->msg = new_msg;
    parser->msg->result = mc_res_ok;
    char* version = parser->msg->value.str = (char *) &parser->msg[1];
    parser->msg->value.len = tlen;
    strncpy(version, ts, tlen);
    version[tlen] = '\0';
  }
  fin);

reply := get_reply | lease_get | metaget_reply |
        reply_set | reply_add | reply_replace | reply_lease_set |
        reply_append | reply_prepend | reply_cas | delete_reply |
        arithmetic_reply |
        stats_reply | ok | version_reply | error;

# requests

set = 'set' @ { parser->msg->op = mc_op_set; };
add = 'add' @ { parser->msg->op = mc_op_add; };
replace = 'replace' @ { parser->msg->op = mc_op_replace; };
leaseset = 'lease-set' @ { parser->msg->op = mc_op_lease_set; };
append = 'append' @ { parser->msg->op = mc_op_append; };
prepend = 'prepend' @ { parser->msg->op = mc_op_prepend; };

set_op = set | add | replace | append | prepend;

delete_op = 'delete' @ { parser->msg->op = mc_op_delete; };

get_op = 'get' @ { parser->msg->op = mc_op_get; } |
         'gets' @ {parser->msg->op = mc_op_gets; } |
         'lease-get' @ { parser->msg->op = mc_op_lease_get; } |
         'metaget' @ { parser->msg->op = mc_op_metaget; } ;

arithmetic_op = 'incr' @ { parser->msg->op = mc_op_incr; } |
                'decr' @ { parser->msg->op = mc_op_decr; };

version_op = 'version' @ { parser->msg->op = mc_op_version; };

quit_op = 'quit' @ { parser->msg->op = mc_op_quit;  };

stats_op = 'stats' @ { parser->msg->op = mc_op_stats; };

exec_op = ('exec' | 'admin') @ { parser->msg->op = mc_op_exec; };

shutdown_op = 'shutdown' @ { parser->msg->op = mc_op_shutdown; };

flush_all_op = 'flush_all' @ { parser->msg->op = mc_op_flushall; };
flush_regex_op = 'flush_regex' @ { parser->msg->op = mc_op_flushre; };

cas = 'cas' %{
  parser->msg->op = mc_op_cas;
} ' '+ key ' '+ flags ' '+ exptime ' '+ value_bytes ' '+ casid ' '* nl
  @rx_data fin;

store = (set_op ' '+ key | leaseset ' '+ key ' '+ lease_token) ' '+ flags ' '+ exptime ' '+ value_bytes (' '+ noreply)? ' '* nl
  @rx_data fin;

get = get_op (' '+ key %get_req_is_ready)+ ' '* fin >{
  parser->msg->op = mc_op_end;
};

arithmetic = arithmetic_op ' '+ key ' '+ delta (' '+ noreply)? ' '* fin;

delete = delete_op ' '+ key (' '+ exptime)? (' '+ noreply)? ' '* fin;

quit = quit_op ' '* fin;

version = version_op ' '* fin;

stats = stats_op (' '+ multi_token)? ' '* fin;

exec = exec_op ' '+ multi_token ' '* fin;

shutdown = shutdown_op (' '+ number)? ' '* fin;

flush_all = flush_all_op (' '+ number)? ' '* fin;

flush_regex = flush_regex_op ' ' key ' '* fin;

request := get | store | cas | delete | arithmetic
  | quit | version | stats | shutdown | flush_all | flush_regex | exec;

request_reply := get | store | cas | delete | arithmetic
  | quit | version | stats | shutdown | flush_all | flush_regex | exec |
  get_reply | lease_get | metaget_reply |
  reply_set | reply_add | reply_replace | reply_lease_set |
  reply_append | reply_prepend | reply_cas | delete_reply |
  arithmetic_reply |
  stats_reply | ok | version_reply | error;

write data;
}%%

int get_reply_ragel_start_state() {
  return ascii_en_reply;
}

int get_request_ragel_start_state() {
  return ascii_en_request;
}

int get_request_reply_ragel_start_state() {
  return ascii_en_request_reply;
}

/** ascii reply parsing
    This is the real workhorse. It consumes buf, resuming the partial reply if
    one exists. As each reply is fully received, we call mcc_req_complete().
    If there is a parse error or remote error, we handle that as well. If
    there is a parse error or unrecoverable remote error, we call
    mcc_on_down() and the rest of the buffer is ignored.
    @return 0 on error */
int _on_ascii_rx(mc_parser_t* parser, char* buf, size_t nbuf) {
  FBI_ASSERT(parser);
  FBI_ASSERT(buf);
  dbg_fentry("parser=%p nbuf=%lu", parser, nbuf);

  int success = 0;

  // ragel variables
  int cs = ascii_error; // current state
  char* p = buf;
  char* pe = buf + nbuf;
  char* eof = NULL;
  %% write init nocs;

  char* ts = NULL; // token start
  size_t tlen = 0; // token length

  int nmsgs = 0;

  /* each iteration of this loop processes one msg, or in the case of value
     msgs the header of the msg, the body of the msg, or the tail of
     the msg. If we reach the end of buf before the end of a msg (whether
     in data or header/tail), we have a partial read and will resume when the
     next packet arrives. */
  while (p < pe) {
    /* sanity check: if memcached has a bug, or we somehow lost a msg, log
       the whatever's left and disconnect */
    if (parser->parser_state == parser_idle) {
      dbg_error("Unexpected msg '%s'", p);
      parser->parse_error(parser->context, parser->error);
      break;
    }

    FBI_ASSERT(parser->parser_state == parser_msg_header ||
           parser->parser_state == parser_body ||
           parser->parser_state == parser_partial);

    /* prepare to resume. If we're not in the middle of a msg, these will
       not be used */
    cs = parser->ragel_state;

    if (parser->parser_state == parser_msg_header) {
      if (parser->msg != NULL) {
        dbg_error("parser->msg already exists at start of new message. "
                  "Probable memory leak.");
      }

      // new msg
      parser->msg = mc_msg_new(0);
      if (parser->msg == NULL) {
        parser->error = parser_out_of_memory;
        parser->resid = parser->off = 0;
        parser->parse_error(parser->context, parser->error);
        success = 0;
        goto epilogue;
      }

      cs = parser->ragel_start_state;
      parser->parser_state = parser_partial;
    }
    FBI_ASSERT(parser->msg);

    dbg_low("resid=%lu msg=%p parser_state=%s cs=%d",
            pe - p, parser->msg,
            parser_state_to_string(parser->parser_state), cs);

    if (parser->parser_state == parser_body) {
      FBI_ASSERT(pe >= p);
      size_t n = MIN(parser->resid, (size_t)(pe - p));
      memcpy(&parser->msg->value.str[parser->off], p, n);
      p += n;
      parser->off += n;
      FBI_ASSERT(parser->resid >= n);
      parser->resid -= n;
      if (parser->resid == 0) {
        FBI_ASSERT(parser->off == parser->msg->value.len);
        parser->parser_state = parser_partial;
      }
    } else {
      if (parser->partial_token) {
        // we're in the middle of a token
        ts = p;
      } else {
        ts = NULL;
      }

      /* this is where ragel emits the state machine code */
      %% write exec;

      /* if msg == NULL, we finished the msg */
      if (parser->msg) {
        parser->ragel_state = cs;
      } else {
        /* sentinel values */
        parser->ragel_state = ascii_error;
      }

      if (ts && parser->partial_token) {
        // we're in the middle of a token, stuff it in our cheeks
        int n = p - ts;
        if(mc_parser_ensure_tbuf(parser, n) == 0) {
          strncpy(parser->te, ts, n);
          parser->te += n;
        } else {
          if (parser->in_key) {
            parser->bad_key = 1;
          } else {
            // fake fgoto error here
            parser->error = parser_malformed_request;
            cs = ascii_error;
          }
        }
      }

      if (cs == ascii_error) {
        if (parser->msg != NULL) {
          mc_msg_decref(parser->msg);
          parser->msg = NULL;
        }
        parser->resid = parser->off = 0;
        parser->parse_error(parser->context, parser->error);
        goto epilogue;
      }
    }

    /* We may have adjusted pe and eof to simulate EOF at the end of a msg.
       Restore them for continued parsing of additional msgs. */
    pe = buf + nbuf;
    eof = NULL;
  }

  success = 1;

epilogue:
  dbg_fexit("%d msgs finished. parser_state=%s cs=%d",
            nmsgs, parser_state_to_string(parser->parser_state), cs);
  return success;
}

%%{
# vim: ft=ragel
}%%