LWS* before credentials.

LWS* before credentials.

// Copyright (c) 2009 - Decho Corp.

#include "mordor/common/http/parser.h"

#include <string>

#include "mordor/common/version.h"

#ifdef ABSOLUTE
#undef ABSOLUTE
#endif
#ifdef RELATIVE
#undef RELATIVE
#endif

#ifdef WINDOWS
#define strtoull _strtoui64
#endif

// From uri.rl
std::string unescape(const std::string& str);

static
HTTP::Version
parseVersion(const char *str)
{
    HTTP::Version ver;
    ver.major = atoi(str + 5);
    ver.minor = atoi(strchr(str + 5, '.') + 1);
    return ver;
}

static
HTTP::Method
parseMethod(const char *str, const char *end)
{
    for(size_t i = 0; i < 8; ++i) {
        if (strnicmp(str, HTTP::methods[i], end - str) == 0) {
            return (HTTP::Method)i;
        }
    }
    throw std::invalid_argument("Unrecognized method");
}

static
std::string
unfold(char *p, char *pe)
{
    char *start = p;
    char *pw = p;

    while (p < pe) {
        // Skip leading whitespace
        if (pw == start) {
            if (*p == ' ' || *p == '\t' || *p == '\r' || *p == '\n') {
                ++p; ++pw; ++start;
                continue;
            }
        }
        // Only copy if necessary
        if (pw != p) {
            *pw = *p;
        }
        ++p; ++pw;
    }
    // Remove trailing whitespace
    do {
        --pw;
    } while ((*pw == ' ' || *pw == '\t' || *pw == '\r' || *pw == '\n') && pw >= start);
    ++pw;
    return std::string(start, pw - start);
}

static
std::string
unquote(char *p, char *pe)
{
    if (pe == p || *p != '"')
        return std::string(p, pe - p);
    char *start = p;
    char *pw = p;

    assert(*p == '"');
    assert(*(pe - 1) == '"');
    ++p; ++pw; ++start;
    --pe;
    bool escaping = false;
    while (p < pe) {
        if (escaping) {
            escaping = false;
        } else if (*p == '\\') {
            escaping = true;
            ++p;
            continue;
        }
        // Only copy if necessary
        if (pw != p) {
            *pw = *p;
        }
        ++p; ++pw;
    }
    return std::string(start, pw - start);
}

%%{
    machine http_parser;

    # See RFC 2616: http://www.w3.org/Protocols/rfc2616/rfc2616.html

    action mark { mark = fpc; }
    action done { fbreak; }

    # basic character types
    OCTET = any;
    CHAR = ascii;
    UPALPHA = "A".."Z";
    LOALPHA = "a".."z";
    ALPHA = alpha;
    DIGIT = digit;
    CTL = cntrl | 127;
    CR = "\r";
    LF = "\n";
    SP = " ";
    HT = "\t";

    # almost-basic character types
    # note that we allow a single LF for a CR LF
    CRLF = CR LF;
    LWS = CRLF? ( SP | HT )+;
    TEXT = LWS | (OCTET -- CTL);
    HEX = xdigit;

    # some basic tokens
    separators = "(" | ")" | "<" | ">" | "@" | "," | ";" | ":" | "\\" | "\"" | "/" | "[" | "]" | "?" | "=" | "{" | "}" | SP | HT;
    token = (CHAR -- (separators | CTL))+;
    quoted_pair = "\\" CHAR;
    ctext = TEXT -- ("(" | ")");
    base_comment = "(" ( ctext | quoted_pair )* ")";
    comment = "(" ( ctext | quoted_pair | base_comment )* ")";
    qdtext = TEXT -- "\"";
    quoted_string = "\"" ( qdtext | quoted_pair )* "\"";

    action parse_HTTP_Version {
        *m_ver = parseVersion(mark);
        mark = NULL;
    }

    HTTP_Version = ("HTTP/" DIGIT+ "." DIGIT+) >mark %parse_HTTP_Version;

    delta_seconds = DIGIT+;

    product_version = token;
    product = token ("/" product_version)?;

    qvalue = ('0' ('.' DIGIT{0,3})?) | ('1' ('.' '0'{0,3})?);

    subtag = ALPHA{1,8};
    primary_tag = ALPHA{1,8};
    language_tag = primary_tag ("-" subtag);

    weak = "W/";
    opaque_tag = quoted_string;
    entity_tag = (weak)? opaque_tag;

    bytes_unit = "bytes";
    other_range_unit = token;
    range_unit = bytes_unit | other_range_unit;

    action save_field_name {
        m_temp1 = std::string(mark, fpc - mark);
        mark = NULL;
    }
    action save_field_value {
        std::string fieldValue = unfold((char*)mark, (char*)fpc);

        StringMap::iterator it = m_entity->extension.find(m_temp1);
        if (it == m_entity->extension.end()) {
            m_entity->extension[m_temp1] = fieldValue;
        } else {
            it->second.append(", ");
            it->second.append(fieldValue);
        }
        mark = NULL;
    }

    field_chars = OCTET -- (CTL | CR LF SP HT);
    field_name = token >mark %save_field_name;
    field_value = TEXT* >mark %save_field_value;
    message_header = field_name ":" field_value;

    action save_string {
        *m_string = std::string(mark, fpc - mark);
        mark = NULL;
    }
    action save_ulong {
        *m_ulong = strtoull(mark, NULL, 10);
        mark = NULL;
    }

    action save_element {
        assert(m_list || m_set);
        if (m_list)
            m_list->push_back(std::string(mark, fpc - mark));
        else
            m_set->insert(std::string(mark, fpc - mark));
        mark = NULL;
    }
    element = token >mark %save_element;
    list = (LWS* element ( LWS* ',' LWS* element)* LWS*);

    action save_parameterized_list_element {
        ValueWithParameters vp;
        vp.value = std::string(mark, fpc - mark);
        m_parameterizedList->push_back(vp);
        m_parameters = &m_parameterizedList->back().parameters;
        mark = NULL;
    }

    action save_parameter_attribute {
        m_temp1 = std::string(mark, fpc - mark);
        mark = NULL;
    }

    action save_parameter_value {
        (*m_parameters)[m_temp1] = unquote((char*)mark, (char*)fpc);
        mark = NULL;
    }

    attribute = (token - 'q'i) >mark %save_parameter_attribute; # q separates params from accept-params
    value = (token | quoted_string) >mark %save_parameter_value;
    parameter = attribute '=' value;
    parameterizedListElement = token >mark %save_parameterized_list_element (';' parameter)*;
    parameterizedList = LWS* parameterizedListElement ( LWS* ',' LWS* parameterizedListElement)* LWS*;

    action save_auth_scheme {
        if (m_parameterizedList && ((!m_parameterizedList->empty() && m_auth == &m_parameterizedList->back())
            || m_parameterizedList->empty())) {
            ValueWithParameters vp;
            m_parameterizedList->push_back(vp);
            m_auth = &m_parameterizedList->back();
        }
        m_auth->value = std::string(mark, fpc - mark);
        m_parameters = &m_auth->parameters;
        mark = NULL;
    }

    auth_param = attribute ('=' value)?;
    auth_scheme = token;
    challenge = auth_scheme >mark %save_auth_scheme SP LWS* auth_param ( LWS* ',' LWS* auth_param )* LWS*;
    credentials = auth_scheme >mark %save_auth_scheme (LWS+ auth_param)? (LWS* ',' LWS* auth_param )* LWS*;
    challengeList = LWS* challenge ( LWS* ',' LWS* challenge)* LWS*;

    action set_connection {
        m_set = &m_general->connection;
        m_list = NULL;
    }

    action set_trailer {
        m_set = &m_general->trailer;
        m_list = NULL;
    }

    action set_transfer_encoding {
        m_parameterizedList = &m_general->transferEncoding;
    }

    Connection = 'Connection:'i @set_connection list;
    Trailer = 'Trailer:'i @set_trailer list;
    Transfer_Encoding = 'Transfer-Encoding:'i @set_transfer_encoding parameterizedList;

    general_header = Connection | Trailer | Transfer_Encoding;
    general_header_names = 'Connection'i | 'Trailer'i | 'Transfer-Encoding'i;

    action set_content_encoding {
        m_list = &m_entity->contentEncoding;
        m_set = NULL;
    }

    action set_content_length {
        m_ulong = &m_entity->contentLength;
    }

    action save_cr_first_byte_pos {
        m_entity->contentRange.first = strtoull(mark, NULL, 10);
        mark = NULL;
    }

    action save_cr_last_byte_pos {
        m_entity->contentRange.last = strtoull(mark, NULL, 10);
        mark = NULL;
    }

    action save_blank_cr {
        m_entity->contentRange.last = 0;
    }

    action save_instance_length {
        m_entity->contentRange.instance = strtoull(mark, NULL, 10);
        mark = NULL;
    }

    action set_content_type
    {
        m_parameters = &m_entity->contentType.parameters;
    }
    action save_type
    {
        m_entity->contentType.type = std::string(mark, fpc - mark);
        mark = NULL;
    }
    action save_subtype
    {
        m_entity->contentType.subtype = std::string(mark, fpc - mark);
        mark = NULL;
    }

    Content_Encoding = 'Content-Encoding:'i @set_content_encoding list;
    Content_Length = 'Content-Length:'i @set_content_length LWS* DIGIT+ >mark %save_ulong LWS*;

    byte_range_resp_spec = (DIGIT+ >mark %save_cr_first_byte_pos '-' DIGIT+ >mark %save_cr_last_byte_pos) | '*' %save_blank_cr;
    content_range_spec = bytes_unit SP byte_range_resp_spec '/' ( DIGIT+ >mark %save_instance_length | '*');
    Content_Range = 'Content-Range:'i LWS* content_range_spec LWS*;

    type = token >mark %save_type;
    subtype = token >mark %save_subtype;
    media_type = type'/' subtype (';' LWS* parameter)*;
    Content_Type = 'Content-Type:'i @set_content_type LWS* media_type LWS*;

    entity_header = Content_Encoding | Content_Length | Content_Range | Content_Type; # | extension_header;
    entity_header_names = 'Content-Encoding'i | 'Content-Length'i | 'Content-Range'i | 'Content-Type'i;

}%%

%%{
    machine http_request_parser;
    include http_parser;
    include uri_parser "../uri.rl";

    action parse_Method {
        m_request->requestLine.method = parseMethod(mark, fpc);
        mark = NULL;
    }

    action set_authorization {
        m_parameterizedList = NULL;
        m_auth = &m_request->request.authorization;
    }

    action set_host {
        m_string = &m_request->request.host;
    }

    action set_proxy_authorization {
        m_parameterizedList = NULL;
        m_auth = &m_request->request.proxyAuthorization;
    }

    action save_expectation {
        KeyValueWithParameters kvp;
        kvp.key = std::string(mark, fpc - mark);
        m_request->request.expect.push_back(kvp);
        mark = NULL;
    }
    action save_expectation_value {
        m_request->request.expect.back().value = unquote((char*)mark, (char*)fpc);
        mark = NULL;
    }
    action save_expectation_param {
        m_temp1 = std::string(mark, fpc - mark);
        m_request->request.expect.back().parameters[m_temp1] = "";
        mark = NULL;
    }
    action save_expectation_param_value {
        m_request->request.expect.back().parameters[m_temp1] = unquote((char*)mark, (char*)fpc);
        mark = NULL;
    }

    action save_first_byte_pos {
        m_request->request.range.push_back(RangeSet::value_type(
            strtoull(mark, NULL, 10), ~0ull));
        mark = NULL;
    }

    action save_last_byte_pos {
        if (mark != NULL) {
            m_request->request.range.back().second = strtoull(mark, NULL, 10);
        }
        mark = NULL;
    }

    action save_suffix_byte_pos {
        m_request->request.range.push_back(RangeSet::value_type(
            ~0ull, strtoull(mark, NULL, 10)));
        mark = NULL;
    }

    action save_accept_list_element {
        AcceptValueWithParameters avp;
        avp.value = std::string(mark, fpc - mark);
        m_acceptList->push_back(avp);
        m_parameters = &m_acceptList->back().parameters;
        mark = NULL;
    }

    action save_qvalue {
        m_acceptList->back().qvalue = 0;
        size_t i = 0;
        unsigned int curPlace = 1000;
        for (; i < 5 && mark < fpc; ++i, ++mark) {
            if (i == 1)
                continue;
            unsigned int cur = *mark - '0';
            m_acceptList->back().qvalue += cur * curPlace;
            curPlace /= 10;
        }
        mark = NULL;
    }

    action save_accept_attribute {
        m_temp1 = std::string(mark, fpc - mark);
        m_acceptList->back().acceptParams[m_temp1] = "";
        mark = NULL;
    }

    action save_accept_value {
        m_acceptList->back().acceptParams[m_temp1] = unquote((char*)mark, (char*)fpc);
        mark = NULL;
    }

    action set_te {
        m_acceptList = &m_request->request.te;
    }

    Authorization = 'Authorization:'i @set_authorization LWS* credentials;

    Host = 'Host:'i @set_host LWS* (host (':' port)?) >mark %save_string LWS*;

    expect_params = ';' token >mark %save_expectation_param ( '=' (token | quoted_string) >mark %save_expectation_param_value )?;
    expectation = token >mark %save_expectation ( '=' (token | quoted_string) >mark %save_expectation_value expect_params* )?;
    Expect = 'Expect:'i LWS* expectation ( LWS* ',' LWS* expectation )* LWS*;

    Proxy_Authorization = 'Proxy-Authorization:'i @set_proxy_authorization credentials;

    byte_range_spec = DIGIT+ >mark %save_first_byte_pos '-' (DIGIT+ >mark %save_last_byte_pos)?;
    suffix_byte_range_spec = '-' DIGIT+ > mark %save_suffix_byte_pos;
    byte_range_set = LWS* (byte_range_spec | suffix_byte_range_spec) ( LWS* ',' LWS* (byte_range_spec | suffix_byte_range_spec))* LWS*;
    ranges_specifier = bytes_unit '=' byte_range_set;
    Range = 'Range:'i LWS* ranges_specifier;

    accept_extension = ';' token >mark %save_accept_attribute ('=' (token | quoted_string) >mark %save_accept_value)?;
    accept_params = ';q='i qvalue >mark %save_qvalue (accept_extension)*;
    acceptListElement = token >mark %save_accept_list_element (';' parameter)* (accept_params)?;
    acceptList = LWS* acceptListElement ( LWS* ',' LWS* acceptListElement)* LWS*;
    TE = 'TE:'i @set_te acceptList;

    request_header = Authorization | Host | Expect | Proxy_Authorization | Range | TE;
    request_header_names = 'Authorization'i | 'Host'i | 'Expect'i | 'Proxy-Authorization'i | 'Range'i | 'TE'i;

    extension_header = (token - (general_header_names | request_header_names | entity_header_names)) >mark %save_field_name
        ':' field_value;

    Method = token >mark %parse_Method;
    # we explicitly add query to hier_part, because the URI spec changed from RFC 2396 to RFC 3986
    # with the query not being part of hier_part
    Request_URI = ( "*" | absolute_URI | (hier_part ( "?" query )?) | authority);
    Request_Line = Method SP Request_URI SP HTTP_Version CRLF;
    Request = Request_Line ((general_header | request_header | entity_header | extension_header) CRLF)* CRLF @done;

    main := Request;
    write data;
}%%

void
HTTP::HTTPParser::init()
{
    m_string = NULL;
    m_set = NULL;
    m_list = NULL;
    m_parameterizedList = NULL;
    m_parameters = NULL;
    m_auth = NULL;
    m_ulong = NULL;
    RagelParser::init();
}

HTTP::RequestParser::RequestParser(Request& request)
: m_request(&request),
  m_ver(&request.requestLine.ver),
  m_uri(&request.requestLine.uri),
  m_path(&request.requestLine.uri.path),
  m_general(&request.general),
  m_entity(&request.entity)
{}

void
HTTP::RequestParser::init()
{
    HTTPParser::init();
    %% write init;
}

bool
HTTP::RequestParser::complete() const
{
    return cs >= http_request_parser_first_final;
}

bool
HTTP::RequestParser::error() const
{
    return cs == http_request_parser_error;
}

void
HTTP::RequestParser::exec()
{
#ifdef MSVC
#pragma warning(push)
#pragma warning(disable : 4244)
#endif
    %% write exec;
#ifdef MSVC
#pragma warning(pop)
#endif
}

%%{
    machine http_response_parser;
    include http_parser;
    include uri_parser "../uri.rl";

    action parse_Status_Code {
        m_response->status.status = (HTTP::Status)atoi(mark);
        mark = NULL;
    }

    action parse_Reason_Phrase {
        m_response->status.reason = std::string(mark, fpc - mark);
        mark = NULL;
    }

    action set_accept_ranges
    {
        m_set = &m_response->response.acceptRanges;
        m_list = NULL;
    }
    action set_proxy_authenticate {
        m_parameterizedList = &m_response->response.proxyAuthenticate;
    }
    action set_www_authenticate {
        m_parameterizedList = &m_response->response.wwwAuthenticate;
    }

    Accept_Ranges = 'Accept-Ranges:'i @set_accept_ranges list;
    Location = 'Location:'i LWS* absolute_URI LWS*;
    Proxy_Authenticate = 'Proxy-Authenticate:'i @set_proxy_authenticate challengeList;
    WWW_Authenticate = 'WWW-Authenticate:'i @set_www_authenticate challengeList;

    response_header = Accept_Ranges | Location | Proxy_Authenticate | WWW_Authenticate;
    response_header_names = 'Accept-Ranges'i | 'Location'i | 'Proxy-Authenticate'i | 'WWW-Authenticate'i;

    extension_header = (token - (general_header_names | response_header_names | entity_header_names)) >mark %save_field_name
        ':' field_value;

    Status_Code = DIGIT{3} > mark %parse_Status_Code;
    Reason_Phrase = (TEXT -- (CR | LF))* >mark %parse_Reason_Phrase;
    Status_Line = HTTP_Version SP Status_Code SP Reason_Phrase CRLF;
    Response = Status_Line ((general_header | response_header | entity_header | extension_header) CRLF)* CRLF @done;

    main := Response;

    write data;
}%%

HTTP::ResponseParser::ResponseParser(Response& response)
: m_response(&response),
  m_ver(&response.status.ver),
  m_uri(&response.response.location),
  m_path(&response.response.location.path),
  m_general(&response.general),
  m_entity(&response.entity)
{}

void
HTTP::ResponseParser::init()
{
    HTTPParser::init();
    %% write init;
}

bool
HTTP::ResponseParser::complete() const
{
    return cs >= http_response_parser_first_final;
}

bool
HTTP::ResponseParser::error() const
{
    return cs == http_response_parser_error;
}

void
HTTP::ResponseParser::exec()
{
#ifdef MSVC
#pragma warning(push)
#pragma warning(disable : 4244)
#endif
    %% write exec;
#ifdef MSVC
#pragma warning(pop)
#endif
}

%%{
    machine http_trailer_parser;
    include http_parser;

    extension_header = (token - (entity_header_names)) >mark %save_field_name ':' field_value;

    trailer = (entity_header | extension_header CRLF)*;

    main := trailer CRLF @done;

    write data;
}%%

HTTP::TrailerParser::TrailerParser(EntityHeaders& entity)
: m_entity(&entity)
{}

void
HTTP::TrailerParser::init()
{
    HTTPParser::init();
    %% write init;
}

bool
HTTP::TrailerParser::complete() const
{
    return cs >= http_trailer_parser_first_final;
}

bool
HTTP::TrailerParser::error() const
{
    return cs == http_trailer_parser_error;
}

void
HTTP::TrailerParser::exec()
{
#ifdef MSVC
#pragma warning(push)
#pragma warning(disable : 4244)
#endif
    %% write exec;
#ifdef MSVC
#pragma warning(pop)
#endif
}

// Copyright (c) 2009 - Decho Corp.

#include "mordor/common/http/parser.h"

#include <string>

#include "mordor/common/version.h"

#ifdef ABSOLUTE
#undef ABSOLUTE
#endif
#ifdef RELATIVE
#undef RELATIVE
#endif

#ifdef WINDOWS
#define strtoull _strtoui64
#endif

// From uri.rl
std::string unescape(const std::string& str);

static
HTTP::Version
parseVersion(const char *str)
{
    HTTP::Version ver;
    ver.major = atoi(str + 5);
    ver.minor = atoi(strchr(str + 5, '.') + 1);
    return ver;
}

static
HTTP::Method
parseMethod(const char *str, const char *end)
{
    for(size_t i = 0; i < 8; ++i) {
        if (strnicmp(str, HTTP::methods[i], end - str) == 0) {
            return (HTTP::Method)i;
        }
    }
    throw std::invalid_argument("Unrecognized method");
}

static
std::string
unfold(char *p, char *pe)
{
    char *start = p;
    char *pw = p;

    while (p < pe) {
        // Skip leading whitespace
        if (pw == start) {
            if (*p == ' ' || *p == '\t' || *p == '\r' || *p == '\n') {
                ++p; ++pw; ++start;
                continue;
            }
        }
        // Only copy if necessary
        if (pw != p) {
            *pw = *p;
        }
        ++p; ++pw;
    }
    // Remove trailing whitespace
    do {
        --pw;
    } while ((*pw == ' ' || *pw == '\t' || *pw == '\r' || *pw == '\n') && pw >= start);
    ++pw;
    return std::string(start, pw - start);
}

static
std::string
unquote(char *p, char *pe)
{
    if (pe == p || *p != '"')
        return std::string(p, pe - p);
    char *start = p;
    char *pw = p;

    assert(*p == '"');
    assert(*(pe - 1) == '"');
    ++p; ++pw; ++start;
    --pe;
    bool escaping = false;
    while (p < pe) {
        if (escaping) {
            escaping = false;
        } else if (*p == '\\') {
            escaping = true;
            ++p;
            continue;
        }
        // Only copy if necessary
        if (pw != p) {
            *pw = *p;
        }
        ++p; ++pw;
    }
    return std::string(start, pw - start);
}

%%{
    machine http_parser;

    # See RFC 2616: http://www.w3.org/Protocols/rfc2616/rfc2616.html

    action mark { mark = fpc; }
    action done { fbreak; }

    # basic character types
    OCTET = any;
    CHAR = ascii;
    UPALPHA = "A".."Z";
    LOALPHA = "a".."z";
    ALPHA = alpha;
    DIGIT = digit;
    CTL = cntrl | 127;
    CR = "\r";
    LF = "\n";
    SP = " ";
    HT = "\t";

    # almost-basic character types
    # note that we allow a single LF for a CR LF
    CRLF = CR LF;
    LWS = CRLF? ( SP | HT )+;
    TEXT = LWS | (OCTET -- CTL);
    HEX = xdigit;

    # some basic tokens
    separators = "(" | ")" | "<" | ">" | "@" | "," | ";" | ":" | "\\" | "\"" | "/" | "[" | "]" | "?" | "=" | "{" | "}" | SP | HT;
    token = (CHAR -- (separators | CTL))+;
    quoted_pair = "\\" CHAR;
    ctext = TEXT -- ("(" | ")");
    base_comment = "(" ( ctext | quoted_pair )* ")";
    comment = "(" ( ctext | quoted_pair | base_comment )* ")";
    qdtext = TEXT -- "\"";
    quoted_string = "\"" ( qdtext | quoted_pair )* "\"";

    action parse_HTTP_Version {
        *m_ver = parseVersion(mark);
        mark = NULL;
    }

    HTTP_Version = ("HTTP/" DIGIT+ "." DIGIT+) >mark %parse_HTTP_Version;

    delta_seconds = DIGIT+;

    product_version = token;
    product = token ("/" product_version)?;

    qvalue = ('0' ('.' DIGIT{0,3})?) | ('1' ('.' '0'{0,3})?);

    subtag = ALPHA{1,8};
    primary_tag = ALPHA{1,8};
    language_tag = primary_tag ("-" subtag);

    weak = "W/";
    opaque_tag = quoted_string;
    entity_tag = (weak)? opaque_tag;

    bytes_unit = "bytes";
    other_range_unit = token;
    range_unit = bytes_unit | other_range_unit;

    action save_field_name {
        m_temp1 = std::string(mark, fpc - mark);
        mark = NULL;
    }
    action save_field_value {
        std::string fieldValue = unfold((char*)mark, (char*)fpc);

        StringMap::iterator it = m_entity->extension.find(m_temp1);
        if (it == m_entity->extension.end()) {
            m_entity->extension[m_temp1] = fieldValue;
        } else {
            it->second.append(", ");
            it->second.append(fieldValue);
        }
        mark = NULL;
    }

    field_chars = OCTET -- (CTL | CR LF SP HT);
    field_name = token >mark %save_field_name;
    field_value = TEXT* >mark %save_field_value;
    message_header = field_name ":" field_value;

    action save_string {
        *m_string = std::string(mark, fpc - mark);
        mark = NULL;
    }
    action save_ulong {
        *m_ulong = strtoull(mark, NULL, 10);
        mark = NULL;
    }

    action save_element {
        assert(m_list || m_set);
        if (m_list)
            m_list->push_back(std::string(mark, fpc - mark));
        else
            m_set->insert(std::string(mark, fpc - mark));
        mark = NULL;
    }
    element = token >mark %save_element;
    list = (LWS* element ( LWS* ',' LWS* element)* LWS*);

    action save_parameterized_list_element {
        ValueWithParameters vp;
        vp.value = std::string(mark, fpc - mark);
        m_parameterizedList->push_back(vp);
        m_parameters = &m_parameterizedList->back().parameters;
        mark = NULL;
    }

    action save_parameter_attribute {
        m_temp1 = std::string(mark, fpc - mark);
        mark = NULL;
    }

    action save_parameter_value {
        (*m_parameters)[m_temp1] = unquote((char*)mark, (char*)fpc);
        mark = NULL;
    }

    attribute = (token - 'q'i) >mark %save_parameter_attribute; # q separates params from accept-params
    value = (token | quoted_string) >mark %save_parameter_value;
    parameter = attribute '=' value;
    parameterizedListElement = token >mark %save_parameterized_list_element (';' parameter)*;
    parameterizedList = LWS* parameterizedListElement ( LWS* ',' LWS* parameterizedListElement)* LWS*;

    action save_auth_scheme {
        if (m_parameterizedList && ((!m_parameterizedList->empty() && m_auth == &m_parameterizedList->back())
            || m_parameterizedList->empty())) {
            ValueWithParameters vp;
            m_parameterizedList->push_back(vp);
            m_auth = &m_parameterizedList->back();
        }
        m_auth->value = std::string(mark, fpc - mark);
        m_parameters = &m_auth->parameters;
        mark = NULL;
    }

    auth_param = attribute ('=' value)?;
    auth_scheme = token;
    challenge = auth_scheme >mark %save_auth_scheme SP LWS* auth_param ( LWS* ',' LWS* auth_param )* LWS*;
    credentials = auth_scheme >mark %save_auth_scheme (LWS+ auth_param)? (LWS* ',' LWS* auth_param )* LWS*;
    challengeList = LWS* challenge ( LWS* ',' LWS* challenge)* LWS*;

    action set_connection {
        m_set = &m_general->connection;
        m_list = NULL;
    }

    action set_trailer {
        m_set = &m_general->trailer;
        m_list = NULL;
    }

    action set_transfer_encoding {
        m_parameterizedList = &m_general->transferEncoding;
    }

    Connection = 'Connection:'i @set_connection list;
    Trailer = 'Trailer:'i @set_trailer list;
    Transfer_Encoding = 'Transfer-Encoding:'i @set_transfer_encoding parameterizedList;

    general_header = Connection | Trailer | Transfer_Encoding;
    general_header_names = 'Connection'i | 'Trailer'i | 'Transfer-Encoding'i;

    action set_content_encoding {
        m_list = &m_entity->contentEncoding;
        m_set = NULL;
    }

    action set_content_length {
        m_ulong = &m_entity->contentLength;
    }

    action save_cr_first_byte_pos {
        m_entity->contentRange.first = strtoull(mark, NULL, 10);
        mark = NULL;
    }

    action save_cr_last_byte_pos {
        m_entity->contentRange.last = strtoull(mark, NULL, 10);
        mark = NULL;
    }

    action save_blank_cr {
        m_entity->contentRange.last = 0;
    }

    action save_instance_length {
        m_entity->contentRange.instance = strtoull(mark, NULL, 10);
        mark = NULL;
    }

    action set_content_type
    {
        m_parameters = &m_entity->contentType.parameters;
    }
    action save_type
    {
        m_entity->contentType.type = std::string(mark, fpc - mark);
        mark = NULL;
    }
    action save_subtype
    {
        m_entity->contentType.subtype = std::string(mark, fpc - mark);
        mark = NULL;
    }

    Content_Encoding = 'Content-Encoding:'i @set_content_encoding list;
    Content_Length = 'Content-Length:'i @set_content_length LWS* DIGIT+ >mark %save_ulong LWS*;

    byte_range_resp_spec = (DIGIT+ >mark %save_cr_first_byte_pos '-' DIGIT+ >mark %save_cr_last_byte_pos) | '*' %save_blank_cr;
    content_range_spec = bytes_unit SP byte_range_resp_spec '/' ( DIGIT+ >mark %save_instance_length | '*');
    Content_Range = 'Content-Range:'i LWS* content_range_spec LWS*;

    type = token >mark %save_type;
    subtype = token >mark %save_subtype;
    media_type = type'/' subtype (';' LWS* parameter)*;
    Content_Type = 'Content-Type:'i @set_content_type LWS* media_type LWS*;

    entity_header = Content_Encoding | Content_Length | Content_Range | Content_Type; # | extension_header;
    entity_header_names = 'Content-Encoding'i | 'Content-Length'i | 'Content-Range'i | 'Content-Type'i;

}%%

%%{
    machine http_request_parser;
    include http_parser;
    include uri_parser "../uri.rl";

    action parse_Method {
        m_request->requestLine.method = parseMethod(mark, fpc);
        mark = NULL;
    }

    action set_authorization {
        m_parameterizedList = NULL;
        m_auth = &m_request->request.authorization;
    }

    action set_host {
        m_string = &m_request->request.host;
    }

    action set_proxy_authorization {
        m_parameterizedList = NULL;
        m_auth = &m_request->request.proxyAuthorization;
    }

    action save_expectation {
        KeyValueWithParameters kvp;
        kvp.key = std::string(mark, fpc - mark);
        m_request->request.expect.push_back(kvp);
        mark = NULL;
    }
    action save_expectation_value {
        m_request->request.expect.back().value = unquote((char*)mark, (char*)fpc);
        mark = NULL;
    }
    action save_expectation_param {
        m_temp1 = std::string(mark, fpc - mark);
        m_request->request.expect.back().parameters[m_temp1] = "";
        mark = NULL;
    }
    action save_expectation_param_value {
        m_request->request.expect.back().parameters[m_temp1] = unquote((char*)mark, (char*)fpc);
        mark = NULL;
    }

    action save_first_byte_pos {
        m_request->request.range.push_back(RangeSet::value_type(
            strtoull(mark, NULL, 10), ~0ull));
        mark = NULL;
    }

    action save_last_byte_pos {
        if (mark != NULL) {
            m_request->request.range.back().second = strtoull(mark, NULL, 10);
        }
        mark = NULL;
    }

    action save_suffix_byte_pos {
        m_request->request.range.push_back(RangeSet::value_type(
            ~0ull, strtoull(mark, NULL, 10)));
        mark = NULL;
    }

    action save_accept_list_element {
        AcceptValueWithParameters avp;
        avp.value = std::string(mark, fpc - mark);
        m_acceptList->push_back(avp);
        m_parameters = &m_acceptList->back().parameters;
        mark = NULL;
    }

    action save_qvalue {
        m_acceptList->back().qvalue = 0;
        size_t i = 0;
        unsigned int curPlace = 1000;
        for (; i < 5 && mark < fpc; ++i, ++mark) {
            if (i == 1)
                continue;
            unsigned int cur = *mark - '0';
            m_acceptList->back().qvalue += cur * curPlace;
            curPlace /= 10;
        }
        mark = NULL;
    }

    action save_accept_attribute {
        m_temp1 = std::string(mark, fpc - mark);
        m_acceptList->back().acceptParams[m_temp1] = "";
        mark = NULL;
    }

    action save_accept_value {
        m_acceptList->back().acceptParams[m_temp1] = unquote((char*)mark, (char*)fpc);
        mark = NULL;
    }

    action set_te {
        m_acceptList = &m_request->request.te;
    }

    Authorization = 'Authorization:'i @set_authorization credentials;

    Host = 'Host:'i @set_host LWS* (host (':' port)?) >mark %save_string LWS*;

    expect_params = ';' token >mark %save_expectation_param ( '=' (token | quoted_string) >mark %save_expectation_param_value )?;
    expectation = token >mark %save_expectation ( '=' (token | quoted_string) >mark %save_expectation_value expect_params* )?;
    Expect = 'Expect:'i LWS* expectation ( LWS* ',' LWS* expectation )* LWS*;

    Proxy_Authorization = 'Proxy-Authorization:'i @set_proxy_authorization credentials;

    byte_range_spec = DIGIT+ >mark %save_first_byte_pos '-' (DIGIT+ >mark %save_last_byte_pos)?;
    suffix_byte_range_spec = '-' DIGIT+ > mark %save_suffix_byte_pos;
    byte_range_set = LWS* (byte_range_spec | suffix_byte_range_spec) ( LWS* ',' LWS* (byte_range_spec | suffix_byte_range_spec))* LWS*;
    ranges_specifier = bytes_unit '=' byte_range_set;
    Range = 'Range:'i LWS* ranges_specifier;

    accept_extension = ';' token >mark %save_accept_attribute ('=' (token | quoted_string) >mark %save_accept_value)?;
    accept_params = ';q='i qvalue >mark %save_qvalue (accept_extension)*;
    acceptListElement = token >mark %save_accept_list_element (';' parameter)* (accept_params)?;
    acceptList = LWS* acceptListElement ( LWS* ',' LWS* acceptListElement)* LWS*;
    TE = 'TE:'i @set_te acceptList;

    request_header = Authorization | Host | Expect | Proxy_Authorization | Range | TE;
    request_header_names = 'Authorization'i | 'Host'i | 'Expect'i | 'Proxy-Authorization'i | 'Range'i | 'TE'i;

    extension_header = (token - (general_header_names | request_header_names | entity_header_names)) >mark %save_field_name
        ':' field_value;

    Method = token >mark %parse_Method;
    # we explicitly add query to hier_part, because the URI spec changed from RFC 2396 to RFC 3986
    # with the query not being part of hier_part
    Request_URI = ( "*" | absolute_URI | (hier_part ( "?" query )?) | authority);
    Request_Line = Method SP Request_URI SP HTTP_Version CRLF;
    Request = Request_Line ((general_header | request_header | entity_header | extension_header) CRLF)* CRLF @done;

    main := Request;
    write data;
}%%

void
HTTP::HTTPParser::init()
{
    m_string = NULL;
    m_set = NULL;
    m_list = NULL;
    m_parameterizedList = NULL;
    m_parameters = NULL;
    m_auth = NULL;
    m_ulong = NULL;
    RagelParser::init();
}

HTTP::RequestParser::RequestParser(Request& request)
: m_request(&request),
  m_ver(&request.requestLine.ver),
  m_uri(&request.requestLine.uri),
  m_path(&request.requestLine.uri.path),
  m_general(&request.general),
  m_entity(&request.entity)
{}

void
HTTP::RequestParser::init()
{
    HTTPParser::init();
    %% write init;
}

bool
HTTP::RequestParser::complete() const
{
    return cs >= http_request_parser_first_final;
}

bool
HTTP::RequestParser::error() const
{
    return cs == http_request_parser_error;
}

void
HTTP::RequestParser::exec()
{
#ifdef MSVC
#pragma warning(push)
#pragma warning(disable : 4244)
#endif
    %% write exec;
#ifdef MSVC
#pragma warning(pop)
#endif
}

%%{
    machine http_response_parser;
    include http_parser;
    include uri_parser "../uri.rl";

    action parse_Status_Code {
        m_response->status.status = (HTTP::Status)atoi(mark);
        mark = NULL;
    }

    action parse_Reason_Phrase {
        m_response->status.reason = std::string(mark, fpc - mark);
        mark = NULL;
    }

    action set_accept_ranges
    {
        m_set = &m_response->response.acceptRanges;
        m_list = NULL;
    }
    action set_proxy_authenticate {
        m_parameterizedList = &m_response->response.proxyAuthenticate;
    }
    action set_www_authenticate {
        m_parameterizedList = &m_response->response.wwwAuthenticate;
    }

    Accept_Ranges = 'Accept-Ranges:'i @set_accept_ranges list;
    Location = 'Location:'i LWS* absolute_URI LWS*;
    Proxy_Authenticate = 'Proxy-Authenticate:'i @set_proxy_authenticate challengeList;
    WWW_Authenticate = 'WWW-Authenticate:'i @set_www_authenticate challengeList;

    response_header = Accept_Ranges | Location | Proxy_Authenticate | WWW_Authenticate;
    response_header_names = 'Accept-Ranges'i | 'Location'i | 'Proxy-Authenticate'i | 'WWW-Authenticate'i;

    extension_header = (token - (general_header_names | response_header_names | entity_header_names)) >mark %save_field_name
        ':' field_value;

    Status_Code = DIGIT{3} > mark %parse_Status_Code;
    Reason_Phrase = (TEXT -- (CR | LF))* >mark %parse_Reason_Phrase;
    Status_Line = HTTP_Version SP Status_Code SP Reason_Phrase CRLF;
    Response = Status_Line ((general_header | response_header | entity_header | extension_header) CRLF)* CRLF @done;

    main := Response;

    write data;
}%%

HTTP::ResponseParser::ResponseParser(Response& response)
: m_response(&response),
  m_ver(&response.status.ver),
  m_uri(&response.response.location),
  m_path(&response.response.location.path),
  m_general(&response.general),
  m_entity(&response.entity)
{}

void
HTTP::ResponseParser::init()
{
    HTTPParser::init();
    %% write init;
}

bool
HTTP::ResponseParser::complete() const
{
    return cs >= http_response_parser_first_final;
}

bool
HTTP::ResponseParser::error() const
{
    return cs == http_response_parser_error;
}

void
HTTP::ResponseParser::exec()
{
#ifdef MSVC
#pragma warning(push)
#pragma warning(disable : 4244)
#endif
    %% write exec;
#ifdef MSVC
#pragma warning(pop)
#endif
}

%%{
    machine http_trailer_parser;
    include http_parser;

    extension_header = (token - (entity_header_names)) >mark %save_field_name ':' field_value;

    trailer = (entity_header | extension_header CRLF)*;

    main := trailer CRLF @done;

    write data;
}%%

HTTP::TrailerParser::TrailerParser(EntityHeaders& entity)
: m_entity(&entity)
{}

void
HTTP::TrailerParser::init()
{
    HTTPParser::init();
    %% write init;
}

bool
HTTP::TrailerParser::complete() const
{
    return cs >= http_trailer_parser_first_final;
}

bool
HTTP::TrailerParser::error() const
{
    return cs == http_trailer_parser_error;
}

void
HTTP::TrailerParser::exec()
{
#ifdef MSVC
#pragma warning(push)
#pragma warning(disable : 4244)
#endif
    %% write exec;
#ifdef MSVC
#pragma warning(pop)
#endif
}


Fix continuing actions

Fix continuing actions

/*
 Copyright (c) 2017 Tuukka Norri
 This code is licensed under MIT license (see LICENSE for details).
 */

#include <vcf2multialign/util.hh>
#include <vcf2multialign/vcf_reader.hh>

%% machine vcf_parser;
%% write data;

// We would like to post-process strings that have been read and then pass
// them to the variant object with a simple function call at the call site.
// Solution: Templates can have member function pointers as parameters but of a specific type.
#define HANDLE_STRING_END(FN, ...)	caller <decltype(FN)>::handle_string_end <FN>(*this, ##__VA_ARGS__)
#define HANDLE_INTEGER_END(FN, ...)	caller <decltype(FN)>::handle_integer_end <FN>(*this, ##__VA_ARGS__)

namespace vcf2multialign { namespace detail {

	template <typename t_enum>
	constexpr typename std::underlying_type <t_enum>::type to_underlying(t_enum e)
	{
		return static_cast <typename std::underlying_type <t_enum>::type>(e);
	}

	// Split a t_string into string_views.
	template <bool t_emplace_back, typename t_string, typename t_invalid_pos>
	size_t read_fields(
		t_string const &string,
		t_invalid_pos const invalid_pos,
		char const *string_start,
		char const *sep,
		size_t const count,
		std::vector <std::string_view> &res
	)
	{
		char const *string_ptr(string_start);

		if (t_emplace_back)
			res.clear();
		else
			res.resize(count);

		typename t_string::size_type sep_pos(0);
		size_t i(0);
		bool should_break(false);
		while (i < count)
		{
			sep_pos = string.find(sep, sep_pos);
			if (invalid_pos == sep_pos)
			{
				should_break = true;
				sep_pos = string.size();
			}

			if (t_emplace_back)
				res.emplace_back(string_ptr, sep_pos - (string_ptr - string_start));
			else
			{
				std::string_view sv(string_ptr, sep_pos - (string_ptr - string_start));
				res[i] = std::move(sv);
			}

			++i;
			++sep_pos;
			string_ptr = string_start + sep_pos;

			if (should_break)
				break;
		}
		return i;
	}

	// Split a string into string_views.
	template <bool t_emplace_back>
	size_t read_fields(
		std::string const &string,
		char const *sep,
		size_t const count,
		std::vector <std::string_view> &res
	)
	{
		return read_fields <t_emplace_back>(string, std::string::npos, string.c_str(), sep, count, res);
	}

	template <bool t_emplace_back>
	size_t read_fields(
		std::string_view const &sv,
		char const *sep,
		size_t const count,
		std::vector <std::string_view> &res
	)
	{
		return read_fields <t_emplace_back>(sv, std::string_view::npos, static_cast <char const *>(sv.data()), sep, count, res);
	}
}}

namespace vcf2multialign {

	template <typename t_fnt>
	struct vcf_reader::caller
	{
		template <t_fnt t_fn, typename ... t_args>
		static void handle_string_end(vcf_reader &r, t_args ... args)
		{
			std::string_view sv(r.m_start, r.m_fsm.p - r.m_start);
			(r.m_current_variant.*(t_fn))(sv, std::forward <t_args>(args)...);
		}

		template <t_fnt t_fn, typename ... t_args>
		static void handle_integer_end(vcf_reader &r, t_args ... args)
		{
			(r.m_current_variant.*(t_fn))(r.m_integer, std::forward <t_args>(args)...);
		}
	};

	template <int t_continue, int t_break>
	int vcf_reader::check_max_field(vcf_field const field, int const target, callback_fn const &cb)
	{
		if (field <= m_max_parsed_field)
			return target;

		skip_to_next_nl();
		if (!cb(m_current_variant))
			return t_break;

		return t_continue;
	}

	void vcf_reader::skip_to_next_nl()
	{
		std::string_view sv(m_fsm.p, m_fsm.pe - m_fsm.p);
		auto const pos(sv.find('\n'));
		always_assert(std::string_view::npos != pos, "Unable to find the next newline");
		m_fsm.p += pos;
	}

	// Seek to the beginning of the records.
	void vcf_reader::reset()
	{
		m_stream->clear();
		m_stream->seekg(m_first_variant_offset);
		m_lineno = m_last_header_lineno;
		m_len = 0;
		m_pos = 0;
	}

	// Return the 1-based number of the given sample.
	size_t vcf_reader::sample_no(std::string const &sample_name) const
	{
		auto const it(m_sample_names.find(sample_name));
		if (it == m_sample_names.cend())
			return 0;
		return it->second;
	}

	void vcf_reader::read_header()
	{
		// For now, just skip lines that begin with "##".
		std::string line;
		while (std::getline(*m_stream, line))
		{
			++m_lineno;
			if (! ('#' == line[0] && '#' == line[1]))
				break;

			// FIXME: header handling goes here.
		}

		// Check for column headers.
		{
			auto const prefix(std::string("#CHROM"));
			auto const res(mismatch(line.cbegin(), line.cend(), prefix.cbegin(), prefix.cend()));
			if (prefix.cend() != res.second)
			{
				std::cerr << "Expected the next line to start with '#CHROM', instead got: '" << line << "'" << std::endl;
				exit(1);
			}
		}

		// Read sample names.
		std::vector <std::string_view> header_names;
		detail::read_fields <true>(line, "\t", -1, header_names);
		std::size_t i(1);
		for (auto it(header_names.cbegin() + detail::to_underlying(vcf_field::ALL)), end(header_names.cend()); it != end; ++it)
		{
			std::string str(*it);
			auto const res(m_sample_names.emplace(std::move(str), i));
			always_assert(res.second, "Duplicate sample name");
			++i;
		}

		// stream now points to the first variant.
		m_first_variant_offset = m_stream->tellg();
		m_last_header_lineno = m_lineno;

		// Instantiate a variant.
		transient_variant var(sample_count());
		using std::swap;
		swap(m_current_variant, var);
	}

	void vcf_reader::fill_buffer()
	{
		// Copy the remainder to the beginning.
		if (m_pos + 1 < m_len)
		{
			char *data_start(m_buffer.data());
			char const *start(data_start + m_pos + 1);
			char const *end(data_start + m_len);
			std::copy(start, end, data_start);
			m_len -= m_pos + 1;
		}
		else
		{
			m_len = 0;
		}

		// Read until there's at least one newline in the buffer.
		while (true)
		{
			char *data_start(m_buffer.data());
			char *data(data_start + m_len);

			std::size_t space(m_buffer.size() - m_len);
			m_stream->read(data, space);
			std::size_t const read_len(m_stream->gcount());
			m_len += read_len;

			if (m_stream->eof())
			{
				m_pos = m_len;
				m_fsm.p = data_start;
				m_fsm.pe = data_start + m_len;
				m_fsm.eof = m_fsm.pe;
				return;
			}

			// Try to find the last newline in the new part.
			std::string_view sv(data, read_len);
			m_pos = sv.rfind('\n');
			if (std::string_view::npos != m_pos)
			{
				m_pos += (data - data_start);
				m_fsm.p = data_start;
				m_fsm.pe = m_fsm.p + m_pos + 1;
				return;
			}

			m_buffer.resize(2 * m_buffer.size());
		}
	}

	bool vcf_reader::parse(callback_fn &&callback)
	{
		return parse(callback);
	}

	bool vcf_reader::parse(callback_fn const &cb)
	{
		typedef variant_tpl <std::string_view> vc;
		bool retval(true);
		int cs(0);
		// FIXME: add throwing EOF actions?
		%%{
			variable p		m_fsm.p;
			variable pe		m_fsm.pe;
			variable eof	m_fsm.eof;

			action start_string {
				m_start = fpc;
			}

			action start_integer {
				m_integer = 0;
			}

			action update_integer {
				// Shift and add the current number.
				m_integer *= 10;
				m_integer += fc - '0';
			}

			action end_sample_field {
				// Check that the current field is followed by another field, another sample or a new record.
				switch (fc)
				{
					case ':':
						fgoto sample_rec_f;

					case '\t':
					{
						always_assert(m_format.size() == m_format_idx, "Not all fields present in the sample");
						m_format_idx = 0;
						fgoto sample_rec_f;
					}

					case '\n':
					{
						always_assert(m_format.size() == m_format_idx, "Not all fields present in the sample");

						if(!cb(m_current_variant))
						{
							fhold;
							fbreak;
						}

						fgoto main;
					}

					default:
						fail("Unexpected character");
				}
			}

			action error {
				fail("Unexpected character");
			}

			tab			= '\t';

			id_string	= '<' alnum+ '>';

			chrom_id	= (alnum+)
				>(start_string)
				%{ HANDLE_STRING_END(&vc::set_chrom_id); };

			pos			= (digit+)
				>(start_integer)
				$(update_integer)
				%{ HANDLE_INTEGER_END(&vc::set_pos); };

			id_part		= (alnum+)
				>(start_string)
				%{ HANDLE_STRING_END(&vc::set_id, m_idx++); };
			id_rec		= (id_part (';' id_part)*) >{ m_idx = 0; };

			ref			= ([ACGTN]+)
				>(start_string)
				%{ HANDLE_STRING_END(&vc::set_ref); };

			# FIXME: add breakends.
			simple_alt	= ([ACGTN]+);
			complex_alt	= ([*]+) %{ m_alt_is_complex = true; };
			alt_part	= (simple_alt | complex_alt)
				>(start_string)
				%{ HANDLE_STRING_END(&vc::set_alt, m_idx++, m_alt_is_complex); };
			alt			= (alt_part (',' alt_part)*) >{ m_idx = 0; };

			qual		= (digit+)
				>(start_integer)
				$(update_integer)
				%{ HANDLE_INTEGER_END(&vc::set_qual); };

			# FIXME: add actions.
			filter_pass	= 'PASS';
			filter_part	= alnum+;
			filter		= (filter_pass | (filter_part (';' filter_part)*));

			# FIXME: add actions.
			info_key	= (alnum | '_') +;
			info_str	= (alnum | [_.]) +;
			info_val	= info_str (',' info_str)*;
			info_part	= info_key ('=' info_val)?;
			info		= (info_part (';' info_part)*);

			# FIXME: handle other formats.
			format_gt	= ('GT') @{ m_format.emplace_back(format_field::GT); };
			format_dp	= ('DP') @{ m_format.emplace_back(format_field::DP); };
			format_gq	= ('GQ') @{ m_format.emplace_back(format_field::GQ); };
			format_ps	= ('PS') @{ m_format.emplace_back(format_field::PS); };
			format_pq	= ('PQ') @{ m_format.emplace_back(format_field::PQ); };
			format_mq	= ('MQ') @{ m_format.emplace_back(format_field::MQ); };
			format_part	= format_gt | format_dp | format_gq | format_ps | format_pq | format_mq;
			format		= ((format_part (':' format_part)*) >{ m_format.clear(); });

			# Genotype field in sample.
			sample_gt_null_allele	= '.'
				%{
					m_integer = NULL_ALLELE;
					HANDLE_INTEGER_END(&vc::set_gt, m_sample_idx, m_idx++, m_gt_is_phased);
				};

			sample_gt_allele_idx	= (digit+)
				>(start_integer)
				$(update_integer)
				%{
					HANDLE_INTEGER_END(&vc::set_gt, m_sample_idx, m_idx++, m_gt_is_phased);
					m_gt_is_phased = false;
				};

			sample_gt_part			= (sample_gt_null_allele | sample_gt_allele_idx);
			sample_gt_p				= sample_gt_part ('|' ${ m_gt_is_phased = true; } sample_gt_part)+;
			sample_gt_up			= sample_gt_part ('/' sample_gt_part)+;
			sample_gt				= (sample_gt_part | sample_gt_p | sample_gt_up)
				>{
					m_idx = 0;
				};

			# FIXME: add actions.
			sample_dp		= (digit+);
			sample_gq		= (digit+);
			sample_ps		= (digit+);
			sample_pq		= (digit+);
			sample_mq		= (digit+);

			sep				= '\t';		# Field separator
			ssep			= [\t\n:];	# Sample separator

			# Handle a newline and continue.
			# The newline gets eaten before its checked, though, so use any instead.
			main_nl := any @{ fhold; fgoto main; };
			break_nl := any @{ fhold; fbreak; };

			# Line start.
			# Apparently main has to be able to read a character, so use fhold.
			# Allow EOF, though, with '?'.
			main := any?
				${
					fhold;

					m_idx = 0;
					m_format_idx = 0;
					m_sample_idx = 0;
					m_alt_is_complex = false;
					++m_lineno;
					m_current_variant.reset();
					m_current_variant.set_lineno(m_lineno);

					fgoto *check_max_field <fentry(main_nl), fentry(break_nl)>(vcf_field::CHROM, fentry(chrom_id_f), cb);
				}
				$err(error)
				$eof{ retval = false; };

			# #CHROM
			chrom_id_f :=
				(chrom_id sep)
				@{
					fgoto *check_max_field <fentry(main_nl), fentry(break_nl)>(vcf_field::POS, fentry(pos_f), cb);
				}
				$err(error);

			# POS
			pos_f :=
				(pos sep)
				@{
					fgoto *check_max_field <fentry(main_nl), fentry(break_nl)>(vcf_field::ID, fentry(id_f), cb);
				}
				$err(error);

			# ID
			id_f :=
				(id_rec sep)
				@{
					fgoto *check_max_field <fentry(main_nl), fentry(break_nl)>(vcf_field::REF, fentry(ref_f), cb);
				}
				$err(error);

			# REF
			ref_f :=
				(ref sep)
				@{
					fgoto *check_max_field <fentry(main_nl), fentry(break_nl)>(vcf_field::ALT, fentry(alt_f), cb);
				}
				$err(error);

			# ALT
			alt_f :=
				(alt sep)
				@{
					fgoto *check_max_field <fentry(main_nl), fentry(break_nl)>(vcf_field::QUAL, fentry(qual_f), cb);
				}
				$err(error);

			# QUAL
			qual_f :=
				(qual sep)
				@{
					fgoto *check_max_field <fentry(main_nl), fentry(break_nl)>(vcf_field::FILTER, fentry(filter_f), cb);
				}
				$err(error);

			# FILTER
			filter_f :=
				(filter sep)
				@{
					fgoto *check_max_field <fentry(main_nl), fentry(break_nl)>(vcf_field::INFO, fentry(info_f), cb);
				}
				$err(error);

			# INFO
			info_f :=
				(info sep)
				@{
					fgoto *check_max_field <fentry(main_nl), fentry(break_nl)>(vcf_field::FORMAT, fentry(format_f), cb);
				}
				$err(error);

			# FORMAT
			# FIXME: try to skip parsing the field and compare to the existing format?
			format_f :=
				(format sep)
				@{
					fgoto *check_max_field <fentry(main_nl), fentry(break_nl)>(vcf_field::ALL, fentry(sample_rec_f), cb);
				}
				$err(error);

			# Sample fields
			sample_gt_f := ((sample_gt) ssep @(end_sample_field)) $err(error);
			sample_dp_f := ((sample_dp) ssep @(end_sample_field)) $err(error);
			sample_gq_f := ((sample_gq) ssep @(end_sample_field)) $err(error);
			sample_ps_f := ((sample_ps) ssep @(end_sample_field)) $err(error);
			sample_pq_f := ((sample_pq) ssep @(end_sample_field)) $err(error);
			sample_mq_f := ((sample_mq) ssep @(end_sample_field)) $err(error);

			# Sample record
			sample_rec_f := "" >to{
				always_assert(m_format_idx < m_format.size(), "Format does not match the sample");

				// Sample index 0 is reserved for the reference.
				++m_sample_idx;

				// Parse according to the format field.
				auto const idx(m_format_idx);
				++m_format_idx;
				auto const field(m_format[idx]);
				switch (field)
				{
					case format_field::GT:
						fgoto sample_gt_f;

					case format_field::DP:
						fgoto sample_dp_f;

					case format_field::GQ:
						fgoto sample_gq_f;

					case format_field::PS:
						fgoto sample_ps_f;

					case format_field::PQ:
						fgoto sample_pq_f;

					case format_field::MQ:
						fgoto sample_mq_f;

					default:
						fail("Unexpected format value");
				}
			};

			dummy := any
				@{
					// Dummy action,
					// make sure that all the machines are instantiated.
					if (false)
					{
						fgoto chrom_id_f;
						fgoto pos_f;
						fgoto id_f;
						fgoto ref_f;
						fgoto alt_f;
						fgoto qual_f;
						fgoto filter_f;
						fgoto info_f;
						fgoto format_f;
						fgoto sample_rec_f;

						fgoto sample_gt_f;
						fgoto sample_dp_f;
						fgoto sample_gq_f;
						fgoto sample_ps_f;
						fgoto sample_pq_f;
						fgoto sample_mq_f;
					}
				};

			write init;
			write exec;
		}%%

		return retval;
	}
}

/*
 Copyright (c) 2017 Tuukka Norri
 This code is licensed under MIT license (see LICENSE for details).
 */

#include <vcf2multialign/util.hh>
#include <vcf2multialign/vcf_reader.hh>

%% machine vcf_parser;
%% write data;

// We would like to post-process strings that have been read and then pass
// them to the variant object with a simple function call at the call site.
// Solution: Templates can have member function pointers as parameters but of a specific type.
#define HANDLE_STRING_END(FN, ...)	caller <decltype(FN)>::handle_string_end <FN>(*this, ##__VA_ARGS__)
#define HANDLE_INTEGER_END(FN, ...)	caller <decltype(FN)>::handle_integer_end <FN>(*this, ##__VA_ARGS__)

namespace vcf2multialign { namespace detail {

	template <typename t_enum>
	constexpr typename std::underlying_type <t_enum>::type to_underlying(t_enum e)
	{
		return static_cast <typename std::underlying_type <t_enum>::type>(e);
	}

	// Split a t_string into string_views.
	template <bool t_emplace_back, typename t_string, typename t_invalid_pos>
	size_t read_fields(
		t_string const &string,
		t_invalid_pos const invalid_pos,
		char const *string_start,
		char const *sep,
		size_t const count,
		std::vector <std::string_view> &res
	)
	{
		char const *string_ptr(string_start);

		if (t_emplace_back)
			res.clear();
		else
			res.resize(count);

		typename t_string::size_type sep_pos(0);
		size_t i(0);
		bool should_break(false);
		while (i < count)
		{
			sep_pos = string.find(sep, sep_pos);
			if (invalid_pos == sep_pos)
			{
				should_break = true;
				sep_pos = string.size();
			}

			if (t_emplace_back)
				res.emplace_back(string_ptr, sep_pos - (string_ptr - string_start));
			else
			{
				std::string_view sv(string_ptr, sep_pos - (string_ptr - string_start));
				res[i] = std::move(sv);
			}

			++i;
			++sep_pos;
			string_ptr = string_start + sep_pos;

			if (should_break)
				break;
		}
		return i;
	}

	// Split a string into string_views.
	template <bool t_emplace_back>
	size_t read_fields(
		std::string const &string,
		char const *sep,
		size_t const count,
		std::vector <std::string_view> &res
	)
	{
		return read_fields <t_emplace_back>(string, std::string::npos, string.c_str(), sep, count, res);
	}

	template <bool t_emplace_back>
	size_t read_fields(
		std::string_view const &sv,
		char const *sep,
		size_t const count,
		std::vector <std::string_view> &res
	)
	{
		return read_fields <t_emplace_back>(sv, std::string_view::npos, static_cast <char const *>(sv.data()), sep, count, res);
	}
}}

namespace vcf2multialign {

	template <typename t_fnt>
	struct vcf_reader::caller
	{
		template <t_fnt t_fn, typename ... t_args>
		static void handle_string_end(vcf_reader &r, t_args ... args)
		{
			std::string_view sv(r.m_start, r.m_fsm.p - r.m_start);
			(r.m_current_variant.*(t_fn))(sv, std::forward <t_args>(args)...);
		}

		template <t_fnt t_fn, typename ... t_args>
		static void handle_integer_end(vcf_reader &r, t_args ... args)
		{
			(r.m_current_variant.*(t_fn))(r.m_integer, std::forward <t_args>(args)...);
		}
	};

	template <int t_continue, int t_break>
	int vcf_reader::check_max_field(vcf_field const field, int const target, callback_fn const &cb)
	{
		if (field <= m_max_parsed_field)
			return target;

		skip_to_next_nl();
		if (!cb(m_current_variant))
			return t_break;

		return t_continue;
	}

	void vcf_reader::skip_to_next_nl()
	{
		std::string_view sv(m_fsm.p, m_fsm.pe - m_fsm.p);
		auto const pos(sv.find('\n'));
		always_assert(std::string_view::npos != pos, "Unable to find the next newline");
		m_fsm.p += pos;
	}

	// Seek to the beginning of the records.
	void vcf_reader::reset()
	{
		m_stream->clear();
		m_stream->seekg(m_first_variant_offset);
		m_lineno = m_last_header_lineno;
		m_len = 0;
		m_pos = 0;
	}

	// Return the 1-based number of the given sample.
	size_t vcf_reader::sample_no(std::string const &sample_name) const
	{
		auto const it(m_sample_names.find(sample_name));
		if (it == m_sample_names.cend())
			return 0;
		return it->second;
	}

	void vcf_reader::read_header()
	{
		// For now, just skip lines that begin with "##".
		std::string line;
		while (std::getline(*m_stream, line))
		{
			++m_lineno;
			if (! ('#' == line[0] && '#' == line[1]))
				break;

			// FIXME: header handling goes here.
		}

		// Check for column headers.
		{
			auto const prefix(std::string("#CHROM"));
			auto const res(mismatch(line.cbegin(), line.cend(), prefix.cbegin(), prefix.cend()));
			if (prefix.cend() != res.second)
			{
				std::cerr << "Expected the next line to start with '#CHROM', instead got: '" << line << "'" << std::endl;
				exit(1);
			}
		}

		// Read sample names.
		std::vector <std::string_view> header_names;
		detail::read_fields <true>(line, "\t", -1, header_names);
		std::size_t i(1);
		for (auto it(header_names.cbegin() + detail::to_underlying(vcf_field::ALL)), end(header_names.cend()); it != end; ++it)
		{
			std::string str(*it);
			auto const res(m_sample_names.emplace(std::move(str), i));
			always_assert(res.second, "Duplicate sample name");
			++i;
		}

		// stream now points to the first variant.
		m_first_variant_offset = m_stream->tellg();
		m_last_header_lineno = m_lineno;

		// Instantiate a variant.
		transient_variant var(sample_count());
		using std::swap;
		swap(m_current_variant, var);
	}

	void vcf_reader::fill_buffer()
	{
		// Copy the remainder to the beginning.
		if (m_pos + 1 < m_len)
		{
			char *data_start(m_buffer.data());
			char const *start(data_start + m_pos + 1);
			char const *end(data_start + m_len);
			std::copy(start, end, data_start);
			m_len -= m_pos + 1;
		}
		else
		{
			m_len = 0;
		}

		// Read until there's at least one newline in the buffer.
		while (true)
		{
			char *data_start(m_buffer.data());
			char *data(data_start + m_len);

			std::size_t space(m_buffer.size() - m_len);
			m_stream->read(data, space);
			std::size_t const read_len(m_stream->gcount());
			m_len += read_len;

			if (m_stream->eof())
			{
				m_pos = m_len;
				m_fsm.p = data_start;
				m_fsm.pe = data_start + m_len;
				m_fsm.eof = m_fsm.pe;
				return;
			}

			// Try to find the last newline in the new part.
			std::string_view sv(data, read_len);
			m_pos = sv.rfind('\n');
			if (std::string_view::npos != m_pos)
			{
				m_pos += (data - data_start);
				m_fsm.p = data_start;
				m_fsm.pe = m_fsm.p + m_pos + 1;
				return;
			}

			m_buffer.resize(2 * m_buffer.size());
		}
	}

	bool vcf_reader::parse(callback_fn &&callback)
	{
		return parse(callback);
	}

	bool vcf_reader::parse(callback_fn const &cb)
	{
		typedef variant_tpl <std::string_view> vc;
		bool retval(true);
		int cs(0);
		// FIXME: add throwing EOF actions?
		%%{
			variable p		m_fsm.p;
			variable pe		m_fsm.pe;
			variable eof	m_fsm.eof;

			action start_string {
				m_start = fpc;
			}

			action start_integer {
				m_integer = 0;
			}

			action update_integer {
				// Shift and add the current number.
				m_integer *= 10;
				m_integer += fc - '0';
			}

			action end_sample_field {
				// Check that the current field is followed by another field, another sample or a new record.
				switch (fc)
				{
					case ':':
						fgoto sample_rec_f;

					case '\t':
					{
						always_assert(m_format.size() == m_format_idx, "Not all fields present in the sample");
						m_format_idx = 0;
						fgoto sample_rec_f;
					}

					case '\n':
					{
						always_assert(m_format.size() == m_format_idx, "Not all fields present in the sample");

						if(!cb(m_current_variant))
						{
							fhold;
							fbreak;
						}

						fgoto main;
					}

					default:
						fail("Unexpected character");
				}
			}

			action error {
				fail("Unexpected character");
			}

			tab			= '\t';

			id_string	= '<' alnum+ '>';

			chrom_id	= (alnum+)
				>(start_string)
				%{ HANDLE_STRING_END(&vc::set_chrom_id); };

			pos			= (digit+)
				>(start_integer)
				$(update_integer)
				%{ HANDLE_INTEGER_END(&vc::set_pos); };

			id_part		= (alnum+)
				>(start_string)
				%{ HANDLE_STRING_END(&vc::set_id, m_idx++); };
			id_rec		= (id_part (';' id_part)*) >{ m_idx = 0; };

			ref			= ([ACGTN]+)
				>(start_string)
				%{ HANDLE_STRING_END(&vc::set_ref); };

			# FIXME: add breakends.
			simple_alt	= ([ACGTN]+);
			complex_alt	= ([*]+) %{ m_alt_is_complex = true; };
			alt_part	= (simple_alt | complex_alt)
				>(start_string)
				%{ HANDLE_STRING_END(&vc::set_alt, m_idx++, m_alt_is_complex); };
			alt			= (alt_part (',' alt_part)*) >{ m_idx = 0; };

			qual		= (digit+)
				>(start_integer)
				$(update_integer)
				%{ HANDLE_INTEGER_END(&vc::set_qual); };

			# FIXME: add actions.
			filter_pass	= 'PASS';
			filter_part	= alnum+;
			filter		= (filter_pass | (filter_part (';' filter_part)*));

			# FIXME: add actions.
			info_key	= (alnum | '_') +;
			info_str	= (alnum | [_.]) +;
			info_val	= info_str (',' info_str)*;
			info_part	= info_key ('=' info_val)?;
			info		= (info_part (';' info_part)*);

			# FIXME: handle other formats.
			format_gt	= ('GT') @{ m_format.emplace_back(format_field::GT); };
			format_dp	= ('DP') @{ m_format.emplace_back(format_field::DP); };
			format_gq	= ('GQ') @{ m_format.emplace_back(format_field::GQ); };
			format_ps	= ('PS') @{ m_format.emplace_back(format_field::PS); };
			format_pq	= ('PQ') @{ m_format.emplace_back(format_field::PQ); };
			format_mq	= ('MQ') @{ m_format.emplace_back(format_field::MQ); };
			format_part	= format_gt | format_dp | format_gq | format_ps | format_pq | format_mq;
			format		= ((format_part (':' format_part)*) >{ m_format.clear(); });

			# Genotype field in sample.
			sample_gt_null_allele	= '.'
				%{
					m_integer = NULL_ALLELE;
					HANDLE_INTEGER_END(&vc::set_gt, m_sample_idx, m_idx++, m_gt_is_phased);
				};

			sample_gt_allele_idx	= (digit+)
				>(start_integer)
				$(update_integer)
				%{
					HANDLE_INTEGER_END(&vc::set_gt, m_sample_idx, m_idx++, m_gt_is_phased);
					m_gt_is_phased = false;
				};

			sample_gt_part			= (sample_gt_null_allele | sample_gt_allele_idx);
			sample_gt_p				= sample_gt_part ('|' ${ m_gt_is_phased = true; } sample_gt_part)+;
			sample_gt_up			= sample_gt_part ('/' sample_gt_part)+;
			sample_gt				= (sample_gt_part | sample_gt_p | sample_gt_up)
				>{
					m_idx = 0;
				};

			# FIXME: add actions.
			sample_dp		= (digit+);
			sample_gq		= (digit+);
			sample_ps		= (digit+);
			sample_pq		= (digit+);
			sample_mq		= (digit+);

			sep				= '\t';		# Field separator
			ssep			= [\t\n:];	# Sample separator

			# Handle a newline and continue.
			main_nl := '\n' %{ fgoto main; };
			break_nl := '\n' %{ fbreak; };

			# Line start.
			# Apparently main has to be able to read a character, so use fhold.
			# Allow EOF, though, with '?'.
			main := any?
				${
					fhold;

					m_idx = 0;
					m_format_idx = 0;
					m_sample_idx = 0;
					m_alt_is_complex = false;
					++m_lineno;
					m_current_variant.reset();
					m_current_variant.set_lineno(m_lineno);

					fgoto *check_max_field <fentry(main_nl), fentry(break_nl)>(vcf_field::CHROM, fentry(chrom_id_f), cb);
				}
				$err(error)
				$eof{ retval = false; };

			# #CHROM
			chrom_id_f :=
				(chrom_id sep)
				@{
					fgoto *check_max_field <fentry(main_nl), fentry(break_nl)>(vcf_field::POS, fentry(pos_f), cb);
				}
				$err(error);

			# POS
			pos_f :=
				(pos sep)
				@{
					fgoto *check_max_field <fentry(main_nl), fentry(break_nl)>(vcf_field::ID, fentry(id_f), cb);
				}
				$err(error);

			# ID
			id_f :=
				(id_rec sep)
				@{
					fgoto *check_max_field <fentry(main_nl), fentry(break_nl)>(vcf_field::REF, fentry(ref_f), cb);
				}
				$err(error);

			# REF
			ref_f :=
				(ref sep)
				@{
					fgoto *check_max_field <fentry(main_nl), fentry(break_nl)>(vcf_field::ALT, fentry(alt_f), cb);
				}
				$err(error);

			# ALT
			alt_f :=
				(alt sep)
				@{
					fgoto *check_max_field <fentry(main_nl), fentry(break_nl)>(vcf_field::QUAL, fentry(qual_f), cb);
				}
				$err(error);

			# QUAL
			qual_f :=
				(qual sep)
				@{
					fgoto *check_max_field <fentry(main_nl), fentry(break_nl)>(vcf_field::FILTER, fentry(filter_f), cb);
				}
				$err(error);

			# FILTER
			filter_f :=
				(filter sep)
				@{
					fgoto *check_max_field <fentry(main_nl), fentry(break_nl)>(vcf_field::INFO, fentry(info_f), cb);
				}
				$err(error);

			# INFO
			info_f :=
				(info sep)
				@{
					fgoto *check_max_field <fentry(main_nl), fentry(break_nl)>(vcf_field::FORMAT, fentry(format_f), cb);
				}
				$err(error);

			# FORMAT
			# FIXME: try to skip parsing the field and compare to the existing format?
			format_f :=
				(format sep)
				@{
					fgoto *check_max_field <fentry(main_nl), fentry(break_nl)>(vcf_field::ALL, fentry(sample_rec_f), cb);
				}
				$err(error);

			# Sample fields
			sample_gt_f := ((sample_gt) ssep @(end_sample_field)) $err(error);
			sample_dp_f := ((sample_dp) ssep @(end_sample_field)) $err(error);
			sample_gq_f := ((sample_gq) ssep @(end_sample_field)) $err(error);
			sample_ps_f := ((sample_ps) ssep @(end_sample_field)) $err(error);
			sample_pq_f := ((sample_pq) ssep @(end_sample_field)) $err(error);
			sample_mq_f := ((sample_mq) ssep @(end_sample_field)) $err(error);

			# Sample record
			sample_rec_f := "" >to{
				always_assert(m_format_idx < m_format.size(), "Format does not match the sample");

				// Sample index 0 is reserved for the reference.
				++m_sample_idx;

				// Parse according to the format field.
				auto const idx(m_format_idx);
				++m_format_idx;
				auto const field(m_format[idx]);
				switch (field)
				{
					case format_field::GT:
						fgoto sample_gt_f;

					case format_field::DP:
						fgoto sample_dp_f;

					case format_field::GQ:
						fgoto sample_gq_f;

					case format_field::PS:
						fgoto sample_ps_f;

					case format_field::PQ:
						fgoto sample_pq_f;

					case format_field::MQ:
						fgoto sample_mq_f;

					default:
						fail("Unexpected format value");
				}
			};

			dummy := any
				@{
					// Dummy action,
					// make sure that all the machines are instantiated.
					if (false)
					{
						fgoto chrom_id_f;
						fgoto pos_f;
						fgoto id_f;
						fgoto ref_f;
						fgoto alt_f;
						fgoto qual_f;
						fgoto filter_f;
						fgoto info_f;
						fgoto format_f;
						fgoto sample_rec_f;

						fgoto sample_gt_f;
						fgoto sample_dp_f;
						fgoto sample_gq_f;
						fgoto sample_ps_f;
						fgoto sample_pq_f;
						fgoto sample_mq_f;
					}
				};

			write init;
			write exec;
		}%%

		return retval;
	}
}