Fix for parsing empty querystring.

Fix for parsing empty querystring.

Change-Id: Ibd251d37cb49c88ddc1846532f59e909c7dd0616
Reviewed-on: https://gerrit.dechocorp.com/6692
Reviewed-by: Russ Simons <c93a72122f3199b175c9b2766a6096fa95d517d4@mozy.com>

// Copyright (c) 2009 - Mozy, Inc.
/* To compile to .cpp:
   ragel uri.rl -G2 -o uri.cpp
*/

#include "mordor/pch.h"

#include "mordor/uri.h"

#include <sstream>

#include "mordor/ragel.h"
#include "mordor/string.h"
#include "mordor/version.h"

namespace Mordor {

static const std::string unreserved("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-._~");
static const std::string sub_delims("!$&'()*+,;=");
static const std::string scheme("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789+-.");
static const std::string userinfo("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-._~" "!$&'()*+,;=" ":");
static const std::string host("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-._~" "!$&'()*+,;=" ":");
static const std::string pchar("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-._~" "!$&'()*+,;=" ":@");
static const std::string path("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-._~" "!$&'()*+,;=" ":@" "/");
static const std::string segment_nc("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-._~" "!$&'()*+,;=" "@");
static const std::string query("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-._~" "!$&'()*+,;=" ":@" "/?");
static const std::string queryString("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-._~" "!$'()*+," ":@" "/?");

static std::string escape(const std::string& str, const std::string& allowedChars, bool spaceAsPlus = false)
{
    const char *hexdigits = "0123456789ABCDEF";
    std::string result(str);

    const char *c = str.c_str();
    const char *end = c + str.length();
    bool differed = false;
    while(c < end)
    {
        if (allowedChars.find(*c) == std::string::npos) {
            if (!differed) {
                result.resize(c - str.c_str());
                differed = true;
            }
            if (*c == ' ' && spaceAsPlus) {
                result.append(1, '+');
            } else {
                result.append(1, '%');
                result.append(1, hexdigits[(unsigned char)*c >> 4]);
                result.append(1, hexdigits[*c & 0xf]);
            }
        } else {
            if (differed) {
                result.append(1, *c);
            }
        }
        ++c;
    }

    if (differed) {
        MORDOR_ASSERT(result.length() >= str.length());
    } else {
        MORDOR_ASSERT(result == str);
    }
    return result;
}

std::string unescape(const std::string& str, bool spaceAsPlus = false)
{
    std::string result = str;

    const char *c = str.c_str();
    const char *end = c + str.length();
    bool differed = false;
    while (c < end)
    {
        if (*c == '%') {
            MORDOR_ASSERT(c + 2 < end);
            if (!differed) {
                result.resize(c - str.c_str());
                differed = true;
            }
            char decoded;
            ++c;
            if (*c >= 'a' && *c <= 'f')
                decoded = (*c - 'a' + 10) << 4;
            else if (*c >= 'A' && *c <= 'F')
                decoded = (*c - 'A' + 10) << 4;
            else {
                MORDOR_ASSERT(*c >= '0' && *c <='9');
                decoded = (*c - '0') << 4;
            }
            ++c;
            if (*c >= 'a' && *c <= 'f')
                decoded |= *c - 'a' + 10;
            else if (*c >= 'A' && *c <= 'F')
                decoded |= *c - 'A' + 10;
            else {
                MORDOR_ASSERT(*c >= '0' && *c <='9');
                decoded |= *c - '0';
            }
            result.append(1, decoded);
        } else if (*c == '+' && spaceAsPlus) {
            if (!differed) {
                result.resize(c - str.c_str());
                differed = true;
            }
            result.append(1, ' ');
        } else if (differed) {
            result.append(1, *c);
        }
        ++c;
    }
    return result;
}

std::string
URI::encode(const std::string &str, CharacterClass charClass)
{
    switch (charClass) {
        case UNRESERVED:
            return escape(str, unreserved, false);
        case QUERYSTRING:
            return escape(str, Mordor::queryString, true);
        default:
            MORDOR_NOTREACHED();
    }
}

std::string
URI::decode(const std::string &str, CharacterClass charClass)
{
    switch (charClass) {
        case UNRESERVED:
            return unescape(str, false);
        case QUERYSTRING:
            return unescape(str, true);
        default:
            MORDOR_NOTREACHED();
    }
}

%%{
    # See RFC 3986: http://www.ietf.org/rfc/rfc3986.txt

    machine uri_parser;

    gen_delims = ":" | "/" | "?" | "#" | "[" | "]" | "@";
    sub_delims = "!" | "$" | "&" | "'" | "(" | ")" | "*" | "+" | "," | ";" | "=";
    reserved = gen_delims | sub_delims;
    unreserved = alpha | digit | "-" | "." | "_" | "~";
    pct_encoded = "%" xdigit xdigit;

    action marku { mark = fpc; }
    action save_scheme
    {
        m_uri->scheme(unescape(std::string(mark, fpc - mark)));
        mark = NULL;
    }

    scheme = (alpha | digit | "+" | "-" | ".")+ >marku %save_scheme;

    action save_port
    {
        if (fpc == mark)
            m_uri->authority.port(-1);
        else
            m_uri->authority.port(atoi(mark));
        mark = NULL;
    }
    action save_userinfo
    {
        m_uri->authority.userinfo(unescape(std::string(mark, fpc - mark)));
        mark = NULL;
    }
    action save_host
    {
        if (mark != NULL) {
            m_uri->authority.host(unescape(std::string(mark, fpc - mark)));
            mark = NULL;
        }
    }

    userinfo = (unreserved | pct_encoded | sub_delims | ":")*;
    dec_octet = digit | [1-9] digit | "1" digit{2} | 2 [0-4] digit | "25" [0-5];
    IPv4address = dec_octet "." dec_octet "." dec_octet "." dec_octet;
    h16 = xdigit{1,4};
    ls32 = (h16 ":" h16) | IPv4address;
    IPv6address = (                         (h16 ":"){6} ls32) |
                  (                    "::" (h16 ":"){5} ls32) |
                  ((             h16)? "::" (h16 ":"){4} ls32) |
                  (((h16 ":"){1} h16)? "::" (h16 ":"){3} ls32) |
                  (((h16 ":"){2} h16)? "::" (h16 ":"){2} ls32) |
                  (((h16 ":"){3} h16)? "::" (h16 ":"){1} ls32) |
                  (((h16 ":"){4} h16)? "::"              ls32) |
                  (((h16 ":"){5} h16)? "::"              h16 ) |
                  (((h16 ":"){6} h16)? "::"                  );
    IPvFuture = "v" xdigit+ "." (unreserved | sub_delims | ":")+;
    IP_literal = "[" (IPv6address | IPvFuture) "]";
    reg_name = (unreserved | pct_encoded | sub_delims)*;
    host = IP_literal | IPv4address | reg_name;
    port = digit*;

    authority = ( (userinfo %save_userinfo "@")? host >marku %save_host (":" port >marku %save_port)? ) >marku;

    action save_segment
    {
        m_path->segments.push_back(unescape(std::string(mark, fpc - mark)));
        mark = NULL;
    }

    pchar = unreserved | pct_encoded | sub_delims | ":" | "@";
    segment = pchar* >marku %save_segment;
    segment_nz = pchar+ >marku %save_segment;
    segment_nz_nc = (pchar - ":")+ >marku %save_segment;

    action set_absolute
    {
        m_path->type = URI::Path::ABSOLUTE;
    }
    action set_relative
    {
        m_path->type = URI::Path::RELATIVE;
    }

    path_abempty = ("/" segment >set_absolute)*;
    path_absolute = "/" (segment_nz ("/" segment)*)?  >set_absolute;
    path_noscheme = segment_nz_nc >set_relative ("/" segment)*;
    path_rootless = segment_nz >set_relative ("/" segment)*;
    path_empty = "" %set_relative;
    path = (path_abempty | path_absolute | path_noscheme | path_rootless | path_empty);

    action save_query
    {
        m_uri->m_query = std::string(mark, fpc - mark);
        m_uri->m_queryDefined = true;
        mark = NULL;
    }
    action save_fragment
    {
        m_uri->fragment(unescape(std::string(mark, fpc - mark)));
        mark = NULL;
    }

    query = (pchar | "/" | "?")* >marku %save_query;
    fragment = (pchar | "/" | "?")* >marku %save_fragment;

    hier_part = ("//" authority path_abempty) | path_absolute | path_rootless | path_empty;

    relative_part = ("//" authority path_abempty) | path_absolute | path_noscheme | path_empty;
    relative_ref = relative_part ( "?" query )? ( "#" fragment )?;

    absolute_URI = scheme ":" hier_part ( "?" query )? ;
    # Obsolete, but referenced from HTTP, so we translate
    relative_URI = relative_part ( "?" query )?;

    URI = scheme ":" hier_part ( "?" query )? ( "#" fragment )?;
    URI_reference = URI | relative_ref;
}%%

%%{
        machine uri_parser_proper;
        include uri_parser;
        main := URI_reference;
        write data;
}%%

class URIParser : public RagelParser
{
public:
    URIParser(URI& uri)
    {
        m_uri = &uri;
        m_path = &m_uri->path;
    }

    void init()
    {
        RagelParser::init();
        %% write init;
    }

protected:
    void exec()
    {
#ifdef MSVC
#pragma warning(push)
#pragma warning(disable : 4244)
#endif
        %% write exec;
#ifdef MSVC
#pragma warning(pop)
#endif
    }

public:
    bool complete() const
    {
        return false;
    }

    bool final() const
    {
        return cs >= uri_parser_proper_first_final;
    }

    bool error() const
    {
        return cs == uri_parser_proper_error;
    }

private:
    URI *m_uri;
    URI::Path *m_path;
};

%%{
    machine uri_path_parser;
    include uri_parser;
    main := path;
    write data;
}%%
class URIPathParser : public RagelParser
{
public:
    URIPathParser(URI::Path& path)
    {
        m_path = &path;
    }

    void init()
    {
        RagelParser::init();
        %% write init;
    }

protected:
    void exec()
    {
#ifdef MSVC
#pragma warning(push)
#pragma warning(disable : 4244)
#endif
        %% write exec;
#ifdef MSVC
#pragma warning(pop)
#endif
    }

public:
    bool complete() const
    {
        return false;
    }

    bool final() const
    {
        return cs >= uri_path_parser_first_final;
    }

    bool error() const
    {
        return cs == uri_path_parser_error;
    }

private:
    URI::Path *m_path;
};

URI::URI()
{
    reset();
}

URI::URI(const std::string& uri)
{
    reset();
    *this = uri;
}

URI::URI(const char *uri)
{
    reset();
    *this = uri;
}

URI::URI(const Buffer &uri)
{
    reset();
    *this = uri;
}

URI&
URI::operator=(const std::string& uri)
{
    URIParser parser(*this);
    parser.run(uri);
    if (parser.error() || !parser.final())
        throw std::invalid_argument("uri");
    return *this;
}

URI&
URI::operator=(const Buffer &uri)
{
    URIParser parser(*this);
    parser.run(uri);
    if (parser.error() || !parser.final())
        throw std::invalid_argument("uri");
    return *this;
}

void
URI::reset()
{
    schemeDefined(false);
    authority.hostDefined(false);
    path.type = Path::RELATIVE;
    path.segments.clear();
    queryDefined(false);
    fragmentDefined(false);
}

URI::Authority::Authority()
{
    userinfoDefined(false);
    hostDefined(false);
    portDefined(false);
}

void
URI::Authority::normalize(const std::string& defaultHost, bool emptyHostValid,
    int defaultPort, bool emptyPortValid)
{
    for(size_t i = 0; i < m_host.length(); ++i)
        m_host[i] = tolower(m_host[i]);
    if (m_port == defaultPort)
        m_port = -1;
    if (m_port == -1 && !emptyPortValid)
        m_portDefined = false;
    if (m_host == defaultHost)
        m_host.clear();
    if (m_host.empty() && !emptyHostValid && !m_userinfoDefined && !m_portDefined)
        m_hostDefined = false;
}

std::string
URI::Authority::toString() const
{
    std::ostringstream os;
    os << *this;
    return os.str();
}

std::string
URI::Path::toString() const
{
    std::ostringstream os;
    os << *this;
    return os.str();
}

static int boolcmp(bool lhs, bool rhs)
{
    if (!lhs && rhs)
        return -1;
    if (lhs && !rhs)
        return 1;
    return 0;
}

int
URI::Authority::cmp(const Authority &rhs) const
{
    int x = boolcmp(m_hostDefined, rhs.m_hostDefined);
    if (x != 0) return x;
    x = strcmp(m_host.c_str(), rhs.m_host.c_str());
    if (x != 0) return x;
    x = boolcmp(m_portDefined, rhs.m_portDefined);
    if (x != 0) return x;
    x = m_port - rhs.m_port;
    if (x != 0) return x;
    x = boolcmp(m_userinfoDefined, rhs.m_userinfoDefined);
    if (x != 0) return x;
    return strcmp(m_userinfo.c_str(), rhs.m_userinfo.c_str());
}

bool
URI::Authority::operator==(const Authority &rhs) const
{
    return cmp(rhs) == 0;
}

std::ostream&
operator<<(std::ostream& os, const URI::Authority& authority)
{
    MORDOR_ASSERT(authority.hostDefined());
    if (authority.userinfoDefined()) {
        os << escape(authority.userinfo(), userinfo) << "@";
    }
    os << escape(authority.host(), host);
    if (authority.portDefined()) {
        os << ":";
        if (authority.port() > 0) {
            os << authority.port();
        }
    }
    return os;
}

URI::Path::Path()
{
    type = RELATIVE;
}

URI::Path::Path(const char *path)
{
    *this = path;
}

URI::Path::Path(const std::string& path)
{
    *this = path;
}

URI::Path&
URI::Path::operator=(const std::string& path)
{
    URIPathParser parser(*this);
    parser.run(path);
    if (parser.error() || !parser.final())
        throw std::invalid_argument("uri");
    return *this;
}

void
URI::Path::removeDotComponents()
{
    for(size_t i = 0; i < segments.size(); ++i) {
        if (segments[i] == ".") {
            if (i + 1 == segments.size()) {
                segments[i].clear();
                continue;
            } else {
                segments.erase(segments.begin() + i);
                --i;
                continue;
            }
        }
        if (segments[i] == "..") {
            if (i == 0) {
                segments.erase(segments.begin());
                --i;
                continue;
            }
            if (i + 1 == segments.size()) {
                segments.resize(segments.size() - 1);
                segments.back().clear();
                --i;
                continue;
            }
            segments.erase(segments.begin() + i - 1, segments.begin() + i + 1);
            i -= 2;
            continue;
        }
    }
}

void
URI::Path::normalize(bool emptyPathValid)
{
    removeDotComponents();
    if (segments.empty() && !emptyPathValid)
        type = ABSOLUTE;
}

void
URI::Path::merge(const Path& rhs)
{
    MORDOR_ASSERT(rhs.type == RELATIVE);
    if (!segments.empty()) {
        segments.pop_back();
        segments.insert(segments.end(), rhs.segments.begin(), rhs.segments.end());
    } else {
        segments = rhs.segments;
    }
}

URI::Path::path_serializer
URI::Path::serialize(bool schemeless) const
{
    path_serializer result;
    result.p = this;
    result.schemeless = schemeless;
    return result;
}

std::ostream&
operator<<(std::ostream& os, const URI::Path::path_serializer &p)
{
    if (p.p->segments.empty() && p.p->type == URI::Path::ABSOLUTE) {
        return os << "/";
    }
    for (size_t i = 0; i < p.p->segments.size(); ++i) {
        if (i != 0 || p.p->type == URI::Path::ABSOLUTE) {
            os << "/";
        }
        if (i == 0 && p.p->type == URI::Path::RELATIVE && p.schemeless) {
            os << escape(p.p->segments[i], segment_nc);
        } else {
            os << escape(p.p->segments[i], pchar);
        }
    }
    return os;
}

std::ostream&
operator<<(std::ostream& os, const URI::Path& path)
{
    return os << path.serialize();
}

int
URI::Path::cmp(const Path &rhs) const
{
    if (type == ABSOLUTE && rhs.type == RELATIVE)
        return -1;
    if (type == RELATIVE && rhs.type == ABSOLUTE)
        return 1;
    std::vector<std::string>::const_iterator itl, itr;
    itl = segments.begin(); itr = rhs.segments.begin();
    while (true) {
        if (itl == segments.end() && itr != rhs.segments.end())
            return -1;
        if (itl != segments.end() && itr == rhs.segments.end())
            return 1;
        if (itl == segments.end() && itr == rhs.segments.end())
            return 0;
        int x = strcmp(itl->c_str(), itr->c_str());
        if (x != 0) return x;
        ++itl; ++itr;
    }
}

bool
URI::Path::operator==(const Path &rhs) const
{
    return type == rhs.type && segments == rhs.segments;
}

void
URI::normalize()
{
    for (size_t i = 0; i < m_scheme.size(); ++i)
        m_scheme[i] = tolower(m_scheme[i]);

    if (m_scheme == "http" || m_scheme == "https") {
        authority.normalize("", false, m_scheme.size() == 4 ? 80 : 443, false);
        path.normalize();
    } else if (m_scheme == "file") {
        authority.normalize("localhost", true);
        path.normalize();
    } else {
        authority.normalize();
        path.normalize();
    }
}

std::string
URI::query() const
{
    MORDOR_ASSERT(m_queryDefined);
    return unescape(m_query);
}

void
URI::query(const std::string &q)
{
    m_queryDefined = true;
    m_query = escape(q, Mordor::query);
}

std::string
URI::toString() const
{
    std::ostringstream os;
    os << *this;
    return os.str();
}

std::ostream&
operator<<(std::ostream& os, const URI& uri)
{
    if (uri.schemeDefined()) {
        os << escape(uri.scheme(), scheme) << ":";
    }

    if (uri.authority.hostDefined()) {
        os << "//" << uri.authority;
    }

    // Has scheme, but no authority, must ensure that an absolute path
    // doesn't begin with an empty segment (or could be mistaken for authority)
    if (uri.schemeDefined() && !uri.authority.hostDefined() &&
        uri.path.type == URI::Path::ABSOLUTE &&
        uri.path.segments.size() > 0 && uri.path.segments.front().empty()) {
        os << "/";
    }
    os << uri.path.serialize(!uri.schemeDefined());

    if (uri.queryDefined()) {
        os << "?" << escape(uri.query(), query);
    }

    if (uri.fragmentDefined()) {
        os << "#" << escape(uri.fragment(), query);
    }
    return os;
}

URI
URI::transform(const URI& base, const URI& relative)
{
    MORDOR_ASSERT(base.schemeDefined());

    URI target;
    if (relative.schemeDefined()) {
        target.scheme(relative.scheme());
        target.authority = relative.authority;
        target.path = relative.path;
        target.path.removeDotComponents();
        target.m_query = relative.m_query;
        target.m_queryDefined = relative.m_queryDefined;
    } else {
        if (relative.authority.hostDefined()) {
            target.authority = relative.authority;
            target.path = relative.path;
            target.path.removeDotComponents();
            target.m_query = relative.m_query;
            target.m_queryDefined = relative.m_queryDefined;
        } else {
            if (relative.path.isEmpty()) {
                target.path = base.path;
                if (relative.queryDefined()) {
                    target.query(relative.query());
                } else {
                    target.m_query = base.m_query;
                    target.m_queryDefined = base.m_queryDefined;
                }
            } else {
                if (relative.path.type == Path::ABSOLUTE) {
                    target.path = relative.path;
                } else {
                    target.path = base.path;
                    target.path.merge(relative.path);
                    if (!base.authority.hostDefined())
                        target.path.type = Path::ABSOLUTE;
                }
                target.path.removeDotComponents();
                target.m_query = relative.m_query;
                target.m_queryDefined = relative.m_queryDefined;
            }
            target.authority = base.authority;
        }
        target.scheme(base.scheme());
    }
    target.m_fragment = relative.m_fragment;
    target.m_fragmentDefined = relative.m_fragmentDefined;
    return target;
}

int
URI::cmp(const URI &rhs) const
{
    int x = boolcmp(m_schemeDefined, rhs.m_schemeDefined);
    if (x != 0) return x;
    x = strcmp(m_scheme.c_str(), rhs.m_scheme.c_str());
    if (x != 0) return x;
    x = authority.cmp(rhs.authority);
    if (x != 0) return x;
    x = path.cmp(rhs.path);
    if (x != 0) return x;
    x = boolcmp(m_queryDefined, rhs.m_queryDefined);
    if (x != 0) return x;
    x = strcmp(m_query.c_str(), rhs.m_query.c_str());
    if (x != 0) return x;
    x = boolcmp(m_fragmentDefined, rhs.m_fragmentDefined);
    if (x != 0) return x;
    return strcmp(m_fragment.c_str(), rhs.m_fragment.c_str());
}

bool
URI::operator<(const URI &rhs) const
{
    return cmp(rhs) < 0;
}

bool
URI::operator==(const URI &rhs) const
{
    return cmp(rhs) == 0;
}

URI::QueryString &
URI::QueryString::operator =(const std::string &str)
{
    clear();
    if (str.empty())
        return *this;
    std::vector<std::string> pairs = split(str, "&;");
    for (std::vector<std::string>::iterator it = pairs.begin();
        it != pairs.end();
        ++it) {
        std::vector<std::string> keyValue = split(*it, '=', 2);
        insert(value_type(unescape(keyValue[0], true), keyValue.size() == 2 ? unescape(keyValue[1], true) : ""));
    }
    return *this;
}

std::string
URI::QueryString::toString() const
{
    std::ostringstream os;
    for (const_iterator it = begin();
        it != end();
        ++it) {
        if (it != begin()) {
            os << '&';
        }
        os << escape(it->first, Mordor::queryString, true);
        if (!it->second.empty())
            os << '=' << escape(it->second, Mordor::queryString, true);
    }
    return os.str();
}

}

// Copyright (c) 2009 - Mozy, Inc.
/* To compile to .cpp:
   ragel uri.rl -G2 -o uri.cpp
*/

#include "mordor/pch.h"

#include "mordor/uri.h"

#include <sstream>

#include "mordor/ragel.h"
#include "mordor/string.h"
#include "mordor/version.h"

namespace Mordor {

static const std::string unreserved("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-._~");
static const std::string sub_delims("!$&'()*+,;=");
static const std::string scheme("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789+-.");
static const std::string userinfo("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-._~" "!$&'()*+,;=" ":");
static const std::string host("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-._~" "!$&'()*+,;=" ":");
static const std::string pchar("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-._~" "!$&'()*+,;=" ":@");
static const std::string path("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-._~" "!$&'()*+,;=" ":@" "/");
static const std::string segment_nc("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-._~" "!$&'()*+,;=" "@");
static const std::string query("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-._~" "!$&'()*+,;=" ":@" "/?");
static const std::string queryString("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-._~" "!$'()*+," ":@" "/?");

static std::string escape(const std::string& str, const std::string& allowedChars, bool spaceAsPlus = false)
{
    const char *hexdigits = "0123456789ABCDEF";
    std::string result(str);

    const char *c = str.c_str();
    const char *end = c + str.length();
    bool differed = false;
    while(c < end)
    {
        if (allowedChars.find(*c) == std::string::npos) {
            if (!differed) {
                result.resize(c - str.c_str());
                differed = true;
            }
            if (*c == ' ' && spaceAsPlus) {
                result.append(1, '+');
            } else {
                result.append(1, '%');
                result.append(1, hexdigits[(unsigned char)*c >> 4]);
                result.append(1, hexdigits[*c & 0xf]);
            }
        } else {
            if (differed) {
                result.append(1, *c);
            }
        }
        ++c;
    }

    if (differed) {
        MORDOR_ASSERT(result.length() >= str.length());
    } else {
        MORDOR_ASSERT(result == str);
    }
    return result;
}

std::string unescape(const std::string& str, bool spaceAsPlus = false)
{
    std::string result = str;

    const char *c = str.c_str();
    const char *end = c + str.length();
    bool differed = false;
    while (c < end)
    {
        if (*c == '%') {
            MORDOR_ASSERT(c + 2 < end);
            if (!differed) {
                result.resize(c - str.c_str());
                differed = true;
            }
            char decoded;
            ++c;
            if (*c >= 'a' && *c <= 'f')
                decoded = (*c - 'a' + 10) << 4;
            else if (*c >= 'A' && *c <= 'F')
                decoded = (*c - 'A' + 10) << 4;
            else {
                MORDOR_ASSERT(*c >= '0' && *c <='9');
                decoded = (*c - '0') << 4;
            }
            ++c;
            if (*c >= 'a' && *c <= 'f')
                decoded |= *c - 'a' + 10;
            else if (*c >= 'A' && *c <= 'F')
                decoded |= *c - 'A' + 10;
            else {
                MORDOR_ASSERT(*c >= '0' && *c <='9');
                decoded |= *c - '0';
            }
            result.append(1, decoded);
        } else if (*c == '+' && spaceAsPlus) {
            if (!differed) {
                result.resize(c - str.c_str());
                differed = true;
            }
            result.append(1, ' ');
        } else if (differed) {
            result.append(1, *c);
        }
        ++c;
    }
    return result;
}

std::string
URI::encode(const std::string &str, CharacterClass charClass)
{
    switch (charClass) {
        case UNRESERVED:
            return escape(str, unreserved, false);
        case QUERYSTRING:
            return escape(str, Mordor::queryString, true);
        default:
            MORDOR_NOTREACHED();
    }
}

std::string
URI::decode(const std::string &str, CharacterClass charClass)
{
    switch (charClass) {
        case UNRESERVED:
            return unescape(str, false);
        case QUERYSTRING:
            return unescape(str, true);
        default:
            MORDOR_NOTREACHED();
    }
}

%%{
    # See RFC 3986: http://www.ietf.org/rfc/rfc3986.txt

    machine uri_parser;

    gen_delims = ":" | "/" | "?" | "#" | "[" | "]" | "@";
    sub_delims = "!" | "$" | "&" | "'" | "(" | ")" | "*" | "+" | "," | ";" | "=";
    reserved = gen_delims | sub_delims;
    unreserved = alpha | digit | "-" | "." | "_" | "~";
    pct_encoded = "%" xdigit xdigit;

    action marku { mark = fpc; }
    action save_scheme
    {
        m_uri->scheme(unescape(std::string(mark, fpc - mark)));
        mark = NULL;
    }

    scheme = (alpha | digit | "+" | "-" | ".")+ >marku %save_scheme;

    action save_port
    {
        if (fpc == mark)
            m_uri->authority.port(-1);
        else
            m_uri->authority.port(atoi(mark));
        mark = NULL;
    }
    action save_userinfo
    {
        m_uri->authority.userinfo(unescape(std::string(mark, fpc - mark)));
        mark = NULL;
    }
    action save_host
    {
        if (mark != NULL) {
            m_uri->authority.host(unescape(std::string(mark, fpc - mark)));
            mark = NULL;
        }
    }

    userinfo = (unreserved | pct_encoded | sub_delims | ":")*;
    dec_octet = digit | [1-9] digit | "1" digit{2} | 2 [0-4] digit | "25" [0-5];
    IPv4address = dec_octet "." dec_octet "." dec_octet "." dec_octet;
    h16 = xdigit{1,4};
    ls32 = (h16 ":" h16) | IPv4address;
    IPv6address = (                         (h16 ":"){6} ls32) |
                  (                    "::" (h16 ":"){5} ls32) |
                  ((             h16)? "::" (h16 ":"){4} ls32) |
                  (((h16 ":"){1} h16)? "::" (h16 ":"){3} ls32) |
                  (((h16 ":"){2} h16)? "::" (h16 ":"){2} ls32) |
                  (((h16 ":"){3} h16)? "::" (h16 ":"){1} ls32) |
                  (((h16 ":"){4} h16)? "::"              ls32) |
                  (((h16 ":"){5} h16)? "::"              h16 ) |
                  (((h16 ":"){6} h16)? "::"                  );
    IPvFuture = "v" xdigit+ "." (unreserved | sub_delims | ":")+;
    IP_literal = "[" (IPv6address | IPvFuture) "]";
    reg_name = (unreserved | pct_encoded | sub_delims)*;
    host = IP_literal | IPv4address | reg_name;
    port = digit*;

    authority = ( (userinfo %save_userinfo "@")? host >marku %save_host (":" port >marku %save_port)? ) >marku;

    action save_segment
    {
        m_path->segments.push_back(unescape(std::string(mark, fpc - mark)));
        mark = NULL;
    }

    pchar = unreserved | pct_encoded | sub_delims | ":" | "@";
    segment = pchar* >marku %save_segment;
    segment_nz = pchar+ >marku %save_segment;
    segment_nz_nc = (pchar - ":")+ >marku %save_segment;

    action set_absolute
    {
        m_path->type = URI::Path::ABSOLUTE;
    }
    action set_relative
    {
        m_path->type = URI::Path::RELATIVE;
    }

    path_abempty = ("/" segment >set_absolute)*;
    path_absolute = "/" (segment_nz ("/" segment)*)?  >set_absolute;
    path_noscheme = segment_nz_nc >set_relative ("/" segment)*;
    path_rootless = segment_nz >set_relative ("/" segment)*;
    path_empty = "" %set_relative;
    path = (path_abempty | path_absolute | path_noscheme | path_rootless | path_empty);

    action save_query
    {
        m_uri->m_query = std::string(mark, fpc - mark);
        m_uri->m_queryDefined = true;
        mark = NULL;
    }
    action save_fragment
    {
        m_uri->fragment(unescape(std::string(mark, fpc - mark)));
        mark = NULL;
    }

    query = (pchar | "/" | "?")* >marku %save_query;
    fragment = (pchar | "/" | "?")* >marku %save_fragment;

    hier_part = ("//" authority path_abempty) | path_absolute | path_rootless | path_empty;

    relative_part = ("//" authority path_abempty) | path_absolute | path_noscheme | path_empty;
    relative_ref = relative_part ( "?" query )? ( "#" fragment )?;

    absolute_URI = scheme ":" hier_part ( "?" query )? ;
    # Obsolete, but referenced from HTTP, so we translate
    relative_URI = relative_part ( "?" query )?;

    URI = scheme ":" hier_part ( "?" query )? ( "#" fragment )?;
    URI_reference = URI | relative_ref;
}%%

%%{
        machine uri_parser_proper;
        include uri_parser;
        main := URI_reference;
        write data;
}%%

class URIParser : public RagelParser
{
public:
    URIParser(URI& uri)
    {
        m_uri = &uri;
        m_path = &m_uri->path;
    }

    void init()
    {
        RagelParser::init();
        %% write init;
    }

protected:
    void exec()
    {
#ifdef MSVC
#pragma warning(push)
#pragma warning(disable : 4244)
#endif
        %% write exec;
#ifdef MSVC
#pragma warning(pop)
#endif
    }

public:
    bool complete() const
    {
        return false;
    }

    bool final() const
    {
        return cs >= uri_parser_proper_first_final;
    }

    bool error() const
    {
        return cs == uri_parser_proper_error;
    }

private:
    URI *m_uri;
    URI::Path *m_path;
};

%%{
    machine uri_path_parser;
    include uri_parser;
    main := path;
    write data;
}%%
class URIPathParser : public RagelParser
{
public:
    URIPathParser(URI::Path& path)
    {
        m_path = &path;
    }

    void init()
    {
        RagelParser::init();
        %% write init;
    }

protected:
    void exec()
    {
#ifdef MSVC
#pragma warning(push)
#pragma warning(disable : 4244)
#endif
        %% write exec;
#ifdef MSVC
#pragma warning(pop)
#endif
    }

public:
    bool complete() const
    {
        return false;
    }

    bool final() const
    {
        return cs >= uri_path_parser_first_final;
    }

    bool error() const
    {
        return cs == uri_path_parser_error;
    }

private:
    URI::Path *m_path;
};

URI::URI()
{
    reset();
}

URI::URI(const std::string& uri)
{
    reset();
    *this = uri;
}

URI::URI(const char *uri)
{
    reset();
    *this = uri;
}

URI::URI(const Buffer &uri)
{
    reset();
    *this = uri;
}

URI&
URI::operator=(const std::string& uri)
{
    URIParser parser(*this);
    parser.run(uri);
    if (parser.error() || !parser.final())
        throw std::invalid_argument("uri");
    return *this;
}

URI&
URI::operator=(const Buffer &uri)
{
    URIParser parser(*this);
    parser.run(uri);
    if (parser.error() || !parser.final())
        throw std::invalid_argument("uri");
    return *this;
}

void
URI::reset()
{
    schemeDefined(false);
    authority.hostDefined(false);
    path.type = Path::RELATIVE;
    path.segments.clear();
    queryDefined(false);
    fragmentDefined(false);
}

URI::Authority::Authority()
{
    userinfoDefined(false);
    hostDefined(false);
    portDefined(false);
}

void
URI::Authority::normalize(const std::string& defaultHost, bool emptyHostValid,
    int defaultPort, bool emptyPortValid)
{
    for(size_t i = 0; i < m_host.length(); ++i)
        m_host[i] = tolower(m_host[i]);
    if (m_port == defaultPort)
        m_port = -1;
    if (m_port == -1 && !emptyPortValid)
        m_portDefined = false;
    if (m_host == defaultHost)
        m_host.clear();
    if (m_host.empty() && !emptyHostValid && !m_userinfoDefined && !m_portDefined)
        m_hostDefined = false;
}

std::string
URI::Authority::toString() const
{
    std::ostringstream os;
    os << *this;
    return os.str();
}

std::string
URI::Path::toString() const
{
    std::ostringstream os;
    os << *this;
    return os.str();
}

static int boolcmp(bool lhs, bool rhs)
{
    if (!lhs && rhs)
        return -1;
    if (lhs && !rhs)
        return 1;
    return 0;
}

int
URI::Authority::cmp(const Authority &rhs) const
{
    int x = boolcmp(m_hostDefined, rhs.m_hostDefined);
    if (x != 0) return x;
    x = strcmp(m_host.c_str(), rhs.m_host.c_str());
    if (x != 0) return x;
    x = boolcmp(m_portDefined, rhs.m_portDefined);
    if (x != 0) return x;
    x = m_port - rhs.m_port;
    if (x != 0) return x;
    x = boolcmp(m_userinfoDefined, rhs.m_userinfoDefined);
    if (x != 0) return x;
    return strcmp(m_userinfo.c_str(), rhs.m_userinfo.c_str());
}

bool
URI::Authority::operator==(const Authority &rhs) const
{
    return cmp(rhs) == 0;
}

std::ostream&
operator<<(std::ostream& os, const URI::Authority& authority)
{
    MORDOR_ASSERT(authority.hostDefined());
    if (authority.userinfoDefined()) {
        os << escape(authority.userinfo(), userinfo) << "@";
    }
    os << escape(authority.host(), host);
    if (authority.portDefined()) {
        os << ":";
        if (authority.port() > 0) {
            os << authority.port();
        }
    }
    return os;
}

URI::Path::Path()
{
    type = RELATIVE;
}

URI::Path::Path(const char *path)
{
    *this = path;
}

URI::Path::Path(const std::string& path)
{
    *this = path;
}

URI::Path&
URI::Path::operator=(const std::string& path)
{
    URIPathParser parser(*this);
    parser.run(path);
    if (parser.error() || !parser.final())
        throw std::invalid_argument("uri");
    return *this;
}

void
URI::Path::removeDotComponents()
{
    for(size_t i = 0; i < segments.size(); ++i) {
        if (segments[i] == ".") {
            if (i + 1 == segments.size()) {
                segments[i].clear();
                continue;
            } else {
                segments.erase(segments.begin() + i);
                --i;
                continue;
            }
        }
        if (segments[i] == "..") {
            if (i == 0) {
                segments.erase(segments.begin());
                --i;
                continue;
            }
            if (i + 1 == segments.size()) {
                segments.resize(segments.size() - 1);
                segments.back().clear();
                --i;
                continue;
            }
            segments.erase(segments.begin() + i - 1, segments.begin() + i + 1);
            i -= 2;
            continue;
        }
    }
}

void
URI::Path::normalize(bool emptyPathValid)
{
    removeDotComponents();
    if (segments.empty() && !emptyPathValid)
        type = ABSOLUTE;
}

void
URI::Path::merge(const Path& rhs)
{
    MORDOR_ASSERT(rhs.type == RELATIVE);
    if (!segments.empty()) {
        segments.pop_back();
        segments.insert(segments.end(), rhs.segments.begin(), rhs.segments.end());
    } else {
        segments = rhs.segments;
    }
}

URI::Path::path_serializer
URI::Path::serialize(bool schemeless) const
{
    path_serializer result;
    result.p = this;
    result.schemeless = schemeless;
    return result;
}

std::ostream&
operator<<(std::ostream& os, const URI::Path::path_serializer &p)
{
    if (p.p->segments.empty() && p.p->type == URI::Path::ABSOLUTE) {
        return os << "/";
    }
    for (size_t i = 0; i < p.p->segments.size(); ++i) {
        if (i != 0 || p.p->type == URI::Path::ABSOLUTE) {
            os << "/";
        }
        if (i == 0 && p.p->type == URI::Path::RELATIVE && p.schemeless) {
            os << escape(p.p->segments[i], segment_nc);
        } else {
            os << escape(p.p->segments[i], pchar);
        }
    }
    return os;
}

std::ostream&
operator<<(std::ostream& os, const URI::Path& path)
{
    return os << path.serialize();
}

int
URI::Path::cmp(const Path &rhs) const
{
    if (type == ABSOLUTE && rhs.type == RELATIVE)
        return -1;
    if (type == RELATIVE && rhs.type == ABSOLUTE)
        return 1;
    std::vector<std::string>::const_iterator itl, itr;
    itl = segments.begin(); itr = rhs.segments.begin();
    while (true) {
        if (itl == segments.end() && itr != rhs.segments.end())
            return -1;
        if (itl != segments.end() && itr == rhs.segments.end())
            return 1;
        if (itl == segments.end() && itr == rhs.segments.end())
            return 0;
        int x = strcmp(itl->c_str(), itr->c_str());
        if (x != 0) return x;
        ++itl; ++itr;
    }
}

bool
URI::Path::operator==(const Path &rhs) const
{
    return type == rhs.type && segments == rhs.segments;
}

void
URI::normalize()
{
    for (size_t i = 0; i < m_scheme.size(); ++i)
        m_scheme[i] = tolower(m_scheme[i]);

    if (m_scheme == "http" || m_scheme == "https") {
        authority.normalize("", false, m_scheme.size() == 4 ? 80 : 443, false);
        path.normalize();
    } else if (m_scheme == "file") {
        authority.normalize("localhost", true);
        path.normalize();
    } else {
        authority.normalize();
        path.normalize();
    }
}

std::string
URI::query() const
{
    MORDOR_ASSERT(m_queryDefined);
    return unescape(m_query);
}

void
URI::query(const std::string &q)
{
    m_queryDefined = true;
    m_query = escape(q, Mordor::query);
}

std::string
URI::toString() const
{
    std::ostringstream os;
    os << *this;
    return os.str();
}

std::ostream&
operator<<(std::ostream& os, const URI& uri)
{
    if (uri.schemeDefined()) {
        os << escape(uri.scheme(), scheme) << ":";
    }

    if (uri.authority.hostDefined()) {
        os << "//" << uri.authority;
    }

    // Has scheme, but no authority, must ensure that an absolute path
    // doesn't begin with an empty segment (or could be mistaken for authority)
    if (uri.schemeDefined() && !uri.authority.hostDefined() &&
        uri.path.type == URI::Path::ABSOLUTE &&
        uri.path.segments.size() > 0 && uri.path.segments.front().empty()) {
        os << "/";
    }
    os << uri.path.serialize(!uri.schemeDefined());

    if (uri.queryDefined()) {
        os << "?" << escape(uri.query(), query);
    }

    if (uri.fragmentDefined()) {
        os << "#" << escape(uri.fragment(), query);
    }
    return os;
}

URI
URI::transform(const URI& base, const URI& relative)
{
    MORDOR_ASSERT(base.schemeDefined());

    URI target;
    if (relative.schemeDefined()) {
        target.scheme(relative.scheme());
        target.authority = relative.authority;
        target.path = relative.path;
        target.path.removeDotComponents();
        target.m_query = relative.m_query;
        target.m_queryDefined = relative.m_queryDefined;
    } else {
        if (relative.authority.hostDefined()) {
            target.authority = relative.authority;
            target.path = relative.path;
            target.path.removeDotComponents();
            target.m_query = relative.m_query;
            target.m_queryDefined = relative.m_queryDefined;
        } else {
            if (relative.path.isEmpty()) {
                target.path = base.path;
                if (relative.queryDefined()) {
                    target.query(relative.query());
                } else {
                    target.m_query = base.m_query;
                    target.m_queryDefined = base.m_queryDefined;
                }
            } else {
                if (relative.path.type == Path::ABSOLUTE) {
                    target.path = relative.path;
                } else {
                    target.path = base.path;
                    target.path.merge(relative.path);
                    if (!base.authority.hostDefined())
                        target.path.type = Path::ABSOLUTE;
                }
                target.path.removeDotComponents();
                target.m_query = relative.m_query;
                target.m_queryDefined = relative.m_queryDefined;
            }
            target.authority = base.authority;
        }
        target.scheme(base.scheme());
    }
    target.m_fragment = relative.m_fragment;
    target.m_fragmentDefined = relative.m_fragmentDefined;
    return target;
}

int
URI::cmp(const URI &rhs) const
{
    int x = boolcmp(m_schemeDefined, rhs.m_schemeDefined);
    if (x != 0) return x;
    x = strcmp(m_scheme.c_str(), rhs.m_scheme.c_str());
    if (x != 0) return x;
    x = authority.cmp(rhs.authority);
    if (x != 0) return x;
    x = path.cmp(rhs.path);
    if (x != 0) return x;
    x = boolcmp(m_queryDefined, rhs.m_queryDefined);
    if (x != 0) return x;
    x = strcmp(m_query.c_str(), rhs.m_query.c_str());
    if (x != 0) return x;
    x = boolcmp(m_fragmentDefined, rhs.m_fragmentDefined);
    if (x != 0) return x;
    return strcmp(m_fragment.c_str(), rhs.m_fragment.c_str());
}

bool
URI::operator<(const URI &rhs) const
{
    return cmp(rhs) < 0;
}

bool
URI::operator==(const URI &rhs) const
{
    return cmp(rhs) == 0;
}

URI::QueryString &
URI::QueryString::operator =(const std::string &str)
{
    clear();
    std::vector<std::string> pairs = split(str, "&;");
    for (std::vector<std::string>::iterator it = pairs.begin();
        it != pairs.end();
        ++it) {
        std::vector<std::string> keyValue = split(*it, '=', 2);
        insert(value_type(unescape(keyValue[0], true), keyValue.size() == 2 ? unescape(keyValue[1], true) : ""));
    }
    return *this;
}

std::string
URI::QueryString::toString() const
{
    std::ostringstream os;
    for (const_iterator it = begin();
        it != end();
        ++it) {
        if (it != begin()) {
            os << '&';
        }
        os << escape(it->first, Mordor::queryString, true);
        if (!it->second.empty())
            os << '=' << escape(it->second, Mordor::queryString, true);
    }
    return os.str();
}

}


Fixed an unintentional concatenation of two patterns due to a missing semi-colon. The error was not covered by the (meagre) test input.

/*
 * @LANG: c++
 */

#include <iostream>
#include <string.h>
using namespace std;

extern char buf[];

struct Scanner
{
	int cs, act;
	char *tokstart, *tokend;

	// Initialize the machine. Invokes any init statement blocks. Returns 0
	// if the machine begins in a non-accepting state and 1 if the machine
	// begins in an accepting state.
	void init( );

	// Execute the machine on a block of data. Returns -1 if after processing
	// the data, the machine is in the error state and can never accept, 0 if
	// the machine is in a non-accepting state and 1 if the machine is in an
	// accepting state.
	int execute( char *data, int len );

	// Indicate that there is no more data. Returns -1 if the machine finishes
	// in the error state and does not accept, 0 if the machine finishes
	// in any other non-accepting state and 1 if the machine finishes in an
	// accepting state.
	int finish( );
};

%%{
	machine Scanner;

	action to_act {
		cout << "to:   fc = ";
		if ( fc == '\'' )
			cout << (int)fc;
		else
			cout << fc;
		cout << " tokstart = " << ( tokstart == 0 ? -1 : tokstart-buf ) << endl;
	}
	action from_act {
		cout << "from: fc = ";
		if ( fc == '\'' )
			cout << (int)fc;
		else
			cout << fc;
		cout << " tokstart = " << ( tokstart == 0 ? -1 : tokstart-buf ) << endl;
	}

	c_comm := ( any* $0 '*/' @1 @{ fgoto main; } ) $~to_act $*from_act;
	cxx_comm := ( any* $0 '\n' @1 @{ fgoto main; } ) $~to_act $*from_act;

	main := |*

	# Single and double literals.
	( 'L'? "'" ( [^'\\\n] | /\\./ )* "'" ) $~ to_act $* from_act;
	( 'L'? '"' ( [^"\\\n] | /\\./ )* '"' ) $~ to_act $* from_act;

	# Identifiers
	( [a-zA-Z_] [a-zA-Z0-9_]* ) $~ to_act $* from_act;

	# Floating literals.
	fract_const = digit* '.' digit+ | digit+ '.';
	exponent = [eE] [+\-]? digit+;
	float_suffix = [flFL];

	( fract_const exponent? float_suffix? |
		digit+ exponent float_suffix? ) $~ to_act $* from_act;

	# Integer decimal. Leading part buffered by float.
	( ( '0' | [1-9] [0-9]* ) [ulUL]{0,3} ) $~ to_act $* from_act;

	# Integer octal. Leading part buffered by float.
	( '0' [0-9]+ [ulUL]{0,2} ) $~ to_act $* from_act;

	# Integer hex. Leading 0 buffered by float.
	( '0x' [0-9a-fA-F]+ [ulUL]{0,2} ) $~ to_act $* from_act;

	# Three char compounds, first item already buffered. */
	( '...' ) $~ to_act $* from_act;

	# Single char symbols.
	( punct - [_"'] ) $~ to_act $* from_act;

	# Comments and whitespace.
	( '/*' ) $~ to_act $* from_act { fgoto c_comm; };
	( '//' ) $~ to_act $* from_act { fgoto cxx_comm; };

	( any - 33..126 )+ $~ to_act $* from_act;

	*|;
}%%

%% write data;

void Scanner::init( )
{
	%% write init;
}

int Scanner::execute( char *data, int len )
{
	char *p = data;
	char *pe = data + len;

	%% write exec;

	int have = 0;
	if ( tokstart != 0 ) {
		have = pe - tokstart;
		memmove( data, tokstart, have );
	}
	return have;
}

int Scanner::finish( )
{
	%% write eof;
	if ( cs == Scanner_error )
		return -1;
	if ( cs >= Scanner_first_final )
		return 1;
	return 0;
}

void test( )
{
	int len = strlen( buf );
	Scanner scanner;

	scanner.init();
	scanner.execute( buf, len );
	if ( scanner.cs == Scanner_error ) {
		/* Machine failed before finding a token. */
		cout << "PARSE ERROR" << endl;
	}
	scanner.finish();
}

char buf[4096];

int main()
{
	strcpy( buf,
		"a b 0.98 /*\n"
		"9 */'\\''//hi\n"
		"there\n"
	);
	test();
	return 0;
}

#ifdef _____OUTPUT_____
from: fc = a tokstart = 0
to:   fc = a tokstart = 0
from: fc =   tokstart = 0
to:   fc = a tokstart = -1
from: fc =   tokstart = 1
to:   fc =   tokstart = 1
from: fc = b tokstart = 1
to:   fc =   tokstart = -1
from: fc = b tokstart = 2
to:   fc = b tokstart = 2
from: fc =   tokstart = 2
to:   fc = b tokstart = -1
from: fc =   tokstart = 3
to:   fc =   tokstart = 3
from: fc = 0 tokstart = 3
to:   fc =   tokstart = -1
from: fc = 0 tokstart = 4
to:   fc = 0 tokstart = 4
from: fc = . tokstart = 4
to:   fc = . tokstart = 4
from: fc = 9 tokstart = 4
to:   fc = 9 tokstart = 4
from: fc = 8 tokstart = 4
to:   fc = 8 tokstart = 4
from: fc =   tokstart = 4
to:   fc = 8 tokstart = -1
from: fc =   tokstart = 8
to:   fc =   tokstart = 8
from: fc = / tokstart = 8
to:   fc =   tokstart = -1
from: fc = / tokstart = 9
to:   fc = / tokstart = 9
from: fc = * tokstart = 9
to:   fc = * tokstart = -1
from: fc =
 tokstart = -1
to:   fc =
 tokstart = -1
from: fc = 9 tokstart = -1
to:   fc = 9 tokstart = -1
from: fc =   tokstart = -1
to:   fc =   tokstart = -1
from: fc = * tokstart = -1
to:   fc = * tokstart = -1
from: fc = / tokstart = -1
to:   fc = / tokstart = -1
from: fc = 39 tokstart = 16
to:   fc = 39 tokstart = 16
from: fc = \ tokstart = 16
to:   fc = \ tokstart = 16
from: fc = 39 tokstart = 16
to:   fc = 39 tokstart = 16
from: fc = 39 tokstart = 16
to:   fc = 39 tokstart = -1
from: fc = / tokstart = 20
to:   fc = / tokstart = 20
from: fc = / tokstart = 20
to:   fc = / tokstart = -1
from: fc = h tokstart = -1
to:   fc = h tokstart = -1
from: fc = i tokstart = -1
to:   fc = i tokstart = -1
from: fc =
 tokstart = -1
to:   fc =
 tokstart = -1
from: fc = t tokstart = 25
to:   fc = t tokstart = 25
from: fc = h tokstart = 25
to:   fc = h tokstart = 25
from: fc = e tokstart = 25
to:   fc = e tokstart = 25
from: fc = r tokstart = 25
to:   fc = r tokstart = 25
from: fc = e tokstart = 25
to:   fc = e tokstart = 25
from: fc =
 tokstart = 25
to:   fc = e tokstart = -1
from: fc =
 tokstart = 30
to:   fc =
 tokstart = 30
#endif

/*
 * @LANG: c++
 */

#include <iostream>
#include <string.h>
using namespace std;

extern char buf[];

struct Scanner
{
	int cs, act;
	char *tokstart, *tokend;

	// Initialize the machine. Invokes any init statement blocks. Returns 0
	// if the machine begins in a non-accepting state and 1 if the machine
	// begins in an accepting state.
	void init( );

	// Execute the machine on a block of data. Returns -1 if after processing
	// the data, the machine is in the error state and can never accept, 0 if
	// the machine is in a non-accepting state and 1 if the machine is in an
	// accepting state.
	int execute( char *data, int len );

	// Indicate that there is no more data. Returns -1 if the machine finishes
	// in the error state and does not accept, 0 if the machine finishes
	// in any other non-accepting state and 1 if the machine finishes in an
	// accepting state.
	int finish( );
};

%%{
	machine Scanner;

	action to_act {
		cout << "to:   fc = ";
		if ( fc == '\'' )
			cout << (int)fc;
		else
			cout << fc;
		cout << " tokstart = " << ( tokstart == 0 ? -1 : tokstart-buf ) << endl;
	}
	action from_act {
		cout << "from: fc = ";
		if ( fc == '\'' )
			cout << (int)fc;
		else
			cout << fc;
		cout << " tokstart = " << ( tokstart == 0 ? -1 : tokstart-buf ) << endl;
	}

	c_comm := ( any* $0 '*/' @1 @{ fgoto main; } ) $~to_act $*from_act;
	cxx_comm := ( any* $0 '\n' @1 @{ fgoto main; } ) $~to_act $*from_act;

	main := |*

	# Single and double literals.
	( 'L'? "'" ( [^'\\\n] | /\\./ )* "'" ) $~ to_act $* from_act;
	( 'L'? '"' ( [^"\\\n] | /\\./ )* '"' ) $~ to_act $* from_act;

	# Identifiers
	( [a-zA-Z_] [a-zA-Z0-9_]* ) $~ to_act $* from_act;

	# Floating literals.
	fract_const = digit* '.' digit+ | digit+ '.';
	exponent = [eE] [+\-]? digit+;
	float_suffix = [flFL];

	( fract_const exponent? float_suffix? |
		digit+ exponent float_suffix? ) $~ to_act $* from_act;

	# Integer decimal. Leading part buffered by float.
	( ( '0' | [1-9] [0-9]* ) [ulUL]{0,3} ) $~ to_act $* from_act

	# Integer octal. Leading part buffered by float.
	( '0' [0-9]+ [ulUL]{0,2} ) $~ to_act $* from_act;

	# Integer hex. Leading 0 buffered by float.
	( '0x' [0-9a-fA-F]+ [ulUL]{0,2} ) $~ to_act $* from_act;

	# Three char compounds, first item already buffered. */
	( '...' ) $~ to_act $* from_act;

	# Single char symbols.
	( punct - [_"'] ) $~ to_act $* from_act;

	# Comments and whitespace.
	( '/*' ) $~ to_act $* from_act { fgoto c_comm; };
	( '//' ) $~ to_act $* from_act { fgoto cxx_comm; };

	( any - 33..126 )+ $~ to_act $* from_act;

	*|;
}%%

%% write data;

void Scanner::init( )
{
	%% write init;
}

int Scanner::execute( char *data, int len )
{
	char *p = data;
	char *pe = data + len;

	%% write exec;

	int have = 0;
	if ( tokstart != 0 ) {
		have = pe - tokstart;
		memmove( data, tokstart, have );
	}
	return have;
}

int Scanner::finish( )
{
	%% write eof;
	if ( cs == Scanner_error )
		return -1;
	if ( cs >= Scanner_first_final )
		return 1;
	return 0;
}

void test( )
{
	int len = strlen( buf );
	Scanner scanner;

	scanner.init();
	scanner.execute( buf, len );
	if ( scanner.cs == Scanner_error ) {
		/* Machine failed before finding a token. */
		cout << "PARSE ERROR" << endl;
	}
	scanner.finish();
}

char buf[4096];

int main()
{
	strcpy( buf,
		"a b 0.98 /*\n"
		"9 */'\\''//hi\n"
		"there\n"
	);
	test();
	return 0;
}

#ifdef _____OUTPUT_____
from: fc = a tokstart = 0
to:   fc = a tokstart = 0
from: fc =   tokstart = 0
to:   fc = a tokstart = -1
from: fc =   tokstart = 1
to:   fc =   tokstart = 1
from: fc = b tokstart = 1
to:   fc =   tokstart = -1
from: fc = b tokstart = 2
to:   fc = b tokstart = 2
from: fc =   tokstart = 2
to:   fc = b tokstart = -1
from: fc =   tokstart = 3
to:   fc =   tokstart = 3
from: fc = 0 tokstart = 3
to:   fc =   tokstart = -1
from: fc = 0 tokstart = 4
to:   fc = 0 tokstart = 4
from: fc = . tokstart = 4
to:   fc = . tokstart = 4
from: fc = 9 tokstart = 4
to:   fc = 9 tokstart = 4
from: fc = 8 tokstart = 4
to:   fc = 8 tokstart = 4
from: fc =   tokstart = 4
to:   fc = 8 tokstart = -1
from: fc =   tokstart = 8
to:   fc =   tokstart = 8
from: fc = / tokstart = 8
to:   fc =   tokstart = -1
from: fc = / tokstart = 9
to:   fc = / tokstart = 9
from: fc = * tokstart = 9
to:   fc = * tokstart = -1
from: fc =
 tokstart = -1
to:   fc =
 tokstart = -1
from: fc = 9 tokstart = -1
to:   fc = 9 tokstart = -1
from: fc =   tokstart = -1
to:   fc =   tokstart = -1
from: fc = * tokstart = -1
to:   fc = * tokstart = -1
from: fc = / tokstart = -1
to:   fc = / tokstart = -1
from: fc = 39 tokstart = 16
to:   fc = 39 tokstart = 16
from: fc = \ tokstart = 16
to:   fc = \ tokstart = 16
from: fc = 39 tokstart = 16
to:   fc = 39 tokstart = 16
from: fc = 39 tokstart = 16
to:   fc = 39 tokstart = -1
from: fc = / tokstart = 20
to:   fc = / tokstart = 20
from: fc = / tokstart = 20
to:   fc = / tokstart = -1
from: fc = h tokstart = -1
to:   fc = h tokstart = -1
from: fc = i tokstart = -1
to:   fc = i tokstart = -1
from: fc =
 tokstart = -1
to:   fc =
 tokstart = -1
from: fc = t tokstart = 25
to:   fc = t tokstart = 25
from: fc = h tokstart = 25
to:   fc = h tokstart = 25
from: fc = e tokstart = 25
to:   fc = e tokstart = 25
from: fc = r tokstart = 25
to:   fc = r tokstart = 25
from: fc = e tokstart = 25
to:   fc = e tokstart = 25
from: fc =
 tokstart = 25
to:   fc = e tokstart = -1
from: fc =
 tokstart = 30
to:   fc =
 tokstart = 30
#endif


Remove the rl file

Remove the rl file

package dns

// Parse private key files

import (
    "os"
    "strings"
)

%%{
        machine k;
        write data;
}%%

// PrivateKey parses a private key file as defined in XXX.
// A map[string]string is returned with the values. All the keys are
// converted to lowercase. All values are returned as-is, except
// the algorithm [e.g. 5 (RSASHA1)] is returned as: m[algorithm] = "RSASHA1"
func (kp *Parser) PrivateKey() (m map[string]string, err os.Error) {
        m = make(map[string]string)
        var (
            k, data string
            cs, p, pe int
        )
        lines := strings.SplitAfter(string(kp.buf), "\n", -1)
        for _, data := range lines {
            cs, p, pe = 0, 0, len(data)
            mark := 0

        %%{
                action mark      { mark = p }
                action setKey    { k = strings.ToLower(data[mark:p]) }
                action setValue  { m[k] = data[mark:p] }
                action setAlg    { m[k] = strings.ToUpper(data[mark:p-1]) }

                bl = [ \t]+;
                base64any = [a-zA-Z0-9.\\/+=() ]+ >mark;
                algorithm = ( 'RSASHA1'i | 'RSASHA256'i ) >mark;
                comment = /^;/;

                key = (
                      ('Private-key-format'i)
                    | ('Algorithm'i)
                    | ('Modulus'i)
                    | ('PublicExponent'i)
                    | ('PrivateExponent'i)      # RSA, RFC ....?
                    | ('GostAsn1'i)             # GOST, RFC 5933
                    | ('PrivateKey'i)           # ECDSA, RFC xxxx (TBA)
                    | ('Prime1'i)
                    | ('Prime2'i)
                    | ('Exponent1'i)
                    | ('Exponent2'i)
                    | ('Coefficient'i)
                    | ('Created'i)
                    | ('Publish'i)
                    | ('Activate'i)
                ) >mark %setKey;

                value = ( base64any %setValue | digit+ bl '(' algorithm ')' %setAlg );

                line = ( key ': ' value | comment );
                main := ( line '\n' )*;

                write init;
                write exec;
        }%%
        }

        if cs < k_first_final {
                if p == pe {
                        //return nil, os.ErrorString("unexpected eof")
                        println("err unexp eof")
                        return m, nil
                } else {
                        //return nil, os.ErrorString(fmt.Sprintf("error at position %d", p))
                        println("err ", p, "data:", data[p])
                        return nil, nil
                }
        }
        return m, nil
}


Moved some of the CDATA docs around.

Moved some of the CDATA docs around.

%%machine lexer; # %

module Oga
  ##
  #
  class Lexer
    %% write data; # %

    # Lazy way of forwarding instance method calls used internally by Ragel to
    # their corresponding class methods.
    private_methods.grep(/^_lexer_/).each do |name|
      define_method(name) do
        return self.class.send(name)
      end

      private(name)
    end

    def initialize
      reset
    end

    def reset
      @line   = 1
      @column = 1
      @data   = nil
      @ts     = nil
      @te     = nil
      @tokens = []
    end

    def lex(data)
      @data       = data
      lexer_start = self.class.lexer_start
      eof         = data.length

      %% write init;
      %% write exec;

      tokens = @tokens

      reset

      return tokens
    end

    private

    def advance_line
      @line  += 1
      @column = 1
    end

    def advance_column(length = 1)
      @column += length
    end

    def t(type, start = @ts, stop = @te)
      value = @data[start...stop]
      token = [type, value, @line, @column]

      advance_column(value.length)

      @tokens << token
    end

    %%{
      # Use instance variables for `ts` and friends.
      access @;

      newline    = '\n';
      whitespace = [ \t];

      any_escaped = /\\./;

      smaller  = '<';
      greater  = '>';
      slash    = '/';
      bang     = '!';
      equals   = '=';
      colon    = ':';
      dash     = '-';
      lbracket = '[';
      rbracket = ']';

      s_quote = "'";
      d_quote = '"';

      # FIXME: there really should be a better way of doing this.
      text = (any - s_quote - d_quote - equals - bang - slash -
        greater - smaller - whitespace - newline - colon - dash -
        lbracket - rbracket)+;

      # DOCTYPES
      #
      # http://www.w3.org/TR/html-markup/syntax.html#doctype-syntax
      #
      # Doctypes are treated with some extra care on lexer level to make the
      # parser's life easier. If they were treated as regular text it would be
      # a pain to specify a proper doctype in Racc since it can't match on a
      # token's value (only on its type).
      #
      # Doctype parsing is also relaxed compared to the W3 specification. For
      # example, the specification defines 4 doctype formats each having
      # different rules. Because Oga doesn't really use the doctype for
      # anything we'll just slap all the formats into a single rule. Easy
      # enough.
      doctype = smaller whitespace* bang whitespace* 'DOCTYPE'i whitespace*
        'HTML'i whitespace* any* greater;

      # CDATA
      cdata_start = smaller bang lbracket 'CDATA' lbracket;
      cdata_end   = rbracket rbracket greater;

      main := |*
        whitespace => { t(:T_SPACE) };
        newline    => { t(:T_NEWLINE); advance_line };

        doctype  => { t(:T_DOCTYPE) };

        # CDATA
        #
        # http://www.w3.org/TR/html-markup/syntax.html#cdata-sections
        #
        # CDATA tags are broken up into 3 parts: the start, the content and the
        # end tag.
        #
        # In HTML CDATA tags have no meaning/are not supported. Oga does
        # support them but treats their contents as plain text.
        #
        cdata_start
          %{
            @cdata_start = p
            t(:T_CDATA_START, @ts, p)
          }

        # Consume everything except ], which is the start of the ending tag.
        (any - rbracket)+
          %{
            t(:T_TEXT, @cdata_start, p)

            @cdata_start = nil
          }

        cdata_end
          >{
            t(:T_CDATA_END, p, pe)
          };

        # General rules and actions.
        smaller  => { t(:T_SMALLER) };
        greater  => { t(:T_GREATER) };
        slash    => { t(:T_SLASH) };
        d_quote  => { t(:T_DQUOTE) };
        s_quote  => { t(:T_SQUOTE) };
        dash     => { t(:T_DASH) };
        rbracket => { t(:T_RBRACKET) };
        lbracket => { t(:T_LBRACKET) };
        colon    => { t(:T_COLON) };
        bang     => { t(:T_BANG) };
        equals   => { t(:T_EQUALS) };
        text     => { t(:T_TEXT) };
      *|;
    }%%
  end # Lexer
end # Oga

%%machine lexer; # %

module Oga
  ##
  #
  class Lexer
    %% write data; # %

    # Lazy way of forwarding instance method calls used internally by Ragel to
    # their corresponding class methods.
    private_methods.grep(/^_lexer_/).each do |name|
      define_method(name) do
        return self.class.send(name)
      end

      private(name)
    end

    def initialize
      reset
    end

    def reset
      @line   = 1
      @column = 1
      @data   = nil
      @ts     = nil
      @te     = nil
      @tokens = []
    end

    def lex(data)
      @data       = data
      lexer_start = self.class.lexer_start
      eof         = data.length

      %% write init;
      %% write exec;

      tokens = @tokens

      reset

      return tokens
    end

    private

    def advance_line
      @line  += 1
      @column = 1
    end

    def advance_column(length = 1)
      @column += length
    end

    def t(type, start = @ts, stop = @te)
      value = @data[start...stop]
      token = [type, value, @line, @column]

      advance_column(value.length)

      @tokens << token
    end

    %%{
      # Use instance variables for `ts` and friends.
      access @;

      newline    = '\n';
      whitespace = [ \t];

      any_escaped = /\\./;

      smaller  = '<';
      greater  = '>';
      slash    = '/';
      bang     = '!';
      equals   = '=';
      colon    = ':';
      dash     = '-';
      lbracket = '[';
      rbracket = ']';

      s_quote = "'";
      d_quote = '"';

      # FIXME: there really should be a better way of doing this.
      text = (any - s_quote - d_quote - equals - bang - slash -
        greater - smaller - whitespace - newline - colon - dash -
        lbracket - rbracket)+;

      # DOCTYPES
      #
      # http://www.w3.org/TR/html-markup/syntax.html#doctype-syntax
      #
      # Doctypes are treated with some extra care on lexer level to make the
      # parser's life easier. If they were treated as regular text it would be
      # a pain to specify a proper doctype in Racc since it can't match on a
      # token's value (only on its type).
      #
      # Doctype parsing is also relaxed compared to the W3 specification. For
      # example, the specification defines 4 doctype formats each having
      # different rules. Because Oga doesn't really use the doctype for
      # anything we'll just slap all the formats into a single rule. Easy
      # enough.
      doctype = smaller whitespace* bang whitespace* 'DOCTYPE'i whitespace*
        'HTML'i whitespace* any* greater;

      # CDATA
      #
      # http://www.w3.org/TR/html-markup/syntax.html#cdata-sections
      #
      # CDATA tags are broken up into 3 parts: the start, the content and the
      # end tag.
      #
      # In HTML CDATA tags have no meaning/are not supported. Oga does support
      # them but treats their contents as plain text.
      #
      cdata_start = smaller bang lbracket 'CDATA' lbracket;
      cdata_end   = rbracket rbracket greater;

      main := |*
        whitespace => { t(:T_SPACE) };
        newline    => { t(:T_NEWLINE); advance_line };

        doctype  => { t(:T_DOCTYPE) };

        # CDATA
        #
        # When processing CDATA patterns we'll emit tokens for the start tag,
        # the content and the end tag.
        #
        cdata_start
          %{
            @cdata_start = p
            t(:T_CDATA_START, @ts, p)
          }

        # Consume everything except ], which is the start of the ending tag.
        (any - rbracket)+
          %{
            t(:T_TEXT, @cdata_start, p)

            @cdata_start = nil
          }

        cdata_end
          >{
            t(:T_CDATA_END, p, pe)
          };

        # General rules and actions.
        smaller  => { t(:T_SMALLER) };
        greater  => { t(:T_GREATER) };
        slash    => { t(:T_SLASH) };
        d_quote  => { t(:T_DQUOTE) };
        s_quote  => { t(:T_SQUOTE) };
        dash     => { t(:T_DASH) };
        rbracket => { t(:T_RBRACKET) };
        lbracket => { t(:T_LBRACKET) };
        colon    => { t(:T_COLON) };
        bang     => { t(:T_BANG) };
        equals   => { t(:T_EQUALS) };
        text     => { t(:T_TEXT) };
      *|;
    }%%
  end # Lexer
end # Oga


remove hextoi - now embedded in state machine

#include "chunked_message.h"
#include <stdio.h>
#include <assert.h>

#ifndef MIN
# define MIN(a,b) (a < b ? a : b)
#endif

#define LEN (p - parser->mark)
#define REMAINING (pe - p)
%%{
  machine chunked_parser;

  action mark { parser->mark = p; }

  action add_to_chunk_size {
    parser->chunk_size *= 16;

    if( 'A' <= *p && *p <= 'F')
      parser->chunk_size += *p - 'A' + 10;
    else if( 'a' <= *p && *p <= 'f')
      parser->chunk_size += *p - 'a' + 10;
    else if( '0' <= *p && *p <= '9')
      parser->chunk_size += *p - '0';
    else
      assert(0 && "bad hex char");

  }

  action start_field {;}
  action write_field {;}
  action start_value {;}
  action write_value {;}

  action skip_data {
    //printf("chunk_size: %d\n", parser->chunk_size);
    if(parser->chunk_size > REMAINING) {
      parser->eating = 1;
      parser->chunk_handler(parser->data, p, REMAINING);
      parser->chunk_size -= REMAINING;
      fhold;
      fbreak;
    } else {
      parser->chunk_handler(parser->data, p, parser->chunk_size);
      p += parser->chunk_size;
      parser->chunk_size = 0;
      parser->eating = 0;
      fhold;
      fgoto chunk_end;
    }
  }

  include http11 "http11.rl";

  trailer = message_header*;

  nonzero_xdigit = [1-9a-fA-F];
  #chunk_ext_val   = token | quoted_string;
  chunk_ext_val   = token*;

  chunk_ext_name  = token*;
  chunk_extension = ( ";" " "* chunk_ext_name ("=" chunk_ext_val)? )*;
  last_chunk = "0"+ chunk_extension CRLF;
  chunk_size = (xdigit* nonzero_xdigit xdigit*) $add_to_chunk_size;
  chunk_end  = CRLF;
  chunk_body = any >skip_data;
  chunk_begin  = chunk_size chunk_extension CRLF;
  chunk        = chunk_begin chunk_body chunk_end;
  Chunked_Body = chunk* last_chunk trailer CRLF;

main := Chunked_Body;

}%%

%% write data;

void chunked_parser_init
  ( chunked_parser *parser
  )
{
  int cs = 0;
  %% write init;
  parser->cs = cs;
  parser->chunk_size = 0;
  parser->eating = 0;

  parser->mark = NULL;
  parser->nread = 0;
  parser->chunk_handler = NULL;
}

/** exec **/
size_t chunked_parser_execute
  ( chunked_parser *parser
  , const char *buffer
  , size_t len
  )
{
  const char *p, *pe;
  int cs = parser->cs;

  p = buffer;
  pe = buffer+len;

  if(0 < parser->chunk_size && parser->eating) {
    size_t eat = MIN(len, parser->chunk_size);
    if(eat == parser->chunk_size) {
      parser->eating = 0;
    }
    parser->chunk_handler(parser->data, p, eat);
    p += eat;
    parser->chunk_size -= eat;
    //printf("eat: %d\n", eat);
  }

  %% write exec;

  parser->cs = cs;
  parser->nread += p - buffer;

  assert(p <= pe && "buffer overflow after parsing execute");

  if(parser->mark)
    assert(parser->mark < pe && "mark is after buffer end");

  return(p - buffer);
}

int chunked_parser_has_error
  ( chunked_parser *parser
  )
{
  return parser->cs == chunked_parser_error;
}

int chunked_parser_is_finished
  ( chunked_parser *parser
  )
{
  return parser->cs == chunked_parser_first_final;
}

#ifdef UNITTEST
#include <string.h>

static chunked_parser parser;
static char test_buf[200];

void chunk_handler(void *data, const char *p, size_t len)
{
  //printf("chunk_handler: '%s'", test_buf);
  strncat(test_buf, p, len);
  //printf(" -> '%s'\n", test_buf);
}

int test_string(const char *buf)
{
  int traversed = 0;
  test_buf[0] = 0;

  chunked_parser_init(&parser);
  parser.chunk_handler = chunk_handler;
  traversed = chunked_parser_execute(&parser, buf, strlen(buf));

  if(!chunked_parser_is_finished(&parser))
    return -1;
  if(chunked_parser_has_error(&parser))
    return -2;

  return 1;
}

int test_split(const char *buf1, const char *buf2)
{
  int traversed = 0;
  test_buf[0] = 0;

  chunked_parser_init(&parser);
  parser.chunk_handler = chunk_handler;
  traversed = chunked_parser_execute(&parser, buf1, strlen(buf1));

  //printf("test_buf: %s\n", test_buf);

  if(chunked_parser_is_finished(&parser))
    return -1;
  if(chunked_parser_has_error(&parser))
    return -2;

  traversed += chunked_parser_execute(&parser, buf2, strlen(buf2));

  if(chunked_parser_has_error(&parser))
    return -3;
  if(!chunked_parser_is_finished(&parser))
    return -4;

  //printf("test_buf: %s\n", test_buf);

  return 1;
}

int main()
{
  assert(0 < test_string("5\r\nhello\r\n0\r\n\r\n"));
  assert(0 == strcmp("hello", test_buf));

  assert(0 < test_string("5\r\nhello\r\n6\r\n world\r\n0\r\n\r\n"));
  assert(0 == strcmp("hello world", test_buf));

  // with trailing headers. blech.
  assert(0 < test_string("5\r\nhello\r\n6\r\n world\r\n0\r\nVary: *\r\nContent-Type: text/plain\r\n\r\n"));
  assert(0 == strcmp("hello world", test_buf));

  // with bullshit after the length
  assert(0 < test_string("5; ihatew3;whatthefuck=aretheseparametersfor\r\nhello\r\n6; blahblah; blah\r\n world\r\n0\r\n\r\n"));
  assert(0 == strcmp("hello world", test_buf));

  assert(0 < test_split("5\r\nhello\r", "\n6\r\n world\r\n0\r\n\r\n"));
  assert(0 == strcmp("hello world", test_buf));

  assert(0 < test_split("5\r\nhello", "\r\n6\r\n world\r\n0\r\n\r\n"));
  assert(0 == strcmp("hello world", test_buf));

  assert(0 < test_split("5\r\nhel", "lo\r\n6\r\n world\r\n0\r\n\r\n"));
  assert(0 == strcmp("hello world", test_buf));

  // now work with "all your base are belong to us"
  // because it is two digits in length (0x1e = 30)

  assert(0 < test_string("1e\r\nall your base are belong to us\r\n0\r\n\r\n"));
  assert(0 == strcmp("all your base are belong to us", test_buf));

  assert(0 < test_split("1e\r\nall your", " base are belong to us\r\n0\r\n\r\n"));
  assert(0 == strcmp("all your base are belong to us", test_buf));

  assert(0 < test_split("1e\r", "\nall your base are belong to us\r\n0\r\n\r\n"));
  assert(0 == strcmp("all your base are belong to us", test_buf));

  assert(0 < test_split("1", "e\r\nall your base are belong to us\r\n0\r\n\r\n"));
  assert(0 == strcmp("all your base are belong to us", test_buf));

  printf("okay\n");
  return 0;
}

#endif

#include "chunked_message.h"
#include <stdio.h>
#include <assert.h>

#ifndef MIN
# define MIN(a,b) (a < b ? a : b)
#endif

static unsigned int hextoi
  ( const char *p
  , size_t len
  )
{
  unsigned int i, total = 0;
  int n;
  for(i = 0; i < len; i++) {
    switch(p[i]) {
      case '0': n =  0; break;
      case '1': n =  1; break;
      case '2': n =  2; break;
      case '3': n =  3; break;
      case '4': n =  4; break;
      case '5': n =  5; break;
      case '6': n =  6; break;
      case '7': n =  7; break;
      case '8': n =  8; break;
      case '9': n =  9; break;
      case 'A':
      case 'a': n = 10; break;
      case 'b':
      case 'B': n = 11; break;
      case 'c':
      case 'C': n = 12; break;
      case 'd':
      case 'D': n = 13; break;
      case 'e':
      case 'E': n = 14; break;
      case 'f':
      case 'F': n = 15; break;
      default: assert(0 && "bad hex char");
    }
    total *= 16;
    total += n;
  }
  return total;
}

#define LEN (p - parser->mark)
#define REMAINING (pe - p)
%%{
  machine chunked_parser;

  action mark { parser->mark = p; }

  action add_to_chunk_size {
    parser->chunk_size *= 16;

    if( 'A' <= *p && *p <= 'F')
      parser->chunk_size += *p - 'A' + 10;
    else if( 'a' <= *p && *p <= 'f')
      parser->chunk_size += *p - 'a' + 10;
    else if( '0' <= *p && *p <= '9')
      parser->chunk_size += *p - '0';
    else
      assert(0 && "bad hex char");

  }

  action start_field {;}
  action write_field {;}
  action start_value {;}
  action write_value {;}

  action skip_data {
    //printf("chunk_size: %d\n", parser->chunk_size);
    if(parser->chunk_size > REMAINING) {
      parser->eating = 1;
      parser->chunk_handler(parser->data, p, REMAINING);
      parser->chunk_size -= REMAINING;
      fhold;
      fbreak;
    } else {
      parser->chunk_handler(parser->data, p, parser->chunk_size);
      p += parser->chunk_size;
      parser->chunk_size = 0;
      parser->eating = 0;
      fhold;
      fgoto chunk_end;
    }
  }

  include http11 "http11.rl";

  trailer = message_header*;

  nonzero_xdigit = [1-9a-fA-F];
  #chunk_ext_val   = token | quoted_string;
  chunk_ext_val   = token*;

  chunk_ext_name  = token*;
  chunk_extension = ( ";" " "* chunk_ext_name ("=" chunk_ext_val)? )*;
  last_chunk = "0"+ chunk_extension CRLF;
  chunk_size = (xdigit* nonzero_xdigit xdigit*) $add_to_chunk_size;
  chunk_end  = CRLF;
  chunk_body = any >skip_data;
  chunk_begin  = chunk_size chunk_extension CRLF;
  chunk        = chunk_begin chunk_body chunk_end;
  Chunked_Body = chunk* last_chunk trailer CRLF;

main := Chunked_Body;

}%%

%% write data;

void chunked_parser_init
  ( chunked_parser *parser
  )
{
  int cs = 0;
  %% write init;
  parser->cs = cs;
  parser->chunk_size = 0;
  parser->eating = 0;

  parser->mark = NULL;
  parser->nread = 0;
  parser->chunk_handler = NULL;
}

/** exec **/
size_t chunked_parser_execute
  ( chunked_parser *parser
  , const char *buffer
  , size_t len
  )
{
  const char *p, *pe;
  int cs = parser->cs;

  p = buffer;
  pe = buffer+len;

  if(0 < parser->chunk_size && parser->eating) {
    size_t eat = MIN(len, parser->chunk_size);
    if(eat == parser->chunk_size) {
      parser->eating = 0;
    }
    parser->chunk_handler(parser->data, p, eat);
    p += eat;
    parser->chunk_size -= eat;
    //printf("eat: %d\n", eat);
  }

  %% write exec;

  parser->cs = cs;
  parser->nread += p - buffer;

  assert(p <= pe && "buffer overflow after parsing execute");

  if(parser->mark)
    assert(parser->mark < pe && "mark is after buffer end");

  return(p - buffer);
}

int chunked_parser_has_error
  ( chunked_parser *parser
  )
{
  return parser->cs == chunked_parser_error;
}

int chunked_parser_is_finished
  ( chunked_parser *parser
  )
{
  return parser->cs == chunked_parser_first_final;
}

#ifdef UNITTEST
#include <string.h>

static chunked_parser parser;
static char test_buf[200];

void chunk_handler(void *data, const char *p, size_t len)
{
  //printf("chunk_handler: '%s'", test_buf);
  strncat(test_buf, p, len);
  //printf(" -> '%s'\n", test_buf);
}

int test_string(const char *buf)
{
  int traversed = 0;
  test_buf[0] = 0;

  chunked_parser_init(&parser);
  parser.chunk_handler = chunk_handler;
  traversed = chunked_parser_execute(&parser, buf, strlen(buf));

  if(!chunked_parser_is_finished(&parser))
    return -1;
  if(chunked_parser_has_error(&parser))
    return -2;

  return 1;
}

int test_split(const char *buf1, const char *buf2)
{
  int traversed = 0;
  test_buf[0] = 0;

  chunked_parser_init(&parser);
  parser.chunk_handler = chunk_handler;
  traversed = chunked_parser_execute(&parser, buf1, strlen(buf1));

  //printf("test_buf: %s\n", test_buf);

  if(chunked_parser_is_finished(&parser))
    return -1;
  if(chunked_parser_has_error(&parser))
    return -2;

  traversed += chunked_parser_execute(&parser, buf2, strlen(buf2));

  if(chunked_parser_has_error(&parser))
    return -3;
  if(!chunked_parser_is_finished(&parser))
    return -4;

  //printf("test_buf: %s\n", test_buf);

  return 1;
}

int main()
{
  assert(3416 == hextoi("d58", 3));
  assert(0 == hextoi("0", 1));
  assert(2748 == hextoi("ABC", 3));

  assert(0 < test_string("5\r\nhello\r\n0\r\n\r\n"));
  assert(0 == strcmp("hello", test_buf));

  assert(0 < test_string("5\r\nhello\r\n6\r\n world\r\n0\r\n\r\n"));
  assert(0 == strcmp("hello world", test_buf));

  // with trailing headers. blech.
  assert(0 < test_string("5\r\nhello\r\n6\r\n world\r\n0\r\nVary: *\r\nContent-Type: text/plain\r\n\r\n"));
  assert(0 == strcmp("hello world", test_buf));

  // with bullshit after the length
  assert(0 < test_string("5; ihatew3;whatthefuck=aretheseparametersfor\r\nhello\r\n6; blahblah; blah\r\n world\r\n0\r\n\r\n"));
  assert(0 == strcmp("hello world", test_buf));

  assert(0 < test_split("5\r\nhello\r", "\n6\r\n world\r\n0\r\n\r\n"));
  assert(0 == strcmp("hello world", test_buf));

  assert(0 < test_split("5\r\nhello", "\r\n6\r\n world\r\n0\r\n\r\n"));
  assert(0 == strcmp("hello world", test_buf));

  assert(0 < test_split("5\r\nhel", "lo\r\n6\r\n world\r\n0\r\n\r\n"));
  assert(0 == strcmp("hello world", test_buf));

  // now work with "all your base are belong to us"
  // because it is two digits in length (0x1e = 30)

  assert(0 < test_string("1e\r\nall your base are belong to us\r\n0\r\n\r\n"));
  assert(0 == strcmp("all your base are belong to us", test_buf));

  assert(0 < test_split("1e\r\nall your", " base are belong to us\r\n0\r\n\r\n"));
  assert(0 == strcmp("all your base are belong to us", test_buf));

  assert(0 < test_split("1e\r", "\nall your base are belong to us\r\n0\r\n\r\n"));
  assert(0 == strcmp("all your base are belong to us", test_buf));

  assert(0 < test_split("1", "e\r\nall your base are belong to us\r\n0\r\n\r\n"));
  assert(0 == strcmp("all your base are belong to us", test_buf));

  printf("okay\n");
  return 0;
}

#endif


Lexer support for JRuby.

Lexer support for JRuby.

JRuby now passes all tests. Benchmark wise it completes the big XML benchmark
in about 500-600 milliseconds.

package org.liboga.xml;

%%machine java_lexer;

import java.io.IOException;

import org.jcodings.Encoding;

import org.jruby.Ruby;
import org.jruby.RubyModule;
import org.jruby.RubyClass;
import org.jruby.RubyObject;
import org.jruby.RubyString;
import org.jruby.util.ByteList;
import org.jruby.anno.JRubyClass;
import org.jruby.anno.JRubyMethod;
import org.jruby.runtime.ThreadContext;
import org.jruby.runtime.ObjectAllocator;
import org.jruby.runtime.builtin.IRubyObject;

@JRubyClass(name="Oga::XML::Lexer", parent="Object")
public class Lexer extends RubyObject
{
    private Ruby runtime;

    %% write data;

    public static void load(Ruby runtime)
    {
        RubyModule xml = (RubyModule) runtime.getModule("Oga")
            .getConstant("XML");

        RubyClass lexer = xml.defineClassUnder(
            "Lexer",
            runtime.getObject(),
            ALLOCATOR
        );

        lexer.defineAnnotatedMethods(Lexer.class);
    }

    private static final ObjectAllocator ALLOCATOR = new ObjectAllocator()
    {
        public IRubyObject allocate(Ruby runtime, RubyClass klass)
        {
            return new org.liboga.xml.Lexer(runtime, klass);
        }
    };

    public Lexer(Ruby runtime, RubyClass klass)
    {
        super(runtime, klass);

        this.runtime = runtime;
    }

    @JRubyMethod
    public IRubyObject advance_native(ThreadContext context)
    {
        // Pull the data in from Ruby land.
        RubyString rb_str = (RubyString) this.getInstanceVariable("@data");
        Encoding encoding = rb_str.getEncoding();

        byte[] data = rb_str.getBytes();

        int act = 0;
        int cs  = 0;
        int ts  = 0;
        int te  = 0;
        int p   = 0;
        int pe  = data.length;
        int eof = data.length;
        int top = 0;

        int[] stack = new int[8];

        %% write init;
        %% write exec;

        return context.nil;
    }

    public void callback(String name, byte[] data, Encoding enc, int ts, int te)
    {
        ByteList bytelist = new ByteList(data, ts, te - ts, enc, true);

        RubyString value = this.runtime.newString(bytelist);

        ThreadContext context = this.runtime.getCurrentContext();

        this.callMethod(context, name, value);
    }

    public void callback_simple(String name)
    {
        ThreadContext context = this.runtime.getCurrentContext();

        this.callMethod(context, name);
    }
}

%%{
    include base_lexer "base_lexer.rl";
}%%

package org.liboga.xml;

%%machine java_lexer;

import java.io.IOException;

import org.jcodings.Encoding;

import org.jruby.Ruby;
import org.jruby.RubyModule;
import org.jruby.RubyClass;
import org.jruby.RubyObject;
import org.jruby.RubyString;
import org.jruby.anno.JRubyClass;
import org.jruby.anno.JRubyMethod;
import org.jruby.runtime.ThreadContext;
import org.jruby.runtime.ObjectAllocator;
import org.jruby.runtime.builtin.IRubyObject;

@JRubyClass(name="Oga::XML::Lexer", parent="Object")
public class Lexer extends RubyObject
{
    %% write data;

    public static void load(Ruby runtime)
    {
        RubyModule xml = (RubyModule) runtime.getModule("Oga")
            .getConstant("XML");

        RubyClass lexer = xml.defineClassUnder(
            "Lexer",
            runtime.getObject(),
            ALLOCATOR
        );

        lexer.defineAnnotatedMethods(Lexer.class);
    }

    private static final ObjectAllocator ALLOCATOR = new ObjectAllocator()
    {
        public IRubyObject allocate(Ruby runtime, RubyClass klass)
        {
            return new org.liboga.xml.Lexer(runtime, klass);
        }
    };

    public Lexer(Ruby runtime, RubyClass klass)
    {
        super(runtime, klass);
    }

    @JRubyMethod
    public IRubyObject advance_native(ThreadContext context)
    {
        // Pull the data in from Ruby land.
        RubyString rb_str = (RubyString) this.getInstanceVariable("@data");
        Encoding encoding = rb_str.getEncoding();

        byte[] data = rb_str.getBytes();

        int act = 0;
        int cs  = 0;
        int ts  = 0;
        int te  = 0;
        int p   = 0;
        int pe  = data.length;
        int eof = 0;
        int top = 0;

        int[] stack = new int[8];

        %% write init;
        %% write exec;

        return context.nil;
    }

    public void callback(String name, byte[] data, Encoding enc, int ts, int te)
    {

    }

    public void callback_simple(String name)
    {

    }
}

%%{
    include base_lexer "base_lexer.rl";
}%%