DOH

DOH

Right concept, wrong method name

// Copyright (c) 2009 - Mozy, Inc.
/* To compile to .cpp:
   ragel uri.rl -G2 -o uri.cpp
*/

#include "mordor/pch.h"

#include "mordor/uri.h"

#include <sstream>

#include "mordor/ragel.h"
#include "mordor/string.h"
#include "mordor/version.h"

namespace Mordor {

static const std::string unreserved("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-._~");
static const std::string sub_delims("!$&'()*+,;=");
static const std::string scheme("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789+-.");
static const std::string userinfo("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-._~" "!$&'()*+,;=" ":");
static const std::string host("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-._~" "!$&'()*+,;=" ":");
static const std::string pchar("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-._~" "!$&'()*+,;=" ":@");
static const std::string path("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-._~" "!$&'()*+,;=" ":@" "/");
static const std::string segment_nc("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-._~" "!$&'()*+,;=" "@");
static const std::string query("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-._~" "!$&'()*+,;=" ":@" "/?");
static const std::string queryString("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-._~" "!$'()*," ":@" "/?");

static std::string escape(const std::string& str, const std::string& allowedChars, bool spaceAsPlus = false)
{
    const char *hexdigits = "0123456789ABCDEF";
    std::string result(str);

    const char *c = str.c_str();
    const char *end = c + str.length();
    bool differed = false;
    while(c < end)
    {
        if (allowedChars.find(*c) == std::string::npos) {
            if (!differed) {
                result.resize(c - str.c_str());
                differed = true;
            }
            if (*c == ' ' && spaceAsPlus) {
                result.append(1, '+');
            } else {
                result.append(1, '%');
                result.append(1, hexdigits[(unsigned char)*c >> 4]);
                result.append(1, hexdigits[*c & 0xf]);
            }
        } else {
            if (differed) {
                result.append(1, *c);
            }
        }
        ++c;
    }

    if (differed) {
        MORDOR_ASSERT(result.length() >= str.length());
    } else {
        MORDOR_ASSERT(result == str);
    }
    return result;
}

std::string unescape(const std::string& str, bool spaceAsPlus = false)
{
    std::string result = str;

    const char *c = str.c_str();
    const char *end = c + str.length();
    bool differed = false;
    while (c < end)
    {
        if (*c == '%') {
            MORDOR_ASSERT(c + 2 < end);
            if (!differed) {
                result.resize(c - str.c_str());
                differed = true;
            }
            char decoded;
            ++c;
            if (*c >= 'a' && *c <= 'f')
                decoded = (*c - 'a' + 10) << 4;
            else if (*c >= 'A' && *c <= 'F')
                decoded = (*c - 'A' + 10) << 4;
            else {
                MORDOR_ASSERT(*c >= '0' && *c <='9');
                decoded = (*c - '0') << 4;
            }
            ++c;
            if (*c >= 'a' && *c <= 'f')
                decoded |= *c - 'a' + 10;
            else if (*c >= 'A' && *c <= 'F')
                decoded |= *c - 'A' + 10;
            else {
                MORDOR_ASSERT(*c >= '0' && *c <='9');
                decoded |= *c - '0';
            }
            result.append(1, decoded);
        } else if (*c == '+' && spaceAsPlus) {
            if (!differed) {
                result.resize(c - str.c_str());
                differed = true;
            }
            result.append(1, ' ');
        } else if (differed) {
            result.append(1, *c);
        }
        ++c;
    }
    return result;
}

std::string
URI::encode(const std::string &str, CharacterClass charClass)
{
    switch (charClass) {
        case UNRESERVED:
            return escape(str, unreserved, false);
        case QUERYSTRING:
            return escape(str, Mordor::queryString, true);
        default:
            MORDOR_NOTREACHED();
    }
}

std::string
URI::decode(const std::string &str, CharacterClass charClass)
{
    switch (charClass) {
        case UNRESERVED:
            return unescape(str, false);
        case QUERYSTRING:
            return unescape(str, true);
        default:
            MORDOR_NOTREACHED();
    }
}

%%{
    # See RFC 3986: http://www.ietf.org/rfc/rfc3986.txt

    machine uri_parser;

    gen_delims = ":" | "/" | "?" | "#" | "[" | "]" | "@";
    sub_delims = "!" | "$" | "&" | "'" | "(" | ")" | "*" | "+" | "," | ";" | "=";
    reserved = gen_delims | sub_delims;
    unreserved = alpha | digit | "-" | "." | "_" | "~";
    pct_encoded = "%" xdigit xdigit;

    action marku { mark = fpc; }
    action save_scheme
    {
        m_uri->scheme(unescape(std::string(mark, fpc - mark)));
        mark = NULL;
    }

    scheme = (alpha | digit | "+" | "-" | ".")+ >marku %save_scheme;

    action save_port
    {
        if (fpc == mark)
            m_uri->authority.port(-1);
        else
            m_uri->authority.port(atoi(mark));
        mark = NULL;
    }
    action save_userinfo
    {
        m_uri->authority.userinfo(unescape(std::string(mark, fpc - mark)));
        mark = NULL;
    }
    action save_host
    {
        if (mark != NULL) {
            m_uri->authority.host(unescape(std::string(mark, fpc - mark)));
            mark = NULL;
        }
    }

    userinfo = (unreserved | pct_encoded | sub_delims | ":")*;
    dec_octet = digit | [1-9] digit | "1" digit{2} | 2 [0-4] digit | "25" [0-5];
    IPv4address = dec_octet "." dec_octet "." dec_octet "." dec_octet;
    h16 = xdigit{1,4};
    ls32 = (h16 ":" h16) | IPv4address;
    IPv6address = (                         (h16 ":"){6} ls32) |
                  (                    "::" (h16 ":"){5} ls32) |
                  ((             h16)? "::" (h16 ":"){4} ls32) |
                  (((h16 ":"){1} h16)? "::" (h16 ":"){3} ls32) |
                  (((h16 ":"){2} h16)? "::" (h16 ":"){2} ls32) |
                  (((h16 ":"){3} h16)? "::" (h16 ":"){1} ls32) |
                  (((h16 ":"){4} h16)? "::"              ls32) |
                  (((h16 ":"){5} h16)? "::"              h16 ) |
                  (((h16 ":"){6} h16)? "::"                  );
    IPvFuture = "v" xdigit+ "." (unreserved | sub_delims | ":")+;
    IP_literal = "[" (IPv6address | IPvFuture) "]";
    reg_name = (unreserved | pct_encoded | sub_delims)*;
    host = IP_literal | IPv4address | reg_name;
    port = digit*;

    authority = ( (userinfo %save_userinfo "@")? host >marku %save_host (":" port >marku %save_port)? ) >marku;

    action save_segment
    {
        m_path->segments.push_back(unescape(std::string(mark, fpc - mark)));
        mark = NULL;
    }

    pchar = unreserved | pct_encoded | sub_delims | ":" | "@";
    segment = pchar* >marku %save_segment;
    segment_nz = pchar+ >marku %save_segment;
    segment_nz_nc = (pchar - ":")+ >marku %save_segment;

    action set_absolute
    {
        m_path->type = URI::Path::ABSOLUTE;
    }
    action set_relative
    {
        m_path->type = URI::Path::RELATIVE;
    }

    path_abempty = ("/" segment >set_absolute)*;
    path_absolute = "/" (segment_nz ("/" segment)*)?  >set_absolute;
    path_noscheme = segment_nz_nc >set_relative ("/" segment)*;
    path_rootless = segment_nz >set_relative ("/" segment)*;
    path_empty = "" %set_relative;
    path = (path_abempty | path_absolute | path_noscheme | path_rootless | path_empty);

    action save_query
    {
        m_uri->m_query = std::string(mark, fpc - mark);
        m_uri->m_queryDefined = true;
        mark = NULL;
    }
    action save_fragment
    {
        m_uri->fragment(unescape(std::string(mark, fpc - mark)));
        mark = NULL;
    }

    query = (pchar | "/" | "?")* >marku %save_query;
    fragment = (pchar | "/" | "?")* >marku %save_fragment;

    hier_part = ("//" authority path_abempty) | path_absolute | path_rootless | path_empty;

    relative_part = ("//" authority path_abempty) | path_absolute | path_noscheme | path_empty;
    relative_ref = relative_part ( "?" query )? ( "#" fragment )?;

    absolute_URI = scheme ":" hier_part ( "?" query )? ;
    # Obsolete, but referenced from HTTP, so we translate
    relative_URI = relative_part ( "?" query )?;

    URI = scheme ":" hier_part ( "?" query )? ( "#" fragment )?;
    URI_reference = URI | relative_ref;
}%%

%%{
        machine uri_parser_proper;
        include uri_parser;
        main := URI_reference;
        write data;
}%%

class URIParser : public RagelParser
{
public:
    URIParser(URI& uri)
    {
        m_uri = &uri;
        m_path = &m_uri->path;
    }

    void init()
    {
        RagelParser::init();
        %% write init;
    }

protected:
    void exec()
    {
#ifdef MSVC
#pragma warning(push)
#pragma warning(disable : 4244)
#endif
        %% write exec;
#ifdef MSVC
#pragma warning(pop)
#endif
    }

public:
    bool complete() const
    {
        return false;
    }

    bool final() const
    {
        return cs >= uri_parser_proper_first_final;
    }

    bool error() const
    {
        return cs == uri_parser_proper_error;
    }

private:
    URI *m_uri;
    URI::Path *m_path;
};

%%{
    machine uri_path_parser;
    include uri_parser;
    main := path;
    write data;
}%%
class URIPathParser : public RagelParser
{
public:
    URIPathParser(URI::Path& path)
    {
        m_path = &path;
    }

    void init()
    {
        RagelParser::init();
        %% write init;
    }

protected:
    void exec()
    {
#ifdef MSVC
#pragma warning(push)
#pragma warning(disable : 4244)
#endif
        %% write exec;
#ifdef MSVC
#pragma warning(pop)
#endif
    }

public:
    bool complete() const
    {
        return false;
    }

    bool final() const
    {
        return cs >= uri_path_parser_first_final;
    }

    bool error() const
    {
        return cs == uri_path_parser_error;
    }

private:
    URI::Path *m_path;
};

URI::URI()
{
    reset();
}

URI::URI(const std::string& uri)
{
    reset();
    *this = uri;
}

URI::URI(const char *uri)
{
    reset();
    *this = uri;
}

URI::URI(const Buffer &uri)
{
    reset();
    *this = uri;
}

URI&
URI::operator=(const std::string& uri)
{
    reset();
    URIParser parser(*this);
    parser.run(uri);
    if (parser.error() || !parser.final())
        throw std::invalid_argument("uri");
    return *this;
}

URI&
URI::operator=(const Buffer &uri)
{
    reset();
    URIParser parser(*this);
    parser.run(uri);
    if (parser.error() || !parser.final())
        throw std::invalid_argument("uri");
    return *this;
}

void
URI::reset()
{
    schemeDefined(false);
    authority.hostDefined(false);
    path.type = Path::RELATIVE;
    path.segments.clear();
    queryDefined(false);
    fragmentDefined(false);
}

URI::Authority::Authority()
{
    userinfoDefined(false);
    hostDefined(false);
    portDefined(false);
}

void
URI::Authority::normalize(const std::string& defaultHost, bool emptyHostValid,
    int defaultPort, bool emptyPortValid)
{
    for(size_t i = 0; i < m_host.length(); ++i)
        m_host[i] = tolower(m_host[i]);
    if (m_port == defaultPort)
        m_port = -1;
    if (m_port == -1 && !emptyPortValid)
        m_portDefined = false;
    if (m_host == defaultHost)
        m_host.clear();
    if (m_host.empty() && !emptyHostValid && !m_userinfoDefined && !m_portDefined)
        m_hostDefined = false;
}

std::string
URI::Authority::toString() const
{
    std::ostringstream os;
    os << *this;
    return os.str();
}

std::string
URI::Path::toString() const
{
    std::ostringstream os;
    os << *this;
    return os.str();
}

static int boolcmp(bool lhs, bool rhs)
{
    if (!lhs && rhs)
        return -1;
    if (lhs && !rhs)
        return 1;
    return 0;
}

int
URI::Authority::cmp(const Authority &rhs) const
{
    int x = boolcmp(m_hostDefined, rhs.m_hostDefined);
    if (x != 0) return x;
    x = strcmp(m_host.c_str(), rhs.m_host.c_str());
    if (x != 0) return x;
    x = boolcmp(m_portDefined, rhs.m_portDefined);
    if (x != 0) return x;
    x = m_port - rhs.m_port;
    if (x != 0) return x;
    x = boolcmp(m_userinfoDefined, rhs.m_userinfoDefined);
    if (x != 0) return x;
    return strcmp(m_userinfo.c_str(), rhs.m_userinfo.c_str());
}

bool
URI::Authority::operator==(const Authority &rhs) const
{
    return cmp(rhs) == 0;
}

std::ostream&
operator<<(std::ostream& os, const URI::Authority& authority)
{
    MORDOR_ASSERT(authority.hostDefined());
    if (authority.userinfoDefined()) {
        os << escape(authority.userinfo(), userinfo) << "@";
    }
    os << escape(authority.host(), host);
    if (authority.portDefined()) {
        os << ":";
        if (authority.port() > 0) {
            os << authority.port();
        }
    }
    return os;
}

URI::Path::Path()
{
    type = RELATIVE;
}

URI::Path::Path(const char *path)
{
    *this = path;
}

URI::Path::Path(const std::string& path)
{
    *this = path;
}

URI::Path&
URI::Path::operator=(const std::string& path)
{
    type = RELATIVE;
    segments.clear();
    URIPathParser parser(*this);
    parser.run(path);
    if (parser.error() || !parser.final())
        throw std::invalid_argument("uri");
    return *this;
}

void
URI::Path::removeDotComponents()
{
    for(size_t i = 0; i < segments.size(); ++i) {
        if (segments[i] == ".") {
            if (i + 1 == segments.size()) {
                segments[i].clear();
                continue;
            } else {
                segments.erase(segments.begin() + i);
                --i;
                continue;
            }
        }
        if (segments[i] == "..") {
            if (i == 0) {
                segments.erase(segments.begin());
                --i;
                continue;
            }
            if (i + 1 == segments.size()) {
                segments.resize(segments.size() - 1);
                segments.back().clear();
                --i;
                continue;
            }
            segments.erase(segments.begin() + i - 1, segments.begin() + i + 1);
            i -= 2;
            continue;
        }
    }
}

void
URI::Path::normalize(bool emptyPathValid)
{
    removeDotComponents();
    if (segments.empty() && !emptyPathValid)
        type = ABSOLUTE;
}

void
URI::Path::merge(const Path& rhs)
{
    MORDOR_ASSERT(rhs.type == RELATIVE);
    if (!segments.empty()) {
        segments.pop_back();
        segments.insert(segments.end(), rhs.segments.begin(), rhs.segments.end());
    } else {
        segments = rhs.segments;
    }
}

URI::Path::path_serializer
URI::Path::serialize(bool schemeless) const
{
    path_serializer result;
    result.p = this;
    result.schemeless = schemeless;
    return result;
}

std::ostream&
operator<<(std::ostream& os, const URI::Path::path_serializer &p)
{
    if (p.p->segments.empty() && p.p->type == URI::Path::ABSOLUTE) {
        return os << "/";
    }
    for (size_t i = 0; i < p.p->segments.size(); ++i) {
        if (i != 0 || p.p->type == URI::Path::ABSOLUTE) {
            os << "/";
        }
        if (i == 0 && p.p->type == URI::Path::RELATIVE && p.schemeless) {
            os << escape(p.p->segments[i], segment_nc);
        } else {
            os << escape(p.p->segments[i], pchar);
        }
    }
    return os;
}

std::ostream&
operator<<(std::ostream& os, const URI::Path& path)
{
    return os << path.serialize();
}

int
URI::Path::cmp(const Path &rhs) const
{
    if (type == ABSOLUTE && rhs.type == RELATIVE)
        return -1;
    if (type == RELATIVE && rhs.type == ABSOLUTE)
        return 1;
    std::vector<std::string>::const_iterator itl, itr;
    itl = segments.begin(); itr = rhs.segments.begin();
    while (true) {
        if (itl == segments.end() && itr != rhs.segments.end())
            return -1;
        if (itl != segments.end() && itr == rhs.segments.end())
            return 1;
        if (itl == segments.end() && itr == rhs.segments.end())
            return 0;
        int x = strcmp(itl->c_str(), itr->c_str());
        if (x != 0) return x;
        ++itl; ++itr;
    }
}

bool
URI::Path::operator==(const Path &rhs) const
{
    return type == rhs.type && segments == rhs.segments;
}

void
URI::normalize()
{
    for (size_t i = 0; i < m_scheme.size(); ++i)
        m_scheme[i] = tolower(m_scheme[i]);

    if (m_scheme == "http" || m_scheme == "https") {
        authority.normalize("", false, m_scheme.size() == 4 ? 80 : 443, false);
        path.normalize();
    } else if (m_scheme == "file") {
        authority.normalize("localhost", true);
        path.normalize();
    } else {
        authority.normalize();
        path.normalize();
    }
}

std::string
URI::query() const
{
    MORDOR_ASSERT(m_queryDefined);
    return unescape(m_query);
}

void
URI::query(const std::string &q)
{
    m_queryDefined = true;
    m_query = escape(q, Mordor::query);
}

std::string
URI::toString() const
{
    std::ostringstream os;
    os << *this;
    return os.str();
}

std::ostream&
operator<<(std::ostream& os, const URI& uri)
{
    if (uri.schemeDefined()) {
        os << escape(uri.scheme(), scheme) << ":";
    }

    if (uri.authority.hostDefined()) {
        os << "//" << uri.authority;
    }

    // Has scheme, but no authority, must ensure that an absolute path
    // doesn't begin with an empty segment (or could be mistaken for authority)
    if (uri.schemeDefined() && !uri.authority.hostDefined() &&
        uri.path.type == URI::Path::ABSOLUTE &&
        uri.path.segments.size() > 0 && uri.path.segments.front().empty()) {
        os << "/";
    }
    os << uri.path.serialize(!uri.schemeDefined());

    if (uri.queryDefined()) {
        os << "?" << uri.m_query;
    }

    if (uri.fragmentDefined()) {
        os << "#" << escape(uri.fragment(), query);
    }
    return os;
}

URI
URI::transform(const URI& base, const URI& relative)
{
    MORDOR_ASSERT(base.schemeDefined());

    URI target;
    if (relative.schemeDefined()) {
        target.scheme(relative.scheme());
        target.authority = relative.authority;
        target.path = relative.path;
        target.path.removeDotComponents();
        target.m_query = relative.m_query;
        target.m_queryDefined = relative.m_queryDefined;
    } else {
        if (relative.authority.hostDefined()) {
            target.authority = relative.authority;
            target.path = relative.path;
            target.path.removeDotComponents();
            target.m_query = relative.m_query;
            target.m_queryDefined = relative.m_queryDefined;
        } else {
            if (relative.path.isEmpty()) {
                target.path = base.path;
                if (relative.queryDefined()) {
                    target.query(relative.query());
                } else {
                    target.m_query = base.m_query;
                    target.m_queryDefined = base.m_queryDefined;
                }
            } else {
                if (relative.path.type == Path::ABSOLUTE) {
                    target.path = relative.path;
                } else {
                    target.path = base.path;
                    target.path.merge(relative.path);
                    if (!base.authority.hostDefined())
                        target.path.type = Path::ABSOLUTE;
                }
                target.path.removeDotComponents();
                target.m_query = relative.m_query;
                target.m_queryDefined = relative.m_queryDefined;
            }
            target.authority = base.authority;
        }
        target.scheme(base.scheme());
    }
    target.m_fragment = relative.m_fragment;
    target.m_fragmentDefined = relative.m_fragmentDefined;
    return target;
}

int
URI::cmp(const URI &rhs) const
{
    int x = boolcmp(m_schemeDefined, rhs.m_schemeDefined);
    if (x != 0) return x;
    x = strcmp(m_scheme.c_str(), rhs.m_scheme.c_str());
    if (x != 0) return x;
    x = authority.cmp(rhs.authority);
    if (x != 0) return x;
    x = path.cmp(rhs.path);
    if (x != 0) return x;
    x = boolcmp(m_queryDefined, rhs.m_queryDefined);
    if (x != 0) return x;
    x = strcmp(m_query.c_str(), rhs.m_query.c_str());
    if (x != 0) return x;
    x = boolcmp(m_fragmentDefined, rhs.m_fragmentDefined);
    if (x != 0) return x;
    return strcmp(m_fragment.c_str(), rhs.m_fragment.c_str());
}

bool
URI::operator<(const URI &rhs) const
{
    return cmp(rhs) < 0;
}

bool
URI::operator==(const URI &rhs) const
{
    return cmp(rhs) == 0;
}

%%{
    machine querystring_parser;

    action mark { mark = fpc; }
    action saveKey {
        m_iterator = m_qs.insert(std::make_pair(
            unescape(std::string(mark, fpc - mark), true), std::string()));
        mark = NULL;
    }
    action saveValue {
        MORDOR_ASSERT(m_iterator != m_qs.end());
        if (fpc - mark == 0 && m_iterator->first.empty())
            m_qs.erase(m_iterator);
        else
            m_iterator->second = unescape(std::string(mark, fpc - mark), true);
        m_iterator = m_qs.end();
        mark = NULL;
    }
    action saveNoValue {
        if (m_iterator != m_qs.end() && m_iterator->first.empty()) {
            m_qs.erase(m_iterator);
            mark = NULL;
        }
    }

    sub_delims = "!" | "$" | "&" | "'" | "(" | ")" | "*" | "+" | "," | ";";
    unreserved = alpha | digit | "-" | "." | "_" | "~";
    pct_encoded = "%" xdigit xdigit;
    pchar = unreserved | pct_encoded | sub_delims | ":" | "@";
    querychar = (pchar | "/" | "?") -- '&' -- ';';
    key = querychar*;
    value = (querychar | '=')*;
    keyValue = key >mark %saveKey ('=' value >mark %saveValue)? %saveNoValue;
    main := keyValue? ( ('&' | ';') keyValue? )*;
    write data;
}%%

class QueryStringParser : public RagelParser
{
public:
    QueryStringParser(URI::QueryString &qs)
    : m_qs(qs),
      m_iterator(m_qs.end())
    {}

    void init()
    {
        RagelParser::init();
        %% write init;
    }

    void exec()
    {
#ifdef MSVC
#pragma warning(push)
#pragma warning(disable : 4244)
#endif
        %% write exec;
#ifdef MSVC
#pragma warning(pop)
#endif
    }

    bool complete() const { return false; }
    bool final() const
    {
        return cs >= querystring_parser_first_final;
    }

    bool error() const
    {
        return cs == querystring_parser_error;
    }

private:
    URI::QueryString &m_qs;
    URI::QueryString::iterator m_iterator;
};

URI::QueryString &
URI::QueryString::operator =(const std::string &string)
{
    clear();
    QueryStringParser parser(*this);
    parser.run(string);
    if (!parser.final() || parser.error())
        MORDOR_THROW_EXCEPTION(std::invalid_argument("Invalid QueryString"));
    return *this;
}

URI::QueryString &
URI::QueryString::operator =(Stream &stream)
{
    clear();
    QueryStringParser parser(*this);
    parser.run(stream);
    if (!parser.final() || parser.error())
        MORDOR_THROW_EXCEPTION(std::invalid_argument("Invalid QueryString"));
    return *this;
}

std::string
URI::QueryString::toString() const
{
    std::ostringstream os;
    for (const_iterator it = begin();
        it != end();
        ++it) {
        if (it != begin()) {
            os << '&';
        }
        os << escape(it->first, Mordor::queryString, true);
        if (!it->second.empty())
            os << '=' << escape(it->second, Mordor::queryString, true);
    }
    return os.str();
}

}

// Copyright (c) 2009 - Mozy, Inc.
/* To compile to .cpp:
   ragel uri.rl -G2 -o uri.cpp
*/

#include "mordor/pch.h"

#include "mordor/uri.h"

#include <sstream>

#include "mordor/ragel.h"
#include "mordor/string.h"
#include "mordor/version.h"

namespace Mordor {

static const std::string unreserved("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-._~");
static const std::string sub_delims("!$&'()*+,;=");
static const std::string scheme("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789+-.");
static const std::string userinfo("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-._~" "!$&'()*+,;=" ":");
static const std::string host("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-._~" "!$&'()*+,;=" ":");
static const std::string pchar("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-._~" "!$&'()*+,;=" ":@");
static const std::string path("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-._~" "!$&'()*+,;=" ":@" "/");
static const std::string segment_nc("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-._~" "!$&'()*+,;=" "@");
static const std::string query("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-._~" "!$&'()*+,;=" ":@" "/?");
static const std::string queryString("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-._~" "!$'()*," ":@" "/?");

static std::string escape(const std::string& str, const std::string& allowedChars, bool spaceAsPlus = false)
{
    const char *hexdigits = "0123456789ABCDEF";
    std::string result(str);

    const char *c = str.c_str();
    const char *end = c + str.length();
    bool differed = false;
    while(c < end)
    {
        if (allowedChars.find(*c) == std::string::npos) {
            if (!differed) {
                result.resize(c - str.c_str());
                differed = true;
            }
            if (*c == ' ' && spaceAsPlus) {
                result.append(1, '+');
            } else {
                result.append(1, '%');
                result.append(1, hexdigits[(unsigned char)*c >> 4]);
                result.append(1, hexdigits[*c & 0xf]);
            }
        } else {
            if (differed) {
                result.append(1, *c);
            }
        }
        ++c;
    }

    if (differed) {
        MORDOR_ASSERT(result.length() >= str.length());
    } else {
        MORDOR_ASSERT(result == str);
    }
    return result;
}

std::string unescape(const std::string& str, bool spaceAsPlus = false)
{
    std::string result = str;

    const char *c = str.c_str();
    const char *end = c + str.length();
    bool differed = false;
    while (c < end)
    {
        if (*c == '%') {
            MORDOR_ASSERT(c + 2 < end);
            if (!differed) {
                result.resize(c - str.c_str());
                differed = true;
            }
            char decoded;
            ++c;
            if (*c >= 'a' && *c <= 'f')
                decoded = (*c - 'a' + 10) << 4;
            else if (*c >= 'A' && *c <= 'F')
                decoded = (*c - 'A' + 10) << 4;
            else {
                MORDOR_ASSERT(*c >= '0' && *c <='9');
                decoded = (*c - '0') << 4;
            }
            ++c;
            if (*c >= 'a' && *c <= 'f')
                decoded |= *c - 'a' + 10;
            else if (*c >= 'A' && *c <= 'F')
                decoded |= *c - 'A' + 10;
            else {
                MORDOR_ASSERT(*c >= '0' && *c <='9');
                decoded |= *c - '0';
            }
            result.append(1, decoded);
        } else if (*c == '+' && spaceAsPlus) {
            if (!differed) {
                result.resize(c - str.c_str());
                differed = true;
            }
            result.append(1, ' ');
        } else if (differed) {
            result.append(1, *c);
        }
        ++c;
    }
    return result;
}

std::string
URI::encode(const std::string &str, CharacterClass charClass)
{
    switch (charClass) {
        case UNRESERVED:
            return escape(str, unreserved, false);
        case QUERYSTRING:
            return escape(str, Mordor::queryString, true);
        default:
            MORDOR_NOTREACHED();
    }
}

std::string
URI::decode(const std::string &str, CharacterClass charClass)
{
    switch (charClass) {
        case UNRESERVED:
            return unescape(str, false);
        case QUERYSTRING:
            return unescape(str, true);
        default:
            MORDOR_NOTREACHED();
    }
}

%%{
    # See RFC 3986: http://www.ietf.org/rfc/rfc3986.txt

    machine uri_parser;

    gen_delims = ":" | "/" | "?" | "#" | "[" | "]" | "@";
    sub_delims = "!" | "$" | "&" | "'" | "(" | ")" | "*" | "+" | "," | ";" | "=";
    reserved = gen_delims | sub_delims;
    unreserved = alpha | digit | "-" | "." | "_" | "~";
    pct_encoded = "%" xdigit xdigit;

    action marku { mark = fpc; }
    action save_scheme
    {
        m_uri->scheme(unescape(std::string(mark, fpc - mark)));
        mark = NULL;
    }

    scheme = (alpha | digit | "+" | "-" | ".")+ >marku %save_scheme;

    action save_port
    {
        if (fpc == mark)
            m_uri->authority.port(-1);
        else
            m_uri->authority.port(atoi(mark));
        mark = NULL;
    }
    action save_userinfo
    {
        m_uri->authority.userinfo(unescape(std::string(mark, fpc - mark)));
        mark = NULL;
    }
    action save_host
    {
        if (mark != NULL) {
            m_uri->authority.host(unescape(std::string(mark, fpc - mark)));
            mark = NULL;
        }
    }

    userinfo = (unreserved | pct_encoded | sub_delims | ":")*;
    dec_octet = digit | [1-9] digit | "1" digit{2} | 2 [0-4] digit | "25" [0-5];
    IPv4address = dec_octet "." dec_octet "." dec_octet "." dec_octet;
    h16 = xdigit{1,4};
    ls32 = (h16 ":" h16) | IPv4address;
    IPv6address = (                         (h16 ":"){6} ls32) |
                  (                    "::" (h16 ":"){5} ls32) |
                  ((             h16)? "::" (h16 ":"){4} ls32) |
                  (((h16 ":"){1} h16)? "::" (h16 ":"){3} ls32) |
                  (((h16 ":"){2} h16)? "::" (h16 ":"){2} ls32) |
                  (((h16 ":"){3} h16)? "::" (h16 ":"){1} ls32) |
                  (((h16 ":"){4} h16)? "::"              ls32) |
                  (((h16 ":"){5} h16)? "::"              h16 ) |
                  (((h16 ":"){6} h16)? "::"                  );
    IPvFuture = "v" xdigit+ "." (unreserved | sub_delims | ":")+;
    IP_literal = "[" (IPv6address | IPvFuture) "]";
    reg_name = (unreserved | pct_encoded | sub_delims)*;
    host = IP_literal | IPv4address | reg_name;
    port = digit*;

    authority = ( (userinfo %save_userinfo "@")? host >marku %save_host (":" port >marku %save_port)? ) >marku;

    action save_segment
    {
        m_path->segments.push_back(unescape(std::string(mark, fpc - mark)));
        mark = NULL;
    }

    pchar = unreserved | pct_encoded | sub_delims | ":" | "@";
    segment = pchar* >marku %save_segment;
    segment_nz = pchar+ >marku %save_segment;
    segment_nz_nc = (pchar - ":")+ >marku %save_segment;

    action set_absolute
    {
        m_path->type = URI::Path::ABSOLUTE;
    }
    action set_relative
    {
        m_path->type = URI::Path::RELATIVE;
    }

    path_abempty = ("/" segment >set_absolute)*;
    path_absolute = "/" (segment_nz ("/" segment)*)?  >set_absolute;
    path_noscheme = segment_nz_nc >set_relative ("/" segment)*;
    path_rootless = segment_nz >set_relative ("/" segment)*;
    path_empty = "" %set_relative;
    path = (path_abempty | path_absolute | path_noscheme | path_rootless | path_empty);

    action save_query
    {
        m_uri->m_query = std::string(mark, fpc - mark);
        m_uri->m_queryDefined = true;
        mark = NULL;
    }
    action save_fragment
    {
        m_uri->fragment(unescape(std::string(mark, fpc - mark)));
        mark = NULL;
    }

    query = (pchar | "/" | "?")* >marku %save_query;
    fragment = (pchar | "/" | "?")* >marku %save_fragment;

    hier_part = ("//" authority path_abempty) | path_absolute | path_rootless | path_empty;

    relative_part = ("//" authority path_abempty) | path_absolute | path_noscheme | path_empty;
    relative_ref = relative_part ( "?" query )? ( "#" fragment )?;

    absolute_URI = scheme ":" hier_part ( "?" query )? ;
    # Obsolete, but referenced from HTTP, so we translate
    relative_URI = relative_part ( "?" query )?;

    URI = scheme ":" hier_part ( "?" query )? ( "#" fragment )?;
    URI_reference = URI | relative_ref;
}%%

%%{
        machine uri_parser_proper;
        include uri_parser;
        main := URI_reference;
        write data;
}%%

class URIParser : public RagelParser
{
public:
    URIParser(URI& uri)
    {
        m_uri = &uri;
        m_path = &m_uri->path;
    }

    void init()
    {
        RagelParser::init();
        %% write init;
    }

protected:
    void exec()
    {
#ifdef MSVC
#pragma warning(push)
#pragma warning(disable : 4244)
#endif
        %% write exec;
#ifdef MSVC
#pragma warning(pop)
#endif
    }

public:
    bool complete() const
    {
        return false;
    }

    bool final() const
    {
        return cs >= uri_parser_proper_first_final;
    }

    bool error() const
    {
        return cs == uri_parser_proper_error;
    }

private:
    URI *m_uri;
    URI::Path *m_path;
};

%%{
    machine uri_path_parser;
    include uri_parser;
    main := path;
    write data;
}%%
class URIPathParser : public RagelParser
{
public:
    URIPathParser(URI::Path& path)
    {
        m_path = &path;
    }

    void init()
    {
        RagelParser::init();
        %% write init;
    }

protected:
    void exec()
    {
#ifdef MSVC
#pragma warning(push)
#pragma warning(disable : 4244)
#endif
        %% write exec;
#ifdef MSVC
#pragma warning(pop)
#endif
    }

public:
    bool complete() const
    {
        return false;
    }

    bool final() const
    {
        return cs >= uri_path_parser_first_final;
    }

    bool error() const
    {
        return cs == uri_path_parser_error;
    }

private:
    URI::Path *m_path;
};

URI::URI()
{
    reset();
}

URI::URI(const std::string& uri)
{
    reset();
    *this = uri;
}

URI::URI(const char *uri)
{
    reset();
    *this = uri;
}

URI::URI(const Buffer &uri)
{
    reset();
    *this = uri;
}

URI&
URI::operator=(const std::string& uri)
{
    clear();
    URIParser parser(*this);
    parser.run(uri);
    if (parser.error() || !parser.final())
        throw std::invalid_argument("uri");
    return *this;
}

URI&
URI::operator=(const Buffer &uri)
{
    clear();
    URIParser parser(*this);
    parser.run(uri);
    if (parser.error() || !parser.final())
        throw std::invalid_argument("uri");
    return *this;
}

void
URI::reset()
{
    schemeDefined(false);
    authority.hostDefined(false);
    path.type = Path::RELATIVE;
    path.segments.clear();
    queryDefined(false);
    fragmentDefined(false);
}

URI::Authority::Authority()
{
    userinfoDefined(false);
    hostDefined(false);
    portDefined(false);
}

void
URI::Authority::normalize(const std::string& defaultHost, bool emptyHostValid,
    int defaultPort, bool emptyPortValid)
{
    for(size_t i = 0; i < m_host.length(); ++i)
        m_host[i] = tolower(m_host[i]);
    if (m_port == defaultPort)
        m_port = -1;
    if (m_port == -1 && !emptyPortValid)
        m_portDefined = false;
    if (m_host == defaultHost)
        m_host.clear();
    if (m_host.empty() && !emptyHostValid && !m_userinfoDefined && !m_portDefined)
        m_hostDefined = false;
}

std::string
URI::Authority::toString() const
{
    std::ostringstream os;
    os << *this;
    return os.str();
}

std::string
URI::Path::toString() const
{
    std::ostringstream os;
    os << *this;
    return os.str();
}

static int boolcmp(bool lhs, bool rhs)
{
    if (!lhs && rhs)
        return -1;
    if (lhs && !rhs)
        return 1;
    return 0;
}

int
URI::Authority::cmp(const Authority &rhs) const
{
    int x = boolcmp(m_hostDefined, rhs.m_hostDefined);
    if (x != 0) return x;
    x = strcmp(m_host.c_str(), rhs.m_host.c_str());
    if (x != 0) return x;
    x = boolcmp(m_portDefined, rhs.m_portDefined);
    if (x != 0) return x;
    x = m_port - rhs.m_port;
    if (x != 0) return x;
    x = boolcmp(m_userinfoDefined, rhs.m_userinfoDefined);
    if (x != 0) return x;
    return strcmp(m_userinfo.c_str(), rhs.m_userinfo.c_str());
}

bool
URI::Authority::operator==(const Authority &rhs) const
{
    return cmp(rhs) == 0;
}

std::ostream&
operator<<(std::ostream& os, const URI::Authority& authority)
{
    MORDOR_ASSERT(authority.hostDefined());
    if (authority.userinfoDefined()) {
        os << escape(authority.userinfo(), userinfo) << "@";
    }
    os << escape(authority.host(), host);
    if (authority.portDefined()) {
        os << ":";
        if (authority.port() > 0) {
            os << authority.port();
        }
    }
    return os;
}

URI::Path::Path()
{
    type = RELATIVE;
}

URI::Path::Path(const char *path)
{
    *this = path;
}

URI::Path::Path(const std::string& path)
{
    *this = path;
}

URI::Path&
URI::Path::operator=(const std::string& path)
{
    type = RELATIVE;
    segments.clear();
    URIPathParser parser(*this);
    parser.run(path);
    if (parser.error() || !parser.final())
        throw std::invalid_argument("uri");
    return *this;
}

void
URI::Path::removeDotComponents()
{
    for(size_t i = 0; i < segments.size(); ++i) {
        if (segments[i] == ".") {
            if (i + 1 == segments.size()) {
                segments[i].clear();
                continue;
            } else {
                segments.erase(segments.begin() + i);
                --i;
                continue;
            }
        }
        if (segments[i] == "..") {
            if (i == 0) {
                segments.erase(segments.begin());
                --i;
                continue;
            }
            if (i + 1 == segments.size()) {
                segments.resize(segments.size() - 1);
                segments.back().clear();
                --i;
                continue;
            }
            segments.erase(segments.begin() + i - 1, segments.begin() + i + 1);
            i -= 2;
            continue;
        }
    }
}

void
URI::Path::normalize(bool emptyPathValid)
{
    removeDotComponents();
    if (segments.empty() && !emptyPathValid)
        type = ABSOLUTE;
}

void
URI::Path::merge(const Path& rhs)
{
    MORDOR_ASSERT(rhs.type == RELATIVE);
    if (!segments.empty()) {
        segments.pop_back();
        segments.insert(segments.end(), rhs.segments.begin(), rhs.segments.end());
    } else {
        segments = rhs.segments;
    }
}

URI::Path::path_serializer
URI::Path::serialize(bool schemeless) const
{
    path_serializer result;
    result.p = this;
    result.schemeless = schemeless;
    return result;
}

std::ostream&
operator<<(std::ostream& os, const URI::Path::path_serializer &p)
{
    if (p.p->segments.empty() && p.p->type == URI::Path::ABSOLUTE) {
        return os << "/";
    }
    for (size_t i = 0; i < p.p->segments.size(); ++i) {
        if (i != 0 || p.p->type == URI::Path::ABSOLUTE) {
            os << "/";
        }
        if (i == 0 && p.p->type == URI::Path::RELATIVE && p.schemeless) {
            os << escape(p.p->segments[i], segment_nc);
        } else {
            os << escape(p.p->segments[i], pchar);
        }
    }
    return os;
}

std::ostream&
operator<<(std::ostream& os, const URI::Path& path)
{
    return os << path.serialize();
}

int
URI::Path::cmp(const Path &rhs) const
{
    if (type == ABSOLUTE && rhs.type == RELATIVE)
        return -1;
    if (type == RELATIVE && rhs.type == ABSOLUTE)
        return 1;
    std::vector<std::string>::const_iterator itl, itr;
    itl = segments.begin(); itr = rhs.segments.begin();
    while (true) {
        if (itl == segments.end() && itr != rhs.segments.end())
            return -1;
        if (itl != segments.end() && itr == rhs.segments.end())
            return 1;
        if (itl == segments.end() && itr == rhs.segments.end())
            return 0;
        int x = strcmp(itl->c_str(), itr->c_str());
        if (x != 0) return x;
        ++itl; ++itr;
    }
}

bool
URI::Path::operator==(const Path &rhs) const
{
    return type == rhs.type && segments == rhs.segments;
}

void
URI::normalize()
{
    for (size_t i = 0; i < m_scheme.size(); ++i)
        m_scheme[i] = tolower(m_scheme[i]);

    if (m_scheme == "http" || m_scheme == "https") {
        authority.normalize("", false, m_scheme.size() == 4 ? 80 : 443, false);
        path.normalize();
    } else if (m_scheme == "file") {
        authority.normalize("localhost", true);
        path.normalize();
    } else {
        authority.normalize();
        path.normalize();
    }
}

std::string
URI::query() const
{
    MORDOR_ASSERT(m_queryDefined);
    return unescape(m_query);
}

void
URI::query(const std::string &q)
{
    m_queryDefined = true;
    m_query = escape(q, Mordor::query);
}

std::string
URI::toString() const
{
    std::ostringstream os;
    os << *this;
    return os.str();
}

std::ostream&
operator<<(std::ostream& os, const URI& uri)
{
    if (uri.schemeDefined()) {
        os << escape(uri.scheme(), scheme) << ":";
    }

    if (uri.authority.hostDefined()) {
        os << "//" << uri.authority;
    }

    // Has scheme, but no authority, must ensure that an absolute path
    // doesn't begin with an empty segment (or could be mistaken for authority)
    if (uri.schemeDefined() && !uri.authority.hostDefined() &&
        uri.path.type == URI::Path::ABSOLUTE &&
        uri.path.segments.size() > 0 && uri.path.segments.front().empty()) {
        os << "/";
    }
    os << uri.path.serialize(!uri.schemeDefined());

    if (uri.queryDefined()) {
        os << "?" << uri.m_query;
    }

    if (uri.fragmentDefined()) {
        os << "#" << escape(uri.fragment(), query);
    }
    return os;
}

URI
URI::transform(const URI& base, const URI& relative)
{
    MORDOR_ASSERT(base.schemeDefined());

    URI target;
    if (relative.schemeDefined()) {
        target.scheme(relative.scheme());
        target.authority = relative.authority;
        target.path = relative.path;
        target.path.removeDotComponents();
        target.m_query = relative.m_query;
        target.m_queryDefined = relative.m_queryDefined;
    } else {
        if (relative.authority.hostDefined()) {
            target.authority = relative.authority;
            target.path = relative.path;
            target.path.removeDotComponents();
            target.m_query = relative.m_query;
            target.m_queryDefined = relative.m_queryDefined;
        } else {
            if (relative.path.isEmpty()) {
                target.path = base.path;
                if (relative.queryDefined()) {
                    target.query(relative.query());
                } else {
                    target.m_query = base.m_query;
                    target.m_queryDefined = base.m_queryDefined;
                }
            } else {
                if (relative.path.type == Path::ABSOLUTE) {
                    target.path = relative.path;
                } else {
                    target.path = base.path;
                    target.path.merge(relative.path);
                    if (!base.authority.hostDefined())
                        target.path.type = Path::ABSOLUTE;
                }
                target.path.removeDotComponents();
                target.m_query = relative.m_query;
                target.m_queryDefined = relative.m_queryDefined;
            }
            target.authority = base.authority;
        }
        target.scheme(base.scheme());
    }
    target.m_fragment = relative.m_fragment;
    target.m_fragmentDefined = relative.m_fragmentDefined;
    return target;
}

int
URI::cmp(const URI &rhs) const
{
    int x = boolcmp(m_schemeDefined, rhs.m_schemeDefined);
    if (x != 0) return x;
    x = strcmp(m_scheme.c_str(), rhs.m_scheme.c_str());
    if (x != 0) return x;
    x = authority.cmp(rhs.authority);
    if (x != 0) return x;
    x = path.cmp(rhs.path);
    if (x != 0) return x;
    x = boolcmp(m_queryDefined, rhs.m_queryDefined);
    if (x != 0) return x;
    x = strcmp(m_query.c_str(), rhs.m_query.c_str());
    if (x != 0) return x;
    x = boolcmp(m_fragmentDefined, rhs.m_fragmentDefined);
    if (x != 0) return x;
    return strcmp(m_fragment.c_str(), rhs.m_fragment.c_str());
}

bool
URI::operator<(const URI &rhs) const
{
    return cmp(rhs) < 0;
}

bool
URI::operator==(const URI &rhs) const
{
    return cmp(rhs) == 0;
}

%%{
    machine querystring_parser;

    action mark { mark = fpc; }
    action saveKey {
        m_iterator = m_qs.insert(std::make_pair(
            unescape(std::string(mark, fpc - mark), true), std::string()));
        mark = NULL;
    }
    action saveValue {
        MORDOR_ASSERT(m_iterator != m_qs.end());
        if (fpc - mark == 0 && m_iterator->first.empty())
            m_qs.erase(m_iterator);
        else
            m_iterator->second = unescape(std::string(mark, fpc - mark), true);
        m_iterator = m_qs.end();
        mark = NULL;
    }
    action saveNoValue {
        if (m_iterator != m_qs.end() && m_iterator->first.empty()) {
            m_qs.erase(m_iterator);
            mark = NULL;
        }
    }

    sub_delims = "!" | "$" | "&" | "'" | "(" | ")" | "*" | "+" | "," | ";";
    unreserved = alpha | digit | "-" | "." | "_" | "~";
    pct_encoded = "%" xdigit xdigit;
    pchar = unreserved | pct_encoded | sub_delims | ":" | "@";
    querychar = (pchar | "/" | "?") -- '&' -- ';';
    key = querychar*;
    value = (querychar | '=')*;
    keyValue = key >mark %saveKey ('=' value >mark %saveValue)? %saveNoValue;
    main := keyValue? ( ('&' | ';') keyValue? )*;
    write data;
}%%

class QueryStringParser : public RagelParser
{
public:
    QueryStringParser(URI::QueryString &qs)
    : m_qs(qs),
      m_iterator(m_qs.end())
    {}

    void init()
    {
        RagelParser::init();
        %% write init;
    }

    void exec()
    {
#ifdef MSVC
#pragma warning(push)
#pragma warning(disable : 4244)
#endif
        %% write exec;
#ifdef MSVC
#pragma warning(pop)
#endif
    }

    bool complete() const { return false; }
    bool final() const
    {
        return cs >= querystring_parser_first_final;
    }

    bool error() const
    {
        return cs == querystring_parser_error;
    }

private:
    URI::QueryString &m_qs;
    URI::QueryString::iterator m_iterator;
};

URI::QueryString &
URI::QueryString::operator =(const std::string &string)
{
    clear();
    QueryStringParser parser(*this);
    parser.run(string);
    if (!parser.final() || parser.error())
        MORDOR_THROW_EXCEPTION(std::invalid_argument("Invalid QueryString"));
    return *this;
}

URI::QueryString &
URI::QueryString::operator =(Stream &stream)
{
    clear();
    QueryStringParser parser(*this);
    parser.run(stream);
    if (!parser.final() || parser.error())
        MORDOR_THROW_EXCEPTION(std::invalid_argument("Invalid QueryString"));
    return *this;
}

std::string
URI::QueryString::toString() const
{
    std::ostringstream os;
    for (const_iterator it = begin();
        it != end();
        ++it) {
        if (it != begin()) {
            os << '&';
        }
        os << escape(it->first, Mordor::queryString, true);
        if (!it->second.empty())
            os << '=' << escape(it->second, Mordor::queryString, true);
    }
    return os.str();
}

}


improved comment on how error's are handled within lexer

#ifndef HEXT_LEXER_H
#define HEXT_LEXER_H

#include <string>
#include <sstream>
#include <stdexcept>
#include <utility>

#include "hext/token.h"
#include "hext/lexer-util.h"

namespace hext {

namespace ragel {
  %%{
    machine hext;
    include "hext.rl";
    write data;
  }%%
} // namespace ragel

class lexer
{
public:
  class lex_error : public std::runtime_error
  {
  public:
    explicit lex_error(const std::string& msg)
    : std::runtime_error(msg) {}
  };

  lexer(const char * begin, const char * end)
  : p_begin(begin),
    p(begin),
    pe(end),
    eof(end),
    cs(0)
  {
    using namespace ragel;
    %% write init;
  }

  std::vector<token> lex()
  {
    using namespace ragel;

    std::vector<token> tokens;

    // this calls throw_error on lexing error
    %% write exec;

    return tokens;
  }

  void throw_error() const
  {
    std::pair<size_t, size_t> pos =
      get_char_position(this->p, this->p_begin, this->pe);

    std::string char_name;
    if( this->p == this->pe )
    {
      char_name = "[eof]";
    }
    else
    {
      if( this->p != nullptr )
        char_name = get_char_name(*this->p);
      else
        char_name = "[unknown]";
    }

    std::stringstream error_msg;
    error_msg << "Error at line "
              << pos.first + 1
              << ", char "
              << pos.second + 1
              << ", unexpected character '"
              << char_name
              << "'";

    throw lex_error(error_msg.str());
  }

private:
  const char * p_begin;
  const char * p;
  const char * pe;
  const char * eof;
  int cs;
};

} // namespace hext

#endif // HEXT_LEXER_H

#ifndef HEXT_LEXER_H
#define HEXT_LEXER_H

#include <string>
#include <sstream>
#include <stdexcept>
#include <utility>

#include "hext/token.h"
#include "hext/lexer-util.h"

namespace hext {

namespace ragel {
  %%{
    machine hext;
    include "hext.rl";
    write data;
  }%%
} // namespace ragel

class lexer
{
public:
  class lex_error : public std::runtime_error
  {
  public:
    explicit lex_error(const std::string& msg)
    : std::runtime_error(msg) {}
  };

  lexer(const char * begin, const char * end)
  : p_begin(begin),
    p(begin),
    pe(end),
    eof(end),
    cs(0)
  {
    using namespace ragel;
    %% write init;
  }

  std::vector<token> lex()
  {
    using namespace ragel;

    std::vector<token> tokens;

    // this throws on lexing error
    %% write exec;

    return tokens;
  }

  void throw_error() const
  {
    std::pair<size_t, size_t> pos =
      get_char_position(this->p, this->p_begin, this->pe);

    std::string char_name;
    if( this->p == this->pe )
    {
      char_name = "[eof]";
    }
    else
    {
      if( this->p != nullptr )
        char_name = get_char_name(*this->p);
      else
        char_name = "[unknown]";
    }

    std::stringstream error_msg;
    error_msg << "Error at line "
              << pos.first + 1
              << ", char "
              << pos.second + 1
              << ", unexpected character '"
              << char_name
              << "'";

    throw lex_error(error_msg.str());
  }

private:
  const char * p_begin;
  const char * p;
  const char * pe;
  const char * eof;
  int cs;
};

} // namespace hext

#endif // HEXT_LEXER_H


added lexer exception lex_error

added lexer exception lex_error

#ifndef HEXT_LEXER_H
#define HEXT_LEXER_H

#include <iostream>
#include <iterator>
#include <algorithm>
#include <string>
#include <cassert>
#include <stdexcept>

#include "hext/token.h"

#define LX_TK_START(tk_id) \
  token tok;               \
  tok.tid = tk_id;         \
  tok.tok_begin = p;       \
  tok.tok_end = nullptr;   \
  tokens.push_back(tok);

#define LX_TK_STOP                          \
  assert(tokens.size() > 0);                \
  tokens.at(tokens.size() - 1).tok_end = p;

namespace hext {

namespace ragel {
  %%{
    machine hext;
    include "hext.rl";
    write data;
  }%%
}

class lexer
{
public:
  class lex_error : public std::runtime_error
  {
  public:
    lex_error(const char * msg)
    : std::runtime_error(msg) {}
  };

  lexer(const char * begin, const char * end)
  : p_begin(begin),
    p(begin),
    pe(end),
    eof(end),
    cs(0)
  {
    using namespace ragel;
    %% write init;
  }

  std::vector<token> lex()
  {
    using namespace ragel;

    std::vector<token> tokens;

    // this throws on lexing error
    %% write exec;

    return tokens;
  }

  void print_error()
  {
    typedef
      std::iterator_traits<std::string::const_iterator>::difference_type
      iter_diff_type;

    std::string all_to_error(this->p_begin, this->p);
    iter_diff_type line_nr =
      std::count(all_to_error.begin(), all_to_error.end(), '\n');

    size_t char_nr = 0;
    size_t line_offset = 0;
    if( line_nr )
    {
      line_offset = all_to_error.find_last_of('\n');
      if( line_offset == std::string::npos )
        line_offset = 1;
      // move past \n
      line_offset += 1;
    }
    char_nr = std::distance(this->p_begin, this->p) - line_offset;

    std::string error_token;
    if( *p == '\n' )
      error_token = "[newline]";
    else if( *p == ' ' )
      error_token = "[space]";
    else
      error_token = *p;

    std::cerr << "Error at line "
              << line_nr + 1
              << ", char "
              << char_nr + 1
              << ", unexpected character '"
              << error_token
              << "'\n";
  }

private:
  const char * p_begin;
  const char * p;
  const char * pe;
  const char * eof;
  int cs;
};

} // namespace hext

#endif // HEXT_LEXER_H

#ifndef HEXT_LEXER_H
#define HEXT_LEXER_H

#include <iostream>
#include <iterator>
#include <algorithm>
#include <string>
#include <cassert>

#include "hext/token.h"

#define LX_TK_START(tk_id) \
  token tok;               \
  tok.tid = tk_id;         \
  tok.tok_begin = p;       \
  tok.tok_end = nullptr;   \
  tokens.push_back(tok);

#define LX_TK_STOP                          \
  assert(tokens.size() > 0);                \
  tokens.at(tokens.size() - 1).tok_end = p;

namespace hext {

namespace ragel {
  %%{
    machine hext;
    include "hext.rl";
    write data;
  }%%
}

class lexer
{
public:
  lexer(const char * begin, const char * end)
  : p_begin(begin),
    p(begin),
    pe(end),
    eof(end),
    cs(0)
  {
    using namespace ragel;
    %% write init;
  }

  std::vector<token> lex()
  {
    using namespace ragel;

    std::vector<token> tokens;

    // this throws on lexing error
    %% write exec;

    return tokens;
  }

  void print_error()
  {
    typedef
      std::iterator_traits<std::string::const_iterator>::difference_type
      iter_diff_type;

    std::string all_to_error(this->p_begin, this->p);
    iter_diff_type line_nr =
      std::count(all_to_error.begin(), all_to_error.end(), '\n');

    size_t char_nr = 0;
    size_t line_offset = 0;
    if( line_nr )
    {
      line_offset = all_to_error.find_last_of('\n');
      if( line_offset == std::string::npos )
        line_offset = 1;
      // move past \n
      line_offset += 1;
    }
    char_nr = std::distance(this->p_begin, this->p) - line_offset;

    std::string error_token;
    if( *p == '\n' )
      error_token = "[newline]";
    else if( *p == ' ' )
      error_token = "[space]";
    else
      error_token = *p;

    std::cerr << "Error at line "
              << line_nr + 1
              << ", char "
              << char_nr + 1
              << ", unexpected character '"
              << error_token
              << "'\n";
  }

private:
  const char * p_begin;
  const char * p;
  const char * pe;
  const char * eof;
  int cs;
};

} // namespace hext

#endif // HEXT_LEXER_H


rename yield action with yield_ prefix

rename yield action with yield_ prefix

Conveys the intention better.

# begin: ragel
=begin
%%{
  machine bel;

  include 'common.rl';
  include 'identifier.rl';
  include 'string.rl';
  include 'list.rl';

  DEFINE_KW     = [dD][eE][fF][iI][nN][eE];
  ANNOTATION_KW = [aA][nN][nN][oO][tT][aA][tT][iI][oO][nN];
  AS_KW         = [aA][sS];
  LIST_KW       = [lL][iI][sS][tT];
  PATTERN_KW    = [pP][aA][tT][tT][eE][rR][nN];
  URL_KW        = [uU][rR][lL];

  action annotation_keyword {
    @buffers[:define_annotation] = s(:define_annotation)
  }

  action keyword {
    @buffers[:define_annotation] = s(:define_annotation,
                                     s(:keyword, @buffers[:ident]))
  }

  action list_keyword {
    @buffers[:define_annotation] = @buffers[:define_annotation] << s(:domain)
  }

  action pattern_keyword {
    @buffers[:define_annotation] = @buffers[:define_annotation] << s(:domain, s(:pattern))
  }

  action url_keyword {
    @buffers[:define_annotation] = @buffers[:define_annotation] << s(:domain, s(:url))
  }

  action pattern {
    keyword, domain              = @buffers[:define_annotation].children
    domain                       = s(:domain,
                                     domain.children[0] << @buffers[:string])
    @buffers[:define_annotation] = s(:define_annotation, keyword, domain)
  }

  action url {
    keyword, domain              = @buffers[:define_annotation].children
    domain                       = s(:domain,
                                     domain.children[0] << @buffers[:string])
    @buffers[:define_annotation] = s(:define_annotation, keyword, domain)
  }

  action list {
    keyword, domain              = @buffers[:define_annotation].children
    domain                       = s(:domain,
                                     @buffers[:list])
    @buffers[:define_annotation] = s(:define_annotation, keyword, domain)
  }

  action yield_define_annotation {
    yield @buffers[:define_annotation]
  }

  # Define FSM
  define_annotation :=
    DEFINE_KW SP+ ANNOTATION_KW @annotation_keyword SP+
      IDENT %keyword SP+
    AS_KW SP+
      (
        (LIST_KW    %list_keyword    SP+ LIST   %list    SP* NL @yield_define_annotation) |
        (PATTERN_KW %pattern_keyword SP+ STRING %pattern SP* NL @yield_define_annotation) |
        (URL_KW     %url_keyword     SP+ STRING %url     SP* NL @yield_define_annotation)
      );
}%%
=end
# end: ragel

require          'ast'
require_relative '../mixin/buffer'
require_relative '../nonblocking_io_wrapper'

module BEL
  module Parsers
    module BELScript
      module DefineAnnotation

        class << self

          MAX_LENGTH = 1024 * 128 # 128K

          def parse(content)
            return nil unless content

            Parser.new(content).each do |obj|
              yield obj
            end
          end
        end

        private

        class Parser
          include Enumerable
          include AST::Sexp
          include BEL::Parser::Buffer

          def initialize(content)
            @content = content
      # begin: ragel
            %% write data;
      # end: ragel
          end

          def each
            @buffers = {}
            data     = @content.unpack('C*')
            p        = 0
            pe       = data.length

      # begin: ragel
            %% write init;
            %% write exec;
      # end: ragel
          end
        end
      end
    end
  end
end

if __FILE__ == $0
  $stdin.each_line do |line|
    BEL::Parsers::BELScript::DefineAnnotation.parse(line) { |obj|
      puts obj.inspect
    }
  end
end

# vim: ft=ruby ts=2 sw=2:
# encoding: utf-8

# begin: ragel
=begin
%%{
  machine bel;

  include 'common.rl';
  include 'identifier.rl';
  include 'string.rl';
  include 'list.rl';

  DEFINE_KW     = [dD][eE][fF][iI][nN][eE];
  ANNOTATION_KW = [aA][nN][nN][oO][tT][aA][tT][iI][oO][nN];
  AS_KW         = [aA][sS];
  LIST_KW       = [lL][iI][sS][tT];
  PATTERN_KW    = [pP][aA][tT][tT][eE][rR][nN];
  URL_KW        = [uU][rR][lL];

  action annotation_keyword {
    @buffers[:define_annotation] = s(:define_annotation)
  }

  action keyword {
    @buffers[:define_annotation] = s(:define_annotation,
                                     s(:keyword, @buffers[:ident]))
  }

  action list_keyword {
    @buffers[:define_annotation] = @buffers[:define_annotation] << s(:domain)
  }

  action pattern_keyword {
    @buffers[:define_annotation] = @buffers[:define_annotation] << s(:domain, s(:pattern))
  }

  action url_keyword {
    @buffers[:define_annotation] = @buffers[:define_annotation] << s(:domain, s(:url))
  }

  action pattern {
    keyword, domain              = @buffers[:define_annotation].children
    domain                       = s(:domain,
                                     domain.children[0] << @buffers[:string])
    @buffers[:define_annotation] = s(:define_annotation, keyword, domain)
  }

  action url {
    keyword, domain              = @buffers[:define_annotation].children
    domain                       = s(:domain,
                                     domain.children[0] << @buffers[:string])
    @buffers[:define_annotation] = s(:define_annotation, keyword, domain)
  }

  action list {
    keyword, domain              = @buffers[:define_annotation].children
    domain                       = s(:domain,
                                     @buffers[:list])
    @buffers[:define_annotation] = s(:define_annotation, keyword, domain)
  }

  action define_annotation {
    yield @buffers[:define_annotation]
  }

  # Define FSM
  define_annotation :=
    DEFINE_KW SP+ ANNOTATION_KW @annotation_keyword SP+
      IDENT %keyword SP+
    AS_KW SP+
      (
        (LIST_KW    %list_keyword    SP+ LIST   %list    SP* NL @define_annotation) |
        (PATTERN_KW %pattern_keyword SP+ STRING %pattern SP* NL @define_annotation) |
        (URL_KW     %url_keyword     SP+ STRING %url     SP* NL @define_annotation)
      );
}%%
=end
# end: ragel

require          'ast'
require_relative '../mixin/buffer'
require_relative '../nonblocking_io_wrapper'

module BEL
  module Parsers
    module BELScript
      module DefineAnnotation

        class << self

          MAX_LENGTH = 1024 * 128 # 128K

          def parse(content)
            return nil unless content

            Parser.new(content).each do |obj|
              yield obj
            end
          end
        end

        private

        class Parser
          include Enumerable
          include AST::Sexp
          include BEL::Parser::Buffer

          def initialize(content)
            @content = content
      # begin: ragel
            %% write data;
      # end: ragel
          end

          def each
            @buffers = {}
            data     = @content.unpack('C*')
            p        = 0
            pe       = data.length

      # begin: ragel
            %% write init;
            %% write exec;
      # end: ragel
          end
        end
      end
    end
  end
end

if __FILE__ == $0
  $stdin.each_line do |line|
    BEL::Parsers::BELScript::DefineAnnotation.parse(line) { |obj|
      puts obj.inspect
    }
  end
end

# vim: ft=ruby ts=2 sw=2:
# encoding: utf-8


[axl_re] fix: don't sign-extend `other` chars in a pattern

//..............................................................................
//
//  This file is part of the AXL library.
//
//  AXL is distributed under the MIT license.
//  For details see accompanying license.txt file,
//  the public copy of which is also available at:
//  http://tibbo.com/downloads/archive/axl/license.txt
//
//..............................................................................

// warning C4065: switch statement contains 'default' but no 'case' labels

#pragma warning (disable: 4065)

namespace axl {
namespace re {

//..............................................................................

%%{

machine axl_re;
write data;

#. . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . .
#
#  standard definitions
#

oct = [0-7];
dec = [0-9];
hex = [0-9a-fA-F];
ws  = [ \t\r];

utf8_1 = 0x00 .. 0x7f;
utf8_2 = 0xc0 .. 0xdf;
utf8_3 = 0xe0 .. 0xef;
utf8_4 = 0xf0 .. 0xf7;
utf8_c = 0x80 .. 0xbf;

#. . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . .
#
#  main machine
#

main := |*

'^'    { createToken(TokenKind_AnchorBeginLine); };
'$'    { createToken(TokenKind_AnchorEndLine); };
'\\A'  { createToken(TokenKind_AnchorBeginText); };
'\\z'  { createToken(TokenKind_AnchorEndText); };
'\\b'  { createToken(TokenKind_AnchorWordBoundary); };
'\\B'  { createToken(TokenKind_AnchorNotWordBoundary); };

'[^'   { createToken(TokenKind_NegatedCharClass); fgoto char_class; };
'['    { createToken(TokenKind_CharClass); fgoto char_class; };
'(?:'  { createToken(TokenKind_NonCapturingGroup); };
'('    { createToken(TokenKind_Group); };
')'    { createToken(TokenKind_EndGroup); };
'??'   { createToken(TokenKind_NonGreedyQuestion); };
'?'    { createToken(TokenKind_Question); };
'*?'   { createToken(TokenKind_NonGreedyStar); };
'*'    { createToken(TokenKind_Star); };
'+?'   { createToken(TokenKind_NonGreedyPlus); };
'+'    { createToken(TokenKind_Plus); };
'|'    { createToken(TokenKind_Pipe); };
'{'    { createToken(TokenKind_Quantifier); fgoto quantifier; };
'.'    { createToken(TokenKind_AnyChar); };

'\\d'  { createToken(TokenKind_StdCharClassDigit); };
'\\D'  { createToken(TokenKind_StdCharClassNonDigit); };
'\\h'  { createToken(TokenKind_StdCharClassHex); };
'\\H'  { createToken(TokenKind_StdCharClassNonHex); };
'\\w'  { createToken(TokenKind_StdCharClassWord); };
'\\W'  { createToken(TokenKind_StdCharClassNonWord); };
'\\s'  { createToken(TokenKind_StdCharClassSpace); };
'\\S'  { createToken(TokenKind_StdCharClassNonSpace); };

'\\0'  { createCharToken(0); };
'\\a'  { createCharToken('\a'); };
'\\b'  { createCharToken('\b'); };
'\\e'  { createCharToken('\x1b'); };
'\\f'  { createCharToken('\f'); };
'\\n'  { createCharToken('\n'); };
'\\r'  { createCharToken('\r'); };
'\\t'  { createCharToken('\t'); };
'\\v'  { createCharToken('\v'); };

'\\x' hex{2}   { createHexCharToken_2(ts + 2); };
'\\u' hex{4}   { createHexCharToken_4(ts + 2); };
'\\U' hex{8}   { createHexCharToken_8(ts + 2); };
'\\'  oct{3}   { createOctCharToken(ts + 1); };
'\\'  any      { createCharToken(ts[1]); };

utf8_1            { createCharToken(ts[0]); };
utf8_2 utf8_c     { createUtf8CharToken_2(ts); };
utf8_3 utf8_c{2}  { createUtf8CharToken_3(ts); };
utf8_4 utf8_c{3}  { createUtf8CharToken_4(ts); };

any               { createCharToken((uchar_t)ts[0]); };

*|;

#. . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . .
#
#  char class machine
#

char_class := |*

'-'    { createToken(TokenKind_Dash); };
']'    { createToken(TokenKind_EndCharClass); fgoto main; };

# alas, Ragel doesn't allow injecting sub-scanners, hence, copy-paste...

'\\d'  { createToken(TokenKind_StdCharClassDigit); };
'\\D'  { createToken(TokenKind_StdCharClassNonDigit); };
'\\h'  { createToken(TokenKind_StdCharClassHex); };
'\\H'  { createToken(TokenKind_StdCharClassNonHex); };
'\\w'  { createToken(TokenKind_StdCharClassWord); };
'\\W'  { createToken(TokenKind_StdCharClassNonWord); };
'\\s'  { createToken(TokenKind_StdCharClassSpace); };
'\\S'  { createToken(TokenKind_StdCharClassNonSpace); };

'\\0'  { createCharToken(0); };
'\\a'  { createCharToken('\a'); };
'\\b'  { createCharToken('\b'); };
'\\e'  { createCharToken('\x1b'); };
'\\f'  { createCharToken('\f'); };
'\\n'  { createCharToken('\n'); };
'\\r'  { createCharToken('\r'); };
'\\t'  { createCharToken('\t'); };
'\\v'  { createCharToken('\v'); };

'\\x' hex{2}   { createHexCharToken_2(ts + 2); };
'\\u' hex{4}   { createHexCharToken_4(ts + 2); };
'\\U' hex{8}   { createHexCharToken_8(ts + 2); };
'\\'  oct{3}   { createOctCharToken(ts + 1); };
'\\'  any      { createCharToken(ts[1]); };

utf8_1            { createCharToken(ts[0]); };
utf8_2 utf8_c     { createUtf8CharToken_2(ts); };
utf8_3 utf8_c{2}  { createUtf8CharToken_3(ts); };
utf8_4 utf8_c{3}  { createUtf8CharToken_4(ts); };

any               { createCharToken(ts[0]); };

*|;

#. . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . .
#
#  quantifier machine
#

quantifier := |*

dec+  { createNumberToken(atoi(ts)); };
','   { createToken(TokenKind_Comma); };
'}'   { createToken(TokenKind_EndQuantifier); fgoto main; };

ws;

*|;

}%%

//..............................................................................

void
Lexer::init () {
	%% write init;
}

bool
Lexer::exec () {
	%% write exec;
	return cs != axl_re_error;
}

//..............................................................................

} // namespace re
} // namespace axl

//..............................................................................
//
//  This file is part of the AXL library.
//
//  AXL is distributed under the MIT license.
//  For details see accompanying license.txt file,
//  the public copy of which is also available at:
//  http://tibbo.com/downloads/archive/axl/license.txt
//
//..............................................................................

// warning C4065: switch statement contains 'default' but no 'case' labels

#pragma warning (disable: 4065)

namespace axl {
namespace re {

//..............................................................................

%%{

machine axl_re;
write data;

#. . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . .
#
#  standard definitions
#

oct = [0-7];
dec = [0-9];
hex = [0-9a-fA-F];
ws  = [ \t\r];

utf8_1 = 0x00 .. 0x7f;
utf8_2 = 0xc0 .. 0xdf;
utf8_3 = 0xe0 .. 0xef;
utf8_4 = 0xf0 .. 0xf7;
utf8_c = 0x80 .. 0xbf;

#. . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . .
#
#  main machine
#

main := |*

'^'    { createToken(TokenKind_AnchorBeginLine); };
'$'    { createToken(TokenKind_AnchorEndLine); };
'\\A'  { createToken(TokenKind_AnchorBeginText); };
'\\z'  { createToken(TokenKind_AnchorEndText); };
'\\b'  { createToken(TokenKind_AnchorWordBoundary); };
'\\B'  { createToken(TokenKind_AnchorNotWordBoundary); };

'[^'   { createToken(TokenKind_NegatedCharClass); fgoto char_class; };
'['    { createToken(TokenKind_CharClass); fgoto char_class; };
'(?:'  { createToken(TokenKind_NonCapturingGroup); };
'('    { createToken(TokenKind_Group); };
')'    { createToken(TokenKind_EndGroup); };
'??'   { createToken(TokenKind_NonGreedyQuestion); };
'?'    { createToken(TokenKind_Question); };
'*?'   { createToken(TokenKind_NonGreedyStar); };
'*'    { createToken(TokenKind_Star); };
'+?'   { createToken(TokenKind_NonGreedyPlus); };
'+'    { createToken(TokenKind_Plus); };
'|'    { createToken(TokenKind_Pipe); };
'{'    { createToken(TokenKind_Quantifier); fgoto quantifier; };
'.'    { createToken(TokenKind_AnyChar); };

'\\d'  { createToken(TokenKind_StdCharClassDigit); };
'\\D'  { createToken(TokenKind_StdCharClassNonDigit); };
'\\h'  { createToken(TokenKind_StdCharClassHex); };
'\\H'  { createToken(TokenKind_StdCharClassNonHex); };
'\\w'  { createToken(TokenKind_StdCharClassWord); };
'\\W'  { createToken(TokenKind_StdCharClassNonWord); };
'\\s'  { createToken(TokenKind_StdCharClassSpace); };
'\\S'  { createToken(TokenKind_StdCharClassNonSpace); };

'\\0'  { createCharToken(0); };
'\\a'  { createCharToken('\a'); };
'\\b'  { createCharToken('\b'); };
'\\e'  { createCharToken('\x1b'); };
'\\f'  { createCharToken('\f'); };
'\\n'  { createCharToken('\n'); };
'\\r'  { createCharToken('\r'); };
'\\t'  { createCharToken('\t'); };
'\\v'  { createCharToken('\v'); };

'\\x' hex{2}   { createHexCharToken_2(ts + 2); };
'\\u' hex{4}   { createHexCharToken_4(ts + 2); };
'\\U' hex{8}   { createHexCharToken_8(ts + 2); };
'\\'  oct{3}   { createOctCharToken(ts + 1); };
'\\'  any      { createCharToken(ts[1]); };

utf8_1            { createCharToken(ts[0]); };
utf8_2 utf8_c     { createUtf8CharToken_2(ts); };
utf8_3 utf8_c{2}  { createUtf8CharToken_3(ts); };
utf8_4 utf8_c{3}  { createUtf8CharToken_4(ts); };

any               { createCharToken(ts[0]); };

*|;

#. . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . .
#
#  char class machine
#

char_class := |*

'-'    { createToken(TokenKind_Dash); };
']'    { createToken(TokenKind_EndCharClass); fgoto main; };

# alas, Ragel doesn't allow injecting sub-scanners, hence, copy-paste...

'\\d'  { createToken(TokenKind_StdCharClassDigit); };
'\\D'  { createToken(TokenKind_StdCharClassNonDigit); };
'\\h'  { createToken(TokenKind_StdCharClassHex); };
'\\H'  { createToken(TokenKind_StdCharClassNonHex); };
'\\w'  { createToken(TokenKind_StdCharClassWord); };
'\\W'  { createToken(TokenKind_StdCharClassNonWord); };
'\\s'  { createToken(TokenKind_StdCharClassSpace); };
'\\S'  { createToken(TokenKind_StdCharClassNonSpace); };

'\\0'  { createCharToken(0); };
'\\a'  { createCharToken('\a'); };
'\\b'  { createCharToken('\b'); };
'\\e'  { createCharToken('\x1b'); };
'\\f'  { createCharToken('\f'); };
'\\n'  { createCharToken('\n'); };
'\\r'  { createCharToken('\r'); };
'\\t'  { createCharToken('\t'); };
'\\v'  { createCharToken('\v'); };

'\\x' hex{2}   { createHexCharToken_2(ts + 2); };
'\\u' hex{4}   { createHexCharToken_4(ts + 2); };
'\\U' hex{8}   { createHexCharToken_8(ts + 2); };
'\\'  oct{3}   { createOctCharToken(ts + 1); };
'\\'  any      { createCharToken(ts[1]); };

utf8_1            { createCharToken(ts[0]); };
utf8_2 utf8_c     { createUtf8CharToken_2(ts); };
utf8_3 utf8_c{2}  { createUtf8CharToken_3(ts); };
utf8_4 utf8_c{3}  { createUtf8CharToken_4(ts); };

any               { createCharToken(ts[0]); };

*|;

#. . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . .
#
#  quantifier machine
#

quantifier := |*

dec+  { createNumberToken(atoi(ts)); };
','   { createToken(TokenKind_Comma); };
'}'   { createToken(TokenKind_EndQuantifier); fgoto main; };

ws;

*|;

}%%

//..............................................................................

void
Lexer::init () {
	%% write init;
}

bool
Lexer::exec () {
	%% write exec;
	return cs != axl_re_error;
}

//..............................................................................

} // namespace re
} // namespace axl


partial work for feature #217 not including option to postprocess warnings.log as this is likely to cause PARSE_ERRORS all the time however, left the option of postprocessing a single file only.

#include "config.h"

#include <string>
#include <fstream>
#include <iostream>
#include <cstdlib>
// New for directory recursion
#ifdef WIN32
	#include <dirent_msvc.h>
#else
	#include <dirent.h>
#endif

using std::cin;
using std::cout;
using std::cerr;
using std::ifstream;
using std::ofstream;
using std::string;
using std::ios_base;
using std::endl;

%%{
  machine PostProcess;
  write data;

  # end of file character
  EOF = 0;

  # all but end of file character
  default = ^0;

  # end of line
  EOL = ('\r'? . '\n') @{line++;};

  # matlab identifier
  IDENT = [A-Z\\a-z\-_][A-Z\\a-z0-9\-_]*;

  # matlab ifdentifier without underscore characters
  IDENT_WO_US = [A-Z\\a-z\-][A-Z\\a-z0-9\-]*;

  ANY_TAG = '<' . [^>]* . '>';

  action echo {
    fout << fpc;
    if (!quiet)
      cout << fpc;
  }

  action st_tok { tmp_p = p; }

  action echo_tok { fout.write(tmp_p, p - tmp_p);
                    if (!quiet)
                      cout.write(tmp_p, p-tmp_p); }

  rettype:= |*

    # matlab is typeless, so discard the type
    ('matlabtypesubstitute' . (' '?)) => {};

    # replace all "::" by "."
    ('::' . [A-Z\\a-z\-_]) => { fout << '.' << *(te-1); };

    # a word
    (any - [\n <>\&\$:,\t])+ => { fout.write(ts, te-ts); };

    # word separators
    ([\n <>\&\$:\t]) => {fout << *ts;};

    # bugfix: allow '>' in the end of typenames for Daniel's generic types
    (('<'|'$<$') . [^&$]* . ('$>$'|'>')) => { fout.write(ts, te-ts); };

    # comma or > end the type
    (',') => { fhold; fret; };
    ('>'|'$>$') => { p -=4; fret; };
  *|;

  # reconstruct return values
  retval:= |*
    (('<'|'$<$') . ('::')?) => { fcall rettype; };

    # matlab identifier (1 return value)
#    ('<' . (default - [,&])*) => { cerr.write(ts+4, te - ts-4); cerr << std::endl; fout.write(ts+4, te - ts-4); };

    (',' . (default - ('\&'|'\$'))*) => {
      if (*p == '\&')
        fout << " ";
      fout.write(ts+1, te-ts-1);
      if (*p == '\&')
        fout << "";
    };

    # end of return value
    ('>'|'$>$') => {
                  if(only_retval) { fout << " ="; }
                  fret;
                };

    # white spaces
    ([ \t\n]*) => { fout.write(ts, te - ts); };

    # typebreak
    ('
') => {};

    # other tags
    #(ANY_TAG) => { fout.write(ts, te-ts); };

    ('\\\*') => {};

  *|;

  retvals := |*

    ('<' | '$<$') => {};

    ('mlhs' . '\\\-'? . 'Inner' . '\\\-'? . 'Subst' . '\\\*'?) => { fcall retval; };

    ('>' | '$>$') => { fout << "] ="; fgoto main; };

    # white spaces and commata
    ([ \t\n]*) => { fout.write(ts, te - ts); };
    ('
' . [ \t\n]* . ',') => { fout << ",
\n"; };
    (',') => { fout << ", "; };

    # other tags
    (ANY_TAG) => { fout.write(ts, te-ts); };

    ('\\\*') => {};

  *|;

  # function name
  mtocsubst:= |*
    ('_' | '\\_');

    (IDENT_WO_US) => { fout.write(ts, te-ts); };

    ('_m_tsbus_cotm_' )  => { fout << ">"; fgoto main; };

    ( '\\-m\\-\\_\\-tsbus\\-\\_\\-cotm') => { fgoto main; };
  *|;

  main:= |*
   # list of return values
   ('mlhs' . ('\\-')? . 'Subst') => { fout << "function ["; only_retval = false; fgoto retvals; };

   # one return value
   ('mlhs' . ('\\-')? . 'Inner' . ('\\-')? . 'Subst' . ('\\\*')?) => { fout << "function "; only_retval = true; fcall retval; fout << " ="; };

   # no return values
   ('noret::' . ('\\-')? . 'substitute' . ('\\\*')?) => {fout << "function ";};

   # function name
   ('mtoc_subst_' | 'mtoc\\-\\_\\-subst\\-\\_\\-') => { fgoto mtocsubst; };

   ('::mtoc_subst_' | ('\\-'? . '::' . '\\-' . 'mtoc\\-\\_\\-subst\\-\\_')) => { fout << '.'; fgoto mtocsubst; };

   # matlab is typeless, so discard the type
   ('matlabtypesubstitute'. (' '?)) => {};

   # remove leading "::" (global namespace identifier)
   ([(,>] . '::') => { fout.write(ts, 1); };
   ( ('<' '$<$') . '::') => { fout.write(ts, 4); };

   # replace all "::" by "."
   ('::' . [A-Z\\a-z\-_]) => { fout << '.' << *(te-1); };

   # a word
   (any - [\n <>()[\]{}\&\$:.,;_\t\-])+ => { fout.write(ts, te-ts); };

   (')=0') => { fout << ")"; };

   # word separators
   ([\n <>()[\]{}\t:.;,_\&\$\-]) => {fout << *ts;};

   # a single dot stays a dot
   ('.') => {fout << '.';};
   *|;
}%%

class PostProcess {

private:
  string docdir_;
  int          line            , col;
  char        *ts              , *te;
  int          act             , have;
  int          cs;
  int          top;
  int          stack[5];
  bool         only_retval;
  bool         quiet_;
  bool         dry_run_;

public:
  /**
   * @class PostProcess
   *
   * @change{1,2,dw,2011-11-04} Changed the postprocessor interface from taking a single file argument to
   * assuming the passed string to be a folder whos contents are to be postprocessed.
   */
  // constructor
  PostProcess(const string &docdir, const bool quiet_flag, const bool dry_run_flag) :
    docdir_(docdir),
    line(1),
    ts(0), te(0), have(0),
    top(0), only_retval(false),
    quiet_(quiet_flag),
    dry_run_(dry_run_flag)
  { }

  int execute()
  {
    DIR *dp;
    if ((dp  = opendir(docdir_.c_str())) == NULL) {
      cerr << "Error opening directory " << docdir_ << endl;
      return -1;
    }

    struct dirent* dirp;
    string file;
    while ((dirp = readdir(dp)) != NULL) {
      file = string(dirp->d_name);
      // Process only html files
      if (file.substr(file.find_last_of(".") + 1) == "tex" || (file.substr(file.find_last_of(".") + 1) == "html" && file.find("8rl") == string::npos)) {
        postprocess(docdir_ + string("/") + file);
      }
    }
    closedir(dp);
    return 0;
  }

  // run postprocessor
  int postprocess(string file)
  {
    std::ios::sync_with_stdio(false);

    %% write init;

    ifstream is;
    try {
      is.open(file.c_str());
    } catch (std::ifstream::failure e) {
      cerr << "Exception opening/reading file";
      exit(-1);
    }

    is.seekg(0, ios_base::end);
    int length = is.tellg();
    is.seekg(0, ios_base::beg);

    char* buf = new char[(int)(1.1*length)];
    char* p = buf;
//    char * tmp_p = p;

    is.read(buf, length);
    is.close();

    ofstream fout2;
    if (!dry_run_)
    {
      try {
        fout2.open(file.c_str(), ios_base::trunc);
      } catch (std::ofstream::failure e) {
        cerr << "Exception opening/writing file";
        exit(-1);
      }
    }

    std::ostream * fout_ptr;

    if (dry_run_)
      fout_ptr = &std::cout;
    else
      fout_ptr = &fout2;
    std::ostream & fout = *fout_ptr;

    int len = is.gcount();
    char *pe = p + len;
    char *eof = pe;

    %% write exec;

    /* Check if we failed. */
    if ( cs == PostProcess_error )
    {
      /* Machine failed before finding a token. */
      cerr << file << ": PARSE ERROR " << endl;
      cerr.write(p, 100);
      exit(-1);
    }

    if (!dry_run_)
      fout2.close();
    delete buf;

    return 0;
  }
};

void usage()
{
  cout << "mtocpp_post Version " << MTOCPP_VERSION_MAJOR << "."
    << MTOCPP_VERSION_MINOR << endl;
  cout << "Usage: mtocpp_post [-q] target" << endl;
  cout << "\nOptions:\n  -q\t\tsuppresses debug output.\n" << endl;
  cout << "  -f\t\tsingle file argument instead of directory." << endl;
}

/**
 * @change{dw,1,4,2012-11-19} Added the possibility to directly specify a file target instead of a whole folder.
 */
int main(int argc, char ** argv)
{
  bool quiet = false;
  bool dry_run = false;
  bool isfile = false;
  string docdir;
  if(argc >= 2)
  {
    if (std::string("--help") == std::string(argv[1]))
    {
      usage();
      return 0;
    }
    if (argc == 3 && std::string("-q") == std::string(argv[1]))
    {
      quiet = true;
      docdir = argv[2];
    }
    else if (argc == 3 && std::string("-d") == std::string(argv[1]))
    {
      dry_run = true;
      docdir = argv[2];
    }
    else if (argc == 3 && std::string("-f") == std::string(argv[1]))
	{
	  isfile = true;
	  docdir = argv[2];
	}
    else if (argc == 3 && (std::string("-qf") == std::string(argv[1]) || std::string("-fq") == std::string(argv[1])))
	{
	  isfile = true;
	  quiet = true;
	  docdir = argv[2];
	}
    else if(argc == 2)
      docdir = argv[1];
    else
    {
      cerr << "wrong arguments!" << endl;
      usage();
      exit(-2);
    }
  }
  else
  {
    cerr << "wrong number of arguments!" << endl;
    usage();
    exit(-2);
  }

  if (!quiet) {
	  if (isfile) {
		  cout << "Running mtoc++ postprocessor on file " << docdir << endl;
	  } else {
		  cout << "Running mtoc++ postprocessor on directory " << docdir << endl;
	  }
  }

  PostProcess scanner(docdir, quiet, dry_run);
  if (isfile) {
	  scanner.postprocess(docdir);
  } else {
	  scanner.execute();
  }
  return 0;
}

/* vim: set et sw=2 ft=ragel: */

#include "config.h"

#include <string>
#include <fstream>
#include <iostream>
#include <cstdlib>
// New for directory recursion
#ifdef WIN32
	#include <dirent_msvc.h>
#else
	#include <dirent.h>
#endif

using std::cin;
using std::cout;
using std::cerr;
using std::ifstream;
using std::ofstream;
using std::string;
using std::ios_base;
using std::endl;

%%{
  machine PostProcess;
  write data;

  # end of file character
  EOF = 0;

  # all but end of file character
  default = ^0;

  # end of line
  EOL = ('\r'? . '\n') @{line++;};

  # matlab identifier
  IDENT = [A-Z\\a-z\-_][A-Z\\a-z0-9\-_]*;

  # matlab ifdentifier without underscore characters
  IDENT_WO_US = [A-Z\\a-z\-][A-Z\\a-z0-9\-]*;

  ANY_TAG = '<' . [^>]* . '>';

  action echo {
    fout << fpc;
    if (!quiet)
      cout << fpc;
  }

  action st_tok { tmp_p = p; }

  action echo_tok { fout.write(tmp_p, p - tmp_p);
                    if (!quiet)
                      cout.write(tmp_p, p-tmp_p); }

  rettype:= |*

    # matlab is typeless, so discard the type
    ('matlabtypesubstitute' . (' '?)) => {};

    # replace all "::" by "."
    ('::' . [A-Z\\a-z\-_]) => { fout << '.' << *(te-1); };

    # a word
    (any - [\n <>\&\$:,\t])+ => { fout.write(ts, te-ts); };

    # word separators
    ([\n <>\&\$:\t]) => {fout << *ts;};

    # bugfix: allow '>' in the end of typenames for Daniel's generic types
    (('<'|'$<$') . [^&$]* . ('$>$'|'>')) => { fout.write(ts, te-ts); };

    # comma or > end the type
    (',') => { fhold; fret; };
    ('>'|'$>$') => { p -=4; fret; };
  *|;

  # reconstruct return values
  retval:= |*
    (('<'|'$<$') . ('::')?) => { fcall rettype; };

    # matlab identifier (1 return value)
#    ('<' . (default - [,&])*) => { cerr.write(ts+4, te - ts-4); cerr << std::endl; fout.write(ts+4, te - ts-4); };

    (',' . (default - ('\&'|'\$'))*) => {
      if (*p == '\&')
        fout << " ";
      fout.write(ts+1, te-ts-1);
      if (*p == '\&')
        fout << "";
    };

    # end of return value
    ('>'|'$>$') => {
                  if(only_retval) { fout << " ="; }
                  fret;
                };

    # white spaces
    ([ \t\n]*) => { fout.write(ts, te - ts); };

    # typebreak
    ('
') => {};

    # other tags
    #(ANY_TAG) => { fout.write(ts, te-ts); };

    ('\\\*') => {};

  *|;

  retvals := |*

    ('<' | '$<$') => {};

    ('mlhs' . '\\\-'? . 'Inner' . '\\\-'? . 'Subst' . '\\\*'?) => { fcall retval; };

    ('>' | '$>$') => { fout << "] ="; fgoto main; };

    # white spaces and commata
    ([ \t\n]*) => { fout.write(ts, te - ts); };
    ('
' . [ \t\n]* . ',') => { fout << ",
\n"; };
    (',') => { fout << ", "; };

    # other tags
    (ANY_TAG) => { fout.write(ts, te-ts); };

    ('\\\*') => {};

  *|;

  # function name
  mtocsubst:= |*
    ('_' | '\\_');

    (IDENT_WO_US) => { fout.write(ts, te-ts); };

    ('_m_tsbus_cotm_' )  => { fout << ">"; fgoto main; };

    ( '\\-m\\-\\_\\-tsbus\\-\\_\\-cotm') => { fgoto main; };
  *|;

  main:= |*
   # list of return values
   ('mlhs' . ('\\-')? . 'Subst') => { fout << "function ["; only_retval = false; fgoto retvals; };

   # one return value
   ('mlhs' . ('\\-')? . 'Inner' . ('\\-')? . 'Subst' . ('\\\*')?) => { fout << "function "; only_retval = true; fcall retval; fout << " ="; };

   # no return values
   ('noret::' . ('\\-')? . 'substitute' . ('\\\*')?) => {fout << "function ";};

   # function name
   ('mtoc_subst_' | 'mtoc\\-\\_\\-subst\\-\\_\\-') => { fgoto mtocsubst; };

   ('::mtoc_subst_' | ('\\-'? . '::' . '\\-' . 'mtoc\\-\\_\\-subst\\-\\_')) => { fout << '.'; fgoto mtocsubst; };

   # matlab is typeless, so discard the type
   ('matlabtypesubstitute'. (' '?)) => {};

   # remove leading "::" (global namespace identifier)
   ([(,>] . '::') => { fout.write(ts, 1); };
   ( ('<' '$<$') . '::') => { fout.write(ts, 4); };

   # replace all "::" by "."
   ('::' . [A-Z\\a-z\-_]) => { fout << '.' << *(te-1); };

   # a word
   (any - [\n <>()[\]{}\&\$:.,;_\t\-])+ => { fout.write(ts, te-ts); };

   (')=0') => { fout << ")"; };

   # word separators
   ([\n <>()[\]{}\t:.;,_\&\$\-]) => {fout << *ts;};

   # a single dot stays a dot
   ('.') => {fout << '.';};
   *|;
}%%

class PostProcess {

private:
  string docdir_;
  int          line            , col;
  char        *ts              , *te;
  int          act             , have;
  int          cs;
  int          top;
  int          stack[5];
  bool         only_retval;
  bool         quiet_;
  bool         dry_run_;

public:
  /**
   * @class PostProcess
   *
   * @change{1,2,dw,2011-11-04} Changed the postprocessor interface from taking a single file argument to
   * assuming the passed string to be a folder whos contents are to be postprocessed.
   */
  // constructor
  PostProcess(const string &docdir, const bool quiet_flag, const bool dry_run_flag) :
    docdir_(docdir),
    line(1),
    ts(0), te(0), have(0),
    top(0), only_retval(false),
    quiet_(quiet_flag),
    dry_run_(dry_run_flag)
  { }

  int execute()
  {
    DIR *dp;
    if ((dp  = opendir(docdir_.c_str())) == NULL) {
      cerr << "Error opening directory " << docdir_ << endl;
      return -1;
    }

    struct dirent* dirp;
    string file;
    while ((dirp = readdir(dp)) != NULL) {
      file = string(dirp->d_name);
      // Process only html files
      if (file.substr(file.find_last_of(".") + 1) == "tex" || (file.substr(file.find_last_of(".") + 1) == "html" && file.find("8rl") == string::npos)) {
        postprocess(docdir_ + string("/") + file);
      }
    }
    closedir(dp);
    return 0;
  }

  // run postprocessor
  int postprocess(string file)
  {
    std::ios::sync_with_stdio(false);

    %% write init;

    ifstream is;
    try {
      is.open(file.c_str());
    } catch (std::ifstream::failure e) {
      cerr << "Exception opening/reading file";
      exit(-1);
    }

    is.seekg(0, ios_base::end);
    int length = is.tellg();
    is.seekg(0, ios_base::beg);

    char* buf = new char[(int)(1.1*length)];
    char* p = buf;
//    char * tmp_p = p;

    is.read(buf, length);
    is.close();

    ofstream fout2;
    if (!dry_run_)
    {
      try {
        fout2.open(file.c_str(), ios_base::trunc);
      } catch (std::ofstream::failure e) {
        cerr << "Exception opening/writing file";
        exit(-1);
      }
    }

    std::ostream * fout_ptr;

    if (dry_run_)
      fout_ptr = &std::cout;
    else
      fout_ptr = &fout2;
    std::ostream & fout = *fout_ptr;

    int len = is.gcount();
    char *pe = p + len;
    char *eof = pe;

    %% write exec;

    /* Check if we failed. */
    if ( cs == PostProcess_error )
    {
      /* Machine failed before finding a token. */
      cerr << file << ": PARSE ERROR " << endl;
      cerr.write(p, 100);
      exit(-1);
    }

    if (!dry_run_)
      fout2.close();
    delete buf;

    return 0;
  }
};

void usage()
{
  cout << "mtocpp_post Version " << MTOCPP_VERSION_MAJOR << "."
    << MTOCPP_VERSION_MINOR << endl;
  cout << "Usage: mtocpp_post [-q] filename" << endl;
  cout << "\nOptions:\n  -q\t\tsuppresses debug output." << endl;
}

int main(int argc, char ** argv)
{
  bool quiet = false;
  bool dry_run = false;
  string docdir;
  if(argc >= 2)
  {
    if (std::string("--help") == std::string(argv[1]))
    {
      usage();
      return 0;
    }
    if (argc == 3 && std::string("-q") == std::string(argv[1]))
    {
      quiet = true;
      docdir = argv[2];
    }
    else if (argc == 3 && std::string("-d") == std::string(argv[1]))
    {
      dry_run = true;
      docdir = argv[2];
    }
    else if(argc == 2)
      docdir = argv[1];
    else
    {
      cerr << "wrong arguments!" << endl;
      usage();
      exit(-2);
    }
  }
  else
  {
    cerr << "wrong number of arguments!" << endl;
    usage();
    exit(-2);
  }

  if (!quiet)
    cout << "Running mtoc++ postprocessor on directory " << docdir << endl;

  PostProcess scanner(docdir, quiet, dry_run);
  scanner.execute();
  return 0;
}

/* vim: set et sw=2 ft=ragel: */