<commit_before>/**
 * @file rpc_manager.cpp
 *
 * @date Feb 7, 2017
 */

#include <cassert>
#include <iostream>

#include <derecho/core/detail/rpc_manager.hpp>

namespace derecho {

namespace rpc {

thread_local bool _in_rpc_handler = false;

RPCManager::~RPCManager() {
    thread_shutdown = true;
    if(rpc_thread.joinable()) {
        rpc_thread.join();
    }
}

void RPCManager::create_connections() {
    connections = std::make_unique<sst::P2PConnections>(sst::P2PParams{nid, {nid}, view_manager.view_max_window_size, view_manager.view_max_payload_size + sizeof(header)});
}

void RPCManager::destroy_remote_invocable_class(uint32_t instance_id) {
    //Delete receiver functions that were added by this class/subgroup
    for(auto receivers_iterator = receivers->begin();
        receivers_iterator != receivers->end();) {
        if(receivers_iterator->first.subgroup_id == instance_id) {
            receivers_iterator = receivers->erase(receivers_iterator);
        } else {
            receivers_iterator++;
        }
    }
    //Deliver a node_removed_from_shard_exception to the QueryResults for this class
    //Important: This only works because the Replicated destructor runs before the
    //wrapped_this member is destroyed; otherwise the PendingResults we're referencing
    //would already have been deleted.
    std::lock_guard<std::mutex> lock(pending_results_mutex);
    while(!pending_results_to_fulfill[instance_id].empty()) {
        pending_results_to_fulfill[instance_id].front().get().set_exception_for_caller_removed();
        pending_results_to_fulfill[instance_id].pop();
    }
    while(!fulfilled_pending_results[instance_id].empty()) {
        fulfilled_pending_results[instance_id].front().get().set_exception_for_caller_removed();
        fulfilled_pending_results[instance_id].pop_front();
    }
}

void RPCManager::start_listening() {
    std::lock_guard<std::mutex> lock(thread_start_mutex);
    thread_start = true;
    thread_start_cv.notify_all();
}

std::exception_ptr RPCManager::receive_message(
        const Opcode& indx, const node_id_t& received_from, char const* const buf,
        std::size_t payload_size, const std::function<char*(int)>& out_alloc) {
    using namespace remote_invocation_utilities;
    assert(payload_size);
    auto receiver_function_entry = receivers->find(indx);
    if(receiver_function_entry == receivers->end()) {
        dbg_default_error("Received an RPC message with an invalid RPC opcode! Opcode was ({}, {}, {}, {}).",
                          indx.class_id, indx.subgroup_id, indx.function_id, indx.is_reply);
        //TODO: We should reply with some kind of "no such method" error in this case
        return std::exception_ptr{};
    }
    std::size_t reply_header_size = header_space();
    recv_ret reply_return = receiver_function_entry->second(
            &rdv, received_from, buf,
            [&out_alloc, &reply_header_size](std::size_t size) {
                return out_alloc(size + reply_header_size) + reply_header_size;
            });
    auto* reply_buf = reply_return.payload;
    if(reply_buf) {
        reply_buf -= reply_header_size;
        const auto id = reply_return.opcode;
        const auto size = reply_return.size;
        populate_header(reply_buf, size, id, nid, 0);
    }
    return reply_return.possible_exception;
}

std::exception_ptr RPCManager::parse_and_receive(char* buf, std::size_t size,
                                                 const std::function<char*(int)>& out_alloc) {
    using namespace remote_invocation_utilities;
    assert(size >= header_space());
    std::size_t payload_size = size;
    Opcode indx;
    node_id_t received_from;
    uint32_t flags;
    retrieve_header(&rdv, buf, payload_size, indx, received_from, flags);
    return receive_message(indx, received_from, buf + header_space(),
                           payload_size, out_alloc);
}

void RPCManager::rpc_message_handler(subgroup_id_t subgroup_id, node_id_t sender_id,
                                     char* msg_buf, uint32_t buffer_size) {
    // WARNING: This assumes the current view doesn't change during execution!
    // (It accesses curr_view without a lock).

    // set the thread local rpc_handler context
    _in_rpc_handler = true;

    //Use the reply-buffer allocation lambda to detect whether parse_and_receive generated a reply
    size_t reply_size = 0;
    char* reply_buf;
    parse_and_receive(msg_buf, buffer_size,
                      [this, &reply_buf, &reply_size, &sender_id](size_t size) -> char* {
                          reply_size = size;
                          if(reply_size <= connections->get_max_p2p_size()) {
                              reply_buf = (char*)connections->get_sendbuffer_ptr(
                                      connections->get_node_rank(sender_id), sst::REQUEST_TYPE::RPC_REPLY);
                              return reply_buf;
                          } else {
                              // the reply size is too large - not part of the design to handle it
                              return nullptr;
                          }
                      });
    if(sender_id == nid) {
        //This is a self-receive of an RPC message I sent, so I have a reply-map that needs fulfilling
        int my_shard = view_manager.curr_view->multicast_group->get_subgroup_settings().at(subgroup_id).shard_num;
        std::unique_lock<std::mutex> lock(pending_results_mutex);
        // because of a race condition, toFulfillQueue can genuinely be empty
        // so we shouldn't assert that it is empty
        // instead we should sleep on a condition variable and let the main thread that called the orderedSend signal us
        // although the race condition is infinitely rare
        pending_results_cv.wait(lock, [&]() { return !pending_results_to_fulfill[subgroup_id].empty(); });
        //We now know the membership of "all nodes in my shard of the subgroup" in the current view
        pending_results_to_fulfill[subgroup_id].front().get().fulfill_map(
                view_manager.curr_view->subgroup_shard_views.at(subgroup_id).at(my_shard).members);
        fulfilled_pending_results[subgroup_id].push_back(
                std::move(pending_results_to_fulfill[subgroup_id].front()));
        pending_results_to_fulfill[subgroup_id].pop();
        if(reply_size > 0) {
            //Since this was a self-receive, the reply also goes to myself
            parse_and_receive(
                    reply_buf, reply_size,
                    [](size_t size) -> char* { assert_always(false); });
        }
    } else if(reply_size > 0) {
        //Otherwise, the only thing to do is send the reply (if there was one)
        connections->send(connections->get_node_rank(sender_id));
    }

    // clear the thread local rpc_handler context
    _in_rpc_handler = false;
}

void RPCManager::p2p_message_handler(node_id_t sender_id, char* msg_buf, uint32_t buffer_size) {
    using namespace remote_invocation_utilities;
    const std::size_t header_size = header_space();
    std::size_t payload_size;
    Opcode indx;
    node_id_t received_from;
    uint32_t flags;
    retrieve_header(nullptr, msg_buf, payload_size, indx, received_from, flags);
    size_t reply_size = 0;
    if(indx.is_reply) {
        // REPLYs can be handled here because they do not block.
        receive_message(indx, received_from, msg_buf + header_size, payload_size,
                        [this, &msg_buf, &buffer_size, &reply_size, &sender_id](size_t _size) -> char* {
                            reply_size = _size;
                            if(reply_size <= buffer_size) {
                                return (char*)connections->get_sendbuffer_ptr(
                                        connections->get_node_rank(sender_id), sst::REQUEST_TYPE::P2P_REPLY);
                            }
                            return nullptr;
                        });
        if(reply_size > 0) {
            connections->send(connections->get_node_rank(sender_id));
        }
    } else if(RPC_HEADER_FLAG_TST(flags, CASCADE)) {
        // TODO: what is the lifetime of msg_buf? discuss with Sagar to make
        // sure the buffers are safely managed.
        // for cascading messages, we create a new thread.
        throw derecho::derecho_exception("Cascading P2P Send/Queries to be implemented!");
    } else {
        // send to fifo queue.
        std::unique_lock<std::mutex> lock(fifo_queue_mutex);
        fifo_queue.emplace(sender_id, msg_buf, buffer_size);
        fifo_queue_cv.notify_one();
    }
}

void RPCManager::new_view_callback(const View& new_view) {
    std::lock_guard<std::mutex> connections_lock(p2p_connections_mutex);
    connections = std::make_unique<sst::P2PConnections>(std::move(*connections), new_view.members);
    dbg_default_debug("Created new connections among the new view members");
    std::lock_guard<std::mutex> lock(pending_results_mutex);
    for(auto& fulfilled_pending_results_pair : fulfilled_pending_results) {
        const subgroup_id_t subgroup_id = fulfilled_pending_results_pair.first;
        //For each PendingResults in this subgroup, check the departed list of each shard
        //the subgroup, and call set_exception_for_removed_node for the departed nodes
        for(auto pending_results_iter = fulfilled_pending_results_pair.second.begin();
            pending_results_iter != fulfilled_pending_results_pair.second.end();) {
            //Garbage-collect PendingResults references that are obsolete
            if(pending_results_iter->get().all_responded()) {
                pending_results_iter = fulfilled_pending_results_pair.second.erase(pending_results_iter);
            } else {
                for(uint32_t shard_num = 0;
                    shard_num < new_view.subgroup_shard_views[subgroup_id].size();
                    ++shard_num) {
                    for(auto removed_id : new_view.subgroup_shard_views[subgroup_id][shard_num].departed) {
                        //This will do nothing if removed_id was never in the
                        //shard this PendingResult corresponds to
                        dbg_default_debug("Setting exception for removed node {} on PendingResults for subgroup {}, shard {}", removed_id, subgroup_id, shard_num);
                        pending_results_iter->get().set_exception_for_removed_node(removed_id);
                    }
                }
                pending_results_iter++;
            }
        }
    }
}

bool RPCManager::finish_rpc_send(subgroup_id_t subgroup_id, PendingBase& pending_results_handle) {
    std::lock_guard<std::mutex> lock(pending_results_mutex);
    pending_results_to_fulfill[subgroup_id].push(pending_results_handle);
    pending_results_cv.notify_all();
    return true;
}

volatile char* RPCManager::get_sendbuffer_ptr(uint32_t dest_id, sst::REQUEST_TYPE type) {
    auto dest_rank = connections->get_node_rank(dest_id);
    volatile char* buf;
    do {
        buf = connections->get_sendbuffer_ptr(dest_rank, type);
    } while(!buf);
    return buf;
}

void RPCManager::finish_p2p_send(node_id_t dest_id, subgroup_id_t dest_subgroup_id, PendingBase& pending_results_handle) {
    connections->send(connections->get_node_rank(dest_id));
    pending_results_handle.fulfill_map({dest_id});
    std::lock_guard<std::mutex> lock(pending_results_mutex);
    fulfilled_pending_results[dest_subgroup_id].push_back(pending_results_handle);
}

void RPCManager::fifo_worker() {
    pthread_setname_np(pthread_self(), "fifo_thread");
    using namespace remote_invocation_utilities;
    const std::size_t header_size = header_space();
    std::size_t payload_size;
    Opcode indx;
    node_id_t received_from;
    uint32_t flags;
    size_t reply_size = 0;
    fifo_req request;

    while(!thread_shutdown) {
        {
            std::unique_lock<std::mutex> lock(fifo_queue_mutex);
            fifo_queue_cv.wait(lock, [&]() { return !fifo_queue.empty() || thread_shutdown; });
            if(thread_shutdown) {
                break;
            }
            request = fifo_queue.front();
            fifo_queue.pop();
        }
        retrieve_header(nullptr, request.msg_buf, payload_size, indx, received_from, flags);
        if(indx.is_reply || RPC_HEADER_FLAG_TST(flags, CASCADE)) {
            dbg_default_error("Invalid rpc message in fifo queue: is_reply={}, is_cascading={}",
                              indx.is_reply, RPC_HEADER_FLAG_TST(flags, CASCADE));
            throw derecho::derecho_exception("invalid rpc message in fifo queue...crash.");
        }
        receive_message(indx, received_from, request.msg_buf + header_size, payload_size,
                        [this, &reply_size, &request](size_t _size) -> char* {
                            reply_size = _size;
                            if(reply_size <= request.buffer_size) {
                                return (char*)connections->get_sendbuffer_ptr(
                                        connections->get_node_rank(request.sender_id), sst::REQUEST_TYPE::P2P_REPLY);
                            }
                            return nullptr;
                        });
        if(reply_size > 0) {
            connections->send(connections->get_node_rank(request.sender_id));
        } else {
            // hack for now to "simulate" a reply for p2p_sends to functions that do not generate a reply
            char* buf = connections->get_sendbuffer_ptr(connections->get_node_rank(request.sender_id), sst::REQUEST_TYPE::P2P_REPLY);
            buf[0] = 0;
            connections->send(connections->get_node_rank(request.sender_id));
        }
    }
}

void RPCManager::p2p_receive_loop() {
    pthread_setname_np(pthread_self(), "rpc_thread");
    uint64_t max_payload_size = getConfUInt64(CONF_SUBGROUP_DEFAULT_MAX_PAYLOAD_SIZE);
    // set the thread local rpc_handler context
    _in_rpc_handler = true;

    while(!thread_start) {
        std::unique_lock<std::mutex> lock(thread_start_mutex);
        thread_start_cv.wait(lock, [this]() { return thread_start; });
    }
    dbg_default_debug("P2P listening thread started");
    // start the fifo worker thread
    fifo_worker_thread = std::thread(&RPCManager::fifo_worker, this);
    // loop event
    while(!thread_shutdown) {
        std::lock_guard<std::mutex> connections_lock(p2p_connections_mutex);
        auto optional_reply_pair = connections->probe_all();
        if(optional_reply_pair) {
            auto reply_pair = optional_reply_pair.value();
            p2p_message_handler(reply_pair.first, (char*)reply_pair.second, max_payload_size);
        }
    }
    // stop fifo worker.
    fifo_queue_cv.notify_one();
    fifo_worker_thread.join();
}

bool in_rpc_handler() {
    return _in_rpc_handler;
}
}  // namespace rpc
}  // namespace derecho
<commit_msg>Fixed a potential deadlock in rpc_message_handler<commit_after>/**
 * @file rpc_manager.cpp
 *
 * @date Feb 7, 2017
 */

#include <cassert>
#include <iostream>

#include <derecho/core/detail/rpc_manager.hpp>

namespace derecho {

namespace rpc {

thread_local bool _in_rpc_handler = false;

RPCManager::~RPCManager() {
    thread_shutdown = true;
    if(rpc_thread.joinable()) {
        rpc_thread.join();
    }
}

void RPCManager::create_connections() {
    connections = std::make_unique<sst::P2PConnections>(sst::P2PParams{nid, {nid}, view_manager.view_max_window_size, view_manager.view_max_payload_size + sizeof(header)});
}

void RPCManager::destroy_remote_invocable_class(uint32_t instance_id) {
    //Delete receiver functions that were added by this class/subgroup
    for(auto receivers_iterator = receivers->begin();
        receivers_iterator != receivers->end();) {
        if(receivers_iterator->first.subgroup_id == instance_id) {
            receivers_iterator = receivers->erase(receivers_iterator);
        } else {
            receivers_iterator++;
        }
    }
    //Deliver a node_removed_from_shard_exception to the QueryResults for this class
    //Important: This only works because the Replicated destructor runs before the
    //wrapped_this member is destroyed; otherwise the PendingResults we're referencing
    //would already have been deleted.
    std::lock_guard<std::mutex> lock(pending_results_mutex);
    while(!pending_results_to_fulfill[instance_id].empty()) {
        pending_results_to_fulfill[instance_id].front().get().set_exception_for_caller_removed();
        pending_results_to_fulfill[instance_id].pop();
    }
    while(!fulfilled_pending_results[instance_id].empty()) {
        fulfilled_pending_results[instance_id].front().get().set_exception_for_caller_removed();
        fulfilled_pending_results[instance_id].pop_front();
    }
}

void RPCManager::start_listening() {
    std::lock_guard<std::mutex> lock(thread_start_mutex);
    thread_start = true;
    thread_start_cv.notify_all();
}

std::exception_ptr RPCManager::receive_message(
        const Opcode& indx, const node_id_t& received_from, char const* const buf,
        std::size_t payload_size, const std::function<char*(int)>& out_alloc) {
    using namespace remote_invocation_utilities;
    assert(payload_size);
    auto receiver_function_entry = receivers->find(indx);
    if(receiver_function_entry == receivers->end()) {
        dbg_default_error("Received an RPC message with an invalid RPC opcode! Opcode was ({}, {}, {}, {}).",
                          indx.class_id, indx.subgroup_id, indx.function_id, indx.is_reply);
        //TODO: We should reply with some kind of "no such method" error in this case
        return std::exception_ptr{};
    }
    std::size_t reply_header_size = header_space();
    recv_ret reply_return = receiver_function_entry->second(
            &rdv, received_from, buf,
            [&out_alloc, &reply_header_size](std::size_t size) {
                return out_alloc(size + reply_header_size) + reply_header_size;
            });
    auto* reply_buf = reply_return.payload;
    if(reply_buf) {
        reply_buf -= reply_header_size;
        const auto id = reply_return.opcode;
        const auto size = reply_return.size;
        populate_header(reply_buf, size, id, nid, 0);
    }
    return reply_return.possible_exception;
}

std::exception_ptr RPCManager::parse_and_receive(char* buf, std::size_t size,
                                                 const std::function<char*(int)>& out_alloc) {
    using namespace remote_invocation_utilities;
    assert(size >= header_space());
    std::size_t payload_size = size;
    Opcode indx;
    node_id_t received_from;
    uint32_t flags;
    retrieve_header(&rdv, buf, payload_size, indx, received_from, flags);
    return receive_message(indx, received_from, buf + header_space(),
                           payload_size, out_alloc);
}

void RPCManager::rpc_message_handler(subgroup_id_t subgroup_id, node_id_t sender_id,
                                     char* msg_buf, uint32_t buffer_size) {
    // WARNING: This assumes the current view doesn't change during execution!
    // (It accesses curr_view without a lock).

    // set the thread local rpc_handler context
    _in_rpc_handler = true;

    //Use the reply-buffer allocation lambda to detect whether parse_and_receive generated a reply
    size_t reply_size = 0;
    char* reply_buf;
    parse_and_receive(msg_buf, buffer_size,
                      [this, &reply_buf, &reply_size, &sender_id](size_t size) -> char* {
                          reply_size = size;
                          if(reply_size <= connections->get_max_p2p_size()) {
                              reply_buf = (char*)connections->get_sendbuffer_ptr(
                                      connections->get_node_rank(sender_id), sst::REQUEST_TYPE::RPC_REPLY);
                              return reply_buf;
                          } else {
                              // the reply size is too large - not part of the design to handle it
                              return nullptr;
                          }
                      });
    if(sender_id == nid) {
        //This is a self-receive of an RPC message I sent, so I have a reply-map that needs fulfilling
        const uint32_t my_shard = view_manager.curr_view->my_subgroups.at(subgroup_id);
        {
            std::unique_lock<std::mutex> lock(pending_results_mutex);
            // because of a race condition, pending_results_to_fulfill can genuinely be empty
            // so before accessing it we should sleep on a condition variable and let the main
            // thread that called the orderedSend signal us
            // although the race condition is infinitely rare
            pending_results_cv.wait(lock, [&]() { return !pending_results_to_fulfill[subgroup_id].empty(); });
            //We now know the membership of "all nodes in my shard of the subgroup" in the current view
            pending_results_to_fulfill[subgroup_id].front().get().fulfill_map(
                    view_manager.curr_view->subgroup_shard_views.at(subgroup_id).at(my_shard).members);
            fulfilled_pending_results[subgroup_id].push_back(
                    std::move(pending_results_to_fulfill[subgroup_id].front()));
            pending_results_to_fulfill[subgroup_id].pop();
        } //release pending_results_mutex
        if(reply_size > 0) {
            //Since this was a self-receive, the reply also goes to myself
            parse_and_receive(
                    reply_buf, reply_size,
                    [](size_t size) -> char* { assert_always(false); });
        }
    } else if(reply_size > 0) {
        //Otherwise, the only thing to do is send the reply (if there was one)
        connections->send(connections->get_node_rank(sender_id));
    }

    // clear the thread local rpc_handler context
    _in_rpc_handler = false;
}

void RPCManager::p2p_message_handler(node_id_t sender_id, char* msg_buf, uint32_t buffer_size) {
    using namespace remote_invocation_utilities;
    const std::size_t header_size = header_space();
    std::size_t payload_size;
    Opcode indx;
    node_id_t received_from;
    uint32_t flags;
    retrieve_header(nullptr, msg_buf, payload_size, indx, received_from, flags);
    size_t reply_size = 0;
    if(indx.is_reply) {
        // REPLYs can be handled here because they do not block.
        receive_message(indx, received_from, msg_buf + header_size, payload_size,
                        [this, &msg_buf, &buffer_size, &reply_size, &sender_id](size_t _size) -> char* {
                            reply_size = _size;
                            if(reply_size <= buffer_size) {
                                return (char*)connections->get_sendbuffer_ptr(
                                        connections->get_node_rank(sender_id), sst::REQUEST_TYPE::P2P_REPLY);
                            }
                            return nullptr;
                        });
        if(reply_size > 0) {
            connections->send(connections->get_node_rank(sender_id));
        }
    } else if(RPC_HEADER_FLAG_TST(flags, CASCADE)) {
        // TODO: what is the lifetime of msg_buf? discuss with Sagar to make
        // sure the buffers are safely managed.
        // for cascading messages, we create a new thread.
        throw derecho::derecho_exception("Cascading P2P Send/Queries to be implemented!");
    } else {
        // send to fifo queue.
        std::unique_lock<std::mutex> lock(fifo_queue_mutex);
        fifo_queue.emplace(sender_id, msg_buf, buffer_size);
        fifo_queue_cv.notify_one();
    }
}

void RPCManager::new_view_callback(const View& new_view) {
    std::lock_guard<std::mutex> connections_lock(p2p_connections_mutex);
    connections = std::make_unique<sst::P2PConnections>(std::move(*connections), new_view.members);
    dbg_default_debug("Created new connections among the new view members");
    std::lock_guard<std::mutex> lock(pending_results_mutex);
    for(auto& fulfilled_pending_results_pair : fulfilled_pending_results) {
        const subgroup_id_t subgroup_id = fulfilled_pending_results_pair.first;
        //For each PendingResults in this subgroup, check the departed list of each shard
        //the subgroup, and call set_exception_for_removed_node for the departed nodes
        for(auto pending_results_iter = fulfilled_pending_results_pair.second.begin();
            pending_results_iter != fulfilled_pending_results_pair.second.end();) {
            //Garbage-collect PendingResults references that are obsolete
            if(pending_results_iter->get().all_responded()) {
                pending_results_iter = fulfilled_pending_results_pair.second.erase(pending_results_iter);
            } else {
                for(uint32_t shard_num = 0;
                    shard_num < new_view.subgroup_shard_views[subgroup_id].size();
                    ++shard_num) {
                    for(auto removed_id : new_view.subgroup_shard_views[subgroup_id][shard_num].departed) {
                        //This will do nothing if removed_id was never in the
                        //shard this PendingResult corresponds to
                        dbg_default_debug("Setting exception for removed node {} on PendingResults for subgroup {}, shard {}", removed_id, subgroup_id, shard_num);
                        pending_results_iter->get().set_exception_for_removed_node(removed_id);
                    }
                }
                pending_results_iter++;
            }
        }
    }
}

bool RPCManager::finish_rpc_send(subgroup_id_t subgroup_id, PendingBase& pending_results_handle) {
    std::lock_guard<std::mutex> lock(pending_results_mutex);
    pending_results_to_fulfill[subgroup_id].push(pending_results_handle);
    pending_results_cv.notify_all();
    return true;
}

volatile char* RPCManager::get_sendbuffer_ptr(uint32_t dest_id, sst::REQUEST_TYPE type) {
    auto dest_rank = connections->get_node_rank(dest_id);
    volatile char* buf;
    do {
        buf = connections->get_sendbuffer_ptr(dest_rank, type);
    } while(!buf);
    return buf;
}

void RPCManager::finish_p2p_send(node_id_t dest_id, subgroup_id_t dest_subgroup_id, PendingBase& pending_results_handle) {
    connections->send(connections->get_node_rank(dest_id));
    pending_results_handle.fulfill_map({dest_id});
    std::lock_guard<std::mutex> lock(pending_results_mutex);
    fulfilled_pending_results[dest_subgroup_id].push_back(pending_results_handle);
}

void RPCManager::fifo_worker() {
    pthread_setname_np(pthread_self(), "fifo_thread");
    using namespace remote_invocation_utilities;
    const std::size_t header_size = header_space();
    std::size_t payload_size;
    Opcode indx;
    node_id_t received_from;
    uint32_t flags;
    size_t reply_size = 0;
    fifo_req request;

    while(!thread_shutdown) {
        {
            std::unique_lock<std::mutex> lock(fifo_queue_mutex);
            fifo_queue_cv.wait(lock, [&]() { return !fifo_queue.empty() || thread_shutdown; });
            if(thread_shutdown) {
                break;
            }
            request = fifo_queue.front();
            fifo_queue.pop();
        }
        retrieve_header(nullptr, request.msg_buf, payload_size, indx, received_from, flags);
        if(indx.is_reply || RPC_HEADER_FLAG_TST(flags, CASCADE)) {
            dbg_default_error("Invalid rpc message in fifo queue: is_reply={}, is_cascading={}",
                              indx.is_reply, RPC_HEADER_FLAG_TST(flags, CASCADE));
            throw derecho::derecho_exception("invalid rpc message in fifo queue...crash.");
        }
        receive_message(indx, received_from, request.msg_buf + header_size, payload_size,
                        [this, &reply_size, &request](size_t _size) -> char* {
                            reply_size = _size;
                            if(reply_size <= request.buffer_size) {
                                return (char*)connections->get_sendbuffer_ptr(
                                        connections->get_node_rank(request.sender_id), sst::REQUEST_TYPE::P2P_REPLY);
                            }
                            return nullptr;
                        });
        if(reply_size > 0) {
            connections->send(connections->get_node_rank(request.sender_id));
        } else {
            // hack for now to "simulate" a reply for p2p_sends to functions that do not generate a reply
            char* buf = connections->get_sendbuffer_ptr(connections->get_node_rank(request.sender_id), sst::REQUEST_TYPE::P2P_REPLY);
            buf[0] = 0;
            connections->send(connections->get_node_rank(request.sender_id));
        }
    }
}

void RPCManager::p2p_receive_loop() {
    pthread_setname_np(pthread_self(), "rpc_thread");
    uint64_t max_payload_size = getConfUInt64(CONF_SUBGROUP_DEFAULT_MAX_PAYLOAD_SIZE);
    // set the thread local rpc_handler context
    _in_rpc_handler = true;

    while(!thread_start) {
        std::unique_lock<std::mutex> lock(thread_start_mutex);
        thread_start_cv.wait(lock, [this]() { return thread_start; });
    }
    dbg_default_debug("P2P listening thread started");
    // start the fifo worker thread
    fifo_worker_thread = std::thread(&RPCManager::fifo_worker, this);
    // loop event
    while(!thread_shutdown) {
        std::lock_guard<std::mutex> connections_lock(p2p_connections_mutex);
        auto optional_reply_pair = connections->probe_all();
        if(optional_reply_pair) {
            auto reply_pair = optional_reply_pair.value();
            p2p_message_handler(reply_pair.first, (char*)reply_pair.second, max_payload_size);
        }
    }
    // stop fifo worker.
    fifo_queue_cv.notify_one();
    fifo_worker_thread.join();
}

bool in_rpc_handler() {
    return _in_rpc_handler;
}
}  // namespace rpc
}  // namespace derecho
<|endoftext|>


<commit_before>#include <cstdio>
#include <dxjson/dxjson.h>

int main() {
  using namespace dx;
  using namespace std;
  try
  {
    JSON j1(JSON_OBJECT);

    std::cout<<"Eps = "<<JSON::getEpsilon();
    j1["key"] = 12;
    JSON j2(JSON_OBJECT);
    j2["blah"] = "sdsdsd";
    j2["blah"] = "key";
    j2["lala"] = j1[j2["blah"]];
    std::cout<<"\nj2 = \n"<<j2.toString()<<"\n";
    j2.erase("lala");
    std::cout<<"\n j2 after erasing 'lala' = \n"<<j2.toString()<<"\n";
    // JSON parse tests
    JSON j3 = JSON::parse("{\"blah\": [  21,232,\"foo\" , {\"key\": \"val1\"}, true, false, null]}");
    j3["blah"].push_back(1.23456789101112);
    j3["blah"].push_back("dsdsd");
    j3["blah"].push_back(Null());
    j3["blah"].push_back(12.212);
    j3["foo"] = vector<int>(5,5);
    std::map<std::string, int> mp;
    mp["lala"] = 12.11e-212;
    mp["dsdsd"] = 1212;
    j3["map"] = mp;
    std::cout<<"\nj3 = "<<j3.toString();
    std::cout<<"\nj3[blah] = "<<j3["blah"].toString();
    std::cout<<"\nj3[blah][2] = "<<j3["blah"][2].toString()<<"\n";

    j3["blah"].erase(2);
    std::cout<<"\nBlah after erasing indx = 2\n"<<j3["blah"].toString()<<"\n";

    JSON j4;
    std::string str = "{\"清华大学\": [\"this should look like second element\", \"\\u6e05\\u534e\\u5927\\u5b66\", \"\\n\\b\\t\\\"\"]    }";
    j4 = JSON::parse(str);

    std::cout<<"j4 = "<<j4.toString()<<"\n";

    JSON j5(JSON_BOOLEAN);
    j5 = true;
    std::cout<<"\nj5 = "<<j5.toString()<<"\n";

    // Equality tests
    std::cout<<"\nj4 == j5: "<<((j4==j5)?"true":"false");

    JSON j5_copy = j5;
    std::cout<<"\nj5_copy == j5: "<<((j5_copy==j5) ? "true" : "false")<<"\n";

    JSON j6 = 12.21;
    JSON j7 = 12.22;
    assert(j6 != j7);

    JSON j8(JSON_ARRAY);
    j8.push_back(12.21);
    j8.push_back("hello");
    j8.push_back(j8);
    std::cout<<"\nj8 = "<<j8.toString()<<"\n";
    JSON j9 = j8;
    assert(j9 == j8);
    j9.erase(2);
    assert(j9 != j8);
    assert(JSON(JSON_NULL) == JSON(JSON_NULL));

    // JSON_UNDEFINED != JSON_UNDEFINED
    assert(JSON() != JSON());

    // Iterator test
    int i=0;
    JSON j10(JSON_OBJECT);
    j10["key1"] = 12;

    j10["key2"] = 13;
    j10["key3"] = j8;

    j10["key4"] = j8;

    std::cout<<"\nChecking forward iterators now ... j10 = "<<j10.toString()<<"\n";
    for (JSON::array_iterator it = j8.array_begin();it != j8.array_end(); ++it, ++i) {
      assert(j8[i] == *(it));
    }

    for (JSON::object_iterator it = j10.object_begin();it != j10.object_end(); ++it) {
      assert(j10[it->first] == it->second);
      std::cout<<"Key = "<<it->first<<", Value = "<<it->second.toString()<<endl;
    }
    std::cout<<"\nChecking reverse now ...\n";
    i = j8.size() - 1;
    for (JSON::array_reverse_iterator it = j8.array_rbegin();it != j8.array_rend(); ++it, --i) {
      assert(j8[i] == *(it));
    }

    for (JSON::object_reverse_iterator it = j10.object_rbegin();it != j10.object_rend(); ++it) {
      assert(j10[it->first] == it->second);
      std::cout<<"Key = "<<it->first<<", Value = "<<it->second.toString()<<endl;
    }

    //
    //typedef std::map<std::string, JSON> ObjectIterator
    //ObjectIterator it = j4.ObjectBegin();
    // JSON Iterators (different class);

    //for (; it != j4.end(); ++it) {

  //  }

     // Check implicit cast operators
     JSON j11(JSON_OBJECT);
     j11["1"] = 1;
     j11["2"] = 12.33;
     j11["3"] = true;
     j11["4"] = 212l;
     j11["4.1"] = "blahh";
     j11["5"] = vector<int>(5,0);
     j11["6"] = "1";

     assert(j11["5"][0.9] == 0);
     assert(j11["5"][j11["1"]] == 0);

     assert(j11.has("1"));
     assert(!j11.has("random"));
     assert(j11["5"].has(0));
     assert(j11["5"].has(1));
     assert(j11["5"].has(j11["5"][0]));
     assert(j11.has(j11["6"]) && j11[j11["6"]] == 1);
     assert(j11["5"][j11["1"]] == 0);
     assert(double(j11["1"]) == 1);
     assert(double(j11["2"]) == 12.33);
     assert(bool(j11["3"]) == true);
     assert(j11["4.1"].toString() == "\"blahh\"");
     assert(long(j11["4"]) == 212l);
     assert(double(j11["1"]) < double(j11["2"]));

     const JSON j12(j11);

     assert(j12["5"][0.9] == 0);
     assert(j12["5"][j11["1"]] == 0);

     assert(j12["5"][j11["1"]] == 0);
     assert(double(j12["1"]) == 1);
     assert(double(j12["2"]) == 12.33);
     assert(bool(j12["3"]) == true);
     assert(j12["4.1"].toString() == "\"blahh\"");
     assert(long(j12["4"]) == 212l);
     assert(double(j12["1"]) < double(j11["2"]));

     JSON j13(JSON_OBJECT);
     j13["foo"] = "blah";
     j13["foo2"] = 12;
     j13["foo3"] = 12.32;
     std::string str1 = j13["foo"].get<std::string>();
     assert(str1 == "blah");
     assert(j13["foo2"].get<int>() == 12);
     assert(j13["foo3"].get<double>() == 12.32);
     assert(j13["foo3"].get<bool>() == true);

     JSON j14(JSON_NULL);
     assert(j14 == JSON_NULL);

     std::cout<<"\nAll assertions performed succesfully\n";
  }
  catch (exception &e)
  {
    std::cout<<"\nErrror occured: \n"<<e.what()<<"\n";
  }

  return 0;
}
<commit_msg>Made dxjson/trial.cpp compilable on clang<commit_after>#include <cstdio>
#include "dxjson.h"

int main() {
  using namespace dx;
  using namespace std;
  try
  {
    JSON j1(JSON_OBJECT);

    std::cout<<"Eps = "<<JSON::getEpsilon();
    j1["key"] = 12;
    JSON j2(JSON_OBJECT);
    j2["blah"] = "sdsdsd";
    j2["blah"] = "key";
    j2["lala"] = j1[j2["blah"]];
    std::cout<<"\nj2 = \n"<<j2.toString()<<"\n";
    j2.erase("lala");
    std::cout<<"\n j2 after erasing 'lala' = \n"<<j2.toString()<<"\n";
    // JSON parse tests
    JSON j3 = JSON::parse("{\"blah\": [  21,232,\"foo\" , {\"key\": \"val1\"}, true, false, null]}");
    j3["blah"].push_back(1.23456789101112);
    j3["blah"].push_back("dsdsd");
    j3["blah"].push_back(Null());
    j3["blah"].push_back(12.212);
    j3["foo"] = vector<int>(5,5);
    std::map<std::string, int> mp;
    mp["lala"] = 12.11e-212;
    mp["dsdsd"] = 1212;
    j3["map"] = mp;
    std::cout<<"\nj3 = "<<j3.toString();
    std::cout<<"\nj3[blah] = "<<j3["blah"].toString();
    std::cout<<"\nj3[blah][2] = "<<j3["blah"][2].toString()<<"\n";

    j3["blah"].erase(2);
    std::cout<<"\nBlah after erasing indx = 2\n"<<j3["blah"].toString()<<"\n";

    JSON j4;
    std::string str = "{\"清华大学\": [\"this should look like second element\", \"\\u6e05\\u534e\\u5927\\u5b66\", \"\\n\\b\\t\\\"\"]    }";
    j4 = JSON::parse(str);

    std::cout<<"j4 = "<<j4.toString()<<"\n";

    JSON j5(JSON_BOOLEAN);
    j5 = true;
    std::cout<<"\nj5 = "<<j5.toString()<<"\n";

    // Equality tests
    std::cout<<"\nj4 == j5: "<<((j4==j5)?"true":"false");

    JSON j5_copy = j5;
    std::cout<<"\nj5_copy == j5: "<<((j5_copy==j5) ? "true" : "false")<<"\n";

    JSON j6 = 12.21;
    JSON j7 = 12.22;
    assert(j6 != j7);

    JSON j8(JSON_ARRAY);
    j8.push_back(12.21);
    j8.push_back("hello");
    j8.push_back(j8);
    std::cout<<"\nj8 = "<<j8.toString()<<"\n";
    JSON j9 = j8;
    assert(j9 == j8);
    j9.erase(2);
    assert(j9 != j8);
    assert(JSON(JSON_NULL) == JSON(JSON_NULL));

    // JSON_UNDEFINED != JSON_UNDEFINED
    assert(JSON() != JSON());

    // Iterator test
    int i=0;
    JSON j10(JSON_OBJECT);
    j10["key1"] = 12;

    j10["key2"] = 13;
    j10["key3"] = j8;

    j10["key4"] = j8;

    std::cout<<"\nChecking forward iterators now ... j10 = "<<j10.toString()<<"\n";
    for (JSON::array_iterator it = j8.array_begin();it != j8.array_end(); ++it, ++i) {
      assert(j8[i] == *(it));
    }

    for (JSON::object_iterator it = j10.object_begin();it != j10.object_end(); ++it) {
      assert(j10[it->first] == it->second);
      std::cout<<"Key = "<<it->first<<", Value = "<<it->second.toString()<<endl;
    }
    std::cout<<"\nChecking reverse now ...\n";
    i = j8.size() - 1;
    for (JSON::array_reverse_iterator it = j8.array_rbegin();it != j8.array_rend(); ++it, --i) {
      assert(j8[i] == *(it));
    }

    for (JSON::object_reverse_iterator it = j10.object_rbegin();it != j10.object_rend(); ++it) {
      assert(j10[it->first] == it->second);
      std::cout<<"Key = "<<it->first<<", Value = "<<it->second.toString()<<endl;
    }

    //
    //typedef std::map<std::string, JSON> ObjectIterator
    //ObjectIterator it = j4.ObjectBegin();
    // JSON Iterators (different class);

    //for (; it != j4.end(); ++it) {

  //  }

     // Check implicit cast operators
     JSON j11(JSON_OBJECT);
     j11["1"] = 1;
     j11["2"] = 12.33;
     j11["3"] = true;
     j11["4"] = 212l;
     j11["4.1"] = "blahh";
     j11["5"] = vector<int>(5,0);
     j11["6"] = "1";

     assert(j11["5"][0.9].get<int>() == 0);
     assert(j11["5"][j11["1"]].get<int>() == 0);

     assert(j11.has("1"));
     assert(!j11.has("random"));
     assert(j11["5"].has(0));
     assert(j11["5"].has(1));
     assert(j11["5"].has(j11["5"][0]));
     assert(j11.has(j11["6"]) && j11[j11["6"]].get<int>() == 1);
     assert(j11["5"][j11["1"]].get<int>() == 0);
     assert(double(j11["1"]) == 1);
     assert(double(j11["2"]) == 12.33);
     assert(bool(j11["3"]) == true);
     assert(j11["4.1"].toString() == "\"blahh\"");
     assert(long(j11["4"]) == 212l);
     assert(double(j11["1"]) < double(j11["2"]));

     const JSON j12(j11);

     assert(j12["5"][0.9].get<int>() == 0);
     assert(j12["5"][j11["1"]].get<int>() == 0);

     assert(j12["5"][j11["1"]].get<int>() == 0);
     assert(double(j12["1"]) == 1);
     assert(double(j12["2"]) == 12.33);
     assert(bool(j12["3"]) == true);
     assert(j12["4.1"].toString() == "\"blahh\"");
     assert(long(j12["4"]) == 212l);
     assert(double(j12["1"]) < double(j11["2"]));

     JSON j13(JSON_OBJECT);
     j13["foo"] = "blah";
     j13["foo2"] = 12;
     j13["foo3"] = 12.32;
     std::string str1 = j13["foo"].get<std::string>();
     assert(str1 == "blah");
     assert(j13["foo2"].get<int>() == 12);
     assert(j13["foo3"].get<double>() == 12.32);
     assert(j13["foo3"].get<bool>() == true);

     JSON j14(JSON_NULL);
     assert(j14 == JSON(JSON_NULL));

     std::cout<<"\nAll assertions performed succesfully\n";
  }
  catch (exception &e)
  {
    std::cout<<"\nErrror occured: \n"<<e.what()<<"\n";
  }

  return 0;
}
<|endoftext|>


<commit_before>//
// Created by Dawid Drozd aka Gelldur on 7/19/16.
//

#include "Preferences.h"

#include<cstdio>

#include <Poco/Data/Session.h>
//#include <Poco/Data/SQLite/Connector.h>

#include <log.h>
#include <acme/MakeUnique.h>

Preferences::Preferences(const std::string& databaseName, const std::string& tableName)
		: TABLE_NAME(tableName)
		, DATABSE_FILE_NAME(databaseName)
{
	//	Poco::Data::SQLite::Connector::registerConnector();

	recreate();
}

Preferences::~Preferences()
{
	if (_session != nullptr)
	{
		_session->close();
	}
	//	Poco::Data::SQLite::Connector::unregisterConnector();
}

void Preferences::setValue(const std::string& key, const std::string& value)
{
	_cache[key] = value;

	if (_session == nullptr)
	{
		ELOG("Session not created\nNo storage");
		return;
	}

	Poco::Data::Session& session = *_session;

	Poco::Data::Statement insert(session);
	insert << "INSERT OR REPLACE INTO " << TABLE_NAME << " VALUES(?, ?)",
			Poco::Data::Keywords::useRef(key),
			Poco::Data::Keywords::useRef(value);
	DLOG("Preferences: %s\n(%s)=(%s)", insert.toString().c_str(), key.c_str(), value.c_str());

	insert.execute();
}

std::string Preferences::getValue(const std::string& key, const std::string& defaultValue)
{
	auto found = _cache.find(key);
	if (found != _cache.end())
	{
		return found->second;
	}

	if (_session == nullptr)
	{
		ELOG("Session not created\nNo storage");
		_cache[key] = defaultValue;
		return defaultValue;
	}

	std::string returnedValue = defaultValue;

	Poco::Data::Session& session = *_session;

	// a simple query
	Poco::Data::Statement select(session);
	select << "SELECT value FROM " << TABLE_NAME << " WHERE key=?",
			Poco::Data::Keywords::useRef(key),
			Poco::Data::Keywords::into(returnedValue); //  iterate over result set one row at a time

	DLOG("Select: %s\nkey:%s", select.toString().c_str(), key.c_str());
	select.execute();

	_cache[key] = returnedValue;
	return returnedValue;
}

void Preferences::clean()
{
	_cache.clear();
	if (_session == nullptr)
	{
		ELOG("Session not created\nNo storage");
		return;
	}

	_session->close();

	if (std::remove(DATABSE_FILE_NAME.c_str()) != 0)
	{
		ELOG("Can't remove database: %s", DATABSE_FILE_NAME.c_str());
	}
	recreate();
}

void Preferences::recreate()
{
	try
	{
		_session = std::make_unique<Poco::Data::Session>("SQLite", DATABSE_FILE_NAME.c_str());
	}
	catch (const std::exception& ex)
	{
		ELOG("Can't create session: %s", ex.what());
	}
	catch (...)
	{
		ELOG("Can't create session");
	}
	if (_session == nullptr)
	{
		ELOG("Session not created\nNo storage");
		return;
	}

	Poco::Data::Session& session = *_session;

	session << "CREATE TABLE IF NOT EXISTS `" << TABLE_NAME << "`("
			"`key` TEXT NOT NULL UNIQUE,"
			"`value` TEXT,"
			"PRIMARY KEY(key)"
			");", Poco::Data::Keywords::now;
}
<commit_msg>Fix Preferences (after android changes)<commit_after>//
// Created by Dawid Drozd aka Gelldur on 7/19/16.
//

#include "Preferences.h"

#include<cstdio>

#include <Poco/Data/Session.h>
#include <Poco/Data/SQLite/Connector.h>

#include <log.h>
#include <acme/MakeUnique.h>

Preferences::Preferences(const std::string& databaseName, const std::string& tableName)
		: TABLE_NAME(tableName)
		, DATABSE_FILE_NAME(databaseName)
{
	Poco::Data::SQLite::Connector::registerConnector();

	recreate();
}

Preferences::~Preferences()
{
	if (_session != nullptr)
	{
		_session->close();
	}
	Poco::Data::SQLite::Connector::unregisterConnector();
}

void Preferences::setValue(const std::string& key, const std::string& value)
{
	_cache[key] = value;

	if (_session == nullptr)
	{
		ELOG("Session not created\nNo storage");
		return;
	}

	Poco::Data::Session& session = *_session;

	Poco::Data::Statement insert(session);
	insert << "INSERT OR REPLACE INTO " << TABLE_NAME << " VALUES(?, ?)",
			Poco::Data::Keywords::useRef(key),
			Poco::Data::Keywords::useRef(value);
	DLOG("Preferences: %s\n(%s)=(%s)", insert.toString().c_str(), key.c_str(), value.c_str());

	insert.execute();
}

std::string Preferences::getValue(const std::string& key, const std::string& defaultValue)
{
	auto found = _cache.find(key);
	if (found != _cache.end())
	{
		return found->second;
	}

	if (_session == nullptr)
	{
		ELOG("Session not created\nNo storage");
		_cache[key] = defaultValue;
		return defaultValue;
	}

	std::string returnedValue = defaultValue;

	Poco::Data::Session& session = *_session;

	// a simple query
	Poco::Data::Statement select(session);
	select << "SELECT value FROM " << TABLE_NAME << " WHERE key=?",
			Poco::Data::Keywords::useRef(key),
			Poco::Data::Keywords::into(returnedValue); //  iterate over result set one row at a time

	DLOG("Select: %s\nkey:%s", select.toString().c_str(), key.c_str());
	select.execute();

	_cache[key] = returnedValue;
	return returnedValue;
}

void Preferences::clean()
{
	_cache.clear();
	if (_session == nullptr)
	{
		ELOG("Session not created\nNo storage");
		return;
	}

	_session->close();

	if (std::remove(DATABSE_FILE_NAME.c_str()) != 0)
	{
		ELOG("Can't remove database: %s", DATABSE_FILE_NAME.c_str());
	}
	recreate();
}

void Preferences::recreate()
{
	try
	{
		_session = std::make_unique<Poco::Data::Session>("SQLite", DATABSE_FILE_NAME.c_str());
	}
	catch (const std::exception& ex)
	{
		ELOG("Can't create session: %s", ex.what());
	}
	catch (...)
	{
		ELOG("Can't create session");
	}
	if (_session == nullptr)
	{
		ELOG("Session not created\nNo storage");
		return;
	}

	Poco::Data::Session& session = *_session;

	session << "CREATE TABLE IF NOT EXISTS `" << TABLE_NAME << "`("
			"`key` TEXT NOT NULL UNIQUE,"
			"`value` TEXT,"
			"PRIMARY KEY(key)"
			");", Poco::Data::Keywords::now;
}
<|endoftext|>


<commit_before>/*****************************************************************************
 *
 * This file is part of Mapnik (c++ mapping toolkit)
 *
 * Copyright (C) 2006 Artem Pavlenko
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2.1 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with this library; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
 *
 *****************************************************************************/
//$Id: datasource_cache.cpp 23 2005-03-22 22:16:34Z pavlenko $

// mapnik
#include <mapnik/datasource_cache.hpp>

#include <mapnik/config_error.hpp>

// boost
#include <boost/thread/mutex.hpp>
#include <boost/filesystem/operations.hpp>
#include <boost/algorithm/string.hpp>

// ltdl
#include <ltdl.h>

// stl
#include <algorithm>
#include <stdexcept>

namespace mapnik
{
   using namespace std;
   using namespace boost;

   bool is_input_plugin (std::string const& filename)
   {
      return boost::algorithm::ends_with(filename,std::string(".input"));
   }

   datasource_cache::datasource_cache()
   {
      if (lt_dlinit()) throw std::runtime_error("lt_dlinit() failed");
   }

   datasource_cache::~datasource_cache()
   {
      lt_dlexit();
   }

   std::map<string,boost::shared_ptr<PluginInfo> > datasource_cache::plugins_;
   bool datasource_cache::registered_=false;

   datasource_ptr datasource_cache::create(const parameters& params)
   {
       boost::optional<std::string> type = params.get<std::string>("type");
       if ( ! type)
       {
           throw config_error(string("Could not create datasource. Required ") +
                   "parameter 'type' is missing");
       }

       datasource_ptr ds;
       map<string,boost::shared_ptr<PluginInfo> >::iterator itr=plugins_.find(*type);
       if ( itr == plugins_.end() )
       {
           throw config_error(string("Could not create datasource. No plugin ") +
                   "found for type '" + * type + "'");
       }
       if ( ! itr->second->handle())
       {
           throw std::runtime_error(string("Cannot load library: ") +
                   lt_dlerror());
       }

       create_ds* create_datasource =
           (create_ds*) lt_dlsym(itr->second->handle(), "create");

       if ( ! create_datasource)
       {
           throw std::runtime_error(string("Cannot load symbols: ") +
                   lt_dlerror());
       }
       std::cout << "size = " << params.size() << "\n";
       parameters::const_iterator i = params.begin();
       for (;i!=params.end();++i)
       {
          std::cout << i->first << "=" << i->second << "\n";
       }
       ds=datasource_ptr(create_datasource(params), datasource_deleter());

#ifdef MAPNIK_DEBUG
       std::clog<<"datasource="<<ds<<" type="<<type<<std::endl;
#endif
       return ds;
   }

   bool datasource_cache::insert(const std::string& type,const lt_dlhandle module)
   {
      return plugins_.insert(make_pair(type,boost::shared_ptr<PluginInfo>
                                       (new PluginInfo(type,module)))).second;
   }

   void datasource_cache::register_datasources(const std::string& str)
   {
      mutex::scoped_lock lock(mapnik::singleton<mapnik::datasource_cache,
                              mapnik::CreateStatic>::mutex_);
      filesystem::path path(str);
      filesystem::directory_iterator end_itr;
      if (exists(path) && is_directory(path))
      {
         for (filesystem::directory_iterator itr(path);itr!=end_itr;++itr )
         {
            if (!is_directory( *itr )  && is_input_plugin(itr->leaf()))
            {
               try
               {
                  lt_dlhandle module=lt_dlopen(itr->string().c_str());
                  if (module)
                  {
                     datasource_name* ds_name =
                        (datasource_name*) lt_dlsym(module, "datasource_name");
                     if (ds_name && insert(ds_name(),module))
                     {
#ifdef MAPNIK_DEBUG
                        std::clog<<"registered datasource : "<<ds_name()<<std::endl;
#endif
                        registered_=true;
                     }
                  }
                  else
                  {
                     std::clog << lt_dlerror() << "\n";
                  }
               }
               catch (...) {}
            }
         }
      }
   }
}
<commit_msg>only print to clog when MAPNIK_DEBUG is defined<commit_after>/*****************************************************************************
 *
 * This file is part of Mapnik (c++ mapping toolkit)
 *
 * Copyright (C) 2006 Artem Pavlenko
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2.1 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with this library; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
 *
 *****************************************************************************/
//$Id: datasource_cache.cpp 23 2005-03-22 22:16:34Z pavlenko $

// mapnik
#include <mapnik/datasource_cache.hpp>

#include <mapnik/config_error.hpp>

// boost
#include <boost/thread/mutex.hpp>
#include <boost/filesystem/operations.hpp>
#include <boost/algorithm/string.hpp>

// ltdl
#include <ltdl.h>

// stl
#include <algorithm>
#include <stdexcept>

namespace mapnik
{
   using namespace std;
   using namespace boost;

   bool is_input_plugin (std::string const& filename)
   {
      return boost::algorithm::ends_with(filename,std::string(".input"));
   }

   datasource_cache::datasource_cache()
   {
      if (lt_dlinit()) throw std::runtime_error("lt_dlinit() failed");
   }

   datasource_cache::~datasource_cache()
   {
      lt_dlexit();
   }

   std::map<string,boost::shared_ptr<PluginInfo> > datasource_cache::plugins_;
   bool datasource_cache::registered_=false;

   datasource_ptr datasource_cache::create(const parameters& params)
   {
       boost::optional<std::string> type = params.get<std::string>("type");
       if ( ! type)
       {
           throw config_error(string("Could not create datasource. Required ") +
                   "parameter 'type' is missing");
       }

       datasource_ptr ds;
       map<string,boost::shared_ptr<PluginInfo> >::iterator itr=plugins_.find(*type);
       if ( itr == plugins_.end() )
       {
           throw config_error(string("Could not create datasource. No plugin ") +
                   "found for type '" + * type + "'");
       }
       if ( ! itr->second->handle())
       {
           throw std::runtime_error(string("Cannot load library: ") +
                   lt_dlerror());
       }

       create_ds* create_datasource =
           (create_ds*) lt_dlsym(itr->second->handle(), "create");

       if ( ! create_datasource)
       {
           throw std::runtime_error(string("Cannot load symbols: ") +
                   lt_dlerror());
       }
#ifdef MAPNIK_DEBUG
       std::clog << "size = " << params.size() << "\n";
       parameters::const_iterator i = params.begin();
       for (;i!=params.end();++i)
       {
          std::clog << i->first << "=" << i->second << "\n";
       }
#endif
       ds=datasource_ptr(create_datasource(params), datasource_deleter());

#ifdef MAPNIK_DEBUG
       std::clog<<"datasource="<<ds<<" type="<<type<<std::endl;
#endif
       return ds;
   }

   bool datasource_cache::insert(const std::string& type,const lt_dlhandle module)
   {
      return plugins_.insert(make_pair(type,boost::shared_ptr<PluginInfo>
                                       (new PluginInfo(type,module)))).second;
   }

   void datasource_cache::register_datasources(const std::string& str)
   {
      mutex::scoped_lock lock(mapnik::singleton<mapnik::datasource_cache,
                              mapnik::CreateStatic>::mutex_);
      filesystem::path path(str);
      filesystem::directory_iterator end_itr;
      if (exists(path) && is_directory(path))
      {
         for (filesystem::directory_iterator itr(path);itr!=end_itr;++itr )
         {
            if (!is_directory( *itr )  && is_input_plugin(itr->leaf()))
            {
               try
               {
                  lt_dlhandle module=lt_dlopen(itr->string().c_str());
                  if (module)
                  {
                     datasource_name* ds_name =
                        (datasource_name*) lt_dlsym(module, "datasource_name");
                     if (ds_name && insert(ds_name(),module))
                     {
#ifdef MAPNIK_DEBUG
                        std::clog<<"registered datasource : "<<ds_name()<<std::endl;
#endif
                        registered_=true;
                     }
                  }
                  else
                  {
                     std::clog << lt_dlerror() << "\n";
                  }
               }
               catch (...) {}
            }
         }
      }
   }
}
<|endoftext|>


<commit_before><commit_msg>Improve IAS Zone enrollment<commit_after><|endoftext|>


<commit_before>// Copyright (c) 2021 ASMlover. All rights reserved.
//
//  ______             __                  ___
// /\__  _\           /\ \                /\_ \
// \/_/\ \/    __     \_\ \  _____     ___\//\ \      __
//    \ \ \  /'__`\   /'_` \/\ '__`\  / __`\\ \ \   /'__`\
//     \ \ \/\ \L\.\_/\ \L\ \ \ \L\ \/\ \L\ \\_\ \_/\  __/
//      \ \_\ \__/.\_\ \___,_\ \ ,__/\ \____//\____\ \____\
//       \/_/\/__/\/_/\/__,_ /\ \ \/  \/___/ \/____/\/____/
//                             \ \_\
//                              \/_/
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions
// are met:
//
//  * Redistributions of source code must retain the above copyright
//    notice, this list ofconditions and the following disclaimer.
//
//  * Redistributions in binary form must reproduce the above copyright
//    notice, this list of conditions and the following disclaimer in
//    the documentation and/or other materialsprovided with the
//    distribution.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
// FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
// COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
// INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
// BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
// LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
// ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
// POSSIBILITY OF SUCH DAMAGE.
#include <Object/FunctionObject.hh>
#include <Compiler/FunctionCompiler.hh>

namespace Tadpole::Compiler {

int FunctionCompiler::resolve_local(const Lex::Token& name, const ErrorFn& errfn) {
  for (int i = locals_count() - 1; i >= 0; --i) {
    const LocalVar& local = locals_[i];
    if (local.name == name && local.depth == -1)
      errfn(Common::from_fmt("cannot load local variable `%s` in its own initializer", name.as_cstring()));
  }
  return -1;
}

int FunctionCompiler::add_upvalue(u8_t index, bool is_local) {
  for (sz_t i = 0; i < fn_->upvalues_count(); ++i) {
    if (upvalues_[i].is_equal(index, is_local))
      return Common::as_type<int>(i);
  }

  upvalues_.push_back({index, is_local});
  return Common::as_type<int>(fn_->inc_upvalues_count());
}

int FunctionCompiler::resolve_upvalue(const Lex::Token& name, const ErrorFn& errfn) {
  if (enclosing_ == nullptr)
    return -1;

  if (int local = enclosing_->resolve_local(name, errfn); local != -1) {
    enclosing_->locals_[local].is_upvalue = true;
    return add_upvalue(Common::as_type<u8_t>(local), true);
  }
  if (int upvalue = enclosing_->resolve_upvalue(name, errfn); upvalue != -1)
    return add_upvalue(Common::as_type<u8_t>(upvalue), false);

  return -1;
}

void FunctionCompiler::declare_localvar(const Lex::Token& name, const ErrorFn& errfn) {
  // TODO:
}

}
<commit_msg>:construction: chore(function-compiler): finished the implementation of function compiler<commit_after>// Copyright (c) 2021 ASMlover. All rights reserved.
//
//  ______             __                  ___
// /\__  _\           /\ \                /\_ \
// \/_/\ \/    __     \_\ \  _____     ___\//\ \      __
//    \ \ \  /'__`\   /'_` \/\ '__`\  / __`\\ \ \   /'__`\
//     \ \ \/\ \L\.\_/\ \L\ \ \ \L\ \/\ \L\ \\_\ \_/\  __/
//      \ \_\ \__/.\_\ \___,_\ \ ,__/\ \____//\____\ \____\
//       \/_/\/__/\/_/\/__,_ /\ \ \/  \/___/ \/____/\/____/
//                             \ \_\
//                              \/_/
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions
// are met:
//
//  * Redistributions of source code must retain the above copyright
//    notice, this list ofconditions and the following disclaimer.
//
//  * Redistributions in binary form must reproduce the above copyright
//    notice, this list of conditions and the following disclaimer in
//    the documentation and/or other materialsprovided with the
//    distribution.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
// FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
// COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
// INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
// BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
// LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
// ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
// POSSIBILITY OF SUCH DAMAGE.
#include <Object/FunctionObject.hh>
#include <Compiler/FunctionCompiler.hh>

namespace Tadpole::Compiler {

int FunctionCompiler::resolve_local(const Lex::Token& name, const ErrorFn& errfn) {
  for (int i = locals_count() - 1; i >= 0; --i) {
    const LocalVar& local = locals_[i];
    if (local.name == name && local.depth == -1)
      errfn(Common::from_fmt("cannot load local variable `%s` in its own initializer", name.as_cstring()));
  }
  return -1;
}

int FunctionCompiler::add_upvalue(u8_t index, bool is_local) {
  for (sz_t i = 0; i < fn_->upvalues_count(); ++i) {
    if (upvalues_[i].is_equal(index, is_local))
      return Common::as_type<int>(i);
  }

  upvalues_.push_back({index, is_local});
  return Common::as_type<int>(fn_->inc_upvalues_count());
}

int FunctionCompiler::resolve_upvalue(const Lex::Token& name, const ErrorFn& errfn) {
  if (enclosing_ == nullptr)
    return -1;

  if (int local = enclosing_->resolve_local(name, errfn); local != -1) {
    enclosing_->locals_[local].is_upvalue = true;
    return add_upvalue(Common::as_type<u8_t>(local), true);
  }
  if (int upvalue = enclosing_->resolve_upvalue(name, errfn); upvalue != -1)
    return add_upvalue(Common::as_type<u8_t>(upvalue), false);

  return -1;
}

void FunctionCompiler::declare_localvar(const Lex::Token& name, const ErrorFn& errfn) {
  if (scope_depth_ == 0)
    return;

  for (auto it = locals_.rbegin(); it != locals_.rend(); ++it) {
    if (it->depth != -1 && it->depth < scope_depth_)
      break;

    if (it->name == name)
      errfn(Common::from_fmt("name `%s` is redefined", name.as_cstring()));
  }
  locals_.push_back({name, -1, false});
}

}
<|endoftext|>


<commit_before>/*
 * Copyright (C) 2019 The Android Open Source Project
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

#include "src/trace_processor/sqlite/db_sqlite_table.h"

#include "src/trace_processor/sqlite/sqlite_utils.h"

namespace perfetto {
namespace trace_processor {

namespace {

FilterOp SqliteOpToFilterOp(int sqlite_op) {
  switch (sqlite_op) {
    case SQLITE_INDEX_CONSTRAINT_EQ:
    case SQLITE_INDEX_CONSTRAINT_IS:
      return FilterOp::kEq;
    case SQLITE_INDEX_CONSTRAINT_GT:
      return FilterOp::kGt;
    case SQLITE_INDEX_CONSTRAINT_LT:
      return FilterOp::kLt;
    case SQLITE_INDEX_CONSTRAINT_ISNOT:
    case SQLITE_INDEX_CONSTRAINT_NE:
      return FilterOp::kNe;
    case SQLITE_INDEX_CONSTRAINT_GE:
      return FilterOp::kGe;
    case SQLITE_INDEX_CONSTRAINT_LE:
      return FilterOp::kLe;
    case SQLITE_INDEX_CONSTRAINT_ISNULL:
      return FilterOp::kIsNull;
    case SQLITE_INDEX_CONSTRAINT_ISNOTNULL:
      return FilterOp::kIsNotNull;
    case SQLITE_INDEX_CONSTRAINT_LIKE:
      return FilterOp::kLike;
    default:
      PERFETTO_FATAL("Currently unsupported constraint");
  }
}

SqlValue SqliteValueToSqlValue(sqlite3_value* sqlite_val) {
  auto col_type = sqlite3_value_type(sqlite_val);
  SqlValue value;
  switch (col_type) {
    case SQLITE_INTEGER:
      value.type = SqlValue::kLong;
      value.long_value = sqlite3_value_int64(sqlite_val);
      break;
    case SQLITE_TEXT:
      value.type = SqlValue::kString;
      value.string_value =
          reinterpret_cast<const char*>(sqlite3_value_text(sqlite_val));
      break;
    case SQLITE_FLOAT:
      value.type = SqlValue::kDouble;
      value.double_value = sqlite3_value_double(sqlite_val);
      break;
    case SQLITE_BLOB:
      value.type = SqlValue::kBytes;
      value.bytes_value = sqlite3_value_blob(sqlite_val);
      value.bytes_count = static_cast<size_t>(sqlite3_value_bytes(sqlite_val));
      break;
    case SQLITE_NULL:
      value.type = SqlValue::kNull;
      break;
  }
  return value;
}

}  // namespace

DbSqliteTable::DbSqliteTable(sqlite3*, const Table* table) : table_(table) {}
DbSqliteTable::~DbSqliteTable() = default;

void DbSqliteTable::RegisterTable(sqlite3* db,
                                  const Table* table,
                                  const std::string& name) {
  SqliteTable::Register<DbSqliteTable, const Table*>(db, table, name);
}

util::Status DbSqliteTable::Init(int, const char* const*, Schema* schema) {
  std::vector<SqliteTable::Column> schema_cols;
  for (uint32_t i = 0; i < table_->GetColumnCount(); ++i) {
    const auto& col = table_->GetColumn(i);
    schema_cols.emplace_back(i, col.name(), col.type());
  }
  // TODO(lalitm): this is hardcoded to be the id column but change this to be
  // more generic in the future.
  auto opt_idx = table_->FindColumnIdxByName("id");
  if (!opt_idx) {
    PERFETTO_FATAL(
        "id column not found in %s. Currently all db Tables need to contain an "
        "id column; this constraint will be relaxed in the future.",
        name().c_str());
  }

  std::vector<size_t> primary_keys;
  primary_keys.emplace_back(*opt_idx);

  *schema = Schema(std::move(schema_cols), std::move(primary_keys));
  return util::OkStatus();
}

int DbSqliteTable::BestIndex(const QueryConstraints& qc, BestIndexInfo* info) {
  // TODO(lalitm): investigate SQLITE_INDEX_SCAN_UNIQUE for id columns.
  auto cost_and_rows = EstimateCost(*table_, qc);
  info->estimated_cost = cost_and_rows.cost;
  info->estimated_rows = cost_and_rows.rows;
  return SQLITE_OK;
}

int DbSqliteTable::ModifyConstraints(QueryConstraints* qc) {
  using C = QueryConstraints::Constraint;

  // Reorder constraints to consider the constraints on columns which are
  // cheaper to filter first.
  auto* cs = qc->mutable_constraints();
  std::sort(cs->begin(), cs->end(), [this](const C& a, const C& b) {
    uint32_t a_idx = static_cast<uint32_t>(a.column);
    uint32_t b_idx = static_cast<uint32_t>(b.column);
    const auto& a_col = table_->GetColumn(a_idx);
    const auto& b_col = table_->GetColumn(b_idx);

    // Id columns are always very cheap to filter on so try and get them
    // first.
    if (a_col.IsId() && !b_col.IsId())
      return true;

    // Sorted columns are also quite cheap to filter so order them after
    // any id columns.
    if (a_col.IsSorted() && !b_col.IsSorted())
      return true;

    // TODO(lalitm): introduce more orderings here based on empirical data.
    return false;
  });

  // Remove any order by constraints which also have an equality constraint.
  auto* ob = qc->mutable_order_by();
  {
    auto p = [&cs](const QueryConstraints::OrderBy& o) {
      auto inner_p = [&o](const QueryConstraints::Constraint& c) {
        return c.column == o.iColumn && sqlite_utils::IsOpEq(c.op);
      };
      return std::any_of(cs->begin(), cs->end(), inner_p);
    };
    auto remove_it = std::remove_if(ob->begin(), ob->end(), p);
    ob->erase(remove_it, ob->end());
  }

  // Go through the order by constraints in reverse order and eliminate
  // constraints until the first non-sorted column or the first order by in
  // descending order.
  {
    auto p = [this](const QueryConstraints::OrderBy& o) {
      const auto& col = table_->GetColumn(static_cast<uint32_t>(o.iColumn));
      return o.desc || !col.IsSorted();
    };
    auto first_non_sorted_it = std::find_if(ob->rbegin(), ob->rend(), p);
    auto pop_count = std::distance(ob->rbegin(), first_non_sorted_it);
    ob->resize(ob->size() - static_cast<uint32_t>(pop_count));
  }

  return SQLITE_OK;
}

DbSqliteTable::QueryCost DbSqliteTable::EstimateCost(
    const Table& table,
    const QueryConstraints& qc) {
  // Currently our cost estimation algorithm is quite simplistic but is good
  // enough for the simplest cases.
  // TODO(lalitm): replace hardcoded constants with either more heuristics
  // based on the exact type of constraint or profiling the queries themselves.

  // We estimate the fixed cost of set-up and tear-down of a query in terms of
  // the number of rows scanned.
  constexpr double kFixedQueryCost = 1000.0;

  // Setup the variables for estimating the number of rows we will have at the
  // end of filtering. Note that |current_row_count| should always be at least 1
  // unless we are absolutely certain that we will return no rows as otherwise
  // SQLite can make some bad choices.
  uint32_t current_row_count = table.size();

  // If the table is empty, any constraint set only pays the fixed cost. Also we
  // can return 0 as the row count as we are certain that we will return no
  // rows.
  if (current_row_count == 0)
    return QueryCost{kFixedQueryCost, 0};

  // Setup the variables for estimating the cost of filtering.
  double filter_cost = 0.0;
  const auto& cs = qc.constraints();
  for (const auto& c : cs) {
    const auto& col = table.GetColumn(static_cast<uint32_t>(c.column));
    if (sqlite_utils::IsOpEq(c.op) && col.IsId()) {
      // If we have an id equality constraint, it's a bit expensive to find
      // the exact row but it filters down to a single row.
      filter_cost += 100;
      current_row_count = 1;
    } else if (sqlite_utils::IsOpEq(c.op)) {
      // If there is only a single equality constraint, we have special logic
      // to sort by that column and then binary search if we see the constraint
      // set often. Model this by dividing but the log of the number of rows as
      // a good approximation. Otherwise, we'll need to do a full table scan.
      filter_cost += cs.size() == 1
                         ? (2 * current_row_count) / log2(current_row_count)
                         : current_row_count;

      // We assume that an equalty constraint will cut down the number of rows
      // by approximate log of the number of rows.
      double estimated_rows = current_row_count / log2(current_row_count);
      current_row_count = std::max(static_cast<uint32_t>(estimated_rows), 1u);
    } else {
      // Otherwise, we will need to do a full table scan and we estimate we will
      // maybe (at best) halve the number of rows.
      filter_cost += current_row_count;
      current_row_count = std::max(current_row_count / 2u, 1u);
    }
  }

  // Now, to figure out the cost of sorting, multiply the final row count
  // by |qc.order_by().size()| * log(row count). This should act as a crude
  // estimation of the cost.
  double sort_cost =
      qc.order_by().size() * current_row_count * log2(current_row_count);

  // The cost of iterating rows is more expensive than filtering the rows
  // so multiply by an appropriate factor.
  double iteration_cost = current_row_count * 2.0;

  // To get the final cost, add up all the individual components.
  double final_cost =
      kFixedQueryCost + filter_cost + sort_cost + iteration_cost;
  return QueryCost{final_cost, current_row_count};
}

std::unique_ptr<SqliteTable::Cursor> DbSqliteTable::CreateCursor() {
  return std::unique_ptr<Cursor>(new Cursor(this));
}

DbSqliteTable::Cursor::Cursor(DbSqliteTable* table)
    : SqliteTable::Cursor(table), initial_db_table_(table->table_) {}

int DbSqliteTable::Cursor::Filter(const QueryConstraints& qc,
                                  sqlite3_value** argv,
                                  FilterHistory history) {
  // Clear out the iterator before filtering to ensure the destructor is run
  // before the table's destructor.
  iterator_ = base::nullopt;

  if (history == FilterHistory::kSame && qc.constraints().size() == 1 &&
      sqlite_utils::IsOpEq(qc.constraints().front().op)) {
    // If we've seen the same constraint set with a single equality constraint
    // more than |kRepeatedThreshold| times, we assume we will see it more
    // in the future and thus cache a table sorted on the column. That way,
    // future equality constraints can binary search for the value instead of
    // doing a full table scan.
    constexpr uint32_t kRepeatedThreshold = 3;
    if (!sorted_cache_table_ && repeated_cache_count_++ > kRepeatedThreshold) {
      const auto& c = qc.constraints().front();
      uint32_t col = static_cast<uint32_t>(c.column);
      sorted_cache_table_ = initial_db_table_->Sort({Order{col, false}});
    }
  } else {
    sorted_cache_table_ = base::nullopt;
    repeated_cache_count_ = 0;
  }

  // We reuse this vector to reduce memory allocations on nested subqueries.
  constraints_.resize(qc.constraints().size());
  for (size_t i = 0; i < qc.constraints().size(); ++i) {
    const auto& cs = qc.constraints()[i];
    uint32_t col = static_cast<uint32_t>(cs.column);

    FilterOp op = SqliteOpToFilterOp(cs.op);
    SqlValue value = SqliteValueToSqlValue(argv[i]);

    constraints_[i] = Constraint{col, op, value};
  }

  // We reuse this vector to reduce memory allocations on nested subqueries.
  orders_.resize(qc.order_by().size());
  for (size_t i = 0; i < qc.order_by().size(); ++i) {
    const auto& ob = qc.order_by()[i];
    uint32_t col = static_cast<uint32_t>(ob.iColumn);
    orders_[i] = Order{col, static_cast<bool>(ob.desc)};
  }

  // Try and use the sorted cache table (if it exists) to speed up the sorting.
  // Otherwise, just use the original table.
  auto* source =
      sorted_cache_table_ ? &*sorted_cache_table_ : &*initial_db_table_;
  db_table_ = source->Filter(constraints_).Sort(orders_);
  iterator_ = db_table_->IterateRows();

  return SQLITE_OK;
}

int DbSqliteTable::Cursor::Next() {
  iterator_->Next();
  return SQLITE_OK;
}

int DbSqliteTable::Cursor::Eof() {
  return !*iterator_;
}

int DbSqliteTable::Cursor::Column(sqlite3_context* ctx, int raw_col) {
  uint32_t column = static_cast<uint32_t>(raw_col);
  SqlValue value = iterator_->Get(column);
  switch (value.type) {
    case SqlValue::Type::kLong:
      sqlite3_result_int64(ctx, value.long_value);
      break;
    case SqlValue::Type::kDouble:
      sqlite3_result_double(ctx, value.double_value);
      break;
    case SqlValue::Type::kString: {
      // We can say kSqliteStatic here because all strings are expected to
      // come from the string pool and thus will be valid for the lifetime
      // of trace processor.
      sqlite3_result_text(ctx, value.string_value, -1,
                          sqlite_utils::kSqliteStatic);
      break;
    }
    case SqlValue::Type::kBytes: {
      // We can say kSqliteStatic here because for our iterator will hold
      // onto the pointer as long as we don't call Next() but that only
      // happens with Next() is called on the Cursor itself at which point
      // SQLite no longer cares about the bytes pointer.
      sqlite3_result_blob(ctx, value.bytes_value,
                          static_cast<int>(value.bytes_count),
                          sqlite_utils::kSqliteStatic);
      break;
    }
    case SqlValue::Type::kNull:
      sqlite3_result_null(ctx);
      break;
  }
  return SQLITE_OK;
}

}  // namespace trace_processor
}  // namespace perfetto
<commit_msg>Fix cost estimation for single row am: 9faab56544<commit_after>/*
 * Copyright (C) 2019 The Android Open Source Project
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

#include "src/trace_processor/sqlite/db_sqlite_table.h"

#include "src/trace_processor/sqlite/sqlite_utils.h"

namespace perfetto {
namespace trace_processor {

namespace {

FilterOp SqliteOpToFilterOp(int sqlite_op) {
  switch (sqlite_op) {
    case SQLITE_INDEX_CONSTRAINT_EQ:
    case SQLITE_INDEX_CONSTRAINT_IS:
      return FilterOp::kEq;
    case SQLITE_INDEX_CONSTRAINT_GT:
      return FilterOp::kGt;
    case SQLITE_INDEX_CONSTRAINT_LT:
      return FilterOp::kLt;
    case SQLITE_INDEX_CONSTRAINT_ISNOT:
    case SQLITE_INDEX_CONSTRAINT_NE:
      return FilterOp::kNe;
    case SQLITE_INDEX_CONSTRAINT_GE:
      return FilterOp::kGe;
    case SQLITE_INDEX_CONSTRAINT_LE:
      return FilterOp::kLe;
    case SQLITE_INDEX_CONSTRAINT_ISNULL:
      return FilterOp::kIsNull;
    case SQLITE_INDEX_CONSTRAINT_ISNOTNULL:
      return FilterOp::kIsNotNull;
    case SQLITE_INDEX_CONSTRAINT_LIKE:
      return FilterOp::kLike;
    default:
      PERFETTO_FATAL("Currently unsupported constraint");
  }
}

SqlValue SqliteValueToSqlValue(sqlite3_value* sqlite_val) {
  auto col_type = sqlite3_value_type(sqlite_val);
  SqlValue value;
  switch (col_type) {
    case SQLITE_INTEGER:
      value.type = SqlValue::kLong;
      value.long_value = sqlite3_value_int64(sqlite_val);
      break;
    case SQLITE_TEXT:
      value.type = SqlValue::kString;
      value.string_value =
          reinterpret_cast<const char*>(sqlite3_value_text(sqlite_val));
      break;
    case SQLITE_FLOAT:
      value.type = SqlValue::kDouble;
      value.double_value = sqlite3_value_double(sqlite_val);
      break;
    case SQLITE_BLOB:
      value.type = SqlValue::kBytes;
      value.bytes_value = sqlite3_value_blob(sqlite_val);
      value.bytes_count = static_cast<size_t>(sqlite3_value_bytes(sqlite_val));
      break;
    case SQLITE_NULL:
      value.type = SqlValue::kNull;
      break;
  }
  return value;
}

}  // namespace

DbSqliteTable::DbSqliteTable(sqlite3*, const Table* table) : table_(table) {}
DbSqliteTable::~DbSqliteTable() = default;

void DbSqliteTable::RegisterTable(sqlite3* db,
                                  const Table* table,
                                  const std::string& name) {
  SqliteTable::Register<DbSqliteTable, const Table*>(db, table, name);
}

util::Status DbSqliteTable::Init(int, const char* const*, Schema* schema) {
  std::vector<SqliteTable::Column> schema_cols;
  for (uint32_t i = 0; i < table_->GetColumnCount(); ++i) {
    const auto& col = table_->GetColumn(i);
    schema_cols.emplace_back(i, col.name(), col.type());
  }
  // TODO(lalitm): this is hardcoded to be the id column but change this to be
  // more generic in the future.
  auto opt_idx = table_->FindColumnIdxByName("id");
  if (!opt_idx) {
    PERFETTO_FATAL(
        "id column not found in %s. Currently all db Tables need to contain an "
        "id column; this constraint will be relaxed in the future.",
        name().c_str());
  }

  std::vector<size_t> primary_keys;
  primary_keys.emplace_back(*opt_idx);

  *schema = Schema(std::move(schema_cols), std::move(primary_keys));
  return util::OkStatus();
}

int DbSqliteTable::BestIndex(const QueryConstraints& qc, BestIndexInfo* info) {
  // TODO(lalitm): investigate SQLITE_INDEX_SCAN_UNIQUE for id columns.
  auto cost_and_rows = EstimateCost(*table_, qc);
  info->estimated_cost = cost_and_rows.cost;
  info->estimated_rows = cost_and_rows.rows;
  return SQLITE_OK;
}

int DbSqliteTable::ModifyConstraints(QueryConstraints* qc) {
  using C = QueryConstraints::Constraint;

  // Reorder constraints to consider the constraints on columns which are
  // cheaper to filter first.
  auto* cs = qc->mutable_constraints();
  std::sort(cs->begin(), cs->end(), [this](const C& a, const C& b) {
    uint32_t a_idx = static_cast<uint32_t>(a.column);
    uint32_t b_idx = static_cast<uint32_t>(b.column);
    const auto& a_col = table_->GetColumn(a_idx);
    const auto& b_col = table_->GetColumn(b_idx);

    // Id columns are always very cheap to filter on so try and get them
    // first.
    if (a_col.IsId() && !b_col.IsId())
      return true;

    // Sorted columns are also quite cheap to filter so order them after
    // any id columns.
    if (a_col.IsSorted() && !b_col.IsSorted())
      return true;

    // TODO(lalitm): introduce more orderings here based on empirical data.
    return false;
  });

  // Remove any order by constraints which also have an equality constraint.
  auto* ob = qc->mutable_order_by();
  {
    auto p = [&cs](const QueryConstraints::OrderBy& o) {
      auto inner_p = [&o](const QueryConstraints::Constraint& c) {
        return c.column == o.iColumn && sqlite_utils::IsOpEq(c.op);
      };
      return std::any_of(cs->begin(), cs->end(), inner_p);
    };
    auto remove_it = std::remove_if(ob->begin(), ob->end(), p);
    ob->erase(remove_it, ob->end());
  }

  // Go through the order by constraints in reverse order and eliminate
  // constraints until the first non-sorted column or the first order by in
  // descending order.
  {
    auto p = [this](const QueryConstraints::OrderBy& o) {
      const auto& col = table_->GetColumn(static_cast<uint32_t>(o.iColumn));
      return o.desc || !col.IsSorted();
    };
    auto first_non_sorted_it = std::find_if(ob->rbegin(), ob->rend(), p);
    auto pop_count = std::distance(ob->rbegin(), first_non_sorted_it);
    ob->resize(ob->size() - static_cast<uint32_t>(pop_count));
  }

  return SQLITE_OK;
}

DbSqliteTable::QueryCost DbSqliteTable::EstimateCost(
    const Table& table,
    const QueryConstraints& qc) {
  // Currently our cost estimation algorithm is quite simplistic but is good
  // enough for the simplest cases.
  // TODO(lalitm): replace hardcoded constants with either more heuristics
  // based on the exact type of constraint or profiling the queries themselves.

  // We estimate the fixed cost of set-up and tear-down of a query in terms of
  // the number of rows scanned.
  constexpr double kFixedQueryCost = 1000.0;

  // Setup the variables for estimating the number of rows we will have at the
  // end of filtering. Note that |current_row_count| should always be at least 1
  // unless we are absolutely certain that we will return no rows as otherwise
  // SQLite can make some bad choices.
  uint32_t current_row_count = table.size();

  // If the table is empty, any constraint set only pays the fixed cost. Also we
  // can return 0 as the row count as we are certain that we will return no
  // rows.
  if (current_row_count == 0)
    return QueryCost{kFixedQueryCost, 0};

  // Setup the variables for estimating the cost of filtering.
  double filter_cost = 0.0;
  const auto& cs = qc.constraints();
  for (const auto& c : cs) {
    if (current_row_count < 2)
      break;
    const auto& col = table.GetColumn(static_cast<uint32_t>(c.column));
    if (sqlite_utils::IsOpEq(c.op) && col.IsId()) {
      // If we have an id equality constraint, it's a bit expensive to find
      // the exact row but it filters down to a single row.
      filter_cost += 100;
      current_row_count = 1;
    } else if (sqlite_utils::IsOpEq(c.op)) {
      // If there is only a single equality constraint, we have special logic
      // to sort by that column and then binary search if we see the constraint
      // set often. Model this by dividing but the log of the number of rows as
      // a good approximation. Otherwise, we'll need to do a full table scan.
      filter_cost += cs.size() == 1
                         ? (2 * current_row_count) / log2(current_row_count)
                         : current_row_count;

      // We assume that an equalty constraint will cut down the number of rows
      // by approximate log of the number of rows.
      double estimated_rows = current_row_count / log2(current_row_count);
      current_row_count = std::max(static_cast<uint32_t>(estimated_rows), 1u);
    } else {
      // Otherwise, we will need to do a full table scan and we estimate we will
      // maybe (at best) halve the number of rows.
      filter_cost += current_row_count;
      current_row_count = std::max(current_row_count / 2u, 1u);
    }
  }

  // Now, to figure out the cost of sorting, multiply the final row count
  // by |qc.order_by().size()| * log(row count). This should act as a crude
  // estimation of the cost.
  double sort_cost =
      qc.order_by().size() * current_row_count * log2(current_row_count);

  // The cost of iterating rows is more expensive than filtering the rows
  // so multiply by an appropriate factor.
  double iteration_cost = current_row_count * 2.0;

  // To get the final cost, add up all the individual components.
  double final_cost =
      kFixedQueryCost + filter_cost + sort_cost + iteration_cost;
  return QueryCost{final_cost, current_row_count};
}

std::unique_ptr<SqliteTable::Cursor> DbSqliteTable::CreateCursor() {
  return std::unique_ptr<Cursor>(new Cursor(this));
}

DbSqliteTable::Cursor::Cursor(DbSqliteTable* table)
    : SqliteTable::Cursor(table), initial_db_table_(table->table_) {}

int DbSqliteTable::Cursor::Filter(const QueryConstraints& qc,
                                  sqlite3_value** argv,
                                  FilterHistory history) {
  // Clear out the iterator before filtering to ensure the destructor is run
  // before the table's destructor.
  iterator_ = base::nullopt;

  if (history == FilterHistory::kSame && qc.constraints().size() == 1 &&
      sqlite_utils::IsOpEq(qc.constraints().front().op)) {
    // If we've seen the same constraint set with a single equality constraint
    // more than |kRepeatedThreshold| times, we assume we will see it more
    // in the future and thus cache a table sorted on the column. That way,
    // future equality constraints can binary search for the value instead of
    // doing a full table scan.
    constexpr uint32_t kRepeatedThreshold = 3;
    if (!sorted_cache_table_ && repeated_cache_count_++ > kRepeatedThreshold) {
      const auto& c = qc.constraints().front();
      uint32_t col = static_cast<uint32_t>(c.column);
      sorted_cache_table_ = initial_db_table_->Sort({Order{col, false}});
    }
  } else {
    sorted_cache_table_ = base::nullopt;
    repeated_cache_count_ = 0;
  }

  // We reuse this vector to reduce memory allocations on nested subqueries.
  constraints_.resize(qc.constraints().size());
  for (size_t i = 0; i < qc.constraints().size(); ++i) {
    const auto& cs = qc.constraints()[i];
    uint32_t col = static_cast<uint32_t>(cs.column);

    FilterOp op = SqliteOpToFilterOp(cs.op);
    SqlValue value = SqliteValueToSqlValue(argv[i]);

    constraints_[i] = Constraint{col, op, value};
  }

  // We reuse this vector to reduce memory allocations on nested subqueries.
  orders_.resize(qc.order_by().size());
  for (size_t i = 0; i < qc.order_by().size(); ++i) {
    const auto& ob = qc.order_by()[i];
    uint32_t col = static_cast<uint32_t>(ob.iColumn);
    orders_[i] = Order{col, static_cast<bool>(ob.desc)};
  }

  // Try and use the sorted cache table (if it exists) to speed up the sorting.
  // Otherwise, just use the original table.
  auto* source =
      sorted_cache_table_ ? &*sorted_cache_table_ : &*initial_db_table_;
  db_table_ = source->Filter(constraints_).Sort(orders_);
  iterator_ = db_table_->IterateRows();

  return SQLITE_OK;
}

int DbSqliteTable::Cursor::Next() {
  iterator_->Next();
  return SQLITE_OK;
}

int DbSqliteTable::Cursor::Eof() {
  return !*iterator_;
}

int DbSqliteTable::Cursor::Column(sqlite3_context* ctx, int raw_col) {
  uint32_t column = static_cast<uint32_t>(raw_col);
  SqlValue value = iterator_->Get(column);
  switch (value.type) {
    case SqlValue::Type::kLong:
      sqlite3_result_int64(ctx, value.long_value);
      break;
    case SqlValue::Type::kDouble:
      sqlite3_result_double(ctx, value.double_value);
      break;
    case SqlValue::Type::kString: {
      // We can say kSqliteStatic here because all strings are expected to
      // come from the string pool and thus will be valid for the lifetime
      // of trace processor.
      sqlite3_result_text(ctx, value.string_value, -1,
                          sqlite_utils::kSqliteStatic);
      break;
    }
    case SqlValue::Type::kBytes: {
      // We can say kSqliteStatic here because for our iterator will hold
      // onto the pointer as long as we don't call Next() but that only
      // happens with Next() is called on the Cursor itself at which point
      // SQLite no longer cares about the bytes pointer.
      sqlite3_result_blob(ctx, value.bytes_value,
                          static_cast<int>(value.bytes_count),
                          sqlite_utils::kSqliteStatic);
      break;
    }
    case SqlValue::Type::kNull:
      sqlite3_result_null(ctx);
      break;
  }
  return SQLITE_OK;
}

}  // namespace trace_processor
}  // namespace perfetto
<|endoftext|>