<commit_before>//*****************************************************************************
// Copyright 2017-2019 Intel Corporation
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//*****************************************************************************

#pragma once

#include <memory>

#include "ngraph/axis_set.hpp"
#include "ngraph/graph_util.hpp"
#include "ngraph/op/op.hpp"

namespace ngraph
{
    namespace op
    {
        //brief Computes indices of top k maximum/minimum index along a specified axis for a given tensor
        class TopK : public Op
        {
        public:
            enum class SortType
            {
                // Returned values are not sorted
                NONE,
                // Sort result based on element indices
                SORT_INDICES,
                // Sort result based on element values
                SORT_VALUES,
            };

            NGRAPH_API
            static const std::string type_name;
            const std::string& description() const override { return type_name; }
            /// \brief Constructs a TopK operation
            TopK() = default;
            /// \brief Constructs a TopK operation.
            ///
            /// \param arg The input tensor
            /// \param top_k_axis The axis along which to compute top k indices
            /// \param index_element_type produce indices. Currently, only int64 or int32 are supported
            /// \param k Number of top indices to compute. Compute all indices if k = 0
            /// \param compute_max Compute top k max or top k min?
            /// \param sort SortType for sorting results, default - NONE
            TopK(const Output<Node>& arg,
                 size_t top_k_axis,
                 const element::Type& index_element_type,
                 size_t k = 0,
                 bool compute_max = true,
                 SortType sort = SortType::NONE);
            /// \brief Constructs a TopK operation.
            ///
            /// \param arg The input tensor
            /// \param k Number of top indices to compute. Compute all indices if k = 0
            /// \param top_k_axis The axis along which to compute top k indices
            /// \param index_element_type produce indices. Currently, only int64 or int32 are supported
            /// \param compute_max Compute top k max or top k min?
            /// \param sort SortType for sorting results, default - NONE
            TopK(const Output<Node>& arg,
                 const Output<Node>& k,
                 size_t top_k_axis,
                 const element::Type& index_element_type,
                 bool compute_max = true,
                 SortType sort = SortType::NONE);

            void validate_and_infer_types() override;

            virtual std::shared_ptr<Node>
                copy_with_new_args(const NodeVector& new_args) const override;

            size_t get_k() const;
            void set_k(size_t k);

            size_t get_top_k_axis() const { return m_top_k_axis; }
            element::Type get_index_element_type() const { return m_index_element_type; }
            bool get_compute_max() const { return m_compute_max; }
            SortType get_sort() const { return m_sort; }
        protected:
            size_t m_top_k_axis{0};
            element::Type m_index_element_type;
            bool m_compute_max{false};
            SortType m_sort;
            virtual void generate_adjoints(autodiff::Adjoints& adjoints,
                                           const NodeVector& deltas) override;
        };
    }
}
<commit_msg>#3413 for master (#3414)<commit_after>//*****************************************************************************
// Copyright 2017-2019 Intel Corporation
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//*****************************************************************************

#pragma once

#include <memory>

#include "ngraph/axis_set.hpp"
#include "ngraph/graph_util.hpp"
#include "ngraph/op/op.hpp"

namespace ngraph
{
    namespace op
    {
        //brief Computes indices of top k maximum/minimum index along a specified axis for a given tensor
        class TopK : public Op
        {
        public:
            enum class SortType
            {
                // Returned values are not sorted
                NONE,
                // Sort result based on element indices
                SORT_INDICES,
                // Sort result based on element values
                SORT_VALUES,
            };

            NGRAPH_API
            static const std::string type_name;
            const std::string& description() const override { return type_name; }
            /// \brief Constructs a TopK operation
            TopK() = default;
            /// \brief Constructs a TopK operation.
            ///
            /// \param arg The input tensor
            /// \param top_k_axis The axis along which to compute top k indices
            /// \param index_element_type produce indices. Currently, only int64 or int32 are supported
            /// \param k Number of top indices to compute. Compute all indices if k = 0
            /// \param compute_max Compute top k max or top k min?
            /// \param sort SortType for sorting results, default - NONE
            TopK(const Output<Node>& arg,
                 size_t top_k_axis,
                 const element::Type& index_element_type,
                 size_t k = 0,
                 bool compute_max = true,
                 SortType sort = SortType::NONE);
            /// \brief Constructs a TopK operation.
            ///
            /// \param arg The input tensor
            /// \param k Number of top indices to compute. Compute all indices if k = 0
            /// \param top_k_axis The axis along which to compute top k indices
            /// \param index_element_type produce indices. Currently, only int64 or int32 are supported
            /// \param compute_max Compute top k max or top k min?
            /// \param sort SortType for sorting results, default - NONE
            TopK(const Output<Node>& arg,
                 const Output<Node>& k,
                 size_t top_k_axis,
                 const element::Type& index_element_type,
                 bool compute_max = true,
                 SortType sort = SortType::NONE);

            void validate_and_infer_types() override;

            virtual std::shared_ptr<Node>
                copy_with_new_args(const NodeVector& new_args) const override;

            size_t get_k() const;
            void set_k(size_t k);

            size_t get_top_k_axis() const { return m_top_k_axis; }
            element::Type get_index_element_type() const { return m_index_element_type; }
            bool get_compute_max() const { return m_compute_max; }
            SortType get_sort() const { return m_sort; }
        protected:
            size_t m_top_k_axis{0};
            element::Type m_index_element_type;
            bool m_compute_max{false};
            SortType m_sort{SortType::NONE};
            virtual void generate_adjoints(autodiff::Adjoints& adjoints,
                                           const NodeVector& deltas) override;
        };
    }
}
<|endoftext|>


<commit_before>/*
 +----------------------------------------------------------------------+
 | Swoole                                                               |
 +----------------------------------------------------------------------+
 | This source file is subject to version 2.0 of the Apache license,    |
 | that is bundled with this package in the file LICENSE, and is        |
 | available through the world-wide-web at the following url:           |
 | http://www.apache.org/licenses/LICENSE-2.0.html                      |
 | If you did not receive a copy of the Apache2.0 license and are unable|
 | to obtain it through the world-wide-web, please send a note to       |
 | license@swoole.com so we can mail you a copy immediately.            |
 +----------------------------------------------------------------------+
 | Author: Tianfeng Han  <mikan.tenny@gmail.com>                        |
 +----------------------------------------------------------------------+
 */

#include "swoole_api.h"
#include "async.h"

#include <thread>
#include <atomic>
#include <unordered_map>
#include <chrono>
#include <condition_variable>
#include <mutex>
#include <queue>

using namespace std;

typedef swAio_event AsyncEvent;

static mutex init_lock;
static atomic<int> refcount(0);

static void aio_thread_release(AsyncEvent *event);

namespace swoole { namespace async {
//-------------------------------------------------------------------------------
class EventQueue
{
public:
    inline void push(AsyncEvent *event)
    {
        unique_lock<mutex> lock(_mutex);
        _queue.push(event);
    }

    inline AsyncEvent* pop()
    {
        unique_lock<mutex> lock(_mutex);
        if (_queue.empty())
        {
            return nullptr;
        }
        AsyncEvent* retval = _queue.front();
        _queue.pop();
        return retval;
    }

    inline double get_max_wait_time()
    {
        unique_lock<mutex> lock(_mutex);
        if (_queue.empty())
        {
            return 0;
        }
        else
        {
            AsyncEvent* event = _queue.front();
            return swoole_microtime() - event->timestamp;
        }
    }

    inline size_t count()
    {
        unique_lock<mutex> lock(_mutex);
        return _queue.size();
    }

private:
    queue<AsyncEvent *> _queue;
    mutex _mutex;
};

class ThreadPool
{
public:
    ThreadPool(size_t _core_worker_num, size_t _worker_num, double _max_wait_time, double _max_idle_time)
    {
        running = false;

        core_worker_num = _core_worker_num == 0 ? SW_CPU_NUM : SW_MAX(1, _core_worker_num);
        worker_num = _worker_num == 0 ? SW_CPU_NUM * SW_AIO_THREAD_NUM_MULTIPLE : SW_MAX(core_worker_num, _worker_num);
        max_wait_time = _max_wait_time == 0 ? SW_AIO_TASK_MAX_WAIT_TIME : _max_wait_time;
        max_idle_time = _max_idle_time == 0 ? SW_AIO_THREAD_MAX_IDLE_TIME : _max_idle_time;

        current_pid = getpid();
    }

    ~ThreadPool()
    {
        shutdown();
    }

    bool start()
    {
        running = true;
        current_task_id = 0;
        n_waiting = 0;
        n_closing = 0;
        for (size_t i = 0; i < core_worker_num; i++)
        {
            create_thread(true);
        }
        return true;
    }

    bool shutdown()
    {
        if (!running)
        {
            return false;
        }
        running = false;

        event_mutex.lock();
        _cv.notify_all();
        event_mutex.unlock();

        for (auto &i : threads)
        {
            thread *_thread = i.second;
            if (_thread->joinable())
            {
                _thread->join();
            }
            delete _thread;
        }

        return true;
    }

    void schedule()
    {
        if (n_waiting == 0 && threads.size() < worker_num && max_wait_time > 0)
        {
            double _max_wait_time = _queue.get_max_wait_time();
            if (_max_wait_time > max_wait_time)
            {
                size_t n = 1;
                /**
                 * maybe we can find a better strategy
                 */
                if (threads.size() + n > worker_num)
                {
                    n = worker_num - threads.size();
                }
                swTraceLog(SW_TRACE_AIO, "Create %zu thread due to wait %fs, we will have %zu threads", n, _max_wait_time, threads.size() + n);
                while (n--)
                {
                    create_thread();
                }
            }
        }
    }

    AsyncEvent* dispatch(const AsyncEvent *request)
    {
        if (SwooleTG.aio_schedule)
        {
            schedule();
        }
        auto _event_copy = new AsyncEvent(*request);
        _event_copy->task_id = current_task_id++;
        _event_copy->timestamp = swoole_microtime();
        _event_copy->pipe_fd = SwooleTG.aio_pipe_write;
        _queue.push(_event_copy);
        _cv.notify_one();
        swDebug("push and notify one: %f", swoole_microtime());
        return _event_copy;
    }

    inline size_t worker_count()
    {
        return threads.size();
    }

    inline size_t queue_count()
    {
        return _queue.count();
    }

    pid_t current_pid;

    void release_thread(thread::id tid)
    {
        auto i = threads.find(tid);
        if (i == threads.end())
        {
            swWarn("AIO thread#%zu is missing", tid);
            return;
        }
        else
        {
            thread *_thread = i->second;
            swTraceLog(SW_TRACE_AIO, "release idle thread#%zu, we have %zu now", tid, threads.size() - 1);
            if (_thread->joinable())
            {
                _thread->join();
            }
            threads.erase(i);
            delete _thread;
        }
    }

#ifdef SW_DEBUG
    void notify_one()
    {
        _cv.notify_one();
    }
#endif

private:
    void create_thread(const bool is_core_worker = false);

    size_t core_worker_num;
    size_t worker_num;
    double max_wait_time;
    double max_idle_time;

    bool running;

    atomic<size_t> n_waiting;
    atomic<size_t> n_closing;
    size_t current_task_id = 0;

    unordered_map<thread::id, thread *> threads;
    EventQueue _queue;
    mutex event_mutex;
    condition_variable _cv;
};
//-------------------------------------------------------------------------------
}};

static swoole::async::ThreadPool *pool = nullptr;

void swoole::async::ThreadPool::create_thread(const bool is_core_worker)
{
    try
    {
        thread *_thread = new thread([this, is_core_worker]()
        {
            bool exit_flag = false;

            SwooleTG.buffer_stack = swString_new(SW_STACK_BUFFER_SIZE);
            if (SwooleTG.buffer_stack == nullptr)
            {
                return;
            }

            swSignal_none();

            while (running)
            {
                AsyncEvent *event = _queue.pop();

                swDebug("%s: %f", event ? "pop 1 event" : "no event", swoole_microtime());

                if (event)
                {
                    if (sw_unlikely(event->handler == nullptr))
                    {
                        event->error = SW_ERROR_AIO_BAD_REQUEST;
                        event->ret = -1;
                    }
                    else if (sw_unlikely(event->canceled))
                    {
                        event->error = SW_ERROR_AIO_CANCELED;
                        event->ret = -1;
                    }
                    else
                    {
                        event->handler(event);
                    }

                    swTraceLog(SW_TRACE_AIO, "aio_thread %s. ret=%d, error=%d", event->ret > 0 ? "ok" : "failed", event->ret, event->error);

                    _send_event:
                    while (true)
                    {
                        int ret = write(event->pipe_fd, &event, sizeof(event));
                        if (ret < 0)
                        {
                            if (errno == EAGAIN)
                            {
                                swSocket_wait(SwooleTG.aio_pipe_write, 1000, SW_EVENT_WRITE);
                                continue;
                            }
                            else if (errno == EINTR)
                            {
                                continue;
                            }
                            else
                            {
                                swSysWarn("sendto swoole_aio_pipe_write failed");
                            }
                        }
                        break;
                    }

                    // exit
                    if (exit_flag)
                    {
                        n_closing--;
                        break;
                    }
                }
                else if (running)
                {
                    unique_lock<mutex> lock(event_mutex);
                    ++n_waiting;
                    if (is_core_worker || max_idle_time <= 0)
                    {
                        _cv.wait(lock);
                    }
                    else
                    {
                        while (true)
                        {
                            if (_cv.wait_for(lock, chrono::microseconds((size_t) (max_idle_time * 1000 * 1000))) == cv_status::timeout)
                            {
                                if (running && n_closing != 0)
                                {
                                    // wait for the next round
                                    continue;
                                }
                                /* notifies the main thread to release this thread */
                                event = new AsyncEvent;
                                event->object = new thread::id(this_thread::get_id());
                                event->callback = aio_thread_release;
                                event->pipe_fd = SwooleG.aio_default_pipe_fd;

                                --n_waiting;
                                ++n_closing;
                                exit_flag = true;
                                goto _send_event;
                            }
                            break;
                        }
                    }
                    --n_waiting;
                }
            }
        });
        threads[_thread->get_id()] = _thread;
    }
    catch (const std::system_error& e)
    {
        swSysNotice("create aio thread failed, please check your system configuration or adjust aio_worker_num");
        return;
    }
}

static void aio_thread_release(swAio_event *event)
{
    thread::id *tid = static_cast<thread::id *>(event->object);
    pool->release_thread(*tid);
    delete tid;
    // balance
    SwooleTG.aio_task_num++;
}

static void swAio_free(void *private_data)
{
    if (!SwooleTG.aio_init)
    {
        return;
    }
    SwooleTG.aio_init = 0;
    swoole_event_del(SwooleTG.aio_pipe_read);
    SwooleTG.aio_pipe.close(&SwooleTG.aio_pipe);
    if (pool->current_pid == getpid())
    {
        if ((--refcount) == 0)
        {
            delete pool;
            pool = nullptr;
        }
    }
}

static int swAio_init()
{
    if (SwooleTG.aio_init)
    {
        swWarn("aio_thread_pool has already been initialized");
        return SW_ERR;
    }
    if (!SwooleTG.reactor)
    {
        swWarn("no event loop, cannot initialized");
        return SW_ERR;
    }

    if (swPipeBase_create(&SwooleTG.aio_pipe, 0) < 0)
    {
        swoole_throw_error(SW_ERROR_SYSTEM_CALL_FAIL);
    }

    SwooleTG.aio_pipe_read = SwooleTG.aio_pipe.getFd(&SwooleTG.aio_pipe, 0);
    SwooleTG.aio_pipe_write = SwooleTG.aio_pipe.getFd(&SwooleTG.aio_pipe, 1);
    swoole_event_add(SwooleTG.aio_pipe_read, SW_EVENT_READ, SW_FD_AIO);
    swReactor_add_destroy_callback(SwooleTG.reactor, swAio_free, nullptr);

    init_lock.lock();
    if ((refcount++) == 0)
    {
        pool = new swoole::async::ThreadPool(
            SwooleG.aio_core_worker_num, SwooleG.aio_worker_num,
            SwooleG.aio_max_wait_time, SwooleG.aio_max_idle_time
        );
        pool->start();
        SwooleTG.aio_schedule = 1;
        SwooleG.aio_default_pipe_fd = SwooleTG.aio_pipe_write;
    }
    SwooleTG.aio_init = 1;
    init_lock.unlock();

    return SW_OK;
}

size_t swAio_thread_count()
{
    return pool ? pool->worker_count() : 0;
}

ssize_t swAio_dispatch(const swAio_event *request)
{
    AsyncEvent *event = swAio_dispatch2(request);
    if (event == nullptr)
    {
        return -1;
    }
    return event->task_id;
}

swAio_event* swAio_dispatch2(const swAio_event *request)
{
    if (sw_unlikely(!SwooleTG.aio_init))
    {
        swAio_init();
    }
    AsyncEvent *event = pool->dispatch(request);
    if (sw_likely(event))
    {
        SwooleTG.aio_task_num++;
    }
    return event;
}

int swAio_callback(swReactor *reactor, swEvent *event)
{
    if (SwooleTG.aio_schedule)
    {
        pool->schedule();
    }

    AsyncEvent *events[SW_AIO_EVENT_NUM];
    ssize_t n = read(event->fd, events, sizeof(AsyncEvent *) * SW_AIO_EVENT_NUM);
    if (n < 0)
    {
        swSysWarn("read() aio events failed");
        return SW_ERR;
    }
    for (size_t i = 0; i < n / sizeof(AsyncEvent *); i++)
    {
        AsyncEvent *event = events[i];
        if (!event->canceled)
        {
            event->callback(events[i]);
        }
        SwooleTG.aio_task_num--;
        delete event;
    }

    return SW_OK;
}

#ifdef SW_DEBUG
void swAio_notify_one()
{
    if (pool)
    {
        pool->notify_one();
    }
}
#endif
<commit_msg>Lock event_mutex<commit_after>/*
 +----------------------------------------------------------------------+
 | Swoole                                                               |
 +----------------------------------------------------------------------+
 | This source file is subject to version 2.0 of the Apache license,    |
 | that is bundled with this package in the file LICENSE, and is        |
 | available through the world-wide-web at the following url:           |
 | http://www.apache.org/licenses/LICENSE-2.0.html                      |
 | If you did not receive a copy of the Apache2.0 license and are unable|
 | to obtain it through the world-wide-web, please send a note to       |
 | license@swoole.com so we can mail you a copy immediately.            |
 +----------------------------------------------------------------------+
 | Author: Tianfeng Han  <mikan.tenny@gmail.com>                        |
 +----------------------------------------------------------------------+
 */

#include "swoole_api.h"
#include "async.h"

#include <thread>
#include <atomic>
#include <unordered_map>
#include <chrono>
#include <condition_variable>
#include <mutex>
#include <queue>

using namespace std;

typedef swAio_event AsyncEvent;

static mutex init_lock;
static atomic<int> refcount(0);

static void aio_thread_release(AsyncEvent *event);

namespace swoole { namespace async {
//-------------------------------------------------------------------------------
class EventQueue
{
public:
    inline void push(AsyncEvent *event)
    {
        unique_lock<mutex> lock(_mutex);
        _queue.push(event);
    }

    inline AsyncEvent* pop()
    {
        unique_lock<mutex> lock(_mutex);
        if (_queue.empty())
        {
            return nullptr;
        }
        AsyncEvent* retval = _queue.front();
        _queue.pop();
        return retval;
    }

    inline double get_max_wait_time()
    {
        unique_lock<mutex> lock(_mutex);
        if (_queue.empty())
        {
            return 0;
        }
        else
        {
            AsyncEvent* event = _queue.front();
            return swoole_microtime() - event->timestamp;
        }
    }

    inline size_t count()
    {
        unique_lock<mutex> lock(_mutex);
        return _queue.size();
    }

private:
    queue<AsyncEvent *> _queue;
    mutex _mutex;
};

class ThreadPool
{
public:
    ThreadPool(size_t _core_worker_num, size_t _worker_num, double _max_wait_time, double _max_idle_time)
    {
        running = false;

        core_worker_num = _core_worker_num == 0 ? SW_CPU_NUM : SW_MAX(1, _core_worker_num);
        worker_num = _worker_num == 0 ? SW_CPU_NUM * SW_AIO_THREAD_NUM_MULTIPLE : SW_MAX(core_worker_num, _worker_num);
        max_wait_time = _max_wait_time == 0 ? SW_AIO_TASK_MAX_WAIT_TIME : _max_wait_time;
        max_idle_time = _max_idle_time == 0 ? SW_AIO_THREAD_MAX_IDLE_TIME : _max_idle_time;

        current_pid = getpid();
    }

    ~ThreadPool()
    {
        shutdown();
    }

    bool start()
    {
        running = true;
        current_task_id = 0;
        n_waiting = 0;
        n_closing = 0;
        for (size_t i = 0; i < core_worker_num; i++)
        {
            create_thread(true);
        }
        return true;
    }

    bool shutdown()
    {
        if (!running)
        {
            return false;
        }
        running = false;

        event_mutex.lock();
        _cv.notify_all();
        event_mutex.unlock();

        for (auto &i : threads)
        {
            thread *_thread = i.second;
            if (_thread->joinable())
            {
                _thread->join();
            }
            delete _thread;
        }

        return true;
    }

    void schedule()
    {
        if (n_waiting == 0 && threads.size() < worker_num && max_wait_time > 0)
        {
            double _max_wait_time = _queue.get_max_wait_time();
            if (_max_wait_time > max_wait_time)
            {
                size_t n = 1;
                /**
                 * maybe we can find a better strategy
                 */
                if (threads.size() + n > worker_num)
                {
                    n = worker_num - threads.size();
                }
                swTraceLog(SW_TRACE_AIO, "Create %zu thread due to wait %fs, we will have %zu threads", n, _max_wait_time, threads.size() + n);
                while (n--)
                {
                    create_thread();
                }
            }
        }
    }

    AsyncEvent* dispatch(const AsyncEvent *request)
    {
        if (SwooleTG.aio_schedule)
        {
            schedule();
        }
        auto _event_copy = new AsyncEvent(*request);
        _event_copy->task_id = current_task_id++;
        _event_copy->timestamp = swoole_microtime();
        _event_copy->pipe_fd = SwooleTG.aio_pipe_write;
        _queue.push(_event_copy);
        event_mutex.lock();
        _cv.notify_one();
        event_mutex.unlock();
        swDebug("push and notify one: %f", swoole_microtime());
        return _event_copy;
    }

    inline size_t worker_count()
    {
        return threads.size();
    }

    inline size_t queue_count()
    {
        return _queue.count();
    }

    pid_t current_pid;

    void release_thread(thread::id tid)
    {
        auto i = threads.find(tid);
        if (i == threads.end())
        {
            swWarn("AIO thread#%zu is missing", tid);
            return;
        }
        else
        {
            thread *_thread = i->second;
            swTraceLog(SW_TRACE_AIO, "release idle thread#%zu, we have %zu now", tid, threads.size() - 1);
            if (_thread->joinable())
            {
                _thread->join();
            }
            threads.erase(i);
            delete _thread;
        }
    }

#ifdef SW_DEBUG
    void notify_one()
    {
        _cv.notify_one();
    }
#endif

private:
    void create_thread(const bool is_core_worker = false);

    size_t core_worker_num;
    size_t worker_num;
    double max_wait_time;
    double max_idle_time;

    bool running;

    atomic<size_t> n_waiting;
    atomic<size_t> n_closing;
    size_t current_task_id = 0;

    unordered_map<thread::id, thread *> threads;
    EventQueue _queue;
    mutex event_mutex;
    condition_variable _cv;
};
//-------------------------------------------------------------------------------
}};

static swoole::async::ThreadPool *pool = nullptr;

void swoole::async::ThreadPool::create_thread(const bool is_core_worker)
{
    try
    {
        thread *_thread = new thread([this, is_core_worker]()
        {
            bool exit_flag = false;

            SwooleTG.buffer_stack = swString_new(SW_STACK_BUFFER_SIZE);
            if (SwooleTG.buffer_stack == nullptr)
            {
                return;
            }

            swSignal_none();

            while (running)
            {
                AsyncEvent *event = _queue.pop();

                swDebug("%s: %f", event ? "pop 1 event" : "no event", swoole_microtime());

                if (event)
                {
                    if (sw_unlikely(event->handler == nullptr))
                    {
                        event->error = SW_ERROR_AIO_BAD_REQUEST;
                        event->ret = -1;
                    }
                    else if (sw_unlikely(event->canceled))
                    {
                        event->error = SW_ERROR_AIO_CANCELED;
                        event->ret = -1;
                    }
                    else
                    {
                        event->handler(event);
                    }

                    swTraceLog(SW_TRACE_AIO, "aio_thread %s. ret=%d, error=%d", event->ret > 0 ? "ok" : "failed", event->ret, event->error);

                    _send_event:
                    while (true)
                    {
                        int ret = write(event->pipe_fd, &event, sizeof(event));
                        if (ret < 0)
                        {
                            if (errno == EAGAIN)
                            {
                                swSocket_wait(SwooleTG.aio_pipe_write, 1000, SW_EVENT_WRITE);
                                continue;
                            }
                            else if (errno == EINTR)
                            {
                                continue;
                            }
                            else
                            {
                                swSysWarn("sendto swoole_aio_pipe_write failed");
                            }
                        }
                        break;
                    }

                    // exit
                    if (exit_flag)
                    {
                        n_closing--;
                        break;
                    }
                }
                else if (running)
                {
                    unique_lock<mutex> lock(event_mutex);
                    ++n_waiting;
                    if (is_core_worker || max_idle_time <= 0)
                    {
                        _cv.wait(lock);
                    }
                    else
                    {
                        while (true)
                        {
                            if (_cv.wait_for(lock, chrono::microseconds((size_t) (max_idle_time * 1000 * 1000))) == cv_status::timeout)
                            {
                                if (running && n_closing != 0)
                                {
                                    // wait for the next round
                                    continue;
                                }
                                /* notifies the main thread to release this thread */
                                event = new AsyncEvent;
                                event->object = new thread::id(this_thread::get_id());
                                event->callback = aio_thread_release;
                                event->pipe_fd = SwooleG.aio_default_pipe_fd;

                                --n_waiting;
                                ++n_closing;
                                exit_flag = true;
                                goto _send_event;
                            }
                            break;
                        }
                    }
                    --n_waiting;
                }
            }
        });
        threads[_thread->get_id()] = _thread;
    }
    catch (const std::system_error& e)
    {
        swSysNotice("create aio thread failed, please check your system configuration or adjust aio_worker_num");
        return;
    }
}

static void aio_thread_release(swAio_event *event)
{
    thread::id *tid = static_cast<thread::id *>(event->object);
    pool->release_thread(*tid);
    delete tid;
    // balance
    SwooleTG.aio_task_num++;
}

static void swAio_free(void *private_data)
{
    if (!SwooleTG.aio_init)
    {
        return;
    }
    SwooleTG.aio_init = 0;
    swoole_event_del(SwooleTG.aio_pipe_read);
    SwooleTG.aio_pipe.close(&SwooleTG.aio_pipe);
    if (pool->current_pid == getpid())
    {
        if ((--refcount) == 0)
        {
            delete pool;
            pool = nullptr;
        }
    }
}

static int swAio_init()
{
    if (SwooleTG.aio_init)
    {
        swWarn("aio_thread_pool has already been initialized");
        return SW_ERR;
    }
    if (!SwooleTG.reactor)
    {
        swWarn("no event loop, cannot initialized");
        return SW_ERR;
    }

    if (swPipeBase_create(&SwooleTG.aio_pipe, 0) < 0)
    {
        swoole_throw_error(SW_ERROR_SYSTEM_CALL_FAIL);
    }

    SwooleTG.aio_pipe_read = SwooleTG.aio_pipe.getFd(&SwooleTG.aio_pipe, 0);
    SwooleTG.aio_pipe_write = SwooleTG.aio_pipe.getFd(&SwooleTG.aio_pipe, 1);
    swoole_event_add(SwooleTG.aio_pipe_read, SW_EVENT_READ, SW_FD_AIO);
    swReactor_add_destroy_callback(SwooleTG.reactor, swAio_free, nullptr);

    init_lock.lock();
    if ((refcount++) == 0)
    {
        pool = new swoole::async::ThreadPool(
            SwooleG.aio_core_worker_num, SwooleG.aio_worker_num,
            SwooleG.aio_max_wait_time, SwooleG.aio_max_idle_time
        );
        pool->start();
        SwooleTG.aio_schedule = 1;
        SwooleG.aio_default_pipe_fd = SwooleTG.aio_pipe_write;
    }
    SwooleTG.aio_init = 1;
    init_lock.unlock();

    return SW_OK;
}

size_t swAio_thread_count()
{
    return pool ? pool->worker_count() : 0;
}

ssize_t swAio_dispatch(const swAio_event *request)
{
    AsyncEvent *event = swAio_dispatch2(request);
    if (event == nullptr)
    {
        return -1;
    }
    return event->task_id;
}

swAio_event* swAio_dispatch2(const swAio_event *request)
{
    if (sw_unlikely(!SwooleTG.aio_init))
    {
        swAio_init();
    }
    AsyncEvent *event = pool->dispatch(request);
    if (sw_likely(event))
    {
        SwooleTG.aio_task_num++;
    }
    return event;
}

int swAio_callback(swReactor *reactor, swEvent *event)
{
    if (SwooleTG.aio_schedule)
    {
        pool->schedule();
    }

    AsyncEvent *events[SW_AIO_EVENT_NUM];
    ssize_t n = read(event->fd, events, sizeof(AsyncEvent *) * SW_AIO_EVENT_NUM);
    if (n < 0)
    {
        swSysWarn("read() aio events failed");
        return SW_ERR;
    }
    for (size_t i = 0; i < n / sizeof(AsyncEvent *); i++)
    {
        AsyncEvent *event = events[i];
        if (!event->canceled)
        {
            event->callback(events[i]);
        }
        SwooleTG.aio_task_num--;
        delete event;
    }

    return SW_OK;
}

#ifdef SW_DEBUG
void swAio_notify_one()
{
    if (pool)
    {
        pool->notify_one();
    }
}
#endif
<|endoftext|>


<commit_before>/*******************************************************************************
 * include/wx_ppc_ss.hpp
 *
 * Copyright (C) 2017-2018 Florian Kurpicz <florian.kurpicz@tu-dortmund.de>
 *
 * All rights reserved. Published under the BSD-2 license in the LICENSE file.
 ******************************************************************************/

#pragma once

#include <omp.h>

#include "construction/ctx_compute_borders.hpp"
#include "construction/pc_ss.hpp"
#include "construction/wavelet_structure.hpp"
#include "construction/building_blocks.hpp"

template <typename AlphabteType, bool is_tree_>
class wx_ppc_ss {

public:
  static constexpr bool is_parallel = true;
  static constexpr bool is_tree     = is_tree_;
  static constexpr uint8_t word_width = sizeof(AlphabteType);
  static constexpr bool  is_huffman_shaped = false;

  using ctx_t = ctx_compute_borders<is_tree>;

  template <typename InputType>
  static wavelet_structure compute(const InputType& text, const uint64_t size,
    const uint64_t levels) {

    if(size == 0) {
      if constexpr (ctx_t::compute_zeros) { return wavelet_structure_matrix(); }
      else { return wavelet_structure_tree(); }
    }

    const auto rho = rho_dispatch<is_tree>::create(levels);
    auto ctx = ctx_t(size, levels, rho);
    auto& bv = ctx.bv();
    auto& zeros = ctx.zeros();

    const uint64_t alphabet_size = (1 << levels);

    std::vector<std::vector<uint64_t>> all_hists;

    #pragma omp parallel
    {
      const auto omp_rank = omp_get_thread_num();
      const auto omp_size = omp_get_num_threads();

      #pragma omp single
      all_hists = std::vector<std::vector<uint64_t>>(omp_size,
        std::vector<uint64_t>(alphabet_size + 1, 0));

      #pragma omp for
      for (uint64_t cur_pos = 0; cur_pos <= size - 64; cur_pos += 64) {
        uint64_t word = 0ULL;
        for (uint64_t i = 0; i < 64; ++i) {
          //++ctx.hist(omp_rank, text[cur_pos + i]);
          ++all_hists[omp_rank][text[cur_pos + i]];
          word <<= 1;
          word |= ((text[cur_pos + i] >> (levels - 1)) & 1ULL);
        }
        bv[0][cur_pos >> 6] = word;
      }
      if ((size & 63ULL) && ((omp_rank + 1) == omp_size)) {
        uint64_t word = 0ULL;
        for (uint64_t i = 0; i < (size & 63ULL); ++i) {
          //++ctx.hist(omp_rank, text[size - (size & 63ULL) + i]);
          ++all_hists[omp_rank][text[size - (size & 63ULL) + i]];
          word <<= 1;
          word |= ((text[size - (size & 63ULL) + i] >> (levels - 1)) & 1ULL);
        }
        word <<= (64 - (size & 63ULL));
        bv[0][size >> 6] = word;
      }
    }

    for (uint64_t i = 0; i < all_hists.size(); ++i) {
      for (uint64_t j = 0; j < alphabet_size; ++j) {
        ctx.hist(levels, j) += all_hists[i][j];
      }
    }
    if constexpr (ctx_t::compute_zeros) {
      for (uint64_t i = 0; i < alphabet_size; i += 2) {
        zeros[levels - 1] += ctx.hist(levels, i);
      }
    }

    bottom_up_compute_hist_and_borders_and_optional_zeros(size, levels, ctx);

    // TODO: Is this correct?
    #pragma omp parallel num_threads(levels)
    {
      uint64_t level = omp_get_thread_num();
      for (uint64_t i = 0; i < size; ++i) {
          const uint64_t prefix_shift = (levels - level);
          const uint64_t cur_bit_shift = prefix_shift - 1;
          const uint64_t pos = ctx.borders(level, text[i] >> prefix_shift)++;
          bv[level][pos >> 6] |= (((text[i] >> cur_bit_shift) & 1ULL)
            << (63ULL - (pos & 63ULL)));
      }
    }

    if constexpr (ctx_t::compute_zeros) {
      return wavelet_structure_matrix(std::move(ctx.bv()), std::move(ctx.zeros()));
    } else { return wavelet_structure_tree(std::move(ctx.bv())); }
  }
}; // class wx_ppc_ss

/******************************************************************************/
<commit_msg>Fix indent<commit_after>/*******************************************************************************
 * include/wx_ppc_ss.hpp
 *
 * Copyright (C) 2017-2018 Florian Kurpicz <florian.kurpicz@tu-dortmund.de>
 *
 * All rights reserved. Published under the BSD-2 license in the LICENSE file.
 ******************************************************************************/

#pragma once

#include <omp.h>

#include "construction/ctx_compute_borders.hpp"
#include "construction/pc_ss.hpp"
#include "construction/wavelet_structure.hpp"
#include "construction/building_blocks.hpp"

template <typename AlphabteType, bool is_tree_>
class wx_ppc_ss {

public:
  static constexpr bool is_parallel = true;
  static constexpr bool is_tree     = is_tree_;
  static constexpr uint8_t word_width = sizeof(AlphabteType);
  static constexpr bool  is_huffman_shaped = false;

  using ctx_t = ctx_compute_borders<is_tree>;

  template <typename InputType>
  static wavelet_structure compute(const InputType& text, const uint64_t size,
    const uint64_t levels) {

    if(size == 0) {
      if constexpr (ctx_t::compute_zeros) { return wavelet_structure_matrix(); }
      else { return wavelet_structure_tree(); }
    }

    const auto rho = rho_dispatch<is_tree>::create(levels);
    auto ctx = ctx_t(size, levels, rho);
    auto& bv = ctx.bv();
    auto& zeros = ctx.zeros();

    const uint64_t alphabet_size = (1 << levels);

    std::vector<std::vector<uint64_t>> all_hists;

    #pragma omp parallel
    {
      const auto omp_rank = omp_get_thread_num();
      const auto omp_size = omp_get_num_threads();

      #pragma omp single
      all_hists = std::vector<std::vector<uint64_t>>(omp_size,
        std::vector<uint64_t>(alphabet_size + 1, 0));

      #pragma omp for
      for (uint64_t cur_pos = 0; cur_pos <= size - 64; cur_pos += 64) {
        uint64_t word = 0ULL;
        for (uint64_t i = 0; i < 64; ++i) {
          //++ctx.hist(omp_rank, text[cur_pos + i]);
          ++all_hists[omp_rank][text[cur_pos + i]];
          word <<= 1;
          word |= ((text[cur_pos + i] >> (levels - 1)) & 1ULL);
        }
        bv[0][cur_pos >> 6] = word;
      }
      if ((size & 63ULL) && ((omp_rank + 1) == omp_size)) {
        uint64_t word = 0ULL;
        for (uint64_t i = 0; i < (size & 63ULL); ++i) {
          //++ctx.hist(omp_rank, text[size - (size & 63ULL) + i]);
          ++all_hists[omp_rank][text[size - (size & 63ULL) + i]];
          word <<= 1;
          word |= ((text[size - (size & 63ULL) + i] >> (levels - 1)) & 1ULL);
        }
        word <<= (64 - (size & 63ULL));
        bv[0][size >> 6] = word;
      }
    }

    for (uint64_t i = 0; i < all_hists.size(); ++i) {
      for (uint64_t j = 0; j < alphabet_size; ++j) {
        ctx.hist(levels, j) += all_hists[i][j];
      }
    }
    if constexpr (ctx_t::compute_zeros) {
      for (uint64_t i = 0; i < alphabet_size; i += 2) {
        zeros[levels - 1] += ctx.hist(levels, i);
      }
    }

    bottom_up_compute_hist_and_borders_and_optional_zeros(size, levels, ctx);

    // TODO: Is this correct?
    #pragma omp parallel num_threads(levels)
    {
      uint64_t level = omp_get_thread_num();
      for (uint64_t i = 0; i < size; ++i) {
        const uint64_t prefix_shift = (levels - level);
        const uint64_t cur_bit_shift = prefix_shift - 1;
        const uint64_t pos = ctx.borders(level, text[i] >> prefix_shift)++;
        bv[level][pos >> 6] |= (((text[i] >> cur_bit_shift) & 1ULL)
          << (63ULL - (pos & 63ULL)));
      }
    }

    if constexpr (ctx_t::compute_zeros) {
      return wavelet_structure_matrix(std::move(ctx.bv()), std::move(ctx.zeros()));
    } else { return wavelet_structure_tree(std::move(ctx.bv())); }
  }
}; // class wx_ppc_ss

/******************************************************************************/
<|endoftext|>


<commit_before>/*
 * Copyright(c) Sophist Solutions, Inc. 1990-2018.  All rights reserved
 */
//  TEST    Foundation::Caching
#include "Stroika/Foundation/StroikaPreComp.h"

#include "Stroika/Foundation/Cache/LRUCache.h"
#include "Stroika/Foundation/Cache/Memoizer.h"
#include "Stroika/Foundation/Cache/TimedCache.h"
#include "Stroika/Foundation/Debug/Assertions.h"
#include "Stroika/Foundation/Debug/TimingTrace.h"
#include "Stroika/Foundation/Debug/Trace.h"
#include "Stroika/Foundation/Execution/Synchronized.h"
#include "Stroika/Foundation/Memory/Optional.h"

#include "../TestHarness/SimpleClass.h"
#include "../TestHarness/TestHarness.h"

using namespace Stroika;
using namespace Stroika::Foundation;
using namespace Stroika::Foundation::Cache;

namespace {
    namespace Test1_Simple_ {
        namespace Private_ {
            void T1_ ()
            {
                LRUCache<string, string> tmp (3);
                tmp.Add ("a", "1");
                tmp.Add ("b", "2");
                tmp.Add ("c", "3");
                tmp.Add ("d", "4");
                VerifyTestResult (not tmp.Lookup ("a").has_value ());
                VerifyTestResult (tmp.Lookup ("b") == "2");
                VerifyTestResult (tmp.Lookup ("d") == "4");

                LRUCache<string, string> tmp2 = tmp;
                VerifyTestResult (not tmp2.Lookup ("a").has_value ());
                VerifyTestResult (tmp2.Lookup ("b") == "2");
                VerifyTestResult (tmp2.Lookup ("d") == "4");
            }
            void T2_ ()
            {
                using CACHE = LRUCache<string, string, equal_to<string>, hash<string>>;
                CACHE tmp (3, equal_to<string>{}, 10);
                tmp.Add ("a", "1");
                tmp.Add ("b", "2");
                tmp.Add ("c", "3");
                tmp.Add ("d", "4");

                VerifyTestResult (not tmp.Lookup ("a").has_value () or *tmp.Lookup ("a") == "1"); // could be missing or found but if found same value
                VerifyTestResult (tmp.Lookup ("b") == "2");
                VerifyTestResult (tmp.Lookup ("d") == "4");

                CACHE tmp2 = tmp;
                VerifyTestResult (not tmp2.Lookup ("a").has_value () or *tmp2.Lookup ("a") == "1"); // could be missing or found but if found same value
                VerifyTestResult (tmp2.Lookup ("b") == "2");
                VerifyTestResult (tmp2.Lookup ("d") == "4");
            }
            void T3_ ()
            {
                // using C++17 deduction guides
                LRUCache tmp (pair<string, string>{}, 3, 10, hash<string>{});
                tmp.Add ("a", "1");
                tmp.Add ("b", "2");
                tmp.Add ("c", "3");
                tmp.Add ("d", "4");

                VerifyTestResult (not tmp.Lookup ("a").has_value () or *tmp.Lookup ("a") == "1"); // could be missing or found but if found same value
                VerifyTestResult (tmp.Lookup ("b") == "2");
                VerifyTestResult (tmp.Lookup ("d") == "4");

                LRUCache tmp2 = tmp;
                VerifyTestResult (not tmp2.Lookup ("a").has_value () or *tmp2.Lookup ("a") == "1"); // could be missing or found but if found same value
                VerifyTestResult (tmp2.Lookup ("b") == "2");
                VerifyTestResult (tmp2.Lookup ("d") == "4");
            }
        }
        void DoIt ()
        {
            Private_::T1_ ();
            Private_::T2_ ();
            Private_::T3_ ();
        }
    }
}

namespace {
    namespace Test2_LRUCache_ObjWithNoArgCTORs_ {
        namespace Private_ {
            struct TNoCTOR_ {
                TNoCTOR_ (int) {}
                TNoCTOR_ () = delete;
                bool operator== ([[maybe_unused]] const TNoCTOR_& rhs) const { return true; }
            };
        }
        void DoIt ()
        {
            using Private_::TNoCTOR_;
            LRUCache<TNoCTOR_, TNoCTOR_> test (10);
            test.Add (TNoCTOR_ (1), TNoCTOR_ (1));
            (void)test.Lookup (TNoCTOR_ (1));
        }
    }
}

namespace {
    namespace Test3_LRUCache_Elements {
        namespace Private_ {
            struct TNoCTOR_ {
                TNoCTOR_ (int) {}
                TNoCTOR_ () = delete;
                bool operator== ([[maybe_unused]] const TNoCTOR_& rhs) const { return true; }
            };
        }
        void DoIt ()
        {
            LRUCache<string, string> tmp (3);
            tmp.Add ("a", "1");
            tmp.Add ("b", "2");
            tmp.Add ("c", "3");
            tmp.Add ("d", "4");
            Containers::Mapping<string, string> x = tmp.Elements (); // subtle cuz converting traits type
            VerifyTestResult (x.length () <= 4);
            for (auto i : tmp.Elements ()) {
                VerifyTestResult (i.fKey == "a" or i.fKey == "b" or i.fKey == "c" or i.fKey == "d");
                VerifyTestResult (i.fValue == "1" or i.fValue == "2" or i.fValue == "3" or i.fValue == "4");
            }
        }
    }
}

namespace {
    namespace Test4_TimedCache_ {
        // FROM Example Usage in TimedCache<>
        namespace Private_ {
            using Characters::String;
            namespace Example1_ {
                struct DiskSpaceUsageType {
                    int size;
                };
                // do the actual lookup part which maybe slow
                auto LookupDiskStats_ ([[maybe_unused]] const String& filename) -> DiskSpaceUsageType { return DiskSpaceUsageType{33}; };

                Cache::TimedCache<String, DiskSpaceUsageType> sDiskUsageCache_{5.0};
                // explicitly caller maintaining the cache
                optional<DiskSpaceUsageType> LookupDiskStats_Try1 (String diskName)
                {
                    optional<DiskSpaceUsageType> o = sDiskUsageCache_.Lookup (diskName);
                    if (not o.has_value ()) {
                        o = LookupDiskStats_ (diskName);
                        if (o) {
                            sDiskUsageCache_.Add (diskName, *o);
                        }
                    }
                    return o;
                }
                // more automatic maintainance of that update pattern
                DiskSpaceUsageType LookupDiskStats_Try2 (String diskName)
                {
                    return sDiskUsageCache_.Lookup (diskName,
                                                    [](String diskName) -> DiskSpaceUsageType {
                                                        return LookupDiskStats_ (diskName);
                                                    });
                }
                // or still simpler
                DiskSpaceUsageType LookupDiskStats_Try3 (String diskName)
                {
                    return sDiskUsageCache_.Lookup (diskName, LookupDiskStats_);
                }
                void DoIt ()
                {
                    VerifyTestResult (Memory::ValueOrDefault (LookupDiskStats_Try1 (L"xx")).size == 33);
                    VerifyTestResult (LookupDiskStats_Try2 (L"xx").size == 33);
                    VerifyTestResult (LookupDiskStats_Try3 (L"xx").size == 33);
                }
            }
            namespace Example2_ {
                using Execution::Synchronized;
                using Time::DurationSecondsType;

                using ScanFolderKey_ = String;
                static constexpr DurationSecondsType kAgeForScanPersistenceCache_{5 * 60.0};
                struct FolderDetails_ {
                    int size; // ...info to cache about a folder
                };
                Synchronized<Cache::TimedCache<
                    ScanFolderKey_,
                    shared_ptr<FolderDetails_>,
                    TimedCacheSupport::DefaultTraits<ScanFolderKey_, shared_ptr<FolderDetails_>, less<ScanFolderKey_>, true>>>
                    sCachedScanFoldersDetails_{kAgeForScanPersistenceCache_};

                shared_ptr<FolderDetails_> AccessFolder_ (const ScanFolderKey_& folder)
                {
                    auto lockedCache = sCachedScanFoldersDetails_.rwget ();
                    if (optional<shared_ptr<FolderDetails_>> o = lockedCache->Lookup (folder)) {
                        return *o;
                    }
                    else {
                        shared_ptr<FolderDetails_> fd = make_shared<FolderDetails_> (); // and fill in default values looking at disk
                        lockedCache->Add (folder, fd);
                        return move (fd);
                    }
                }

                void DoIt ()
                {
                    auto f1 = AccessFolder_ (L"folder1");
                    auto f2 = AccessFolder_ (L"folder2");
                    auto f1again = AccessFolder_ (L"folder1");  // if you trace through the debug code you'll see this is a cache hit
                }
            }
        }
        void DoIt ()
        {
            Private_::Example1_::DoIt ();
            Private_::Example2_::DoIt ();
        }
    }
}

namespace {
    namespace Test5_Memoizer_ {
        // FROM Example Usage in ???
        namespace Private_ {
        }
        void DoIt ()
        {
            {
                unsigned int totalCallsCount{};
#if qCompilerAndStdLib_template_template_argument_as_different_template_paramters_Buggy
                Memoizer<int, MemoizerSupport::DEFAULT_CACHE, int, int> memoizer{[&totalCallsCount](int a, int b) { totalCallsCount++;  return a + b; }};
#else
                Memoizer<int, LRUCache, int, int> memoizer{[&totalCallsCount](int a, int b) { totalCallsCount++;  return a + b; }};
#endif
                VerifyTestResult (memoizer.Compute (1, 1) == 2 and totalCallsCount == 1);
                VerifyTestResult (memoizer.Compute (1, 1) == 2 and totalCallsCount == 1);
            }
#if 0
            {
                // trying to figure out how to do...
                unsigned int totalCallsCount{};
                Memoizer     memoizer{[&totalCallsCount](int a, int b) { totalCallsCount++;  return a + b; }};
                VerifyTestResult (memoizer.Compute (1, 1) == 2 and totalCallsCount == 1);
                VerifyTestResult (memoizer.Compute (1, 1) == 2 and totalCallsCount == 1);
            }
#endif
        }
    }
}

namespace {
    void DoRegressionTests_ ()
    {
        Test1_Simple_::DoIt ();
        Test2_LRUCache_ObjWithNoArgCTORs_::DoIt ();
        Test3_LRUCache_Elements::DoIt ();
        Test4_TimedCache_::DoIt ();
        Test5_Memoizer_::DoIt ();
    }
}

int main ([[maybe_unused]] int argc, [[maybe_unused]] const char* argv[])
{
    Stroika::TestHarness::Setup ();
    return Stroika::TestHarness::PrintPassOrFail (DoRegressionTests_);
}
<commit_msg>make format<commit_after>/*
 * Copyright(c) Sophist Solutions, Inc. 1990-2018.  All rights reserved
 */
//  TEST    Foundation::Caching
#include "Stroika/Foundation/StroikaPreComp.h"

#include "Stroika/Foundation/Cache/LRUCache.h"
#include "Stroika/Foundation/Cache/Memoizer.h"
#include "Stroika/Foundation/Cache/TimedCache.h"
#include "Stroika/Foundation/Debug/Assertions.h"
#include "Stroika/Foundation/Debug/TimingTrace.h"
#include "Stroika/Foundation/Debug/Trace.h"
#include "Stroika/Foundation/Execution/Synchronized.h"
#include "Stroika/Foundation/Memory/Optional.h"

#include "../TestHarness/SimpleClass.h"
#include "../TestHarness/TestHarness.h"

using namespace Stroika;
using namespace Stroika::Foundation;
using namespace Stroika::Foundation::Cache;

namespace {
    namespace Test1_Simple_ {
        namespace Private_ {
            void T1_ ()
            {
                LRUCache<string, string> tmp (3);
                tmp.Add ("a", "1");
                tmp.Add ("b", "2");
                tmp.Add ("c", "3");
                tmp.Add ("d", "4");
                VerifyTestResult (not tmp.Lookup ("a").has_value ());
                VerifyTestResult (tmp.Lookup ("b") == "2");
                VerifyTestResult (tmp.Lookup ("d") == "4");

                LRUCache<string, string> tmp2 = tmp;
                VerifyTestResult (not tmp2.Lookup ("a").has_value ());
                VerifyTestResult (tmp2.Lookup ("b") == "2");
                VerifyTestResult (tmp2.Lookup ("d") == "4");
            }
            void T2_ ()
            {
                using CACHE = LRUCache<string, string, equal_to<string>, hash<string>>;
                CACHE tmp (3, equal_to<string>{}, 10);
                tmp.Add ("a", "1");
                tmp.Add ("b", "2");
                tmp.Add ("c", "3");
                tmp.Add ("d", "4");

                VerifyTestResult (not tmp.Lookup ("a").has_value () or *tmp.Lookup ("a") == "1"); // could be missing or found but if found same value
                VerifyTestResult (tmp.Lookup ("b") == "2");
                VerifyTestResult (tmp.Lookup ("d") == "4");

                CACHE tmp2 = tmp;
                VerifyTestResult (not tmp2.Lookup ("a").has_value () or *tmp2.Lookup ("a") == "1"); // could be missing or found but if found same value
                VerifyTestResult (tmp2.Lookup ("b") == "2");
                VerifyTestResult (tmp2.Lookup ("d") == "4");
            }
            void T3_ ()
            {
                // using C++17 deduction guides
                LRUCache tmp (pair<string, string>{}, 3, 10, hash<string>{});
                tmp.Add ("a", "1");
                tmp.Add ("b", "2");
                tmp.Add ("c", "3");
                tmp.Add ("d", "4");

                VerifyTestResult (not tmp.Lookup ("a").has_value () or *tmp.Lookup ("a") == "1"); // could be missing or found but if found same value
                VerifyTestResult (tmp.Lookup ("b") == "2");
                VerifyTestResult (tmp.Lookup ("d") == "4");

                LRUCache tmp2 = tmp;
                VerifyTestResult (not tmp2.Lookup ("a").has_value () or *tmp2.Lookup ("a") == "1"); // could be missing or found but if found same value
                VerifyTestResult (tmp2.Lookup ("b") == "2");
                VerifyTestResult (tmp2.Lookup ("d") == "4");
            }
        }
        void DoIt ()
        {
            Private_::T1_ ();
            Private_::T2_ ();
            Private_::T3_ ();
        }
    }
}

namespace {
    namespace Test2_LRUCache_ObjWithNoArgCTORs_ {
        namespace Private_ {
            struct TNoCTOR_ {
                TNoCTOR_ (int) {}
                TNoCTOR_ () = delete;
                bool operator== ([[maybe_unused]] const TNoCTOR_& rhs) const { return true; }
            };
        }
        void DoIt ()
        {
            using Private_::TNoCTOR_;
            LRUCache<TNoCTOR_, TNoCTOR_> test (10);
            test.Add (TNoCTOR_ (1), TNoCTOR_ (1));
            (void)test.Lookup (TNoCTOR_ (1));
        }
    }
}

namespace {
    namespace Test3_LRUCache_Elements {
        namespace Private_ {
            struct TNoCTOR_ {
                TNoCTOR_ (int) {}
                TNoCTOR_ () = delete;
                bool operator== ([[maybe_unused]] const TNoCTOR_& rhs) const { return true; }
            };
        }
        void DoIt ()
        {
            LRUCache<string, string> tmp (3);
            tmp.Add ("a", "1");
            tmp.Add ("b", "2");
            tmp.Add ("c", "3");
            tmp.Add ("d", "4");
            Containers::Mapping<string, string> x = tmp.Elements (); // subtle cuz converting traits type
            VerifyTestResult (x.length () <= 4);
            for (auto i : tmp.Elements ()) {
                VerifyTestResult (i.fKey == "a" or i.fKey == "b" or i.fKey == "c" or i.fKey == "d");
                VerifyTestResult (i.fValue == "1" or i.fValue == "2" or i.fValue == "3" or i.fValue == "4");
            }
        }
    }
}

namespace {
    namespace Test4_TimedCache_ {
        // FROM Example Usage in TimedCache<>
        namespace Private_ {
            using Characters::String;
            namespace Example1_ {
                struct DiskSpaceUsageType {
                    int size;
                };
                // do the actual lookup part which maybe slow
                auto LookupDiskStats_ ([[maybe_unused]] const String& filename) -> DiskSpaceUsageType { return DiskSpaceUsageType{33}; };

                Cache::TimedCache<String, DiskSpaceUsageType> sDiskUsageCache_{5.0};
                // explicitly caller maintaining the cache
                optional<DiskSpaceUsageType> LookupDiskStats_Try1 (String diskName)
                {
                    optional<DiskSpaceUsageType> o = sDiskUsageCache_.Lookup (diskName);
                    if (not o.has_value ()) {
                        o = LookupDiskStats_ (diskName);
                        if (o) {
                            sDiskUsageCache_.Add (diskName, *o);
                        }
                    }
                    return o;
                }
                // more automatic maintainance of that update pattern
                DiskSpaceUsageType LookupDiskStats_Try2 (String diskName)
                {
                    return sDiskUsageCache_.Lookup (diskName,
                                                    [](String diskName) -> DiskSpaceUsageType {
                                                        return LookupDiskStats_ (diskName);
                                                    });
                }
                // or still simpler
                DiskSpaceUsageType LookupDiskStats_Try3 (String diskName)
                {
                    return sDiskUsageCache_.Lookup (diskName, LookupDiskStats_);
                }
                void DoIt ()
                {
                    VerifyTestResult (Memory::ValueOrDefault (LookupDiskStats_Try1 (L"xx")).size == 33);
                    VerifyTestResult (LookupDiskStats_Try2 (L"xx").size == 33);
                    VerifyTestResult (LookupDiskStats_Try3 (L"xx").size == 33);
                }
            }
            namespace Example2_ {
                using Execution::Synchronized;
                using Time::DurationSecondsType;

                using ScanFolderKey_ = String;
                static constexpr DurationSecondsType kAgeForScanPersistenceCache_{5 * 60.0};
                struct FolderDetails_ {
                    int size; // ...info to cache about a folder
                };
                Synchronized<Cache::TimedCache<
                    ScanFolderKey_,
                    shared_ptr<FolderDetails_>,
                    TimedCacheSupport::DefaultTraits<ScanFolderKey_, shared_ptr<FolderDetails_>, less<ScanFolderKey_>, true>>>
                    sCachedScanFoldersDetails_{kAgeForScanPersistenceCache_};

                shared_ptr<FolderDetails_> AccessFolder_ (const ScanFolderKey_& folder)
                {
                    auto lockedCache = sCachedScanFoldersDetails_.rwget ();
                    if (optional<shared_ptr<FolderDetails_>> o = lockedCache->Lookup (folder)) {
                        return *o;
                    }
                    else {
                        shared_ptr<FolderDetails_> fd = make_shared<FolderDetails_> (); // and fill in default values looking at disk
                        lockedCache->Add (folder, fd);
                        return move (fd);
                    }
                }

                void DoIt ()
                {
                    auto f1      = AccessFolder_ (L"folder1");
                    auto f2      = AccessFolder_ (L"folder2");
                    auto f1again = AccessFolder_ (L"folder1"); // if you trace through the debug code you'll see this is a cache hit
                }
            }
        }
        void DoIt ()
        {
            Private_::Example1_::DoIt ();
            Private_::Example2_::DoIt ();
        }
    }
}

namespace {
    namespace Test5_Memoizer_ {
        // FROM Example Usage in ???
        namespace Private_ {
        }
        void DoIt ()
        {
            {
                unsigned int totalCallsCount{};
#if qCompilerAndStdLib_template_template_argument_as_different_template_paramters_Buggy
                Memoizer<int, MemoizerSupport::DEFAULT_CACHE, int, int> memoizer{[&totalCallsCount](int a, int b) { totalCallsCount++;  return a + b; }};
#else
                Memoizer<int, LRUCache, int, int> memoizer{[&totalCallsCount](int a, int b) { totalCallsCount++;  return a + b; }};
#endif
                VerifyTestResult (memoizer.Compute (1, 1) == 2 and totalCallsCount == 1);
                VerifyTestResult (memoizer.Compute (1, 1) == 2 and totalCallsCount == 1);
            }
#if 0
            {
                // trying to figure out how to do...
                unsigned int totalCallsCount{};
                Memoizer     memoizer{[&totalCallsCount](int a, int b) { totalCallsCount++;  return a + b; }};
                VerifyTestResult (memoizer.Compute (1, 1) == 2 and totalCallsCount == 1);
                VerifyTestResult (memoizer.Compute (1, 1) == 2 and totalCallsCount == 1);
            }
#endif
        }
    }
}

namespace {
    void DoRegressionTests_ ()
    {
        Test1_Simple_::DoIt ();
        Test2_LRUCache_ObjWithNoArgCTORs_::DoIt ();
        Test3_LRUCache_Elements::DoIt ();
        Test4_TimedCache_::DoIt ();
        Test5_Memoizer_::DoIt ();
    }
}

int main ([[maybe_unused]] int argc, [[maybe_unused]] const char* argv[])
{
    Stroika::TestHarness::Setup ();
    return Stroika::TestHarness::PrintPassOrFail (DoRegressionTests_);
}
<|endoftext|>


<commit_before>/*
 * Copyright(c) Sophist Solutions, Inc. 1990-2016.  All rights reserved
 */
//  TEST    Foundation::IO::Network::Transfer
#include    "Stroika/Foundation/StroikaPreComp.h"

#include    <iostream>
#include    <random>

#include    "Stroika/Foundation/Cryptography/Encoding/Algorithm/Base64.h"
#include    "Stroika/Foundation/DataExchange/JSON/Reader.h"
#include    "Stroika/Foundation/Debug/Trace.h"
#include    "Stroika/Foundation/Execution/RequiredComponentMissingException.h"
#include    "Stroika/Foundation/IO/Network/Transfer/Client.h"
#if     qHasFeature_LibCurl
#include    "Stroika/Foundation/IO/Network/Transfer/Client_libcurl.h"
#endif
#if     qHasFeature_WinHTTP
#include    "Stroika/Foundation/IO/Network/Transfer/Client_WinHTTP.h"
#endif

#include    "../TestHarness/TestHarness.h"

using   namespace   Stroika::Foundation;
using   namespace   Stroika::Foundation::DataExchange;
using   namespace   Stroika::Foundation::IO;
using   namespace   Stroika::Foundation::IO::Network;
using   namespace   Stroika::Foundation::IO::Network::Transfer;

namespace {
    namespace Test_1_SimpleConnnectionTests_ {
        namespace Private_ {
            void    Test_1_SimpleFetch_Google_C_ (Connection c)
            {
                c.SetURL (URL::Parse (L"http://www.google.com"));
                Response    r   =   c.GET ();
                VerifyTestResult (r.GetSucceeded ());
                VerifyTestResult (r.GetData ().size () > 1);
            }
            void    Test_2_SimpleFetch_SSL_Google_C_ (Connection c)
            {
                try {
                    c.SetURL (URL::Parse (L"https://www.google.com"));
                    Response    r   =   c.GET ();
                    VerifyTestResult (r.GetSucceeded ());
                    VerifyTestResult (r.GetData ().size () > 1);
                }
#if     qHasFeature_LibCurl
                catch (const LibCurlException& lce) {
#if     !qHasFeature_OpenSSL
                    if (lce.GetCode () == 1/*CURLE_UNSUPPORTED_PROTOCOL*/) {
                        DbgTrace ("Warning - ignored exception doing LibCurl/ssl - for now probably just no SSL support with libcurl");
                        return;
                    }
#endif
                    throw;
                }
#else
                catch (...) {
                    throw;
                }
#endif
            }
            void    DoRegressionTests_ForConnectionFactory_ (Connection (*factory) ())
            {
                Test_1_SimpleFetch_Google_C_ (factory ());
                Test_2_SimpleFetch_SSL_Google_C_ (factory ());
            }
        }
        void    DoTests_ ()
        {
            Debug::TraceContextBumper ctx ("{}::Test_1_SimpleConnnectionTests_");
            using namespace Private_;
            try {
                DoRegressionTests_ForConnectionFactory_ ([]() -> Connection { return CreateConnection (); });
            }
            catch (const Execution::RequiredComponentMissingException&) {
#if     !qHasFeature_LibCurl && !qHasFeature_WinHTTP
                // OK to ignore. We don't wnat to call this failing a test, because there is nothing to fix.
                // This is more like the absence of a feature beacuse of the missing component.
#else
                Execution::ReThrow ();
#endif
            }

#if     qHasFeature_LibCurl
            DoRegressionTests_ForConnectionFactory_ ([]() -> Connection { return Connection_LibCurl (); });
#endif
#if     qHasFeature_WinHTTP
            DoRegressionTests_ForConnectionFactory_ ([]() -> Connection { return Connection_WinHTTP (); });
#endif
        }
    }
}

namespace {
    namespace Test_2_SimpleFetch_httpbin_ {
        namespace Private_ {
            void    T1_httpbin_SimpleGET_ (Connection c)
            {
                Debug::TraceContextBumper ctx ("T1_httpbin_SimpleGET_");
                c.SetURL (URL::Parse (L"http://httpbin.org/get"));
                Response    r   =   c.GET ();
                VerifyTestResult (r.GetSucceeded ());
                VerifyTestResult (r.GetData ().size () > 1);
                {
                    VariantValue v = JSON::Reader ().Read (r.GetDataBinaryInputStream ());
                    Mapping<String, VariantValue> vv = v.As<Mapping<String, VariantValue>> ();
                    VerifyTestResult (vv.ContainsKey (L"args"));
                    VerifyTestResult (vv[L"url"] == L"http://httpbin.org/get");
                }
            }
            void    T2_httpbin_SimplePOST_ (Connection c)
            {
                Debug::TraceContextBumper ctx ("T2_httpbin_SimplePOST_");
                using   Memory::BLOB;

                static   mt19937 sRNG_;

                c.SetURL (URL::Parse (L"http://httpbin.org/post"));
                BLOB    roundTripTestData = [] () {
                    Memory::SmallStackBuffer<Byte> buf (1024);
                    for (size_t i = 0; i < buf.GetSize (); ++i) {
                        buf[i] = static_cast<Byte> (uniform_int_distribution<unsigned short>()(sRNG_));
                    }
                    return BLOB (buf.begin (), buf.end ());
                } ();
                Response    r   =   c.POST (roundTripTestData, DataExchange::PredefinedInternetMediaType::OctetStream_CT ());
                VerifyTestResult (r.GetSucceeded ());
                {
                    VariantValue v = JSON::Reader ().Read (r.GetDataBinaryInputStream ());
                    Mapping<String, VariantValue> vv = v.As<Mapping<String, VariantValue>> ();
                    DbgTrace ("POST parsed response:");
                    for (auto i : vv) {
                        DbgTrace (L"%s : %s", i.fKey.c_str (), i.fValue.As<String> ().c_str ());
                    }
                    String  dataValueString = vv.Lookup (L"data").Value ().As<String> ();
                    {
                        size_t i = dataValueString.Find (',');
                        if (i != -1) {
                            dataValueString = dataValueString.SubString (i + 1);
                        }
                    }
                    BLOB    resultBLOB = Cryptography::Encoding::Algorithm::DecodeBase64 (dataValueString.AsUTF8 ());
                    VerifyTestResult (resultBLOB == roundTripTestData);
                }
            }
            void    T3_httpbin_SimplePUT_ (Connection c)
            {
                Debug::TraceContextBumper ctx ("T3_httpbin_SimplePUT_");
                using   Memory::BLOB;

                static   mt19937 sRNG_;

                c.SetURL (URL::Parse (L"http://httpbin.org/put"));
                BLOB    roundTripTestData = [] () {
                    Memory::SmallStackBuffer<Byte> buf (1024);
                    for (size_t i = 0; i < buf.GetSize (); ++i) {
                        buf[i] = static_cast<Byte> (uniform_int_distribution<unsigned short>()(sRNG_));
                    }
                    return BLOB (buf.begin (), buf.end ());
                } ();
                Response    r   =   c.PUT (roundTripTestData, DataExchange::PredefinedInternetMediaType::OctetStream_CT ());
                VerifyTestResult (r.GetSucceeded ());
                {
                    VariantValue v = JSON::Reader ().Read (r.GetDataBinaryInputStream ());
                    Mapping<String, VariantValue> vv = v.As<Mapping<String, VariantValue>> ();
                    DbgTrace ("PUT parsed response:");
                    for (auto i : vv) {
                        DbgTrace (L"%s : %s", i.fKey.c_str (), i.fValue.As<String> ().c_str ());
                    }
                    String  dataValueString = vv.Lookup (L"data").Value ().As<String> ();
                    {
                        size_t i = dataValueString.Find (',');
                        if (i != -1) {
                            dataValueString = dataValueString.SubString (i + 1);
                        }
                    }
                    BLOB    resultBLOB = Cryptography::Encoding::Algorithm::DecodeBase64 (dataValueString.AsUTF8 ());
                    VerifyTestResult (resultBLOB == roundTripTestData);
                }
            }
            void    DoRegressionTests_ForConnectionFactory_ (Connection (*factory) ())
            {
                {
                    T1_httpbin_SimpleGET_ (factory ());
                    T2_httpbin_SimplePOST_ (factory ());
                }
                {
                    // Connection re-use
                    Connection conn = factory ();
                    T1_httpbin_SimpleGET_ (conn);
                    T2_httpbin_SimplePOST_ (conn);
                    T3_httpbin_SimplePUT_ (conn);
                }
            }
        }
        void    DoTests_ ()
        {
            Debug::TraceContextBumper ctx ("{}::Test_2_SimpleFetch_httpbin_");
            using namespace Private_;
            try {
                DoRegressionTests_ForConnectionFactory_ ([]() -> Connection { return CreateConnection (); });
            }
            catch (const Execution::RequiredComponentMissingException&) {
#if     !qHasFeature_LibCurl && !qHasFeature_WinHTTP
                // OK to ignore. We don't wnat to call this failing a test, because there is nothing to fix.
                // This is more like the absence of a feature beacuse of the missing component.
#else
                Execution::ReThrow ();
#endif
            }

#if     qHasFeature_LibCurl
            DoRegressionTests_ForConnectionFactory_ ([]() -> Connection { return Connection_LibCurl (); });
#endif
#if     qHasFeature_WinHTTP
            DoRegressionTests_ForConnectionFactory_ ([]() -> Connection { return Connection_WinHTTP (); });
#endif
        }
    }
}

namespace {
    namespace Test3_TextStreamResponse_ {
        namespace Private_ {
            void    Test_1_SimpleFetch_Google_C_ (Connection c)
            {
                c.SetURL (URL::Parse (L"http://www.google.com"));
                Response    r   =   c.GET ();
                VerifyTestResult (r.GetSucceeded ());
                for (auto i : r.GetHeaders ()) {
                    DbgTrace (L"%s=%s", i.fKey.c_str (), i.fValue.c_str ());
                }
                InternetMediaType contentType = r.GetContentType ();
                String responseText = r.GetDataTextInputStream ().ReadAll ();
                DbgTrace (L"responseText = %s", responseText.c_str ());
                VerifyTestResult (responseText.Contains (L"google"));
            }
            void    DoRegressionTests_ForConnectionFactory_ (Connection (*factory) ())
            {
                Test_1_SimpleFetch_Google_C_ (factory ());
            }
        }
        void    DoTests_ ()
        {
            Debug::TraceContextBumper ctx ("{}::Test3_TextStreamResponse_");
            using namespace Private_;
            try {
                DoRegressionTests_ForConnectionFactory_ ([]() -> Connection { return CreateConnection (); });
            }
            catch (const Execution::RequiredComponentMissingException&) {
#if     !qHasFeature_LibCurl && !qHasFeature_WinHTTP
                // OK to ignore. We don't wnat to call this failing a test, because there is nothing to fix.
                // This is more like the absence of a feature beacuse of the missing component.
#else
                Execution::ReThrow ();
#endif
            }

#if     qHasFeature_LibCurl
            DoRegressionTests_ForConnectionFactory_ ([]() -> Connection { return Connection_LibCurl (); });
#endif
#if     qHasFeature_WinHTTP
            DoRegressionTests_ForConnectionFactory_ ([]() -> Connection { return Connection_WinHTTP (); });
#endif
        }
    }
}

namespace {
    namespace Test_4_RefDocsTests_ {
        namespace Private_ {
            void    T1_get_ ()
            {
                Connection  c   =   IO::Network::Transfer::CreateConnection ();
                c.SetURL (URL::Parse (L"http://www.google.com"));
                Response    r   =   c.GET ();
                VerifyTestResult (r.GetSucceeded ());
                VerifyTestResult (r.GetData ().size () > 1);
            }

        }
        void    DoTests_ ()
        {
            Debug::TraceContextBumper ctx ("{}::Test_4_RefDocsTests_");
            try {
                Private_::T1_get_ ();
            }
            catch (const Execution::RequiredComponentMissingException&) {
#if     !qHasFeature_LibCurl && !qHasFeature_WinHTTP
                // OK to ignore. We don't wnat to call this failing a test, because there is nothing to fix.
                // This is more like the absence of a feature beacuse of the missing component.
#else
                Execution::ReThrow ();
#endif
            }
        }
    }
}

namespace {
    namespace Test_5_SSLCertCheckTests_ {
        namespace Private_ {
            void    T1_get_ (Connection::Options o)
            {
                Connection  c   =   IO::Network::Transfer::CreateConnection (o);
                try {
                    c.SetURL (URL::Parse (L"https://testssl-valid.disig.sk/index.en.html"));
                    Response    r   =   c.GET ();
                    VerifyTestResult (r.GetSucceeded ());
                    VerifyTestResult (r.GetData ().size () > 1);
                }
#if     qHasFeature_LibCurl
                catch (const LibCurlException& lce) {
#if     !qHasFeature_OpenSSL
                    if (lce.GetCode () == 1/*CURLE_UNSUPPORTED_PROTOCOL*/) {
                        DbgTrace ("Warning - ignored exception doing LibCurl/ssl - for now probably just no SSL support with libcurl");
                        return;
                    }
#endif
                    throw;
                }
#else
                catch (...) {
                    throw;
                }
#endif
            }
        }
        void    DoTests_ ()
        {
            Debug::TraceContextBumper ctx ("{}::Test_5_SSLCertCheckTests_");
            Connection::Options o;
            try {
                o.fFailConnectionIfSSLCertificateInvalid = true;
                Private_::T1_get_ (o);
#if     qHasFeature_LibCurl && !qHasFeature_OpenSSL
                // ingore this case, since we allow the failed connect above...
#else
                VerifyTestResult (false);
#endif
            }
            catch (const Execution::RequiredComponentMissingException&) {
#if     !qHasFeature_LibCurl && !qHasFeature_WinHTTP
                // OK to ignore. We don't wnat to call this failing a test, because there is nothing to fix.
                // This is more like the absence of a feature beacuse of the missing component.
#else
                Execution::ReThrow ();
#endif
            }
            catch (...) {
                // Good - this should fail
            }
            try {
                o.fFailConnectionIfSSLCertificateInvalid = false;
                Private_::T1_get_ (o);
            }
            catch (const Execution::RequiredComponentMissingException&) {
#if     !qHasFeature_LibCurl && !qHasFeature_WinHTTP
                // OK to ignore. We don't wnat to call this failing a test, because there is nothing to fix.
                // This is more like the absence of a feature beacuse of the missing component.
#else
                Execution::ReThrow ();
#endif
            }
            catch (...) {
                VerifyTestResult (false);
            }
        }
    }
}

namespace   {
    void    DoRegressionTests_ ()
    {
        Test_1_SimpleConnnectionTests_::DoTests_ ();
        Test_2_SimpleFetch_httpbin_::DoTests_ ();
        Test3_TextStreamResponse_::DoTests_ ();
        Test_4_RefDocsTests_::DoTests_ ();
        Test_5_SSLCertCheckTests_::DoTests_ ();
    }
}

int main (int argc, const char* argv[])
{
    Stroika::TestHarness::Setup ();
    Stroika::TestHarness::PrintPassOrFail (DoRegressionTests_);
    return EXIT_SUCCESS;
}
<commit_msg>fixed io transer network connection regtest to use fMaxAutomaticRedirects = 1, since many of them use redirects<commit_after>/*
 * Copyright(c) Sophist Solutions, Inc. 1990-2016.  All rights reserved
 */
//  TEST    Foundation::IO::Network::Transfer
#include    "Stroika/Foundation/StroikaPreComp.h"

#include    <iostream>
#include    <random>

#include    "Stroika/Foundation/Cryptography/Encoding/Algorithm/Base64.h"
#include    "Stroika/Foundation/DataExchange/JSON/Reader.h"
#include    "Stroika/Foundation/Debug/Trace.h"
#include    "Stroika/Foundation/Execution/RequiredComponentMissingException.h"
#include    "Stroika/Foundation/IO/Network/Transfer/Client.h"
#if     qHasFeature_LibCurl
#include    "Stroika/Foundation/IO/Network/Transfer/Client_libcurl.h"
#endif
#if     qHasFeature_WinHTTP
#include    "Stroika/Foundation/IO/Network/Transfer/Client_WinHTTP.h"
#endif

#include    "../TestHarness/TestHarness.h"

using   namespace   Stroika::Foundation;
using   namespace   Stroika::Foundation::DataExchange;
using   namespace   Stroika::Foundation::IO;
using   namespace   Stroika::Foundation::IO::Network;
using   namespace   Stroika::Foundation::IO::Network::Transfer;

namespace {
    const   Connection::Options kDefaultTestOptions_ = [] ()
    {
        Connection::Options o;
        o.fMaxAutomaticRedirects = 1;
        return o;
    } ();
}

namespace {
    namespace Test_1_SimpleConnnectionTests_ {
        namespace Private_ {
            void    Test_1_SimpleFetch_Google_C_ (Connection c)
            {
                c.SetURL (URL::Parse (L"http://www.google.com"));
                Response    r   =   c.GET ();
                VerifyTestResult (r.GetSucceeded ());
                VerifyTestResult (r.GetData ().size () > 1);
            }
            void    Test_2_SimpleFetch_SSL_Google_C_ (Connection c)
            {
                try {
                    c.SetURL (URL::Parse (L"https://www.google.com"));
                    Response    r   =   c.GET ();
                    VerifyTestResult (r.GetSucceeded ());
                    VerifyTestResult (r.GetData ().size () > 1);
                }
#if     qHasFeature_LibCurl
                catch (const LibCurlException& lce) {
#if     !qHasFeature_OpenSSL
                    if (lce.GetCode () == 1/*CURLE_UNSUPPORTED_PROTOCOL*/) {
                        DbgTrace ("Warning - ignored exception doing LibCurl/ssl - for now probably just no SSL support with libcurl");
                        return;
                    }
#endif
                    throw;
                }
#else
                catch (...) {
                    throw;
                }
#endif
            }
            void    DoRegressionTests_ForConnectionFactory_ (Connection (*factory) ())
            {
                Test_1_SimpleFetch_Google_C_ (factory ());
                Test_2_SimpleFetch_SSL_Google_C_ (factory ());
            }
        }
        void    DoTests_ ()
        {
            Debug::TraceContextBumper ctx ("{}::Test_1_SimpleConnnectionTests_");
            using namespace Private_;
            try {
                DoRegressionTests_ForConnectionFactory_ ([]() -> Connection { return CreateConnection (kDefaultTestOptions_); });
            }
            catch (const Execution::RequiredComponentMissingException&) {
#if     !qHasFeature_LibCurl && !qHasFeature_WinHTTP
                // OK to ignore. We don't wnat to call this failing a test, because there is nothing to fix.
                // This is more like the absence of a feature beacuse of the missing component.
#else
                Execution::ReThrow ();
#endif
            }

#if     qHasFeature_LibCurl
            DoRegressionTests_ForConnectionFactory_ ([]() -> Connection { return Connection_LibCurl (kDefaultTestOptions_); });
#endif
#if     qHasFeature_WinHTTP
            DoRegressionTests_ForConnectionFactory_ ([]() -> Connection { return Connection_WinHTTP (kDefaultTestOptions_); });
#endif
        }
    }
}

namespace {
    namespace Test_2_SimpleFetch_httpbin_ {
        namespace Private_ {
            void    T1_httpbin_SimpleGET_ (Connection c)
            {
                Debug::TraceContextBumper ctx ("T1_httpbin_SimpleGET_");
                c.SetURL (URL::Parse (L"http://httpbin.org/get"));
                Response    r   =   c.GET ();
                VerifyTestResult (r.GetSucceeded ());
                VerifyTestResult (r.GetData ().size () > 1);
                {
                    VariantValue v = JSON::Reader ().Read (r.GetDataBinaryInputStream ());
                    Mapping<String, VariantValue> vv = v.As<Mapping<String, VariantValue>> ();
                    VerifyTestResult (vv.ContainsKey (L"args"));
                    VerifyTestResult (vv[L"url"] == L"http://httpbin.org/get");
                }
            }
            void    T2_httpbin_SimplePOST_ (Connection c)
            {
                Debug::TraceContextBumper ctx ("T2_httpbin_SimplePOST_");
                using   Memory::BLOB;

                static   mt19937 sRNG_;

                c.SetURL (URL::Parse (L"http://httpbin.org/post"));
                BLOB    roundTripTestData = [] () {
                    Memory::SmallStackBuffer<Byte> buf (1024);
                    for (size_t i = 0; i < buf.GetSize (); ++i) {
                        buf[i] = static_cast<Byte> (uniform_int_distribution<unsigned short>()(sRNG_));
                    }
                    return BLOB (buf.begin (), buf.end ());
                } ();
                Response    r   =   c.POST (roundTripTestData, DataExchange::PredefinedInternetMediaType::OctetStream_CT ());
                VerifyTestResult (r.GetSucceeded ());
                {
                    VariantValue v = JSON::Reader ().Read (r.GetDataBinaryInputStream ());
                    Mapping<String, VariantValue> vv = v.As<Mapping<String, VariantValue>> ();
                    DbgTrace ("POST parsed response:");
                    for (auto i : vv) {
                        DbgTrace (L"%s : %s", i.fKey.c_str (), i.fValue.As<String> ().c_str ());
                    }
                    String  dataValueString = vv.Lookup (L"data").Value ().As<String> ();
                    {
                        size_t i = dataValueString.Find (',');
                        if (i != -1) {
                            dataValueString = dataValueString.SubString (i + 1);
                        }
                    }
                    BLOB    resultBLOB = Cryptography::Encoding::Algorithm::DecodeBase64 (dataValueString.AsUTF8 ());
                    VerifyTestResult (resultBLOB == roundTripTestData);
                }
            }
            void    T3_httpbin_SimplePUT_ (Connection c)
            {
                Debug::TraceContextBumper ctx ("T3_httpbin_SimplePUT_");
                using   Memory::BLOB;

                static   mt19937 sRNG_;

                c.SetURL (URL::Parse (L"http://httpbin.org/put"));
                BLOB    roundTripTestData = [] () {
                    Memory::SmallStackBuffer<Byte> buf (1024);
                    for (size_t i = 0; i < buf.GetSize (); ++i) {
                        buf[i] = static_cast<Byte> (uniform_int_distribution<unsigned short>()(sRNG_));
                    }
                    return BLOB (buf.begin (), buf.end ());
                } ();
                Response    r   =   c.PUT (roundTripTestData, DataExchange::PredefinedInternetMediaType::OctetStream_CT ());
                VerifyTestResult (r.GetSucceeded ());
                {
                    VariantValue v = JSON::Reader ().Read (r.GetDataBinaryInputStream ());
                    Mapping<String, VariantValue> vv = v.As<Mapping<String, VariantValue>> ();
                    DbgTrace ("PUT parsed response:");
                    for (auto i : vv) {
                        DbgTrace (L"%s : %s", i.fKey.c_str (), i.fValue.As<String> ().c_str ());
                    }
                    String  dataValueString = vv.Lookup (L"data").Value ().As<String> ();
                    {
                        size_t i = dataValueString.Find (',');
                        if (i != -1) {
                            dataValueString = dataValueString.SubString (i + 1);
                        }
                    }
                    BLOB    resultBLOB = Cryptography::Encoding::Algorithm::DecodeBase64 (dataValueString.AsUTF8 ());
                    VerifyTestResult (resultBLOB == roundTripTestData);
                }
            }
            void    DoRegressionTests_ForConnectionFactory_ (Connection (*factory) ())
            {
                {
                    T1_httpbin_SimpleGET_ (factory ());
                    T2_httpbin_SimplePOST_ (factory ());
                }
                {
                    // Connection re-use
                    Connection conn = factory ();
                    T1_httpbin_SimpleGET_ (conn);
                    T2_httpbin_SimplePOST_ (conn);
                    T3_httpbin_SimplePUT_ (conn);
                }
            }
        }
        void    DoTests_ ()
        {
            Debug::TraceContextBumper ctx ("{}::Test_2_SimpleFetch_httpbin_");
            using namespace Private_;
            try {
                DoRegressionTests_ForConnectionFactory_ ([]() -> Connection { return CreateConnection (kDefaultTestOptions_); });
            }
            catch (const Execution::RequiredComponentMissingException&) {
#if     !qHasFeature_LibCurl && !qHasFeature_WinHTTP
                // OK to ignore. We don't wnat to call this failing a test, because there is nothing to fix.
                // This is more like the absence of a feature beacuse of the missing component.
#else
                Execution::ReThrow ();
#endif
            }

#if     qHasFeature_LibCurl
            DoRegressionTests_ForConnectionFactory_ ([]() -> Connection { return Connection_LibCurl (kDefaultTestOptions_); });
#endif
#if     qHasFeature_WinHTTP
            DoRegressionTests_ForConnectionFactory_ ([]() -> Connection { return Connection_WinHTTP (kDefaultTestOptions_); });
#endif
        }
    }
}

namespace {
    namespace Test3_TextStreamResponse_ {
        namespace Private_ {
            void    Test_1_SimpleFetch_Google_C_ (Connection c)
            {
                c.SetURL (URL::Parse (L"http://www.google.com"));
                Response    r   =   c.GET ();
                VerifyTestResult (r.GetSucceeded ());
                for (auto i : r.GetHeaders ()) {
                    DbgTrace (L"%s=%s", i.fKey.c_str (), i.fValue.c_str ());
                }
                InternetMediaType contentType = r.GetContentType ();
                String responseText = r.GetDataTextInputStream ().ReadAll ();
                DbgTrace (L"responseText = %s", responseText.c_str ());
                VerifyTestResult (responseText.Contains (L"google"));
            }
            void    DoRegressionTests_ForConnectionFactory_ (Connection (*factory) ())
            {
                Test_1_SimpleFetch_Google_C_ (factory ());
            }
        }
        void    DoTests_ ()
        {
            Debug::TraceContextBumper ctx ("{}::Test3_TextStreamResponse_");
            using namespace Private_;
            try {
                DoRegressionTests_ForConnectionFactory_ ([]() -> Connection { return CreateConnection (kDefaultTestOptions_); });
            }
            catch (const Execution::RequiredComponentMissingException&) {
#if     !qHasFeature_LibCurl && !qHasFeature_WinHTTP
                // OK to ignore. We don't wnat to call this failing a test, because there is nothing to fix.
                // This is more like the absence of a feature beacuse of the missing component.
#else
                Execution::ReThrow ();
#endif
            }

#if     qHasFeature_LibCurl
            DoRegressionTests_ForConnectionFactory_ ([]() -> Connection { return Connection_LibCurl (kDefaultTestOptions_); });
#endif
#if     qHasFeature_WinHTTP
            DoRegressionTests_ForConnectionFactory_ ([]() -> Connection { return Connection_WinHTTP (kDefaultTestOptions_); });
#endif
        }
    }
}

namespace {
    namespace Test_4_RefDocsTests_ {
        namespace Private_ {
            void    T1_get_ ()
            {
                Connection  c   =   IO::Network::Transfer::CreateConnection (kDefaultTestOptions_);
                c.SetURL (URL::Parse (L"http://www.google.com"));
                Response    r   =   c.GET ();
                VerifyTestResult (r.GetSucceeded ());
                VerifyTestResult (r.GetData ().size () > 1);
            }

        }
        void    DoTests_ ()
        {
            Debug::TraceContextBumper ctx ("{}::Test_4_RefDocsTests_");
            try {
                Private_::T1_get_ ();
            }
            catch (const Execution::RequiredComponentMissingException&) {
#if     !qHasFeature_LibCurl && !qHasFeature_WinHTTP
                // OK to ignore. We don't wnat to call this failing a test, because there is nothing to fix.
                // This is more like the absence of a feature beacuse of the missing component.
#else
                Execution::ReThrow ();
#endif
            }
        }
    }
}

namespace {
    namespace Test_5_SSLCertCheckTests_ {
        namespace Private_ {
            void    T1_get_ (Connection::Options o)
            {
                Connection  c   =   IO::Network::Transfer::CreateConnection (o);
                try {
                    c.SetURL (URL::Parse (L"https://testssl-valid.disig.sk/index.en.html"));
                    Response    r   =   c.GET ();
                    VerifyTestResult (r.GetSucceeded ());
                    VerifyTestResult (r.GetData ().size () > 1);
                }
#if     qHasFeature_LibCurl
                catch (const LibCurlException& lce) {
#if     !qHasFeature_OpenSSL
                    if (lce.GetCode () == 1/*CURLE_UNSUPPORTED_PROTOCOL*/) {
                        DbgTrace ("Warning - ignored exception doing LibCurl/ssl - for now probably just no SSL support with libcurl");
                        return;
                    }
#endif
                    throw;
                }
#else
                catch (...) {
                    throw;
                }
#endif
            }
        }
        void    DoTests_ ()
        {
            Debug::TraceContextBumper ctx ("{}::Test_5_SSLCertCheckTests_");
            Connection::Options o = kDefaultTestOptions_;
            try {
                o.fFailConnectionIfSSLCertificateInvalid = true;
                Private_::T1_get_ (o);
#if     qHasFeature_LibCurl && !qHasFeature_OpenSSL
                // ingore this case, since we allow the failed connect above...
#else
                VerifyTestResult (false);
#endif
            }
            catch (const Execution::RequiredComponentMissingException&) {
#if     !qHasFeature_LibCurl && !qHasFeature_WinHTTP
                // OK to ignore. We don't wnat to call this failing a test, because there is nothing to fix.
                // This is more like the absence of a feature beacuse of the missing component.
#else
                Execution::ReThrow ();
#endif
            }
            catch (...) {
                // Good - this should fail
            }
            try {
                o.fFailConnectionIfSSLCertificateInvalid = false;
                Private_::T1_get_ (o);
            }
            catch (const Execution::RequiredComponentMissingException&) {
#if     !qHasFeature_LibCurl && !qHasFeature_WinHTTP
                // OK to ignore. We don't wnat to call this failing a test, because there is nothing to fix.
                // This is more like the absence of a feature beacuse of the missing component.
#else
                Execution::ReThrow ();
#endif
            }
            catch (...) {
                VerifyTestResult (false);
            }
        }
    }
}

namespace   {
    void    DoRegressionTests_ ()
    {
        Test_1_SimpleConnnectionTests_::DoTests_ ();
        Test_2_SimpleFetch_httpbin_::DoTests_ ();
        Test3_TextStreamResponse_::DoTests_ ();
        Test_4_RefDocsTests_::DoTests_ ();
        Test_5_SSLCertCheckTests_::DoTests_ ();
    }
}

int main (int argc, const char* argv[])
{
    Stroika::TestHarness::Setup ();
    Stroika::TestHarness::PrintPassOrFail (DoRegressionTests_);
    return EXIT_SUCCESS;
}
<|endoftext|>