<commit_before>/* IBM_PROLOG_BEGIN_TAG                                                   */
/* This is an automatically generated prolog.                             */
/*                                                                        */
/* $Source: src/import/chips/p9/procedures/hwp/memory/lib/power_thermal/throttle.H $ */
/*                                                                        */
/* OpenPOWER HostBoot Project                                             */
/*                                                                        */
/* Contributors Listed Below - COPYRIGHT 2016,2019                        */
/* [+] International Business Machines Corp.                              */
/*                                                                        */
/*                                                                        */
/* Licensed under the Apache License, Version 2.0 (the "License");        */
/* you may not use this file except in compliance with the License.       */
/* You may obtain a copy of the License at                                */
/*                                                                        */
/*     http://www.apache.org/licenses/LICENSE-2.0                         */
/*                                                                        */
/* Unless required by applicable law or agreed to in writing, software    */
/* distributed under the License is distributed on an "AS IS" BASIS,      */
/* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or        */
/* implied. See the License for the specific language governing           */
/* permissions and limitations under the License.                         */
/*                                                                        */
/* IBM_PROLOG_END_TAG                                                     */
///
/// @file p9_mss_utils_to_throttle.H
/// @brief throttle API
///

// *HWP HWP Owner: Andre Marin <aamarin@us.ibm.com>
// *HWP HWP Backup: Brian Silver <bsilver@us.ibm.com>
// *HWP Team: Memory
// *HWP Level: 2
// *HWP Consumed by: FSP:HB

#ifndef _MSS_POWER_THROTTLE_
#define _MSS_POWER_THROTTLE_

namespace mss
{

enum throttle_const : size_t
{
    // Dram data bus utilization is 4X the address bus utilization
    DRAM_BUS_UTILS = 4,
    PERCENT_CONVERSION = 100,
};

///
/// @brief Calculate N (address operations) allowed within a window of M DRAM clocks
/// @param[in] i_databus_util databus utilization percentage (e.g. 5% = 5)
/// @param[in] i_num_dram_clocks window of M DRAM clocks
/// @return number of throttled commands allowed
/// @note Uses N/M Throttling.
/// Equation:  (DRAM data bus utilization Percent / 100 ) = ((N * 4) / M )
/// Dram data bus utilization is 4X the address bus utilization
///
inline uint32_t throttled_cmds(const uint32_t i_databus_util, const uint32_t i_num_dram_clocks)
{
    constexpr uint64_t l_divisor = DRAM_BUS_UTILS * PERCENT_CONVERSION;
    const uint64_t l_dividend = i_databus_util * i_num_dram_clocks;
    const uint64_t l_quotient = l_dividend / l_divisor;
    const uint64_t l_remainder = l_dividend % l_divisor;

    return l_quotient + (l_remainder == 0  ? 0 : 1);
}

///
/// @brief Set ATTR_MSS_CHANNEL_PAIR_MAXPOWER and  ATTR_MSS_MEM_THROTTLED_N_COMMANDS_PER_SLOT,
/// @param[in] Vector of MCS 's on the same VDDR domain
/// @return fapi2::ReturnCode - FAPI2_RC_SUCCESS iff get is OK
/// @note Called in p9_mss_bulk_pwr_throttles
/// @note determines the throttle levels based off of the port's power curve,
/// sets the slot throttles to the same
///
fapi2::ReturnCode bulk_power_regulator_throttles (const std::vector<fapi2::Target<fapi2::TARGET_TYPE_MCS>>&
        i_targets);

///
/// @brief Set ATTR_MSS_MEM_THROTTLED_N_COMMANDS_PER_SLOT,
/// @param[in] Vector of MCS 's on the same VDDR domain
/// @return fapi2::ReturnCode - FAPI2_RC_SUCCESS iff get is OK
/// @note Called in p9_mss_bulk_pwr_throttles
/// @note Sets the throttle levels based off of the dimm's thermal limits
/// @note both DIMM's on a port are set to the same throttle level
///
fapi2::ReturnCode bulk_thermal_throttles (const std::vector<fapi2::Target<fapi2::TARGET_TYPE_MCS>>& i_targets);

}// mss

#endif
<commit_msg>Started implementation of bulk_pwr_throttles<commit_after>/* IBM_PROLOG_BEGIN_TAG                                                   */
/* This is an automatically generated prolog.                             */
/*                                                                        */
/* $Source: src/import/chips/p9/procedures/hwp/memory/lib/power_thermal/throttle.H $ */
/*                                                                        */
/* OpenPOWER HostBoot Project                                             */
/*                                                                        */
/* Contributors Listed Below - COPYRIGHT 2016,2019                        */
/* [+] International Business Machines Corp.                              */
/*                                                                        */
/*                                                                        */
/* Licensed under the Apache License, Version 2.0 (the "License");        */
/* you may not use this file except in compliance with the License.       */
/* You may obtain a copy of the License at                                */
/*                                                                        */
/*     http://www.apache.org/licenses/LICENSE-2.0                         */
/*                                                                        */
/* Unless required by applicable law or agreed to in writing, software    */
/* distributed under the License is distributed on an "AS IS" BASIS,      */
/* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or        */
/* implied. See the License for the specific language governing           */
/* permissions and limitations under the License.                         */
/*                                                                        */
/* IBM_PROLOG_END_TAG                                                     */
///
/// @file p9_mss_utils_to_throttle.H
/// @brief throttle API
///

// *HWP HWP Owner: Jacob Harvey <jlharvey@us.ibm.com>
// *HWP HWP Backup: Brian Silver <bsilver@us.ibm.com>
// *HWP Team: Memory
// *HWP Level: 2
// *HWP Consumed by: FSP:HB

#ifndef _MSS_POWER_THROTTLE_
#define _MSS_POWER_THROTTLE_

#include <fapi2.H>
#include <mss.H>
namespace mss
{
namespace power_thermal
{
enum throttle_const : size_t
{
    // Dram data bus utilization is bus utilization / 4
    DRAM_BUS_UTILS = 4,
    UTIL_CONVERSION = 10000,
    PERCENT_CONVERSION = 100,
    MIN_UTIL = 1,
    IDLE_UTIL = 0,
};
///
/// @class throttle
/// @brief Determine power_thermal throttles for memory
///
class throttle
{
    private:

    public:
        const fapi2::Target<fapi2::TARGET_TYPE_MCA>& iv_target;
        //dimm level
        uint32_t iv_databus_port_max;

        uint8_t iv_power_uplift_idle;
        uint8_t iv_power_uplift;

        uint32_t iv_runtime_n_slot;
        uint32_t iv_runtime_n_port;
        uint32_t iv_m_clocks;
        uint32_t iv_dimm_thermal_limit[MAX_DIMM_PER_PORT] = {};
        uint16_t iv_pwr_slope[MAX_DIMM_PER_PORT] = {};
        uint16_t iv_pwr_int[MAX_DIMM_PER_PORT] = {};
        uint32_t iv_n_slot;
        uint32_t iv_n_port;
        uint32_t iv_port_power_limit;
        uint32_t iv_port_maxpower;
        uint32_t iv_calc_port_maxpower;

        //default ctor deleted
        throttle() = delete;

        ///
        /// @brief Constructor
        /// @param[in] i_target MCA target to call power thermal stuff on
        /// @param[out] o_rc fapi2::ReturnCode fapi2::FAPI2_RC_SUCCESS iff ctor was successful
        ///
        throttle( const fapi2::Target<fapi2::TARGET_TYPE_MCA>& i_mca, fapi2::ReturnCode o_rc);

        //
        // @brief Destructor
        //
        ~throttle() = default;

        ///
        /// @brief Calculates the min and max power usage for a port based off of power curves and utilizati
        /// @param[in] i_idle_util the utilization of the databus in idle mode
        /// @param[in] i_max_util  the utilization of the port at maximum possible (mrw or calculated)
        /// @param[out] o_port_power_idle max value of port power in cW
        /// @param[out] o_port_power_max max value of port power in cW
        /// @note Called twice in p9_mss_bulk_pwr_throttles
        /// @note uses dimm power curves from class variables
        ///
        void calc_port_power(const double i_idle_util [MAX_DIMM_PER_PORT],
                             const double i_max_util [MAX_DIMM_PER_PORT],
                             double& o_port_power_idle,
                             double& o_port_power_max);
        ///
        /// @brief Calculate the port power curve in order to calculate the port utilization
        /// @paramp[in] i_port_power_calc_idle double of the port's power consumption at idle
        /// @paramp[in] i_port_power_calc_max double of the port's power consumption at max utilization
        /// @paramp[out] o_port_power_slope
        /// @paramp[out] o_port_power_int
        /// @return FAPI2_RC_SUCCESS iff success
        /// @note Called in p9_mss_bulk_pwr_throttles
        /// @note Port power curve needed to calculate the port utilization
        ///
        fapi2::ReturnCode calc_port_power_curve(const double& i_port_power_calc_idle,
                                                const double& i_port_power_calc_max,
                                                uint32_t& o_port_power_slope,
                                                uint32_t& o_port_power_int);
        ///
        /// @brief Calculate the port's databus utilization given the port's power curve
        /// @paramp[in] i_port_power_slope
        /// @paramp[in] i_port_power_int
        /// @paramp[out] o_port_util the port's databus utilization
        /// @note Called in p9_mss_bulk_pwr_throttles
        /// @note Chooses worst case between the maximum allowed databus utilization and the calculated value
        /// @notes makes sure that the utilization isn't 0
        ///
        void calc_port_util_usage(const uint32_t& i_port_power_slope,
                                  const uint32_t& i_port_power_int,
                                  double& o_port_util);

        ///
        /// @brief Calculates the power max and idle for each dimm using power curves and databus utilization
        /// @param[out] o_dimm_power_idle double type for precision, the DIMM power limit in idle state (0 utilization)
        /// @param[out] o_dimm_power_max double type for precision, the DIMM power limit at max utilization
        /// @note Called in p9_mss_bulk_pwr_throttle for thermal_throttles, eff_config_thermal
        /// @note power values are as if dimm is alone on port, using port_databus_util_max
        ///
        void calc_dimm_power(double o_dimm_power_idle [MAX_DIMM_PER_PORT],
                             double o_dimm_power_max [MAX_DIMM_PER_PORT]);
        ///
        /// @brief Converts the port maximum databus to a dimm level based on powerslopes and dimms installed
        /// @param[in] i_databus_port_max max databus utilization for the port (either calculated or mrw)
        /// @param[out] o_databus_dimm_max array of dimm utilization values
        /// @return fapi2::ReturnCode - FAPI2_RC_SUCCESS iff the split is OK
        /// @note Called in p9_mss_bulk_pwr_throttles
        /// @used to calculate the port power based off of DIMM power curves
        ///
        fapi2::ReturnCode calc_databus(const double& i_databus_port_max,
                                       double o_databus_dimm_max [MAX_DIMM_PER_PORT]);

        ///
        /// @brief Set ATTR_MSS_CHANNEL_PAIR_MAXPOWER and  ATTR_MSS_MEM_THROTTLED_N_COMMANDS_PER_SLOT,
        /// @return fapi2::ReturnCode - FAPI2_RC_SUCCESS iff get is OK
        /// @note Called in p9_mss_bulk_pwr_throttles
        /// @note determines the throttle levels based off of the port's power curve,
        /// sets the slot throttles to the same
        ///
        fapi2::ReturnCode power_regulator_throttles ();

        ///
        /// @brief Set ATTR_MSS_MEM_THROTTLED_N_COMMANDS_PER_SLOT,
        /// @return fapi2::ReturnCode - FAPI2_RC_SUCCESS iff get is OK
        /// @note Called in p9_mss_bulk_pwr_throttles
        /// @note Sets the throttle levels based off of the dimm's thermal limits
        /// @note both DIMM's on a port are set to the same throttle level
        ///
        fapi2::ReturnCode thermal_throttles ();
};

///
/// @brief Calculate N (address operations) allowed within a window of M DRAM clocks
/// @param[in] i_databus_util databus utilization percentage (e.g. 5% = 5)
/// @param[in] i_num_dram_clocks window of M DRAM clocks
/// @return number of throttled commands allowed
/// @note Uses N/M Throttling.
/// Equation:  (DRAM data bus utilization Percent / 10000 ) = ((N * M) / 4 )
/// Equation: N = (DRAM data bus utilization Percent * M) / (4 * 10000)
inline uint32_t throttled_cmds(const uint32_t i_databus_util, const uint32_t i_num_dram_clocks)
{
    constexpr uint64_t l_divisor = DRAM_BUS_UTILS * UTIL_CONVERSION;
    const uint64_t l_dividend = i_databus_util * i_num_dram_clocks;
    const uint64_t l_result = l_dividend / l_divisor;

    return l_result;
}

///
/// @brief Calculate the port databus utilization based off of N throttles and M dram clocks
/// @param[in] i_n_throttles N (address operations) allowed within a window of M DRAM clocks
/// @param[in] i_num_dram_clocks window of M DRAM clocks
/// @return number of throttled commands allowed
/// @note Uses N/M Throttling.
/// Equation: databus utilization = (N * 4 * 100000) / M
///
inline double calc_util_from_throttles(const uint16_t i_n_throttles, const uint32_t i_num_dram_clocks)
{
    constexpr uint32_t l_multiplier = DRAM_BUS_UTILS * UTIL_CONVERSION;
    return  ( (double(i_n_throttles) * l_multiplier) / i_num_dram_clocks);

}

///
/// @brief Perform thermal calculations as part of the effective configuration
/// @param[in] i_target the MCS target in which the runtime throttles will be reset
/// @return FAPI2_RC_SUCCESS iff ok
///
fapi2::ReturnCode restore_runtime_throttles( const fapi2::Target<fapi2::TARGET_TYPE_MCS>& i_target );

}//power_thermal
}// mss

#endif
<|endoftext|>


<commit_before>#ifndef __RANK_FILTER__
#define __RANK_FILTER__

#include <list>
#include <set>
#include <cmath>
#include <cassert>
#include <iostream>
#include <iterator>
#include <type_traits>
#include <utility>

#include <vigra/multi_array.hxx>
#include <vigra/linear_algebra.hxx>

namespace vigra
{

template<unsigned int N,
        class T1, class S1,
        class T2, class S2,
        typename std::enable_if<(N == 1), int>::type = 0>
inline void lineRankOrderFilterND(const vigra::MultiArrayView <N, T1, S1> &src,
        vigra::MultiArrayView <N, T2, S2> dest,
        unsigned int half_length, float rank, unsigned int axis = N - 1)
{
    // Will ignore boundaries initially.
    // Then will try adding reflection.

    // Rank must be in the range 0 to 1
    assert((0 <= rank) && (rank <= 1));

    const int rank_pos = round(rank * (2 * half_length));

    // The position of the
    typename vigra::MultiArrayView<N, T1, S1>::difference_type_1 window_begin(0);
    std::multiset<T1> sorted_window;
    std::list< typename std::multiset<T1>::iterator > window_iters;

    // Get the initial sorted window.
    // Include the reflection.
    for (typename vigra::MultiArrayView<N, T1, S1>::difference_type_1 j(half_length); j > 0; j--)
    {
        window_iters.push_back(sorted_window.insert(src[window_begin + j]));
    }
    for (typename vigra::MultiArrayView<N, T1, S1>::difference_type_1 j(0); j <= half_length; j++)
    {
        window_iters.push_back(sorted_window.insert(src[window_begin + j]));
    }

    typename std::multiset<T1>::iterator rank_point = sorted_window.begin();

    for (int i = 0; i < rank_pos; i++)
    {
        rank_point++;
    }

    while ( window_begin < src.size() )
    {
        dest[window_begin] = *rank_point;

        typename std::multiset<T1>::iterator prev_iter(window_iters.front());
        T1 prev_value = *prev_iter;
        window_iters.pop_front();

        window_begin++;

        T1 next_value;
        if ( window_begin < (src.size() - half_length) )
        {
            next_value = src[window_begin + half_length];
        }
        else
        {
            next_value = src[2 * src.size() - (window_begin + half_length + 2)];
        }

        if ( ( *rank_point < *prev_iter ) && ( *rank_point <= next_value ) )
        {
            sorted_window.erase(prev_iter);
            window_iters.push_back(sorted_window.insert(next_value));
        }
        else if ( ( *rank_point >= *prev_iter ) && ( *rank_point > next_value ) )
        {
            if ( rank_point == prev_iter )
            {
                window_iters.push_back(sorted_window.insert(next_value));
                rank_point--;

                sorted_window.erase(prev_iter);
            }
            else
            {
                sorted_window.erase(prev_iter);
                window_iters.push_back(sorted_window.insert(next_value));
            }
        }
        else if ( ( *rank_point < *prev_iter ) && ( *rank_point > next_value ) )
        {
            sorted_window.erase(prev_iter);
            window_iters.push_back(sorted_window.insert(next_value));

            rank_point--;
        }
        else if ( ( *rank_point >= *prev_iter ) && ( *rank_point <= next_value ) )
        {
            if (rank_point == prev_iter)
            {
                window_iters.push_back(sorted_window.insert(next_value));
                rank_point++;

                sorted_window.erase(prev_iter);
            }
            else
            {
                sorted_window.erase(prev_iter);
                window_iters.push_back(sorted_window.insert(next_value));

                rank_point++;
            }
        }
    }
}

template<unsigned int N,
        class T1, class S1,
        class T2, class S2,
        typename std::enable_if<(N > 1), int>::type = 0>
inline void lineRankOrderFilterND(const vigra::MultiArrayView <N, T1, S1> &src,
        vigra::MultiArrayView <N, T2, S2> dest,
        unsigned int half_length, float rank, unsigned int axis = N - 1)
{
    typename vigra::MultiArrayView<N, T1, S1>::difference_type transposed_axes;

    for (unsigned int i = 0; i < N; i++)
    {
        transposed_axes[i] = i;
    }

    std::swap(transposed_axes[N - 1], transposed_axes[axis]);

    vigra::MultiArray<N, T1> src_transposed(src.transpose(transposed_axes));

    vigra::MultiArrayView<N, T1, S1> dest_transposed(dest.transpose(transposed_axes));

    typename vigra::MultiArrayView<N - 1, T1, S1>::difference_type pos;
    pos = 0;

    bool done = false;

    while (!done)
    {
        lineRankOrderFilter(src_transposed.bindInner(pos), dest_transposed.bindInner(pos), half_length, rank);

        bool carry = true;
        for (unsigned int i = 0; ( carry && (i < (N - 1)) ); i++)
        {
            if ( (++pos[N - (i + 2)]) < src_transposed.shape(N - (i + 2)) )
            {
                carry = false;
            }
            else
            {
                pos[N - (i + 2)] = 0;
                carry = true;
            }
        }

        done = carry;
    }
}

template<unsigned int N,
        class T1, class S1,
        class T2, class S2>
inline void lineRankOrderFilter(const vigra::MultiArrayView <N, T1, S1> &src,
        vigra::MultiArrayView <N, T2, S2> dest,
        unsigned int half_length, float rank, unsigned int axis = N - 1)
{
    lineRankOrderFilterND(src, dest, half_length, rank, axis);
}

}

#endif //__RANK_FILTER__
<commit_msg>rank_filter.hxx: Fixed call in lineRankOrderFilterND to be a call to lineRankOrderFilterND instead of lineRankOrderFilteND instead of lineRankOrderFilter.<commit_after>#ifndef __RANK_FILTER__
#define __RANK_FILTER__

#include <list>
#include <set>
#include <cmath>
#include <cassert>
#include <iostream>
#include <iterator>
#include <type_traits>
#include <utility>

#include <vigra/multi_array.hxx>
#include <vigra/linear_algebra.hxx>

namespace vigra
{

template<unsigned int N,
        class T1, class S1,
        class T2, class S2,
        typename std::enable_if<(N == 1), int>::type = 0>
inline void lineRankOrderFilterND(const vigra::MultiArrayView <N, T1, S1> &src,
        vigra::MultiArrayView <N, T2, S2> dest,
        unsigned int half_length, float rank, unsigned int axis = N - 1)
{
    // Will ignore boundaries initially.
    // Then will try adding reflection.

    // Rank must be in the range 0 to 1
    assert((0 <= rank) && (rank <= 1));

    const int rank_pos = round(rank * (2 * half_length));

    // The position of the
    typename vigra::MultiArrayView<N, T1, S1>::difference_type_1 window_begin(0);
    std::multiset<T1> sorted_window;
    std::list< typename std::multiset<T1>::iterator > window_iters;

    // Get the initial sorted window.
    // Include the reflection.
    for (typename vigra::MultiArrayView<N, T1, S1>::difference_type_1 j(half_length); j > 0; j--)
    {
        window_iters.push_back(sorted_window.insert(src[window_begin + j]));
    }
    for (typename vigra::MultiArrayView<N, T1, S1>::difference_type_1 j(0); j <= half_length; j++)
    {
        window_iters.push_back(sorted_window.insert(src[window_begin + j]));
    }

    typename std::multiset<T1>::iterator rank_point = sorted_window.begin();

    for (int i = 0; i < rank_pos; i++)
    {
        rank_point++;
    }

    while ( window_begin < src.size() )
    {
        dest[window_begin] = *rank_point;

        typename std::multiset<T1>::iterator prev_iter(window_iters.front());
        T1 prev_value = *prev_iter;
        window_iters.pop_front();

        window_begin++;

        T1 next_value;
        if ( window_begin < (src.size() - half_length) )
        {
            next_value = src[window_begin + half_length];
        }
        else
        {
            next_value = src[2 * src.size() - (window_begin + half_length + 2)];
        }

        if ( ( *rank_point < *prev_iter ) && ( *rank_point <= next_value ) )
        {
            sorted_window.erase(prev_iter);
            window_iters.push_back(sorted_window.insert(next_value));
        }
        else if ( ( *rank_point >= *prev_iter ) && ( *rank_point > next_value ) )
        {
            if ( rank_point == prev_iter )
            {
                window_iters.push_back(sorted_window.insert(next_value));
                rank_point--;

                sorted_window.erase(prev_iter);
            }
            else
            {
                sorted_window.erase(prev_iter);
                window_iters.push_back(sorted_window.insert(next_value));
            }
        }
        else if ( ( *rank_point < *prev_iter ) && ( *rank_point > next_value ) )
        {
            sorted_window.erase(prev_iter);
            window_iters.push_back(sorted_window.insert(next_value));

            rank_point--;
        }
        else if ( ( *rank_point >= *prev_iter ) && ( *rank_point <= next_value ) )
        {
            if (rank_point == prev_iter)
            {
                window_iters.push_back(sorted_window.insert(next_value));
                rank_point++;

                sorted_window.erase(prev_iter);
            }
            else
            {
                sorted_window.erase(prev_iter);
                window_iters.push_back(sorted_window.insert(next_value));

                rank_point++;
            }
        }
    }
}

template<unsigned int N,
        class T1, class S1,
        class T2, class S2,
        typename std::enable_if<(N > 1), int>::type = 0>
inline void lineRankOrderFilterND(const vigra::MultiArrayView <N, T1, S1> &src,
        vigra::MultiArrayView <N, T2, S2> dest,
        unsigned int half_length, float rank, unsigned int axis = N - 1)
{
    typename vigra::MultiArrayView<N, T1, S1>::difference_type transposed_axes;

    for (unsigned int i = 0; i < N; i++)
    {
        transposed_axes[i] = i;
    }

    std::swap(transposed_axes[N - 1], transposed_axes[axis]);

    vigra::MultiArray<N, T1> src_transposed(src.transpose(transposed_axes));

    vigra::MultiArrayView<N, T1, S1> dest_transposed(dest.transpose(transposed_axes));

    typename vigra::MultiArrayView<N - 1, T1, S1>::difference_type pos;
    pos = 0;

    bool done = false;

    while (!done)
    {
        lineRankOrderFilterND(src_transposed.bindInner(pos), dest_transposed.bindInner(pos), half_length, rank);

        bool carry = true;
        for (unsigned int i = 0; ( carry && (i < (N - 1)) ); i++)
        {
            if ( (++pos[N - (i + 2)]) < src_transposed.shape(N - (i + 2)) )
            {
                carry = false;
            }
            else
            {
                pos[N - (i + 2)] = 0;
                carry = true;
            }
        }

        done = carry;
    }
}

template<unsigned int N,
        class T1, class S1,
        class T2, class S2>
inline void lineRankOrderFilter(const vigra::MultiArrayView <N, T1, S1> &src,
        vigra::MultiArrayView <N, T2, S2> dest,
        unsigned int half_length, float rank, unsigned int axis = N - 1)
{
    lineRankOrderFilterND(src, dest, half_length, rank, axis);
}

}

#endif //__RANK_FILTER__
<|endoftext|>


<commit_before>#ifndef SM_LOGGING_LEVELS_HPP
#define SM_LOGGING_LEVELS_HPP

// These allow you to compile-out everything below a certain severity level if necessary
#define SMCONSOLE_SEVERITY_ALL 0
#define SMCONSOLE_SEVERITY_FINEST 1
#define SMCONSOLE_SEVERITY_VERBOSE 2
#define SMCONSOLE_SEVERITY_FINER 3
#define SMCONSOLE_SEVERITY_TRACE 4
#define SMCONSOLE_SEVERITY_FINE 5
#define SMCONSOLE_SEVERITY_DEBUG 6
#define SMCONSOLE_SEVERITY_INFO 7
#define SMCONSOLE_SEVERITY_WARN 8
#define SMCONSOLE_SEVERITY_ERROR 9
#define SMCONSOLE_SEVERITY_FATAL 10
#define SMCONSOLE_SEVERITY_NONE 11

#include <string>

namespace sm {
    namespace logging {

        namespace levels
        {
            enum Level
            {
                All,
                Finest,
                Verbose,
                Finer,
                Trace,
                Fine,
                Debug,
                Info,
                Warn,
                Error,
                Fatal,

                Count
            };

            Level fromString(const std::string& l);

        }

        typedef levels::Level Level;

    } // namespace logging
} // namespace sm

#endif /* SM_LOGGING_LEVELS_HPP */
<commit_msg>Update Levels.hpp<commit_after>#ifndef SM_LOGGING_LEVELS_HPP
#define SM_LOGGING_LEVELS_HPP

// These allow you to compile-out everything below a certain severity level if necessary
#define SMCONSOLE_SEVERITY_ALL 0
#define SMCONSOLE_SEVERITY_FINEST 1
#define SMCONSOLE_SEVERITY_VERBOSE 2
#define SMCONSOLE_SEVERITY_FINER 3
#define SMCONSOLE_SEVERITY_TRACE 4
#define SMCONSOLE_SEVERITY_FINE 5
#define SMCONSOLE_SEVERITY_DEBUG 6
#define SMCONSOLE_SEVERITY_INFO 7
#define SMCONSOLE_SEVERITY_WARN 8
#define SMCONSOLE_SEVERITY_ERROR 9
#define SMCONSOLE_SEVERITY_FATAL 10
#define SMCONSOLE_SEVERITY_NONE 11

#include <string>

namespace sm {
    namespace logging {

        namespace levels
        {
            enum Level
            {
                All,
                Finest,
                Verbose,
                Finer,
                Trace,
                Fine,
                Debug,
                Info,
                Warn,
                Error,
                Fatal,

                Count
            };

            Level fromString(const std::string& level);

        }

        typedef levels::Level Level;

    } // namespace logging
} // namespace sm

#endif /* SM_LOGGING_LEVELS_HPP */
<|endoftext|>


<commit_before>/****************************************************************************
 *
 *   Copyright (C) 2012 PX4 Development Team. All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 *
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in
 *    the documentation and/or other materials provided with the
 *    distribution.
 * 3. Neither the name PX4 nor the names of its contributors may be
 *    used to endorse or promote products derived from this software
 *    without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
 * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS
 * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
 * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 * POSSIBILITY OF SUCH DAMAGE.
 *
 ****************************************************************************/

/**
 * @file Base class for devices connected via SPI.
 *
 * @todo Work out if caching the mode/frequency would save any time.
 */

#include "spi.h"

#ifndef CONFIG_SPI_EXCHANGE
# error This driver requires CONFIG_SPI_EXCHANGE
#endif

namespace device
{

SPI::SPI(const char *name,
	 const char *devname,
	 int bus,
	 enum spi_dev_e device,
	 enum spi_mode_e mode,
	 uint32_t frequency,
	 int irq) :
	// base class
	CDev(name, devname, irq),
	// public
	// protected
	// private
	_bus(bus),
	_device(device),
	_mode(mode),
	_frequency(frequency),
	_dev(nullptr)
{
}

SPI::~SPI()
{
	// XXX no way to let go of the bus...
}

int
SPI::init()
{
	int ret = OK;

	// attach to the spi bus
	if (_dev == nullptr)
		_dev = up_spiinitialize(_bus);

	if (_dev == nullptr) {
		debug("failed to init SPI");
		ret = -ENOENT;
		goto out;
	}

	// call the probe function to check whether the device is present
	ret = probe();

	if (ret != OK) {
		debug("probe failed");
		goto out;
	}

	// do base class init, which will create the device node, etc.
	ret = CDev::init();

	if (ret != OK) {
		debug("cdev init failed");
		goto out;
	}

	// tell the workd where we are
	log("on bus %d at %d", _bus, _device);

out:
	return ret;
}

int
SPI::probe()
{
	// assume the device is too stupid to be discoverable
	return OK;
}

int
SPI::transfer(uint8_t *send, uint8_t *recv, unsigned len)
{

	if ((send == nullptr) && (recv == nullptr))
		return -EINVAL;

	/* do common setup */
	SPI_LOCK(_dev, true);
	SPI_SETFREQUENCY(_dev, _frequency);
	SPI_SETMODE(_dev, _mode);
	SPI_SELECT(_dev, _device, true);

	/* do the transfer */
	SPI_EXCHANGE(_dev, send, recv, len);

	/* and clean up */
	SPI_SELECT(_dev, _device, false);
	SPI_LOCK(_dev, false);

	return OK;
}

} // namespace device<commit_msg>force 8-bit SPI transactions for device::SPI drivers.<commit_after>/****************************************************************************
 *
 *   Copyright (C) 2012 PX4 Development Team. All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 *
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in
 *    the documentation and/or other materials provided with the
 *    distribution.
 * 3. Neither the name PX4 nor the names of its contributors may be
 *    used to endorse or promote products derived from this software
 *    without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
 * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS
 * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
 * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 * POSSIBILITY OF SUCH DAMAGE.
 *
 ****************************************************************************/

/**
 * @file Base class for devices connected via SPI.
 *
 * @todo Work out if caching the mode/frequency would save any time.
 */

#include "spi.h"

#ifndef CONFIG_SPI_EXCHANGE
# error This driver requires CONFIG_SPI_EXCHANGE
#endif

namespace device
{

SPI::SPI(const char *name,
	 const char *devname,
	 int bus,
	 enum spi_dev_e device,
	 enum spi_mode_e mode,
	 uint32_t frequency,
	 int irq) :
	// base class
	CDev(name, devname, irq),
	// public
	// protected
	// private
	_bus(bus),
	_device(device),
	_mode(mode),
	_frequency(frequency),
	_dev(nullptr)
{
}

SPI::~SPI()
{
	// XXX no way to let go of the bus...
}

int
SPI::init()
{
	int ret = OK;

	// attach to the spi bus
	if (_dev == nullptr)
		_dev = up_spiinitialize(_bus);

	if (_dev == nullptr) {
		debug("failed to init SPI");
		ret = -ENOENT;
		goto out;
	}

	// call the probe function to check whether the device is present
	ret = probe();

	if (ret != OK) {
		debug("probe failed");
		goto out;
	}

	// do base class init, which will create the device node, etc.
	ret = CDev::init();

	if (ret != OK) {
		debug("cdev init failed");
		goto out;
	}

	// tell the workd where we are
	log("on bus %d at %d", _bus, _device);

out:
	return ret;
}

int
SPI::probe()
{
	// assume the device is too stupid to be discoverable
	return OK;
}

int
SPI::transfer(uint8_t *send, uint8_t *recv, unsigned len)
{

	if ((send == nullptr) && (recv == nullptr))
		return -EINVAL;

	/* do common setup */
	SPI_LOCK(_dev, true);
	SPI_SETFREQUENCY(_dev, _frequency);
	SPI_SETMODE(_dev, _mode);
	SPI_SETBITS(_dev, 8);
	SPI_SELECT(_dev, _device, true);

	/* do the transfer */
	SPI_EXCHANGE(_dev, send, recv, len);

	/* and clean up */
	SPI_SELECT(_dev, _device, false);
	SPI_LOCK(_dev, false);

	return OK;
}

} // namespace device
<|endoftext|>


<commit_before>// Copyright (c) 2010 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

#include "net/test/test_server.h"

#include <algorithm>
#include <string>
#include <vector>

#include "build/build_config.h"

#if defined(OS_MACOSX)
#include "net/base/x509_certificate.h"
#endif

#include "base/command_line.h"
#include "base/file_util.h"
#include "base/leak_annotations.h"
#include "base/logging.h"
#include "base/path_service.h"
#include "base/string_number_conversions.h"
#include "base/utf_string_conversions.h"
#include "googleurl/src/gurl.h"
#include "net/base/cert_test_util.h"
#include "net/base/host_port_pair.h"
#include "net/base/host_resolver.h"
#include "net/base/test_completion_callback.h"
#include "net/socket/tcp_client_socket.h"
#include "net/test/python_utils.h"
#include "testing/platform_test.h"

namespace {

// Number of connection attempts for tests.
const int kServerConnectionAttempts = 10;

// Connection timeout in milliseconds for tests.
const int kServerConnectionTimeoutMs = 1000;

const char kTestServerShardFlag[] = "test-server-shard";

int GetPortBase(net::TestServer::Type type) {
  switch (type) {
    case net::TestServer::TYPE_FTP:
      return 3117;
    case net::TestServer::TYPE_HTTP:
      return 1337;
    case net::TestServer::TYPE_HTTPS:
      return 9443;
    case net::TestServer::TYPE_HTTPS_CLIENT_AUTH:
      return 9543;
    case net::TestServer::TYPE_HTTPS_EXPIRED_CERTIFICATE:
      // TODO(phajdan.jr): Some tests rely on this hardcoded value.
      // Some uses of this are actually in .html/.js files.
      return 9666;
    case net::TestServer::TYPE_HTTPS_MISMATCHED_HOSTNAME:
      return 9643;
    default:
      NOTREACHED();
  }
  return -1;
}

int GetPort(net::TestServer::Type type) {
  int port = GetPortBase(type);
  if (CommandLine::ForCurrentProcess()->HasSwitch(kTestServerShardFlag)) {
    std::string shard_str(CommandLine::ForCurrentProcess()->GetSwitchValueASCII(
                              kTestServerShardFlag));
    int shard = -1;
    if (base::StringToInt(shard_str, &shard)) {
      port += shard;
    } else {
      LOG(FATAL) << "Got invalid " << kTestServerShardFlag << " flag value. "
                 << "An integer is expected.";
    }
  }
  return port;
}

std::string GetHostname(net::TestServer::Type type) {
  if (type == net::TestServer::TYPE_HTTPS_MISMATCHED_HOSTNAME) {
    // Return a different hostname string that resolves to the same hostname.
    return "localhost";
  }

  return "127.0.0.1";
}

}  // namespace

namespace net {

#if defined(OS_MACOSX)
void SetMacTestCertificate(X509Certificate* cert);
#endif

TestServer::TestServer(Type type, const FilePath& document_root)
    : host_port_pair_(GetHostname(type), GetPort(type)),
      process_handle_(base::kNullProcessHandle),
      type_(type) {
  FilePath src_dir;
  PathService::Get(base::DIR_SOURCE_ROOT, &src_dir);

  document_root_ = src_dir.Append(document_root);

  certificates_dir_ = src_dir.Append(FILE_PATH_LITERAL("net"))
                       .Append(FILE_PATH_LITERAL("data"))
                       .Append(FILE_PATH_LITERAL("ssl"))
                       .Append(FILE_PATH_LITERAL("certificates"));
}

TestServer::~TestServer() {
#if defined(OS_MACOSX)
  SetMacTestCertificate(NULL);
#endif
  Stop();
}

bool TestServer::Start() {
  if (GetScheme() == "https") {
    if (!LoadTestRootCert())
      return false;
    if (!CheckCATrusted())
      return false;
  }

  // Get path to python server script
  FilePath testserver_path;
  if (!PathService::Get(base::DIR_SOURCE_ROOT, &testserver_path)) {
    LOG(ERROR) << "Failed to get DIR_SOURCE_ROOT";
    return false;
  }
  testserver_path = testserver_path
      .Append(FILE_PATH_LITERAL("net"))
      .Append(FILE_PATH_LITERAL("tools"))
      .Append(FILE_PATH_LITERAL("testserver"))
      .Append(FILE_PATH_LITERAL("testserver.py"));

  if (!SetPythonPath())
    return false;

  if (!LaunchPython(testserver_path))
    return false;

  if (!WaitToStart()) {
    Stop();
    return false;
  }

  return true;
}

bool TestServer::Stop() {
  if (!process_handle_)
    return true;

  // First check if the process has already terminated.
  bool ret = base::WaitForSingleProcess(process_handle_, 0);
  if (!ret)
    ret = base::KillProcess(process_handle_, 1, true);

  if (ret) {
    base::CloseProcessHandle(process_handle_);
    process_handle_ = base::kNullProcessHandle;
  } else {
    LOG(INFO) << "Kill failed?";
  }

  return ret;
}

std::string TestServer::GetScheme() const {
  switch (type_) {
    case TYPE_FTP:
      return "ftp";
    case TYPE_HTTP:
      return "http";
    case TYPE_HTTPS:
    case TYPE_HTTPS_CLIENT_AUTH:
    case TYPE_HTTPS_MISMATCHED_HOSTNAME:
    case TYPE_HTTPS_EXPIRED_CERTIFICATE:
      return "https";
    default:
      NOTREACHED();
  }
  return std::string();
}

bool TestServer::GetAddressList(AddressList* address_list) const {
  DCHECK(address_list);

  scoped_ptr<HostResolver> resolver(
      CreateSystemHostResolver(HostResolver::kDefaultParallelism, NULL));
  HostResolver::RequestInfo info(host_port_pair_);
  int rv = resolver->Resolve(info, address_list, NULL, NULL, BoundNetLog());
  if (rv != net::OK) {
    LOG(ERROR) << "Failed to resolve hostname: " << host_port_pair_.host();
    return false;
  }
  return true;
}

GURL TestServer::GetURL(const std::string& path) {
  return GURL(GetScheme() + "://" + host_port_pair_.ToString() +
              "/" + path);
}

GURL TestServer::GetURLWithUser(const std::string& path,
                                const std::string& user) {
  return GURL(GetScheme() + "://" + user + "@" +
              host_port_pair_.ToString() +
              "/" + path);
}

GURL TestServer::GetURLWithUserAndPassword(const std::string& path,
                                           const std::string& user,
                                           const std::string& password) {
  return GURL(GetScheme() + "://" + user + ":" + password +
              "@" + host_port_pair_.ToString() +
              "/" + path);
}

bool TestServer::SetPythonPath() {
  FilePath third_party_dir;
  if (!PathService::Get(base::DIR_SOURCE_ROOT, &third_party_dir)) {
    LOG(ERROR) << "Failed to get DIR_SOURCE_ROOT";
    return false;
  }
  third_party_dir = third_party_dir.Append(FILE_PATH_LITERAL("third_party"));

  AppendToPythonPath(third_party_dir.Append(FILE_PATH_LITERAL("tlslite")));
  AppendToPythonPath(third_party_dir.Append(FILE_PATH_LITERAL("pyftpdlib")));

  // Locate the Python code generated by the protocol buffers compiler.
  FilePath generated_code_dir;
  if (!PathService::Get(base::DIR_EXE, &generated_code_dir)) {
    LOG(ERROR) << "Failed to get DIR_EXE";
    return false;
  }
  generated_code_dir = generated_code_dir.Append(FILE_PATH_LITERAL("pyproto"));
  AppendToPythonPath(generated_code_dir);
  AppendToPythonPath(generated_code_dir.Append(FILE_PATH_LITERAL("sync_pb")));

  return true;
}

FilePath TestServer::GetRootCertificatePath() {
  return certificates_dir_.AppendASCII("root_ca_cert.crt");
}

bool TestServer::LoadTestRootCert() {
#if defined(USE_NSS)
  if (cert_)
    return true;

  // TODO(dkegel): figure out how to get this to only happen once?

  // This currently leaks a little memory.
  // TODO(dkegel): fix the leak and remove the entry in
  // tools/valgrind/memcheck/suppressions.txt
  ANNOTATE_SCOPED_MEMORY_LEAK;  // Tell heap checker about the leak.
  cert_ = LoadTemporaryRootCert(GetRootCertificatePath());
  return (cert_ != NULL);
#elif defined(OS_MACOSX)
  X509Certificate* cert = LoadTemporaryRootCert(GetRootCertificatePath());
  if (!cert)
    return false;
  SetMacTestCertificate(cert);
  return true;
#else
  return true;
#endif
}

FilePath TestServer::GetCertificatePath() {
  switch (type_) {
    case TYPE_FTP:
    case TYPE_HTTP:
      return FilePath();
    case TYPE_HTTPS:
    case TYPE_HTTPS_CLIENT_AUTH:
    case TYPE_HTTPS_MISMATCHED_HOSTNAME:
      return certificates_dir_.AppendASCII("ok_cert.pem");
    case TYPE_HTTPS_EXPIRED_CERTIFICATE:
      return certificates_dir_.AppendASCII("expired_cert.pem");
    default:
      NOTREACHED();
  }
  return FilePath();
}

}  // namespace net
<commit_msg>testserver: try harder to find the pyproto directory on OS X<commit_after>// Copyright (c) 2010 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

#include "net/test/test_server.h"

#include <algorithm>
#include <string>
#include <vector>

#include "build/build_config.h"

#if defined(OS_MACOSX)
#include "net/base/x509_certificate.h"
#endif

#include "base/command_line.h"
#include "base/file_util.h"
#include "base/leak_annotations.h"
#include "base/logging.h"
#include "base/path_service.h"
#include "base/string_number_conversions.h"
#include "base/utf_string_conversions.h"
#include "googleurl/src/gurl.h"
#include "net/base/cert_test_util.h"
#include "net/base/host_port_pair.h"
#include "net/base/host_resolver.h"
#include "net/base/test_completion_callback.h"
#include "net/socket/tcp_client_socket.h"
#include "net/test/python_utils.h"
#include "testing/platform_test.h"

namespace {

// Number of connection attempts for tests.
const int kServerConnectionAttempts = 10;

// Connection timeout in milliseconds for tests.
const int kServerConnectionTimeoutMs = 1000;

const char kTestServerShardFlag[] = "test-server-shard";

int GetPortBase(net::TestServer::Type type) {
  switch (type) {
    case net::TestServer::TYPE_FTP:
      return 3117;
    case net::TestServer::TYPE_HTTP:
      return 1337;
    case net::TestServer::TYPE_HTTPS:
      return 9443;
    case net::TestServer::TYPE_HTTPS_CLIENT_AUTH:
      return 9543;
    case net::TestServer::TYPE_HTTPS_EXPIRED_CERTIFICATE:
      // TODO(phajdan.jr): Some tests rely on this hardcoded value.
      // Some uses of this are actually in .html/.js files.
      return 9666;
    case net::TestServer::TYPE_HTTPS_MISMATCHED_HOSTNAME:
      return 9643;
    default:
      NOTREACHED();
  }
  return -1;
}

int GetPort(net::TestServer::Type type) {
  int port = GetPortBase(type);
  if (CommandLine::ForCurrentProcess()->HasSwitch(kTestServerShardFlag)) {
    std::string shard_str(CommandLine::ForCurrentProcess()->GetSwitchValueASCII(
                              kTestServerShardFlag));
    int shard = -1;
    if (base::StringToInt(shard_str, &shard)) {
      port += shard;
    } else {
      LOG(FATAL) << "Got invalid " << kTestServerShardFlag << " flag value. "
                 << "An integer is expected.";
    }
  }
  return port;
}

std::string GetHostname(net::TestServer::Type type) {
  if (type == net::TestServer::TYPE_HTTPS_MISMATCHED_HOSTNAME) {
    // Return a different hostname string that resolves to the same hostname.
    return "localhost";
  }

  return "127.0.0.1";
}

}  // namespace

namespace net {

#if defined(OS_MACOSX)
void SetMacTestCertificate(X509Certificate* cert);
#endif

TestServer::TestServer(Type type, const FilePath& document_root)
    : host_port_pair_(GetHostname(type), GetPort(type)),
      process_handle_(base::kNullProcessHandle),
      type_(type) {
  FilePath src_dir;
  PathService::Get(base::DIR_SOURCE_ROOT, &src_dir);

  document_root_ = src_dir.Append(document_root);

  certificates_dir_ = src_dir.Append(FILE_PATH_LITERAL("net"))
                       .Append(FILE_PATH_LITERAL("data"))
                       .Append(FILE_PATH_LITERAL("ssl"))
                       .Append(FILE_PATH_LITERAL("certificates"));
}

TestServer::~TestServer() {
#if defined(OS_MACOSX)
  SetMacTestCertificate(NULL);
#endif
  Stop();
}

bool TestServer::Start() {
  if (GetScheme() == "https") {
    if (!LoadTestRootCert())
      return false;
    if (!CheckCATrusted())
      return false;
  }

  // Get path to python server script
  FilePath testserver_path;
  if (!PathService::Get(base::DIR_SOURCE_ROOT, &testserver_path)) {
    LOG(ERROR) << "Failed to get DIR_SOURCE_ROOT";
    return false;
  }
  testserver_path = testserver_path
      .Append(FILE_PATH_LITERAL("net"))
      .Append(FILE_PATH_LITERAL("tools"))
      .Append(FILE_PATH_LITERAL("testserver"))
      .Append(FILE_PATH_LITERAL("testserver.py"));

  if (!SetPythonPath())
    return false;

  if (!LaunchPython(testserver_path))
    return false;

  if (!WaitToStart()) {
    Stop();
    return false;
  }

  return true;
}

bool TestServer::Stop() {
  if (!process_handle_)
    return true;

  // First check if the process has already terminated.
  bool ret = base::WaitForSingleProcess(process_handle_, 0);
  if (!ret)
    ret = base::KillProcess(process_handle_, 1, true);

  if (ret) {
    base::CloseProcessHandle(process_handle_);
    process_handle_ = base::kNullProcessHandle;
  } else {
    LOG(INFO) << "Kill failed?";
  }

  return ret;
}

std::string TestServer::GetScheme() const {
  switch (type_) {
    case TYPE_FTP:
      return "ftp";
    case TYPE_HTTP:
      return "http";
    case TYPE_HTTPS:
    case TYPE_HTTPS_CLIENT_AUTH:
    case TYPE_HTTPS_MISMATCHED_HOSTNAME:
    case TYPE_HTTPS_EXPIRED_CERTIFICATE:
      return "https";
    default:
      NOTREACHED();
  }
  return std::string();
}

bool TestServer::GetAddressList(AddressList* address_list) const {
  DCHECK(address_list);

  scoped_ptr<HostResolver> resolver(
      CreateSystemHostResolver(HostResolver::kDefaultParallelism, NULL));
  HostResolver::RequestInfo info(host_port_pair_);
  int rv = resolver->Resolve(info, address_list, NULL, NULL, BoundNetLog());
  if (rv != net::OK) {
    LOG(ERROR) << "Failed to resolve hostname: " << host_port_pair_.host();
    return false;
  }
  return true;
}

GURL TestServer::GetURL(const std::string& path) {
  return GURL(GetScheme() + "://" + host_port_pair_.ToString() +
              "/" + path);
}

GURL TestServer::GetURLWithUser(const std::string& path,
                                const std::string& user) {
  return GURL(GetScheme() + "://" + user + "@" +
              host_port_pair_.ToString() +
              "/" + path);
}

GURL TestServer::GetURLWithUserAndPassword(const std::string& path,
                                           const std::string& user,
                                           const std::string& password) {
  return GURL(GetScheme() + "://" + user + ":" + password +
              "@" + host_port_pair_.ToString() +
              "/" + path);
}

bool TestServer::SetPythonPath() {
  FilePath third_party_dir;
  if (!PathService::Get(base::DIR_SOURCE_ROOT, &third_party_dir)) {
    LOG(ERROR) << "Failed to get DIR_SOURCE_ROOT";
    return false;
  }
  third_party_dir = third_party_dir.Append(FILE_PATH_LITERAL("third_party"));

  AppendToPythonPath(third_party_dir.Append(FILE_PATH_LITERAL("tlslite")));
  AppendToPythonPath(third_party_dir.Append(FILE_PATH_LITERAL("pyftpdlib")));

  // Locate the Python code generated by the protocol buffers compiler.
  FilePath generated_code_dir;
  if (!PathService::Get(base::DIR_EXE, &generated_code_dir)) {
    LOG(ERROR) << "Failed to get DIR_EXE";
    return false;
  }

  static const FilePath kPyProto(FILE_PATH_LITERAL("pyproto"));

#if defined(OS_MACOSX)
  // On Mac, DIR_EXE might be pointing deep into the Release/ (or Debug/)
  // directory and we can't depend on how far down it goes. So we walk upwards
  // from DIR_EXE until we find a likely looking spot.
  while (!file_util::DirectoryExists(generated_code_dir.Append(kPyProto))) {
    FilePath parent = generated_code_dir.DirName();
    if (parent == generated_code_dir) {
      // We hit the root directory. Maybe we didn't build any targets which
      // produced Python protocol buffers.
      PathService::Get(base::DIR_EXE, &generated_code_dir);
      break;
    }
    generated_code_dir = parent;
  }
#endif

  AppendToPythonPath(generated_code_dir.Append(kPyProto));
  AppendToPythonPath(generated_code_dir.Append(kPyProto).
                     Append(FILE_PATH_LITERAL("sync_pb")));

  return true;
}

FilePath TestServer::GetRootCertificatePath() {
  return certificates_dir_.AppendASCII("root_ca_cert.crt");
}

bool TestServer::LoadTestRootCert() {
#if defined(USE_NSS)
  if (cert_)
    return true;

  // TODO(dkegel): figure out how to get this to only happen once?

  // This currently leaks a little memory.
  // TODO(dkegel): fix the leak and remove the entry in
  // tools/valgrind/memcheck/suppressions.txt
  ANNOTATE_SCOPED_MEMORY_LEAK;  // Tell heap checker about the leak.
  cert_ = LoadTemporaryRootCert(GetRootCertificatePath());
  return (cert_ != NULL);
#elif defined(OS_MACOSX)
  X509Certificate* cert = LoadTemporaryRootCert(GetRootCertificatePath());
  if (!cert)
    return false;
  SetMacTestCertificate(cert);
  return true;
#else
  return true;
#endif
}

FilePath TestServer::GetCertificatePath() {
  switch (type_) {
    case TYPE_FTP:
    case TYPE_HTTP:
      return FilePath();
    case TYPE_HTTPS:
    case TYPE_HTTPS_CLIENT_AUTH:
    case TYPE_HTTPS_MISMATCHED_HOSTNAME:
      return certificates_dir_.AppendASCII("ok_cert.pem");
    case TYPE_HTTPS_EXPIRED_CERTIFICATE:
      return certificates_dir_.AppendASCII("expired_cert.pem");
    default:
      NOTREACHED();
  }
  return FilePath();
}

}  // namespace net
<|endoftext|>


<commit_before>#pragma once
//=========================================================================//
/*! @file
    @brief  UDP/TCP 共通クラス @n
			Copyright 2017 Kunihito Hiramatsu
    @author 平松邦仁 (hira@rvf-rc45.net)
*/
//=========================================================================//
#include "common/fixed_block.hpp"
#include "net2/net_st.hpp"
#include "net2/memory.hpp"

#define UDP_TCP_DEBUG

namespace net {

	//+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++//
	/*!
		@brief  UDP/TCP 共通テンプレート
		@param[in]	CTX	コンテキスト
		@param[in]	NMAX	管理最大数
	*/
	//+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++//
	template<class CTX, uint32_t NMAX>
	class udp_tcp_common {

#ifndef UDP_TCP_DEBUG
		typedef utils::null_format debug_format;
#else
		typedef utils::format debug_format;
#endif

		typedef utils::fixed_block<CTX, NMAX> BLOCKS;
		BLOCKS	blocks_;

	public:
		//-----------------------------------------------------------------//
		/*!
			@brief  コンストラクター
		*/
		//-----------------------------------------------------------------//
		udp_tcp_common() noexcept : blocks_() { }

		//-----------------------------------------------------------------//
		/*!
			@brief  MAC アドレス取得の検査
			@param[in]	ctx		コンテキスト
			@param[in]	info	ネット情報
			@return MAC アドレス取得済みなら「true」
		*/
		//-----------------------------------------------------------------//
		bool check_mac(CTX& ctx, net_info& info) noexcept
		{
			const auto& cash = info.get_cash();
			auto idx = cash.lookup(ctx.adrs_);
			if(cash.is_valid(idx)) {
				std::memcpy(ctx.mac_, cash[idx].mac, 6);
				debug_format("UDP/TCP MAC lookup: %s at %s\n")
					% ctx.adrs_.c_str()
					% tools::mac_str(cash[idx].mac);
				return true;
			}
			return false;
		}

		//-----------------------------------------------------------------//
		/*!
			@brief  コンテキスト・ブロックの参照
			@return コンテキスト・ブロック
		*/
		//-----------------------------------------------------------------//
		inline BLOCKS& at_blocks() noexcept
		{
			return blocks_;
		}

		//-----------------------------------------------------------------//
		/*!
			@brief  送信
			@param[in]	desc	ディスクリプタ
			@param[in]	src		ソース
			@param[in]	len		送信バイト数
			@return 送信バイト（負の値はエラー）
		*/
		//-----------------------------------------------------------------//
		int send(int desc, const void* src, uint16_t len) noexcept
		{
			uint32_t idx = static_cast<uint32_t>(desc);
			if(!blocks_.is_alloc(idx)) return -1;
			if(blocks_.is_lock(idx)) return -1;

			CTX& ctx = blocks_.at(idx);
			uint16_t spc = ctx.send_.size() - ctx.send_.length() - 1;
			if(spc < len) {
				len = spc;
			}
			ctx.send_.put(src, len);

			return len;
		}

		//-----------------------------------------------------------------//
		/*!
			@brief  送信バッファの残量取得
			@param[in]	desc	ディスクリプタ
			@return 送信バッファの残量（負の値はエラー）
		*/
		//-----------------------------------------------------------------//
		int get_send_length(int desc) const noexcept
		{
			uint32_t idx = static_cast<uint32_t>(desc);
			if(!blocks_.is_alloc(idx)) return -1;
			if(blocks_.is_lock(idx)) return -1;

			CTX& ctx = blocks_.at(idx);
			return ctx.send_.length();
		}

		//-----------------------------------------------------------------//
		/*!
			@brief  受信
			@param[in]	desc	ディスクリプタ
			@param[in]	dst		ソース
			@param[in]	len		受信バイト数
			@return 受信バイト（負の値はエラー）
		*/
		//-----------------------------------------------------------------//
		int recv(int desc, void* dst, uint16_t len) noexcept
		{
			uint32_t idx = static_cast<uint32_t>(desc);
			if(!blocks_.is_alloc(idx)) return -1;
			if(blocks_.is_lock(idx)) return -1;

			CTX& ctx = blocks_.at(idx);
			int rlen = ctx.recv_.length();
			if(rlen == 0) {  // 読み込むデータが無い
				return rlen;
			}

			if(rlen > len) {
				rlen = len;
			}
			ctx.recv_.get(dst, rlen);
			return rlen;
		}

		//-----------------------------------------------------------------//
		/*!
			@brief  受信バッファの残量取得
			@param[in]	desc	ディスクリプタ
			@return 受信バッファの残量（負の値はエラー）
		*/
		//-----------------------------------------------------------------//
		int get_recv_length(int desc) const noexcept
		{
			uint32_t idx = static_cast<uint32_t>(desc);
			if(!blocks_.is_alloc(idx)) return -1;
			if(blocks_.is_lock(idx)) return -1;

			CTX& ctx = blocks_.at(idx);
			return ctx.recv_.length();
		}
	};
}
<commit_msg>fix const<commit_after>#pragma once
//=========================================================================//
/*! @file
    @brief  UDP/TCP 共通クラス @n
			Copyright 2017 Kunihito Hiramatsu
    @author 平松邦仁 (hira@rvf-rc45.net)
*/
//=========================================================================//
#include "common/fixed_block.hpp"
#include "net2/net_st.hpp"
#include "net2/memory.hpp"

#define UDP_TCP_DEBUG

namespace net {

	//+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++//
	/*!
		@brief  UDP/TCP 共通テンプレート
		@param[in]	CTX	コンテキスト
		@param[in]	NMAX	管理最大数
	*/
	//+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++//
	template<class CTX, uint32_t NMAX>
	class udp_tcp_common {

#ifndef UDP_TCP_DEBUG
		typedef utils::null_format debug_format;
#else
		typedef utils::format debug_format;
#endif

		typedef utils::fixed_block<CTX, NMAX> BLOCKS;
		BLOCKS	blocks_;

	public:
		//-----------------------------------------------------------------//
		/*!
			@brief  コンストラクター
		*/
		//-----------------------------------------------------------------//
		udp_tcp_common() noexcept : blocks_() { }

		//-----------------------------------------------------------------//
		/*!
			@brief  MAC アドレス取得の検査
			@param[in]	ctx		コンテキスト
			@param[in]	info	ネット情報
			@return MAC アドレス取得済みなら「true」
		*/
		//-----------------------------------------------------------------//
		bool check_mac(CTX& ctx, net_info& info) noexcept
		{
			const auto& cash = info.get_cash();
			auto idx = cash.lookup(ctx.adrs_);
			if(cash.is_valid(idx)) {
				std::memcpy(ctx.mac_, cash[idx].mac, 6);
				debug_format("UDP/TCP MAC lookup: %s at %s\n")
					% ctx.adrs_.c_str()
					% tools::mac_str(cash[idx].mac);
				return true;
			}
			return false;
		}

		//-----------------------------------------------------------------//
		/*!
			@brief  コンテキスト・ブロックの参照
			@return コンテキスト・ブロック
		*/
		//-----------------------------------------------------------------//
		inline BLOCKS& at_blocks() noexcept
		{
			return blocks_;
		}

		//-----------------------------------------------------------------//
		/*!
			@brief  送信
			@param[in]	desc	ディスクリプタ
			@param[in]	src		ソース
			@param[in]	len		送信バイト数
			@return 送信バイト（負の値はエラー）
		*/
		//-----------------------------------------------------------------//
		int send(int desc, const void* src, uint16_t len) noexcept
		{
			uint32_t idx = static_cast<uint32_t>(desc);
			if(!blocks_.is_alloc(idx)) return -1;
			if(blocks_.is_lock(idx)) return -1;

			CTX& ctx = blocks_.at(idx);
			uint16_t spc = ctx.send_.size() - ctx.send_.length() - 1;
			if(spc < len) {
				len = spc;
			}
			ctx.send_.put(src, len);

			return len;
		}

		//-----------------------------------------------------------------//
		/*!
			@brief  送信バッファの残量取得
			@param[in]	desc	ディスクリプタ
			@return 送信バッファの残量（負の値はエラー）
		*/
		//-----------------------------------------------------------------//
		int get_send_length(int desc) const noexcept
		{
			uint32_t idx = static_cast<uint32_t>(desc);
			if(!blocks_.is_alloc(idx)) return -1;
			if(blocks_.is_lock(idx)) return -1;

			const CTX& ctx = blocks_.get(idx);
			return ctx.send_.length();
		}

		//-----------------------------------------------------------------//
		/*!
			@brief  受信
			@param[in]	desc	ディスクリプタ
			@param[in]	dst		ソース
			@param[in]	len		受信バイト数
			@return 受信バイト（負の値はエラー）
		*/
		//-----------------------------------------------------------------//
		int recv(int desc, void* dst, uint16_t len) noexcept
		{
			uint32_t idx = static_cast<uint32_t>(desc);
			if(!blocks_.is_alloc(idx)) return -1;
			if(blocks_.is_lock(idx)) return -1;

			CTX& ctx = blocks_.at(idx);
			int rlen = ctx.recv_.length();
			if(rlen == 0) {  // 読み込むデータが無い
				return rlen;
			}

			if(rlen > len) {
				rlen = len;
			}
			ctx.recv_.get(dst, rlen);
			return rlen;
		}

		//-----------------------------------------------------------------//
		/*!
			@brief  受信バッファの残量取得
			@param[in]	desc	ディスクリプタ
			@return 受信バッファの残量（負の値はエラー）
		*/
		//-----------------------------------------------------------------//
		int get_recv_length(int desc) const noexcept
		{
			uint32_t idx = static_cast<uint32_t>(desc);
			if(!blocks_.is_alloc(idx)) return -1;
			if(blocks_.is_lock(idx)) return -1;

			const CTX& ctx = blocks_.get(idx);
			return ctx.recv_.length();
		}
	};
}
<|endoftext|>


<commit_before>#include "hiconfig.h"
#include "hpoolserver.h"

using namespace std;

class PoolException : public std::exception
{
	std::string m_message;
	public:

	PoolException(std::string message) {
		m_message = message;
	}

	std::string what() {
		return m_message;
	}

	~PoolException() throw() {
		m_message.clear();
	}
};

hPoolServer::Connection::Connection(std::string _ip, int _port, int _sock,
			boost::function<void(int)> _onClose,
			boost::function<void(int)> _onSetWrite):
		create_ts(time(0)),
		ip(_ip),
		port(_port),
		m_sock(_sock),
		readmode(true),
		closing(false),
		m_onClose(_onClose),
		m_onSetWrite(_onSetWrite) {
}

hPoolServer::Connection::~Connection() {
	if (!closing)
		close();
}

uint64_t hPoolServer::Connection::getCreateTs() {
	return create_ts;
}

void hPoolServer::Connection::close() {
	std::cout << "pool connection closing\n";
	closing = true;
	m_onClose(m_sock);
}

void hPoolServer::Connection::setWriteMode() {

	if (readmode) {
		m_onSetWrite(m_sock);
		readmode = false;
	}
}

hPoolServer::hPoolServer(TaskLauncherPtr launcher,
					boost::function<void(ConnectionPtr)> _onRead,
					boost::function<void(ConnectionPtr)> _onWrite,
					boost::function<void(ConnectionPtr)> _onError) {
	m_launcher = launcher;
	m_onRead = _onRead;
	m_onWrite = _onWrite;
	m_onError = _onError;

	m_istopped = 0;
	m_events_watcher.reset(new EventWatcher(
			boost::bind(&hPoolServer::onRead, this, _1, _2),
			boost::bind(&hPoolServer::onWrite, this, _1, _2),
			boost::bind(&hPoolServer::onError, this, _1, _2),
			boost::bind(&hPoolServer::onAccept, this, _1, _2)));
}

hPoolServer::~hPoolServer() {
	stop();
}

void hPoolServer::onCloseConnection(int _sock_fd) {
	{
		hLockTicketPtr ticket = m_connections_lock.lock();

		hiaux::hashtable<int, ConnectionPtr>::iterator it = m_connections.find(_sock_fd);
		if (it != m_connections.end())
			m_connections.erase(it);
	}
	hLockTicketPtr _ticket = m_sockets_to_close_q_lock.lock();
	m_sockets_to_close_q.push(_sock_fd);

	//m_events_watcher->delSocket(_sock_fd, NULL);
}

void hPoolServer::onRead(int _sock_fd, void *_opaque_info) {

	hLockTicketPtr ticket = m_connections_lock.lock();
	hiaux::hashtable<int, ConnectionPtr>::iterator it = m_connections.find(_sock_fd);
	if (it != m_connections.end()) {
		ticket->unlock();
		m_onRead(it->second);
	}
}

/*
TaskLauncher::TaskRet hPoolServer::handleReadTask(ConnectionPtr _conn) {
	m_handler(_conn);
	return TaskLauncher::NO_RELAUNCH;
}*/

void hPoolServer::onWrite(int _sock_fd, void *_opaque_info) {

	hLockTicketPtr ticket = m_connections_lock.lock();
	hiaux::hashtable<int, ConnectionPtr>::iterator it = m_connections.find(_sock_fd);
	if (it != m_connections.end()) {
		ticket->unlock();
		m_onWrite(it->second);
	}
}

void hPoolServer::onError(int _sock_fd, void *_opaque_info) {

	hLockTicketPtr ticket = m_connections_lock.lock();
	hiaux::hashtable<int, ConnectionPtr>::iterator it = m_connections.find(_sock_fd);
	if (it != m_connections.end()) {
		ticket->unlock();
		m_onError(it->second);
	}
}

void hPoolServer::onSetWrite(int _sock) {

//	m_events_watcher->delSocket(sock_fd, NULL);
//	m_events_watcher->delSocket(sock_fd, NULL);

}

TaskLauncher::TaskRet hPoolServer::readThread() {

	while (m_isrunning) {
		//std::cout << "hPoolServer::readThread handleEvents\n";
		m_events_watcher->handleEvents();
		//std::cout << "hPoolServer::readThread handleEvents finished\n";

		{
			hLockTicketPtr ticket = m_sockets_to_close_q_lock.lock();
			while (!m_sockets_to_close_q.empty()) {
				std::cout << "removing socket from m_events_watcher\n";
				int sock_fd = m_sockets_to_close_q.front();
				m_events_watcher->delSocket(sock_fd, NULL);
				std::cout << "removed socket from m_events_watcher\n";
				::close(sock_fd);
				::shutdown(sock_fd, SHUT_RDWR);
				m_sockets_to_close_q.pop();
			}
		}
	}
	m_istopped.fetch_add(1);
	return TaskLauncher::NO_RELAUNCH;
}

hPoolServer::ConnectionPtr hPoolServer::getConnection(int _fd) {

	hLockTicketPtr ticket = m_connections_lock.lock();

	hiaux::hashtable<int, ConnectionPtr>::iterator it = m_connections.find(_fd);
	if (it != m_connections.end())
		return it->second;
	std::cout << "hPoolServer::getConnection no such connection\n";
	return hPoolServer::ConnectionPtr();
}

void hPoolServer::onAccept(int _sock_fd, void *_opaque_info) {

	struct sockaddr_in cli_addr;
	size_t clilen = sizeof(cli_addr);
	int accepted_socket = accept(_sock_fd,
			 (struct sockaddr *) &cli_addr,
			 (socklen_t*)&clilen);

	if (accepted_socket < 0)
		return;

	setSocketNonBlock(accepted_socket);

	ConnectionPtr connection(new Connection(inet_ntoa(cli_addr.sin_addr),
							cli_addr.sin_port,
							accepted_socket,
							boost::bind(&hPoolServer::onCloseConnection, this, _1),
							boost::bind(&hPoolServer::onSetWrite, this, _1)));

	{
		hLockTicketPtr ticket = m_connections_lock.lock();
		m_connections.insert(std::pair<int, ConnectionPtr>(connection->m_sock,
													connection) );
	}

	m_events_watcher->addSocketRead(connection->m_sock, NULL);
}

void hPoolServer::start(int port) {
	m_isrunning = true;
	m_listen_socket = startServer(port);
	m_events_watcher->addSocketAccept(m_listen_socket, NULL);
	m_launcher->addTask(new boost::function<TaskLauncher::TaskRet()> (
			boost::bind(&hPoolServer::readThread, this)));
}

void hPoolServer::stop() {
    shutdown(m_listen_socket, SHUT_RDWR);
    m_isrunning = 0;

	while (m_istopped.load() == 0) {
	}
}

void hPoolServer::setSocketNonBlock(int _fd) {

	int flags = fcntl(_fd, F_GETFL);
	fcntl(_fd, F_SETFL, flags | O_NONBLOCK);
}

int hPoolServer::startServer(int port)
{
	char bf[255];
	struct sockaddr_in serv_addr;

	int sockfd = socket(AF_INET, SOCK_STREAM, 0);
	setSocketNonBlock(sockfd);

	if (sockfd < 0)
			throw new std::string("hsock_t::server: ERROR opening server socket");
	serv_addr.sin_family = AF_INET;
	serv_addr.sin_port = htons(port);
	serv_addr.sin_addr.s_addr = INADDR_ANY;

	int yes = 1;
	if (setsockopt(sockfd, SOL_SOCKET, SO_REUSEADDR, &yes, sizeof(yes)) == -1)  {
			perror("setsockopt");
			exit(1);
	}

	if (setsockopt(sockfd, IPPROTO_TCP, TCP_NODELAY, &yes, sizeof(yes)) == -1)  {
			perror("setsockopt");
			exit(1);
	}

	if (bind(sockfd, (struct sockaddr *) &serv_addr, sizeof(serv_addr))<0)
	throw new std::string("hsock_t::server: Error server binding");

	listen(sockfd, POOLSERVER_SOCKET_CONN_QUEUE_SIZE);
	return sockfd;
}

int hPoolServer::startClient(const std::string &_ip, int portno)
{
	int sockfd;
	struct sockaddr_in serv_addr;
	struct hostent *server = gethostbyname(_ip.c_str());

	sockfd = socket(AF_INET, SOCK_STREAM, 0);

	if (sockfd < 0)
			throw new std::string("hsock_t::client: ERROR opening client socket");

	int yes = 1;
	if (setsockopt(sockfd, IPPROTO_TCP, TCP_NODELAY, &yes, sizeof(yes)) == -1)  {
	}

	memcpy(&serv_addr.sin_addr.s_addr, server->h_addr, server->h_length);
	serv_addr.sin_family = AF_INET;
	serv_addr.sin_port = htons(portno);

	if (connect(sockfd, (struct sockaddr *) &serv_addr, sizeof(serv_addr)))
			throw new std::string("hsock_t::client: ERROR connecting");
	return sockfd;
}
<commit_msg>remove fcntl<commit_after>#include "hiconfig.h"
#include "hpoolserver.h"

using namespace std;

class PoolException : public std::exception
{
	std::string m_message;
	public:

	PoolException(std::string message) {
		m_message = message;
	}

	std::string what() {
		return m_message;
	}

	~PoolException() throw() {
		m_message.clear();
	}
};

hPoolServer::Connection::Connection(std::string _ip, int _port, int _sock,
			boost::function<void(int)> _onClose,
			boost::function<void(int)> _onSetWrite):
		create_ts(time(0)),
		ip(_ip),
		port(_port),
		m_sock(_sock),
		readmode(true),
		closing(false),
		m_onClose(_onClose),
		m_onSetWrite(_onSetWrite) {
}

hPoolServer::Connection::~Connection() {
	if (!closing)
		close();
}

uint64_t hPoolServer::Connection::getCreateTs() {
	return create_ts;
}

void hPoolServer::Connection::close() {
	std::cout << "pool connection closing\n";
	closing = true;
	m_onClose(m_sock);
}

void hPoolServer::Connection::setWriteMode() {

	if (readmode) {
		m_onSetWrite(m_sock);
		readmode = false;
	}
}

hPoolServer::hPoolServer(TaskLauncherPtr launcher,
					boost::function<void(ConnectionPtr)> _onRead,
					boost::function<void(ConnectionPtr)> _onWrite,
					boost::function<void(ConnectionPtr)> _onError) {
	m_launcher = launcher;
	m_onRead = _onRead;
	m_onWrite = _onWrite;
	m_onError = _onError;

	m_istopped = 0;
	m_events_watcher.reset(new EventWatcher(
			boost::bind(&hPoolServer::onRead, this, _1, _2),
			boost::bind(&hPoolServer::onWrite, this, _1, _2),
			boost::bind(&hPoolServer::onError, this, _1, _2),
			boost::bind(&hPoolServer::onAccept, this, _1, _2)));
}

hPoolServer::~hPoolServer() {
	stop();
}

void hPoolServer::onCloseConnection(int _sock_fd) {
	{
		hLockTicketPtr ticket = m_connections_lock.lock();

		hiaux::hashtable<int, ConnectionPtr>::iterator it = m_connections.find(_sock_fd);
		if (it != m_connections.end())
			m_connections.erase(it);
	}
	hLockTicketPtr _ticket = m_sockets_to_close_q_lock.lock();
	m_sockets_to_close_q.push(_sock_fd);

	//m_events_watcher->delSocket(_sock_fd, NULL);
}

void hPoolServer::onRead(int _sock_fd, void *_opaque_info) {

	hLockTicketPtr ticket = m_connections_lock.lock();
	hiaux::hashtable<int, ConnectionPtr>::iterator it = m_connections.find(_sock_fd);
	if (it != m_connections.end()) {
		ticket->unlock();
		m_onRead(it->second);
	}
}

/*
TaskLauncher::TaskRet hPoolServer::handleReadTask(ConnectionPtr _conn) {
	m_handler(_conn);
	return TaskLauncher::NO_RELAUNCH;
}*/

void hPoolServer::onWrite(int _sock_fd, void *_opaque_info) {

	hLockTicketPtr ticket = m_connections_lock.lock();
	hiaux::hashtable<int, ConnectionPtr>::iterator it = m_connections.find(_sock_fd);
	if (it != m_connections.end()) {
		ticket->unlock();
		m_onWrite(it->second);
	}
}

void hPoolServer::onError(int _sock_fd, void *_opaque_info) {

	hLockTicketPtr ticket = m_connections_lock.lock();
	hiaux::hashtable<int, ConnectionPtr>::iterator it = m_connections.find(_sock_fd);
	if (it != m_connections.end()) {
		ticket->unlock();
		m_onError(it->second);
	}
}

void hPoolServer::onSetWrite(int _sock) {

//	m_events_watcher->delSocket(sock_fd, NULL);
//	m_events_watcher->delSocket(sock_fd, NULL);

}

TaskLauncher::TaskRet hPoolServer::readThread() {

	while (m_isrunning) {
		//std::cout << "hPoolServer::readThread handleEvents\n";
		m_events_watcher->handleEvents();
		//std::cout << "hPoolServer::readThread handleEvents finished\n";

		{
			hLockTicketPtr ticket = m_sockets_to_close_q_lock.lock();
			while (!m_sockets_to_close_q.empty()) {
				std::cout << "removing socket from m_events_watcher\n";
				int sock_fd = m_sockets_to_close_q.front();
				m_events_watcher->delSocket(sock_fd, NULL);
				std::cout << "removed socket from m_events_watcher\n";
				::close(sock_fd);
				::shutdown(sock_fd, SHUT_RDWR);
				m_sockets_to_close_q.pop();
			}
		}
	}
	m_istopped.fetch_add(1);
	return TaskLauncher::NO_RELAUNCH;
}

hPoolServer::ConnectionPtr hPoolServer::getConnection(int _fd) {

	hLockTicketPtr ticket = m_connections_lock.lock();

	hiaux::hashtable<int, ConnectionPtr>::iterator it = m_connections.find(_fd);
	if (it != m_connections.end())
		return it->second;
	std::cout << "hPoolServer::getConnection no such connection\n";
	return hPoolServer::ConnectionPtr();
}

void hPoolServer::onAccept(int _sock_fd, void *_opaque_info) {

	struct sockaddr_in cli_addr;
	size_t clilen = sizeof(cli_addr);
	int accepted_socket = accept(_sock_fd,
			 (struct sockaddr *) &cli_addr,
			 (socklen_t*)&clilen);

	if (accepted_socket < 0)
		return;

	setSocketNonBlock(accepted_socket);

	ConnectionPtr connection(new Connection(inet_ntoa(cli_addr.sin_addr),
							cli_addr.sin_port,
							accepted_socket,
							boost::bind(&hPoolServer::onCloseConnection, this, _1),
							boost::bind(&hPoolServer::onSetWrite, this, _1)));

	{
		hLockTicketPtr ticket = m_connections_lock.lock();
		m_connections.insert(std::pair<int, ConnectionPtr>(connection->m_sock,
													connection) );
	}

	m_events_watcher->addSocketRead(connection->m_sock, NULL);
}

void hPoolServer::start(int port) {
	m_isrunning = true;
	m_listen_socket = startServer(port);
	m_events_watcher->addSocketAccept(m_listen_socket, NULL);
	m_launcher->addTask(new boost::function<TaskLauncher::TaskRet()> (
			boost::bind(&hPoolServer::readThread, this)));
}

void hPoolServer::stop() {
    shutdown(m_listen_socket, SHUT_RDWR);
    m_isrunning = 0;

	while (m_istopped.load() == 0) {
	}
}

void hPoolServer::setSocketNonBlock(int _fd) {

//	int flags = fcntl(_fd, F_GETFL);
//	fcntl(_fd, F_SETFL, flags | O_NONBLOCK);
}

int hPoolServer::startServer(int port)
{
	char bf[255];
	struct sockaddr_in serv_addr;

	int sockfd = socket(AF_INET, SOCK_STREAM, 0);
	setSocketNonBlock(sockfd);

	if (sockfd < 0)
			throw new std::string("hsock_t::server: ERROR opening server socket");
	serv_addr.sin_family = AF_INET;
	serv_addr.sin_port = htons(port);
	serv_addr.sin_addr.s_addr = INADDR_ANY;

	int yes = 1;
	if (setsockopt(sockfd, SOL_SOCKET, SO_REUSEADDR, &yes, sizeof(yes)) == -1)  {
			perror("setsockopt");
			exit(1);
	}

	if (setsockopt(sockfd, IPPROTO_TCP, TCP_NODELAY, &yes, sizeof(yes)) == -1)  {
			perror("setsockopt");
			exit(1);
	}

	if (bind(sockfd, (struct sockaddr *) &serv_addr, sizeof(serv_addr))<0)
	throw new std::string("hsock_t::server: Error server binding");

	listen(sockfd, POOLSERVER_SOCKET_CONN_QUEUE_SIZE);
	return sockfd;
}

int hPoolServer::startClient(const std::string &_ip, int portno)
{
	int sockfd;
	struct sockaddr_in serv_addr;
	struct hostent *server = gethostbyname(_ip.c_str());

	sockfd = socket(AF_INET, SOCK_STREAM, 0);

	if (sockfd < 0)
			throw new std::string("hsock_t::client: ERROR opening client socket");

	int yes = 1;
	if (setsockopt(sockfd, IPPROTO_TCP, TCP_NODELAY, &yes, sizeof(yes)) == -1)  {
	}

	memcpy(&serv_addr.sin_addr.s_addr, server->h_addr, server->h_length);
	serv_addr.sin_family = AF_INET;
	serv_addr.sin_port = htons(portno);

	if (connect(sockfd, (struct sockaddr *) &serv_addr, sizeof(serv_addr)))
			throw new std::string("hsock_t::client: ERROR connecting");
	return sockfd;
}
<|endoftext|>


<commit_before>/**
 * Copyright (c) 2016 zScale Technology GmbH <legal@zscale.io>
 * Authors:
 *   - Paul Asmuth <paul@zscale.io>
 *   - Laura Schlimmer <laura@zscale.io>
 *
 * This program is free software: you can redistribute it and/or modify it under
 * the terms of the GNU Affero General Public License ("the license") as
 * published by the Free Software Foundation, either version 3 of the License,
 * or any later version.
 *
 * In accordance with Section 7(e) of the license, the licensing of the Program
 * under the license does not imply a trademark license. Therefore any rights,
 * title and interest in our trademarks remain entirely with us.
 *
 * This program is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
 * FOR A PARTICULAR PURPOSE. See the license for more details.
 *
 * You can be released from the requirements of the license by purchasing a
 * commercial license. Buying such a license is mandatory as soon as you develop
 * commercial activities involving this program without disclosing the source
 * code of your own applications
 */
#include <eventql/eventql.h>
#include <eventql/util/stdtypes.h>
#include <eventql/util/application.h>
#include <eventql/util/cli/flagparser.h>
#include <eventql/util/csv/BinaryCSVOutputStream.h>
#include <eventql/util/http/httpclient.h>
#include <eventql/util/json/jsonoutputstream.h>
#include "eventql/util/thread/FixedSizeThreadPool.h"
#include "eventql/util/util/SimpleRateLimit.h"
#include <eventql/util/mysql/MySQL.h>
#include <eventql/util/mysql//MySQLConnection.h>

using namespace eventql;

void run(const cli::FlagParser& flags) {
  auto source_table = flags.getString("source_table");
  auto destination_table = flags.getString("destination_table");
  //String api_url = "http://api.zscale.io/api/v1";
  auto shard_size = flags.getInt("shard_size");
  auto mysql_addr = flags.getString("mysql");
  auto host = flags.getString("host");
  auto port = flags.getInt("port");

  logInfo("evql-mysql-import", "Connecting to MySQL Server...");

  util::mysql::mysqlInit();
  auto mysql_conn = util::mysql::MySQLConnection::openConnection(URI(mysql_addr));

  logInfo(
      "mysql-upload",
      "Analyzing the input table. This might take a few minutes...");

  auto schema = mysql_conn->getTableSchema(source_table);
  logDebug("mysql-upload", "Table Schema:\n$0", schema->toString());

  auto count_rows_qry = StringUtil::format(
      "SELECT count(*) FROM `$0`;",
      source_table);

  size_t num_rows = 0;
  size_t num_rows_uploaded = 0;
  mysql_conn->executeQuery(
      count_rows_qry,
      [&num_rows] (const Vector<String>& row) -> bool {
    if (row.size() == 1) num_rows = std::stoull(row[0]);
    return false;
  });

  if (num_rows == 0) {
    logError(
        "mysql-upload",
        "Table '$0' appears to be empty",
        source_table);
    return;
  }

  auto num_shards = (num_rows + shard_size - 1) / shard_size;
  logDebug("mysql-upload", "Splitting into $0 shards", num_shards);

  http::HTTPMessage::HeaderList auth_headers;
  if (flags.isSet("auth_token")) {
    auth_headers.emplace_back(
        "Authorization",
        StringUtil::format("Token $0", flags.getString("auth_token")));
  //} else if (!cfg_.getPassword().isEmpty()) {
  //  auth_headers.emplace_back(
  //      "Authorization",
  //      StringUtil::format("Basic $0",
  //          util::Base64::encode(
  //              cfg_.getUser() + ":" + cfg_.getPassword().get())));
  }

  /* build create table request */
  //Buffer create_req;
  //{
  //  json::JSONOutputStream j(BufferOutputStream::fromBuffer(&create_req));
  //  j.beginObject();
  //  j.addObjectEntry("table_name");
  //  j.addString(destination_table);
  //  j.addComma();
  //  j.addObjectEntry("update");
  //  j.addTrue();
  //  j.addComma();
  //  j.addObjectEntry("num_shards");
  //  j.addInteger(num_shards);
  //  j.addComma();
  //  j.addObjectEntry("schema");
  //  schema->toJSON(&j);
  //  j.endObject();
  //}

  //{
  //  http::HTTPClient http_client;
  //  auto create_res = http_client.executeRequest(
  //      http::HTTPRequest::mkPost(
  //          api_url + "/tables/create_table",
  //          create_req,
  //          auth_headers));

  //  if (create_res.statusCode() != 201) {
  //    RAISE(kRuntimeError, create_res.body().toString());
  //  }
  //}

  /* status line */
  util::SimpleRateLimitedFn status_line(
      kMicrosPerSecond,
      [&num_rows_uploaded, num_rows] () {
    logInfo(
        "mysql-upload",
        "[$0%] Uploading... $1/$2 rows",
        (size_t) ((num_rows_uploaded / (double) num_rows) * 100),
        num_rows_uploaded,
        num_rows);
  });

  Vector<String> columns;
  for (const auto& field : schema->fields()) {
    columns.emplace_back(field.name);
  }

  thread::FixedSizeThreadPool tpool(thread::ThreadPoolOptions{}, 4);
  tpool.start();

  Buffer shard_data;
  BinaryCSVOutputStream shard_csv(BufferOutputStream::fromBuffer(&shard_data));
  shard_csv.appendRow(columns);

  size_t nshard = 0;
  auto upload_shard = [&] {
    logDebug(
        "mysql-upload",
        "Shard $0 ready for upload; size=$1MB",
        nshard,
        shard_data.size() / 1000000.0);

    auto upload_uri = StringUtil::format(
        "http://$0:$1/api/v1/tables/upload_table?table=$2&shard=$3",
        host,
        port,
        URI::urlEncode(destination_table),
        nshard++);

    tpool.run([upload_uri, shard_data, auth_headers] () {
      logDebug("mysql-upload", "Uploading: $0", upload_uri);

      http::HTTPClient http_client;
      auto upload_res = http_client.executeRequest(
          http::HTTPRequest::mkPost(
              upload_uri,
              shard_data,
              auth_headers));

      logDebug("mysql-upload", "Upload finished: $0", upload_uri);
      if (upload_res.statusCode() != 201) {
        RAISE(kRuntimeError, upload_res.body().toString());
      }
    });

    shard_data.clear();
    shard_csv.appendRow(columns);
  };

  size_t num_rows_shard = 0;
  auto get_rows_qry = StringUtil::format("SELECT * FROM `$0`;", source_table);
  mysql_conn->executeQuery(
      get_rows_qry,
      [&] (const Vector<String>& row) -> bool {
    shard_csv.appendRow(row);
    for (const auto& r : row) {
      iputs("cell $0", r);
    }

    if (++num_rows_shard == shard_size) {
      upload_shard();
      num_rows_shard = 0;
    }

    if (++num_rows_uploaded < num_rows) {
      status_line.runMaybe();
      return true;
    } else {
      return false;
    }
  });

  if (num_rows_shard > 0) {
    upload_shard();
  }

  status_line.runForce();
  tpool.stop();

  logInfo("dx-mysql-upload", "Upload finished successfully :)");
  exit(0);
}

int main(int argc, const char** argv) {
  Application::init();
  Application::logToStderr("evql-mysql-import");

  cli::FlagParser flags;

  flags.defineFlag(
      "loglevel",
      cli::FlagParser::T_STRING,
      false,
      NULL,
      "INFO",
      "loglevel",
      "<level>");

  flags.defineFlag(
      "source_table",
      cli::FlagParser::T_STRING,
      true,
      "t",
      NULL,
      "table name",
      "<name>");

  flags.defineFlag(
      "destination_table",
      cli::FlagParser::T_STRING,
      true,
      "t",
      NULL,
      "table name",
      "<name>");

  flags.defineFlag(
      "host",
      cli::FlagParser::T_STRING,
      false,
      "h",
      "localhost",
      "eventql server hostname",
      "<host>");

  flags.defineFlag(
      "port",
      cli::FlagParser::T_INTEGER,
      false,
      "p",
      "9175",
      "eventql server port",
      "<port>");

  flags.defineFlag(
      "auth_token",
      cli::FlagParser::T_STRING,
      false,
      NULL,
      NULL,
      "auth token",
      "<token>");

  flags.defineFlag(
      "mysql",
      cli::FlagParser::T_STRING,
      true,
      "x",
      "mysql://localhost:3306/mydb?user=root",
      "MySQL URI",
      "<url>");

  flags.defineFlag(
      "shard_size",
      cli::FlagParser::T_INTEGER,
      false,
      NULL,
      "262144",
      "shard size",
      "<num>");

  flags.parseArgv(argc, argv);

  Logger::get()->setMinimumLogLevel(
      strToLogLevel(flags.getString("loglevel")));

  try {
    run(flags);
  } catch (const StandardException& e) {
    logError("mysql-upload", "[FATAL ERROR] $0", e.what());
  }

  return 0;
}
<commit_msg>mysql import: send request to insert rows<commit_after>/**
 * Copyright (c) 2016 zScale Technology GmbH <legal@zscale.io>
 * Authors:
 *   - Paul Asmuth <paul@zscale.io>
 *   - Laura Schlimmer <laura@zscale.io>
 *
 * This program is free software: you can redistribute it and/or modify it under
 * the terms of the GNU Affero General Public License ("the license") as
 * published by the Free Software Foundation, either version 3 of the License,
 * or any later version.
 *
 * In accordance with Section 7(e) of the license, the licensing of the Program
 * under the license does not imply a trademark license. Therefore any rights,
 * title and interest in our trademarks remain entirely with us.
 *
 * This program is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
 * FOR A PARTICULAR PURPOSE. See the license for more details.
 *
 * You can be released from the requirements of the license by purchasing a
 * commercial license. Buying such a license is mandatory as soon as you develop
 * commercial activities involving this program without disclosing the source
 * code of your own applications
 */
#include <eventql/eventql.h>
#include <eventql/util/stdtypes.h>
#include <eventql/util/application.h>
#include <eventql/util/cli/flagparser.h>
#include <eventql/util/csv/BinaryCSVOutputStream.h>
#include <eventql/util/http/httpclient.h>
#include <eventql/util/json/jsonoutputstream.h>
#include "eventql/util/thread/FixedSizeThreadPool.h"
#include "eventql/util/util/SimpleRateLimit.h"
#include <eventql/util/mysql/MySQL.h>
#include <eventql/util/mysql//MySQLConnection.h>

using namespace eventql;

void run(const cli::FlagParser& flags) {
  auto source_table = flags.getString("source_table");
  auto destination_table = flags.getString("destination_table");
  auto shard_size = flags.getInt("shard_size");
  auto mysql_addr = flags.getString("mysql");
  auto host = flags.getString("host");
  auto port = flags.getInt("port");
  auto db = flags.getString("database");

  logInfo("evql-mysql-import", "Connecting to MySQL Server...");

  util::mysql::mysqlInit();
  auto mysql_conn = util::mysql::MySQLConnection::openConnection(URI(mysql_addr));

  logInfo(
      "mysql-upload",
      "Analyzing the input table. This might take a few minutes...");

  auto schema = mysql_conn->getTableSchema(source_table);
  logDebug("mysql-upload", "Table Schema:\n$0", schema->toString());
  Vector<String> column_names;
  for (const auto& field : schema->fields()) {
    column_names.emplace_back(field.name);
  }

  /* count rows */
  auto count_rows_qry = StringUtil::format(
      "SELECT count(*) FROM `$0`;",
      source_table);

  size_t num_rows = 0;
  size_t num_rows_uploaded = 0;
  mysql_conn->executeQuery(
      count_rows_qry,
      [&num_rows] (const Vector<String>& row) -> bool {
    if (row.size() == 1) num_rows = std::stoull(row[0]);
    return false;
  });

  if (num_rows == 0) {
    logError(
        "mysql-upload",
        "Table '$0' appears to be empty",
        source_table);
    return;
  }

  auto num_shards = (num_rows + shard_size - 1) / shard_size;
  logDebug("mysql-upload", "Splitting into $0 shards", num_shards);

  /* status line */
  util::SimpleRateLimitedFn status_line(
      kMicrosPerSecond,
      [&num_rows_uploaded, num_rows] () {
    logInfo(
        "mysql-upload",
        "[$0%] Uploading... $1/$2 rows",
        (size_t) ((num_rows_uploaded / (double) num_rows) * 100),
        num_rows_uploaded,
        num_rows);
  });

  /* upload rows */
  size_t nshard = 0;
  size_t num_rows_shard = 0;
  Buffer shard_data;
  json::JSONOutputStream json(BufferOutputStream::fromBuffer(&shard_data));

  auto get_rows_qry = StringUtil::format("SELECT * FROM `$0`;", source_table);
  mysql_conn->executeQuery(
      get_rows_qry,
      [&] (const Vector<String>& column_values) -> bool {
    ++num_rows_shard;
    ++num_rows_uploaded;

    if (num_rows_shard > 1) {
      json.addComma();
    }

    json.beginObject();
    json.addObjectEntry("database");
    json.addString(db);
    json.addComma();
    json.addObjectEntry("table");
    json.addString(destination_table);
    json.addComma();
    json.addObjectEntry("data");
    json.beginObject();

    for (size_t i = 0; i < column_names.size() && i < column_values.size(); ++i) {
      if (i > 0 ){
        json.addComma();
      }

      json.addObjectEntry(column_names[i]);
      json.addString(column_values[i]);
    }

    json.endObject();
    json.endObject();

    if (num_rows_shard == shard_size ||
        num_rows_uploaded == num_rows) {
      logDebug(
          "mysql-upload",
          "Uploading shard $0; size=$1MB",
          nshard + 1,
          shard_data.size() / 1000000.0);

      auto insert_uri = StringUtil::format(
          "http://$0:$1/api/v1/tables/insert",
          host,
          port);

      http::HTTPMessage::HeaderList auth_headers;
      if (flags.isSet("auth_token")) {
        auth_headers.emplace_back(
            "Authorization",
            StringUtil::format("Token $0", flags.getString("auth_token")));
      //} else if (!cfg_.getPassword().isEmpty()) {
      //  auth_headers.emplace_back(
      //      "Authorization",
      //      StringUtil::format("Basic $0",
      //          util::Base64::encode(
      //              cfg_.getUser() + ":" + cfg_.getPassword().get())));
      }

      http::HTTPClient http_client;
      auto upload_res = http_client.executeRequest(
          http::HTTPRequest::mkPost(
              insert_uri,
              "[" + shard_data.toString() + "]",
              auth_headers));

      logDebug("mysql-upload", "Upload finished: $0", insert_uri);
      if (upload_res.statusCode() != 201) {
        RAISE(kRuntimeError, upload_res.body().toString());
      }

      shard_data.clear();
      ++nshard;
    }

    if (num_rows_uploaded < num_rows) {
      status_line.runMaybe();
      return true;
    } else {
      return false;
    }
  });

  status_line.runForce();

  logInfo("dx-mysql-upload", "Upload finished successfully :)");
  exit(0);
}

int main(int argc, const char** argv) {
  Application::init();
  Application::logToStderr("evql-mysql-import");

  cli::FlagParser flags;

  flags.defineFlag(
      "loglevel",
      cli::FlagParser::T_STRING,
      false,
      NULL,
      "INFO",
      "loglevel",
      "<level>");

  flags.defineFlag(
      "source_table",
      cli::FlagParser::T_STRING,
      true,
      "t",
      NULL,
      "table name",
      "<name>");

  flags.defineFlag(
      "destination_table",
      cli::FlagParser::T_STRING,
      true,
      "t",
      NULL,
      "table name",
      "<name>");

  flags.defineFlag(
      "host",
      cli::FlagParser::T_STRING,
      false,
      "h",
      "localhost",
      "eventql server hostname",
      "<host>");

  flags.defineFlag(
      "port",
      cli::FlagParser::T_INTEGER,
      false,
      "p",
      "9175",
      "eventql server port",
      "<port>");

  flags.defineFlag(
      "database",
      cli::FlagParser::T_STRING,
      false,
      "db",
      "",
      "eventql database",
      "<databse>");

  flags.defineFlag(
      "auth_token",
      cli::FlagParser::T_STRING,
      false,
      NULL,
      NULL,
      "auth token",
      "<token>");

  flags.defineFlag(
      "mysql",
      cli::FlagParser::T_STRING,
      true,
      "x",
      "mysql://localhost:3306/mydb?user=root",
      "MySQL URI",
      "<url>");

  flags.defineFlag(
      "shard_size",
      cli::FlagParser::T_INTEGER,
      false,
      NULL,
      "262144",
      "shard size",
      "<num>");

  flags.parseArgv(argc, argv);

  Logger::get()->setMinimumLogLevel(
      strToLogLevel(flags.getString("loglevel")));

  try {
    run(flags);
  } catch (const StandardException& e) {
    logError("mysql-upload", "[FATAL ERROR] $0", e.what());
  }

  return 0;
}
<|endoftext|>