<commit_before>/*
	yqlcpp is a lightweight C++ API for performing Yahoo Query Language queries.
	At this time it is intended to be a simple interface with minimal error
	checking and no concern for thread safety. Built on top of and requires libcurl.

    Author: Kevin Hawkins <kevindhawkins@gmail.com>
	Copyright (c) 2014 Kevin Hawkins. All rights reserved.
	https://github.com/kevindhawkins/yqlcpp

	Licensed under the MIT license.

	Permission is hereby granted, free of charge, to any person obtaining a copy
	of this software and associated documentation files (the "Software"), to deal
	in the Software without restriction, including without limitation the rights
	to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
	copies of the Software, and to permit persons to whom the Software is
	furnished to do so, subject to the following conditions:

	The above copyright notice and this permission notice shall be included in
	all copies or substantial portions of the Software.

	THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
	IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
	FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
	AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
	LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
	OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
	THE SOFTWARE.
*/

#ifndef __YQLCPP_H
#define __YQLCPP_H

#include <string>
#include <fstream>
#include <curl/curl.h>

namespace yqlcpp
{
	const std::string baseUrl = "http://query.yahooapis.com/v1/public/yql";
	const std::string envCmd = "&env=store://datatables.org/alltableswithkeys";

	class yqlquery
	{
	public:

		enum class yqlformat { JSON, XML };

		yqlquery(const std::string& cmd, const yqlformat& format = yqlformat::JSON) : m_command(cmd)
		{
			m_curl = curl_easy_init();
			m_format = formatToStr(format);
		}

		virtual ~yqlquery() { curl_easy_cleanup(m_curl); }

		//!< Execute the YQL query.
		bool execute()
		{
			if (m_curl)
			{
				std::string strRequest = "q=" + m_command + envCmd + "&format=" + m_format;

				curl_easy_setopt(m_curl, CURLOPT_URL, baseUrl.c_str());
				curl_easy_setopt(m_curl, CURLOPT_POST, 1);
				curl_easy_setopt(m_curl, CURLOPT_POSTFIELDS, strRequest.c_str());
				curl_easy_setopt(m_curl, CURLOPT_WRITEFUNCTION, responseWriter);

				CURLcode result = curl_easy_perform(m_curl);
				return (result == CURLE_OK);
			}

			return false;
		}

		//!< Get the YQL query response as a string.
		std::string getResponse() { return (*m_response); }

		//!< Write the YQL response to a file.
		void toFile(const std::string& filename)
		{
			if (m_response)
			{
				std::ofstream out(filename);
				out << (*m_response);
				out.close();
			}
		}

	private:
		yqlquery() {}

		CURL* m_curl;				//!< Our curl handle.
		std::string m_command;		//!< Query command sent to YQL.
		std::string m_format;		//!< Format of the response data.

		static std::string* m_response;		//!< Response data, provided and managed by curl.

		///////////////////////////////////////////////////////////////////////

		//!<  Writes curl response data to the response string.
		static size_t responseWriter(char* contents, size_t size, size_t nmemb, std::string* data)
		{
			if (data)
			{
				size_t realSize = size * nmemb;

				data->append(contents, realSize);
				m_response = data;

				return realSize;
			}

			return 0;
		}

		///////////////////////////////////////////////////////////////////////

		std::string formatToStr(const yqlformat& format)
		{
			switch(format)
			{
			case yqlformat::JSON:
				return "json";
			case yqlformat::XML:
				return "xml";
			default:
				return "json";
			}
		}
	};

	std::string* yqlquery::m_response = NULL;

}

#endif
<commit_msg>Cleanup tabs<commit_after>/*
    yqlcpp is a lightweight C++ API for performing Yahoo Query Language queries.
    At this time it is intended to be a simple interface with minimal error
    checking and no concern for thread safety. Built on top of and requires libcurl.

    Author: Kevin Hawkins <kevindhawkins@gmail.com>
    Copyright (c) 2014 Kevin Hawkins. All rights reserved.
    https://github.com/kevindhawkins/yqlcpp

    Licensed under the MIT license.

    Permission is hereby granted, free of charge, to any person obtaining a copy
    of this software and associated documentation files (the "Software"), to deal
    in the Software without restriction, including without limitation the rights
    to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
    copies of the Software, and to permit persons to whom the Software is
    furnished to do so, subject to the following conditions:

    The above copyright notice and this permission notice shall be included in
    all copies or substantial portions of the Software.

    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
    OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
    THE SOFTWARE.
*/

#ifndef __YQLCPP_H
#define __YQLCPP_H

#include <string>
#include <fstream>
#include <curl/curl.h>

namespace yqlcpp
{
    const std::string baseUrl = "http://query.yahooapis.com/v1/public/yql";
    const std::string envCmd = "&env=store://datatables.org/alltableswithkeys";

    class yqlquery
    {
    public:

        enum class yqlformat { JSON, XML };

        yqlquery(const std::string& cmd, const yqlformat& format = yqlformat::JSON) : m_command(cmd)
        {
            m_curl = curl_easy_init();
            m_format = formatToStr(format);
        }

        virtual ~yqlquery() { curl_easy_cleanup(m_curl); }

        //!< Execute the YQL query.
        bool execute()
        {
            if (m_curl)
            {
                std::string strRequest = "q=" + m_command + envCmd + "&format=" + m_format;

                curl_easy_setopt(m_curl, CURLOPT_URL, baseUrl.c_str());
                curl_easy_setopt(m_curl, CURLOPT_POST, 1);
                curl_easy_setopt(m_curl, CURLOPT_POSTFIELDS, strRequest.c_str());
                curl_easy_setopt(m_curl, CURLOPT_WRITEFUNCTION, responseWriter);

                CURLcode result = curl_easy_perform(m_curl);
                return (result == CURLE_OK);
            }

            return false;
        }

        //!< Get the YQL query response as a string.
        std::string getResponse() { return (*m_response); }

        //!< Write the YQL response to a file.
        void toFile(const std::string& filename)
        {
            if (m_response)
            {
                std::ofstream out(filename);
                out << (*m_response);
                out.close();
            }
        }

    private:
        yqlquery() {}

        CURL* m_curl;				//!< Our curl handle.
        std::string m_command;		//!< Query command sent to YQL.
        std::string m_format;		//!< Format of the response data.

        static std::string* m_response;		//!< Response data, provided and managed by curl.

        ///////////////////////////////////////////////////////////////////////

        //!<  Writes curl response data to the response string.
        static size_t responseWriter(char* contents, size_t size, size_t nmemb, std::string* data)
        {
            if (data)
            {
                size_t realSize = size * nmemb;

                data->append(contents, realSize);
                m_response = data;

                return realSize;
            }

            return 0;
        }

        ///////////////////////////////////////////////////////////////////////

        std::string formatToStr(const yqlformat& format)
        {
            switch(format)
            {
            case yqlformat::JSON:
                return "json";
            case yqlformat::XML:
                return "xml";
            default:
                return "json";
            }
        }
    };

    std::string* yqlquery::m_response = NULL;

}

#endif
<|endoftext|>


<commit_before>// $Id: directory.C,v 1.11 2000/06/27 09:08:52 oliver Exp $

#include <dirent.h>
#include <stdio.h>
#include <string.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <unistd.h>
#include <iostream>
#include <errno.h>
#include <BALL/SYSTEM/directory.h>

namespace BALL
{
	Directory::Directory()
	{
		dir_ = 0;
		dirent_ = 0;
		char* buffer_;
		if ((buffer_ = ::getcwd(NULL, 64)) != NULL)	directory_path_ = buffer_;
		else directory_path_ = "";
	}

	Directory::Directory(const String& directory_path, bool set_current)
	{
		if (!set(directory_path, set_current)) directory_path_ = "";
	}

	Directory::Directory(const Directory& directory)
	{
		set(directory);
	}

	Directory::~Directory()
	{
		if (dir_ != 0) ::closedir(dir_);
	}

	void Directory::swap(Directory& directory)
	{
		DIR* temp_dir = dir_;
		dir_ = directory.dir_;
		directory.dir_ = temp_dir;

		dirent* temp_dirent = dirent_;
		dirent_ = directory.dirent_;
		directory.dirent_ = temp_dirent;

		backup_path_.swap(directory.backup_path_);
		directory_path_.swap(directory.directory_path_);
	}

	bool Directory::getFirstEntry(String& entry)
	{
		synchronize_();
		if (dir_ != 0) ::closedir(dir_);
		dir_ = ::opendir(directory_path_.data());
		if (dir_ == 0) return desynchronize_(false);
		dirent_ = ::readdir(dir_);
		if (dirent_ == 0)
		{
			::closedir(dir_);
			dir_ = 0;
			return desynchronize_(false);
		}
		else
		{
			entry = dirent_->d_name;
			return desynchronize_(true);
		}
	}

	bool Directory::getNextEntry(String& entry)
	{
		synchronize_();
		if (dir_ == 0) dir_ = ::opendir(directory_path_.data());
		if (dir_ == 0) return desynchronize_(false);

		dirent_ = ::readdir(dir_);
		if (dirent_ == 0)
		{
			::closedir(dir_);
			dir_ = 0;
			return desynchronize_(false);
		}
		entry = dirent_->d_name;
		return desynchronize_(true);
	}

	Size Directory::countItems()
	{
		synchronize_();
		Size size = 0;
		DIR* dir = ::opendir(directory_path_.data());
		if (dir == 0)
		{
			desynchronize_();
			return 0;
		}
		while(::readdir(dir) != 0) ++size;
		::closedir(dir);
		desynchronize_();
		return (size - 2);
	}	// ignore current (.) and parent directory entry (..)

	Size Directory::countFiles()
	{
		synchronize_();
		struct stat stats;
		Size size = 0;
		dirent* myDirent;
		DIR* dir = ::opendir(directory_path_.data());
		if (dir == 0)
		{
			desynchronize_();
			return 0;
		}
		while((myDirent = ::readdir(dir)) != 0)
		{
			if (lstat(myDirent->d_name, &stats) < 0)	continue;
			if (S_ISDIR(stats.st_mode) == 0) ++size;
		}
		::closedir(dir);
		desynchronize_();
		return size;
	}

	Size Directory::countDirectories()
	{
		synchronize_();
		struct stat stats;
		Size size = 0;
		dirent* myDirent;
		DIR *dir = ::opendir(directory_path_.data());
		if (dir == 0)
		{
			desynchronize_();
			return 0;
		}
		while((myDirent = ::readdir(dir)) != 0)
		{
			if (lstat(myDirent->d_name, &stats) < 0)	continue;
			if (S_ISDIR(stats.st_mode) != 0) ++size;
		}
		::closedir(dir);
		desynchronize_();
		return (size - 2);
	}		// ignore current (.) and parent directory entry (..)

	bool Directory::isCurrent() const
	{
		char* buffer_;
		if ((buffer_ = ::getcwd(NULL, 64)) == 0) return false;
		return (buffer_ == directory_path_);
	}

	bool Directory::has(const String& item) //const
	{
		synchronize_();
		String entry;
		while (getNextEntry(entry))
		{
			if (entry == item) return desynchronize_(true);
		}
		return desynchronize_(false);
	}

	bool Directory::find(const String& item, String& filepath)
	{
		if (has(item))
		{
			filepath = directory_path_;
			return true; // no sync needed...
		}
		synchronize_();
		struct stat stats;
		dirent* myDirent;
		Directory directory;
		String s;
		DIR* dir = ::opendir(FileSystem::CURRENT_DIRECTORY);
		if (dir == 0)	return desynchronize_(false);

		while((myDirent = ::readdir(dir)) != 0)
		{
			if (lstat(myDirent->d_name, &stats) < 0) continue;
			if (S_ISDIR(stats.st_mode) != 0 &&
					strcmp(myDirent->d_name, FileSystem::CURRENT_DIRECTORY) != 0 &&
			    strcmp(myDirent->d_name, FileSystem::PARENT_DIRECTORY ) != 0)
			{
				directory =Directory(myDirent->d_name);
				if (directory.find(item, s))
				{
					filepath = s;
					::closedir(dir);
					return desynchronize_(true);
				}
			}
		}
		::closedir(dir);
		return desynchronize_(false);
	}

	bool Directory::set(const String& directory_path, bool set_current)
	{
		dir_ = 0;
		dirent_ = 0;
		backup_path_ = "";
		char* buffer_;
		if (directory_path[0] == '/')		//absolute path
		{
			directory_path_ = directory_path;
			FileSystem::canonizePath(directory_path_);
			return isValid();
		}
		if ((buffer_ = ::getcwd(NULL, 64)) != NULL)
		{
			directory_path_ = buffer_;
			directory_path_ += FileSystem::PATH_SEPARATOR;
			directory_path_ += directory_path;
			FileSystem::canonizePath(directory_path_);
			if (directory_path_.hasSuffix(String(FileSystem::PATH_SEPARATOR)))
			{
				directory_path_.truncate(directory_path_.size() - 1);
			}
		}
		else
		{
			directory_path_ = "";
			return false;
		}
		if (!isValid())	return false;
		if (set_current) return (::chdir(directory_path_.data()) == 0);
		return true;
	}

	bool Directory::remove()
	{
		synchronize_();
		if (::chdir("..") != 0) return desynchronize_(false);
		bool result1 = (::rmdir(directory_path_.data()) == 0);
		bool result2;
		if (backup_path_ != "")
		{
			result2 = ::chdir(backup_path_.data());
			backup_path_ = "";
		}
		dir_ = 0;
		dirent_ = 0;
		directory_path_ = "";
		return (result1 && result2);
	}

	bool Directory::renameTo(String new_path)
	{
		synchronize_();
		FileSystem::canonizePath(new_path);
		if (::chdir("..") != 0)	return desynchronize_(false);
		if (::rename(directory_path_.data(), new_path.data()) == 0)
		{
			directory_path_ = new_path;
			return desynchronize_(true);
		}
		return desynchronize_(false);
	}

#       ifdef BALL_NO_INLINE_FUNCTIONS
#               include <BALL/SYSTEM/directory.iC>
#       endif

} // namespace BALL
<commit_msg>fixed:redefinition of default argument       Directory::Directory(const String& directory_path, bool set_current = false)<commit_after>// $Id: directory.C,v 1.12 2000/06/27 10:40:51 amoll Exp $

#include <dirent.h>
#include <stdio.h>
#include <string.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <unistd.h>
#include <iostream>
#include <errno.h>
#include <BALL/SYSTEM/directory.h>

namespace BALL
{
	Directory::Directory()
	{
		dir_ = 0;
		dirent_ = 0;
		char* buffer_;
		if ((buffer_ = ::getcwd(NULL, 64)) != NULL)	directory_path_ = buffer_;
		else directory_path_ = "";
	}

	Directory::Directory(const String& directory_path, bool set_current)
	{
		if (!set(directory_path, set_current)) directory_path_ = "";
	}

	Directory::Directory(const Directory& directory)
	{
		set(directory);
	}

	Directory::~Directory()
	{
		if (dir_ != 0) ::closedir(dir_);
	}

	void Directory::swap(Directory& directory)
	{
		DIR* temp_dir = dir_;
		dir_ = directory.dir_;
		directory.dir_ = temp_dir;

		dirent* temp_dirent = dirent_;
		dirent_ = directory.dirent_;
		directory.dirent_ = temp_dirent;

		backup_path_.swap(directory.backup_path_);
		directory_path_.swap(directory.directory_path_);
	}

	bool Directory::getFirstEntry(String& entry)
	{
		synchronize_();
		if (dir_ != 0) ::closedir(dir_);
		dir_ = ::opendir(directory_path_.data());
		if (dir_ == 0) return desynchronize_(false);
		dirent_ = ::readdir(dir_);
		if (dirent_ == 0)
		{
			::closedir(dir_);
			dir_ = 0;
			return desynchronize_(false);
		}
		else
		{
			entry = dirent_->d_name;
			return desynchronize_(true);
		}
	}

	bool Directory::getNextEntry(String& entry)
	{
		synchronize_();
		if (dir_ == 0) dir_ = ::opendir(directory_path_.data());
		if (dir_ == 0) return desynchronize_(false);

		dirent_ = ::readdir(dir_);
		if (dirent_ == 0)
		{
			::closedir(dir_);
			dir_ = 0;
			return desynchronize_(false);
		}
		entry = dirent_->d_name;
		return desynchronize_(true);
	}

	Size Directory::countItems()
	{
		synchronize_();
		Size size = 0;
		DIR* dir = ::opendir(directory_path_.data());
		if (dir == 0)
		{
			desynchronize_();
			return 0;
		}
		while(::readdir(dir) != 0) ++size;
		::closedir(dir);
		desynchronize_();
		return (size - 2);
	}	// ignore current (.) and parent directory entry (..)

	Size Directory::countFiles()
	{
		synchronize_();
		struct stat stats;
		Size size = 0;
		dirent* myDirent;
		DIR* dir = ::opendir(directory_path_.data());
		if (dir == 0)
		{
			desynchronize_();
			return 0;
		}
		while((myDirent = ::readdir(dir)) != 0)
		{
			if (lstat(myDirent->d_name, &stats) < 0)	continue;
			if (S_ISDIR(stats.st_mode) == 0) ++size;
		}
		::closedir(dir);
		desynchronize_();
		return size;
	}

	Size Directory::countDirectories()
	{
		synchronize_();
		struct stat stats;
		Size size = 0;
		dirent* myDirent;
		DIR *dir = ::opendir(directory_path_.data());
		if (dir == 0)
		{
			desynchronize_();
			return 0;
		}
		while((myDirent = ::readdir(dir)) != 0)
		{
			if (lstat(myDirent->d_name, &stats) < 0)	continue;
			if (S_ISDIR(stats.st_mode) != 0) ++size;
		}
		::closedir(dir);
		desynchronize_();
		return (size - 2);
	}		// ignore current (.) and parent directory entry (..)

	bool Directory::isCurrent() const
	{
		char* buffer_;
		if ((buffer_ = ::getcwd(NULL, 64)) == 0) return false;
		return (buffer_ == directory_path_);
	}

	bool Directory::has(const String& item) //const
	{
		synchronize_();
		String entry;
		while (getNextEntry(entry))
		{
			if (entry == item) return desynchronize_(true);
		}
		return desynchronize_(false);
	}

	bool Directory::find(const String& item, String& filepath)
	{
		if (has(item))
		{
			filepath = directory_path_;
			return true; // no sync needed...
		}
		synchronize_();
		struct stat stats;
		dirent* myDirent;
		Directory directory;
		String s;
		DIR* dir = ::opendir(FileSystem::CURRENT_DIRECTORY);
		if (dir == 0)	return desynchronize_(false);

		while((myDirent = ::readdir(dir)) != 0)
		{
			if (lstat(myDirent->d_name, &stats) < 0) continue;
			if (S_ISDIR(stats.st_mode) != 0 &&
					strcmp(myDirent->d_name, FileSystem::CURRENT_DIRECTORY) != 0 &&
			    strcmp(myDirent->d_name, FileSystem::PARENT_DIRECTORY ) != 0)
			{
				directory =Directory(myDirent->d_name);
				if (directory.find(item, s))
				{
					filepath = s;
					::closedir(dir);
					return desynchronize_(true);
				}
			}
		}
		::closedir(dir);
		return desynchronize_(false);
	}

	bool Directory::set(const String& directory_path, bool set_current)
	{
		dir_ = 0;
		dirent_ = 0;
		backup_path_ = "";
		char* buffer_;
		if (directory_path[0] == '/')		//absolute path
		{
			directory_path_ = directory_path;
			FileSystem::canonizePath(directory_path_);
			return isValid();
		}
		if ((buffer_ = ::getcwd(NULL, 64)) != NULL)
		{
			directory_path_ = buffer_;
			directory_path_ += FileSystem::PATH_SEPARATOR;
			directory_path_ += directory_path;
			FileSystem::canonizePath(directory_path_);
			if (directory_path_.hasSuffix(String(FileSystem::PATH_SEPARATOR)))
			{
				directory_path_.truncate(directory_path_.size() - 1);
			}
		}
		else
		{
			directory_path_ = "";
			return false;
		}
		if (!isValid())	return false;
		if (set_current) return (::chdir(directory_path_.data()) == 0);
		return true;
	}

	bool Directory::remove()
	{
		synchronize_();
		if (::chdir("..") != 0) return desynchronize_(false);
		bool result1 = (::rmdir(directory_path_.data()) == 0);
		bool result2;
		if (backup_path_ != "")
		{
			result2 = ::chdir(backup_path_.data());
			backup_path_ = "";
		}
		dir_ = 0;
		dirent_ = 0;
		directory_path_ = "";
		return (result1 && result2);
	}

	bool Directory::renameTo(String new_path)
	{
		synchronize_();
		FileSystem::canonizePath(new_path);
		if (::chdir("..") != 0)	return desynchronize_(false);
		if (::rename(directory_path_.data(), new_path.data()) == 0)
		{
			directory_path_ = new_path;
			return desynchronize_(true);
		}
		return desynchronize_(false);
	}

#       ifdef BALL_NO_INLINE_FUNCTIONS
#               include <BALL/SYSTEM/directory.iC>
#       endif

} // namespace BALL
<|endoftext|>


<commit_before>#include <thread>
#include <chrono>
#include <stdexcept>
#include <cctype>
#include <algorithm>
#include <vector>
#include <map>
#include <sstream>

#include "ogrsf_frmts.h"

#include "common/config.h"
#include "common/stringutils.h"
#include "server/worker.h"

using namespace Batyr;

struct OgrField
{
    std::string name;
    unsigned int index;
    OGRFieldType type;
};
typedef std::map<std::string, OgrField> OgrFieldMap;

Worker::Worker(Configuration::Ptr _configuration, std::shared_ptr<JobStorage> _jobs)
    :   logger(Poco::Logger::get("Worker")),
        configuration(_configuration),
        jobs(_jobs),
        db(_configuration)
{
    poco_debug(logger, "Creating Worker");
}

Worker::~Worker()
{
    poco_debug(logger, "Destroying Worker");
}

void
Worker::pull(Job::Ptr job)
{
    if (job->getFilter().empty()) {
        poco_information(logger, "pulling layer \""+job->getLayerName()+"\"");
    }
    else {
        poco_information(logger, "pulling layer \""+job->getLayerName()+"\" using filter \""+job->getFilter()+"\"");
    }

    auto layer = configuration->getLayer(job->getLayerName());

    // open the dataset
    std::unique_ptr<OGRDataSource, void (*)(OGRDataSource*)> ogrDataset(
        OGRSFDriverRegistrar::Open(layer->source.c_str(), false),  OGRDataSource::DestroyDataSource);
    if (!ogrDataset) {
        throw WorkerError("Could not open dataset for layer \"" + layer->name + "\"");
    }

    // find the layer
    auto ogrLayer = ogrDataset->GetLayerByName(layer->source_layer.c_str());
    if (ogrLayer == nullptr) {
        throw WorkerError("source_layer \"" +layer->source_layer+ "\" in dataset for layer \""
                                + layer->name + "\" not found");
    }
    ogrLayer->ResetReading();

    // TODO: set filter

    auto ogrFeatureDefn = ogrLayer->GetLayerDefn();

#if GDAL_VERSION_MAJOR > 1
    if (ogrFeatureDefn->GetGeomFieldCount() != 1) {
        std::string msg = "The source provides " + std::to_string(ogrFeatureDefn->GetGeomFieldCount()) +
                "geometry fields. Currently only sources with on geoemtry field are supported";
        throw WorkerError(msg);
    }
#endif

    // collect the columns of the dataset
    OgrFieldMap ogrFields;
    for(int i=0; i<ogrFeatureDefn->GetFieldCount(); i++) {
        auto ogrFieldDefn = ogrFeatureDefn->GetFieldDefn(i);

        // lowercase column names -- TODO: this may cause problems when postgresqls column names
        // contain uppercase letters, but will do for a start
        std::string fieldNameCased = std::string(ogrFieldDefn->GetNameRef());
        std::string fieldName;
        std::transform(fieldNameCased.begin(), fieldNameCased.end(), std::back_inserter(fieldName), ::tolower);

#ifdef _DEBUG
        {
            std::string msg = "ogr layer provides the column " + fieldName;
            poco_debug(logger, msg.c_str());
        }
#endif
        auto entry = &ogrFields[fieldName];
        entry->index = i;
        entry->type = ogrFieldDefn->GetType();
    }

    // perform the work in an transaction
    if (auto transaction = db.getTransaction()) {
        int numPulled = 0;
        int numCreated = 0;
        int numUpdated = 0;
        int numDeleted = 0;

        // build a unique name for the temporary table
        std::string tempTableName = "batyr_" + job->getId();

        // create a temp table to write the data to
        transaction->createTempTable(layer->target_table_schema, layer->target_table_name, tempTableName);

        // fetch the column list from the target_table as the tempTable
        // does not have the constraints of the original table
        auto tableFields = transaction->getTableFields(layer->target_table_schema, layer->target_table_name);

        // check if the requirements of the primary key are satisfied
        // TODO: allow overriding the primarykey from the configfile
        std::vector<std::string> primaryKeyColumns;
        std::string geometryColumn;
        std::vector<std::string> insertColumns;
        std::vector<std::string> updateColumns;
        for(auto tableFieldPair : tableFields) {
            if (tableFieldPair.second.isPrimaryKey) {
                primaryKeyColumns.push_back(tableFieldPair.second.name);
            }
            else {
                updateColumns.push_back(tableFieldPair.second.name);
            }
            if (tableFieldPair.second.pgTypeName == "geometry") {
                if (!geometryColumn.empty()) {
                    throw WorkerError("Layer \"" + job->getLayerName() + "\" has multiple geometry columns. Currently only one is supported");
                }
                geometryColumn = tableFieldPair.second.name;
                insertColumns.push_back(tableFieldPair.second.name);
            }
            if (ogrFields.find(tableFieldPair.second.name) != ogrFields.end()) {
                insertColumns.push_back(tableFieldPair.second.name);
            }
        }
        if (primaryKeyColumns.empty()) {
            throw WorkerError("Got no primarykey for layer \"" + job->getLayerName() + "\"");
        }
        std::vector<std::string> missingPrimaryKeysSource;
        for( auto primaryKeyCol : primaryKeyColumns) {
            if (ogrFields.find(primaryKeyCol) == ogrFields.end()) {
                missingPrimaryKeysSource.push_back(primaryKeyCol);
            }
        }
        if (!missingPrimaryKeysSource.empty()) {
            throw WorkerError("The source for layer \"" + job->getLayerName() + "\" is missing the following fields required "+
                    "by the primary key: " + StringUtils::join(missingPrimaryKeysSource, ", "));
        }

        // prepare an insert query into the temporary table
        std::vector<std::string> insertQueryValues;
        unsigned int idxColumn = 1;
        for (std::string insertColumn : insertColumns) {
            std::stringstream colStream;
            colStream << "$" << idxColumn << "::" << tableFields[insertColumn].pgTypeName;
            insertQueryValues.push_back(colStream.str());
            idxColumn++;
        }
        std::stringstream insertQueryStream;
        // TODO: include st_transform statement into insert if original table has a srid set in geometry_columns
        insertQueryStream   << "insert into \"" << tempTableName << "\" (\""
                            << StringUtils::join(insertColumns, "\", \"")
                            << "\") values ("
                            << StringUtils::join(insertQueryValues, ", ")
                            << ")";
        poco_debug(logger, insertQueryStream.str().c_str());
        std::string insertStmtName = "batyr_insert" + job->getId();
        auto resInsertStmt = transaction->prepare(insertStmtName, insertQueryStream.str(), insertColumns.size(), NULL);

        OGRFeature * ogrFeature = 0;
        while( (ogrFeature = ogrLayer->GetNextFeature()) != nullptr) {
            std::vector<std::string> strValues;

            for (std::string insertColumn : insertColumns) {
                auto tableField = &tableFields[insertColumn];

                if (tableField->pgTypeName == "geometry") {
                    // TODO: Maybe use the implementation from OGRPGLayer::GeometryToHex
                    GByte * buffer;

                    auto ogrGeometry = ogrFeature->GetGeometryRef();
                    int bufferSize = ogrGeometry->WkbSize();

                    buffer = (GByte *) CPLMalloc(bufferSize);
                    if (buffer == nullptr) {
                        throw WorkerError("Unable to allocate memory to export geometry");
                    }
                    if (ogrGeometry->exportToWkb(wkbNDR, buffer) != OGRERR_NONE) {
                        OGRFree(buffer);
                        throw WorkerError("Could not export the geometry from feature #" + std::to_string(numPulled));
                    }
                    char * hexBuffer = CPLBinaryToHex(bufferSize, buffer);
                    if (hexBuffer == nullptr) {
                        OGRFree(buffer);
                        throw WorkerError("Unable to allocate memory to convert geometry to hex");
                    }
                    OGRFree(buffer);
                    strValues.push_back(std::string(hexBuffer));
                    CPLFree(hexBuffer);
                }
                else {
                    auto ogrField = &ogrFields[insertColumn];
                    switch (ogrField->type) {
                        case OFTString:
                            strValues.push_back(std::string(ogrFeature->GetFieldAsString(ogrField->index)));
                            break;
                        case OFTInteger:
                            strValues.push_back(std::to_string(ogrFeature->GetFieldAsInteger(ogrField->index)));
                            break;
                        case OFTReal:
                            strValues.push_back(std::to_string(ogrFeature->GetFieldAsDouble(ogrField->index)));
                            break;
                        // TODO: implment all of the OGRFieldType types
                        default:
                            throw WorkerError("Unsupported OGR field type: " + std::to_string(static_cast<int>(ogrField->type)));
                    }
                }
            }

            // convert to an array of c strings
            std::vector<const char*> cStrValues;
            std::vector<int> cStrValueLenghts;
            std::transform(strValues.begin(), strValues.end(), std::back_inserter(cStrValues),
                        [](std::string & s){ return s.c_str();});
            std::transform(strValues.begin(), strValues.end(), std::back_inserter(cStrValueLenghts),
                        [](std::string & s){ return s.length();});

            transaction->execPrepared(insertStmtName, cStrValues.size(), &cStrValues[0], &cStrValueLenghts[0],
                        NULL, 1);

            numPulled++;
        }
        job->setStatistics(numPulled, numCreated, numUpdated, numDeleted);

        // update the existing table only touching rows which have differences to prevent
        // slowdowns by triggers
        std::stringstream updateStmt;
        updateStmt          << "update \"" << layer->target_table_schema << "\".\"" << layer->target_table_name << "\" "
                            << " set ";
        bool firstIter = true;
        for (std::string updateColumn : updateColumns) {
            if (!firstIter) {
                updateStmt << ", ";
            }
            updateStmt << "\"" << updateColumn << "\" = \"" << tempTableName << "\".\"" << updateColumn << "\" ";
            firstIter = false;
        }
        updateStmt          << " from \"" << tempTableName << "\""
                            << " where (";
        firstIter = true;
        for (std::string primaryKeyColumn : primaryKeyColumns) {
            if (!firstIter) {
                updateStmt << " and ";
            }
            updateStmt  << "\""  << layer->target_table_name << "\".\"" << primaryKeyColumn
                        << "\" is not distinct from \"" << tempTableName << "\".\"" << primaryKeyColumn << "\"";
            firstIter = false;
        }
        updateStmt          << ") and (";
        firstIter = true;
        for (std::string updateColumn : updateColumns) {
            if (!firstIter) {
                updateStmt << " or ";
            }
            updateStmt  << "(\"" << layer->target_table_name << "\".\"" << updateColumn
                        << "\" is distinct from  \""
                        << tempTableName << "\".\"" << updateColumn << "\")";
            firstIter = false;
        }
        updateStmt          << ")";
        auto updateRes = transaction->exec(updateStmt.str());
        numUpdated = std::atoi(PQcmdTuples(updateRes.get()));
        updateRes.reset(NULL); // immediately dispose the result

        // insert missing rows in the exisiting table
        std::stringstream insertMissingStmt;
        insertMissingStmt   << "insert into \"" << layer->target_table_schema << "\".\"" << layer->target_table_name << "\" "
                            << " ( \"" << StringUtils::join(insertColumns, "\", \"") << "\") "
                            << " select \"" << StringUtils::join(insertColumns, "\", \"") << "\" "
                            << " from \"" << tempTableName << "\""
                            << " where (\"" << StringUtils::join(primaryKeyColumns, "\", \"") << "\") not in ("
                            << " select \"" << StringUtils::join(primaryKeyColumns, "\",\"") << "\" "
                            << "       from \"" << layer->target_table_schema << "\".\""  << layer->target_table_name << "\""
                            << ")";
        auto insertMissingRes = transaction->exec(insertMissingStmt.str());
        numCreated = std::atoi(PQcmdTuples(insertMissingRes.get()));
        insertMissingRes.reset(NULL); // immediately dispose the result

        // delete deprecated rows from the exisiting table
        // TODO: make this optional and skip when a filter is used
        std::stringstream deleteRemovedStmt;
        deleteRemovedStmt   << "delete from \"" << layer->target_table_schema << "\".\"" << layer->target_table_name << "\" "
                            << " where (\"" << StringUtils::join(primaryKeyColumns, "\", \"") << "\") not in ("
                            << " select \"" << StringUtils::join(primaryKeyColumns, "\",\"") << "\" "
                            << "       from \"" << tempTableName << "\""
                            << ")";
        auto deleteRemovedRes = transaction->exec(deleteRemovedStmt.str());
        numDeleted = std::atoi(PQcmdTuples(deleteRemovedRes.get()));
        deleteRemovedRes.reset(NULL); // immediately dispose the result

        job->setStatus(Job::Status::FINISHED);
        job->setStatistics(numPulled, numCreated, numUpdated, numDeleted);
    }
    else {
        std::string msg("Could not start a database transaction");
        poco_error(logger, msg.c_str());
        job->setStatus(Job::Status::FAILED);
        job->setMessage(msg);
    }

}

void
Worker::run()
{
    while (true) {
        Job::Ptr job;
        try {
            bool got_job = jobs->pop(job);
            if (!got_job) {
                // no job means the queue recieved a quit command, so the worker
                // can be shut down
                break;
            }
            poco_debug(logger, "Got job from queue");

            job->setStatus(Job::Status::IN_PROCESS);

            // check if we got a working database connection
            // or block until we got one
            size_t reconnectAttempts = 0;
            while(!db.reconnect(true)) {
                if (reconnectAttempts == 0) {
                    // set job message to inform clients we are waiting here
                    job->setMessage("Waiting to aquire a database connection");
                }
                reconnectAttempts++;
                std::this_thread::sleep_for( std::chrono::milliseconds( SERVER_DB_RECONNECT_WAIT ) );
            }

            pull(job);
        }
        catch (Batyr::Db::DbError &e) {
            poco_error(logger, e.what());
            job->setStatus(Job::Status::FAILED);
            job->setMessage(e.what());
        }
        catch (WorkerError &e) {
            poco_error(logger, e.what());
            job->setStatus(Job::Status::FAILED);
            job->setMessage(e.what());
        }
        catch (std::runtime_error &e) {
            poco_error(logger, e.what());
            job->setStatus(Job::Status::FAILED);
            job->setMessage(e.what());

            // do not know how this exception was caused as it
            // was not handled by one of the earlier catch blocks
            throw;
        }
    }
    poco_debug(logger, "leaving run method");
}
<commit_msg>support for ogr attribute filters<commit_after>#include <thread>
#include <chrono>
#include <stdexcept>
#include <cctype>
#include <algorithm>
#include <vector>
#include <map>
#include <sstream>

#include "ogrsf_frmts.h"

#include "common/config.h"
#include "common/stringutils.h"
#include "server/worker.h"

using namespace Batyr;

struct OgrField
{
    std::string name;
    unsigned int index;
    OGRFieldType type;
};
typedef std::map<std::string, OgrField> OgrFieldMap;

Worker::Worker(Configuration::Ptr _configuration, std::shared_ptr<JobStorage> _jobs)
    :   logger(Poco::Logger::get("Worker")),
        configuration(_configuration),
        jobs(_jobs),
        db(_configuration)
{
    poco_debug(logger, "Creating Worker");
}

Worker::~Worker()
{
    poco_debug(logger, "Destroying Worker");
}

void
Worker::pull(Job::Ptr job)
{
    if (job->getFilter().empty()) {
        poco_information(logger, "pulling layer \""+job->getLayerName()+"\"");
    }
    else {
        poco_information(logger, "pulling layer \""+job->getLayerName()+"\" using filter \""+job->getFilter()+"\"");
    }

    auto layer = configuration->getLayer(job->getLayerName());

    // open the dataset
    std::unique_ptr<OGRDataSource, void (*)(OGRDataSource*)> ogrDataset(
        OGRSFDriverRegistrar::Open(layer->source.c_str(), false),  OGRDataSource::DestroyDataSource);
    if (!ogrDataset) {
        throw WorkerError("Could not open dataset for layer \"" + layer->name + "\"");
    }

    // find the layer
    auto ogrLayer = ogrDataset->GetLayerByName(layer->source_layer.c_str());
    if (ogrLayer == nullptr) {
        throw WorkerError("source_layer \"" +layer->source_layer+ "\" in dataset for layer \""
                                + layer->name + "\" not found");
    }
    ogrLayer->ResetReading();

    // set filter if set
    std::string filterString = job->getFilter();
    if (!filterString.empty()) {
        CPLErrorReset();
        if (ogrLayer->SetAttributeFilter(filterString.c_str()) != OGRERR_NONE) {
            std::stringstream msgstream;
            msgstream   << "The given filter for layer \""
                        << layer->name
                        << "\" is invalid";
            if (CPLGetLastErrorMsg()) {
                msgstream   << ": " << CPLGetLastErrorMsg();
            }
            else {
                msgstream   << ".";
            }
            msgstream   << " The applied filter was [ "
                        << filterString
                        << " ]";
            CPLErrorReset();
            throw WorkerError(msgstream.str());
        }
    }

    auto ogrFeatureDefn = ogrLayer->GetLayerDefn();

#if GDAL_VERSION_MAJOR > 1
    if (ogrFeatureDefn->GetGeomFieldCount() != 1) {
        std::string msg = "The source provides " + std::to_string(ogrFeatureDefn->GetGeomFieldCount()) +
                "geometry fields. Currently only sources with on geoemtry field are supported";
        throw WorkerError(msg);
    }
#endif

    // collect the columns of the dataset
    OgrFieldMap ogrFields;
    for(int i=0; i<ogrFeatureDefn->GetFieldCount(); i++) {
        auto ogrFieldDefn = ogrFeatureDefn->GetFieldDefn(i);

        // lowercase column names -- TODO: this may cause problems when postgresqls column names
        // contain uppercase letters, but will do for a start
        std::string fieldNameCased = std::string(ogrFieldDefn->GetNameRef());
        std::string fieldName;
        std::transform(fieldNameCased.begin(), fieldNameCased.end(), std::back_inserter(fieldName), ::tolower);

#ifdef _DEBUG
        {
            std::string msg = "ogr layer provides the column " + fieldName;
            poco_debug(logger, msg.c_str());
        }
#endif
        auto entry = &ogrFields[fieldName];
        entry->index = i;
        entry->type = ogrFieldDefn->GetType();
    }

    // perform the work in an transaction
    if (auto transaction = db.getTransaction()) {
        int numPulled = 0;
        int numCreated = 0;
        int numUpdated = 0;
        int numDeleted = 0;

        // build a unique name for the temporary table
        std::string tempTableName = "batyr_" + job->getId();

        // create a temp table to write the data to
        transaction->createTempTable(layer->target_table_schema, layer->target_table_name, tempTableName);

        // fetch the column list from the target_table as the tempTable
        // does not have the constraints of the original table
        auto tableFields = transaction->getTableFields(layer->target_table_schema, layer->target_table_name);

        // check if the requirements of the primary key are satisfied
        // TODO: allow overriding the primarykey from the configfile
        std::vector<std::string> primaryKeyColumns;
        std::string geometryColumn;
        std::vector<std::string> insertColumns;
        std::vector<std::string> updateColumns;
        for(auto tableFieldPair : tableFields) {
            if (tableFieldPair.second.isPrimaryKey) {
                primaryKeyColumns.push_back(tableFieldPair.second.name);
            }
            else {
                updateColumns.push_back(tableFieldPair.second.name);
            }
            if (tableFieldPair.second.pgTypeName == "geometry") {
                if (!geometryColumn.empty()) {
                    throw WorkerError("Layer \"" + job->getLayerName() + "\" has multiple geometry columns. Currently only one is supported");
                }
                geometryColumn = tableFieldPair.second.name;
                insertColumns.push_back(tableFieldPair.second.name);
            }
            if (ogrFields.find(tableFieldPair.second.name) != ogrFields.end()) {
                insertColumns.push_back(tableFieldPair.second.name);
            }
        }
        if (primaryKeyColumns.empty()) {
            throw WorkerError("Got no primarykey for layer \"" + job->getLayerName() + "\"");
        }
        std::vector<std::string> missingPrimaryKeysSource;
        for( auto primaryKeyCol : primaryKeyColumns) {
            if (ogrFields.find(primaryKeyCol) == ogrFields.end()) {
                missingPrimaryKeysSource.push_back(primaryKeyCol);
            }
        }
        if (!missingPrimaryKeysSource.empty()) {
            throw WorkerError("The source for layer \"" + job->getLayerName() + "\" is missing the following fields required "+
                    "by the primary key: " + StringUtils::join(missingPrimaryKeysSource, ", "));
        }

        // prepare an insert query into the temporary table
        std::vector<std::string> insertQueryValues;
        unsigned int idxColumn = 1;
        for (std::string insertColumn : insertColumns) {
            std::stringstream colStream;
            colStream << "$" << idxColumn << "::" << tableFields[insertColumn].pgTypeName;
            insertQueryValues.push_back(colStream.str());
            idxColumn++;
        }
        std::stringstream insertQueryStream;
        // TODO: include st_transform statement into insert if original table has a srid set in geometry_columns
        insertQueryStream   << "insert into \"" << tempTableName << "\" (\""
                            << StringUtils::join(insertColumns, "\", \"")
                            << "\") values ("
                            << StringUtils::join(insertQueryValues, ", ")
                            << ")";
        poco_debug(logger, insertQueryStream.str().c_str());
        std::string insertStmtName = "batyr_insert" + job->getId();
        auto resInsertStmt = transaction->prepare(insertStmtName, insertQueryStream.str(), insertColumns.size(), NULL);

        OGRFeature * ogrFeature = 0;
        while( (ogrFeature = ogrLayer->GetNextFeature()) != nullptr) {
            std::vector<std::string> strValues;

            for (std::string insertColumn : insertColumns) {
                auto tableField = &tableFields[insertColumn];

                if (tableField->pgTypeName == "geometry") {
                    // TODO: Maybe use the implementation from OGRPGLayer::GeometryToHex
                    GByte * buffer;

                    auto ogrGeometry = ogrFeature->GetGeometryRef();
                    int bufferSize = ogrGeometry->WkbSize();

                    buffer = (GByte *) CPLMalloc(bufferSize);
                    if (buffer == nullptr) {
                        throw WorkerError("Unable to allocate memory to export geometry");
                    }
                    if (ogrGeometry->exportToWkb(wkbNDR, buffer) != OGRERR_NONE) {
                        OGRFree(buffer);
                        throw WorkerError("Could not export the geometry from feature #" + std::to_string(numPulled));
                    }
                    char * hexBuffer = CPLBinaryToHex(bufferSize, buffer);
                    if (hexBuffer == nullptr) {
                        OGRFree(buffer);
                        throw WorkerError("Unable to allocate memory to convert geometry to hex");
                    }
                    OGRFree(buffer);
                    strValues.push_back(std::string(hexBuffer));
                    CPLFree(hexBuffer);
                }
                else {
                    auto ogrField = &ogrFields[insertColumn];
                    switch (ogrField->type) {
                        case OFTString:
                            strValues.push_back(std::string(ogrFeature->GetFieldAsString(ogrField->index)));
                            break;
                        case OFTInteger:
                            strValues.push_back(std::to_string(ogrFeature->GetFieldAsInteger(ogrField->index)));
                            break;
                        case OFTReal:
                            strValues.push_back(std::to_string(ogrFeature->GetFieldAsDouble(ogrField->index)));
                            break;
                        // TODO: implment all of the OGRFieldType types
                        default:
                            throw WorkerError("Unsupported OGR field type: " + std::to_string(static_cast<int>(ogrField->type)));
                    }
                }
            }

            // convert to an array of c strings
            std::vector<const char*> cStrValues;
            std::vector<int> cStrValueLenghts;
            std::transform(strValues.begin(), strValues.end(), std::back_inserter(cStrValues),
                        [](std::string & s){ return s.c_str();});
            std::transform(strValues.begin(), strValues.end(), std::back_inserter(cStrValueLenghts),
                        [](std::string & s){ return s.length();});

            transaction->execPrepared(insertStmtName, cStrValues.size(), &cStrValues[0], &cStrValueLenghts[0],
                        NULL, 1);

            numPulled++;
        }
        job->setStatistics(numPulled, numCreated, numUpdated, numDeleted);

        // update the existing table only touching rows which have differences to prevent
        // slowdowns by triggers
        std::stringstream updateStmt;
        updateStmt          << "update \"" << layer->target_table_schema << "\".\"" << layer->target_table_name << "\" "
                            << " set ";
        bool firstIter = true;
        for (std::string updateColumn : updateColumns) {
            if (!firstIter) {
                updateStmt << ", ";
            }
            updateStmt << "\"" << updateColumn << "\" = \"" << tempTableName << "\".\"" << updateColumn << "\" ";
            firstIter = false;
        }
        updateStmt          << " from \"" << tempTableName << "\""
                            << " where (";
        firstIter = true;
        for (std::string primaryKeyColumn : primaryKeyColumns) {
            if (!firstIter) {
                updateStmt << " and ";
            }
            updateStmt  << "\""  << layer->target_table_name << "\".\"" << primaryKeyColumn
                        << "\" is not distinct from \"" << tempTableName << "\".\"" << primaryKeyColumn << "\"";
            firstIter = false;
        }
        updateStmt          << ") and (";
        firstIter = true;
        for (std::string updateColumn : updateColumns) {
            if (!firstIter) {
                updateStmt << " or ";
            }
            updateStmt  << "(\"" << layer->target_table_name << "\".\"" << updateColumn
                        << "\" is distinct from  \""
                        << tempTableName << "\".\"" << updateColumn << "\")";
            firstIter = false;
        }
        updateStmt          << ")";
        auto updateRes = transaction->exec(updateStmt.str());
        numUpdated = std::atoi(PQcmdTuples(updateRes.get()));
        updateRes.reset(NULL); // immediately dispose the result

        // insert missing rows in the exisiting table
        std::stringstream insertMissingStmt;
        insertMissingStmt   << "insert into \"" << layer->target_table_schema << "\".\"" << layer->target_table_name << "\" "
                            << " ( \"" << StringUtils::join(insertColumns, "\", \"") << "\") "
                            << " select \"" << StringUtils::join(insertColumns, "\", \"") << "\" "
                            << " from \"" << tempTableName << "\""
                            << " where (\"" << StringUtils::join(primaryKeyColumns, "\", \"") << "\") not in ("
                            << " select \"" << StringUtils::join(primaryKeyColumns, "\",\"") << "\" "
                            << "       from \"" << layer->target_table_schema << "\".\""  << layer->target_table_name << "\""
                            << ")";
        auto insertMissingRes = transaction->exec(insertMissingStmt.str());
        numCreated = std::atoi(PQcmdTuples(insertMissingRes.get()));
        insertMissingRes.reset(NULL); // immediately dispose the result

        // delete deprecated rows from the exisiting table
        // TODO: make this optional and skip when a filter is used
        std::stringstream deleteRemovedStmt;
        deleteRemovedStmt   << "delete from \"" << layer->target_table_schema << "\".\"" << layer->target_table_name << "\" "
                            << " where (\"" << StringUtils::join(primaryKeyColumns, "\", \"") << "\") not in ("
                            << " select \"" << StringUtils::join(primaryKeyColumns, "\",\"") << "\" "
                            << "       from \"" << tempTableName << "\""
                            << ")";
        auto deleteRemovedRes = transaction->exec(deleteRemovedStmt.str());
        numDeleted = std::atoi(PQcmdTuples(deleteRemovedRes.get()));
        deleteRemovedRes.reset(NULL); // immediately dispose the result

        job->setStatus(Job::Status::FINISHED);
        job->setStatistics(numPulled, numCreated, numUpdated, numDeleted);
    }
    else {
        std::string msg("Could not start a database transaction");
        poco_error(logger, msg.c_str());
        job->setStatus(Job::Status::FAILED);
        job->setMessage(msg);
    }

}

void
Worker::run()
{
    while (true) {
        Job::Ptr job;
        try {
            bool got_job = jobs->pop(job);
            if (!got_job) {
                // no job means the queue recieved a quit command, so the worker
                // can be shut down
                break;
            }
            poco_debug(logger, "Got job from queue");

            job->setStatus(Job::Status::IN_PROCESS);

            // check if we got a working database connection
            // or block until we got one
            size_t reconnectAttempts = 0;
            while(!db.reconnect(true)) {
                if (reconnectAttempts == 0) {
                    // set job message to inform clients we are waiting here
                    job->setMessage("Waiting to aquire a database connection");
                }
                reconnectAttempts++;
                std::this_thread::sleep_for( std::chrono::milliseconds( SERVER_DB_RECONNECT_WAIT ) );
            }

            pull(job);
        }
        catch (Batyr::Db::DbError &e) {
            poco_error(logger, e.what());
            job->setStatus(Job::Status::FAILED);
            job->setMessage(e.what());
        }
        catch (WorkerError &e) {
            poco_error(logger, e.what());
            job->setStatus(Job::Status::FAILED);
            job->setMessage(e.what());
        }
        catch (std::runtime_error &e) {
            poco_error(logger, e.what());
            job->setStatus(Job::Status::FAILED);
            job->setMessage(e.what());

            // do not know how this exception was caused as it
            // was not handled by one of the earlier catch blocks
            throw;
        }
    }
    poco_debug(logger, "leaving run method");
}
<|endoftext|>


<commit_before>#include "adaptiverandom.h"

#include <qmath.h>
#include <cstdlib>

namespace
{
	AdaptiveRandom instance;
}

RpnOperand AdaptiveRandom::calculate(FunctionCalculator *calculator, QList<RpnOperand> actualArguments)
{
	m_calculator = calculator;
	m_functionName = actualArguments[0].value.value<QString>();

	m_sourcePoint = actualArguments[1].value.value<QList<Number> >();
	if (m_calculator->functionArguments(m_functionName).size() != m_sourcePoint.size()) {
		THROW(EWrongParametersCount(QObject::tr("Source point"), m_calculator->functionArguments(m_functionName).size()));
	}

	m_acceleration = actualArguments[2].value.value<Number>();
	m_decrease = actualArguments[3].value.value<Number>();
	m_wrongStepsCount = actualArguments[4].value.value<Number>();
	m_iterationsCount = actualArguments[5].value.value<Number>();
	m_minimumStepSize = actualArguments[6].value.value<Number>();
	m_stepSize = 1;

	// Initialize random
	srand(time(NULL));

	RpnOperand result;
	result.type = RpnOperandVector;
	result.value = QVariant::fromValue(findMinimum());
	return result;
}

QList<RpnArgument> AdaptiveRandom::requiredArguments()
{
	QList<RpnArgument> arguments;
	arguments
		// NOTE: QVariant() shows that number of arguments is not fixed, maybe there is other way
		<< RpnArgument(RpnOperandFunctionName, QString(), QVariant())
		<< RpnArgument(RpnOperandVector)
		<< RpnArgument(RpnOperandNumber)
		<< RpnArgument(RpnOperandNumber)
		<< RpnArgument(RpnOperandNumber)
		<< RpnArgument(RpnOperandNumber)
		<< RpnArgument(RpnOperandNumber);

	return arguments;
}

QList<Number> AdaptiveRandom::findMinimum()
{
	int failCount = 1;
	int iterationCount = 0;

	forever {
		QList<Number> randomPoint = generateRandomNumbers(m_sourcePoint.size(), -1, 1);

		QList<Number> currentPoint = sumListList(
			m_sourcePoint,
			productListNumber(
				quotientListNumber(randomPoint, modulusList(randomPoint)),
				m_stepSize
			)
		);

		if (countFunction(currentPoint) < countFunction(m_sourcePoint)) {
			QList<Number> newPoint = sumListList(
				m_sourcePoint,
				productListNumber(
					diffListList(currentPoint, m_sourcePoint),
					m_acceleration
				)
			);

			if (countFunction(newPoint) < countFunction(m_sourcePoint)) {
				m_sourcePoint = newPoint;
				m_stepSize *= m_acceleration;
				iterationCount++;

				if (iterationCount < m_iterationsCount) {
					failCount = 1;
					continue;
				}
				else {
					// Exit condition
					return m_sourcePoint;
				}
			}
		}

		if (failCount < m_wrongStepsCount) {
			failCount++;
		}
		else if (m_stepSize <= m_minimumStepSize) {
			// Exit condition
			return m_sourcePoint;
		}
		else {
			m_stepSize *= m_decrease;
			failCount = 1;
		}
	}
}

QList<Number> AdaptiveRandom::generateRandomNumbers(int count, Number lowerLimit, Number higherLimit)
{
	QList<Number> result;

	for (int i = 0; i < count; i++) {
		result << getRandomNumber(qAbs(lowerLimit - higherLimit)) + lowerLimit;
	}

	return result;
}

// Return a random number between 0 and limit
Number AdaptiveRandom::getRandomNumber(Number limit)
{
	Number result;

	do {
		result = rand() / (RAND_MAX / (limit + 1));
	} while (result > limit);

	return result;
}

QList<Number> AdaptiveRandom::productListNumber(QList<Number> list, Number number)
{
	QList<Number> result;

	foreach (Number element, list) {
		result << element * number;
	}

	return result;
}

QList<Number> AdaptiveRandom::diffListList(QList<Number> source, QList<Number> subtractin)
{
	QList<Number> result;

	for (int i = 0; i < source.size(); i++) {
		result << source[i] - subtractin[i];
	}

	return result;
}

QList<Number> AdaptiveRandom::sumListList(QList<Number> source, QList<Number> item)
{
	QList<Number> result;

	for (int i = 0; i < source.size(); i++) {
		result << source[i] + item[i];
	}

	return result;
}

Number AdaptiveRandom::productListList(QList<Number> source, QList<Number> item)
{
	Number result = 0;

	for (int i = 0; i < source.size(); i++) {
		result += source[i] * item[i];
	}

	return result;
}

Number AdaptiveRandom::modulusList(QList<Number> list)
{
	Number result = 0;

	foreach (Number element, list) {
		result += qPow(element, 2);
	}

	return qSqrt(result);
}

QList<Number> AdaptiveRandom::quotientListNumber(QList<Number> source, Number divisor)
{
	QList<Number> result;

	foreach (Number element, source) {
		result << element / divisor;
	}

	return result;
}

Number AdaptiveRandom::countFunction(QList<Number> arguments)
{
	QList<RpnOperand> functionArguments;

	foreach (Number argument, arguments) {
		RpnOperand functionArgument(RpnOperandNumber, argument);
		functionArguments << functionArgument;
	}

	RpnOperand result = m_calculator->calculate(m_functionName, functionArguments);
	return result.value.value<Number>();
}
<commit_msg>Including ctime instead of cstdlib for using time() function, because MSVC won't find time() in cstdlib.<commit_after>#include "adaptiverandom.h"

#include <qmath.h>
#include <ctime>

namespace
{
	AdaptiveRandom instance;
}

RpnOperand AdaptiveRandom::calculate(FunctionCalculator *calculator, QList<RpnOperand> actualArguments)
{
	m_calculator = calculator;
	m_functionName = actualArguments[0].value.value<QString>();

	m_sourcePoint = actualArguments[1].value.value<QList<Number> >();
	if (m_calculator->functionArguments(m_functionName).size() != m_sourcePoint.size()) {
		THROW(EWrongParametersCount(QObject::tr("Source point"), m_calculator->functionArguments(m_functionName).size()));
	}

	m_acceleration = actualArguments[2].value.value<Number>();
	m_decrease = actualArguments[3].value.value<Number>();
	m_wrongStepsCount = actualArguments[4].value.value<Number>();
	m_iterationsCount = actualArguments[5].value.value<Number>();
	m_minimumStepSize = actualArguments[6].value.value<Number>();
	m_stepSize = 1;

	// Initialize random
	srand(time(NULL));

	RpnOperand result;
	result.type = RpnOperandVector;
	result.value = QVariant::fromValue(findMinimum());
	return result;
}

QList<RpnArgument> AdaptiveRandom::requiredArguments()
{
	QList<RpnArgument> arguments;
	arguments
		// NOTE: QVariant() shows that number of arguments is not fixed, maybe there is other way
		<< RpnArgument(RpnOperandFunctionName, QString(), QVariant())
		<< RpnArgument(RpnOperandVector)
		<< RpnArgument(RpnOperandNumber)
		<< RpnArgument(RpnOperandNumber)
		<< RpnArgument(RpnOperandNumber)
		<< RpnArgument(RpnOperandNumber)
		<< RpnArgument(RpnOperandNumber);

	return arguments;
}

QList<Number> AdaptiveRandom::findMinimum()
{
	int failCount = 1;
	int iterationCount = 0;

	forever {
		QList<Number> randomPoint = generateRandomNumbers(m_sourcePoint.size(), -1, 1);

		QList<Number> currentPoint = sumListList(
			m_sourcePoint,
			productListNumber(
				quotientListNumber(randomPoint, modulusList(randomPoint)),
				m_stepSize
			)
		);

		if (countFunction(currentPoint) < countFunction(m_sourcePoint)) {
			QList<Number> newPoint = sumListList(
				m_sourcePoint,
				productListNumber(
					diffListList(currentPoint, m_sourcePoint),
					m_acceleration
				)
			);

			if (countFunction(newPoint) < countFunction(m_sourcePoint)) {
				m_sourcePoint = newPoint;
				m_stepSize *= m_acceleration;
				iterationCount++;

				if (iterationCount < m_iterationsCount) {
					failCount = 1;
					continue;
				}
				else {
					// Exit condition
					return m_sourcePoint;
				}
			}
		}

		if (failCount < m_wrongStepsCount) {
			failCount++;
		}
		else if (m_stepSize <= m_minimumStepSize) {
			// Exit condition
			return m_sourcePoint;
		}
		else {
			m_stepSize *= m_decrease;
			failCount = 1;
		}
	}
}

QList<Number> AdaptiveRandom::generateRandomNumbers(int count, Number lowerLimit, Number higherLimit)
{
	QList<Number> result;

	for (int i = 0; i < count; i++) {
		result << getRandomNumber(qAbs(lowerLimit - higherLimit)) + lowerLimit;
	}

	return result;
}

// Return a random number between 0 and limit
Number AdaptiveRandom::getRandomNumber(Number limit)
{
	Number result;

	do {
		result = rand() / (RAND_MAX / (limit + 1));
	} while (result > limit);

	return result;
}

QList<Number> AdaptiveRandom::productListNumber(QList<Number> list, Number number)
{
	QList<Number> result;

	foreach (Number element, list) {
		result << element * number;
	}

	return result;
}

QList<Number> AdaptiveRandom::diffListList(QList<Number> source, QList<Number> subtractin)
{
	QList<Number> result;

	for (int i = 0; i < source.size(); i++) {
		result << source[i] - subtractin[i];
	}

	return result;
}

QList<Number> AdaptiveRandom::sumListList(QList<Number> source, QList<Number> item)
{
	QList<Number> result;

	for (int i = 0; i < source.size(); i++) {
		result << source[i] + item[i];
	}

	return result;
}

Number AdaptiveRandom::productListList(QList<Number> source, QList<Number> item)
{
	Number result = 0;

	for (int i = 0; i < source.size(); i++) {
		result += source[i] * item[i];
	}

	return result;
}

Number AdaptiveRandom::modulusList(QList<Number> list)
{
	Number result = 0;

	foreach (Number element, list) {
		result += qPow(element, 2);
	}

	return qSqrt(result);
}

QList<Number> AdaptiveRandom::quotientListNumber(QList<Number> source, Number divisor)
{
	QList<Number> result;

	foreach (Number element, source) {
		result << element / divisor;
	}

	return result;
}

Number AdaptiveRandom::countFunction(QList<Number> arguments)
{
	QList<RpnOperand> functionArguments;

	foreach (Number argument, arguments) {
		RpnOperand functionArgument(RpnOperandNumber, argument);
		functionArguments << functionArgument;
	}

	RpnOperand result = m_calculator->calculate(m_functionName, functionArguments);
	return result.value.value<Number>();
}
<|endoftext|>


<commit_before>/*
    vcflib C++ library for parsing and manipulating VCF files

    Copyright © 2010-2020 Erik Garrison
    Copyright © 2020      Pjotr Prins

    This software is published under the MIT License. See the LICENSE file.
*/

#include "Variant.h"
#include <set>

using namespace std;
using namespace vcflib;

int main(int argc, char** argv) {
if (argc == 2) {
  string h_flag = argv[1];
  if (h_flag == "-h" || h_flag == "--help") {
      cerr << R"(

List unique alleles For each record, remove any duplicate alternate
alleles that may have resulted from merging separate VCF files.

Usage: vcfuniqalleles <vcf file>

Type: filter

      )";
      exit(1);
    }
  }

    VariantCallFile variantFile;

    if (argc > 1) {
        string filename = argv[1];
        variantFile.open(filename);
    } else {
        variantFile.open(std::cin);
    }

    if (!variantFile.is_open()) {
        return 1;
    }

    cout << variantFile.header << endl;

    string lastsn;
    long int lastpos;
    string lastref;
    vector<string> lastalt;

    Variant var(variantFile);
    while (variantFile.getNextVariant(var)) {
        set<string> alleles;
        vector<string> alleles_to_remove;
        for (vector<string>::iterator a = var.alt.begin(); a != var.alt.end(); ++a) {
            if (*a != var.ref) {
                if (alleles.find(*a) == alleles.end()) {
                    alleles.insert(*a);
                } else {
                    alleles_to_remove.push_back(*a);
                }
            } else {
                alleles_to_remove.push_back(*a); // same as ref
            }
        }
        for (vector<string>::iterator a = alleles_to_remove.begin(); a != alleles_to_remove.end(); ++a) {
            cerr << "removing " << *a << " from " << var.sequenceName << ":" << var.position << endl;
            var.removeAlt(*a);
        }
        cout << var << endl;
    }

    return 0;

}
<commit_msg>vcfuniqalleles: remove unused variables.<commit_after>/*
    vcflib C++ library for parsing and manipulating VCF files

    Copyright © 2010-2020 Erik Garrison
    Copyright © 2020      Pjotr Prins

    This software is published under the MIT License. See the LICENSE file.
*/

#include "Variant.h"
#include <set>

using namespace std;
using namespace vcflib;

int main(int argc, char** argv) {
if (argc == 2) {
  string h_flag = argv[1];
  if (h_flag == "-h" || h_flag == "--help") {
      cerr << R"(

List unique alleles For each record, remove any duplicate alternate
alleles that may have resulted from merging separate VCF files.

Usage: vcfuniqalleles <vcf file>

Type: filter

      )";
      exit(1);
    }
  }

    VariantCallFile variantFile;

    if (argc > 1) {
        string filename = argv[1];
        variantFile.open(filename);
    } else {
        variantFile.open(std::cin);
    }

    if (!variantFile.is_open()) {
        return 1;
    }

    cout << variantFile.header << endl;

    Variant var(variantFile);
    while (variantFile.getNextVariant(var)) {
        set<string> alleles;
        vector<string> alleles_to_remove;
        for (vector<string>::iterator a = var.alt.begin(); a != var.alt.end(); ++a) {
            if (*a != var.ref) {
                if (alleles.find(*a) == alleles.end()) {
                    alleles.insert(*a);
                } else {
                    alleles_to_remove.push_back(*a);
                }
            } else {
                alleles_to_remove.push_back(*a); // same as ref
            }
        }
        for (vector<string>::iterator a = alleles_to_remove.begin(); a != alleles_to_remove.end(); ++a) {
            cerr << "removing " << *a << " from " << var.sequenceName << ":" << var.position << endl;
            var.removeAlt(*a);
        }
        cout << var << endl;
    }

    return 0;

}
<|endoftext|>


<commit_before>#include "addgroupwindow.h"
#include "functions.h"

/**
 * Constructor of the AddGroupWindow class, generating its window.
 * @param	favorites	List of favorites tags, needed for coloration
 * @param	parent		The parent window
 */
AddGroupWindow::AddGroupWindow(QStringList sites, QStringList favorites, mainWindow *parent) : QWidget(parent), m_parent(parent), m_sites(sites)
{
	QVBoxLayout *layout = new QVBoxLayout;
		QFormLayout *formLayout = new QFormLayout;
			m_comboSites = new QComboBox;
				m_comboSites->setMaxVisibleItems(20);
				m_comboSites->addItems(m_sites);
				formLayout->addRow(tr("&Site"), m_comboSites);
			m_lineTags = new TextEdit(favorites, this);
				m_lineTags->setContextMenuPolicy(Qt::CustomContextMenu);
				QStringList completion;
					QFile words("words.txt");
					if (words.open(QIODevice::ReadOnly | QIODevice::Text))
					{
						while (!words.atEnd())
						{
							QByteArray line = words.readLine();
							completion.append(QString(line).remove("\r\n").remove("\n").split(" ", QString::SkipEmptyParts));
						}
						QCompleter *completer = new QCompleter(completion, this);
						completer->setCaseSensitivity(Qt::CaseInsensitive);
						m_lineTags->setCompleter(completer);
					}
				formLayout->addRow(tr("&Tags"), m_lineTags);
			m_spinPage = new QSpinBox;
				m_spinPage->setRange(1, 1000);
				m_spinPage->setValue(1);
				formLayout->addRow(tr("&Page"), m_spinPage);
			m_spinPP = new QSpinBox;
				m_spinPP->setRange(1, 1000);
				m_spinPP->setValue(100);
				formLayout->addRow(tr("&Images par page"), m_spinPP);
			m_spinLimit = new QSpinBox;
				m_spinLimit->setRange(1, 1000000);
				m_spinLimit->setValue(100);
				formLayout->addRow(tr("&Limite d'images"), m_spinLimit);
			m_comboDwl = new QComboBox;
				m_comboDwl->setMaxVisibleItems(20);
				m_comboDwl->addItems(QStringList() << tr("Oui") << tr("Non"));
				m_comboDwl->setCurrentIndex(1);
				formLayout->addRow(tr("&Tlcharger les image de la liste noire"), m_comboDwl);
			layout->addLayout(formLayout);
		QHBoxLayout *layoutButtons = new QHBoxLayout;
			QPushButton *buttonOk = new QPushButton(tr("Ok"));
				connect(buttonOk, SIGNAL(clicked()), this, SLOT(ok()));
				layoutButtons->addWidget(buttonOk);
			QPushButton *buttonClose = new QPushButton(tr("Fermer"));
				connect(buttonClose, SIGNAL(clicked()), this, SLOT(close()));
				layoutButtons->addWidget(buttonClose);
			layout->addLayout(layoutButtons);
	this->setLayout(layout);
	this->setWindowIcon(QIcon(":/images/icon.ico"));
	this->setWindowTitle(tr("Grabber")+" - "+tr("Ajouter groupe"));
	this->setWindowFlags(Qt::Window);
	this->resize(QSize(400, 0));
}

/**
 * Relays the informations to the parent window.
 */
void AddGroupWindow::ok()
{
	QSettings *settings = new QSettings(savePath("settings.ini"), QSettings::IniFormat);
	QStringList bools = QStringList() << "true" << "false";
	QStringList values = QStringList() << m_lineTags->toPlainText() << QString::number(m_spinPage->value()) << QString::number(m_spinPP->value()) << QString::number(m_spinLimit->value()) << bools.at(m_comboDwl->currentIndex()) << m_sites.at(m_comboSites->currentIndex()) << "false" << settings->value("Save/filename").toString() << settings->value("Save/path").toString() << "";
	m_parent->batchAddGroup(values);
	this->close();
}
<commit_msg>Fixed addgroupwindow.cpp bug (remaining popular boolean removed)<commit_after>#include "addgroupwindow.h"
#include "functions.h"

/**
 * Constructor of the AddGroupWindow class, generating its window.
 * @param	favorites	List of favorites tags, needed for coloration
 * @param	parent		The parent window
 */
AddGroupWindow::AddGroupWindow(QStringList sites, QStringList favorites, mainWindow *parent) : QWidget(parent), m_parent(parent), m_sites(sites)
{
	QVBoxLayout *layout = new QVBoxLayout;
		QFormLayout *formLayout = new QFormLayout;
			m_comboSites = new QComboBox;
				m_comboSites->setMaxVisibleItems(20);
				m_comboSites->addItems(m_sites);
				formLayout->addRow(tr("&Site"), m_comboSites);
			m_lineTags = new TextEdit(favorites, this);
				m_lineTags->setContextMenuPolicy(Qt::CustomContextMenu);
				QStringList completion;
					QFile words("words.txt");
					if (words.open(QIODevice::ReadOnly | QIODevice::Text))
					{
						while (!words.atEnd())
						{
							QByteArray line = words.readLine();
							completion.append(QString(line).remove("\r\n").remove("\n").split(" ", QString::SkipEmptyParts));
						}
						QCompleter *completer = new QCompleter(completion, this);
						completer->setCaseSensitivity(Qt::CaseInsensitive);
						m_lineTags->setCompleter(completer);
					}
				formLayout->addRow(tr("&Tags"), m_lineTags);
			m_spinPage = new QSpinBox;
				m_spinPage->setRange(1, 1000);
				m_spinPage->setValue(1);
				formLayout->addRow(tr("&Page"), m_spinPage);
			m_spinPP = new QSpinBox;
				m_spinPP->setRange(1, 1000);
				m_spinPP->setValue(100);
				formLayout->addRow(tr("&Images par page"), m_spinPP);
			m_spinLimit = new QSpinBox;
				m_spinLimit->setRange(1, 1000000);
				m_spinLimit->setValue(100);
				formLayout->addRow(tr("&Limite d'images"), m_spinLimit);
			m_comboDwl = new QComboBox;
				m_comboDwl->setMaxVisibleItems(20);
				m_comboDwl->addItems(QStringList() << tr("Oui") << tr("Non"));
				m_comboDwl->setCurrentIndex(1);
				formLayout->addRow(tr("&Tlcharger les image de la liste noire"), m_comboDwl);
			layout->addLayout(formLayout);
		QHBoxLayout *layoutButtons = new QHBoxLayout;
			QPushButton *buttonOk = new QPushButton(tr("Ok"));
				connect(buttonOk, SIGNAL(clicked()), this, SLOT(ok()));
				layoutButtons->addWidget(buttonOk);
			QPushButton *buttonClose = new QPushButton(tr("Fermer"));
				connect(buttonClose, SIGNAL(clicked()), this, SLOT(close()));
				layoutButtons->addWidget(buttonClose);
			layout->addLayout(layoutButtons);
	this->setLayout(layout);
	this->setWindowIcon(QIcon(":/images/icon.ico"));
	this->setWindowTitle(tr("Grabber")+" - "+tr("Ajouter groupe"));
	this->setWindowFlags(Qt::Window);
	this->resize(QSize(400, 0));
}

/**
 * Relays the informations to the parent window.
 */
void AddGroupWindow::ok()
{
	QSettings *settings = new QSettings(savePath("settings.ini"), QSettings::IniFormat);
	QStringList bools = QStringList() << "true" << "false";
	QStringList values = QStringList() << m_lineTags->toPlainText() << QString::number(m_spinPage->value()) << QString::number(m_spinPP->value()) << QString::number(m_spinLimit->value()) << bools.at(m_comboDwl->currentIndex()) << m_sites.at(m_comboSites->currentIndex()) << settings->value("Save/filename").toString() << settings->value("Save/path").toString() << "";
	m_parent->batchAddGroup(values);
	this->close();
}
<|endoftext|>


<commit_before>/*******************************************************************************
 * src/benchmark.cpp
 *
 * Copyright (C) 2017 Florian Kurpicz <florian.kurpicz@tu-dortmund.de>
 *
 * All rights reserved. Published under the BSD-2 license in the LICENSE file.
 ******************************************************************************/

#include <tclap/CmdLine.h>
#include <vector>

#include "benchmark/algorithm.hpp"
#include "util/alphabet_util.hpp"
#include "util/file_util.hpp"

#ifdef MALLOC_COUNT
#include "benchmark/malloc_count.h"
#endif // MALLOC_COUNT

auto filter_parallel(bool only_parallel, bool is_parallel) {
  return (!only_parallel || is_parallel);
}

auto filter_sequential(bool sequential, bool is_parallel) {
  return (!sequential || !is_parallel);
}

auto filter_wavelet_type(bool is_tree, bool no_trees, bool no_matrices) {
  return (is_tree ? !no_trees : !no_matrices);
}

int32_t main(int32_t argc, char const* argv[]) {
  TCLAP::CmdLine cmd("Benchmark for wavelet tree (and matrix) construction",
    ' ', "0.2");

  TCLAP::SwitchArg list_all_algorithms("l", "list",
    "Print the name and description of all registered algorithms", false);
  cmd.add(list_all_algorithms);
  TCLAP::MultiArg<std::string> file_path_arg("f", "file",
    "Path to the text file.", false, "string");
  cmd.add(file_path_arg);
  TCLAP::ValueArg<std::string> filter_arg("n", "name",
    "Runs all algorithms that contain the <name> in their name", false, "",
    "string");
  cmd.add(filter_arg);
  TCLAP::ValueArg<int32_t> word_width_arg("b", "byte",
    "Bytes per char in the input text.", false, 1, "uint8_t");
  cmd.add(word_width_arg);
  TCLAP::ValueArg<int32_t> nr_runs_arg("r", "runs",
    "Number of repetitions of the construction algorithm.",
    false, 5, "int32_t");
  cmd.add(nr_runs_arg);
  TCLAP::SwitchArg run_only_parallel_arg("p", "parallel",
    "Run only parallel construction algorithms.", false);
  cmd.add(run_only_parallel_arg);
  TCLAP::SwitchArg run_only_sequential_arg("s", "sequential",
    "Run only sequential construction algorithms.", false);
  cmd.add(run_only_sequential_arg);
  TCLAP::SwitchArg no_trees_arg("m", "no_trees",
    "Skip all wavelet trees construction algorithms.", false);
  cmd.add(no_trees_arg);
  TCLAP::SwitchArg no_matrices_arg("t", "no_matrices",
    "Skip all wavelet matrices construction algorithms.", false);
  cmd.add(no_matrices_arg);
  TCLAP::SwitchArg memory_arg("", "memory",
    "Compute peak memory during construction.", false);
  cmd.add(memory_arg);
  cmd.parse( argc, argv );

  auto& algo_list = algorithm_list::get_algorithm_list();
  if (list_all_algorithms.getValue()) {
    for (const auto& a : algo_list) {
      a->print_info();
    }
    return 0;
  }

  const std::vector<std::string> file_paths = file_path_arg.getValue();
  std::string filter = filter_arg.getValue();
  const int32_t word_width = word_width_arg.getValue();
  const int32_t nr_runs = nr_runs_arg.getValue();
  const bool run_only_parallel = run_only_parallel_arg.getValue();
  const bool run_only_sequential = run_only_sequential_arg.getValue();
  const bool no_trees = no_trees_arg.getValue();
  const bool no_matrices = no_matrices_arg.getValue();
  const bool memory = memory_arg.getValue();

  for (const auto& path : file_paths) {
    std::cout << std::endl << "Text: " << path << std::endl;
    void* txt_prt = nullptr;
    uint64_t text_size = 0;
    uint64_t max_char = 0;
    uint64_t levels = 0;
    std::vector<uint8_t> text_uint8;
    std::vector<uint16_t> text_uint16;
    std::vector<uint32_t> text_uint32;
    std::vector<uint64_t> text_uint64;
#ifdef MALLOC_COUNT
    malloc_count_reset_peak();
#endif
    if (word_width == 1) {
      text_uint8 = file_to_vector<1>(path);
      text_size = text_uint8.size();
      max_char = reduce_alphabet(text_uint8);
      levels = levels_for_max_char(max_char);
      txt_prt = &text_uint8;
    } else if (word_width == 2) {
      text_uint16 = file_to_vector<2>(path);
      text_size = text_uint16.size();
      max_char = reduce_alphabet(text_uint16);
      levels = levels_for_max_char(max_char);
      txt_prt = &text_uint16;
    } else if (word_width == 4) {
      text_uint32 = file_to_vector<4>(path);
      text_size = text_uint32.size();
      max_char = reduce_alphabet(text_uint32);
      levels = levels_for_max_char(max_char);
      txt_prt = &text_uint32;
    } else if (word_width == 8) {
      text_uint64 = file_to_vector<8>(path);
      text_size = text_uint64.size();
      max_char = reduce_alphabet(text_uint64);
      levels = levels_for_max_char(max_char);
      txt_prt = &text_uint64;
    } else {
      std::cerr << "You entered an invalid number of bytes per character "
                   "(parameter 'b')." << std::endl;
      return -1;
    }
    std::cout << "Characters: " << text_size << std::endl;
#ifdef MALLOC_COUNT
    std::cout << "Memory peak text: " << malloc_count_peak() << ", MB: "
              << malloc_count_peak() / (1024 * 1024) << std::endl;
#endif // MALLOC_COUNT
    for (const auto& a : algo_list) {
      if (filter == "" || (a->name().find(filter) != std::string::npos)) {
        if (a->word_width() == word_width) {
          if (filter_parallel(run_only_parallel, a->is_parallel())) {
            if (filter_sequential(run_only_sequential, a->is_parallel())) {
              if (filter_wavelet_type(a->is_tree(), no_trees, no_matrices)) {
                a->print_info();
                if (memory) {
#ifdef MALLOC_COUNT
                  malloc_count_reset_peak();
                  a->memory_peak(txt_prt, text_size, levels);
                  std::cout << malloc_count_peak() << ", MB: "
                            << malloc_count_peak() / (1024 * 1024) << std::endl;
#else
                  std::cout << "Memory measurement is NOT enabled."
                            << std::endl;
#endif // MALLOC_COUNT
                } else {
                  std::cout << a->median_time(
                    txt_prt, text_size, levels, nr_runs) << std::endl;
                }
              }
            }
          }
        }
      }
    }
  }
  return 0;
}

/******************************************************************************/
<commit_msg>fix confusing memory output<commit_after>/*******************************************************************************
 * src/benchmark.cpp
 *
 * Copyright (C) 2017 Florian Kurpicz <florian.kurpicz@tu-dortmund.de>
 *
 * All rights reserved. Published under the BSD-2 license in the LICENSE file.
 ******************************************************************************/

#include <tclap/CmdLine.h>
#include <vector>

#include "benchmark/algorithm.hpp"
#include "util/alphabet_util.hpp"
#include "util/file_util.hpp"

#ifdef MALLOC_COUNT
#include "benchmark/malloc_count.h"
#endif // MALLOC_COUNT

auto filter_parallel(bool only_parallel, bool is_parallel) {
  return (!only_parallel || is_parallel);
}

auto filter_sequential(bool sequential, bool is_parallel) {
  return (!sequential || !is_parallel);
}

auto filter_wavelet_type(bool is_tree, bool no_trees, bool no_matrices) {
  return (is_tree ? !no_trees : !no_matrices);
}

int32_t main(int32_t argc, char const* argv[]) {
  TCLAP::CmdLine cmd("Benchmark for wavelet tree (and matrix) construction",
    ' ', "0.2");

  TCLAP::SwitchArg list_all_algorithms("l", "list",
    "Print the name and description of all registered algorithms", false);
  cmd.add(list_all_algorithms);
  TCLAP::MultiArg<std::string> file_path_arg("f", "file",
    "Path to the text file.", false, "string");
  cmd.add(file_path_arg);
  TCLAP::ValueArg<std::string> filter_arg("n", "name",
    "Runs all algorithms that contain the <name> in their name", false, "",
    "string");
  cmd.add(filter_arg);
  TCLAP::ValueArg<int32_t> word_width_arg("b", "byte",
    "Bytes per char in the input text.", false, 1, "uint8_t");
  cmd.add(word_width_arg);
  TCLAP::ValueArg<int32_t> nr_runs_arg("r", "runs",
    "Number of repetitions of the construction algorithm.",
    false, 5, "int32_t");
  cmd.add(nr_runs_arg);
  TCLAP::SwitchArg run_only_parallel_arg("p", "parallel",
    "Run only parallel construction algorithms.", false);
  cmd.add(run_only_parallel_arg);
  TCLAP::SwitchArg run_only_sequential_arg("s", "sequential",
    "Run only sequential construction algorithms.", false);
  cmd.add(run_only_sequential_arg);
  TCLAP::SwitchArg no_trees_arg("m", "no_trees",
    "Skip all wavelet trees construction algorithms.", false);
  cmd.add(no_trees_arg);
  TCLAP::SwitchArg no_matrices_arg("t", "no_matrices",
    "Skip all wavelet matrices construction algorithms.", false);
  cmd.add(no_matrices_arg);
  TCLAP::SwitchArg memory_arg("", "memory",
    "Compute peak memory during construction.", false);
  cmd.add(memory_arg);
  cmd.parse( argc, argv );

  auto& algo_list = algorithm_list::get_algorithm_list();
  if (list_all_algorithms.getValue()) {
    for (const auto& a : algo_list) {
      a->print_info();
    }
    return 0;
  }

  const std::vector<std::string> file_paths = file_path_arg.getValue();
  std::string filter = filter_arg.getValue();
  const int32_t word_width = word_width_arg.getValue();
  const int32_t nr_runs = nr_runs_arg.getValue();
  const bool run_only_parallel = run_only_parallel_arg.getValue();
  const bool run_only_sequential = run_only_sequential_arg.getValue();
  const bool no_trees = no_trees_arg.getValue();
  const bool no_matrices = no_matrices_arg.getValue();
  const bool memory = memory_arg.getValue();

  for (const auto& path : file_paths) {
    std::cout << std::endl << "Text: " << path << std::endl;
    void* txt_prt = nullptr;
    uint64_t text_size = 0;
    uint64_t max_char = 0;
    uint64_t levels = 0;
    std::vector<uint8_t> text_uint8;
    std::vector<uint16_t> text_uint16;
    std::vector<uint32_t> text_uint32;
    std::vector<uint64_t> text_uint64;
#ifdef MALLOC_COUNT
    malloc_count_reset_peak();
#endif
    if (word_width == 1) {
      text_uint8 = file_to_vector<1>(path);
      text_size = text_uint8.size();
      max_char = reduce_alphabet(text_uint8);
      levels = levels_for_max_char(max_char);
      txt_prt = &text_uint8;
    } else if (word_width == 2) {
      text_uint16 = file_to_vector<2>(path);
      text_size = text_uint16.size();
      max_char = reduce_alphabet(text_uint16);
      levels = levels_for_max_char(max_char);
      txt_prt = &text_uint16;
    } else if (word_width == 4) {
      text_uint32 = file_to_vector<4>(path);
      text_size = text_uint32.size();
      max_char = reduce_alphabet(text_uint32);
      levels = levels_for_max_char(max_char);
      txt_prt = &text_uint32;
    } else if (word_width == 8) {
      text_uint64 = file_to_vector<8>(path);
      text_size = text_uint64.size();
      max_char = reduce_alphabet(text_uint64);
      levels = levels_for_max_char(max_char);
      txt_prt = &text_uint64;
    } else {
      std::cerr << "You entered an invalid number of bytes per character "
                   "(parameter 'b')." << std::endl;
      return -1;
    }
    std::cout << "Characters: " << text_size << std::endl;
#ifdef MALLOC_COUNT
    std::cout << "Memory peak text: " << malloc_count_peak() << ", MB: "
              << malloc_count_peak() / (1024 * 1024) << std::endl;
#endif // MALLOC_COUNT
    for (const auto& a : algo_list) {
      if (filter == "" || (a->name().find(filter) != std::string::npos)) {
        if (a->word_width() == word_width) {
          if (filter_parallel(run_only_parallel, a->is_parallel())) {
            if (filter_sequential(run_only_sequential, a->is_parallel())) {
              if (filter_wavelet_type(a->is_tree(), no_trees, no_matrices)) {
                a->print_info();
                if (memory) {
#ifdef MALLOC_COUNT
                  malloc_count_reset_peak();
                  a->memory_peak(txt_prt, text_size, levels);
                  std::cout << "B: "
                            << malloc_count_peak() << ", MB: "
                            << malloc_count_peak() / (1024 * 1024) << std::endl;
#else
                  std::cout << "Memory measurement is NOT enabled."
                            << std::endl;
#endif // MALLOC_COUNT
                } else {
                  std::cout << a->median_time(
                    txt_prt, text_size, levels, nr_runs) << std::endl;
                }
              }
            }
          }
        }
      }
    }
  }
  return 0;
}

/******************************************************************************/
<|endoftext|>