<commit_before>/*
Copyright (c) 2013 Microsoft Corporation. All rights reserved.
Released under Apache 2.0 license as described in the file LICENSE.

Author: Leonardo de Moura
*/
#if defined(LEAN_WINDOWS) && !defined(LEAN_CYGWIN)
#include <windows.h>
#endif
#include <string>
#include <cstdlib>
#include <fstream>
#include <vector>
#include <sys/types.h>
#include <sys/stat.h>
#include <dirent.h>
#include "util/exception.h"
#include "util/sstream.h"
#include "util/name.h"
#include "util/optional.h"
#include "util/realpath.h"
#include "util/lean_path.h"

#ifndef LEAN_DEFAULT_MODULE_FILE_NAME
#define LEAN_DEFAULT_MODULE_FILE_NAME "default"
#endif

namespace lean {
file_not_found_exception::file_not_found_exception(std::string const & fname):
    exception(sstream() << "file '" << fname << "' not found in the LEAN_PATH"),
    m_fname(fname) {}

static std::string * g_default_file_name             = nullptr;
static std::string *              g_lean_path        = nullptr;
static std::vector<std::string> * g_lean_path_vector = nullptr;

bool is_directory(char const * pathname) {
    struct stat info;
    if (stat(pathname, &info) != 0 )
        return false; // failed to access pathname
    return info.st_mode & S_IFDIR;
}

#if defined(LEAN_EMSCRIPTEN)
// emscripten version
static char g_path_sep     = ':';
static char g_sep          = '/';
static char g_bad_sep      = '\\';
bool is_path_sep(char c) { return c == g_path_sep; }
#elif defined(LEAN_WINDOWS) && !defined(LEAN_CYGWIN)
// Windows version
static char g_path_sep     = ';';
static char g_path_alt_sep = ':';
static char g_sep          = '\\';
static char g_bad_sep      = '/';
std::string get_exe_location() {
    HMODULE hModule = GetModuleHandleW(NULL);
    WCHAR path[MAX_PATH];
    GetModuleFileNameW(hModule, path, MAX_PATH);
    std::wstring pathstr(path);
    return std::string(pathstr.begin(), pathstr.end());
}
bool is_path_sep(char c) { return c == g_path_sep || c == g_path_alt_sep; }
#elif defined(__APPLE__)
// OSX version
#include <mach-o/dyld.h>
#include <limits.h>
#include <stdlib.h>
static char g_path_sep     = ':';
static char g_sep          = '/';
static char g_bad_sep      = '\\';
std::string get_exe_location() {
    char buf1[PATH_MAX];
    char buf2[PATH_MAX];
    uint32_t bufsize = PATH_MAX;
    if (_NSGetExecutablePath(buf1, &bufsize) != 0)
        throw exception("failed to locate Lean executable location");
    if (!realpath(buf1, buf2))
        throw exception("failed to resolve symbolic links in " + std::string(buf1));
    return std::string(buf2);
}
bool is_path_sep(char c) { return c == g_path_sep; }
#else
// Linux version
#include <unistd.h>
#include <string.h>
#include <limits.h> // NOLINT
#include <stdio.h>
static char g_path_sep     = ':';
static char g_sep          = '/';
static char g_bad_sep      = '\\';
std::string get_exe_location() {
    char path[PATH_MAX];
    char dest[PATH_MAX];
    memset(dest, 0, PATH_MAX);
    pid_t pid = getpid();
    snprintf(path, PATH_MAX, "/proc/%d/exe", pid);
    if (readlink(path, dest, PATH_MAX) == -1) {
        throw exception("failed to locate Lean executable location");
    } else {
        return std::string(dest);
    }
}
bool is_path_sep(char c) { return c == g_path_sep; }
#endif

std::string normalize_path(std::string f) {
    for (auto & c : f) {
        if (c == g_bad_sep)
            c = g_sep;
    }
    return f;
}

std::string get_path(std::string f) {
    while (true) {
        if (f.empty())
            throw exception("failed to locate Lean executable location");
        if (f.back() == g_sep) {
            f.pop_back();
            return f;
        }
        f.pop_back();
    }
}

void init_lean_path() {
    char * r = nullptr;
    r = getenv("LEAN_PATH");
    if (r == nullptr) {
#ifdef LEAN_EMSCRIPTEN
        *g_lean_path = "/library";
#else
        std::string exe_path = get_path(get_exe_location());
        *g_lean_path  = exe_path + g_sep + ".." + g_sep + "library";
        *g_lean_path += g_path_sep;
        *g_lean_path += exe_path + g_sep + ".." + g_sep + "lib" + g_sep + "lean" + g_sep + "library";
        *g_lean_path += g_path_sep;
        *g_lean_path += ".";
#endif
    } else {
        *g_lean_path = r;
    }
    g_lean_path_vector->clear();
    *g_lean_path = normalize_path(*g_lean_path);
    unsigned i  = 0;
    unsigned j  = 0;
    unsigned sz = g_lean_path->size();
    for (; j < sz; j++) {
        if (is_path_sep((*g_lean_path)[j])) {
            if (j > i)
                g_lean_path_vector->push_back(g_lean_path->substr(i, j - i));
            i = j + 1;
        }
    }
    if (j > i)
        g_lean_path_vector->push_back(g_lean_path->substr(i, j - i));
}

static char g_sep_str[2];

void initialize_lean_path() {
    if (g_default_file_name != nullptr)
        finalize_lean_path();
    g_default_file_name = new std::string(LEAN_DEFAULT_MODULE_FILE_NAME);
    g_lean_path         = new std::string();
    g_lean_path_vector  = new std::vector<std::string>();
    g_sep_str[0]        = g_sep;
    g_sep_str[1]        = 0;
    init_lean_path();
}

void finalize_lean_path() {
    delete g_lean_path_vector;
    delete g_lean_path;
    delete g_default_file_name;
}

bool has_file_ext(std::string const & fname, char const * ext) {
    unsigned ext_len = strlen(ext);
    return fname.size() > ext_len && fname.substr(fname.size() - ext_len, ext_len) == ext;
}

bool is_lean_file(std::string const & fname) {
    return has_file_ext(fname, ".lean");
}

bool is_olean_file(std::string const & fname) {
    return has_file_ext(fname, ".olean");
}

bool is_known_file_ext(std::string const & fname) {
    return is_lean_file(fname) || is_olean_file(fname);
}

optional<std::string> check_file_core(std::string file, char const * ext) {
    if (ext)
        file += ext;
    std::ifstream ifile(file);
    if (ifile)
        return optional<std::string>(lrealpath(file.c_str()));
    else
        return optional<std::string>();
}

optional<std::string> check_file(std::string const & path, std::string const & fname, char const * ext = nullptr) {
    std::string file = path + g_sep + fname;
    if (is_directory(file.c_str())) {
        std::string default_file = file + g_sep + *g_default_file_name;
        if (auto r1 = check_file_core(default_file, ext)) {
            if (auto r2 = check_file_core(file, ext))
                throw exception(sstream() << "ambiguous import, it can be '" << *r1 << "' or '" << *r2 << "'");
            return r1;
        }
    }
    return check_file_core(file, ext);
}

std::string name_to_file(name const & fname) {
    return fname.to_string(g_sep_str);
}

std::string find_file(std::string fname, std::initializer_list<char const *> const & extensions) {
    bool is_known = is_known_file_ext(fname);
    fname = normalize_path(fname);
    for (auto const & path : *g_lean_path_vector) {
        if (is_known) {
            if (auto r = check_file(path, fname))
                return *r;
        } else {
            for (auto ext : extensions) {
                if (auto r = check_file(path, fname, ext))
                    return *r;
            }
        }
    }
    throw file_not_found_exception(fname);
}

std::string find_file(std::string const & base, optional<unsigned> const & rel, name const & fname,
                      std::initializer_list<char const *> const & extensions) {
    if (!rel) {
        return find_file(fname.to_string(g_sep_str), extensions);
    } else {
        auto path = base;
        for (unsigned i = 0; i < *rel; i++) {
            path += g_sep;
            path += "..";
        }
        for (auto ext : extensions) {
            if (auto r = check_file(path, fname.to_string(g_sep_str), ext))
                return *r;
        }
        throw file_not_found_exception(fname.to_string());
    }
}

std::string find_file(std::string const & base, optional<unsigned> const & k, name const & fname, char const * ext) {
    return find_file(base, k, fname, {ext});
}

std::string find_file(std::string fname) {
    return find_file(fname, {".olean", ".lean"});
}

std::string find_file(name const & fname) {
    return find_file(fname.to_string(g_sep_str));
}

std::string find_file(name const & fname, std::initializer_list<char const *> const & exts) {
    return find_file(fname.to_string(g_sep_str), exts);
}

void find_files(std::string const & base, char const * ext, std::vector<std::string> & files) {
    for (auto & fn : read_dir(base)) {
        if (auto i_d = is_dir(fn)) {
            if (*i_d) {
                find_files(fn, ext, files);
            } else if (has_file_ext(fn, ext)) {
                files.push_back(fn);
            }
        }
    }
}

void find_imports_core(std::string const & base, optional<unsigned> const & k,
                       std::vector<pair<std::string, std::string>> & imports) {
    std::vector<std::string> files;
    find_files(base, ".lean", files);

    for (auto const & file : files) {
        auto import = file.substr(base.size() + 1, file.size() - base.size() - 1 - std::string(".lean").size());
        std::replace(import.begin(), import.end(), g_sep, '.');
        if (k)
            import = std::string(*k + 1, '.') + import;
        imports.push_back({import, file});
    }
}

void find_imports(std::string const & base, optional<unsigned> const & k,
                  std::vector<pair<std::string, std::string>> & imports) {
    if (!k) {
        for (auto const & base : *g_lean_path_vector)
            if (is_dir(base))
                find_imports_core(base, k, imports);
    } else {
        auto path = base;
        for (unsigned i = 0; i < *k; i++) {
            path += g_sep;
            path += "..";
        }
        find_imports_core(path, k, imports);
    }
}

char const * get_lean_path() {
    return g_lean_path->c_str();
}

void display_path(std::ostream & out, std::string const & fname) {
    out << fname;
}

std::string dirname(char const * fname) {
    if (fname == nullptr)
        return ".";
    std::string nfname = normalize_path(std::string(fname));
    fname = nfname.c_str();
    unsigned i        = 0;
    unsigned last_sep = 0;
    bool found_sep    = false;
    char const * it   = fname;
    while (*it) {
        if (*it == g_sep) {
            found_sep = true;
            last_sep  = i;
        }
        ++i;
        ++it;
    }
    if (!found_sep) {
        return ".";
    } else {
        std::string r;
        for (unsigned i = 0; i < last_sep; i++)
            r.push_back(fname[i]);
        return r;
    }
}

std::string path_append(char const * p1, char const * p2) {
    std::string r(p1);
    r += g_sep;
    r += p2;
    return r;
}

std::string olean_of_lean(std::string const & lean_fn) {
    if (lean_fn.size() > 5 && lean_fn.substr(lean_fn.size() - 5) == ".lean") {
        return lean_fn.substr(0, lean_fn.size() - 5) + ".olean";
    } else {
        throw exception(sstream() << "not a .lean file: " << lean_fn);
    }
}

std::string olean_file_to_lean_file(std::string const &olean) {
    lean_assert(is_olean_file(olean));
    std::string lean = olean;
    lean.erase(lean.size() - std::string("olean").size(), 1);
    return lean;
}

std::string read_file(std::string const & fname, std::ios_base::openmode mode) {
    std::ifstream in(fname, mode);
    if (!in.good()) throw file_not_found_exception(fname);
    std::stringstream buf;
    buf << in.rdbuf();
    return buf.str();
}

time_t get_mtime(std::string const &fname) {
    struct stat st;
    if (stat(fname.c_str(), &st) != 0)
        return -1;
    return st.st_mtime;
}

optional<bool> is_dir(std::string const & fn) {
    struct stat st;
    if (stat(fn.c_str(), &st) == 0)
        return optional<bool>(S_ISDIR(st.st_mode));
    return optional<bool>();
}

std::vector<std::string> read_dir(std::string const &dirname) {
    auto dir = opendir(dirname.c_str());
    if (!dir) throw exception(sstream() << "could not open directory " << dirname << ": " << std::strerror(errno));

    std::vector<std::string> files;
    while (auto ep = readdir(dir)) { // NOLINT
        // ^^ disabling readdir_r lint because glibc recommends using readdir now.

        std::string fn = ep->d_name;
        if (fn == "." || fn == "..") continue;
        files.push_back(path_append(dirname.c_str(), fn.c_str()));
    }
    closedir(dir);
    return files;
}

}
<commit_msg>fix(util/lean_path): find_imports: .olean files and default modules<commit_after>/*
Copyright (c) 2013 Microsoft Corporation. All rights reserved.
Released under Apache 2.0 license as described in the file LICENSE.

Author: Leonardo de Moura
*/
#if defined(LEAN_WINDOWS) && !defined(LEAN_CYGWIN)
#include <windows.h>
#endif
#include <string>
#include <cstdlib>
#include <fstream>
#include <vector>
#include <sys/types.h>
#include <sys/stat.h>
#include <dirent.h>
#include "util/exception.h"
#include "util/sstream.h"
#include "util/name.h"
#include "util/optional.h"
#include "util/realpath.h"
#include "util/lean_path.h"

#ifndef LEAN_DEFAULT_MODULE_FILE_NAME
#define LEAN_DEFAULT_MODULE_FILE_NAME "default"
#endif

namespace lean {
file_not_found_exception::file_not_found_exception(std::string const & fname):
    exception(sstream() << "file '" << fname << "' not found in the LEAN_PATH"),
    m_fname(fname) {}

static std::string * g_default_file_name             = nullptr;
static std::string *              g_lean_path        = nullptr;
static std::vector<std::string> * g_lean_path_vector = nullptr;

bool is_directory(char const * pathname) {
    struct stat info;
    if (stat(pathname, &info) != 0 )
        return false; // failed to access pathname
    return info.st_mode & S_IFDIR;
}

#if defined(LEAN_EMSCRIPTEN)
// emscripten version
static char g_path_sep     = ':';
static char g_sep          = '/';
static char g_bad_sep      = '\\';
bool is_path_sep(char c) { return c == g_path_sep; }
#elif defined(LEAN_WINDOWS) && !defined(LEAN_CYGWIN)
// Windows version
static char g_path_sep     = ';';
static char g_path_alt_sep = ':';
static char g_sep          = '\\';
static char g_bad_sep      = '/';
std::string get_exe_location() {
    HMODULE hModule = GetModuleHandleW(NULL);
    WCHAR path[MAX_PATH];
    GetModuleFileNameW(hModule, path, MAX_PATH);
    std::wstring pathstr(path);
    return std::string(pathstr.begin(), pathstr.end());
}
bool is_path_sep(char c) { return c == g_path_sep || c == g_path_alt_sep; }
#elif defined(__APPLE__)
// OSX version
#include <mach-o/dyld.h>
#include <limits.h>
#include <stdlib.h>
static char g_path_sep     = ':';
static char g_sep          = '/';
static char g_bad_sep      = '\\';
std::string get_exe_location() {
    char buf1[PATH_MAX];
    char buf2[PATH_MAX];
    uint32_t bufsize = PATH_MAX;
    if (_NSGetExecutablePath(buf1, &bufsize) != 0)
        throw exception("failed to locate Lean executable location");
    if (!realpath(buf1, buf2))
        throw exception("failed to resolve symbolic links in " + std::string(buf1));
    return std::string(buf2);
}
bool is_path_sep(char c) { return c == g_path_sep; }
#else
// Linux version
#include <unistd.h>
#include <string.h>
#include <limits.h> // NOLINT
#include <stdio.h>
static char g_path_sep     = ':';
static char g_sep          = '/';
static char g_bad_sep      = '\\';
std::string get_exe_location() {
    char path[PATH_MAX];
    char dest[PATH_MAX];
    memset(dest, 0, PATH_MAX);
    pid_t pid = getpid();
    snprintf(path, PATH_MAX, "/proc/%d/exe", pid);
    if (readlink(path, dest, PATH_MAX) == -1) {
        throw exception("failed to locate Lean executable location");
    } else {
        return std::string(dest);
    }
}
bool is_path_sep(char c) { return c == g_path_sep; }
#endif

std::string normalize_path(std::string f) {
    for (auto & c : f) {
        if (c == g_bad_sep)
            c = g_sep;
    }
    return f;
}

std::string get_path(std::string f) {
    while (true) {
        if (f.empty())
            throw exception("failed to locate Lean executable location");
        if (f.back() == g_sep) {
            f.pop_back();
            return f;
        }
        f.pop_back();
    }
}

void init_lean_path() {
    char * r = nullptr;
    r = getenv("LEAN_PATH");
    if (r == nullptr) {
#ifdef LEAN_EMSCRIPTEN
        *g_lean_path = "/library";
#else
        std::string exe_path = get_path(get_exe_location());
        *g_lean_path  = exe_path + g_sep + ".." + g_sep + "library";
        *g_lean_path += g_path_sep;
        *g_lean_path += exe_path + g_sep + ".." + g_sep + "lib" + g_sep + "lean" + g_sep + "library";
        *g_lean_path += g_path_sep;
        *g_lean_path += ".";
#endif
    } else {
        *g_lean_path = r;
    }
    g_lean_path_vector->clear();
    *g_lean_path = normalize_path(*g_lean_path);
    unsigned i  = 0;
    unsigned j  = 0;
    unsigned sz = g_lean_path->size();
    for (; j < sz; j++) {
        if (is_path_sep((*g_lean_path)[j])) {
            if (j > i)
                g_lean_path_vector->push_back(g_lean_path->substr(i, j - i));
            i = j + 1;
        }
    }
    if (j > i)
        g_lean_path_vector->push_back(g_lean_path->substr(i, j - i));
}

static char g_sep_str[2];

void initialize_lean_path() {
    if (g_default_file_name != nullptr)
        finalize_lean_path();
    g_default_file_name = new std::string(LEAN_DEFAULT_MODULE_FILE_NAME);
    g_lean_path         = new std::string();
    g_lean_path_vector  = new std::vector<std::string>();
    g_sep_str[0]        = g_sep;
    g_sep_str[1]        = 0;
    init_lean_path();
}

void finalize_lean_path() {
    delete g_lean_path_vector;
    delete g_lean_path;
    delete g_default_file_name;
}

bool has_file_ext(std::string const & fname, char const * ext) {
    unsigned ext_len = strlen(ext);
    return fname.size() > ext_len && fname.substr(fname.size() - ext_len, ext_len) == ext;
}

bool is_lean_file(std::string const & fname) {
    return has_file_ext(fname, ".lean");
}

bool is_olean_file(std::string const & fname) {
    return has_file_ext(fname, ".olean");
}

bool is_known_file_ext(std::string const & fname) {
    return is_lean_file(fname) || is_olean_file(fname);
}

optional<std::string> check_file_core(std::string file, char const * ext) {
    if (ext)
        file += ext;
    std::ifstream ifile(file);
    if (ifile)
        return optional<std::string>(lrealpath(file.c_str()));
    else
        return optional<std::string>();
}

optional<std::string> check_file(std::string const & path, std::string const & fname, char const * ext = nullptr) {
    std::string file = path + g_sep + fname;
    if (is_directory(file.c_str())) {
        std::string default_file = file + g_sep + *g_default_file_name;
        if (auto r1 = check_file_core(default_file, ext)) {
            if (auto r2 = check_file_core(file, ext))
                throw exception(sstream() << "ambiguous import, it can be '" << *r1 << "' or '" << *r2 << "'");
            return r1;
        }
    }
    return check_file_core(file, ext);
}

std::string name_to_file(name const & fname) {
    return fname.to_string(g_sep_str);
}

std::string find_file(std::string fname, std::initializer_list<char const *> const & extensions) {
    bool is_known = is_known_file_ext(fname);
    fname = normalize_path(fname);
    for (auto const & path : *g_lean_path_vector) {
        if (is_known) {
            if (auto r = check_file(path, fname))
                return *r;
        } else {
            for (auto ext : extensions) {
                if (auto r = check_file(path, fname, ext))
                    return *r;
            }
        }
    }
    throw file_not_found_exception(fname);
}

std::string find_file(std::string const & base, optional<unsigned> const & rel, name const & fname,
                      std::initializer_list<char const *> const & extensions) {
    if (!rel) {
        return find_file(fname.to_string(g_sep_str), extensions);
    } else {
        auto path = base;
        for (unsigned i = 0; i < *rel; i++) {
            path += g_sep;
            path += "..";
        }
        for (auto ext : extensions) {
            if (auto r = check_file(path, fname.to_string(g_sep_str), ext))
                return *r;
        }
        throw file_not_found_exception(fname.to_string());
    }
}

std::string find_file(std::string const & base, optional<unsigned> const & k, name const & fname, char const * ext) {
    return find_file(base, k, fname, {ext});
}

std::string find_file(std::string fname) {
    return find_file(fname, {".olean", ".lean"});
}

std::string find_file(name const & fname) {
    return find_file(fname.to_string(g_sep_str));
}

std::string find_file(name const & fname, std::initializer_list<char const *> const & exts) {
    return find_file(fname.to_string(g_sep_str), exts);
}

void find_files(std::string const & base, char const * ext, std::vector<std::string> & files) {
    for (auto & fn : read_dir(base)) {
        if (auto i_d = is_dir(fn)) {
            if (*i_d) {
                find_files(fn, ext, files);
            } else if (has_file_ext(fn, ext)) {
                files.push_back(fn);
            }
        }
    }
}

void find_imports_core(std::string const & base, optional<unsigned> const & k,
                       std::vector<pair<std::string, std::string>> & imports) {
    std::vector<std::string> files;
    find_files(base, ".lean", files);
    find_files(base, ".olean", files);

    for (auto const & file : files) {
        auto import = file.substr(base.size() + 1, file.rfind('.') - (base.size() + 1));
        std::replace(import.begin(), import.end(), g_sep, '.');
        if (k)
            import = std::string(*k + 1, '.') + import;
        auto n = import.rfind(".default");
        if (n != -1 && n == import.size() - std::string(".default").size())
            import = import.substr(0, n);
        imports.push_back({import, file});
    }
}

void find_imports(std::string const & base, optional<unsigned> const & k,
                  std::vector<pair<std::string, std::string>> & imports) {
    if (!k) {
        for (auto const & base : *g_lean_path_vector)
            if (is_dir(base))
                find_imports_core(base, k, imports);
    } else {
        auto path = base;
        for (unsigned i = 0; i < *k; i++) {
            path += g_sep;
            path += "..";
        }
        find_imports_core(path, k, imports);
    }
}

char const * get_lean_path() {
    return g_lean_path->c_str();
}

void display_path(std::ostream & out, std::string const & fname) {
    out << fname;
}

std::string dirname(char const * fname) {
    if (fname == nullptr)
        return ".";
    std::string nfname = normalize_path(std::string(fname));
    fname = nfname.c_str();
    unsigned i        = 0;
    unsigned last_sep = 0;
    bool found_sep    = false;
    char const * it   = fname;
    while (*it) {
        if (*it == g_sep) {
            found_sep = true;
            last_sep  = i;
        }
        ++i;
        ++it;
    }
    if (!found_sep) {
        return ".";
    } else {
        std::string r;
        for (unsigned i = 0; i < last_sep; i++)
            r.push_back(fname[i]);
        return r;
    }
}

std::string path_append(char const * p1, char const * p2) {
    std::string r(p1);
    r += g_sep;
    r += p2;
    return r;
}

std::string olean_of_lean(std::string const & lean_fn) {
    if (lean_fn.size() > 5 && lean_fn.substr(lean_fn.size() - 5) == ".lean") {
        return lean_fn.substr(0, lean_fn.size() - 5) + ".olean";
    } else {
        throw exception(sstream() << "not a .lean file: " << lean_fn);
    }
}

std::string olean_file_to_lean_file(std::string const &olean) {
    lean_assert(is_olean_file(olean));
    std::string lean = olean;
    lean.erase(lean.size() - std::string("olean").size(), 1);
    return lean;
}

std::string read_file(std::string const & fname, std::ios_base::openmode mode) {
    std::ifstream in(fname, mode);
    if (!in.good()) throw file_not_found_exception(fname);
    std::stringstream buf;
    buf << in.rdbuf();
    return buf.str();
}

time_t get_mtime(std::string const &fname) {
    struct stat st;
    if (stat(fname.c_str(), &st) != 0)
        return -1;
    return st.st_mtime;
}

optional<bool> is_dir(std::string const & fn) {
    struct stat st;
    if (stat(fn.c_str(), &st) == 0)
        return optional<bool>(S_ISDIR(st.st_mode));
    return optional<bool>();
}

std::vector<std::string> read_dir(std::string const &dirname) {
    auto dir = opendir(dirname.c_str());
    if (!dir) throw exception(sstream() << "could not open directory " << dirname << ": " << std::strerror(errno));

    std::vector<std::string> files;
    while (auto ep = readdir(dir)) { // NOLINT
        // ^^ disabling readdir_r lint because glibc recommends using readdir now.

        std::string fn = ep->d_name;
        if (fn == "." || fn == "..") continue;
        files.push_back(path_append(dirname.c_str(), fn.c_str()));
    }
    closedir(dir);
    return files;
}

}
<|endoftext|>


<commit_before>/*
 * Copyright 2006-2008 The FLWOR Foundation.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

#ifndef WIN32
# include <cstdio>                      /* for strerror */
# include <fcntl.h>                     /* for open(2) */
# include <sys/mman.h>                  /* for mmap(2) */
# include <sys/stat.h>                  /* for stat(2) */
# include <unistd.h>                    /* for close(2) */
#endif /* WIN32 */
#include <sstream>

#include "zorbaerrors/error_manager.h"
#include "mmap_file.h"

#ifndef WIN32
# define ERROR_STRING ::strerror( errno )
#else
# define ERROR_STRING ::GetLastError()
#endif /* WIN32 */

#define THROW_IOEXCEPTION(func) {                                           \
          ostringstream oss;                                                \
          oss << #func "(\"" << path_ << "\") failed: " << ERROR_STRING;    \
          ZORBA_ERROR_DESC( XQP0013_SYSTEM_MMFILE_IOEXCEPTION, oss.str() ); \
        }

using namespace std;

namespace zorba {

void mmap_file::close() {
#ifndef WIN32
  if ( addr_ && ::munmap( static_cast<char*>( addr_ ), size_ ) == -1 )
    THROW_IOEXCEPTION( munmap );
  if ( fd_ != -1 )
    ::close( fd_ );
#else /* WIN32 */
  if ( addr_ && !::UnmapViewOfFile( addr_ ) )
    THROW_IOEXCEPTION( UnmapViewOfFile );
  if ( mapping_ )
    ::CloseHandle( mapping_ );
  if ( fd_ != INVALID_HANDLE_VALUE )
    ::CloseHandle( fd_ );
#endif /* WIN32 */
  init();
}

void mmap_file::init() {
#ifndef WIN32
  fd_ = -1;
#else
  fd_ = INVALID_HANDLE_VALUE;
#endif /* WIN32 */
  size_ = 0;
  addr_ = 0;
}

void mmap_file::open( char const *path, ios::openmode mode ) {
  close();
  path_ = path;

#ifndef WIN32

  struct stat stat_buf;
  if ( ::stat( path, &stat_buf ) == -1 )
    THROW_IOEXCEPTION( stat );
  size_ = stat_buf.st_size;

  int flags = 0;
  int prot  = PROT_NONE;

  if ( mode & ios::in ) {
    flags |= O_RDONLY;
    prot  |= PROT_READ;
  }
  if ( mode & ios::out ) {
    flags |= O_WRONLY;
    prot  |= PROT_WRITE;
  }

  if ( (fd_ = ::open( path, flags )) == -1 )
    THROW_IOEXCEPTION( open );

  if ( (addr_ = ::mmap( 0, size_, prot, MAP_SHARED, fd_, 0 )) == MAP_FAILED ) {
    addr_ = 0;
    THROW_IOEXCEPTION( mmap );
  }

#else /* WIN32 */

#ifdef UNICODE
  TCHAR wPath[ 1024 ];
  ::MultiByteToWideChar(
    CP_ACP /* or CP_UTF8 */, 0, path, -1, wPath,
    sizeof( wPath ) / sizeof( wPath[0] )
  );
#else
  char const *const wPath = path;
#endif /* UNICODE */

  DWORD createAccess = 0;
  DWORD protect      = 0;
  DWORD shareMode    = 0;
  DWORD mapAccess    = 0;

  if ( mode & ios::in ) {
    createAccess |= GENERIC_READ;
    shareMode    |= FILE_SHARE_READ;
    mapAccess    |= FILE_MAP_READ;
    protect       = PAGE_READONLY;
  }
  if ( mode & ios::out ) {
    createAccess |= GENERIC_WRITE;
    shareMode    |= FILE_SHARE_WRITE;
    mapAccess    |= FILE_MAP_WRITE;
    protect       = PAGE_READWRITE;
  }

  fd_ = ::CreateFile(
    wPath, createAccess, shareMode, NULL, OPEN_EXISTING, 0, NULL
  );
  if ( fd_ == INVALID_HANDLE_VALUE )
    THROW_IOEXCEPTION( CreateFile );

  if ( (size_ = ::GetFileSize( fd_, NULL )) == -1 )
    THROW_IOEXCEPTION( GetFileSize );

  if ( !(mapping_ = ::CreateFileMapping( fd_, NULL, protect, 0, 0, NULL )) )
    THROW_IOEXCEPTION( CreateFileMapping );

  if ( !(addr_ = ::MapViewOfFile( mapping_, mapAccess, 0, 0, 0 )) )
    THROW_IOEXCEPTION( MapViewOfFile );
#endif /* WIN32 */
}

} // namespace zorba
/* vim:set et sw=2 ts=2: */
<commit_msg>Added #include <cerrno><commit_after>/*
 * Copyright 2006-2008 The FLWOR Foundation.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

#ifndef WIN32
# include <cerrno>                      /* for errno */
# include <cstdio>                      /* for strerror */
# include <fcntl.h>                     /* for open(2) */
# include <sys/mman.h>                  /* for mmap(2) */
# include <sys/stat.h>                  /* for stat(2) */
# include <unistd.h>                    /* for close(2) */
#endif /* WIN32 */
#include <sstream>

#include "zorbaerrors/error_manager.h"
#include "mmap_file.h"

#ifndef WIN32
# define ERROR_STRING ::strerror( errno )
#else
# define ERROR_STRING ::GetLastError()
#endif /* WIN32 */

#define THROW_IOEXCEPTION(func) {                                           \
          ostringstream oss;                                                \
          oss << #func "(\"" << path_ << "\") failed: " << ERROR_STRING;    \
          ZORBA_ERROR_DESC( XQP0013_SYSTEM_MMFILE_IOEXCEPTION, oss.str() ); \
        }

using namespace std;

namespace zorba {

void mmap_file::close() {
#ifndef WIN32
  if ( addr_ && ::munmap( static_cast<char*>( addr_ ), size_ ) == -1 )
    THROW_IOEXCEPTION( munmap );
  if ( fd_ != -1 )
    ::close( fd_ );
#else /* WIN32 */
  if ( addr_ && !::UnmapViewOfFile( addr_ ) )
    THROW_IOEXCEPTION( UnmapViewOfFile );
  if ( mapping_ )
    ::CloseHandle( mapping_ );
  if ( fd_ != INVALID_HANDLE_VALUE )
    ::CloseHandle( fd_ );
#endif /* WIN32 */
  init();
}

void mmap_file::init() {
#ifndef WIN32
  fd_ = -1;
#else
  fd_ = INVALID_HANDLE_VALUE;
#endif /* WIN32 */
  size_ = 0;
  addr_ = 0;
}

void mmap_file::open( char const *path, ios::openmode mode ) {
  close();
  path_ = path;

#ifndef WIN32

  struct stat stat_buf;
  if ( ::stat( path, &stat_buf ) == -1 )
    THROW_IOEXCEPTION( stat );
  size_ = stat_buf.st_size;

  int flags = 0;
  int prot  = PROT_NONE;

  if ( mode & ios::in ) {
    flags |= O_RDONLY;
    prot  |= PROT_READ;
  }
  if ( mode & ios::out ) {
    flags |= O_WRONLY;
    prot  |= PROT_WRITE;
  }

  if ( (fd_ = ::open( path, flags )) == -1 )
    THROW_IOEXCEPTION( open );

  if ( (addr_ = ::mmap( 0, size_, prot, MAP_SHARED, fd_, 0 )) == MAP_FAILED ) {
    addr_ = 0;
    THROW_IOEXCEPTION( mmap );
  }

#else /* WIN32 */

#ifdef UNICODE
  TCHAR wPath[ 1024 ];
  ::MultiByteToWideChar(
    CP_ACP /* or CP_UTF8 */, 0, path, -1, wPath,
    sizeof( wPath ) / sizeof( wPath[0] )
  );
#else
  char const *const wPath = path;
#endif /* UNICODE */

  DWORD createAccess = 0;
  DWORD protect      = 0;
  DWORD shareMode    = 0;
  DWORD mapAccess    = 0;

  if ( mode & ios::in ) {
    createAccess |= GENERIC_READ;
    shareMode    |= FILE_SHARE_READ;
    mapAccess    |= FILE_MAP_READ;
    protect       = PAGE_READONLY;
  }
  if ( mode & ios::out ) {
    createAccess |= GENERIC_WRITE;
    shareMode    |= FILE_SHARE_WRITE;
    mapAccess    |= FILE_MAP_WRITE;
    protect       = PAGE_READWRITE;
  }

  fd_ = ::CreateFile(
    wPath, createAccess, shareMode, NULL, OPEN_EXISTING, 0, NULL
  );
  if ( fd_ == INVALID_HANDLE_VALUE )
    THROW_IOEXCEPTION( CreateFile );

  if ( (size_ = ::GetFileSize( fd_, NULL )) == -1 )
    THROW_IOEXCEPTION( GetFileSize );

  if ( !(mapping_ = ::CreateFileMapping( fd_, NULL, protect, 0, 0, NULL )) )
    THROW_IOEXCEPTION( CreateFileMapping );

  if ( !(addr_ = ::MapViewOfFile( mapping_, mapAccess, 0, 0, 0 )) )
    THROW_IOEXCEPTION( MapViewOfFile );
#endif /* WIN32 */
}

} // namespace zorba
/* vim:set et sw=2 ts=2: */
<|endoftext|>


<commit_before>/*
 * Copyright 2006-2008 The FLWOR Foundation.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

#ifndef ZORBA_NO_UNICODE
#include <unicode/ustring.h>
#endif /* ZORBA_NO_UNICODE */

#include "utf8_util.h"
#include "zorbaerrors/error_manager.h"
#include "zorbatypes/zorbatypesError.h"

#ifndef ZORBA_NO_UNICODE
U_NAMESPACE_USE
#endif /* ZORBA_NO_UNICODE */

unsigned const Mask1Byte   = 0x80;
unsigned const Mask2Bytes  = 0xC0;
unsigned const Mask3Bytes  = 0xE0;
unsigned const Mask4Bytes  = 0xF0;
unsigned const Mask5Bytes  = 0xF8;
unsigned const Mask6Bytes  = 0xFC;

namespace zorba {
namespace utf8 {

unicode::code_point const SubChar = 0xFFFD;

size_type byte_pos( storage_type const *s, size_type char_pos ) {
  if ( char_pos == npos )
    return npos;
  storage_type const *p = s;
  for ( ; char_pos > 0; --char_pos ) {
    if ( !*p )
      return npos;
    p += char_length( *p );
  }
  return p - s;
}

size_type byte_pos( storage_type const *s, size_type s_size,
                    size_type char_pos ) {
  if ( char_pos == npos )
    return npos;
  storage_type const *p = s;
  storage_type const *const end = s + s_size;
  for ( ; char_pos > 0; --char_pos ) {
    if ( p >= end )
      return npos;
    p += char_length( *p );
  }
  return p - s;
}

size_type char_length( storage_type lead ) {
  static char const length_table[] = {
    /*      0 1 2 3 4 5 6 7 8 9 A B C D E F */
    /* 0 */ 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
    /* 1 */ 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
    /* 2 */ 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
    /* 3 */ 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
    /* 4 */ 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
    /* 5 */ 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
    /* 6 */ 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
    /* 7 */ 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
    /* 8 */ 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
    /* 9 */ 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
    /* A */ 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
    /* B */ 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
    /* C */ 2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,
    /* D */ 2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,
    /* E */ 3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,
    /* F */ 4,4,4,4,4,4,4,4,5,5,5,5,6,6,0,0
  };

  size_type const len = length_table[ static_cast<unsigned>( lead ) & 0xFF ];
  if ( !len )
    ZORBA_ERROR( XQP0034_ILLEGAL_UTF8_BYTE );
  return len;
}

size_type char_pos( storage_type const *s, storage_type const *p ) {
  size_type pos = 0;
  while ( s < p ) {
    s += char_length( *s );
    ++pos;
  }
  return pos;
}

size_type encode( unicode::code_point c, storage_type **ps ) {
  if ( !unicode::is_valid( c ) )
    ZORBA_ERROR( FOCH0001 );
  unsigned const n = c & 0xFFFFFFFF;
  storage_type *&p = *ps, *const p0 = p;
  if ( n < 0x80 ) {
    // 0xxxxxxx
    *p++ = static_cast<storage_type>( n );
  } else if ( n < 0x800 ) {
    // 110xxxxx 10xxxxxx
    *p++ = static_cast<storage_type>( Mask2Bytes | ( n >> 6 )         );
    *p++ = static_cast<storage_type>( Mask1Byte  | ( n        & 0x3F) );
  } else if ( n < 0x10000 ) {
    // 1110xxxx 10xxxxxx 10xxxxxx
    *p++ = static_cast<storage_type>( Mask3Bytes |  (n >> 12)         );
    *p++ = static_cast<storage_type>( Mask1Byte  | ((n >>  6) & 0x3F) );
    *p++ = static_cast<storage_type>( Mask1Byte  | ( n        & 0x3F) );
  } else if ( n < 0x200000 ) {
    // 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx
    *p++ = static_cast<storage_type>( Mask4Bytes |  (n >> 18)         );
    *p++ = static_cast<storage_type>( Mask1Byte  | ((n >> 12) & 0x3F) );
    *p++ = static_cast<storage_type>( Mask1Byte  | ((n >>  6) & 0x3F) );
    *p++ = static_cast<storage_type>( Mask1Byte  | ( n        & 0x3F) );
  } else if ( n < 0x4000000 ) {
    // 111110xx 10xxxxxx 10xxxxxx 10xxxxxx 10xxxxxx
    *p++ = static_cast<storage_type>( Mask5Bytes |  (n >> 24)         );
    *p++ = static_cast<storage_type>( Mask1Byte  | ((n >> 18) & 0x3F) );
    *p++ = static_cast<storage_type>( Mask1Byte  | ((n >> 12) & 0x3F) );
    *p++ = static_cast<storage_type>( Mask1Byte  | ((n >>  6) & 0x3F) );
    *p++ = static_cast<storage_type>( Mask1Byte  | ( n        & 0x3F) );
  } else if ( n < 0x8000000 ) {
    // 1111110x 10xxxxxx 10xxxxxx 10xxxxxx 10xxxxxx 10xxxxxx
    *p++ = static_cast<storage_type>( Mask6Bytes |  (n >> 30)         );
    *p++ = static_cast<storage_type>( Mask1Byte  | ((n >> 24) & 0x3F) );
    *p++ = static_cast<storage_type>( Mask1Byte  | ((n >> 18) & 0x3F) );
    *p++ = static_cast<storage_type>( Mask1Byte  | ((n >> 12) & 0x3F) );
    *p++ = static_cast<storage_type>( Mask1Byte  | ((n >>  6) & 0x3F) );
    *p++ = static_cast<storage_type>( Mask1Byte  | ( n        & 0x3F) );
  }
  return p - p0;
}

size_type length( storage_type const *s ) {
  size_type len = 0;
  while ( *s ) {
    s += char_length( *s );
    ++len;
  }
  return len;
}

size_type length( storage_type const *begin, storage_type const *end ) {
  size_type len = 0;
  while ( begin < end && *begin ) {
    begin += char_length( *begin );
    ++len;
  }
  return len;
}

#ifndef ZORBA_NO_UNICODE

bool to_string( unicode::char_type const *in, size_type in_len,
                storage_type **out, size_type *out_len ) {
  UErrorCode err = U_ZERO_ERROR;
  unicode::size_type utf8_len;
  u_strToUTF8WithSub(                   // pre-flight to get utf8_len
    NULL, 0, &utf8_len, in, in_len, SubChar, NULL, &err
  );
  // must not check "err" here since ICU always returns "buffer overflow"

  storage_type *const utf8_buf = new storage_type[ utf8_len ];
  err = U_ZERO_ERROR;
  u_strToUTF8WithSub(
    utf8_buf, utf8_len, NULL, in, in_len, SubChar, NULL, &err
  );
  if ( U_FAILURE( err ) ) {
    delete[] utf8_buf;
    return false;
  }
  *out = utf8_buf;
  *out_len = utf8_len;
  return true;
}

#ifndef WIN32
bool to_string( wchar_t const *in, size_type in_len, storage_type **out,
                size_type *out_len ) {
  bool result = false;
  unicode::string temp;
  if ( unicode::to_string( in, in_len, &temp ) ) {
    size_type const temp_len = temp.length();
    unicode::char_type *const buf = temp.getBuffer( -1 );
    result = utf8::to_string( buf, temp_len, out, out_len );
    temp.releaseBuffer();
  }
  return result;
}
#endif /* WIN32 */

#endif /* ZORBA_NO_UNICODE */

} // namespace utf8
} // namespace zorba

/* vim:set et sw=2 ts=2: */
<commit_msg>Made SubChar local.<commit_after>/*
 * Copyright 2006-2008 The FLWOR Foundation.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

#ifndef ZORBA_NO_UNICODE
#include <unicode/ustring.h>
#endif /* ZORBA_NO_UNICODE */

#include "utf8_util.h"
#include "zorbaerrors/error_manager.h"
#include "zorbatypes/zorbatypesError.h"

#ifndef ZORBA_NO_UNICODE
U_NAMESPACE_USE
#endif /* ZORBA_NO_UNICODE */

unsigned const Mask1Byte   = 0x80;
unsigned const Mask2Bytes  = 0xC0;
unsigned const Mask3Bytes  = 0xE0;
unsigned const Mask4Bytes  = 0xF0;
unsigned const Mask5Bytes  = 0xF8;
unsigned const Mask6Bytes  = 0xFC;

namespace zorba {
namespace utf8 {

size_type byte_pos( storage_type const *s, size_type char_pos ) {
  if ( char_pos == npos )
    return npos;
  storage_type const *p = s;
  for ( ; char_pos > 0; --char_pos ) {
    if ( !*p )
      return npos;
    p += char_length( *p );
  }
  return p - s;
}

size_type byte_pos( storage_type const *s, size_type s_size,
                    size_type char_pos ) {
  if ( char_pos == npos )
    return npos;
  storage_type const *p = s;
  storage_type const *const end = s + s_size;
  for ( ; char_pos > 0; --char_pos ) {
    if ( p >= end )
      return npos;
    p += char_length( *p );
  }
  return p - s;
}

size_type char_length( storage_type lead ) {
  static char const length_table[] = {
    /*      0 1 2 3 4 5 6 7 8 9 A B C D E F */
    /* 0 */ 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
    /* 1 */ 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
    /* 2 */ 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
    /* 3 */ 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
    /* 4 */ 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
    /* 5 */ 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
    /* 6 */ 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
    /* 7 */ 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
    /* 8 */ 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
    /* 9 */ 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
    /* A */ 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
    /* B */ 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
    /* C */ 2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,
    /* D */ 2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,
    /* E */ 3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,
    /* F */ 4,4,4,4,4,4,4,4,5,5,5,5,6,6,0,0
  };

  size_type const len = length_table[ static_cast<unsigned>( lead ) & 0xFF ];
  if ( !len )
    ZORBA_ERROR( XQP0034_ILLEGAL_UTF8_BYTE );
  return len;
}

size_type char_pos( storage_type const *s, storage_type const *p ) {
  size_type pos = 0;
  while ( s < p ) {
    s += char_length( *s );
    ++pos;
  }
  return pos;
}

size_type encode( unicode::code_point c, storage_type **ps ) {
  if ( !unicode::is_valid( c ) )
    ZORBA_ERROR( FOCH0001 );
  unsigned const n = c & 0xFFFFFFFF;
  storage_type *&p = *ps, *const p0 = p;
  if ( n < 0x80 ) {
    // 0xxxxxxx
    *p++ = static_cast<storage_type>( n );
  } else if ( n < 0x800 ) {
    // 110xxxxx 10xxxxxx
    *p++ = static_cast<storage_type>( Mask2Bytes | ( n >> 6 )         );
    *p++ = static_cast<storage_type>( Mask1Byte  | ( n        & 0x3F) );
  } else if ( n < 0x10000 ) {
    // 1110xxxx 10xxxxxx 10xxxxxx
    *p++ = static_cast<storage_type>( Mask3Bytes |  (n >> 12)         );
    *p++ = static_cast<storage_type>( Mask1Byte  | ((n >>  6) & 0x3F) );
    *p++ = static_cast<storage_type>( Mask1Byte  | ( n        & 0x3F) );
  } else if ( n < 0x200000 ) {
    // 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx
    *p++ = static_cast<storage_type>( Mask4Bytes |  (n >> 18)         );
    *p++ = static_cast<storage_type>( Mask1Byte  | ((n >> 12) & 0x3F) );
    *p++ = static_cast<storage_type>( Mask1Byte  | ((n >>  6) & 0x3F) );
    *p++ = static_cast<storage_type>( Mask1Byte  | ( n        & 0x3F) );
  } else if ( n < 0x4000000 ) {
    // 111110xx 10xxxxxx 10xxxxxx 10xxxxxx 10xxxxxx
    *p++ = static_cast<storage_type>( Mask5Bytes |  (n >> 24)         );
    *p++ = static_cast<storage_type>( Mask1Byte  | ((n >> 18) & 0x3F) );
    *p++ = static_cast<storage_type>( Mask1Byte  | ((n >> 12) & 0x3F) );
    *p++ = static_cast<storage_type>( Mask1Byte  | ((n >>  6) & 0x3F) );
    *p++ = static_cast<storage_type>( Mask1Byte  | ( n        & 0x3F) );
  } else if ( n < 0x8000000 ) {
    // 1111110x 10xxxxxx 10xxxxxx 10xxxxxx 10xxxxxx 10xxxxxx
    *p++ = static_cast<storage_type>( Mask6Bytes |  (n >> 30)         );
    *p++ = static_cast<storage_type>( Mask1Byte  | ((n >> 24) & 0x3F) );
    *p++ = static_cast<storage_type>( Mask1Byte  | ((n >> 18) & 0x3F) );
    *p++ = static_cast<storage_type>( Mask1Byte  | ((n >> 12) & 0x3F) );
    *p++ = static_cast<storage_type>( Mask1Byte  | ((n >>  6) & 0x3F) );
    *p++ = static_cast<storage_type>( Mask1Byte  | ( n        & 0x3F) );
  }
  return p - p0;
}

size_type length( storage_type const *s ) {
  size_type len = 0;
  while ( *s ) {
    s += char_length( *s );
    ++len;
  }
  return len;
}

size_type length( storage_type const *begin, storage_type const *end ) {
  size_type len = 0;
  while ( begin < end && *begin ) {
    begin += char_length( *begin );
    ++len;
  }
  return len;
}

#ifndef ZORBA_NO_UNICODE

bool to_string( unicode::char_type const *in, size_type in_len,
                storage_type **out, size_type *out_len ) {
  unicode::code_point const SubChar = 0xFFFD;

  UErrorCode err = U_ZERO_ERROR;
  unicode::size_type utf8_len;
  u_strToUTF8WithSub(                   // pre-flight to get utf8_len
    NULL, 0, &utf8_len, in, in_len, SubChar, NULL, &err
  );
  // must not check "err" here since ICU always returns "buffer overflow"

  storage_type *const utf8_buf = new storage_type[ utf8_len ];
  err = U_ZERO_ERROR;
  u_strToUTF8WithSub(
    utf8_buf, utf8_len, NULL, in, in_len, SubChar, NULL, &err
  );
  if ( U_FAILURE( err ) ) {
    delete[] utf8_buf;
    return false;
  }
  *out = utf8_buf;
  *out_len = utf8_len;
  return true;
}

#ifndef WIN32
bool to_string( wchar_t const *in, size_type in_len, storage_type **out,
                size_type *out_len ) {
  bool result = false;
  unicode::string temp;
  if ( unicode::to_string( in, in_len, &temp ) ) {
    size_type const temp_len = temp.length();
    unicode::char_type *const buf = temp.getBuffer( -1 );
    result = utf8::to_string( buf, temp_len, out, out_len );
    temp.releaseBuffer();
  }
  return result;
}
#endif /* WIN32 */

#endif /* ZORBA_NO_UNICODE */

} // namespace utf8
} // namespace zorba

/* vim:set et sw=2 ts=2: */
<|endoftext|>


<commit_before>#include <bitcoin/utility/logger.hpp>

#include <iostream>

namespace libbitcoin {

std::string level_repr(log_level level)
{
    switch (level)
    {
        case log_level::null:
            return "NULL";
        case log_level::debug:
            return "DEBUG";
        case log_level::info:
            return "INFO";
        case log_level::warning:
            return "WARNING";
        case log_level::error:
            return "ERROR";
        case log_level::fatal:
            return "FATAL";
    }
    return "";
}

void output_to_ostream(std::ostream& ostr, log_level level,
    const std::string& domain, const std::string& body)
{
    std::cerr << level_repr(level);
    if (!domain.empty())
        std::cerr << " [" << domain << "]";
    std::cerr << ": " << body << std::endl;
}

void output_to_cout(log_level level,
    const std::string& domain, const std::string& body)
{
    output_to_ostream(std::cout, level, domain, body);
}
void output_to_cerr(log_level level,
    const std::string& domain, const std::string& body)
{
    output_to_ostream(std::cout, level, domain, body);
}

logger_wrapper::destination_map logger_wrapper::dests_{
    std::make_pair(log_level::debug, output_to_cout),
    std::make_pair(log_level::info, output_to_cout),
    std::make_pair(log_level::warning, output_to_cerr),
    std::make_pair(log_level::error, output_to_cerr),
    std::make_pair(log_level::fatal, output_to_cerr)
};

logger_wrapper::logger_wrapper(log_level lev, const std::string& domain)
  : level_(lev), domain_(domain)
{
}
// g++ bug in initializer list.
// It should be: stream_(std::move(other.stream_))
// http://gcc.gnu.org/bugzilla/show_bug.cgi?id=54316
logger_wrapper::logger_wrapper(logger_wrapper&& other)
  : level_(other.level_), stream_(other.stream_.str()),
    domain_(std::move(other.domain_))
{
}
logger_wrapper::~logger_wrapper()
{
    if (!dests_.count(level_))
        return;
    auto& outfunc = dests_[level_];
    outfunc(level_, domain_, stream_.str());
}

void logger_wrapper::set_output_function(logger_output_func outfunc)
{
    dests_[level_] = outfunc;
}

logger_wrapper log_debug(const std::string& domain)
{
    return logger_wrapper(log_level::debug, domain);
}

logger_wrapper log_info(const std::string& domain)
{
    return logger_wrapper(log_level::info, domain);
}

logger_wrapper log_warning(const std::string& domain)
{
    return logger_wrapper(log_level::warning, domain);
}

logger_wrapper log_error(const std::string& domain)
{
    return logger_wrapper(log_level::error, domain);
}

logger_wrapper log_fatal(const std::string& domain)
{
    return logger_wrapper(log_level::fatal, domain);
}

} // namespace libbitcoin

<commit_msg>nicer function names for logger (output_cout() instead of output_to_cout())<commit_after>#include <bitcoin/utility/logger.hpp>

#include <iostream>

namespace libbitcoin {

std::string level_repr(log_level level)
{
    switch (level)
    {
        case log_level::null:
            return "NULL";
        case log_level::debug:
            return "DEBUG";
        case log_level::info:
            return "INFO";
        case log_level::warning:
            return "WARNING";
        case log_level::error:
            return "ERROR";
        case log_level::fatal:
            return "FATAL";
    }
    return "";
}

void output_ostream(std::ostream& ostr, log_level level,
    const std::string& domain, const std::string& body)
{
    std::cerr << level_repr(level);
    if (!domain.empty())
        std::cerr << " [" << domain << "]";
    std::cerr << ": " << body << std::endl;
}

void output_cout(log_level level,
    const std::string& domain, const std::string& body)
{
    output_ostream(std::cout, level, domain, body);
}
void output_cerr(log_level level,
    const std::string& domain, const std::string& body)
{
    output_ostream(std::cout, level, domain, body);
}

logger_wrapper::destination_map logger_wrapper::dests_{
    std::make_pair(log_level::debug, output_cout),
    std::make_pair(log_level::info, output_cout),
    std::make_pair(log_level::warning, output_cerr),
    std::make_pair(log_level::error, output_cerr),
    std::make_pair(log_level::fatal, output_cerr)
};

logger_wrapper::logger_wrapper(log_level lev, const std::string& domain)
  : level_(lev), domain_(domain)
{
}
// g++ bug in initializer list.
// It should be: stream_(std::move(other.stream_))
// http://gcc.gnu.org/bugzilla/show_bug.cgi?id=54316
logger_wrapper::logger_wrapper(logger_wrapper&& other)
  : level_(other.level_), stream_(other.stream_.str()),
    domain_(std::move(other.domain_))
{
}
logger_wrapper::~logger_wrapper()
{
    if (!dests_.count(level_))
        return;
    auto& outfunc = dests_[level_];
    outfunc(level_, domain_, stream_.str());
}

void logger_wrapper::set_output_function(logger_output_func outfunc)
{
    dests_[level_] = outfunc;
}

logger_wrapper log_debug(const std::string& domain)
{
    return logger_wrapper(log_level::debug, domain);
}

logger_wrapper log_info(const std::string& domain)
{
    return logger_wrapper(log_level::info, domain);
}

logger_wrapper log_warning(const std::string& domain)
{
    return logger_wrapper(log_level::warning, domain);
}

logger_wrapper log_error(const std::string& domain)
{
    return logger_wrapper(log_level::error, domain);
}

logger_wrapper log_fatal(const std::string& domain)
{
    return logger_wrapper(log_level::fatal, domain);
}

} // namespace libbitcoin

<|endoftext|>


<commit_before>#include	<cstdlib>
#include	<cmath>
#include	<fstream>
#include	<sstream>
#include	<iostream>
#include	<iomanip>
#include	<string>
#include	<vector>
#include	<algorithm>
#include	<exception>
#include	<sys/time.h>
#include	"modules/htmTree.h"
#include	"modules/kdTree.h"
#include        "misc.h"
#include        "feat.h"
#include        "structs.h"
#include        "collision.h"
#include        "global.h"
//reduce redistributes, updates  07/02/15 rnc
int main(int argc, char **argv) {
	//// Initializations ---------------------------------------------
	srand48(1234); // Make sure we have reproducability
	check_args(argc);
	Time t, time; // t for global, time for local
	init_time(t);
	Feat F;
    MTL M;

	// Read parameters file //
	F.readInputFile(argv[1]);
	printFile(argv[1]);
	// Read galaxies
	Gals G;
    if(F.Ascii){
        G=read_galaxies_ascii(F);}
    else{
        G = read_galaxies(F);
    }
	F.Ngal = G.size();
	printf("# Read %s galaxies from %s \n",f(F.Ngal).c_str(),F.galFile.c_str());
    std::vector<int> count;
    count=count_galaxies(G);
    printf(" Number of galaxies by type, QSO-Ly-a, QSO-tracers, LRG, ELG, fake QSO, fake LRG, SS, SF\n");
    for(int i=0;i<8;i++){printf (" type %d number  %d  \n",i, count[i]);}
    //read the three input files
    MTL Targ=read_MTLfile(F.Targfile,F,0,0);
    MTL SStars=read_MTLfile(F.SStarsfile,F,1,0);
    MTL SkyF=read_MTLfile(F.SkyFfile,F,0,1);
    //combine the three input files

    M=Targ;
    M.reserve(Targ.size()+SStars.size()+SkyF.size());
    printf(" M size %d \n",M.size());
    M.insert(M.end(),SStars.begin(),SStars.end());
    printf(" M size %d \n",M.size());
    M.insert(M.end(),SkyF.begin(),SkyF.end());
    printf(" M size %d \n",M.size());
    //need to fix priority list, combining those from targets, standard stars, skyfibers
    std::vector<int> p_list=Targ.priority_list;
    //p_list.insert(p_list.end(),SStars.priority_list.begin(),SStars.priority_list.end());
    //p_list.insert(p_list.end(),SkyF.priority_list.begin(),SkyF.priority_list.end());
    //std::sort(p_list.begin(),p_list.end());

    //M.priority_list.clear();
    //M.priority_list.push_back(p_list[0]);
    //for(int i=1;i<p_list.size();++i){
       // if(p_list[i]!=p_list[i-1]){
         //   M.priority_list.push_back(p_list[i]);
           // }
    //}

    assign_priority_class(M);

    //establish priority classes
    std::vector <int> count_class(Targ.priority_list.size(),0);
    printf("Number in each priority class.  The last two are SF and SS.\n");
    for(int i;i<Targ.size();++i){
        count_class[Targ[i].priority_class]+=1;
    }
    for(int i;i<Targ.priority_list.size();++i){
        printf("  class  %d  number  %d\n",i,count_class[i]);
    }

	PP pp;
	pp.read_fiber_positions(F);
	F.Nfiber = pp.fp.size()/2;
	F.Npetal = max(pp.spectrom)+1;
    F.Nfbp = (int) (F.Nfiber/F.Npetal);// fibers per petal = 500
	pp.get_neighbors(F); pp.compute_fibsofsp(F);
	Plates OP = read_plate_centers(F);
    Plates P;
    printf(" future plates %d\n",P.size());
    F.ONplate=OP.size();
	printf("# Read %s plate centers from %s and %d fibers from %s\n",f(F.ONplate).c_str(),F.tileFile.c_str(),F.Nfiber,F.fibFile.c_str());

	// Computes geometries of cb and fh: pieces of positioner - used to determine possible collisions
	F.cb = create_cb(); // cb=central body
	F.fh = create_fh(); // fh=fiber holder

	//// Collect available galaxies <-> tilefibers --------------------
	// HTM Tree of galaxies
	const double MinTreeSize = 0.01;
	init_time_at(time,"# Start building HTM tree",t);
	htmTree<struct target> T(M,MinTreeSize);
	print_time(time,"# ... took :");//T.stats();

	// For plates/fibers, collect available galaxies; done in parallel
    collect_galaxies_for_all(M,T,OP,pp,F);

	// For each galaxy, computes available tilefibers  G[i].av_tfs = [(j1,k1),(j2,k2),..]
	collect_available_tilefibers(M,OP,F);

	//results_on_inputs("doc/figs/",G,P,F,true);

	//// Assignment |||||||||||||||||||||||||||||||||||||||||||||||||||
    printf(" Nplate %d  Ngal %d   Nfiber %d \n", F.ONplate, F.Ngal, F.Nfiber);
	Assignment A(M,F);
    printf(" Nplate %d  Ngal %d   Nfiber %d \n", F.ONplate, F.Ngal, F.Nfiber);
	print_time(t,"# Start assignment at : ");

	// Make a plan ----------------------------------------------------
    // Plans whole survey without sky fibers, standard stars
    // assumes maximum number of observations needed for QSOs, LRGs

    simple_assign(M,OP,pp,F,A);

    //check to see if there are tiles with no galaxies
    for (int j=0;j<F.ONplate ;++j){

        OP[j].is_used=false;
        bool not_done=true;
        for(int k=0;k<F.Nfiber && not_done;++k){
            if(A.TF[j][k]!=-1){
                OP[j].is_used=true;
                P.push_back(OP[j]);

                not_done=false;
            }
        }
    }
    F.Nplate=P.size();
    printf(" Plates after screening %d \n",F.Nplate);
    if(F.diagnose)diagnostic(M,G,F,A);

    print_hist("Unused fibers",5,histogram(A.unused_fbp(pp,F),5),false); // Hist of unused fibs

	// Smooth out distribution of free fibers, and increase the number of assignments

	for (int i=0; i<1; i++) redistribute_tf(M,P,pp,F,A);// more iterations will improve performance slightly
	for (int i=0; i<2; i++) {
        improve(M,P,pp,F,A);
		//redistribute_tf(M,P,pp,F,A);
		redistribute_tf(M,P,pp,F,A);
	}

	print_hist("Unused fibers",5,histogram(A.unused_fbp(pp,F),5),false);
    //try assigning SF and SS before real time assignment
    for (int j=0;j<F.Nplate;++j){
        A.next_plate=j;
        assign_sf_ss(j,M,P,pp,F,A); // Assign SS and SF for each tile
        assign_unused(j,M,P,pp,F,A);
    }

    init_time_at(time,"# Begin real time assignment",t);

	//Execute plan, updating targets at intervals

    for(int i=0;i<F.pass_intervals.size();++i){
        printf(" before pass = %d  at %d  tiles\n",i,F.pass_intervals[i]);
        //display_results("doc/figs/",G,P,pp,F,A,true);
        //execute this phase (i) of survey
        A.next_plate=F.pass_intervals[i];
        for (int jj=F.pass_intervals[i]; jj<F.Nplate&&jj<F.Nplate; jj++) {
            int j = A.next_plate;
            printf("  next plate is %d \n",j);
            assign_sf_ss(j,M,P,pp,F,A); // Assign SS and SF
            assign_unused(j,M,P,pp,F,A);
            A.next_plate++;
        }
        //update target information for this interval
        A.next_plate=F.pass_intervals[i];
        for (int jj=F.pass_intervals[i]; jj<F.pass_intervals[i+1]&&jj<F.Nplate; jj++) {
            int j = A.next_plate;

            // Update corrects all future occurrences of wrong QSOs etc and tries to observe something else
            if (0<=j-F.Analysis) update_plan_from_one_obs(G,M,P,pp,F,A,F.Nplate-1); else printf("\n");
            A.next_plate++;
        }
        /*
        if(A.next_plate<F.Nplate){
        redistribute_tf(M,P,pp,F,A);
        redistribute_tf(M,P,pp,F,A);
        improve(M,P,pp,F,A);
        redistribute_tf(M,P,pp,F,A);
        }
         */
        if(F.diagnose)diagnostic(M,G,F,A);
    }

	// Results -------------------------------------------------------
    if (F.PrintAscii) for (int j=0; j<F.Nplate; j++){
        write_FAtile_ascii(j,F.outDir,M,P,pp,F,A);
    }

    if (F.PrintFits) for (int j=0; j<F.Nplate; j++){
        fa_write(j,F.outDir,M,P,pp,F,A); // Write output
    }

	display_results("doc/figs/",G,M,P,pp,F,A,true);
	if (F.Verif) A.verif(P,M,pp,F); // Verification that the assignment is sane

	print_time(t,"# Finished !... in");

	return(0);

}
<commit_msg>more diagnostics<commit_after>#include	<cstdlib>
#include	<cmath>
#include	<fstream>
#include	<sstream>
#include	<iostream>
#include	<iomanip>
#include	<string>
#include	<vector>
#include	<algorithm>
#include	<exception>
#include	<sys/time.h>
#include	"modules/htmTree.h"
#include	"modules/kdTree.h"
#include        "misc.h"
#include        "feat.h"
#include        "structs.h"
#include        "collision.h"
#include        "global.h"
//reduce redistributes, updates  07/02/15 rnc
int main(int argc, char **argv) {
	//// Initializations ---------------------------------------------
	srand48(1234); // Make sure we have reproducability
	check_args(argc);
	Time t, time; // t for global, time for local
	init_time(t);
	Feat F;
    MTL M;

	// Read parameters file //
	F.readInputFile(argv[1]);
	printFile(argv[1]);
	// Read galaxies
	Gals G;
    if(F.Ascii){
        G=read_galaxies_ascii(F);}
    else{
        G = read_galaxies(F);
    }
	F.Ngal = G.size();
	printf("# Read %s galaxies from %s \n",f(F.Ngal).c_str(),F.galFile.c_str());
    std::vector<int> count;
    count=count_galaxies(G);
    printf(" Number of galaxies by type, QSO-Ly-a, QSO-tracers, LRG, ELG, fake QSO, fake LRG, SS, SF\n");
    for(int i=0;i<8;i++){printf (" type %d number  %d  \n",i, count[i]);}
    //read the three input files
    MTL Targ=read_MTLfile(F.Targfile,F,0,0);
    MTL SStars=read_MTLfile(F.SStarsfile,F,1,0);
    MTL SkyF=read_MTLfile(F.SkyFfile,F,0,1);
    //combine the three input files

    M=Targ;
    M.reserve(Targ.size()+SStars.size()+SkyF.size());
    printf(" M size %d \n",M.size());
    M.insert(M.end(),SStars.begin(),SStars.end());
    printf(" M size %d \n",M.size());
    M.insert(M.end(),SkyF.begin(),SkyF.end());
    printf(" M size %d \n",M.size());
    //need to fix priority list, combining those from targets, standard stars, skyfibers
    std::vector<int> p_list=Targ.priority_list;
    //p_list.insert(p_list.end(),SStars.priority_list.begin(),SStars.priority_list.end());
    //p_list.insert(p_list.end(),SkyF.priority_list.begin(),SkyF.priority_list.end());
    //std::sort(p_list.begin(),p_list.end());

    //M.priority_list.clear();
    //M.priority_list.push_back(p_list[0]);
    //for(int i=1;i<p_list.size();++i){
       // if(p_list[i]!=p_list[i-1]){
         //   M.priority_list.push_back(p_list[i]);
           // }
    //}

    assign_priority_class(M);

    //establish priority classes
    std::vector <int> count_class(Targ.priority_list.size(),0);
    printf("Number in each priority class.  The last two are SF and SS.\n");
    for(int i;i<Targ.size();++i){
        count_class[Targ[i].priority_class]+=1;
    }
    for(int i;i<Targ.priority_list.size();++i){
        printf("  class  %d  number  %d\n",i,count_class[i]);
    }

	PP pp;
	pp.read_fiber_positions(F);
	F.Nfiber = pp.fp.size()/2;
	F.Npetal = max(pp.spectrom)+1;
    F.Nfbp = (int) (F.Nfiber/F.Npetal);// fibers per petal = 500
	pp.get_neighbors(F); pp.compute_fibsofsp(F);
	Plates OP = read_plate_centers(F);
    Plates P;
    printf(" future plates %d\n",P.size());
    F.ONplate=OP.size();
	printf("# Read %s plate centers from %s and %d fibers from %s\n",f(F.ONplate).c_str(),F.tileFile.c_str(),F.Nfiber,F.fibFile.c_str());

	// Computes geometries of cb and fh: pieces of positioner - used to determine possible collisions
	F.cb = create_cb(); // cb=central body
	F.fh = create_fh(); // fh=fiber holder

	//// Collect available galaxies <-> tilefibers --------------------
	// HTM Tree of galaxies
	const double MinTreeSize = 0.01;
	init_time_at(time,"# Start building HTM tree",t);
	htmTree<struct target> T(M,MinTreeSize);
	print_time(time,"# ... took :");//T.stats();

	// For plates/fibers, collect available galaxies; done in parallel
    collect_galaxies_for_all(M,T,OP,pp,F);

	// For each galaxy, computes available tilefibers  G[i].av_tfs = [(j1,k1),(j2,k2),..]
	collect_available_tilefibers(M,OP,F);

	//results_on_inputs("doc/figs/",G,P,F,true);

	//// Assignment |||||||||||||||||||||||||||||||||||||||||||||||||||
    Assignment A(M,F);
    printf(" Nplate %d  Ngal %d   Nfiber %d \n", F.ONplate, F.Ngal, F.Nfiber);
	print_time(t,"# Start assignment at : ");

	// Make a plan ----------------------------------------------------
    // Plans whole survey without sky fibers, standard stars
    // assumes maximum number of observations needed for QSOs, LRGs

    simple_assign(M,OP,pp,F,A);

    //check to see if there are tiles with no galaxies
    for (int j=0;j<F.ONplate ;++j){

        OP[j].is_used=false;
        bool not_done=true;
        for(int k=0;k<F.Nfiber && not_done;++k){
            if(A.TF[j][k]!=-1){
                OP[j].is_used=true;
                P.push_back(OP[j]);
                printf("j  %d   k  %d \n",j,k);
                not_done=false;
            }
        }
    }
    F.Nplate=P.size();
    printf(" Plates after screening %d \n",F.Nplate);
    if(F.diagnose)diagnostic(M,G,F,A);

    print_hist("Unused fibers",5,histogram(A.unused_fbp(pp,F),5),false); // Hist of unused fibs

	// Smooth out distribution of free fibers, and increase the number of assignments

	for (int i=0; i<1; i++) redistribute_tf(M,P,pp,F,A);// more iterations will improve performance slightly
	for (int i=0; i<2; i++) {
        improve(M,P,pp,F,A);
		//redistribute_tf(M,P,pp,F,A);
		redistribute_tf(M,P,pp,F,A);
	}

	print_hist("Unused fibers",5,histogram(A.unused_fbp(pp,F),5),false);
    //try assigning SF and SS before real time assignment
    for (int j=0;j<F.Nplate;++j){
        A.next_plate=j;
        assign_sf_ss(j,M,P,pp,F,A); // Assign SS and SF for each tile
        assign_unused(j,M,P,pp,F,A);
    }

    init_time_at(time,"# Begin real time assignment",t);

	//Execute plan, updating targets at intervals

    for(int i=0;i<F.pass_intervals.size();++i){
        printf(" before pass = %d  at %d  tiles\n",i,F.pass_intervals[i]);
        //display_results("doc/figs/",G,P,pp,F,A,true);
        //execute this phase (i) of survey
        A.next_plate=F.pass_intervals[i];
        for (int jj=F.pass_intervals[i]; jj<F.Nplate&&jj<F.Nplate; jj++) {
            int j = A.next_plate;
            printf("  next plate is %d \n",j);
            assign_sf_ss(j,M,P,pp,F,A); // Assign SS and SF
            assign_unused(j,M,P,pp,F,A);
            A.next_plate++;
        }
        //update target information for this interval
        A.next_plate=F.pass_intervals[i];
        for (int jj=F.pass_intervals[i]; jj<F.pass_intervals[i+1]&&jj<F.Nplate; jj++) {
            int j = A.next_plate;

            // Update corrects all future occurrences of wrong QSOs etc and tries to observe something else
            if (0<=j-F.Analysis) update_plan_from_one_obs(G,M,P,pp,F,A,F.Nplate-1); else printf("\n");
            A.next_plate++;
        }
        /*
        if(A.next_plate<F.Nplate){
        redistribute_tf(M,P,pp,F,A);
        redistribute_tf(M,P,pp,F,A);
        improve(M,P,pp,F,A);
        redistribute_tf(M,P,pp,F,A);
        }
         */
        if(F.diagnose)diagnostic(M,G,F,A);
    }

	// Results -------------------------------------------------------
    if (F.PrintAscii) for (int j=0; j<F.Nplate; j++){
        write_FAtile_ascii(j,F.outDir,M,P,pp,F,A);
    }

    if (F.PrintFits) for (int j=0; j<F.Nplate; j++){
        fa_write(j,F.outDir,M,P,pp,F,A); // Write output
    }

	display_results("doc/figs/",G,M,P,pp,F,A,true);
	if (F.Verif) A.verif(P,M,pp,F); // Verification that the assignment is sane

	print_time(t,"# Finished !... in");

	return(0);

}
<|endoftext|>


<commit_before>#include	<cstdlib>
#include	<cmath>
#include	<fstream>
#include	<sstream>
#include	<iostream>
#include	<iomanip>
#include	<string>
#include	<vector>
#include	<algorithm>
#include	<exception>
#include	<sys/time.h>
#include	"modules/htmTree.h"
#include	"modules/kdTree.h"
#include        "misc.h"
#include        "feat.h"
#include        "structs.h"
#include        "collision.h"
#include        "global.h"
//reduce redistributes, updates  07/02/15 rnc
int main(int argc, char **argv) {
	//// Initializations ---------------------------------------------
	srand48(1234); // Make sure we have reproducability
	check_args(argc);
	Time t, time; // t for global, time for local
	init_time(t);
	Feat F;
    MTL M;

	// Read parameters file //
	F.readInputFile(argv[1]);
	printFile(argv[1]);
	// Read galaxies
    /*
	Gals G;
    if(F.Ascii){
        G=read_galaxies_ascii(F);}
    else{
        G = read_galaxies(F);
    }
	F.Ngal = G.size();
     */

    // Secret contains the identity of each target: QSO-Ly-a, QSO-tracers, LRG, ELG, fake QSO, fake LRG, SS, SF
    Gals Secret;
    printf("# Read %s galaxies from %s \n",f(F.Ngal).c_str(),F.Secretfile.c_str());
    Secret=read_Secretfile(F.Secretfile,F);
    F.Ngal= Secret.size();

    std::vector<int> count;
    count=count_galaxies(Secret);
    printf(" Number of galaxies by type, QSO-Ly-a, QSO-tracers, LRG, ELG, fake QSO, fake LRG, SS, SF\n");
    for(int i=0;i<8;i++){printf (" type %d number  %d  \n",i, count[i]);}
    //read the three input files
    MTL Targ=read_MTLfile(F.Targfile,F,0,0);
    MTL SStars=read_MTLfile(F.SStarsfile,F,1,0);
    MTL SkyF=read_MTLfile(F.SkyFfile,F,0,1);
    //combine the three input files

    M=Targ;

    printf(" M size %d \n",M.size());
    M.insert(M.end(),SStars.begin(),SStars.end());
    printf(" M size %d \n",M.size());
    M.insert(M.end(),SkyF.begin(),SkyF.end());
    printf(" M size %d \n",M.size());

    assign_priority_class(Targ);

    //establish priority classes
    std::vector <int> count_class(Targ.priority_list.size(),0);
    printf("Number in each priority class.  The last two are SF and SS.\n");
    for(int i;i<Targ.size();++i){
        count_class[Targ[i].priority_class]+=1;
    }
    for(int i;i<Targ.priority_list.size();++i){
        printf("  class  %d  number  %d\n",i,count_class[i]);
    }

    PP pp;
	pp.read_fiber_positions(F);
	F.Nfiber = pp.fp.size()/2;
	F.Npetal = max(pp.spectrom)+1;
    F.Nfbp = (int) (F.Nfiber/F.Npetal);// fibers per petal = 500
	pp.get_neighbors(F); pp.compute_fibsofsp(F);
	Plates OP = read_plate_centers(F);
    Plates P;
    printf(" future plates %d\n",P.size());
    F.ONplate=OP.size();
	printf("# Read %s plate centers from %s and %d fibers from %s\n",f(F.ONplate).c_str(),F.tileFile.c_str(),F.Nfiber,F.fibFile.c_str());

	// Computes geometries of cb and fh: pieces of positioner - used to determine possible collisions
	F.cb = create_cb(); // cb=central body
	F.fh = create_fh(); // fh=fiber holder

	//// Collect available galaxies <-> tilefibers --------------------
	// HTM Tree of galaxies
	const double MinTreeSize = 0.01;
	init_time_at(time,"# Start building HTM tree",t);
	htmTree<struct target> T(M,MinTreeSize);
	print_time(time,"# ... took :");//T.stats();

	// For plates/fibers, collect available galaxies; done in parallel
    collect_galaxies_for_all(M,T,OP,pp,F);

	// For each galaxy, computes available tilefibers  G[i].av_tfs = [(j1,k1),(j2,k2),..]
	collect_available_tilefibers(M,OP,F);

	//results_on_inputs("doc/figs/",G,P,F,true);

	//// Assignment |||||||||||||||||||||||||||||||||||||||||||||||||||
    printf(" Nplate %d  Ngal %d   Nfiber %d \n", F.ONplate, F.Ngal, F.Nfiber);
    Assignment A(M,F);

	print_time(t,"# Start assignment at : ");

	// Make a plan ----------------------------------------------------
    // Plans whole survey without sky fibers, standard stars
    // assumes maximum number of observations needed for QSOs, LRGs

    simple_assign(M,OP,pp,F,A);

    //check to see if there are tiles with no galaxies
    //need to keep mapping of old tile list to new tile list
    for (int j=0;j<F.ONplate ;++j){
        bool not_done=true;
        for(int k=0;k<F.Nfiber && not_done;++k){
            if(A.TF[j][k]!=-1){
                 P.push_back(OP[j]);
                A.suborder.push_back(j);
                not_done=false;
            }
        }
    }
    F.Nplate=P.size();
    printf(" Plates after screening %d \n",F.Nplate);
    //if(F.diagnose)diagnostic(M,G,F,A);
    //diagnostic
    for (int j=0;j<F.Nplate;++j){
        for (int k=0;k<F.Nfiber;++k){
            if(k%500==0){
                int g=A.TF[A.suborder[j]][k];
                if(g!=-1){
                    dpair test_projection=projection(g,j,M,P);
                    if (test_projection.f>500. || test_projection.s>500.){
                        printf(" g %d  j %d test_projection.f  %f test_projection.s  %f\n",g,j,test_projection.f,test_projection.s);
                    }
                }
            }
        }
    }

    print_hist("Unused fibers",5,histogram(A.unused_fbp(pp,F),5),false); // Hist of unused fibs

	// Smooth out distribution of free fibers, and increase the number of assignments

	for (int i=0; i<1; i++) redistribute_tf(M,P,pp,F,A);// more iterations will improve performance slightly
	for (int i=0; i<1; i++) {
        improve(M,P,pp,F,A);
		redistribute_tf(M,P,pp,F,A);
	}

	print_hist("Unused fibers",5,histogram(A.unused_fbp(pp,F),5),false);
    //try assigning SF and SS before real time assignment
    for (int j=0;j<F.Nplate;++j){
        int js=A.suborder[j];
        A.next_plate=js;
        assign_sf_ss(js,M,P,pp,F,A); // Assign SS and SF for each tile
        assign_unused(js,M,P,pp,F,A);
    }

    init_time_at(time,"# Begin real time assignment",t);

	//Execute plan, updating targets at intervals

    for(int i=0;i<F.pass_intervals.size()&&F.pass_intervals[i]<F.Nplate;++i){
        printf(" before pass = %d  at %d  tiles\n",i,F.pass_intervals[i]);
        //display_results("doc/figs/",G,P,pp,F,A,true);
        //execute this phase (i) of survey
        //A.next_plate=F.pass_intervals[i];
        for (int jj=F.pass_intervals[i]; jj<F.Nplate; jj++) {
            int j = A.suborder[A.next_plate];
            printf("  next plate is %d \n",j);
            assign_sf_ss(j,M,P,pp,F,A); // Assign SS and SF
            assign_unused(j,M,P,pp,F,A);
            //A.next_plate++;
        }
        //update target information for this interval
        //A.next_plate=F.pass_intervals[i];
        for (int jj=F.pass_intervals[i]; jj<F.pass_intervals[i+1]&&jj<F.Nplate; jj++) {
            //int j = A.suborder[A.next_plate];
            int js=A.suborder[jj];
            // Update corrects all future occurrences of wrong QSOs etc and tries to observe something else
            if (0<=js-F.Analysis) update_plan_from_one_obs(Secret,M,P,pp,F,A,F.Nplate-1); else printf("\n");
            //A.next_plate++;
        }
        /*
        if(A.next_plate<F.Nplate){
        redistribute_tf(M,P,pp,F,A);
        redistribute_tf(M,P,pp,F,A);
        improve(M,P,pp,F,A);
        redistribute_tf(M,P,pp,F,A);
        }
         */
        if(F.diagnose)diagnostic(M,Secret,F,A);
    }

	// Results -------------------------------------------------------
    if (F.PrintAscii) for (int j=0; j<F.Nplate; j++){
        write_FAtile_ascii(j,F.outDir,M,P,pp,F,A);
    }

    if (F.PrintFits) for (int j=0; j<F.Nplate; j++){
        fa_write(j,F.outDir,M,P,pp,F,A); // Write output
    }

	display_results("doc/figs/",Secret,M,P,pp,F,A,true);
	if (F.Verif) A.verif(P,M,pp,F); // Verification that the assignment is sane

	print_time(t,"# Finished !... in");

	return(0);

}
<commit_msg>new diagnostic<commit_after>#include	<cstdlib>
#include	<cmath>
#include	<fstream>
#include	<sstream>
#include	<iostream>
#include	<iomanip>
#include	<string>
#include	<vector>
#include	<algorithm>
#include	<exception>
#include	<sys/time.h>
#include	"modules/htmTree.h"
#include	"modules/kdTree.h"
#include        "misc.h"
#include        "feat.h"
#include        "structs.h"
#include        "collision.h"
#include        "global.h"
//reduce redistributes, updates  07/02/15 rnc
int main(int argc, char **argv) {
	//// Initializations ---------------------------------------------
	srand48(1234); // Make sure we have reproducability
	check_args(argc);
	Time t, time; // t for global, time for local
	init_time(t);
	Feat F;
    MTL M;

	// Read parameters file //
	F.readInputFile(argv[1]);
	printFile(argv[1]);
	// Read galaxies
    /*
	Gals G;
    if(F.Ascii){
        G=read_galaxies_ascii(F);}
    else{
        G = read_galaxies(F);
    }
	F.Ngal = G.size();
     */

    // Secret contains the identity of each target: QSO-Ly-a, QSO-tracers, LRG, ELG, fake QSO, fake LRG, SS, SF
    Gals Secret;
    printf("# Read %s galaxies from %s \n",f(F.Ngal).c_str(),F.Secretfile.c_str());
    Secret=read_Secretfile(F.Secretfile,F);
    F.Ngal= Secret.size();

    std::vector<int> count;
    count=count_galaxies(Secret);
    printf(" Number of galaxies by type, QSO-Ly-a, QSO-tracers, LRG, ELG, fake QSO, fake LRG, SS, SF\n");
    for(int i=0;i<8;i++){printf (" type %d number  %d  \n",i, count[i]);}
    //read the three input files
    MTL Targ=read_MTLfile(F.Targfile,F,0,0);
    MTL SStars=read_MTLfile(F.SStarsfile,F,1,0);
    MTL SkyF=read_MTLfile(F.SkyFfile,F,0,1);
    //combine the three input files

    M=Targ;

    printf(" M size %d \n",M.size());
    M.insert(M.end(),SStars.begin(),SStars.end());
    printf(" M size %d \n",M.size());
    M.insert(M.end(),SkyF.begin(),SkyF.end());
    printf(" M size %d \n",M.size());

    assign_priority_class(Targ);

    //establish priority classes
    std::vector <int> count_class(Targ.priority_list.size(),0);
    printf("Number in each priority class.  The last two are SF and SS.\n");
    for(int i;i<Targ.size();++i){
        count_class[Targ[i].priority_class]+=1;
    }
    for(int i;i<Targ.priority_list.size();++i){
        printf("  class  %d  number  %d\n",i,count_class[i]);
    }

    PP pp;
	pp.read_fiber_positions(F);
	F.Nfiber = pp.fp.size()/2;
	F.Npetal = max(pp.spectrom)+1;
    F.Nfbp = (int) (F.Nfiber/F.Npetal);// fibers per petal = 500
	pp.get_neighbors(F); pp.compute_fibsofsp(F);
	Plates OP = read_plate_centers(F);
    Plates P;
    printf(" future plates %d\n",P.size());
    F.ONplate=OP.size();
	printf("# Read %s plate centers from %s and %d fibers from %s\n",f(F.ONplate).c_str(),F.tileFile.c_str(),F.Nfiber,F.fibFile.c_str());

	// Computes geometries of cb and fh: pieces of positioner - used to determine possible collisions
	F.cb = create_cb(); // cb=central body
	F.fh = create_fh(); // fh=fiber holder

	//// Collect available galaxies <-> tilefibers --------------------
	// HTM Tree of galaxies
	const double MinTreeSize = 0.01;
	init_time_at(time,"# Start building HTM tree",t);
	htmTree<struct target> T(M,MinTreeSize);
	print_time(time,"# ... took :");//T.stats();

	// For plates/fibers, collect available galaxies; done in parallel
    collect_galaxies_for_all(M,T,OP,pp,F);

	// For each galaxy, computes available tilefibers  G[i].av_tfs = [(j1,k1),(j2,k2),..]
	collect_available_tilefibers(M,OP,F);

	//results_on_inputs("doc/figs/",G,P,F,true);

	//// Assignment |||||||||||||||||||||||||||||||||||||||||||||||||||
    printf(" Nplate %d  Ngal %d   Nfiber %d \n", F.ONplate, F.Ngal, F.Nfiber);
    Assignment A(M,F);

	print_time(t,"# Start assignment at : ");

	// Make a plan ----------------------------------------------------
    // Plans whole survey without sky fibers, standard stars
    // assumes maximum number of observations needed for QSOs, LRGs

    simple_assign(M,OP,pp,F,A);

    //check to see if there are tiles with no galaxies
    //need to keep mapping of old tile list to new tile list
    for (int j=0;j<F.ONplate ;++j){
        bool not_done=true;
        for(int k=0;k<F.Nfiber && not_done;++k){
            if(A.TF[j][k]!=-1){
                 P.push_back(OP[j]);
                A.suborder.push_back(j);
                not_done=false;
            }
        }
    }
    F.Nplate=P.size();
    printf(" Plates after screening %d \n",F.Nplate);
    //if(F.diagnose)diagnostic(M,G,F,A);
    //diagnostic
    for (int j=0;j<F.Nplate;++j){
        for (int k=0;k<F.Nfiber;++k){
            if(k%50==0){
                int g=A.TF[A.suborder[j]][k];
                if(g!=-1){
                    dpair test_projection=projection(g,j,M,P);
                    if (test_projection.f>500. || test_projection.s>500.){
                        printf(" g %d  j %d test_projection.f  %f test_projection.s  %f\n",g,j,test_projection.f,test_projection.s);
                    }
                }
            }
        }
    }

    print_hist("Unused fibers",5,histogram(A.unused_fbp(pp,F),5),false); // Hist of unused fibs

	// Smooth out distribution of free fibers, and increase the number of assignments

	for (int i=0; i<1; i++) redistribute_tf(M,P,pp,F,A);// more iterations will improve performance slightly
	for (int i=0; i<1; i++) {
        improve(M,P,pp,F,A);
		redistribute_tf(M,P,pp,F,A);
	}

	print_hist("Unused fibers",5,histogram(A.unused_fbp(pp,F),5),false);
    //try assigning SF and SS before real time assignment
    for (int j=0;j<F.Nplate;++j){
        int js=A.suborder[j];
        A.next_plate=js;
        assign_sf_ss(js,M,P,pp,F,A); // Assign SS and SF for each tile
        assign_unused(js,M,P,pp,F,A);
    }

    init_time_at(time,"# Begin real time assignment",t);

	//Execute plan, updating targets at intervals

    for(int i=0;i<F.pass_intervals.size()&&F.pass_intervals[i]<F.Nplate;++i){
        printf(" before pass = %d  at %d  tiles\n",i,F.pass_intervals[i]);
        //display_results("doc/figs/",G,P,pp,F,A,true);
        //execute this phase (i) of survey
        //A.next_plate=F.pass_intervals[i];
        for (int jj=F.pass_intervals[i]; jj<F.Nplate; jj++) {
            int j = A.suborder[A.next_plate];
            printf("  next plate is %d \n",j);
            assign_sf_ss(j,M,P,pp,F,A); // Assign SS and SF
            assign_unused(j,M,P,pp,F,A);
            //A.next_plate++;
        }
        //update target information for this interval
        //A.next_plate=F.pass_intervals[i];
        for (int jj=F.pass_intervals[i]; jj<F.pass_intervals[i+1]&&jj<F.Nplate; jj++) {
            //int j = A.suborder[A.next_plate];
            int js=A.suborder[jj];
            // Update corrects all future occurrences of wrong QSOs etc and tries to observe something else
            if (0<=js-F.Analysis) update_plan_from_one_obs(Secret,M,P,pp,F,A,F.Nplate-1); else printf("\n");
            //A.next_plate++;
        }
        /*
        if(A.next_plate<F.Nplate){
        redistribute_tf(M,P,pp,F,A);
        redistribute_tf(M,P,pp,F,A);
        improve(M,P,pp,F,A);
        redistribute_tf(M,P,pp,F,A);
        }
         */
        if(F.diagnose)diagnostic(M,Secret,F,A);
    }

	// Results -------------------------------------------------------
    if (F.PrintAscii) for (int j=0; j<F.Nplate; j++){
        write_FAtile_ascii(j,F.outDir,M,P,pp,F,A);
    }

    if (F.PrintFits) for (int j=0; j<F.Nplate; j++){
        fa_write(j,F.outDir,M,P,pp,F,A); // Write output
    }

	display_results("doc/figs/",Secret,M,P,pp,F,A,true);
	if (F.Verif) A.verif(P,M,pp,F); // Verification that the assignment is sane

	print_time(t,"# Finished !... in");

	return(0);

}
<|endoftext|>