@@ -1169,7 +1169,8 @@ dynamo_shared_exit(thread_record_t *toexit /* must ==cur thread for Linux */
     return SUCCESS;
 }
 
-int
+/* NOINLINE because dynamorio_app_exit is a stopping point. */
+NOINLINE int
 dynamorio_app_exit(void)
 {
     return dynamo_process_exit();
/* **********************************************************
 * Copyright (c) 2010-2019 Google, Inc.  All rights reserved.
 * Copyright (c) 2000-2010 VMware, Inc.  All rights reserved.
 * **********************************************************/

/*
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are met:
 *
 * * Redistributions of source code must retain the above copyright notice,
 *   this list of conditions and the following disclaimer.
 *
 * * Redistributions in binary form must reproduce the above copyright notice,
 *   this list of conditions and the following disclaimer in the documentation
 *   and/or other materials provided with the distribution.
 *
 * * Neither the name of VMware, Inc. nor the names of its contributors may be
 *   used to endorse or promote products derived from this software without
 *   specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
 * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
 * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
 * DAMAGE.
 */

/* Copyright (c) 2003-2007 Determina Corp. */
/* Copyright (c) 2001-2003 Massachusetts Institute of Technology */
/* Copyright (c) 2000-2001 Hewlett-Packard Company */

/*
 * dynamo.c -- initialization and cleanup routines for DynamoRIO
 */

#include "globals.h"
#include "configure_defines.h"
#include "link.h"
#include "fragment.h"
#include "fcache.h"
#include "emit.h"
#include "dispatch.h"
#include "utils.h"
#include "monitor.h"
#include "vmareas.h"
#ifdef SIDELINE
#    include "sideline.h"
#endif
#ifdef PAPI
#    include "perfctr.h"
#endif
#ifdef CLIENT_INTERFACE
#    include "instrument.h"
#endif
#include "hotpatch.h"
#include "moduledb.h"
#include "module_shared.h"
#include "synch.h"
#include "native_exec.h"
#include "jit_opt.h"

#ifdef ANNOTATIONS
#    include "annotations.h"
#endif

#ifdef WINDOWS
/* for close handle, duplicate handle, free memory and constants associated with them
 */
/* also for nt_terminate_process_for_app() */
#    include "ntdll.h"
#    include "nudge.h" /* to get generic_nudge_target() address for an assert */
#endif

#ifdef RCT_IND_BRANCH
#    include "rct.h"
#endif

#include "perscache.h"

#ifdef VMX86_SERVER
#    include "vmkuw.h"
#endif

#ifndef STANDALONE_UNIT_TEST
#    ifdef __AVX512F__
#        error "DynamoRIO core should run without AVX-512 instructions to remain \
portable and to avoid frequency scaling."
#    endif
#endif

/* global thread-shared variables */
bool dynamo_initialized = false;
bool dynamo_heap_initialized = false;
bool dynamo_started = false;
bool automatic_startup = false;
bool control_all_threads = false;
#ifdef WINDOWS
bool dr_early_injected = false;
int dr_early_injected_location = INJECT_LOCATION_Invalid;
bool dr_earliest_injected = false;
static void *dr_earliest_inject_args;

/* should be set if we are controlling the primary thread, either by
 * injecting initially (!dr_injected_secondary_thread), or by retaking
 * over (dr_late_injected_primary_thread).  Used only for debugging
 * purposes, yet can't rely on !dr_injected_secondary_thread very
 * early in the process
 */
bool dr_injected_primary_thread = false;
bool dr_injected_secondary_thread = false;

/* should be set once we retakeover the primary thread for -inject_primary */
bool dr_late_injected_primary_thread = false;
#endif /* WINDOWS */
/* flags to indicate when DR is being initialized / exited using the API */
bool dr_api_entry = false;
bool dr_api_exit = false;
#ifdef RETURN_AFTER_CALL
bool dr_preinjected = false;
#endif /* RETURN_AFTER_CALL */
#ifdef UNIX
static bool dynamo_exiting = false;
#endif
bool dynamo_exited = false;
bool dynamo_exited_all_other_threads = false;
bool dynamo_exited_and_cleaned = false;
#ifdef DEBUG
bool dynamo_exited_log_and_stats = false;
#endif
/* Only used in release build to decide whether synch is needed, justifying
 * its placement in .nspdata.  If we use it for more we should protect it.
 */
DECLARE_NEVERPROT_VAR(bool dynamo_all_threads_synched, false);
bool dynamo_resetting = false;
#if defined(CLIENT_INTERFACE) || defined(STANDALONE_UNIT_TEST)
bool standalone_library = false;
#endif
#ifdef UNIX
bool post_execve = false;
#endif
/* initial stack so we don't have to use app's */
byte *d_r_initstack;

event_t dr_app_started;
event_t dr_attach_finished;

#ifdef WINDOWS
/* PR203701: separate stack for error reporting when the dstack is exhausted */
#    define EXCEPTION_STACK_SIZE (2 * PAGE_SIZE)
DECLARE_NEVERPROT_VAR(byte *exception_stack, NULL);
#endif

/*******************************************************/
/* separate segment of Non-Self-Protected data to avoid data section
 * protection issues -- we need to write to these vars in bootstrapping
 * spots where we cannot unprotect first
 */
START_DATA_SECTION(NEVER_PROTECTED_SECTION, "w");

/* spinlock used in assembly trampolines when we can't spare registers for more */
mutex_t initstack_mutex VAR_IN_SECTION(NEVER_PROTECTED_SECTION) =
    INIT_SPINLOCK_FREE(initstack_mutex);
byte *initstack_app_xsp VAR_IN_SECTION(NEVER_PROTECTED_SECTION) = 0;
/* keeps track of how many threads are in cleanup_and_terminate */
volatile int exiting_thread_count VAR_IN_SECTION(NEVER_PROTECTED_SECTION) = 0;
/* Tracks newly created threads not yet on the all_threads list. */
volatile int uninit_thread_count VAR_IN_SECTION(NEVER_PROTECTED_SECTION) = 0;

/* This is unprotected to allow stats to be written while the data
 * segment is still protected (right now the only ones are selfmod stats)
 */
static dr_statistics_t nonshared_stats VAR_IN_SECTION(NEVER_PROTECTED_SECTION) = {
    { 0 },
};

/* Each lock protects its corresponding datasec_start, datasec_end, and
 * datasec_writable variables.
 */
static mutex_t
    datasec_lock[DATASEC_NUM] VAR_IN_SECTION(NEVER_PROTECTED_SECTION) = { { 0 } };

/* back to normal section */
END_DATA_SECTION()
/*******************************************************/

/* Like a recursive lock: 0==readonly, 1+=writable.
 * This would be a simple array, but we need each in its own protected
 * section, as this could be exploited.
 */
const uint datasec_writable_neverprot = 1; /* always writable */
uint datasec_writable_rareprot = 1;
DECLARE_FREQPROT_VAR(uint datasec_writable_freqprot, 1);
DECLARE_CXTSWPROT_VAR(uint datasec_writable_cxtswprot, 1);

static app_pc datasec_start[DATASEC_NUM];
static app_pc datasec_end[DATASEC_NUM];

const uint DATASEC_SELFPROT[] = {
    0,
    SELFPROT_DATA_RARE,
    SELFPROT_DATA_FREQ,
    SELFPROT_DATA_CXTSW,
};

const char *const DATASEC_NAMES[] = {
    NEVER_PROTECTED_SECTION,
    RARELY_PROTECTED_SECTION,
    FREQ_PROTECTED_SECTION,
    CXTSW_PROTECTED_SECTION,
};

/* kept in unprotected heap to avoid issues w/ data segment being RO */
typedef struct _protect_info_t {
    /* FIXME: this needs to be a recursive lock to handle signals
     * and exceptions!
     */
    mutex_t lock;
    int num_threads_unprot; /* # threads in DR code */
    int num_threads_suspended;
} protect_info_t;
static protect_info_t *protect_info;

static void
data_section_init(void);
static void
data_section_exit(void);

#ifdef DEBUG /*************************/

#    include <time.h>

/* FIXME: not all dynamo_options references are #ifdef DEBUG
 * are we trying to hardcode the options for a release build?
 */
#    ifdef UNIX
/* linux include files for mmap stuff*/
#        include <sys/ipc.h>
#        include <sys/types.h>
#        include <unistd.h>
#    endif

static uint starttime;

file_t main_logfile = INVALID_FILE;

#endif /* DEBUG ****************************/

dr_statistics_t *d_r_stats = NULL;

DECLARE_FREQPROT_VAR(static int num_known_threads, 0);
#ifdef UNIX
/* i#237/PR 498284: vfork threads that execve need to be separately delay-freed */
DECLARE_FREQPROT_VAR(int num_execve_threads, 0);
#endif
DECLARE_FREQPROT_VAR(static uint threads_ever_count, 0);

/* FIXME : not static so os.c can hand walk it for dump core */
/* FIXME: use new generic_table_t and generic_hash_* routines */
thread_record_t **all_threads; /* ALL_THREADS_HASH_BITS-bit addressed hash table */

/* these locks are used often enough that we put them in .cspdata: */

/* not static so can be referenced in win32/os.c for SuspendThread handling,
 * FIXME : is almost completely redundant in usage with thread_initexit_lock
 * maybe replace this lock with thread_initexit_lock? */
DECLARE_CXTSWPROT_VAR(mutex_t all_threads_lock, INIT_LOCK_FREE(all_threads_lock));
/* used for synch to prevent thread creation/deletion in critical periods
 * due to its use for flushing, this lock cannot be held while couldbelinking!
 */
DECLARE_CXTSWPROT_VAR(mutex_t thread_initexit_lock, INIT_LOCK_FREE(thread_initexit_lock));

/* recursive to handle signals/exceptions while in DR code */
DECLARE_CXTSWPROT_VAR(static recursive_lock_t thread_in_DR_exclusion,
                      INIT_RECURSIVE_LOCK(thread_in_DR_exclusion));

static thread_synch_state_t
exit_synch_state(void);

static void
synch_with_threads_at_exit(thread_synch_state_t synch_res, bool pre_exit);

/****************************************************************************/
#ifdef DEBUG

static const char *
main_logfile_name(void)
{
    return get_app_name_for_path();
}

static const char *
thread_logfile_name(void)
{
    return "log";
}

#endif /* DEBUG */
/****************************************************************************/

static void
statistics_pre_init(void)
{
    /* until it's set up for real, point at static var
     * really only logmask and loglevel are meaningful, so be careful!
     * statistics_init and create_log_directory are the only routines that
     * use stats before it's set up for real, currently
     */
    /* The indirection here is left over from when we used to allow alternative
     * locations for stats (namely shared memory for the old MIT gui). */
    d_r_stats = &nonshared_stats;
    d_r_stats->process_id = get_process_id();
    strncpy(d_r_stats->process_name, get_application_name(), MAXIMUM_PATH);
    d_r_stats->process_name[MAXIMUM_PATH - 1] = '\0';
    ASSERT(strlen(d_r_stats->process_name) > 0);
    d_r_stats->num_stats = 0;
}

static void
statistics_init(void)
{
    /* should have called statistics_pre_init() first */
    ASSERT(d_r_stats == &nonshared_stats);
    ASSERT(d_r_stats->num_stats == 0);
#ifndef DEBUG
    if (!DYNAMO_OPTION(global_rstats)) {
        /* references to stat values should return 0 (static var) */
        return;
    }
#endif
    d_r_stats->num_stats = 0
#ifdef DEBUG
#    define STATS_DEF(desc, name) +1
#else
#    define RSTATS_DEF(desc, name) +1
#endif
#include "statsx.h"
#undef STATS_DEF
#undef RSTATS_DEF
        ;
    /* We inline the stat description to make it easy for external processes
     * to view our stats: they don't have to chase pointers, and we could put
     * this in shared memory easily.  However, we do waste some memory, but
     * not much in release build.
     */
#ifdef DEBUG
#    define STATS_DEF(desc, statname)                                   \
        strncpy(d_r_stats->statname##_pair.name, desc,                  \
                BUFFER_SIZE_ELEMENTS(d_r_stats->statname##_pair.name)); \
        NULL_TERMINATE_BUFFER(d_r_stats->statname##_pair.name);
#else
#    define RSTATS_DEF(desc, statname)                                  \
        strncpy(d_r_stats->statname##_pair.name, desc,                  \
                BUFFER_SIZE_ELEMENTS(d_r_stats->statname##_pair.name)); \
        NULL_TERMINATE_BUFFER(d_r_stats->statname##_pair.name);
#endif
#include "statsx.h"
#undef STATS_DEF
#undef RSTATS_DEF
}

static void
statistics_exit(void)
{
    if (doing_detach)
        memset(d_r_stats, 0, sizeof(*d_r_stats)); /* for possible re-attach */
    d_r_stats = NULL;
}

dr_statistics_t *
get_dr_stats(void)
{
    return d_r_stats;
}

/* initialize per-process dynamo state; this must be called before any
 * threads are created and before any other API calls are made;
 * returns zero on success, non-zero on failure
 */
DYNAMORIO_EXPORT int
dynamorio_app_init(void)
{
    int size;

    if (!dynamo_initialized /* we do enter if nullcalls is on */) {

#ifdef UNIX
        os_page_size_init((const char **)our_environ, is_our_environ_followed_by_auxv());
#endif
#ifdef WINDOWS
        /* MUST do this before making any system calls */
        syscalls_init();
#endif
        /* avoid time() for libc independence */
        DODEBUG(starttime = query_time_seconds(););

#ifdef UNIX
        if (getenv(DYNAMORIO_VAR_EXECVE) != NULL) {
            post_execve = true;
#    ifdef VMX86_SERVER
            /* PR 458917: our gdt slot was not cleared on exec so we need to
             * clear it now to ensure we don't leak it and eventually run out of
             * slots.  We could alternatively call os_tls_exit() prior to
             * execve, since syscalls use thread-private fcache_enter, but
             * complex to recover from execve failure, so instead we pass which
             * TLS index we had.
             */
            os_tls_pre_init(atoi(getenv(DYNAMORIO_VAR_EXECVE)));
#    endif
            /* important to remove it, don't want to propagate to forked children, etc. */
            /* i#909: unsetenv is unsafe as it messes up auxv access, so we disable */
            disable_env(DYNAMORIO_VAR_EXECVE);
            /* check that it's gone: we've had problems with unsetenv */
            ASSERT(getenv(DYNAMORIO_VAR_EXECVE) == NULL);
        } else
            post_execve = false;
#endif

            /* default non-zero dynamo settings (options structure is
             * initialized to 0 automatically)
             */
#ifdef DEBUG
#    ifndef INTERNAL
        nonshared_stats.logmask = LOG_ALL_RELEASE;
#    else
        nonshared_stats.logmask = LOG_ALL;
#    endif
        statistics_pre_init();
#endif

        d_r_config_init();
        options_init();
#ifdef WINDOWS
        syscalls_init_options_read(); /* must be called after options_init
                                       * but before init_syscall_trampolines */
#endif
        utils_init();
        data_section_init();

#ifdef DEBUG
        /* decision: nullcalls WILL create a dynamorio.log file and
         * fill it with perfctr stats!
         */
        if (d_r_stats->loglevel > 0) {
            main_logfile = open_log_file(main_logfile_name(), NULL, 0);
            LOG(GLOBAL, LOG_TOP, 1, "global log file fd=%d\n", main_logfile);
        } else {
            /* loglevel 0 means we don't create a log file!
             * if the loglevel is later raised, too bad!  it all goes to stderr!
             * N.B.: when checking for no logdir, we check for empty string or
             * first char '<'!
             */
            strncpy(d_r_stats->logdir, "<none (loglevel was 0 on startup)>",
                    MAXIMUM_PATH - 1);
            d_r_stats->logdir[MAXIMUM_PATH - 1] = '\0'; /* if max no null */
            main_logfile = INVALID_FILE;
        }

#    ifdef PAPI
        /* setup hardware performance counting */
        hardware_perfctr_init();
#    endif

        DOLOG(1, LOG_TOP, { print_version_and_app_info(GLOBAL); });

        /* now exit if nullcalls, now that perfctrs are set up */
        if (INTERNAL_OPTION(nullcalls)) {
            print_file(main_logfile,
                       "** nullcalls is set, NOT taking over execution **\n\n");
            return SUCCESS;
        }

        LOG(GLOBAL, LOG_TOP, 1, PRODUCT_NAME "'s stack size: %d Kb\n",
            DYNAMORIO_STACK_SIZE / 1024);
#endif /* !DEBUG */

        /* set up exported statistics struct */
#ifndef DEBUG
        statistics_pre_init();
#endif
        statistics_init();

#ifdef VMX86_SERVER
        /* Must be before {vmm,d_r}_heap_init() */
        vmk_init_lib();
#endif

        /* initialize components (CAUTION: order is important here) */
        vmm_heap_init(); /* must be called even if not using vmm heap */
#ifdef CLIENT_INTERFACE
        /* PR 200207: load the client lib before callback_interception_init
         * since the client library load would hit our own hooks (xref hotpatch
         * cases about that) -- though -private_loader removes that issue.
         */
        instrument_load_client_libs();
#endif
        d_r_heap_init();
        dynamo_heap_initialized = true;

        /* The process start event should be done after d_r_os_init() but before
         * process_control_int() because the former initializes event logging
         * and the latter can kill the process if a violation occurs.
         */
        SYSLOG(SYSLOG_INFORMATION,
               IF_CLIENT_INTERFACE_ELSE(INFO_PROCESS_START_CLIENT, INFO_PROCESS_START),
               IF_CLIENT_INTERFACE_ELSE(2, 3), get_application_name(),
               get_application_pid() _IF_NOT_CLIENT_INTERFACE(get_application_md5()));

#ifdef PROCESS_CONTROL
        if (IS_PROCESS_CONTROL_ON()) /* Case 8594. */
            process_control_init();
#endif

#ifdef WINDOWS
        /* Now that DR is set up, perform any final clean-up, before
         * we do our address space scans.
         */
        if (dr_earliest_injected)
            earliest_inject_cleanup(dr_earliest_inject_args);
#endif

        dynamo_vm_areas_init();
        d_r_decode_init();
        proc_init();
        modules_init(); /* before vm_areas_init() */
        d_r_os_init();
        config_heap_init(); /* after heap_init */

        /* Setup for handling faults in loader_init() */
        /* initial stack so we don't have to use app's
         * N.B.: we never de-allocate d_r_initstack (see comments in app_exit)
         */
        d_r_initstack = (byte *)stack_alloc(DYNAMORIO_STACK_SIZE, NULL);
        LOG(GLOBAL, LOG_SYNCH, 2, "d_r_initstack is " PFX "-" PFX "\n",
            d_r_initstack - DYNAMORIO_STACK_SIZE, d_r_initstack);

#ifdef WINDOWS
        /* PR203701: separate stack for error reporting when the
         * dstack is exhausted
         */
        exception_stack = (byte *)stack_alloc(EXCEPTION_STACK_SIZE, NULL);
#endif
#ifdef WINDOWS
        if (!INTERNAL_OPTION(noasynch)) {
            /* We split the hooks up: first we put in just Ki* to catch
             * exceptions in client init routines (PR 200207), but we don't want
             * syscall hooks so client init can scan syscalls.
             * Xref PR 216934 where this was originally down below 1st thread init,
             * before we had GLOBAL_DCONTEXT.
             */
            callback_interception_init_start();
        }
#endif /* WINDOWS */

#ifdef WINDOWS
        /* loader initialization, finalize the private lib load.
         * i#338: this must be before d_r_arch_init() for Windows, but Linux
         * wants it later (i#2751).
         */
        loader_init();
#endif
        d_r_arch_init();
        synch_init();

#ifdef KSTATS
        kstat_init();
#endif
        d_r_monitor_init();
        fcache_init();
        d_r_link_init();
        fragment_init();
        moduledb_init();    /* before vm_areas_init, after heap_init */
        perscache_init();   /* before vm_areas_init */
        native_exec_init(); /* before vm_areas_init, after arch_init */

        if (!DYNAMO_OPTION(thin_client)) {
#ifdef HOT_PATCHING_INTERFACE
            /* must init hotp before vm_areas_init() calls find_executable_vm_areas() */
            if (DYNAMO_OPTION(hot_patching))
                hotp_init();
#endif
        }

#ifdef INTERNAL
        {
            char initial_options[MAX_OPTIONS_STRING];
            get_dynamo_options_string(&dynamo_options, initial_options,
                                      sizeof(initial_options), true);
            SYSLOG_INTERNAL_INFO("Initial options = %s", initial_options);
            DOLOG(1, LOG_TOP, {
                get_pcache_dynamo_options_string(&dynamo_options, initial_options,
                                                 sizeof(initial_options),
                                                 OP_PCACHE_LOCAL);
                LOG(GLOBAL, LOG_TOP, 1, "Initial pcache-affecting options = %s\n",
                    initial_options);
            });
        }
#endif /* INTERNAL */

        LOG(GLOBAL, LOG_TOP, 1, "\n");

        /* initialize thread hashtable */
        /* Note: for thin_client, this isn't needed if it is only going to
         * look for spawned processes; however, if we plan to promote from
         * thin_client to hotp_only mode (highly likely), this would be needed.
         * For now, leave it in there unless thin_client footprint becomes an
         * issue.
         */
        size = HASHTABLE_SIZE(ALL_THREADS_HASH_BITS) * sizeof(thread_record_t *);
        all_threads =
            (thread_record_t **)global_heap_alloc(size HEAPACCT(ACCT_THREAD_MGT));
        memset(all_threads, 0, size);
        if (!INTERNAL_OPTION(nop_initial_bblock) IF_WINDOWS(
                || !check_sole_thread())) /* some other thread is already here! */
            bb_lock_start = true;

#ifdef SIDELINE
        /* initialize sideline thread after thread table is set up */
        if (dynamo_options.sideline)
            sideline_init();
#endif

        /* thread-specific initialization for the first thread we inject in
         * (in a race with injected threads, sometimes it is not the primary thread)
         */
        /* i#117/PR 395156: it'd be nice to have mc here but would
         * require changing start/stop API
         */
        dynamo_thread_init(NULL, NULL, NULL _IF_CLIENT_INTERFACE(false));
#ifndef WINDOWS
        /* i#2751: we need TLS to be set up to relocate and call init funcs. */
        loader_init();
#endif

        /* We move vm_areas_init() below dynamo_thread_init() so we can have
         * two things: 1) a dcontext and 2) a SIGSEGV handler, for TRY/EXCEPT
         * inside vm_areas_init() for PR 361594's probes and for d_r_safe_read().
         * This means vm_areas_thread_init() runs before vm_areas_init().
         */
        if (!DYNAMO_OPTION(thin_client)) {
            vm_areas_init();
#ifdef RCT_IND_BRANCH
            /* relies on is_in_dynamo_dll() which needs vm_areas_init */
            rct_init();
#endif
        } else {
            /* This is needed to handle exceptions in thin_client mode, mostly
             * internal ones, but can be app ones too. */
            dynamo_vm_areas_lock();
            find_dynamo_library_vm_areas();
            dynamo_vm_areas_unlock();
        }

#ifdef ANNOTATIONS
        annotation_init();
#endif
        jitopt_init();

        dr_attach_finished = create_broadcast_event();

        /* New client threads rely on dr_app_started being initialized, so do
         * that before initializing clients.
         */
        dr_app_started = create_broadcast_event();
#ifdef CLIENT_INTERFACE
        /* client last, in case it depends on other inits: must be after
         * dynamo_thread_init so the client can use a dcontext (PR 216936).
         * Note that we *load* the client library before installing our hooks,
         * but call the client's init routine afterward so that we correctly
         * report crashes (PR 200207).
         * Note: DllMain in client libraries can crash and we still won't
         *       report; better document that client libraries shouldn't have
         *       DllMain.
         */
        instrument_init();
        /* To give clients a chance to process pcaches as we load them, we
         * delay the loading until we've initialized the clients.
         */
        vm_area_delay_load_coarse_units();
#endif

#ifdef WINDOWS
        if (!INTERNAL_OPTION(noasynch))
            callback_interception_init_finish(); /* split for PR 200207: see above */
#endif

        if (SELF_PROTECT_ON_CXT_SWITCH) {
            protect_info = (protect_info_t *)global_unprotected_heap_alloc(
                sizeof(protect_info_t) HEAPACCT(ACCT_OTHER));
            ASSIGN_INIT_LOCK_FREE(protect_info->lock, protect_info);
            protect_info->num_threads_unprot = 0; /* ENTERING_DR() below will inc to 1 */
            protect_info->num_threads_suspended = 0;
            if (INTERNAL_OPTION(single_privileged_thread)) {
                /* FIXME: thread_initexit_lock must be a recursive lock! */
                ASSERT_NOT_IMPLEMENTED(false);
                /* grab the lock now -- the thread that is in dynamo must be holding
                 * the lock, and we are the initial thread in dynamo!
                 */
                d_r_mutex_lock(&thread_initexit_lock);
            }
            /* ENTERING_DR will increment, so decrement first
             * FIXME: waste of protection change since will nop-unprotect!
             */
            if (TEST(SELFPROT_DATA_CXTSW, DYNAMO_OPTION(protect_mask)))
                datasec_writable_cxtswprot = 0;
            /* FIXME case 8073: remove once freqprot not every cxt sw */
            if (TEST(SELFPROT_DATA_FREQ, DYNAMO_OPTION(protect_mask)))
                datasec_writable_freqprot = 0;
        }
        /* this thread is now entering DR */
        ENTERING_DR();

#ifdef WINDOWS
        if (DYNAMO_OPTION(early_inject)) {
            /* AFTER callback_interception_init and self protect init and
             * ENTERING_DR() */
            early_inject_init();
        }
#endif
    }

    dynamo_initialized = true;

    /* Protect .data, assuming all vars there have been initialized. */
    SELF_PROTECT_DATASEC(DATASEC_RARELY_PROT);

    /* internal-only options for testing run-once (case 3990) */
    if (INTERNAL_OPTION(unsafe_crash_process)) {
        SYSLOG_INTERNAL_ERROR("Crashing the process deliberately!");
        *((int *)PTR_UINT_MINUS_1) = 0;
    }
    if (INTERNAL_OPTION(unsafe_hang_process)) {
        event_t never_signaled = create_event();
        SYSLOG_INTERNAL_ERROR("Hanging the process deliberately!");
        wait_for_event(never_signaled, 0);
        destroy_event(never_signaled);
    }

    return SUCCESS;
}

#ifdef UNIX
void
dynamorio_fork_init(dcontext_t *dcontext)
{
    /* on a fork we want to re-initialize some data structures, especially
     * log files, which we want a separate directory for
     */
    thread_record_t **threads;
    int i, num_threads;
#    ifdef DEBUG
    char parent_logdir[MAXIMUM_PATH];
#    endif

    /* re-cache app name, etc. that are using parent pid before we
     * create log dirs (xref i#189/PR 452168)
     */
    os_fork_init(dcontext);

    /* sanity check, plus need to set this for statistics_init:
     * even if parent did an execve, env var should be reset by now
     */
    post_execve = (getenv(DYNAMORIO_VAR_EXECVE) != NULL);
    ASSERT(!post_execve);

#    ifdef DEBUG
    /* copy d_r_stats->logdir
     * d_r_stats->logdir is static, so current copy is fine, don't need
     * frozen copy
     */
    strncpy(parent_logdir, d_r_stats->logdir, MAXIMUM_PATH - 1);
    d_r_stats->logdir[MAXIMUM_PATH - 1] = '\0'; /* if max no null */
#    endif

    if (get_log_dir(PROCESS_DIR, NULL, NULL)) {
        /* we want brand new log dir  */
        enable_new_log_dir();
        create_log_dir(PROCESS_DIR);
    }

#    ifdef DEBUG
    /* just like dynamorio_app_init, create main_logfile before stats */
    if (d_r_stats->loglevel > 0) {
        /* we want brand new log files.  os_fork_init() closed inherited files. */
        main_logfile = open_log_file(main_logfile_name(), NULL, 0);
        print_file(main_logfile, "%s\n", dynamorio_version_string);
        print_file(main_logfile, "New log file for child %d forked by parent %d\n",
                   d_r_get_thread_id(), get_parent_id());
        print_file(main_logfile, "Parent's log dir: %s\n", parent_logdir);
    }

    d_r_stats->process_id = get_process_id();

    if (d_r_stats->loglevel > 0) {
        /* FIXME: share these few lines of code w/ dynamorio_app_init? */
        LOG(GLOBAL, LOG_TOP, 1, "Running: %s\n", d_r_stats->process_name);
#        ifndef _WIN32_WCE
        LOG(GLOBAL, LOG_TOP, 1, "DYNAMORIO_OPTIONS: %s\n", option_string);
#        endif
    }
#    endif /* DEBUG */

    /* must re-hash parent entry in threads table, plus no longer have any
     * other threads (fork -> we're alone in address space), so clear
     * out entire thread table, then add child
     */
    d_r_mutex_lock(&thread_initexit_lock);
    get_list_of_threads_ex(&threads, &num_threads, true /*include execve*/);
    for (i = 0; i < num_threads; i++) {
        if (threads[i] == dcontext->thread_record)
            remove_thread(threads[i]->id);
        else
            dynamo_other_thread_exit(threads[i]);
    }
    d_r_mutex_unlock(&thread_initexit_lock);
    global_heap_free(threads,
                     num_threads * sizeof(thread_record_t *) HEAPACCT(ACCT_THREAD_MGT));

    add_thread(get_process_id(), d_r_get_thread_id(), true /*under dynamo control*/,
               dcontext);

    GLOBAL_STAT(num_threads) = 1;
#    ifdef DEBUG
    if (d_r_stats->loglevel > 0) {
        /* need a new thread-local logfile */
        dcontext->logfile = open_log_file(thread_logfile_name(), NULL, 0);
        print_file(dcontext->logfile, "%s\n", dynamorio_version_string);
        print_file(dcontext->logfile, "New log file for child %d forked by parent %d\n",
                   d_r_get_thread_id(), get_parent_id());
        LOG(THREAD, LOG_TOP | LOG_THREADS, 1, "THREAD %d (dcontext " PFX ")\n\n",
            d_r_get_thread_id(), dcontext);
    }
#    endif
    num_threads = 1;

    /* FIXME: maybe should have a callback list for who wants to be notified
     * on a fork -- probably everyone who makes a log file on init.
     */
    fragment_fork_init(dcontext);
    /* this must be called after dynamo_other_thread_exit() above */
    signal_fork_init(dcontext);

#    ifdef CLIENT_INTERFACE
    if (CLIENTS_EXIST()) {
        instrument_fork_init(dcontext);
    }
#    endif
}
#endif /* UNIX */

#if defined(CLIENT_INTERFACE) || defined(STANDALONE_UNIT_TEST)
/* To make DynamoRIO useful as a library for a standalone client
 * application (as opposed to a client library that works with
 * DynamoRIO in executing a target application).  This makes DynamoRIO
 * useful as an IA-32 disassembly library, etc.
 */
dcontext_t *
standalone_init(void)
{
    dcontext_t *dcontext;
    if (dynamo_initialized)
        return GLOBAL_DCONTEXT;
    standalone_library = true;
    /* We have release-build stats now so this is not just DEBUG */
    d_r_stats = &nonshared_stats;
    /* No reason to limit heap size when there's no code cache. */
    IF_X64(dynamo_options.reachable_heap = false;)
    dynamo_options.vm_base_near_app = false;
#    if defined(INTERNAL) && defined(DEADLOCK_AVOIDANCE)
    /* avoid issues w/ GLOBAL_DCONTEXT instead of thread dcontext */
    dynamo_options.deadlock_avoidance = false;
#    endif
#    ifdef UNIX
    os_page_size_init((const char **)our_environ, is_our_environ_followed_by_auxv());
#    endif
#    ifdef WINDOWS
    /* MUST do this before making any system calls */
    if (!syscalls_init())
        return NULL; /* typically b/c of unsupported OS version */
#    endif
    d_r_config_init();
    options_init();
    vmm_heap_init();
    d_r_heap_init();
    dynamo_heap_initialized = true;
    dynamo_vm_areas_init();
    d_r_decode_init();
    proc_init();
    d_r_os_init();
    config_heap_init();

#    ifdef STANDALONE_UNIT_TEST
    os_tls_init();
    dcontext = create_new_dynamo_context(true /*initial*/, NULL, NULL);
    set_thread_private_dcontext(dcontext);
    /* sanity check */
    ASSERT(get_thread_private_dcontext() == dcontext);

    heap_thread_init(dcontext);

#        ifdef DEBUG
    /* FIXME: share code w/ main init routine? */
    nonshared_stats.logmask = LOG_ALL;
    options_init();
    if (d_r_stats->loglevel > 0) {
        char initial_options[MAX_OPTIONS_STRING];
        main_logfile = open_log_file(main_logfile_name(), NULL, 0);
        print_file(main_logfile, "%s\n", dynamorio_version_string);
        print_file(main_logfile, "Log file for standalone unit test\n");
        get_dynamo_options_string(&dynamo_options, initial_options,
                                  sizeof(initial_options), true);
        SYSLOG_INTERNAL_INFO("Initial options = %s", initial_options);
        print_file(main_logfile, "\n");
    }
#        endif /* DEBUG */
#    else
    /* rather than ask the user to call some thread-init routine in
     * every thread, we just use global dcontext everywhere (i#548)
     */
    dcontext = GLOBAL_DCONTEXT;
#    endif

    /* since we do not export any dr_standalone_exit(), we clean up any .1config
     * file right now.  the only loss if that we can't synch options: but that
     * should be less important for standalone.  we disabling synching.
     */
    /* options are never made read-only for standalone */
    dynamo_options.dynamic_options = false;

    dynamo_initialized = true;

    return dcontext;
}

void
standalone_exit(void)
{
    /* should clean up here */
    d_r_config_exit();
}
#endif

/* Perform exit tasks that require full thread data structs, which we have
 * already cleaned up by the time we reach dynamo_shared_exit() for both
 * debug and detach paths.
 */
void
dynamo_process_exit_with_thread_info(void)
{
    perscache_fast_exit(); /* "fast" b/c called in release as well */
}

/* shared between app_exit and detach */
int
dynamo_shared_exit(thread_record_t *toexit /* must ==cur thread for Linux */
                       _IF_WINDOWS(bool detach_stacked_callbacks))
{
    DEBUG_DECLARE(uint endtime);
    /* set this now, could already be set */
    dynamo_exited = true;

    /* avoid time() for libc independence */
    DODEBUG(endtime = query_time_seconds(););
    LOG(GLOBAL, LOG_STATS, 1, "\n#### Statistics for entire process:\n");
    LOG(GLOBAL, LOG_STATS, 1, "Total running time: %d seconds\n", endtime - starttime);

#ifdef PAPI
    hardware_perfctr_exit();
#endif
#ifdef DEBUG
#    if defined(INTERNAL) && defined(X86)
    print_optimization_stats();
#    endif /* INTERNAL && X86 */
    DOLOG(1, LOG_STATS, { dump_global_stats(false); });
#endif /* DEBUG */

    if (SELF_PROTECT_ON_CXT_SWITCH) {
        DELETE_LOCK(protect_info->lock);
        global_unprotected_heap_free(protect_info,
                                     sizeof(protect_info_t) HEAPACCT(ACCT_OTHER));
    }

    /* call all component exit routines (CAUTION: order is important here) */

    DELETE_RECURSIVE_LOCK(thread_in_DR_exclusion);
    DOSTATS({
        LOG(GLOBAL, LOG_TOP | LOG_THREADS, 1,
            "fcache_stats_exit: before fragment cleanup\n");
        DOLOG(1, LOG_CACHE, fcache_stats_exit(););
    });
#ifdef RCT_IND_BRANCH
    if (!DYNAMO_OPTION(thin_client))
        rct_exit();
#endif
    fragment_exit();
#ifdef ANNOTATIONS
    annotation_exit();
#endif
    jitopt_exit();
#ifdef CLIENT_INTERFACE
    /* We tell the client as soon as possible in case it wants to use services from other
     * components.  Must be after fragment_exit() so that the client gets all the
     * fragment_deleted() callbacks (xref PR 228156). FIXME - might be issues with the
     * client trying to use api routines that depend on fragment state.
     */
    instrument_exit();
#    ifdef CLIENT_SIDELINE
    /* We only need do a second synch-all if there are sideline client threads. */
    if (d_r_get_num_threads() > 1)
        synch_with_threads_at_exit(exit_synch_state(), false /*post-exit*/);
    /* only current thread is alive */
    dynamo_exited_all_other_threads = true;
#    endif /* CLIENT_SIDELINE */
    /* Some lock can only be deleted if only one thread left. */
    instrument_exit_post_sideline();
#endif /* CLIENT_INTERFACE */
    fragment_exit_post_sideline();

    /* The dynamo_exited_and_cleaned should be set after the second synch-all.
     * If it is set earlier after the first synch-all, some client thread may
     * have memory leak due to dynamo_thread_exit_pre_client being skipped in
     * dynamo_thread_exit_common called from exiting client threads.
     */
    dynamo_exited_and_cleaned = true;

    destroy_event(dr_app_started);
    destroy_event(dr_attach_finished);

    /* we want dcontext around for loader_exit() */
    if (get_thread_private_dcontext() != NULL)
        loader_thread_exit(get_thread_private_dcontext());
    loader_exit();

    if (toexit != NULL) {
        /* free detaching thread's dcontext */
#ifdef WINDOWS
        /* If we use dynamo_thread_exit() when toexit is the current thread,
         * it results in asserts in the win32.tls test, so we stick with this.
         */
        d_r_mutex_lock(&thread_initexit_lock);
        dynamo_other_thread_exit(toexit, false);
        d_r_mutex_unlock(&thread_initexit_lock);
#else
        /* On Linux, restoring segment registers can only be done
         * on the current thread, which must be toexit.
         */
        ASSERT(toexit->id == d_r_get_thread_id());
        dynamo_thread_exit();
#endif
    }

    if (IF_WINDOWS_ELSE(!detach_stacked_callbacks, true)) {
        /* We don't fully free cur thread until after client exit event (PR 536058) */
        if (thread_lookup(d_r_get_thread_id()) == NULL) {
            LOG(GLOBAL, LOG_TOP | LOG_THREADS, 1,
                "Current thread never under DynamoRIO control, not exiting it\n");
        } else {
            /* call thread_exit even if !under_dynamo_control, could have
             * been at one time
             */
            /* exit this thread now */
            dynamo_thread_exit();
        }
    }
    /* now that the final thread is exited, free the all_threads memory */
    d_r_mutex_lock(&all_threads_lock);
    global_heap_free(all_threads,
                     HASHTABLE_SIZE(ALL_THREADS_HASH_BITS) *
                         sizeof(thread_record_t *) HEAPACCT(ACCT_THREAD_MGT));
    all_threads = NULL;
    d_r_mutex_unlock(&all_threads_lock);

#ifdef WINDOWS
#    ifdef CLIENT_INTERFACE
    /* for -private_loader we do this here to catch more exit-time crashes */
    if (!INTERNAL_OPTION(noasynch) && INTERNAL_OPTION(private_loader) && !doing_detach)
        callback_interception_unintercept();
#    endif
    /* callback_interception_exit must be after fragment exit for CLIENT_INTERFACE so
     * that fragment_exit->frees fragments->instrument_fragment_deleted->
     * hide_tag_from_fragment->is_intercepted_app_pc won't crash. xref PR 228156 */
    if (!INTERNAL_OPTION(noasynch)) {
        callback_interception_exit();
    }
#endif
    d_r_link_exit();
    fcache_exit();
    d_r_monitor_exit();
    synch_exit();
    d_r_arch_exit(IF_WINDOWS(detach_stacked_callbacks));
#ifdef CALL_PROFILE
    /* above os_exit to avoid eventlog_mutex trigger if we're the first to
     * create a log file
     */
    profile_callers_exit();
#endif
    os_fast_exit();
    os_slow_exit();
    native_exec_exit(); /* before vm_areas_exit for using dynamo_areas */
    vm_areas_exit();
    perscache_slow_exit(); /* fast called in dynamo_process_exit_with_thread_info() */
    modules_exit();        /* after aslr_exit() from os_slow_exit(),
                            * after vm_areas & perscache exits */
    moduledb_exit();       /* before heap_exit */
#ifdef HOT_PATCHING_INTERFACE
    if (DYNAMO_OPTION(hot_patching))
        hotp_exit();
#endif
#ifdef WINDOWS
    /* Free exception stack before calling heap_exit */
    stack_free(exception_stack, EXCEPTION_STACK_SIZE);
    exception_stack = NULL;
#endif
    config_heap_exit();
    d_r_heap_exit();
    vmm_heap_exit();
    diagnost_exit();
    data_section_exit();
    /* funny dependences: options exit just frees lock, not destroying
     * any options that are needed for other exits, so do it prior to
     * checking locks in debug build
     */
    options_exit();
    utils_exit();
    d_r_config_exit();

#ifdef KSTATS
    kstat_exit();
#endif

    DELETE_LOCK(all_threads_lock);
    DELETE_LOCK(thread_initexit_lock);

    DOLOG(1, LOG_STATS, {
        /* dump after cleaning up to make it easy to check if stats that
         * are inc-ed and dec-ed actually come down to 0
         */
        dump_global_stats(false);
    });
    if (INTERNAL_OPTION(rstats_to_stderr))
        dump_global_rstats_to_stderr();

    statistics_exit();
#ifdef DEBUG
#    ifdef DEADLOCK_AVOIDANCE
    ASSERT(locks_not_closed() == 0);
#    endif
    dynamo_exited_log_and_stats = true;
    if (main_logfile != STDERR) {
        /* do it this way just in case someone tries to log to the global file
         * right now */
        file_t file_temp = main_logfile;
        main_logfile = INVALID_FILE;
        close_log_file(file_temp);
    }
#else
#    ifdef DEADLOCK_AVOIDANCE
    ASSERT(locks_not_closed() == 0);
#    endif
#endif /* DEBUG */

    dynamo_initialized = false;
    dynamo_started = false;
    return SUCCESS;
}

int
dynamorio_app_exit(void)
{
    return dynamo_process_exit();
}

/* synchs with all threads using synch type synch_res.
 * also sets dynamo_exited to true.
 * does not resume the threads but does release the thread_initexit_lock.
 */
static void
synch_with_threads_at_exit(thread_synch_state_t synch_res, bool pre_exit)
{
    int num_threads;
    thread_record_t **threads;
    DEBUG_DECLARE(bool ok;)
    /* If we fail to suspend a thread (e.g., privilege
     * problems) ignore it. XXX: retry instead?
     */
    uint flags = THREAD_SYNCH_SUSPEND_FAILURE_IGNORE;
    if (pre_exit) {
        /* i#297: we only synch client threads after process exit event. */
        flags |= THREAD_SYNCH_SKIP_CLIENT_THREAD;
    }
    LOG(GLOBAL, LOG_TOP | LOG_THREADS, 1,
        "\nsynch_with_threads_at_exit: cleaning up %d un-terminated threads\n",
        d_r_get_num_threads());

#if defined(CLIENT_INTERFACE) && defined(WINDOWS)
    /* make sure client nudges are finished */
    wait_for_outstanding_nudges();
#endif

    /* xref case 8747, requesting suspended is preferable to terminated and it
     * doesn't make a difference here which we use (since the process is about
     * to die).
     * On Linux, however, we do not have dependencies on OS thread
     * properties like we do on Windows (TEB, etc.), and our suspended
     * threads use their sigstacks and ostd data structs, making cleanup
     * while still catching other leaks more difficult: thus it's
     * simpler to terminate and then clean up.  FIXME: by terminating
     * we'll raise SIGCHLD that may not have been raised natively if the
     * whole group went down in a single SYS_exit_group.  Instead we
     * could have the suspended thread move from the sigstack-reliant
     * loop to a stack-free loop (xref i#95).
     */
    IF_UNIX(dynamo_exiting = true;) /* include execve-exited vfork threads */
    DEBUG_DECLARE(ok =)
    synch_with_all_threads(synch_res, &threads, &num_threads,
                           /* Case 6821: other synch-all-thread uses that
                            * only care about threads carrying fcache
                            * state can ignore us
                            */
                           THREAD_SYNCH_NO_LOCKS_NO_XFER, flags);
    ASSERT(ok);
    ASSERT(threads == NULL && num_threads == 0); /* We asked for CLEANED */
    /* the synch_with_all_threads function grabbed the
     * thread_initexit_lock for us! */
    /* do this now after all threads we know about are killed and
     * while we hold the thread_initexit_lock so any new threads that
     * are waiting on it won't get in our way (see thread_init()) */
    dynamo_exited = true;
    end_synch_with_all_threads(threads, num_threads, false /*don't resume*/);
}

static thread_synch_state_t
exit_synch_state(void)
{
    thread_synch_state_t synch_res = IF_WINDOWS_ELSE(THREAD_SYNCH_SUSPENDED_AND_CLEANED,
                                                     THREAD_SYNCH_TERMINATED_AND_CLEANED);
#if defined(DR_APP_EXPORTS) && defined(UNIX)
    if (dr_api_exit) {
        /* Don't terminate the app's threads in case the app plans to continue
         * after dr_app_cleanup().  Note that today we don't fully support that
         * anyway: the app should use dr_app_stop_and_cleanup() whose detach
         * code won't come here.
         */
        synch_res = THREAD_SYNCH_SUSPENDED_AND_CLEANED;
    }
#endif
    return synch_res;
}

#ifdef DEBUG
/* cleanup after the application has exited */
static int
dynamo_process_exit_cleanup(void)
{
    /* CAUTION: this should only be invoked after all app threads have stopped */
    if (!dynamo_exited && !INTERNAL_OPTION(nullcalls)) {
        dcontext_t *dcontext;

        APP_EXPORT_ASSERT(dynamo_initialized, "Improper DynamoRIO initialization");

        dcontext = get_thread_private_dcontext();

        /* we deliberately do NOT clean up d_r_initstack (which was
         * allocated using a separate mmap and so is not part of some
         * large unit that is de-allocated), as it is used in special
         * circumstances to call us...FIXME: is this memory leak ok?
         * is there a better solution besides assuming the app stack?
         */

#    ifdef SIDELINE
        if (dynamo_options.sideline) {
            /* exit now to make thread cleanup simpler */
            sideline_exit();
        }
#    endif

        /* perform exit tasks that require full thread data structs */
        dynamo_process_exit_with_thread_info();

        if (INTERNAL_OPTION(single_privileged_thread)) {
            d_r_mutex_unlock(&thread_initexit_lock);
        }

        /* if ExitProcess called before all threads terminated, they won't
         * all have gone through dynamo_thread_exit, so clean them up now
         * so we can get stats about them
         *
         * we don't check control_all_threads b/c we're just killing
         * the threads we know about here
         */
        synch_with_threads_at_exit(exit_synch_state(), true /*pre-exit*/);
#    ifndef CLIENT_SIDELINE
        /* no sideline thread, synchall done */
        dynamo_exited_all_other_threads = true;
#    endif
        /* now that APC interception point is unpatched and
         * dynamorio_exited is set and we've killed all the theads we know
         * about, assumption is that no other threads will be running in
         * dynamorio code from here on out (esp. when we get into shared exit)
         * that will do anything that could be dangerous (could possibly be
         * a thread in the APC interception code prior to reaching thread_init
         * but it will only global log and do thread_lookup which should be
         * safe throughout) */

        /* In order to pass the client a dcontext in the process exit event
         * we do some thread cleanup early for the final thread so we can delay
         * the rest (PR 536058).  This is a little risky in that we
         * clean up dcontext->fragment_field, which is used for lots of
         * things like couldbelinking (and thus we have to disable some API
         * routines in the thread exit event: i#1989).
         */
        dynamo_thread_exit_pre_client(get_thread_private_dcontext(), d_r_get_thread_id());

#    ifdef WINDOWS
        /* FIXME : our call un-interception isn't atomic so (miniscule) chance
         * of something going wrong if new thread is just hitting its init APC
         */
        /* w/ the app's loader we must remove our LdrUnloadDll hook
         * before we unload the client lib (and thus we miss client
         * exit crashes): xref PR 200207.
         */
        if (!INTERNAL_OPTION(noasynch)
                IF_CLIENT_INTERFACE(&&!INTERNAL_OPTION(private_loader))) {
            callback_interception_unintercept();
        }
#    else  /* UNIX */
        unhook_vsyscall();
#    endif /* UNIX */

        return dynamo_shared_exit(NULL /* not detaching */
                                      _IF_WINDOWS(false /* not detaching */));
    }
    return SUCCESS;
}
#endif /* DEBUG */

int
dynamo_nullcalls_exit(void)
{
    /* this routine is used when nullcalls is turned on
     * simply to get perfctr numbers in a log file
     */
    ASSERT(INTERNAL_OPTION(nullcalls));
#ifdef PAPI
    hardware_perfctr_exit();
#endif

#ifdef DEBUG
    if (main_logfile != STDERR) {
        close_log_file(main_logfile);
        main_logfile = INVALID_FILE;
    }
#endif /* DEBUG */

    dynamo_exited = true;
    return SUCCESS;
}

/* called when we see that the process is about to exit */
int
dynamo_process_exit(void)
{
#ifndef DEBUG
    bool each_thread;
#endif
    SELF_UNPROTECT_DATASEC(DATASEC_RARELY_PROT);
    synchronize_dynamic_options();
    SYSLOG(SYSLOG_INFORMATION, INFO_PROCESS_STOP, 2, get_application_name(),
           get_application_pid());
#ifdef DEBUG
    if (!dynamo_exited) {
        if (INTERNAL_OPTION(nullcalls)) {
            /* if nullcalls is on we still do perfctr stats, and this is
             * the only place we can print them out and exit
             */
            dynamo_nullcalls_exit();
        } else {
            /* we don't check automatic_startup -- even if the app_
             * interface is used, we are about to be gone from the process
             * address space, so we clean up now
             */
            LOG(GLOBAL, LOG_TOP, 1,
                "\ndynamo_process_exit from thread " TIDFMT " -- cleaning up dynamo\n",
                d_r_get_thread_id());
            dynamo_process_exit_cleanup();
        }
    }

    return SUCCESS;

#else
    if (dynamo_exited)
        return SUCCESS;

    /* don't need to do much!
     * we didn't create any IPC objects or anything that might be persistent
     * beyond our death, we're not holding any systemwide locks, etc.
     */

    /* It is not clear whether the Event Log service handles unterminated connections */

    /* Do we need profile data for each thread?
     * Note that windows prof_pcs duplicates the thread walk in d_r_os_exit()
     * FIXME: should combine that thread walk with this one
     */
    each_thread = TRACEDUMP_ENABLED();
#    ifdef UNIX
    each_thread = each_thread || INTERNAL_OPTION(profile_pcs);
#    endif
#    ifdef KSTATS
    each_thread = each_thread || DYNAMO_OPTION(kstats);
#    endif
#    ifdef CLIENT_INTERFACE
    each_thread = each_thread ||
        /* If we don't need a thread exit event, avoid the possibility of
         * racy crashes (PR 470957) by not calling instrument_thread_exit()
         */
        (!INTERNAL_OPTION(nullcalls) && dr_thread_exit_hook_exists() &&
         !DYNAMO_OPTION(skip_thread_exit_at_exit));
#    endif

    if (DYNAMO_OPTION(synch_at_exit)
        /* by default we synch if any exit event exists */
        IF_CLIENT_INTERFACE(
            || (!DYNAMO_OPTION(multi_thread_exit) && dr_exit_hook_exists()) ||
            (!DYNAMO_OPTION(skip_thread_exit_at_exit) && dr_thread_exit_hook_exists()))) {
        /* needed primarily for CLIENT_INTERFACE but technically all configurations
         * can have racy crashes at exit time (xref PR 470957)
         */
        synch_with_threads_at_exit(exit_synch_state(), true /*pre-exit*/);
#    ifndef CLIENT_SIDELINE
        dynamo_exited_all_other_threads = true;
#    endif
    } else
        dynamo_exited = true;

    if (each_thread) {
        thread_record_t **threads;
        int num, i;
        d_r_mutex_lock(&thread_initexit_lock);
        get_list_of_threads(&threads, &num);

        for (i = 0; i < num; i++) {
#    ifdef CLIENT_SIDELINE
            if (IS_CLIENT_THREAD(threads[i]->dcontext))
                continue;
#    endif
            /* FIXME: separate trace dump from rest of fragment cleanup code */
            if (TRACEDUMP_ENABLED() IF_CLIENT_INTERFACE(|| true)) {
                /* We always want to call this for CI builds so we can get the
                 * dr_fragment_deleted() callbacks.
                 */
                fragment_thread_exit(threads[i]->dcontext);
            }
#    ifdef UNIX
            if (INTERNAL_OPTION(profile_pcs))
                pcprofile_thread_exit(threads[i]->dcontext);
#    endif
#    ifdef KSTATS
            if (DYNAMO_OPTION(kstats))
                kstat_thread_exit(threads[i]->dcontext);
#    endif
#    ifdef CLIENT_INTERFACE
            /* Inform client of all thread exits */
            if (!INTERNAL_OPTION(nullcalls) && !DYNAMO_OPTION(skip_thread_exit_at_exit)) {
                instrument_thread_exit_event(threads[i]->dcontext);
                /* i#1617: ensure we do all cleanup of priv libs */
                if (threads[i]->id != d_r_get_thread_id()) /* i#1617: must delay this */
                    loader_thread_exit(threads[i]->dcontext);
            }
#    endif
        }
        global_heap_free(threads,
                         num * sizeof(thread_record_t *) HEAPACCT(ACCT_THREAD_MGT));
        d_r_mutex_unlock(&thread_initexit_lock);
    }

    /* PR 522783: must be before we clear dcontext (if CLIENT_INTERFACE)! */
    /* must also be prior to fragment_exit so we actually freeze pcaches (i#703) */
    dynamo_process_exit_with_thread_info();

    /* FIXME: separate trace dump from rest of fragment cleanup code.  For client
     * interface we need to call fragment_exit to get all the fragment deleted events. */
    if (TRACEDUMP_ENABLED() IF_CLIENT_INTERFACE(|| dr_fragment_deleted_hook_exists()))
        fragment_exit();

        /* Inform client of process exit */
#    ifdef CLIENT_INTERFACE
    if (!INTERNAL_OPTION(nullcalls)) {
#        ifdef WINDOWS
        /* instrument_exit() unloads the client library, so make sure
         * LdrUnloadDll isn't hooked if using the app loader.
         */
        if (!INTERNAL_OPTION(noasynch)
                IF_CLIENT_INTERFACE(&&!INTERNAL_OPTION(private_loader))) {
            callback_interception_unintercept();
        }
#        endif
#        ifdef UNIX
        /* i#2976: unhook prior to client exit if modules are being watched */
        if (dr_modload_hook_exists())
            unhook_vsyscall();
#        endif
        /* Must be after fragment_exit() so that the client gets all the
         * fragment_deleted() callbacks (xref PR 228156).  FIXME - might be issues
         * with the client trying to use api routines that depend on fragment state.
         */
        instrument_exit();

#        ifdef CLIENT_SIDELINE
        /* We only need do a second synch-all if there are sideline client threads. */
        if (d_r_get_num_threads() > 1)
            synch_with_threads_at_exit(exit_synch_state(), false /*post-exit*/);
        dynamo_exited_all_other_threads = true;
#        endif
        /* Some lock can only be deleted if one thread left. */
        instrument_exit_post_sideline();

        /* i#1617: We need to call client library fini routines for global
         * destructors, etc.
         */
        if (!INTERNAL_OPTION(nullcalls) && !DYNAMO_OPTION(skip_thread_exit_at_exit))
            loader_thread_exit(get_thread_private_dcontext());
        loader_exit();

        /* for -private_loader we do this here to catch more exit-time crashes */
#        ifdef WINDOWS
        if (!INTERNAL_OPTION(noasynch)
                IF_CLIENT_INTERFACE(&&INTERNAL_OPTION(private_loader)))
            callback_interception_unintercept();
#        endif
    }
#    endif /* CLIENT_INTERFACE */
    fragment_exit_post_sideline();

#    ifdef CALL_PROFILE
    profile_callers_exit();
#    endif
#    ifdef KSTATS
    if (DYNAMO_OPTION(kstats))
        kstat_exit();
#    endif
    /* so make sure eventlog connection is terminated (if present)  */
    os_fast_exit();

    if (INTERNAL_OPTION(rstats_to_stderr))
        dump_global_rstats_to_stderr();

    return SUCCESS;
#endif /* !DEBUG */
}

void
dynamo_exit_post_detach(void)
{
    /* i#2157: best-effort re-init in case of re-attach */

    do_once_generation++; /* Increment the generation in case we re-attach */

    dynamo_initialized = false;
    dynamo_heap_initialized = false;
    automatic_startup = false;
    control_all_threads = false;
    dr_api_entry = false;
    dr_api_exit = false;
#ifdef UNIX
    dynamo_exiting = false;
#endif
    dynamo_exited = false;
    dynamo_exited_all_other_threads = false;
    dynamo_exited_and_cleaned = false;
#ifdef DEBUG
    dynamo_exited_log_and_stats = false;
#endif
    dynamo_resetting = false;
#ifdef UNIX
    post_execve = false;
#endif
    vm_areas_post_exit();
    heap_post_exit();
}

dcontext_t *
create_new_dynamo_context(bool initial, byte *dstack_in, priv_mcontext_t *mc)
{
    dcontext_t *dcontext;
    size_t alloc = sizeof(dcontext_t) + proc_get_cache_line_size();
    void *alloc_start =
        (void *)((TEST(SELFPROT_GLOBAL, dynamo_options.protect_mask) &&
                  !TEST(SELFPROT_DCONTEXT, dynamo_options.protect_mask))
                     ?
                     /* if protecting global but not dcontext, put whole thing in unprot
                        mem */
                     global_unprotected_heap_alloc(alloc HEAPACCT(ACCT_OTHER))
                     : global_heap_alloc(alloc HEAPACCT(ACCT_OTHER)));
    dcontext = (dcontext_t *)proc_bump_to_end_of_cache_line((ptr_uint_t)alloc_start);
    ASSERT(proc_is_cache_aligned(dcontext));
#ifdef X86
    /* 264138: ensure xmm/ymm slots are aligned so we can use vmovdqa */
    ASSERT(ALIGNED(get_mcontext(dcontext)->simd, ZMM_REG_SIZE));
    /* also ensure we don't have extra padding beyond x86.asm defines */
    ASSERT(sizeof(priv_mcontext_t) ==
           IF_X64_ELSE(18, 10) * sizeof(reg_t) + PRE_XMM_PADDING +
               MCXT_TOTAL_SIMD_SLOTS_SIZE);
#elif defined(ARM)
    /* FIXME i#1551: add arm alignment check if any */
#endif /* X86/ARM */

    /* Put here all one-time dcontext field initialization
     * Make sure to update create_callback_dcontext to shared
     * fields across callback dcontexts for the same thread.
     */
    /* must set to 0 so can tell if initialized for callbacks! */
    memset(dcontext, 0x0, sizeof(dcontext_t));
    dcontext->allocated_start = alloc_start;

    /* we share a single dstack across all callbacks */
    if (initial) {
        /* DrMi#1723: our dstack needs to be at a higher address than the app
         * stack.  If mc passed, use its xsp; else use cur xsp (initial thread
         * is on the app stack here: xref i#1105), for lower bound for dstack.
         */
        byte *app_xsp;
        if (mc == NULL)
            GET_STACK_PTR(app_xsp);
        else
            app_xsp = (byte *)mc->xsp;
        if (dstack_in == NULL) {
            dcontext->dstack = (byte *)stack_alloc(DYNAMORIO_STACK_SIZE, app_xsp);
        } else
            dcontext->dstack = dstack_in; /* xref i#149/PR 403015 */
#ifdef WINDOWS
        DOCHECK(1, {
            if (dcontext->dstack < app_xsp)
                SYSLOG_INTERNAL_WARNING_ONCE("dstack is below app xsp");
        });
#endif
    } else {
        /* dstack may be pre-allocated only at thread init, not at callback */
        ASSERT(dstack_in == NULL);
    }
    if (TEST(SELFPROT_DCONTEXT, dynamo_options.protect_mask)) {
        dcontext->upcontext.separate_upcontext = global_unprotected_heap_alloc(
            sizeof(unprotected_context_t) HEAPACCT(ACCT_OTHER));
        /* don't need to initialize upcontext */
        LOG(GLOBAL, LOG_TOP, 2, "new dcontext=" PFX ", dcontext->upcontext=" PFX "\n",
            dcontext, dcontext->upcontext.separate_upcontext);
        dcontext->upcontext_ptr = dcontext->upcontext.separate_upcontext;
    } else
        dcontext->upcontext_ptr = &(dcontext->upcontext.upcontext);
#ifdef HOT_PATCHING_INTERFACE
    /* Set the hot patch exception state to be empty/unused. */
    DODEBUG(memset(&dcontext->hotp_excpt_state, -1, sizeof(dr_jmp_buf_t)););
#endif
    ASSERT(dcontext->try_except.try_except_state == NULL);

    DODEBUG({ dcontext->logfile = INVALID_FILE; });
    dcontext->owning_thread = d_r_get_thread_id();
#ifdef UNIX
    dcontext->owning_process = get_process_id();
#endif
    /* thread_record is set in add_thread */
    /* all of the thread-private fcache and hashtable fields are shared
     * among all dcontext instances of a thread, so the caller must
     * set those fields
     */
    /* rest of dcontext initialization happens in initialize_dynamo_context(),
     * which is executed for each dr_app_start() and each
     * callback start
     */
    return dcontext;
}

static void
delete_dynamo_context(dcontext_t *dcontext, bool free_stack)
{
    if (free_stack) {
        ASSERT(dcontext->dstack != NULL);
        ASSERT(!is_currently_on_dstack(dcontext));
        LOG(GLOBAL, LOG_THREADS, 1, "Freeing DR stack " PFX "\n", dcontext->dstack);
        stack_free(dcontext->dstack, DYNAMORIO_STACK_SIZE);
    } /* else will be cleaned up by caller */

    ASSERT(dcontext->try_except.try_except_state == NULL);

    if (TEST(SELFPROT_DCONTEXT, dynamo_options.protect_mask)) {
        global_unprotected_heap_free(dcontext->upcontext.separate_upcontext,
                                     sizeof(unprotected_context_t) HEAPACCT(ACCT_OTHER));
    }
    if (TEST(SELFPROT_GLOBAL, dynamo_options.protect_mask) &&
        !TEST(SELFPROT_DCONTEXT, dynamo_options.protect_mask)) {
        /* if protecting global but not dcontext, we put whole thing in unprot mem */
        global_unprotected_heap_free(dcontext->allocated_start,
                                     sizeof(dcontext_t) +
                                         proc_get_cache_line_size() HEAPACCT(ACCT_OTHER));
    } else {
        global_heap_free(dcontext->allocated_start,
                         sizeof(dcontext_t) +
                             proc_get_cache_line_size() HEAPACCT(ACCT_OTHER));
    }
}

/* This routine is called not only at thread initialization,
 * but for every callback, etc. that gets a fresh execution
 * environment!
 */
void
initialize_dynamo_context(dcontext_t *dcontext)
{
    /* we can't just zero out the whole thing b/c we have persistent state
     * (fields kept across callbacks, like dstack, module-private fields, next &
     * prev, etc.)
     */
    memset(dcontext->upcontext_ptr, 0, sizeof(unprotected_context_t));
    dcontext->initialized = true;
    dcontext->whereami = DR_WHERE_APP;
    dcontext->next_tag = NULL;
    dcontext->native_exec_postsyscall = NULL;
    memset(dcontext->native_retstack, 0, sizeof(dcontext->native_retstack));
    dcontext->native_retstack_cur = 0;
    dcontext->isa_mode = DEFAULT_ISA_MODE;
#ifdef ARM
    dcontext->encode_state[0] = 0;
    dcontext->encode_state[1] = 0;
    dcontext->decode_state[0] = 0;
    dcontext->decode_state[1] = 0;
#endif
    dcontext->sys_num = 0;
#ifdef WINDOWS
#    ifdef CLIENT_INTERFACE
    dcontext->app_errno = 0;
#        ifdef DEBUG
    dcontext->is_client_thread_exiting = false;
#        endif
#    endif
    dcontext->sys_param_base = NULL;
    /* always initialize aslr_context */
    dcontext->aslr_context.sys_aslr_clobbered = 0;
    dcontext->aslr_context.randomized_section_handle = INVALID_HANDLE_VALUE;
    dcontext->aslr_context.original_image_section_handle = INVALID_HANDLE_VALUE;
    dcontext->aslr_context.original_section_base = ASLR_INVALID_SECTION_BASE;
#    ifdef DEBUG
    dcontext->aslr_context.last_app_section_handle = INVALID_HANDLE_VALUE;
#    endif
    /* note that aslr_context.last_child_padded is preserved across callbacks */
    dcontext->ignore_enterexit = false;
#else
    dcontext->sys_param0 = 0;
    dcontext->sys_param1 = 0;
    dcontext->sys_param2 = 0;
#endif

#ifdef UNIX
    dcontext->signals_pending = false;
#endif

    /* all thread-private fields are initialized in dynamo_thread_init
     * or in create_callback_dcontext because they must be initialized differently
     * in those two cases
     */

    set_last_exit(dcontext, (linkstub_t *)get_starting_linkstub());

#ifdef PROFILE_RDTSC
    dcontext->start_time = (uint64)0;
    dcontext->prev_fragment = NULL;
    dcontext->cache_frag_count = (uint64)0;
    {
        int i;
        for (i = 0; i < 10; i++) {
            dcontext->cache_time[i] = (uint64)0;
            dcontext->cache_count[i] = (uint64)0;
        }
    }
#endif
#ifdef DEBUG
    dcontext->in_opnd_disassemble = false;
#endif
#ifdef WINDOWS
    /* Other pieces of DR -- callback & APC handling, detach -- test
     * asynch_target to determine where the next app pc to execute is
     * stored. Init it to 0 to indicate that this context's most recent
     * syscall was not executed from handle_system_call().
     */
    dcontext->asynch_target = NULL;
    /* next_saved and prev_unused are zeroed out when dcontext is
     * created; we shouldn't zero them here, they may have valid data
     */
    dcontext->valid = true;
#endif
#ifdef HOT_PATCHING_INTERFACE
    dcontext->nudge_thread = false; /* Fix for case 5367. */
#endif
#ifdef CHECK_RETURNS_SSE2
    /* initialize sse2 index with 0
     * go ahead and use eax, it's dead (about to return)
     */
#    ifdef UNIX
    asm("movl $0, %eax");
    asm("pinsrw $7,%eax,%xmm7");
#    else
#        error NYI
#    endif
#endif
    /* We don't need to initialize dcontext->coarse_exit as it is only
     * read when last_exit indicates a coarse exit, which sets the fields.
     */
    dcontext->go_native = false;
}

#ifdef WINDOWS
/* on windows we use a new dcontext for each callback context */
dcontext_t *
create_callback_dcontext(dcontext_t *old_dcontext)
{
    dcontext_t *new_dcontext = create_new_dynamo_context(false, NULL, NULL);
    new_dcontext->valid = false;
    /* all of these fields are shared among all dcontexts of a thread: */
    new_dcontext->owning_thread = old_dcontext->owning_thread;
#    ifdef UNIX
    new_dcontext->owning_process = old_dcontext->owning_process;
#    endif
    new_dcontext->thread_record = old_dcontext->thread_record;
    /* now that we have clean stack usage we can share a single stack */
    ASSERT(old_dcontext->dstack != NULL);
    new_dcontext->dstack = old_dcontext->dstack;
    new_dcontext->isa_mode = old_dcontext->isa_mode;
    new_dcontext->link_field = old_dcontext->link_field;
    new_dcontext->monitor_field = old_dcontext->monitor_field;
    new_dcontext->fcache_field = old_dcontext->fcache_field;
    new_dcontext->fragment_field = old_dcontext->fragment_field;
    new_dcontext->heap_field = old_dcontext->heap_field;
    new_dcontext->vm_areas_field = old_dcontext->vm_areas_field;
    new_dcontext->os_field = old_dcontext->os_field;
    new_dcontext->synch_field = old_dcontext->synch_field;
    /* case 8958: copy win32_start_addr in case we produce a forensics file
     * from within a callback.
     */
    new_dcontext->win32_start_addr = old_dcontext->win32_start_addr;
#    ifdef CLIENT_INTERFACE
    /* FlsData is persistent across callbacks */
    new_dcontext->app_fls_data = old_dcontext->app_fls_data;
    new_dcontext->priv_fls_data = old_dcontext->priv_fls_data;
    new_dcontext->app_nt_rpc = old_dcontext->app_nt_rpc;
    new_dcontext->priv_nt_rpc = old_dcontext->priv_nt_rpc;
    new_dcontext->app_nls_cache = old_dcontext->app_nls_cache;
    new_dcontext->priv_nls_cache = old_dcontext->priv_nls_cache;
#    endif
    new_dcontext->app_stack_limit = old_dcontext->app_stack_limit;
    new_dcontext->app_stack_base = old_dcontext->app_stack_base;
    new_dcontext->teb_base = old_dcontext->teb_base;
#    ifdef UNIX
    new_dcontext->signal_field = old_dcontext->signal_field;
    new_dcontext->pcprofile_field = old_dcontext->pcprofile_field;
#    endif
    new_dcontext->private_code = old_dcontext->private_code;
#    ifdef CLIENT_INTERFACE
    new_dcontext->client_data = old_dcontext->client_data;
#    endif
#    ifdef DEBUG
    new_dcontext->logfile = old_dcontext->logfile;
    new_dcontext->thread_stats = old_dcontext->thread_stats;
#    endif
#    ifdef DEADLOCK_AVOIDANCE
    new_dcontext->thread_owned_locks = old_dcontext->thread_owned_locks;
#    endif
#    ifdef KSTATS
    new_dcontext->thread_kstats = old_dcontext->thread_kstats;
#    endif
    /* at_syscall is real time based, not app context based, so shared
     *
     * FIXME: Yes need to share when swapping at NtCallbackReturn, but
     * want to keep old so when return from cb will do post-syscall for
     * syscall that triggered cb in the first place!
     * Plus, new cb calls initialize_dynamo_context(), which clears this field
     * anyway!  This all works now b/c we don't have alertable syscalls
     * that we do post-syscall processing on.
     */
    new_dcontext->upcontext_ptr->at_syscall = old_dcontext->upcontext_ptr->at_syscall;
#    ifdef HOT_PATCHING_INTERFACE /* Fix for case 5367. */
    /* hotp_excpt_state should be unused at this point.  If it is used, it can
     * be only because a hot patch made a system call with a callback.  This is
     * a bug because hot patches can't do system calls, let alone one with
     * callbacks.
     */
    DOCHECK(1, {
        dr_jmp_buf_t empty;
        memset(&empty, -1, sizeof(dr_jmp_buf_t));
        ASSERT(memcmp(&old_dcontext->hotp_excpt_state, &empty, sizeof(dr_jmp_buf_t)) ==
               0);
    });
    new_dcontext->nudge_thread = old_dcontext->nudge_thread;
#    endif
    /* our exceptions should be handled within one DR context switch */
    ASSERT(old_dcontext->try_except.try_except_state == NULL);
    new_dcontext->local_state = old_dcontext->local_state;
#    ifdef WINDOWS
    new_dcontext->aslr_context.last_child_padded =
        old_dcontext->aslr_context.last_child_padded;
#    endif

    LOG(new_dcontext->logfile, LOG_TOP, 2, "made new dcontext " PFX " (old=" PFX ")\n",
        new_dcontext, old_dcontext);
    return new_dcontext;
}
#endif

bool
is_thread_initialized(void)
{
#if defined(UNIX) && defined(HAVE_TLS)
    /* We don't want to pay the d_r_get_thread_id() cost on every
     * get_thread_private_dcontext() when we only really need the
     * check for this call here, so we explicitly check.
     */
    if (get_tls_thread_id() != get_sys_thread_id())
        return false;
#endif
    return (get_thread_private_dcontext() != NULL);
}

bool
is_thread_known(thread_id_t tid)
{
    return (thread_lookup(tid) != NULL);
}

#ifdef UNIX
/* i#237/PR 498284: a thread about to execute SYS_execve should be considered
 * exited, but we can't easily clean up it for real immediately
 */
void
mark_thread_execve(thread_record_t *tr, bool execve)
{
    ASSERT((execve && !tr->execve) || (!execve && tr->execve));
    tr->execve = execve;
    d_r_mutex_lock(&all_threads_lock);
    if (execve) {
        /* since we free on a second vfork we should never accumulate
         * more than one
         */
        ASSERT(num_execve_threads == 0);
        num_execve_threads++;
    } else {
        ASSERT(num_execve_threads > 0);
        num_execve_threads--;
    }
    d_r_mutex_unlock(&all_threads_lock);
}
#endif /* UNIX */

int
d_r_get_num_threads(void)
{
    return num_known_threads IF_UNIX(-num_execve_threads);
}

bool
is_last_app_thread(void)
{
    return (d_r_get_num_threads() == IF_CLIENT_INTERFACE(get_num_client_threads() +) 1);
}

/* This routine takes a snapshot of all the threads known to DR,
 * NOT LIMITED to those currently under DR control!
 * It returns an array of thread_record_t* and the length of the array
 * The caller must free the array using global_heap_free
 * The caller must hold the thread_initexit_lock to ensure that threads
 * are not created or destroyed before the caller is done with the list
 * The caller CANNOT be could_be_linking, else a deadlock with flushing
 * can occur (unless the caller is the one flushing)
 */
static void
get_list_of_threads_common(thread_record_t ***list,
                           int *num _IF_UNIX(bool include_execve))
{
    int i, cur = 0, max_num;
    thread_record_t *tr;
    thread_record_t **mylist;

    /* Only a flushing thread can get the thread snapshot while being
     * couldbelinking -- else a deadlock w/ flush!
     * FIXME: this assert should be on any acquisition of thread_initexit_lock!
     */
    ASSERT(is_self_flushing() || !is_self_couldbelinking());
    ASSERT(all_threads != NULL);
    ASSERT_OWN_MUTEX(true, &thread_initexit_lock);

    d_r_mutex_lock(&all_threads_lock);
    /* Do not include vfork threads that exited via execve, unless we're exiting */
    max_num = IF_UNIX_ELSE((include_execve || dynamo_exiting) ? num_known_threads
                                                              : d_r_get_num_threads(),
                           d_r_get_num_threads());
    mylist = (thread_record_t **)global_heap_alloc(
        max_num * sizeof(thread_record_t *) HEAPACCT(ACCT_THREAD_MGT));
    for (i = 0; i < HASHTABLE_SIZE(ALL_THREADS_HASH_BITS); i++) {
        for (tr = all_threads[i]; tr != NULL; tr = tr->next) {
            /* include those for which !tr->under_dynamo_control */
            /* don't include those that exited for execve.  there should be
             * no race b/c vfork suspends the parent.  xref i#237/PR 498284.
             */
            if (IF_UNIX_ELSE(!tr->execve || include_execve || dynamo_exiting, true)) {
                mylist[cur] = tr;
                cur++;
            }
        }
    }

    ASSERT(cur > 0);
    IF_WINDOWS(ASSERT(cur == max_num));
    if (cur < max_num) {
        mylist = (thread_record_t **)global_heap_realloc(
            mylist, max_num, cur, sizeof(thread_record_t *) HEAPACCT(ACCT_THREAD_MGT));
    }

    *num = cur;
    *list = mylist;
    d_r_mutex_unlock(&all_threads_lock);
}

void
get_list_of_threads(thread_record_t ***list, int *num)
{
    get_list_of_threads_common(list, num _IF_UNIX(false));
}

#ifdef UNIX
void
get_list_of_threads_ex(thread_record_t ***list, int *num, bool include_execve)
{
    get_list_of_threads_common(list, num, include_execve);
}
#endif

/* assumes caller can ensure that thread is either suspended or self to
 * avoid races
 */
thread_record_t *
thread_lookup(thread_id_t tid)
{
    thread_record_t *tr;
    uint hindex;

    /* check that caller is self or has initexit_lock
     * FIXME: no way to tell who has initexit_lock
     */
    ASSERT(mutex_testlock(&thread_initexit_lock) || tid == d_r_get_thread_id());

    hindex = HASH_FUNC_BITS(tid, ALL_THREADS_HASH_BITS);
    d_r_mutex_lock(&all_threads_lock);
    if (all_threads == NULL) {
        tr = NULL;
    } else {
        tr = all_threads[hindex];
    }
    while (tr != NULL) {
        if (tr->id == tid) {
            d_r_mutex_unlock(&all_threads_lock);
            return tr;
        }
        tr = tr->next;
    }
    d_r_mutex_unlock(&all_threads_lock);
    return NULL;
}

/* assumes caller can ensure that thread is either suspended or self to
 * avoid races
 */
uint
get_thread_num(thread_id_t tid)
{
    thread_record_t *tr = thread_lookup(tid);
    if (tr != NULL)
        return tr->num;
    else
        return 0; /* yes can't distinguish from 1st thread, who cares */
}

void
add_thread(IF_WINDOWS_ELSE_NP(HANDLE hthread, process_id_t pid), thread_id_t tid,
           bool under_dynamo_control, dcontext_t *dcontext)
{
    thread_record_t *tr;
    uint hindex;

    ASSERT(all_threads != NULL);

    /* add entry to thread hashtable */
    tr = (thread_record_t *)global_heap_alloc(sizeof(thread_record_t)
                                                  HEAPACCT(ACCT_THREAD_MGT));
#ifdef WINDOWS
    /* we duplicate the thread pseudo-handle, this should give us full rights
     * Note that instead asking explicitly for THREAD_ALL_ACCESS or just for
     * THREAD_TERMINATE|THREAD_SUSPEND_RESUME|THREAD_GET_CONTEXT|THREAD_SET_CONTEXT
     * does not seem able to acquire more rights than simply duplicating the
     * app handle gives.
     */
    LOG(GLOBAL, LOG_THREADS, 1, "Thread %d app handle rights: " PFX "\n", tid,
        nt_get_handle_access_rights(hthread));
    duplicate_handle(NT_CURRENT_PROCESS, hthread, NT_CURRENT_PROCESS, &tr->handle, 0, 0,
                     DUPLICATE_SAME_ACCESS | DUPLICATE_SAME_ATTRIBUTES);
    /* We prob. only need TERMINATE (for kill thread), SUSPEND/RESUME/GET_CONTEXT
     * (for synchronizing), and SET_CONTEXT (+ synchronizing requirements, for
     * detach).  All access includes this and quite a bit more. */
#    if 0
    /* eventually should be a real assert, but until we have a story for the
     * injected detach threads, have to ifdef out even the ASSERT_CURIOSITY
     * (even a syslog internal warning is prob. to noisy for QA) */
    ASSERT_CURIOSITY(TESTALL(THREAD_ALL_ACCESS, nt_get_handle_access_rights(tr->handle)));
#    endif
    LOG(GLOBAL, LOG_THREADS, 1, "Thread %d our handle rights: " PFX "\n", tid,
        nt_get_handle_access_rights(tr->handle));
    tr->retakeover = false;
#else
    tr->pid = pid;
    tr->execve = false;
#endif
    tr->id = tid;
    ASSERT(tid != INVALID_THREAD_ID); /* ensure os never assigns invalid id to a thread */
    tr->under_dynamo_control = under_dynamo_control;
    tr->dcontext = dcontext;
    if (dcontext != NULL) /* we allow NULL for dr_create_client_thread() */
        dcontext->thread_record = tr;

    d_r_mutex_lock(&all_threads_lock);
    tr->num = threads_ever_count++;
    hindex = HASH_FUNC_BITS(tr->id, ALL_THREADS_HASH_BITS);
    tr->next = all_threads[hindex];
    all_threads[hindex] = tr;
    /* must be inside all_threads_lock to avoid race w/ get_list_of_threads */
    RSTATS_ADD_PEAK(num_threads, 1);
    RSTATS_INC(num_threads_created);
    num_known_threads++;
    d_r_mutex_unlock(&all_threads_lock);
}

/* return false if couldn't find the thread */
bool
remove_thread(IF_WINDOWS_(HANDLE hthread) thread_id_t tid)
{
    thread_record_t *tr = NULL, *prevtr;
    uint hindex = HASH_FUNC_BITS(tid, ALL_THREADS_HASH_BITS);

    ASSERT(all_threads != NULL);

    d_r_mutex_lock(&all_threads_lock);
    for (tr = all_threads[hindex], prevtr = NULL; tr; prevtr = tr, tr = tr->next) {
        if (tr->id == tid) {
            if (prevtr)
                prevtr->next = tr->next;
            else
                all_threads[hindex] = tr->next;
            /* must be inside all_threads_lock to avoid race w/ get_list_of_threads */
            RSTATS_DEC(num_threads);
#ifdef UNIX
            if (tr->execve) {
                ASSERT(num_execve_threads > 0);
                num_execve_threads--;
            }
#endif
            num_known_threads--;
#ifdef WINDOWS
            close_handle(tr->handle);
#endif
            global_heap_free(tr, sizeof(thread_record_t) HEAPACCT(ACCT_THREAD_MGT));
            break;
        }
    }
    d_r_mutex_unlock(&all_threads_lock);
    return (tr != NULL);
}

/* this bool is protected by reset_pending_lock */
DECLARE_FREQPROT_VAR(static bool reset_at_nth_thread_triggered, false);

/* thread-specific initialization
 * if dstack_in is NULL, then a dstack is allocated; else dstack_in is used
 * as the thread's dstack
 * mc can be NULL for the initial thread
 * returns -1 if current thread has already been initialized
 */
/* On UNIX, if dstack_in != NULL, the parent of this new thread must have
 * increased uninit_thread_count.
 */
int
dynamo_thread_init(byte *dstack_in, priv_mcontext_t *mc,
                   void *os_data _IF_CLIENT_INTERFACE(bool client_thread))
{
    dcontext_t *dcontext;
    /* due to lock issues (see below) we need another var */
    bool reset_at_nth_thread_pending = false;
    bool under_dynamo_control = true;
    APP_EXPORT_ASSERT(dynamo_initialized || dynamo_exited ||
                          d_r_get_num_threads() ==
                              0 IF_CLIENT_INTERFACE(|| client_thread),
                      PRODUCT_NAME " not initialized");
    if (INTERNAL_OPTION(nullcalls)) {
        ASSERT(uninit_thread_count == 0);
        return SUCCESS;
    }

    /* note that ENTERING_DR is assumed to have already happened: in apc handler
     * for win32, in new_thread_setup for linux, in main init for 1st thread
     */
#if defined(WINDOWS) && defined(DR_APP_EXPORTS)
    /* We need to identify a thread we intercepted in its APC when we
     * take over all threads on dr_app_start().  Stack and pc checks aren't
     * simple b/c it can be in ntdll waiting on a lock.
     */
    if (dr_api_entry)
        os_take_over_mark_thread(d_r_get_thread_id());
#endif

    /* Try to handle externally injected threads */
    if (dynamo_initialized && !bb_lock_start)
        pre_second_thread();

    /* synch point so thread creation can be prevented for critical periods */
    d_r_mutex_lock(&thread_initexit_lock);

    /* XXX i#2611: during detach, there is a race where a thread can
     * reach here on Windows despite init_apc_go_native (i#2600).
     */
    ASSERT_BUG_NUM(2611, !doing_detach);

    /* The assumption is that if dynamo_exited, then we are about to exit and
     * clean up, initializing this thread then would be dangerous, better to
     * wait here for the app to die.
     */
    /* under current implementation of process exit, can happen only under
     * debug build, or app_start app_exit interface */
    while (dynamo_exited) {
        /* logging should be safe, though might not actually result in log
         * message */
        DODEBUG_ONCE(LOG(GLOBAL, LOG_THREADS, 1,
                         "Thread %d reached initialization point while dynamo exiting, "
                         "waiting for app to exit\n",
                         d_r_get_thread_id()););
        d_r_mutex_unlock(&thread_initexit_lock);
        os_thread_yield();
        /* just in case we want to support exited and then restarted at some
         * point */
        d_r_mutex_lock(&thread_initexit_lock);
    }

    if (is_thread_initialized()) {
        d_r_mutex_unlock(&thread_initexit_lock);
#if defined(WINDOWS) && defined(DR_APP_EXPORTS)
        if (dr_api_entry)
            os_take_over_unmark_thread(d_r_get_thread_id());
#endif
        return -1;
    }

    os_tls_init();
    dcontext = create_new_dynamo_context(true /*initial*/, dstack_in, mc);
    initialize_dynamo_context(dcontext);
    set_thread_private_dcontext(dcontext);
    /* sanity check */
    ASSERT(get_thread_private_dcontext() == dcontext);

    /* set local state pointer for access from other threads */
    dcontext->local_state = get_local_state();

    /* set initial mcontext, if known */
    if (mc != NULL)
        *get_mcontext(dcontext) = *mc;

    /* For hotp_only, the thread should run native, not under dr.  However,
     * the core should still get control of the thread at hook points to track
     * what the application is doing & at patched points to execute hot patches.
     * It is the same for thin_client except that there are fewer hooks, only to
     * follow children.
     */
    if (RUNNING_WITHOUT_CODE_CACHE())
        under_dynamo_control = false;

    /* add entry to thread hashtable before creating logdir so have thread num.
     * otherwise we'd like to do this only after we'd fully initialized the thread, but we
     * hold the thread_initexit_lock, so nobody should be listing us -- thread_lookup
     * on other than self, or a thread list, should only be done while the initexit_lock
     * is held.  CHECK: is this always correct?  thread_lookup does have an assert
     * to try and enforce but cannot tell who has the lock.
     */
    add_thread(IF_WINDOWS_ELSE(NT_CURRENT_THREAD, get_process_id()), d_r_get_thread_id(),
               under_dynamo_control, dcontext);
#ifdef UNIX /* i#2600: Not easy on Windows: we rely on init_apc_go_native there. */
    if (dstack_in != NULL) { /* Else not a thread creation we observed */
        ASSERT(uninit_thread_count > 0);
        ATOMIC_DEC(int, uninit_thread_count);
    }
#endif
#if defined(WINDOWS) && defined(DR_APP_EXPORTS)
    /* Now that the thread is in the main thread table we don't need to remember it */
    if (dr_api_entry)
        os_take_over_unmark_thread(d_r_get_thread_id());
#endif

    LOG(GLOBAL, LOG_TOP | LOG_THREADS, 1,
        "\ndynamo_thread_init: %d thread(s) now, dcontext=" PFX ", #=%d, id=" TIDFMT
        ", pid=" PIDFMT "\n\n",
        GLOBAL_STAT(num_threads), dcontext, get_thread_num(d_r_get_thread_id()),
        d_r_get_thread_id(), get_process_id());

    DOLOG(1, LOG_STATS, { dump_global_stats(false); });
#ifdef DEBUG
    if (d_r_stats->loglevel > 0) {
        dcontext->logfile = open_log_file(thread_logfile_name(), NULL, 0);
        print_file(dcontext->logfile, "%s\n", dynamorio_version_string);
    } else {
        dcontext->logfile = INVALID_FILE;
    }
    DOLOG(1, LOG_TOP | LOG_THREADS, {
        LOG(THREAD, LOG_TOP | LOG_THREADS, 1, PRODUCT_NAME " built with: %s\n",
            DYNAMORIO_DEFINES);
        LOG(THREAD, LOG_TOP | LOG_THREADS, 1, PRODUCT_NAME " built on: %s\n",
            dynamorio_buildmark);
    });

    LOG(THREAD, LOG_TOP | LOG_THREADS, 1, "%sTHREAD %d (dcontext " PFX ")\n\n",
        IF_CLIENT_INTERFACE_ELSE(client_thread ? "CLIENT " : "", ""), d_r_get_thread_id(),
        dcontext);
    LOG(THREAD, LOG_TOP | LOG_THREADS, 1,
        "DR stack is " PFX "-" PFX " (passed in " PFX ")\n",
        dcontext->dstack - DYNAMORIO_STACK_SIZE, dcontext->dstack, dstack_in);
#endif

#ifdef DEADLOCK_AVOIDANCE
    locks_thread_init(dcontext);
#endif
    heap_thread_init(dcontext);
    DOSTATS({ stats_thread_init(dcontext); });
#ifdef KSTATS
    kstat_thread_init(dcontext);
#endif
    os_thread_init(dcontext, os_data);
    arch_thread_init(dcontext);
    synch_thread_init(dcontext);

    if (!DYNAMO_OPTION(thin_client))
        vm_areas_thread_init(dcontext);

    monitor_thread_init(dcontext);
    fcache_thread_init(dcontext);
    link_thread_init(dcontext);
    fragment_thread_init(dcontext);

    /* OS thread init after synch_thread_init and other setup can handle signals, etc. */
    os_thread_init_finalize(dcontext, os_data);

    /* This lock has served its purposes: A) a barrier to thread creation for those
     * iterating over threads, B) mutex for add_thread, and C) mutex for synch_field
     * to be set up.
     * So we release it to shrink the time spent w/ this big lock, in particular
     * to avoid holding it while running private lib thread init code (i#875).
     */
    d_r_mutex_unlock(&thread_initexit_lock);

#ifdef CLIENT_INTERFACE
    /* Set up client data needed in loader_thread_init for IS_CLIENT_THREAD */
    instrument_client_thread_init(dcontext, client_thread);
#endif

    loader_thread_init(dcontext);

    if (!DYNAMO_OPTION(thin_client)) {
#ifdef CLIENT_INTERFACE
        /* put client last, may depend on other thread inits.
         * Note that we are calling this prior to instrument_init()
         * now (PR 216936), which is required to initialize
         * the client dcontext field prior to instrument_init().
         */
        instrument_thread_init(dcontext, client_thread, mc != NULL);
#endif

#ifdef SIDELINE
        if (dynamo_options.sideline) {
            /* wake up sideline thread -- ok to call if thread already awake */
            sideline_start();
        }
#endif
    }

    /* must check # threads while holding thread_initexit_lock, yet cannot
     * call fcache_reset_all_caches_proactively while holding it due to
     * rank order of reset_pending_lock which we must also hold -- so we
     * set a local bool reset_at_nth_thread_pending
     */
    if (DYNAMO_OPTION(reset_at_nth_thread) != 0 && !reset_at_nth_thread_triggered &&
        (uint)d_r_get_num_threads() == DYNAMO_OPTION(reset_at_nth_thread)) {
        d_r_mutex_lock(&reset_pending_lock);
        if (!reset_at_nth_thread_triggered) {
            reset_at_nth_thread_triggered = true;
            reset_at_nth_thread_pending = true;
        }
        d_r_mutex_unlock(&reset_pending_lock);
    }

    DOLOG(1, LOG_STATS, { dump_thread_stats(dcontext, false); });

    if (reset_at_nth_thread_pending) {
        d_r_mutex_lock(&reset_pending_lock);
        /* fcache_reset_all_caches_proactively() will unlock */
        fcache_reset_all_caches_proactively(RESET_ALL);
    }
    return SUCCESS;
}

/* We don't free cur thread until after client exit event (PR 536058) except for
 * fragment_thread_exit().  Since this is called outside of dynamo_thread_exit()
 * on process exit we assume fine to skip enter_threadexit().
 */
void
dynamo_thread_exit_pre_client(dcontext_t *dcontext, thread_id_t id)
{
    /* fcache stats needs to examine fragment state, so run it before
     * fragment exit, but real fcache exit needs to be after fragment exit
     */
#ifdef DEBUG
    fcache_thread_exit_stats(dcontext);
#endif
    /* must abort now to avoid deleting possibly un-deletable fragments
     * monitor_thread_exit remains later b/c of monitor_remove_fragment calls
     */
    trace_abort_and_delete(dcontext);
    fragment_thread_exit(dcontext);
#ifdef CLIENT_INTERFACE
    IF_WINDOWS(loader_pre_client_thread_exit(dcontext));
    instrument_thread_exit_event(dcontext);
#endif
}

/* thread-specific cleanup */
/* Note : if this routine is not called by thread id, then other_thread should
 * be true and the calling thread should hold the thread_initexit_lock
 */
static int
dynamo_thread_exit_common(dcontext_t *dcontext, thread_id_t id,
                          IF_WINDOWS_(bool detach_stacked_callbacks) bool other_thread)
{
    dcontext_t *dcontext_tmp;
#ifdef WINDOWS
    dcontext_t *dcontext_next;
    int num_dcontext;
#endif
    bool on_dstack = !other_thread && is_currently_on_dstack(dcontext);
    /* cache this now for use after freeing dcontext */
    local_state_t *local_state = dcontext->local_state;

    if (INTERNAL_OPTION(nullcalls) || dcontext == NULL)
        return SUCCESS;

    /* make sure don't get into deadlock w/ flusher */
    enter_threadexit(dcontext);

    /* synch point so thread exiting can be prevented for critical periods */
    /* see comment at start of method for other thread exit */
    if (!other_thread)
        d_r_mutex_lock(&thread_initexit_lock);

    ASSERT_OWN_MUTEX(true, &thread_initexit_lock);
#ifdef WINDOWS
    /* need to clean up thread stack before clean up other thread data, but
     * after we're made nolinking
     */
    os_thread_stack_exit(dcontext);
    /* free the thread's application stack if requested */
    if (dcontext->free_app_stack) {
        byte *base;
        /* only used for nudge threads currently */
        ASSERT(dcontext->nudge_target == generic_nudge_target);
        if (get_stack_bounds(dcontext, &base, NULL)) {
            NTSTATUS res;
            ASSERT(base != NULL);
            res = nt_free_virtual_memory(base);
            ASSERT(NT_SUCCESS(res));
        } else {
            /* stack should be available here */
            ASSERT_NOT_REACHED();
        }
    }
#endif

#ifdef SIDELINE
    /* N.B.: do not clean up any data structures while sideline thread
     * is still running!  put it to sleep for duration of this routine!
     */
    if (!DYNAMO_OPTION(thin_client)) {
        if (dynamo_options.sideline) {
            /* put sideline thread to sleep */
            sideline_stop();
            /* sideline_stop will not return until sideline thread is asleep */
        }
    }
#endif

    LOG(GLOBAL, LOG_TOP | LOG_THREADS, 1,
        "\ndynamo_thread_exit (thread #%d id=" TIDFMT "): %d thread(s) now\n\n",
        get_thread_num(id), id, GLOBAL_STAT(num_threads) - 1);

    DOLOG(1, LOG_STATS, { dump_global_stats(false); });

    LOG(THREAD, LOG_STATS | LOG_THREADS, 1, "\n## Statistics for this thread:\n");

#ifdef PROFILE_RDTSC
    if (dynamo_options.profile_times) {
        int i;
        ASSERT(dcontext);
        LOG(THREAD, LOG_STATS | LOG_THREADS, 1, "\nTop ten cache times:\n");
        for (i = 0; i < 10; i++) {
            if (dcontext->cache_time[i] > (uint64)0) {
                uint top_part, bottom_part;
                divide_int64_print(dcontext->cache_time[i], kilo_hertz, false, 3,
                                   &top_part, &bottom_part);
                LOG(THREAD, LOG_STATS | LOG_THREADS, 1,
                    "\t#%2d = %6u.%.3u ms, %9d hits\n", i + 1, top_part, bottom_part,
                    (int)dcontext->cache_count[i]);
            }
        }
        LOG(THREAD, LOG_STATS | LOG_THREADS, 1, "\n");
    }
#endif

    /* In order to pass the client a dcontext in the process exit event
     * we do some thread cleanup early for the final thread so we can delay
     * the rest (PR 536058)
     */
    if (!dynamo_exited_and_cleaned)
        dynamo_thread_exit_pre_client(dcontext, id);
#ifdef CLIENT_INTERFACE
    /* PR 243759: don't free client_data until after all fragment deletion events */
    if (!DYNAMO_OPTION(thin_client))
        instrument_thread_exit(dcontext);
#endif

    /* i#920: we can't take segment/timer/asynch actions for other threads.
     * This must be called after dynamo_thread_exit_pre_client where
     * we called event callbacks.
     */
    if (!other_thread)
        dynamo_thread_not_under_dynamo(dcontext);

    /* We clean up priv libs prior to setting tls dc to NULL so we can use
     * TRY_EXCEPT when calling the priv lib entry routine
     */
    if (!dynamo_exited ||
        (other_thread &&
         (IF_WINDOWS_ELSE(!doing_detach, true) ||
          dcontext->owning_thread != d_r_get_thread_id()))) /* else already did this */
        loader_thread_exit(dcontext);

    /* set tls dc to NULL prior to cleanup, to avoid problems handling
     * alarm signals received during cleanup (we'll suppress if tls
     * dc==NULL which seems the right thing to do: not worth our
     * effort to pass to another thread if thread-group-shared alarm,
     * and if thread-private then thread would have exited soon
     * anyway).  see PR 596127.
     */
    /* make sure we invalidate the dcontext before releasing the memory  */
    /* when cleaning up other threads, we cannot set their dcs to null,
     * but we only do this at dynamorio_app_exit so who cares
     */
    /* This must be called after instrument_thread_exit, which uses
     * get_thread_private_dcontext for app/dr state checks.
     */
    if (id == d_r_get_thread_id())
        set_thread_private_dcontext(NULL);

    fcache_thread_exit(dcontext);
    link_thread_exit(dcontext);
    monitor_thread_exit(dcontext);
    if (!DYNAMO_OPTION(thin_client))
        vm_areas_thread_exit(dcontext);
    synch_thread_exit(dcontext);
    arch_thread_exit(dcontext _IF_WINDOWS(detach_stacked_callbacks));
    os_thread_exit(dcontext, other_thread);
    DOLOG(1, LOG_STATS, { dump_thread_stats(dcontext, false); });
#ifdef KSTATS
    kstat_thread_exit(dcontext);
#endif
    DOSTATS({ stats_thread_exit(dcontext); });
    heap_thread_exit(dcontext);
#ifdef DEADLOCK_AVOIDANCE
    locks_thread_exit(dcontext);
#endif

#ifdef DEBUG
    if (dcontext->logfile != INVALID_FILE) {
        os_flush(dcontext->logfile);
        close_log_file(dcontext->logfile);
    }
#endif

    /* remove thread from threads hashtable */
    remove_thread(IF_WINDOWS_(NT_CURRENT_THREAD) id);

    dcontext_tmp = dcontext;
#ifdef WINDOWS
    /* clean up all the dcs */
    num_dcontext = 0;
#    ifdef DCONTEXT_IN_EDI
    /* go to one end of list */
    while (dcontext_tmp->next_saved)
        dcontext_tmp = dcontext_tmp->next_saved;
#    else
    /* already at one end of list */
#    endif

    /* delete through to other end */
    while (dcontext_tmp) {
        num_dcontext++;
        dcontext_next = dcontext_tmp->prev_unused;
        delete_dynamo_context(dcontext_tmp,
                              dcontext_tmp == dcontext /*do not free dup cb stacks*/
                                  && !on_dstack /*do not free own stack*/);
        dcontext_tmp = dcontext_next;
    }
    LOG(GLOBAL, LOG_STATS | LOG_THREADS, 1, "\tdynamo contexts used: %d\n", num_dcontext);
#else  /* UNIX */
    delete_dynamo_context(dcontext_tmp, !on_dstack /*do not free own stack*/);
#endif /* UNIX */
    os_tls_exit(local_state, other_thread);

#ifdef SIDELINE
    /* see notes above -- we can now wake up sideline thread */
    if (dynamo_options.sideline && d_r_get_num_threads() > 0) {
        sideline_start();
    }
#endif
    if (!other_thread) {
        d_r_mutex_unlock(&thread_initexit_lock);
        /* FIXME: once thread_initexit_lock is released, we're not on
         * thread list, and a terminate targeting us could kill us in the middle
         * of this call -- but this can't come before the unlock b/c the lock's
         * in the data segment!  (see case 3121)
         * (note we do not re-protect for process exit, see !dynamo_exited check
         * in exiting_dynamorio)
         */
        if (!on_dstack) {
            EXITING_DR();
            /* else, caller will clean up stack and then call EXITING_DR(),
             * probably via dynamo_thread_stack_free_and_exit(), as the stack free
             * must be done before the exit
             */
        }
    }

    return SUCCESS;
}

int
dynamo_thread_exit(void)
{
    dcontext_t *dcontext = get_thread_private_dcontext();
    return dynamo_thread_exit_common(dcontext, d_r_get_thread_id(),
                                     IF_WINDOWS_(false) false);
}

/* NOTE : you must hold thread_initexit_lock to call this function! */
int
dynamo_other_thread_exit(thread_record_t *tr _IF_WINDOWS(bool detach_stacked_callbacks))
{
    /* FIXME: Usually a safe spot for cleaning other threads should be
     * under num_exits_dir_syscall, but for now rewinding all the way
     */
    KSTOP_REWIND_DC(tr->dcontext, thread_measured);
    KSTART_DC(tr->dcontext, thread_measured);
    return dynamo_thread_exit_common(tr->dcontext, tr->id,
                                     IF_WINDOWS_(detach_stacked_callbacks) true);
}

/* Called from another stack to finish cleaning up a thread.
 * The final steps are to free the stack and perform the exit hook.
 */
void
dynamo_thread_stack_free_and_exit(byte *stack)
{
    if (stack != NULL) {
        stack_free(stack, DYNAMORIO_STACK_SIZE);
        /* ASSUMPTION: if stack is NULL here, the exit was done earlier
         * (fixes case 6967)
         */
        EXITING_DR();
    }
}

#ifdef DR_APP_EXPORTS
/* API routine to initialize DR */
DR_APP_API int
dr_app_setup(void)
{
    /* FIXME: we either have to disallow the client calling this with
     * more than one thread running, or we have to suspend all the threads.
     * We should share the suspend-and-takeover loop (and for dr_app_setup_and_start
     * share the takeover portion) from dr_app_start().
     */
    int res;
    dcontext_t *dcontext;
    dr_api_entry = true;
    res = dynamorio_app_init();
    /* For dr_api_entry, we do not install signal handlers during init (to avoid
     * races: i#2335): we delay until dr_app_start().  Plus the vsyscall hook is
     * not set up until we find out the syscall method.  Thus we're already
     * "os_process_not_under_dynamorio".
     * We can't as easily avoid initializing the thread TLS and then dropping
     * it, however, as parts of init assume we have TLS.
     */
    dcontext = get_thread_private_dcontext();
    dynamo_thread_not_under_dynamo(dcontext);
    return res;
}

/* API routine to exit DR */
DR_APP_API int
dr_app_cleanup(void)
{
    thread_record_t *tr;
    SELF_UNPROTECT_DATASEC(DATASEC_RARELY_PROT);
    dr_api_exit = true;
    SELF_PROTECT_DATASEC(DATASEC_RARELY_PROT); /* to keep properly nested */

    /* XXX: The dynamo_thread_[not_]under_dynamo() routines are not idempotent,
     * and must be balanced!  On Linux, they track the shared itimer refcount,
     * so a mismatch will lead to a refleak or negative refcount.
     * dynamorio_app_exit() will call dynamo_thread_not_under_dynamo(), so we
     * must ensure that we are under DR before calling it.  Therefore, we
     * require that the caller call dr_app_stop() before calling
     * dr_app_cleanup().  However, we cannot make a usage assertion to that
     * effect without addressing the FIXME comments in
     * dynamo_thread_not_under_dynamo() about updating tr->under_dynamo_control.
     */
    tr = thread_lookup(d_r_get_thread_id());
    if (tr != NULL && tr->dcontext != NULL) {
        os_process_under_dynamorio_initiate(tr->dcontext);
        os_process_under_dynamorio_complete(tr->dcontext);
        dynamo_thread_under_dynamo(tr->dcontext);
    }
    return dynamorio_app_exit();
}

/* Called by dr_app_start in arch-specific assembly file */
void
dr_app_start_helper(priv_mcontext_t *mc)
{
    apicheck(dynamo_initialized, PRODUCT_NAME " not initialized");
    LOG(GLOBAL, LOG_TOP, 1, "dr_app_start in thread " TIDFMT "\n", d_r_get_thread_id());
    LOG(THREAD_GET, LOG_TOP, 1, "dr_app_start\n");

    if (!INTERNAL_OPTION(nullcalls)) {
        /* Adjust the app stack to account for the return address + alignment.
         * See dr_app_start in x86.asm.
         */
        mc->xsp += DYNAMO_START_XSP_ADJUST;
        dynamo_start(mc);
        /* the interpreter takes over from here */
    }
}

/* dummy routine that returns control to the app if it is currently
 * under dynamo control
 */
DR_APP_API void
dr_app_stop(void)
{
    /* the application regains control in here */
}

DR_APP_API void
dr_app_stop_and_cleanup(void)
{
    dr_app_stop_and_cleanup_with_stats(NULL);
}

DR_APP_API void
dr_app_stop_and_cleanup_with_stats(dr_stats_t *drstats)
{
    /* XXX i#95: today this is a full detach, while a separated dr_app_cleanup()
     * is not.  We should try and have dr_app_cleanup() take this detach path
     * here (and then we can simplify exit_synch_state()) but it's more complicated
     * and we need to resolve the unbounded dr_app_stop() time.
     */
    if (dynamo_initialized && !dynamo_exited && !doing_detach) {
        detach_on_permanent_stack(true /*internal*/, true /*do cleanup*/, drstats);
    }
    /* the application regains control in here */
}

DR_APP_API int
dr_app_setup_and_start(void)
{
    int r = dr_app_setup();
    if (r == SUCCESS)
        dr_app_start();
    return r;
}
#endif

/* For use by threads that start and stop whether dynamo controls them.
 */
void
dynamo_thread_under_dynamo(dcontext_t *dcontext)
{
    LOG(THREAD, LOG_ASYNCH, 2, "thread %d under DR control\n", dcontext->owning_thread);
    ASSERT(dcontext != NULL);
    /* FIXME: mark under_dynamo_control?
     * see comments in not routine below
     */
    os_thread_under_dynamo(dcontext);
#ifdef SIDELINE
    if (dynamo_options.sideline) {
        /* wake up sideline thread -- ok to call if thread already awake */
        sideline_start();
    }
#endif
    dcontext->currently_stopped = false;
    dcontext->go_native = false;
}

/* For use by threads that start and stop whether dynamo controls them.
 * This must be called by the owner of dcontext and not another
 * non-executing thread.
 */
void
dynamo_thread_not_under_dynamo(dcontext_t *dcontext)
{
    ASSERT_MESSAGE(CHKLVL_ASSERTS + 1 /*expensive*/, "can only act on executing thread",
                   dcontext == get_thread_private_dcontext());
    if (dcontext == NULL)
        return;
    LOG(THREAD, LOG_ASYNCH, 2, "thread %d not under DR control\n",
        dcontext->owning_thread);
    dcontext->currently_stopped = true;
    os_thread_not_under_dynamo(dcontext);
#ifdef SIDELINE
    /* FIXME: if # active threads is 0, then put sideline thread to sleep! */
    if (dynamo_options.sideline) {
        /* put sideline thread to sleep */
        sideline_stop();
    }
#endif
#ifdef DEBUG
    os_flush(dcontext->logfile);
#endif
}

#define MAX_TAKE_OVER_ATTEMPTS 8

/* Mark this thread as under DR, and take over other threads in the current process.
 */
void
dynamorio_take_over_threads(dcontext_t *dcontext)
{
    /* We repeatedly check if there are other threads in the process, since
     * while we're checking one may be spawning additional threads.
     */
    bool found_threads;
    uint attempts = 0;

    os_process_under_dynamorio_initiate(dcontext);
    /* We can start this thread now that we've set up process-wide actions such
     * as handling signals.
     */
    dynamo_thread_under_dynamo(dcontext);
    signal_event(dr_app_started);
    SELF_UNPROTECT_DATASEC(DATASEC_RARELY_PROT);
    dynamo_started = true;
    SELF_PROTECT_DATASEC(DATASEC_RARELY_PROT);
    /* XXX i#1305: we should suspend all the other threads for DR init to
     * satisfy the parts of the init process that assume there are no races.
     */
    do {
        found_threads = os_take_over_all_unknown_threads(dcontext);
        attempts++;
        if (found_threads && !bb_lock_start)
            bb_lock_start = true;
    } while (found_threads && attempts < MAX_TAKE_OVER_ATTEMPTS);
    os_process_under_dynamorio_complete(dcontext);
    /* End the barrier to new threads. */
    signal_event(dr_attach_finished);

    if (found_threads) {
        REPORT_FATAL_ERROR_AND_EXIT(FAILED_TO_TAKE_OVER_THREADS, 2,
                                    get_application_name(), get_application_pid());
    }
    char buf[16];
    int num_threads = d_r_get_num_threads();
    if (num_threads > 1) { /* avoid for early injection */
        snprintf(buf, BUFFER_SIZE_ELEMENTS(buf), "%d", num_threads);
        NULL_TERMINATE_BUFFER(buf);
        SYSLOG(SYSLOG_INFORMATION, INFO_ATTACHED, 3, buf, get_application_name(),
               get_application_pid());
    }
}

/* Called by dynamorio_app_take_over in arch-specific assembly file */
void
dynamorio_app_take_over_helper(priv_mcontext_t *mc)
{
    static bool have_taken_over = false; /* ASSUMPTION: not an actual write */
    SELF_UNPROTECT_DATASEC(DATASEC_RARELY_PROT);
    APP_EXPORT_ASSERT(dynamo_initialized, PRODUCT_NAME " not initialized");
#ifdef RETURN_AFTER_CALL
    /* FIXME : this is set after dynamo_initialized, so a slight race with
     * an injected thread turning on .C protection before the main thread
     * sets this. */
    dr_preinjected = true; /* currently only relevant on Win32 */
#endif
    LOG(GLOBAL, LOG_TOP, 1, "taking over via preinject in %s\n", __FUNCTION__);

    if (!INTERNAL_OPTION(nullcalls) && !have_taken_over) {
        have_taken_over = true;
        LOG(GLOBAL, LOG_TOP, 1, "dynamorio_app_take_over\n");
        /* set this flag to indicate that we should run until the program dies: */
        automatic_startup = true;

        if (DYNAMO_OPTION(inject_primary))
            take_over_primary_thread();

        /* who knows when this was called -- no guarantee we control all threads --
         * unless we were auto-injected (preinject library calls this routine)
         */
        control_all_threads = automatic_startup;
        SELF_PROTECT_DATASEC(DATASEC_RARELY_PROT);

        /* Adjust the app stack to account for the return address + alignment.
         * See dynamorio_app_take_over in x86.asm.
         */
        mc->xsp += DYNAMO_START_XSP_ADJUST;

        /* For hotp_only and thin_client, the app should run native, except
         * for our hooks.
         * This is where apps hooked using appinit key are let go native.
         * Even though control is going to native app code, we want
         * automatic_startup and control_all_threads set.
         */
        if (!RUNNING_WITHOUT_CODE_CACHE())
            dynamo_start(mc);
        /* the interpreter takes over from here */
    } else
        SELF_PROTECT_DATASEC(DATASEC_RARELY_PROT);
}

#ifdef WINDOWS
extern app_pc parent_early_inject_address; /* from os.c */

/* in arch-specific assembly file */
void
dynamorio_app_take_over(void);

DYNAMORIO_EXPORT void
dynamorio_app_init_and_early_takeover(uint inject_location, void *restore_code)
{
    int res;
    ASSERT(!dynamo_initialized && !dynamo_exited);
    /* This routine combines dynamorio_app_init() and dynamrio_app_takeover into
     * a single routine that also handles any early injection cleanup needed. */
    ASSERT_NOT_IMPLEMENTED(inject_location != INJECT_LOCATION_KiUserApc);
    /* currently only Ldr* hook points are known to work */
    ASSERT_CURIOSITY(INJECT_LOCATION_IS_LDR(inject_location));
    /* See notes in os.c DLLMain. When early injected we are unable to find
     * the address of LdrpLoadDll so we use the parent's value which is passed
     * to us at the start of restore_code. FIXME - if we start using multiple
     * inject locations we'll probably have to ensure we always pass this.
     */
    if (INJECT_LOCATION_IS_LDR(inject_location)) {
        parent_early_inject_address = *(app_pc *)restore_code;
    }
    dr_early_injected = true;
    dr_early_injected_location = inject_location;
    res = dynamorio_app_init();
    ASSERT(res == SUCCESS);
    ASSERT(dynamo_initialized && !dynamo_exited);
    LOG(GLOBAL, LOG_TOP, 1, "taking over via early injection in %s\n", __FUNCTION__);
    /* FIXME - restore code needs to be freed, but we have to return through it
     * first... could instead duplicate its tail here if we wrap this
     * routine in asm or eqv. pass the continuation state in as args. */
    ASSERT(inject_location != INJECT_LOCATION_KiUserApc);
    dynamorio_app_take_over();
}

/* Called with DR library mapped in but without its imports processed.
 */
void
dynamorio_earliest_init_takeover_C(byte *arg_ptr)
{
    int res;
    bool earliest_inject;

    /* Windows-specific code for the most part */
    earliest_inject = earliest_inject_init(arg_ptr);

    /* Initialize now that DR dll imports are hooked up */
    if (earliest_inject) {
        dr_earliest_injected = true;
        dr_earliest_inject_args = arg_ptr;
    } else
        dr_early_injected = true;
    res = dynamorio_app_init();
    ASSERT(res == SUCCESS);
    ASSERT(dynamo_initialized && !dynamo_exited);
    LOG(GLOBAL, LOG_TOP, 1, "taking over via earliest injection in %s\n", __FUNCTION__);

    /* earliest_inject_cleanup() is called within dynamorio_app_init() to avoid
     * confusing the exec areas scan
     */

    /* Take over at retaddr
     *
     * XXX i#626: app_takeover sets preinjected for rct (should prob. rename)
     * which needs to be done whenever we takeover not at the bottom of the
     * callstack.  For earliest won't need to set this if we takeover
     * in such a way as to handle the return back to our hook code without a
     * violation -- though currently we will see 3 rets (return from
     * dynamorio_app_take_over(), return from here, and return from
     * dynamorio_earliest_init_takeover() to app hook code).
     * Should we have dynamorio_earliest_init_takeover() set up an
     * mcontext that we can go to directly instead of interpreting
     * the returns in our own code?  That would make tools that shadow
     * callstacks simpler too.
     */
    dynamorio_app_take_over();
}
#endif /* WINDOWS */

/***************************************************************************
 * SELF-PROTECTION
 */

/* FIXME: even with -single_privileged_thread, we aren't fully protected,
 * because there's a window between us resuming the other threads and
 * returning to our caller where another thread could clobber our return
 * address or something.
 */
static void
dynamorio_protect(void)
{
    ASSERT(SELF_PROTECT_ON_CXT_SWITCH);
    LOG(GLOBAL, LOG_DISPATCH, 4, "dynamorio_protect thread=" TIDFMT "\n",
        d_r_get_thread_id());
    /* we don't protect local heap here, that's done lazily */

    d_r_mutex_lock(&protect_info->lock);
    ASSERT(protect_info->num_threads_unprot > 0);
    /* FIXME: nice to also catch double enters but would need to track more info */
    if (protect_info->num_threads_unprot <= 0) {
        /* Defensive code to prevent crashes from double exits (the theory
         * for case 7631/8030).  However, this precludes an extra exit+enter
         * pair from working properly (though an extra enter+exit will continue
         * to work), though such a pair would have crashed if another thread
         * had entered in the interim anyway.
         */
        protect_info->num_threads_unprot = 0;
        d_r_mutex_unlock(&protect_info->lock);
        return;
    }
    protect_info->num_threads_unprot--;
    if (protect_info->num_threads_unprot > 0) {
        /* other threads still in DR, cannot protect global memory */
        LOG(GLOBAL, LOG_DISPATCH, 4, "dynamorio_protect: not last thread => nop\n");
        d_r_mutex_unlock(&protect_info->lock);
        return;
    }

    SELF_PROTECT_GLOBAL(READONLY);

    if (INTERNAL_OPTION(single_privileged_thread)) {
        /* FIXME: want to resume threads and allow thread creation only
         * _after_ protect data segment, but lock is in data segment!
         */
        if (protect_info->num_threads_suspended > 0) {
            thread_record_t *tr;
            int i, num = 0;
            /* we do not need to grab the all_threads_lock because
             * no threads can be added or removed so who cares if we
             * access the data structure simultaneously with another
             * reader of it
             */
            for (i = 0; i < HASHTABLE_SIZE(ALL_THREADS_HASH_BITS); i++) {
                for (tr = all_threads[i]; tr; tr = tr->next) {
                    if (tr->under_dynamo_control) {
                        os_thread_resume(all_threads[i]);
                        num++;
                    }
                }
            }
            ASSERT(num == protect_info->num_threads_suspended);
            protect_info->num_threads_suspended = 0;
        }

        /* thread init/exit can proceed now */
        d_r_mutex_unlock(&thread_initexit_lock);
    }

    /* FIXME case 8073: temporary until we put in unprots in the
     * right places.  if we were to leave this here we'd want to combine
     * .fspdata and .cspdata for more efficient prot changes.
     */
    SELF_PROTECT_DATASEC(DATASEC_FREQ_PROT);
    SELF_PROTECT_DATASEC(DATASEC_CXTSW_PROT);

    d_r_mutex_unlock(&protect_info->lock);
}

static void
dynamorio_unprotect(void)
{
    ASSERT(SELF_PROTECT_ON_CXT_SWITCH);

    d_r_mutex_lock(
        &protect_info->lock); /* lock in unprot heap, not data segment, so safe! */
    protect_info->num_threads_unprot++;
    if (protect_info->num_threads_unprot == 1) {
        /* was protected, so we need to do the unprotection */
        SELF_UNPROTECT_DATASEC(DATASEC_CXTSW_PROT);
        /* FIXME case 8073: temporary until we put in unprots in the
         * right places.  if we were to leave this here we'd want to combine
         * .fspdata and .cspdata for more efficient prot changes.
         */
        SELF_UNPROTECT_DATASEC(DATASEC_FREQ_PROT);

        if (INTERNAL_OPTION(single_privileged_thread)) {
            /* FIXME: want to suspend all other threads _before_ unprotecting anything,
             * but need to guarantee no new threads while we're suspending them,
             * and can't do that without setting a lock => need data segment!
             */
            d_r_mutex_lock(&thread_initexit_lock);

            if (d_r_get_num_threads() > 1) {
                thread_record_t *tr;
                int i;
                /* current multiple-thread solution: suspend all other threads! */
                ASSERT(protect_info->num_threads_suspended == 0);
                /* we do not need to grab the all_threads_lock because
                 * no threads can be added or removed so who cares if we
                 * access the data structure simultaneously with another
                 * reader of it
                 */
                for (i = 0; i < HASHTABLE_SIZE(ALL_THREADS_HASH_BITS); i++) {
                    for (tr = all_threads[i]; tr; tr = tr->next) {
                        if (tr->under_dynamo_control) {
                            DEBUG_DECLARE(bool ok =)
                            os_thread_suspend(all_threads[i]);
                            ASSERT(ok);
                            protect_info->num_threads_suspended++;
                        }
                    }
                }
            }
            /* we don't unlock or resume threads until we re-enter cache */
        }

        SELF_PROTECT_GLOBAL(WRITABLE);
    }
    /* we don't re-protect local heap here, that's done at points where
     * it was protected lazily
     */
    d_r_mutex_unlock(&protect_info->lock);
    LOG(GLOBAL, LOG_DISPATCH, 4, "dynamorio_unprotect thread=" TIDFMT "\n",
        d_r_get_thread_id());
}

#ifdef DEBUG
const char *
get_data_section_name(app_pc pc)
{
    uint i;
    for (i = 0; i < DATASEC_NUM; i++) {
        if (pc >= datasec_start[i] && pc < datasec_end[i])
            return DATASEC_NAMES[i];
    }
    return NULL;
}

bool
check_should_be_protected(uint sec)
{
    /* Blindly asserting that a data section is protected is racy as
     * another thread could be in an unprot window.  We use some
     * heuristics to try and identify bugs where a section is left
     * unprot, but it's not easy.
     */
    if (/* case 8107: for INJECT_LOCATION_LdrpLoadImportModule we
         * load a helper library and end up in d_r_dispatch() for
         * syscall_while_native before DR is initialized.
         */
        !dynamo_initialized ||
#    ifdef WINDOWS
        /* case 8113: detach currently unprots .data prior to its
         * thread synch, so don't count anything after that
         */
        doing_detach ||
#    endif
        !TEST(DATASEC_SELFPROT[sec], DYNAMO_OPTION(protect_mask)) ||
        DATASEC_PROTECTED(sec))
        return true;
    STATS_INC(datasec_not_prot);
    /* FIXME: even checking d_r_get_num_threads()==1 is still racy as a thread could
     * exit, and it's not worth grabbing thread_initexit_lock here..
     */
    if (threads_ever_count == 1
#    ifdef DR_APP_EXPORTS
        /* For start/stop, can be other threads running around so we bail on
         * perfect protection
         */
        && !dr_api_entry
#    endif
    )
        return false;
    /* FIXME: no count of threads in DR or anything so can't conclude much
     * Just return true and hope developer looks at datasec_not_prot stats.
     * We do have an ASSERT_CURIOSITY on the stat in data_section_exit().
     */
    return true;
}

#    ifdef WINDOWS
/* Assumed to only be called about DR dll writable regions */
bool
data_sections_enclose_region(app_pc start, app_pc end)
{
    /* Rather than solve the general enclose problem by sorting,
     * we subtract each piece we find.
     * It used to be that on 32-bit .data|.fspdata|.cspdata|.nspdata formed
     * the only writable region, with .pdata between .data and .fspdata on 64.
     * But building with VS2012, I'm seeing the sections in other orders (i#1075).
     * And with x64 reachability we moved the interception buffer in .data,
     * and marking it +rx results in sub-section calls to here.
     */
    int i;
    bool found_start = false, found_end = false;
    ssize_t sz = end - start;
    for (i = 0; i < DATASEC_NUM; i++) {
        if (datasec_start[i] <= end && datasec_end[i] >= start) {
            byte *overlap_start = MAX(datasec_start[i], start);
            byte *overlap_end = MIN(datasec_end[i], end);
            sz -= overlap_end - overlap_start;
        }
    }
    return sz == 0;
}
#    endif /* WINDOWS */
#endif     /* DEBUG */

static void
get_data_section_bounds(uint sec)
{
    /* FIXME: on linux we should include .got and .dynamic in one of our
     * sections, requiring specifying the order of sections (case 3789)!
     * Should use an ld script to ensure that .nspdata is last, or find a unique
     * attribute to force separation (perhaps mark as rwx, then
     * remove the x at init time?)  ld 2.15 puts it at the end, but
     * ld 2.13 puts .got and .dynamic after it!  For now we simply
     * don't protect subsequent guys.
     * On win32 there are no other rw sections, fortunately.
     */
    ASSERT(sec >= 0 && sec < DATASEC_NUM);
    /* for DEBUG we use for data_sections_enclose_region() */
    ASSERT(IF_WINDOWS(IF_DEBUG(true ||))
               TEST(DATASEC_SELFPROT[sec], dynamo_options.protect_mask));
    d_r_mutex_lock(&datasec_lock[sec]);
    ASSERT(datasec_start[sec] == NULL);
    get_named_section_bounds(get_dynamorio_dll_start(), DATASEC_NAMES[sec],
                             &datasec_start[sec], &datasec_end[sec]);
    d_r_mutex_unlock(&datasec_lock[sec]);
    ASSERT(ALIGNED(datasec_start[sec], PAGE_SIZE));
    ASSERT(ALIGNED(datasec_end[sec], PAGE_SIZE));
    ASSERT(datasec_start[sec] < datasec_end[sec]);
#ifdef WINDOWS
    if (IF_DEBUG(true ||) TEST(DATASEC_SELFPROT[sec], dynamo_options.protect_mask))
        merge_writecopy_pages(datasec_start[sec], datasec_end[sec]);
#endif
}

#ifdef UNIX
/* We get into problems if we keep a .section open across string literals, etc.
 * (such as when wrapping a function to get its local-scope statics in that section),
 * but the VAR_IN_SECTION does the real work for us, just so long as we have one
 * .section decl somewhere.
 */
DECLARE_DATA_SECTION(RARELY_PROTECTED_SECTION, "w")
DECLARE_DATA_SECTION(FREQ_PROTECTED_SECTION, "w")
DECLARE_DATA_SECTION(NEVER_PROTECTED_SECTION, "w")
END_DATA_SECTION_DECLARATIONS()
#endif

static void
data_section_init(void)
{
    uint i;
    for (i = 0; i < DATASEC_NUM; i++) {
        if (datasec_start[i] != NULL) {
            /* We were called early due to an early syslog.
             * We still retain our slightly later normal init position so we can
             * log, etc. in normal runs.
             */
            return;
        }
        ASSIGN_INIT_LOCK_FREE(datasec_lock[i], datasec_selfprot_lock);
        /* for DEBUG we use for data_sections_enclose_region() */
        if (IF_WINDOWS(IF_DEBUG(true ||))
                TEST(DATASEC_SELFPROT[i], dynamo_options.protect_mask)) {
            get_data_section_bounds(i);
        }
    }
    DOCHECK(1, {
        /* ensure no overlaps */
        uint j;
        for (i = 0; i < DATASEC_NUM; i++) {
            for (j = i + 1; j < DATASEC_NUM; j++) {
                ASSERT(datasec_start[i] >= datasec_end[j] ||
                       datasec_start[j] >= datasec_end[i]);
            }
        }
    });
}

static void
data_section_exit(void)
{
    uint i;
    DOSTATS({
        /* There can't have been that many races.
         * A failure to re-protect should result in a ton of d_r_dispatch
         * entrances w/ .data unprot, so should show up here.
         * However, an app with threads that are initializing in DR and thus
         * unprotected .data while other threads are running new code (such as
         * on attach) can easily rack up hundreds of unprot cache entrances.
         */
        ASSERT_CURIOSITY(GLOBAL_STAT(datasec_not_prot) < 5000);
    });
    for (i = 0; i < DATASEC_NUM; i++)
        DELETE_LOCK(datasec_lock[i]);
}

#define DATASEC_WRITABLE_MOD(which, op)                 \
    ((which) == DATASEC_RARELY_PROT                     \
         ? (datasec_writable_rareprot op)               \
         : ((which) == DATASEC_CXTSW_PROT               \
                ? (datasec_writable_cxtswprot op)       \
                : ((which) == DATASEC_FREQ_PROT         \
                       ? (datasec_writable_freqprot op) \
                       : (ASSERT_NOT_REACHED(), datasec_writable_neverprot))))

/* WARNING: any DO_ONCE will call this routine, so don't call anything here
 * that has a DO_ONCE, to avoid deadlock!
 */
void
protect_data_section(uint sec, bool writable)
{
    ASSERT(sec >= 0 && sec < DATASEC_NUM);
    ASSERT(TEST(DATASEC_SELFPROT[sec], dynamo_options.protect_mask));
    /* We can be called very early before data_section_init() so init here
     * (data_section_init() has no dependences).
     */
    if (datasec_start[sec] == NULL) {
        /* should only happen early in init */
        ASSERT(!dynamo_initialized);
        data_section_init();
    }
    d_r_mutex_lock(&datasec_lock[sec]);
    ASSERT(datasec_start[sec] != NULL);
    /* if using libc, we cannot print while data segment is read-only!
     * thus, if making it writable, do that first, otherwise do it last.
     * w/ ntdll this is not a problem.
     */
    /* Remember that multiple threads can be doing (unprotect,protect) pairs of
     * calls simultaneously.  The datasec_lock makes each individual call atomic,
     * and if all calls are properly nested, our use of counters should result in
     * the proper protection only after the final protect call and not in the
     * middle of some other thread's writes to the data section.
     */
    if (writable) {
        /* On-context-switch protection has a separate mechanism for
         * only protecting when the final thread leaves DR
         */
        ASSERT_CURIOSITY(DATASEC_WRITABLE(sec) <= 2); /* shouldn't nest too deep! */
        if (DATASEC_WRITABLE(sec) == 0) {
            make_writable(datasec_start[sec], datasec_end[sec] - datasec_start[sec]);
            STATS_INC(datasec_prot_changes);
        } else
            STATS_INC(datasec_prot_wasted_calls);
        (void)DATASEC_WRITABLE_MOD(sec, ++);
    }
    LOG(TEST(DATASEC_SELFPROT[sec], SELFPROT_ON_CXT_SWITCH) ? THREAD_GET : GLOBAL,
        LOG_VMAREAS, TEST(DATASEC_SELFPROT[sec], SELFPROT_ON_CXT_SWITCH) ? 3U : 2U,
        "protect_data_section: thread " TIDFMT " %s (recur %d, stat %d) %s %s %d\n",
        d_r_get_thread_id(), DATASEC_WRITABLE(sec) == 1 ? "changing" : "nop",
        DATASEC_WRITABLE(sec), GLOBAL_STAT(datasec_not_prot), DATASEC_NAMES[sec],
        writable ? "rw" : "r", DATASEC_WRITABLE(sec));
    if (!writable) {
        ASSERT(DATASEC_WRITABLE(sec) > 0);
        (void)DATASEC_WRITABLE_MOD(sec, --);
        if (DATASEC_WRITABLE(sec) == 0) {
            make_unwritable(datasec_start[sec], datasec_end[sec] - datasec_start[sec]);
            STATS_INC(datasec_prot_changes);
        } else
            STATS_INC(datasec_prot_wasted_calls);
    }
    d_r_mutex_unlock(&datasec_lock[sec]);
}

/* enter/exit DR hooks */
void
entering_dynamorio(void)
{
    if (SELF_PROTECT_ON_CXT_SWITCH)
        dynamorio_unprotect();
    ASSERT(HOOK_ENABLED);
    LOG(GLOBAL, LOG_DISPATCH, 3, "entering_dynamorio thread=" TIDFMT "\n",
        d_r_get_thread_id());
    STATS_INC(num_entering_DR);
    if (INTERNAL_OPTION(single_thread_in_DR)) {
        acquire_recursive_lock(&thread_in_DR_exclusion);
        LOG(GLOBAL, LOG_DISPATCH, 3, "entering_dynamorio thread=" TIDFMT " count=%d\n",
            d_r_get_thread_id(), thread_in_DR_exclusion.count);
    }
}

void
exiting_dynamorio(void)
{
    ASSERT(HOOK_ENABLED);
    LOG(GLOBAL, LOG_DISPATCH, 3, "exiting_dynamorio thread=" TIDFMT "\n",
        d_r_get_thread_id());
    STATS_INC(num_exiting_DR);
    if (INTERNAL_OPTION(single_thread_in_DR)) {
        /* thread init/exit can proceed now */
        LOG(GLOBAL, LOG_DISPATCH, 3, "exiting_dynamorio thread=" TIDFMT " count=%d\n",
            d_r_get_thread_id(), thread_in_DR_exclusion.count - 1);
        release_recursive_lock(&thread_in_DR_exclusion);
    }
    if (SELF_PROTECT_ON_CXT_SWITCH && !dynamo_exited)
        dynamorio_protect();
}

/* Note this includes any stack guard pages */
bool
is_on_initstack(byte *esp)
{
    return (esp <= d_r_initstack && esp > d_r_initstack - DYNAMORIO_STACK_SIZE);
}

/* Note this includes any stack guard pages */
bool
is_on_dstack(dcontext_t *dcontext, byte *esp)
{
    return (esp <= dcontext->dstack && esp > dcontext->dstack - DYNAMORIO_STACK_SIZE);
}

bool
is_currently_on_dstack(dcontext_t *dcontext)
{
    byte *cur_esp;
    GET_STACK_PTR(cur_esp);
    return is_on_dstack(dcontext, cur_esp);
}

void
pre_second_thread(void)
{
    /* i#1111: nop-out bb_building_lock until 2nd thread created.
     * While normally we'll call this in the primary thread while not holding
     * the lock, it's possible on Windows for an externally injected thread
     * (or for a thread sneakily created by some native_exec code w/o going
     * through ntdll wrappers) to appear.  We solve the problem of the main
     * thread currently holding bb_building_lock and us turning its
     * unlock into an error by the bb_lock_would_have bool in
     * SHARED_BB_UNLOCK().
     */
    if (!bb_lock_start) {
        d_r_mutex_lock(&bb_building_lock);
        SELF_UNPROTECT_DATASEC(DATASEC_RARELY_PROT);
        bb_lock_start = true;
        SELF_PROTECT_DATASEC(DATASEC_RARELY_PROT);
        d_r_mutex_unlock(&bb_building_lock);
    }
}

> Those scenarios would need their own optimizer-specific solution. Not for this PR but a future one -- We should add that to the docs: though we never made particular docs for static-library DR so we need to do that first. Maybe we should file an issue on doing that.