text: /*
   * %CopyrightBegin%
   *
   * Copyright Ericsson AB 1996-2018. All Rights Reserved.
   *
   * Licensed under the Apache License, Version 2.0 (the "License");
   * you may not use this file except in compliance with the License.
   * You may obtain a copy of the License at
   *
   *     http://www.apache.org/licenses/LICENSE-2.0
   *
   * Unless required by applicable law or agreed to in writing, software
   * distributed under the License is distributed on an "AS IS" BASIS,
   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   * See the License for the specific language governing permissions and
   * limitations under the License.
   *
   * %CopyrightEnd%
   */
  
  #ifndef __PROCESS_H__
  #define __PROCESS_H__
  
  #include "sys.h"
  
  #undef ERTS_INCLUDE_SCHEDULER_INTERNALS
  #if (defined(ERL_PROCESS_C__) \
       || defined(ERL_PORT_TASK_C__) \
       || (ERTS_GLB_INLINE_INCL_FUNC_DEF \
  	 && defined(ERTS_DO_INCL_GLB_INLINE_FUNC_DEF)))
  #define ERTS_INCLUDE_SCHEDULER_INTERNALS
  #endif
  
  /* #define ERTS_DO_VERIFY_UNUSED_TEMP_ALLOC */
  
  #if !defined(ERTS_DO_VERIFY_UNUSED_TEMP_ALLOC) && defined(DEBUG)
  #  define ERTS_DO_VERIFY_UNUSED_TEMP_ALLOC
  #endif
  
  typedef struct process Process;
  
  #define ERTS_PROCESS_LOCK_ONLY_PROC_LOCK_TYPE__
  #include "erl_process_lock.h" /* Only pull out important types... */
  #undef ERTS_PROCESS_LOCK_ONLY_PROC_LOCK_TYPE__
  
  #define ERL_PORT_GET_PORT_TYPE_ONLY__
  #include "erl_port.h"
  #undef ERL_PORT_GET_PORT_TYPE_ONLY__
  #include "erl_vm.h"
  #include "erl_message.h"
  #include "erl_process_dict.h"
  #include "erl_node_container_utils.h"
  #include "erl_node_tables.h"
  #include "erl_monitor_link.h"
  #include "erl_hl_timer.h"
  #include "erl_time.h"
  #include "erl_atom_table.h"
  #include "external.h"
  #include "erl_mseg.h"
  #include "erl_async.h"
  #include "erl_gc.h"
  #define ERTS_ONLY_INCLUDE_TRACE_FLAGS
  #include "erl_trace.h"
  #undef ERTS_ONLY_INCLUDE_TRACE_FLAGS
  #define ERTS_ONLY_SCHED_SPEC_ETS_DATA
  #include "erl_db.h"
  #undef ERTS_ONLY_SCHED_SPEC_ETS_DATA
  
  #ifdef HIPE
  #include "hipe_process.h"
  #endif
  
  #undef ERL_THR_PROGRESS_TSD_TYPE_ONLY
  #define ERL_THR_PROGRESS_TSD_TYPE_ONLY
  #include "erl_thr_progress.h"
  #undef ERL_THR_PROGRESS_TSD_TYPE_ONLY
  
  #define ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT_OPT	0
  #define ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT		0
  
  #define ERTS_MAX_NO_OF_SCHEDULERS 1024
  #define ERTS_MAX_NO_OF_DIRTY_CPU_SCHEDULERS ERTS_MAX_NO_OF_SCHEDULERS
  #define ERTS_MAX_NO_OF_DIRTY_IO_SCHEDULERS ERTS_MAX_NO_OF_SCHEDULERS
  
  #define ERTS_DEFAULT_MAX_PROCESSES (1 << 18)
  
  #define ERTS_HEAP_ALLOC(Type, Size)					\
       erts_alloc((Type), (Size))
  
  #define ERTS_HEAP_REALLOC(Type, Ptr, OldSize, NewSize)			\
       erts_realloc((Type), (Ptr), (NewSize))
  
  #define ERTS_HEAP_FREE(Type, Ptr, Size)					\
       erts_free((Type), (Ptr))
  
  #include "export.h"
  
  struct saved_calls {
     int len;
     int n;
     int cur;
     Export *ct[1];
  };
  
  extern Export exp_send, exp_receive, exp_timeout;
  extern int ERTS_WRITE_UNLIKELY(erts_sched_compact_load);
  extern int ERTS_WRITE_UNLIKELY(erts_sched_balance_util);
  extern Uint ERTS_WRITE_UNLIKELY(erts_no_schedulers);
  extern Uint ERTS_WRITE_UNLIKELY(erts_no_total_schedulers);
  extern Uint ERTS_WRITE_UNLIKELY(erts_no_dirty_cpu_schedulers);
  extern Uint ERTS_WRITE_UNLIKELY(erts_no_dirty_io_schedulers);
  extern Uint ERTS_WRITE_UNLIKELY(erts_no_run_queues);
  extern int erts_sched_thread_suggested_stack_size;
  extern int erts_dcpu_sched_thread_suggested_stack_size;
  extern int erts_dio_sched_thread_suggested_stack_size;
  #define ERTS_SCHED_THREAD_MIN_STACK_SIZE 20	/* Kilo words */
  #define ERTS_SCHED_THREAD_MAX_STACK_SIZE 8192	/* Kilo words */
  
  #include "erl_bits.h"
  
  /* process priorities */
  #define PRIORITY_MAX          0
  #define PRIORITY_HIGH         1
  #define PRIORITY_NORMAL       2
  #define PRIORITY_LOW          3
  #define ERTS_NO_PROC_PRIO_LEVELS      4
  #define ERTS_NO_PROC_PRIO_QUEUES      3
  
  #define ERTS_PORT_PRIO_LEVEL ERTS_NO_PROC_PRIO_LEVELS
  #define ERTS_NO_PRIO_LEVELS (ERTS_NO_PROC_PRIO_LEVELS + 1)
  
  #define ERTS_RUNQ_FLGS_PROCS_QMASK \
    ((((Uint32) 1) << ERTS_NO_PROC_PRIO_LEVELS) - 1)
  
  #define ERTS_RUNQ_FLGS_QMASK \
    ((((Uint32) 1) << ERTS_NO_PRIO_LEVELS) - 1)
  
  #define ERTS_RUNQ_FLGS_EMIGRATE_SHFT \
    ERTS_NO_PRIO_LEVELS
  #define ERTS_RUNQ_FLGS_IMMIGRATE_SHFT \
    (ERTS_RUNQ_FLGS_EMIGRATE_SHFT + ERTS_NO_PRIO_LEVELS)
  #define ERTS_RUNQ_FLGS_EVACUATE_SHFT \
    (ERTS_RUNQ_FLGS_IMMIGRATE_SHFT + ERTS_NO_PRIO_LEVELS)
  #define ERTS_RUNQ_FLGS_EMIGRATE_QMASK \
    (ERTS_RUNQ_FLGS_QMASK << ERTS_RUNQ_FLGS_EMIGRATE_SHFT)
  #define ERTS_RUNQ_FLGS_IMMIGRATE_QMASK \
    (ERTS_RUNQ_FLGS_QMASK << ERTS_RUNQ_FLGS_IMMIGRATE_SHFT)
  #define ERTS_RUNQ_FLGS_EVACUATE_QMASK \
    (ERTS_RUNQ_FLGS_QMASK << ERTS_RUNQ_FLGS_EVACUATE_SHFT)
  
  #define ERTS_RUNQ_FLG_BASE2 \
    (ERTS_RUNQ_FLGS_EVACUATE_SHFT + ERTS_NO_PRIO_LEVELS)
  
  #define ERTS_RUNQ_FLG_OUT_OF_WORK \
    (((Uint32) 1) << (ERTS_RUNQ_FLG_BASE2 + 0))
  #define ERTS_RUNQ_FLG_HALFTIME_OUT_OF_WORK \
    (((Uint32) 1) << (ERTS_RUNQ_FLG_BASE2 + 1))
  #define ERTS_RUNQ_FLG_SUSPENDED \
    (((Uint32) 1) << (ERTS_RUNQ_FLG_BASE2 + 2))
  #define ERTS_RUNQ_FLG_CHK_CPU_BIND \
    (((Uint32) 1) << (ERTS_RUNQ_FLG_BASE2 + 3))
  #define ERTS_RUNQ_FLG_INACTIVE \
    (((Uint32) 1) << (ERTS_RUNQ_FLG_BASE2 + 4))
  #define ERTS_RUNQ_FLG_NONEMPTY \
    (((Uint32) 1) << (ERTS_RUNQ_FLG_BASE2 + 5))
  #define ERTS_RUNQ_FLG_PROTECTED \
    (((Uint32) 1) << (ERTS_RUNQ_FLG_BASE2 + 6))
  #define ERTS_RUNQ_FLG_EXEC \
    (((Uint32) 1) << (ERTS_RUNQ_FLG_BASE2 + 7))
  #define ERTS_RUNQ_FLG_MSB_EXEC \
    (((Uint32) 1) << (ERTS_RUNQ_FLG_BASE2 + 8))
  #define ERTS_RUNQ_FLG_MISC_OP \
    (((Uint32) 1) << (ERTS_RUNQ_FLG_BASE2 + 9))
  #define ERTS_RUNQ_FLG_HALTING \
    (((Uint32) 1) << (ERTS_RUNQ_FLG_BASE2 + 10))
  
  #define ERTS_RUNQ_FLG_MAX (ERTS_RUNQ_FLG_BASE2 + 12)
  
  #define ERTS_RUNQ_FLGS_MIGRATION_QMASKS	\
    (ERTS_RUNQ_FLGS_EMIGRATE_QMASK	\
     | ERTS_RUNQ_FLGS_IMMIGRATE_QMASK	\
     | ERTS_RUNQ_FLGS_EVACUATE_QMASK)
  
  #define ERTS_RUNQ_FLGS_MIGRATION_INFO \
    (ERTS_RUNQ_FLG_INACTIVE \
     | ERTS_RUNQ_FLG_OUT_OF_WORK \
     | ERTS_RUNQ_FLG_HALFTIME_OUT_OF_WORK)
  
  #define ERTS_RUNQ_FLG_EMIGRATE(PRIO) \
    (((Uint32) 1) << (ERTS_RUNQ_FLGS_EMIGRATE_SHFT + (PRIO)))
  #define ERTS_CHK_RUNQ_FLG_EMIGRATE(FLGS, PRIO) \
    ((FLGS) & ERTS_RUNQ_FLG_EMIGRATE((PRIO)))
  #define ERTS_SET_RUNQ_FLG_EMIGRATE(FLGS, PRIO) \
    ((FLGS) |= ERTS_RUNQ_FLG_EMIGRATE((PRIO)))
  #define ERTS_UNSET_RUNQ_FLG_EMIGRATE(FLGS, PRIO) \
    ((FLGS) &= ~ERTS_RUNQ_FLG_EMIGRATE((PRIO)))
  
  #define ERTS_RUNQ_FLG_IMMIGRATE(PRIO) \
    (((Uint32) 1) << (ERTS_RUNQ_FLGS_IMMIGRATE_SHFT + (PRIO)))
  #define ERTS_CHK_RUNQ_FLG_IMMIGRATE(FLGS, PRIO) \
    ((FLGS) & ERTS_RUNQ_FLG_IMMIGRATE((PRIO)))
  #define ERTS_SET_RUNQ_FLG_IMMIGRATE(FLGS, PRIO) \
    ((FLGS) |= ERTS_RUNQ_FLG_IMMIGRATE((PRIO)))
  #define ERTS_UNSET_RUNQ_FLG_IMMIGRATE(FLGS, PRIO) \
    ((FLGS) &= ~ERTS_RUNQ_FLG_IMMIGRATE((PRIO)))
  
  #define ERTS_RUNQ_FLG_EVACUATE(PRIO) \
    (((Uint32) 1) << (ERTS_RUNQ_FLGS_EVACUATE_SHFT + (PRIO)))
  #define ERTS_CHK_RUNQ_FLG_EVACUATE(FLGS, PRIO) \
    ((FLGS) & ERTS_RUNQ_FLG_EVACUATE((PRIO)))
  #define ERTS_SET_RUNQ_FLG_EVACUATE(FLGS, PRIO) \
    ((FLGS) |= ERTS_RUNQ_FLG_EVACUATE((PRIO)))
  #define ERTS_UNSET_RUNQ_FLG_EVACUATE(FLGS, PRIO) \
    ((FLGS) &= ~ERTS_RUNQ_FLG_EVACUATE((PRIO)))
  
  #define ERTS_RUNQ_FLGS_INIT(RQ, INIT)					\
      erts_atomic32_init_nob(&(RQ)->flags, (erts_aint32_t) (INIT))
  #define ERTS_RUNQ_FLGS_SET(RQ, FLGS)					\
      ((Uint32) erts_atomic32_read_bor_relb(&(RQ)->flags,		\
  					      (erts_aint32_t) (FLGS)))
  #define ERTS_RUNQ_FLGS_SET_NOB(RQ, FLGS)				\
      ((Uint32) erts_atomic32_read_bor_nob(&(RQ)->flags,		\
  					     (erts_aint32_t) (FLGS)))
  #define ERTS_RUNQ_FLGS_BSET(RQ, MSK, FLGS)				\
      ((Uint32) erts_atomic32_read_bset_relb(&(RQ)->flags,		\
  					       (erts_aint32_t) (MSK),	\
  					       (erts_aint32_t) (FLGS)))
  #define ERTS_RUNQ_FLGS_UNSET(RQ, FLGS)					\
      ((Uint32) erts_atomic32_read_band_relb(&(RQ)->flags,		\
  					       (erts_aint32_t) ~(FLGS)))
  #define ERTS_RUNQ_FLGS_UNSET_NOB(RQ, FLGS)					\
      ((Uint32) erts_atomic32_read_band_nob(&(RQ)->flags,		\
  					      (erts_aint32_t) ~(FLGS)))
  #define ERTS_RUNQ_FLGS_GET(RQ)						\
      ((Uint32) erts_atomic32_read_acqb(&(RQ)->flags))
  #define ERTS_RUNQ_FLGS_GET_NOB(RQ)					\
      ((Uint32) erts_atomic32_read_nob(&(RQ)->flags))
  #define ERTS_RUNQ_FLGS_GET_MB(RQ)					\
      ((Uint32) erts_atomic32_read_mb(&(RQ)->flags))
  #define ERTS_RUNQ_FLGS_READ_BSET(RQ, MSK, FLGS)		  		\
      ((Uint32) erts_atomic32_read_bset_relb(&(RQ)->flags, 		\
  					       (erts_aint32_t) (MSK),	\
  					       (erts_aint32_t) (FLGS)))
  
  #define ERTS_RUNQ_POINTER_MASK  (~((erts_aint_t) 3))
  #define ERTS_RUNQ_BOUND_FLAG    ((erts_aint_t) 1)
  
  typedef enum {
      ERTS_SCHDLR_SSPND_DONE_MSCHED_BLOCKED,
      ERTS_SCHDLR_SSPND_DONE_NMSCHED_BLOCKED,
      ERTS_SCHDLR_SSPND_YIELD_DONE_MSCHED_BLOCKED,
      ERTS_SCHDLR_SSPND_YIELD_DONE_NMSCHED_BLOCKED,
      ERTS_SCHDLR_SSPND_DONE,
      ERTS_SCHDLR_SSPND_YIELD_RESTART,
      ERTS_SCHDLR_SSPND_YIELD_DONE,
      ERTS_SCHDLR_SSPND_EINVAL
  } ErtsSchedSuspendResult;
  
  typedef enum {
      ERTS_MIGRATE_SUCCESS,
      ERTS_MIGRATE_FAILED_NOT_IN_RUNQ,
      ERTS_MIGRATE_FAILED_RUNQ_CHANGED,
      ERTS_MIGRATE_FAILED_RUNQ_SUSPENDED
  } ErtsMigrateResult;
  
  #define ERTS_SSI_FLG_SLEEPING		(((erts_aint32_t) 1) << 0)
  #define ERTS_SSI_FLG_POLL_SLEEPING 	(((erts_aint32_t) 1) << 1)
  #define ERTS_SSI_FLG_TSE_SLEEPING 	(((erts_aint32_t) 1) << 2)
  #define ERTS_SSI_FLG_WAITING		(((erts_aint32_t) 1) << 3)
  #define ERTS_SSI_FLG_SUSPENDED	 	(((erts_aint32_t) 1) << 4)
  #define ERTS_SSI_FLG_MSB_EXEC	 	(((erts_aint32_t) 1) << 5)
  
  #define ERTS_SSI_FLGS_MAX                                       6
  
  #define ERTS_SSI_FLGS_SLEEP_TYPE			\
   (ERTS_SSI_FLG_TSE_SLEEPING|ERTS_SSI_FLG_POLL_SLEEPING)
  
  #define ERTS_SSI_FLGS_SLEEP				\
   (ERTS_SSI_FLG_SLEEPING|ERTS_SSI_FLGS_SLEEP_TYPE)
  
  #define ERTS_SSI_FLGS_ALL				\
   (ERTS_SSI_FLGS_SLEEP					\
    | ERTS_SSI_FLG_WAITING				\
    | ERTS_SSI_FLG_SUSPENDED                              \
    | ERTS_SSI_FLG_MSB_EXEC)
  
  /*
   * Keep ERTS_SSI_AUX_WORK flags ordered in expected frequency
   * order relative eachother. Most frequent at lowest at lowest
   * index.
   *
   * ERTS_SSI_AUX_WORK_DEBUG_WAIT_COMPLETED_IX *need* to be
   * highest index...
   *
   * Remember to update description in erts_pre_init_process()
   * and etp-commands when adding new flags...
   */
  
  typedef enum {
      ERTS_SSI_AUX_WORK_DELAYED_AW_WAKEUP_IX,
      ERTS_SSI_AUX_WORK_DD_IX,
      ERTS_SSI_AUX_WORK_DD_THR_PRGR_IX,
      ERTS_SSI_AUX_WORK_FIX_ALLOC_DEALLOC_IX,
      ERTS_SSI_AUX_WORK_FIX_ALLOC_LOWER_LIM_IX,
      ERTS_SSI_AUX_WORK_THR_PRGR_LATER_OP_IX,
      ERTS_SSI_AUX_WORK_CNCLD_TMRS_IX,
      ERTS_SSI_AUX_WORK_CNCLD_TMRS_THR_PRGR_IX,
      ERTS_SSI_AUX_WORK_ASYNC_READY_IX,
      ERTS_SSI_AUX_WORK_ASYNC_READY_CLEAN_IX,
      ERTS_SSI_AUX_WORK_MISC_THR_PRGR_IX,
      ERTS_SSI_AUX_WORK_MISC_IX,
      ERTS_SSI_AUX_WORK_SET_TMO_IX,
      ERTS_SSI_AUX_WORK_MSEG_CACHE_CHECK_IX,
      ERTS_SSI_AUX_WORK_YIELD_IX,
      ERTS_SSI_AUX_WORK_REAP_PORTS_IX,
      ERTS_SSI_AUX_WORK_DEBUG_WAIT_COMPLETED_IX, /* SHOULD be last flag index */
  
      ERTS_SSI_AUX_WORK_NO_FLAGS /* Not a flag index... */
  } ErtsSsiAuxWorkFlagIndex;
  
  #define ERTS_SSI_AUX_WORK_DELAYED_AW_WAKEUP \
      (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_DELAYED_AW_WAKEUP_IX)
  #define ERTS_SSI_AUX_WORK_DD \
      (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_DD_IX)
  #define ERTS_SSI_AUX_WORK_DD_THR_PRGR \
      (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_DD_THR_PRGR_IX)
  #define ERTS_SSI_AUX_WORK_FIX_ALLOC_DEALLOC \
      (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_FIX_ALLOC_DEALLOC_IX)
  #define ERTS_SSI_AUX_WORK_FIX_ALLOC_LOWER_LIM \
      (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_FIX_ALLOC_LOWER_LIM_IX)
  #define ERTS_SSI_AUX_WORK_THR_PRGR_LATER_OP \
      (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_THR_PRGR_LATER_OP_IX)
  #define ERTS_SSI_AUX_WORK_CNCLD_TMRS \
      (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_CNCLD_TMRS_IX)
  #define ERTS_SSI_AUX_WORK_CNCLD_TMRS_THR_PRGR \
      (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_CNCLD_TMRS_THR_PRGR_IX)
  #define ERTS_SSI_AUX_WORK_ASYNC_READY \
      (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_ASYNC_READY_IX)
  #define ERTS_SSI_AUX_WORK_ASYNC_READY_CLEAN \
      (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_ASYNC_READY_CLEAN_IX)
  #define ERTS_SSI_AUX_WORK_MISC_THR_PRGR \
      (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_MISC_THR_PRGR_IX)
  #define ERTS_SSI_AUX_WORK_MISC \
      (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_MISC_IX)
  #define ERTS_SSI_AUX_WORK_SET_TMO \
      (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_SET_TMO_IX)
  #define ERTS_SSI_AUX_WORK_MSEG_CACHE_CHECK \
      (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_MSEG_CACHE_CHECK_IX)
  #define ERTS_SSI_AUX_WORK_YIELD \
      (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_YIELD_IX)
  #define ERTS_SSI_AUX_WORK_REAP_PORTS \
      (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_REAP_PORTS_IX)
  #define ERTS_SSI_AUX_WORK_DEBUG_WAIT_COMPLETED \
      (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_DEBUG_WAIT_COMPLETED_IX)
  
  typedef struct ErtsSchedulerSleepInfo_ ErtsSchedulerSleepInfo;
  
  typedef struct {
      erts_spinlock_t lock;
      ErtsSchedulerSleepInfo *list; /* circular lifo list; points to last out */
  } ErtsSchedulerSleepList;
  
  struct ErtsSchedulerSleepInfo_ {
      struct ErtsSchedulerData_ *esdp;
      ErtsSchedulerSleepInfo *next;
      ErtsSchedulerSleepInfo *prev;
      erts_atomic32_t flags;
      erts_tse_t *event;
      struct erts_poll_thread *psi;
      erts_atomic32_t aux_work;
  };
  
  /* times to reschedule low prio process before running */
  #define RESCHEDULE_LOW        8
  
  #define ERTS_MAX_MISC_OPS 5
  
  #define ERTS_FULL_REDS_HISTORY_AVG_SHFT 3
  #define ERTS_FULL_REDS_HISTORY_SIZE \
     ((1 << ERTS_FULL_REDS_HISTORY_AVG_SHFT) - 1)
  
  typedef struct ErtsProcList_ ErtsProcList;
  struct ErtsProcList_ {
      union {
          Eterm pid;
          Process *p;
      } u;
      Uint64 started_interval;
      ErtsProcList* next;
      ErtsProcList* prev;
  };
  
  typedef struct ErtsMiscOpList_ ErtsMiscOpList;
  struct ErtsMiscOpList_ {
      ErtsMiscOpList *next;
      void (*func)(void *arg);
      void *arg;
  };
  
  typedef struct {
      Process* first;
      Process* last;
  } ErtsRunPrioQueue;
  
  typedef enum {
      ERTS_SCHED_NORMAL = 0,
      ERTS_SCHED_DIRTY_CPU = 1,
      ERTS_SCHED_DIRTY_IO = 2,
  
      ERTS_SCHED_TYPE_FIRST = ERTS_SCHED_NORMAL,
      ERTS_SCHED_TYPE_LAST = ERTS_SCHED_DIRTY_IO
  } ErtsSchedType;
  
  typedef struct ErtsSchedulerData_ ErtsSchedulerData;
  
  typedef struct ErtsRunQueue_ ErtsRunQueue;
  
  typedef struct {
      erts_atomic32_t len;
      erts_aint32_t max_len;
      int reds;
  } ErtsRunQueueInfo;
  
  
  #ifdef ERTS_HAVE_OS_MONOTONIC_TIME_SUPPORT
  #  undef ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT_OPT
  #  define ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT_OPT 1
  #endif
  
  
  #undef ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT
  #define ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT_OPT
  
  typedef erts_atomic64_t ErtsAtomicSchedTime;
  
  #if ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT
  typedef struct {
      ErtsAtomicSchedTime last;
      struct {
  	Uint64 short_interval;
  	Uint64 long_interval;
      } worktime;
      int is_working;
  } ErtsRunQueueSchedUtil;
  #endif
  
  typedef struct {
  #if ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT
      int sched_util;
  #endif
      Uint32 flags;
      ErtsRunQueue *misc_evac_runq;
      struct {
  	struct {
  	    int this;
  	    int other;
  	} limit;
  	ErtsRunQueue *runq;
  	Uint32 flags;
      } prio[ERTS_NO_PRIO_LEVELS];
  } ErtsMigrationPath;
  
  typedef struct ErtsMigrationPaths_ ErtsMigrationPaths;
  
  struct ErtsMigrationPaths_ {
      void *block;
      ErtsMigrationPaths *next;
      ErtsThrPrgrVal thr_prgr;
      ErtsMigrationPath mpath[1];
  };
  
  
  struct ErtsRunQueue_ {
      int ix;
  
      erts_mtx_t mtx;
      erts_cnd_t cnd;
  
      ErtsSchedulerSleepList sleepers;
  
      ErtsSchedulerData *scheduler;
      int waiting; /* < 0 in sys schedule; > 0 on cnd variable */
      int woken;
      erts_atomic32_t flags;
      int check_balance_reds;
      int full_reds_history_sum;
      int full_reds_history[ERTS_FULL_REDS_HISTORY_SIZE];
      int out_of_work_count;
      erts_aint32_t max_len;
      erts_atomic32_t len;
      int wakeup_other;
      int wakeup_other_reds;
  
      struct {
  	Uint context_switches;
  	Uint reductions;
  
  	ErtsRunQueueInfo prio_info[ERTS_NO_PROC_PRIO_LEVELS];
  
  	/* We use the same prio queue for low and
  	   normal prio processes */
  	ErtsRunPrioQueue prio[ERTS_NO_PROC_PRIO_LEVELS-1];
      } procs;
  
      struct {
  	ErtsMiscOpList *start;
  	ErtsMiscOpList *end;
  	erts_atomic_t evac_runq;
      } misc;
  
      struct {
  	ErtsRunQueueInfo info;
  	Port *start;
  	Port *end;
      } ports;
  #if ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT
      ErtsRunQueueSchedUtil sched_util;
  #endif
  };
  
  extern long erts_runq_supervision_interval;
  
  typedef union {
      ErtsRunQueue runq;
      char align[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(ErtsRunQueue))];
  } ErtsAlignedRunQueue;
  
  extern ErtsAlignedRunQueue * ERTS_WRITE_UNLIKELY(erts_aligned_run_queues);
  
  #define ERTS_PROC_REDUCTIONS_EXECUTED(SD, RQ, PRIO, REDS, AREDS)\
  do {								\
      (RQ)->procs.reductions += (AREDS);				\
      (RQ)->procs.prio_info[(PRIO)].reds += (REDS);		\
      (RQ)->check_balance_reds -= (REDS);				\
      (RQ)->wakeup_other_reds += (AREDS);				\
      (SD)->check_time_reds += (AREDS);				\
  } while (0)
  
  #define ERTS_PORT_REDUCTIONS_EXECUTED(SD, RQ, REDS)		\
  do {								\
      (RQ)->ports.info.reds += (REDS);				\
      (RQ)->check_balance_reds -= (REDS);				\
      (RQ)->wakeup_other_reds += (REDS);				\
      (SD)->check_time_reds += (REDS);				\
  } while (0)
  
  typedef struct {
      union {
          erts_atomic32_t mod; /* on dirty schedulers */
          int need; /* "+sbu true" or scheduler_wall_time enabled */
      } u;
      int enabled;
      Uint64 start;
      struct {
  	Uint64 total;
  	Uint64 start;
      } working;
  } ErtsSchedWallTime;
  
  typedef struct {
      int sched;
      erts_aint32_t aux_work;
  } ErtsDelayedAuxWorkWakeupJob;
  
  typedef struct {
      int sched_id;
      ErtsSchedulerData *esdp;
      ErtsSchedulerSleepInfo *ssi;
      ErtsThrPrgrVal current_thr_prgr;
      ErtsThrPrgrVal latest_wakeup;
      struct {
  	int ix;
  	ErtsThrPrgrVal thr_prgr;
      } misc;
      struct {
  	ErtsThrPrgrVal thr_prgr;
      } dd;
      struct {
  	ErtsThrPrgrVal thr_prgr;
      } cncld_tmrs;
      struct {
  	ErtsThrPrgrVal thr_prgr;
  	UWord size;
  	ErtsThrPrgrLaterOp *first;
  	ErtsThrPrgrLaterOp *last;
      } later_op;
      struct {
  	int need_thr_prgr;
  	ErtsThrPrgrVal thr_prgr;
  	void *queue;
      } async_ready;
      struct {
  	Uint64 next;
  	int *sched2jix;
  	int jix;
  	ErtsDelayedAuxWorkWakeupJob *job;
      } delayed_wakeup;
      struct {
          ErtsAlcuBlockscanYieldData alcu_blockscan;
          ErtsEtsAllYieldData ets_all;
          /* Other yielding operations... */
      } yield;
      struct {
  	struct {
  	    erts_aint32_t flags;
  	    void (*callback)(void *);
  	    void *arg;
  	} wait_completed;
      } debug;
  } ErtsAuxWorkData;
  
  #define ERTS_SCHED_AUX_YIELD_DATA(ESDP, NAME) \
      (&(ESDP)->aux_work_data.yield.NAME)
  void erts_notify_new_aux_yield_work(ErtsSchedulerData *esdp);
  
  typedef enum {
      ERTS_DIRTY_CPU_SCHEDULER,
      ERTS_DIRTY_IO_SCHEDULER
  } ErtsDirtySchedulerType;
  
  
  struct ErtsSchedulerData_ {
      /*
       * Keep X registers first (so we get as many low
       * numbered registers as possible in the same cache
       * line).
       */
      Eterm* x_reg_array;		/* X registers */
      FloatDef* f_reg_array;	/* Floating point registers. */
  
      ErtsTimerWheel *timer_wheel;
      ErtsNextTimeoutRef next_tmo_ref;
      ErtsHLTimerService *timer_service;
      ethr_tid tid;		/* Thread id */
      struct erl_bits_state erl_bits_state; /* erl_bits.c state */
      void *match_pseudo_process; /* erl_db_util.c:db_prog_match() */
      Process *free_process;
      ErtsThrPrgrData thr_progress_data;
      ErtsSchedulerSleepInfo *ssi;
      Process *current_process;
      ErtsSchedType type;
      Uint no;			/* Scheduler number for normal schedulers */
      Uint dirty_no;  /* Scheduler number for dirty schedulers */
      int flxctr_slot_no; /* slot nr when a flxctr is used */
      struct enif_environment_t *current_nif;
      Process *dirty_shadow_process;
      Port *current_port;
      ErtsRunQueue *run_queue;
      int virtual_reds;
      int cpu_id;			/* >= 0 when bound */
      ErtsAuxWorkData aux_work_data;
      ErtsAtomCacheMap atom_cache_map;
  
      ErtsMonotonicTime last_monotonic_time;
      int check_time_reds;
  
      Uint32 thr_id;
      Uint64 unique;
      Uint64 ref;
  
      ErtsSchedAllocData alloc_data;
  
      struct {
  	Uint64 out;
  	Uint64 in;
      } io;
      struct {
          ErtsSignal* sig;
          Eterm to;
  #ifdef DEBUG
  	Process* dbg_from;
  #endif
      } pending_signal;
  
      Uint64 reductions;
      ErtsSchedWallTime sched_wall_time;
      ErtsGCInfo gc_info;
      ErtsPortTaskHandle nosuspend_port_task_handle;
      ErtsEtsTables ets_tables;
  #ifdef ERTS_DO_VERIFY_UNUSED_TEMP_ALLOC
      erts_alloc_verify_func_t verify_unused_temp_alloc;
      Allctr_t *verify_unused_temp_alloc_data;
  #endif
  };
  
  typedef union {
      ErtsSchedulerData esd;
      char align[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(ErtsSchedulerData))];
  } ErtsAlignedSchedulerData;
  
  extern ErtsAlignedSchedulerData * ERTS_WRITE_UNLIKELY(erts_aligned_scheduler_data);
  extern ErtsAlignedSchedulerData * ERTS_WRITE_UNLIKELY(erts_aligned_dirty_cpu_scheduler_data);
  extern ErtsAlignedSchedulerData * ERTS_WRITE_UNLIKELY(erts_aligned_dirty_io_scheduler_data);
  
  
  #if defined(ERTS_ENABLE_LOCK_CHECK)
  int erts_lc_runq_is_locked(ErtsRunQueue *);
  #endif
  
  void
  erts_debug_later_op_foreach(void (*callback)(void*),
                              void (*func)(void *, ErtsThrPrgrVal, void *),
                              void *arg);
  
  #ifdef ERTS_INCLUDE_SCHEDULER_INTERNALS
  
  void erts_empty_runq(ErtsRunQueue *rq);
  void erts_non_empty_runq(ErtsRunQueue *rq);
  
  
  /*
   * Run queue locked during modifications. We use atomic ops since
   * other threads peek at values without run queue lock.
   */
  
  ERTS_GLB_INLINE void erts_inc_runq_len(ErtsRunQueue *rq, ErtsRunQueueInfo *rqi, int prio);
  ERTS_GLB_INLINE void erts_dec_runq_len(ErtsRunQueue *rq, ErtsRunQueueInfo *rqi, int prio);
  ERTS_GLB_INLINE void erts_reset_max_len(ErtsRunQueue *rq, ErtsRunQueueInfo *rqi);
  
  #if ERTS_GLB_INLINE_INCL_FUNC_DEF
  
  ERTS_GLB_INLINE void
  erts_inc_runq_len(ErtsRunQueue *rq, ErtsRunQueueInfo *rqi, int prio)
  {
      erts_aint32_t len;
  
      ERTS_LC_ASSERT(erts_lc_runq_is_locked(rq));
  
      len = erts_atomic32_read_dirty(&rq->len);
  
      if (len == 0)
  	erts_non_empty_runq(rq);
      len++;
      if (rq->max_len < len)
  	rq->max_len = len;
      ASSERT(len > 0);
      erts_atomic32_set_nob(&rq->len, len);
  
      len = erts_atomic32_read_dirty(&rqi->len);
      ASSERT(len >= 0);
      if (len == 0) {
  	ASSERT((erts_atomic32_read_nob(&rq->flags)
  		& ((erts_aint32_t) (1 << prio))) == 0);
  	erts_atomic32_read_bor_nob(&rq->flags,
  				       (erts_aint32_t) (1 << prio));
      }
      len++;
      if (rqi->max_len < len)
  	rqi->max_len = len;
  
      erts_atomic32_set_relb(&rqi->len, len);
  }
  
  ERTS_GLB_INLINE void
  erts_dec_runq_len(ErtsRunQueue *rq, ErtsRunQueueInfo *rqi, int prio)
  {
      erts_aint32_t len;
  
      ERTS_LC_ASSERT(erts_lc_runq_is_locked(rq));
  
      len = erts_atomic32_read_dirty(&rq->len);
      len--;
      ASSERT(len >= 0);
      erts_atomic32_set_nob(&rq->len, len);
  
      len = erts_atomic32_read_dirty(&rqi->len);
      len--;
      ASSERT(len >= 0);
      if (len == 0) {
  	ASSERT((erts_atomic32_read_nob(&rq->flags)
  		& ((erts_aint32_t) (1 << prio))));
  	erts_atomic32_read_band_nob(&rq->flags,
  					~((erts_aint32_t) (1 << prio)));
      }
      erts_atomic32_set_relb(&rqi->len, len);
  
  }
  
  ERTS_GLB_INLINE void
  erts_reset_max_len(ErtsRunQueue *rq, ErtsRunQueueInfo *rqi)
  {
      erts_aint32_t len;
  
      ERTS_LC_ASSERT(erts_lc_runq_is_locked(rq));
  
      len = erts_atomic32_read_dirty(&rqi->len);
      ASSERT(rqi->max_len >= len);
      rqi->max_len = len;
  }
  
  #endif /* ERTS_GLB_INLINE_INCL_FUNC_DEF */
  
  #define RUNQ_READ_LEN(X) erts_atomic32_read_nob((X))
  
  #endif /* ERTS_INCLUDE_SCHEDULER_INTERNALS */
  
  /*
   * Process Specific Data.
   *
   * NOTE: Only use PSD for very rarely used data.
   */
  
  #define ERTS_PSD_ERROR_HANDLER			0
  #define ERTS_PSD_SAVED_CALLS_BUF		1
  #define ERTS_PSD_SCHED_ID			2
  #define ERTS_PSD_CALL_TIME_BP			3
  #define ERTS_PSD_DELAYED_GC_TASK_QS		4
  #define ERTS_PSD_NIF_TRAP_EXPORT		5
  #define ERTS_PSD_ETS_OWNED_TABLES               6
  #define ERTS_PSD_ETS_FIXED_TABLES               7
  #define ERTS_PSD_DIST_ENTRY	                8
  #define ERTS_PSD_PENDING_SUSPEND                9
  #define ERTS_PSD_SUSPENDED_SAVED_CALLS_BUF	10 /* keep last... */
  
  #define ERTS_PSD_SIZE				11
  
  #if !defined(HIPE)
  #  undef ERTS_PSD_SUSPENDED_SAVED_CALLS_BUF
  #  undef ERTS_PSD_SIZE
  #  define ERTS_PSD_SIZE 10
  #endif
  
  typedef struct {
      void *data[ERTS_PSD_SIZE];
  } ErtsPSD;
  
  #ifdef ERTS_ENABLE_LOCK_CHECK
  #define ERTS_LC_PSD_ANY_LOCK (~ERTS_PROC_LOCKS_ALL)
  
  #define ERTS_PSD_ERROR_HANDLER_BUF_GET_LOCKS ERTS_PROC_LOCK_MAIN
  #define ERTS_PSD_ERROR_HANDLER_BUF_SET_LOCKS ERTS_PROC_LOCK_MAIN
  
  #define ERTS_PSD_SAVED_CALLS_BUF_GET_LOCKS ((ErtsProcLocks) 0)
  #define ERTS_PSD_SAVED_CALLS_BUF_SET_LOCKS ((ErtsProcLocks) 0)
  
  #define ERTS_PSD_SCHED_ID_GET_LOCKS ERTS_PROC_LOCK_STATUS
  #define ERTS_PSD_SCHED_ID_SET_LOCKS ERTS_PROC_LOCK_STATUS
  
  #define ERTS_PSD_CALL_TIME_BP_GET_LOCKS ERTS_PROC_LOCK_MAIN
  #define ERTS_PSD_CALL_TIME_BP_SET_LOCKS ERTS_PROC_LOCK_MAIN
  
  #define ERTS_PSD_DELAYED_GC_TASK_QS_GET_LOCKS ERTS_PROC_LOCK_MAIN
  #define ERTS_PSD_DELAYED_GC_TASK_QS_SET_LOCKS ERTS_PROC_LOCK_MAIN
  
  #define ERTS_PSD_NIF_TRAP_EXPORT_GET_LOCKS ERTS_PROC_LOCK_MAIN
  #define ERTS_PSD_NIF_TRAP_EXPORT_SET_LOCKS ERTS_PROC_LOCK_MAIN
  
  #define ERTS_PSD_ETS_OWNED_TABLES_GET_LOCKS ERTS_PROC_LOCK_STATUS
  #define ERTS_PSD_ETS_OWNED_TABLES_SET_LOCKS ERTS_PROC_LOCK_STATUS
  
  #define ERTS_PSD_ETS_FIXED_TABLES_GET_LOCKS ERTS_PROC_LOCK_MAIN
  #define ERTS_PSD_ETS_FIXED_TABLES_SET_LOCKS ERTS_PROC_LOCK_MAIN
  
  #define ERTS_PSD_DIST_ENTRY_GET_LOCKS ERTS_PROC_LOCK_MAIN
  #define ERTS_PSD_DIST_ENTRY_SET_LOCKS ERTS_PROC_LOCK_MAIN
  
  #define ERTS_PSD_PENDING_SUSPEND_GET_LOCKS ERTS_PROC_LOCK_MAIN
  #define ERTS_PSD_PENDING_SUSPEND_SET_LOCKS ERTS_PROC_LOCK_MAIN
  
  typedef struct {
      ErtsProcLocks get_locks;
      ErtsProcLocks set_locks;
  } ErtsLcPSDLocks;
  
  extern ErtsLcPSDLocks erts_psd_required_locks[ERTS_PSD_SIZE];
  
  #endif
  
  #define ERTS_SCHED_STAT_MODIFY_DISABLE		1
  #define ERTS_SCHED_STAT_MODIFY_ENABLE		2
  #define ERTS_SCHED_STAT_MODIFY_CLEAR		3
  
  typedef struct {
      erts_spinlock_t lock;
      int enabled;
      struct {
  	Eterm name;
  	Uint total_executed;
  	Uint executed;
  	Uint total_migrated;
  	Uint migrated;
      } prio[ERTS_NO_PRIO_LEVELS];
  } erts_sched_stat_t;
  
  extern erts_sched_stat_t erts_sched_stat;
  
  typedef struct {
      Eterm reason;
      ErlHeapFragment *bp;
  } ErtsPendExit;
  
  typedef struct ErtsProcSysTask_ ErtsProcSysTask;
  typedef struct ErtsProcSysTaskQs_ ErtsProcSysTaskQs;
  
  /* Defines to ease the change of memory architecture */
  #  define HEAP_START(p)     (p)->heap
  #  define HEAP_TOP(p)       (p)->htop
  #  define HEAP_LIMIT(p)     (p)->stop
  #  define HEAP_END(p)       (p)->hend
  #  define HEAP_SIZE(p)      (p)->heap_sz
  #  define STACK_START(p)    (p)->hend
  #  define STACK_TOP(p)      (p)->stop
  #  define STACK_END(p)      (p)->htop
  #  define HIGH_WATER(p)     (p)->high_water
  #  define OLD_HEND(p)       (p)->old_hend
  #  define OLD_HTOP(p)       (p)->old_htop
  #  define OLD_HEAP(p)       (p)->old_heap
  #  define GEN_GCS(p)        (p)->gen_gcs
  #  define MAX_GEN_GCS(p)    (p)->max_gen_gcs
  #  define FLAGS(p)          (p)->flags
  #  define MBUF(p)           (p)->mbuf
  #  define HALLOC_MBUF(p)    (p)->halloc_mbuf
  #  define MBUF_SIZE(p)      (p)->mbuf_sz
  #  define MSO(p)            (p)->off_heap
  #  define MIN_HEAP_SIZE(p)  (p)->min_heap_size
  
  #  define MIN_VHEAP_SIZE(p)   (p)->min_vheap_size
  #  define BIN_VHEAP_SZ(p)     (p)->bin_vheap_sz
  #  define BIN_OLD_VHEAP_SZ(p) (p)->bin_old_vheap_sz
  #  define BIN_OLD_VHEAP(p)    (p)->bin_old_vheap
  
  #  define MAX_HEAP_SIZE_GET(p)     ((p)->max_heap_size >> 2)
  #  define MAX_HEAP_SIZE_SET(p, sz) ((p)->max_heap_size = ((sz) << 2) |  \
                                      MAX_HEAP_SIZE_FLAGS_GET(p))
  #  define MAX_HEAP_SIZE_FLAGS_GET(p)          ((p)->max_heap_size & 0x3)
  #  define MAX_HEAP_SIZE_FLAGS_SET(p, flags)   ((p)->max_heap_size = flags | \
                                                 ((p)->max_heap_size & ~0x3))
  #  define MAX_HEAP_SIZE_KILL 1
  #  define MAX_HEAP_SIZE_LOG  2
  
  struct process {
      ErtsPTabElementCommon common; /* *Need* to be first in struct */
  
      /* All fields in the PCB that differs between different heap
       * architectures, have been moved to the end of this struct to
       * make sure that as few offsets as possible differ. Different
       * offsets between memory architectures in this struct, means that
       * native code have to use functions instead of constants.
       */
  
      Eterm* htop;		/* Heap top */
      Eterm* stop;		/* Stack top */
      Eterm* heap;		/* Heap start */
      Eterm* hend;		/* Heap end */
      Eterm* abandoned_heap;
      Uint heap_sz;		/* Size of heap in words */
      Uint min_heap_size;         /* Minimum size of heap (in words). */
      Uint min_vheap_size;        /* Minimum size of virtual heap (in words). */
      Uint max_heap_size;         /* Maximum size of heap (in words). */
  
  #if !defined(NO_FPE_SIGNALS) || defined(HIPE)
      volatile unsigned long fp_exception;
  #endif
  
  #ifdef HIPE
      /* HiPE-specific process fields. Put it early in struct process,
         to enable smaller & faster addressing modes on the x86. */
      struct hipe_process_state hipe;
  #endif
  
      /*
       * Saved x registers.
       */
      Uint arity;			/* Number of live argument registers (only valid
  				 * when process is *not* running).
  				 */
      Eterm* arg_reg;		/* Pointer to argument registers. */
      unsigned max_arg_reg;	/* Maximum number of argument registers available. */
      Eterm def_arg_reg[6];	/* Default array for argument registers. */
  
      BeamInstr* cp;		/* (untagged) Continuation pointer (for threaded code). */
      BeamInstr* i;		/* Program counter for threaded code. */
      Sint catches;		/* Number of catches on stack */
      Sint fcalls;		/* 
  				 * Number of reductions left to execute.
  				 * Only valid for the current process.
  				 */
      Uint32 rcount;		/* suspend count */
      int  schedule_count;	/* Times left to reschedule a low prio process */
      Uint reds;			/* No of reductions for this process  */
      Eterm group_leader;		/* Pid in charge (can be boxed) */
      Uint flags;			/* Trap exit, etc (no trace flags anymore) */
      Eterm fvalue;		/* Exit & Throw value (failure reason) */
      Uint freason;		/* Reason for detected failure */
      Eterm ftrace;		/* Latest exception stack trace dump */
  
      Process *next;		/* Pointer to next process in run queue */
  
      ErtsSignalPrivQueues sig_qs; /* Signal queues */
      ErtsBifTimers *bif_timers;	/* Bif timers aiming at this process */
  
      ProcDict  *dictionary;       /* Process dictionary, may be NULL */
  
      Uint seq_trace_clock;
      Uint seq_trace_lastcnt;
      Eterm seq_trace_token;	/* Sequential trace token (tuple size 5 see below) */
  
  #ifdef USE_VM_PROBES
      Eterm dt_utag;              /* Place to store the dynamc trace user tag */
      Uint dt_utag_flags;         /* flag field for the dt_utag */
  #endif
      union {
  	void *terminate;
  	ErtsCodeMFA initial;	/* Initial module(0), function(1), arity(2),
                                     often used instead of pointer to funcinfo
                                     instruction. */
      } u;
      ErtsCodeMFA* current;	/* Current Erlang function, part of the funcinfo:
  				 * module(0), function(1), arity(2)
  				 * (module and functions are tagged atoms;
  				 * arity an untagged integer).
  				 */
  
      /*
       * Information mainly for post-mortem use (erl crash dump).
       */
      Eterm parent;		/* Pid of process that created this process. */
  
      Uint32 static_flags;        /* Flags that do *not* change */
  
      /* This is the place, where all fields that differs between memory
       * architectures, have gone to.
       */
  
      Eterm *high_water;
      Eterm *old_hend;            /* Heap pointers for generational GC. */
      Eterm *old_htop;
      Eterm *old_heap;
      Uint16 gen_gcs;		/* Number of (minor) generational GCs. */
      Uint16 max_gen_gcs;		/* Max minor gen GCs before fullsweep. */
      ErlOffHeap off_heap;	/* Off-heap data updated by copy_struct(). */
      ErlHeapFragment* mbuf;	/* Pointer to heap fragment list */
      ErlHeapFragment* live_hf_end;
      ErtsMessage *msg_frag;	/* Pointer to message fragment list */
      Uint mbuf_sz;		/* Total size of heap fragments and message fragments */
      erts_atomic_t psd;		/* Rarely used process specific data */
  
      Uint64 bin_vheap_sz;	/* Virtual heap block size for binaries */
      Uint64 bin_old_vheap_sz;	/* Virtual old heap block size for binaries */
      Uint64 bin_old_vheap;	/* Virtual old heap size for binaries */
  
      ErtsProcSysTaskQs *sys_task_qs;
      ErtsProcSysTask *dirty_sys_tasks;
  
      erts_atomic32_t state;  /* Process state flags (see ERTS_PSFLG_*) */
      erts_atomic32_t dirty_state; /* Process dirty state flags (see ERTS_PDSFLG_*) */
  
      ErtsSignalInQueue sig_inq;
      ErlTraceMessageQueue *trace_msg_q;
      erts_proc_lock_t lock;
      ErtsSchedulerData *scheduler_data;
      erts_atomic_t run_queue;
  
  #ifdef CHECK_FOR_HOLES
      Eterm* last_htop;		/* No need to scan the heap below this point. */
      ErlHeapFragment* last_mbuf;	/* No need to scan beyond this mbuf. */
      ErlHeapFragment* heap_hfrag; /* Heap abandoned, htop now lives in this frag */
  #endif
  
  #ifdef DEBUG
      Eterm* last_old_htop;	/*
  				 * No need to scan the old heap below this point
  				 * when looking for invalid pointers into the new heap or
  				 * heap fragments.
  				 */
  #endif
  
  #ifdef FORCE_HEAP_FRAGS
      Uint space_verified;        /* Avoid HAlloc forcing heap fragments when */ 
      Eterm* space_verified_from; /* we rely on available heap space (TestHeap) */
  #endif
  
  #ifdef DEBUG
      Uint debug_reds_in;
  #endif
  };
  
  extern Eterm erts_init_process_id; /* pid of init process */
  extern const Process erts_invalid_process;
  
  #ifdef CHECK_FOR_HOLES
  # define INIT_HOLE_CHECK(p)			\
  do {						\
    (p)->last_htop = 0;				\
    (p)->last_mbuf = 0;				\
    (p)->heap_hfrag = NULL;			\
  } while (0)
  
  # define ERTS_HOLE_CHECK(p) erts_check_for_holes((p))
  void erts_check_for_holes(Process* p);
  #else
  # define INIT_HOLE_CHECK(p)
  # define ERTS_HOLE_CHECK(p)
  #endif
  
  /*
   * The MBUF_GC_FACTOR decides how easily a process is subject to GC 
   * due to message buffers allocated outside the heap.
   * The larger the factor, the easier the process gets GCed.
   * On a small memory system with lots of processes, this makes a significant 
   * difference, especially since the GCs help fragmentation quite a bit too.
   */
  #if defined(SMALL_MEMORY)
  #define MBUF_GC_FACTOR 4
  #else
  #define MBUF_GC_FACTOR 1
  #endif
  
  #define SEQ_TRACE_TOKEN(p)  ((p)->seq_trace_token)
  
  #if ERTS_NO_PROC_PRIO_LEVELS > 4
  #  error "Need to increase ERTS_PSFLG_PRIO_SHIFT"
  #endif
  
  #define ERTS_PSFLGS_PRIO_BITS 2
  #define ERTS_PSFLGS_PRIO_MASK \
      ((((erts_aint32_t) 1) << ERTS_PSFLGS_PRIO_BITS) - 1)
  
  #define ERTS_PSFLGS_ACT_PRIO_OFFSET (0*ERTS_PSFLGS_PRIO_BITS)
  #define ERTS_PSFLGS_USR_PRIO_OFFSET (1*ERTS_PSFLGS_PRIO_BITS)
  #define ERTS_PSFLGS_PRQ_PRIO_OFFSET (2*ERTS_PSFLGS_PRIO_BITS)
  #define ERTS_PSFLGS_ZERO_BIT_OFFSET (3*ERTS_PSFLGS_PRIO_BITS)
  
  #define ERTS_PSFLGS_QMASK_BITS 4
  #define ERTS_PSFLGS_QMASK \
      ((((erts_aint32_t) 1) << ERTS_PSFLGS_QMASK_BITS) - 1)
  #define ERTS_PSFLGS_IN_PRQ_MASK_OFFSET \
      ERTS_PSFLGS_ZERO_BIT_OFFSET
  
  #define ERTS_PSFLG_BIT(N) \
      (((erts_aint32_t) 1) << (ERTS_PSFLGS_ZERO_BIT_OFFSET + (N)))
  
  /*
   * ACT_PRIO -> Active prio, i.e., currently active prio. This
   *             prio may be higher than user prio.
   * USR_PRIO -> User prio. i.e., prio the user has set.
   * PRQ_PRIO -> Prio queue prio, i.e., prio queue currently
   *             enqueued in. 
   *
   * Update etp-proc-state-int in $ERL_TOP/erts/etc/unix/etp-commands.in
   * when changing ERTS_PSFLG_*.
   */
  #define ERTS_PSFLGS_ACT_PRIO_MASK \
      (ERTS_PSFLGS_PRIO_MASK << ERTS_PSFLGS_ACT_PRIO_OFFSET)
  #define ERTS_PSFLGS_USR_PRIO_MASK \
      (ERTS_PSFLGS_PRIO_MASK << ERTS_PSFLGS_USR_PRIO_OFFSET)
  #define ERTS_PSFLGS_PRQ_PRIO_MASK \
      (ERTS_PSFLGS_PRIO_MASK << ERTS_PSFLGS_PRQ_PRIO_OFFSET)
  #define ERTS_PSFLG_IN_PRQ_MAX 		ERTS_PSFLG_BIT(0)
  #define ERTS_PSFLG_IN_PRQ_HIGH		ERTS_PSFLG_BIT(1)
  #define ERTS_PSFLG_IN_PRQ_NORMAL	ERTS_PSFLG_BIT(2)
  #define ERTS_PSFLG_IN_PRQ_LOW 		ERTS_PSFLG_BIT(3)
  #define ERTS_PSFLG_FREE			ERTS_PSFLG_BIT(4)
  #define ERTS_PSFLG_EXITING		ERTS_PSFLG_BIT(5)
  #define ERTS_PSFLG_UNUSED		ERTS_PSFLG_BIT(6)
  #define ERTS_PSFLG_ACTIVE		ERTS_PSFLG_BIT(7)
  #define ERTS_PSFLG_IN_RUNQ		ERTS_PSFLG_BIT(8)
  #define ERTS_PSFLG_RUNNING		ERTS_PSFLG_BIT(9)
  #define ERTS_PSFLG_SUSPENDED		ERTS_PSFLG_BIT(10)
  #define ERTS_PSFLG_GC			ERTS_PSFLG_BIT(11)
  #define ERTS_PSFLG_SYS_TASKS		ERTS_PSFLG_BIT(12)
  #define ERTS_PSFLG_SIG_IN_Q		ERTS_PSFLG_BIT(13)
  #define ERTS_PSFLG_ACTIVE_SYS		ERTS_PSFLG_BIT(14)
  #define ERTS_PSFLG_RUNNING_SYS		ERTS_PSFLG_BIT(15)
  #define ERTS_PSFLG_PROXY		ERTS_PSFLG_BIT(16)
  #define ERTS_PSFLG_DELAYED_SYS		ERTS_PSFLG_BIT(17)
  #define ERTS_PSFLG_OFF_HEAP_MSGQ	ERTS_PSFLG_BIT(18)
  #define ERTS_PSFLG_SIG_Q		ERTS_PSFLG_BIT(19)
  #define ERTS_PSFLG_DIRTY_CPU_PROC	ERTS_PSFLG_BIT(20)
  #define ERTS_PSFLG_DIRTY_IO_PROC	ERTS_PSFLG_BIT(21)
  #define ERTS_PSFLG_DIRTY_ACTIVE_SYS	ERTS_PSFLG_BIT(22)
  #define ERTS_PSFLG_DIRTY_RUNNING	ERTS_PSFLG_BIT(23)
  #define ERTS_PSFLG_DIRTY_RUNNING_SYS	ERTS_PSFLG_BIT(24)
  
  #define ERTS_PSFLG_MAX  (ERTS_PSFLGS_ZERO_BIT_OFFSET + 24)
  
  #define ERTS_PSFLGS_DIRTY_WORK		(ERTS_PSFLG_DIRTY_CPU_PROC	\
  					 | ERTS_PSFLG_DIRTY_IO_PROC	\
  					 | ERTS_PSFLG_DIRTY_ACTIVE_SYS)
  
  #define ERTS_PSFLGS_IN_PRQ_MASK 	(ERTS_PSFLG_IN_PRQ_MAX		\
  					 | ERTS_PSFLG_IN_PRQ_HIGH	\
  					 | ERTS_PSFLG_IN_PRQ_NORMAL	\
  					 | ERTS_PSFLG_IN_PRQ_LOW)
  
  #define ERTS_PSFLGS_VOLATILE_HEAP	(ERTS_PSFLG_EXITING		\
  					 | ERTS_PSFLG_DIRTY_RUNNING	\
  					 | ERTS_PSFLG_DIRTY_RUNNING_SYS)
  
  #define ERTS_PSFLGS_GET_ACT_PRIO(PSFLGS) \
      (((PSFLGS) >> ERTS_PSFLGS_ACT_PRIO_OFFSET) & ERTS_PSFLGS_PRIO_MASK)
  #define ERTS_PSFLGS_GET_USR_PRIO(PSFLGS) \
      (((PSFLGS) >> ERTS_PSFLGS_USR_PRIO_OFFSET) & ERTS_PSFLGS_PRIO_MASK)
  #define ERTS_PSFLGS_GET_PRQ_PRIO(PSFLGS) \
      (((PSFLGS) >> ERTS_PSFLGS_PRQ_PRIO_OFFSET) & ERTS_PSFLGS_PRIO_MASK)
  
  
  /*
   * Flags in the dirty_state field.
   */
  
  #define ERTS_PDSFLG_IN_CPU_PRQ_MAX 	(((erts_aint32_t) 1) << 0)
  #define ERTS_PDSFLG_IN_CPU_PRQ_HIGH	(((erts_aint32_t) 1) << 1)
  #define ERTS_PDSFLG_IN_CPU_PRQ_NORMAL	(((erts_aint32_t) 1) << 2)
  #define ERTS_PDSFLG_IN_CPU_PRQ_LOW 	(((erts_aint32_t) 1) << 3)
  #define ERTS_PDSFLG_IN_IO_PRQ_MAX 	(((erts_aint32_t) 1) << 4)
  #define ERTS_PDSFLG_IN_IO_PRQ_HIGH	(((erts_aint32_t) 1) << 5)
  #define ERTS_PDSFLG_IN_IO_PRQ_NORMAL	(((erts_aint32_t) 1) << 6)
  #define ERTS_PDSFLG_IN_IO_PRQ_LOW 	(((erts_aint32_t) 1) << 7)
  
  #define ERTS_PDSFLGS_QMASK 		ERTS_PSFLGS_QMASK
  #define ERTS_PDSFLGS_IN_CPU_PRQ_MASK_OFFSET 0
  #define ERTS_PDSFLGS_IN_IO_PRQ_MASK_OFFSET ERTS_PSFLGS_QMASK_BITS
  
  #define ERTS_PDSFLG_IN_CPU_PRQ_MASK 	(ERTS_PDSFLG_IN_CPU_PRQ_MAX	\
  					 | ERTS_PDSFLG_IN_CPU_PRQ_HIGH	\
  					 | ERTS_PDSFLG_IN_CPU_PRQ_NORMAL\
  					 | ERTS_PDSFLG_IN_CPU_PRQ_LOW)
  #define ERTS_PDSFLG_IN_IO_PRQ_MASK 	(ERTS_PDSFLG_IN_CPU_PRQ_MAX	\
  					 | ERTS_PDSFLG_IN_CPU_PRQ_HIGH	\
  					 | ERTS_PDSFLG_IN_CPU_PRQ_NORMAL\
  					 | ERTS_PDSFLG_IN_CPU_PRQ_LOW)
  
  
  /*
   * Static flags that do not change after process creation.
   */
  #define ERTS_STC_FLG_SYSTEM_PROC	(((Uint32) 1) << 0)
  #define ERTS_STC_FLG_SHADOW_PROC	(((Uint32) 1) << 1)
  
  /* The sequential tracing token is a tuple of size 5:
   *
   *    {Flags, Label, Serial, Sender}
   */
  
  #define SEQ_TRACE_TOKEN_ARITY(p)    (arityval(*(tuple_val(SEQ_TRACE_TOKEN(p)))))
  #define SEQ_TRACE_TOKEN_FLAGS(p)    (*(tuple_val(SEQ_TRACE_TOKEN(p)) + 1))
  #define SEQ_TRACE_TOKEN_LABEL(p)    (*(tuple_val(SEQ_TRACE_TOKEN(p)) + 2))
  #define SEQ_TRACE_TOKEN_SERIAL(p)   (*(tuple_val(SEQ_TRACE_TOKEN(p)) + 3))
  #define SEQ_TRACE_TOKEN_SENDER(p)   (*(tuple_val(SEQ_TRACE_TOKEN(p)) + 4))
  #define SEQ_TRACE_TOKEN_LASTCNT(p)  (*(tuple_val(SEQ_TRACE_TOKEN(p)) + 5))
  
  /* used when we have unit32 token */
  #define SEQ_TRACE_T_ARITY(token)    (arityval(*(tuple_val(token))))
  #define SEQ_TRACE_T_FLAGS(token)    (*(tuple_val(token) + 1))
  #define SEQ_TRACE_T_LABEL(token)    (*(tuple_val(token) + 2))
  #define SEQ_TRACE_T_SERIAL(token)   (*(tuple_val(token) + 3))
  #define SEQ_TRACE_T_SENDER(token)   (*(tuple_val(token) + 4))
  #define SEQ_TRACE_T_LASTCNT(token)  (*(tuple_val(token) + 5))
  
  #ifdef USE_VM_PROBES
  /* The dtrace probe for seq_trace only supports 'int' labels, so we represent
   * all values that won't fit into a 32-bit signed integer as ERTS_SINT32_MIN
   * (bigints, tuples, etc). */
  
  #define SEQ_TRACE_T_DTRACE_LABEL(token) \
      DTRACE_SEQ_TRACE_LABEL__(SEQ_TRACE_T_LABEL(token))
  
  #define DTRACE_SEQ_TRACE_LABEL__(label_term) \
      (is_small((label_term)) ? \
          ((signed_val((label_term)) <= ERTS_SINT32_MAX && \
            signed_val((label_term)) >= ERTS_SINT32_MIN) ? \
               signed_val((label_term)) : ERTS_SINT32_MIN) \
          : ERTS_SINT32_MIN)
  #endif
  
  /*
  
   * Possible flags for the flags field in ErlSpawnOpts below.
   */
  
  #define SPO_LINK 1
  #define SPO_USE_ARGS 2
  #define SPO_MONITOR 4
  #define SPO_SYSTEM_PROC 8
  #define SPO_OFF_HEAP_MSGQ 16
  #define SPO_ON_HEAP_MSGQ 32
  
  extern int ERTS_WRITE_UNLIKELY(erts_default_spo_flags);
  
  /*
   * The following struct contains options for a process to be spawned.
   */
  typedef struct {
      int flags;
      int error_code;		/* Error code returned from create_process(). */
      Eterm mref;			/* Monitor ref returned (if SPO_MONITOR was given). */
  
      /*
       * The following items are only initialized if the SPO_USE_ARGS flag is set.
       */
      Uint min_heap_size;		/* Minimum heap size (must be a valued returned
  				 * from next_heap_size()).  */
      Uint min_vheap_size;	/* Minimum virtual heap size  */
      int priority;		/* Priority for process. */
      Uint16 max_gen_gcs;		/* Maximum number of gen GCs before fullsweep. */
      Uint max_heap_size;         /* Maximum heap size in words */
      Uint max_heap_flags;        /* Maximum heap flags (kill | log) */
      int scheduler;
  } ErlSpawnOpts;
  
  /*
   * The KILL_CATCHES(p) macro kills pending catches for process p.
   */
  
  #define KILL_CATCHES(p) (p)->catches = -1
  
  /* Shrink heap fragment from _last_ HAlloc.
  */
  ERTS_GLB_INLINE void erts_heap_frag_shrink(Process* p, Eterm* hp);
  #if ERTS_GLB_INLINE_INCL_FUNC_DEF
  ERTS_GLB_INLINE void erts_heap_frag_shrink(Process* p, Eterm* hp)
  {
      ErlHeapFragment* hf = MBUF(p);
      Uint sz;
  
      ASSERT(hf!=NULL && (hp - hf->mem < hf->alloc_size));
  
      sz = hp - hf->mem;
      p->mbuf_sz -= hf->used_size - sz;
      hf->used_size = sz;
  }	
  #endif /* inline */
  
  Eterm* erts_heap_alloc(Process* p, Uint need, Uint xtra);
  
  extern erts_rwmtx_t erts_cpu_bind_rwmtx;
  /* If any of the erts_system_monitor_* variables are set (enabled),
  ** erts_system_monitor must be != NIL, to allow testing on just
  ** the erts_system_monitor_* variables.
  */
  extern Eterm ERTS_WRITE_UNLIKELY(erts_system_monitor);
  extern Uint ERTS_WRITE_UNLIKELY(erts_system_monitor_long_gc);
  extern Uint ERTS_WRITE_UNLIKELY(erts_system_monitor_long_schedule);
  extern Uint ERTS_WRITE_UNLIKELY(erts_system_monitor_large_heap);
  struct erts_system_monitor_flags_t {
  	 unsigned int busy_port : 1;
      unsigned int busy_dist_port : 1;
  };
  extern struct erts_system_monitor_flags_t erts_system_monitor_flags;
  
  /* system_profile, same rules as for system_monitor.
  	erts_profile must be != NIL when 
  	erts_profile_* is set. */
  
  extern Eterm erts_system_profile;
  struct erts_system_profile_flags_t {
      unsigned int scheduler : 1;
      unsigned int runnable_procs : 1;
      unsigned int runnable_ports : 1;
      unsigned int exclusive : 1;
  };
  extern struct erts_system_profile_flags_t erts_system_profile_flags;
  extern int erts_system_profile_ts_type;
  
  /* process flags */
  #define F_HIBERNATE_SCHED    (1 <<  0) /* Schedule out after hibernate op */
  #define F_INSLPQUEUE         (1 <<  1) /* Set if in timer queue */
  #define F_TIMO               (1 <<  2) /* Set if timeout */
  #define F_HEAP_GROW          (1 <<  3)
  #define F_NEED_FULLSWEEP     (1 <<  4)
  #define F_USING_DB           (1 <<  5) /* If have created tables */
  #define F_DISTRIBUTION       (1 <<  6) /* Process used in distribution */
  #define F_USING_DDLL         (1 <<  7) /* Process has used the DDLL interface */
  #define F_HAVE_BLCKD_MSCHED  (1 <<  8) /* Process has blocked multi-scheduling */
  #define F_UNUSED             (1 <<  9)
  #define F_FORCE_GC           (1 << 10) /* Force gc at process in-scheduling */
  #define F_DISABLE_GC         (1 << 11) /* Disable GC (see below) */
  #define F_OFF_HEAP_MSGQ      (1 << 12) /* Off heap msg queue */
  #define F_ON_HEAP_MSGQ       (1 << 13) /* On heap msg queue */
  #define F_OFF_HEAP_MSGQ_CHNG (1 << 14) /* Off heap msg queue changing */
  #define F_ABANDONED_HEAP_USE (1 << 15) /* Have usage of abandoned heap */
  #define F_DELAY_GC           (1 << 16) /* Similar to disable GC (see below) */
  #define F_SCHDLR_ONLN_WAITQ  (1 << 17) /* Process enqueued waiting to change schedulers online */
  #define F_HAVE_BLCKD_NMSCHED (1 << 18) /* Process has blocked normal multi-scheduling */
  #define F_HIPE_MODE          (1 << 19) /* Process is executing in HiPE mode */
  #define F_DELAYED_DEL_PROC   (1 << 20) /* Delay delete process (dirty proc exit case) */
  #define F_DIRTY_CLA          (1 << 21) /* Dirty copy literal area scheduled */
  #define F_DIRTY_GC_HIBERNATE (1 << 22) /* Dirty GC hibernate scheduled */
  #define F_DIRTY_MAJOR_GC     (1 << 23) /* Dirty major GC scheduled */
  #define F_DIRTY_MINOR_GC     (1 << 24) /* Dirty minor GC scheduled */
  #define F_HIBERNATED         (1 << 25) /* Hibernated */
  #define F_LOCAL_SIGS_ONLY    (1 << 26) /* Handle privq sigs only */
  #define F_TRAP_EXIT          (1 << 27) /* Trapping exit */
  #define F_DEFERRED_SAVED_LAST (1 << 28) /* Deferred sig_qs.saved_last */
  #define F_DELAYED_PSIGQS_LEN (1 << 29) /* Delayed update of sig_qs.len */
  #define F_HIPE_RECV_LOCKED   (1 << 30) /* HiPE message queue locked */
  #define F_HIPE_RECV_YIELD    (1 << 31) /* HiPE receive yield */
  
  /*
   * F_DISABLE_GC and F_DELAY_GC are similar. Both will prevent
   * GC of the process, but it is important to use the right
   * one:
   * - F_DISABLE_GC should *only* be used by BIFs. This when
   *   the BIF needs to yield while preventig a GC.
   * - F_DELAY_GC should only be used when GC is temporarily
   *   disabled while the process is scheduled. A process must
   *   not be scheduled out while F_DELAY_GC is set.
   */
  
  #define ERTS_TRACE_FLAGS_TS_TYPE_SHIFT			0
  
  #define F_TRACE_FLAG(N)      (1 << (ERTS_TRACE_TS_TYPE_BITS + (N)))
  
  /* process trace_flags */
  #define F_NOW_TS             (ERTS_TRACE_FLG_NOW_TIMESTAMP \
  			      << ERTS_TRACE_FLAGS_TS_TYPE_SHIFT)
  #define F_STRICT_MON_TS      (ERTS_TRACE_FLG_STRICT_MONOTONIC_TIMESTAMP \
  			      << ERTS_TRACE_FLAGS_TS_TYPE_SHIFT)
  #define F_MON_TS             (ERTS_TRACE_FLG_MONOTONIC_TIMESTAMP \
  			      << ERTS_TRACE_FLAGS_TS_TYPE_SHIFT)
  #define F_SENSITIVE          F_TRACE_FLAG(0)
  #define F_TRACE_SEND         F_TRACE_FLAG(1)
  #define F_TRACE_RECEIVE      F_TRACE_FLAG(2)
  #define F_TRACE_SOS          F_TRACE_FLAG(3) /* Set on spawn       */
  #define F_TRACE_SOS1         F_TRACE_FLAG(4) /* Set on first spawn */
  #define F_TRACE_SOL          F_TRACE_FLAG(5) /* Set on link        */
  #define F_TRACE_SOL1         F_TRACE_FLAG(6) /* Set on first link  */
  #define F_TRACE_CALLS        F_TRACE_FLAG(7)
  #define F_TRACE_PROCS        F_TRACE_FLAG(8)
  #define F_TRACE_FIRST_CHILD  F_TRACE_FLAG(9)
  #define F_TRACE_SCHED        F_TRACE_FLAG(10)
  #define F_TRACE_GC           F_TRACE_FLAG(11)
  #define F_TRACE_ARITY_ONLY   F_TRACE_FLAG(12)
  #define F_TRACE_RETURN_TO    F_TRACE_FLAG(13) /* Return_to trace when breakpoint tracing */
  #define F_TRACE_SILENT       F_TRACE_FLAG(14) /* No call trace msg suppress */
  #define F_EXCEPTION_TRACE    F_TRACE_FLAG(15) /* May have exception trace on stack */
  
  /* port trace flags, currently the same as process trace flags */
  #define F_TRACE_SCHED_PORTS  F_TRACE_FLAG(17) /* Trace of port scheduling */
  #define F_TRACE_SCHED_PROCS  F_TRACE_FLAG(18) /* With virtual scheduling */
  #define F_TRACE_PORTS	     F_TRACE_FLAG(19) /* Ports equivalent to F_TRACE_PROCS */
  #define F_TRACE_SCHED_NO     F_TRACE_FLAG(20) /* Trace with scheduler id */
  #define F_TRACE_SCHED_EXIT   F_TRACE_FLAG(21)
  
  #define F_NUM_FLAGS          (ERTS_TRACE_TS_TYPE_BITS + 22)
  #ifdef DEBUG
  #  define F_INITIAL_TRACE_FLAGS (5 << F_NUM_FLAGS)
  #else
  #  define F_INITIAL_TRACE_FLAGS 0
  #endif
  
  /* F_TIMESTAMP_MASK is a bit-field of all all timestamp types */
  #define F_TIMESTAMP_MASK \
      (ERTS_TRACE_TS_TYPE_MASK << ERTS_TRACE_FLAGS_TS_TYPE_SHIFT)
  
  #define TRACEE_FLAGS ( F_TRACE_PROCS | F_TRACE_CALLS \
  		     | F_TRACE_SOS |  F_TRACE_SOS1| F_TRACE_RECEIVE  \
  		     | F_TRACE_SOL | F_TRACE_SOL1 | F_TRACE_SEND \
  		     | F_TRACE_SCHED | F_TIMESTAMP_MASK | F_TRACE_GC \
  		     | F_TRACE_ARITY_ONLY | F_TRACE_RETURN_TO \
                       | F_TRACE_SILENT | F_TRACE_SCHED_PROCS | F_TRACE_PORTS \
  		     | F_TRACE_SCHED_PORTS | F_TRACE_SCHED_NO \
  		     | F_TRACE_SCHED_EXIT )
  
  
  #define ERTS_TRACEE_MODIFIER_FLAGS \
      (F_TRACE_SILENT | F_TIMESTAMP_MASK | F_TRACE_SCHED_NO \
       | F_TRACE_RECEIVE | F_TRACE_SEND)
  #define ERTS_PORT_TRACEE_FLAGS                                     \
      (ERTS_TRACEE_MODIFIER_FLAGS | F_TRACE_PORTS | F_TRACE_SCHED_PORTS)
  #define ERTS_PROC_TRACEE_FLAGS \
      ((TRACEE_FLAGS & ~ERTS_PORT_TRACEE_FLAGS) | ERTS_TRACEE_MODIFIER_FLAGS)
  
  #define SEQ_TRACE_FLAG(N)        (1 << (ERTS_TRACE_TS_TYPE_BITS + (N)))
  
  #define ERTS_SIG_ENABLE_TRACE_FLAGS \
      ( F_TRACE_RECEIVE | F_TRACE_PROCS)
  
  /*
   * F_TRACE_RECEIVE is always enabled/disable via signaling.
   * F_TRACE_PROCS enable/disable F_TRACE_PROCS_SIG via signaling.
   */
  
  /* Sequential trace flags */
  
  /* SEQ_TRACE_TIMESTAMP_MASK is a bit-field */
  #define SEQ_TRACE_TIMESTAMP_MASK \
      (ERTS_TRACE_TS_TYPE_MASK << ERTS_SEQ_TRACE_FLAGS_TS_TYPE_SHIFT)
  
  #define SEQ_TRACE_SEND     (1 << 0)
  #define SEQ_TRACE_RECEIVE  (1 << 1)
  #define SEQ_TRACE_PRINT    (1 << 2)
  
  #define ERTS_SEQ_TRACE_FLAGS_TS_TYPE_SHIFT 3
  
  #define SEQ_TRACE_NOW_TS   (ERTS_TRACE_FLG_NOW_TIMESTAMP \
  			    << ERTS_SEQ_TRACE_FLAGS_TS_TYPE_SHIFT)
  #define SEQ_TRACE_STRICT_MON_TS (ERTS_TRACE_FLG_STRICT_MONOTONIC_TIMESTAMP \
  				 << ERTS_SEQ_TRACE_FLAGS_TS_TYPE_SHIFT)
  #define SEQ_TRACE_MON_TS (ERTS_TRACE_FLG_MONOTONIC_TIMESTAMP \
  			  << ERTS_SEQ_TRACE_FLAGS_TS_TYPE_SHIFT)
  
  #ifdef USE_VM_PROBES
  #define DT_UTAG_PERMANENT (1 << 0)
  #define DT_UTAG_SPREADING (1 << 1)
  #define DT_UTAG(P) ((P)->dt_utag)
  #define DT_UTAG_FLAGS(P)  ((P)->dt_utag_flags) 
  #endif
  
  #define CANCEL_TIMER(P)					\
      do {						\
  	if ((P)->flags & (F_INSLPQUEUE|F_TIMO)) {	\
  	    if ((P)->flags & F_INSLPQUEUE)		\
  		erts_cancel_proc_timer((P));		\
  	    else					\
  		(P)->flags &= ~F_TIMO;			\
  	}						\
      } while (0)
  
  #define ERTS_NUM_DIRTY_CPU_RUNQS 1
  #define ERTS_NUM_DIRTY_IO_RUNQS 1
  
  #define ERTS_NUM_DIRTY_RUNQS (ERTS_NUM_DIRTY_CPU_RUNQS+ERTS_NUM_DIRTY_IO_RUNQS)
  
  #define ERTS_RUNQ_IX(IX)						\
    (ASSERT(0 <= (IX) && (IX) < erts_no_run_queues+ERTS_NUM_DIRTY_RUNQS), \
     &erts_aligned_run_queues[(IX)].runq)
  #define ERTS_RUNQ_IX_IS_DIRTY(IX)					\
    (ASSERT(0 <= (IX) && (IX) < erts_no_run_queues+ERTS_NUM_DIRTY_RUNQS), \
     (erts_no_run_queues <= (IX)))
  #define ERTS_DIRTY_RUNQ_IX(IX)						\
    (ASSERT(ERTS_RUNQ_IX_IS_DIRTY(IX)),					\
     &erts_aligned_run_queues[(IX)].runq)
  #define ERTS_DIRTY_CPU_RUNQ (&erts_aligned_run_queues[erts_no_run_queues].runq)
  #define ERTS_DIRTY_IO_RUNQ  (&erts_aligned_run_queues[erts_no_run_queues+1].runq)
  #define ERTS_RUNQ_IS_DIRTY_CPU_RUNQ(RQ) ((RQ) == ERTS_DIRTY_CPU_RUNQ)
  #define ERTS_RUNQ_IS_DIRTY_IO_RUNQ(RQ) ((RQ) == ERTS_DIRTY_IO_RUNQ)
  #define ERTS_SCHEDULER_IX(IX)						\
    (ASSERT(0 <= (IX) && (IX) < erts_no_schedulers),			\
     &erts_aligned_scheduler_data[(IX)].esd)
  #define ERTS_DIRTY_CPU_SCHEDULER_IX(IX)					\
    (ASSERT(0 <= (IX) && (IX) < erts_no_dirty_cpu_schedulers),		\
     &erts_aligned_dirty_cpu_scheduler_data[(IX)].esd)
  #define ERTS_DIRTY_IO_SCHEDULER_IX(IX)					\
    (ASSERT(0 <= (IX) && (IX) < erts_no_dirty_io_schedulers),		\
     &erts_aligned_dirty_io_scheduler_data[(IX)].esd)
  #define ERTS_SCHEDULER_IS_DIRTY(ESDP)					\
    ((ESDP)->type != ERTS_SCHED_NORMAL)
  #define ERTS_SCHEDULER_IS_DIRTY_CPU(ESDP)				\
      ((ESDP)->type == ERTS_SCHED_DIRTY_CPU)
  #define ERTS_SCHEDULER_IS_DIRTY_IO(ESDP)				\
      ((ESDP)->type == ERTS_SCHED_DIRTY_IO)
  
  void erts_pre_init_process(void);
  void erts_late_init_process(void);
  void erts_early_init_scheduling(int);
  void erts_init_scheduling(int, int, int, int, int, int);
  void erts_execute_dirty_system_task(Process *c_p);
  int erts_set_gc_state(Process *c_p, int enable);
  Eterm erts_sched_wall_time_request(Process *c_p, int set, int enable,
                                     int dirty_cpu, int want_dirty_io);
  Eterm erts_system_check_request(Process *c_p);
  Eterm erts_gc_info_request(Process *c_p);
  Uint64 erts_get_proc_interval(void);
  Uint64 erts_ensure_later_proc_interval(Uint64);
  Uint64 erts_step_proc_interval(void);
  
  ErtsProcList *erts_proclist_create(Process *);
  ErtsProcList *erts_proclist_copy(ErtsProcList *);
  void erts_proclist_destroy(ErtsProcList *);
  void erts_proclist_dump(fmtfn_t to, void *to_arg, ErtsProcList*);
  
  ERTS_GLB_INLINE int erts_proclist_same(ErtsProcList *, Process *);
  ERTS_GLB_INLINE void erts_proclist_store_first(ErtsProcList **, ErtsProcList *);
  ERTS_GLB_INLINE void erts_proclist_store_last(ErtsProcList **, ErtsProcList *);
  ERTS_GLB_INLINE ErtsProcList *erts_proclist_peek_first(ErtsProcList *);
  ERTS_GLB_INLINE ErtsProcList *erts_proclist_peek_last(ErtsProcList *);
  ERTS_GLB_INLINE ErtsProcList *erts_proclist_peek_next(ErtsProcList *, ErtsProcList *);
  ERTS_GLB_INLINE ErtsProcList *erts_proclist_peek_prev(ErtsProcList *, ErtsProcList *);
  ERTS_GLB_INLINE ErtsProcList *erts_proclist_fetch_first(ErtsProcList **);
  ERTS_GLB_INLINE ErtsProcList *erts_proclist_fetch_last(ErtsProcList **);
  ERTS_GLB_INLINE int erts_proclist_fetch(ErtsProcList **, ErtsProcList **);
  ERTS_GLB_INLINE void erts_proclist_remove(ErtsProcList **, ErtsProcList *);
  ERTS_GLB_INLINE int erts_proclist_is_empty(ErtsProcList *);
  ERTS_GLB_INLINE int erts_proclist_is_first(ErtsProcList *, ErtsProcList *);
  ERTS_GLB_INLINE int erts_proclist_is_last(ErtsProcList *, ErtsProcList *);
  
  #if ERTS_GLB_INLINE_INCL_FUNC_DEF
  
  ERTS_GLB_INLINE int
  erts_proclist_same(ErtsProcList *plp, Process *p)
  {
      return ((plp->u.pid == p->common.id || plp->u.p == p)
  	    && (plp->started_interval
  		== p->common.u.alive.started_interval));
  }
  
  ERTS_GLB_INLINE void erts_proclist_store_first(ErtsProcList **list,
  					       ErtsProcList *element)
  {
      if (!*list)
  	element->next = element->prev = element;
      else {
  	element->prev = (*list)->prev;
  	element->next = *list;
  	element->prev->next = element;
  	element->next->prev = element;
      }
      *list = element;
  }
  
  ERTS_GLB_INLINE void erts_proclist_store_last(ErtsProcList **list,
  					      ErtsProcList *element)
  {
      if (!*list) {
  	element->next = element->prev = element;
  	*list = element;
      }
      else {
  	element->prev = (*list)->prev;
  	element->next = *list;
  	element->prev->next = element;
  	element->next->prev = element;
      }
  }
  
  ERTS_GLB_INLINE ErtsProcList *erts_proclist_peek_first(ErtsProcList *list)
  {
      return list;
  }
  
  ERTS_GLB_INLINE ErtsProcList *erts_proclist_peek_last(ErtsProcList *list)
  {
      if (!list)
  	return NULL;
      else
  	return list->prev;
  }
  
  ERTS_GLB_INLINE ErtsProcList *erts_proclist_peek_next(ErtsProcList *list,
  						      ErtsProcList *element)
  {
      ErtsProcList *next;
      ASSERT(list && element);
      next = element->next;
      return list == next ? NULL : next;
  }
  
  ERTS_GLB_INLINE ErtsProcList *erts_proclist_peek_prev(ErtsProcList *list,
  						      ErtsProcList *element)
  {
      ErtsProcList *prev;
      ASSERT(list && element);
      prev = element->prev;
      return list == element ? NULL : prev;
  }
  
  ERTS_GLB_INLINE ErtsProcList *erts_proclist_fetch_first(ErtsProcList **list)
  {
      if (!*list)
  	return NULL;
      else {
  	ErtsProcList *res = *list;
  	if (res->next == *list)
  	    *list = NULL;
  	else
  	    *list = res->next;
  	res->next->prev = res->prev;
  	res->prev->next = res->next;
  	return res;
      }
  }
  
  ERTS_GLB_INLINE ErtsProcList *erts_proclist_fetch_last(ErtsProcList **list)
  {
      if (!*list)
  	return NULL;
      else {
  	ErtsProcList *res = (*list)->prev;
  	if (res == *list)
  	    *list = NULL;
  	res->next->prev = res->prev;
  	res->prev->next = res->next;
  	return res;
      }
  }
  
  ERTS_GLB_INLINE int erts_proclist_fetch(ErtsProcList **list_first,
  					ErtsProcList **list_last)
  {
      if (!*list_first) {
  	if (list_last)
  	    *list_last = NULL;
  	return 0;
      }
      else {
  	if (list_last)
  	    *list_last = (*list_first)->prev;
  	(*list_first)->prev->next = NULL;
  	(*list_first)->prev = NULL;
  	return !0;
      }
  }
  
  ERTS_GLB_INLINE void erts_proclist_remove(ErtsProcList **list,
  					  ErtsProcList *element)
  {
      ASSERT(list && *list);
      if (*list == element) {
  	*list = element->next;
  	if (*list == element)
  	    *list = NULL;
      }
      element->next->prev = element->prev;
      element->prev->next = element->next;
  }
  
  ERTS_GLB_INLINE int erts_proclist_is_empty(ErtsProcList *list)
  {
      return list == NULL;
  }
  
  ERTS_GLB_INLINE int erts_proclist_is_first(ErtsProcList *list,
  					   ErtsProcList *element)
  {
      ASSERT(list && element);
      return list == element;
  }
  
  ERTS_GLB_INLINE int erts_proclist_is_last(ErtsProcList *list,
  					  ErtsProcList *element)
  {
      ASSERT(list && element);
      return list->prev == element;
  }
  
  #endif
  
  int erts_sched_set_wakeup_other_threshold(ErtsSchedType sched_type, char *str);
  int erts_sched_set_wakeup_other_type(ErtsSchedType sched_type, char *str);
  int erts_sched_set_busy_wait_threshold(ErtsSchedType sched_type, char *str);
  int erts_sched_set_wake_cleanup_threshold(char *);
  
  void erts_schedule_thr_prgr_later_op(void (*)(void *),
  				     void *,
  				     ErtsThrPrgrLaterOp *);
  void erts_schedule_thr_prgr_later_cleanup_op(void (*)(void *),
  					     void *,
  					     ErtsThrPrgrLaterOp *,
  					     UWord);
  void erts_schedule_complete_off_heap_message_queue_change(Eterm pid);
  struct db_fixation;
  void erts_schedule_ets_free_fixation(Eterm pid, struct db_fixation*);
  void erts_schedule_flush_trace_messages(Process *proc, int force_on_proc);
  int erts_flush_trace_messages(Process *c_p, ErtsProcLocks locks);
  int erts_sig_prio(Eterm pid, int prio);
  
  #if defined(ERTS_ENABLE_LOCK_CHECK)
  int erts_dbg_check_halloc_lock(Process *p);
  #endif
  void
  erts_schedulers_state(Uint *, Uint *, Uint *, Uint *, Uint *, Uint *, Uint *, Uint *);
  ErtsSchedSuspendResult
  erts_set_schedulers_online(Process *p,
  			   ErtsProcLocks plocks,
  			   Sint new_no,
  			   Sint *old_no,
  			   int dirty_only);
  ErtsSchedSuspendResult
  erts_block_multi_scheduling(Process *, ErtsProcLocks, int, int, int);
  int erts_is_multi_scheduling_blocked(void);
  Eterm erts_multi_scheduling_blockers(Process *, int);
  void erts_start_schedulers(void);
  void erts_alloc_notify_delayed_dealloc(int);
  void erts_alloc_ensure_handle_delayed_dealloc_call(int);
  void erts_notify_canceled_timer(ErtsSchedulerData *, int);
  void erts_notify_check_async_ready_queue(void *);
  void erts_notify_code_ix_activation(Process* p, ErtsThrPrgrVal later);
  void erts_notify_finish_breakpointing(Process* p);
  void erts_schedule_misc_aux_work(int sched_id,
  				 void (*func)(void *),
  				 void *arg);
  void erts_schedule_multi_misc_aux_work(int ignore_self,
  				       int max_sched,
  				       void (*func)(void *),
  				       void *arg);
  erts_aint32_t erts_set_aux_work_timeout(int, erts_aint32_t, int);
  void erts_aux_work_timeout_late_init(ErtsSchedulerData *esdp);
  void erts_sched_notify_check_cpu_bind(void);
  Uint erts_active_schedulers(void);
  void erts_init_process(int, int, int);
  Eterm erts_process_state2status(erts_aint32_t);
  Eterm erts_process_status(Process *, Eterm);
  Uint erts_run_queues_len(Uint *, int, int, int);
  void erts_add_to_runq(Process *);
  Eterm erts_bound_schedulers_term(Process *c_p);
  Eterm erts_get_cpu_topology_term(Process *c_p, Eterm which);
  Eterm erts_get_schedulers_binds(Process *c_p);
  Eterm erts_set_cpu_topology(Process *c_p, Eterm term);
  Eterm erts_bind_schedulers(Process *c_p, Eterm how);
  ErtsRunQueue *erts_schedid2runq(Uint);
  Process *erts_schedule(ErtsSchedulerData *, Process*, int);
  void erts_schedule_misc_op(void (*)(void *), void *);
  Eterm erl_create_process(Process*, Eterm, Eterm, Eterm, ErlSpawnOpts*);
  void erts_set_self_exiting(Process *, Eterm);
  void erts_do_exit_process(Process*, Eterm);
  void erts_continue_exit_process(Process *);
  void erts_proc_exit_link(Process *, ErtsLink *, Uint16, Eterm, Eterm);
  /* Begin System profile */
  Uint erts_runnable_process_count(void);
  /* End System profile */
  void erts_init_empty_process(Process *p);
  void erts_cleanup_empty_process(Process* p);
  #ifdef DEBUG
  void erts_debug_verify_clean_empty_process(Process* p);
  #endif
  void erts_stack_dump(fmtfn_t to, void *to_arg, Process *);
  void erts_limited_stack_trace(fmtfn_t to, void *to_arg, Process *);
  void erts_program_counter_info(fmtfn_t to, void *to_arg, Process *);
  void erts_print_scheduler_info(fmtfn_t to, void *to_arg, ErtsSchedulerData *esdp);
  void erts_print_run_queue_info(fmtfn_t, void *to_arg, ErtsRunQueue*);
  void erts_dump_extended_process_state(fmtfn_t to, void *to_arg, erts_aint32_t psflg);
  void erts_dump_process_state(fmtfn_t to, void *to_arg, erts_aint32_t psflg);
  Eterm erts_process_info(Process *c_p, ErtsHeapFactory *hfact,
                          Process *rp, ErtsProcLocks rp_locks,
                          int *item_ix, int item_ix_len,
                          int flags, Uint reserve_size, Uint *reds);
  
  typedef struct {
      Process *c_p;
      Eterm reason;
      ErtsLink *dist_links;
      ErtsMonitor *dist_monitors;
      Eterm dist_state;
  } ErtsProcExitContext;
  int erts_proc_exit_handle_monitor(ErtsMonitor *mon, void *vctxt, Sint reds);
  int erts_proc_exit_handle_link(ErtsLink *lnk, void *vctxt, Sint reds);
  
  Eterm erts_get_process_priority(erts_aint32_t state);
  Eterm erts_set_process_priority(Process *p, Eterm prio);
  
  Uint erts_get_total_context_switches(void);
  void erts_get_total_reductions(Uint *, Uint *);
  void erts_get_exact_total_reductions(Process *, Uint *, Uint *);
  
  Eterm erts_fake_scheduler_bindings(Process *p, Eterm how);
  
  void erts_sched_stat_modify(int what);
  Eterm erts_sched_stat_term(Process *p, int total);
  
  void erts_free_proc(Process *);
  
  void erts_suspend(Process*, ErtsProcLocks, Port*);
  void erts_resume(Process*, ErtsProcLocks);
  int erts_resume_processes(ErtsProcList *);
  
  void erts_deep_process_dump(fmtfn_t, void *);
  
  Eterm erts_get_reader_groups_map(Process *c_p);
  Eterm erts_get_decentralized_counter_groups_map(Process *c_p);
  Eterm erts_debug_reader_groups_map(Process *c_p, int groups);
  
  Uint erts_debug_nbalance(void);
  
  #define ERTS_DEBUG_WAIT_COMPLETED_DEALLOCATIONS		(1 << 0)
  #define ERTS_DEBUG_WAIT_COMPLETED_TIMER_CANCELLATIONS	(1 << 1)
  #define ERTS_DEBUG_WAIT_COMPLETED_AUX_WORK		(1 << 2)
  
  int erts_debug_wait_completed(Process *c_p, int flags);
  
  Uint erts_process_memory(Process *c_p, int include_sigs_in_transit);
  
  #ifdef ERTS_DO_VERIFY_UNUSED_TEMP_ALLOC
  #  define ERTS_VERIFY_UNUSED_TEMP_ALLOC(P)					\
  do {										\
      ErtsSchedulerData *esdp__ = ((P)						\
  				 ? erts_proc_sched_data((Process *) (P))	\
  				 : erts_get_scheduler_data());			\
      if (esdp__ && !ERTS_SCHEDULER_IS_DIRTY(esdp__))				\
  	esdp__->verify_unused_temp_alloc(					\
  	    esdp__->verify_unused_temp_alloc_data);				\
  } while (0)
  #else
  #  define ERTS_VERIFY_UNUSED_TEMP_ALLOC(ESDP)
  #endif
  
  ErtsSchedulerData *erts_get_scheduler_data(void);
  
  void erts_schedule_process(Process *, erts_aint32_t, ErtsProcLocks);
  erts_aint32_t erts_proc_sys_schedule(Process *p, erts_aint32_t state,
                                       erts_aint32_t enable_flag);
  
  ERTS_GLB_INLINE void erts_proc_notify_new_message(Process *p, ErtsProcLocks locks);
  ERTS_GLB_INLINE void erts_schedule_dirty_sys_execution(Process *c_p);
  
  #if ERTS_GLB_INLINE_INCL_FUNC_DEF
  ERTS_GLB_INLINE void
  erts_proc_notify_new_message(Process *p, ErtsProcLocks locks)
  {
      /* No barrier needed, due to msg lock */
      erts_aint32_t state = erts_atomic32_read_nob(&p->state);
      if (!(state & ERTS_PSFLG_ACTIVE))
  	erts_schedule_process(p, state, locks);
  }
  
  ERTS_GLB_INLINE void
  erts_schedule_dirty_sys_execution(Process *c_p)
  {
      erts_aint32_t a, n, e;
  
      a = erts_atomic32_read_nob(&c_p->state);
  
      /*
       * Only a currently executing process schedules
       * itself for dirty-sys execution...
       */
  
      ASSERT(a & (ERTS_PSFLG_RUNNING|ERTS_PSFLG_RUNNING_SYS));
  
      /* Don't set dirty-active-sys if we are about to exit... */
  
      while (!(a & (ERTS_PSFLG_DIRTY_ACTIVE_SYS
                    | ERTS_PSFLG_EXITING))) {
          e = a;
          n = a | ERTS_PSFLG_DIRTY_ACTIVE_SYS;
          a = erts_atomic32_cmpxchg_mb(&c_p->state, n, e);
          if (a == e)
              break; /* dirty-active-sys set */
      }
  }
  
  #endif
  
  #if defined(ERTS_ENABLE_LOCK_CHECK)
  
  #define ERTS_PROCESS_LOCK_ONLY_LOCK_CHECK_PROTO__
  #include "erl_process_lock.h"
  #undef ERTS_PROCESS_LOCK_ONLY_LOCK_CHECK_PROTO__
  
  #define ERTS_LC_CHK_RUNQ_LOCK(RQ, L)				\
  do {									\
      if ((L))								\
  	ERTS_LC_ASSERT(erts_lc_runq_is_locked((RQ)));		\
      else								\
  	ERTS_LC_ASSERT(!erts_lc_runq_is_locked((RQ)));		\
  } while (0)
  #else
  #define ERTS_LC_CHK_RUNQ_LOCK(RQ, L)
  #endif
  
  void *erts_psd_set_init(Process *p, int ix, void *data);
  
  ERTS_GLB_INLINE void *
  erts_psd_get(Process *p, int ix);
  ERTS_GLB_INLINE void *
  erts_psd_set(Process *p, int ix, void *new);
  
  #if ERTS_GLB_INLINE_INCL_FUNC_DEF
  
  ERTS_GLB_INLINE void *
  erts_psd_get(Process *p, int ix)
  {
      ErtsPSD *psd;
  #if defined(ERTS_ENABLE_LOCK_CHECK)
      ErtsProcLocks locks = erts_proc_lc_my_proc_locks(p);
      if (ERTS_LC_PSD_ANY_LOCK == erts_psd_required_locks[ix].get_locks)
  	ERTS_LC_ASSERT(locks || erts_thr_progress_is_blocking());
      else {
  	locks &= erts_psd_required_locks[ix].get_locks;
  	ERTS_LC_ASSERT(erts_psd_required_locks[ix].get_locks == locks
  			   || erts_thr_progress_is_blocking());
      }
  #endif
  
      psd = (ErtsPSD *) erts_atomic_read_nob(&p->psd);
      ASSERT(0 <= ix && ix < ERTS_PSD_SIZE);
      if (!psd)
  	return NULL;
      ERTS_THR_DATA_DEPENDENCY_READ_MEMORY_BARRIER;
      return psd->data[ix];
  }
  
  ERTS_GLB_INLINE void *
  erts_psd_set(Process *p, int ix, void *data)
  {
      ErtsPSD *psd;
  #if defined(ERTS_ENABLE_LOCK_CHECK)
      ErtsProcLocks locks = erts_proc_lc_my_proc_locks(p);
      erts_aint32_t state = state = erts_atomic32_read_nob(&p->state);
      if (!(state & ERTS_PSFLG_FREE)) {
  	if (ERTS_LC_PSD_ANY_LOCK == erts_psd_required_locks[ix].set_locks)
  	    ERTS_LC_ASSERT(locks || erts_thr_progress_is_blocking());
  	else {
  	    locks &= erts_psd_required_locks[ix].set_locks;
  	    ERTS_LC_ASSERT(erts_psd_required_locks[ix].set_locks == locks
  			       || erts_thr_progress_is_blocking());
  	}
      }
  #endif
      psd = (ErtsPSD *) erts_atomic_read_nob(&p->psd);
      ASSERT(0 <= ix && ix < ERTS_PSD_SIZE);
      if (psd) {
  	void *old;
  #ifdef ETHR_ORDERED_READ_DEPEND
  	ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore);
  #else
  	ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreStore);
  #endif
  	old = psd->data[ix];
  	psd->data[ix] = data;
  	return old;
      }
  
      if (!data)
  	return NULL;
  
      return erts_psd_set_init(p, ix, data);
  }
  
  #endif
  
  #define ERTS_PROC_SCHED_ID(P, ID) \
    ((UWord) erts_psd_set((P), ERTS_PSD_SCHED_ID, (void *) (ID)))
  
  #define ERTS_PROC_GET_SAVED_CALLS_BUF(P) \
    ((struct saved_calls *) erts_psd_get((P), ERTS_PSD_SAVED_CALLS_BUF))
  #define ERTS_PROC_SET_SAVED_CALLS_BUF(P, SCB) \
    ((struct saved_calls *) erts_psd_set((P), ERTS_PSD_SAVED_CALLS_BUF, (void *) (SCB)))
  
  #define ERTS_PROC_GET_CALL_TIME(P) \
    ((process_breakpoint_time_t *) erts_psd_get((P), ERTS_PSD_CALL_TIME_BP))
  #define ERTS_PROC_SET_CALL_TIME(P, PBT) \
    ((process_breakpoint_time_t *) erts_psd_set((P), ERTS_PSD_CALL_TIME_BP, (void *) (PBT)))
  
  #define ERTS_PROC_GET_DELAYED_GC_TASK_QS(P) \
      ((ErtsProcSysTaskQs *) erts_psd_get((P), ERTS_PSD_DELAYED_GC_TASK_QS))
  #define ERTS_PROC_SET_DELAYED_GC_TASK_QS(P, PBT) \
      ((ErtsProcSysTaskQs *) erts_psd_set((P), ERTS_PSD_DELAYED_GC_TASK_QS, (void *) (PBT)))
  
  #define ERTS_PROC_GET_NIF_TRAP_EXPORT(P) \
      erts_psd_get((P), ERTS_PSD_NIF_TRAP_EXPORT)
  #define ERTS_PROC_SET_NIF_TRAP_EXPORT(P, NTE) \
      erts_psd_set((P), ERTS_PSD_NIF_TRAP_EXPORT, (void *) (NTE))
  
  #define ERTS_PROC_GET_DIST_ENTRY(P) \
      ((DistEntry *) erts_psd_get((P), ERTS_PSD_DIST_ENTRY))
  #define ERTS_PROC_SET_DIST_ENTRY(P, DE) \
      ((DistEntry *) erts_psd_set((P), ERTS_PSD_DIST_ENTRY, (void *) (DE)))
  
  #define ERTS_PROC_GET_PENDING_SUSPEND(P) \
      ((void *) erts_psd_get((P), ERTS_PSD_PENDING_SUSPEND))
  #define ERTS_PROC_SET_PENDING_SUSPEND(P, PS) \
      ((void *) erts_psd_set((P), ERTS_PSD_PENDING_SUSPEND, (void *) (PS)))
  
  #ifdef HIPE
  #define ERTS_PROC_GET_SUSPENDED_SAVED_CALLS_BUF(P) \
    ((struct saved_calls *) erts_psd_get((P), ERTS_PSD_SUSPENDED_SAVED_CALLS_BUF))
  #define ERTS_PROC_SET_SUSPENDED_SAVED_CALLS_BUF(P, SCB) \
    ((struct saved_calls *) erts_psd_set((P), ERTS_PSD_SUSPENDED_SAVED_CALLS_BUF, (void *) (SCB)))
  #endif
  
  ERTS_GLB_INLINE Eterm erts_proc_get_error_handler(Process *p);
  ERTS_GLB_INLINE Eterm erts_proc_set_error_handler(Process *p, Eterm handler);
  
  #if ERTS_GLB_INLINE_INCL_FUNC_DEF
  ERTS_GLB_INLINE Eterm
  erts_proc_get_error_handler(Process *p)
  {
      void *val = erts_psd_get(p, ERTS_PSD_ERROR_HANDLER);
      if (!val)
  	return am_error_handler;
      else {
  	ASSERT(is_atom(((Eterm) (UWord) val)));
  	return (Eterm) (UWord) val;
      }
  }
  
  ERTS_GLB_INLINE Eterm
  erts_proc_set_error_handler(Process *p, Eterm handler)
  {
      void *old_val;
      void *new_val;
      ASSERT(is_atom(handler));
      new_val = (handler == am_error_handler) ? NULL : (void *) (UWord) handler;
      old_val = erts_psd_set(p, ERTS_PSD_ERROR_HANDLER, new_val);
      if (!old_val)
  	return am_error_handler;
      else {
  	ASSERT(is_atom(((Eterm) (UWord) old_val)));
  	return (Eterm) (UWord) old_val;
      }
  }
  
  #endif
  
  #ifdef ERTS_INCLUDE_SCHEDULER_INTERNALS
  
  
  #include "erl_thr_progress.h"
  
  extern erts_atomic_t erts_migration_paths;
  
  #if ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT
  int erts_get_sched_util(ErtsRunQueue *rq,
  			int initially_locked,
  			int short_interval);
  #endif
  
  
  ERTS_GLB_INLINE ErtsMigrationPaths *erts_get_migration_paths_managed(void);
  ERTS_GLB_INLINE ErtsMigrationPaths *erts_get_migration_paths(void);
  ERTS_GLB_INLINE ErtsRunQueue *erts_check_emigration_need(ErtsRunQueue *c_rq,
  							 int prio);
  #if ERTS_GLB_INLINE_INCL_FUNC_DEF
  
  ERTS_GLB_INLINE ErtsMigrationPaths *
  erts_get_migration_paths_managed(void)
  {
      return (ErtsMigrationPaths *) erts_atomic_read_ddrb(&erts_migration_paths);
  }
  
  ERTS_GLB_INLINE ErtsMigrationPaths *
  erts_get_migration_paths(void)
  {
      if (erts_thr_progress_is_managed_thread())
  	return erts_get_migration_paths_managed();
      else
  	return NULL;
  }
  
  ERTS_GLB_INLINE ErtsRunQueue *
  erts_check_emigration_need(ErtsRunQueue *c_rq, int prio)
  {
      ErtsMigrationPaths *mps = erts_get_migration_paths();
      ErtsMigrationPath *mp;
      Uint32 flags;
  
      if (!mps)
  	return NULL;
  
      mp = &mps->mpath[c_rq->ix];
      flags = mp->flags;
  
      if (ERTS_CHK_RUNQ_FLG_EMIGRATE(flags, prio)) {
  	int len;
  
  	if (ERTS_CHK_RUNQ_FLG_EVACUATE(flags, prio)) {
  	    /* force emigration */
  	    return mp->prio[prio].runq;
  	}
  
  	if (flags & ERTS_RUNQ_FLG_INACTIVE) {
  	    /*
  	     * Run queue was inactive at last balance. Verify that
  	     * it still is before forcing emigration.
  	     */
  	    if (ERTS_RUNQ_FLGS_GET(c_rq) & ERTS_RUNQ_FLG_INACTIVE)
  		return mp->prio[prio].runq;
  	}
  
  #if ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT
  	if (mp->sched_util) {
  	    ErtsRunQueue *rq = mp->prio[prio].runq;
  	    /* No migration if other is non-empty */
  	    if (!(ERTS_RUNQ_FLGS_GET(rq) & ERTS_RUNQ_FLG_NONEMPTY)
  		&& erts_get_sched_util(rq, 0, 1) < mp->prio[prio].limit.other
  		&& erts_get_sched_util(c_rq, 0, 1) > mp->prio[prio].limit.this) {
  		return rq;
  	    }
  	}
  	else
  #endif
  	{
  
  	    if (prio == ERTS_PORT_PRIO_LEVEL)
  		len = RUNQ_READ_LEN(&c_rq->ports.info.len);
  	    else
  		len = RUNQ_READ_LEN(&c_rq->procs.prio_info[prio].len);
  
  	    if (len > mp->prio[prio].limit.this) {
  		ErtsRunQueue *n_rq = mp->prio[prio].runq;
  		if (n_rq) {
  		    if (prio == ERTS_PORT_PRIO_LEVEL)
  			len = RUNQ_READ_LEN(&n_rq->ports.info.len);
  		    else
  			len = RUNQ_READ_LEN(&n_rq->procs.prio_info[prio].len);
  
  		    if (len < mp->prio[prio].limit.other)
  			return n_rq;
  		}
  	    }
  	}
      }
      return NULL;
  }
  
  #endif
  
  
  #endif
  
  ERTS_GLB_INLINE ErtsSchedulerData *erts_proc_sched_data(Process *c_p);
  ERTS_GLB_INLINE int erts_is_scheduler_bound(ErtsSchedulerData *esdp);
  ERTS_GLB_INLINE Process *erts_get_current_process(void);
  ERTS_GLB_INLINE Eterm erts_get_current_pid(void);
  ERTS_GLB_INLINE Uint erts_get_scheduler_id(void);
  ERTS_GLB_INLINE void erts_init_runq_proc(Process *p, ErtsRunQueue *rq, int bnd);
  ERTS_GLB_INLINE ErtsRunQueue *erts_set_runq_proc(Process *p, ErtsRunQueue *rq, int *boundp);
  ERTS_GLB_INLINE int erts_try_change_runq_proc(Process *p, ErtsRunQueue *rq);
  ERTS_GLB_INLINE ErtsRunQueue *erts_bind_runq_proc(Process *p, int bind);
  ERTS_GLB_INLINE int erts_proc_runq_is_bound(Process *p);
  ERTS_GLB_INLINE ErtsRunQueue *erts_get_runq_proc(Process *p, int *boundp);
  ERTS_GLB_INLINE ErtsRunQueue *erts_get_runq_current(ErtsSchedulerData *esdp);
  ERTS_GLB_INLINE void erts_runq_lock(ErtsRunQueue *rq);
  ERTS_GLB_INLINE int erts_runq_trylock(ErtsRunQueue *rq);
  ERTS_GLB_INLINE void erts_runq_unlock(ErtsRunQueue *rq);
  ERTS_GLB_INLINE void erts_xrunq_lock(ErtsRunQueue *rq, ErtsRunQueue *xrq);
  ERTS_GLB_INLINE void erts_xrunq_unlock(ErtsRunQueue *rq, ErtsRunQueue *xrq);
  ERTS_GLB_INLINE void erts_runqs_lock(ErtsRunQueue *rq1, ErtsRunQueue *rq2);
  ERTS_GLB_INLINE void erts_runqs_unlock(ErtsRunQueue *rq1, ErtsRunQueue *rq2);
  
  ERTS_GLB_INLINE ErtsMessage *erts_alloc_message_heap_state(Process *pp,
  							   erts_aint32_t *psp,
  							   ErtsProcLocks *plp,
  							   Uint sz,
  							   Eterm **hpp,
  							   ErlOffHeap **ohpp);
  ERTS_GLB_INLINE ErtsMessage *erts_alloc_message_heap(Process *pp,
  						     ErtsProcLocks *plp,
  						     Uint sz,
  						     Eterm **hpp,
  						     ErlOffHeap **ohpp);
  
  ERTS_GLB_INLINE void erts_shrink_message_heap(ErtsMessage **msgpp, Process *pp,
  					      Eterm *start_hp, Eterm *used_hp, Eterm *end_hp,
  					      Eterm *brefs, Uint brefs_size);
  
  #if ERTS_GLB_INLINE_INCL_FUNC_DEF
  
  ERTS_GLB_INLINE
  ErtsSchedulerData *erts_proc_sched_data(Process *c_p)
  {
      ErtsSchedulerData *esdp;
      ASSERT(c_p);
      esdp = c_p->scheduler_data;
      if (esdp) {
  	ASSERT(esdp == erts_get_scheduler_data());
  	ASSERT(!ERTS_SCHEDULER_IS_DIRTY(esdp));
      }
      else {
  	esdp = erts_get_scheduler_data();
  	ASSERT(esdp);
  	ASSERT(ERTS_SCHEDULER_IS_DIRTY(esdp));
      }
      ASSERT(esdp);
      return esdp;
  }
  
  ERTS_GLB_INLINE
  int erts_is_scheduler_bound(ErtsSchedulerData *esdp)
  {
      if (!esdp)
  	esdp = erts_get_scheduler_data();
      ASSERT(esdp);
      return esdp->cpu_id >= 0;
  }
  
  ERTS_GLB_INLINE
  Process *erts_get_current_process(void)
  {
      ErtsSchedulerData *esdp = erts_get_scheduler_data();
      return esdp ? esdp->current_process : NULL;
  }
  
  ERTS_GLB_INLINE
  Eterm erts_get_current_pid(void)
  {
      Process *proc = erts_get_current_process();
      return proc ? proc->common.id : THE_NON_VALUE;
  }
  
  ERTS_GLB_INLINE
  Uint erts_get_scheduler_id(void)
  {
      ErtsSchedulerData *esdp = erts_get_scheduler_data();
      if (esdp && ERTS_SCHEDULER_IS_DIRTY(esdp))
  	return 0;
      else
  	return esdp ? esdp->no : (Uint) 0;
  }
  
  /**
   * Init run-queue of process.
   *
   * @param p[in,out]     Process
   * @param rq[in]        Run-queue that process will be assigned to
   * @param bnd[in,out]   If non-zero binds process to run-queue.
   */
  
  ERTS_GLB_INLINE void
  erts_init_runq_proc(Process *p, ErtsRunQueue *rq, int bnd)
  {
      erts_aint_t rqint = (erts_aint_t) rq;
      if (bnd)
          rqint |= ERTS_RUNQ_BOUND_FLAG;
      erts_atomic_init_nob(&p->run_queue, rqint);
  }
  
  /**
   * Forcibly set run-queue of process.
   *
   * @param p[in,out]     Process
   * @param rq[in]        Run-queue that process will be assigned to
   * @param bndp[in,out]  Pointer to integer. On input non-zero
   *                      value causes the process to be bound to
   *                      the run-queue. On output, indicating
   *                      wether process previously was bound or
   *                      not.
   * @return              Previous run-queue.
   */
  
  ERTS_GLB_INLINE ErtsRunQueue *
  erts_set_runq_proc(Process *p, ErtsRunQueue *rq, int *bndp)
  {
      erts_aint_t rqint = (erts_aint_t) rq;
      ASSERT(bndp);
      ASSERT(rq);
      if (*bndp)
          rqint |= ERTS_RUNQ_BOUND_FLAG;
      rqint = erts_atomic_xchg_nob(&p->run_queue, rqint);
      *bndp = (int) (rqint & ERTS_RUNQ_BOUND_FLAG);
      return (ErtsRunQueue *) (rqint & ERTS_RUNQ_POINTER_MASK);
  }
  
  /**
   * Try to change run-queue assignment of a process.
   *
   * @param p[in,out]     Process
   * @param rq[int]       Run-queue that process will be assigned to
   * @return              Non-zero if the run-queue assignment was
   *                      successfully changed.
   */
  
  ERTS_GLB_INLINE int
  erts_try_change_runq_proc(Process *p, ErtsRunQueue *rq)
  {
      erts_aint_t old_rqint, new_rqint;
  
      ASSERT(rq);
  
      new_rqint = (erts_aint_t) rq;
      old_rqint = (erts_aint_t) erts_atomic_read_nob(&p->run_queue);
      while (1) {
          erts_aint_t act_rqint;
  
          if (old_rqint & ERTS_RUNQ_BOUND_FLAG)
              return 0;
  
          act_rqint = erts_atomic_cmpxchg_nob(&p->run_queue,
                                              new_rqint,
                                              old_rqint);
          if (act_rqint == old_rqint)
              return !0;
  
          old_rqint = act_rqint;
      }
  }
  
  /**
   *
   * Bind or unbind process to/from currently used run-queue.
   *
   * @param p             Process
   * @param bind          Bind if non-zero; otherwise unbind
   * @return              Pointer to previously bound run-queue,
   *                      or NULL if previously unbound
   */
  
  ERTS_GLB_INLINE ErtsRunQueue *
  erts_bind_runq_proc(Process *p, int bind)
  {
      erts_aint_t rqint;
      if (bind)
          rqint = erts_atomic_read_bor_nob(&p->run_queue,
                                           ERTS_RUNQ_BOUND_FLAG);
      else
          rqint = erts_atomic_read_band_nob(&p->run_queue,
                                            ~ERTS_RUNQ_BOUND_FLAG);
      if (rqint & ERTS_RUNQ_BOUND_FLAG)
          return (ErtsRunQueue *) (rqint & ERTS_RUNQ_POINTER_MASK);
      else
          return NULL;
  }
  
  /**
   * Determine wether a process is bound to a run-queue or not.
   *
   * @return              Returns a non-zero value if bound,
   *                      and zero of not bound.
   */
  
  ERTS_GLB_INLINE int
  erts_proc_runq_is_bound(Process *p)
  {
      erts_aint_t rqint = erts_atomic_read_nob(&p->run_queue);
      return (int) (rqint & ERTS_RUNQ_BOUND_FLAG);
  }
  
  /**
   * Set run-queue of process.
   *
   * @param p[in,out]     Process
   * @param bndp[out]     Pointer to integer. If non-NULL pointer,
   *                      the integer will be set to a non-zero
   *                      value if the process is bound to the
   *                      run-queue.
   * @return              Pointer to the normal run-queue that
   *                      the process currently is assigend to.
   *                      A process is always assigned to a
   *                      normal run-queue.
   */
  
  ERTS_GLB_INLINE ErtsRunQueue *
  erts_get_runq_proc(Process *p, int *bndp)
  {
      erts_aint_t rqint = erts_atomic_read_nob(&p->run_queue);
      ErtsRunQueue *rq;
      if (bndp)
          *bndp = (int) (rqint & ERTS_RUNQ_BOUND_FLAG);
      rqint &= ERTS_RUNQ_POINTER_MASK;
      rq = (ErtsRunQueue *) rqint;
      ASSERT(rq);
      return rq;
  }
  
  ERTS_GLB_INLINE ErtsRunQueue *
  erts_get_runq_current(ErtsSchedulerData *esdp)
  {
      ASSERT(!esdp || esdp == erts_get_scheduler_data());
      if (!esdp)
  	esdp = erts_get_scheduler_data();
      return esdp->run_queue;
  }
  
  ERTS_GLB_INLINE void
  erts_runq_lock(ErtsRunQueue *rq)
  {
      erts_mtx_lock(&rq->mtx);
  }
  
  ERTS_GLB_INLINE int
  erts_runq_trylock(ErtsRunQueue *rq)
  {
      return erts_mtx_trylock(&rq->mtx);
  }
  
  ERTS_GLB_INLINE void
  erts_runq_unlock(ErtsRunQueue *rq)
  {
      erts_mtx_unlock(&rq->mtx);
  }
  
  ERTS_GLB_INLINE void
  erts_xrunq_lock(ErtsRunQueue *rq, ErtsRunQueue *xrq)
  {
      ERTS_LC_ASSERT(erts_lc_mtx_is_locked(&rq->mtx));
      if (xrq != rq) {
  	if (erts_mtx_trylock(&xrq->mtx) == EBUSY) {
  	    if (rq < xrq)
  		erts_mtx_lock(&xrq->mtx);
  	    else {
  		erts_mtx_unlock(&rq->mtx);
  		erts_mtx_lock(&xrq->mtx);
  		erts_mtx_lock(&rq->mtx);
  	    }
  	}
      }
  }
  
  ERTS_GLB_INLINE void
  erts_xrunq_unlock(ErtsRunQueue *rq, ErtsRunQueue *xrq)
  {
      if (xrq != rq)
  	erts_mtx_unlock(&xrq->mtx);
  }
  
  ERTS_GLB_INLINE void
  erts_runqs_lock(ErtsRunQueue *rq1, ErtsRunQueue *rq2)
  {
      ASSERT(rq1 && rq2);
      if (rq1 == rq2)
  	erts_mtx_lock(&rq1->mtx);
      else if (rq1 < rq2) {
  	erts_mtx_lock(&rq1->mtx);
  	erts_mtx_lock(&rq2->mtx);
      }
      else {
  	erts_mtx_lock(&rq2->mtx);
  	erts_mtx_lock(&rq1->mtx);
      }
  }
  
  ERTS_GLB_INLINE void
  erts_runqs_unlock(ErtsRunQueue *rq1, ErtsRunQueue *rq2)
  {
      ASSERT(rq1 && rq2);
      erts_mtx_unlock(&rq1->mtx);
      if (rq1 != rq2)
  	erts_mtx_unlock(&rq2->mtx);
  }
  
  ERTS_GLB_INLINE ErtsMessage *
  erts_alloc_message_heap_state(Process *pp,
  			      erts_aint32_t *psp,
  			      ErtsProcLocks *plp,
  			      Uint sz,
  			      Eterm **hpp,
  			      ErlOffHeap **ohpp)
  {
      int on_heap;
      ErtsMessage *mp;
  
      if ((*psp) & ERTS_PSFLG_OFF_HEAP_MSGQ) {
  	mp = erts_alloc_message(sz, hpp);
  	*ohpp = sz == 0 ? NULL : &mp->hfrag.off_heap;
  	return mp;
      }
  
      mp = erts_try_alloc_message_on_heap(pp, psp, plp, sz, hpp, ohpp, &on_heap);
      ASSERT(pp || !on_heap);
      return mp;
  }
  
  ERTS_GLB_INLINE ErtsMessage *
  erts_alloc_message_heap(Process *pp,
  			ErtsProcLocks *plp,
  			Uint sz,
  			Eterm **hpp,
  			ErlOffHeap **ohpp)
  {
      erts_aint32_t state = pp ? erts_atomic32_read_nob(&pp->state) : 0;
      return erts_alloc_message_heap_state(pp, &state, plp, sz, hpp, ohpp);
  }
  
  ERTS_GLB_INLINE void
  erts_shrink_message_heap(ErtsMessage **msgpp, Process *pp,
  			 Eterm *start_hp, Eterm *used_hp, Eterm *end_hp,
  			 Eterm *brefs, Uint brefs_size)
  {
      ASSERT(start_hp <= used_hp && used_hp <= end_hp);
      if ((*msgpp)->data.attached == ERTS_MSG_COMBINED_HFRAG)
  	*msgpp = erts_shrink_message(*msgpp, used_hp - start_hp,
  				     brefs, brefs_size);
      else if (!(*msgpp)->data.attached) {
  	ERTS_LC_ASSERT(ERTS_PROC_LOCK_MAIN
  			   & erts_proc_lc_my_proc_locks(pp));
  	HRelease(pp, end_hp, used_hp);
      }
      else {
  	ErlHeapFragment *hfrag = (*msgpp)->data.heap_frag;
  	if (start_hp != used_hp)
  	    hfrag = erts_resize_message_buffer(hfrag, used_hp - start_hp,
  					       brefs, brefs_size);
  	else {
  	    free_message_buffer(hfrag);
  	    hfrag = NULL;
  	}
  	(*msgpp)->data.heap_frag = hfrag;
      }
  }
  
  #endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */
  
  ERTS_GLB_INLINE ErtsAtomCacheMap *erts_get_atom_cache_map(Process *c_p);
  
  #if ERTS_GLB_INLINE_INCL_FUNC_DEF
  ERTS_GLB_INLINE ErtsAtomCacheMap *
  erts_get_atom_cache_map(Process *c_p)
  {
      ErtsSchedulerData *esdp = (c_p
  			       ? erts_proc_sched_data(c_p)
  			       : erts_get_scheduler_data());
      ASSERT(esdp);
      return &esdp->atom_cache_map;
  }
  #endif
  
  #ifdef __WIN32__
  /*
   * Don't want erts_time2reds() inlined in beam_emu.c on windows since
   * it is compiled with gcc which fails on it. Implementation is in
   * erl_process.c on windows.
   */
  #  define ERTS_TIME2REDS_IMPL__ erts_time2reds__
  #else
  #  define ERTS_TIME2REDS_IMPL__ erts_time2reds
  #endif
  
  ERTS_GLB_INLINE Sint64 ERTS_TIME2REDS_IMPL__(ErtsMonotonicTime start,
  					     ErtsMonotonicTime end);
  
  #if ERTS_GLB_INLINE_INCL_FUNC_DEF
  ERTS_GLB_INLINE Sint64
  ERTS_TIME2REDS_IMPL__(ErtsMonotonicTime start, ErtsMonotonicTime end)
  {
      ErtsMonotonicTime time = end - start;
      ASSERT(time >= 0);
      time = ERTS_MONOTONIC_TO_USEC(time);
      if (time == 0)
  	return (Sint64) 1; /* At least one reduction */
      /* Currently two reductions per micro second */
      time *= (CONTEXT_REDS-1)/1000 + 1;
      return (Sint64) time;
  }
  #endif
  
  Process *erts_try_lock_sig_free_proc(Eterm pid,
                                       ErtsProcLocks locks,
                                       erts_aint32_t *statep);
  
  #ifdef DEBUG
  #define ERTS_ASSERT_IS_NOT_EXITING(P) \
      do { ASSERT(!ERTS_PROC_IS_EXITING((P))); } while (0)
  #else
  #define ERTS_ASSERT_IS_NOT_EXITING(P)
  #endif
  
  
  #define ERTS_PROC_IS_EXITING(P) \
      (ERTS_PSFLG_EXITING & erts_atomic32_read_acqb(&(P)->state))
  
  
  /* Minimum NUMBER of processes for a small system to start */
  #define ERTS_MIN_PROCESSES		1024
  #if ERTS_MIN_PROCESSES < ERTS_NO_OF_PIX_LOCKS
  #undef ERTS_MIN_PROCESSES
  #define ERTS_MIN_PROCESSES		ERTS_NO_OF_PIX_LOCKS
  #endif
  
  void erts_notify_inc_runq(ErtsRunQueue *runq);
  
  void erts_sched_finish_poke(ErtsSchedulerSleepInfo *, erts_aint32_t);
  ERTS_GLB_INLINE void erts_sched_poke(ErtsSchedulerSleepInfo *ssi);
  void erts_aux_thread_poke(void);
  
  #if ERTS_GLB_INLINE_INCL_FUNC_DEF
  
  ERTS_GLB_INLINE void
  erts_sched_poke(ErtsSchedulerSleepInfo *ssi)
  {
      erts_aint32_t flags;
      ERTS_THR_MEMORY_BARRIER;
      flags = erts_atomic32_read_nob(&ssi->flags);
      if (flags & ERTS_SSI_FLG_SLEEPING) {
  	flags = erts_atomic32_read_band_nob(&ssi->flags, ~ERTS_SSI_FLGS_SLEEP);
  	erts_sched_finish_poke(ssi, flags);
      }
  }
  
  
  #endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */
  
  
  #include "erl_process_lock.h"
  
  #undef ERTS_INCLUDE_SCHEDULER_INTERNALS
  
  #endif
  
  
  void erts_halt(int code);
  extern erts_atomic32_t erts_halt_progress;
  extern int erts_halt_code;
  