text: /*
   * %CopyrightBegin%
   *
   * Copyright Ericsson AB 2007-2018. All Rights Reserved.
   *
   * Licensed under the Apache License, Version 2.0 (the "License");
   * you may not use this file except in compliance with the License.
   * You may obtain a copy of the License at
   *
   *     http://www.apache.org/licenses/LICENSE-2.0
   *
   * Unless required by applicable law or agreed to in writing, software
   * distributed under the License is distributed on an "AS IS" BASIS,
   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   * See the License for the specific language governing permissions and
   * limitations under the License.
   *
   * %CopyrightEnd%
   */
  
  
  /*
   * Description:	Impementation of Erlang process locks.
   *
   * Author: 	Rickard Green
   */
  
  /*
   * A short explanation of the process lock implementation:
   *     Each process has a lock bitfield and a number of lock wait
   *   queues.
   *     The bit field contains of a number of lock flags (L1, L2, ...)
   *   and a number of wait flags (W1, W2, ...). Each lock flag has a
   *   corresponding wait flag. The bit field isn't guarranteed to be
   *   larger than 32-bits which sets a maximum of 16 different locks
   *   per process. Currently, only 4 locks per process are used. The
   *   bit field is operated on by use of atomic operations (custom
   *   made bitwise atomic operations). When a lock is locked the
   *   corresponding lock bit is set. When a thread is waiting on a
   *   lock the wait flag for the lock is set.
   *     The process table is protected by pix (process index) locks
   *   which is spinlocks that protects a number of process indices in
   *   the process table. The pix locks also protects the lock queues
   *   and modifications of wait flags.
   *     When acquiring a process lock we first try to set the lock
   *   flag. If we are able to set the lock flag and the wait flag
   *   isn't set we are done. If the lock flag was already set we
   *   have to acquire the pix lock, set the wait flag, and put
   *   ourselves in the wait queue.
   *   Process locks will always be acquired in fifo order.
   *     When releasing a process lock we first unset all lock flags
   *   whose corresponding wait flag is clear (which will succeed).
   *   If wait flags were set for the locks being released, we acquire
   *   the pix lock, and transfer the lock to the first thread
   *   in the wait queue.
   *     Note that wait flags may be read without the pix lock, but
   *   it is important that wait flags only are modified when the pix
   *   lock is held.
   *     This implementation assumes that erts_atomic_or_retold()
   *   provides necessary memorybarriers for a lock operation, and that
   *   erts_atomic_and_retold() provides necessary memorybarriers
   *   for an unlock operation.
   */
  
  #ifdef HAVE_CONFIG_H
  #  include "config.h"
  #endif
  
  #include "erl_process.h"
  #include "erl_thr_progress.h"
  
  
  #if ERTS_PROC_LOCK_OWN_IMPL
  
  #define ERTS_PROC_LOCK_SPIN_COUNT_MAX  2000
  #define ERTS_PROC_LOCK_SPIN_COUNT_SCHED_INC 32
  #define ERTS_PROC_LOCK_SPIN_COUNT_BASE 1000
  #define ERTS_PROC_LOCK_AUX_SPIN_COUNT 50
  
  #define ERTS_PROC_LOCK_SPIN_UNTIL_YIELD 25
  
  #ifdef ERTS_PROC_LOCK_DEBUG
  #define ERTS_PROC_LOCK_HARD_DEBUG
  #endif
  
  #ifdef ERTS_PROC_LOCK_HARD_DEBUG
  static void check_queue(erts_proc_lock_t *lck);
  #endif
  
  #if SIZEOF_INT < 4
  #error "The size of the 'uflgs' field of the erts_tse_t type is too small"
  #endif
  
  static int proc_lock_spin_count;
  static int aux_thr_proc_lock_spin_count;
  
  static void cleanup_tse(void);
  
  #endif /* ERTS_PROC_LOCK_OWN_IMPL */
  
  #ifdef ERTS_ENABLE_LOCK_CHECK
  static struct {
      Sint16 proc_lock_main;
      Sint16 proc_lock_msgq;
      Sint16 proc_lock_btm;
      Sint16 proc_lock_status;
      Sint16 proc_lock_trace;
  } lc_id;
  #endif
  
  erts_pix_lock_t erts_pix_locks[ERTS_NO_OF_PIX_LOCKS];
  
  void
  erts_init_proc_lock(int cpus)
  {
      int i;
      for (i = 0; i < ERTS_NO_OF_PIX_LOCKS; i++) {
          erts_mtx_init(&erts_pix_locks[i].u.mtx, "pix_lock", make_small(i),
              ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_PROCESS);
      }
  #if ERTS_PROC_LOCK_OWN_IMPL
      erts_thr_install_exit_handler(cleanup_tse);
      if (cpus > 1) {
  	proc_lock_spin_count = ERTS_PROC_LOCK_SPIN_COUNT_BASE;
  	proc_lock_spin_count += (ERTS_PROC_LOCK_SPIN_COUNT_SCHED_INC
  				 * ((int) erts_no_schedulers));
  	aux_thr_proc_lock_spin_count = ERTS_PROC_LOCK_AUX_SPIN_COUNT;
      }
      else if (cpus == 1) {
  	proc_lock_spin_count = 0;
  	aux_thr_proc_lock_spin_count = 0;
      }
      else { /* No of cpus unknown. Assume multi proc, but be conservative. */
  	proc_lock_spin_count = ERTS_PROC_LOCK_SPIN_COUNT_BASE/2;
  	aux_thr_proc_lock_spin_count = ERTS_PROC_LOCK_AUX_SPIN_COUNT/2;
      }
      if (proc_lock_spin_count > ERTS_PROC_LOCK_SPIN_COUNT_MAX)
  	proc_lock_spin_count = ERTS_PROC_LOCK_SPIN_COUNT_MAX;
  #endif
  #ifdef ERTS_ENABLE_LOCK_CHECK
      lc_id.proc_lock_main	= erts_lc_get_lock_order_id("proc_main");
      lc_id.proc_lock_msgq	= erts_lc_get_lock_order_id("proc_msgq");
      lc_id.proc_lock_btm		= erts_lc_get_lock_order_id("proc_btm");
      lc_id.proc_lock_status	= erts_lc_get_lock_order_id("proc_status");
      lc_id.proc_lock_trace	= erts_lc_get_lock_order_id("proc_trace");
  #endif
  }
  
  #if ERTS_PROC_LOCK_OWN_IMPL
  
  #ifdef ERTS_ENABLE_LOCK_CHECK
  #define CHECK_UNUSED_TSE(W) ERTS_LC_ASSERT((W)->uflgs == 0)
  #else
  #define CHECK_UNUSED_TSE(W)
  #endif
  
  static ERTS_INLINE erts_tse_t *
  tse_fetch(erts_pix_lock_t *pix_lock)
  {
      erts_tse_t *tse = erts_tse_fetch();
      tse->uflgs = 0;
      return tse;
  }
  
  static ERTS_INLINE void
  tse_return(erts_tse_t *tse)
  {
      CHECK_UNUSED_TSE(tse);
      erts_tse_return(tse);
  }
  
  static void
  cleanup_tse(void)
  {
      erts_tse_t *tse = erts_tse_fetch();
      if (tse)
  	erts_tse_return(tse);
  }
  
  
  /*
   * Waiters are queued in a circular double linked list;
   * where lck->queue[lock_ix] is the first waiter in queue, and
   * lck->queue[lock_ix]->prev is the last waiter in queue.
   */
  
  static ERTS_INLINE void
  enqueue_waiter(erts_proc_lock_t *lck, int ix, erts_tse_t *wtr)
  {
      if (!lck->queue[ix]) {
  	lck->queue[ix] = wtr;
  	wtr->next = wtr;
  	wtr->prev = wtr;
      }
      else {
  	ERTS_LC_ASSERT(lck->queue[ix]->next && lck->queue[ix]->prev);
  	wtr->next = lck->queue[ix];
  	wtr->prev = lck->queue[ix]->prev;
  	wtr->prev->next = wtr;
  	lck->queue[ix]->prev = wtr;
      }
  }
  
  static erts_tse_t *
  dequeue_waiter(erts_proc_lock_t *lck, int ix)
  {
      erts_tse_t *wtr = lck->queue[ix];
      ERTS_LC_ASSERT(lck->queue[ix]);
      if (wtr->next == wtr) {
  	ERTS_LC_ASSERT(lck->queue[ix]->prev == wtr);
  	lck->queue[ix] = NULL;
      }
      else {
  	ERTS_LC_ASSERT(wtr->next != wtr);
  	ERTS_LC_ASSERT(wtr->prev != wtr);
  	wtr->next->prev = wtr->prev;
  	wtr->prev->next = wtr->next;
  	lck->queue[ix] = wtr->next;
      }
      return wtr;
  }
  
  /*
   * Tries to aquire as many locks as possible in lock order,
   * and sets the wait flag on the first lock not possible to
   * aquire.
   *
   * Note: We need the pix lock during this operation. Wait
   *       flags are only allowed to be manipulated under pix
   *       lock.
   */
  static ERTS_INLINE void
  try_aquire(erts_proc_lock_t *lck, erts_tse_t *wtr)
  {
      ErtsProcLocks got_locks = (ErtsProcLocks) 0;
      ErtsProcLocks locks = wtr->uflgs;
      int lock_no;
  
      ERTS_LC_ASSERT(got_locks != locks);
  
      for (lock_no = 0; lock_no <= ERTS_PROC_LOCK_MAX_BIT; lock_no++) {
  	ErtsProcLocks lock = ((ErtsProcLocks) 1) << lock_no;
  	if (locks & lock) {
  	    ErtsProcLocks wflg, old_lflgs;
  	    if (lck->queue[lock_no]) {
  		/* Others already waiting */
  	    enqueue:
  		ERTS_LC_ASSERT(ERTS_PROC_LOCK_FLGS_READ_(lck)
  			       & (lock << ERTS_PROC_LOCK_WAITER_SHIFT));
  		enqueue_waiter(lck, lock_no, wtr);
  		break;
  	    }
  	    wflg = lock << ERTS_PROC_LOCK_WAITER_SHIFT;
  	    old_lflgs = ERTS_PROC_LOCK_FLGS_BOR_ACQB_(lck, wflg | lock);
  	    if (old_lflgs & lock) {
  		/* Didn't get the lock */
  		goto enqueue;
  	    }
  	    else {
  		/* Got the lock */
  		got_locks |= lock;
  		ERTS_LC_ASSERT(!(old_lflgs & wflg));
  		/* No one else can be waiting for the lock; remove wait flag */
  		(void) ERTS_PROC_LOCK_FLGS_BAND_(lck, ~wflg);
  		if (got_locks == locks)
  		    break;
  	    }
  	}
      }
  
      wtr->uflgs &= ~got_locks;
  }
  
  /*
   * Transfer 'trnsfr_lcks' held by this executing thread to other
   * threads waiting for the locks. When a lock has been transferred
   * we also have to try to aquire as many lock as possible for the
   * other thread.
   */
  static int
  transfer_locks(Process *p,
  	       ErtsProcLocks trnsfr_lcks,
  	       erts_pix_lock_t *pix_lock,
  	       int unlock)
  {
      int transferred = 0;
      erts_tse_t *wake = NULL;
      erts_tse_t *wtr;
      ErtsProcLocks unset_waiter = 0;
      ErtsProcLocks tlocks = trnsfr_lcks;
      int lock_no;
  
      ERTS_LC_ASSERT(erts_lc_pix_lock_is_locked(pix_lock));
  
  #ifdef ERTS_PROC_LOCK_HARD_DEBUG
      check_queue(&p->lock);
  #endif
  
      for (lock_no = 0; tlocks && lock_no <= ERTS_PROC_LOCK_MAX_BIT; lock_no++) {
  	ErtsProcLocks lock = ((ErtsProcLocks) 1) << lock_no;
  	if (tlocks & lock) {
  	    /* Transfer lock */
  #ifdef ERTS_ENABLE_LOCK_CHECK
  	    tlocks &= ~lock;
  #endif
  	    ERTS_LC_ASSERT(ERTS_PROC_LOCK_FLGS_READ_(&p->lock)
  			   & (lock << ERTS_PROC_LOCK_WAITER_SHIFT));
  	    transferred++;
  	    wtr = dequeue_waiter(&p->lock, lock_no);
  	    ERTS_LC_ASSERT(wtr);
  	    if (!p->lock.queue[lock_no])
  		unset_waiter |= lock;
  	    ERTS_LC_ASSERT(wtr->uflgs & lock);
  	    wtr->uflgs &= ~lock;
  	    if (wtr->uflgs)
  		try_aquire(&p->lock, wtr);
  	    if (!wtr->uflgs) {
  		/*
  		 * The other thread got all locks it needs;
  		 * need to wake it up.
  		 */
  		wtr->next = wake;
  		wake = wtr;
  	    }
  	}
  
      }
  
      if (unset_waiter) {
  	unset_waiter <<= ERTS_PROC_LOCK_WAITER_SHIFT;
  	(void) ERTS_PROC_LOCK_FLGS_BAND_(&p->lock, ~unset_waiter);
      }
  
  #ifdef ERTS_PROC_LOCK_HARD_DEBUG
      check_queue(&p->lock);
  #endif
  
      ERTS_LC_ASSERT(tlocks == 0); /* We should have transferred all of them */
  
      if (!wake) {
  	if (unlock)
  	    erts_pix_unlock(pix_lock);
      }
      else {
  	erts_pix_unlock(pix_lock);
      
  	do {
  	    erts_tse_t *tmp = wake;
  	    wake = wake->next;
  	    erts_atomic32_set_nob(&tmp->uaflgs, 0);
  	    erts_tse_set(tmp);
  	} while (wake);
  
  	if (!unlock)
  	    erts_pix_lock(pix_lock);
      }
      return transferred;
  }
  
  /*
   * Determine which locks in 'need_locks' are not currently locked in
   * 'in_use', but do not return any locks "above" some lock we need,
   * so we do not attempt to grab locks out of order.
   *
   * For example, if we want to lock 10111, and 00100 was already locked, this
   * would return 00011, indicating we should not try for 10000 yet because
   * that would be a lock-ordering violation.
   */
  static ERTS_INLINE ErtsProcLocks
  in_order_locks(ErtsProcLocks in_use, ErtsProcLocks need_locks)
  {
      /* All locks we want that are already locked by someone else. */
      ErtsProcLocks busy = in_use & need_locks;
  
      /* Just the lowest numbered lock we want that's in use; 0 if none. */
      ErtsProcLocks lowest_busy = busy & -busy;
  
      /* All locks below the lowest one we want that's in use already. */
      return need_locks & (lowest_busy - 1);
  }
  
  /*
   * Try to grab locks one at a time in lock order and wait on the lowest
   * lock we fail to grab, if any.
   *
   * If successful, this returns 0 and all locks in 'need_locks' are held.
   *
   * On entry, the pix lock is held iff !ERTS_PROC_LOCK_ATOMIC_IMPL.
   * On exit it is not held.
   */
  static void
  wait_for_locks(Process *p,
                 erts_pix_lock_t *pixlck,
  	       ErtsProcLocks locks,
                 ErtsProcLocks need_locks,
                 ErtsProcLocks olflgs)
  {
      erts_pix_lock_t *pix_lock = pixlck ? pixlck : ERTS_PID2PIXLOCK(p->common.id);
      erts_tse_t *wtr;
  
      /* Acquire a waiter object on which this thread can wait. */
      wtr = tse_fetch(pix_lock);
      
      /* Record which locks this waiter needs. */
      wtr->uflgs = need_locks;
  
      ASSERT((wtr->uflgs & ~ERTS_PROC_LOCKS_ALL) == 0);
  
  #if ERTS_PROC_LOCK_ATOMIC_IMPL
      erts_pix_lock(pix_lock);
  #endif
  
      ERTS_LC_ASSERT(erts_lc_pix_lock_is_locked(pix_lock));
  
  #ifdef ERTS_PROC_LOCK_HARD_DEBUG
      check_queue(&p->lock);
  #endif
  
      /* Try to aquire locks one at a time in lock order and set wait flag */
      try_aquire(&p->lock, wtr);
  
      ASSERT((wtr->uflgs & ~ERTS_PROC_LOCKS_ALL) == 0);
  
  #ifdef ERTS_PROC_LOCK_HARD_DEBUG
      check_queue(&p->lock);
  #endif
  
      if (wtr->uflgs == 0)
  	erts_pix_unlock(pix_lock);
      else {
  	/* We didn't get them all; need to wait... */
  
  	ASSERT((wtr->uflgs & ~ERTS_PROC_LOCKS_ALL) == 0);
  
  	erts_atomic32_set_nob(&wtr->uaflgs, 1);
  	erts_pix_unlock(pix_lock);
  
  	while (1) {
  	    int res;
  	    erts_tse_reset(wtr);
  
  	    if (erts_atomic32_read_nob(&wtr->uaflgs) == 0)
  		break;
  
  	    /*
  	     * Wait for needed locks. When we are woken all needed locks have
  	     * have been acquired by other threads and transfered to us.
  	     * However, we need to be prepared for spurious wakeups.
  	     */
  	    do {
  		res = erts_tse_wait(wtr); /* might return EINTR */
  	    } while (res != 0);
  	}
  
  	ASSERT(wtr->uflgs == 0);
      }
  
      ERTS_LC_ASSERT(locks == (ERTS_PROC_LOCK_FLGS_READ_(&p->lock) & locks));
  
      tse_return(wtr);
  }
  
  /*
   * erts_proc_lock_failed() is called when erts_proc_lock()
   * wasn't able to lock all locks. We may need to transfer locks
   * to waiters and wait for our turn on locks.
   *
   * Iff !ERTS_PROC_LOCK_ATOMIC_IMPL, the pix lock is locked on entry.
   *
   * This always returns with the pix lock unlocked.
   */
  void
  erts_proc_lock_failed(Process *p,
  		      erts_pix_lock_t *pixlck,
  		      ErtsProcLocks locks,
  		      ErtsProcLocks old_lflgs)
  {
      int until_yield = ERTS_PROC_LOCK_SPIN_UNTIL_YIELD;
      int thr_spin_count;
      int spin_count;
      ErtsProcLocks need_locks = locks;
      ErtsProcLocks olflgs = old_lflgs;
  
      if (erts_thr_get_main_status())
  	thr_spin_count = proc_lock_spin_count;
      else
  	thr_spin_count = aux_thr_proc_lock_spin_count;
  
      spin_count = thr_spin_count;
  
      while (need_locks != 0) {
          ErtsProcLocks can_grab;
  
  	can_grab = in_order_locks(olflgs, need_locks);
  
          if (can_grab == 0) {
              /* Someone already has the lowest-numbered lock we want. */
  
              if (spin_count-- <= 0) {
                  /* Too many retries, give up and sleep for the lock. */
                  wait_for_locks(p, pixlck, locks, need_locks, olflgs);
                  return;
              }
  
  	    ERTS_SPIN_BODY;
  
  	    if (--until_yield == 0) {
  		until_yield = ERTS_PROC_LOCK_SPIN_UNTIL_YIELD;
  		erts_thr_yield();
  	    }
  
              olflgs = ERTS_PROC_LOCK_FLGS_READ_(&p->lock);
          }
          else {
              /* Try to grab all of the grabbable locks at once with cmpxchg. */
              ErtsProcLocks grabbed = olflgs | can_grab;
              ErtsProcLocks nflgs =
                  ERTS_PROC_LOCK_FLGS_CMPXCHG_ACQB_(&p->lock, grabbed, olflgs);
  
              if (nflgs == olflgs) {
                  /* Success! We grabbed the 'can_grab' locks. */
                  olflgs = grabbed;
                  need_locks &= ~can_grab;
  
                  /* Since we made progress, reset the spin count. */
                  spin_count = thr_spin_count;
              }
              else {
                  /* Compare-and-exchange failed, try again. */
                  olflgs = nflgs;
              }
          }
      }
  
     /* Now we have all of the locks we wanted. */
  
  #if !ERTS_PROC_LOCK_ATOMIC_IMPL
      erts_pix_unlock(pixlck);
  #endif
  }
  
  /*
   * erts_proc_unlock_failed() is called when erts_proc_unlock()
   * wasn't able to unlock all locks. We may need to transfer locks
   * to waiters.
   */
  void
  erts_proc_unlock_failed(Process *p,
  			erts_pix_lock_t *pixlck,
  			ErtsProcLocks wait_locks)
  {
      erts_pix_lock_t *pix_lock = pixlck ? pixlck : ERTS_PID2PIXLOCK(p->common.id);
  
  #if ERTS_PROC_LOCK_ATOMIC_IMPL
      erts_pix_lock(pix_lock);
  #endif
  
      transfer_locks(p, wait_locks, pix_lock, 1); /* unlocks pix_lock */
  }
  
  #endif /* ERTS_PROC_LOCK_OWN_IMPL */
  
  void
  erts_proc_lock_prepare_proc_lock_waiter(void)
  {
  #if ERTS_PROC_LOCK_OWN_IMPL
      tse_return(tse_fetch(NULL));
  #endif
  }
  
  /*
   * proc_safelock() locks process locks on two processes. In order
   * to avoid a deadlock, proc_safelock() unlocks those locks that
   * needs to be unlocked,  and then acquires locks in lock order
   * (including the previously unlocked ones).
   */
  
  static void
  proc_safelock(int is_managed,
  	      Process *a_proc,
  	      ErtsProcLocks a_have_locks,
  	      ErtsProcLocks a_need_locks,
  	      Process *b_proc,
  	      ErtsProcLocks b_have_locks,
  	      ErtsProcLocks b_need_locks)
  {
      Process *p1, *p2;
  #ifdef ERTS_ENABLE_LOCK_CHECK
      Eterm pid1, pid2;
  #endif
      ErtsProcLocks need_locks1, have_locks1, need_locks2, have_locks2;
      ErtsProcLocks unlock_mask;
      int lock_no, refc1 = 0, refc2 = 0;
  
      ERTS_LC_ASSERT(b_proc);
  
  
      /* Determine inter process lock order...
       * Locks with the same lock order should be locked on p1 before p2.
       */
      if (a_proc) {
  	if (a_proc->common.id < b_proc->common.id) {
  	    p1 = a_proc;
  #ifdef ERTS_ENABLE_LOCK_CHECK
  	    pid1 = a_proc->common.id;
  #endif
  	    need_locks1 = a_need_locks;
  	    have_locks1 = a_have_locks;
  	    p2 = b_proc;
  #ifdef ERTS_ENABLE_LOCK_CHECK
  	    pid2 = b_proc->common.id;
  #endif
  	    need_locks2 = b_need_locks;
  	    have_locks2 = b_have_locks;
  	}
  	else if (a_proc->common.id > b_proc->common.id) {
  	    p1 = b_proc;
  #ifdef ERTS_ENABLE_LOCK_CHECK
  	    pid1 = b_proc->common.id;
  #endif
  	    need_locks1 = b_need_locks;
  	    have_locks1 = b_have_locks;
  	    p2 = a_proc;
  #ifdef ERTS_ENABLE_LOCK_CHECK
  	    pid2 = a_proc->common.id;
  #endif
  	    need_locks2 = a_need_locks;
  	    have_locks2 = a_have_locks;
  	}
  	else {
  	    ERTS_LC_ASSERT(a_proc == b_proc);
  	    ERTS_LC_ASSERT(a_proc->common.id == b_proc->common.id);
  	    p1 = a_proc;
  #ifdef ERTS_ENABLE_LOCK_CHECK
  	    pid1 = a_proc->common.id;
  #endif
  	    need_locks1 = a_need_locks | b_need_locks;
  	    have_locks1 = a_have_locks | b_have_locks;
  	    p2 = NULL;
  #ifdef ERTS_ENABLE_LOCK_CHECK
  	    pid2 = 0;
  #endif
  	    need_locks2 = 0;
  	    have_locks2 = 0;
  	}
      }
      else {
  	p1 = b_proc;
  #ifdef ERTS_ENABLE_LOCK_CHECK
  	pid1 = b_proc->common.id;
  #endif
  	need_locks1 = b_need_locks;
  	have_locks1 = b_have_locks;
  	p2 = NULL;
  #ifdef ERTS_ENABLE_LOCK_CHECK
  	pid2 = 0;
  #endif
  	need_locks2 = 0;
  	have_locks2 = 0;
  #ifdef ERTS_ENABLE_LOCK_CHECK
  	a_need_locks = 0;
  	a_have_locks = 0;
  #endif
      }
  
  #ifdef ERTS_ENABLE_LOCK_CHECK
      if (p1)
  	erts_proc_lc_chk_proc_locks(p1, have_locks1);
      if (p2)
  	erts_proc_lc_chk_proc_locks(p2, have_locks2);
  
      if ((need_locks1 & have_locks1) != have_locks1)
  	erts_lc_fail("Thread tries to release process lock(s) "
  		     "on %T via erts_proc_safelock().", pid1);
      if ((need_locks2 & have_locks2) != have_locks2)
  	erts_lc_fail("Thread tries to release process lock(s) "
  		     "on %T via erts_proc_safelock().",
  		     pid2);
  #endif
  
  
      need_locks1 &= ~have_locks1;
      need_locks2 &= ~have_locks2;
  
      /* Figure out the range of locks that needs to be unlocked... */
      unlock_mask = ERTS_PROC_LOCKS_ALL;
      for (lock_no = 0;
  	 lock_no <= ERTS_PROC_LOCK_MAX_BIT;
  	 lock_no++) {
  	ErtsProcLocks lock = (1 << lock_no);
  	if (lock & need_locks1)
  	    break;
  	unlock_mask &= ~lock;
  	if (lock & need_locks2)
  	    break;
      }
  
      /* ... and unlock locks in that range... */
      if (have_locks1 || have_locks2) {
  	ErtsProcLocks unlock_locks;
  	unlock_locks = unlock_mask & have_locks1;
  	if (unlock_locks) {
  	    have_locks1 &= ~unlock_locks;
  	    need_locks1 |= unlock_locks;
  	    if (!is_managed && !have_locks1) {
  		refc1 = 1;
  		erts_proc_inc_refc(p1);
  	    }
  	    erts_proc_unlock(p1, unlock_locks);
  	}
  	unlock_locks = unlock_mask & have_locks2;
  	if (unlock_locks) {
  	    have_locks2 &= ~unlock_locks;
  	    need_locks2 |= unlock_locks;
  	    if (!is_managed && !have_locks2) {
  		refc2 = 1;
  		erts_proc_inc_refc(p2);
  	    }
  	    erts_proc_unlock(p2, unlock_locks);
  	}
      }
  
      /*
       * lock_no equals the number of the first lock to lock on
       * either p1 *or* p2.
       */
  
  
  #ifdef ERTS_ENABLE_LOCK_CHECK
      if (p1)
  	erts_proc_lc_chk_proc_locks(p1, have_locks1);
      if (p2)
  	erts_proc_lc_chk_proc_locks(p2, have_locks2);
  #endif
  
      /* Lock locks in lock order... */
      while (lock_no <= ERTS_PROC_LOCK_MAX_BIT) {
  	ErtsProcLocks locks;
  	ErtsProcLocks lock = (1 << lock_no);
  	ErtsProcLocks lock_mask = 0;
  	if (need_locks1 & lock) {
  	    do {
  		lock = (1 << lock_no++);
  		lock_mask |= lock;
  	    } while (lock_no <= ERTS_PROC_LOCK_MAX_BIT
  		     && !(need_locks2 & lock));
  	    if (need_locks2 & lock)
  		lock_no--;
  	    locks = need_locks1 & lock_mask;
  	    erts_proc_lock(p1, locks);
  	    have_locks1 |= locks;
  	    need_locks1 &= ~locks;
  	}
  	else if (need_locks2 & lock) {
  	    while (lock_no <= ERTS_PROC_LOCK_MAX_BIT
  		   && !(need_locks1 & lock)) {
  		lock_mask |= lock;
  		lock = (1 << ++lock_no);
  	    }
  	    locks = need_locks2 & lock_mask;
  	    erts_proc_lock(p2, locks);
  	    have_locks2 |= locks;
  	    need_locks2 &= ~locks;
  	}
  	else
  	    lock_no++;
      }
  
  #ifdef ERTS_ENABLE_LOCK_CHECK
      if (p1)
  	erts_proc_lc_chk_proc_locks(p1, have_locks1);
      if (p2)
  	erts_proc_lc_chk_proc_locks(p2, have_locks2);
  
      if (p1 && p2) {
  	if (p1 == a_proc) {
  	    ERTS_LC_ASSERT(a_need_locks == have_locks1);
  	    ERTS_LC_ASSERT(b_need_locks == have_locks2);
  	}
  	else {
  	    ERTS_LC_ASSERT(a_need_locks == have_locks2);
  	    ERTS_LC_ASSERT(b_need_locks == have_locks1);
  	}
      }
      else {
  	ERTS_LC_ASSERT(p1);
  	if (a_proc) {
  	    ERTS_LC_ASSERT(have_locks1 == (a_need_locks | b_need_locks));
  	}
  	else {
  	    ERTS_LC_ASSERT(have_locks1 == b_need_locks);
  	}
      }
  #endif
  
      if (!is_managed) {
  	if (refc1)
  	    erts_proc_dec_refc(p1);
  	if (refc2)
  	    erts_proc_dec_refc(p2);
      }
  }
  
  void
  erts_proc_safelock(Process *a_proc,
  		   ErtsProcLocks a_have_locks,
  		   ErtsProcLocks a_need_locks,
  		   Process *b_proc,
  		   ErtsProcLocks b_have_locks,
  		   ErtsProcLocks b_need_locks)
  {
      proc_safelock(erts_get_scheduler_id() != 0,
  		  a_proc,
  		  a_have_locks,
  		  a_need_locks,
  		  b_proc,
  		  b_have_locks,
  		  b_need_locks);
  }
  
  Process *
  erts_pid2proc_opt(Process *c_p,
  		  ErtsProcLocks c_p_have_locks,
  		  Eterm pid,
  		  ErtsProcLocks pid_need_locks,
  		  int flags)
  {
      Process *dec_refc_proc = NULL;
      ErtsThrPrgrDelayHandle dhndl;
      ErtsProcLocks need_locks;
      Uint pix;
      Process *proc;
  #if ERTS_PROC_LOCK_OWN_IMPL && defined(ERTS_ENABLE_LOCK_COUNT)
      ErtsProcLocks lcnt_locks;
  #endif
  
  #ifdef ERTS_ENABLE_LOCK_CHECK
      if (c_p) {
  	ErtsProcLocks might_unlock = c_p_have_locks & pid_need_locks;
  	if (might_unlock)
  	    erts_proc_lc_might_unlock(c_p, might_unlock);
      }
  #endif
  
      if (is_not_internal_pid(pid))
  	return NULL;
      pix = internal_pid_index(pid);
  
      ERTS_LC_ASSERT((pid_need_locks & ERTS_PROC_LOCKS_ALL) == pid_need_locks);
      need_locks = pid_need_locks;
  
      if (c_p && c_p->common.id == pid) {
  	ASSERT(c_p->common.id != ERTS_INVALID_PID);
  	ASSERT(c_p == erts_pix2proc(pix));
  
  	if (!(flags & ERTS_P2P_FLG_ALLOW_OTHER_X)
  	    && ERTS_PROC_IS_EXITING(c_p))
  	    return NULL;
  	need_locks &= ~c_p_have_locks;
  	if (!need_locks) {
  	    if (flags & ERTS_P2P_FLG_INC_REFC)
  		erts_proc_inc_refc(c_p);
  	    return c_p;
  	}
      }
  
      dhndl = erts_thr_progress_unmanaged_delay();
  
      proc = (Process *) erts_ptab_pix2intptr_ddrb(&erts_proc, pix);
  
      if (proc) {
  	if (proc->common.id != pid)
  	    proc = NULL;
  	else if (!need_locks) {
  	    if (flags & ERTS_P2P_FLG_INC_REFC)
  		erts_proc_inc_refc(proc);
  	}
  	else {
  	    int busy;
  
  #if ERTS_PROC_LOCK_OWN_IMPL
  #ifdef ERTS_ENABLE_LOCK_COUNT
  	    lcnt_locks = need_locks;
  	    if (!(flags & ERTS_P2P_FLG_TRY_LOCK)) {
  	    	erts_lcnt_proc_lock(&proc->lock, need_locks);
  	    }
  #endif
  
  #ifdef ERTS_ENABLE_LOCK_CHECK
  	    /* Make sure erts_pid2proc_safelock() is enough to handle
  	       a potential lock order violation situation... */
  	    busy = erts_proc_lc_trylock_force_busy(proc, need_locks);
  	    if (!busy)
  #endif
  #endif /* ERTS_PROC_LOCK_OWN_IMPL */
  	    {
  		/* Try a quick trylock to grab all the locks we need. */
  		busy = (int) erts_proc_raw_trylock__(proc, need_locks);
  
  #if ERTS_PROC_LOCK_OWN_IMPL && defined(ERTS_ENABLE_LOCK_CHECK)
  		erts_proc_lc_trylock(proc, need_locks, !busy, __FILE__,__LINE__);
  #endif
  #ifdef ERTS_PROC_LOCK_DEBUG
  		if (!busy)
  		    erts_proc_lock_op_debug(proc, need_locks, 1);
  #endif
  	    }
  
  #if ERTS_PROC_LOCK_OWN_IMPL && defined(ERTS_ENABLE_LOCK_COUNT)
  	    if (flags & ERTS_P2P_FLG_TRY_LOCK)
  		erts_lcnt_proc_trylock(&proc->lock, need_locks,
  				       busy ? EBUSY : 0);
  #endif
  
  	    if (!busy) {
  		if (flags & ERTS_P2P_FLG_INC_REFC)
  		    erts_proc_inc_refc(proc);
  
  #if ERTS_PROC_LOCK_OWN_IMPL && defined(ERTS_ENABLE_LOCK_COUNT)
  	    	/* all is great */
  	    	if (!(flags & ERTS_P2P_FLG_TRY_LOCK))
  		    erts_lcnt_proc_lock_post_x(&proc->lock, lcnt_locks,
  					       __FILE__, __LINE__);
  #endif
  
  	    }
  	    else {
  		if (flags & ERTS_P2P_FLG_TRY_LOCK)
  		    proc = ERTS_PROC_LOCK_BUSY;
  		else {
  		    int managed;
  		    if (flags & ERTS_P2P_FLG_INC_REFC)
  			erts_proc_inc_refc(proc);
  
  #if ERTS_PROC_LOCK_OWN_IMPL && defined(ERTS_ENABLE_LOCK_COUNT)
  		    erts_lcnt_proc_lock_unacquire(&proc->lock, lcnt_locks);
  #endif
  
  		    managed = dhndl == ERTS_THR_PRGR_DHANDLE_MANAGED;
  		    if (!managed) {
  			erts_proc_inc_refc(proc);
  			erts_thr_progress_unmanaged_continue(dhndl);
  			dec_refc_proc = proc;
  
  			/*
  			 * We don't want to call
  			 * erts_thr_progress_unmanaged_continue()
  			 * again.
  			 */
  			dhndl = ERTS_THR_PRGR_DHANDLE_MANAGED;
  		    }
  
  		    proc_safelock(managed,
  				  c_p,
  				  c_p_have_locks,
  				  c_p_have_locks,
  				  proc,
  				  0,
  				  need_locks);
  		}
  	    }
          }
      }
  
      if (dhndl != ERTS_THR_PRGR_DHANDLE_MANAGED)
  	erts_thr_progress_unmanaged_continue(dhndl);
  
      if (need_locks
  	&& proc
  	&& proc != ERTS_PROC_LOCK_BUSY
  	&& (!(flags & ERTS_P2P_FLG_ALLOW_OTHER_X)
  	    ? ERTS_PROC_IS_EXITING(proc)
  	    : (proc
  	       != (Process *) erts_ptab_pix2intptr_nob(&erts_proc, pix)))) {
  
  	erts_proc_unlock(proc, need_locks);
  
  	if (flags & ERTS_P2P_FLG_INC_REFC)
  	    dec_refc_proc = proc;
  	proc = NULL;
  
      }
  
      if (dec_refc_proc)
  	erts_proc_dec_refc(dec_refc_proc);
  
  #if ERTS_PROC_LOCK_OWN_IMPL && defined(ERTS_PROC_LOCK_DEBUG)
      ERTS_LC_ASSERT(!proc
  		   || proc == ERTS_PROC_LOCK_BUSY
  		   || (pid_need_locks ==
  		       (ERTS_PROC_LOCK_FLGS_READ_(&proc->lock)
  			& pid_need_locks)));
  #endif
  
      return proc;
  }
  
  static ERTS_INLINE
  Process *proc_lookup_inc_refc(Eterm pid, int allow_exit)
  {
      Process *proc;
      ErtsThrPrgrDelayHandle dhndl;
  
      dhndl = erts_thr_progress_unmanaged_delay();
  
      proc = erts_proc_lookup_raw(pid);
      if (proc) {
          if (!allow_exit && ERTS_PROC_IS_EXITING(proc))
              proc = NULL;
          else
              erts_proc_inc_refc(proc);
      }
  
      erts_thr_progress_unmanaged_continue(dhndl);
  
      return proc;
  }
  
  Process *erts_proc_lookup_inc_refc(Eterm pid)
  {
      return proc_lookup_inc_refc(pid, 0);
  }
  
  Process *erts_proc_lookup_raw_inc_refc(Eterm pid)
  {
      return proc_lookup_inc_refc(pid, 1);
  }
  
  void
  erts_proc_lock_init(Process *p)
  {
  #if ERTS_PROC_LOCK_OWN_IMPL || defined(ERTS_PROC_LOCK_DEBUG)
      int i;
  #endif
  #if ERTS_PROC_LOCK_OWN_IMPL
      /* We always start with all locks locked */
  #if ERTS_PROC_LOCK_ATOMIC_IMPL
      erts_atomic32_init_nob(&p->lock.flags,
  			       (erts_aint32_t) ERTS_PROC_LOCKS_ALL);
  #else
      p->lock.flags = ERTS_PROC_LOCKS_ALL;
  #endif
      for (i = 0; i <= ERTS_PROC_LOCK_MAX_BIT; i++)
  	p->lock.queue[i] = NULL;
  #ifdef ERTS_ENABLE_LOCK_CHECK
      erts_proc_lc_trylock(p, ERTS_PROC_LOCKS_ALL, 1,__FILE__,__LINE__);
  #endif
  #elif ERTS_PROC_LOCK_RAW_MUTEX_IMPL
  
      erts_mtx_init(&p->lock.main, "proc_main", p->common.id,
          ERTS_LOCK_FLAGS_CATEGORY_PROCESS);
      ethr_mutex_lock(&p->lock.main.mtx);
  #ifdef ERTS_ENABLE_LOCK_CHECK
      erts_lc_trylock(1, &p->lock.main.lc);
  #endif
      erts_mtx_init(&p->lock.msgq, "proc_msgq", p->common.id,
          ERTS_LOCK_FLAGS_CATEGORY_PROCESS);
      ethr_mutex_lock(&p->lock.msgq.mtx);
  #ifdef ERTS_ENABLE_LOCK_CHECK
      erts_lc_trylock(1, &p->lock.msgq.lc);
  #endif
      erts_mtx_init(&p->lock.btm, "proc_btm", p->common.id,
          ERTS_LOCK_FLAGS_CATEGORY_PROCESS);
      ethr_mutex_lock(&p->lock.btm.mtx);
  #ifdef ERTS_ENABLE_LOCK_CHECK
      erts_lc_trylock(1, &p->lock.btm.lc);
  #endif
      erts_mtx_init(&p->lock.status, "proc_status", p->common.id,
          ERTS_LOCK_FLAGS_CATEGORY_PROCESS);
      ethr_mutex_lock(&p->lock.status.mtx);
  #ifdef ERTS_ENABLE_LOCK_CHECK
      erts_lc_trylock(1, &p->lock.status.lc);
  #endif
      erts_mtx_init(&p->lock.trace, "proc_trace", p->common.id,
          ERTS_LOCK_FLAGS_CATEGORY_PROCESS);
      ethr_mutex_lock(&p->lock.trace.mtx);
  #ifdef ERTS_ENABLE_LOCK_CHECK
      erts_lc_trylock(1, &p->lock.trace.lc);
  #endif
  #endif
  #ifdef ERTS_PROC_LOCK_DEBUG
      for (i = 0; i <= ERTS_PROC_LOCK_MAX_BIT; i++)
  	erts_atomic32_init_nob(&p->lock.locked[i], (erts_aint32_t) 1);
  #endif
  #ifdef ERTS_ENABLE_LOCK_COUNT
      erts_lcnt_proc_lock_init(p);
      erts_lcnt_proc_lock(&(p->lock), ERTS_PROC_LOCKS_ALL);
      erts_lcnt_proc_lock_post_x(&(p->lock), ERTS_PROC_LOCKS_ALL, __FILE__, __LINE__);
  #endif
  }
  
  void
  erts_proc_lock_fin(Process *p)
  {
  #if ERTS_PROC_LOCK_RAW_MUTEX_IMPL
      erts_mtx_destroy(&p->lock.main);
      erts_mtx_destroy(&p->lock.msgq);
      erts_mtx_destroy(&p->lock.btm);
      erts_mtx_destroy(&p->lock.status);
      erts_mtx_destroy(&p->lock.trace);
  #endif
  #if defined(ERTS_ENABLE_LOCK_COUNT)
      erts_lcnt_proc_lock_destroy(p);
  #endif
  }
  
  /* --- Process lock counting ----------------------------------------------- */
  
  #if ERTS_PROC_LOCK_OWN_IMPL && defined(ERTS_ENABLE_LOCK_COUNT)
  
  void erts_lcnt_proc_lock_init(Process *p) {
      erts_lcnt_init_ref(&p->lock.lcnt_carrier);
  
      if(erts_lcnt_check_enabled(ERTS_LOCK_FLAGS_CATEGORY_PROCESS)) {
          erts_lcnt_enable_proc_lock_count(p, 1);
      }
  } /* logic reversed */
  
  void erts_lcnt_proc_lock_destroy(Process *p) {
      erts_lcnt_uninstall(&p->lock.lcnt_carrier);
  }
  
  void erts_lcnt_enable_proc_lock_count(Process *proc, int enable) {
      if(proc->common.id == ERTS_INVALID_PID) {
          /* Locks without an id are more trouble than they're worth; there's no
           * way to look them up and we can't track them with _STATIC since it's
           * too early to tell whether we're a system process (proc->static_flags
           * hasn't been not set yet). */
      } else if(!enable) {
          erts_lcnt_proc_lock_destroy(proc);
      } else if(!erts_lcnt_check_ref_installed(&proc->lock.lcnt_carrier)) {
          erts_lcnt_lock_info_carrier_t *carrier;
  
          carrier = erts_lcnt_create_lock_info_carrier(ERTS_LCNT_PROCLOCK_COUNT);
  
          erts_lcnt_init_lock_info_idx(carrier, ERTS_LCNT_PROCLOCK_IDX_MAIN,
              "proc_main", proc->common.id, ERTS_LOCK_TYPE_PROCLOCK);
          erts_lcnt_init_lock_info_idx(carrier, ERTS_LCNT_PROCLOCK_IDX_MSGQ,
              "proc_msgq", proc->common.id, ERTS_LOCK_TYPE_PROCLOCK);
          erts_lcnt_init_lock_info_idx(carrier, ERTS_LCNT_PROCLOCK_IDX_BTM,
              "proc_btm", proc->common.id, ERTS_LOCK_TYPE_PROCLOCK);
          erts_lcnt_init_lock_info_idx(carrier, ERTS_LCNT_PROCLOCK_IDX_STATUS,
              "proc_status",proc->common.id, ERTS_LOCK_TYPE_PROCLOCK);
          erts_lcnt_init_lock_info_idx(carrier, ERTS_LCNT_PROCLOCK_IDX_TRACE,
              "proc_trace", proc->common.id, ERTS_LOCK_TYPE_PROCLOCK);
  
          erts_lcnt_install(&proc->lock.lcnt_carrier, carrier);
      }
  }
  
  void erts_lcnt_update_process_locks(int enable) {
      int i, max;
  
      max = erts_ptab_max(&erts_proc);
  
      for(i = 0; i < max; i++) {
          int delay_handle;
          Process *proc;
  
          delay_handle = erts_thr_progress_unmanaged_delay();
          proc = erts_pix2proc(i);
  
          if(proc != NULL) {
              erts_lcnt_enable_proc_lock_count(proc, enable);
          }
  
          if(delay_handle != ERTS_THR_PRGR_DHANDLE_MANAGED) {
              erts_thr_progress_unmanaged_continue(delay_handle);
          }
      }
  }
  
  #endif /* ERTS_ENABLE_LOCK_COUNT */
  
  
  /* --- Process lock checking ----------------------------------------------- */
  
  #ifdef ERTS_ENABLE_LOCK_CHECK
  
  #if ERTS_PROC_LOCK_OWN_IMPL
  
  void
  erts_proc_lc_lock(Process *p, ErtsProcLocks locks, char *file, unsigned int line)
  {
      erts_lc_lock_t lck = ERTS_LC_LOCK_INIT(-1,
  					   p->common.id,
  					   ERTS_LOCK_TYPE_PROCLOCK);
      if (locks & ERTS_PROC_LOCK_MAIN) {
  	lck.id = lc_id.proc_lock_main;
  	erts_lc_lock_x(&lck,file,line);
      }
      if (locks & ERTS_PROC_LOCK_MSGQ) {
  	lck.id = lc_id.proc_lock_msgq;
  	erts_lc_lock_x(&lck,file,line);
      }
      if (locks & ERTS_PROC_LOCK_BTM) {
  	lck.id = lc_id.proc_lock_btm;
  	erts_lc_lock_x(&lck,file,line);
      }
      if (locks & ERTS_PROC_LOCK_STATUS) {
  	lck.id = lc_id.proc_lock_status;
  	erts_lc_lock_x(&lck,file,line);
      }
      if (locks & ERTS_PROC_LOCK_TRACE) {
  	lck.id = lc_id.proc_lock_trace;
  	erts_lc_lock_x(&lck,file,line);
      }
  }
  
  void
  erts_proc_lc_trylock(Process *p, ErtsProcLocks locks, int locked,
  		     char* file, unsigned int line)
  {
      erts_lc_lock_t lck = ERTS_LC_LOCK_INIT(-1,
  					   p->common.id,
  					   ERTS_LOCK_TYPE_PROCLOCK);
      if (locks & ERTS_PROC_LOCK_MAIN) {
  	lck.id = lc_id.proc_lock_main;
  	erts_lc_trylock_x(locked, &lck, file, line);
      }
      if (locks & ERTS_PROC_LOCK_MSGQ) {
  	lck.id = lc_id.proc_lock_msgq;
  	erts_lc_trylock_x(locked, &lck, file, line);
      }
      if (locks & ERTS_PROC_LOCK_BTM) {
  	lck.id = lc_id.proc_lock_btm;
  	erts_lc_trylock_x(locked, &lck, file, line);
      }
      if (locks & ERTS_PROC_LOCK_STATUS) {
  	lck.id = lc_id.proc_lock_status;
  	erts_lc_trylock_x(locked, &lck, file, line);
      }
      if (locks & ERTS_PROC_LOCK_TRACE) {
  	lck.id = lc_id.proc_lock_trace;
  	erts_lc_trylock_x(locked, &lck, file, line);
      }
  }
  
  void
  erts_proc_lc_unlock(Process *p, ErtsProcLocks locks)
  {
      erts_lc_lock_t lck = ERTS_LC_LOCK_INIT(-1,
  					   p->common.id,
  					   ERTS_LOCK_TYPE_PROCLOCK);
      if (locks & ERTS_PROC_LOCK_TRACE) {
  	lck.id = lc_id.proc_lock_trace;
  	erts_lc_unlock(&lck);
      }
      if (locks & ERTS_PROC_LOCK_STATUS) {
  	lck.id = lc_id.proc_lock_status;
  	erts_lc_unlock(&lck);
      }
      if (locks & ERTS_PROC_LOCK_BTM) {
  	lck.id = lc_id.proc_lock_btm;
  	erts_lc_unlock(&lck);
      }
      if (locks & ERTS_PROC_LOCK_MSGQ) {
  	lck.id = lc_id.proc_lock_msgq;
  	erts_lc_unlock(&lck);
      }
      if (locks & ERTS_PROC_LOCK_MAIN) {
  	lck.id = lc_id.proc_lock_main;
  	erts_lc_unlock(&lck);
      }
  }
  
  #endif /* ERTS_PROC_LOCK_OWN_IMPL */
  
  void
  erts_proc_lc_might_unlock(Process *p, ErtsProcLocks locks)
  {
  #if ERTS_PROC_LOCK_OWN_IMPL
      erts_lc_lock_t lck = ERTS_LC_LOCK_INIT(-1,
  					   p->common.id,
  					   ERTS_LOCK_TYPE_PROCLOCK);
      if (locks & ERTS_PROC_LOCK_TRACE) {
  	lck.id = lc_id.proc_lock_trace;
  	erts_lc_might_unlock(&lck);
      }
      if (locks & ERTS_PROC_LOCK_STATUS) {
  	lck.id = lc_id.proc_lock_status;
  	erts_lc_might_unlock(&lck);
      }
      if (locks & ERTS_PROC_LOCK_BTM) {
  	lck.id = lc_id.proc_lock_btm;
  	erts_lc_might_unlock(&lck);
      }
      if (locks & ERTS_PROC_LOCK_MSGQ) {
  	lck.id = lc_id.proc_lock_msgq;
  	erts_lc_might_unlock(&lck);
      }
      if (locks & ERTS_PROC_LOCK_MAIN) {
  	lck.id = lc_id.proc_lock_main;
  	erts_lc_might_unlock(&lck);
      }
  #elif ERTS_PROC_LOCK_RAW_MUTEX_IMPL
      if (locks & ERTS_PROC_LOCK_MAIN)
  	erts_lc_might_unlock(&p->lock.main.lc);
      if (locks & ERTS_PROC_LOCK_MSGQ)
  	erts_lc_might_unlock(&p->lock.msgq.lc);
      if (locks & ERTS_PROC_LOCK_BTM)
  	erts_lc_might_unlock(&p->lock.btm.lc);
      if (locks & ERTS_PROC_LOCK_STATUS)
  	erts_lc_might_unlock(&p->lock.status.lc);
      if (locks & ERTS_PROC_LOCK_TRACE)
  	erts_lc_might_unlock(&p->lock.trace.lc);
  #endif
  }
  
  void
  erts_proc_lc_require_lock(Process *p, ErtsProcLocks locks, char *file,
  			  unsigned int line)
  {
  #if ERTS_PROC_LOCK_OWN_IMPL
      erts_lc_lock_t lck = ERTS_LC_LOCK_INIT(-1,
  					   p->common.id,
  					   ERTS_LOCK_TYPE_PROCLOCK);
      if (locks & ERTS_PROC_LOCK_MAIN) {
  	lck.id = lc_id.proc_lock_main;
  	erts_lc_require_lock(&lck, file, line);
      }
      if (locks & ERTS_PROC_LOCK_MSGQ) {
  	lck.id = lc_id.proc_lock_msgq;
  	erts_lc_require_lock(&lck, file, line);
      }
      if (locks & ERTS_PROC_LOCK_BTM) {
  	lck.id = lc_id.proc_lock_btm;
  	erts_lc_require_lock(&lck, file, line);
      }
      if (locks & ERTS_PROC_LOCK_STATUS) {
  	lck.id = lc_id.proc_lock_status;
  	erts_lc_require_lock(&lck, file, line);
      }
      if (locks & ERTS_PROC_LOCK_TRACE) {
  	lck.id = lc_id.proc_lock_trace;
  	erts_lc_require_lock(&lck, file, line);
      }
  #elif ERTS_PROC_LOCK_RAW_MUTEX_IMPL
      if (locks & ERTS_PROC_LOCK_MAIN)
  	erts_lc_require_lock(&p->lock.main.lc, file, line);
      if (locks & ERTS_PROC_LOCK_MSGQ)
  	erts_lc_require_lock(&p->lock.msgq.lc, file, line);
      if (locks & ERTS_PROC_LOCK_BTM)
  	erts_lc_require_lock(&p->lock.btm.lc, file, line);
      if (locks & ERTS_PROC_LOCK_STATUS)
  	erts_lc_require_lock(&p->lock.status.lc, file, line);
      if (locks & ERTS_PROC_LOCK_TRACE)
  	erts_lc_require_lock(&p->lock.trace.lc, file, line);
  #endif
  }
  
  void
  erts_proc_lc_unrequire_lock(Process *p, ErtsProcLocks locks)
  {
  #if ERTS_PROC_LOCK_OWN_IMPL
      erts_lc_lock_t lck = ERTS_LC_LOCK_INIT(-1,
  					   p->common.id,
  					   ERTS_LOCK_TYPE_PROCLOCK);
      if (locks & ERTS_PROC_LOCK_TRACE) {
  	lck.id = lc_id.proc_lock_trace;
  	erts_lc_unrequire_lock(&lck);
      }
      if (locks & ERTS_PROC_LOCK_STATUS) {
  	lck.id = lc_id.proc_lock_status;
  	erts_lc_unrequire_lock(&lck);
      }
      if (locks & ERTS_PROC_LOCK_BTM) {
  	lck.id = lc_id.proc_lock_btm;
  	erts_lc_unrequire_lock(&lck);
      }
      if (locks & ERTS_PROC_LOCK_MSGQ) {
  	lck.id = lc_id.proc_lock_msgq;
  	erts_lc_unrequire_lock(&lck);
      }
      if (locks & ERTS_PROC_LOCK_MAIN) {
  	lck.id = lc_id.proc_lock_main;
  	erts_lc_unrequire_lock(&lck);
      }
  #elif ERTS_PROC_LOCK_RAW_MUTEX_IMPL
      if (locks & ERTS_PROC_LOCK_MAIN)
  	erts_lc_unrequire_lock(&p->lock.main.lc);
      if (locks & ERTS_PROC_LOCK_MSGQ)
  	erts_lc_unrequire_lock(&p->lock.msgq.lc);
      if (locks & ERTS_PROC_LOCK_BTM)
  	erts_lc_unrequire_lock(&p->lock.btm.lc);
      if (locks & ERTS_PROC_LOCK_STATUS)
  	erts_lc_unrequire_lock(&p->lock.status.lc);
      if (locks & ERTS_PROC_LOCK_TRACE)
  	erts_lc_unrequire_lock(&p->lock.trace.lc);
  #endif
  }
  
  #if ERTS_PROC_LOCK_OWN_IMPL
  
  int
  erts_proc_lc_trylock_force_busy(Process *p, ErtsProcLocks locks)
  {
      if (locks & ERTS_PROC_LOCKS_ALL) {
  	erts_lc_lock_t lck = ERTS_LC_LOCK_INIT(-1,
  					       p->common.id,
  					       ERTS_LOCK_TYPE_PROCLOCK);
  
  	if (locks & ERTS_PROC_LOCK_MAIN)
  	    lck.id = lc_id.proc_lock_main;
  	else if (locks & ERTS_PROC_LOCK_MSGQ)
  	    lck.id = lc_id.proc_lock_msgq;
  	else if (locks & ERTS_PROC_LOCK_BTM)
  	    lck.id = lc_id.proc_lock_btm;
  	else if (locks & ERTS_PROC_LOCK_STATUS)
  	    lck.id = lc_id.proc_lock_status;
  	else if (locks & ERTS_PROC_LOCK_TRACE)
  	    lck.id = lc_id.proc_lock_trace;
  	else
  	    erts_lc_fail("Unknown proc lock found");
  
  	return erts_lc_trylock_force_busy(&lck);
      }
      return 0;
  }
  
  #endif /* ERTS_PROC_LOCK_OWN_IMPL */
  
  void erts_proc_lc_chk_only_proc_main(Process *p)
  {
      erts_proc_lc_chk_only_proc(p, ERTS_PROC_LOCK_MAIN);
  }
  
  #if ERTS_PROC_LOCK_OWN_IMPL
  #define ERTS_PROC_LC_EMPTY_LOCK_INIT \
    ERTS_LC_LOCK_INIT(-1, THE_NON_VALUE, ERTS_LOCK_TYPE_PROCLOCK)
  #endif /* ERTS_PROC_LOCK_OWN_IMPL */
  
  void erts_proc_lc_chk_only_proc(Process *p, ErtsProcLocks locks)
  {
      int have_locks_len = 0;
  #if ERTS_PROC_LOCK_OWN_IMPL
      erts_lc_lock_t have_locks[6] = {ERTS_PROC_LC_EMPTY_LOCK_INIT,
  				    ERTS_PROC_LC_EMPTY_LOCK_INIT,
  				    ERTS_PROC_LC_EMPTY_LOCK_INIT,
  				    ERTS_PROC_LC_EMPTY_LOCK_INIT,
  				    ERTS_PROC_LC_EMPTY_LOCK_INIT,
                                      ERTS_PROC_LC_EMPTY_LOCK_INIT};
      if (locks & ERTS_PROC_LOCK_MAIN) {
  	have_locks[have_locks_len].id = lc_id.proc_lock_main;
  	have_locks[have_locks_len++].extra = p->common.id;
      }
      if (locks & ERTS_PROC_LOCK_MSGQ) {
  	have_locks[have_locks_len].id = lc_id.proc_lock_msgq;
  	have_locks[have_locks_len++].extra = p->common.id;
      }
      if (locks & ERTS_PROC_LOCK_BTM) {
  	have_locks[have_locks_len].id = lc_id.proc_lock_btm;
  	have_locks[have_locks_len++].extra = p->common.id;
      }
      if (locks & ERTS_PROC_LOCK_STATUS) {
  	have_locks[have_locks_len].id = lc_id.proc_lock_status;
  	have_locks[have_locks_len++].extra = p->common.id;
      }
      if (locks & ERTS_PROC_LOCK_TRACE) {
  	have_locks[have_locks_len].id = lc_id.proc_lock_trace;
  	have_locks[have_locks_len++].extra = p->common.id;
      }
  #elif ERTS_PROC_LOCK_RAW_MUTEX_IMPL
      erts_lc_lock_t have_locks[6];
      if (locks & ERTS_PROC_LOCK_MAIN)
  	have_locks[have_locks_len++] = p->lock.main.lc;
      if (locks & ERTS_PROC_LOCK_MSGQ)
  	have_locks[have_locks_len++] = p->lock.msgq.lc;
      if (locks & ERTS_PROC_LOCK_BTM)
  	have_locks[have_locks_len++] = p->lock.btm.lc;
      if (locks & ERTS_PROC_LOCK_STATUS)
  	have_locks[have_locks_len++] = p->lock.status.lc;
      if (locks & ERTS_PROC_LOCK_TRACE)
  	have_locks[have_locks_len++] = p->lock.trace.lc;
  #endif
      erts_lc_check_exact(have_locks, have_locks_len);
  }
  
  void
  erts_proc_lc_chk_have_proc_locks(Process *p, ErtsProcLocks locks)
  {
      int have_locks_len = 0;
  #if ERTS_PROC_LOCK_OWN_IMPL
      erts_lc_lock_t have_locks[6] = {ERTS_PROC_LC_EMPTY_LOCK_INIT,
  				    ERTS_PROC_LC_EMPTY_LOCK_INIT,
  				    ERTS_PROC_LC_EMPTY_LOCK_INIT,
                                      ERTS_PROC_LC_EMPTY_LOCK_INIT,
  				    ERTS_PROC_LC_EMPTY_LOCK_INIT,
  				    ERTS_PROC_LC_EMPTY_LOCK_INIT};
      if (locks & ERTS_PROC_LOCK_MAIN) {
  	have_locks[have_locks_len].id = lc_id.proc_lock_main;
  	have_locks[have_locks_len++].extra = p->common.id;
      }
      if (locks & ERTS_PROC_LOCK_MSGQ) {
  	have_locks[have_locks_len].id = lc_id.proc_lock_msgq;
  	have_locks[have_locks_len++].extra = p->common.id;
      }
      if (locks & ERTS_PROC_LOCK_BTM) {
  	have_locks[have_locks_len].id = lc_id.proc_lock_btm;
  	have_locks[have_locks_len++].extra = p->common.id;
      }
      if (locks & ERTS_PROC_LOCK_STATUS) {
  	have_locks[have_locks_len].id = lc_id.proc_lock_status;
  	have_locks[have_locks_len++].extra = p->common.id;
      }
      if (locks & ERTS_PROC_LOCK_TRACE) {
  	have_locks[have_locks_len].id = lc_id.proc_lock_trace;
  	have_locks[have_locks_len++].extra = p->common.id;
      }
  #elif ERTS_PROC_LOCK_RAW_MUTEX_IMPL
      erts_lc_lock_t have_locks[6];
      if (locks & ERTS_PROC_LOCK_MAIN)
  	have_locks[have_locks_len++] = p->lock.main.lc;
      if (locks & ERTS_PROC_LOCK_MSGQ)
  	have_locks[have_locks_len++] = p->lock.msgq.lc;
      if (locks & ERTS_PROC_LOCK_BTM)
  	have_locks[have_locks_len++] = p->lock.btm.lc;
      if (locks & ERTS_PROC_LOCK_STATUS)
  	have_locks[have_locks_len++] = p->lock.status.lc;
      if (locks & ERTS_PROC_LOCK_TRACE)
  	have_locks[have_locks_len++] = p->lock.trace.lc;
  #endif
      erts_lc_check(have_locks, have_locks_len, NULL, 0);
  }
  
  void
  erts_proc_lc_chk_proc_locks(Process *p, ErtsProcLocks locks)
  {
      int have_locks_len = 0;
      int have_not_locks_len = 0;
  #if ERTS_PROC_LOCK_OWN_IMPL
      erts_lc_lock_t have_locks[6] = {ERTS_PROC_LC_EMPTY_LOCK_INIT,
  				    ERTS_PROC_LC_EMPTY_LOCK_INIT,
                                      ERTS_PROC_LC_EMPTY_LOCK_INIT,
  				    ERTS_PROC_LC_EMPTY_LOCK_INIT,
  				    ERTS_PROC_LC_EMPTY_LOCK_INIT};
      erts_lc_lock_t have_not_locks[6] = {ERTS_PROC_LC_EMPTY_LOCK_INIT,
  					ERTS_PROC_LC_EMPTY_LOCK_INIT,
                                          ERTS_PROC_LC_EMPTY_LOCK_INIT,
  					ERTS_PROC_LC_EMPTY_LOCK_INIT,
  					ERTS_PROC_LC_EMPTY_LOCK_INIT};
  
      if (locks & ERTS_PROC_LOCK_MAIN) {
  	have_locks[have_locks_len].id = lc_id.proc_lock_main;
  	have_locks[have_locks_len++].extra = p->common.id;
      }
      else {
  	have_not_locks[have_not_locks_len].id = lc_id.proc_lock_main;
  	have_not_locks[have_not_locks_len++].extra = p->common.id;
      }
      if (locks & ERTS_PROC_LOCK_MSGQ) {
  	have_locks[have_locks_len].id = lc_id.proc_lock_msgq;
  	have_locks[have_locks_len++].extra = p->common.id;
      }
      else {
  	have_not_locks[have_not_locks_len].id = lc_id.proc_lock_msgq;
  	have_not_locks[have_not_locks_len++].extra = p->common.id;
      }
      if (locks & ERTS_PROC_LOCK_BTM) {
  	have_locks[have_locks_len].id = lc_id.proc_lock_btm;
  	have_locks[have_locks_len++].extra = p->common.id;
      }
      else {
  	have_not_locks[have_not_locks_len].id = lc_id.proc_lock_btm;
  	have_not_locks[have_not_locks_len++].extra = p->common.id;
      }
      if (locks & ERTS_PROC_LOCK_STATUS) {
  	have_locks[have_locks_len].id = lc_id.proc_lock_status;
  	have_locks[have_locks_len++].extra = p->common.id;
      }
      else {
  	have_not_locks[have_not_locks_len].id = lc_id.proc_lock_status;
  	have_not_locks[have_not_locks_len++].extra = p->common.id;
      }
      if (locks & ERTS_PROC_LOCK_TRACE) {
  	have_locks[have_locks_len].id = lc_id.proc_lock_trace;
  	have_locks[have_locks_len++].extra = p->common.id;
      }
      else {
  	have_not_locks[have_not_locks_len].id = lc_id.proc_lock_trace;
  	have_not_locks[have_not_locks_len++].extra = p->common.id;
      }
  #elif ERTS_PROC_LOCK_RAW_MUTEX_IMPL
      erts_lc_lock_t have_locks[6];
      erts_lc_lock_t have_not_locks[6];
  
      if (locks & ERTS_PROC_LOCK_MAIN)
  	have_locks[have_locks_len++] = p->lock.main.lc;
      else
  	have_not_locks[have_not_locks_len++] = p->lock.main.lc;
      if (locks & ERTS_PROC_LOCK_MSGQ)
  	have_locks[have_locks_len++] = p->lock.msgq.lc;
      else
  	have_not_locks[have_not_locks_len++] = p->lock.msgq.lc;
      if (locks & ERTS_PROC_LOCK_BTM)
  	have_locks[have_locks_len++] = p->lock.btm.lc;
      else
  	have_not_locks[have_not_locks_len++] = p->lock.btm.lc;
      if (locks & ERTS_PROC_LOCK_STATUS)
  	have_locks[have_locks_len++] = p->lock.status.lc;
      else
  	have_not_locks[have_not_locks_len++] = p->lock.status.lc;
      if (locks & ERTS_PROC_LOCK_TRACE)
  	have_locks[have_locks_len++] = p->lock.trace.lc;
      else
  	have_not_locks[have_not_locks_len++] = p->lock.trace.lc;
  #endif
  
      erts_lc_check(have_locks, have_locks_len,
  		  have_not_locks, have_not_locks_len);
  }
  
  ErtsProcLocks
  erts_proc_lc_my_proc_locks(Process *p)
  {
      int resv[5];
      ErtsProcLocks res = 0;
  #if ERTS_PROC_LOCK_OWN_IMPL
      erts_lc_lock_t locks[5] = {ERTS_LC_LOCK_INIT(lc_id.proc_lock_main,
  						 p->common.id,
  						 ERTS_LOCK_TYPE_PROCLOCK),
  			       ERTS_LC_LOCK_INIT(lc_id.proc_lock_msgq,
  						 p->common.id,
  						 ERTS_LOCK_TYPE_PROCLOCK),
  			       ERTS_LC_LOCK_INIT(lc_id.proc_lock_btm,
  						 p->common.id,
  						 ERTS_LOCK_TYPE_PROCLOCK),
  			       ERTS_LC_LOCK_INIT(lc_id.proc_lock_status,
  						 p->common.id,
  						 ERTS_LOCK_TYPE_PROCLOCK),
  			       ERTS_LC_LOCK_INIT(lc_id.proc_lock_trace,
  						 p->common.id,
  						 ERTS_LOCK_TYPE_PROCLOCK)};
  #elif ERTS_PROC_LOCK_RAW_MUTEX_IMPL
      erts_lc_lock_t locks[5] = {p->lock.main.lc,
  			       p->lock.msgq.lc,
  			       p->lock.btm.lc,
  			       p->lock.status.lc,
  			       p->lock.trace.lc};
  #endif
  
      erts_lc_have_locks(resv, locks, 5);
      if (resv[0])
  	res |= ERTS_PROC_LOCK_MAIN;
      if (resv[1])
  	res |= ERTS_PROC_LOCK_MSGQ;
      if (resv[2])
  	res |= ERTS_PROC_LOCK_BTM;
      if (resv[3])
  	res |= ERTS_PROC_LOCK_STATUS;
      if (resv[4])
  	res |= ERTS_PROC_LOCK_TRACE;
  
      return res;
  }
  
  void
  erts_proc_lc_chk_no_proc_locks(char *file, int line)
  {
      int resv[5];
      int ids[5] = {lc_id.proc_lock_main,
  		  lc_id.proc_lock_msgq,
  		  lc_id.proc_lock_btm,
  		  lc_id.proc_lock_status,
  		  lc_id.proc_lock_trace};
      erts_lc_have_lock_ids(resv, ids, 5);
      if (!ERTS_IS_CRASH_DUMPING && (resv[0] || resv[1] || resv[2] || resv[3] || resv[4])) {
  	erts_lc_fail("%s:%d: Thread has process locks locked when expected "
  		     "not to have any process locks locked",
  		     file, line);
      }
  }
  
  #endif /* #ifdef ERTS_ENABLE_LOCK_CHECK */
  
  #if ERTS_PROC_LOCK_OWN_IMPL && defined(ERTS_PROC_LOCK_HARD_DEBUG)
  void
  check_queue(erts_proc_lock_t *lck)
  {
      int lock_no;
      ErtsProcLocks lflgs = ERTS_PROC_LOCK_FLGS_READ_(lck);
  
      for (lock_no = 0; lock_no <= ERTS_PROC_LOCK_MAX_BIT; lock_no++) {
  	ErtsProcLocks wtr;
  	wtr = (((ErtsProcLocks) 1) << lock_no) << ERTS_PROC_LOCK_WAITER_SHIFT;
  	if (lflgs & wtr) {
  	    int n;
  	    erts_tse_t *wtr;
  	    ERTS_LC_ASSERT(lck->queue[lock_no]);
  	    wtr = lck->queue[lock_no];
  	    n = 0;
  	    do {
  		wtr = wtr->next;
  		n++;
  	    } while (wtr != lck->queue[lock_no]);
  	    do {
  		wtr = wtr->prev;
  		n--;
  	    } while (wtr != lck->queue[lock_no]);
  	    ERTS_LC_ASSERT(n == 0);
  	}
  	else {
  	    ERTS_LC_ASSERT(!lck->queue[lock_no]);
  	}
      }
  }
  #endif
  
  