text: /*
   * CDDL HEADER START
   *
   * The contents of this file are subject to the terms of the
   * Common Development and Distribution License (the "License").
   * You may not use this file except in compliance with the License.
   *
   * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   * or http://www.opensolaris.org/os/licensing.
   * See the License for the specific language governing permissions
   * and limitations under the License.
   *
   * When distributing Covered Code, include this CDDL HEADER in each
   * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
   * If applicable, add the following below this CDDL HEADER, with the
   * fields enclosed by brackets "[]" replaced with your own identifying
   * information: Portions Copyright [yyyy] [name of copyright owner]
   *
   * CDDL HEADER END
   */
  
  /*
   * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
   * Use is subject to license terms.
   *
   * Copyright 2017 RackTop Systems.
   */
  
  #ifndef	_LGRP_H
  #define	_LGRP_H
  
  /*
   * locality group definitions for kernel
   */
  
  #include <sys/types.h>
  
  #ifdef	__cplusplus
  extern "C" {
  #endif
  
  #define	LGRP_NONE	(-1)		/* non-existent lgroup ID */
  
  #if !defined(_KERNEL) && !defined(_FAKE_KERNEL) && !defined(_KMEMUSER)
  typedef struct lgrp_mem_policy_info { int opaque[2]; }	lgrp_mem_policy_info_t;
  #endif	/* !_KERNEL && !_FAKE_KERNEL && !_KMEMUSER */
  
  #if defined(_KERNEL) || defined(_FAKE_KERNEL) || defined(_KMEMUSER)
  #include <sys/cpuvar.h>
  #include <sys/bitmap.h>
  #include <sys/vnode.h>
  #include <vm/anon.h>
  #include <vm/seg.h>
  #include <sys/lgrp_user.h>
  #include <sys/param.h>
  
  typedef	uint32_t	lgrp_load_t;	/* lgrp_loadavg type */
  typedef uintptr_t	lgrp_handle_t;	/* lgrp handle */
  
  #define	LGRP_NONE_SUCH		LGRP_NONE	/* non-existent lgroup ID */
  /* null platform handle */
  #define	LGRP_NULL_HANDLE	((lgrp_handle_t)0xbadbad)
  #define	LGRP_DEFAULT_HANDLE	((lgrp_handle_t)0xbabecafe) /* uma handle */
  #define	LGRP_ROOTID		(0)		/* root lgroup ID */
  
  /*
   * Maximum number of lgrps a platform may define.
   */
  #define	NLGRPS_MAX		64
  #define	LGRP_LOADAVG_MAX	UINT32_MAX
  
  /*
   * The load-average we expect for one cpu-bound thread's worth of load
   */
  #define	LGRP_LOADAVG_THREAD_MAX		65516
  
  /*
   * The input to the load-average generating function for one cpu-bound thread's
   * worth of load
   */
  
  #define	LGRP_LOADAVG_IN_THREAD_MAX	128
  
  /*
   * LPL actions
   */
  
  typedef enum {
  	LPL_INCREMENT,
  	LPL_DECREMENT
  } lpl_act_t;
  
  /*
   * lgroup statistics.  Most of these are counters that are updated
   * dynamically so they are hashed to CPU buckets to reduce cache
   * interference.  The remaining statistics are snapshots of kernel
   * data, so they aren't stored in the array of counter stats.
   *
   * For the hashed stats to make sense, you have to sum all the buckets for
   * that stat, hence macros are provided to read the stats.
   */
  
  #define	LGRP_NUM_CPU_BUCKETS	8	/* must be power of 2 */
  #define	LGRP_CPU_BUCKET_MASK	(LGRP_NUM_CPU_BUCKETS - 1)
  
  /*
   * Flags for what to do with lgroup memory policy
   * Used for heap and stack where policy is extended to new segments added to
   * the end
   */
  #define	LGRP_MP_FLAG_EXTEND_UP		0x1	/* policy should extend up */
  #define	LGRP_MP_FLAG_EXTEND_DOWN	0x2	/* policy should extend down */
  
  #define	LGRP_STAT(stats, bucket, whichstat) \
  	((stats)->ls_data[bucket][whichstat])
  
  /* Return a pointer suitable for an atomic 64-bit op on the bucket */
  #define	LGRP_STAT_WRITE_PTR(stats, whichstat) \
  	(&LGRP_STAT(stats, (CPU->cpu_id) & LGRP_CPU_BUCKET_MASK, \
  	    whichstat))
  
  /* Sum up all the buckets and return the value in 'val' */
  #define	LGRP_STAT_READ(stats, whichstat, val) {				\
  	int bkt;							\
  	for (val = 0, bkt = 0; bkt < LGRP_NUM_CPU_BUCKETS; bkt++)	\
  		val += LGRP_STAT(stats, bkt, whichstat);		\
  }
  
  /* Reset all buckets for the stat to 0 */
  #define	LGRP_STAT_RESET(stats, stat) {					\
  	int i;								\
  	for (i = 0; i < LGRP_NUM_CPU_BUCKETS; i++)			\
  		LGRP_STAT(stats, i, stat) = 0;				\
  }
  
  /*
   * Define all of the statistics that are kept for lgrp kstats,
   * and their corresponding text names.
   */
  
  typedef enum lgrp_stat_types {
  	LGRP_NUM_MIGR,		/* # migrations away from this lgrp */
  	LGRP_NUM_ALLOC_FAIL,	/* # times alloc fails for chosen lgrp */
  	LGRP_PM_SRC_PGS,	/* # pages migrated from this lgrp */
  	LGRP_PM_DEST_PGS,	/* # pages migrated to this lgrp */
  	LGRP_PM_FAIL_ALLOC_PGS,	/* # pages failed to migrate to this lgrp */
  	LGRP_PM_FAIL_LOCK_PGS,	/* # pages failed to migrate from this lgrp */
  	LGRP_PMM_PGS,		/* # pages marked to migrate from this lgrp */
  	LGRP_PMM_FAIL_PGS,	/* # pages marked to migrate from this lgrp */
  	LGRP_NUM_DEFAULT,	/* # of times default policy applied */
  	LGRP_NUM_NEXT,		/* # of times next touch policy applied */
  	LGRP_NUM_RANDOM,	/* # of times random policy applied */
  	LGRP_NUM_RANDOM_PROC,	/* # of times random proc policy applied */
  	LGRP_NUM_RANDOM_PSET,	/* # of times random pset policy applied */
  	LGRP_NUM_ROUNDROBIN,	/* # of times round robin policy applied */
  	LGRP_NUM_NEXT_SEG,	/* # of times next to seg policy applied */
  	LGRP_NUM_COUNTER_STATS,	/* always last */
  	LGRP_CTR_STATS_ALLOC = 16	/* cache-align pad - multiple of 8 */
  				/* always keep >= LGRP_NUM_COUNTER_STATS */
  } lgrp_stat_t;
  
  typedef enum lgrp_snap_stat_types {
  	LGRP_NUM_CPUS,		/* number of CPUs */
  	LGRP_NUM_PG_FREE,	/* # of free pages */
  	LGRP_NUM_PG_AVAIL,	/* # of allocatable physical pages */
  	LGRP_NUM_PG_INSTALL,	/* # of installed physical pages */
  	LGRP_LOADAVG,		/* unscaled load average of this lgrp */
  	LGRP_LOADAVG_SCALE,	/* load unit of one CPU bound thread */
  	LGRP_NUM_SNAPSHOT_STATS	/* always last */
  } lgrp_snap_stat_t;
  
  #define	LGRP_KSTAT_NAMES		\
  static char *lgrp_kstat_names[] = {	\
  					\
  	/* Counter stats */		\
  	"lwp migrations",		\
  	"alloc fail",			\
  	"pages migrated from",		\
  	"pages migrated to",		\
  	"pages failed to migrate to",	\
  	"pages failed to migrate from",	\
  	"pages marked for migration",	\
  	"pages failed to mark",		\
  	"default policy",		\
  	"next-touch policy",		\
  	"random policy",		\
  	"span process policy",		\
  	"span psrset policy",		\
  	"round robin policy",		\
  	"next-seg policy",		\
  					\
  	/* Snapshot stats */		\
  	"cpus",				\
  	"pages free",			\
  	"pages avail",			\
  	"pages installed",		\
  	"load average",			\
  	"loadscale"			\
  }
  
  #define	LGRP_NUM_STATS	((int)LGRP_NUM_COUNTER_STATS +			\
  	(int)LGRP_NUM_SNAPSHOT_STATS)
  
  /*
   * The contents of this structure are opaque and should only be
   * accessed through the LGRP_STAT macro.
   */
  struct lgrp_stats {
  	int64_t ls_data[LGRP_NUM_CPU_BUCKETS][LGRP_CTR_STATS_ALLOC];
  };
  
  /* The kernel's version of a bitmap of lgroups */
  typedef uint64_t klgrpset_t;
  
  /*
   * This really belongs in memnode.h, but it must be defined here to avoid
   * recursive inclusion problems. Note that memnode.h includes this header.
   */
  typedef	uint64_t	mnodeset_t;
  
  /*
   * lgroup structure
   *
   * Visible to generic code and contains the lgroup ID, CPUs in this lgroup,
   * and a platform handle used to identify this lgroup to the lgroup platform
   * support code
   */
  typedef struct lgrp {
  
  	lgrp_id_t 	lgrp_id;	/* which lgroup	*/
  	int		lgrp_latency;
  	lgrp_handle_t  	lgrp_plathand;	/* handle for platform calls */
  	struct lgrp	*lgrp_parent;	/* parent lgroup */
  	uint_t		lgrp_reserved1;	/* filler */
  	uint_t		lgrp_childcnt;	/* number of children lgroups */
  	klgrpset_t	lgrp_children;	/* children lgroups */
  	klgrpset_t	lgrp_leaves;	/* (direct decendant) leaf lgroups */
  
  	/*
  	 * set of lgroups containing a given type of resource
  	 * at this level of locality
  	 */
  	klgrpset_t	lgrp_set[LGRP_RSRC_COUNT];
  
  	mnodeset_t	lgrp_mnodes;	/* set of memory nodes in this lgroup */
  	uint_t		lgrp_nmnodes;	/* number of memnodes */
  	uint_t		lgrp_reserved2;	/* filler */
  
  	struct cpu	*lgrp_cpu;	/* pointer to a cpu may be null */
  	uint_t		lgrp_cpucnt;	/* number of cpus in this lgrp	*/
  	kstat_t		*lgrp_kstat;	/* per-lgrp kstats */
  } lgrp_t;
  
  /*
   * lgroup load average structure
   */
  
  typedef struct lgrp_ld {
  	lgrp_load_t	lpl_loadavg;	/* load average		*/
  	uint_t		lpl_ncpu;	/* how many cpus	*/
  	lgrp_id_t	lpl_lgrpid;	/* which group this lpl part of */
  	lgrp_t		*lpl_lgrp;	/* ptr to lpl's lgrp */
  	struct lgrp_ld	*lpl_parent;	/* lpl of parent lgrp */
  	struct cpu	*lpl_cpus;	/* list of cpus in lpl */
  					/* NULL for non-leaf lgrps */
  	uint_t		lpl_nrset;	/* no. of leaf lpls for lgrp */
  	hrtime_t	lpl_homed_time;	/* time of last homing to this lpl */
  	uint_t		lpl_rset_sz;	/* Resource set capacity */
  	struct lgrp_ld	**lpl_rset;	/* leaf lpls for lgrp */
  					/* contains ptr to self for leaf lgrp */
  	int		*lpl_id2rset;	/* mapping of lgrpid to rset index */
  } lpl_t;
  
  /*
   * 1 << LGRP_MAX_EFFECT_SHFT ==  lgrp_loadavg_max_effect
   */
  #define	LGRP_MAX_EFFECT_SHFT 16
  
  /*
   * Operations handled by lgrp_config()
   */
  typedef enum lgrp_config_flag {
  	LGRP_CONFIG_NOP,
  	LGRP_CONFIG_CPU_ADD,
  	LGRP_CONFIG_CPU_DEL,
  	LGRP_CONFIG_CPU_ONLINE,
  	LGRP_CONFIG_CPU_OFFLINE,
  	LGRP_CONFIG_CPUPART_ADD,
  	LGRP_CONFIG_CPUPART_DEL,
  	LGRP_CONFIG_MEM_ADD,
  	LGRP_CONFIG_MEM_DEL,
  	LGRP_CONFIG_MEM_RENAME,
  	LGRP_CONFIG_GEN_UPDATE,
  	LGRP_CONFIG_FLATTEN,
  	LGRP_CONFIG_LAT_CHANGE_ALL,
  	LGRP_CONFIG_LAT_CHANGE
  } lgrp_config_flag_t;
  
  /*
   * Stages of lgroup framework initialization (done through lgrp_init()):
   *
   * 1) Initialize common and platform specific code (called in mlsetup())
   *
   * 2) Setup root lgroup and add CPU 0 to lgroup(s) (called near beginning of
   *    main() before startup())
   *
   * 3) Probe from CPU 0 and copy and release any BOP_ALLOC-ed memory temporarily
   *    allocated before kernel memory allocator is setup (called in main()
   *    after startup(), gethrtime() is setup, and before interrupts enabled)
   *
   * 4) Check for null proc LPA on Starcat, collapse lgroup topology (if
   *    necessary), setup lgroup kstats, etc. (called before start_other_cpus())
   *
   * 5) Finish any lgroup initialization needed including updating lgroup
   *    topology after all CPUs started (called after start_other_cpus())
   */
  typedef enum lgrp_init_stages {
  	LGRP_INIT_STAGE1,
  	LGRP_INIT_STAGE2,
  	LGRP_INIT_STAGE3,
  	LGRP_INIT_STAGE4,
  	LGRP_INIT_STAGE5
  } lgrp_init_stages_t;
  
  /*
   * Memory allocation policies
   */
  typedef enum lgrp_mem_policy {
  	LGRP_MEM_POLICY_DEFAULT,
  	LGRP_MEM_POLICY_NEXT,		/* near LWP to next touch */
  	LGRP_MEM_POLICY_RANDOM_PROC,	/* randomly across process */
  	LGRP_MEM_POLICY_RANDOM_PSET,	/* randomly across processor set */
  	LGRP_MEM_POLICY_RANDOM,		/* randomly across all lgroups */
  	LGRP_MEM_POLICY_ROUNDROBIN,	/* round robin across all lgroups */
  	LGRP_MEM_POLICY_NEXT_CPU,	/* Near next CPU to touch memory */
  	LGRP_MEM_POLICY_NEXT_SEG,	/* lgrp specified directly by seg */
  	LGRP_NUM_MEM_POLICIES
  } lgrp_mem_policy_t;
  
  /*
   * Search scopes for finding resouces
   */
  typedef	enum lgrp_res_ss {
  	LGRP_SRCH_LOCAL,		/* Search local lgroup only */
  	LGRP_SRCH_HIER			/* Search entire hierarchy */
  } lgrp_res_ss_t;
  
  /*
   * Cookie used for lgrp mnode selection
   */
  typedef struct lgrp_mnode_cookie {
  	lgrp_t		*lmc_lgrp;	/* lgrp under consideration */
  	mnodeset_t	lmc_nodes;	/* nodes not yet tried in lgrp */
  	int		lmc_cnt;	/* how many nodes in untried set */
  	mnodeset_t	lmc_tried;	/* nodes already tried */
  	int		lmc_ntried;	/* how many nodes in tried set */
  	lgrp_res_ss_t	lmc_scope;	/* consider non-local nodes? */
  	ushort_t	lmc_rand;	/* a "random" number */
  } lgrp_mnode_cookie_t;
  
  /*
   * Information needed to implement memory allocation policy
   */
  typedef struct lgrp_mem_policy_info {
  	int		mem_policy;		/* memory allocation policy */
  	lgrp_id_t	mem_lgrpid;		/* lgroup id */
  } lgrp_mem_policy_info_t;
  
  /*
   * Shared memory policy segment
   */
  typedef struct lgrp_shm_policy_seg {
  	u_offset_t		shm_off;	/* offset into shared object */
  	size_t			shm_size;	/* size of segment */
  	lgrp_mem_policy_info_t	shm_policy;	/* memory allocation policy */
  	avl_node_t		shm_tree;	/* AVL tree */
  } lgrp_shm_policy_seg_t;
  
  /*
   * Shared memory locality info
   */
  typedef struct lgrp_shm_locality {
  	size_t		loc_count;		/* reference count */
  	avl_tree_t	*loc_tree;		/* policy segment tree */
  	krwlock_t	loc_lock;		/* protects tree */
  } lgrp_shm_locality_t;
  
  /*
   * Queries that may be made to determine lgroup memory size
   */
  typedef enum {
  	LGRP_MEM_SIZE_FREE,		/* number of free pages */
  	LGRP_MEM_SIZE_AVAIL,		/* number of pages in phys_avail */
  	LGRP_MEM_SIZE_INSTALL		/* number of pages in phys_install */
  } lgrp_mem_query_t;
  
  /*
   * Argument for the memory copy-rename operation, contains the source and the
   * destination platform handles.
   */
  typedef struct lgrp_config_mem_rename {
  	lgrp_handle_t lmem_rename_from;
  	lgrp_handle_t lmem_rename_to;
  } lgrp_config_mem_rename_t;
  
  /* Macro to clear an lgroup bitmap */
  #define	klgrpset_clear(klgrpset) \
  	(klgrpset) = (klgrpset_t)0
  
  /* Macro to fill an lgroup bitmap */
  #define	klgrpset_fill(klgrpset) \
  	(klgrpset) = (klgrpset_t)(-1)
  
  /* Macro to add an lgroup to an lgroup bitmap */
  #define	klgrpset_add(klgrpset, lgrpid) \
  	(klgrpset) |= ((klgrpset_t)1 << (lgrpid))
  
  /* Macro to delete an lgroup from an lgroup bitmap */
  #define	klgrpset_del(klgrpset, lgrpid) \
  	(klgrpset) &= ~((klgrpset_t)1 << (lgrpid))
  
  /* Macro to copy a klgrpset into another klgrpset */
  #define	klgrpset_copy(klgrpset_to, klgrpset_from) \
  	(klgrpset_to) = (klgrpset_from)
  
  /* Macro to perform an 'and' operation on a pair of lgroup bitmaps */
  #define	klgrpset_and(klgrpset_rslt, klgrpset_arg) \
  	(klgrpset_rslt) &= (klgrpset_arg)
  
  /* Macro to perform an 'or' operation on a pair of lgroup bitmaps */
  #define	klgrpset_or(klgrpset_rslt, klgrpset_arg) \
  	(klgrpset_rslt) |= (klgrpset_arg)
  
  /* Macro to perform a 'diff' operation on a pair of lgroup bitmaps */
  #define	klgrpset_diff(klgrpset_rslt, klgrpset_arg) \
  	(klgrpset_rslt) &= ~(klgrpset_arg)
  
  /* Macro to check if an lgroup is a member of an lgrpset */
  #define	klgrpset_ismember(klgrpset, lgrpid) \
  	((klgrpset) & ((klgrpset_t)1 << (lgrpid)))
  
  /* Macro to check if an lgroup bitmap is empty */
  #define	klgrpset_isempty(klgrpset) \
  	((klgrpset) == (klgrpset_t)0)
  
  /* Macro to check if two lgrpsets intersect */
  #define	klgrpset_intersects(klgrpset1, klgrpset2) \
  	((klgrpset1) & (klgrpset2))
  
  /* Macro to count the number of members in an lgrpset */
  #define	klgrpset_nlgrps(klgrpset, count)				\
  {									\
  	lgrp_id_t	lgrpid;						\
  	for (lgrpid = 0, count = 0; lgrpid <= lgrp_alloc_max; lgrpid++) {\
  		if (klgrpset_ismember(klgrpset, lgrpid))		\
  			count++;					\
  	}								\
  }
  
  /* Macro to get total memory size (in bytes) of a given set of lgroups */
  #define	klgrpset_totalsize(klgrpset, size)				\
  {									\
  	lgrp_handle_t	hand;						\
  	lgrp_id_t	lgrpid;						\
  									\
  	for (lgrpid = 0, size = 0; lgrpid <= lgrp_alloc_max; lgrpid++) {\
  		if (klgrpset_ismember(klgrpset, lgrpid) &&		\
  		    lgrp_table[lgrpid])	{				\
  			hand = lgrp_table[lgrpid]->lgrp_plathand;	\
  			size += lgrp_plat_mem_size(hand,		\
  			    LGRP_MEM_SIZE_AVAIL) * PAGESIZE;		\
  		}							\
  	}								\
  }
  
  /*
   * Does this lgroup exist?
   */
  #define	LGRP_EXISTS(lgrp)	\
  	(lgrp != NULL && lgrp->lgrp_id != LGRP_NONE)
  
  /*
   * Macro for testing if a CPU is contained in an lgrp.
   */
  #define	LGRP_CONTAINS_CPU(lgrp, cpu)	\
  	(klgrpset_ismember(lgrp->lgrp_set[LGRP_RSRC_CPU],	\
  	    cpu->cpu_lpl->lpl_lgrpid))
  
  /*
   * Initialize an lgrp_mnode_cookie
   */
  #define	LGRP_MNODE_COOKIE_INIT(c, lgrp, scope)	\
  {							\
  	bzero(&(c), sizeof (lgrp_mnode_cookie_t));	\
  	(&(c))->lmc_lgrp = lgrp;			\
  	(&(c))->lmc_nodes = lgrp->lgrp_mnodes;		\
  	(&(c))->lmc_cnt = lgrp->lgrp_nmnodes;		\
  	(&(c))->lmc_scope = scope;			\
  	(&(c))->lmc_rand = (ushort_t)gethrtime_unscaled() >> 4;	\
  }
  
  /*
   * Upgrade cookie scope from LGRP_SRCH_LOCAL to LGRP_SRCH_HIER.
   */
  #define	LGRP_MNODE_COOKIE_UPGRADE(c)	\
  {							\
  	ASSERT((&(c))->lmc_scope == LGRP_SRCH_LOCAL);	\
  	(&(c))->lmc_scope = LGRP_SRCH_HIER;		\
  }
  
  /*
   * Macro to see whether memory allocation policy can be reapplied
   */
  #define	LGRP_MEM_POLICY_REAPPLICABLE(p) \
  	(p == LGRP_MEM_POLICY_NEXT)
  
  /*
   * Return true if lgrp has CPU resources in the cpupart
   */
  #define	LGRP_CPUS_IN_PART(lgrpid, cpupart) \
  	(cpupart->cp_lgrploads[lgrpid].lpl_ncpu > 0)
  
  extern int	lgrp_alloc_max;
  extern lgrp_t	*lgrp_table[NLGRPS_MAX];	/* indexed by lgrp_id */
  extern int		nlgrps;		/* number of lgroups in machine */
  extern int		nlgrpsmax;	/* max number of lgroups on platform */
  extern lgrp_gen_t	lgrp_gen;	/* generation of lgroup hierarchy */
  extern int		lgrp_initialized; /* single-CPU initialization done */
  extern int		lgrp_topo_initialized; /* lgrp topology constructed */
  extern lgrp_t		*lgrp_root;	/* root lgroup */
  extern unsigned int	lgrp_topo_levels;
  extern lpl_t		*lpl_bootstrap;	/* bootstrap lpl for non-active CPUs */
  
  
  /* generic interfaces */
  
  /*
   * lgroup management
   */
  int	lgrp_optimizations(void);
  void	lgrp_init(lgrp_init_stages_t);
  lgrp_t	*lgrp_create(void);
  void	lgrp_destroy(lgrp_t *);
  void	lgrp_config(lgrp_config_flag_t, uintptr_t, uintptr_t);
  lgrp_t	*lgrp_hand_to_lgrp(lgrp_handle_t);
  
  /*
   * lgroup stats
   */
  void	lgrp_kstat_create(struct cpu *);
  void	lgrp_kstat_destroy(struct cpu *);
  void	lgrp_stat_add(lgrp_id_t, lgrp_stat_t, int64_t);
  int64_t lgrp_stat_read(lgrp_id_t, lgrp_stat_t);
  
  /*
   * lgroup memory
   */
  lgrp_mem_policy_t	lgrp_madv_to_policy(uchar_t, size_t, int);
  pgcnt_t	lgrp_mem_size(lgrp_id_t, lgrp_mem_query_t);
  lgrp_t	*lgrp_mem_choose(struct seg *, caddr_t, size_t);
  int	lgrp_memnode_choose(lgrp_mnode_cookie_t *);
  lgrp_mem_policy_t	lgrp_mem_policy_default(size_t, int);
  int	lgrp_mnode_update(klgrpset_t, klgrpset_t *);
  lgrp_t	*lgrp_pfn_to_lgrp(pfn_t);
  lgrp_t	*lgrp_phys_to_lgrp(u_longlong_t);	/* used by numat driver */
  int	lgrp_privm_policy_set(lgrp_mem_policy_t, lgrp_mem_policy_info_t *,
      size_t);
  void	lgrp_shm_policy_init(struct anon_map *, vnode_t *);
  void	lgrp_shm_policy_fini(struct anon_map *, vnode_t *);
  lgrp_mem_policy_info_t	*lgrp_shm_policy_get(struct anon_map *, ulong_t,
      vnode_t *, u_offset_t);
  int	lgrp_shm_policy_set(lgrp_mem_policy_t, struct anon_map *, ulong_t,
      vnode_t *, u_offset_t, size_t);
  
  /*
   * Used by numat driver
   */
  int	lgrp_query_cpu(processorid_t, lgrp_id_t *);
  int	lgrp_query_load(processorid_t, lgrp_load_t *);
  
  /*
   * lgroup thread placement
   */
  lpl_t	*lgrp_affinity_best(kthread_t *, struct cpupart *, lgrp_id_t,
      boolean_t);
  void	lgrp_affinity_init(lgrp_affinity_t **);
  void	lgrp_affinity_free(lgrp_affinity_t **);
  lpl_t	*lgrp_choose(kthread_t *t, struct cpupart *);
  lgrp_t	*lgrp_home_lgrp(void);
  lgrp_id_t	lgrp_home_id(kthread_t *);
  void	lgrp_loadavg(lpl_t *, uint_t, int);
  void	lgrp_move_thread(kthread_t *, lpl_t *, int);
  uint64_t lgrp_get_trthr_migrations(void);
  void 	lgrp_update_trthr_migrations(uint64_t);
  
  /*
   * lgroup topology
   */
  int	lgrp_leaf_add(lgrp_t *, lgrp_t **, int, klgrpset_t *);
  int	lgrp_leaf_delete(lgrp_t *, lgrp_t **, int, klgrpset_t *);
  int	lgrp_rsets_empty(klgrpset_t *);
  int	lgrp_rsets_member(klgrpset_t *, lgrp_id_t);
  int	lgrp_topo_flatten(int, lgrp_t **, int, klgrpset_t *);
  int	lgrp_topo_ht_limit(void);
  int	lgrp_topo_ht_limit_default(void);
  int	lgrp_topo_ht_limit_set(int);
  int	lgrp_topo_update(lgrp_t **, int, klgrpset_t *);
  
  /*
   * lpl topology
   */
  void	lpl_topo_bootstrap(lpl_t *, int);
  int	lpl_topo_flatten(int);
  int	lpl_topo_verify(struct cpupart *);
  
  
  /* platform interfaces */
  void	lgrp_plat_init(lgrp_init_stages_t);
  lgrp_t	*lgrp_plat_alloc(lgrp_id_t lgrpid);
  void	lgrp_plat_config(lgrp_config_flag_t, uintptr_t);
  lgrp_handle_t	lgrp_plat_cpu_to_hand(processorid_t);
  lgrp_handle_t	lgrp_plat_pfn_to_hand(pfn_t);
  int	lgrp_plat_max_lgrps(void);
  pgcnt_t	lgrp_plat_mem_size(lgrp_handle_t, lgrp_mem_query_t);
  int	lgrp_plat_latency(lgrp_handle_t, lgrp_handle_t);
  lgrp_handle_t	lgrp_plat_root_hand(void);
  
  extern uint32_t		lgrp_expand_proc_thresh;
  extern uint32_t		lgrp_expand_proc_diff;
  extern pgcnt_t		lgrp_mem_free_thresh;
  extern uint32_t		lgrp_loadavg_tolerance;
  extern uint32_t		lgrp_loadavg_max_effect;
  extern uint32_t		lgrp_load_thresh;
  extern lgrp_mem_policy_t lgrp_mem_policy_root;
  
  #endif	/* _KERNEL || _FAKE_KERNEL || _KMEMUSER */
  
  #ifdef	__cplusplus
  }
  #endif
  
  #endif /* _LGRP_H */
  