text: /*
   * CDDL HEADER START
   *
   * The contents of this file are subject to the terms of the
   * Common Development and Distribution License (the "License").
   * You may not use this file except in compliance with the License.
   *
   * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   * or http://www.opensolaris.org/os/licensing.
   * See the License for the specific language governing permissions
   * and limitations under the License.
   *
   * When distributing Covered Code, include this CDDL HEADER in each
   * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
   * If applicable, add the following below this CDDL HEADER, with the
   * fields enclosed by brackets "[]" replaced with your own identifying
   * information: Portions Copyright [yyyy] [name of copyright owner]
   *
   * CDDL HEADER END
   */
  /*
   * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
   * Use is subject to license terms.
   */
  
  /*
   * VM - Hardware Address Translation management.
   *
   * This file describes the contents of the sun reference mmu (sfmmu)
   * specific hat data structures and the sfmmu specific hat procedures.
   * The machine independent interface is described in <vm/hat.h>.
   */
  
  #ifndef _VM_MACH_SFMMU_H
  #define	_VM_MACH_SFMMU_H
  
  #include <sys/x_call.h>
  #include <sys/hypervisor_api.h>
  
  #ifdef	__cplusplus
  extern "C" {
  #endif
  
  /*
   * Define UTSB_PHYS if user TSB is always accessed via physical address.
   * On sun4v platform, user TSB is accessed via physical address.
   */
  #define	UTSB_PHYS	1
  
  /*
   * Hypervisor TSB info
   */
  #define	NHV_TSB_INFO	4
  
  #ifndef _ASM
  
  struct hv_tsb_block {
  	uint64_t	hv_tsb_info_pa;	/* hypervisor TSB info PA */
  	uint64_t	hv_tsb_info_cnt; /* hypervisor TSB info count */
  	hv_tsb_info_t	hv_tsb_info[NHV_TSB_INFO]; /* hypervisor TSB info */
  };
  
  #endif /* _ASM */
  
  #ifdef _ASM
  
  /*
   * This macro is used to set private/shared secondary context register in
   * sfmmu_alloc_ctx().
   * Input:
   * cnum     = cnum
   * is_shctx = sfmmu private/shared flag (0: private, 1: shared)
   * tmp2 is only used in the sun4u version of this macro
   */
  #define	SET_SECCTX(cnum, is_shctx, tmp1, tmp2, label)			\
  	mov	MMU_SCONTEXT, tmp1;					\
  	movrnz	is_shctx, MMU_SCONTEXT1, tmp1;				\
  	stxa    cnum, [tmp1]ASI_MMU_CTX;  /* set 2nd ctx reg. */	\
  	membar  #Sync;							\
  
  /*
   * This macro is used in the MMU code to check if TL should be lowered from
   * 2 to 1 to pop trapstat's state.  See the block comment in trapstat.c
   * for details.
   */
  
  #define	TSTAT_CHECK_TL1(label, scr1, scr2)			\
  	rdpr	%tpc, scr1;					\
  	sethi	%hi(KERNELBASE), scr2;				\
  	or	scr2, %lo(KERNELBASE), scr2; 			\
  	cmp	scr1, scr2; 					\
  	bgeu	%xcc, 9f;					\
  	nop;							\
  	wrpr	%g0, 1, %gl;					\
  	ba	label;						\
  	wrpr	%g0, 1, %tl;					\
  9:
  
  /*
   * The following macros allow us to share majority of the
   * SFMMU code between sun4u and sun4v platforms.
   */
  
  #define	SETUP_TSB_ASI(qlp, tmp)
  
  #define	SETUP_UTSB_ATOMIC_ASI(tmp1, tmp2)
  
  /*
   * Macro to swtich to alternate global register on sun4u platforms
   * (not applicable to sun4v platforms)
   */
  #define	USE_ALTERNATE_GLOBALS(scr)
  
  /*
   * Macro to set %gl register value on sun4v platforms
   * (not applicable to sun4u platforms)
   */
  #define	SET_GL_REG(val)						\
  	wrpr	%g0, val, %gl
  
  /*
   * Get pseudo-tagacc value and context from the MMU fault area.  Pseudo-tagacc
   * is the faulting virtual address OR'd with 0 for KCONTEXT, INVALID_CONTEXT
   * (1) for invalid context, and USER_CONTEXT (2) for user context.
   *
   * In:
   *   tagacc, ctxtype = scratch registers
   * Out:
   *   tagacc = MMU data tag access register value
   *   ctx = context type (KCONTEXT, INVALID_CONTEXT or USER_CONTEXT)
   */
  #define	GET_MMU_D_PTAGACC_CTXTYPE(ptagacc, ctxtype)			\
  	MMU_FAULT_STATUS_AREA(ctxtype);					\
  	ldx	[ctxtype + MMFSA_D_ADDR], ptagacc;			\
  	ldx	[ctxtype + MMFSA_D_CTX], ctxtype;			\
  	srlx	ptagacc, MMU_PAGESHIFT, ptagacc; /* align to page boundary */ \
  	cmp	ctxtype, USER_CONTEXT_TYPE;				\
  	sllx	ptagacc, MMU_PAGESHIFT, ptagacc;			\
  	movgu	%icc, USER_CONTEXT_TYPE, ctxtype;			\
  	or	ptagacc, ctxtype, ptagacc
  
  /*
   * Synthesize/get data tag access register value from the MMU fault area
   *
   * In:
   *   tagacc, scr1 = scratch registers
   * Out:
   *   tagacc = MMU data tag access register value
   */
  #define	GET_MMU_D_TAGACC(tagacc, scr1)				\
  	GET_MMU_D_PTAGACC_CTXTYPE(tagacc, scr1)
  
  /*
   * Synthesize/get data tag target register value from the MMU fault area
   *
   * In:
   *   ttarget, scr1 = scratch registers
   * Out:
   *   ttarget = MMU data tag target register value
   */
  #define	GET_MMU_D_TTARGET(ttarget, scr1)			\
  	MMU_FAULT_STATUS_AREA(ttarget);				\
  	ldx	[ttarget + MMFSA_D_CTX], scr1;			\
  	sllx	scr1, TTARGET_CTX_SHIFT, scr1;			\
  	ldx	[ttarget + MMFSA_D_ADDR], ttarget;		\
  	srlx	ttarget, TTARGET_VA_SHIFT, ttarget;		\
  	or	ttarget, scr1, ttarget
  
  /*
   * Synthesize/get data/instruction psuedo tag access register values
   * from the MMU fault area (context is 0 for kernel, 1 for invalid, 2 for user)
   *
   * In:
   *   dtagacc, itagacc, scr1, scr2 = scratch registers
   * Out:
   *   dtagacc = MMU data tag access register value w/psuedo-context
   *   itagacc = MMU instruction tag access register value w/pseudo-context
   */
  #define	GET_MMU_BOTH_TAGACC(dtagacc, itagacc, scr1, scr2)	\
  	MMU_FAULT_STATUS_AREA(scr1);				\
  	ldx	[scr1 + MMFSA_D_ADDR], scr2;			\
  	ldx	[scr1 + MMFSA_D_CTX], dtagacc;			\
  	srlx	scr2, MMU_PAGESHIFT, scr2;	/* align to page boundary */ \
  	cmp	dtagacc, USER_CONTEXT_TYPE;			\
  	sllx	scr2, MMU_PAGESHIFT, scr2;			\
  	movgu	%icc, USER_CONTEXT_TYPE, dtagacc;		\
  	or	scr2, dtagacc, dtagacc;				\
  	ldx	[scr1 + MMFSA_I_ADDR], scr2;			\
  	ldx	[scr1 + MMFSA_I_CTX], itagacc;			\
  	srlx	scr2, MMU_PAGESHIFT, scr2;	/* align to page boundry */ \
  	cmp	itagacc, USER_CONTEXT_TYPE;			\
  	sllx	scr2, MMU_PAGESHIFT, scr2;			\
  	movgu	%icc, USER_CONTEXT_TYPE, itagacc;		\
  	or	scr2, itagacc, itagacc
  
  /*
   * Synthesize/get MMU data fault address from the MMU fault area
   *
   * In:
   *   daddr, scr1 = scratch registers
   * Out:
   *   daddr = MMU data fault address
   */
  #define	GET_MMU_D_ADDR(daddr, scr1)				\
  	MMU_FAULT_STATUS_AREA(scr1);				\
  	ldx	[scr1 + MMFSA_D_ADDR], daddr
  
  /*
   * Get pseudo-tagacc value and context from the MMU fault area.  Pseudo-tagacc
   * is the faulting virtual address OR'd with 0 for KCONTEXT, INVALID_CONTEXT
   * (1) for invalid context, and USER_CONTEXT (2) for user context.
   *
   * In:
   *   tagacc, ctxtype = scratch registers
   * Out:
   *   tagacc = MMU instruction tag access register value
   *   ctxtype = context type (KCONTEXT, INVALID_CONTEXT or USER_CONTEXT)
   */
  #define	GET_MMU_I_PTAGACC_CTXTYPE(ptagacc, ctxtype)			\
  	MMU_FAULT_STATUS_AREA(ctxtype);					\
  	ldx	[ctxtype + MMFSA_I_ADDR], ptagacc;			\
  	ldx	[ctxtype + MMFSA_I_CTX], ctxtype;			\
  	srlx	ptagacc, MMU_PAGESHIFT, ptagacc; /* align to page boundary */ \
  	cmp	ctxtype, USER_CONTEXT_TYPE;				\
  	sllx	ptagacc, MMU_PAGESHIFT, ptagacc;			\
  	movgu	%icc, USER_CONTEXT_TYPE, ctxtype;			\
  	or	ptagacc, ctxtype, ptagacc
  
  /*
   * Load ITLB entry
   *
   * In:
   *   tte = reg containing tte
   *   scr1, scr2, scr3, scr4 = scratch registers
   */
  #define	ITLB_STUFF(tte, scr1, scr2, scr3, scr4)		\
  	mov	%o0, scr1;				\
  	mov	%o1, scr2;				\
  	mov	%o2, scr3;				\
  	mov	%o3, scr4;				\
  	MMU_FAULT_STATUS_AREA(%o2);			\
  	ldx	[%o2 + MMFSA_I_ADDR], %o0;		\
  	ldx	[%o2 + MMFSA_I_CTX], %o1;		\
  	mov	tte, %o2;				\
  	mov	MAP_ITLB, %o3;				\
  	ta	MMU_MAP_ADDR;				\
  	/* BEGIN CSTYLED */				\
  	brnz,a,pn %o0, ptl1_panic;			\
  	  mov	PTL1_BAD_HCALL, %g1;			\
  	/* END CSTYLED */				\
  	mov	scr1, %o0;				\
  	mov	scr2, %o1;				\
  	mov	scr3, %o2;				\
  	mov	scr4, %o3
  
  /*
   * Load DTLB entry
   *
   * In:
   *   tte = reg containing tte
   *   scr1, scr2, scr3, scr4 = scratch registers
   */
  #define	DTLB_STUFF(tte, scr1, scr2, scr3, scr4)		\
  	mov	%o0, scr1;				\
  	mov	%o1, scr2;				\
  	mov	%o2, scr3;				\
  	mov	%o3, scr4;				\
  	MMU_FAULT_STATUS_AREA(%o2);			\
  	ldx	[%o2 + MMFSA_D_ADDR], %o0;		\
  	ldx	[%o2 + MMFSA_D_CTX], %o1;		\
  	mov	tte, %o2;				\
  	mov	MAP_DTLB, %o3;				\
  	ta	MMU_MAP_ADDR;				\
  	/* BEGIN CSTYLED */				\
  	brnz,a,pn %o0, ptl1_panic;			\
  	  mov	PTL1_BAD_HCALL, %g1;			\
  	/* END CSTYLED */				\
  	mov	scr1, %o0;				\
  	mov	scr2, %o1;				\
  	mov	scr3, %o2;				\
  	mov	scr4, %o3
  
  /*
   * Returns PFN given the TTE and vaddr
   *
   * In:
   *   tte = reg containing tte
   *   vaddr = reg containing vaddr
   *   scr1, scr2, scr3 = scratch registers
   * Out:
   *   tte = PFN value
   */
  #define	TTETOPFN(tte, vaddr, label, scr1, scr2, scr3)			\
  	and	tte, TTE_SZ_BITS, scr1;		/* scr1 = ttesz */	\
  	sllx	tte, TTE_PA_LSHIFT, tte;				\
  	sllx	scr1, 1, scr2;						\
  	add	scr2, scr1, scr2;		/* mulx 3 */		\
  	add	scr2, MMU_PAGESHIFT + TTE_PA_LSHIFT, scr3;		\
  	/* CSTYLED */							\
  	brz,pt	scr2, label/**/1;					\
  	srlx	tte, scr3, tte;						\
  	sllx	tte, scr2, tte;						\
  	set	1, scr1;						\
  	add	scr2, MMU_PAGESHIFT, scr3;				\
  	sllx	scr1, scr3, scr1;					\
  	sub	scr1, 1, scr1;	/* scr1=TTE_PAGE_OFFSET(ttesz) */	\
  	and	vaddr, scr1, scr2;					\
  	srln	scr2, MMU_PAGESHIFT, scr2;				\
  	or	tte, scr2, tte;						\
  	/* CSTYLED */							\
  label/**/1:
  
  /*
   * TTE_SET_REF_ML is a macro that updates the reference bit if it is
   * not already set.
   *
   * Parameters:
   * tte      = reg containing tte
   * ttepa    = physical pointer to tte
   * tsbarea  = tsb miss area
   * tmp1     = tmp reg
   * tmp2     = tmp reg
   * label    = temporary label
   */
  
  #define	TTE_SET_REF_ML(tte, ttepa, tsbarea, tmp1, tmp2, label)		\
  	/* BEGIN CSTYLED */						\
  	/* check reference bit */					\
  	btst	TTE_REF_INT, tte;					\
  	bnz,pt	%xcc, label/**/2;	/* if ref bit set-skip ahead */	\
  	nop;								\
  	/* update reference bit */					\
  label/**/1:								\
  	or	tte, TTE_REF_INT, tmp1;					\
  	casxa	[ttepa]ASI_MEM, tte, tmp1; 	/* update ref bit */	\
  	cmp	tte, tmp1;						\
  	bne,a,pn %xcc, label/**/1;					\
  	ldxa	[ttepa]ASI_MEM, tte;	/* MMU_READTTE through pa */	\
  	or	tte, TTE_REF_INT, tte;					\
  label/**/2:								\
  	/* END CSTYLED */
  
  
  /*
   * TTE_SET_REFMOD_ML is a macro that updates the reference and modify bits
   * if not already set.
   *
   * Parameters:
   * tte      = reg containing tte
   * ttepa    = physical pointer to tte
   * tsbarea  = tsb miss area
   * tmp1     = tmp reg
   * tmp2     = tmp reg
   * label    = temporary label
   * exitlabel = label where to jump to if write perm bit not set.
   */
  
  #define	TTE_SET_REFMOD_ML(tte, ttepa, tsbarea, tmp1, tmp2, label,	\
  	exitlabel)							\
  	/* BEGIN CSTYLED */						\
  	/* check reference bit */					\
  	btst	TTE_WRPRM_INT, tte;					\
  	bz,pn	%xcc, exitlabel;	/* exit if wr_perm no set */	\
  	  btst	TTE_HWWR_INT, tte;					\
  	bnz,pn	%xcc, label/**/2;	/* nothing to do */		\
  	  nop;								\
  	/* update reference bit */					\
  label/**/1:								\
  	or	tte, TTE_HWWR_INT | TTE_REF_INT, tmp1;			\
  	casxa	[ttepa]ASI_MEM, tte, tmp1; /* update ref/mod bit */	\
  	cmp	tte, tmp1;						\
  	bne,a,pn %xcc, label/**/1;					\
  	  ldxa	[ttepa]ASI_MEM, tte;	/* MMU_READTTE through pa */	\
  	or	tte, TTE_HWWR_INT | TTE_REF_INT, tte;			\
  label/**/2:								\
  	/* END CSTYLED */
  /*
   * Get TSB base register from the scratchpad for
   * shared contexts
   *
   * In:
   *   tsbmiss = pointer to tsbmiss area
   *   tsbmissoffset = offset to right tsb pointer
   *   tsbreg = scratch
   * Out:
   *   tsbreg = tsbreg from the specified scratchpad register
   */
  #define	GET_UTSBREG_SHCTX(tsbmiss, tsbmissoffset, tsbreg)		\
  	ldx	[tsbmiss + tsbmissoffset], tsbreg
  
  
  /*
   * Get the location of the TSB entry in the first TSB to probe
   *
   * In:
   *   tagacc = tag access register (not clobbered)
   *   tsbe, tmp1, tmp2 = scratch registers
   * Out:
   *   tsbe = pointer to the tsbe in the 1st TSB
   */
  
  #define	GET_1ST_TSBE_PTR(tagacc, tsbe, tmp1, tmp2)			\
  	/* BEGIN CSTYLED */						\
  	mov	SCRATCHPAD_UTSBREG1, tmp1				;\
  	ldxa	[tmp1]ASI_SCRATCHPAD, tsbe	/* get tsbreg */	;\
  	and	tsbe, TSB_SOFTSZ_MASK, tmp2	/* tmp2=szc */		;\
  	andn	tsbe, TSB_SOFTSZ_MASK, tsbe	/* tsbbase */		;\
  	mov	TSB_ENTRIES(0), tmp1	/* nentries in TSB size 0 */	;\
  	sllx	tmp1, tmp2, tmp1	/* tmp1 = nentries in TSB */	;\
  	sub	tmp1, 1, tmp1		/* mask = nentries - 1 */	;\
  	srlx	tagacc, MMU_PAGESHIFT, tmp2 				;\
  	and	tmp2, tmp1, tmp1	/* tsbent = virtpage & mask */	;\
  	sllx	tmp1, TSB_ENTRY_SHIFT, tmp1	/* entry num --> ptr */	;\
  	add	tsbe, tmp1, tsbe	/* add entry offset to TSB base */ ;\
  	/* END CSTYLED */
  
  
  /*
   * Will probe the first TSB, and if it finds a match, will insert it
   * into the TLB and retry.
   *
   * tsbe_ptr = precomputed first TSB entry pointer (in, ro)
   * vpg_4m = 4M virtual page number for tag matching  (in, ro)
   * label = where to branch to if this is a miss (text)
   * %asi = atomic ASI to use for the TSB access
   *
   * For trapstat, we have to explicily use these registers.
   * g4 = location tag will be retrieved into from TSB (out)
   * g5 = location data(tte) will be retrieved into from TSB (out)
   */
  #define	PROBE_1ST_DTSB(tsbe_ptr, vpg_4m, label)	/* g4/g5 clobbered */	\
  	/* BEGIN CSTYLED */						\
  	ldda	[tsbe_ptr]ASI_QUAD_LDD_PHYS, %g4 /* g4 = tag, g5 = data */ ;\
  	cmp	%g4, vpg_4m		/* compare tag w/ TSB */	;\
  	bne,pn	%xcc, label/**/1	/* branch if !match */		;\
  	  nop								;\
  	brgez,pn %g5, label/**/1					;\
  	  nop								;\
  	TT_TRACE(trace_tsbhit)						;\
  	DTLB_STUFF(%g5, %g1, %g2, %g3, %g4)				;\
  	/* trapstat expects tte in %g5 */				;\
  	retry				/* retry faulted instruction */	;\
  label/**/1:								\
  	/* END CSTYLED */
  
  
  /*
   * Same as above, only if the TTE doesn't have the execute
   * bit set, will branch to exec_fault directly.
   */
  #define	PROBE_1ST_ITSB(tsbe_ptr, vpg_4m, label)				\
  	/* BEGIN CSTYLED */						\
  	ldda	[tsbe_ptr]ASI_QUAD_LDD_PHYS, %g4 /* g4 = tag, g5 = data */ ;\
  	cmp	%g4, vpg_4m		/* compare tag w/ TSB */	;\
  	bne,pn	%xcc, label/**/1	/* branch if !match */		;\
  	  nop								;\
  	brgez,pn %g5, label/**/1					;\
  	  nop								;\
  	andcc	%g5, TTE_EXECPRM_INT, %g0  /* check execute bit */	;\
  	bz,pn	%icc, exec_fault					;\
  	  nop								;\
  	TT_TRACE(trace_tsbhit)						;\
  	ITLB_STUFF(%g5, %g1, %g2, %g3, %g4)				;\
  	retry				/* retry faulted instruction */	;\
  label/**/1:								\
  	/* END CSTYLED */
  
  /*
   * vpg_4m = 4M virtual page number for tag matching (in)
   * tsbe_ptr = precomputed second TSB entry pointer (in)
   * label = label to use to make branch targets unique (text)
   *
   * For trapstat, we have to explicity use these registers.
   * g4 = tag portion of TSBE (out)
   * g5 = data portion of TSBE (out)
   */
  #define	PROBE_2ND_DTSB(tsbe_ptr, vpg_4m, label)				\
  	/* BEGIN CSTYLED */						\
  	ldda	[tsbe_ptr]ASI_QUAD_LDD_PHYS, %g4  /* g4 = tag, g5 = data */ ;\
  	/* since we are looking at 2nd tsb, if it's valid, it must be 4M */ ;\
  	cmp	%g4, vpg_4m						;\
  	bne,pn	%xcc, label/**/1					;\
  	  nop								;\
  	brgez,pn %g5, label/**/1					;\
  	  nop								;\
  	mov	tsbe_ptr, %g1		/* trace_tsbhit wants ptr in %g1 */ ;\
  	TT_TRACE(trace_tsbhit)						;\
  	DTLB_STUFF(%g5, %g1, %g2, %g3, %g4)				;\
  	/* trapstat expects tte in %g5 */				;\
  	retry				/* retry faulted instruction */	;\
  label/**/1:								\
  	/* END CSTYLED */
  
  
  /*
   * Same as above, with the following additions:
   * If the TTE found is not executable, branch directly
   * to exec_fault.  If a TSB miss, branch to TSB miss handler.
   */
  #define	PROBE_2ND_ITSB(tsbe_ptr, vpg_4m)				\
  	/* BEGIN CSTYLED */						\
  	ldda	[tsbe_ptr]ASI_QUAD_LDD_PHYS, %g4 /* g4 = tag, g5 = data */ ;\
  	cmp	%g4, vpg_4m		/* compare tag w/ TSB */	;\
  	bne,pn	%xcc, sfmmu_tsb_miss_tt	/* branch if !match */		;\
  	  nop								;\
  	brgez,pn %g5, sfmmu_tsb_miss_tt					;\
  	  nop								;\
  	andcc	%g5, TTE_EXECPRM_INT, %g0  /* check execute bit */	;\
  	bz,pn	%icc, exec_fault					;\
  	  mov	tsbe_ptr, %g1		/* trap trace wants ptr in %g1 */ ;\
  	TT_TRACE(trace_tsbhit)						;\
  	ITLB_STUFF(%g5, %g1, %g2, %g3, %g4)				;\
  	retry				/* retry faulted instruction */	\
  	/* END CSTYLED */
  
  /*
   * 1. Get ctx1. The traptype is supplied by caller.
   * 2. If iTSB miss, store in MMFSA_I_CTX
   * 3. if dTSB miss, store in MMFSA_D_CTX
   * 4. Thus the [D|I]TLB_STUFF will work as expected.
   */
  #define	SAVE_CTX1(traptype, ctx1, tmp, label)				\
  	/* BEGIN CSTYLED */						\
  	mov	MMU_SCONTEXT1, tmp					;\
  	ldxa	[tmp]ASI_MMU_CTX, ctx1					;\
  	MMU_FAULT_STATUS_AREA(tmp)					;\
  	cmp     traptype, FAST_IMMU_MISS_TT				;\
  	be,a,pn %icc, label						;\
  	  stx	ctx1, [tmp + MMFSA_I_CTX] 				;\
  	cmp     traptype, T_INSTR_MMU_MISS				;\
  	be,a,pn %icc, label						;\
  	  stx	ctx1, [tmp + MMFSA_I_CTX]				;\
  	stx	ctx1, [tmp + MMFSA_D_CTX]				;\
  label:
  	/* END CSTYLED */
  
  #endif /* _ASM */
  
  #ifdef	__cplusplus
  }
  #endif
  
  #endif	/* _VM_MACH_SFMMU_H */
  