text: /*
   * Copyright (c) 1993 The Regents of the University of California.
   * Copyright (c) 2008 The DragonFly Project.
   * All rights reserved.
   *
   * Redistribution and use in source and binary forms, with or without
   * modification, are permitted provided that the following conditions
   * are met:
   * 1. Redistributions of source code must retain the above copyright
   *    notice, this list of conditions and the following disclaimer.
   * 2. Redistributions in binary form must reproduce the above copyright
   *    notice, this list of conditions and the following disclaimer in the
   *    documentation and/or other materials provided with the distribution.
   * 3. Neither the name of the University nor the names of its contributors
   *    may be used to endorse or promote products derived from this software
   *    without specific prior written permission.
   *
   * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
   * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
   * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
   * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
   * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
   * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
   * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
   * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
   * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
   * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
   * SUCH DAMAGE.
   *
   * $FreeBSD: src/sys/amd64/include/asmacros.h,v 1.32 2006/10/28 06:04:29 bde Exp $
   */
  
  #ifndef _CPU_ASMACROS_H_
  #define _CPU_ASMACROS_H_
  
  #include <sys/cdefs.h>
  #include <machine/specialreg.h>
  
  /* XXX too much duplication in various asm*.h's. */
  
  /*
   * CNAME is used to manage the relationship between symbol names in C
   * and the equivalent assembly language names.  CNAME is given a name as
   * it would be used in a C program.  It expands to the equivalent assembly
   * language name.
   */
  #define CNAME(csym)		csym
  
  #define ALIGN_DATA	.p2align 3	/* 8 byte alignment, zero filled */
  #define ALIGN_TEXT	.p2align 4,0x90	/* 16-byte alignment, nop filled */
  #define SUPERALIGN_TEXT	.p2align 4,0x90	/* 16-byte alignment, nop filled */
  
  #define GEN_ENTRY(name)		ALIGN_TEXT; .globl CNAME(name); \
  				.type CNAME(name),@function; CNAME(name):
  #define NON_GPROF_ENTRY(name)	GEN_ENTRY(name)
  #define NON_GPROF_RET		.byte 0xc3	/* opcode for `ret' */
  
  #define	END(name)		.size name, . - name
  
  /*
   * ALTENTRY() has to align because it is before a corresponding ENTRY().
   * ENTRY() has to align to because there may be no ALTENTRY() before it.
   * If there is a previous ALTENTRY() then the alignment code for ENTRY()
   * is empty.
   */
  #define ALTENTRY(name)		GEN_ENTRY(name)
  #define	CROSSJUMP(jtrue, label, jfalse)	jtrue label
  #define	CROSSJUMPTARGET(label)
  #define ENTRY(name)		GEN_ENTRY(name)
  #define FAKE_MCOUNT(caller)
  #define MCOUNT
  #define MCOUNT_LABEL(name)
  #define MEXITCOUNT
  
  #ifdef LOCORE
  /*
   * Convenience macro for declaring interrupt entry points.
   */
  #define	IDTVEC(name)	ALIGN_TEXT; .globl __CONCAT(X,name); \
  			.type __CONCAT(X,name),@function; __CONCAT(X,name):
  
  /*
   * stack frame macro support - supports mmu isolation, swapgs, and
   * stack frame pushing and popping.
   */
  
  /*
   * Kernel pmap isolation to work-around the massive Intel mmu bug
   * that allows kernel memory to be sussed out due to speculative memory
   * reads and instruction execution creating timing differences that can
   * be detected by userland.  e.g. force speculative read, speculatively
   * execute a cmp/branch sequence, detect timing.  Iterate cmp $values
   * to suss-out content of speculatively read kernel memory.
   *
   * We do this by creating a trampoline area for all user->kernel and
   * kernel->user transitions.  The trampoline area allows us to limit
   * the reach the kernel map in the isolated version of the user pmap
   * to JUST the trampoline area (for all cpus), tss, and vector area.
   *
   * It is very important that these transitions not access any memory
   * outside of the trampoline page while the isolated user process pmap
   * is active in %cr3.
   *
   * The trampoline does not add much overhead when pmap isolation is
   * disabled, so we just run with it regardless.  Of course, when pmap
   * isolation is enabled, the %cr3 loads add 150-250ns to every system
   * call as well as (without PCID) smash the TLB.
   *
   * KMMUENTER -	Executed by the trampoline when a user->kernel transition
   *		is detected.  The stack pointer points into the pcpu
   *		trampoline space and is available for register save/restore.
   *		Other registers have not yet been saved.  %gs points at
   *		the kernel pcpu structure.
   *
   *		Caller has already determined that a transition is in
   *		progress and has already issued the swapgs.  hwtf indicates
   *		how much hardware has already pushed.
   *
   * KMMUEXIT  -	Executed when a kernel->user transition is made.  The stack
   *		pointer points into the pcpu trampoline space and we are
   *		almost ready to iretq.  %gs still points at the kernel pcpu
   *		structure.
   *
   *		Caller has already determined that a transition is in
   *		progress.  hwtf indicates how much hardware has already
   *		pushed.
   */
  
  /*
   * KMMUENTER_CORE - Handles ISOMMU, IBRS, and IBPB.  Caller has already
   *		    saved %rcx and %rdx.  We have to deal with %rax.
   *
   *		    XXX If IBPB is not supported, try to clear the
   *		    call return hw cache w/ many x chained call sequence?
   *
   * NOTE - IBRS2 - We are leaving IBRS on full-time.  However, Intel
   *		  believes it is not safe unless the MSR is poked on each
   *		  user->kernel transition, so poke the MSR for both IBRS1
   *		  and IBRS2.
   */
  #define KMMUENTER_CORE							\
  	testq	$PCB_ISOMMU,PCPU(trampoline)+TR_PCB_FLAGS ; 		\
  	je	40f ;							\
  	movq	PCPU(trampoline)+TR_PCB_CR3,%rcx ;			\
  	movq	%rcx,%cr3 ;						\
  40:	movl	PCPU(trampoline)+TR_PCB_SPEC_CTRL,%edx ;		\
  	testq	%rdx, %rdx ;						\
  	je	43f ;							\
  	movq	%rax, PCPU(trampoline)+TR_RAX ;				\
  	testq	$SPEC_CTRL_DUMMY_ENABLE,%rdx ;				\
  	je	41f ;							\
  	movq	%rdx, %rax ;						\
  	andq	$SPEC_CTRL_IBRS|SPEC_CTRL_STIBP, %rax ;			\
  	movq	$MSR_SPEC_CTRL,%rcx ;					\
  	xorl	%edx,%edx ;						\
  	wrmsr ;								\
  	movl	PCPU(trampoline)+TR_PCB_SPEC_CTRL,%edx ;		\
  41:	testq	$SPEC_CTRL_DUMMY_IBPB,%rdx ;				\
  	je	42f ;							\
  	movl	$MSR_PRED_CMD,%ecx ;					\
  	movl	$1,%eax ;						\
  	xorl	%edx,%edx ;						\
  	wrmsr ;								\
  42:	movq	PCPU(trampoline)+TR_RAX, %rax ;				\
  43:									\
  
  /*
   * Enter with trampoline, hardware pushed up to %rip
   */
  #define KMMUENTER_TFRIP							\
  	subq	$TR_RIP, %rsp ;						\
  	movq	%rcx, TR_RCX(%rsp) ;					\
  	movq	%rdx, TR_RDX(%rsp) ;					\
  	KMMUENTER_CORE ;						\
  	movq	%rsp, %rcx ;		/* trampoline rsp */		\
  	movq	PCPU(trampoline)+TR_PCB_RSP,%rsp ; /* kstack rsp */	\
  	movq	TR_SS(%rcx), %rdx ;					\
  	pushq	%rdx ;							\
  	movq	TR_RSP(%rcx), %rdx ;					\
  	pushq	%rdx ;							\
  	movq	TR_RFLAGS(%rcx), %rdx ;					\
  	pushq	%rdx ;							\
  	movq	TR_CS(%rcx), %rdx ;					\
  	pushq	%rdx ;							\
  	movq	TR_RIP(%rcx), %rdx ;					\
  	pushq	%rdx ;							\
  	movq	TR_RDX(%rcx), %rdx ;					\
  	movq	TR_RCX(%rcx), %rcx					\
  
  /*
   * Enter with trampoline, hardware pushed up to ERR
   */
  #define KMMUENTER_TFERR							\
  	subq	$TR_ERR, %rsp ;						\
  	movq	%rcx, TR_RCX(%rsp) ;					\
  	movq	%rdx, TR_RDX(%rsp) ;					\
  	KMMUENTER_CORE ;						\
  	movq	%rsp, %rcx ;		/* trampoline rsp */		\
  	movq	PCPU(trampoline)+TR_PCB_RSP,%rsp ; /* kstack rsp */	\
  	movq	TR_SS(%rcx), %rdx ;					\
  	pushq	%rdx ;							\
  	movq	TR_RSP(%rcx), %rdx ;					\
  	pushq	%rdx ;							\
  	movq	TR_RFLAGS(%rcx), %rdx ;					\
  	pushq	%rdx ;							\
  	movq	TR_CS(%rcx), %rdx ;					\
  	pushq	%rdx ;							\
  	movq	TR_RIP(%rcx), %rdx ;					\
  	pushq	%rdx ;							\
  	movq	TR_ERR(%rcx), %rdx ;					\
  	pushq	%rdx ;							\
  	movq	TR_RDX(%rcx), %rdx ;					\
  	movq	TR_RCX(%rcx), %rcx					\
  
  /*
   * Enter with trampoline, hardware pushed up to ERR and
   * we need to save %cr2 early (before potentially reloading %cr3).
   */
  #define KMMUENTER_TFERR_SAVECR2						\
  	subq	$TR_ERR, %rsp ;						\
  	movq	%rcx, TR_RCX(%rsp) ;					\
  	movq	%rdx, TR_RDX(%rsp) ;					\
  	movq	%cr2, %rcx ;						\
  	movq	%rcx, PCPU(trampoline)+TR_CR2 ;				\
  	KMMUENTER_CORE ;						\
  	movq	%rsp, %rcx ;		/* trampoline rsp */		\
  	movq	PCPU(trampoline)+TR_PCB_RSP,%rsp ; /* kstack rsp */	\
  	movq	TR_SS(%rcx), %rdx ;					\
  	pushq	%rdx ;							\
  	movq	TR_RSP(%rcx), %rdx ;					\
  	pushq	%rdx ;							\
  	movq	TR_RFLAGS(%rcx), %rdx ;					\
  	pushq	%rdx ;							\
  	movq	TR_CS(%rcx), %rdx ;					\
  	pushq	%rdx ;							\
  	movq	TR_RIP(%rcx), %rdx ;					\
  	pushq	%rdx ;							\
  	movq	TR_ERR(%rcx), %rdx ;					\
  	pushq	%rdx ;							\
  	movq	TR_RDX(%rcx), %rdx ;					\
  	movq	TR_RCX(%rcx), %rcx					\
  
  /*
   * Set %cr3 if necessary on syscall entry.  No registers may be
   * disturbed.
   *
   * NOTE: TR_CR2 is used by the caller to save %rsp, we cannot use it here.
   */
  #define KMMUENTER_SYSCALL						\
  	movq	%rcx, PCPU(trampoline)+TR_RCX ;				\
  	movq	%rdx, PCPU(trampoline)+TR_RDX ;				\
  	KMMUENTER_CORE ;						\
  	movq	PCPU(trampoline)+TR_RDX, %rdx ;				\
  	movq	PCPU(trampoline)+TR_RCX, %rcx 				\
  
  /*
   * KMMUEXIT_CORE handles IBRS, STIBP, and MDS, but not ISOMMU
   *
   * We don't re-execute the IBPB barrier on exit atm.
   *
   * The MDS barrier (Microarchitectural Data Sampling) should be executed
   * prior to any return to user-mode, if supported and enabled.  This is
   * Intel-only.
   *
   * WARNING! %rsp may not be usable (it could be pointing to the user
   *	    stack at this point).  And we must save/restore any registers
   *	    we use.
   */
  #define KMMUEXIT_CORE							\
  	testl	$SPEC_CTRL_DUMMY_ENABLE|SPEC_CTRL_MDS_ENABLE, PCPU(trampoline)+TR_PCB_SPEC_CTRL+4 ; \
  	je	43f ;							\
  	movq	%rax, PCPU(trampoline)+TR_RAX ;				\
  	movl	PCPU(trampoline)+TR_PCB_SPEC_CTRL+4, %eax ;		\
  	testq	$SPEC_CTRL_MDS_ENABLE, %rax ;				\
  	je	41f ;							\
  	movq	$GSEL(GDATA_SEL, SEL_KPL), PCPU(trampoline)+TR_RCX ;	\
  	verw	PCPU(trampoline)+TR_RCX ;				\
  41:	testq	$SPEC_CTRL_DUMMY_ENABLE, %rax ;				\
  	je	42f ;							\
  	movq	%rcx, PCPU(trampoline)+TR_RCX ;				\
  	movq	%rdx, PCPU(trampoline)+TR_RDX ;				\
  	andq	$SPEC_CTRL_IBRS|SPEC_CTRL_STIBP, %rax ;			\
  	movq	$MSR_SPEC_CTRL,%rcx ;					\
  	xorl	%edx,%edx ;						\
  	wrmsr ;								\
  	movq	PCPU(trampoline)+TR_RDX, %rdx ;				\
  	movq	PCPU(trampoline)+TR_RCX, %rcx ;				\
  42:	movq	PCPU(trampoline)+TR_RAX, %rax ;				\
  43:
  
  /*
   * We are positioned at the base of the trapframe.  Advance the trapframe
   * and handle MMU isolation.  MMU isolation requires us to copy the
   * hardware frame to the trampoline area before setting %cr3 to the
   * isolated map.  We then set the %rsp for iretq to TR_RIP in the
   * trampoline area (after restoring the register we saved in TR_ERR).
   */
  #define KMMUEXIT							\
  	addq	$TF_RIP,%rsp ;						\
  	KMMUEXIT_CORE ;							\
  	testq	$PCB_ISOMMU,PCPU(trampoline)+TR_PCB_FLAGS ; 		\
  	je	50f ;							\
  	movq	%rcx, PCPU(trampoline)+TR_ERR ;	/* save in TR_ERR */	\
  	popq	%rcx ;				/* copy %rip */		\
  	movq	%rcx, PCPU(trampoline)+TR_RIP ;				\
  	popq	%rcx ;				/* copy %cs */		\
  	movq	%rcx, PCPU(trampoline)+TR_CS ;				\
  	popq	%rcx ;				/* copy %rflags */	\
  	movq	%rcx, PCPU(trampoline)+TR_RFLAGS ;			\
  	popq	%rcx ;				/* copy %rsp */		\
  	movq	%rcx, PCPU(trampoline)+TR_RSP ;				\
  	popq	%rcx ;				/* copy %ss */		\
  	movq	%rcx, PCPU(trampoline)+TR_SS ;				\
  	movq	%gs:0,%rcx ;						\
  	addq	$GD_TRAMPOLINE+TR_ERR,%rcx ;				\
  	movq	%rcx,%rsp ;						\
  	movq	PCPU(trampoline)+TR_PCB_CR3_ISO,%rcx ;			\
  	movq	%rcx,%cr3 ;						\
  	popq	%rcx ;		/* positioned at TR_RIP after this */	\
  50:									\
  
  /*
   * Warning: user stack pointer already loaded into %rsp at this
   * point.  We still have the kernel %gs.
   *
   * Caller will sysexit, we do not have to copy anything to the
   * trampoline area.
   */
  #define KMMUEXIT_SYSCALL						\
  	KMMUEXIT_CORE ;							\
  	testq	$PCB_ISOMMU,PCPU(trampoline)+TR_PCB_FLAGS ; 		\
  	je	50f ;							\
  	movq	%rcx, PCPU(trampoline)+TR_RCX ;				\
  	movq	PCPU(trampoline)+TR_PCB_CR3_ISO,%rcx ;			\
  	movq	%rcx,%cr3 ;						\
  	movq	PCPU(trampoline)+TR_RCX, %rcx ;				\
  50:									\
  
  /*
   * Macros to create and destroy a trap frame.  rsp has already been shifted
   * to the base of the trapframe in the thread structure.
   */
  #define PUSH_FRAME_REGS							\
  	movq	%rdi,TF_RDI(%rsp) ;					\
  	movq	%rsi,TF_RSI(%rsp) ;					\
  	movq	%rdx,TF_RDX(%rsp) ;					\
  	movq	%rcx,TF_RCX(%rsp) ;					\
  	movq	%r8,TF_R8(%rsp) ;					\
  	movq	%r9,TF_R9(%rsp) ;					\
  	movq	%rax,TF_RAX(%rsp) ;					\
  	movq	%rbx,TF_RBX(%rsp) ;					\
  	movq	%rbp,TF_RBP(%rsp) ;					\
  	movq	%r10,TF_R10(%rsp) ;					\
  	movq	%r11,TF_R11(%rsp) ;					\
  	movq	%r12,TF_R12(%rsp) ;					\
  	movq	%r13,TF_R13(%rsp) ;					\
  	movq	%r14,TF_R14(%rsp) ;					\
  	movq	%r15,TF_R15(%rsp) ;					\
  					/* SECURITY CLEAR REGS */	\
  	xorq	%rax,%rax ;						\
  	movq	%rax,%rbx ;						\
  	movq	%rax,%rcx ;						\
  	movq	%rax,%rdx ;						\
  	movq	%rax,%rdi ;						\
  	movq	%rax,%rsi ;						\
  	movq	%rax,%rbp ;						\
  	movq	%rax,%r8 ;						\
  	movq	%rax,%r9 ;						\
  	movq	%rax,%r10 ;						\
  	movq	%rax,%r11 ;						\
  	movq	%rax,%r12 ;						\
  	movq	%rax,%r13 ;						\
  	movq	%rax,%r14 ;						\
  	movq	%rax,%r15 						\
  
  
  /*
   * PUSH_FRAME is the first thing executed upon interrupt entry.  We are
   * responsible for swapgs execution and the KMMUENTER dispatch.
   *
   * NOTE - PUSH_FRAME code doesn't mess with %gs or the stack, or assume it can
   *	  use PCPU(trampoline), if the trap/exception is from supevisor mode.
   *	  It only messes with that stuff when the trap/exception is from user
   *	  mode.  Our DBG and NMI code depend on this behavior.
   */
  #define PUSH_FRAME_TFRIP						\
  	testb	$SEL_RPL_MASK,TF_CS-TF_RIP(%rsp) ; /* from userland? */	\
  	jz	1f ;							\
  	swapgs ;		/* from userland */			\
  	KMMUENTER_TFRIP ;	/* from userland */			\
  1:									\
  	subq	$TF_RIP,%rsp ;						\
  	PUSH_FRAME_REGS ; 						\
  	SMAP_CLOSE							\
  
  #define PUSH_FRAME_TFERR						\
  	testb	$SEL_RPL_MASK,TF_CS-TF_ERR(%rsp) ; /* from userland? */	\
  	jz	1f ;							\
  	swapgs ;		/* from userland */			\
  	KMMUENTER_TFERR ;	/* from userland */			\
  1:									\
  	subq	$TF_ERR,%rsp ;						\
  	PUSH_FRAME_REGS ;						\
  	SMAP_CLOSE							\
  
  #define PUSH_FRAME_TFERR_SAVECR2					\
  	testb	$SEL_RPL_MASK,TF_CS-TF_ERR(%rsp) ;			\
  	jz	1f ;							\
  	swapgs ;		/* from userland */			\
  	KMMUENTER_TFERR_SAVECR2 ;/* from userland */			\
  	subq	$TF_ERR,%rsp ;						\
  	PUSH_FRAME_REGS ;						\
  	movq	PCPU(trampoline)+TR_CR2, %r10 ;				\
  	jmp 2f ;							\
  1:									\
  	subq	$TF_ERR,%rsp ;						\
  	PUSH_FRAME_REGS ;						\
  	movq	%cr2, %r10 ;						\
  2:									\
  	movq	%r10, TF_ADDR(%rsp) ;					\
  	SMAP_CLOSE							\
  
  /*
   * POP_FRAME is issued just prior to the iretq, or just prior to a
   * jmp doreti_iret.  These must be passed in to the macro.
   */
  #define POP_FRAME(lastinsn)						\
  	movq	TF_RDI(%rsp),%rdi ;					\
  	movq	TF_RSI(%rsp),%rsi ;					\
  	movq	TF_RDX(%rsp),%rdx ;					\
  	movq	TF_RCX(%rsp),%rcx ;					\
  	movq	TF_R8(%rsp),%r8 ;					\
  	movq	TF_R9(%rsp),%r9 ;					\
  	movq	TF_RAX(%rsp),%rax ;					\
  	movq	TF_RBX(%rsp),%rbx ;					\
  	movq	TF_RBP(%rsp),%rbp ;					\
  	movq	TF_R10(%rsp),%r10 ;					\
  	movq	TF_R11(%rsp),%r11 ;					\
  	movq	TF_R12(%rsp),%r12 ;					\
  	movq	TF_R13(%rsp),%r13 ;					\
  	movq	TF_R14(%rsp),%r14 ;					\
  	movq	TF_R15(%rsp),%r15 ;					\
  	cli ;								\
  	testb	$SEL_RPL_MASK,TF_CS(%rsp) ; /* return to user? */	\
  	jz	1f ;							\
  	KMMUEXIT ;		/* return to user */			\
  	swapgs ;		/* return to user */			\
  	jmp	2f ;							\
  1:									\
  	addq	$TF_RIP,%rsp ;	/* setup for iretq */			\
  2:									\
  	lastinsn
  
  /*
   * Access per-CPU data.
   */
  #define	PCPU(member)		%gs:gd_ ## member
  #define PCPU_E8(member,idx)	%gs:gd_ ## member(,idx,8)
  #define	PCPU_ADDR(member, reg)					\
  	movq %gs:PC_PRVSPACE, reg ;				\
  	addq $PC_ ## member, reg
  
  /*
   * This adds a code placemarker (the 3-byte NOP) and references
   * it from a section, allowing it to be replaced with a STAC or CLAC
   * instruction (also 3-byte instructions) when the system supports
   * SMAP.
   */
  #define SMAP_OPEN						\
  	.globl	__start_set_smap_open ;				\
  	.globl	__stop_set_smap_open ;				\
  60:	nop	%eax ;						\
  	.section set_smap_open,"aw" ;				\
  	.quad	60b ;						\
  	.previous						\
  
  #define SMAP_CLOSE						\
  	.globl	__start_set_smap_close ;			\
  	.globl	__stop_set_smap_close ;				\
  60:	nop	%eax ;						\
  	.section set_smap_close,"aw" ;				\
  	.quad	60b ;						\
  	.previous						\
  
  #endif /* LOCORE */
  
  #endif /* !_CPU_ASMACROS_H_ */
  