text: // SPDX-License-Identifier: GPL-2.0-or-later
  
  #include <linux/context_tracking.h>
  #include <linux/err.h>
  #include <linux/compat.h>
  
  #include <asm/asm-prototypes.h>
  #include <asm/kup.h>
  #include <asm/cputime.h>
  #include <asm/interrupt.h>
  #include <asm/hw_irq.h>
  #include <asm/interrupt.h>
  #include <asm/kprobes.h>
  #include <asm/paca.h>
  #include <asm/ptrace.h>
  #include <asm/reg.h>
  #include <asm/signal.h>
  #include <asm/switch_to.h>
  #include <asm/syscall.h>
  #include <asm/time.h>
  #include <asm/unistd.h>
  
  typedef long (*syscall_fn)(long, long, long, long, long, long);
  
  /* Has to run notrace because it is entered not completely "reconciled" */
  notrace long system_call_exception(long r3, long r4, long r5,
  				   long r6, long r7, long r8,
  				   unsigned long r0, struct pt_regs *regs)
  {
  	syscall_fn f;
  
  	regs->orig_gpr3 = r3;
  
  	if (IS_ENABLED(CONFIG_PPC_IRQ_SOFT_MASK_DEBUG))
  		BUG_ON(irq_soft_mask_return() != IRQS_ALL_DISABLED);
  
  	trace_hardirqs_off(); /* finish reconciling */
  
  	CT_WARN_ON(ct_state() == CONTEXT_KERNEL);
  	user_exit_irqoff();
  
  	if (!IS_ENABLED(CONFIG_BOOKE) && !IS_ENABLED(CONFIG_40x))
  		BUG_ON(!(regs->msr & MSR_RI));
  	BUG_ON(!(regs->msr & MSR_PR));
  	BUG_ON(!FULL_REGS(regs));
  	BUG_ON(arch_irq_disabled_regs(regs));
  
  #ifdef CONFIG_PPC_PKEY
  	if (mmu_has_feature(MMU_FTR_PKEY)) {
  		unsigned long amr, iamr;
  		bool flush_needed = false;
  		/*
  		 * When entering from userspace we mostly have the AMR/IAMR
  		 * different from kernel default values. Hence don't compare.
  		 */
  		amr = mfspr(SPRN_AMR);
  		iamr = mfspr(SPRN_IAMR);
  		regs->amr  = amr;
  		regs->iamr = iamr;
  		if (mmu_has_feature(MMU_FTR_BOOK3S_KUAP)) {
  			mtspr(SPRN_AMR, AMR_KUAP_BLOCKED);
  			flush_needed = true;
  		}
  		if (mmu_has_feature(MMU_FTR_BOOK3S_KUEP)) {
  			mtspr(SPRN_IAMR, AMR_KUEP_BLOCKED);
  			flush_needed = true;
  		}
  		if (flush_needed)
  			isync();
  	} else
  #endif
  #ifdef CONFIG_PPC64
  		kuap_check_amr();
  #endif
  
  	booke_restore_dbcr0();
  
  	account_cpu_user_entry();
  
  	account_stolen_time();
  
  	/*
  	 * This is not required for the syscall exit path, but makes the
  	 * stack frame look nicer. If this was initialised in the first stack
  	 * frame, or if the unwinder was taught the first stack frame always
  	 * returns to user with IRQS_ENABLED, this store could be avoided!
  	 */
  	irq_soft_mask_regs_set_state(regs, IRQS_ENABLED);
  
  	local_irq_enable();
  
  	if (unlikely(current_thread_info()->flags & _TIF_SYSCALL_DOTRACE)) {
  		if (unlikely(trap_is_unsupported_scv(regs))) {
  			/* Unsupported scv vector */
  			_exception(SIGILL, regs, ILL_ILLOPC, regs->nip);
  			return regs->gpr[3];
  		}
  		/*
  		 * We use the return value of do_syscall_trace_enter() as the
  		 * syscall number. If the syscall was rejected for any reason
  		 * do_syscall_trace_enter() returns an invalid syscall number
  		 * and the test against NR_syscalls will fail and the return
  		 * value to be used is in regs->gpr[3].
  		 */
  		r0 = do_syscall_trace_enter(regs);
  		if (unlikely(r0 >= NR_syscalls))
  			return regs->gpr[3];
  		r3 = regs->gpr[3];
  		r4 = regs->gpr[4];
  		r5 = regs->gpr[5];
  		r6 = regs->gpr[6];
  		r7 = regs->gpr[7];
  		r8 = regs->gpr[8];
  
  	} else if (unlikely(r0 >= NR_syscalls)) {
  		if (unlikely(trap_is_unsupported_scv(regs))) {
  			/* Unsupported scv vector */
  			_exception(SIGILL, regs, ILL_ILLOPC, regs->nip);
  			return regs->gpr[3];
  		}
  		return -ENOSYS;
  	}
  
  	/* May be faster to do array_index_nospec? */
  	barrier_nospec();
  
  	if (unlikely(is_compat_task())) {
  		f = (void *)compat_sys_call_table[r0];
  
  		r3 &= 0x00000000ffffffffULL;
  		r4 &= 0x00000000ffffffffULL;
  		r5 &= 0x00000000ffffffffULL;
  		r6 &= 0x00000000ffffffffULL;
  		r7 &= 0x00000000ffffffffULL;
  		r8 &= 0x00000000ffffffffULL;
  
  	} else {
  		f = (void *)sys_call_table[r0];
  	}
  
  	return f(r3, r4, r5, r6, r7, r8);
  }
  
  /*
   * local irqs must be disabled. Returns false if the caller must re-enable
   * them, check for new work, and try again.
   *
   * This should be called with local irqs disabled, but if they were previously
   * enabled when the interrupt handler returns (indicating a process-context /
   * synchronous interrupt) then irqs_enabled should be true.
   */
  static notrace __always_inline bool __prep_irq_for_enabled_exit(bool clear_ri)
  {
  	/* This must be done with RI=1 because tracing may touch vmaps */
  	trace_hardirqs_on();
  
  	/* This pattern matches prep_irq_for_idle */
  	if (clear_ri)
  		__hard_EE_RI_disable();
  	else
  		__hard_irq_disable();
  #ifdef CONFIG_PPC64
  	if (unlikely(lazy_irq_pending_nocheck())) {
  		/* Took an interrupt, may have more exit work to do. */
  		if (clear_ri)
  			__hard_RI_enable();
  		trace_hardirqs_off();
  		local_paca->irq_happened |= PACA_IRQ_HARD_DIS;
  
  		return false;
  	}
  	local_paca->irq_happened = 0;
  	irq_soft_mask_set(IRQS_ENABLED);
  #endif
  	return true;
  }
  
  static notrace inline bool prep_irq_for_enabled_exit(bool clear_ri, bool irqs_enabled)
  {
  	if (__prep_irq_for_enabled_exit(clear_ri))
  		return true;
  
  	/*
  	 * Must replay pending soft-masked interrupts now. Don't just
  	 * local_irq_enabe(); local_irq_disable(); because if we are
  	 * returning from an asynchronous interrupt here, another one
  	 * might hit after irqs are enabled, and it would exit via this
  	 * same path allowing another to fire, and so on unbounded.
  	 *
  	 * If interrupts were enabled when this interrupt exited,
  	 * indicating a process context (synchronous) interrupt,
  	 * local_irq_enable/disable can be used, which will enable
  	 * interrupts rather than keeping them masked (unclear how
  	 * much benefit this is over just replaying for all cases,
  	 * because we immediately disable again, so all we're really
  	 * doing is allowing hard interrupts to execute directly for
  	 * a very small time, rather than being masked and replayed).
  	 */
  	if (irqs_enabled) {
  		local_irq_enable();
  		local_irq_disable();
  	} else {
  		replay_soft_interrupts();
  	}
  
  	return false;
  }
  
  static notrace void booke_load_dbcr0(void)
  {
  #ifdef CONFIG_PPC_ADV_DEBUG_REGS
  	unsigned long dbcr0 = current->thread.debug.dbcr0;
  
  	if (likely(!(dbcr0 & DBCR0_IDM)))
  		return;
  
  	/*
  	 * Check to see if the dbcr0 register is set up to debug.
  	 * Use the internal debug mode bit to do this.
  	 */
  	mtmsr(mfmsr() & ~MSR_DE);
  	if (IS_ENABLED(CONFIG_PPC32)) {
  		isync();
  		global_dbcr0[smp_processor_id()] = mfspr(SPRN_DBCR0);
  	}
  	mtspr(SPRN_DBCR0, dbcr0);
  	mtspr(SPRN_DBSR, -1);
  #endif
  }
  
  /*
   * This should be called after a syscall returns, with r3 the return value
   * from the syscall. If this function returns non-zero, the system call
   * exit assembly should additionally load all GPR registers and CTR and XER
   * from the interrupt frame.
   *
   * The function graph tracer can not trace the return side of this function,
   * because RI=0 and soft mask state is "unreconciled", so it is marked notrace.
   */
  notrace unsigned long syscall_exit_prepare(unsigned long r3,
  					   struct pt_regs *regs,
  					   long scv)
  {
  	unsigned long ti_flags;
  	unsigned long ret = 0;
  	bool is_not_scv = !IS_ENABLED(CONFIG_PPC_BOOK3S_64) || !scv;
  
  	CT_WARN_ON(ct_state() == CONTEXT_USER);
  
  #ifdef CONFIG_PPC64
  	kuap_check_amr();
  #endif
  
  	regs->result = r3;
  
  	/* Check whether the syscall is issued inside a restartable sequence */
  	rseq_syscall(regs);
  
  	ti_flags = current_thread_info()->flags;
  
  	if (unlikely(r3 >= (unsigned long)-MAX_ERRNO) && is_not_scv) {
  		if (likely(!(ti_flags & (_TIF_NOERROR | _TIF_RESTOREALL)))) {
  			r3 = -r3;
  			regs->ccr |= 0x10000000; /* Set SO bit in CR */
  		}
  	}
  
  	if (unlikely(ti_flags & _TIF_PERSYSCALL_MASK)) {
  		if (ti_flags & _TIF_RESTOREALL)
  			ret = _TIF_RESTOREALL;
  		else
  			regs->gpr[3] = r3;
  		clear_bits(_TIF_PERSYSCALL_MASK, &current_thread_info()->flags);
  	} else {
  		regs->gpr[3] = r3;
  	}
  
  	if (unlikely(ti_flags & _TIF_SYSCALL_DOTRACE)) {
  		do_syscall_trace_leave(regs);
  		ret |= _TIF_RESTOREALL;
  	}
  
  	local_irq_disable();
  
  again:
  	ti_flags = READ_ONCE(current_thread_info()->flags);
  	while (unlikely(ti_flags & (_TIF_USER_WORK_MASK & ~_TIF_RESTORE_TM))) {
  		local_irq_enable();
  		if (ti_flags & _TIF_NEED_RESCHED) {
  			schedule();
  		} else {
  			/*
  			 * SIGPENDING must restore signal handler function
  			 * argument GPRs, and some non-volatiles (e.g., r1).
  			 * Restore all for now. This could be made lighter.
  			 */
  			if (ti_flags & _TIF_SIGPENDING)
  				ret |= _TIF_RESTOREALL;
  			do_notify_resume(regs, ti_flags);
  		}
  		local_irq_disable();
  		ti_flags = READ_ONCE(current_thread_info()->flags);
  	}
  
  	if (IS_ENABLED(CONFIG_PPC_BOOK3S) && IS_ENABLED(CONFIG_PPC_FPU)) {
  		if (IS_ENABLED(CONFIG_PPC_TRANSACTIONAL_MEM) &&
  				unlikely((ti_flags & _TIF_RESTORE_TM))) {
  			restore_tm_state(regs);
  		} else {
  			unsigned long mathflags = MSR_FP;
  
  			if (cpu_has_feature(CPU_FTR_VSX))
  				mathflags |= MSR_VEC | MSR_VSX;
  			else if (cpu_has_feature(CPU_FTR_ALTIVEC))
  				mathflags |= MSR_VEC;
  
  			/*
  			 * If userspace MSR has all available FP bits set,
  			 * then they are live and no need to restore. If not,
  			 * it means the regs were given up and restore_math
  			 * may decide to restore them (to avoid taking an FP
  			 * fault).
  			 */
  			if ((regs->msr & mathflags) != mathflags)
  				restore_math(regs);
  		}
  	}
  
  	user_enter_irqoff();
  
  	/* scv need not set RI=0 because SRRs are not used */
  	if (unlikely(!__prep_irq_for_enabled_exit(is_not_scv))) {
  		user_exit_irqoff();
  		local_irq_enable();
  		local_irq_disable();
  		goto again;
  	}
  
  #ifdef CONFIG_PPC_TRANSACTIONAL_MEM
  	local_paca->tm_scratch = regs->msr;
  #endif
  
  	booke_load_dbcr0();
  
  	account_cpu_user_exit();
  
  #ifdef CONFIG_PPC_BOOK3S_64 /* BOOK3E and ppc32 not using this */
  	/*
  	 * We do this at the end so that we do context switch with KERNEL AMR
  	 */
  	kuap_user_restore(regs);
  #endif
  	return ret;
  }
  
  #ifndef CONFIG_PPC_BOOK3E_64 /* BOOK3E not yet using this */
  notrace unsigned long interrupt_exit_user_prepare(struct pt_regs *regs, unsigned long msr)
  {
  	unsigned long ti_flags;
  	unsigned long flags;
  	unsigned long ret = 0;
  
  	if (!IS_ENABLED(CONFIG_BOOKE) && !IS_ENABLED(CONFIG_40x))
  		BUG_ON(!(regs->msr & MSR_RI));
  	BUG_ON(!(regs->msr & MSR_PR));
  	BUG_ON(!FULL_REGS(regs));
  	BUG_ON(arch_irq_disabled_regs(regs));
  	CT_WARN_ON(ct_state() == CONTEXT_USER);
  
  	/*
  	 * We don't need to restore AMR on the way back to userspace for KUAP.
  	 * AMR can only have been unlocked if we interrupted the kernel.
  	 */
  #ifdef CONFIG_PPC64
  	kuap_check_amr();
  #endif
  
  	local_irq_save(flags);
  
  again:
  	ti_flags = READ_ONCE(current_thread_info()->flags);
  	while (unlikely(ti_flags & (_TIF_USER_WORK_MASK & ~_TIF_RESTORE_TM))) {
  		local_irq_enable(); /* returning to user: may enable */
  		if (ti_flags & _TIF_NEED_RESCHED) {
  			schedule();
  		} else {
  			if (ti_flags & _TIF_SIGPENDING)
  				ret |= _TIF_RESTOREALL;
  			do_notify_resume(regs, ti_flags);
  		}
  		local_irq_disable();
  		ti_flags = READ_ONCE(current_thread_info()->flags);
  	}
  
  	if (IS_ENABLED(CONFIG_PPC_BOOK3S) && IS_ENABLED(CONFIG_PPC_FPU)) {
  		if (IS_ENABLED(CONFIG_PPC_TRANSACTIONAL_MEM) &&
  				unlikely((ti_flags & _TIF_RESTORE_TM))) {
  			restore_tm_state(regs);
  		} else {
  			unsigned long mathflags = MSR_FP;
  
  			if (cpu_has_feature(CPU_FTR_VSX))
  				mathflags |= MSR_VEC | MSR_VSX;
  			else if (cpu_has_feature(CPU_FTR_ALTIVEC))
  				mathflags |= MSR_VEC;
  
  			/* See above restore_math comment */
  			if ((regs->msr & mathflags) != mathflags)
  				restore_math(regs);
  		}
  	}
  
  	user_enter_irqoff();
  
  	if (unlikely(!__prep_irq_for_enabled_exit(true))) {
  		user_exit_irqoff();
  		local_irq_enable();
  		local_irq_disable();
  		goto again;
  	}
  
  	booke_load_dbcr0();
  
  #ifdef CONFIG_PPC_TRANSACTIONAL_MEM
  	local_paca->tm_scratch = regs->msr;
  #endif
  
  	account_cpu_user_exit();
  
  	/*
  	 * We do this at the end so that we do context switch with KERNEL AMR
  	 */
  #ifdef CONFIG_PPC64
  	kuap_user_restore(regs);
  #endif
  	return ret;
  }
  
  void preempt_schedule_irq(void);
  
  notrace unsigned long interrupt_exit_kernel_prepare(struct pt_regs *regs, unsigned long msr)
  {
  	unsigned long flags;
  	unsigned long ret = 0;
  #ifdef CONFIG_PPC64
  	unsigned long amr;
  #endif
  
  	if (!IS_ENABLED(CONFIG_BOOKE) && !IS_ENABLED(CONFIG_40x) &&
  	    unlikely(!(regs->msr & MSR_RI)))
  		unrecoverable_exception(regs);
  	BUG_ON(regs->msr & MSR_PR);
  	BUG_ON(!FULL_REGS(regs));
  	/*
  	 * CT_WARN_ON comes here via program_check_exception,
  	 * so avoid recursion.
  	 */
  	if (TRAP(regs) != 0x700)
  		CT_WARN_ON(ct_state() == CONTEXT_USER);
  
  #ifdef CONFIG_PPC64
  	amr = kuap_get_and_check_amr();
  #endif
  
  	if (unlikely(current_thread_info()->flags & _TIF_EMULATE_STACK_STORE)) {
  		clear_bits(_TIF_EMULATE_STACK_STORE, &current_thread_info()->flags);
  		ret = 1;
  	}
  
  	local_irq_save(flags);
  
  	if (!arch_irq_disabled_regs(regs)) {
  		/* Returning to a kernel context with local irqs enabled. */
  		WARN_ON_ONCE(!(regs->msr & MSR_EE));
  again:
  		if (IS_ENABLED(CONFIG_PREEMPT)) {
  			/* Return to preemptible kernel context */
  			if (unlikely(current_thread_info()->flags & _TIF_NEED_RESCHED)) {
  				if (preempt_count() == 0)
  					preempt_schedule_irq();
  			}
  		}
  
  		if (unlikely(!prep_irq_for_enabled_exit(true, !irqs_disabled_flags(flags))))
  			goto again;
  	} else {
  		/* Returning to a kernel context with local irqs disabled. */
  		__hard_EE_RI_disable();
  #ifdef CONFIG_PPC64
  		if (regs->msr & MSR_EE)
  			local_paca->irq_happened &= ~PACA_IRQ_HARD_DIS;
  #endif
  	}
  
  
  #ifdef CONFIG_PPC_TRANSACTIONAL_MEM
  	local_paca->tm_scratch = regs->msr;
  #endif
  
  	/*
  	 * Don't want to mfspr(SPRN_AMR) here, because this comes after mtmsr,
  	 * which would cause Read-After-Write stalls. Hence, we take the AMR
  	 * value from the check above.
  	 */
  #ifdef CONFIG_PPC64
  	kuap_kernel_restore(regs, amr);
  #endif
  
  	return ret;
  }
  #endif
  