text: // SPDX-License-Identifier: GPL-2.0
  /*
   * AMD Encrypted Register State Support
   *
   * Author: Joerg Roedel <jroedel@suse.de>
   *
   * This file is not compiled stand-alone. It contains code shared
   * between the pre-decompression boot code and the running Linux kernel
   * and is included directly into both code-bases.
   */
  
  #ifndef __BOOT_COMPRESSED
  #define error(v)	pr_err(v)
  #define has_cpuflag(f)	boot_cpu_has(f)
  #endif
  
  static bool __init sev_es_check_cpu_features(void)
  {
  	if (!has_cpuflag(X86_FEATURE_RDRAND)) {
  		error("RDRAND instruction not supported - no trusted source of randomness available\n");
  		return false;
  	}
  
  	return true;
  }
  
  static void sev_es_terminate(unsigned int reason)
  {
  	u64 val = GHCB_SEV_TERMINATE;
  
  	/*
  	 * Tell the hypervisor what went wrong - only reason-set 0 is
  	 * currently supported.
  	 */
  	val |= GHCB_SEV_TERMINATE_REASON(0, reason);
  
  	/* Request Guest Termination from Hypvervisor */
  	sev_es_wr_ghcb_msr(val);
  	VMGEXIT();
  
  	while (true)
  		asm volatile("hlt\n" : : : "memory");
  }
  
  static bool sev_es_negotiate_protocol(void)
  {
  	u64 val;
  
  	/* Do the GHCB protocol version negotiation */
  	sev_es_wr_ghcb_msr(GHCB_SEV_INFO_REQ);
  	VMGEXIT();
  	val = sev_es_rd_ghcb_msr();
  
  	if (GHCB_INFO(val) != GHCB_SEV_INFO)
  		return false;
  
  	if (GHCB_PROTO_MAX(val) < GHCB_PROTO_OUR ||
  	    GHCB_PROTO_MIN(val) > GHCB_PROTO_OUR)
  		return false;
  
  	return true;
  }
  
  static __always_inline void vc_ghcb_invalidate(struct ghcb *ghcb)
  {
  	memset(ghcb->save.valid_bitmap, 0, sizeof(ghcb->save.valid_bitmap));
  }
  
  static bool vc_decoding_needed(unsigned long exit_code)
  {
  	/* Exceptions don't require to decode the instruction */
  	return !(exit_code >= SVM_EXIT_EXCP_BASE &&
  		 exit_code <= SVM_EXIT_LAST_EXCP);
  }
  
  static enum es_result vc_init_em_ctxt(struct es_em_ctxt *ctxt,
  				      struct pt_regs *regs,
  				      unsigned long exit_code)
  {
  	enum es_result ret = ES_OK;
  
  	memset(ctxt, 0, sizeof(*ctxt));
  	ctxt->regs = regs;
  
  	if (vc_decoding_needed(exit_code))
  		ret = vc_decode_insn(ctxt);
  
  	return ret;
  }
  
  static void vc_finish_insn(struct es_em_ctxt *ctxt)
  {
  	ctxt->regs->ip += ctxt->insn.length;
  }
  
  static enum es_result sev_es_ghcb_hv_call(struct ghcb *ghcb,
  					  struct es_em_ctxt *ctxt,
  					  u64 exit_code, u64 exit_info_1,
  					  u64 exit_info_2)
  {
  	enum es_result ret;
  
  	/* Fill in protocol and format specifiers */
  	ghcb->protocol_version = GHCB_PROTOCOL_MAX;
  	ghcb->ghcb_usage       = GHCB_DEFAULT_USAGE;
  
  	ghcb_set_sw_exit_code(ghcb, exit_code);
  	ghcb_set_sw_exit_info_1(ghcb, exit_info_1);
  	ghcb_set_sw_exit_info_2(ghcb, exit_info_2);
  
  	sev_es_wr_ghcb_msr(__pa(ghcb));
  	VMGEXIT();
  
  	if ((ghcb->save.sw_exit_info_1 & 0xffffffff) == 1) {
  		u64 info = ghcb->save.sw_exit_info_2;
  		unsigned long v;
  
  		info = ghcb->save.sw_exit_info_2;
  		v = info & SVM_EVTINJ_VEC_MASK;
  
  		/* Check if exception information from hypervisor is sane. */
  		if ((info & SVM_EVTINJ_VALID) &&
  		    ((v == X86_TRAP_GP) || (v == X86_TRAP_UD)) &&
  		    ((info & SVM_EVTINJ_TYPE_MASK) == SVM_EVTINJ_TYPE_EXEPT)) {
  			ctxt->fi.vector = v;
  			if (info & SVM_EVTINJ_VALID_ERR)
  				ctxt->fi.error_code = info >> 32;
  			ret = ES_EXCEPTION;
  		} else {
  			ret = ES_VMM_ERROR;
  		}
  	} else {
  		ret = ES_OK;
  	}
  
  	return ret;
  }
  
  /*
   * Boot VC Handler - This is the first VC handler during boot, there is no GHCB
   * page yet, so it only supports the MSR based communication with the
   * hypervisor and only the CPUID exit-code.
   */
  void __init do_vc_no_ghcb(struct pt_regs *regs, unsigned long exit_code)
  {
  	unsigned int fn = lower_bits(regs->ax, 32);
  	unsigned long val;
  
  	/* Only CPUID is supported via MSR protocol */
  	if (exit_code != SVM_EXIT_CPUID)
  		goto fail;
  
  	sev_es_wr_ghcb_msr(GHCB_CPUID_REQ(fn, GHCB_CPUID_REQ_EAX));
  	VMGEXIT();
  	val = sev_es_rd_ghcb_msr();
  	if (GHCB_SEV_GHCB_RESP_CODE(val) != GHCB_SEV_CPUID_RESP)
  		goto fail;
  	regs->ax = val >> 32;
  
  	sev_es_wr_ghcb_msr(GHCB_CPUID_REQ(fn, GHCB_CPUID_REQ_EBX));
  	VMGEXIT();
  	val = sev_es_rd_ghcb_msr();
  	if (GHCB_SEV_GHCB_RESP_CODE(val) != GHCB_SEV_CPUID_RESP)
  		goto fail;
  	regs->bx = val >> 32;
  
  	sev_es_wr_ghcb_msr(GHCB_CPUID_REQ(fn, GHCB_CPUID_REQ_ECX));
  	VMGEXIT();
  	val = sev_es_rd_ghcb_msr();
  	if (GHCB_SEV_GHCB_RESP_CODE(val) != GHCB_SEV_CPUID_RESP)
  		goto fail;
  	regs->cx = val >> 32;
  
  	sev_es_wr_ghcb_msr(GHCB_CPUID_REQ(fn, GHCB_CPUID_REQ_EDX));
  	VMGEXIT();
  	val = sev_es_rd_ghcb_msr();
  	if (GHCB_SEV_GHCB_RESP_CODE(val) != GHCB_SEV_CPUID_RESP)
  		goto fail;
  	regs->dx = val >> 32;
  
  	/*
  	 * This is a VC handler and the #VC is only raised when SEV-ES is
  	 * active, which means SEV must be active too. Do sanity checks on the
  	 * CPUID results to make sure the hypervisor does not trick the kernel
  	 * into the no-sev path. This could map sensitive data unencrypted and
  	 * make it accessible to the hypervisor.
  	 *
  	 * In particular, check for:
  	 *	- Hypervisor CPUID bit
  	 *	- Availability of CPUID leaf 0x8000001f
  	 *	- SEV CPUID bit.
  	 *
  	 * The hypervisor might still report the wrong C-bit position, but this
  	 * can't be checked here.
  	 */
  
  	if ((fn == 1 && !(regs->cx & BIT(31))))
  		/* Hypervisor bit */
  		goto fail;
  	else if (fn == 0x80000000 && (regs->ax < 0x8000001f))
  		/* SEV leaf check */
  		goto fail;
  	else if ((fn == 0x8000001f && !(regs->ax & BIT(1))))
  		/* SEV bit */
  		goto fail;
  
  	/* Skip over the CPUID two-byte opcode */
  	regs->ip += 2;
  
  	return;
  
  fail:
  	sev_es_wr_ghcb_msr(GHCB_SEV_TERMINATE);
  	VMGEXIT();
  
  	/* Shouldn't get here - if we do halt the machine */
  	while (true)
  		asm volatile("hlt\n");
  }
  
  static enum es_result vc_insn_string_read(struct es_em_ctxt *ctxt,
  					  void *src, char *buf,
  					  unsigned int data_size,
  					  unsigned int count,
  					  bool backwards)
  {
  	int i, b = backwards ? -1 : 1;
  	enum es_result ret = ES_OK;
  
  	for (i = 0; i < count; i++) {
  		void *s = src + (i * data_size * b);
  		char *d = buf + (i * data_size);
  
  		ret = vc_read_mem(ctxt, s, d, data_size);
  		if (ret != ES_OK)
  			break;
  	}
  
  	return ret;
  }
  
  static enum es_result vc_insn_string_write(struct es_em_ctxt *ctxt,
  					   void *dst, char *buf,
  					   unsigned int data_size,
  					   unsigned int count,
  					   bool backwards)
  {
  	int i, s = backwards ? -1 : 1;
  	enum es_result ret = ES_OK;
  
  	for (i = 0; i < count; i++) {
  		void *d = dst + (i * data_size * s);
  		char *b = buf + (i * data_size);
  
  		ret = vc_write_mem(ctxt, d, b, data_size);
  		if (ret != ES_OK)
  			break;
  	}
  
  	return ret;
  }
  
  #define IOIO_TYPE_STR  BIT(2)
  #define IOIO_TYPE_IN   1
  #define IOIO_TYPE_INS  (IOIO_TYPE_IN | IOIO_TYPE_STR)
  #define IOIO_TYPE_OUT  0
  #define IOIO_TYPE_OUTS (IOIO_TYPE_OUT | IOIO_TYPE_STR)
  
  #define IOIO_REP       BIT(3)
  
  #define IOIO_ADDR_64   BIT(9)
  #define IOIO_ADDR_32   BIT(8)
  #define IOIO_ADDR_16   BIT(7)
  
  #define IOIO_DATA_32   BIT(6)
  #define IOIO_DATA_16   BIT(5)
  #define IOIO_DATA_8    BIT(4)
  
  #define IOIO_SEG_ES    (0 << 10)
  #define IOIO_SEG_DS    (3 << 10)
  
  static enum es_result vc_ioio_exitinfo(struct es_em_ctxt *ctxt, u64 *exitinfo)
  {
  	struct insn *insn = &ctxt->insn;
  	*exitinfo = 0;
  
  	switch (insn->opcode.bytes[0]) {
  	/* INS opcodes */
  	case 0x6c:
  	case 0x6d:
  		*exitinfo |= IOIO_TYPE_INS;
  		*exitinfo |= IOIO_SEG_ES;
  		*exitinfo |= (ctxt->regs->dx & 0xffff) << 16;
  		break;
  
  	/* OUTS opcodes */
  	case 0x6e:
  	case 0x6f:
  		*exitinfo |= IOIO_TYPE_OUTS;
  		*exitinfo |= IOIO_SEG_DS;
  		*exitinfo |= (ctxt->regs->dx & 0xffff) << 16;
  		break;
  
  	/* IN immediate opcodes */
  	case 0xe4:
  	case 0xe5:
  		*exitinfo |= IOIO_TYPE_IN;
  		*exitinfo |= (u8)insn->immediate.value << 16;
  		break;
  
  	/* OUT immediate opcodes */
  	case 0xe6:
  	case 0xe7:
  		*exitinfo |= IOIO_TYPE_OUT;
  		*exitinfo |= (u8)insn->immediate.value << 16;
  		break;
  
  	/* IN register opcodes */
  	case 0xec:
  	case 0xed:
  		*exitinfo |= IOIO_TYPE_IN;
  		*exitinfo |= (ctxt->regs->dx & 0xffff) << 16;
  		break;
  
  	/* OUT register opcodes */
  	case 0xee:
  	case 0xef:
  		*exitinfo |= IOIO_TYPE_OUT;
  		*exitinfo |= (ctxt->regs->dx & 0xffff) << 16;
  		break;
  
  	default:
  		return ES_DECODE_FAILED;
  	}
  
  	switch (insn->opcode.bytes[0]) {
  	case 0x6c:
  	case 0x6e:
  	case 0xe4:
  	case 0xe6:
  	case 0xec:
  	case 0xee:
  		/* Single byte opcodes */
  		*exitinfo |= IOIO_DATA_8;
  		break;
  	default:
  		/* Length determined by instruction parsing */
  		*exitinfo |= (insn->opnd_bytes == 2) ? IOIO_DATA_16
  						     : IOIO_DATA_32;
  	}
  	switch (insn->addr_bytes) {
  	case 2:
  		*exitinfo |= IOIO_ADDR_16;
  		break;
  	case 4:
  		*exitinfo |= IOIO_ADDR_32;
  		break;
  	case 8:
  		*exitinfo |= IOIO_ADDR_64;
  		break;
  	}
  
  	if (insn_has_rep_prefix(insn))
  		*exitinfo |= IOIO_REP;
  
  	return ES_OK;
  }
  
  static enum es_result vc_handle_ioio(struct ghcb *ghcb, struct es_em_ctxt *ctxt)
  {
  	struct pt_regs *regs = ctxt->regs;
  	u64 exit_info_1, exit_info_2;
  	enum es_result ret;
  
  	ret = vc_ioio_exitinfo(ctxt, &exit_info_1);
  	if (ret != ES_OK)
  		return ret;
  
  	if (exit_info_1 & IOIO_TYPE_STR) {
  
  		/* (REP) INS/OUTS */
  
  		bool df = ((regs->flags & X86_EFLAGS_DF) == X86_EFLAGS_DF);
  		unsigned int io_bytes, exit_bytes;
  		unsigned int ghcb_count, op_count;
  		unsigned long es_base;
  		u64 sw_scratch;
  
  		/*
  		 * For the string variants with rep prefix the amount of in/out
  		 * operations per #VC exception is limited so that the kernel
  		 * has a chance to take interrupts and re-schedule while the
  		 * instruction is emulated.
  		 */
  		io_bytes   = (exit_info_1 >> 4) & 0x7;
  		ghcb_count = sizeof(ghcb->shared_buffer) / io_bytes;
  
  		op_count    = (exit_info_1 & IOIO_REP) ? regs->cx : 1;
  		exit_info_2 = min(op_count, ghcb_count);
  		exit_bytes  = exit_info_2 * io_bytes;
  
  		es_base = insn_get_seg_base(ctxt->regs, INAT_SEG_REG_ES);
  
  		/* Read bytes of OUTS into the shared buffer */
  		if (!(exit_info_1 & IOIO_TYPE_IN)) {
  			ret = vc_insn_string_read(ctxt,
  					       (void *)(es_base + regs->si),
  					       ghcb->shared_buffer, io_bytes,
  					       exit_info_2, df);
  			if (ret)
  				return ret;
  		}
  
  		/*
  		 * Issue an VMGEXIT to the HV to consume the bytes from the
  		 * shared buffer or to have it write them into the shared buffer
  		 * depending on the instruction: OUTS or INS.
  		 */
  		sw_scratch = __pa(ghcb) + offsetof(struct ghcb, shared_buffer);
  		ghcb_set_sw_scratch(ghcb, sw_scratch);
  		ret = sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_IOIO,
  					  exit_info_1, exit_info_2);
  		if (ret != ES_OK)
  			return ret;
  
  		/* Read bytes from shared buffer into the guest's destination. */
  		if (exit_info_1 & IOIO_TYPE_IN) {
  			ret = vc_insn_string_write(ctxt,
  						   (void *)(es_base + regs->di),
  						   ghcb->shared_buffer, io_bytes,
  						   exit_info_2, df);
  			if (ret)
  				return ret;
  
  			if (df)
  				regs->di -= exit_bytes;
  			else
  				regs->di += exit_bytes;
  		} else {
  			if (df)
  				regs->si -= exit_bytes;
  			else
  				regs->si += exit_bytes;
  		}
  
  		if (exit_info_1 & IOIO_REP)
  			regs->cx -= exit_info_2;
  
  		ret = regs->cx ? ES_RETRY : ES_OK;
  
  	} else {
  
  		/* IN/OUT into/from rAX */
  
  		int bits = (exit_info_1 & 0x70) >> 1;
  		u64 rax = 0;
  
  		if (!(exit_info_1 & IOIO_TYPE_IN))
  			rax = lower_bits(regs->ax, bits);
  
  		ghcb_set_rax(ghcb, rax);
  
  		ret = sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_IOIO, exit_info_1, 0);
  		if (ret != ES_OK)
  			return ret;
  
  		if (exit_info_1 & IOIO_TYPE_IN) {
  			if (!ghcb_rax_is_valid(ghcb))
  				return ES_VMM_ERROR;
  			regs->ax = lower_bits(ghcb->save.rax, bits);
  		}
  	}
  
  	return ret;
  }
  
  static enum es_result vc_handle_cpuid(struct ghcb *ghcb,
  				      struct es_em_ctxt *ctxt)
  {
  	struct pt_regs *regs = ctxt->regs;
  	u32 cr4 = native_read_cr4();
  	enum es_result ret;
  
  	ghcb_set_rax(ghcb, regs->ax);
  	ghcb_set_rcx(ghcb, regs->cx);
  
  	if (cr4 & X86_CR4_OSXSAVE)
  		/* Safe to read xcr0 */
  		ghcb_set_xcr0(ghcb, xgetbv(XCR_XFEATURE_ENABLED_MASK));
  	else
  		/* xgetbv will cause #GP - use reset value for xcr0 */
  		ghcb_set_xcr0(ghcb, 1);
  
  	ret = sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_CPUID, 0, 0);
  	if (ret != ES_OK)
  		return ret;
  
  	if (!(ghcb_rax_is_valid(ghcb) &&
  	      ghcb_rbx_is_valid(ghcb) &&
  	      ghcb_rcx_is_valid(ghcb) &&
  	      ghcb_rdx_is_valid(ghcb)))
  		return ES_VMM_ERROR;
  
  	regs->ax = ghcb->save.rax;
  	regs->bx = ghcb->save.rbx;
  	regs->cx = ghcb->save.rcx;
  	regs->dx = ghcb->save.rdx;
  
  	return ES_OK;
  }
  
  static enum es_result vc_handle_rdtsc(struct ghcb *ghcb,
  				      struct es_em_ctxt *ctxt,
  				      unsigned long exit_code)
  {
  	bool rdtscp = (exit_code == SVM_EXIT_RDTSCP);
  	enum es_result ret;
  
  	ret = sev_es_ghcb_hv_call(ghcb, ctxt, exit_code, 0, 0);
  	if (ret != ES_OK)
  		return ret;
  
  	if (!(ghcb_rax_is_valid(ghcb) && ghcb_rdx_is_valid(ghcb) &&
  	     (!rdtscp || ghcb_rcx_is_valid(ghcb))))
  		return ES_VMM_ERROR;
  
  	ctxt->regs->ax = ghcb->save.rax;
  	ctxt->regs->dx = ghcb->save.rdx;
  	if (rdtscp)
  		ctxt->regs->cx = ghcb->save.rcx;
  
  	return ES_OK;
  }
  