text: /*-
   * Copyright (c) 2011 NetApp, Inc.
   * Copyright (c) 2015 xhyve developers
   * All rights reserved.
   *
   * Redistribution and use in source and binary forms, with or without
   * modification, are permitted provided that the following conditions
   * are met:
   * 1. Redistributions of source code must retain the above copyright
   *    notice, this list of conditions and the following disclaimer.
   * 2. Redistributions in binary form must reproduce the above copyright
   *    notice, this list of conditions and the following disclaimer in the
   *    documentation and/or other materials provided with the distribution.
   *
   * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND
   * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
   * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
   * ARE DISCLAIMED.  IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE
   * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
   * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
   * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
   * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
   * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
   * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
   * SUCH DAMAGE.
   *
   * $FreeBSD$
   */
  
  #include <stdint.h>
  #include <stdbool.h>
  #include <stdlib.h>
  #include <stdio.h>
  #include <errno.h>
  #include <assert.h>
  #include <Hypervisor/hv.h>
  #include <Hypervisor/hv_vmx.h>
  #include <xhyve/support/misc.h>
  #include <xhyve/support/atomic.h>
  #include <xhyve/support/psl.h>
  #include <xhyve/support/specialreg.h>
  #include <xhyve/vmm/vmm.h>
  #include <xhyve/vmm/vmm_instruction_emul.h>
  #include <xhyve/vmm/vmm_lapic.h>
  #include <xhyve/vmm/vmm_host.h>
  #include <xhyve/vmm/vmm_ktr.h>
  #include <xhyve/vmm/vmm_stat.h>
  #include <xhyve/vmm/io/vatpic.h>
  #include <xhyve/vmm/io/vlapic.h>
  #include <xhyve/vmm/io/vlapic_priv.h>
  #include <xhyve/vmm/intel/vmx.h>
  #include <xhyve/vmm/intel/vmx_msr.h>
  #include <xhyve/vmm/x86.h>
  #include <xhyve/vmm/intel/vmx_controls.h>
  
  #define PROCBASED_CTLS_WINDOW_SETTING \
  	(PROCBASED_INT_WINDOW_EXITING | \
  	 PROCBASED_NMI_WINDOW_EXITING)
  #define PROCBASED_CTLS_ONE_SETTING \
  	(PROCBASED_SECONDARY_CONTROLS | \
  	 PROCBASED_MWAIT_EXITING | \
  	 PROCBASED_MONITOR_EXITING | \
  	 PROCBASED_IO_EXITING | \
  	 PROCBASED_MSR_BITMAPS | \
  	 PROCBASED_CTLS_WINDOW_SETTING | \
  	 PROCBASED_CR8_LOAD_EXITING | \
  	 PROCBASED_CR8_STORE_EXITING | \
  	 PROCBASED_HLT_EXITING | \
  	 PROCBASED_TSC_OFFSET)
  #define PROCBASED_CTLS_ZERO_SETTING \
  	(PROCBASED_CR3_LOAD_EXITING | \
  	 PROCBASED_CR3_STORE_EXITING | \
  	 PROCBASED_IO_BITMAPS | \
  	 PROCBASED_RDTSC_EXITING | \
  	 PROCBASED_USE_TPR_SHADOW | \
  	 PROCBASED_MOV_DR_EXITING | \
  	 PROCBASED_MTF | \
  	 PROCBASED_INVLPG_EXITING | \
  	 PROCBASED_PAUSE_EXITING)
  #define PROCBASED_CTLS2_ONE_SETTING \
  	(PROCBASED2_ENABLE_EPT | \
  	 PROCBASED2_UNRESTRICTED_GUEST | \
  	 PROCBASED2_ENABLE_VPID | \
  	 PROCBASED2_ENABLE_RDTSCP)
  #define PROCBASED_CTLS2_ZERO_SETTING \
  	(PROCBASED2_VIRTUALIZE_APIC_ACCESSES | \
  	 PROCBASED2_DESC_TABLE_EXITING | \
  	 PROCBASED2_WBINVD_EXITING | \
  	 PROCBASED2_PAUSE_LOOP_EXITING /* FIXME */ | \
  	 PROCBASED2_RDRAND_EXITING | \
  	 PROCBASED2_ENABLE_INVPCID /* FIXME */ | \
  	 PROCBASED2_RDSEED_EXITING | \
  	 PROCBASED2_ENABLE_XSAVES_XRSTORS | \
  	 PROCBASED2_VMCS_SHADOW )
  #define PINBASED_CTLS_ONE_SETTING \
  	(PINBASED_EXTINT_EXITING | \
  	 PINBASED_NMI_EXITING | \
  	 PINBASED_VIRTUAL_NMI)
  #define PINBASED_CTLS_ZERO_SETTING \
  	(PINBASED_PREMPTION_TIMER)
  #define VM_ENTRY_CTLS_ONE_SETTING \
  	(VM_ENTRY_LOAD_EFER)
  #define VM_ENTRY_CTLS_ZERO_SETTING \
  	(VM_ENTRY_INTO_SMM | \
  	 VM_ENTRY_DEACTIVATE_DUAL_MONITOR | \
  	 VM_ENTRY_LOAD_PERF_GLOBAL_CTRL | \
  	 VM_ENTRY_GUEST_LMA)
  #define  VM_EXIT_CTLS_ONE_SETTING \
  	(VM_EXIT_HOST_LMA | \
  	 VM_EXIT_LOAD_EFER)
  #define VM_EXIT_CTLS_ZERO_SETTING \
  	(VM_EXIT_SAVE_PREEMPTION_TIMER | \
  	 VM_EXIT_SAVE_EFER | \
  	 VM_EXIT_LOAD_PERF_GLOBAL_CTRL)
  #define NMI_BLOCKING \
  	(VMCS_INTERRUPTIBILITY_NMI_BLOCKING | \
  	 VMCS_INTERRUPTIBILITY_MOVSS_BLOCKING)
  #define HWINTR_BLOCKING \
  	(VMCS_INTERRUPTIBILITY_STI_BLOCKING | \
  	 VMCS_INTERRUPTIBILITY_MOVSS_BLOCKING)
  
  #define	HANDLED		1
  #define	UNHANDLED	0
  
  static uint64_t cr0_ones_mask, cr0_zeros_mask;
  static uint64_t cr4_ones_mask, cr4_zeros_mask;
  
  /*
   * Optional capabilities
   */
  
  static int cap_halt_exit;
  static int cap_pause_exit;
  // static int cap_unrestricted_guest;
  static int cap_monitor_trap;
  // static int cap_invpcid;
  // static int pirvec = -1;
  // static struct unrhdr *vpid_unr;
  // static u_int vpid_alloc_failed;
  
  /*
   * Use the last page below 4GB as the APIC access address. This address is
   * occupied by the boot firmware so it is guaranteed that it will not conflict
   * with a page in system memory.
   */
  // #define	APIC_ACCESS_ADDRESS	0xFFFFF000
  
  static int vmx_getdesc(void *arg, int vcpu, int reg, struct seg_desc *desc);
  static int vmx_getreg(void *arg, int vcpu, int reg, uint64_t *retval);
  
  static __inline uint64_t
  reg_read(int vcpuid, hv_x86_reg_t reg) {
  	uint64_t val;
  
  	hv_vcpu_read_register(((hv_vcpuid_t) vcpuid), reg, &val);
  	return val;
  }
  
  static __inline void
  reg_write(int vcpuid, hv_x86_reg_t reg, uint64_t val) {
  	hv_vcpu_write_register(((hv_vcpuid_t) vcpuid), reg, val);
  }
  
  static void hvdump(int vcpu) {
  	printf("VMCS_PIN_BASED_CTLS:           0x%016llx\n",
  		vmcs_read(vcpu, VMCS_PIN_BASED_CTLS));
  	printf("VMCS_PRI_PROC_BASED_CTLS:      0x%016llx\n",
  		vmcs_read(vcpu, VMCS_PRI_PROC_BASED_CTLS));
  	printf("VMCS_SEC_PROC_BASED_CTLS:      0x%016llx\n",
  		vmcs_read(vcpu, VMCS_SEC_PROC_BASED_CTLS));
  	printf("VMCS_ENTRY_CTLS:               0x%016llx\n",
  		vmcs_read(vcpu, VMCS_ENTRY_CTLS));
  	printf("VMCS_EXCEPTION_BITMAP:         0x%016llx\n",
  		vmcs_read(vcpu, VMCS_EXCEPTION_BITMAP));
  	printf("VMCS_CR0_MASK:                 0x%016llx\n",
  		vmcs_read(vcpu, VMCS_CR0_MASK));
  	printf("VMCS_CR0_SHADOW:               0x%016llx\n",
  		vmcs_read(vcpu, VMCS_CR0_SHADOW));
  	printf("VMCS_CR4_MASK:                 0x%016llx\n",
  		vmcs_read(vcpu, VMCS_CR4_MASK));
  	printf("VMCS_CR4_SHADOW:               0x%016llx\n",
  		vmcs_read(vcpu, VMCS_CR4_SHADOW));
  	printf("VMCS_GUEST_CS_SELECTOR:        0x%016llx\n",
  		vmcs_read(vcpu, VMCS_GUEST_CS_SELECTOR));
  	printf("VMCS_GUEST_CS_LIMIT:           0x%016llx\n",
  		vmcs_read(vcpu, VMCS_GUEST_CS_LIMIT));
  	printf("VMCS_GUEST_CS_ACCESS_RIGHTS:   0x%016llx\n",
  		vmcs_read(vcpu, VMCS_GUEST_CS_ACCESS_RIGHTS));
  	printf("VMCS_GUEST_CS_BASE:            0x%016llx\n",
  		vmcs_read(vcpu, VMCS_GUEST_CS_BASE));
  	printf("VMCS_GUEST_DS_SELECTOR:        0x%016llx\n",
  		vmcs_read(vcpu, VMCS_GUEST_DS_SELECTOR));
  	printf("VMCS_GUEST_DS_LIMIT:           0x%016llx\n",
  		vmcs_read(vcpu, VMCS_GUEST_DS_LIMIT));
  	printf("VMCS_GUEST_DS_ACCESS_RIGHTS:   0x%016llx\n",
  		vmcs_read(vcpu, VMCS_GUEST_DS_ACCESS_RIGHTS));
  	printf("VMCS_GUEST_DS_BASE:            0x%016llx\n",
  		vmcs_read(vcpu, VMCS_GUEST_DS_BASE));
  	printf("VMCS_GUEST_ES_SELECTOR:        0x%016llx\n",
  		vmcs_read(vcpu, VMCS_GUEST_ES_SELECTOR));
  	printf("VMCS_GUEST_ES_LIMIT:           0x%016llx\n",
  		vmcs_read(vcpu, VMCS_GUEST_ES_LIMIT));
  	printf("VMCS_GUEST_ES_ACCESS_RIGHTS:   0x%016llx\n",
  		vmcs_read(vcpu, VMCS_GUEST_ES_ACCESS_RIGHTS));
  	printf("VMCS_GUEST_ES_BASE:            0x%016llx\n",
  		vmcs_read(vcpu, VMCS_GUEST_ES_BASE));
  	printf("VMCS_GUEST_FS_SELECTOR:        0x%016llx\n",
  		vmcs_read(vcpu, VMCS_GUEST_FS_SELECTOR));
  	printf("VMCS_GUEST_FS_LIMIT:           0x%016llx\n",
  		vmcs_read(vcpu, VMCS_GUEST_FS_LIMIT));
  	printf("VMCS_GUEST_FS_ACCESS_RIGHTS:   0x%016llx\n",
  		vmcs_read(vcpu, VMCS_GUEST_FS_ACCESS_RIGHTS));
  	printf("VMCS_GUEST_FS_BASE:            0x%016llx\n",
  		vmcs_read(vcpu, VMCS_GUEST_FS_BASE));
  	printf("VMCS_GUEST_GS_SELECTOR:        0x%016llx\n",
  		vmcs_read(vcpu, VMCS_GUEST_GS_SELECTOR));
  	printf("VMCS_GUEST_GS_LIMIT:           0x%016llx\n",
  		vmcs_read(vcpu, VMCS_GUEST_GS_LIMIT));
  	printf("VMCS_GUEST_GS_ACCESS_RIGHTS:   0x%016llx\n",
  		vmcs_read(vcpu, VMCS_GUEST_GS_ACCESS_RIGHTS));
  	printf("VMCS_GUEST_GS_BASE:            0x%016llx\n",
  		vmcs_read(vcpu, VMCS_GUEST_GS_BASE));
  	printf("VMCS_GUEST_SS_SELECTOR:        0x%016llx\n",
  		vmcs_read(vcpu, VMCS_GUEST_SS_SELECTOR));
  	printf("VMCS_GUEST_SS_LIMIT:           0x%016llx\n",
  		vmcs_read(vcpu, VMCS_GUEST_SS_LIMIT));
  	printf("VMCS_GUEST_SS_ACCESS_RIGHTS:   0x%016llx\n",
  		vmcs_read(vcpu, VMCS_GUEST_SS_ACCESS_RIGHTS));
  	printf("VMCS_GUEST_SS_BASE:            0x%016llx\n",
  		vmcs_read(vcpu, VMCS_GUEST_SS_BASE));
  	printf("VMCS_GUEST_LDTR_SELECTOR:      0x%016llx\n",
  		vmcs_read(vcpu, VMCS_GUEST_LDTR_SELECTOR));
  	printf("VMCS_GUEST_LDTR_LIMIT:         0x%016llx\n",
  		vmcs_read(vcpu, VMCS_GUEST_LDTR_LIMIT));
  	printf("VMCS_GUEST_LDTR_ACCESS_RIGHTS: 0x%016llx\n",
  		vmcs_read(vcpu, VMCS_GUEST_LDTR_ACCESS_RIGHTS));
  	printf("VMCS_GUEST_LDTR_BASE:          0x%016llx\n",
  		vmcs_read(vcpu, VMCS_GUEST_LDTR_BASE));
  	printf("VMCS_GUEST_TR_SELECTOR:        0x%016llx\n",
  		vmcs_read(vcpu, VMCS_GUEST_TR_SELECTOR));
  	printf("VMCS_GUEST_TR_LIMIT:           0x%016llx\n",
  		vmcs_read(vcpu, VMCS_GUEST_TR_LIMIT));
  	printf("VMCS_GUEST_TR_ACCESS_RIGHTS:   0x%016llx\n",
  		vmcs_read(vcpu, VMCS_GUEST_TR_ACCESS_RIGHTS));
  	printf("VMCS_GUEST_TR_BASE:            0x%016llx\n",
  		vmcs_read(vcpu, VMCS_GUEST_TR_BASE));
  	printf("VMCS_GUEST_GDTR_LIMIT:         0x%016llx\n",
  		vmcs_read(vcpu, VMCS_GUEST_GDTR_LIMIT));
  	printf("VMCS_GUEST_GDTR_BASE:          0x%016llx\n",
  		vmcs_read(vcpu, VMCS_GUEST_GDTR_BASE));
  	printf("VMCS_GUEST_IDTR_LIMIT:         0x%016llx\n",
  		vmcs_read(vcpu, VMCS_GUEST_LDTR_LIMIT));
  	printf("VMCS_GUEST_IDTR_BASE:          0x%016llx\n",
  		vmcs_read(vcpu, VMCS_GUEST_LDTR_BASE));
  	printf("VMCS_GUEST_CR0:                0x%016llx\n",
  		vmcs_read(vcpu, VMCS_GUEST_CR0));
  	printf("VMCS_GUEST_CR3:                0x%016llx\n",
  		vmcs_read(vcpu, VMCS_GUEST_CR3));
  	printf("VMCS_GUEST_CR4:                0x%016llx\n",
  		vmcs_read(vcpu, VMCS_GUEST_CR4));
  	printf("VMCS_GUEST_IA32_EFER:          0x%016llx\n",
  		vmcs_read(vcpu, VMCS_GUEST_IA32_EFER));
  	printf("\n");
  	printf("rip: 0x%016llx rfl: 0x%016llx cr2: 0x%016llx\n",
  		reg_read(vcpu, HV_X86_RIP), reg_read(vcpu, HV_X86_RFLAGS),
  		reg_read(vcpu, HV_X86_CR2));
  	printf("rax: 0x%016llx rbx: 0x%016llx rcx: 0x%016llx rdx: 0x%016llx\n",
  		reg_read(vcpu, HV_X86_RAX), reg_read(vcpu, HV_X86_RBX),
  		reg_read(vcpu, HV_X86_RCX), reg_read(vcpu, HV_X86_RDX));
  	printf("rsi: 0x%016llx rdi: 0x%016llx rbp: 0x%016llx rsp: 0x%016llx\n",
  		reg_read(vcpu, HV_X86_RSI), reg_read(vcpu, HV_X86_RDI),
  		reg_read(vcpu, HV_X86_RBP), reg_read(vcpu, HV_X86_RSP));
  	printf("r8:  0x%016llx r9:  0x%016llx r10: 0x%016llx r11: 0x%016llx\n",
  		reg_read(vcpu, HV_X86_R8), reg_read(vcpu, HV_X86_R9),
  		reg_read(vcpu, HV_X86_R10), reg_read(vcpu, HV_X86_R11));
  	printf("r12: 0x%016llx r13: 0x%016llx r14: 0x%016llx r15: 0x%016llx\n",
  		reg_read(vcpu, HV_X86_R12), reg_read(vcpu, HV_X86_R12),
  		reg_read(vcpu, HV_X86_R14), reg_read(vcpu, HV_X86_R15));
  }
  
  #ifdef XHYVE_CONFIG_TRACE
  static const char *
  exit_reason_to_str(int reason)
  {
  	static char reasonbuf[32];
  
  	switch (reason) {
  	case EXIT_REASON_EXCEPTION:
  		return "exception";
  	case EXIT_REASON_EXT_INTR:
  		return "extint";
  	case EXIT_REASON_TRIPLE_FAULT:
  		return "triplefault";
  	case EXIT_REASON_INIT:
  		return "init";
  	case EXIT_REASON_SIPI:
  		return "sipi";
  	case EXIT_REASON_IO_SMI:
  		return "iosmi";
  	case EXIT_REASON_SMI:
  		return "smi";
  	case EXIT_REASON_INTR_WINDOW:
  		return "intrwindow";
  	case EXIT_REASON_NMI_WINDOW:
  		return "nmiwindow";
  	case EXIT_REASON_TASK_SWITCH:
  		return "taskswitch";
  	case EXIT_REASON_CPUID:
  		return "cpuid";
  	case EXIT_REASON_GETSEC:
  		return "getsec";
  	case EXIT_REASON_HLT:
  		return "hlt";
  	case EXIT_REASON_INVD:
  		return "invd";
  	case EXIT_REASON_INVLPG:
  		return "invlpg";
  	case EXIT_REASON_RDPMC:
  		return "rdpmc";
  	case EXIT_REASON_RDTSC:
  		return "rdtsc";
  	case EXIT_REASON_RSM:
  		return "rsm";
  	case EXIT_REASON_VMCALL:
  		return "vmcall";
  	case EXIT_REASON_VMCLEAR:
  		return "vmclear";
  	case EXIT_REASON_VMLAUNCH:
  		return "vmlaunch";
  	case EXIT_REASON_VMPTRLD:
  		return "vmptrld";
  	case EXIT_REASON_VMPTRST:
  		return "vmptrst";
  	case EXIT_REASON_VMREAD:
  		return "vmread";
  	case EXIT_REASON_VMRESUME:
  		return "vmresume";
  	case EXIT_REASON_VMWRITE:
  		return "vmwrite";
  	case EXIT_REASON_VMXOFF:
  		return "vmxoff";
  	case EXIT_REASON_VMXON:
  		return "vmxon";
  	case EXIT_REASON_CR_ACCESS:
  		return "craccess";
  	case EXIT_REASON_DR_ACCESS:
  		return "draccess";
  	case EXIT_REASON_INOUT:
  		return "inout";
  	case EXIT_REASON_RDMSR:
  		return "rdmsr";
  	case EXIT_REASON_WRMSR:
  		return "wrmsr";
  	case EXIT_REASON_INVAL_VMCS:
  		return "invalvmcs";
  	case EXIT_REASON_INVAL_MSR:
  		return "invalmsr";
  	case EXIT_REASON_MWAIT:
  		return "mwait";
  	case EXIT_REASON_MTF:
  		return "mtf";
  	case EXIT_REASON_MONITOR:
  		return "monitor";
  	case EXIT_REASON_PAUSE:
  		return "pause";
  	case EXIT_REASON_MCE_DURING_ENTRY:
  		return "mce-during-entry";
  	case EXIT_REASON_TPR:
  		return "tpr";
  	case EXIT_REASON_APIC_ACCESS:
  		return "apic-access";
  	case EXIT_REASON_GDTR_IDTR:
  		return "gdtridtr";
  	case EXIT_REASON_LDTR_TR:
  		return "ldtrtr";
  	case EXIT_REASON_EPT_FAULT:
  		return "eptfault";
  	case EXIT_REASON_EPT_MISCONFIG:
  		return "eptmisconfig";
  	case EXIT_REASON_INVEPT:
  		return "invept";
  	case EXIT_REASON_RDTSCP:
  		return "rdtscp";
  	case EXIT_REASON_VMX_PREEMPT:
  		return "vmxpreempt";
  	case EXIT_REASON_INVVPID:
  		return "invvpid";
  	case EXIT_REASON_WBINVD:
  		return "wbinvd";
  	case EXIT_REASON_XSETBV:
  		return "xsetbv";
  	case EXIT_REASON_APIC_WRITE:
  		return "apic-write";
  	default:
  		snprintf(reasonbuf, sizeof(reasonbuf), "%d", reason);
  		return (reasonbuf);
  	}
  }
  #endif	/* XHYVE_CONFIG_TRACE */
  
  // static int
  // vmx_allow_x2apic_msrs(struct vmx *vmx)
  // {
  // 	int i, error;
  
  // 	error = 0;
  
  // 	/*
  // 	 * Allow readonly access to the following x2APIC MSRs from the guest.
  // 	 */
  // 	error += guest_msr_ro(vmx, MSR_APIC_ID);
  // 	error += guest_msr_ro(vmx, MSR_APIC_VERSION);
  // 	error += guest_msr_ro(vmx, MSR_APIC_LDR);
  // 	error += guest_msr_ro(vmx, MSR_APIC_SVR);
  
  // 	for (i = 0; i < 8; i++)
  // 		error += guest_msr_ro(vmx, MSR_APIC_ISR0 + i);
  
  // 	for (i = 0; i < 8; i++)
  // 		error += guest_msr_ro(vmx, MSR_APIC_TMR0 + i);
  	
  // 	for (i = 0; i < 8; i++)
  // 		error += guest_msr_ro(vmx, MSR_APIC_IRR0 + i);
  
  // 	error += guest_msr_ro(vmx, MSR_APIC_ESR);
  // 	error += guest_msr_ro(vmx, MSR_APIC_LVT_TIMER);
  // 	error += guest_msr_ro(vmx, MSR_APIC_LVT_THERMAL);
  // 	error += guest_msr_ro(vmx, MSR_APIC_LVT_PCINT);
  // 	error += guest_msr_ro(vmx, MSR_APIC_LVT_LINT0);
  // 	error += guest_msr_ro(vmx, MSR_APIC_LVT_LINT1);
  // 	error += guest_msr_ro(vmx, MSR_APIC_LVT_ERROR);
  // 	error += guest_msr_ro(vmx, MSR_APIC_ICR_TIMER);
  // 	error += guest_msr_ro(vmx, MSR_APIC_DCR_TIMER);
  // 	error += guest_msr_ro(vmx, MSR_APIC_ICR);
  
  // 	/*
  // 	 * Allow TPR, EOI and SELF_IPI MSRs to be read and written by the guest.
  // 	 *
  // 	 * These registers get special treatment described in the section
  // 	 * "Virtualizing MSR-Based APIC Accesses".
  // 	 */
  // 	error += guest_msr_rw(vmx, MSR_APIC_TPR);
  // 	error += guest_msr_rw(vmx, MSR_APIC_EOI);
  // 	error += guest_msr_rw(vmx, MSR_APIC_SELF_IPI);
  
  // 	return (error);
  // }
  
  u_long
  vmx_fix_cr0(u_long cr0)
  {
  	return ((cr0 | cr0_ones_mask) & ~cr0_zeros_mask);
  }
  
  u_long
  vmx_fix_cr4(u_long cr4)
  {
  	return ((cr4 | cr4_ones_mask) & ~cr4_zeros_mask);
  }
  
  static int
  vmx_cleanup(void)
  {
  	return (0);
  }
  
  static int
  vmx_init(void)
  {
  	int error = hv_vm_create(HV_VM_DEFAULT);
  	switch (error) {
  		case HV_SUCCESS:
  			break;
  		case HV_ERROR:
  			/* Don't know if this can happen, report to us */
  			xhyve_abort("hv_vm_create HV_ERROR\n");
  		case HV_BUSY:
  			/* Should never happen, report to us (perhaps we need to retry) */
  			xhyve_abort("hv_vm_create HV_BUSY\n");
  		case HV_BAD_ARGUMENT:
  			/* Should never happen, report to Apple */
  			xhyve_abort("hv_vm_create HV_BAD_ARGUMENT\n");
  		case HV_NO_RESOURCES:
  			/* Don't know if this can happen, report to us */
  			xhyve_abort("hv_vm_create HV_NO_RESOURCES\n");
  		case HV_NO_DEVICE:
  			printf("vmx_init: processor not supported by "
  			       "Hypervisor.framework\n");
  			return (error);
  		case HV_UNSUPPORTED:
  			/* Don't know if this can happen, report to us */
  			xhyve_abort("hv_vm_create HV_UNSUPPORTED\n");
  		default:
  			/* Should never happen, report to Apple */
  			xhyve_abort("hv_vm_create unknown error %d\n", error);
  	}
  
  	/*
  	 * Check support for optional features by testing them
  	 * as individual bits
  	 */
  	cap_halt_exit = 1;
  	cap_monitor_trap = 1;
  	cap_pause_exit = 1;
  	// cap_unrestricted_guest = 1;
  	// cap_invpcid = 1;
  
  	/* FIXME */
    	cr0_ones_mask = cr4_ones_mask = 0;
    	cr0_zeros_mask = cr4_zeros_mask = 0;
  
    	cr0_ones_mask |= (CR0_NE | CR0_ET);
    	cr0_zeros_mask |= (CR0_NW | CR0_CD);
    	cr4_ones_mask = 0x2000;
  
  	vmx_msr_init();
  
  	return (0);
  }
  
  static int
  vmx_setup_cr_shadow(int vcpuid, int which, uint32_t initial)
  {
  	int error, mask_ident, shadow_ident;
  	uint64_t mask_value;
  
  	if (which != 0 && which != 4)
  		xhyve_abort("vmx_setup_cr_shadow: unknown cr%d", which);
  
  	if (which == 0) {
  		mask_ident = VMCS_CR0_MASK;
  		mask_value = (cr0_ones_mask | cr0_zeros_mask) | (CR0_PG | CR0_PE);
  		shadow_ident = VMCS_CR0_SHADOW;
  	} else {
  		mask_ident = VMCS_CR4_MASK;
  		mask_value = cr4_ones_mask | cr4_zeros_mask;
  		shadow_ident = VMCS_CR4_SHADOW;
  	}
  
  	error = vmcs_setreg(vcpuid, VMCS_IDENT(mask_ident), mask_value);
  	if (error)
  		return (error);
  
  	error = vmcs_setreg(vcpuid, VMCS_IDENT(shadow_ident), initial);
  	if (error)
  		return (error);
  
  	return (0);
  }
  #define	vmx_setup_cr0_shadow(vcpuid,init) vmx_setup_cr_shadow(vcpuid, 0, (init))
  #define	vmx_setup_cr4_shadow(vcpuid,init) vmx_setup_cr_shadow(vcpuid, 4, (init))
  
  static void *
  vmx_vm_init(struct vm *vm)
  {
  	struct vmx *vmx;
  
  	vmx = malloc(sizeof(struct vmx));
  	assert(vmx);
  	bzero(vmx, sizeof(struct vmx));
  	vmx->vm = vm;
  
  	return (vmx);
  }
  
  static int
  vmx_vcpu_init(void *arg, int vcpuid) {
  	uint32_t exc_bitmap;
  	struct vmx *vmx;
  	hv_vcpuid_t hvid;
  	int error;
  
  	vmx = (struct vmx *) arg;
  
  	if (hv_vcpu_create(&hvid, HV_VCPU_DEFAULT)) {
  		xhyve_abort("hv_vcpu_create failed\n");
  	}
  
  	if (hvid != ((hv_vcpuid_t) vcpuid)) {
  		/* FIXME */
  		xhyve_abort("vcpu id mismatch\n");
  	}
  
  	if (hv_vcpu_enable_native_msr(hvid, MSR_GSBASE, 1) ||
  		hv_vcpu_enable_native_msr(hvid, MSR_FSBASE, 1) ||
  		hv_vcpu_enable_native_msr(hvid, MSR_SYSENTER_CS_MSR, 1) ||
  		hv_vcpu_enable_native_msr(hvid, MSR_SYSENTER_ESP_MSR, 1) ||
  		hv_vcpu_enable_native_msr(hvid, MSR_SYSENTER_EIP_MSR, 1) ||
  		hv_vcpu_enable_native_msr(hvid, MSR_TSC, 1) ||
  		hv_vcpu_enable_native_msr(hvid, MSR_IA32_TSC_AUX, 1))
  	{
  		xhyve_abort("vmx_vcpu_init: error setting guest msr access\n");
  	}
  
  	vmx_msr_guest_init(vmx, vcpuid);
  
  	uint32_t pinbased_ctls = 0;
  	uint32_t procbased_ctls = 0;
  	uint32_t procbased_ctls2 = 0;
  	uint32_t exit_ctls = 0;
  	vmx->state[vcpuid].entry_ctls = 0;
  	
  	/* Check support for primary processor-based VM-execution controls */
  	error = vmx_set_ctlreg(vcpuid, VMCS_PRI_PROC_BASED_CTLS,
  						   HV_VMX_CAP_PROCBASED,
  						   PROCBASED_CTLS_ONE_SETTING,
  						   PROCBASED_CTLS_ZERO_SETTING, &procbased_ctls);
  	if (error) {
  		printf("vmx_init: processor does not support desired primary "
  			   "processor-based controls\n");
  		return (error);
  	}
  	
  	/* Clear the processor-based ctl bits that are set on demand */
  	procbased_ctls &= ~PROCBASED_CTLS_WINDOW_SETTING;
  	
  	/* Check support for secondary processor-based VM-execution controls */
  	error = vmx_set_ctlreg(vcpuid, VMCS_SEC_PROC_BASED_CTLS, HV_VMX_CAP_PROCBASED2,
  						   PROCBASED_CTLS2_ONE_SETTING,
  						   PROCBASED_CTLS2_ZERO_SETTING, &procbased_ctls2);
  	if (error) {
  		printf("vmx_init: processor does not support desired secondary "
  			   "processor-based controls\n");
  		return (error);
  	}
  	
  	/* Check support for pin-based VM-execution controls */
  	error = vmx_set_ctlreg(vcpuid, VMCS_PIN_BASED_CTLS, HV_VMX_CAP_PINBASED,
  						   PINBASED_CTLS_ONE_SETTING,
  						   PINBASED_CTLS_ZERO_SETTING, &pinbased_ctls);
  	if (error) {
  		printf("vmx_init: processor does not support desired "
  			   "pin-based controls\n");
  		return (error);
  	}
  	
  	/* Check support for VM-exit controls */
  	error = vmx_set_ctlreg(vcpuid, VMCS_EXIT_CTLS, HV_VMX_CAP_EXIT,
  						   VM_EXIT_CTLS_ONE_SETTING,
  						   VM_EXIT_CTLS_ZERO_SETTING,
  						   &exit_ctls);
  	if (error) {
  		printf("vmx_init: processor does not support desired "
  			   "exit controls\n");
  		return (error);
  	}
  	
  	/* Check support for VM-entry controls */
  	error = vmx_set_ctlreg(vcpuid, VMCS_ENTRY_CTLS, HV_VMX_CAP_ENTRY,
  						   VM_ENTRY_CTLS_ONE_SETTING, VM_ENTRY_CTLS_ZERO_SETTING,
  						   &vmx->state[vcpuid].entry_ctls);
  	if (error) {
  		printf("vmx_init: processor does not support desired "
  			   "entry controls\n");
  		return (error);
  	}
  	
  	vmcs_write(vcpuid, VMCS_PIN_BASED_CTLS, pinbased_ctls);
  	vmcs_write(vcpuid, VMCS_PRI_PROC_BASED_CTLS, procbased_ctls);
  	vmcs_write(vcpuid, VMCS_SEC_PROC_BASED_CTLS, procbased_ctls2);
  	vmcs_write(vcpuid, VMCS_EXIT_CTLS, exit_ctls);
  	vmcs_write(vcpuid, VMCS_ENTRY_CTLS, vmx->state[vcpuid].entry_ctls);
  
  	/* exception bitmap */
  	if (vcpu_trace_exceptions())
  		exc_bitmap = 0xffffffff;
  	else
  		exc_bitmap = 1 << IDT_MC;
  
  	vmcs_write(vcpuid, VMCS_EXCEPTION_BITMAP, exc_bitmap);
  
  	vmx->cap[vcpuid].set = 0;
  	vmx->cap[vcpuid].proc_ctls = procbased_ctls;
  	vmx->cap[vcpuid].proc_ctls2 = procbased_ctls2;
  	vmx->state[vcpuid].nextrip = ~(uint64_t) 0;
  
  	/*
  	 * Set up the CR0/4 shadows, and init the read shadow
  	 * to the power-on register value from the Intel Sys Arch.
  	 *  CR0 - 0x60000010
  	 *  CR4 - 0
  	 */
  	error = vmx_setup_cr0_shadow(vcpuid, 0x60000010);
  	if (error != 0)
  		xhyve_abort("vmx_setup_cr0_shadow %d\n", error);
  
  	error = vmx_setup_cr4_shadow(vcpuid, 0);
  
  	if (error != 0)
  		xhyve_abort("vmx_setup_cr4_shadow %d\n", error);
  
  	return (0);
  }
  
  static int
  vmx_handle_cpuid(struct vm *vm, int vcpuid)
  {
  	uint32_t eax, ebx, ecx, edx;
  	int error;
  	
  	eax = (uint32_t) reg_read(vcpuid, HV_X86_RAX);
  	ebx = (uint32_t) reg_read(vcpuid, HV_X86_RBX);
  	ecx = (uint32_t) reg_read(vcpuid, HV_X86_RCX);
  	edx = (uint32_t) reg_read(vcpuid, HV_X86_RDX);
  
  	error = x86_emulate_cpuid(vm, vcpuid, &eax, &ebx, &ecx, &edx);
  
  	reg_write(vcpuid, HV_X86_RAX, eax);
  	reg_write(vcpuid, HV_X86_RBX, ebx);
  	reg_write(vcpuid, HV_X86_RCX, ecx);
  	reg_write(vcpuid, HV_X86_RDX, edx);
  
  	return (error);
  }
  
  static __inline void
  vmx_run_trace(struct vmx *vmx, int vcpu)
  {
  #ifdef XHYVE_CONFIG_TRACE
  	VCPU_CTR1(vmx->vm, vcpu, "Resume execution at %#llx", vmcs_guest_rip(vcpu));
  #else
  	(void) vmx;
  	(void) vcpu;
  #endif
  }
  
  static __inline void
  vmx_exit_trace(struct vmx *vmx, int vcpu, uint64_t rip, uint32_t exit_reason,
  	       int handled)
  {
  #ifdef XHYVE_CONFIG_TRACE
  	VCPU_CTR3(vmx->vm, vcpu, "%s %s vmexit at 0x%0llx",
  		 handled ? "handled" : "unhandled",
  		 exit_reason_to_str((int) exit_reason), rip);
  #else
  	(void) vmx;
  	(void) vcpu;
  	(void) rip;
  	(void) exit_reason;
  	(void) handled;
  #endif
  }
  
  /*
   * We depend on 'procbased_ctls' to have the Interrupt Window Exiting bit set.
   */
  CTASSERT((PROCBASED_CTLS_ONE_SETTING & PROCBASED_INT_WINDOW_EXITING) != 0);
  
  static void __inline
  vmx_set_int_window_exiting(struct vmx *vmx, int vcpu)
  {
  	if ((vmx->cap[vcpu].proc_ctls & PROCBASED_INT_WINDOW_EXITING) == 0) {
  		vmx->cap[vcpu].proc_ctls |= PROCBASED_INT_WINDOW_EXITING;
  		vmcs_write(vcpu, VMCS_PRI_PROC_BASED_CTLS, vmx->cap[vcpu].proc_ctls);
  		VCPU_CTR0(vmx->vm, vcpu, "Enabling interrupt window exiting");
  	}
  }
  
  static void __inline
  vmx_clear_int_window_exiting(struct vmx *vmx, int vcpu)
  {
  	KASSERT((vmx->cap[vcpu].proc_ctls & PROCBASED_INT_WINDOW_EXITING) != 0,
  	    ("intr_window_exiting not set: %#x", vmx->cap[vcpu].proc_ctls));
  	vmx->cap[vcpu].proc_ctls &= ~PROCBASED_INT_WINDOW_EXITING;
  	vmcs_write(vcpu, VMCS_PRI_PROC_BASED_CTLS, vmx->cap[vcpu].proc_ctls);
  	VCPU_CTR0(vmx->vm, vcpu, "Disabling interrupt window exiting");
  }
  
  static void __inline
  vmx_set_nmi_window_exiting(struct vmx *vmx, int vcpu)
  {
  
  	if ((vmx->cap[vcpu].proc_ctls & PROCBASED_NMI_WINDOW_EXITING) == 0) {
  		vmx->cap[vcpu].proc_ctls |= PROCBASED_NMI_WINDOW_EXITING;
  		vmcs_write(vcpu, VMCS_PRI_PROC_BASED_CTLS, vmx->cap[vcpu].proc_ctls);
  		VCPU_CTR0(vmx->vm, vcpu, "Enabling NMI window exiting");
  	}
  }
  
  static void __inline
  vmx_clear_nmi_window_exiting(struct vmx *vmx, int vcpu)
  {
  
  	KASSERT((vmx->cap[vcpu].proc_ctls & PROCBASED_NMI_WINDOW_EXITING) != 0,
  	    ("nmi_window_exiting not set %#x", vmx->cap[vcpu].proc_ctls));
  	vmx->cap[vcpu].proc_ctls &= ~PROCBASED_NMI_WINDOW_EXITING;
  	vmcs_write(vcpu, VMCS_PRI_PROC_BASED_CTLS, vmx->cap[vcpu].proc_ctls);
  	VCPU_CTR0(vmx->vm, vcpu, "Disabling NMI window exiting");
  }
  
  static void
  vmx_inject_nmi(struct vmx *vmx, int vcpu)
  {
  	uint32_t gi, info;
  
  	gi = (uint32_t) vmcs_read(vcpu, VMCS_GUEST_INTERRUPTIBILITY);
  	KASSERT((gi & NMI_BLOCKING) == 0, ("vmx_inject_nmi: invalid guest "
  	    "interruptibility-state %#x", gi));
  
  	info = (uint32_t) vmcs_read(vcpu, VMCS_ENTRY_INTR_INFO);
  	KASSERT((info & VMCS_INTR_VALID) == 0, ("vmx_inject_nmi: invalid "
  	    "VM-entry interruption information %#x", info));
  
  	/*
  	 * Inject the virtual NMI. The vector must be the NMI IDT entry
  	 * or the VMCS entry check will fail.
  	 */
  	info = IDT_NMI | VMCS_INTR_T_NMI | VMCS_INTR_VALID;
  	vmcs_write(vcpu, VMCS_ENTRY_INTR_INFO, info);
  
  	VCPU_CTR0(vmx->vm, vcpu, "Injecting vNMI");
  
  	/* Clear the request */
  	vm_nmi_clear(vmx->vm, vcpu);
  }
  
  static void
  vmx_inject_interrupts(struct vmx *vmx, int vcpu, struct vlapic *vlapic,
      uint64_t guestrip)
  {
  	int vector, need_nmi_exiting, extint_pending;
  	uint64_t rflags, entryinfo;
  	uint32_t gi, info;
  
  	if (vmx->state[vcpu].nextrip != guestrip) {
  		gi = (uint32_t) vmcs_read(vcpu, VMCS_GUEST_INTERRUPTIBILITY);
  		if (gi & HWINTR_BLOCKING) {
  			VCPU_CTR2(vmx->vm, vcpu, "Guest interrupt blocking "
  			    "cleared due to rip change: %#llx/%#llx",
  			    vmx->state[vcpu].nextrip, guestrip);
  			gi &= ~HWINTR_BLOCKING;
  			vmcs_write(vcpu, VMCS_GUEST_INTERRUPTIBILITY, gi);
  		}
  	}
  
  	if (vm_entry_intinfo(vmx->vm, vcpu, &entryinfo)) {
  		KASSERT((entryinfo & VMCS_INTR_VALID) != 0, ("%s: entry "
  		    "intinfo is not valid: %#llx", __func__, entryinfo));
  
  		info = (uint32_t) vmcs_read(vcpu, VMCS_ENTRY_INTR_INFO);
  		KASSERT((info & VMCS_INTR_VALID) == 0, ("%s: cannot inject "
  		     "pending exception: %#llx/%#x", __func__, entryinfo, info));
  
  		info = (uint32_t) entryinfo;
  		vector = info & 0xff;
  		if (vector == IDT_BP || vector == IDT_OF) {
  			/*
  			 * VT-x requires #BP and #OF to be injected as software
  			 * exceptions.
  			 */
  			info &= ~VMCS_INTR_T_MASK;
  			info |= VMCS_INTR_T_SWEXCEPTION;
  		}
  
  		if (info & VMCS_INTR_DEL_ERRCODE)
  			vmcs_write(vcpu, VMCS_ENTRY_EXCEPTION_ERROR, entryinfo >> 32);
  
  		vmcs_write(vcpu, VMCS_ENTRY_INTR_INFO, info);
  	}
  
  	if (vm_nmi_pending(vmx->vm, vcpu)) {
  		/*
  		 * If there are no conditions blocking NMI injection then
  		 * inject it directly here otherwise enable "NMI window
  		 * exiting" to inject it as soon as we can.
  		 *
  		 * We also check for STI_BLOCKING because some implementations
  		 * don't allow NMI injection in this case. If we are running
  		 * on a processor that doesn't have this restriction it will
  		 * immediately exit and the NMI will be injected in the
  		 * "NMI window exiting" handler.
  		 */
  		need_nmi_exiting = 1;
  		gi = (uint32_t) vmcs_read(vcpu, VMCS_GUEST_INTERRUPTIBILITY);
  		if ((gi & (HWINTR_BLOCKING | NMI_BLOCKING)) == 0) {
  			info = (uint32_t) vmcs_read(vcpu, VMCS_ENTRY_INTR_INFO);
  			if ((info & VMCS_INTR_VALID) == 0) {
  				vmx_inject_nmi(vmx, vcpu);
  				need_nmi_exiting = 0;
  			} else {
  				VCPU_CTR1(vmx->vm, vcpu, "Cannot inject NMI "
  				    "due to VM-entry intr info %#x", info);
  			}
  		} else {
  			VCPU_CTR1(vmx->vm, vcpu, "Cannot inject NMI due to "
  			    "Guest Interruptibility-state %#x", gi);
  		}
  
  		if (need_nmi_exiting)
  			vmx_set_nmi_window_exiting(vmx, vcpu);
  	}
  
  	extint_pending = vm_extint_pending(vmx->vm, vcpu);
  
  	/*
  	 * If interrupt-window exiting is already in effect then don't bother
  	 * checking for pending interrupts. This is just an optimization and
  	 * not needed for correctness.
  	 */
  	if ((vmx->cap[vcpu].proc_ctls & PROCBASED_INT_WINDOW_EXITING) != 0) {
  		VCPU_CTR0(vmx->vm, vcpu, "Skip interrupt injection due to "
  		    "pending int_window_exiting");
  		return;
  	}
  
  	if (!extint_pending) {
  		/* Ask the local apic for a vector to inject */
  		if (!vlapic_pending_intr(vlapic, &vector))
  			return;
  
  		/*
  		 * From the Intel SDM, Volume 3, Section "Maskable
  		 * Hardware Interrupts":
  		 * - maskable interrupt vectors [16,255] can be delivered
  		 *   through the local APIC.
  		*/
  		KASSERT(vector >= 16 && vector <= 255,
  		    ("invalid vector %d from local APIC", vector));
  	} else {
  		/* Ask the legacy pic for a vector to inject */
  		vatpic_pending_intr(vmx->vm, &vector);
  
  		/*
  		 * From the Intel SDM, Volume 3, Section "Maskable
  		 * Hardware Interrupts":
  		 * - maskable interrupt vectors [0,255] can be delivered
  		 *   through the INTR pin.
  		 */
  		KASSERT(vector >= 0 && vector <= 255,
  		    ("invalid vector %d from INTR", vector));
  	}
  
  	/* Check RFLAGS.IF and the interruptibility state of the guest */
  	rflags = vmcs_read(vcpu, VMCS_GUEST_RFLAGS);
  	if ((rflags & PSL_I) == 0) {
  		VCPU_CTR2(vmx->vm, vcpu, "Cannot inject vector %d due to "
  		    "rflags %#llx", vector, rflags);
  		goto cantinject;
  	}
  
  	gi = (uint32_t) vmcs_read(vcpu, VMCS_GUEST_INTERRUPTIBILITY);
  	if (gi & HWINTR_BLOCKING) {
  		VCPU_CTR2(vmx->vm, vcpu, "Cannot inject vector %d due to "
  		    "Guest Interruptibility-state %#x", vector, gi);
  		goto cantinject;
  	}
  
  	info = (uint32_t) vmcs_read(vcpu, VMCS_ENTRY_INTR_INFO);
  	if (info & VMCS_INTR_VALID) {
  		/*
  		 * This is expected and could happen for multiple reasons:
  		 * - A vectoring VM-entry was aborted due to astpending
  		 * - A VM-exit happened during event injection.
  		 * - An exception was injected above.
  		 * - An NMI was injected above or after "NMI window exiting"
  		 */
  		VCPU_CTR2(vmx->vm, vcpu, "Cannot inject vector %d due to "
  		    "VM-entry intr info %#x", vector, info);
  		goto cantinject;
  	}
  
  	/* Inject the interrupt */
  	info = VMCS_INTR_T_HWINTR | VMCS_INTR_VALID;
  	info |= (uint32_t) vector;
  	vmcs_write(vcpu, VMCS_ENTRY_INTR_INFO, info);
  
  	if (!extint_pending) {
  		/* Update the Local APIC ISR */
  		vlapic_intr_accepted(vlapic, vector);
  	} else {
  		vm_extint_clear(vmx->vm, vcpu);
  		vatpic_intr_accepted(vmx->vm, vector);
  
  		/*
  		 * After we accepted the current ExtINT the PIC may
  		 * have posted another one.  If that is the case, set
  		 * the Interrupt Window Exiting execution control so
  		 * we can inject that one too.
  		 *
  		 * Also, interrupt window exiting allows us to inject any
  		 * pending APIC vector that was preempted by the ExtINT
  		 * as soon as possible. This applies both for the software
  		 * emulated vlapic and the hardware assisted virtual APIC.
  		 */
  		vmx_set_int_window_exiting(vmx, vcpu);
  	}
  
  	VCPU_CTR1(vmx->vm, vcpu, "Injecting hwintr at vector %d", vector);
  
  	return;
  
  cantinject:
  	/*
  	 * Set the Interrupt Window Exiting execution control so we can inject
  	 * the interrupt as soon as blocking condition goes away.
  	 */
  	vmx_set_int_window_exiting(vmx, vcpu);
  }
  
  /*
   * If the Virtual NMIs execution control is '1' then the logical processor
   * tracks virtual-NMI blocking in the Guest Interruptibility-state field of
   * the VMCS. An IRET instruction in VMX non-root operation will remove any
   * virtual-NMI blocking.
   *
   * This unblocking occurs even if the IRET causes a fault. In this case the
   * hypervisor needs to restore virtual-NMI blocking before resuming the guest.
   */
  static void
  vmx_restore_nmi_blocking(struct vmx *vmx, int vcpuid)
  {
  	uint32_t gi;
  
  	VCPU_CTR0(vmx->vm, vcpuid, "Restore Virtual-NMI blocking");
  	gi = (uint32_t) vmcs_read(vcpuid, VMCS_GUEST_INTERRUPTIBILITY);
  	gi |= VMCS_INTERRUPTIBILITY_NMI_BLOCKING;
  	vmcs_write(vcpuid, VMCS_GUEST_INTERRUPTIBILITY, gi);
  }
  
  static void
  vmx_clear_nmi_blocking(struct vmx *vmx, int vcpuid)
  {
  	uint32_t gi;
  
  	VCPU_CTR0(vmx->vm, vcpuid, "Clear Virtual-NMI blocking");
  	gi = (uint32_t) vmcs_read(vcpuid, VMCS_GUEST_INTERRUPTIBILITY);
  	gi &= ~VMCS_INTERRUPTIBILITY_NMI_BLOCKING;
  	vmcs_write(vcpuid, VMCS_GUEST_INTERRUPTIBILITY, gi);
  }
  
  static void
  vmx_assert_nmi_blocking(int vcpuid)
  {
  	uint32_t gi;
  
  	gi = (uint32_t) vmcs_read(vcpuid, VMCS_GUEST_INTERRUPTIBILITY);
  	KASSERT(gi & VMCS_INTERRUPTIBILITY_NMI_BLOCKING,
  	    ("NMI blocking is not in effect %#x", gi));
  }
  
  static int
  vmx_emulate_xsetbv(struct vmx *vmx, int vcpu)
  {
  	uint64_t xcrval;
  	const struct xsave_limits *limits;
  
  	limits = vmm_get_xsave_limits();
  
  	/*
  	 * Note that the processor raises a GP# fault on its own if
  	 * xsetbv is executed for CPL != 0, so we do not have to
  	 * emulate that fault here.
  	 */
  
  	/* Only xcr0 is supported. */
  	if (reg_read(vcpu, HV_X86_RCX) != 0) {
  		vm_inject_gp(vmx->vm, vcpu);
  		return (HANDLED);
  	}
  
  	/* We only handle xcr0 if both the host and guest have XSAVE enabled. */
  	if (!limits->xsave_enabled ||
  		!(vmcs_read(vcpu, VMCS_GUEST_CR4) & CR4_XSAVE))
  	{
  		vm_inject_ud(vmx->vm, vcpu);
  		return (HANDLED);
  	}
  
  	xcrval = reg_read(vcpu, HV_X86_RDX) << 32
  		| (reg_read(vcpu, HV_X86_RAX) & 0xffffffff);
  
  	if ((xcrval & ~limits->xcr0_allowed) != 0) {
  		vm_inject_gp(vmx->vm, vcpu);
  		return (HANDLED);
  	}
  
  	if (!(xcrval & XFEATURE_ENABLED_X87)) {
  		vm_inject_gp(vmx->vm, vcpu);
  		return (HANDLED);
  	}
  
  	/* AVX (YMM_Hi128) requires SSE. */
  	if (xcrval & XFEATURE_ENABLED_AVX &&
  	    (xcrval & XFEATURE_AVX) != XFEATURE_AVX) {
  		vm_inject_gp(vmx->vm, vcpu);
  		return (HANDLED);
  	}
  
  	/*
  	 * AVX512 requires base AVX (YMM_Hi128) as well as OpMask,
  	 * ZMM_Hi256, and Hi16_ZMM.
  	 */
  	if (xcrval & XFEATURE_AVX512 &&
  	    (xcrval & (XFEATURE_AVX512 | XFEATURE_AVX)) !=
  	    (XFEATURE_AVX512 | XFEATURE_AVX)) {
  		vm_inject_gp(vmx->vm, vcpu);
  		return (HANDLED);
  	}
  
  	/*
  	 * Intel MPX requires both bound register state flags to be
  	 * set.
  	 */
  	if (((xcrval & XFEATURE_ENABLED_BNDREGS) != 0) !=
  	    ((xcrval & XFEATURE_ENABLED_BNDCSR) != 0)) {
  		vm_inject_gp(vmx->vm, vcpu);
  		return (HANDLED);
  	}
  
  	reg_write(vcpu, HV_X86_XCR0, xcrval);
  	return (HANDLED);
  }
  
  static uint64_t
  vmx_get_guest_reg(int vcpu, int ident)
  {
  	switch (ident) {
  	case 0:
  		return (reg_read(vcpu, HV_X86_RAX));
  	case 1:
  		return (reg_read(vcpu, HV_X86_RCX));
  	case 2:
  		return (reg_read(vcpu, HV_X86_RDX));
  	case 3:
  		return (reg_read(vcpu, HV_X86_RBX));
  	case 4:
  		return (vmcs_read(vcpu, VMCS_GUEST_RSP));
  	case 5:
  		return (reg_read(vcpu, HV_X86_RBP));
  	case 6:
  		return (reg_read(vcpu, HV_X86_RSI));
  	case 7:
  		return (reg_read(vcpu, HV_X86_RDI));
  	case 8:
  		return (reg_read(vcpu, HV_X86_R8));
  	case 9:
  		return (reg_read(vcpu, HV_X86_R9));
  	case 10:
  		return (reg_read(vcpu, HV_X86_R10));
  	case 11:
  		return (reg_read(vcpu, HV_X86_R11));
  	case 12:
  		return (reg_read(vcpu, HV_X86_R12));
  	case 13:
  		return (reg_read(vcpu, HV_X86_R13));
  	case 14:
  		return (reg_read(vcpu, HV_X86_R14));
  	case 15:
  		return (reg_read(vcpu, HV_X86_R15));
  	default:
  		xhyve_abort("invalid vmx register %d", ident);
  	}
  }
  
  static void
  vmx_set_guest_reg(int vcpu, int ident, uint64_t regval)
  {
  	switch (ident) {
  	case 0:
  		reg_write(vcpu, HV_X86_RAX, regval);
  		break;
  	case 1:
  		reg_write(vcpu, HV_X86_RCX, regval);
  		break;
  	case 2:
  		reg_write(vcpu, HV_X86_RDX, regval);
  		break;
  	case 3:
  		reg_write(vcpu, HV_X86_RBX, regval);
  		break;
  	case 4:
  		vmcs_write(vcpu, VMCS_GUEST_RSP, regval);
  		break;
  	case 5:
  		reg_write(vcpu, HV_X86_RBP, regval);
  		break;
  	case 6:
  		reg_write(vcpu, HV_X86_RSI, regval);
  		break;
  	case 7:
  		reg_write(vcpu, HV_X86_RDI, regval);
  		break;
  	case 8:
  		reg_write(vcpu, HV_X86_R8, regval);
  		break;
  	case 9:
  		reg_write(vcpu, HV_X86_R9, regval);
  		break;
  	case 10:
  		reg_write(vcpu, HV_X86_R10, regval);
  		break;
  	case 11:
  		reg_write(vcpu, HV_X86_R11, regval);
  		break;
  	case 12:
  		reg_write(vcpu, HV_X86_R12, regval);
  		break;
  	case 13:
  		reg_write(vcpu, HV_X86_R13, regval);
  		break;
  	case 14:
  		reg_write(vcpu, HV_X86_R14, regval);
  		break;
  	case 15:
  		reg_write(vcpu, HV_X86_R15, regval);
  		break;
  	default:
  		xhyve_abort("invalid vmx register %d", ident);
  	}
  }
  
  static int
  vmx_emulate_cr0_access(UNUSED struct vm *vm, int vcpu, uint64_t exitqual)
  {
  	uint64_t crval, efer, entryctls, regval;
  	// *pt;
  
  	/* We only handle mov to %cr0 at this time */
  	if ((exitqual & 0xf0) != 0x00)
  		return (UNHANDLED);
  
  	regval = vmx_get_guest_reg(vcpu, (exitqual >> 8) & 0xf);
  
  	vmcs_write(vcpu, VMCS_CR0_SHADOW, regval);
  
  	crval = regval | cr0_ones_mask;
  	crval &= ~cr0_zeros_mask;
  	// printf("cr0: v:0x%016llx 1:0x%08llx 0:0x%08llx v:0x%016llx\n",
  	// 	regval, cr0_ones_mask, cr0_zeros_mask, crval);
  	vmcs_write(vcpu, VMCS_GUEST_CR0, crval);
  
  	efer = vmcs_read(vcpu, VMCS_GUEST_IA32_EFER);
  
  	if (regval & CR0_PG) {
  		/*
  		 * If CR0.PG is 1 and EFER.LME is 1 then EFER.LMA and
  		 * the "IA-32e mode guest" bit in VM-entry control must be
  		 * equal.
  		 */
  		if (efer & EFER_LME) {
  			efer |= EFER_LMA;
  			vmcs_write(vcpu, VMCS_GUEST_IA32_EFER, efer);
  			entryctls = vmcs_read(vcpu, VMCS_ENTRY_CTLS);
  			entryctls |= VM_ENTRY_GUEST_LMA;
  			vmcs_write(vcpu, VMCS_ENTRY_CTLS, entryctls);
  		}
  
  		// if (vmcs_read(vcpu, VMCS_GUEST_CR4) & CR4_PAE) {
  		// 	if (!(pt = (uint64_t *) vm_gpa2hva(vm,
  		// 		vmcs_read(vcpu, VMCS_GUEST_CR3), sizeof(uint64_t) * 4)))
  		// 	{
  		// 		xhyve_abort("invalid cr3\n");
  		// 	}
  
  		// 	vmcs_write(vcpu, VMCS_GUEST_PDPTE0, pt[0]);
  		// 	vmcs_write(vcpu, VMCS_GUEST_PDPTE1, pt[1]);
  		// 	vmcs_write(vcpu, VMCS_GUEST_PDPTE2, pt[2]);
  		// 	vmcs_write(vcpu, VMCS_GUEST_PDPTE3, pt[3]);
  		// }
  	} else {
  		/*
  		 * If CR0.PG is 0 and EFER.LMA is 1, this is a
  		 * switch out of IA32e mode so emulate that.
  		 */
  		if (efer & EFER_LMA) {
  			efer &= ~(uint64_t)EFER_LMA;
  			vmcs_write(vcpu, VMCS_GUEST_IA32_EFER, efer);
  			entryctls = vmcs_read(vcpu, VMCS_ENTRY_CTLS);
  			entryctls &= ~VM_ENTRY_GUEST_LMA;
  			vmcs_write(vcpu, VMCS_ENTRY_CTLS, entryctls);
  		}
  	}
  
  	return (HANDLED);
  }
  
  static int
  vmx_emulate_cr4_access(int vcpu, uint64_t exitqual)
  {
  	uint64_t crval, regval;
  
  	/* We only handle mov to %cr4 at this time */
  	if ((exitqual & 0xf0) != 0x00)
  		return (UNHANDLED);
  
  	regval = vmx_get_guest_reg(vcpu, (exitqual >> 8) & 0xf);
  
  	vmcs_write(vcpu, VMCS_CR4_SHADOW, regval);
  
  	crval = regval | cr4_ones_mask;
  	crval &= ~cr4_zeros_mask;
  	vmcs_write(vcpu, VMCS_GUEST_CR4, crval);
  
  	return (HANDLED);
  }
  
  static int
  vmx_emulate_cr8_access(struct vmx *vmx, int vcpu, uint64_t exitqual)
  {
  	struct vlapic *vlapic;
  	uint64_t cr8;
  	int regnum;
  
  	/* We only handle mov %cr8 to/from a register at this time. */
  	if ((exitqual & 0xe0) != 0x00) {
  		return (UNHANDLED);
  	}
  
  	vlapic = vm_lapic(vmx->vm, vcpu);
  	regnum = (exitqual >> 8) & 0xf;
  	if (exitqual & 0x10) {
  		cr8 = vlapic_get_cr8(vlapic);
  		vmx_set_guest_reg(vcpu, regnum, cr8);
  	} else {
  		cr8 = vmx_get_guest_reg(vcpu, regnum);
  		vlapic_set_cr8(vlapic, cr8);
  	}
  
  	return (HANDLED);
  }
  
  /*
   * From section "Guest Register State" in the Intel SDM: CPL = SS.DPL
   */
  static int
  vmx_cpl(int vcpu)
  {
  	uint32_t ssar;
  
  	ssar = (uint32_t) vmcs_read(vcpu, VMCS_GUEST_SS_ACCESS_RIGHTS);
  	return ((ssar >> 5) & 0x3);
  }
  
  static enum vm_cpu_mode
  vmx_cpu_mode(int vcpu)
  {
  	uint32_t csar;
  
  	if (vmcs_read(vcpu, VMCS_GUEST_IA32_EFER) & EFER_LMA) {
  		csar = (uint32_t) vmcs_read(vcpu, VMCS_GUEST_CS_ACCESS_RIGHTS);
  		if (csar & 0x2000)
  			return (CPU_MODE_64BIT);	/* CS.L = 1 */
  		else
  			return (CPU_MODE_COMPATIBILITY);
  	} else if (vmcs_read(vcpu, VMCS_GUEST_CR0) & CR0_PE) {
  		return (CPU_MODE_PROTECTED);
  	} else {
  		return (CPU_MODE_REAL);
  	}
  }
  
  static enum vm_paging_mode
  vmx_paging_mode(int vcpu)
  {
  
  	if (!(vmcs_read(vcpu, VMCS_GUEST_CR0) & CR0_PG))
  		return (PAGING_MODE_FLAT);
  	if (!(vmcs_read(vcpu, VMCS_GUEST_CR4) & CR4_PAE))
  		return (PAGING_MODE_32);
  	if (vmcs_read(vcpu, VMCS_GUEST_IA32_EFER) & EFER_LME)
  		return (PAGING_MODE_64);
  	else
  		return (PAGING_MODE_PAE);
  }
  
  static uint64_t
  inout_str_index(struct vmx *vmx, int vcpuid, int in)
  {
  	uint64_t val;
  	int error;
  	enum vm_reg_name reg;
  
  	reg = in ? VM_REG_GUEST_RDI : VM_REG_GUEST_RSI;
  	error = vmx_getreg(vmx, vcpuid, (int) reg, &val);
  	KASSERT(error == 0, ("%s: vmx_getreg error %d", __func__, error));
  	return (val);
  }
  
  static uint64_t
  inout_str_count(struct vmx *vmx, int vcpuid, int rep)
  {
  	uint64_t val;
  	int error;
  
  	if (rep) {
  		error = vmx_getreg(vmx, vcpuid, VM_REG_GUEST_RCX, &val);
  		KASSERT(!error, ("%s: vmx_getreg error %d", __func__, error));
  	} else {
  		val = 1;
  	}
  	return (val);
  }
  
  static int
  inout_str_addrsize(uint32_t inst_info)
  {
  	uint32_t size;
  
  	size = (inst_info >> 7) & 0x7;
  	switch (size) {
  	case 0:
  		return (2);	/* 16 bit */
  	case 1:
  		return (4);	/* 32 bit */
  	case 2:
  		return (8);	/* 64 bit */
  	default:
  		xhyve_abort("%s: invalid size encoding %d", __func__, size);
  	}
  }
  
  static void
  inout_str_seginfo(struct vmx *vmx, int vcpuid, uint32_t inst_info, int in,
      struct vm_inout_str *vis)
  {
  	int error, s;
  
  	if (in) {
  		vis->seg_name = VM_REG_GUEST_ES;
  	} else {
  		s = (inst_info >> 15) & 0x7;
  		vis->seg_name = vm_segment_name(s);
  	}
  
  	error = vmx_getdesc(vmx, vcpuid, (int) vis->seg_name, &vis->seg_desc);
  	KASSERT(error == 0, ("%s: vmx_getdesc error %d", __func__, error));
  }
  
  static void
  vmx_paging_info(struct vm_guest_paging *paging, int vcpu)
  {
  	paging->cr3 = vmcs_guest_cr3(vcpu);
  	paging->cpl = vmx_cpl(vcpu);
  	paging->cpu_mode = vmx_cpu_mode(vcpu);
  	paging->paging_mode = vmx_paging_mode(vcpu);
  }
  
  static void
  vmexit_inst_emul(struct vm_exit *vmexit, uint64_t gpa, uint64_t gla, int vcpu)
  {
  	struct vm_guest_paging *paging;
  	uint32_t csar;
  
  	paging = &vmexit->u.inst_emul.paging;
  
  	vmexit->exitcode = VM_EXITCODE_INST_EMUL;
  	vmexit->u.inst_emul.gpa = gpa;
  	vmexit->u.inst_emul.gla = gla;
  	vmx_paging_info(paging, vcpu);
  	switch (paging->cpu_mode) {
  	case CPU_MODE_REAL:
  		vmexit->u.inst_emul.cs_base = vmcs_read(vcpu, VMCS_GUEST_CS_BASE);
  		vmexit->u.inst_emul.cs_d = 0;
  		break;
  	case CPU_MODE_PROTECTED:
  	case CPU_MODE_COMPATIBILITY:
  		vmexit->u.inst_emul.cs_base = vmcs_read(vcpu, VMCS_GUEST_CS_BASE);
  		csar = (uint32_t) vmcs_read(vcpu, VMCS_GUEST_CS_ACCESS_RIGHTS);
  		vmexit->u.inst_emul.cs_d = SEG_DESC_DEF32(csar);
  		break;
  	case CPU_MODE_64BIT:
  		vmexit->u.inst_emul.cs_base = 0;
  		vmexit->u.inst_emul.cs_d = 0;
  		break;
  	}
  	vie_init(&vmexit->u.inst_emul.vie, NULL, 0);
  }
  
  static int
  ept_fault_type(uint64_t ept_qual)
  {
  	int fault_type;
  
  	if (ept_qual & EPT_VIOLATION_DATA_WRITE)
  		fault_type = XHYVE_PROT_WRITE;
  	else if (ept_qual & EPT_VIOLATION_INST_FETCH)
  		fault_type = XHYVE_PROT_EXECUTE;
  	else
  		fault_type= XHYVE_PROT_READ;
  
  	return (fault_type);
  }
  
  static bool
  ept_emulation_fault(uint64_t ept_qual)
  {
  	int read, write;
  
  	/* EPT fault on an instruction fetch doesn't make sense here */
  	if (ept_qual & EPT_VIOLATION_INST_FETCH)
  		return (FALSE);
  
  	/* EPT fault must be a read fault or a write fault */
  	read = ept_qual & EPT_VIOLATION_DATA_READ ? 1 : 0;
  	write = ept_qual & EPT_VIOLATION_DATA_WRITE ? 1 : 0;
  	if ((read | write) == 0)
  		return (FALSE);
  
  	/*
  	 * The EPT violation must have been caused by accessing a
  	 * guest-physical address that is a translation of a guest-linear
  	 * address.
  	 */
  	if ((ept_qual & EPT_VIOLATION_GLA_VALID) == 0 ||
  	    (ept_qual & EPT_VIOLATION_XLAT_VALID) == 0) {
  		return (FALSE);
  	}
  
  	return (TRUE);
  }
  
  static __inline int
  apic_access_virtualization(struct vmx *vmx, int vcpuid)
  {
  	uint32_t proc_ctls2;
  
  	proc_ctls2 = vmx->cap[vcpuid].proc_ctls2;
  	return ((proc_ctls2 & PROCBASED2_VIRTUALIZE_APIC_ACCESSES) ? 1 : 0);
  }
  
  static __inline int
  x2apic_virtualization(struct vmx *vmx, int vcpuid)
  {
  	uint32_t proc_ctls2;
  
  	proc_ctls2 = vmx->cap[vcpuid].proc_ctls2;
  	return ((proc_ctls2 & PROCBASED2_VIRTUALIZE_X2APIC_MODE) ? 1 : 0);
  }
  
  static int
  vmx_handle_apic_write(struct vmx *vmx, int vcpuid, struct vlapic *vlapic,
      uint64_t qual)
  {
  	int error, handled, offset;
  	uint32_t *apic_regs, vector;
  	bool retu;
  
  	handled = HANDLED;
  	offset = APIC_WRITE_OFFSET(qual);
  
  	if (!apic_access_virtualization(vmx, vcpuid)) {
  		/*
  		 * In general there should not be any APIC write VM-exits
  		 * unless APIC-access virtualization is enabled.
  		 *
  		 * However self-IPI virtualization can legitimately trigger
  		 * an APIC-write VM-exit so treat it specially.
  		 */
  		if (x2apic_virtualization(vmx, vcpuid) &&
  		    offset == APIC_OFFSET_SELF_IPI) {
  			apic_regs = (uint32_t *)(vlapic->apic_page);
  			vector = apic_regs[APIC_OFFSET_SELF_IPI / 4];
  			vlapic_self_ipi_handler(vlapic, vector);
  			return (HANDLED);
  		} else
  			return (UNHANDLED);
  	}
  
  	switch (offset) {
  	case APIC_OFFSET_ID:
  		vlapic_id_write_handler(vlapic);
  		break;
  	case APIC_OFFSET_LDR:
  		vlapic_ldr_write_handler(vlapic);
  		break;
  	case APIC_OFFSET_DFR:
  		vlapic_dfr_write_handler(vlapic);
  		break;
  	case APIC_OFFSET_SVR:
  		vlapic_svr_write_handler(vlapic);
  		break;
  	case APIC_OFFSET_ESR:
  		vlapic_esr_write_handler(vlapic);
  		break;
  	case APIC_OFFSET_ICR_LOW:
  		retu = false;
  		error = vlapic_icrlo_write_handler(vlapic, &retu);
  		if (error != 0 || retu)
  			handled = UNHANDLED;
  		break;
  	case APIC_OFFSET_CMCI_LVT:
  	case APIC_OFFSET_TIMER_LVT:
  	case APIC_OFFSET_THERM_LVT:
  	case APIC_OFFSET_PERF_LVT:
  	case APIC_OFFSET_LINT0_LVT:
  	case APIC_OFFSET_LINT1_LVT:
  	case APIC_OFFSET_ERROR_LVT:
  		vlapic_lvt_write_handler(vlapic, ((uint32_t) offset));
  		break;
  	case APIC_OFFSET_TIMER_ICR:
  		vlapic_icrtmr_write_handler(vlapic);
  		break;
  	case APIC_OFFSET_TIMER_DCR:
  		vlapic_dcr_write_handler(vlapic);
  		break;
  	default:
  		handled = UNHANDLED;
  		break;
  	}
  	return (handled);
  }
  
  static bool
  apic_access_fault(struct vmx *vmx, int vcpuid, uint64_t gpa)
  {
  
  	if (apic_access_virtualization(vmx, vcpuid) &&
  	    (gpa >= DEFAULT_APIC_BASE && gpa < DEFAULT_APIC_BASE + XHYVE_PAGE_SIZE))
  		return (true);
  	else
  		return (false);
  }
  
  static int
  vmx_handle_apic_access(struct vmx *vmx, int vcpuid, struct vm_exit *vmexit)
  {
  	uint64_t qual;
  	int access_type, offset, allowed;
  
  	if (!apic_access_virtualization(vmx, vcpuid))
  		return (UNHANDLED);
  
  	qual = vmexit->u.vmx.exit_qualification;
  	access_type = APIC_ACCESS_TYPE(qual);
  	offset = APIC_ACCESS_OFFSET(qual);
  
  	allowed = 0;
  	if (access_type == 0) {
  		/*
  		 * Read data access to the following registers is expected.
  		 */
  		switch (offset) {
  		case APIC_OFFSET_APR:
  		case APIC_OFFSET_PPR:
  		case APIC_OFFSET_RRR:
  		case APIC_OFFSET_CMCI_LVT:
  		case APIC_OFFSET_TIMER_CCR:
  			allowed = 1;
  			break;
  		default:
  			break;
  		}
  	} else if (access_type == 1) {
  		/*
  		 * Write data access to the following registers is expected.
  		 */
  		switch (offset) {
  		case APIC_OFFSET_VER:
  		case APIC_OFFSET_APR:
  		case APIC_OFFSET_PPR:
  		case APIC_OFFSET_RRR:
  		case APIC_OFFSET_ISR0:
  		case APIC_OFFSET_ISR1:
  		case APIC_OFFSET_ISR2:
  		case APIC_OFFSET_ISR3:
  		case APIC_OFFSET_ISR4:
  		case APIC_OFFSET_ISR5:
  		case APIC_OFFSET_ISR6:
  		case APIC_OFFSET_ISR7:
  		case APIC_OFFSET_TMR0:
  		case APIC_OFFSET_TMR1:
  		case APIC_OFFSET_TMR2:
  		case APIC_OFFSET_TMR3:
  		case APIC_OFFSET_TMR4:
  		case APIC_OFFSET_TMR5:
  		case APIC_OFFSET_TMR6:
  		case APIC_OFFSET_TMR7:
  		case APIC_OFFSET_IRR0:
  		case APIC_OFFSET_IRR1:
  		case APIC_OFFSET_IRR2:
  		case APIC_OFFSET_IRR3:
  		case APIC_OFFSET_IRR4:
  		case APIC_OFFSET_IRR5:
  		case APIC_OFFSET_IRR6:
  		case APIC_OFFSET_IRR7:
  		case APIC_OFFSET_CMCI_LVT:
  		case APIC_OFFSET_TIMER_CCR:
  			allowed = 1;
  			break;
  		default:
  			break;
  		}
  	}
  
  	if (allowed) {
  		vmexit_inst_emul(vmexit, DEFAULT_APIC_BASE + ((uint32_t) offset),
  		    VIE_INVALID_GLA, vcpuid);
  	}
  
  	/*
  	 * Regardless of whether the APIC-access is allowed this handler
  	 * always returns UNHANDLED:
  	 * - if the access is allowed then it is handled by emulating the
  	 *   instruction that caused the VM-exit (outside the critical section)
  	 * - if the access is not allowed then it will be converted to an
  	 *   exitcode of VM_EXITCODE_VMX and will be dealt with in userland.
  	 */
  	return (UNHANDLED);
  }
  
  static enum task_switch_reason
  vmx_task_switch_reason(uint64_t qual)
  {
  	int reason;
  
  	reason = (qual >> 30) & 0x3;
  	switch (reason) {
  	case 0:
  		return (TSR_CALL);
  	case 1:
  		return (TSR_IRET);
  	case 2:
  		return (TSR_JMP);
  	case 3:
  		return (TSR_IDT_GATE);
  	default:
  		xhyve_abort("%s: invalid reason %d", __func__, reason);
  	}
  }
  
  static int
  emulate_wrmsr(struct vmx *vmx, int vcpuid, u_int num, uint64_t val, bool *retu)
  {
  	int error;
  
  	if (lapic_msr(num))
  		error = lapic_wrmsr(vmx->vm, vcpuid, num, val, retu);
  	else
  		error = vmx_wrmsr(vmx, vcpuid, num, val);
  
  	return (error);
  }
  
  static int
  emulate_rdmsr(struct vmx *vmx, int vcpuid, u_int num, bool *retu)
  {
  	uint64_t result;
  	uint32_t eax, edx;
  	int error;
  
  	if (lapic_msr(num))
  		error = lapic_rdmsr(vmx->vm, vcpuid, num, &result, retu);
  	else
  		error = vmx_rdmsr(vmx, vcpuid, num, &result);
  
  	if (error == 0) {
  		eax = (uint32_t) result;
  		reg_write(vcpuid, HV_X86_RAX, eax);
  		edx = (uint32_t) (result >> 32);
  		reg_write(vcpuid, HV_X86_RDX, edx);
  	}
  
  	return (error);
  }
  
  static int
  vmx_exit_process(struct vmx *vmx, int vcpu, struct vm_exit *vmexit)
  {
  	int error, errcode, errcode_valid, handled, in;
  	struct vlapic *vlapic;
  	struct vm_inout_str *vis;
  	struct vm_task_switch *ts;
  	uint32_t eax, ecx, edx, idtvec_info, idtvec_err, intr_info, inst_info;
  	uint32_t intr_type, intr_vec, reason;
  	uint64_t exitintinfo, qual, gpa;
  	bool retu;
  
  	CTASSERT((PINBASED_CTLS_ONE_SETTING & PINBASED_VIRTUAL_NMI) != 0);
  	CTASSERT((PINBASED_CTLS_ONE_SETTING & PINBASED_NMI_EXITING) != 0);
  
  	handled = UNHANDLED;
  
  	qual = vmexit->u.vmx.exit_qualification;
  	reason = vmexit->u.vmx.exit_reason;
  	vmexit->exitcode = VM_EXITCODE_BOGUS;
  
  	vmm_stat_incr(vmx->vm, vcpu, VMEXIT_COUNT, 1);
  
  	/*
  	 * VM exits that can be triggered during event delivery need to
  	 * be handled specially by re-injecting the event if the IDT
  	 * vectoring information field's valid bit is set.
  	 *
  	 * See "Information for VM Exits During Event Delivery" in Intel SDM
  	 * for details.
  	 */
  	idtvec_info = (uint32_t) vmcs_idt_vectoring_info(vcpu);
  	if (idtvec_info & VMCS_IDT_VEC_VALID) {
  		idtvec_info &= ~(1u << 12); /* clear undefined bit */
  		exitintinfo = idtvec_info;
  		if (idtvec_info & VMCS_IDT_VEC_ERRCODE_VALID) {
  			idtvec_err = (uint32_t) vmcs_idt_vectoring_err(vcpu);
  			exitintinfo |= (uint64_t)idtvec_err << 32;
  		}
  		error = vm_exit_intinfo(vmx->vm, vcpu, exitintinfo);
  		KASSERT(error == 0, ("%s: vm_set_intinfo error %d",
  		    __func__, error));
  
  		/*
  		 * If 'virtual NMIs' are being used and the VM-exit
  		 * happened while injecting an NMI during the previous
  		 * VM-entry, then clear "blocking by NMI" in the
  		 * Guest Interruptibility-State so the NMI can be
  		 * reinjected on the subsequent VM-entry.
  		 *
  		 * However, if the NMI was being delivered through a task
  		 * gate, then the new task must start execution with NMIs
  		 * blocked so don't clear NMI blocking in this case.
  		 */
  		intr_type = idtvec_info & VMCS_INTR_T_MASK;
  		if (intr_type == VMCS_INTR_T_NMI) {
  			if (reason != EXIT_REASON_TASK_SWITCH)
  				vmx_clear_nmi_blocking(vmx, vcpu);
  			else
  				vmx_assert_nmi_blocking(vcpu);
  		}
  
  		/*
  		 * Update VM-entry instruction length if the event being
  		 * delivered was a software interrupt or software exception.
  		 */
  		if (intr_type == VMCS_INTR_T_SWINTR ||
  		    intr_type == VMCS_INTR_T_PRIV_SWEXCEPTION ||
  		    intr_type == VMCS_INTR_T_SWEXCEPTION) {
  			vmcs_write(vcpu, VMCS_ENTRY_INST_LENGTH,
  				((uint64_t) vmexit->inst_length));
  		}
  	}
  
  	switch (reason) {
  	case EXIT_REASON_TASK_SWITCH:
  		ts = &vmexit->u.task_switch;
  		ts->tsssel = qual & 0xffff;
  		ts->reason = vmx_task_switch_reason(qual);
  		ts->ext = 0;
  		ts->errcode_valid = 0;
  		vmx_paging_info(&ts->paging, vcpu);
  		/*
  		 * If the task switch was due to a CALL, JMP, IRET, software
  		 * interrupt (INT n) or software exception (INT3, INTO),
  		 * then the saved %rip references the instruction that caused
  		 * the task switch. The instruction length field in the VMCS
  		 * is valid in this case.
  		 *
  		 * In all other cases (e.g., NMI, hardware exception) the
  		 * saved %rip is one that would have been saved in the old TSS
  		 * had the task switch completed normally so the instruction
  		 * length field is not needed in this case and is explicitly
  		 * set to 0.
  		 */
  		if (ts->reason == TSR_IDT_GATE) {
  			KASSERT(idtvec_info & VMCS_IDT_VEC_VALID,
  			    ("invalid idtvec_info %#x for IDT task switch",
  			    idtvec_info));
  			intr_type = idtvec_info & VMCS_INTR_T_MASK;
  			if (intr_type != VMCS_INTR_T_SWINTR &&
  			    intr_type != VMCS_INTR_T_SWEXCEPTION &&
  			    intr_type != VMCS_INTR_T_PRIV_SWEXCEPTION) {
  				/* Task switch triggered by external event */
  				ts->ext = 1;
  				vmexit->inst_length = 0;
  				if (idtvec_info & VMCS_IDT_VEC_ERRCODE_VALID) {
  					ts->errcode_valid = 1;
  					ts->errcode = (uint32_t) vmcs_idt_vectoring_err(vcpu);
  				}
  			}
  		}
  		vmexit->exitcode = VM_EXITCODE_TASK_SWITCH;
  		VCPU_CTR4(vmx->vm, vcpu, "task switch reason %d, tss 0x%04x, "
  		    "%s errcode 0x%016llx", ts->reason, ts->tsssel,
  		    ts->ext ? "external" : "internal",
  		    ((uint64_t)ts->errcode << 32) | ((uint64_t) ts->errcode_valid));
  		break;
  	case EXIT_REASON_CR_ACCESS:
  		vmm_stat_incr(vmx->vm, vcpu, VMEXIT_CR_ACCESS, 1);
  		switch (qual & 0xf) {
  		case 0:
  			handled = vmx_emulate_cr0_access(vmx->vm, vcpu, qual);
  			break;
  		case 4:
  			handled = vmx_emulate_cr4_access(vcpu, qual);
  			break;
  		case 8:
  			handled = vmx_emulate_cr8_access(vmx, vcpu, qual);
  			break;
  		}
  		break;
  	case EXIT_REASON_RDMSR:
  		vmm_stat_incr(vmx->vm, vcpu, VMEXIT_RDMSR, 1);
  		retu = false;
  		ecx = (uint32_t) reg_read(vcpu, HV_X86_RCX);
  		VCPU_CTR1(vmx->vm, vcpu, "rdmsr 0x%08x", ecx);
  		// printf("EXIT_REASON_RDMSR 0x%08x\n", ecx);
  		error = emulate_rdmsr(vmx, vcpu, ecx, &retu);
  		if (error) {
  			vmexit->exitcode = VM_EXITCODE_RDMSR;
  			vmexit->u.msr.code = ecx;
  		} else if (!retu) {
  			handled = HANDLED;
  		} else {
  			/* Return to userspace with a valid exitcode */
  			KASSERT(vmexit->exitcode != VM_EXITCODE_BOGUS,
  			    ("emulate_rdmsr retu with bogus exitcode"));
  		}
  		break;
  	case EXIT_REASON_WRMSR:
  		vmm_stat_incr(vmx->vm, vcpu, VMEXIT_WRMSR, 1);
  		retu = false;
  		eax = (uint32_t) reg_read(vcpu, HV_X86_RAX);
  		ecx = (uint32_t) reg_read(vcpu, HV_X86_RCX);
  		edx = (uint32_t) reg_read(vcpu, HV_X86_RDX);
  		VCPU_CTR2(vmx->vm, vcpu, "wrmsr 0x%08x value 0x%016llx",
  		    ecx, (uint64_t)edx << 32 | eax);
  		// printf("EXIT_REASON_WRMSR 0x%08x value 0x%016llx\n",
  		//     ecx, (uint64_t)edx << 32 | eax);
  		error = emulate_wrmsr(vmx, vcpu, ecx,
  		    (uint64_t)edx << 32 | eax, &retu);
  		if (error) {
  			vmexit->exitcode = VM_EXITCODE_WRMSR;
  			vmexit->u.msr.code = ecx;
  			vmexit->u.msr.wval = (uint64_t)edx << 32 | eax;
  		} else if (!retu) {
  			handled = HANDLED;
  		} else {
  			/* Return to userspace with a valid exitcode */
  			KASSERT(vmexit->exitcode != VM_EXITCODE_BOGUS,
  			    ("emulate_wrmsr retu with bogus exitcode"));
  		}
  		break;
  	case EXIT_REASON_HLT:
  		vmm_stat_incr(vmx->vm, vcpu, VMEXIT_HLT, 1);
  		vmexit->exitcode = VM_EXITCODE_HLT;
  		vmexit->u.hlt.rflags = vmcs_read(vcpu, VMCS_GUEST_RFLAGS);
  		break;
  	case EXIT_REASON_MTF:
  		vmm_stat_incr(vmx->vm, vcpu, VMEXIT_MTRAP, 1);
  		vmexit->exitcode = VM_EXITCODE_MTRAP;
  		vmexit->inst_length = 0;
  		break;
  	case EXIT_REASON_PAUSE:
  		vmm_stat_incr(vmx->vm, vcpu, VMEXIT_PAUSE, 1);
  		vmexit->exitcode = VM_EXITCODE_PAUSE;
  		break;
  	case EXIT_REASON_INTR_WINDOW:
  		vmm_stat_incr(vmx->vm, vcpu, VMEXIT_INTR_WINDOW, 1);
  		vmx_clear_int_window_exiting(vmx, vcpu);
  		return (1);
  	case EXIT_REASON_EXT_INTR:
  		/*
  		 * External interrupts serve only to cause VM exits and allow
  		 * the host interrupt handler to run.
  		 *
  		 * If this external interrupt triggers a virtual interrupt
  		 * to a VM, then that state will be recorded by the
  		 * host interrupt handler in the VM's softc. We will inject
  		 * this virtual interrupt during the subsequent VM enter.
  		 */
  		intr_info = (uint32_t) vmcs_read(vcpu, VMCS_EXIT_INTR_INFO);
  
  		/*
  		 * XXX: Ignore this exit if VMCS_INTR_VALID is not set.
  		 * This appears to be a bug in VMware Fusion?
  		 */
  		if (!(intr_info & VMCS_INTR_VALID))
  			return (1);
  		KASSERT((intr_info & VMCS_INTR_VALID) != 0 &&
  		    (intr_info & VMCS_INTR_T_MASK) == VMCS_INTR_T_HWINTR,
  		    ("VM exit interruption info invalid: %#x", intr_info));
  
  		/*
  		 * This is special. We want to treat this as an 'handled'
  		 * VM-exit but not increment the instruction pointer.
  		 */
  		vmm_stat_incr(vmx->vm, vcpu, VMEXIT_EXTINT, 1);
  		return (1);
  	case EXIT_REASON_NMI_WINDOW:
  		/* Exit to allow the pending virtual NMI to be injected */
  		if (vm_nmi_pending(vmx->vm, vcpu))
  			vmx_inject_nmi(vmx, vcpu);
  		vmx_clear_nmi_window_exiting(vmx, vcpu);
  		vmm_stat_incr(vmx->vm, vcpu, VMEXIT_NMI_WINDOW, 1);
  		return (1);
  	case EXIT_REASON_INOUT:
  		vmm_stat_incr(vmx->vm, vcpu, VMEXIT_INOUT, 1);
  		vmexit->exitcode = VM_EXITCODE_INOUT;
  		vmexit->u.inout.bytes = (qual & 0x7) + 1;
  		vmexit->u.inout.in = in = (qual & 0x8) ? 1 : 0;
  		vmexit->u.inout.string = (qual & 0x10) ? 1 : 0;
  		vmexit->u.inout.rep = (qual & 0x20) ? 1 : 0;
  		vmexit->u.inout.port = (uint16_t)(qual >> 16);
  		vmexit->u.inout.eax = (uint32_t) reg_read(vcpu, HV_X86_RAX);
  		// if ((vmexit->u.inout.port == 0x0020) ||
  		// 	(vmexit->u.inout.port == 0x0021) ||
  		// 	(vmexit->u.inout.port == 0x00a0) ||
  		// 	(vmexit->u.inout.port == 0x00a1))
  		// {
  		// 	printf("EXIT_REASON_INOUT port 0x%03x in %d\n",
  		// 		vmexit->u.inout.port, vmexit->u.inout.in);
  		// }
  		if (vmexit->u.inout.string) {
  			inst_info = (uint32_t) vmcs_read(vcpu, VMCS_EXIT_INSTRUCTION_INFO);
  			vmexit->exitcode = VM_EXITCODE_INOUT_STR;
  			vis = &vmexit->u.inout_str;
  			vmx_paging_info(&vis->paging, vcpu);
  			vis->rflags = vmcs_read(vcpu, VMCS_GUEST_RFLAGS);
  			vis->cr0 = vmcs_read(vcpu, VMCS_GUEST_CR0);
  			vis->index = inout_str_index(vmx, vcpu, in);
  			vis->count = inout_str_count(vmx, vcpu, vis->inout.rep);
  			vis->addrsize = inout_str_addrsize(inst_info);
  			inout_str_seginfo(vmx, vcpu, inst_info, in, vis);
  		}
  		break;
  	case EXIT_REASON_CPUID:
  		vmm_stat_incr(vmx->vm, vcpu, VMEXIT_CPUID, 1);
  		handled = vmx_handle_cpuid(vmx->vm, vcpu);
  		break;
  	case EXIT_REASON_EXCEPTION:
  		vmm_stat_incr(vmx->vm, vcpu, VMEXIT_EXCEPTION, 1);
  		intr_info = (uint32_t) vmcs_read(vcpu, VMCS_EXIT_INTR_INFO);
  		KASSERT((intr_info & VMCS_INTR_VALID) != 0,
  		    ("VM exit interruption info invalid: %#x", intr_info));
  
  		intr_vec = intr_info & 0xff;
  		intr_type = intr_info & VMCS_INTR_T_MASK;
  
  		/*
  		 * If Virtual NMIs control is 1 and the VM-exit is due to a
  		 * fault encountered during the execution of IRET then we must
  		 * restore the state of "virtual-NMI blocking" before resuming
  		 * the guest.
  		 *
  		 * See "Resuming Guest Software after Handling an Exception".
  		 * See "Information for VM Exits Due to Vectored Events".
  		 */
  		if ((idtvec_info & VMCS_IDT_VEC_VALID) == 0 &&
  		    (intr_vec != IDT_DF) &&
  		    (intr_info & EXIT_QUAL_NMIUDTI) != 0)
  			vmx_restore_nmi_blocking(vmx, vcpu);
  
  		/*
  		 * The NMI has already been handled in vmx_exit_handle_nmi().
  		 */
  		if (intr_type == VMCS_INTR_T_NMI)
  			return (1);
  
  		if (intr_vec == IDT_PF) {
  			reg_write(vcpu, HV_X86_CR2, qual);
  		}
  
  		/*
  		 * Software exceptions exhibit trap-like behavior. This in
  		 * turn requires populating the VM-entry instruction length
  		 * so that the %rip in the trap frame is past the INT3/INTO
  		 * instruction.
  		 */
  		if (intr_type == VMCS_INTR_T_SWEXCEPTION)
  			vmcs_write(vcpu, VMCS_ENTRY_INST_LENGTH,
  				((uint64_t) vmexit->inst_length));
  
  		/* Reflect all other exceptions back into the guest */
  		errcode_valid = errcode = 0;
  		if (intr_info & VMCS_INTR_DEL_ERRCODE) {
  			errcode_valid = 1;
  			errcode = (int) vmcs_read(vcpu, VMCS_EXIT_INTR_ERRCODE);
  		}
  		VCPU_CTR2(vmx->vm, vcpu, "Reflecting exception %d/%#x into "
  		    "the guest", intr_vec, errcode);
  		error = vm_inject_exception(vmx->vm, vcpu, ((int) intr_vec),
  		    errcode_valid, ((uint32_t) errcode), 0);
  		KASSERT(error == 0, ("%s: vm_inject_exception error %d",
  		    __func__, error));
  		return (1);
  
  	case EXIT_REASON_EPT_FAULT:
  		/*
  		 * If 'gpa' lies within the address space allocated to
  		 * memory then this must be a nested page fault otherwise
  		 * this must be an instruction that accesses MMIO space.
  		 */
  		gpa = vmcs_gpa(vcpu);
  		if (vm_mem_allocated(vmx->vm, gpa) ||
  		    apic_access_fault(vmx, vcpu, gpa)) {
  			vmexit->exitcode = VM_EXITCODE_PAGING;
  			vmexit->inst_length = 0;
  			vmexit->u.paging.gpa = gpa;
  			vmexit->u.paging.fault_type = ept_fault_type(qual);
  			vmm_stat_incr(vmx->vm, vcpu, VMEXIT_NESTED_FAULT, 1);
  		} else if (ept_emulation_fault(qual)) {
  			vmexit_inst_emul(vmexit, gpa, vmcs_gla(vcpu), vcpu);
  			vmm_stat_incr(vmx->vm, vcpu, VMEXIT_INST_EMUL, 1);
  		}
  		/*
  		 * If Virtual NMIs control is 1 and the VM-exit is due to an
  		 * EPT fault during the execution of IRET then we must restore
  		 * the state of "virtual-NMI blocking" before resuming.
  		 *
  		 * See description of "NMI unblocking due to IRET" in
  		 * "Exit Qualification for EPT Violations".
  		 */
  		if ((idtvec_info & VMCS_IDT_VEC_VALID) == 0 &&
  		    (qual & EXIT_QUAL_NMIUDTI) != 0)
  			vmx_restore_nmi_blocking(vmx, vcpu);
  		break;
  	case EXIT_REASON_VIRTUALIZED_EOI:
  		vmexit->exitcode = VM_EXITCODE_IOAPIC_EOI;
  		vmexit->u.ioapic_eoi.vector = qual & 0xFF;
  		vmexit->inst_length = 0;	/* trap-like */
  		break;
  	case EXIT_REASON_APIC_ACCESS:
  		handled = vmx_handle_apic_access(vmx, vcpu, vmexit);
  		break;
  	case EXIT_REASON_APIC_WRITE:
  		/*
  		 * APIC-write VM exit is trap-like so the %rip is already
  		 * pointing to the next instruction.
  		 */
  		vmexit->inst_length = 0;
  		vlapic = vm_lapic(vmx->vm, vcpu);
  		handled = vmx_handle_apic_write(vmx, vcpu, vlapic, qual);
  		break;
  	case EXIT_REASON_XSETBV:
  		handled = vmx_emulate_xsetbv(vmx, vcpu);
  		break;
  	case EXIT_REASON_MONITOR:
  		vmexit->exitcode = VM_EXITCODE_MONITOR;
  		break;
  	case EXIT_REASON_MWAIT:
  		vmexit->exitcode = VM_EXITCODE_MWAIT;
  		break;
  	default:
  		vmm_stat_incr(vmx->vm, vcpu, VMEXIT_UNKNOWN, 1);
  		break;
  	}
  
  	if (handled) {
  		/*
  		 * It is possible that control is returned to userland
  		 * even though we were able to handle the VM exit in the
  		 * kernel.
  		 *
  		 * In such a case we want to make sure that the userland
  		 * restarts guest execution at the instruction *after*
  		 * the one we just processed. Therefore we update the
  		 * guest rip in the VMCS and in 'vmexit'.
  		 */
  		vmexit->rip += (uint64_t) vmexit->inst_length;
  		vmexit->inst_length = 0;
  		vmcs_write(vcpu, VMCS_GUEST_RIP, vmexit->rip);
  	} else {
  		if (vmexit->exitcode == VM_EXITCODE_BOGUS) {
  			/*
  			 * If this VM exit was not claimed by anybody then
  			 * treat it as a generic VMX exit.
  			 */
  			vmexit->exitcode = VM_EXITCODE_VMX;
  			vmexit->u.vmx.status = VM_SUCCESS;
  			vmexit->u.vmx.inst_type = 0;
  			vmexit->u.vmx.inst_error = 0;
  		} else {
  			/*
  			 * The exitcode and collateral have been populated.
  			 * The VM exit will be processed further in userland.
  			 */
  		}
  	}
  	return (handled);
  }
  
  static int
  vmx_run(void *arg, int vcpu, register_t rip, void *rendezvous_cookie,
  	void *suspend_cookie)
  {
  	int handled;
  	struct vmx *vmx;
  	struct vm *vm;
  	struct vm_exit *vmexit;
  	struct vlapic *vlapic;
  	uint32_t exit_reason;
  	hv_return_t hvr;
  
  	vmx = arg;
  	vm = vmx->vm;
  	vlapic = vm_lapic(vm, vcpu);
  	vmexit = vm_exitinfo(vm, vcpu);
  
  	vmcs_write(vcpu, VMCS_GUEST_RIP, ((uint64_t) rip));
  
  	do {
  		KASSERT(vmcs_guest_rip(vcpu) == ((uint64_t) rip),
  			("%s: vmcs guest rip mismatch %#llx/%#llx",
  				__func__, vmcs_guest_rip(vcpu), ((uint64_t) rip)));
  
  		handled = UNHANDLED;
  
  		vmx_inject_interrupts(vmx, vcpu, vlapic, ((uint64_t) rip));
  
  		/*
  		 * Check for vcpu suspension after injecting events because
  		 * vmx_inject_interrupts() can suspend the vcpu due to a
  		 * triple fault.
  		 */
  		if (vcpu_suspended(suspend_cookie)) {
  			vm_exit_suspended(vmx->vm, vcpu, ((uint64_t) rip));
  			break;
  		}
  
  		if (vcpu_rendezvous_pending(rendezvous_cookie)) {
  			vm_exit_rendezvous(vmx->vm, vcpu, ((uint64_t) rip));
  			break;
  		}
  
  		vmx_run_trace(vmx, vcpu);
  		hvr = hv_vcpu_run((hv_vcpuid_t) vcpu);
  		/* Collect some information for VM exit processing */
  		rip = (register_t) vmcs_guest_rip(vcpu);
  		vmexit->rip = (uint64_t) rip;
  		vmexit->inst_length = (int) vmexit_instruction_length(vcpu);
  		vmexit->u.vmx.exit_reason = exit_reason = vmcs_exit_reason(vcpu);
  		vmexit->u.vmx.exit_qualification = vmcs_exit_qualification(vcpu);
  		/* Update 'nextrip' */
  		vmx->state[vcpu].nextrip = (uint64_t) rip;
  		if (hvr == HV_SUCCESS) {
  			handled = vmx_exit_process(vmx, vcpu, vmexit);
  		} else {
  			hvdump(vcpu);
  			xhyve_abort("vmentry error\n");
  		}
  		vmx_exit_trace(vmx, vcpu, ((uint64_t) rip), exit_reason, handled);
  		rip = (register_t) vmexit->rip;
  	} while (handled);
  
  	/*
  	 * If a VM exit has been handled then the exitcode must be BOGUS
  	 * If a VM exit is not handled then the exitcode must not be BOGUS
  	 */
  	if ((handled && vmexit->exitcode != VM_EXITCODE_BOGUS) ||
  	    (!handled && vmexit->exitcode == VM_EXITCODE_BOGUS)) {
  		xhyve_abort("Mismatch between handled (%d) and exitcode (%d)",
  		      handled, vmexit->exitcode);
  	}
  
  	if (!handled)
  		vmm_stat_incr(vm, vcpu, VMEXIT_USERSPACE, 1);
  
  	VCPU_CTR1(vm, vcpu, "returning from vmx_run: exitcode %d",
  	    vmexit->exitcode);
  
  	return (0);
  }
  
  static void
  vmx_vm_cleanup(void *arg)
  {
  	struct vmx *vmx = arg;
  
  	free(vmx);
  
  	return;
  }
  
  static void
  vmx_vcpu_cleanup(void *arg, int vcpuid) {
  	if (arg || vcpuid) xhyve_abort("vmx_vcpu_cleanup\n");
  }
  
  
  static int
  vmx_get_intr_shadow(int vcpu, uint64_t *retval)
  {
  	uint64_t gi;
  	int error;
  
  	error = vmcs_getreg(vcpu, VMCS_IDENT(VMCS_GUEST_INTERRUPTIBILITY), &gi);
  	*retval = (gi & HWINTR_BLOCKING) ? 1 : 0;
  	return (error);
  }
  
  static int
  vmx_modify_intr_shadow(struct vmx *vmx, int vcpu, uint64_t val)
  {
  	uint64_t gi;
  	int error, ident;
  
  	/*
  	 * Forcing the vcpu into an interrupt shadow is not supported.
  	 */
  	if (val) {
  		error = EINVAL;
  		goto done;
  	}
  
  	ident = VMCS_IDENT(VMCS_GUEST_INTERRUPTIBILITY);
  	error = vmcs_getreg(vcpu, ident, &gi);
  	if (error == 0) {
  		gi &= ~HWINTR_BLOCKING;
  		error = vmcs_setreg(vcpu, ident, gi);
  	}
  done:
  	VCPU_CTR2(vmx->vm, vcpu, "Setting intr_shadow to %#llx %s", val,
  	    error ? "failed" : "succeeded");
  	return (error);
  }
  
  static int
  vmx_shadow_reg(int reg)
  {
  	int shreg;
  
  	shreg = -1;
  
  	switch (reg) {
  	case VM_REG_GUEST_CR0:
  		shreg = VMCS_CR0_SHADOW;
                  break;
          case VM_REG_GUEST_CR4:
  		shreg = VMCS_CR4_SHADOW;
  		break;
  	default:
  		break;
  	}
  
  	return (shreg);
  }
  
  static const hv_x86_reg_t hvregs[] = {
  	HV_X86_RAX,
  	HV_X86_RBX,
  	HV_X86_RCX,
  	HV_X86_RDX,
  	HV_X86_RSI,
  	HV_X86_RDI,
  	HV_X86_RBP,
  	HV_X86_R8,
  	HV_X86_R9,
  	HV_X86_R10,
  	HV_X86_R11,
  	HV_X86_R12,
  	HV_X86_R13,
  	HV_X86_R14,
  	HV_X86_R15,
  	HV_X86_REGISTERS_MAX,
  	HV_X86_REGISTERS_MAX,
  	HV_X86_REGISTERS_MAX,
  	HV_X86_REGISTERS_MAX,
  	HV_X86_REGISTERS_MAX,
  	HV_X86_REGISTERS_MAX,
  	HV_X86_REGISTERS_MAX,
  	HV_X86_REGISTERS_MAX,
  	HV_X86_REGISTERS_MAX,
  	HV_X86_REGISTERS_MAX,
  	HV_X86_REGISTERS_MAX,
  	HV_X86_REGISTERS_MAX,
  	HV_X86_REGISTERS_MAX,
  	HV_X86_REGISTERS_MAX,
  	HV_X86_REGISTERS_MAX,
  	HV_X86_REGISTERS_MAX,
  	HV_X86_REGISTERS_MAX,
  	HV_X86_REGISTERS_MAX,
  	HV_X86_CR2,
  	HV_X86_REGISTERS_MAX,
  	HV_X86_REGISTERS_MAX,
  	HV_X86_REGISTERS_MAX,
  	HV_X86_REGISTERS_MAX,
  	HV_X86_REGISTERS_MAX
  };
  
  static int
  vmx_getreg(UNUSED void *arg, int vcpu, int reg, uint64_t *retval)
  {
  	hv_x86_reg_t hvreg;
  
  	if (reg == VM_REG_GUEST_INTR_SHADOW)
  		return (vmx_get_intr_shadow(vcpu, retval));
  
  	hvreg = hvregs[reg];
  	if (hvreg != HV_X86_REGISTERS_MAX) {
  		*retval = reg_read(vcpu, hvreg);
  		return (0);
  	}
  
  	return (vmcs_getreg(vcpu, reg, retval));
  }
  
  static int
  vmx_setreg(void *arg, int vcpu, int reg, uint64_t val)
  {
  	int error, shadow;
  	uint64_t ctls;
  	hv_x86_reg_t hvreg;
  	struct vmx *vmx = arg;
  
  	if (reg == VM_REG_GUEST_INTR_SHADOW)
  		return (vmx_modify_intr_shadow(vmx, vcpu, val));
  
  	hvreg = hvregs[reg];
  	if (hvreg != HV_X86_REGISTERS_MAX) {
  		reg_write(vcpu, hvreg, val);
  		return (0);
  	}
  
  	error = vmcs_setreg(vcpu, reg, val);
  
  	if (error == 0) {
  		/*
  		 * If the "load EFER" VM-entry control is 1 then the
  		 * value of EFER.LMA must be identical to "IA-32e mode guest"
  		 * bit in the VM-entry control.
  		 */
  		if ((vmx->state[vcpu].entry_ctls & VM_ENTRY_LOAD_EFER) != 0 &&
  			(reg == VM_REG_GUEST_EFER)) {
  			vmcs_getreg(vcpu, VMCS_IDENT(VMCS_ENTRY_CTLS), &ctls);
  			if (val & EFER_LMA)
  				ctls |= VM_ENTRY_GUEST_LMA;
  			else
  				ctls &= ~VM_ENTRY_GUEST_LMA;
  			vmcs_setreg(vcpu, VMCS_IDENT(VMCS_ENTRY_CTLS), ctls);
  		}
  
  		shadow = vmx_shadow_reg(reg);
  		if (shadow > 0) {
  			/*
  			 * Store the unmodified value in the shadow
  			 */			
  			error = vmcs_setreg(vcpu, VMCS_IDENT(shadow), val);
  		}
  
  		if (reg == VM_REG_GUEST_CR3) {
  			/*
  			 * Invalidate the guest vcpu's TLB mappings to emulate
  			 * the behavior of updating %cr3.
  			 */
  			hv_vcpu_invalidate_tlb((hv_vcpuid_t) vcpu);
  		}
  	}
  
  	return (error);
  }
  
  static int
  vmx_getdesc(UNUSED void *arg, int vcpu, int reg, struct seg_desc *desc)
  {
  	return (vmcs_getdesc(vcpu, reg, desc));
  }
  
  static int
  vmx_setdesc(UNUSED void *arg, int vcpu, int reg, struct seg_desc *desc)
  {
  	return (vmcs_setdesc(vcpu, reg, desc));
  }
  
  static int
  vmx_getcap(void *arg, int vcpu, int type, int *retval)
  {
  	struct vmx *vmx = arg;
  	int vcap;
  	int ret;
  
  	ret = ENOENT;
  
  	vcap = vmx->cap[vcpu].set;
  
  	switch (type) {
  	case VM_CAP_HALT_EXIT:
  		if (cap_halt_exit)
  			ret = 0;
  		break;
  	case VM_CAP_PAUSE_EXIT:
  		if (cap_pause_exit)
  			ret = 0;
  		break;
  	case VM_CAP_MTRAP_EXIT:
  		if (cap_monitor_trap)
  			ret = 0;
  		break;
  	default:
  		break;
  	}
  
  	if (ret == 0)
  		*retval = (vcap & (1 << type)) ? 1 : 0;
  
  	return (ret);
  }
  
  static int
  vmx_setcap(void *arg, int vcpu, int type, int val)
  {
  	struct vmx *vmx = arg;
  	uint32_t baseval;
  	uint32_t *pptr;
  	uint32_t reg;
  	uint32_t flag;
  	int retval;
  
  	retval = ENOENT;
  	pptr = NULL;
  	baseval = 0;
  	reg = 0;
  	flag = 0;
  
  	switch (type) {
  	case VM_CAP_HALT_EXIT:
  		if (cap_halt_exit) {
  			retval = 0;
  			pptr = &vmx->cap[vcpu].proc_ctls;
  			baseval = *pptr;
  			flag = PROCBASED_HLT_EXITING;
  			reg = VMCS_PRI_PROC_BASED_CTLS;
  		}
  		break;
  	case VM_CAP_MTRAP_EXIT:
  		if (cap_monitor_trap) {
  			retval = 0;
  			pptr = &vmx->cap[vcpu].proc_ctls;
  			baseval = *pptr;
  			flag = PROCBASED_MTF;
  			reg = VMCS_PRI_PROC_BASED_CTLS;
  		}
  		break;
  	case VM_CAP_PAUSE_EXIT:
  		if (cap_pause_exit) {
  			retval = 0;
  			pptr = &vmx->cap[vcpu].proc_ctls;
  			baseval = *pptr;
  			flag = PROCBASED_PAUSE_EXITING;
  			reg = VMCS_PRI_PROC_BASED_CTLS;
  		}
  		break;
  	default:
  		xhyve_abort("vmx_setcap\n");
  	}
  
  	if (retval == 0) {
  		if (val) {
  			baseval |= flag;
  		} else {
  			baseval &= ~flag;
  		}
  
  		vmcs_write(vcpu, reg, baseval);
  
  		/*
  		 * Update optional stored flags, and record
  		 * setting
  		 */
  		if (pptr != NULL) {
  			*pptr = baseval;
  		}
  
  		if (val) {
  			vmx->cap[vcpu].set |= (1 << type);
  		} else {
  			vmx->cap[vcpu].set &= ~(1 << type);
  		}
  
  	}
  
  	return (retval);
  }
  
  struct vlapic_vtx {
  	struct vlapic vlapic;
  	struct pir_desc *pir_desc;
  	struct vmx *vmx;
  };
  
  // #define	VMX_CTR_PIR(vm, vcpuid, pir_desc, notify, vector, level, msg)	\
  // do {									\
  // 	VCPU_CTR2(vm, vcpuid, msg " assert %s-triggered vector %d",	\
  // 	    level ? "level" : "edge", vector);				\
  // 	VCPU_CTR1(vm, vcpuid, msg " pir0 0x%016lx", pir_desc->pir[0]);	\
  // 	VCPU_CTR1(vm, vcpuid, msg " pir1 0x%016lx", pir_desc->pir[1]);	\
  // 	VCPU_CTR1(vm, vcpuid, msg " pir2 0x%016lx", pir_desc->pir[2]);	\
  // 	VCPU_CTR1(vm, vcpuid, msg " pir3 0x%016lx", pir_desc->pir[3]);	\
  // 	VCPU_CTR1(vm, vcpuid, msg " notify: %s", notify ? "yes" : "no");\
  // } while (0)
  
  // /*
  //  * vlapic->ops handlers that utilize the APICv hardware assist described in
  //  * Chapter 29 of the Intel SDM.
  //  */
  // static int
  // vmx_set_intr_ready(struct vlapic *vlapic, int vector, bool level)
  // {
  // 	struct vlapic_vtx *vlapic_vtx;
  // 	struct pir_desc *pir_desc;
  // 	uint64_t mask;
  // 	int idx, notify;
  
  // 	vlapic_vtx = (struct vlapic_vtx *)vlapic;
  // 	pir_desc = vlapic_vtx->pir_desc;
  
  // 	/*
  // 	 * Keep track of interrupt requests in the PIR descriptor. This is
  // 	 * because the virtual APIC page pointed to by the VMCS cannot be
  // 	 * modified if the vcpu is running.
  // 	 */
  // 	idx = vector / 64;
  // 	mask = 1UL << (vector % 64);
  // 	atomic_set_long(&pir_desc->pir[idx], mask);
  // 	notify = atomic_cmpset_long(&pir_desc->pending, 0, 1);
  
  // 	VMX_CTR_PIR(vlapic->vm, vlapic->vcpuid, pir_desc, notify, vector,
  // 	    level, "vmx_set_intr_ready");
  // 	return (notify);
  // }
  
  // static int
  // vmx_pending_intr(struct vlapic *vlapic, int *vecptr)
  // {
  // 	struct vlapic_vtx *vlapic_vtx;
  // 	struct pir_desc *pir_desc;
  // 	struct LAPIC *lapic;
  // 	uint64_t pending, pirval;
  // 	uint32_t ppr, vpr;
  // 	int i;
  
  // 	/*
  // 	 * This function is only expected to be called from the 'HLT' exit
  // 	 * handler which does not care about the vector that is pending.
  // 	 */
  // 	KASSERT(vecptr == NULL, ("vmx_pending_intr: vecptr must be NULL"));
  
  // 	vlapic_vtx = (struct vlapic_vtx *)vlapic;
  // 	pir_desc = vlapic_vtx->pir_desc;
  
  // 	pending = atomic_load_acq_long(&pir_desc->pending);
  // 	if (!pending)
  // 		return (0);	/* common case */
  
  // 	/*
  // 	 * If there is an interrupt pending then it will be recognized only
  // 	 * if its priority is greater than the processor priority.
  // 	 *
  // 	 * Special case: if the processor priority is zero then any pending
  // 	 * interrupt will be recognized.
  // 	 */
  // 	lapic = vlapic->apic_page;
  // 	ppr = lapic->ppr & 0xf0;
  // 	if (ppr == 0)
  // 		return (1);
  
  // 	VCPU_CTR1(vlapic->vm, vlapic->vcpuid, "HLT with non-zero PPR %d",
  // 	    lapic->ppr);
  
  // 	for (i = 3; i >= 0; i--) {
  // 		pirval = pir_desc->pir[i];
  // 		if (pirval != 0) {
  // 			vpr = (i * 64 + flsl(pirval) - 1) & 0xf0;
  // 			return (vpr > ppr);
  // 		}
  // 	}
  // 	return (0);
  // }
  
  // static void
  // vmx_intr_accepted(struct vlapic *vlapic, int vector)
  // {
  
  // 	xhyve_abort("vmx_intr_accepted: not expected to be called");
  // }
  
  // static void
  // vmx_set_tmr(struct vlapic *vlapic, int vector, bool level)
  // {
  // 	struct vlapic_vtx *vlapic_vtx;
  // 	struct vmx *vmx;
  // 	struct vmcs *vmcs;
  // 	uint64_t mask, val;
  
  // 	KASSERT(vector >= 0 && vector <= 255, ("invalid vector %d", vector));
  // 	KASSERT(!vcpu_is_running(vlapic->vm, vlapic->vcpuid, NULL),
  // 	    ("vmx_set_tmr: vcpu cannot be running"));
  
  // 	vlapic_vtx = (struct vlapic_vtx *)vlapic;
  // 	vmx = vlapic_vtx->vmx;
  // 	vmcs = &vmx->vmcs[vlapic->vcpuid];
  // 	mask = 1UL << (vector % 64);
  
  // 	VMPTRLD(vmcs);
  // 	val = vmcs_read(VMCS_EOI_EXIT(vector));
  // 	if (level)
  // 		val |= mask;
  // 	else
  // 		val &= ~mask;
  // 	vmcs_write(VMCS_EOI_EXIT(vector), val);
  // 	VMCLEAR(vmcs);
  // }
  
  // static void
  // vmx_enable_x2apic_mode(struct vlapic *vlapic)
  // {
  // 	struct vmx *vmx;
  // 	struct vmcs *vmcs;
  // 	uint32_t proc_ctls2;
  // 	int vcpuid, error;
  
  // 	vcpuid = vlapic->vcpuid;
  // 	vmx = ((struct vlapic_vtx *)vlapic)->vmx;
  // 	vmcs = &vmx->vmcs[vcpuid];
  
  // 	proc_ctls2 = vmx->cap[vcpuid].proc_ctls2;
  // 	KASSERT((proc_ctls2 & PROCBASED2_VIRTUALIZE_APIC_ACCESSES) != 0,
  // 	    ("%s: invalid proc_ctls2 %#x", __func__, proc_ctls2));
  
  // 	proc_ctls2 &= ~PROCBASED2_VIRTUALIZE_APIC_ACCESSES;
  // 	proc_ctls2 |= PROCBASED2_VIRTUALIZE_X2APIC_MODE;
  // 	vmx->cap[vcpuid].proc_ctls2 = proc_ctls2;
  
  // 	VMPTRLD(vmcs);
  // 	vmcs_write(VMCS_SEC_PROC_BASED_CTLS, proc_ctls2);
  // 	VMCLEAR(vmcs);
  
  // 	if (vlapic->vcpuid == 0) {
  // 		/*
  // 		 * The nested page table mappings are shared by all vcpus
  // 		 * so unmap the APIC access page just once.
  // 		 */
  // 		error = vm_unmap_mmio(vmx->vm, DEFAULT_APIC_BASE, PAGE_SIZE);
  // 		KASSERT(error == 0, ("%s: vm_unmap_mmio error %d",
  // 		    __func__, error));
  
  // 		/*
  // 		 * The MSR bitmap is shared by all vcpus so modify it only
  // 		 * once in the context of vcpu 0.
  // 		 */
  // 		error = vmx_allow_x2apic_msrs(vmx);
  // 		KASSERT(error == 0, ("%s: vmx_allow_x2apic_msrs error %d",
  // 		    __func__, error));
  // 	}
  // }
  
  static struct vlapic *
  vmx_vlapic_init(void *arg, int vcpuid)
  {
  	struct vmx *vmx;
  	struct vlapic *vlapic;
  	struct vlapic_vtx *vlapic_vtx;
  	
  	vmx = arg;
  
  	vlapic = malloc(sizeof(struct vlapic_vtx));
  	assert(vlapic);
  	bzero(vlapic, sizeof(struct vlapic));
  	vlapic->vm = vmx->vm;
  	vlapic->vcpuid = vcpuid;
  	vlapic->apic_page = (struct LAPIC *)&vmx->apic_page[vcpuid];
  
  	vlapic_vtx = (struct vlapic_vtx *)vlapic;
  	vlapic_vtx->vmx = vmx;
  
  	vlapic_init(vlapic);
  
  	return (vlapic);
  }
  
  static void
  vmx_vlapic_cleanup(UNUSED void *arg, struct vlapic *vlapic)
  {
  	vlapic_cleanup(vlapic);
  	free(vlapic);
  }
  
  static void
  vmx_vcpu_interrupt(int vcpu) {
  	hv_vcpuid_t hvvcpu;
  
  	hvvcpu = (hv_vcpuid_t) vcpu;
  
  	hv_vcpu_interrupt(&hvvcpu, 1);
  }
  
  struct vmm_ops vmm_ops_intel = {
  	vmx_init,
  	vmx_cleanup,
  	vmx_vm_init,
  	vmx_vcpu_init,
  	vmx_run,
  	vmx_vm_cleanup,
  	vmx_vcpu_cleanup,
  	vmx_getreg,
  	vmx_setreg,
  	vmx_getdesc,
  	vmx_setdesc,
  	vmx_getcap,
  	vmx_setcap,
  	vmx_vlapic_init,
  	vmx_vlapic_cleanup,
  	vmx_vcpu_interrupt
  };
  