xref: /freebsd/sys/amd64/vmm/intel/vmx.c (revision 4ec234c813eed05c166859bba82c882e40826eb9)
1 /*-
2  * Copyright (c) 2011 NetApp, Inc.
3  * All rights reserved.
4  *
5  * Redistribution and use in source and binary forms, with or without
6  * modification, are permitted provided that the following conditions
7  * are met:
8  * 1. Redistributions of source code must retain the above copyright
9  *    notice, this list of conditions and the following disclaimer.
10  * 2. Redistributions in binary form must reproduce the above copyright
11  *    notice, this list of conditions and the following disclaimer in the
12  *    documentation and/or other materials provided with the distribution.
13  *
14  * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND
15  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
16  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
17  * ARE DISCLAIMED.  IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE
18  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
19  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
20  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
21  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
22  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
23  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
24  * SUCH DAMAGE.
25  *
26  * $FreeBSD$
27  */
28 
29 #include <sys/cdefs.h>
30 __FBSDID("$FreeBSD$");
31 
32 #include <sys/param.h>
33 #include <sys/systm.h>
34 #include <sys/smp.h>
35 #include <sys/kernel.h>
36 #include <sys/malloc.h>
37 #include <sys/pcpu.h>
38 #include <sys/proc.h>
39 #include <sys/sysctl.h>
40 
41 #include <vm/vm.h>
42 #include <vm/pmap.h>
43 
44 #include <machine/psl.h>
45 #include <machine/cpufunc.h>
46 #include <machine/md_var.h>
47 #include <machine/segments.h>
48 #include <machine/smp.h>
49 #include <machine/specialreg.h>
50 #include <machine/vmparam.h>
51 
52 #include <machine/vmm.h>
53 #include "vmm_host.h"
54 #include "vmm_ipi.h"
55 #include "vmm_msr.h"
56 #include "vmm_ktr.h"
57 #include "vmm_stat.h"
58 #include "vlapic.h"
59 #include "vlapic_priv.h"
60 
61 #include "vmx_msr.h"
62 #include "ept.h"
63 #include "vmx_cpufunc.h"
64 #include "vmx.h"
65 #include "x86.h"
66 #include "vmx_controls.h"
67 
68 #define	PINBASED_CTLS_ONE_SETTING					\
69 	(PINBASED_EXTINT_EXITING	|				\
70 	 PINBASED_NMI_EXITING		|				\
71 	 PINBASED_VIRTUAL_NMI)
72 #define	PINBASED_CTLS_ZERO_SETTING	0
73 
74 #define PROCBASED_CTLS_WINDOW_SETTING					\
75 	(PROCBASED_INT_WINDOW_EXITING	|				\
76 	 PROCBASED_NMI_WINDOW_EXITING)
77 
78 #define	PROCBASED_CTLS_ONE_SETTING 					\
79 	(PROCBASED_SECONDARY_CONTROLS	|				\
80 	 PROCBASED_IO_EXITING		|				\
81 	 PROCBASED_MSR_BITMAPS		|				\
82 	 PROCBASED_CTLS_WINDOW_SETTING)
83 #define	PROCBASED_CTLS_ZERO_SETTING	\
84 	(PROCBASED_CR3_LOAD_EXITING |	\
85 	PROCBASED_CR3_STORE_EXITING |	\
86 	PROCBASED_IO_BITMAPS)
87 
88 #define	PROCBASED_CTLS2_ONE_SETTING	PROCBASED2_ENABLE_EPT
89 #define	PROCBASED_CTLS2_ZERO_SETTING	0
90 
91 #define VM_EXIT_CTLS_ONE_SETTING_NO_PAT					\
92 	(VM_EXIT_HOST_LMA			|			\
93 	VM_EXIT_SAVE_EFER			|			\
94 	VM_EXIT_LOAD_EFER)
95 
96 #define	VM_EXIT_CTLS_ONE_SETTING					\
97 	(VM_EXIT_CTLS_ONE_SETTING_NO_PAT       	|			\
98 	VM_EXIT_ACKNOWLEDGE_INTERRUPT		|			\
99 	VM_EXIT_SAVE_PAT			|			\
100 	VM_EXIT_LOAD_PAT)
101 #define	VM_EXIT_CTLS_ZERO_SETTING	VM_EXIT_SAVE_DEBUG_CONTROLS
102 
103 #define	VM_ENTRY_CTLS_ONE_SETTING_NO_PAT	VM_ENTRY_LOAD_EFER
104 
105 #define	VM_ENTRY_CTLS_ONE_SETTING					\
106 	(VM_ENTRY_CTLS_ONE_SETTING_NO_PAT     	|			\
107 	VM_ENTRY_LOAD_PAT)
108 #define	VM_ENTRY_CTLS_ZERO_SETTING					\
109 	(VM_ENTRY_LOAD_DEBUG_CONTROLS		|			\
110 	VM_ENTRY_INTO_SMM			|			\
111 	VM_ENTRY_DEACTIVATE_DUAL_MONITOR)
112 
113 #define	guest_msr_rw(vmx, msr) \
114 	msr_bitmap_change_access((vmx)->msr_bitmap, (msr), MSR_BITMAP_ACCESS_RW)
115 
116 #define	HANDLED		1
117 #define	UNHANDLED	0
118 
119 static MALLOC_DEFINE(M_VMX, "vmx", "vmx");
120 static MALLOC_DEFINE(M_VLAPIC, "vlapic", "vlapic");
121 
122 SYSCTL_DECL(_hw_vmm);
123 SYSCTL_NODE(_hw_vmm, OID_AUTO, vmx, CTLFLAG_RW, NULL, NULL);
124 
125 int vmxon_enabled[MAXCPU];
126 static char vmxon_region[MAXCPU][PAGE_SIZE] __aligned(PAGE_SIZE);
127 
128 static uint32_t pinbased_ctls, procbased_ctls, procbased_ctls2;
129 static uint32_t exit_ctls, entry_ctls;
130 
131 static uint64_t cr0_ones_mask, cr0_zeros_mask;
132 SYSCTL_ULONG(_hw_vmm_vmx, OID_AUTO, cr0_ones_mask, CTLFLAG_RD,
133 	     &cr0_ones_mask, 0, NULL);
134 SYSCTL_ULONG(_hw_vmm_vmx, OID_AUTO, cr0_zeros_mask, CTLFLAG_RD,
135 	     &cr0_zeros_mask, 0, NULL);
136 
137 static uint64_t cr4_ones_mask, cr4_zeros_mask;
138 SYSCTL_ULONG(_hw_vmm_vmx, OID_AUTO, cr4_ones_mask, CTLFLAG_RD,
139 	     &cr4_ones_mask, 0, NULL);
140 SYSCTL_ULONG(_hw_vmm_vmx, OID_AUTO, cr4_zeros_mask, CTLFLAG_RD,
141 	     &cr4_zeros_mask, 0, NULL);
142 
143 static int vmx_no_patmsr;
144 
145 static int vmx_initialized;
146 SYSCTL_INT(_hw_vmm_vmx, OID_AUTO, initialized, CTLFLAG_RD,
147 	   &vmx_initialized, 0, "Intel VMX initialized");
148 
149 /*
150  * Virtual NMI blocking conditions.
151  *
152  * Some processor implementations also require NMI to be blocked if
153  * the STI_BLOCKING bit is set. It is possible to detect this at runtime
154  * based on the (exit_reason,exit_qual) tuple being set to
155  * (EXIT_REASON_INVAL_VMCS, EXIT_QUAL_NMI_WHILE_STI_BLOCKING).
156  *
157  * We take the easy way out and also include STI_BLOCKING as one of the
158  * gating items for vNMI injection.
159  */
160 static uint64_t nmi_blocking_bits = VMCS_INTERRUPTIBILITY_MOVSS_BLOCKING |
161 				    VMCS_INTERRUPTIBILITY_NMI_BLOCKING |
162 				    VMCS_INTERRUPTIBILITY_STI_BLOCKING;
163 
164 /*
165  * Optional capabilities
166  */
167 static int cap_halt_exit;
168 static int cap_pause_exit;
169 static int cap_unrestricted_guest;
170 static int cap_monitor_trap;
171 static int cap_invpcid;
172 
173 static int virtual_interrupt_delivery;
174 SYSCTL_INT(_hw_vmm_vmx, OID_AUTO, virtual_interrupt_delivery, CTLFLAG_RD,
175     &virtual_interrupt_delivery, 0, "APICv virtual interrupt delivery support");
176 
177 static int posted_interrupts;
178 SYSCTL_INT(_hw_vmm_vmx, OID_AUTO, posted_interrupts, CTLFLAG_RD,
179     &posted_interrupts, 0, "APICv posted interrupt support");
180 
181 static int pirvec;
182 SYSCTL_INT(_hw_vmm_vmx, OID_AUTO, posted_interrupt_vector, CTLFLAG_RD,
183     &pirvec, 0, "APICv posted interrupt vector");
184 
185 static struct unrhdr *vpid_unr;
186 static u_int vpid_alloc_failed;
187 SYSCTL_UINT(_hw_vmm_vmx, OID_AUTO, vpid_alloc_failed, CTLFLAG_RD,
188 	    &vpid_alloc_failed, 0, NULL);
189 
190 /*
191  * Use the last page below 4GB as the APIC access address. This address is
192  * occupied by the boot firmware so it is guaranteed that it will not conflict
193  * with a page in system memory.
194  */
195 #define	APIC_ACCESS_ADDRESS	0xFFFFF000
196 
197 static void vmx_inject_pir(struct vlapic *vlapic);
198 
199 #ifdef KTR
200 static const char *
201 exit_reason_to_str(int reason)
202 {
203 	static char reasonbuf[32];
204 
205 	switch (reason) {
206 	case EXIT_REASON_EXCEPTION:
207 		return "exception";
208 	case EXIT_REASON_EXT_INTR:
209 		return "extint";
210 	case EXIT_REASON_TRIPLE_FAULT:
211 		return "triplefault";
212 	case EXIT_REASON_INIT:
213 		return "init";
214 	case EXIT_REASON_SIPI:
215 		return "sipi";
216 	case EXIT_REASON_IO_SMI:
217 		return "iosmi";
218 	case EXIT_REASON_SMI:
219 		return "smi";
220 	case EXIT_REASON_INTR_WINDOW:
221 		return "intrwindow";
222 	case EXIT_REASON_NMI_WINDOW:
223 		return "nmiwindow";
224 	case EXIT_REASON_TASK_SWITCH:
225 		return "taskswitch";
226 	case EXIT_REASON_CPUID:
227 		return "cpuid";
228 	case EXIT_REASON_GETSEC:
229 		return "getsec";
230 	case EXIT_REASON_HLT:
231 		return "hlt";
232 	case EXIT_REASON_INVD:
233 		return "invd";
234 	case EXIT_REASON_INVLPG:
235 		return "invlpg";
236 	case EXIT_REASON_RDPMC:
237 		return "rdpmc";
238 	case EXIT_REASON_RDTSC:
239 		return "rdtsc";
240 	case EXIT_REASON_RSM:
241 		return "rsm";
242 	case EXIT_REASON_VMCALL:
243 		return "vmcall";
244 	case EXIT_REASON_VMCLEAR:
245 		return "vmclear";
246 	case EXIT_REASON_VMLAUNCH:
247 		return "vmlaunch";
248 	case EXIT_REASON_VMPTRLD:
249 		return "vmptrld";
250 	case EXIT_REASON_VMPTRST:
251 		return "vmptrst";
252 	case EXIT_REASON_VMREAD:
253 		return "vmread";
254 	case EXIT_REASON_VMRESUME:
255 		return "vmresume";
256 	case EXIT_REASON_VMWRITE:
257 		return "vmwrite";
258 	case EXIT_REASON_VMXOFF:
259 		return "vmxoff";
260 	case EXIT_REASON_VMXON:
261 		return "vmxon";
262 	case EXIT_REASON_CR_ACCESS:
263 		return "craccess";
264 	case EXIT_REASON_DR_ACCESS:
265 		return "draccess";
266 	case EXIT_REASON_INOUT:
267 		return "inout";
268 	case EXIT_REASON_RDMSR:
269 		return "rdmsr";
270 	case EXIT_REASON_WRMSR:
271 		return "wrmsr";
272 	case EXIT_REASON_INVAL_VMCS:
273 		return "invalvmcs";
274 	case EXIT_REASON_INVAL_MSR:
275 		return "invalmsr";
276 	case EXIT_REASON_MWAIT:
277 		return "mwait";
278 	case EXIT_REASON_MTF:
279 		return "mtf";
280 	case EXIT_REASON_MONITOR:
281 		return "monitor";
282 	case EXIT_REASON_PAUSE:
283 		return "pause";
284 	case EXIT_REASON_MCE:
285 		return "mce";
286 	case EXIT_REASON_TPR:
287 		return "tpr";
288 	case EXIT_REASON_APIC_ACCESS:
289 		return "apic-access";
290 	case EXIT_REASON_GDTR_IDTR:
291 		return "gdtridtr";
292 	case EXIT_REASON_LDTR_TR:
293 		return "ldtrtr";
294 	case EXIT_REASON_EPT_FAULT:
295 		return "eptfault";
296 	case EXIT_REASON_EPT_MISCONFIG:
297 		return "eptmisconfig";
298 	case EXIT_REASON_INVEPT:
299 		return "invept";
300 	case EXIT_REASON_RDTSCP:
301 		return "rdtscp";
302 	case EXIT_REASON_VMX_PREEMPT:
303 		return "vmxpreempt";
304 	case EXIT_REASON_INVVPID:
305 		return "invvpid";
306 	case EXIT_REASON_WBINVD:
307 		return "wbinvd";
308 	case EXIT_REASON_XSETBV:
309 		return "xsetbv";
310 	case EXIT_REASON_APIC_WRITE:
311 		return "apic-write";
312 	default:
313 		snprintf(reasonbuf, sizeof(reasonbuf), "%d", reason);
314 		return (reasonbuf);
315 	}
316 }
317 #endif	/* KTR */
318 
319 u_long
320 vmx_fix_cr0(u_long cr0)
321 {
322 
323 	return ((cr0 | cr0_ones_mask) & ~cr0_zeros_mask);
324 }
325 
326 u_long
327 vmx_fix_cr4(u_long cr4)
328 {
329 
330 	return ((cr4 | cr4_ones_mask) & ~cr4_zeros_mask);
331 }
332 
333 static void
334 vpid_free(int vpid)
335 {
336 	if (vpid < 0 || vpid > 0xffff)
337 		panic("vpid_free: invalid vpid %d", vpid);
338 
339 	/*
340 	 * VPIDs [0,VM_MAXCPU] are special and are not allocated from
341 	 * the unit number allocator.
342 	 */
343 
344 	if (vpid > VM_MAXCPU)
345 		free_unr(vpid_unr, vpid);
346 }
347 
348 static void
349 vpid_alloc(uint16_t *vpid, int num)
350 {
351 	int i, x;
352 
353 	if (num <= 0 || num > VM_MAXCPU)
354 		panic("invalid number of vpids requested: %d", num);
355 
356 	/*
357 	 * If the "enable vpid" execution control is not enabled then the
358 	 * VPID is required to be 0 for all vcpus.
359 	 */
360 	if ((procbased_ctls2 & PROCBASED2_ENABLE_VPID) == 0) {
361 		for (i = 0; i < num; i++)
362 			vpid[i] = 0;
363 		return;
364 	}
365 
366 	/*
367 	 * Allocate a unique VPID for each vcpu from the unit number allocator.
368 	 */
369 	for (i = 0; i < num; i++) {
370 		x = alloc_unr(vpid_unr);
371 		if (x == -1)
372 			break;
373 		else
374 			vpid[i] = x;
375 	}
376 
377 	if (i < num) {
378 		atomic_add_int(&vpid_alloc_failed, 1);
379 
380 		/*
381 		 * If the unit number allocator does not have enough unique
382 		 * VPIDs then we need to allocate from the [1,VM_MAXCPU] range.
383 		 *
384 		 * These VPIDs are not be unique across VMs but this does not
385 		 * affect correctness because the combined mappings are also
386 		 * tagged with the EP4TA which is unique for each VM.
387 		 *
388 		 * It is still sub-optimal because the invvpid will invalidate
389 		 * combined mappings for a particular VPID across all EP4TAs.
390 		 */
391 		while (i-- > 0)
392 			vpid_free(vpid[i]);
393 
394 		for (i = 0; i < num; i++)
395 			vpid[i] = i + 1;
396 	}
397 }
398 
399 static void
400 vpid_init(void)
401 {
402 	/*
403 	 * VPID 0 is required when the "enable VPID" execution control is
404 	 * disabled.
405 	 *
406 	 * VPIDs [1,VM_MAXCPU] are used as the "overflow namespace" when the
407 	 * unit number allocator does not have sufficient unique VPIDs to
408 	 * satisfy the allocation.
409 	 *
410 	 * The remaining VPIDs are managed by the unit number allocator.
411 	 */
412 	vpid_unr = new_unrhdr(VM_MAXCPU + 1, 0xffff, NULL);
413 }
414 
415 static void
416 msr_save_area_init(struct msr_entry *g_area, int *g_count)
417 {
418 	int cnt;
419 
420 	static struct msr_entry guest_msrs[] = {
421 		{ MSR_KGSBASE, 0, 0 },
422 	};
423 
424 	cnt = sizeof(guest_msrs) / sizeof(guest_msrs[0]);
425 	if (cnt > GUEST_MSR_MAX_ENTRIES)
426 		panic("guest msr save area overrun");
427 	bcopy(guest_msrs, g_area, sizeof(guest_msrs));
428 	*g_count = cnt;
429 }
430 
431 static void
432 vmx_disable(void *arg __unused)
433 {
434 	struct invvpid_desc invvpid_desc = { 0 };
435 	struct invept_desc invept_desc = { 0 };
436 
437 	if (vmxon_enabled[curcpu]) {
438 		/*
439 		 * See sections 25.3.3.3 and 25.3.3.4 in Intel Vol 3b.
440 		 *
441 		 * VMXON or VMXOFF are not required to invalidate any TLB
442 		 * caching structures. This prevents potential retention of
443 		 * cached information in the TLB between distinct VMX episodes.
444 		 */
445 		invvpid(INVVPID_TYPE_ALL_CONTEXTS, invvpid_desc);
446 		invept(INVEPT_TYPE_ALL_CONTEXTS, invept_desc);
447 		vmxoff();
448 	}
449 	load_cr4(rcr4() & ~CR4_VMXE);
450 }
451 
452 static int
453 vmx_cleanup(void)
454 {
455 
456 	if (pirvec != 0)
457 		vmm_ipi_free(pirvec);
458 
459 	if (vpid_unr != NULL) {
460 		delete_unrhdr(vpid_unr);
461 		vpid_unr = NULL;
462 	}
463 
464 	smp_rendezvous(NULL, vmx_disable, NULL, NULL);
465 
466 	return (0);
467 }
468 
469 static void
470 vmx_enable(void *arg __unused)
471 {
472 	int error;
473 
474 	load_cr4(rcr4() | CR4_VMXE);
475 
476 	*(uint32_t *)vmxon_region[curcpu] = vmx_revision();
477 	error = vmxon(vmxon_region[curcpu]);
478 	if (error == 0)
479 		vmxon_enabled[curcpu] = 1;
480 }
481 
482 static void
483 vmx_restore(void)
484 {
485 
486 	if (vmxon_enabled[curcpu])
487 		vmxon(vmxon_region[curcpu]);
488 }
489 
490 static int
491 vmx_init(int ipinum)
492 {
493 	int error, use_tpr_shadow;
494 	uint64_t fixed0, fixed1, feature_control;
495 	uint32_t tmp, procbased2_vid_bits;
496 
497 	/* CPUID.1:ECX[bit 5] must be 1 for processor to support VMX */
498 	if (!(cpu_feature2 & CPUID2_VMX)) {
499 		printf("vmx_init: processor does not support VMX operation\n");
500 		return (ENXIO);
501 	}
502 
503 	/*
504 	 * Verify that MSR_IA32_FEATURE_CONTROL lock and VMXON enable bits
505 	 * are set (bits 0 and 2 respectively).
506 	 */
507 	feature_control = rdmsr(MSR_IA32_FEATURE_CONTROL);
508 	if ((feature_control & IA32_FEATURE_CONTROL_LOCK) == 0 ||
509 	    (feature_control & IA32_FEATURE_CONTROL_VMX_EN) == 0) {
510 		printf("vmx_init: VMX operation disabled by BIOS\n");
511 		return (ENXIO);
512 	}
513 
514 	/* Check support for primary processor-based VM-execution controls */
515 	error = vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS,
516 			       MSR_VMX_TRUE_PROCBASED_CTLS,
517 			       PROCBASED_CTLS_ONE_SETTING,
518 			       PROCBASED_CTLS_ZERO_SETTING, &procbased_ctls);
519 	if (error) {
520 		printf("vmx_init: processor does not support desired primary "
521 		       "processor-based controls\n");
522 		return (error);
523 	}
524 
525 	/* Clear the processor-based ctl bits that are set on demand */
526 	procbased_ctls &= ~PROCBASED_CTLS_WINDOW_SETTING;
527 
528 	/* Check support for secondary processor-based VM-execution controls */
529 	error = vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS2,
530 			       MSR_VMX_PROCBASED_CTLS2,
531 			       PROCBASED_CTLS2_ONE_SETTING,
532 			       PROCBASED_CTLS2_ZERO_SETTING, &procbased_ctls2);
533 	if (error) {
534 		printf("vmx_init: processor does not support desired secondary "
535 		       "processor-based controls\n");
536 		return (error);
537 	}
538 
539 	/* Check support for VPID */
540 	error = vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS2, MSR_VMX_PROCBASED_CTLS2,
541 			       PROCBASED2_ENABLE_VPID, 0, &tmp);
542 	if (error == 0)
543 		procbased_ctls2 |= PROCBASED2_ENABLE_VPID;
544 
545 	/* Check support for pin-based VM-execution controls */
546 	error = vmx_set_ctlreg(MSR_VMX_PINBASED_CTLS,
547 			       MSR_VMX_TRUE_PINBASED_CTLS,
548 			       PINBASED_CTLS_ONE_SETTING,
549 			       PINBASED_CTLS_ZERO_SETTING, &pinbased_ctls);
550 	if (error) {
551 		printf("vmx_init: processor does not support desired "
552 		       "pin-based controls\n");
553 		return (error);
554 	}
555 
556 	/* Check support for VM-exit controls */
557 	error = vmx_set_ctlreg(MSR_VMX_EXIT_CTLS, MSR_VMX_TRUE_EXIT_CTLS,
558 			       VM_EXIT_CTLS_ONE_SETTING,
559 			       VM_EXIT_CTLS_ZERO_SETTING,
560 			       &exit_ctls);
561 	if (error) {
562 		/* Try again without the PAT MSR bits */
563 		error = vmx_set_ctlreg(MSR_VMX_EXIT_CTLS,
564 				       MSR_VMX_TRUE_EXIT_CTLS,
565 				       VM_EXIT_CTLS_ONE_SETTING_NO_PAT,
566 				       VM_EXIT_CTLS_ZERO_SETTING,
567 				       &exit_ctls);
568 		if (error) {
569 			printf("vmx_init: processor does not support desired "
570 			       "exit controls\n");
571 			return (error);
572 		} else {
573 			if (bootverbose)
574 				printf("vmm: PAT MSR access not supported\n");
575 			guest_msr_valid(MSR_PAT);
576 			vmx_no_patmsr = 1;
577 		}
578 	}
579 
580 	/* Check support for VM-entry controls */
581 	if (!vmx_no_patmsr) {
582 		error = vmx_set_ctlreg(MSR_VMX_ENTRY_CTLS,
583 				       MSR_VMX_TRUE_ENTRY_CTLS,
584 				       VM_ENTRY_CTLS_ONE_SETTING,
585 				       VM_ENTRY_CTLS_ZERO_SETTING,
586 				       &entry_ctls);
587 	} else {
588 		error = vmx_set_ctlreg(MSR_VMX_ENTRY_CTLS,
589 				       MSR_VMX_TRUE_ENTRY_CTLS,
590 				       VM_ENTRY_CTLS_ONE_SETTING_NO_PAT,
591 				       VM_ENTRY_CTLS_ZERO_SETTING,
592 				       &entry_ctls);
593 	}
594 
595 	if (error) {
596 		printf("vmx_init: processor does not support desired "
597 		       "entry controls\n");
598 		       return (error);
599 	}
600 
601 	/*
602 	 * Check support for optional features by testing them
603 	 * as individual bits
604 	 */
605 	cap_halt_exit = (vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS,
606 					MSR_VMX_TRUE_PROCBASED_CTLS,
607 					PROCBASED_HLT_EXITING, 0,
608 					&tmp) == 0);
609 
610 	cap_monitor_trap = (vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS,
611 					MSR_VMX_PROCBASED_CTLS,
612 					PROCBASED_MTF, 0,
613 					&tmp) == 0);
614 
615 	cap_pause_exit = (vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS,
616 					 MSR_VMX_TRUE_PROCBASED_CTLS,
617 					 PROCBASED_PAUSE_EXITING, 0,
618 					 &tmp) == 0);
619 
620 	cap_unrestricted_guest = (vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS2,
621 					MSR_VMX_PROCBASED_CTLS2,
622 					PROCBASED2_UNRESTRICTED_GUEST, 0,
623 				        &tmp) == 0);
624 
625 	cap_invpcid = (vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS2,
626 	    MSR_VMX_PROCBASED_CTLS2, PROCBASED2_ENABLE_INVPCID, 0,
627 	    &tmp) == 0);
628 
629 	/*
630 	 * Check support for virtual interrupt delivery.
631 	 */
632 	procbased2_vid_bits = (PROCBASED2_VIRTUALIZE_APIC_ACCESSES |
633 	    PROCBASED2_VIRTUALIZE_X2APIC_MODE |
634 	    PROCBASED2_APIC_REGISTER_VIRTUALIZATION |
635 	    PROCBASED2_VIRTUAL_INTERRUPT_DELIVERY);
636 
637 	use_tpr_shadow = (vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS,
638 	    MSR_VMX_TRUE_PROCBASED_CTLS, PROCBASED_USE_TPR_SHADOW, 0,
639 	    &tmp) == 0);
640 
641 	error = vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS2, MSR_VMX_PROCBASED_CTLS2,
642 	    procbased2_vid_bits, 0, &tmp);
643 	if (error == 0 && use_tpr_shadow) {
644 		virtual_interrupt_delivery = 1;
645 		TUNABLE_INT_FETCH("hw.vmm.vmx.use_apic_vid",
646 		    &virtual_interrupt_delivery);
647 	}
648 
649 	if (virtual_interrupt_delivery) {
650 		procbased_ctls |= PROCBASED_USE_TPR_SHADOW;
651 		procbased_ctls2 |= procbased2_vid_bits;
652 		procbased_ctls2 &= ~PROCBASED2_VIRTUALIZE_X2APIC_MODE;
653 
654 		/*
655 		 * Check for Posted Interrupts only if Virtual Interrupt
656 		 * Delivery is enabled.
657 		 */
658 		error = vmx_set_ctlreg(MSR_VMX_PINBASED_CTLS,
659 		    MSR_VMX_TRUE_PINBASED_CTLS, PINBASED_POSTED_INTERRUPT, 0,
660 		    &tmp);
661 		if (error == 0) {
662 			pirvec = vmm_ipi_alloc();
663 			if (pirvec == 0) {
664 				if (bootverbose) {
665 					printf("vmx_init: unable to allocate "
666 					    "posted interrupt vector\n");
667 				}
668 			} else {
669 				posted_interrupts = 1;
670 				TUNABLE_INT_FETCH("hw.vmm.vmx.use_apic_pir",
671 				    &posted_interrupts);
672 			}
673 		}
674 	}
675 
676 	if (posted_interrupts)
677 		    pinbased_ctls |= PINBASED_POSTED_INTERRUPT;
678 
679 	/* Initialize EPT */
680 	error = ept_init(ipinum);
681 	if (error) {
682 		printf("vmx_init: ept initialization failed (%d)\n", error);
683 		return (error);
684 	}
685 
686 	/*
687 	 * Stash the cr0 and cr4 bits that must be fixed to 0 or 1
688 	 */
689 	fixed0 = rdmsr(MSR_VMX_CR0_FIXED0);
690 	fixed1 = rdmsr(MSR_VMX_CR0_FIXED1);
691 	cr0_ones_mask = fixed0 & fixed1;
692 	cr0_zeros_mask = ~fixed0 & ~fixed1;
693 
694 	/*
695 	 * CR0_PE and CR0_PG can be set to zero in VMX non-root operation
696 	 * if unrestricted guest execution is allowed.
697 	 */
698 	if (cap_unrestricted_guest)
699 		cr0_ones_mask &= ~(CR0_PG | CR0_PE);
700 
701 	/*
702 	 * Do not allow the guest to set CR0_NW or CR0_CD.
703 	 */
704 	cr0_zeros_mask |= (CR0_NW | CR0_CD);
705 
706 	fixed0 = rdmsr(MSR_VMX_CR4_FIXED0);
707 	fixed1 = rdmsr(MSR_VMX_CR4_FIXED1);
708 	cr4_ones_mask = fixed0 & fixed1;
709 	cr4_zeros_mask = ~fixed0 & ~fixed1;
710 
711 	vpid_init();
712 
713 	/* enable VMX operation */
714 	smp_rendezvous(NULL, vmx_enable, NULL, NULL);
715 
716 	vmx_initialized = 1;
717 
718 	return (0);
719 }
720 
721 static void
722 vmx_trigger_hostintr(int vector)
723 {
724 	uintptr_t func;
725 	struct gate_descriptor *gd;
726 
727 	gd = &idt[vector];
728 
729 	KASSERT(vector >= 32 && vector <= 255, ("vmx_trigger_hostintr: "
730 	    "invalid vector %d", vector));
731 	KASSERT(gd->gd_p == 1, ("gate descriptor for vector %d not present",
732 	    vector));
733 	KASSERT(gd->gd_type == SDT_SYSIGT, ("gate descriptor for vector %d "
734 	    "has invalid type %d", vector, gd->gd_type));
735 	KASSERT(gd->gd_dpl == SEL_KPL, ("gate descriptor for vector %d "
736 	    "has invalid dpl %d", vector, gd->gd_dpl));
737 	KASSERT(gd->gd_selector == GSEL(GCODE_SEL, SEL_KPL), ("gate descriptor "
738 	    "for vector %d has invalid selector %d", vector, gd->gd_selector));
739 	KASSERT(gd->gd_ist == 0, ("gate descriptor for vector %d has invalid "
740 	    "IST %d", vector, gd->gd_ist));
741 
742 	func = ((long)gd->gd_hioffset << 16 | gd->gd_looffset);
743 	vmx_call_isr(func);
744 }
745 
746 static int
747 vmx_setup_cr_shadow(int which, struct vmcs *vmcs, uint32_t initial)
748 {
749 	int error, mask_ident, shadow_ident;
750 	uint64_t mask_value;
751 
752 	if (which != 0 && which != 4)
753 		panic("vmx_setup_cr_shadow: unknown cr%d", which);
754 
755 	if (which == 0) {
756 		mask_ident = VMCS_CR0_MASK;
757 		mask_value = cr0_ones_mask | cr0_zeros_mask;
758 		shadow_ident = VMCS_CR0_SHADOW;
759 	} else {
760 		mask_ident = VMCS_CR4_MASK;
761 		mask_value = cr4_ones_mask | cr4_zeros_mask;
762 		shadow_ident = VMCS_CR4_SHADOW;
763 	}
764 
765 	error = vmcs_setreg(vmcs, 0, VMCS_IDENT(mask_ident), mask_value);
766 	if (error)
767 		return (error);
768 
769 	error = vmcs_setreg(vmcs, 0, VMCS_IDENT(shadow_ident), initial);
770 	if (error)
771 		return (error);
772 
773 	return (0);
774 }
775 #define	vmx_setup_cr0_shadow(vmcs,init)	vmx_setup_cr_shadow(0, (vmcs), (init))
776 #define	vmx_setup_cr4_shadow(vmcs,init)	vmx_setup_cr_shadow(4, (vmcs), (init))
777 
778 static void *
779 vmx_vminit(struct vm *vm, pmap_t pmap)
780 {
781 	uint16_t vpid[VM_MAXCPU];
782 	int i, error, guest_msr_count;
783 	struct vmx *vmx;
784 	struct vmcs *vmcs;
785 
786 	vmx = malloc(sizeof(struct vmx), M_VMX, M_WAITOK | M_ZERO);
787 	if ((uintptr_t)vmx & PAGE_MASK) {
788 		panic("malloc of struct vmx not aligned on %d byte boundary",
789 		      PAGE_SIZE);
790 	}
791 	vmx->vm = vm;
792 
793 	vmx->eptp = eptp(vtophys((vm_offset_t)pmap->pm_pml4));
794 
795 	/*
796 	 * Clean up EPTP-tagged guest physical and combined mappings
797 	 *
798 	 * VMX transitions are not required to invalidate any guest physical
799 	 * mappings. So, it may be possible for stale guest physical mappings
800 	 * to be present in the processor TLBs.
801 	 *
802 	 * Combined mappings for this EP4TA are also invalidated for all VPIDs.
803 	 */
804 	ept_invalidate_mappings(vmx->eptp);
805 
806 	msr_bitmap_initialize(vmx->msr_bitmap);
807 
808 	/*
809 	 * It is safe to allow direct access to MSR_GSBASE and MSR_FSBASE.
810 	 * The guest FSBASE and GSBASE are saved and restored during
811 	 * vm-exit and vm-entry respectively. The host FSBASE and GSBASE are
812 	 * always restored from the vmcs host state area on vm-exit.
813 	 *
814 	 * The SYSENTER_CS/ESP/EIP MSRs are identical to FS/GSBASE in
815 	 * how they are saved/restored so can be directly accessed by the
816 	 * guest.
817 	 *
818 	 * Guest KGSBASE is saved and restored in the guest MSR save area.
819 	 * Host KGSBASE is restored before returning to userland from the pcb.
820 	 * There will be a window of time when we are executing in the host
821 	 * kernel context with a value of KGSBASE from the guest. This is ok
822 	 * because the value of KGSBASE is inconsequential in kernel context.
823 	 *
824 	 * MSR_EFER is saved and restored in the guest VMCS area on a
825 	 * VM exit and entry respectively. It is also restored from the
826 	 * host VMCS area on a VM exit.
827 	 */
828 	if (guest_msr_rw(vmx, MSR_GSBASE) ||
829 	    guest_msr_rw(vmx, MSR_FSBASE) ||
830 	    guest_msr_rw(vmx, MSR_SYSENTER_CS_MSR) ||
831 	    guest_msr_rw(vmx, MSR_SYSENTER_ESP_MSR) ||
832 	    guest_msr_rw(vmx, MSR_SYSENTER_EIP_MSR) ||
833 	    guest_msr_rw(vmx, MSR_KGSBASE) ||
834 	    guest_msr_rw(vmx, MSR_EFER))
835 		panic("vmx_vminit: error setting guest msr access");
836 
837 	/*
838 	 * MSR_PAT is saved and restored in the guest VMCS are on a VM exit
839 	 * and entry respectively. It is also restored from the host VMCS
840 	 * area on a VM exit. However, if running on a system with no
841 	 * MSR_PAT save/restore support, leave access disabled so accesses
842 	 * will be trapped.
843 	 */
844 	if (!vmx_no_patmsr && guest_msr_rw(vmx, MSR_PAT))
845 		panic("vmx_vminit: error setting guest pat msr access");
846 
847 	vpid_alloc(vpid, VM_MAXCPU);
848 
849 	if (virtual_interrupt_delivery) {
850 		error = vm_map_mmio(vm, DEFAULT_APIC_BASE, PAGE_SIZE,
851 		    APIC_ACCESS_ADDRESS);
852 		/* XXX this should really return an error to the caller */
853 		KASSERT(error == 0, ("vm_map_mmio(apicbase) error %d", error));
854 	}
855 
856 	for (i = 0; i < VM_MAXCPU; i++) {
857 		vmcs = &vmx->vmcs[i];
858 		vmcs->identifier = vmx_revision();
859 		error = vmclear(vmcs);
860 		if (error != 0) {
861 			panic("vmx_vminit: vmclear error %d on vcpu %d\n",
862 			      error, i);
863 		}
864 
865 		error = vmcs_init(vmcs);
866 		KASSERT(error == 0, ("vmcs_init error %d", error));
867 
868 		VMPTRLD(vmcs);
869 		error = 0;
870 		error += vmwrite(VMCS_HOST_RSP, (u_long)&vmx->ctx[i]);
871 		error += vmwrite(VMCS_EPTP, vmx->eptp);
872 		error += vmwrite(VMCS_PIN_BASED_CTLS, pinbased_ctls);
873 		error += vmwrite(VMCS_PRI_PROC_BASED_CTLS, procbased_ctls);
874 		error += vmwrite(VMCS_SEC_PROC_BASED_CTLS, procbased_ctls2);
875 		error += vmwrite(VMCS_EXIT_CTLS, exit_ctls);
876 		error += vmwrite(VMCS_ENTRY_CTLS, entry_ctls);
877 		error += vmwrite(VMCS_MSR_BITMAP, vtophys(vmx->msr_bitmap));
878 		error += vmwrite(VMCS_VPID, vpid[i]);
879 		if (virtual_interrupt_delivery) {
880 			error += vmwrite(VMCS_APIC_ACCESS, APIC_ACCESS_ADDRESS);
881 			error += vmwrite(VMCS_VIRTUAL_APIC,
882 			    vtophys(&vmx->apic_page[i]));
883 			error += vmwrite(VMCS_EOI_EXIT0, 0);
884 			error += vmwrite(VMCS_EOI_EXIT1, 0);
885 			error += vmwrite(VMCS_EOI_EXIT2, 0);
886 			error += vmwrite(VMCS_EOI_EXIT3, 0);
887 		}
888 		if (posted_interrupts) {
889 			error += vmwrite(VMCS_PIR_VECTOR, pirvec);
890 			error += vmwrite(VMCS_PIR_DESC,
891 			    vtophys(&vmx->pir_desc[i]));
892 		}
893 		VMCLEAR(vmcs);
894 		KASSERT(error == 0, ("vmx_vminit: error customizing the vmcs"));
895 
896 		vmx->cap[i].set = 0;
897 		vmx->cap[i].proc_ctls = procbased_ctls;
898 		vmx->cap[i].proc_ctls2 = procbased_ctls2;
899 
900 		vmx->state[i].lastcpu = -1;
901 		vmx->state[i].vpid = vpid[i];
902 
903 		msr_save_area_init(vmx->guest_msrs[i], &guest_msr_count);
904 
905 		error = vmcs_set_msr_save(vmcs, vtophys(vmx->guest_msrs[i]),
906 		    guest_msr_count);
907 		if (error != 0)
908 			panic("vmcs_set_msr_save error %d", error);
909 
910 		/*
911 		 * Set up the CR0/4 shadows, and init the read shadow
912 		 * to the power-on register value from the Intel Sys Arch.
913 		 *  CR0 - 0x60000010
914 		 *  CR4 - 0
915 		 */
916 		error = vmx_setup_cr0_shadow(vmcs, 0x60000010);
917 		if (error != 0)
918 			panic("vmx_setup_cr0_shadow %d", error);
919 
920 		error = vmx_setup_cr4_shadow(vmcs, 0);
921 		if (error != 0)
922 			panic("vmx_setup_cr4_shadow %d", error);
923 
924 		vmx->ctx[i].pmap = pmap;
925 		vmx->ctx[i].eptp = vmx->eptp;
926 	}
927 
928 	return (vmx);
929 }
930 
931 static int
932 vmx_handle_cpuid(struct vm *vm, int vcpu, struct vmxctx *vmxctx)
933 {
934 	int handled, func;
935 
936 	func = vmxctx->guest_rax;
937 
938 	handled = x86_emulate_cpuid(vm, vcpu,
939 				    (uint32_t*)(&vmxctx->guest_rax),
940 				    (uint32_t*)(&vmxctx->guest_rbx),
941 				    (uint32_t*)(&vmxctx->guest_rcx),
942 				    (uint32_t*)(&vmxctx->guest_rdx));
943 	return (handled);
944 }
945 
946 static __inline void
947 vmx_run_trace(struct vmx *vmx, int vcpu)
948 {
949 #ifdef KTR
950 	VCPU_CTR1(vmx->vm, vcpu, "Resume execution at %#lx", vmcs_guest_rip());
951 #endif
952 }
953 
954 static __inline void
955 vmx_exit_trace(struct vmx *vmx, int vcpu, uint64_t rip, uint32_t exit_reason,
956 	       int handled)
957 {
958 #ifdef KTR
959 	VCPU_CTR3(vmx->vm, vcpu, "%s %s vmexit at 0x%0lx",
960 		 handled ? "handled" : "unhandled",
961 		 exit_reason_to_str(exit_reason), rip);
962 #endif
963 }
964 
965 static __inline void
966 vmx_astpending_trace(struct vmx *vmx, int vcpu, uint64_t rip)
967 {
968 #ifdef KTR
969 	VCPU_CTR1(vmx->vm, vcpu, "astpending vmexit at 0x%0lx", rip);
970 #endif
971 }
972 
973 static void
974 vmx_set_pcpu_defaults(struct vmx *vmx, int vcpu)
975 {
976 	int lastcpu;
977 	struct vmxstate *vmxstate;
978 	struct invvpid_desc invvpid_desc = { 0 };
979 
980 	vmxstate = &vmx->state[vcpu];
981 	lastcpu = vmxstate->lastcpu;
982 	vmxstate->lastcpu = curcpu;
983 
984 	if (lastcpu == curcpu)
985 		return;
986 
987 	vmm_stat_incr(vmx->vm, vcpu, VCPU_MIGRATIONS, 1);
988 
989 	vmcs_write(VMCS_HOST_TR_BASE, vmm_get_host_trbase());
990 	vmcs_write(VMCS_HOST_GDTR_BASE, vmm_get_host_gdtrbase());
991 	vmcs_write(VMCS_HOST_GS_BASE, vmm_get_host_gsbase());
992 
993 	/*
994 	 * If we are using VPIDs then invalidate all mappings tagged with 'vpid'
995 	 *
996 	 * We do this because this vcpu was executing on a different host
997 	 * cpu when it last ran. We do not track whether it invalidated
998 	 * mappings associated with its 'vpid' during that run. So we must
999 	 * assume that the mappings associated with 'vpid' on 'curcpu' are
1000 	 * stale and invalidate them.
1001 	 *
1002 	 * Note that we incur this penalty only when the scheduler chooses to
1003 	 * move the thread associated with this vcpu between host cpus.
1004 	 *
1005 	 * Note also that this will invalidate mappings tagged with 'vpid'
1006 	 * for "all" EP4TAs.
1007 	 */
1008 	if (vmxstate->vpid != 0) {
1009 		invvpid_desc.vpid = vmxstate->vpid;
1010 		invvpid(INVVPID_TYPE_SINGLE_CONTEXT, invvpid_desc);
1011 	}
1012 }
1013 
1014 /*
1015  * We depend on 'procbased_ctls' to have the Interrupt Window Exiting bit set.
1016  */
1017 CTASSERT((PROCBASED_CTLS_ONE_SETTING & PROCBASED_INT_WINDOW_EXITING) != 0);
1018 
1019 static void __inline
1020 vmx_set_int_window_exiting(struct vmx *vmx, int vcpu)
1021 {
1022 
1023 	vmx->cap[vcpu].proc_ctls |= PROCBASED_INT_WINDOW_EXITING;
1024 	vmcs_write(VMCS_PRI_PROC_BASED_CTLS, vmx->cap[vcpu].proc_ctls);
1025 }
1026 
1027 static void __inline
1028 vmx_clear_int_window_exiting(struct vmx *vmx, int vcpu)
1029 {
1030 
1031 	vmx->cap[vcpu].proc_ctls &= ~PROCBASED_INT_WINDOW_EXITING;
1032 	vmcs_write(VMCS_PRI_PROC_BASED_CTLS, vmx->cap[vcpu].proc_ctls);
1033 }
1034 
1035 static void __inline
1036 vmx_set_nmi_window_exiting(struct vmx *vmx, int vcpu)
1037 {
1038 
1039 	vmx->cap[vcpu].proc_ctls |= PROCBASED_NMI_WINDOW_EXITING;
1040 	vmcs_write(VMCS_PRI_PROC_BASED_CTLS, vmx->cap[vcpu].proc_ctls);
1041 }
1042 
1043 static void __inline
1044 vmx_clear_nmi_window_exiting(struct vmx *vmx, int vcpu)
1045 {
1046 
1047 	vmx->cap[vcpu].proc_ctls &= ~PROCBASED_NMI_WINDOW_EXITING;
1048 	vmcs_write(VMCS_PRI_PROC_BASED_CTLS, vmx->cap[vcpu].proc_ctls);
1049 }
1050 
1051 static int
1052 vmx_inject_nmi(struct vmx *vmx, int vcpu)
1053 {
1054 	uint64_t info, interruptibility;
1055 
1056 	/* Bail out if no NMI requested */
1057 	if (!vm_nmi_pending(vmx->vm, vcpu))
1058 		return (0);
1059 
1060 	interruptibility = vmcs_read(VMCS_GUEST_INTERRUPTIBILITY);
1061 	if (interruptibility & nmi_blocking_bits)
1062 		goto nmiblocked;
1063 
1064 	/*
1065 	 * Inject the virtual NMI. The vector must be the NMI IDT entry
1066 	 * or the VMCS entry check will fail.
1067 	 */
1068 	info = VMCS_INTR_INFO_NMI | VMCS_INTR_INFO_VALID;
1069 	info |= IDT_NMI;
1070 	vmcs_write(VMCS_ENTRY_INTR_INFO, info);
1071 
1072 	VCPU_CTR0(vmx->vm, vcpu, "Injecting vNMI");
1073 
1074 	/* Clear the request */
1075 	vm_nmi_clear(vmx->vm, vcpu);
1076 	return (1);
1077 
1078 nmiblocked:
1079 	/*
1080 	 * Set the NMI Window Exiting execution control so we can inject
1081 	 * the virtual NMI as soon as blocking condition goes away.
1082 	 */
1083 	vmx_set_nmi_window_exiting(vmx, vcpu);
1084 
1085 	VCPU_CTR0(vmx->vm, vcpu, "Enabling NMI window exiting");
1086 	return (1);
1087 }
1088 
1089 static void
1090 vmx_inject_interrupts(struct vmx *vmx, int vcpu, struct vlapic *vlapic)
1091 {
1092 	int vector;
1093 	uint64_t info, rflags, interruptibility;
1094 
1095 	const int HWINTR_BLOCKED = VMCS_INTERRUPTIBILITY_STI_BLOCKING |
1096 				   VMCS_INTERRUPTIBILITY_MOVSS_BLOCKING;
1097 
1098 	/*
1099 	 * If there is already an interrupt pending then just return.
1100 	 *
1101 	 * This could happen if an interrupt was injected on a prior
1102 	 * VM entry but the actual entry into guest mode was aborted
1103 	 * because of a pending AST.
1104 	 */
1105 	info = vmcs_read(VMCS_ENTRY_INTR_INFO);
1106 	if (info & VMCS_INTR_INFO_VALID)
1107 		return;
1108 
1109 	/*
1110 	 * NMI injection has priority so deal with those first
1111 	 */
1112 	if (vmx_inject_nmi(vmx, vcpu))
1113 		return;
1114 
1115 	if (virtual_interrupt_delivery) {
1116 		vmx_inject_pir(vlapic);
1117 		return;
1118 	}
1119 
1120 	/* Ask the local apic for a vector to inject */
1121 	if (!vlapic_pending_intr(vlapic, &vector))
1122 		return;
1123 
1124 	if (vector < 32 || vector > 255)
1125 		panic("vmx_inject_interrupts: invalid vector %d\n", vector);
1126 
1127 	/* Check RFLAGS.IF and the interruptibility state of the guest */
1128 	rflags = vmcs_read(VMCS_GUEST_RFLAGS);
1129 	if ((rflags & PSL_I) == 0)
1130 		goto cantinject;
1131 
1132 	interruptibility = vmcs_read(VMCS_GUEST_INTERRUPTIBILITY);
1133 	if (interruptibility & HWINTR_BLOCKED)
1134 		goto cantinject;
1135 
1136 	/* Inject the interrupt */
1137 	info = VMCS_INTR_INFO_HW_INTR | VMCS_INTR_INFO_VALID;
1138 	info |= vector;
1139 	vmcs_write(VMCS_ENTRY_INTR_INFO, info);
1140 
1141 	/* Update the Local APIC ISR */
1142 	vlapic_intr_accepted(vlapic, vector);
1143 
1144 	VCPU_CTR1(vmx->vm, vcpu, "Injecting hwintr at vector %d", vector);
1145 
1146 	return;
1147 
1148 cantinject:
1149 	/*
1150 	 * Set the Interrupt Window Exiting execution control so we can inject
1151 	 * the interrupt as soon as blocking condition goes away.
1152 	 */
1153 	vmx_set_int_window_exiting(vmx, vcpu);
1154 
1155 	VCPU_CTR0(vmx->vm, vcpu, "Enabling interrupt window exiting");
1156 }
1157 
1158 static int
1159 vmx_emulate_cr_access(struct vmx *vmx, int vcpu, uint64_t exitqual)
1160 {
1161 	int cr, vmcs_guest_cr, vmcs_shadow_cr;
1162 	uint64_t crval, regval, ones_mask, zeros_mask;
1163 	const struct vmxctx *vmxctx;
1164 
1165 	/* We only handle mov to %cr0 or %cr4 at this time */
1166 	if ((exitqual & 0xf0) != 0x00)
1167 		return (UNHANDLED);
1168 
1169 	cr = exitqual & 0xf;
1170 	if (cr != 0 && cr != 4)
1171 		return (UNHANDLED);
1172 
1173 	regval = 0; /* silence gcc */
1174 	vmxctx = &vmx->ctx[vcpu];
1175 
1176 	/*
1177 	 * We must use vmcs_write() directly here because vmcs_setreg() will
1178 	 * call vmclear(vmcs) as a side-effect which we certainly don't want.
1179 	 */
1180 	switch ((exitqual >> 8) & 0xf) {
1181 	case 0:
1182 		regval = vmxctx->guest_rax;
1183 		break;
1184 	case 1:
1185 		regval = vmxctx->guest_rcx;
1186 		break;
1187 	case 2:
1188 		regval = vmxctx->guest_rdx;
1189 		break;
1190 	case 3:
1191 		regval = vmxctx->guest_rbx;
1192 		break;
1193 	case 4:
1194 		regval = vmcs_read(VMCS_GUEST_RSP);
1195 		break;
1196 	case 5:
1197 		regval = vmxctx->guest_rbp;
1198 		break;
1199 	case 6:
1200 		regval = vmxctx->guest_rsi;
1201 		break;
1202 	case 7:
1203 		regval = vmxctx->guest_rdi;
1204 		break;
1205 	case 8:
1206 		regval = vmxctx->guest_r8;
1207 		break;
1208 	case 9:
1209 		regval = vmxctx->guest_r9;
1210 		break;
1211 	case 10:
1212 		regval = vmxctx->guest_r10;
1213 		break;
1214 	case 11:
1215 		regval = vmxctx->guest_r11;
1216 		break;
1217 	case 12:
1218 		regval = vmxctx->guest_r12;
1219 		break;
1220 	case 13:
1221 		regval = vmxctx->guest_r13;
1222 		break;
1223 	case 14:
1224 		regval = vmxctx->guest_r14;
1225 		break;
1226 	case 15:
1227 		regval = vmxctx->guest_r15;
1228 		break;
1229 	}
1230 
1231 	if (cr == 0) {
1232 		ones_mask = cr0_ones_mask;
1233 		zeros_mask = cr0_zeros_mask;
1234 		vmcs_guest_cr = VMCS_GUEST_CR0;
1235 		vmcs_shadow_cr = VMCS_CR0_SHADOW;
1236 	} else {
1237 		ones_mask = cr4_ones_mask;
1238 		zeros_mask = cr4_zeros_mask;
1239 		vmcs_guest_cr = VMCS_GUEST_CR4;
1240 		vmcs_shadow_cr = VMCS_CR4_SHADOW;
1241 	}
1242 	vmcs_write(vmcs_shadow_cr, regval);
1243 
1244 	crval = regval | ones_mask;
1245 	crval &= ~zeros_mask;
1246 	vmcs_write(vmcs_guest_cr, crval);
1247 
1248 	if (cr == 0 && regval & CR0_PG) {
1249 		uint64_t efer, entry_ctls;
1250 
1251 		/*
1252 		 * If CR0.PG is 1 and EFER.LME is 1 then EFER.LMA and
1253 		 * the "IA-32e mode guest" bit in VM-entry control must be
1254 		 * equal.
1255 		 */
1256 		efer = vmcs_read(VMCS_GUEST_IA32_EFER);
1257 		if (efer & EFER_LME) {
1258 			efer |= EFER_LMA;
1259 			vmcs_write(VMCS_GUEST_IA32_EFER, efer);
1260 			entry_ctls = vmcs_read(VMCS_ENTRY_CTLS);
1261 			entry_ctls |= VM_ENTRY_GUEST_LMA;
1262 			vmcs_write(VMCS_ENTRY_CTLS, entry_ctls);
1263 		}
1264 	}
1265 
1266 	return (HANDLED);
1267 }
1268 
1269 static int
1270 ept_fault_type(uint64_t ept_qual)
1271 {
1272 	int fault_type;
1273 
1274 	if (ept_qual & EPT_VIOLATION_DATA_WRITE)
1275 		fault_type = VM_PROT_WRITE;
1276 	else if (ept_qual & EPT_VIOLATION_INST_FETCH)
1277 		fault_type = VM_PROT_EXECUTE;
1278 	else
1279 		fault_type= VM_PROT_READ;
1280 
1281 	return (fault_type);
1282 }
1283 
1284 static boolean_t
1285 ept_emulation_fault(uint64_t ept_qual)
1286 {
1287 	int read, write;
1288 
1289 	/* EPT fault on an instruction fetch doesn't make sense here */
1290 	if (ept_qual & EPT_VIOLATION_INST_FETCH)
1291 		return (FALSE);
1292 
1293 	/* EPT fault must be a read fault or a write fault */
1294 	read = ept_qual & EPT_VIOLATION_DATA_READ ? 1 : 0;
1295 	write = ept_qual & EPT_VIOLATION_DATA_WRITE ? 1 : 0;
1296 	if ((read | write) == 0)
1297 		return (FALSE);
1298 
1299 	/*
1300 	 * The EPT violation must have been caused by accessing a
1301 	 * guest-physical address that is a translation of a guest-linear
1302 	 * address.
1303 	 */
1304 	if ((ept_qual & EPT_VIOLATION_GLA_VALID) == 0 ||
1305 	    (ept_qual & EPT_VIOLATION_XLAT_VALID) == 0) {
1306 		return (FALSE);
1307 	}
1308 
1309 	return (TRUE);
1310 }
1311 
1312 static int
1313 vmx_handle_apic_write(struct vlapic *vlapic, uint64_t qual)
1314 {
1315 	int error, handled, offset;
1316 	bool retu;
1317 
1318 	if (!virtual_interrupt_delivery)
1319 		return (UNHANDLED);
1320 
1321 	handled = 1;
1322 	offset = APIC_WRITE_OFFSET(qual);
1323 	switch (offset) {
1324 	case APIC_OFFSET_ID:
1325 		vlapic_id_write_handler(vlapic);
1326 		break;
1327 	case APIC_OFFSET_LDR:
1328 		vlapic_ldr_write_handler(vlapic);
1329 		break;
1330 	case APIC_OFFSET_DFR:
1331 		vlapic_dfr_write_handler(vlapic);
1332 		break;
1333 	case APIC_OFFSET_SVR:
1334 		vlapic_svr_write_handler(vlapic);
1335 		break;
1336 	case APIC_OFFSET_ESR:
1337 		vlapic_esr_write_handler(vlapic);
1338 		break;
1339 	case APIC_OFFSET_ICR_LOW:
1340 		retu = false;
1341 		error = vlapic_icrlo_write_handler(vlapic, &retu);
1342 		if (error != 0 || retu)
1343 			handled = 0;
1344 		break;
1345 	case APIC_OFFSET_CMCI_LVT:
1346 	case APIC_OFFSET_TIMER_LVT ... APIC_OFFSET_ERROR_LVT:
1347 		vlapic_lvt_write_handler(vlapic, offset);
1348 		break;
1349 	case APIC_OFFSET_TIMER_ICR:
1350 		vlapic_icrtmr_write_handler(vlapic);
1351 		break;
1352 	case APIC_OFFSET_TIMER_DCR:
1353 		vlapic_dcr_write_handler(vlapic);
1354 		break;
1355 	default:
1356 		handled = 0;
1357 		break;
1358 	}
1359 	return (handled);
1360 }
1361 
1362 static bool
1363 apic_access_fault(uint64_t gpa)
1364 {
1365 
1366 	if (virtual_interrupt_delivery &&
1367 	    (gpa >= DEFAULT_APIC_BASE && gpa < DEFAULT_APIC_BASE + PAGE_SIZE))
1368 		return (true);
1369 	else
1370 		return (false);
1371 }
1372 
1373 static int
1374 vmx_handle_apic_access(struct vmx *vmx, int vcpuid, struct vm_exit *vmexit)
1375 {
1376 	uint64_t qual;
1377 	int access_type, offset, allowed;
1378 
1379 	if (!virtual_interrupt_delivery)
1380 		return (UNHANDLED);
1381 
1382 	qual = vmexit->u.vmx.exit_qualification;
1383 	access_type = APIC_ACCESS_TYPE(qual);
1384 	offset = APIC_ACCESS_OFFSET(qual);
1385 
1386 	allowed = 0;
1387 	if (access_type == 0) {
1388 		/*
1389 		 * Read data access to the following registers is expected.
1390 		 */
1391 		switch (offset) {
1392 		case APIC_OFFSET_APR:
1393 		case APIC_OFFSET_PPR:
1394 		case APIC_OFFSET_RRR:
1395 		case APIC_OFFSET_CMCI_LVT:
1396 		case APIC_OFFSET_TIMER_CCR:
1397 			allowed = 1;
1398 			break;
1399 		default:
1400 			break;
1401 		}
1402 	} else if (access_type == 1) {
1403 		/*
1404 		 * Write data access to the following registers is expected.
1405 		 */
1406 		switch (offset) {
1407 		case APIC_OFFSET_VER:
1408 		case APIC_OFFSET_APR:
1409 		case APIC_OFFSET_PPR:
1410 		case APIC_OFFSET_RRR:
1411 		case APIC_OFFSET_ISR0 ... APIC_OFFSET_ISR7:
1412 		case APIC_OFFSET_TMR0 ... APIC_OFFSET_TMR7:
1413 		case APIC_OFFSET_IRR0 ... APIC_OFFSET_IRR7:
1414 		case APIC_OFFSET_CMCI_LVT:
1415 		case APIC_OFFSET_TIMER_CCR:
1416 			allowed = 1;
1417 			break;
1418 		default:
1419 			break;
1420 		}
1421 	}
1422 
1423 	if (allowed) {
1424 		vmexit->exitcode = VM_EXITCODE_INST_EMUL;
1425 		vmexit->u.inst_emul.gpa = DEFAULT_APIC_BASE + offset;
1426 		vmexit->u.inst_emul.gla = VIE_INVALID_GLA;
1427 		vmexit->u.inst_emul.cr3 = vmcs_guest_cr3();
1428 	}
1429 
1430 	/*
1431 	 * Regardless of whether the APIC-access is allowed this handler
1432 	 * always returns UNHANDLED:
1433 	 * - if the access is allowed then it is handled by emulating the
1434 	 *   instruction that caused the VM-exit (outside the critical section)
1435 	 * - if the access is not allowed then it will be converted to an
1436 	 *   exitcode of VM_EXITCODE_VMX and will be dealt with in userland.
1437 	 */
1438 	return (UNHANDLED);
1439 }
1440 
1441 static int
1442 vmx_exit_process(struct vmx *vmx, int vcpu, struct vm_exit *vmexit)
1443 {
1444 	int error, handled;
1445 	struct vmxctx *vmxctx;
1446 	struct vlapic *vlapic;
1447 	uint32_t eax, ecx, edx, idtvec_info, idtvec_err, intr_info, reason;
1448 	uint64_t qual, gpa;
1449 	bool retu;
1450 
1451 	handled = 0;
1452 	vmxctx = &vmx->ctx[vcpu];
1453 
1454 	qual = vmexit->u.vmx.exit_qualification;
1455 	reason = vmexit->u.vmx.exit_reason;
1456 	vmexit->exitcode = VM_EXITCODE_BOGUS;
1457 
1458 	vmm_stat_incr(vmx->vm, vcpu, VMEXIT_COUNT, 1);
1459 
1460 	/*
1461 	 * VM exits that could be triggered during event injection on the
1462 	 * previous VM entry need to be handled specially by re-injecting
1463 	 * the event.
1464 	 *
1465 	 * See "Information for VM Exits During Event Delivery" in Intel SDM
1466 	 * for details.
1467 	 */
1468 	switch (reason) {
1469 	case EXIT_REASON_EPT_FAULT:
1470 	case EXIT_REASON_EPT_MISCONFIG:
1471 	case EXIT_REASON_APIC_ACCESS:
1472 	case EXIT_REASON_TASK_SWITCH:
1473 	case EXIT_REASON_EXCEPTION:
1474 		idtvec_info = vmcs_idt_vectoring_info();
1475 		if (idtvec_info & VMCS_IDT_VEC_VALID) {
1476 			idtvec_info &= ~(1 << 12); /* clear undefined bit */
1477 			vmcs_write(VMCS_ENTRY_INTR_INFO, idtvec_info);
1478 			if (idtvec_info & VMCS_IDT_VEC_ERRCODE_VALID) {
1479 				idtvec_err = vmcs_idt_vectoring_err();
1480 				vmcs_write(VMCS_ENTRY_EXCEPTION_ERROR,
1481 				    idtvec_err);
1482 			}
1483 			vmcs_write(VMCS_ENTRY_INST_LENGTH, vmexit->inst_length);
1484 		}
1485 	default:
1486 		break;
1487 	}
1488 
1489 	switch (reason) {
1490 	case EXIT_REASON_CR_ACCESS:
1491 		vmm_stat_incr(vmx->vm, vcpu, VMEXIT_CR_ACCESS, 1);
1492 		handled = vmx_emulate_cr_access(vmx, vcpu, qual);
1493 		break;
1494 	case EXIT_REASON_RDMSR:
1495 		vmm_stat_incr(vmx->vm, vcpu, VMEXIT_RDMSR, 1);
1496 		retu = false;
1497 		ecx = vmxctx->guest_rcx;
1498 		error = emulate_rdmsr(vmx->vm, vcpu, ecx, &retu);
1499 		if (error) {
1500 			vmexit->exitcode = VM_EXITCODE_RDMSR;
1501 			vmexit->u.msr.code = ecx;
1502 		} else if (!retu) {
1503 			handled = 1;
1504 		} else {
1505 			/* Return to userspace with a valid exitcode */
1506 			KASSERT(vmexit->exitcode != VM_EXITCODE_BOGUS,
1507 			    ("emulate_wrmsr retu with bogus exitcode"));
1508 		}
1509 		break;
1510 	case EXIT_REASON_WRMSR:
1511 		vmm_stat_incr(vmx->vm, vcpu, VMEXIT_WRMSR, 1);
1512 		retu = false;
1513 		eax = vmxctx->guest_rax;
1514 		ecx = vmxctx->guest_rcx;
1515 		edx = vmxctx->guest_rdx;
1516 		error = emulate_wrmsr(vmx->vm, vcpu, ecx,
1517 		    (uint64_t)edx << 32 | eax, &retu);
1518 		if (error) {
1519 			vmexit->exitcode = VM_EXITCODE_WRMSR;
1520 			vmexit->u.msr.code = ecx;
1521 			vmexit->u.msr.wval = (uint64_t)edx << 32 | eax;
1522 		} else if (!retu) {
1523 			handled = 1;
1524 		} else {
1525 			/* Return to userspace with a valid exitcode */
1526 			KASSERT(vmexit->exitcode != VM_EXITCODE_BOGUS,
1527 			    ("emulate_wrmsr retu with bogus exitcode"));
1528 		}
1529 		break;
1530 	case EXIT_REASON_HLT:
1531 		vmm_stat_incr(vmx->vm, vcpu, VMEXIT_HLT, 1);
1532 		vmexit->exitcode = VM_EXITCODE_HLT;
1533 		vmexit->u.hlt.rflags = vmcs_read(VMCS_GUEST_RFLAGS);
1534 		break;
1535 	case EXIT_REASON_MTF:
1536 		vmm_stat_incr(vmx->vm, vcpu, VMEXIT_MTRAP, 1);
1537 		vmexit->exitcode = VM_EXITCODE_MTRAP;
1538 		break;
1539 	case EXIT_REASON_PAUSE:
1540 		vmm_stat_incr(vmx->vm, vcpu, VMEXIT_PAUSE, 1);
1541 		vmexit->exitcode = VM_EXITCODE_PAUSE;
1542 		break;
1543 	case EXIT_REASON_INTR_WINDOW:
1544 		vmm_stat_incr(vmx->vm, vcpu, VMEXIT_INTR_WINDOW, 1);
1545 		vmx_clear_int_window_exiting(vmx, vcpu);
1546 		VCPU_CTR0(vmx->vm, vcpu, "Disabling interrupt window exiting");
1547 		return (1);
1548 	case EXIT_REASON_EXT_INTR:
1549 		/*
1550 		 * External interrupts serve only to cause VM exits and allow
1551 		 * the host interrupt handler to run.
1552 		 *
1553 		 * If this external interrupt triggers a virtual interrupt
1554 		 * to a VM, then that state will be recorded by the
1555 		 * host interrupt handler in the VM's softc. We will inject
1556 		 * this virtual interrupt during the subsequent VM enter.
1557 		 */
1558 		intr_info = vmcs_read(VMCS_EXIT_INTR_INFO);
1559 		KASSERT((intr_info & VMCS_INTR_INFO_VALID) != 0 &&
1560 		    VMCS_INTR_INFO_TYPE(intr_info) == 0,
1561 		    ("VM exit interruption info invalid: %#x", intr_info));
1562 		vmx_trigger_hostintr(intr_info & 0xff);
1563 
1564 		/*
1565 		 * This is special. We want to treat this as an 'handled'
1566 		 * VM-exit but not increment the instruction pointer.
1567 		 */
1568 		vmm_stat_incr(vmx->vm, vcpu, VMEXIT_EXTINT, 1);
1569 		return (1);
1570 	case EXIT_REASON_NMI_WINDOW:
1571 		/* Exit to allow the pending virtual NMI to be injected */
1572 		vmm_stat_incr(vmx->vm, vcpu, VMEXIT_NMI_WINDOW, 1);
1573 		vmx_clear_nmi_window_exiting(vmx, vcpu);
1574 		VCPU_CTR0(vmx->vm, vcpu, "Disabling NMI window exiting");
1575 		return (1);
1576 	case EXIT_REASON_INOUT:
1577 		vmm_stat_incr(vmx->vm, vcpu, VMEXIT_INOUT, 1);
1578 		vmexit->exitcode = VM_EXITCODE_INOUT;
1579 		vmexit->u.inout.bytes = (qual & 0x7) + 1;
1580 		vmexit->u.inout.in = (qual & 0x8) ? 1 : 0;
1581 		vmexit->u.inout.string = (qual & 0x10) ? 1 : 0;
1582 		vmexit->u.inout.rep = (qual & 0x20) ? 1 : 0;
1583 		vmexit->u.inout.port = (uint16_t)(qual >> 16);
1584 		vmexit->u.inout.eax = (uint32_t)(vmxctx->guest_rax);
1585 		break;
1586 	case EXIT_REASON_CPUID:
1587 		vmm_stat_incr(vmx->vm, vcpu, VMEXIT_CPUID, 1);
1588 		handled = vmx_handle_cpuid(vmx->vm, vcpu, vmxctx);
1589 		break;
1590 	case EXIT_REASON_EPT_FAULT:
1591 		vmm_stat_incr(vmx->vm, vcpu, VMEXIT_EPT_FAULT, 1);
1592 		/*
1593 		 * If 'gpa' lies within the address space allocated to
1594 		 * memory then this must be a nested page fault otherwise
1595 		 * this must be an instruction that accesses MMIO space.
1596 		 */
1597 		gpa = vmcs_gpa();
1598 		if (vm_mem_allocated(vmx->vm, gpa) || apic_access_fault(gpa)) {
1599 			vmexit->exitcode = VM_EXITCODE_PAGING;
1600 			vmexit->u.paging.gpa = gpa;
1601 			vmexit->u.paging.fault_type = ept_fault_type(qual);
1602 		} else if (ept_emulation_fault(qual)) {
1603 			vmexit->exitcode = VM_EXITCODE_INST_EMUL;
1604 			vmexit->u.inst_emul.gpa = gpa;
1605 			vmexit->u.inst_emul.gla = vmcs_gla();
1606 			vmexit->u.inst_emul.cr3 = vmcs_guest_cr3();
1607 		}
1608 		break;
1609 	case EXIT_REASON_APIC_ACCESS:
1610 		handled = vmx_handle_apic_access(vmx, vcpu, vmexit);
1611 		break;
1612 	case EXIT_REASON_APIC_WRITE:
1613 		/*
1614 		 * APIC-write VM exit is trap-like so the %rip is already
1615 		 * pointing to the next instruction.
1616 		 */
1617 		vmexit->inst_length = 0;
1618 		vlapic = vm_lapic(vmx->vm, vcpu);
1619 		handled = vmx_handle_apic_write(vlapic, qual);
1620 		break;
1621 	default:
1622 		vmm_stat_incr(vmx->vm, vcpu, VMEXIT_UNKNOWN, 1);
1623 		break;
1624 	}
1625 
1626 	if (handled) {
1627 		/*
1628 		 * It is possible that control is returned to userland
1629 		 * even though we were able to handle the VM exit in the
1630 		 * kernel.
1631 		 *
1632 		 * In such a case we want to make sure that the userland
1633 		 * restarts guest execution at the instruction *after*
1634 		 * the one we just processed. Therefore we update the
1635 		 * guest rip in the VMCS and in 'vmexit'.
1636 		 */
1637 		vmexit->rip += vmexit->inst_length;
1638 		vmexit->inst_length = 0;
1639 		vmcs_write(VMCS_GUEST_RIP, vmexit->rip);
1640 	} else {
1641 		if (vmexit->exitcode == VM_EXITCODE_BOGUS) {
1642 			/*
1643 			 * If this VM exit was not claimed by anybody then
1644 			 * treat it as a generic VMX exit.
1645 			 */
1646 			vmexit->exitcode = VM_EXITCODE_VMX;
1647 			vmexit->u.vmx.status = VM_SUCCESS;
1648 		} else {
1649 			/*
1650 			 * The exitcode and collateral have been populated.
1651 			 * The VM exit will be processed further in userland.
1652 			 */
1653 		}
1654 	}
1655 	return (handled);
1656 }
1657 
1658 static __inline int
1659 vmx_exit_astpending(struct vmx *vmx, int vcpu, struct vm_exit *vmexit)
1660 {
1661 
1662 	vmexit->rip = vmcs_guest_rip();
1663 	vmexit->inst_length = 0;
1664 	vmexit->exitcode = VM_EXITCODE_BOGUS;
1665 	vmx_astpending_trace(vmx, vcpu, vmexit->rip);
1666 	vmm_stat_incr(vmx->vm, vcpu, VMEXIT_ASTPENDING, 1);
1667 
1668 	return (HANDLED);
1669 }
1670 
1671 static __inline int
1672 vmx_exit_inst_error(struct vmxctx *vmxctx, int rc, struct vm_exit *vmexit)
1673 {
1674 
1675 	KASSERT(vmxctx->inst_fail_status != VM_SUCCESS,
1676 	    ("vmx_exit_inst_error: invalid inst_fail_status %d",
1677 	    vmxctx->inst_fail_status));
1678 
1679 	vmexit->inst_length = 0;
1680 	vmexit->exitcode = VM_EXITCODE_VMX;
1681 	vmexit->u.vmx.status = vmxctx->inst_fail_status;
1682 	vmexit->u.vmx.inst_error = vmcs_instruction_error();
1683 	vmexit->u.vmx.exit_reason = ~0;
1684 	vmexit->u.vmx.exit_qualification = ~0;
1685 
1686 	switch (rc) {
1687 	case VMX_VMRESUME_ERROR:
1688 	case VMX_VMLAUNCH_ERROR:
1689 	case VMX_INVEPT_ERROR:
1690 		vmexit->u.vmx.inst_type = rc;
1691 		break;
1692 	default:
1693 		panic("vm_exit_inst_error: vmx_enter_guest returned %d", rc);
1694 	}
1695 
1696 	return (UNHANDLED);
1697 }
1698 
1699 static int
1700 vmx_run(void *arg, int vcpu, register_t startrip, pmap_t pmap)
1701 {
1702 	int rc, handled, launched;
1703 	struct vmx *vmx;
1704 	struct vmxctx *vmxctx;
1705 	struct vmcs *vmcs;
1706 	struct vm_exit *vmexit;
1707 	struct vlapic *vlapic;
1708 	uint64_t rip;
1709 	uint32_t exit_reason;
1710 
1711 	vmx = arg;
1712 	vmcs = &vmx->vmcs[vcpu];
1713 	vmxctx = &vmx->ctx[vcpu];
1714 	vlapic = vm_lapic(vmx->vm, vcpu);
1715 	vmexit = vm_exitinfo(vmx->vm, vcpu);
1716 	launched = 0;
1717 
1718 	KASSERT(vmxctx->pmap == pmap,
1719 	    ("pmap %p different than ctx pmap %p", pmap, vmxctx->pmap));
1720 	KASSERT(vmxctx->eptp == vmx->eptp,
1721 	    ("eptp %p different than ctx eptp %#lx", eptp, vmxctx->eptp));
1722 
1723 	VMPTRLD(vmcs);
1724 
1725 	/*
1726 	 * XXX
1727 	 * We do this every time because we may setup the virtual machine
1728 	 * from a different process than the one that actually runs it.
1729 	 *
1730 	 * If the life of a virtual machine was spent entirely in the context
1731 	 * of a single process we could do this once in vmx_vminit().
1732 	 */
1733 	vmcs_write(VMCS_HOST_CR3, rcr3());
1734 
1735 	vmcs_write(VMCS_GUEST_RIP, startrip);
1736 	vmx_set_pcpu_defaults(vmx, vcpu);
1737 	do {
1738 		/*
1739 		 * Interrupts are disabled from this point on until the
1740 		 * guest starts executing. This is done for the following
1741 		 * reasons:
1742 		 *
1743 		 * If an AST is asserted on this thread after the check below,
1744 		 * then the IPI_AST notification will not be lost, because it
1745 		 * will cause a VM exit due to external interrupt as soon as
1746 		 * the guest state is loaded.
1747 		 *
1748 		 * A posted interrupt after 'vmx_inject_interrupts()' will
1749 		 * not be "lost" because it will be held pending in the host
1750 		 * APIC because interrupts are disabled. The pending interrupt
1751 		 * will be recognized as soon as the guest state is loaded.
1752 		 *
1753 		 * The same reasoning applies to the IPI generated by
1754 		 * pmap_invalidate_ept().
1755 		 */
1756 		disable_intr();
1757 		if (curthread->td_flags & (TDF_ASTPENDING | TDF_NEEDRESCHED)) {
1758 			enable_intr();
1759 			handled = vmx_exit_astpending(vmx, vcpu, vmexit);
1760 			break;
1761 		}
1762 
1763 		vmx_inject_interrupts(vmx, vcpu, vlapic);
1764 		vmx_run_trace(vmx, vcpu);
1765 		rc = vmx_enter_guest(vmxctx, launched);
1766 
1767 		enable_intr();
1768 
1769 		/* Collect some information for VM exit processing */
1770 		vmexit->rip = rip = vmcs_guest_rip();
1771 		vmexit->inst_length = vmexit_instruction_length();
1772 		vmexit->u.vmx.exit_reason = exit_reason = vmcs_exit_reason();
1773 		vmexit->u.vmx.exit_qualification = vmcs_exit_qualification();
1774 
1775 		if (rc == VMX_GUEST_VMEXIT) {
1776 			launched = 1;
1777 			handled = vmx_exit_process(vmx, vcpu, vmexit);
1778 		} else {
1779 			handled = vmx_exit_inst_error(vmxctx, rc, vmexit);
1780 		}
1781 
1782 		vmx_exit_trace(vmx, vcpu, rip, exit_reason, handled);
1783 	} while (handled);
1784 
1785 	/*
1786 	 * If a VM exit has been handled then the exitcode must be BOGUS
1787 	 * If a VM exit is not handled then the exitcode must not be BOGUS
1788 	 */
1789 	if ((handled && vmexit->exitcode != VM_EXITCODE_BOGUS) ||
1790 	    (!handled && vmexit->exitcode == VM_EXITCODE_BOGUS)) {
1791 		panic("Mismatch between handled (%d) and exitcode (%d)",
1792 		      handled, vmexit->exitcode);
1793 	}
1794 
1795 	if (!handled)
1796 		vmm_stat_incr(vmx->vm, vcpu, VMEXIT_USERSPACE, 1);
1797 
1798 	VCPU_CTR1(vmx->vm, vcpu, "returning from vmx_run: exitcode %d",
1799 	    vmexit->exitcode);
1800 
1801 	VMCLEAR(vmcs);
1802 	return (0);
1803 }
1804 
1805 static void
1806 vmx_vmcleanup(void *arg)
1807 {
1808 	int i, error;
1809 	struct vmx *vmx = arg;
1810 
1811 	if (virtual_interrupt_delivery)
1812 		vm_unmap_mmio(vmx->vm, DEFAULT_APIC_BASE, PAGE_SIZE);
1813 
1814 	for (i = 0; i < VM_MAXCPU; i++)
1815 		vpid_free(vmx->state[i].vpid);
1816 
1817 	/*
1818 	 * XXXSMP we also need to clear the VMCS active on the other vcpus.
1819 	 */
1820 	error = vmclear(&vmx->vmcs[0]);
1821 	if (error != 0)
1822 		panic("vmx_vmcleanup: vmclear error %d on vcpu 0", error);
1823 
1824 	free(vmx, M_VMX);
1825 
1826 	return;
1827 }
1828 
1829 static register_t *
1830 vmxctx_regptr(struct vmxctx *vmxctx, int reg)
1831 {
1832 
1833 	switch (reg) {
1834 	case VM_REG_GUEST_RAX:
1835 		return (&vmxctx->guest_rax);
1836 	case VM_REG_GUEST_RBX:
1837 		return (&vmxctx->guest_rbx);
1838 	case VM_REG_GUEST_RCX:
1839 		return (&vmxctx->guest_rcx);
1840 	case VM_REG_GUEST_RDX:
1841 		return (&vmxctx->guest_rdx);
1842 	case VM_REG_GUEST_RSI:
1843 		return (&vmxctx->guest_rsi);
1844 	case VM_REG_GUEST_RDI:
1845 		return (&vmxctx->guest_rdi);
1846 	case VM_REG_GUEST_RBP:
1847 		return (&vmxctx->guest_rbp);
1848 	case VM_REG_GUEST_R8:
1849 		return (&vmxctx->guest_r8);
1850 	case VM_REG_GUEST_R9:
1851 		return (&vmxctx->guest_r9);
1852 	case VM_REG_GUEST_R10:
1853 		return (&vmxctx->guest_r10);
1854 	case VM_REG_GUEST_R11:
1855 		return (&vmxctx->guest_r11);
1856 	case VM_REG_GUEST_R12:
1857 		return (&vmxctx->guest_r12);
1858 	case VM_REG_GUEST_R13:
1859 		return (&vmxctx->guest_r13);
1860 	case VM_REG_GUEST_R14:
1861 		return (&vmxctx->guest_r14);
1862 	case VM_REG_GUEST_R15:
1863 		return (&vmxctx->guest_r15);
1864 	default:
1865 		break;
1866 	}
1867 	return (NULL);
1868 }
1869 
1870 static int
1871 vmxctx_getreg(struct vmxctx *vmxctx, int reg, uint64_t *retval)
1872 {
1873 	register_t *regp;
1874 
1875 	if ((regp = vmxctx_regptr(vmxctx, reg)) != NULL) {
1876 		*retval = *regp;
1877 		return (0);
1878 	} else
1879 		return (EINVAL);
1880 }
1881 
1882 static int
1883 vmxctx_setreg(struct vmxctx *vmxctx, int reg, uint64_t val)
1884 {
1885 	register_t *regp;
1886 
1887 	if ((regp = vmxctx_regptr(vmxctx, reg)) != NULL) {
1888 		*regp = val;
1889 		return (0);
1890 	} else
1891 		return (EINVAL);
1892 }
1893 
1894 static int
1895 vmx_shadow_reg(int reg)
1896 {
1897 	int shreg;
1898 
1899 	shreg = -1;
1900 
1901 	switch (reg) {
1902 	case VM_REG_GUEST_CR0:
1903 		shreg = VMCS_CR0_SHADOW;
1904                 break;
1905         case VM_REG_GUEST_CR4:
1906 		shreg = VMCS_CR4_SHADOW;
1907 		break;
1908 	default:
1909 		break;
1910 	}
1911 
1912 	return (shreg);
1913 }
1914 
1915 static int
1916 vmx_getreg(void *arg, int vcpu, int reg, uint64_t *retval)
1917 {
1918 	int running, hostcpu;
1919 	struct vmx *vmx = arg;
1920 
1921 	running = vcpu_is_running(vmx->vm, vcpu, &hostcpu);
1922 	if (running && hostcpu != curcpu)
1923 		panic("vmx_getreg: %s%d is running", vm_name(vmx->vm), vcpu);
1924 
1925 	if (vmxctx_getreg(&vmx->ctx[vcpu], reg, retval) == 0)
1926 		return (0);
1927 
1928 	return (vmcs_getreg(&vmx->vmcs[vcpu], running, reg, retval));
1929 }
1930 
1931 static int
1932 vmx_setreg(void *arg, int vcpu, int reg, uint64_t val)
1933 {
1934 	int error, hostcpu, running, shadow;
1935 	uint64_t ctls;
1936 	struct vmx *vmx = arg;
1937 
1938 	running = vcpu_is_running(vmx->vm, vcpu, &hostcpu);
1939 	if (running && hostcpu != curcpu)
1940 		panic("vmx_setreg: %s%d is running", vm_name(vmx->vm), vcpu);
1941 
1942 	if (vmxctx_setreg(&vmx->ctx[vcpu], reg, val) == 0)
1943 		return (0);
1944 
1945 	error = vmcs_setreg(&vmx->vmcs[vcpu], running, reg, val);
1946 
1947 	if (error == 0) {
1948 		/*
1949 		 * If the "load EFER" VM-entry control is 1 then the
1950 		 * value of EFER.LMA must be identical to "IA-32e mode guest"
1951 		 * bit in the VM-entry control.
1952 		 */
1953 		if ((entry_ctls & VM_ENTRY_LOAD_EFER) != 0 &&
1954 		    (reg == VM_REG_GUEST_EFER)) {
1955 			vmcs_getreg(&vmx->vmcs[vcpu], running,
1956 				    VMCS_IDENT(VMCS_ENTRY_CTLS), &ctls);
1957 			if (val & EFER_LMA)
1958 				ctls |= VM_ENTRY_GUEST_LMA;
1959 			else
1960 				ctls &= ~VM_ENTRY_GUEST_LMA;
1961 			vmcs_setreg(&vmx->vmcs[vcpu], running,
1962 				    VMCS_IDENT(VMCS_ENTRY_CTLS), ctls);
1963 		}
1964 
1965 		shadow = vmx_shadow_reg(reg);
1966 		if (shadow > 0) {
1967 			/*
1968 			 * Store the unmodified value in the shadow
1969 			 */
1970 			error = vmcs_setreg(&vmx->vmcs[vcpu], running,
1971 				    VMCS_IDENT(shadow), val);
1972 		}
1973 	}
1974 
1975 	return (error);
1976 }
1977 
1978 static int
1979 vmx_getdesc(void *arg, int vcpu, int reg, struct seg_desc *desc)
1980 {
1981 	struct vmx *vmx = arg;
1982 
1983 	return (vmcs_getdesc(&vmx->vmcs[vcpu], reg, desc));
1984 }
1985 
1986 static int
1987 vmx_setdesc(void *arg, int vcpu, int reg, struct seg_desc *desc)
1988 {
1989 	struct vmx *vmx = arg;
1990 
1991 	return (vmcs_setdesc(&vmx->vmcs[vcpu], reg, desc));
1992 }
1993 
1994 static int
1995 vmx_inject(void *arg, int vcpu, int type, int vector, uint32_t code,
1996 	   int code_valid)
1997 {
1998 	int error;
1999 	uint64_t info;
2000 	struct vmx *vmx = arg;
2001 	struct vmcs *vmcs = &vmx->vmcs[vcpu];
2002 
2003 	static uint32_t type_map[VM_EVENT_MAX] = {
2004 		0x1,		/* VM_EVENT_NONE */
2005 		0x0,		/* VM_HW_INTR */
2006 		0x2,		/* VM_NMI */
2007 		0x3,		/* VM_HW_EXCEPTION */
2008 		0x4,		/* VM_SW_INTR */
2009 		0x5,		/* VM_PRIV_SW_EXCEPTION */
2010 		0x6,		/* VM_SW_EXCEPTION */
2011 	};
2012 
2013 	/*
2014 	 * If there is already an exception pending to be delivered to the
2015 	 * vcpu then just return.
2016 	 */
2017 	error = vmcs_getreg(vmcs, 0, VMCS_IDENT(VMCS_ENTRY_INTR_INFO), &info);
2018 	if (error)
2019 		return (error);
2020 
2021 	if (info & VMCS_INTR_INFO_VALID)
2022 		return (EAGAIN);
2023 
2024 	info = vector | (type_map[type] << 8) | (code_valid ? 1 << 11 : 0);
2025 	info |= VMCS_INTR_INFO_VALID;
2026 	error = vmcs_setreg(vmcs, 0, VMCS_IDENT(VMCS_ENTRY_INTR_INFO), info);
2027 	if (error != 0)
2028 		return (error);
2029 
2030 	if (code_valid) {
2031 		error = vmcs_setreg(vmcs, 0,
2032 				    VMCS_IDENT(VMCS_ENTRY_EXCEPTION_ERROR),
2033 				    code);
2034 	}
2035 	return (error);
2036 }
2037 
2038 static int
2039 vmx_getcap(void *arg, int vcpu, int type, int *retval)
2040 {
2041 	struct vmx *vmx = arg;
2042 	int vcap;
2043 	int ret;
2044 
2045 	ret = ENOENT;
2046 
2047 	vcap = vmx->cap[vcpu].set;
2048 
2049 	switch (type) {
2050 	case VM_CAP_HALT_EXIT:
2051 		if (cap_halt_exit)
2052 			ret = 0;
2053 		break;
2054 	case VM_CAP_PAUSE_EXIT:
2055 		if (cap_pause_exit)
2056 			ret = 0;
2057 		break;
2058 	case VM_CAP_MTRAP_EXIT:
2059 		if (cap_monitor_trap)
2060 			ret = 0;
2061 		break;
2062 	case VM_CAP_UNRESTRICTED_GUEST:
2063 		if (cap_unrestricted_guest)
2064 			ret = 0;
2065 		break;
2066 	case VM_CAP_ENABLE_INVPCID:
2067 		if (cap_invpcid)
2068 			ret = 0;
2069 		break;
2070 	default:
2071 		break;
2072 	}
2073 
2074 	if (ret == 0)
2075 		*retval = (vcap & (1 << type)) ? 1 : 0;
2076 
2077 	return (ret);
2078 }
2079 
2080 static int
2081 vmx_setcap(void *arg, int vcpu, int type, int val)
2082 {
2083 	struct vmx *vmx = arg;
2084 	struct vmcs *vmcs = &vmx->vmcs[vcpu];
2085 	uint32_t baseval;
2086 	uint32_t *pptr;
2087 	int error;
2088 	int flag;
2089 	int reg;
2090 	int retval;
2091 
2092 	retval = ENOENT;
2093 	pptr = NULL;
2094 
2095 	switch (type) {
2096 	case VM_CAP_HALT_EXIT:
2097 		if (cap_halt_exit) {
2098 			retval = 0;
2099 			pptr = &vmx->cap[vcpu].proc_ctls;
2100 			baseval = *pptr;
2101 			flag = PROCBASED_HLT_EXITING;
2102 			reg = VMCS_PRI_PROC_BASED_CTLS;
2103 		}
2104 		break;
2105 	case VM_CAP_MTRAP_EXIT:
2106 		if (cap_monitor_trap) {
2107 			retval = 0;
2108 			pptr = &vmx->cap[vcpu].proc_ctls;
2109 			baseval = *pptr;
2110 			flag = PROCBASED_MTF;
2111 			reg = VMCS_PRI_PROC_BASED_CTLS;
2112 		}
2113 		break;
2114 	case VM_CAP_PAUSE_EXIT:
2115 		if (cap_pause_exit) {
2116 			retval = 0;
2117 			pptr = &vmx->cap[vcpu].proc_ctls;
2118 			baseval = *pptr;
2119 			flag = PROCBASED_PAUSE_EXITING;
2120 			reg = VMCS_PRI_PROC_BASED_CTLS;
2121 		}
2122 		break;
2123 	case VM_CAP_UNRESTRICTED_GUEST:
2124 		if (cap_unrestricted_guest) {
2125 			retval = 0;
2126 			pptr = &vmx->cap[vcpu].proc_ctls2;
2127 			baseval = *pptr;
2128 			flag = PROCBASED2_UNRESTRICTED_GUEST;
2129 			reg = VMCS_SEC_PROC_BASED_CTLS;
2130 		}
2131 		break;
2132 	case VM_CAP_ENABLE_INVPCID:
2133 		if (cap_invpcid) {
2134 			retval = 0;
2135 			pptr = &vmx->cap[vcpu].proc_ctls2;
2136 			baseval = *pptr;
2137 			flag = PROCBASED2_ENABLE_INVPCID;
2138 			reg = VMCS_SEC_PROC_BASED_CTLS;
2139 		}
2140 		break;
2141 	default:
2142 		break;
2143 	}
2144 
2145 	if (retval == 0) {
2146 		if (val) {
2147 			baseval |= flag;
2148 		} else {
2149 			baseval &= ~flag;
2150 		}
2151 		VMPTRLD(vmcs);
2152 		error = vmwrite(reg, baseval);
2153 		VMCLEAR(vmcs);
2154 
2155 		if (error) {
2156 			retval = error;
2157 		} else {
2158 			/*
2159 			 * Update optional stored flags, and record
2160 			 * setting
2161 			 */
2162 			if (pptr != NULL) {
2163 				*pptr = baseval;
2164 			}
2165 
2166 			if (val) {
2167 				vmx->cap[vcpu].set |= (1 << type);
2168 			} else {
2169 				vmx->cap[vcpu].set &= ~(1 << type);
2170 			}
2171 		}
2172 	}
2173 
2174         return (retval);
2175 }
2176 
2177 struct vlapic_vtx {
2178 	struct vlapic	vlapic;
2179 	struct pir_desc	*pir_desc;
2180 };
2181 
2182 #define	VMX_CTR_PIR(vm, vcpuid, pir_desc, notify, vector, level, msg)	\
2183 do {									\
2184 	VCPU_CTR2(vm, vcpuid, msg " assert %s-triggered vector %d",	\
2185 	    level ? "level" : "edge", vector);				\
2186 	VCPU_CTR1(vm, vcpuid, msg " pir0 0x%016lx", pir_desc->pir[0]);	\
2187 	VCPU_CTR1(vm, vcpuid, msg " pir1 0x%016lx", pir_desc->pir[1]);	\
2188 	VCPU_CTR1(vm, vcpuid, msg " pir2 0x%016lx", pir_desc->pir[2]);	\
2189 	VCPU_CTR1(vm, vcpuid, msg " pir3 0x%016lx", pir_desc->pir[3]);	\
2190 	VCPU_CTR1(vm, vcpuid, msg " notify: %s", notify ? "yes" : "no");\
2191 } while (0)
2192 
2193 /*
2194  * vlapic->ops handlers that utilize the APICv hardware assist described in
2195  * Chapter 29 of the Intel SDM.
2196  */
2197 static int
2198 vmx_set_intr_ready(struct vlapic *vlapic, int vector, bool level)
2199 {
2200 	struct vlapic_vtx *vlapic_vtx;
2201 	struct pir_desc *pir_desc;
2202 	uint64_t mask;
2203 	int idx, notify;
2204 
2205 	/*
2206 	 * XXX need to deal with level triggered interrupts
2207 	 */
2208 	vlapic_vtx = (struct vlapic_vtx *)vlapic;
2209 	pir_desc = vlapic_vtx->pir_desc;
2210 
2211 	/*
2212 	 * Keep track of interrupt requests in the PIR descriptor. This is
2213 	 * because the virtual APIC page pointed to by the VMCS cannot be
2214 	 * modified if the vcpu is running.
2215 	 */
2216 	idx = vector / 64;
2217 	mask = 1UL << (vector % 64);
2218 	atomic_set_long(&pir_desc->pir[idx], mask);
2219 	notify = atomic_cmpset_long(&pir_desc->pending, 0, 1);
2220 
2221 	VMX_CTR_PIR(vlapic->vm, vlapic->vcpuid, pir_desc, notify, vector,
2222 	    level, "vmx_set_intr_ready");
2223 	return (notify);
2224 }
2225 
2226 static int
2227 vmx_pending_intr(struct vlapic *vlapic, int *vecptr)
2228 {
2229 	struct vlapic_vtx *vlapic_vtx;
2230 	struct pir_desc *pir_desc;
2231 	struct LAPIC *lapic;
2232 	uint64_t pending, pirval;
2233 	uint32_t ppr, vpr;
2234 	int i;
2235 
2236 	/*
2237 	 * This function is only expected to be called from the 'HLT' exit
2238 	 * handler which does not care about the vector that is pending.
2239 	 */
2240 	KASSERT(vecptr == NULL, ("vmx_pending_intr: vecptr must be NULL"));
2241 
2242 	vlapic_vtx = (struct vlapic_vtx *)vlapic;
2243 	pir_desc = vlapic_vtx->pir_desc;
2244 
2245 	pending = atomic_load_acq_long(&pir_desc->pending);
2246 	if (!pending)
2247 		return (0);	/* common case */
2248 
2249 	/*
2250 	 * If there is an interrupt pending then it will be recognized only
2251 	 * if its priority is greater than the processor priority.
2252 	 *
2253 	 * Special case: if the processor priority is zero then any pending
2254 	 * interrupt will be recognized.
2255 	 */
2256 	lapic = vlapic->apic_page;
2257 	ppr = lapic->ppr & 0xf0;
2258 	if (ppr == 0)
2259 		return (1);
2260 
2261 	VCPU_CTR1(vlapic->vm, vlapic->vcpuid, "HLT with non-zero PPR %d",
2262 	    lapic->ppr);
2263 
2264 	for (i = 3; i >= 0; i--) {
2265 		pirval = pir_desc->pir[i];
2266 		if (pirval != 0) {
2267 			vpr = (i * 64 + flsl(pirval) - 1) & 0xf0;
2268 			return (vpr > ppr);
2269 		}
2270 	}
2271 	return (0);
2272 }
2273 
2274 static void
2275 vmx_intr_accepted(struct vlapic *vlapic, int vector)
2276 {
2277 
2278 	panic("vmx_intr_accepted: not expected to be called");
2279 }
2280 
2281 static void
2282 vmx_post_intr(struct vlapic *vlapic, int hostcpu)
2283 {
2284 
2285 	ipi_cpu(hostcpu, pirvec);
2286 }
2287 
2288 /*
2289  * Transfer the pending interrupts in the PIR descriptor to the IRR
2290  * in the virtual APIC page.
2291  */
2292 static void
2293 vmx_inject_pir(struct vlapic *vlapic)
2294 {
2295 	struct vlapic_vtx *vlapic_vtx;
2296 	struct pir_desc *pir_desc;
2297 	struct LAPIC *lapic;
2298 	uint64_t val, pirval;
2299 	int rvi, pirbase;
2300 	uint16_t intr_status_old, intr_status_new;
2301 
2302 	vlapic_vtx = (struct vlapic_vtx *)vlapic;
2303 	pir_desc = vlapic_vtx->pir_desc;
2304 	if (atomic_cmpset_long(&pir_desc->pending, 1, 0) == 0) {
2305 		VCPU_CTR0(vlapic->vm, vlapic->vcpuid, "vmx_inject_pir: "
2306 		    "no posted interrupt pending");
2307 		return;
2308 	}
2309 
2310 	pirval = 0;
2311 	lapic = vlapic->apic_page;
2312 
2313 	val = atomic_readandclear_long(&pir_desc->pir[0]);
2314 	if (val != 0) {
2315 		lapic->irr0 |= val;
2316 		lapic->irr1 |= val >> 32;
2317 		pirbase = 0;
2318 		pirval = val;
2319 	}
2320 
2321 	val = atomic_readandclear_long(&pir_desc->pir[1]);
2322 	if (val != 0) {
2323 		lapic->irr2 |= val;
2324 		lapic->irr3 |= val >> 32;
2325 		pirbase = 64;
2326 		pirval = val;
2327 	}
2328 
2329 	val = atomic_readandclear_long(&pir_desc->pir[2]);
2330 	if (val != 0) {
2331 		lapic->irr4 |= val;
2332 		lapic->irr5 |= val >> 32;
2333 		pirbase = 128;
2334 		pirval = val;
2335 	}
2336 
2337 	val = atomic_readandclear_long(&pir_desc->pir[3]);
2338 	if (val != 0) {
2339 		lapic->irr6 |= val;
2340 		lapic->irr7 |= val >> 32;
2341 		pirbase = 192;
2342 		pirval = val;
2343 	}
2344 	VLAPIC_CTR_IRR(vlapic, "vmx_inject_pir");
2345 
2346 	/*
2347 	 * Update RVI so the processor can evaluate pending virtual
2348 	 * interrupts on VM-entry.
2349 	 */
2350 	if (pirval != 0) {
2351 		rvi = pirbase + flsl(pirval) - 1;
2352 		intr_status_old = vmcs_read(VMCS_GUEST_INTR_STATUS);
2353 		intr_status_new = (intr_status_old & 0xFF00) | rvi;
2354 		if (intr_status_new > intr_status_old) {
2355 			vmcs_write(VMCS_GUEST_INTR_STATUS, intr_status_new);
2356 			VCPU_CTR2(vlapic->vm, vlapic->vcpuid, "vmx_inject_pir: "
2357 			    "guest_intr_status changed from 0x%04x to 0x%04x",
2358 			    intr_status_old, intr_status_new);
2359 		}
2360 	}
2361 }
2362 
2363 static struct vlapic *
2364 vmx_vlapic_init(void *arg, int vcpuid)
2365 {
2366 	struct vmx *vmx;
2367 	struct vlapic *vlapic;
2368 	struct vlapic_vtx *vlapic_vtx;
2369 
2370 	vmx = arg;
2371 
2372 	vlapic = malloc(sizeof(struct vlapic_vtx), M_VLAPIC, M_WAITOK | M_ZERO);
2373 	vlapic->vm = vmx->vm;
2374 	vlapic->vcpuid = vcpuid;
2375 	vlapic->apic_page = (struct LAPIC *)&vmx->apic_page[vcpuid];
2376 
2377 	vlapic_vtx = (struct vlapic_vtx *)vlapic;
2378 	vlapic_vtx->pir_desc = &vmx->pir_desc[vcpuid];
2379 
2380 	if (virtual_interrupt_delivery) {
2381 		vlapic->ops.set_intr_ready = vmx_set_intr_ready;
2382 		vlapic->ops.pending_intr = vmx_pending_intr;
2383 		vlapic->ops.intr_accepted = vmx_intr_accepted;
2384 	}
2385 
2386 	if (posted_interrupts)
2387 		vlapic->ops.post_intr = vmx_post_intr;
2388 
2389 	vlapic_init(vlapic);
2390 
2391 	return (vlapic);
2392 }
2393 
2394 static void
2395 vmx_vlapic_cleanup(void *arg, struct vlapic *vlapic)
2396 {
2397 
2398 	vlapic_cleanup(vlapic);
2399 	free(vlapic, M_VLAPIC);
2400 }
2401 
2402 struct vmm_ops vmm_ops_intel = {
2403 	vmx_init,
2404 	vmx_cleanup,
2405 	vmx_restore,
2406 	vmx_vminit,
2407 	vmx_run,
2408 	vmx_vmcleanup,
2409 	vmx_getreg,
2410 	vmx_setreg,
2411 	vmx_getdesc,
2412 	vmx_setdesc,
2413 	vmx_inject,
2414 	vmx_getcap,
2415 	vmx_setcap,
2416 	ept_vmspace_alloc,
2417 	ept_vmspace_free,
2418 	vmx_vlapic_init,
2419 	vmx_vlapic_cleanup,
2420 };
2421