xref: /freebsd/sys/amd64/vmm/amd/amdvi_hw.c (revision 190cef3d52236565eb22e18b33e9e865ec634aa3)
1 /*-
2  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
3  *
4  * Copyright (c) 2016, Anish Gupta (anish@freebsd.org)
5  * All rights reserved.
6  *
7  * Redistribution and use in source and binary forms, with or without
8  * modification, are permitted provided that the following conditions
9  * are met:
10  * 1. Redistributions of source code must retain the above copyright
11  *    notice unmodified, this list of conditions, and the following
12  *    disclaimer.
13  * 2. Redistributions in binary form must reproduce the above copyright
14  *    notice, this list of conditions and the following disclaimer in the
15  *    documentation and/or other materials provided with the distribution.
16  *
17  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
18  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
19  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
20  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
21  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
22  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
23  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
24  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
25  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
26  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
27  */
28 
29 #include <sys/cdefs.h>
30 __FBSDID("$FreeBSD$");
31 
32 #include <sys/param.h>
33 #include <sys/systm.h>
34 #include <sys/bus.h>
35 #include <sys/kernel.h>
36 #include <sys/module.h>
37 #include <sys/malloc.h>
38 #include <sys/pcpu.h>
39 #include <sys/rman.h>
40 #include <sys/smp.h>
41 #include <sys/sysctl.h>
42 
43 #include <vm/vm.h>
44 #include <vm/pmap.h>
45 
46 #include <dev/pci/pcivar.h>
47 #include <dev/pci/pcireg.h>
48 
49 #include <machine/resource.h>
50 #include <machine/vmm.h>
51 #include <machine/pmap.h>
52 #include <machine/vmparam.h>
53 #include <machine/pci_cfgreg.h>
54 
55 #include "pcib_if.h"
56 
57 #include "io/iommu.h"
58 #include "amdvi_priv.h"
59 
60 SYSCTL_DECL(_hw_vmm);
61 SYSCTL_NODE(_hw_vmm, OID_AUTO, amdvi, CTLFLAG_RW, NULL, NULL);
62 
63 #define MOD_INC(a, s, m) (((a) + (s)) % ((m) * (s)))
64 #define MOD_DEC(a, s, m) (((a) - (s)) % ((m) * (s)))
65 
66 /* Print RID or device ID in PCI string format. */
67 #define RID2PCI_STR(d) PCI_RID2BUS(d), PCI_RID2SLOT(d), PCI_RID2FUNC(d)
68 
69 static void amdvi_dump_cmds(struct amdvi_softc *softc);
70 static void amdvi_print_dev_cap(struct amdvi_softc *softc);
71 
72 MALLOC_DEFINE(M_AMDVI, "amdvi", "amdvi");
73 
74 extern device_t *ivhd_devs;
75 
76 extern int ivhd_count;
77 SYSCTL_INT(_hw_vmm_amdvi, OID_AUTO, count, CTLFLAG_RDTUN, &ivhd_count,
78     0, NULL);
79 
80 static int amdvi_enable_user = 0;
81 SYSCTL_INT(_hw_vmm_amdvi, OID_AUTO, enable, CTLFLAG_RDTUN,
82     &amdvi_enable_user, 0, NULL);
83 TUNABLE_INT("hw.vmm.amdvi_enable", &amdvi_enable_user);
84 
85 #ifdef AMDVI_ATS_ENABLE
86 /* XXX: ATS is not tested. */
87 static int amdvi_enable_iotlb = 1;
88 SYSCTL_INT(_hw_vmm_amdvi, OID_AUTO, iotlb_enabled, CTLFLAG_RDTUN,
89     &amdvi_enable_iotlb, 0, NULL);
90 TUNABLE_INT("hw.vmm.enable_iotlb", &amdvi_enable_iotlb);
91 #endif
92 
93 static int amdvi_host_ptp = 1;	/* Use page tables for host. */
94 SYSCTL_INT(_hw_vmm_amdvi, OID_AUTO, host_ptp, CTLFLAG_RDTUN,
95     &amdvi_host_ptp, 0, NULL);
96 TUNABLE_INT("hw.vmm.amdvi.host_ptp", &amdvi_host_ptp);
97 
98 /* Page table level used <= supported by h/w[v1=7]. */
99 static int amdvi_ptp_level = 4;
100 SYSCTL_INT(_hw_vmm_amdvi, OID_AUTO, ptp_level, CTLFLAG_RDTUN,
101     &amdvi_ptp_level, 0, NULL);
102 TUNABLE_INT("hw.vmm.amdvi.ptp_level", &amdvi_ptp_level);
103 
104 /* Disable fault event reporting. */
105 static int amdvi_disable_io_fault = 0;
106 SYSCTL_INT(_hw_vmm_amdvi, OID_AUTO, disable_io_fault, CTLFLAG_RDTUN,
107     &amdvi_disable_io_fault, 0, NULL);
108 TUNABLE_INT("hw.vmm.amdvi.disable_io_fault", &amdvi_disable_io_fault);
109 
110 static uint32_t amdvi_dom_id = 0;	/* 0 is reserved for host. */
111 SYSCTL_UINT(_hw_vmm_amdvi, OID_AUTO, domain_id, CTLFLAG_RD,
112     &amdvi_dom_id, 0, NULL);
113 /*
114  * Device table entry.
115  * Bus(256) x Dev(32) x Fun(8) x DTE(256 bits or 32 bytes).
116  *	= 256 * 2 * PAGE_SIZE.
117  */
118 static struct amdvi_dte amdvi_dte[PCI_NUM_DEV_MAX] __aligned(PAGE_SIZE);
119 CTASSERT(PCI_NUM_DEV_MAX == 0x10000);
120 CTASSERT(sizeof(amdvi_dte) == 0x200000);
121 
122 static SLIST_HEAD (, amdvi_domain) dom_head;
123 
124 static inline uint32_t
125 amdvi_pci_read(struct amdvi_softc *softc, int off)
126 {
127 
128 	return (pci_cfgregread(PCI_RID2BUS(softc->pci_rid),
129 	    PCI_RID2SLOT(softc->pci_rid), PCI_RID2FUNC(softc->pci_rid),
130 	    off, 4));
131 }
132 
133 #ifdef AMDVI_ATS_ENABLE
134 /* XXX: Should be in pci.c */
135 /*
136  * Check if device has ATS capability and its enabled.
137  * If ATS is absent or disabled, return (-1), otherwise ATS
138  * queue length.
139  */
140 static int
141 amdvi_find_ats_qlen(uint16_t devid)
142 {
143 	device_t dev;
144 	uint32_t off, cap;
145 	int qlen = -1;
146 
147 	dev = pci_find_bsf(PCI_RID2BUS(devid), PCI_RID2SLOT(devid),
148 			   PCI_RID2FUNC(devid));
149 
150 	if (!dev) {
151 		return (-1);
152 	}
153 #define PCIM_ATS_EN	BIT(31)
154 
155 	if (pci_find_extcap(dev, PCIZ_ATS, &off) == 0) {
156 		cap = pci_read_config(dev, off + 4, 4);
157 		qlen = (cap & 0x1F);
158 		qlen = qlen ? qlen : 32;
159 		printf("AMD-Vi: PCI device %d.%d.%d ATS %s qlen=%d\n",
160 		       RID2PCI_STR(devid),
161 		       (cap & PCIM_ATS_EN) ? "enabled" : "Disabled",
162 		       qlen);
163 		qlen = (cap & PCIM_ATS_EN) ? qlen : -1;
164 	}
165 
166 	return (qlen);
167 }
168 
169 /*
170  * Check if an endpoint device support device IOTLB or ATS.
171  */
172 static inline bool
173 amdvi_dev_support_iotlb(struct amdvi_softc *softc, uint16_t devid)
174 {
175 	struct ivhd_dev_cfg *cfg;
176 	int qlen, i;
177 	bool pci_ats, ivhd_ats;
178 
179 	qlen = amdvi_find_ats_qlen(devid);
180 	if (qlen < 0)
181 		return (false);
182 
183 	KASSERT(softc, ("softc is NULL"));
184 	cfg = softc->dev_cfg;
185 
186 	ivhd_ats = false;
187 	for (i = 0; i < softc->dev_cfg_cnt; i++) {
188 		if ((cfg->start_id <= devid) && (cfg->end_id >= devid)) {
189 			ivhd_ats = cfg->enable_ats;
190 			break;
191 		}
192 		cfg++;
193 	}
194 
195 	pci_ats = (qlen < 0) ? false : true;
196 	if (pci_ats != ivhd_ats)
197 		device_printf(softc->dev,
198 		    "BIOS bug: mismatch in ATS setting for %d.%d.%d,"
199 		    "ATS inv qlen = %d\n", RID2PCI_STR(devid), qlen);
200 
201 	/* Ignore IVRS setting and respect PCI setting. */
202 	return (pci_ats);
203 }
204 #endif
205 
206 /* Enable IOTLB support for IOMMU if its supported. */
207 static inline void
208 amdvi_hw_enable_iotlb(struct amdvi_softc *softc)
209 {
210 #ifndef AMDVI_ATS_ENABLE
211 	softc->iotlb = false;
212 #else
213 	bool supported;
214 
215 	supported = (softc->ivhd_flag & IVHD_FLAG_IOTLB) ? true : false;
216 
217 	if (softc->pci_cap & AMDVI_PCI_CAP_IOTLB) {
218 		if (!supported)
219 			device_printf(softc->dev, "IOTLB disabled by BIOS.\n");
220 
221 		if (supported && !amdvi_enable_iotlb) {
222 			device_printf(softc->dev, "IOTLB disabled by user.\n");
223 			supported = false;
224 		}
225 	} else
226 		supported = false;
227 
228 	softc->iotlb = supported;
229 
230 #endif
231 }
232 
233 static int
234 amdvi_init_cmd(struct amdvi_softc *softc)
235 {
236 	struct amdvi_ctrl *ctrl = softc->ctrl;
237 
238 	ctrl->cmd.len = 8;	/* Use 256 command buffer entries. */
239 	softc->cmd_max = 1 << ctrl->cmd.len;
240 
241 	softc->cmd = malloc(sizeof(struct amdvi_cmd) *
242 	    softc->cmd_max, M_AMDVI, M_WAITOK | M_ZERO);
243 
244 	if ((uintptr_t)softc->cmd & PAGE_MASK)
245 		panic("AMDVi: Command buffer not aligned on page boundary.");
246 
247 	ctrl->cmd.base = vtophys(softc->cmd) / PAGE_SIZE;
248 	/*
249 	 * XXX: Reset the h/w pointers in case IOMMU is restarting,
250 	 * h/w doesn't clear these pointers based on empirical data.
251 	 */
252 	ctrl->cmd_tail = 0;
253 	ctrl->cmd_head = 0;
254 
255 	return (0);
256 }
257 
258 /*
259  * Note: Update tail pointer after we have written the command since tail
260  * pointer update cause h/w to execute new commands, see section 3.3
261  * of AMD IOMMU spec ver 2.0.
262  */
263 /* Get the command tail pointer w/o updating it. */
264 static struct amdvi_cmd *
265 amdvi_get_cmd_tail(struct amdvi_softc *softc)
266 {
267 	struct amdvi_ctrl *ctrl;
268 	struct amdvi_cmd *tail;
269 
270 	KASSERT(softc, ("softc is NULL"));
271 	KASSERT(softc->cmd != NULL, ("cmd is NULL"));
272 
273 	ctrl = softc->ctrl;
274 	KASSERT(ctrl != NULL, ("ctrl is NULL"));
275 
276 	tail = (struct amdvi_cmd *)((uint8_t *)softc->cmd +
277 	    ctrl->cmd_tail);
278 
279 	return (tail);
280 }
281 
282 /*
283  * Update the command tail pointer which will start command execution.
284  */
285 static void
286 amdvi_update_cmd_tail(struct amdvi_softc *softc)
287 {
288 	struct amdvi_ctrl *ctrl;
289 	int size;
290 
291 	size = sizeof(struct amdvi_cmd);
292 	KASSERT(softc->cmd != NULL, ("cmd is NULL"));
293 
294 	ctrl = softc->ctrl;
295 	KASSERT(ctrl != NULL, ("ctrl is NULL"));
296 
297 	ctrl->cmd_tail = MOD_INC(ctrl->cmd_tail, size, softc->cmd_max);
298 	softc->total_cmd++;
299 
300 #ifdef AMDVI_DEBUG_CMD
301 	device_printf(softc->dev, "cmd_tail: %s Tail:0x%x, Head:0x%x.\n",
302 	    ctrl->cmd_tail,
303 	    ctrl->cmd_head);
304 #endif
305 
306 }
307 
308 /*
309  * Various commands supported by IOMMU.
310  */
311 
312 /* Completion wait command. */
313 static void
314 amdvi_cmd_cmp(struct amdvi_softc *softc, const uint64_t data)
315 {
316 	struct amdvi_cmd *cmd;
317 	uint64_t pa;
318 
319 	cmd = amdvi_get_cmd_tail(softc);
320 	KASSERT(cmd != NULL, ("Cmd is NULL"));
321 
322 	pa = vtophys(&softc->cmp_data);
323 	cmd->opcode = AMDVI_CMP_WAIT_OPCODE;
324 	cmd->word0 = (pa & 0xFFFFFFF8) |
325 	    (AMDVI_CMP_WAIT_STORE);
326 	//(AMDVI_CMP_WAIT_FLUSH | AMDVI_CMP_WAIT_STORE);
327 	cmd->word1 = (pa >> 32) & 0xFFFFF;
328 	cmd->addr = data;
329 
330 	amdvi_update_cmd_tail(softc);
331 }
332 
333 /* Invalidate device table entry. */
334 static void
335 amdvi_cmd_inv_dte(struct amdvi_softc *softc, uint16_t devid)
336 {
337 	struct amdvi_cmd *cmd;
338 
339 	cmd = amdvi_get_cmd_tail(softc);
340 	KASSERT(cmd != NULL, ("Cmd is NULL"));
341 	cmd->opcode = AMDVI_INVD_DTE_OPCODE;
342 	cmd->word0 = devid;
343 	amdvi_update_cmd_tail(softc);
344 #ifdef AMDVI_DEBUG_CMD
345 	device_printf(softc->dev, "Invalidated DTE:0x%x\n", devid);
346 #endif
347 }
348 
349 /* Invalidate IOMMU page, use for invalidation of domain. */
350 static void
351 amdvi_cmd_inv_iommu_pages(struct amdvi_softc *softc, uint16_t domain_id,
352 			  uint64_t addr, bool guest_nested,
353 			  bool pde, bool page)
354 {
355 	struct amdvi_cmd *cmd;
356 
357 	cmd = amdvi_get_cmd_tail(softc);
358 	KASSERT(cmd != NULL, ("Cmd is NULL"));
359 
360 
361 	cmd->opcode = AMDVI_INVD_PAGE_OPCODE;
362 	cmd->word1 = domain_id;
363 	/*
364 	 * Invalidate all addresses for this domain.
365 	 */
366 	cmd->addr = addr;
367 	cmd->addr |= pde ? AMDVI_INVD_PAGE_PDE : 0;
368 	cmd->addr |= page ? AMDVI_INVD_PAGE_S : 0;
369 
370 	amdvi_update_cmd_tail(softc);
371 }
372 
373 #ifdef AMDVI_ATS_ENABLE
374 /* Invalidate device IOTLB. */
375 static void
376 amdvi_cmd_inv_iotlb(struct amdvi_softc *softc, uint16_t devid)
377 {
378 	struct amdvi_cmd *cmd;
379 	int qlen;
380 
381 	if (!softc->iotlb)
382 		return;
383 
384 	qlen = amdvi_find_ats_qlen(devid);
385 	if (qlen < 0) {
386 		panic("AMDVI: Invalid ATS qlen(%d) for device %d.%d.%d\n",
387 		      qlen, RID2PCI_STR(devid));
388 	}
389 	cmd = amdvi_get_cmd_tail(softc);
390 	KASSERT(cmd != NULL, ("Cmd is NULL"));
391 
392 #ifdef AMDVI_DEBUG_CMD
393 	device_printf(softc->dev, "Invalidate IOTLB devID 0x%x"
394 		      " Qlen:%d\n", devid, qlen);
395 #endif
396 	cmd->opcode = AMDVI_INVD_IOTLB_OPCODE;
397 	cmd->word0 = devid;
398 	cmd->word1 = qlen;
399 	cmd->addr = AMDVI_INVD_IOTLB_ALL_ADDR |
400 		AMDVI_INVD_IOTLB_S;
401 	amdvi_update_cmd_tail(softc);
402 }
403 #endif
404 
405 #ifdef notyet				/* For Interrupt Remap. */
406 static void
407 amdvi_cmd_inv_intr_map(struct amdvi_softc *softc,
408 		       uint16_t devid)
409 {
410 	struct amdvi_cmd *cmd;
411 
412 	cmd = amdvi_get_cmd_tail(softc);
413 	KASSERT(cmd != NULL, ("Cmd is NULL"));
414 	cmd->opcode = AMDVI_INVD_INTR_OPCODE;
415 	cmd->word0 = devid;
416 	amdvi_update_cmd_tail(softc);
417 #ifdef AMDVI_DEBUG_CMD
418 	device_printf(softc->dev, "Invalidate INTR map of devID 0x%x\n", devid);
419 #endif
420 }
421 #endif
422 
423 /* Invalidate domain using INVALIDATE_IOMMU_PAGES command. */
424 static void
425 amdvi_inv_domain(struct amdvi_softc *softc, uint16_t domain_id)
426 {
427 	struct amdvi_cmd *cmd;
428 
429 	cmd = amdvi_get_cmd_tail(softc);
430 	KASSERT(cmd != NULL, ("Cmd is NULL"));
431 
432 	/*
433 	 * See section 3.3.3 of IOMMU spec rev 2.0, software note
434 	 * for invalidating domain.
435 	 */
436 	amdvi_cmd_inv_iommu_pages(softc, domain_id, AMDVI_INVD_PAGE_ALL_ADDR,
437 				false, true, true);
438 
439 #ifdef AMDVI_DEBUG_CMD
440 	device_printf(softc->dev, "Invalidate domain:0x%x\n", domain_id);
441 
442 #endif
443 }
444 
445 static	bool
446 amdvi_cmp_wait(struct amdvi_softc *softc)
447 {
448 	struct amdvi_ctrl *ctrl;
449 	const uint64_t VERIFY = 0xA5A5;
450 	volatile uint64_t *read;
451 	int i;
452 	bool status;
453 
454 	ctrl = softc->ctrl;
455 	read = &softc->cmp_data;
456 	*read = 0;
457 	amdvi_cmd_cmp(softc, VERIFY);
458 	/* Wait for h/w to update completion data. */
459 	for (i = 0; i < 100 && (*read != VERIFY); i++) {
460 		DELAY(1000);		/* 1 ms */
461 	}
462 	status = (VERIFY == softc->cmp_data) ? true : false;
463 
464 #ifdef AMDVI_DEBUG_CMD
465 	if (status)
466 		device_printf(softc->dev, "CMD completion DONE Tail:0x%x, "
467 			      "Head:0x%x, loop:%d.\n", ctrl->cmd_tail,
468 			      ctrl->cmd_head, loop);
469 #endif
470 	return (status);
471 }
472 
473 static void
474 amdvi_wait(struct amdvi_softc *softc)
475 {
476 	struct amdvi_ctrl *ctrl;
477 	int i;
478 
479 	KASSERT(softc, ("softc is NULL"));
480 
481 	ctrl = softc->ctrl;
482 	KASSERT(ctrl != NULL, ("ctrl is NULL"));
483 	/* Don't wait if h/w is not enabled. */
484 	if ((ctrl->control & AMDVI_CTRL_EN) == 0)
485 		return;
486 
487 	for (i = 0; i < 10; i++) {
488 		if (amdvi_cmp_wait(softc))
489 			return;
490 	}
491 
492 	device_printf(softc->dev, "Error: completion failed"
493 		      " tail:0x%x, head:0x%x.\n",
494 		      ctrl->cmd_tail, ctrl->cmd_head);
495 	amdvi_dump_cmds(softc);
496 }
497 
498 static void
499 amdvi_dump_cmds(struct amdvi_softc *softc)
500 {
501 	struct amdvi_ctrl *ctrl;
502 	struct amdvi_cmd *cmd;
503 	int off, i;
504 
505 	ctrl = softc->ctrl;
506 	device_printf(softc->dev, "Dump all the commands:\n");
507 	/*
508 	 * If h/w is stuck in completion, it is the previous command,
509 	 * start dumping from previous command onward.
510 	 */
511 	off = MOD_DEC(ctrl->cmd_head, sizeof(struct amdvi_cmd),
512 	    softc->cmd_max);
513 	for (i = 0; off != ctrl->cmd_tail &&
514 	    i < softc->cmd_max; i++) {
515 		cmd = (struct amdvi_cmd *)((uint8_t *)softc->cmd + off);
516 		printf("  [CMD%d, off:0x%x] opcode= 0x%x 0x%x"
517 		    " 0x%x 0x%lx\n", i, off, cmd->opcode,
518 		    cmd->word0, cmd->word1, cmd->addr);
519 		off = (off + sizeof(struct amdvi_cmd)) %
520 		    (softc->cmd_max * sizeof(struct amdvi_cmd));
521 	}
522 }
523 
524 static int
525 amdvi_init_event(struct amdvi_softc *softc)
526 {
527 	struct amdvi_ctrl *ctrl;
528 
529 	ctrl = softc->ctrl;
530 	ctrl->event.len = 8;
531 	softc->event_max = 1 << ctrl->event.len;
532 	softc->event = malloc(sizeof(struct amdvi_event) *
533 	    softc->event_max, M_AMDVI, M_WAITOK | M_ZERO);
534 	if ((uintptr_t)softc->event & PAGE_MASK) {
535 		device_printf(softc->dev, "Event buffer not aligned on page.");
536 		return (false);
537 	}
538 	ctrl->event.base = vtophys(softc->event) / PAGE_SIZE;
539 
540 	/* Reset the pointers. */
541 	ctrl->evt_head = 0;
542 	ctrl->evt_tail = 0;
543 
544 	return (0);
545 }
546 
547 static inline void
548 amdvi_decode_evt_flag(uint16_t flag)
549 {
550 
551 	flag &= AMDVI_EVENT_FLAG_MASK;
552 	printf(" 0x%b]\n", flag,
553 		"\020"
554 		"\001GN"
555 		"\002NX"
556 		"\003US"
557 		"\004I"
558 		"\005PR"
559 		"\006RW"
560 		"\007PE"
561 		"\010RZ"
562 		"\011TR"
563 		);
564 }
565 
566 /* See section 2.5.4 of AMD IOMMU spec ver 2.62.*/
567 static inline void
568 amdvi_decode_evt_flag_type(uint8_t type)
569 {
570 
571 	switch (AMDVI_EVENT_FLAG_TYPE(type)) {
572 	case 0:
573 		printf("RSVD\n");
574 		break;
575 	case 1:
576 		printf("Master Abort\n");
577 		break;
578 	case 2:
579 		printf("Target Abort\n");
580 		break;
581 	case 3:
582 		printf("Data Err\n");
583 		break;
584 	default:
585 		break;
586 	}
587 }
588 
589 static void
590 amdvi_decode_inv_dte_evt(uint16_t devid, uint16_t domid, uint64_t addr,
591     uint16_t flag)
592 {
593 
594 	printf("\t[IO_PAGE_FAULT EVT: devId:0x%x DomId:0x%x"
595 	    " Addr:0x%lx",
596 	    devid, domid, addr);
597 	amdvi_decode_evt_flag(flag);
598 }
599 
600 static void
601 amdvi_decode_pf_evt(uint16_t devid, uint16_t domid, uint64_t addr,
602     uint16_t flag)
603 {
604 
605 	printf("\t[IO_PAGE_FAULT EVT: devId:0x%x DomId:0x%x"
606 	    " Addr:0x%lx",
607 	    devid, domid, addr);
608 	amdvi_decode_evt_flag(flag);
609 }
610 
611 static void
612 amdvi_decode_dte_hwerr_evt(uint16_t devid, uint16_t domid,
613     uint64_t addr, uint16_t flag)
614 {
615 
616 	printf("\t[DEV_TAB_HW_ERR EVT: devId:0x%x DomId:0x%x"
617 	    " Addr:0x%lx", devid, domid, addr);
618 	amdvi_decode_evt_flag(flag);
619 	amdvi_decode_evt_flag_type(flag);
620 }
621 
622 static void
623 amdvi_decode_page_hwerr_evt(uint16_t devid, uint16_t domid, uint64_t addr,
624     uint16_t flag)
625 {
626 
627 	printf("\t[PAGE_TAB_HW_ERR EVT: devId:0x%x DomId:0x%x"
628 	    " Addr:0x%lx", devid, domid, addr);
629 	amdvi_decode_evt_flag(flag);
630 	amdvi_decode_evt_flag_type(AMDVI_EVENT_FLAG_TYPE(flag));
631 }
632 
633 static void
634 amdvi_decode_evt(struct amdvi_event *evt)
635 {
636 	struct amdvi_cmd *cmd;
637 
638 	switch (evt->opcode) {
639 	case AMDVI_EVENT_INVALID_DTE:
640 		amdvi_decode_inv_dte_evt(evt->devid, evt->pasid_domid,
641 		    evt->addr, evt->flag);
642 		break;
643 
644 	case AMDVI_EVENT_PFAULT:
645 		amdvi_decode_pf_evt(evt->devid, evt->pasid_domid,
646 		    evt->addr, evt->flag);
647 		break;
648 
649 	case AMDVI_EVENT_DTE_HW_ERROR:
650 		amdvi_decode_dte_hwerr_evt(evt->devid, evt->pasid_domid,
651 		    evt->addr, evt->flag);
652 		break;
653 
654 	case AMDVI_EVENT_PAGE_HW_ERROR:
655 		amdvi_decode_page_hwerr_evt(evt->devid, evt->pasid_domid,
656 		    evt->addr, evt->flag);
657 		break;
658 
659 	case AMDVI_EVENT_ILLEGAL_CMD:
660 		/* FALL THROUGH */
661 	case AMDVI_EVENT_CMD_HW_ERROR:
662 		printf("\t[%s EVT]\n", (evt->opcode == AMDVI_EVENT_ILLEGAL_CMD) ?
663 		    "ILLEGAL CMD" : "CMD HW ERR");
664 		cmd = (struct amdvi_cmd *)PHYS_TO_DMAP(evt->addr);
665 		printf("\tCMD opcode= 0x%x 0x%x 0x%x 0x%lx\n",
666 		    cmd->opcode, cmd->word0, cmd->word1, cmd->addr);
667 		break;
668 
669 	case AMDVI_EVENT_IOTLB_TIMEOUT:
670 		printf("\t[IOTLB_INV_TIMEOUT devid:0x%x addr:0x%lx]\n",
671 		    evt->devid, evt->addr);
672 		break;
673 
674 	case AMDVI_EVENT_INVALID_DTE_REQ:
675 		printf("\t[INV_DTE devid:0x%x addr:0x%lx type:0x%x tr:%d]\n",
676 		    evt->devid, evt->addr, evt->flag >> 9,
677 		    (evt->flag >> 8) & 1);
678 		break;
679 
680 	case AMDVI_EVENT_INVALID_PPR_REQ:
681 	case AMDVI_EVENT_COUNTER_ZERO:
682 		printf("AMD-Vi: v2 events.\n");
683 		break;
684 
685 	default:
686 		printf("Unsupported AMD-Vi event:%d\n", evt->opcode);
687 	}
688 }
689 
690 static void
691 amdvi_print_events(struct amdvi_softc *softc)
692 {
693 	struct amdvi_ctrl *ctrl;
694 	struct amdvi_event *event;
695 	int i, size;
696 
697 	ctrl = softc->ctrl;
698 	size = sizeof(struct amdvi_event);
699 	for (i = 0; i < softc->event_max; i++) {
700 		event = &softc->event[ctrl->evt_head / size];
701 		if (!event->opcode)
702 			break;
703 		device_printf(softc->dev, "\t[Event%d: Head:0x%x Tail:0x%x]\n",
704 		    i, ctrl->evt_head, ctrl->evt_tail);
705 		amdvi_decode_evt(event);
706 		ctrl->evt_head = MOD_INC(ctrl->evt_head, size,
707 		    softc->event_max);
708 	}
709 }
710 
711 static int
712 amdvi_init_dte(struct amdvi_softc *softc)
713 {
714 	struct amdvi_ctrl *ctrl;
715 
716 	ctrl = softc->ctrl;
717 	ctrl->dte.base = vtophys(amdvi_dte) / PAGE_SIZE;
718 	ctrl->dte.size = 0x1FF;		/* 2MB device table. */
719 
720 	return (0);
721 }
722 
723 /*
724  * Not all capabilities of IOMMU are available in ACPI IVHD flag
725  * or EFR entry, read directly from device.
726  */
727 static int
728 amdvi_print_pci_cap(device_t dev)
729 {
730 	struct amdvi_softc *softc;
731 	uint32_t off, cap;
732 
733 
734 	softc = device_get_softc(dev);
735 	off = softc->cap_off;
736 
737 	/*
738 	 * Section 3.7.1 of IOMMU sepc rev 2.0.
739 	 * Read capability from device.
740 	 */
741 	cap = amdvi_pci_read(softc, off);
742 
743 	/* Make sure capability type[18:16] is 3. */
744 	KASSERT((((cap >> 16) & 0x7) == 0x3),
745 	    ("Not a IOMMU capability 0x%x@0x%x", cap, off));
746 
747 	softc->pci_cap = cap >> 24;
748 	device_printf(softc->dev, "PCI cap 0x%x@0x%x feature:%b\n",
749 	    cap, off, softc->pci_cap,
750 	    "\20\1IOTLB\2HT\3NPCache\4EFR\5CapExt");
751 
752 	return (0);
753 }
754 
755 static void
756 amdvi_event_intr(void *arg)
757 {
758 	struct amdvi_softc *softc;
759 	struct amdvi_ctrl *ctrl;
760 
761 	softc = (struct amdvi_softc *)arg;
762 	ctrl = softc->ctrl;
763 	device_printf(softc->dev, "EVT INTR %ld Status:0x%x"
764 	    " EVT Head:0x%x Tail:0x%x]\n", softc->event_intr_cnt++,
765 	    ctrl->status, ctrl->evt_head, ctrl->evt_tail);
766 	printf("  [CMD Total 0x%lx] Tail:0x%x, Head:0x%x.\n",
767 	    softc->total_cmd, ctrl->cmd_tail, ctrl->cmd_head);
768 
769 	amdvi_print_events(softc);
770 	ctrl->status &= AMDVI_STATUS_EV_OF | AMDVI_STATUS_EV_INTR;
771 }
772 
773 static void
774 amdvi_free_evt_intr_res(device_t dev)
775 {
776 
777 	struct amdvi_softc *softc;
778 
779 	softc = device_get_softc(dev);
780 	if (softc->event_tag != NULL) {
781 		bus_teardown_intr(dev, softc->event_res, softc->event_tag);
782 	}
783 	if (softc->event_res != NULL) {
784 		bus_release_resource(dev, SYS_RES_IRQ, softc->event_rid,
785 		    softc->event_res);
786 	}
787 	bus_delete_resource(dev, SYS_RES_IRQ, softc->event_rid);
788 	PCIB_RELEASE_MSI(device_get_parent(device_get_parent(dev)),
789 	    dev, 1, &softc->event_irq);
790 }
791 
792 static bool
793 amdvi_alloc_intr_resources(struct amdvi_softc *softc)
794 {
795 	struct amdvi_ctrl *ctrl;
796 	device_t dev, pcib;
797 	device_t mmio_dev;
798 	uint64_t msi_addr;
799 	uint32_t msi_data;
800 	int err;
801 
802 	dev = softc->dev;
803 	pcib = device_get_parent(device_get_parent(dev));
804 	mmio_dev = pci_find_bsf(PCI_RID2BUS(softc->pci_rid),
805             PCI_RID2SLOT(softc->pci_rid), PCI_RID2FUNC(softc->pci_rid));
806 	if (device_is_attached(mmio_dev)) {
807 		device_printf(dev,
808 		    "warning: IOMMU device is claimed by another driver %s\n",
809 		    device_get_driver(mmio_dev)->name);
810 	}
811 
812 	softc->event_irq = -1;
813 	softc->event_rid = 0;
814 
815 	/*
816 	 * Section 3.7.1 of IOMMU rev 2.0. With MSI, there is only one
817 	 * interrupt. XXX: Enable MSI/X support.
818 	 */
819 	err = PCIB_ALLOC_MSI(pcib, dev, 1, 1, &softc->event_irq);
820 	if (err) {
821 		device_printf(dev,
822 		    "Couldn't find event MSI IRQ resource.\n");
823 		return (ENOENT);
824 	}
825 
826 	err = bus_set_resource(dev, SYS_RES_IRQ, softc->event_rid,
827 	    softc->event_irq, 1);
828 	if (err) {
829 		device_printf(dev, "Couldn't set event MSI resource.\n");
830 		return (ENXIO);
831 	}
832 
833 	softc->event_res = bus_alloc_resource_any(dev, SYS_RES_IRQ,
834 	    &softc->event_rid, RF_ACTIVE);
835 	if (!softc->event_res) {
836 		device_printf(dev,
837 		    "Unable to allocate event INTR resource.\n");
838 		return (ENOMEM);
839 	}
840 
841 	if (bus_setup_intr(dev, softc->event_res,
842 	    INTR_TYPE_MISC | INTR_MPSAFE, NULL, amdvi_event_intr,
843 	    softc, &softc->event_tag)) {
844 		device_printf(dev, "Fail to setup event intr\n");
845 		bus_release_resource(softc->dev, SYS_RES_IRQ,
846 		    softc->event_rid, softc->event_res);
847 		softc->event_res = NULL;
848 		return (ENXIO);
849 	}
850 
851 	bus_describe_intr(dev, softc->event_res, softc->event_tag,
852 	    "fault");
853 
854 	err = PCIB_MAP_MSI(pcib, dev, softc->event_irq, &msi_addr,
855 	    &msi_data);
856 	if (err) {
857 		device_printf(dev,
858 		    "Event interrupt config failed, err=%d.\n",
859 		    err);
860 		amdvi_free_evt_intr_res(softc->dev);
861 		return (err);
862 	}
863 
864 	/* Clear interrupt status bits. */
865 	ctrl = softc->ctrl;
866 	ctrl->status &= AMDVI_STATUS_EV_OF | AMDVI_STATUS_EV_INTR;
867 
868 	/* Now enable MSI interrupt. */
869 	pci_enable_msi(mmio_dev, msi_addr, msi_data);
870 	return (0);
871 }
872 
873 
874 static void
875 amdvi_print_dev_cap(struct amdvi_softc *softc)
876 {
877 	struct ivhd_dev_cfg *cfg;
878 	int i;
879 
880 	cfg = softc->dev_cfg;
881 	for (i = 0; i < softc->dev_cfg_cnt; i++) {
882 		device_printf(softc->dev, "device [0x%x - 0x%x]"
883 		    "config:%b%s\n", cfg->start_id, cfg->end_id,
884 		    cfg->data,
885 		    "\020\001INIT\002ExtInt\003NMI"
886 		    "\007LINT0\008LINT1",
887 		    cfg->enable_ats ? "ATS enabled" : "");
888 		cfg++;
889 	}
890 }
891 
892 static int
893 amdvi_handle_sysctl(SYSCTL_HANDLER_ARGS)
894 {
895 	struct amdvi_softc *softc;
896 	int result, type, error = 0;
897 
898 	softc = (struct amdvi_softc *)arg1;
899 	type = arg2;
900 
901 	switch (type) {
902 	case 0:
903 		result = softc->ctrl->cmd_head;
904 		error = sysctl_handle_int(oidp, &result, 0,
905 		    req);
906 		break;
907 	case 1:
908 		result = softc->ctrl->cmd_tail;
909 		error = sysctl_handle_int(oidp, &result, 0,
910 		    req);
911 		break;
912 	case 2:
913 		result = softc->ctrl->evt_head;
914 		error = sysctl_handle_int(oidp, &result, 0,
915 		    req);
916 		break;
917 	case 3:
918 		result = softc->ctrl->evt_tail;
919 		error = sysctl_handle_int(oidp, &result, 0,
920 		    req);
921 		break;
922 
923 	default:
924 		device_printf(softc->dev, "Unknown sysctl:%d\n", type);
925 	}
926 
927 	return (error);
928 }
929 
930 static void
931 amdvi_add_sysctl(struct amdvi_softc *softc)
932 {
933 	struct sysctl_oid_list *child;
934 	struct sysctl_ctx_list *ctx;
935 	device_t dev;
936 
937 	dev = softc->dev;
938 	ctx = device_get_sysctl_ctx(dev);
939 	child = SYSCTL_CHILDREN(device_get_sysctl_tree(dev));
940 
941 	SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "event_intr_count", CTLFLAG_RD,
942 	    &softc->event_intr_cnt, "Event interrupt count");
943 	SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "command_count", CTLFLAG_RD,
944 	    &softc->total_cmd, "Command submitted count");
945 	SYSCTL_ADD_U16(ctx, child, OID_AUTO, "pci_rid", CTLFLAG_RD,
946 	    &softc->pci_rid, 0, "IOMMU RID");
947 	SYSCTL_ADD_U16(ctx, child, OID_AUTO, "start_dev_rid", CTLFLAG_RD,
948 	    &softc->start_dev_rid, 0, "Start of device under this IOMMU");
949 	SYSCTL_ADD_U16(ctx, child, OID_AUTO, "end_dev_rid", CTLFLAG_RD,
950 	    &softc->end_dev_rid, 0, "End of device under this IOMMU");
951 	SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "command_head",
952 	    CTLTYPE_UINT | CTLFLAG_RD, softc, 0,
953 	    amdvi_handle_sysctl, "IU", "Command head");
954 	SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "command_tail",
955 	    CTLTYPE_UINT | CTLFLAG_RD, softc, 1,
956 	    amdvi_handle_sysctl, "IU", "Command tail");
957 	SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "event_head",
958 	    CTLTYPE_UINT | CTLFLAG_RD, softc, 2,
959 	    amdvi_handle_sysctl, "IU", "Command head");
960 	SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "event_tail",
961 	    CTLTYPE_UINT | CTLFLAG_RD, softc, 3,
962 	    amdvi_handle_sysctl, "IU", "Command tail");
963 }
964 
965 int
966 amdvi_setup_hw(struct amdvi_softc *softc)
967 {
968 	device_t dev;
969 	int status;
970 
971 	dev = softc->dev;
972 
973 	amdvi_hw_enable_iotlb(softc);
974 
975 	amdvi_print_dev_cap(softc);
976 
977 	if ((status = amdvi_print_pci_cap(dev)) != 0) {
978 		device_printf(dev, "PCI capability.\n");
979 		return (status);
980 	}
981 	if ((status = amdvi_init_cmd(softc)) != 0) {
982 		device_printf(dev, "Couldn't configure command buffer.\n");
983 		return (status);
984 	}
985 	if ((status = amdvi_init_event(softc)) != 0) {
986 		device_printf(dev, "Couldn't configure event buffer.\n");
987 		return (status);
988 	}
989 	if ((status = amdvi_init_dte(softc)) != 0) {
990 		device_printf(dev, "Couldn't configure device table.\n");
991 		return (status);
992 	}
993 	if ((status = amdvi_alloc_intr_resources(softc)) != 0) {
994 		return (status);
995 	}
996 	amdvi_add_sysctl(softc);
997 	return (0);
998 }
999 
1000 int
1001 amdvi_teardown_hw(struct amdvi_softc *softc)
1002 {
1003 	device_t dev;
1004 
1005 	dev = softc->dev;
1006 
1007 	/*
1008 	 * Called after disable, h/w is stopped by now, free all the resources.
1009 	 */
1010 	amdvi_free_evt_intr_res(dev);
1011 
1012 	if (softc->cmd)
1013 		free(softc->cmd, M_AMDVI);
1014 
1015 	if (softc->event)
1016 		free(softc->event, M_AMDVI);
1017 
1018 	return (0);
1019 }
1020 
1021 /*********** bhyve interfaces *********************/
1022 static int
1023 amdvi_init(void)
1024 {
1025 	if (!ivhd_count) {
1026 		return (EIO);
1027 	}
1028 	if (!amdvi_enable_user && ivhd_count) {
1029 		printf("bhyve: Found %d AMD-Vi/IOMMU device(s), "
1030 		    	"use hw.vmm.amdvi.enable=1 to enable pass-through.\n",
1031 		    ivhd_count);
1032 		return (EINVAL);
1033 	}
1034 	return (0);
1035 }
1036 
1037 static void
1038 amdvi_cleanup(void)
1039 {
1040 	/* Nothing. */
1041 }
1042 
1043 static uint16_t
1044 amdvi_domainId(void)
1045 {
1046 
1047 	/*
1048 	 * If we hit maximum domain limit, rollover leaving host
1049 	 * domain(0).
1050 	 * XXX: make sure that this domain is not used.
1051 	 */
1052 	if (amdvi_dom_id == AMDVI_MAX_DOMAIN)
1053 		amdvi_dom_id = 1;
1054 
1055 	return ((uint16_t)amdvi_dom_id++);
1056 }
1057 
1058 static void
1059 amdvi_do_inv_domain(uint16_t domain_id, bool create)
1060 {
1061 	struct amdvi_softc *softc;
1062 	int i;
1063 
1064 	for (i = 0; i < ivhd_count; i++) {
1065 		softc = device_get_softc(ivhd_devs[i]);
1066 		KASSERT(softc, ("softc is NULL"));
1067 		/*
1068 		 * If not present pages are cached, invalidate page after
1069 		 * creating domain.
1070 		 */
1071 #if 0
1072 		if (create && ((softc->pci_cap & AMDVI_PCI_CAP_NPCACHE) == 0))
1073 			continue;
1074 #endif
1075 		amdvi_inv_domain(softc, domain_id);
1076 		amdvi_wait(softc);
1077 	}
1078 }
1079 
1080 static void *
1081 amdvi_create_domain(vm_paddr_t maxaddr)
1082 {
1083 	struct amdvi_domain *dom;
1084 
1085 	dom = malloc(sizeof(struct amdvi_domain), M_AMDVI, M_ZERO | M_WAITOK);
1086 	dom->id = amdvi_domainId();
1087 	//dom->maxaddr = maxaddr;
1088 #ifdef AMDVI_DEBUG_CMD
1089 	printf("Created domain #%d\n", dom->id);
1090 #endif
1091 	/*
1092 	 * Host domain(#0) don't create translation table.
1093 	 */
1094 	if (dom->id || amdvi_host_ptp)
1095 		dom->ptp = malloc(PAGE_SIZE, M_AMDVI, M_WAITOK | M_ZERO);
1096 
1097 	dom->ptp_level = amdvi_ptp_level;
1098 
1099 	amdvi_do_inv_domain(dom->id, true);
1100 	SLIST_INSERT_HEAD(&dom_head, dom, next);
1101 
1102 	return (dom);
1103 }
1104 
1105 static void
1106 amdvi_free_ptp(uint64_t *ptp, int level)
1107 {
1108 	int i;
1109 
1110 	if (level < 1)
1111 		return;
1112 
1113 	for (i = 0; i < NPTEPG ; i++) {
1114 		if ((ptp[i] & AMDVI_PT_PRESENT) == 0)
1115 			continue;
1116 		/* XXX: Add super-page or PTE mapping > 4KB. */
1117 #ifdef notyet
1118 		/* Super-page mapping. */
1119 		if (AMDVI_PD_SUPER(ptp[i]))
1120 			continue;
1121 #endif
1122 
1123 		amdvi_free_ptp((uint64_t *)PHYS_TO_DMAP(ptp[i]
1124 		    & AMDVI_PT_MASK), level - 1);
1125 
1126 	}
1127 
1128 	free(ptp, M_AMDVI);
1129 }
1130 
1131 static void
1132 amdvi_destroy_domain(void *arg)
1133 {
1134 	struct amdvi_domain *domain;
1135 
1136 	domain = (struct amdvi_domain *)arg;
1137 	KASSERT(domain, ("domain is NULL"));
1138 #ifdef AMDVI_DEBUG_CMD
1139 	printf("Destroying domain %d\n", domain->id);
1140 #endif
1141 	if (domain->ptp)
1142 		amdvi_free_ptp(domain->ptp, domain->ptp_level);
1143 
1144 	amdvi_do_inv_domain(domain->id, false);
1145 	SLIST_REMOVE(&dom_head, domain, amdvi_domain, next);
1146 	free(domain, M_AMDVI);
1147 }
1148 
1149 static uint64_t
1150 amdvi_set_pt(uint64_t *pt, int level, vm_paddr_t gpa,
1151     vm_paddr_t hpa, uint64_t pg_size, bool create)
1152 {
1153 	uint64_t *page, pa;
1154 	int shift, index;
1155 	const int PT_SHIFT = 9;
1156 	const int PT_INDEX_MASK = (1 << PT_SHIFT) - 1;	/* Based on PT_SHIFT */
1157 
1158 	if (!pg_size)
1159 		return (0);
1160 
1161 	if (hpa & (pg_size - 1)) {
1162 		printf("HPA is not size aligned.\n");
1163 		return (0);
1164 	}
1165 	if (gpa & (pg_size - 1)) {
1166 		printf("HPA is not size aligned.\n");
1167 		return (0);
1168 	}
1169 	shift = PML4SHIFT;
1170 	while ((shift > PAGE_SHIFT) && (pg_size < (1UL << shift))) {
1171 		index = (gpa >> shift) & PT_INDEX_MASK;
1172 
1173 		if ((pt[index] == 0) && create) {
1174 			page = malloc(PAGE_SIZE, M_AMDVI, M_WAITOK | M_ZERO);
1175 			pa = vtophys(page);
1176 			pt[index] = pa | AMDVI_PT_PRESENT | AMDVI_PT_RW |
1177 			    ((level - 1) << AMDVI_PD_LEVEL_SHIFT);
1178 		}
1179 #ifdef AMDVI_DEBUG_PTE
1180 		if ((gpa % 0x1000000) == 0)
1181 			printf("[level%d, shift = %d]PTE:0x%lx\n",
1182 			    level, shift, pt[index]);
1183 #endif
1184 #define PTE2PA(x)	((uint64_t)(x) & AMDVI_PT_MASK)
1185 		pa = PTE2PA(pt[index]);
1186 		pt = (uint64_t *)PHYS_TO_DMAP(pa);
1187 		shift -= PT_SHIFT;
1188 		level--;
1189 	}
1190 
1191 	/* Leaf entry. */
1192 	index = (gpa >> shift) & PT_INDEX_MASK;
1193 
1194 	if (create) {
1195 		pt[index] = hpa | AMDVI_PT_RW | AMDVI_PT_PRESENT;
1196 	} else
1197 		pt[index] = 0;
1198 
1199 #ifdef AMDVI_DEBUG_PTE
1200 	if ((gpa % 0x1000000) == 0)
1201 		printf("[Last level%d, shift = %d]PTE:0x%lx\n",
1202 		    level, shift, pt[index]);
1203 #endif
1204 	return (1ULL << shift);
1205 }
1206 
1207 static uint64_t
1208 amdvi_update_mapping(struct amdvi_domain *domain, vm_paddr_t gpa,
1209     vm_paddr_t hpa, uint64_t size, bool create)
1210 {
1211 	uint64_t mapped, *ptp, len;
1212 	int level;
1213 
1214 	KASSERT(domain, ("domain is NULL"));
1215 	level = domain->ptp_level;
1216 	KASSERT(level, ("Page table level is 0"));
1217 
1218 	ptp = domain->ptp;
1219 	KASSERT(ptp, ("PTP is NULL"));
1220 	mapped = 0;
1221 	while (mapped < size) {
1222 		len = amdvi_set_pt(ptp, level, gpa + mapped, hpa + mapped,
1223 		    PAGE_SIZE, create);
1224 		if (!len) {
1225 			printf("Error: Couldn't map HPA:0x%lx GPA:0x%lx\n",
1226 			    hpa, gpa);
1227 			return (0);
1228 		}
1229 		mapped += len;
1230 	}
1231 
1232 	return (mapped);
1233 }
1234 
1235 static uint64_t
1236 amdvi_create_mapping(void *arg, vm_paddr_t gpa, vm_paddr_t hpa,
1237     uint64_t len)
1238 {
1239 	struct amdvi_domain *domain;
1240 
1241 	domain = (struct amdvi_domain *)arg;
1242 
1243 	if (domain->id && !domain->ptp) {
1244 		printf("ptp is NULL");
1245 		return (-1);
1246 	}
1247 
1248 	/*
1249 	 * If host domain is created w/o page table, skip IOMMU page
1250 	 * table set-up.
1251 	 */
1252 	if (domain->ptp)
1253 		return (amdvi_update_mapping(domain, gpa, hpa, len, true));
1254 	else
1255 		return (len);
1256 }
1257 
1258 static uint64_t
1259 amdvi_destroy_mapping(void *arg, vm_paddr_t gpa, uint64_t len)
1260 {
1261 	struct amdvi_domain *domain;
1262 
1263 	domain = (struct amdvi_domain *)arg;
1264 	/*
1265 	 * If host domain is created w/o page table, skip IOMMU page
1266 	 * table set-up.
1267 	 */
1268 	if (domain->ptp)
1269 		return (amdvi_update_mapping(domain, gpa, 0, len, false));
1270 	return
1271 	    (len);
1272 }
1273 
1274 static struct amdvi_softc *
1275 amdvi_find_iommu(uint16_t devid)
1276 {
1277 	struct amdvi_softc *softc;
1278 	int i;
1279 
1280 	for (i = 0; i < ivhd_count; i++) {
1281 		softc = device_get_softc(ivhd_devs[i]);
1282 		if ((devid >= softc->start_dev_rid) &&
1283 		    (devid <= softc->end_dev_rid))
1284 			return (softc);
1285 	}
1286 
1287 	/*
1288 	 * XXX: BIOS bug, device not in IVRS table, assume its from first IOMMU.
1289 	 */
1290 	printf("BIOS bug device(%d.%d.%d) doesn't have IVHD entry.\n",
1291 	    RID2PCI_STR(devid));
1292 
1293 	return (device_get_softc(ivhd_devs[0]));
1294 }
1295 
1296 /*
1297  * Set-up device table entry.
1298  * IOMMU spec Rev 2.0, section 3.2.2.2, some of the fields must
1299  * be set concurrently, e.g. read and write bits.
1300  */
1301 static void
1302 amdvi_set_dte(struct amdvi_domain *domain, uint16_t devid, bool enable)
1303 {
1304 	struct amdvi_softc *softc;
1305 	struct amdvi_dte* temp;
1306 
1307 	KASSERT(domain, ("domain is NULL for pci_rid:0x%x\n", devid));
1308 
1309 	softc = amdvi_find_iommu(devid);
1310 	KASSERT(softc, ("softc is NULL for pci_rid:0x%x\n", devid));
1311 
1312 	temp = &amdvi_dte[devid];
1313 
1314 #ifdef AMDVI_ATS_ENABLE
1315 	/* If IOMMU and device support IOTLB, enable it. */
1316 	if (amdvi_dev_support_iotlb(softc, devid) && softc->iotlb)
1317 		temp->iotlb_enable = 1;
1318 #endif
1319 
1320 	/* Avoid duplicate I/O faults. */
1321 	temp->sup_second_io_fault = 1;
1322 	temp->sup_all_io_fault = amdvi_disable_io_fault;
1323 
1324 	temp->dt_valid = 1;
1325 	temp->domain_id = domain->id;
1326 
1327 	if (enable) {
1328 		if (domain->ptp) {
1329 			temp->pt_base = vtophys(domain->ptp) >> 12;
1330 			temp->pt_level = amdvi_ptp_level;
1331 		}
1332 		/*
1333 		 * XXX: Page table valid[TV] bit must be set even if host domain
1334 		 * page tables are not enabled.
1335 		 */
1336 		temp->pt_valid = 1;
1337 		temp->read_allow = 1;
1338 		temp->write_allow = 1;
1339 	}
1340 }
1341 
1342 static void
1343 amdvi_inv_device(uint16_t devid)
1344 {
1345 	struct amdvi_softc *softc;
1346 
1347 	softc = amdvi_find_iommu(devid);
1348 	KASSERT(softc, ("softc is NULL"));
1349 
1350 	amdvi_cmd_inv_dte(softc, devid);
1351 #ifdef AMDVI_ATS_ENABLE
1352 	if (amdvi_dev_support_iotlb(softc, devid))
1353 		amdvi_cmd_inv_iotlb(softc, devid);
1354 #endif
1355 	amdvi_wait(softc);
1356 }
1357 
1358 static void
1359 amdvi_add_device(void *arg, uint16_t devid)
1360 {
1361 	struct amdvi_domain *domain;
1362 
1363 	domain = (struct amdvi_domain *)arg;
1364 	KASSERT(domain != NULL, ("domain is NULL"));
1365 #ifdef AMDVI_DEBUG_CMD
1366 	printf("Assigning device(%d.%d.%d) to domain:%d\n",
1367 	    RID2PCI_STR(devid), domain->id);
1368 #endif
1369 	amdvi_set_dte(domain, devid, true);
1370 	amdvi_inv_device(devid);
1371 }
1372 
1373 static void
1374 amdvi_remove_device(void *arg, uint16_t devid)
1375 {
1376 	struct amdvi_domain *domain;
1377 
1378 	domain = (struct amdvi_domain *)arg;
1379 #ifdef AMDVI_DEBUG_CMD
1380 	printf("Remove device(0x%x) from domain:%d\n",
1381 	       devid, domain->id);
1382 #endif
1383 	amdvi_set_dte(domain, devid, false);
1384 	amdvi_inv_device(devid);
1385 }
1386 
1387 static void
1388 amdvi_enable(void)
1389 {
1390 	struct amdvi_ctrl *ctrl;
1391 	struct amdvi_softc *softc;
1392 	uint64_t val;
1393 	int i;
1394 
1395 	for (i = 0; i < ivhd_count; i++) {
1396 		softc = device_get_softc(ivhd_devs[i]);
1397 		KASSERT(softc, ("softc is NULL\n"));
1398 		ctrl = softc->ctrl;
1399 		KASSERT(ctrl, ("ctrl is NULL\n"));
1400 
1401 		val = (	AMDVI_CTRL_EN 		|
1402 			AMDVI_CTRL_CMD 		|
1403 		    	AMDVI_CTRL_ELOG 	|
1404 		    	AMDVI_CTRL_ELOGINT 	|
1405 		    	AMDVI_CTRL_INV_TO_1S);
1406 
1407 		if (softc->ivhd_flag & IVHD_FLAG_COH)
1408 			val |= AMDVI_CTRL_COH;
1409 		if (softc->ivhd_flag & IVHD_FLAG_HTT)
1410 			val |= AMDVI_CTRL_HTT;
1411 		if (softc->ivhd_flag & IVHD_FLAG_RPPW)
1412 			val |= AMDVI_CTRL_RPPW;
1413 		if (softc->ivhd_flag & IVHD_FLAG_PPW)
1414 			val |= AMDVI_CTRL_PPW;
1415 		if (softc->ivhd_flag & IVHD_FLAG_ISOC)
1416 			val |= AMDVI_CTRL_ISOC;
1417 
1418 		ctrl->control = val;
1419 	}
1420 }
1421 
1422 static void
1423 amdvi_disable(void)
1424 {
1425 	struct amdvi_ctrl *ctrl;
1426 	struct amdvi_softc *softc;
1427 	int i;
1428 
1429 	for (i = 0; i < ivhd_count; i++) {
1430 		softc = device_get_softc(ivhd_devs[i]);
1431 		KASSERT(softc, ("softc is NULL\n"));
1432 		ctrl = softc->ctrl;
1433 		KASSERT(ctrl, ("ctrl is NULL\n"));
1434 
1435 		ctrl->control = 0;
1436 	}
1437 }
1438 
1439 static void
1440 amdvi_inv_tlb(void *arg)
1441 {
1442 	struct amdvi_domain *domain;
1443 
1444 	domain = (struct amdvi_domain *)arg;
1445 	KASSERT(domain, ("domain is NULL"));
1446 	amdvi_do_inv_domain(domain->id, false);
1447 }
1448 
1449 struct iommu_ops iommu_ops_amd = {
1450 	amdvi_init,
1451 	amdvi_cleanup,
1452 	amdvi_enable,
1453 	amdvi_disable,
1454 	amdvi_create_domain,
1455 	amdvi_destroy_domain,
1456 	amdvi_create_mapping,
1457 	amdvi_destroy_mapping,
1458 	amdvi_add_device,
1459 	amdvi_remove_device,
1460 	amdvi_inv_tlb
1461 };
1462