xref: /freebsd/sys/amd64/vmm/amd/amdvi_hw.c (revision 25ecdc7d52770caf1c9b44b5ec11f468f6b636f3)
1 /*-
2  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
3  *
4  * Copyright (c) 2016, Anish Gupta (anish@freebsd.org)
5  * All rights reserved.
6  *
7  * Redistribution and use in source and binary forms, with or without
8  * modification, are permitted provided that the following conditions
9  * are met:
10  * 1. Redistributions of source code must retain the above copyright
11  *    notice unmodified, this list of conditions, and the following
12  *    disclaimer.
13  * 2. Redistributions in binary form must reproduce the above copyright
14  *    notice, this list of conditions and the following disclaimer in the
15  *    documentation and/or other materials provided with the distribution.
16  *
17  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
18  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
19  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
20  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
21  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
22  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
23  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
24  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
25  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
26  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
27  */
28 
29 #include <sys/cdefs.h>
30 __FBSDID("$FreeBSD$");
31 
32 #include <sys/param.h>
33 #include <sys/systm.h>
34 #include <sys/bus.h>
35 #include <sys/kernel.h>
36 #include <sys/module.h>
37 #include <sys/malloc.h>
38 #include <sys/pcpu.h>
39 #include <sys/rman.h>
40 #include <sys/smp.h>
41 #include <sys/sysctl.h>
42 
43 #include <vm/vm.h>
44 #include <vm/pmap.h>
45 
46 #include <dev/pci/pcivar.h>
47 #include <dev/pci/pcireg.h>
48 
49 #include <machine/resource.h>
50 #include <machine/vmm.h>
51 #include <machine/pmap.h>
52 #include <machine/vmparam.h>
53 #include <machine/pci_cfgreg.h>
54 
55 #include "ivhd_if.h"
56 #include "pcib_if.h"
57 
58 #include "io/iommu.h"
59 #include "amdvi_priv.h"
60 
61 SYSCTL_DECL(_hw_vmm);
62 SYSCTL_NODE(_hw_vmm, OID_AUTO, amdvi, CTLFLAG_RW | CTLFLAG_MPSAFE, NULL,
63     NULL);
64 
65 #define MOD_INC(a, s, m) (((a) + (s)) % ((m) * (s)))
66 #define MOD_DEC(a, s, m) (((a) - (s)) % ((m) * (s)))
67 
68 /* Print RID or device ID in PCI string format. */
69 #define RID2PCI_STR(d) PCI_RID2BUS(d), PCI_RID2SLOT(d), PCI_RID2FUNC(d)
70 
71 static void amdvi_dump_cmds(struct amdvi_softc *softc, int count);
72 static void amdvi_print_dev_cap(struct amdvi_softc *softc);
73 
74 MALLOC_DEFINE(M_AMDVI, "amdvi", "amdvi");
75 
76 extern device_t *ivhd_devs;
77 
78 extern int ivhd_count;
79 SYSCTL_INT(_hw_vmm_amdvi, OID_AUTO, count, CTLFLAG_RDTUN, &ivhd_count,
80     0, NULL);
81 
82 static int amdvi_enable_user = 0;
83 SYSCTL_INT(_hw_vmm_amdvi, OID_AUTO, enable, CTLFLAG_RDTUN,
84     &amdvi_enable_user, 0, NULL);
85 TUNABLE_INT("hw.vmm.amdvi_enable", &amdvi_enable_user);
86 
87 #ifdef AMDVI_ATS_ENABLE
88 /* XXX: ATS is not tested. */
89 static int amdvi_enable_iotlb = 1;
90 SYSCTL_INT(_hw_vmm_amdvi, OID_AUTO, iotlb_enabled, CTLFLAG_RDTUN,
91     &amdvi_enable_iotlb, 0, NULL);
92 TUNABLE_INT("hw.vmm.enable_iotlb", &amdvi_enable_iotlb);
93 #endif
94 
95 static int amdvi_host_ptp = 1;	/* Use page tables for host. */
96 SYSCTL_INT(_hw_vmm_amdvi, OID_AUTO, host_ptp, CTLFLAG_RDTUN,
97     &amdvi_host_ptp, 0, NULL);
98 TUNABLE_INT("hw.vmm.amdvi.host_ptp", &amdvi_host_ptp);
99 
100 /* Page table level used <= supported by h/w[v1=7]. */
101 int amdvi_ptp_level = 4;
102 SYSCTL_INT(_hw_vmm_amdvi, OID_AUTO, ptp_level, CTLFLAG_RDTUN,
103     &amdvi_ptp_level, 0, NULL);
104 TUNABLE_INT("hw.vmm.amdvi.ptp_level", &amdvi_ptp_level);
105 
106 /* Disable fault event reporting. */
107 static int amdvi_disable_io_fault = 0;
108 SYSCTL_INT(_hw_vmm_amdvi, OID_AUTO, disable_io_fault, CTLFLAG_RDTUN,
109     &amdvi_disable_io_fault, 0, NULL);
110 TUNABLE_INT("hw.vmm.amdvi.disable_io_fault", &amdvi_disable_io_fault);
111 
112 static uint32_t amdvi_dom_id = 0;	/* 0 is reserved for host. */
113 SYSCTL_UINT(_hw_vmm_amdvi, OID_AUTO, domain_id, CTLFLAG_RD,
114     &amdvi_dom_id, 0, NULL);
115 /*
116  * Device table entry.
117  * Bus(256) x Dev(32) x Fun(8) x DTE(256 bits or 32 bytes).
118  *	= 256 * 2 * PAGE_SIZE.
119  */
120 static struct amdvi_dte amdvi_dte[PCI_NUM_DEV_MAX] __aligned(PAGE_SIZE);
121 CTASSERT(PCI_NUM_DEV_MAX == 0x10000);
122 CTASSERT(sizeof(amdvi_dte) == 0x200000);
123 
124 static SLIST_HEAD (, amdvi_domain) dom_head;
125 
126 static inline uint32_t
127 amdvi_pci_read(struct amdvi_softc *softc, int off)
128 {
129 
130 	return (pci_cfgregread(PCI_RID2BUS(softc->pci_rid),
131 	    PCI_RID2SLOT(softc->pci_rid), PCI_RID2FUNC(softc->pci_rid),
132 	    off, 4));
133 }
134 
135 #ifdef AMDVI_ATS_ENABLE
136 /* XXX: Should be in pci.c */
137 /*
138  * Check if device has ATS capability and its enabled.
139  * If ATS is absent or disabled, return (-1), otherwise ATS
140  * queue length.
141  */
142 static int
143 amdvi_find_ats_qlen(uint16_t devid)
144 {
145 	device_t dev;
146 	uint32_t off, cap;
147 	int qlen = -1;
148 
149 	dev = pci_find_bsf(PCI_RID2BUS(devid), PCI_RID2SLOT(devid),
150 			   PCI_RID2FUNC(devid));
151 
152 	if (!dev) {
153 		return (-1);
154 	}
155 #define PCIM_ATS_EN	BIT(31)
156 
157 	if (pci_find_extcap(dev, PCIZ_ATS, &off) == 0) {
158 		cap = pci_read_config(dev, off + 4, 4);
159 		qlen = (cap & 0x1F);
160 		qlen = qlen ? qlen : 32;
161 		printf("AMD-Vi: PCI device %d.%d.%d ATS %s qlen=%d\n",
162 		       RID2PCI_STR(devid),
163 		       (cap & PCIM_ATS_EN) ? "enabled" : "Disabled",
164 		       qlen);
165 		qlen = (cap & PCIM_ATS_EN) ? qlen : -1;
166 	}
167 
168 	return (qlen);
169 }
170 
171 /*
172  * Check if an endpoint device support device IOTLB or ATS.
173  */
174 static inline bool
175 amdvi_dev_support_iotlb(struct amdvi_softc *softc, uint16_t devid)
176 {
177 	struct ivhd_dev_cfg *cfg;
178 	int qlen, i;
179 	bool pci_ats, ivhd_ats;
180 
181 	qlen = amdvi_find_ats_qlen(devid);
182 	if (qlen < 0)
183 		return (false);
184 
185 	KASSERT(softc, ("softc is NULL"));
186 	cfg = softc->dev_cfg;
187 
188 	ivhd_ats = false;
189 	for (i = 0; i < softc->dev_cfg_cnt; i++) {
190 		if ((cfg->start_id <= devid) && (cfg->end_id >= devid)) {
191 			ivhd_ats = cfg->enable_ats;
192 			break;
193 		}
194 		cfg++;
195 	}
196 
197 	pci_ats = (qlen < 0) ? false : true;
198 	if (pci_ats != ivhd_ats)
199 		device_printf(softc->dev,
200 		    "BIOS bug: mismatch in ATS setting for %d.%d.%d,"
201 		    "ATS inv qlen = %d\n", RID2PCI_STR(devid), qlen);
202 
203 	/* Ignore IVRS setting and respect PCI setting. */
204 	return (pci_ats);
205 }
206 #endif
207 
208 /* Enable IOTLB support for IOMMU if its supported. */
209 static inline void
210 amdvi_hw_enable_iotlb(struct amdvi_softc *softc)
211 {
212 #ifndef AMDVI_ATS_ENABLE
213 	softc->iotlb = false;
214 #else
215 	bool supported;
216 
217 	supported = (softc->ivhd_flag & IVHD_FLAG_IOTLB) ? true : false;
218 
219 	if (softc->pci_cap & AMDVI_PCI_CAP_IOTLB) {
220 		if (!supported)
221 			device_printf(softc->dev, "IOTLB disabled by BIOS.\n");
222 
223 		if (supported && !amdvi_enable_iotlb) {
224 			device_printf(softc->dev, "IOTLB disabled by user.\n");
225 			supported = false;
226 		}
227 	} else
228 		supported = false;
229 
230 	softc->iotlb = supported;
231 
232 #endif
233 }
234 
235 static int
236 amdvi_init_cmd(struct amdvi_softc *softc)
237 {
238 	struct amdvi_ctrl *ctrl = softc->ctrl;
239 
240 	ctrl->cmd.len = 8;	/* Use 256 command buffer entries. */
241 	softc->cmd_max = 1 << ctrl->cmd.len;
242 
243 	softc->cmd = malloc(sizeof(struct amdvi_cmd) *
244 	    softc->cmd_max, M_AMDVI, M_WAITOK | M_ZERO);
245 
246 	if ((uintptr_t)softc->cmd & PAGE_MASK)
247 		panic("AMDVi: Command buffer not aligned on page boundary.");
248 
249 	ctrl->cmd.base = vtophys(softc->cmd) / PAGE_SIZE;
250 	/*
251 	 * XXX: Reset the h/w pointers in case IOMMU is restarting,
252 	 * h/w doesn't clear these pointers based on empirical data.
253 	 */
254 	ctrl->cmd_tail = 0;
255 	ctrl->cmd_head = 0;
256 
257 	return (0);
258 }
259 
260 /*
261  * Note: Update tail pointer after we have written the command since tail
262  * pointer update cause h/w to execute new commands, see section 3.3
263  * of AMD IOMMU spec ver 2.0.
264  */
265 /* Get the command tail pointer w/o updating it. */
266 static struct amdvi_cmd *
267 amdvi_get_cmd_tail(struct amdvi_softc *softc)
268 {
269 	struct amdvi_ctrl *ctrl;
270 	struct amdvi_cmd *tail;
271 
272 	KASSERT(softc, ("softc is NULL"));
273 	KASSERT(softc->cmd != NULL, ("cmd is NULL"));
274 
275 	ctrl = softc->ctrl;
276 	KASSERT(ctrl != NULL, ("ctrl is NULL"));
277 
278 	tail = (struct amdvi_cmd *)((uint8_t *)softc->cmd +
279 	    ctrl->cmd_tail);
280 
281 	return (tail);
282 }
283 
284 /*
285  * Update the command tail pointer which will start command execution.
286  */
287 static void
288 amdvi_update_cmd_tail(struct amdvi_softc *softc)
289 {
290 	struct amdvi_ctrl *ctrl;
291 	int size;
292 
293 	size = sizeof(struct amdvi_cmd);
294 	KASSERT(softc->cmd != NULL, ("cmd is NULL"));
295 
296 	ctrl = softc->ctrl;
297 	KASSERT(ctrl != NULL, ("ctrl is NULL"));
298 
299 	ctrl->cmd_tail = MOD_INC(ctrl->cmd_tail, size, softc->cmd_max);
300 	softc->total_cmd++;
301 
302 #ifdef AMDVI_DEBUG_CMD
303 	device_printf(softc->dev, "cmd_tail: %s Tail:0x%x, Head:0x%x.\n",
304 	    ctrl->cmd_tail,
305 	    ctrl->cmd_head);
306 #endif
307 
308 }
309 
310 /*
311  * Various commands supported by IOMMU.
312  */
313 
314 /* Completion wait command. */
315 static void
316 amdvi_cmd_cmp(struct amdvi_softc *softc, const uint64_t data)
317 {
318 	struct amdvi_cmd *cmd;
319 	uint64_t pa;
320 
321 	cmd = amdvi_get_cmd_tail(softc);
322 	KASSERT(cmd != NULL, ("Cmd is NULL"));
323 
324 	pa = vtophys(&softc->cmp_data);
325 	cmd->opcode = AMDVI_CMP_WAIT_OPCODE;
326 	cmd->word0 = (pa & 0xFFFFFFF8) | AMDVI_CMP_WAIT_STORE;
327 	cmd->word1 = (pa >> 32) & 0xFFFFF;
328 	cmd->addr = data;
329 
330 	amdvi_update_cmd_tail(softc);
331 }
332 
333 /* Invalidate device table entry. */
334 static void
335 amdvi_cmd_inv_dte(struct amdvi_softc *softc, uint16_t devid)
336 {
337 	struct amdvi_cmd *cmd;
338 
339 	cmd = amdvi_get_cmd_tail(softc);
340 	KASSERT(cmd != NULL, ("Cmd is NULL"));
341 	cmd->opcode = AMDVI_INVD_DTE_OPCODE;
342 	cmd->word0 = devid;
343 	amdvi_update_cmd_tail(softc);
344 #ifdef AMDVI_DEBUG_CMD
345 	device_printf(softc->dev, "Invalidated DTE:0x%x\n", devid);
346 #endif
347 }
348 
349 /* Invalidate IOMMU page, use for invalidation of domain. */
350 static void
351 amdvi_cmd_inv_iommu_pages(struct amdvi_softc *softc, uint16_t domain_id,
352 			  uint64_t addr, bool guest_nested,
353 			  bool pde, bool page)
354 {
355 	struct amdvi_cmd *cmd;
356 
357 	cmd = amdvi_get_cmd_tail(softc);
358 	KASSERT(cmd != NULL, ("Cmd is NULL"));
359 
360 	cmd->opcode = AMDVI_INVD_PAGE_OPCODE;
361 	cmd->word1 = domain_id;
362 	/*
363 	 * Invalidate all addresses for this domain.
364 	 */
365 	cmd->addr = addr;
366 	cmd->addr |= pde ? AMDVI_INVD_PAGE_PDE : 0;
367 	cmd->addr |= page ? AMDVI_INVD_PAGE_S : 0;
368 
369 	amdvi_update_cmd_tail(softc);
370 }
371 
372 #ifdef AMDVI_ATS_ENABLE
373 /* Invalidate device IOTLB. */
374 static void
375 amdvi_cmd_inv_iotlb(struct amdvi_softc *softc, uint16_t devid)
376 {
377 	struct amdvi_cmd *cmd;
378 	int qlen;
379 
380 	if (!softc->iotlb)
381 		return;
382 
383 	qlen = amdvi_find_ats_qlen(devid);
384 	if (qlen < 0) {
385 		panic("AMDVI: Invalid ATS qlen(%d) for device %d.%d.%d\n",
386 		      qlen, RID2PCI_STR(devid));
387 	}
388 	cmd = amdvi_get_cmd_tail(softc);
389 	KASSERT(cmd != NULL, ("Cmd is NULL"));
390 
391 #ifdef AMDVI_DEBUG_CMD
392 	device_printf(softc->dev, "Invalidate IOTLB devID 0x%x"
393 		      " Qlen:%d\n", devid, qlen);
394 #endif
395 	cmd->opcode = AMDVI_INVD_IOTLB_OPCODE;
396 	cmd->word0 = devid;
397 	cmd->word1 = qlen;
398 	cmd->addr = AMDVI_INVD_IOTLB_ALL_ADDR |
399 		AMDVI_INVD_IOTLB_S;
400 	amdvi_update_cmd_tail(softc);
401 }
402 #endif
403 
404 #ifdef notyet				/* For Interrupt Remap. */
405 static void
406 amdvi_cmd_inv_intr_map(struct amdvi_softc *softc,
407 		       uint16_t devid)
408 {
409 	struct amdvi_cmd *cmd;
410 
411 	cmd = amdvi_get_cmd_tail(softc);
412 	KASSERT(cmd != NULL, ("Cmd is NULL"));
413 	cmd->opcode = AMDVI_INVD_INTR_OPCODE;
414 	cmd->word0 = devid;
415 	amdvi_update_cmd_tail(softc);
416 #ifdef AMDVI_DEBUG_CMD
417 	device_printf(softc->dev, "Invalidate INTR map of devID 0x%x\n", devid);
418 #endif
419 }
420 #endif
421 
422 /* Invalidate domain using INVALIDATE_IOMMU_PAGES command. */
423 static void
424 amdvi_inv_domain(struct amdvi_softc *softc, uint16_t domain_id)
425 {
426 	struct amdvi_cmd *cmd;
427 
428 	cmd = amdvi_get_cmd_tail(softc);
429 	KASSERT(cmd != NULL, ("Cmd is NULL"));
430 
431 	/*
432 	 * See section 3.3.3 of IOMMU spec rev 2.0, software note
433 	 * for invalidating domain.
434 	 */
435 	amdvi_cmd_inv_iommu_pages(softc, domain_id, AMDVI_INVD_PAGE_ALL_ADDR,
436 				false, true, true);
437 
438 #ifdef AMDVI_DEBUG_CMD
439 	device_printf(softc->dev, "Invalidate domain:0x%x\n", domain_id);
440 
441 #endif
442 }
443 
444 static	bool
445 amdvi_cmp_wait(struct amdvi_softc *softc)
446 {
447 	struct amdvi_ctrl *ctrl;
448 	const uint64_t VERIFY = 0xA5A5;
449 	volatile uint64_t *read;
450 	int i;
451 	bool status;
452 
453 	ctrl = softc->ctrl;
454 	read = &softc->cmp_data;
455 	*read = 0;
456 	amdvi_cmd_cmp(softc, VERIFY);
457 	/* Wait for h/w to update completion data. */
458 	for (i = 0; i < 100 && (*read != VERIFY); i++) {
459 		DELAY(1000);		/* 1 ms */
460 	}
461 	status = (VERIFY == softc->cmp_data) ? true : false;
462 
463 #ifdef AMDVI_DEBUG_CMD
464 	if (status)
465 		device_printf(softc->dev, "CMD completion DONE Tail:0x%x, "
466 			      "Head:0x%x, loop:%d.\n", ctrl->cmd_tail,
467 			      ctrl->cmd_head, loop);
468 #endif
469 	return (status);
470 }
471 
472 static void
473 amdvi_wait(struct amdvi_softc *softc)
474 {
475 	struct amdvi_ctrl *ctrl;
476 	int i;
477 
478 	KASSERT(softc, ("softc is NULL"));
479 
480 	ctrl = softc->ctrl;
481 	KASSERT(ctrl != NULL, ("ctrl is NULL"));
482 	/* Don't wait if h/w is not enabled. */
483 	if ((ctrl->control & AMDVI_CTRL_EN) == 0)
484 		return;
485 
486 	for (i = 0; i < 10; i++) {
487 		if (amdvi_cmp_wait(softc))
488 			return;
489 	}
490 
491 	device_printf(softc->dev, "Error: completion failed"
492 		      " tail:0x%x, head:0x%x.\n",
493 		      ctrl->cmd_tail, ctrl->cmd_head);
494 	/* Dump the last command. */
495 	amdvi_dump_cmds(softc, 1);
496 }
497 
498 static void
499 amdvi_dump_cmds(struct amdvi_softc *softc, int count)
500 {
501 	struct amdvi_ctrl *ctrl;
502 	struct amdvi_cmd *cmd;
503 	int off, i;
504 
505 	ctrl = softc->ctrl;
506 	device_printf(softc->dev, "Dump last %d command(s):\n", count);
507 	/*
508 	 * If h/w is stuck in completion, it is the previous command,
509 	 * start dumping from previous command onward.
510 	 */
511 	off = MOD_DEC(ctrl->cmd_head, sizeof(struct amdvi_cmd),
512 	    softc->cmd_max);
513 	for (i = 0; off != ctrl->cmd_tail && i < count; i++) {
514 		cmd = (struct amdvi_cmd *)((uint8_t *)softc->cmd + off);
515 		printf("  [CMD%d, off:0x%x] opcode= 0x%x 0x%x"
516 		    " 0x%x 0x%lx\n", i, off, cmd->opcode,
517 		    cmd->word0, cmd->word1, cmd->addr);
518 		off = MOD_INC(off, sizeof(struct amdvi_cmd), softc->cmd_max);
519 	}
520 }
521 
522 static int
523 amdvi_init_event(struct amdvi_softc *softc)
524 {
525 	struct amdvi_ctrl *ctrl;
526 
527 	ctrl = softc->ctrl;
528 	ctrl->event.len = 8;
529 	softc->event_max = 1 << ctrl->event.len;
530 	softc->event = malloc(sizeof(struct amdvi_event) *
531 	    softc->event_max, M_AMDVI, M_WAITOK | M_ZERO);
532 	if ((uintptr_t)softc->event & PAGE_MASK) {
533 		device_printf(softc->dev, "Event buffer not aligned on page.");
534 		return (false);
535 	}
536 	ctrl->event.base = vtophys(softc->event) / PAGE_SIZE;
537 
538 	/* Reset the pointers. */
539 	ctrl->evt_head = 0;
540 	ctrl->evt_tail = 0;
541 
542 	return (0);
543 }
544 
545 static inline void
546 amdvi_decode_evt_flag(uint16_t flag)
547 {
548 
549 	flag &= AMDVI_EVENT_FLAG_MASK;
550 	printf(" 0x%b]\n", flag,
551 		"\020"
552 		"\001GN"
553 		"\002NX"
554 		"\003US"
555 		"\004I"
556 		"\005PR"
557 		"\006RW"
558 		"\007PE"
559 		"\010RZ"
560 		"\011TR"
561 		);
562 }
563 
564 /* See section 2.5.4 of AMD IOMMU spec ver 2.62.*/
565 static inline void
566 amdvi_decode_evt_flag_type(uint8_t type)
567 {
568 
569 	switch (AMDVI_EVENT_FLAG_TYPE(type)) {
570 	case 0:
571 		printf("RSVD\n");
572 		break;
573 	case 1:
574 		printf("Master Abort\n");
575 		break;
576 	case 2:
577 		printf("Target Abort\n");
578 		break;
579 	case 3:
580 		printf("Data Err\n");
581 		break;
582 	default:
583 		break;
584 	}
585 }
586 
587 static void
588 amdvi_decode_inv_dte_evt(uint16_t devid, uint16_t domid, uint64_t addr,
589     uint16_t flag)
590 {
591 
592 	printf("\t[IO_PAGE_FAULT EVT: devId:0x%x DomId:0x%x"
593 	    " Addr:0x%lx",
594 	    devid, domid, addr);
595 	amdvi_decode_evt_flag(flag);
596 }
597 
598 static void
599 amdvi_decode_pf_evt(uint16_t devid, uint16_t domid, uint64_t addr,
600     uint16_t flag)
601 {
602 
603 	printf("\t[IO_PAGE_FAULT EVT: devId:0x%x DomId:0x%x"
604 	    " Addr:0x%lx",
605 	    devid, domid, addr);
606 	amdvi_decode_evt_flag(flag);
607 }
608 
609 static void
610 amdvi_decode_dte_hwerr_evt(uint16_t devid, uint16_t domid,
611     uint64_t addr, uint16_t flag)
612 {
613 
614 	printf("\t[DEV_TAB_HW_ERR EVT: devId:0x%x DomId:0x%x"
615 	    " Addr:0x%lx", devid, domid, addr);
616 	amdvi_decode_evt_flag(flag);
617 	amdvi_decode_evt_flag_type(flag);
618 }
619 
620 static void
621 amdvi_decode_page_hwerr_evt(uint16_t devid, uint16_t domid, uint64_t addr,
622     uint16_t flag)
623 {
624 
625 	printf("\t[PAGE_TAB_HW_ERR EVT: devId:0x%x DomId:0x%x"
626 	    " Addr:0x%lx", devid, domid, addr);
627 	amdvi_decode_evt_flag(flag);
628 	amdvi_decode_evt_flag_type(AMDVI_EVENT_FLAG_TYPE(flag));
629 }
630 
631 static void
632 amdvi_decode_evt(struct amdvi_event *evt)
633 {
634 	struct amdvi_cmd *cmd;
635 
636 	switch (evt->opcode) {
637 	case AMDVI_EVENT_INVALID_DTE:
638 		amdvi_decode_inv_dte_evt(evt->devid, evt->pasid_domid,
639 		    evt->addr, evt->flag);
640 		break;
641 
642 	case AMDVI_EVENT_PFAULT:
643 		amdvi_decode_pf_evt(evt->devid, evt->pasid_domid,
644 		    evt->addr, evt->flag);
645 		break;
646 
647 	case AMDVI_EVENT_DTE_HW_ERROR:
648 		amdvi_decode_dte_hwerr_evt(evt->devid, evt->pasid_domid,
649 		    evt->addr, evt->flag);
650 		break;
651 
652 	case AMDVI_EVENT_PAGE_HW_ERROR:
653 		amdvi_decode_page_hwerr_evt(evt->devid, evt->pasid_domid,
654 		    evt->addr, evt->flag);
655 		break;
656 
657 	case AMDVI_EVENT_ILLEGAL_CMD:
658 		/* FALL THROUGH */
659 	case AMDVI_EVENT_CMD_HW_ERROR:
660 		printf("\t[%s EVT]\n", (evt->opcode == AMDVI_EVENT_ILLEGAL_CMD) ?
661 		    "ILLEGAL CMD" : "CMD HW ERR");
662 		cmd = (struct amdvi_cmd *)PHYS_TO_DMAP(evt->addr);
663 		printf("\tCMD opcode= 0x%x 0x%x 0x%x 0x%lx\n",
664 		    cmd->opcode, cmd->word0, cmd->word1, cmd->addr);
665 		break;
666 
667 	case AMDVI_EVENT_IOTLB_TIMEOUT:
668 		printf("\t[IOTLB_INV_TIMEOUT devid:0x%x addr:0x%lx]\n",
669 		    evt->devid, evt->addr);
670 		break;
671 
672 	case AMDVI_EVENT_INVALID_DTE_REQ:
673 		printf("\t[INV_DTE devid:0x%x addr:0x%lx type:0x%x tr:%d]\n",
674 		    evt->devid, evt->addr, evt->flag >> 9,
675 		    (evt->flag >> 8) & 1);
676 		break;
677 
678 	case AMDVI_EVENT_INVALID_PPR_REQ:
679 	case AMDVI_EVENT_COUNTER_ZERO:
680 		printf("AMD-Vi: v2 events.\n");
681 		break;
682 
683 	default:
684 		printf("Unsupported AMD-Vi event:%d\n", evt->opcode);
685 	}
686 }
687 
688 static void
689 amdvi_print_events(struct amdvi_softc *softc)
690 {
691 	struct amdvi_ctrl *ctrl;
692 	struct amdvi_event *event;
693 	int i, size;
694 
695 	ctrl = softc->ctrl;
696 	size = sizeof(struct amdvi_event);
697 	for (i = 0; i < softc->event_max; i++) {
698 		event = &softc->event[ctrl->evt_head / size];
699 		if (!event->opcode)
700 			break;
701 		device_printf(softc->dev, "\t[Event%d: Head:0x%x Tail:0x%x]\n",
702 		    i, ctrl->evt_head, ctrl->evt_tail);
703 		amdvi_decode_evt(event);
704 		ctrl->evt_head = MOD_INC(ctrl->evt_head, size,
705 		    softc->event_max);
706 	}
707 }
708 
709 static int
710 amdvi_init_dte(struct amdvi_softc *softc)
711 {
712 	struct amdvi_ctrl *ctrl;
713 
714 	ctrl = softc->ctrl;
715 	ctrl->dte.base = vtophys(amdvi_dte) / PAGE_SIZE;
716 	ctrl->dte.size = 0x1FF;		/* 2MB device table. */
717 
718 	return (0);
719 }
720 
721 /*
722  * Not all capabilities of IOMMU are available in ACPI IVHD flag
723  * or EFR entry, read directly from device.
724  */
725 static int
726 amdvi_print_pci_cap(device_t dev)
727 {
728 	struct amdvi_softc *softc;
729 	uint32_t off, cap;
730 
731 	softc = device_get_softc(dev);
732 	off = softc->cap_off;
733 
734 	/*
735 	 * Section 3.7.1 of IOMMU sepc rev 2.0.
736 	 * Read capability from device.
737 	 */
738 	cap = amdvi_pci_read(softc, off);
739 
740 	/* Make sure capability type[18:16] is 3. */
741 	KASSERT((((cap >> 16) & 0x7) == 0x3),
742 	    ("Not a IOMMU capability 0x%x@0x%x", cap, off));
743 
744 	softc->pci_cap = cap >> 24;
745 	device_printf(softc->dev, "PCI cap 0x%x@0x%x feature:%b\n",
746 	    cap, off, softc->pci_cap,
747 	    "\20\1IOTLB\2HT\3NPCache\4EFR\5CapExt");
748 
749 	return (0);
750 }
751 
752 static void
753 amdvi_event_intr(void *arg)
754 {
755 	struct amdvi_softc *softc;
756 	struct amdvi_ctrl *ctrl;
757 
758 	softc = (struct amdvi_softc *)arg;
759 	ctrl = softc->ctrl;
760 	device_printf(softc->dev, "EVT INTR %ld Status:0x%x"
761 	    " EVT Head:0x%x Tail:0x%x]\n", softc->event_intr_cnt++,
762 	    ctrl->status, ctrl->evt_head, ctrl->evt_tail);
763 	printf("  [CMD Total 0x%lx] Tail:0x%x, Head:0x%x.\n",
764 	    softc->total_cmd, ctrl->cmd_tail, ctrl->cmd_head);
765 
766 	amdvi_print_events(softc);
767 	ctrl->status &= AMDVI_STATUS_EV_OF | AMDVI_STATUS_EV_INTR;
768 }
769 
770 static void
771 amdvi_free_evt_intr_res(device_t dev)
772 {
773 
774 	struct amdvi_softc *softc;
775 	device_t mmio_dev;
776 
777 	softc = device_get_softc(dev);
778 	mmio_dev = softc->pci_dev;
779 
780 	IVHD_TEARDOWN_INTR(mmio_dev);
781 }
782 
783 static bool
784 amdvi_alloc_intr_resources(struct amdvi_softc *softc)
785 {
786 	struct amdvi_ctrl *ctrl;
787 	device_t dev, mmio_dev;
788 	int err;
789 
790 	dev = softc->dev;
791 	mmio_dev = softc->pci_dev;
792 
793 	/* Clear interrupt status bits. */
794 	ctrl = softc->ctrl;
795 	ctrl->status &= AMDVI_STATUS_EV_OF | AMDVI_STATUS_EV_INTR;
796 
797 	err = IVHD_SETUP_INTR(mmio_dev, amdvi_event_intr, softc, "fault");
798 	if (err)
799 		device_printf(dev, "Interrupt setup failed on %s\n",
800 		    device_get_nameunit(mmio_dev));
801 	return (err);
802 }
803 
804 static void
805 amdvi_print_dev_cap(struct amdvi_softc *softc)
806 {
807 	struct ivhd_dev_cfg *cfg;
808 	int i;
809 
810 	cfg = softc->dev_cfg;
811 	for (i = 0; i < softc->dev_cfg_cnt; i++) {
812 		device_printf(softc->dev, "device [0x%x - 0x%x]"
813 		    "config:%b%s\n", cfg->start_id, cfg->end_id,
814 		    cfg->data,
815 		    "\020\001INIT\002ExtInt\003NMI"
816 		    "\007LINT0\008LINT1",
817 		    cfg->enable_ats ? "ATS enabled" : "");
818 		cfg++;
819 	}
820 }
821 
822 static int
823 amdvi_handle_sysctl(SYSCTL_HANDLER_ARGS)
824 {
825 	struct amdvi_softc *softc;
826 	int result, type, error = 0;
827 
828 	softc = (struct amdvi_softc *)arg1;
829 	type = arg2;
830 
831 	switch (type) {
832 	case 0:
833 		result = softc->ctrl->cmd_head;
834 		error = sysctl_handle_int(oidp, &result, 0,
835 		    req);
836 		break;
837 	case 1:
838 		result = softc->ctrl->cmd_tail;
839 		error = sysctl_handle_int(oidp, &result, 0,
840 		    req);
841 		break;
842 	case 2:
843 		result = softc->ctrl->evt_head;
844 		error = sysctl_handle_int(oidp, &result, 0,
845 		    req);
846 		break;
847 	case 3:
848 		result = softc->ctrl->evt_tail;
849 		error = sysctl_handle_int(oidp, &result, 0,
850 		    req);
851 		break;
852 
853 	default:
854 		device_printf(softc->dev, "Unknown sysctl:%d\n", type);
855 	}
856 
857 	return (error);
858 }
859 
860 static void
861 amdvi_add_sysctl(struct amdvi_softc *softc)
862 {
863 	struct sysctl_oid_list *child;
864 	struct sysctl_ctx_list *ctx;
865 	device_t dev;
866 
867 	dev = softc->dev;
868 	ctx = device_get_sysctl_ctx(dev);
869 	child = SYSCTL_CHILDREN(device_get_sysctl_tree(dev));
870 
871 	SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "event_intr_count", CTLFLAG_RD,
872 	    &softc->event_intr_cnt, "Event interrupt count");
873 	SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "command_count", CTLFLAG_RD,
874 	    &softc->total_cmd, "Command submitted count");
875 	SYSCTL_ADD_U16(ctx, child, OID_AUTO, "pci_rid", CTLFLAG_RD,
876 	    &softc->pci_rid, 0, "IOMMU RID");
877 	SYSCTL_ADD_U16(ctx, child, OID_AUTO, "start_dev_rid", CTLFLAG_RD,
878 	    &softc->start_dev_rid, 0, "Start of device under this IOMMU");
879 	SYSCTL_ADD_U16(ctx, child, OID_AUTO, "end_dev_rid", CTLFLAG_RD,
880 	    &softc->end_dev_rid, 0, "End of device under this IOMMU");
881 	SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "command_head",
882 	    CTLTYPE_UINT | CTLFLAG_RD | CTLFLAG_MPSAFE, softc, 0,
883 	    amdvi_handle_sysctl, "IU", "Command head");
884 	SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "command_tail",
885 	    CTLTYPE_UINT | CTLFLAG_RD | CTLFLAG_MPSAFE, softc, 1,
886 	    amdvi_handle_sysctl, "IU", "Command tail");
887 	SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "event_head",
888 	    CTLTYPE_UINT | CTLFLAG_RD | CTLFLAG_MPSAFE, softc, 2,
889 	    amdvi_handle_sysctl, "IU", "Command head");
890 	SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "event_tail",
891 	    CTLTYPE_UINT | CTLFLAG_RD | CTLFLAG_MPSAFE, softc, 3,
892 	    amdvi_handle_sysctl, "IU", "Command tail");
893 }
894 
895 int
896 amdvi_setup_hw(struct amdvi_softc *softc)
897 {
898 	device_t dev;
899 	int status;
900 
901 	dev = softc->dev;
902 
903 	amdvi_hw_enable_iotlb(softc);
904 
905 	amdvi_print_dev_cap(softc);
906 
907 	if ((status = amdvi_print_pci_cap(dev)) != 0) {
908 		device_printf(dev, "PCI capability.\n");
909 		return (status);
910 	}
911 	if ((status = amdvi_init_cmd(softc)) != 0) {
912 		device_printf(dev, "Couldn't configure command buffer.\n");
913 		return (status);
914 	}
915 	if ((status = amdvi_init_event(softc)) != 0) {
916 		device_printf(dev, "Couldn't configure event buffer.\n");
917 		return (status);
918 	}
919 	if ((status = amdvi_init_dte(softc)) != 0) {
920 		device_printf(dev, "Couldn't configure device table.\n");
921 		return (status);
922 	}
923 	if ((status = amdvi_alloc_intr_resources(softc)) != 0) {
924 		return (status);
925 	}
926 	amdvi_add_sysctl(softc);
927 	return (0);
928 }
929 
930 int
931 amdvi_teardown_hw(struct amdvi_softc *softc)
932 {
933 	device_t dev;
934 
935 	dev = softc->dev;
936 
937 	/*
938 	 * Called after disable, h/w is stopped by now, free all the resources.
939 	 */
940 	amdvi_free_evt_intr_res(dev);
941 
942 	if (softc->cmd)
943 		free(softc->cmd, M_AMDVI);
944 
945 	if (softc->event)
946 		free(softc->event, M_AMDVI);
947 
948 	return (0);
949 }
950 
951 /*********** bhyve interfaces *********************/
952 static int
953 amdvi_init(void)
954 {
955 	if (!ivhd_count) {
956 		return (EIO);
957 	}
958 	if (!amdvi_enable_user && ivhd_count) {
959 		printf("bhyve: Found %d AMD-Vi/IOMMU device(s), "
960 		    	"use hw.vmm.amdvi.enable=1 to enable pass-through.\n",
961 		    ivhd_count);
962 		return (EINVAL);
963 	}
964 	return (0);
965 }
966 
967 static void
968 amdvi_cleanup(void)
969 {
970 	/* Nothing. */
971 }
972 
973 static uint16_t
974 amdvi_domainId(void)
975 {
976 
977 	/*
978 	 * If we hit maximum domain limit, rollover leaving host
979 	 * domain(0).
980 	 * XXX: make sure that this domain is not used.
981 	 */
982 	if (amdvi_dom_id == AMDVI_MAX_DOMAIN)
983 		amdvi_dom_id = 1;
984 
985 	return ((uint16_t)amdvi_dom_id++);
986 }
987 
988 static void
989 amdvi_do_inv_domain(uint16_t domain_id, bool create)
990 {
991 	struct amdvi_softc *softc;
992 	int i;
993 
994 	for (i = 0; i < ivhd_count; i++) {
995 		softc = device_get_softc(ivhd_devs[i]);
996 		KASSERT(softc, ("softc is NULL"));
997 		/*
998 		 * If not present pages are cached, invalidate page after
999 		 * creating domain.
1000 		 */
1001 #if 0
1002 		if (create && ((softc->pci_cap & AMDVI_PCI_CAP_NPCACHE) == 0))
1003 			continue;
1004 #endif
1005 		amdvi_inv_domain(softc, domain_id);
1006 		amdvi_wait(softc);
1007 	}
1008 }
1009 
1010 static void *
1011 amdvi_create_domain(vm_paddr_t maxaddr)
1012 {
1013 	struct amdvi_domain *dom;
1014 
1015 	dom = malloc(sizeof(struct amdvi_domain), M_AMDVI, M_ZERO | M_WAITOK);
1016 	dom->id = amdvi_domainId();
1017 	//dom->maxaddr = maxaddr;
1018 #ifdef AMDVI_DEBUG_CMD
1019 	printf("Created domain #%d\n", dom->id);
1020 #endif
1021 	/*
1022 	 * Host domain(#0) don't create translation table.
1023 	 */
1024 	if (dom->id || amdvi_host_ptp)
1025 		dom->ptp = malloc(PAGE_SIZE, M_AMDVI, M_WAITOK | M_ZERO);
1026 
1027 	dom->ptp_level = amdvi_ptp_level;
1028 
1029 	amdvi_do_inv_domain(dom->id, true);
1030 	SLIST_INSERT_HEAD(&dom_head, dom, next);
1031 
1032 	return (dom);
1033 }
1034 
1035 static void
1036 amdvi_free_ptp(uint64_t *ptp, int level)
1037 {
1038 	int i;
1039 
1040 	if (level < 1)
1041 		return;
1042 
1043 	for (i = 0; i < NPTEPG ; i++) {
1044 		if ((ptp[i] & AMDVI_PT_PRESENT) == 0)
1045 			continue;
1046 		/* XXX: Add super-page or PTE mapping > 4KB. */
1047 #ifdef notyet
1048 		/* Super-page mapping. */
1049 		if (AMDVI_PD_SUPER(ptp[i]))
1050 			continue;
1051 #endif
1052 
1053 		amdvi_free_ptp((uint64_t *)PHYS_TO_DMAP(ptp[i]
1054 		    & AMDVI_PT_MASK), level - 1);
1055 	}
1056 
1057 	free(ptp, M_AMDVI);
1058 }
1059 
1060 static void
1061 amdvi_destroy_domain(void *arg)
1062 {
1063 	struct amdvi_domain *domain;
1064 
1065 	domain = (struct amdvi_domain *)arg;
1066 	KASSERT(domain, ("domain is NULL"));
1067 #ifdef AMDVI_DEBUG_CMD
1068 	printf("Destroying domain %d\n", domain->id);
1069 #endif
1070 	if (domain->ptp)
1071 		amdvi_free_ptp(domain->ptp, domain->ptp_level);
1072 
1073 	amdvi_do_inv_domain(domain->id, false);
1074 	SLIST_REMOVE(&dom_head, domain, amdvi_domain, next);
1075 	free(domain, M_AMDVI);
1076 }
1077 
1078 static uint64_t
1079 amdvi_set_pt(uint64_t *pt, int level, vm_paddr_t gpa,
1080     vm_paddr_t hpa, uint64_t pg_size, bool create)
1081 {
1082 	uint64_t *page, pa;
1083 	int shift, index;
1084 	const int PT_SHIFT = 9;
1085 	const int PT_INDEX_MASK = (1 << PT_SHIFT) - 1;	/* Based on PT_SHIFT */
1086 
1087 	if (!pg_size)
1088 		return (0);
1089 
1090 	if (hpa & (pg_size - 1)) {
1091 		printf("HPA is not size aligned.\n");
1092 		return (0);
1093 	}
1094 	if (gpa & (pg_size - 1)) {
1095 		printf("HPA is not size aligned.\n");
1096 		return (0);
1097 	}
1098 	shift = PML4SHIFT;
1099 	while ((shift > PAGE_SHIFT) && (pg_size < (1UL << shift))) {
1100 		index = (gpa >> shift) & PT_INDEX_MASK;
1101 
1102 		if ((pt[index] == 0) && create) {
1103 			page = malloc(PAGE_SIZE, M_AMDVI, M_WAITOK | M_ZERO);
1104 			pa = vtophys(page);
1105 			pt[index] = pa | AMDVI_PT_PRESENT | AMDVI_PT_RW |
1106 			    ((level - 1) << AMDVI_PD_LEVEL_SHIFT);
1107 		}
1108 #ifdef AMDVI_DEBUG_PTE
1109 		if ((gpa % 0x1000000) == 0)
1110 			printf("[level%d, shift = %d]PTE:0x%lx\n",
1111 			    level, shift, pt[index]);
1112 #endif
1113 #define PTE2PA(x)	((uint64_t)(x) & AMDVI_PT_MASK)
1114 		pa = PTE2PA(pt[index]);
1115 		pt = (uint64_t *)PHYS_TO_DMAP(pa);
1116 		shift -= PT_SHIFT;
1117 		level--;
1118 	}
1119 
1120 	/* Leaf entry. */
1121 	index = (gpa >> shift) & PT_INDEX_MASK;
1122 
1123 	if (create) {
1124 		pt[index] = hpa | AMDVI_PT_RW | AMDVI_PT_PRESENT;
1125 	} else
1126 		pt[index] = 0;
1127 
1128 #ifdef AMDVI_DEBUG_PTE
1129 	if ((gpa % 0x1000000) == 0)
1130 		printf("[Last level%d, shift = %d]PTE:0x%lx\n",
1131 		    level, shift, pt[index]);
1132 #endif
1133 	return (1ULL << shift);
1134 }
1135 
1136 static uint64_t
1137 amdvi_update_mapping(struct amdvi_domain *domain, vm_paddr_t gpa,
1138     vm_paddr_t hpa, uint64_t size, bool create)
1139 {
1140 	uint64_t mapped, *ptp, len;
1141 	int level;
1142 
1143 	KASSERT(domain, ("domain is NULL"));
1144 	level = domain->ptp_level;
1145 	KASSERT(level, ("Page table level is 0"));
1146 
1147 	ptp = domain->ptp;
1148 	KASSERT(ptp, ("PTP is NULL"));
1149 	mapped = 0;
1150 	while (mapped < size) {
1151 		len = amdvi_set_pt(ptp, level, gpa + mapped, hpa + mapped,
1152 		    PAGE_SIZE, create);
1153 		if (!len) {
1154 			printf("Error: Couldn't map HPA:0x%lx GPA:0x%lx\n",
1155 			    hpa, gpa);
1156 			return (0);
1157 		}
1158 		mapped += len;
1159 	}
1160 
1161 	return (mapped);
1162 }
1163 
1164 static uint64_t
1165 amdvi_create_mapping(void *arg, vm_paddr_t gpa, vm_paddr_t hpa,
1166     uint64_t len)
1167 {
1168 	struct amdvi_domain *domain;
1169 
1170 	domain = (struct amdvi_domain *)arg;
1171 
1172 	if (domain->id && !domain->ptp) {
1173 		printf("ptp is NULL");
1174 		return (-1);
1175 	}
1176 
1177 	/*
1178 	 * If host domain is created w/o page table, skip IOMMU page
1179 	 * table set-up.
1180 	 */
1181 	if (domain->ptp)
1182 		return (amdvi_update_mapping(domain, gpa, hpa, len, true));
1183 	else
1184 		return (len);
1185 }
1186 
1187 static uint64_t
1188 amdvi_destroy_mapping(void *arg, vm_paddr_t gpa, uint64_t len)
1189 {
1190 	struct amdvi_domain *domain;
1191 
1192 	domain = (struct amdvi_domain *)arg;
1193 	/*
1194 	 * If host domain is created w/o page table, skip IOMMU page
1195 	 * table set-up.
1196 	 */
1197 	if (domain->ptp)
1198 		return (amdvi_update_mapping(domain, gpa, 0, len, false));
1199 	return
1200 	    (len);
1201 }
1202 
1203 static struct amdvi_softc *
1204 amdvi_find_iommu(uint16_t devid)
1205 {
1206 	struct amdvi_softc *softc;
1207 	int i;
1208 
1209 	for (i = 0; i < ivhd_count; i++) {
1210 		softc = device_get_softc(ivhd_devs[i]);
1211 		if ((devid >= softc->start_dev_rid) &&
1212 		    (devid <= softc->end_dev_rid))
1213 			return (softc);
1214 	}
1215 
1216 	/*
1217 	 * XXX: BIOS bug, device not in IVRS table, assume its from first IOMMU.
1218 	 */
1219 	printf("BIOS bug device(%d.%d.%d) doesn't have IVHD entry.\n",
1220 	    RID2PCI_STR(devid));
1221 
1222 	return (device_get_softc(ivhd_devs[0]));
1223 }
1224 
1225 /*
1226  * Set-up device table entry.
1227  * IOMMU spec Rev 2.0, section 3.2.2.2, some of the fields must
1228  * be set concurrently, e.g. read and write bits.
1229  */
1230 static void
1231 amdvi_set_dte(struct amdvi_domain *domain, uint16_t devid, bool enable)
1232 {
1233 	struct amdvi_softc *softc;
1234 	struct amdvi_dte* temp;
1235 
1236 	KASSERT(domain, ("domain is NULL for pci_rid:0x%x\n", devid));
1237 
1238 	softc = amdvi_find_iommu(devid);
1239 	KASSERT(softc, ("softc is NULL for pci_rid:0x%x\n", devid));
1240 
1241 	temp = &amdvi_dte[devid];
1242 
1243 #ifdef AMDVI_ATS_ENABLE
1244 	/* If IOMMU and device support IOTLB, enable it. */
1245 	if (amdvi_dev_support_iotlb(softc, devid) && softc->iotlb)
1246 		temp->iotlb_enable = 1;
1247 #endif
1248 
1249 	/* Avoid duplicate I/O faults. */
1250 	temp->sup_second_io_fault = 1;
1251 	temp->sup_all_io_fault = amdvi_disable_io_fault;
1252 
1253 	temp->dt_valid = 1;
1254 	temp->domain_id = domain->id;
1255 
1256 	if (enable) {
1257 		if (domain->ptp) {
1258 			temp->pt_base = vtophys(domain->ptp) >> 12;
1259 			temp->pt_level = amdvi_ptp_level;
1260 		}
1261 		/*
1262 		 * XXX: Page table valid[TV] bit must be set even if host domain
1263 		 * page tables are not enabled.
1264 		 */
1265 		temp->pt_valid = 1;
1266 		temp->read_allow = 1;
1267 		temp->write_allow = 1;
1268 	}
1269 }
1270 
1271 static void
1272 amdvi_inv_device(uint16_t devid)
1273 {
1274 	struct amdvi_softc *softc;
1275 
1276 	softc = amdvi_find_iommu(devid);
1277 	KASSERT(softc, ("softc is NULL"));
1278 
1279 	amdvi_cmd_inv_dte(softc, devid);
1280 #ifdef AMDVI_ATS_ENABLE
1281 	if (amdvi_dev_support_iotlb(softc, devid))
1282 		amdvi_cmd_inv_iotlb(softc, devid);
1283 #endif
1284 	amdvi_wait(softc);
1285 }
1286 
1287 static void
1288 amdvi_add_device(void *arg, uint16_t devid)
1289 {
1290 	struct amdvi_domain *domain;
1291 
1292 	domain = (struct amdvi_domain *)arg;
1293 	KASSERT(domain != NULL, ("domain is NULL"));
1294 #ifdef AMDVI_DEBUG_CMD
1295 	printf("Assigning device(%d.%d.%d) to domain:%d\n",
1296 	    RID2PCI_STR(devid), domain->id);
1297 #endif
1298 	amdvi_set_dte(domain, devid, true);
1299 	amdvi_inv_device(devid);
1300 }
1301 
1302 static void
1303 amdvi_remove_device(void *arg, uint16_t devid)
1304 {
1305 	struct amdvi_domain *domain;
1306 
1307 	domain = (struct amdvi_domain *)arg;
1308 #ifdef AMDVI_DEBUG_CMD
1309 	printf("Remove device(0x%x) from domain:%d\n",
1310 	       devid, domain->id);
1311 #endif
1312 	amdvi_set_dte(domain, devid, false);
1313 	amdvi_inv_device(devid);
1314 }
1315 
1316 static void
1317 amdvi_enable(void)
1318 {
1319 	struct amdvi_ctrl *ctrl;
1320 	struct amdvi_softc *softc;
1321 	uint64_t val;
1322 	int i;
1323 
1324 	for (i = 0; i < ivhd_count; i++) {
1325 		softc = device_get_softc(ivhd_devs[i]);
1326 		KASSERT(softc, ("softc is NULL\n"));
1327 		ctrl = softc->ctrl;
1328 		KASSERT(ctrl, ("ctrl is NULL\n"));
1329 
1330 		val = (	AMDVI_CTRL_EN 		|
1331 			AMDVI_CTRL_CMD 		|
1332 		    	AMDVI_CTRL_ELOG 	|
1333 		    	AMDVI_CTRL_ELOGINT 	|
1334 		    	AMDVI_CTRL_INV_TO_1S);
1335 
1336 		if (softc->ivhd_flag & IVHD_FLAG_COH)
1337 			val |= AMDVI_CTRL_COH;
1338 		if (softc->ivhd_flag & IVHD_FLAG_HTT)
1339 			val |= AMDVI_CTRL_HTT;
1340 		if (softc->ivhd_flag & IVHD_FLAG_RPPW)
1341 			val |= AMDVI_CTRL_RPPW;
1342 		if (softc->ivhd_flag & IVHD_FLAG_PPW)
1343 			val |= AMDVI_CTRL_PPW;
1344 		if (softc->ivhd_flag & IVHD_FLAG_ISOC)
1345 			val |= AMDVI_CTRL_ISOC;
1346 
1347 		ctrl->control = val;
1348 	}
1349 }
1350 
1351 static void
1352 amdvi_disable(void)
1353 {
1354 	struct amdvi_ctrl *ctrl;
1355 	struct amdvi_softc *softc;
1356 	int i;
1357 
1358 	for (i = 0; i < ivhd_count; i++) {
1359 		softc = device_get_softc(ivhd_devs[i]);
1360 		KASSERT(softc, ("softc is NULL\n"));
1361 		ctrl = softc->ctrl;
1362 		KASSERT(ctrl, ("ctrl is NULL\n"));
1363 
1364 		ctrl->control = 0;
1365 	}
1366 }
1367 
1368 static void
1369 amdvi_inv_tlb(void *arg)
1370 {
1371 	struct amdvi_domain *domain;
1372 
1373 	domain = (struct amdvi_domain *)arg;
1374 	KASSERT(domain, ("domain is NULL"));
1375 	amdvi_do_inv_domain(domain->id, false);
1376 }
1377 
1378 struct iommu_ops iommu_ops_amd = {
1379 	amdvi_init,
1380 	amdvi_cleanup,
1381 	amdvi_enable,
1382 	amdvi_disable,
1383 	amdvi_create_domain,
1384 	amdvi_destroy_domain,
1385 	amdvi_create_mapping,
1386 	amdvi_destroy_mapping,
1387 	amdvi_add_device,
1388 	amdvi_remove_device,
1389 	amdvi_inv_tlb
1390 };
1391