1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 3 * 4 * Copyright (c) 2016, Anish Gupta (anish@freebsd.org) 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice unmodified, this list of conditions, and the following 12 * disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 18 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 19 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 20 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 21 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 22 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 23 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 24 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 25 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 26 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 27 */ 28 29 #include <sys/cdefs.h> 30 __FBSDID("$FreeBSD$"); 31 32 #include <sys/param.h> 33 #include <sys/systm.h> 34 #include <sys/bus.h> 35 #include <sys/kernel.h> 36 #include <sys/module.h> 37 #include <sys/malloc.h> 38 #include <sys/pcpu.h> 39 #include <sys/rman.h> 40 #include <sys/smp.h> 41 #include <sys/sysctl.h> 42 43 #include <vm/vm.h> 44 #include <vm/pmap.h> 45 46 #include <dev/pci/pcivar.h> 47 #include <dev/pci/pcireg.h> 48 49 #include <machine/resource.h> 50 #include <machine/vmm.h> 51 #include <machine/pmap.h> 52 #include <machine/vmparam.h> 53 #include <machine/pci_cfgreg.h> 54 55 #include "ivhd_if.h" 56 #include "pcib_if.h" 57 58 #include "io/iommu.h" 59 #include "amdvi_priv.h" 60 61 SYSCTL_DECL(_hw_vmm); 62 SYSCTL_NODE(_hw_vmm, OID_AUTO, amdvi, CTLFLAG_RW | CTLFLAG_MPSAFE, NULL, 63 NULL); 64 65 #define MOD_INC(a, s, m) (((a) + (s)) % ((m) * (s))) 66 #define MOD_DEC(a, s, m) (((a) - (s)) % ((m) * (s))) 67 68 /* Print RID or device ID in PCI string format. */ 69 #define RID2PCI_STR(d) PCI_RID2BUS(d), PCI_RID2SLOT(d), PCI_RID2FUNC(d) 70 71 static void amdvi_dump_cmds(struct amdvi_softc *softc, int count); 72 static void amdvi_print_dev_cap(struct amdvi_softc *softc); 73 74 MALLOC_DEFINE(M_AMDVI, "amdvi", "amdvi"); 75 76 extern device_t *ivhd_devs; 77 78 extern int ivhd_count; 79 SYSCTL_INT(_hw_vmm_amdvi, OID_AUTO, count, CTLFLAG_RDTUN, &ivhd_count, 80 0, NULL); 81 82 static int amdvi_enable_user = 0; 83 SYSCTL_INT(_hw_vmm_amdvi, OID_AUTO, enable, CTLFLAG_RDTUN, 84 &amdvi_enable_user, 0, NULL); 85 TUNABLE_INT("hw.vmm.amdvi_enable", &amdvi_enable_user); 86 87 #ifdef AMDVI_ATS_ENABLE 88 /* XXX: ATS is not tested. */ 89 static int amdvi_enable_iotlb = 1; 90 SYSCTL_INT(_hw_vmm_amdvi, OID_AUTO, iotlb_enabled, CTLFLAG_RDTUN, 91 &amdvi_enable_iotlb, 0, NULL); 92 TUNABLE_INT("hw.vmm.enable_iotlb", &amdvi_enable_iotlb); 93 #endif 94 95 static int amdvi_host_ptp = 1; /* Use page tables for host. */ 96 SYSCTL_INT(_hw_vmm_amdvi, OID_AUTO, host_ptp, CTLFLAG_RDTUN, 97 &amdvi_host_ptp, 0, NULL); 98 TUNABLE_INT("hw.vmm.amdvi.host_ptp", &amdvi_host_ptp); 99 100 /* Page table level used <= supported by h/w[v1=7]. */ 101 int amdvi_ptp_level = 4; 102 SYSCTL_INT(_hw_vmm_amdvi, OID_AUTO, ptp_level, CTLFLAG_RDTUN, 103 &amdvi_ptp_level, 0, NULL); 104 TUNABLE_INT("hw.vmm.amdvi.ptp_level", &amdvi_ptp_level); 105 106 /* Disable fault event reporting. */ 107 static int amdvi_disable_io_fault = 0; 108 SYSCTL_INT(_hw_vmm_amdvi, OID_AUTO, disable_io_fault, CTLFLAG_RDTUN, 109 &amdvi_disable_io_fault, 0, NULL); 110 TUNABLE_INT("hw.vmm.amdvi.disable_io_fault", &amdvi_disable_io_fault); 111 112 static uint32_t amdvi_dom_id = 0; /* 0 is reserved for host. */ 113 SYSCTL_UINT(_hw_vmm_amdvi, OID_AUTO, domain_id, CTLFLAG_RD, 114 &amdvi_dom_id, 0, NULL); 115 /* 116 * Device table entry. 117 * Bus(256) x Dev(32) x Fun(8) x DTE(256 bits or 32 bytes). 118 * = 256 * 2 * PAGE_SIZE. 119 */ 120 static struct amdvi_dte amdvi_dte[PCI_NUM_DEV_MAX] __aligned(PAGE_SIZE); 121 CTASSERT(PCI_NUM_DEV_MAX == 0x10000); 122 CTASSERT(sizeof(amdvi_dte) == 0x200000); 123 124 static SLIST_HEAD (, amdvi_domain) dom_head; 125 126 static inline uint32_t 127 amdvi_pci_read(struct amdvi_softc *softc, int off) 128 { 129 130 return (pci_cfgregread(PCI_RID2BUS(softc->pci_rid), 131 PCI_RID2SLOT(softc->pci_rid), PCI_RID2FUNC(softc->pci_rid), 132 off, 4)); 133 } 134 135 #ifdef AMDVI_ATS_ENABLE 136 /* XXX: Should be in pci.c */ 137 /* 138 * Check if device has ATS capability and its enabled. 139 * If ATS is absent or disabled, return (-1), otherwise ATS 140 * queue length. 141 */ 142 static int 143 amdvi_find_ats_qlen(uint16_t devid) 144 { 145 device_t dev; 146 uint32_t off, cap; 147 int qlen = -1; 148 149 dev = pci_find_bsf(PCI_RID2BUS(devid), PCI_RID2SLOT(devid), 150 PCI_RID2FUNC(devid)); 151 152 if (!dev) { 153 return (-1); 154 } 155 #define PCIM_ATS_EN BIT(31) 156 157 if (pci_find_extcap(dev, PCIZ_ATS, &off) == 0) { 158 cap = pci_read_config(dev, off + 4, 4); 159 qlen = (cap & 0x1F); 160 qlen = qlen ? qlen : 32; 161 printf("AMD-Vi: PCI device %d.%d.%d ATS %s qlen=%d\n", 162 RID2PCI_STR(devid), 163 (cap & PCIM_ATS_EN) ? "enabled" : "Disabled", 164 qlen); 165 qlen = (cap & PCIM_ATS_EN) ? qlen : -1; 166 } 167 168 return (qlen); 169 } 170 171 /* 172 * Check if an endpoint device support device IOTLB or ATS. 173 */ 174 static inline bool 175 amdvi_dev_support_iotlb(struct amdvi_softc *softc, uint16_t devid) 176 { 177 struct ivhd_dev_cfg *cfg; 178 int qlen, i; 179 bool pci_ats, ivhd_ats; 180 181 qlen = amdvi_find_ats_qlen(devid); 182 if (qlen < 0) 183 return (false); 184 185 KASSERT(softc, ("softc is NULL")); 186 cfg = softc->dev_cfg; 187 188 ivhd_ats = false; 189 for (i = 0; i < softc->dev_cfg_cnt; i++) { 190 if ((cfg->start_id <= devid) && (cfg->end_id >= devid)) { 191 ivhd_ats = cfg->enable_ats; 192 break; 193 } 194 cfg++; 195 } 196 197 pci_ats = (qlen < 0) ? false : true; 198 if (pci_ats != ivhd_ats) 199 device_printf(softc->dev, 200 "BIOS bug: mismatch in ATS setting for %d.%d.%d," 201 "ATS inv qlen = %d\n", RID2PCI_STR(devid), qlen); 202 203 /* Ignore IVRS setting and respect PCI setting. */ 204 return (pci_ats); 205 } 206 #endif 207 208 /* Enable IOTLB support for IOMMU if its supported. */ 209 static inline void 210 amdvi_hw_enable_iotlb(struct amdvi_softc *softc) 211 { 212 #ifndef AMDVI_ATS_ENABLE 213 softc->iotlb = false; 214 #else 215 bool supported; 216 217 supported = (softc->ivhd_flag & IVHD_FLAG_IOTLB) ? true : false; 218 219 if (softc->pci_cap & AMDVI_PCI_CAP_IOTLB) { 220 if (!supported) 221 device_printf(softc->dev, "IOTLB disabled by BIOS.\n"); 222 223 if (supported && !amdvi_enable_iotlb) { 224 device_printf(softc->dev, "IOTLB disabled by user.\n"); 225 supported = false; 226 } 227 } else 228 supported = false; 229 230 softc->iotlb = supported; 231 232 #endif 233 } 234 235 static int 236 amdvi_init_cmd(struct amdvi_softc *softc) 237 { 238 struct amdvi_ctrl *ctrl = softc->ctrl; 239 240 ctrl->cmd.len = 8; /* Use 256 command buffer entries. */ 241 softc->cmd_max = 1 << ctrl->cmd.len; 242 243 softc->cmd = malloc(sizeof(struct amdvi_cmd) * 244 softc->cmd_max, M_AMDVI, M_WAITOK | M_ZERO); 245 246 if ((uintptr_t)softc->cmd & PAGE_MASK) 247 panic("AMDVi: Command buffer not aligned on page boundary."); 248 249 ctrl->cmd.base = vtophys(softc->cmd) / PAGE_SIZE; 250 /* 251 * XXX: Reset the h/w pointers in case IOMMU is restarting, 252 * h/w doesn't clear these pointers based on empirical data. 253 */ 254 ctrl->cmd_tail = 0; 255 ctrl->cmd_head = 0; 256 257 return (0); 258 } 259 260 /* 261 * Note: Update tail pointer after we have written the command since tail 262 * pointer update cause h/w to execute new commands, see section 3.3 263 * of AMD IOMMU spec ver 2.0. 264 */ 265 /* Get the command tail pointer w/o updating it. */ 266 static struct amdvi_cmd * 267 amdvi_get_cmd_tail(struct amdvi_softc *softc) 268 { 269 struct amdvi_ctrl *ctrl; 270 struct amdvi_cmd *tail; 271 272 KASSERT(softc, ("softc is NULL")); 273 KASSERT(softc->cmd != NULL, ("cmd is NULL")); 274 275 ctrl = softc->ctrl; 276 KASSERT(ctrl != NULL, ("ctrl is NULL")); 277 278 tail = (struct amdvi_cmd *)((uint8_t *)softc->cmd + 279 ctrl->cmd_tail); 280 281 return (tail); 282 } 283 284 /* 285 * Update the command tail pointer which will start command execution. 286 */ 287 static void 288 amdvi_update_cmd_tail(struct amdvi_softc *softc) 289 { 290 struct amdvi_ctrl *ctrl; 291 int size; 292 293 size = sizeof(struct amdvi_cmd); 294 KASSERT(softc->cmd != NULL, ("cmd is NULL")); 295 296 ctrl = softc->ctrl; 297 KASSERT(ctrl != NULL, ("ctrl is NULL")); 298 299 ctrl->cmd_tail = MOD_INC(ctrl->cmd_tail, size, softc->cmd_max); 300 softc->total_cmd++; 301 302 #ifdef AMDVI_DEBUG_CMD 303 device_printf(softc->dev, "cmd_tail: %s Tail:0x%x, Head:0x%x.\n", 304 ctrl->cmd_tail, 305 ctrl->cmd_head); 306 #endif 307 308 } 309 310 /* 311 * Various commands supported by IOMMU. 312 */ 313 314 /* Completion wait command. */ 315 static void 316 amdvi_cmd_cmp(struct amdvi_softc *softc, const uint64_t data) 317 { 318 struct amdvi_cmd *cmd; 319 uint64_t pa; 320 321 cmd = amdvi_get_cmd_tail(softc); 322 KASSERT(cmd != NULL, ("Cmd is NULL")); 323 324 pa = vtophys(&softc->cmp_data); 325 cmd->opcode = AMDVI_CMP_WAIT_OPCODE; 326 cmd->word0 = (pa & 0xFFFFFFF8) | AMDVI_CMP_WAIT_STORE; 327 cmd->word1 = (pa >> 32) & 0xFFFFF; 328 cmd->addr = data; 329 330 amdvi_update_cmd_tail(softc); 331 } 332 333 /* Invalidate device table entry. */ 334 static void 335 amdvi_cmd_inv_dte(struct amdvi_softc *softc, uint16_t devid) 336 { 337 struct amdvi_cmd *cmd; 338 339 cmd = amdvi_get_cmd_tail(softc); 340 KASSERT(cmd != NULL, ("Cmd is NULL")); 341 cmd->opcode = AMDVI_INVD_DTE_OPCODE; 342 cmd->word0 = devid; 343 amdvi_update_cmd_tail(softc); 344 #ifdef AMDVI_DEBUG_CMD 345 device_printf(softc->dev, "Invalidated DTE:0x%x\n", devid); 346 #endif 347 } 348 349 /* Invalidate IOMMU page, use for invalidation of domain. */ 350 static void 351 amdvi_cmd_inv_iommu_pages(struct amdvi_softc *softc, uint16_t domain_id, 352 uint64_t addr, bool guest_nested, 353 bool pde, bool page) 354 { 355 struct amdvi_cmd *cmd; 356 357 cmd = amdvi_get_cmd_tail(softc); 358 KASSERT(cmd != NULL, ("Cmd is NULL")); 359 360 cmd->opcode = AMDVI_INVD_PAGE_OPCODE; 361 cmd->word1 = domain_id; 362 /* 363 * Invalidate all addresses for this domain. 364 */ 365 cmd->addr = addr; 366 cmd->addr |= pde ? AMDVI_INVD_PAGE_PDE : 0; 367 cmd->addr |= page ? AMDVI_INVD_PAGE_S : 0; 368 369 amdvi_update_cmd_tail(softc); 370 } 371 372 #ifdef AMDVI_ATS_ENABLE 373 /* Invalidate device IOTLB. */ 374 static void 375 amdvi_cmd_inv_iotlb(struct amdvi_softc *softc, uint16_t devid) 376 { 377 struct amdvi_cmd *cmd; 378 int qlen; 379 380 if (!softc->iotlb) 381 return; 382 383 qlen = amdvi_find_ats_qlen(devid); 384 if (qlen < 0) { 385 panic("AMDVI: Invalid ATS qlen(%d) for device %d.%d.%d\n", 386 qlen, RID2PCI_STR(devid)); 387 } 388 cmd = amdvi_get_cmd_tail(softc); 389 KASSERT(cmd != NULL, ("Cmd is NULL")); 390 391 #ifdef AMDVI_DEBUG_CMD 392 device_printf(softc->dev, "Invalidate IOTLB devID 0x%x" 393 " Qlen:%d\n", devid, qlen); 394 #endif 395 cmd->opcode = AMDVI_INVD_IOTLB_OPCODE; 396 cmd->word0 = devid; 397 cmd->word1 = qlen; 398 cmd->addr = AMDVI_INVD_IOTLB_ALL_ADDR | 399 AMDVI_INVD_IOTLB_S; 400 amdvi_update_cmd_tail(softc); 401 } 402 #endif 403 404 #ifdef notyet /* For Interrupt Remap. */ 405 static void 406 amdvi_cmd_inv_intr_map(struct amdvi_softc *softc, 407 uint16_t devid) 408 { 409 struct amdvi_cmd *cmd; 410 411 cmd = amdvi_get_cmd_tail(softc); 412 KASSERT(cmd != NULL, ("Cmd is NULL")); 413 cmd->opcode = AMDVI_INVD_INTR_OPCODE; 414 cmd->word0 = devid; 415 amdvi_update_cmd_tail(softc); 416 #ifdef AMDVI_DEBUG_CMD 417 device_printf(softc->dev, "Invalidate INTR map of devID 0x%x\n", devid); 418 #endif 419 } 420 #endif 421 422 /* Invalidate domain using INVALIDATE_IOMMU_PAGES command. */ 423 static void 424 amdvi_inv_domain(struct amdvi_softc *softc, uint16_t domain_id) 425 { 426 struct amdvi_cmd *cmd; 427 428 cmd = amdvi_get_cmd_tail(softc); 429 KASSERT(cmd != NULL, ("Cmd is NULL")); 430 431 /* 432 * See section 3.3.3 of IOMMU spec rev 2.0, software note 433 * for invalidating domain. 434 */ 435 amdvi_cmd_inv_iommu_pages(softc, domain_id, AMDVI_INVD_PAGE_ALL_ADDR, 436 false, true, true); 437 438 #ifdef AMDVI_DEBUG_CMD 439 device_printf(softc->dev, "Invalidate domain:0x%x\n", domain_id); 440 441 #endif 442 } 443 444 static bool 445 amdvi_cmp_wait(struct amdvi_softc *softc) 446 { 447 struct amdvi_ctrl *ctrl; 448 const uint64_t VERIFY = 0xA5A5; 449 volatile uint64_t *read; 450 int i; 451 bool status; 452 453 ctrl = softc->ctrl; 454 read = &softc->cmp_data; 455 *read = 0; 456 amdvi_cmd_cmp(softc, VERIFY); 457 /* Wait for h/w to update completion data. */ 458 for (i = 0; i < 100 && (*read != VERIFY); i++) { 459 DELAY(1000); /* 1 ms */ 460 } 461 status = (VERIFY == softc->cmp_data) ? true : false; 462 463 #ifdef AMDVI_DEBUG_CMD 464 if (status) 465 device_printf(softc->dev, "CMD completion DONE Tail:0x%x, " 466 "Head:0x%x, loop:%d.\n", ctrl->cmd_tail, 467 ctrl->cmd_head, loop); 468 #endif 469 return (status); 470 } 471 472 static void 473 amdvi_wait(struct amdvi_softc *softc) 474 { 475 struct amdvi_ctrl *ctrl; 476 int i; 477 478 KASSERT(softc, ("softc is NULL")); 479 480 ctrl = softc->ctrl; 481 KASSERT(ctrl != NULL, ("ctrl is NULL")); 482 /* Don't wait if h/w is not enabled. */ 483 if ((ctrl->control & AMDVI_CTRL_EN) == 0) 484 return; 485 486 for (i = 0; i < 10; i++) { 487 if (amdvi_cmp_wait(softc)) 488 return; 489 } 490 491 device_printf(softc->dev, "Error: completion failed" 492 " tail:0x%x, head:0x%x.\n", 493 ctrl->cmd_tail, ctrl->cmd_head); 494 /* Dump the last command. */ 495 amdvi_dump_cmds(softc, 1); 496 } 497 498 static void 499 amdvi_dump_cmds(struct amdvi_softc *softc, int count) 500 { 501 struct amdvi_ctrl *ctrl; 502 struct amdvi_cmd *cmd; 503 int off, i; 504 505 ctrl = softc->ctrl; 506 device_printf(softc->dev, "Dump last %d command(s):\n", count); 507 /* 508 * If h/w is stuck in completion, it is the previous command, 509 * start dumping from previous command onward. 510 */ 511 off = MOD_DEC(ctrl->cmd_head, sizeof(struct amdvi_cmd), 512 softc->cmd_max); 513 for (i = 0; off != ctrl->cmd_tail && i < count; i++) { 514 cmd = (struct amdvi_cmd *)((uint8_t *)softc->cmd + off); 515 printf(" [CMD%d, off:0x%x] opcode= 0x%x 0x%x" 516 " 0x%x 0x%lx\n", i, off, cmd->opcode, 517 cmd->word0, cmd->word1, cmd->addr); 518 off = MOD_INC(off, sizeof(struct amdvi_cmd), softc->cmd_max); 519 } 520 } 521 522 static int 523 amdvi_init_event(struct amdvi_softc *softc) 524 { 525 struct amdvi_ctrl *ctrl; 526 527 ctrl = softc->ctrl; 528 ctrl->event.len = 8; 529 softc->event_max = 1 << ctrl->event.len; 530 softc->event = malloc(sizeof(struct amdvi_event) * 531 softc->event_max, M_AMDVI, M_WAITOK | M_ZERO); 532 if ((uintptr_t)softc->event & PAGE_MASK) { 533 device_printf(softc->dev, "Event buffer not aligned on page."); 534 return (false); 535 } 536 ctrl->event.base = vtophys(softc->event) / PAGE_SIZE; 537 538 /* Reset the pointers. */ 539 ctrl->evt_head = 0; 540 ctrl->evt_tail = 0; 541 542 return (0); 543 } 544 545 static inline void 546 amdvi_decode_evt_flag(uint16_t flag) 547 { 548 549 flag &= AMDVI_EVENT_FLAG_MASK; 550 printf(" 0x%b]\n", flag, 551 "\020" 552 "\001GN" 553 "\002NX" 554 "\003US" 555 "\004I" 556 "\005PR" 557 "\006RW" 558 "\007PE" 559 "\010RZ" 560 "\011TR" 561 ); 562 } 563 564 /* See section 2.5.4 of AMD IOMMU spec ver 2.62.*/ 565 static inline void 566 amdvi_decode_evt_flag_type(uint8_t type) 567 { 568 569 switch (AMDVI_EVENT_FLAG_TYPE(type)) { 570 case 0: 571 printf("RSVD\n"); 572 break; 573 case 1: 574 printf("Master Abort\n"); 575 break; 576 case 2: 577 printf("Target Abort\n"); 578 break; 579 case 3: 580 printf("Data Err\n"); 581 break; 582 default: 583 break; 584 } 585 } 586 587 static void 588 amdvi_decode_inv_dte_evt(uint16_t devid, uint16_t domid, uint64_t addr, 589 uint16_t flag) 590 { 591 592 printf("\t[IO_PAGE_FAULT EVT: devId:0x%x DomId:0x%x" 593 " Addr:0x%lx", 594 devid, domid, addr); 595 amdvi_decode_evt_flag(flag); 596 } 597 598 static void 599 amdvi_decode_pf_evt(uint16_t devid, uint16_t domid, uint64_t addr, 600 uint16_t flag) 601 { 602 603 printf("\t[IO_PAGE_FAULT EVT: devId:0x%x DomId:0x%x" 604 " Addr:0x%lx", 605 devid, domid, addr); 606 amdvi_decode_evt_flag(flag); 607 } 608 609 static void 610 amdvi_decode_dte_hwerr_evt(uint16_t devid, uint16_t domid, 611 uint64_t addr, uint16_t flag) 612 { 613 614 printf("\t[DEV_TAB_HW_ERR EVT: devId:0x%x DomId:0x%x" 615 " Addr:0x%lx", devid, domid, addr); 616 amdvi_decode_evt_flag(flag); 617 amdvi_decode_evt_flag_type(flag); 618 } 619 620 static void 621 amdvi_decode_page_hwerr_evt(uint16_t devid, uint16_t domid, uint64_t addr, 622 uint16_t flag) 623 { 624 625 printf("\t[PAGE_TAB_HW_ERR EVT: devId:0x%x DomId:0x%x" 626 " Addr:0x%lx", devid, domid, addr); 627 amdvi_decode_evt_flag(flag); 628 amdvi_decode_evt_flag_type(AMDVI_EVENT_FLAG_TYPE(flag)); 629 } 630 631 static void 632 amdvi_decode_evt(struct amdvi_event *evt) 633 { 634 struct amdvi_cmd *cmd; 635 636 switch (evt->opcode) { 637 case AMDVI_EVENT_INVALID_DTE: 638 amdvi_decode_inv_dte_evt(evt->devid, evt->pasid_domid, 639 evt->addr, evt->flag); 640 break; 641 642 case AMDVI_EVENT_PFAULT: 643 amdvi_decode_pf_evt(evt->devid, evt->pasid_domid, 644 evt->addr, evt->flag); 645 break; 646 647 case AMDVI_EVENT_DTE_HW_ERROR: 648 amdvi_decode_dte_hwerr_evt(evt->devid, evt->pasid_domid, 649 evt->addr, evt->flag); 650 break; 651 652 case AMDVI_EVENT_PAGE_HW_ERROR: 653 amdvi_decode_page_hwerr_evt(evt->devid, evt->pasid_domid, 654 evt->addr, evt->flag); 655 break; 656 657 case AMDVI_EVENT_ILLEGAL_CMD: 658 /* FALL THROUGH */ 659 case AMDVI_EVENT_CMD_HW_ERROR: 660 printf("\t[%s EVT]\n", (evt->opcode == AMDVI_EVENT_ILLEGAL_CMD) ? 661 "ILLEGAL CMD" : "CMD HW ERR"); 662 cmd = (struct amdvi_cmd *)PHYS_TO_DMAP(evt->addr); 663 printf("\tCMD opcode= 0x%x 0x%x 0x%x 0x%lx\n", 664 cmd->opcode, cmd->word0, cmd->word1, cmd->addr); 665 break; 666 667 case AMDVI_EVENT_IOTLB_TIMEOUT: 668 printf("\t[IOTLB_INV_TIMEOUT devid:0x%x addr:0x%lx]\n", 669 evt->devid, evt->addr); 670 break; 671 672 case AMDVI_EVENT_INVALID_DTE_REQ: 673 printf("\t[INV_DTE devid:0x%x addr:0x%lx type:0x%x tr:%d]\n", 674 evt->devid, evt->addr, evt->flag >> 9, 675 (evt->flag >> 8) & 1); 676 break; 677 678 case AMDVI_EVENT_INVALID_PPR_REQ: 679 case AMDVI_EVENT_COUNTER_ZERO: 680 printf("AMD-Vi: v2 events.\n"); 681 break; 682 683 default: 684 printf("Unsupported AMD-Vi event:%d\n", evt->opcode); 685 } 686 } 687 688 static void 689 amdvi_print_events(struct amdvi_softc *softc) 690 { 691 struct amdvi_ctrl *ctrl; 692 struct amdvi_event *event; 693 int i, size; 694 695 ctrl = softc->ctrl; 696 size = sizeof(struct amdvi_event); 697 for (i = 0; i < softc->event_max; i++) { 698 event = &softc->event[ctrl->evt_head / size]; 699 if (!event->opcode) 700 break; 701 device_printf(softc->dev, "\t[Event%d: Head:0x%x Tail:0x%x]\n", 702 i, ctrl->evt_head, ctrl->evt_tail); 703 amdvi_decode_evt(event); 704 ctrl->evt_head = MOD_INC(ctrl->evt_head, size, 705 softc->event_max); 706 } 707 } 708 709 static int 710 amdvi_init_dte(struct amdvi_softc *softc) 711 { 712 struct amdvi_ctrl *ctrl; 713 714 ctrl = softc->ctrl; 715 ctrl->dte.base = vtophys(amdvi_dte) / PAGE_SIZE; 716 ctrl->dte.size = 0x1FF; /* 2MB device table. */ 717 718 return (0); 719 } 720 721 /* 722 * Not all capabilities of IOMMU are available in ACPI IVHD flag 723 * or EFR entry, read directly from device. 724 */ 725 static int 726 amdvi_print_pci_cap(device_t dev) 727 { 728 struct amdvi_softc *softc; 729 uint32_t off, cap; 730 731 softc = device_get_softc(dev); 732 off = softc->cap_off; 733 734 /* 735 * Section 3.7.1 of IOMMU sepc rev 2.0. 736 * Read capability from device. 737 */ 738 cap = amdvi_pci_read(softc, off); 739 740 /* Make sure capability type[18:16] is 3. */ 741 KASSERT((((cap >> 16) & 0x7) == 0x3), 742 ("Not a IOMMU capability 0x%x@0x%x", cap, off)); 743 744 softc->pci_cap = cap >> 24; 745 device_printf(softc->dev, "PCI cap 0x%x@0x%x feature:%b\n", 746 cap, off, softc->pci_cap, 747 "\20\1IOTLB\2HT\3NPCache\4EFR\5CapExt"); 748 749 return (0); 750 } 751 752 static void 753 amdvi_event_intr(void *arg) 754 { 755 struct amdvi_softc *softc; 756 struct amdvi_ctrl *ctrl; 757 758 softc = (struct amdvi_softc *)arg; 759 ctrl = softc->ctrl; 760 device_printf(softc->dev, "EVT INTR %ld Status:0x%x" 761 " EVT Head:0x%x Tail:0x%x]\n", softc->event_intr_cnt++, 762 ctrl->status, ctrl->evt_head, ctrl->evt_tail); 763 printf(" [CMD Total 0x%lx] Tail:0x%x, Head:0x%x.\n", 764 softc->total_cmd, ctrl->cmd_tail, ctrl->cmd_head); 765 766 amdvi_print_events(softc); 767 ctrl->status &= AMDVI_STATUS_EV_OF | AMDVI_STATUS_EV_INTR; 768 } 769 770 static void 771 amdvi_free_evt_intr_res(device_t dev) 772 { 773 774 struct amdvi_softc *softc; 775 device_t mmio_dev; 776 777 softc = device_get_softc(dev); 778 mmio_dev = softc->pci_dev; 779 780 IVHD_TEARDOWN_INTR(mmio_dev); 781 } 782 783 static bool 784 amdvi_alloc_intr_resources(struct amdvi_softc *softc) 785 { 786 struct amdvi_ctrl *ctrl; 787 device_t dev, mmio_dev; 788 int err; 789 790 dev = softc->dev; 791 mmio_dev = softc->pci_dev; 792 793 /* Clear interrupt status bits. */ 794 ctrl = softc->ctrl; 795 ctrl->status &= AMDVI_STATUS_EV_OF | AMDVI_STATUS_EV_INTR; 796 797 err = IVHD_SETUP_INTR(mmio_dev, amdvi_event_intr, softc, "fault"); 798 if (err) 799 device_printf(dev, "Interrupt setup failed on %s\n", 800 device_get_nameunit(mmio_dev)); 801 return (err); 802 } 803 804 static void 805 amdvi_print_dev_cap(struct amdvi_softc *softc) 806 { 807 struct ivhd_dev_cfg *cfg; 808 int i; 809 810 cfg = softc->dev_cfg; 811 for (i = 0; i < softc->dev_cfg_cnt; i++) { 812 device_printf(softc->dev, "device [0x%x - 0x%x] " 813 "config:%b%s\n", cfg->start_id, cfg->end_id, 814 cfg->data, 815 "\020\001INIT\002ExtInt\003NMI" 816 "\007LINT0\010LINT1", 817 cfg->enable_ats ? "ATS enabled" : ""); 818 cfg++; 819 } 820 } 821 822 static int 823 amdvi_handle_sysctl(SYSCTL_HANDLER_ARGS) 824 { 825 struct amdvi_softc *softc; 826 int result, type, error = 0; 827 828 softc = (struct amdvi_softc *)arg1; 829 type = arg2; 830 831 switch (type) { 832 case 0: 833 result = softc->ctrl->cmd_head; 834 error = sysctl_handle_int(oidp, &result, 0, 835 req); 836 break; 837 case 1: 838 result = softc->ctrl->cmd_tail; 839 error = sysctl_handle_int(oidp, &result, 0, 840 req); 841 break; 842 case 2: 843 result = softc->ctrl->evt_head; 844 error = sysctl_handle_int(oidp, &result, 0, 845 req); 846 break; 847 case 3: 848 result = softc->ctrl->evt_tail; 849 error = sysctl_handle_int(oidp, &result, 0, 850 req); 851 break; 852 853 default: 854 device_printf(softc->dev, "Unknown sysctl:%d\n", type); 855 } 856 857 return (error); 858 } 859 860 static void 861 amdvi_add_sysctl(struct amdvi_softc *softc) 862 { 863 struct sysctl_oid_list *child; 864 struct sysctl_ctx_list *ctx; 865 device_t dev; 866 867 dev = softc->dev; 868 ctx = device_get_sysctl_ctx(dev); 869 child = SYSCTL_CHILDREN(device_get_sysctl_tree(dev)); 870 871 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "event_intr_count", CTLFLAG_RD, 872 &softc->event_intr_cnt, "Event interrupt count"); 873 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "command_count", CTLFLAG_RD, 874 &softc->total_cmd, "Command submitted count"); 875 SYSCTL_ADD_U16(ctx, child, OID_AUTO, "pci_rid", CTLFLAG_RD, 876 &softc->pci_rid, 0, "IOMMU RID"); 877 SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "command_head", 878 CTLTYPE_UINT | CTLFLAG_RD | CTLFLAG_MPSAFE, softc, 0, 879 amdvi_handle_sysctl, "IU", "Command head"); 880 SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "command_tail", 881 CTLTYPE_UINT | CTLFLAG_RD | CTLFLAG_MPSAFE, softc, 1, 882 amdvi_handle_sysctl, "IU", "Command tail"); 883 SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "event_head", 884 CTLTYPE_UINT | CTLFLAG_RD | CTLFLAG_MPSAFE, softc, 2, 885 amdvi_handle_sysctl, "IU", "Command head"); 886 SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "event_tail", 887 CTLTYPE_UINT | CTLFLAG_RD | CTLFLAG_MPSAFE, softc, 3, 888 amdvi_handle_sysctl, "IU", "Command tail"); 889 } 890 891 int 892 amdvi_setup_hw(struct amdvi_softc *softc) 893 { 894 device_t dev; 895 int status; 896 897 dev = softc->dev; 898 899 amdvi_hw_enable_iotlb(softc); 900 901 amdvi_print_dev_cap(softc); 902 903 if ((status = amdvi_print_pci_cap(dev)) != 0) { 904 device_printf(dev, "PCI capability.\n"); 905 return (status); 906 } 907 if ((status = amdvi_init_cmd(softc)) != 0) { 908 device_printf(dev, "Couldn't configure command buffer.\n"); 909 return (status); 910 } 911 if ((status = amdvi_init_event(softc)) != 0) { 912 device_printf(dev, "Couldn't configure event buffer.\n"); 913 return (status); 914 } 915 if ((status = amdvi_init_dte(softc)) != 0) { 916 device_printf(dev, "Couldn't configure device table.\n"); 917 return (status); 918 } 919 if ((status = amdvi_alloc_intr_resources(softc)) != 0) { 920 return (status); 921 } 922 amdvi_add_sysctl(softc); 923 return (0); 924 } 925 926 int 927 amdvi_teardown_hw(struct amdvi_softc *softc) 928 { 929 device_t dev; 930 931 dev = softc->dev; 932 933 /* 934 * Called after disable, h/w is stopped by now, free all the resources. 935 */ 936 amdvi_free_evt_intr_res(dev); 937 938 if (softc->cmd) 939 free(softc->cmd, M_AMDVI); 940 941 if (softc->event) 942 free(softc->event, M_AMDVI); 943 944 return (0); 945 } 946 947 /*********** bhyve interfaces *********************/ 948 static int 949 amdvi_init(void) 950 { 951 if (!ivhd_count) { 952 return (EIO); 953 } 954 if (!amdvi_enable_user && ivhd_count) { 955 printf("bhyve: Found %d AMD-Vi/IOMMU device(s), " 956 "use hw.vmm.amdvi.enable=1 to enable pass-through.\n", 957 ivhd_count); 958 return (EINVAL); 959 } 960 return (0); 961 } 962 963 static void 964 amdvi_cleanup(void) 965 { 966 /* Nothing. */ 967 } 968 969 static uint16_t 970 amdvi_domainId(void) 971 { 972 973 /* 974 * If we hit maximum domain limit, rollover leaving host 975 * domain(0). 976 * XXX: make sure that this domain is not used. 977 */ 978 if (amdvi_dom_id == AMDVI_MAX_DOMAIN) 979 amdvi_dom_id = 1; 980 981 return ((uint16_t)amdvi_dom_id++); 982 } 983 984 static void 985 amdvi_do_inv_domain(uint16_t domain_id, bool create) 986 { 987 struct amdvi_softc *softc; 988 int i; 989 990 for (i = 0; i < ivhd_count; i++) { 991 softc = device_get_softc(ivhd_devs[i]); 992 KASSERT(softc, ("softc is NULL")); 993 /* 994 * If not present pages are cached, invalidate page after 995 * creating domain. 996 */ 997 #if 0 998 if (create && ((softc->pci_cap & AMDVI_PCI_CAP_NPCACHE) == 0)) 999 continue; 1000 #endif 1001 amdvi_inv_domain(softc, domain_id); 1002 amdvi_wait(softc); 1003 } 1004 } 1005 1006 static void * 1007 amdvi_create_domain(vm_paddr_t maxaddr) 1008 { 1009 struct amdvi_domain *dom; 1010 1011 dom = malloc(sizeof(struct amdvi_domain), M_AMDVI, M_ZERO | M_WAITOK); 1012 dom->id = amdvi_domainId(); 1013 //dom->maxaddr = maxaddr; 1014 #ifdef AMDVI_DEBUG_CMD 1015 printf("Created domain #%d\n", dom->id); 1016 #endif 1017 /* 1018 * Host domain(#0) don't create translation table. 1019 */ 1020 if (dom->id || amdvi_host_ptp) 1021 dom->ptp = malloc(PAGE_SIZE, M_AMDVI, M_WAITOK | M_ZERO); 1022 1023 dom->ptp_level = amdvi_ptp_level; 1024 1025 amdvi_do_inv_domain(dom->id, true); 1026 SLIST_INSERT_HEAD(&dom_head, dom, next); 1027 1028 return (dom); 1029 } 1030 1031 static void 1032 amdvi_free_ptp(uint64_t *ptp, int level) 1033 { 1034 int i; 1035 1036 if (level < 1) 1037 return; 1038 1039 for (i = 0; i < NPTEPG ; i++) { 1040 if ((ptp[i] & AMDVI_PT_PRESENT) == 0) 1041 continue; 1042 /* XXX: Add super-page or PTE mapping > 4KB. */ 1043 #ifdef notyet 1044 /* Super-page mapping. */ 1045 if (AMDVI_PD_SUPER(ptp[i])) 1046 continue; 1047 #endif 1048 1049 amdvi_free_ptp((uint64_t *)PHYS_TO_DMAP(ptp[i] 1050 & AMDVI_PT_MASK), level - 1); 1051 } 1052 1053 free(ptp, M_AMDVI); 1054 } 1055 1056 static void 1057 amdvi_destroy_domain(void *arg) 1058 { 1059 struct amdvi_domain *domain; 1060 1061 domain = (struct amdvi_domain *)arg; 1062 KASSERT(domain, ("domain is NULL")); 1063 #ifdef AMDVI_DEBUG_CMD 1064 printf("Destroying domain %d\n", domain->id); 1065 #endif 1066 if (domain->ptp) 1067 amdvi_free_ptp(domain->ptp, domain->ptp_level); 1068 1069 amdvi_do_inv_domain(domain->id, false); 1070 SLIST_REMOVE(&dom_head, domain, amdvi_domain, next); 1071 free(domain, M_AMDVI); 1072 } 1073 1074 static uint64_t 1075 amdvi_set_pt(uint64_t *pt, int level, vm_paddr_t gpa, 1076 vm_paddr_t hpa, uint64_t pg_size, bool create) 1077 { 1078 uint64_t *page, pa; 1079 int shift, index; 1080 const int PT_SHIFT = 9; 1081 const int PT_INDEX_MASK = (1 << PT_SHIFT) - 1; /* Based on PT_SHIFT */ 1082 1083 if (!pg_size) 1084 return (0); 1085 1086 if (hpa & (pg_size - 1)) { 1087 printf("HPA is not size aligned.\n"); 1088 return (0); 1089 } 1090 if (gpa & (pg_size - 1)) { 1091 printf("HPA is not size aligned.\n"); 1092 return (0); 1093 } 1094 shift = PML4SHIFT; 1095 while ((shift > PAGE_SHIFT) && (pg_size < (1UL << shift))) { 1096 index = (gpa >> shift) & PT_INDEX_MASK; 1097 1098 if ((pt[index] == 0) && create) { 1099 page = malloc(PAGE_SIZE, M_AMDVI, M_WAITOK | M_ZERO); 1100 pa = vtophys(page); 1101 pt[index] = pa | AMDVI_PT_PRESENT | AMDVI_PT_RW | 1102 ((level - 1) << AMDVI_PD_LEVEL_SHIFT); 1103 } 1104 #ifdef AMDVI_DEBUG_PTE 1105 if ((gpa % 0x1000000) == 0) 1106 printf("[level%d, shift = %d]PTE:0x%lx\n", 1107 level, shift, pt[index]); 1108 #endif 1109 #define PTE2PA(x) ((uint64_t)(x) & AMDVI_PT_MASK) 1110 pa = PTE2PA(pt[index]); 1111 pt = (uint64_t *)PHYS_TO_DMAP(pa); 1112 shift -= PT_SHIFT; 1113 level--; 1114 } 1115 1116 /* Leaf entry. */ 1117 index = (gpa >> shift) & PT_INDEX_MASK; 1118 1119 if (create) { 1120 pt[index] = hpa | AMDVI_PT_RW | AMDVI_PT_PRESENT; 1121 } else 1122 pt[index] = 0; 1123 1124 #ifdef AMDVI_DEBUG_PTE 1125 if ((gpa % 0x1000000) == 0) 1126 printf("[Last level%d, shift = %d]PTE:0x%lx\n", 1127 level, shift, pt[index]); 1128 #endif 1129 return (1ULL << shift); 1130 } 1131 1132 static uint64_t 1133 amdvi_update_mapping(struct amdvi_domain *domain, vm_paddr_t gpa, 1134 vm_paddr_t hpa, uint64_t size, bool create) 1135 { 1136 uint64_t mapped, *ptp, len; 1137 int level; 1138 1139 KASSERT(domain, ("domain is NULL")); 1140 level = domain->ptp_level; 1141 KASSERT(level, ("Page table level is 0")); 1142 1143 ptp = domain->ptp; 1144 KASSERT(ptp, ("PTP is NULL")); 1145 mapped = 0; 1146 while (mapped < size) { 1147 len = amdvi_set_pt(ptp, level, gpa + mapped, hpa + mapped, 1148 PAGE_SIZE, create); 1149 if (!len) { 1150 printf("Error: Couldn't map HPA:0x%lx GPA:0x%lx\n", 1151 hpa, gpa); 1152 return (0); 1153 } 1154 mapped += len; 1155 } 1156 1157 return (mapped); 1158 } 1159 1160 static uint64_t 1161 amdvi_create_mapping(void *arg, vm_paddr_t gpa, vm_paddr_t hpa, 1162 uint64_t len) 1163 { 1164 struct amdvi_domain *domain; 1165 1166 domain = (struct amdvi_domain *)arg; 1167 1168 if (domain->id && !domain->ptp) { 1169 printf("ptp is NULL"); 1170 return (-1); 1171 } 1172 1173 /* 1174 * If host domain is created w/o page table, skip IOMMU page 1175 * table set-up. 1176 */ 1177 if (domain->ptp) 1178 return (amdvi_update_mapping(domain, gpa, hpa, len, true)); 1179 else 1180 return (len); 1181 } 1182 1183 static uint64_t 1184 amdvi_remove_mapping(void *arg, vm_paddr_t gpa, uint64_t len) 1185 { 1186 struct amdvi_domain *domain; 1187 1188 domain = (struct amdvi_domain *)arg; 1189 /* 1190 * If host domain is created w/o page table, skip IOMMU page 1191 * table set-up. 1192 */ 1193 if (domain->ptp) 1194 return (amdvi_update_mapping(domain, gpa, 0, len, false)); 1195 return 1196 (len); 1197 } 1198 1199 static struct amdvi_softc * 1200 amdvi_find_iommu(uint16_t devid) 1201 { 1202 struct amdvi_softc *softc; 1203 int i, j; 1204 1205 for (i = 0; i < ivhd_count; i++) { 1206 softc = device_get_softc(ivhd_devs[i]); 1207 for (j = 0; j < softc->dev_cfg_cnt; j++) 1208 if ((devid >= softc->dev_cfg[j].start_id) && 1209 (devid <= softc->dev_cfg[j].end_id)) 1210 return (softc); 1211 } 1212 1213 return (NULL); 1214 } 1215 1216 /* 1217 * Set-up device table entry. 1218 * IOMMU spec Rev 2.0, section 3.2.2.2, some of the fields must 1219 * be set concurrently, e.g. read and write bits. 1220 */ 1221 static void 1222 amdvi_set_dte(struct amdvi_domain *domain, struct amdvi_softc *softc, 1223 uint16_t devid, bool enable) 1224 { 1225 struct amdvi_dte* temp; 1226 1227 KASSERT(domain, ("domain is NULL for pci_rid:0x%x\n", devid)); 1228 KASSERT(softc, ("softc is NULL for pci_rid:0x%x\n", devid)); 1229 1230 temp = &amdvi_dte[devid]; 1231 1232 #ifdef AMDVI_ATS_ENABLE 1233 /* If IOMMU and device support IOTLB, enable it. */ 1234 if (amdvi_dev_support_iotlb(softc, devid) && softc->iotlb) 1235 temp->iotlb_enable = 1; 1236 #endif 1237 1238 /* Avoid duplicate I/O faults. */ 1239 temp->sup_second_io_fault = 1; 1240 temp->sup_all_io_fault = amdvi_disable_io_fault; 1241 1242 temp->dt_valid = 1; 1243 temp->domain_id = domain->id; 1244 1245 if (enable) { 1246 if (domain->ptp) { 1247 temp->pt_base = vtophys(domain->ptp) >> 12; 1248 temp->pt_level = amdvi_ptp_level; 1249 } 1250 /* 1251 * XXX: Page table valid[TV] bit must be set even if host domain 1252 * page tables are not enabled. 1253 */ 1254 temp->pt_valid = 1; 1255 temp->read_allow = 1; 1256 temp->write_allow = 1; 1257 } 1258 } 1259 1260 static void 1261 amdvi_inv_device(struct amdvi_softc *softc, uint16_t devid) 1262 { 1263 KASSERT(softc, ("softc is NULL")); 1264 1265 amdvi_cmd_inv_dte(softc, devid); 1266 #ifdef AMDVI_ATS_ENABLE 1267 if (amdvi_dev_support_iotlb(softc, devid)) 1268 amdvi_cmd_inv_iotlb(softc, devid); 1269 #endif 1270 amdvi_wait(softc); 1271 } 1272 1273 static void 1274 amdvi_add_device(void *arg, uint16_t devid) 1275 { 1276 struct amdvi_domain *domain; 1277 struct amdvi_softc *softc; 1278 1279 domain = (struct amdvi_domain *)arg; 1280 KASSERT(domain != NULL, ("domain is NULL")); 1281 #ifdef AMDVI_DEBUG_CMD 1282 printf("Assigning device(%d.%d.%d) to domain:%d\n", 1283 RID2PCI_STR(devid), domain->id); 1284 #endif 1285 softc = amdvi_find_iommu(devid); 1286 if (softc == NULL) 1287 return; 1288 amdvi_set_dte(domain, softc, devid, true); 1289 amdvi_inv_device(softc, devid); 1290 } 1291 1292 static void 1293 amdvi_remove_device(void *arg, uint16_t devid) 1294 { 1295 struct amdvi_domain *domain; 1296 struct amdvi_softc *softc; 1297 1298 domain = (struct amdvi_domain *)arg; 1299 #ifdef AMDVI_DEBUG_CMD 1300 printf("Remove device(0x%x) from domain:%d\n", 1301 devid, domain->id); 1302 #endif 1303 softc = amdvi_find_iommu(devid); 1304 if (softc == NULL) 1305 return; 1306 amdvi_set_dte(domain, softc, devid, false); 1307 amdvi_inv_device(softc, devid); 1308 } 1309 1310 static void 1311 amdvi_enable(void) 1312 { 1313 struct amdvi_ctrl *ctrl; 1314 struct amdvi_softc *softc; 1315 uint64_t val; 1316 int i; 1317 1318 for (i = 0; i < ivhd_count; i++) { 1319 softc = device_get_softc(ivhd_devs[i]); 1320 KASSERT(softc, ("softc is NULL\n")); 1321 ctrl = softc->ctrl; 1322 KASSERT(ctrl, ("ctrl is NULL\n")); 1323 1324 val = ( AMDVI_CTRL_EN | 1325 AMDVI_CTRL_CMD | 1326 AMDVI_CTRL_ELOG | 1327 AMDVI_CTRL_ELOGINT | 1328 AMDVI_CTRL_INV_TO_1S); 1329 1330 if (softc->ivhd_flag & IVHD_FLAG_COH) 1331 val |= AMDVI_CTRL_COH; 1332 if (softc->ivhd_flag & IVHD_FLAG_HTT) 1333 val |= AMDVI_CTRL_HTT; 1334 if (softc->ivhd_flag & IVHD_FLAG_RPPW) 1335 val |= AMDVI_CTRL_RPPW; 1336 if (softc->ivhd_flag & IVHD_FLAG_PPW) 1337 val |= AMDVI_CTRL_PPW; 1338 if (softc->ivhd_flag & IVHD_FLAG_ISOC) 1339 val |= AMDVI_CTRL_ISOC; 1340 1341 ctrl->control = val; 1342 } 1343 } 1344 1345 static void 1346 amdvi_disable(void) 1347 { 1348 struct amdvi_ctrl *ctrl; 1349 struct amdvi_softc *softc; 1350 int i; 1351 1352 for (i = 0; i < ivhd_count; i++) { 1353 softc = device_get_softc(ivhd_devs[i]); 1354 KASSERT(softc, ("softc is NULL\n")); 1355 ctrl = softc->ctrl; 1356 KASSERT(ctrl, ("ctrl is NULL\n")); 1357 1358 ctrl->control = 0; 1359 } 1360 } 1361 1362 static void 1363 amdvi_invalidate_tlb(void *arg) 1364 { 1365 struct amdvi_domain *domain; 1366 1367 domain = (struct amdvi_domain *)arg; 1368 KASSERT(domain, ("domain is NULL")); 1369 amdvi_do_inv_domain(domain->id, false); 1370 } 1371 1372 const struct iommu_ops iommu_ops_amd = { 1373 .init = amdvi_init, 1374 .cleanup = amdvi_cleanup, 1375 .enable = amdvi_enable, 1376 .disable = amdvi_disable, 1377 .create_domain = amdvi_create_domain, 1378 .destroy_domain = amdvi_destroy_domain, 1379 .create_mapping = amdvi_create_mapping, 1380 .remove_mapping = amdvi_remove_mapping, 1381 .add_device = amdvi_add_device, 1382 .remove_device = amdvi_remove_device, 1383 .invalidate_tlb = amdvi_invalidate_tlb 1384 }; 1385