1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause 3 * 4 * Copyright (c) 2016, Anish Gupta (anish@freebsd.org) 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice unmodified, this list of conditions, and the following 12 * disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 18 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 19 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 20 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 21 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 22 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 23 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 24 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 25 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 26 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 27 */ 28 29 #include <sys/param.h> 30 #include <sys/systm.h> 31 #include <sys/bus.h> 32 #include <sys/kernel.h> 33 #include <sys/module.h> 34 #include <sys/malloc.h> 35 #include <sys/pcpu.h> 36 #include <sys/rman.h> 37 #include <sys/smp.h> 38 #include <sys/sysctl.h> 39 40 #include <vm/vm.h> 41 #include <vm/pmap.h> 42 43 #include <dev/pci/pcivar.h> 44 #include <dev/pci/pcireg.h> 45 46 #include <machine/resource.h> 47 #include <machine/vmm.h> 48 #include <machine/pmap.h> 49 #include <machine/vmparam.h> 50 #include <machine/pci_cfgreg.h> 51 52 #include "ivhd_if.h" 53 #include "pcib_if.h" 54 55 #include "io/iommu.h" 56 #include "amdvi_priv.h" 57 58 SYSCTL_DECL(_hw_vmm); 59 SYSCTL_NODE(_hw_vmm, OID_AUTO, amdvi, CTLFLAG_RW | CTLFLAG_MPSAFE, NULL, 60 NULL); 61 62 #define MOD_INC(a, s, m) (((a) + (s)) % ((m) * (s))) 63 #define MOD_DEC(a, s, m) (((a) - (s)) % ((m) * (s))) 64 65 /* Print RID or device ID in PCI string format. */ 66 #define RID2PCI_STR(d) PCI_RID2BUS(d), PCI_RID2SLOT(d), PCI_RID2FUNC(d) 67 68 static void amdvi_dump_cmds(struct amdvi_softc *softc, int count); 69 static void amdvi_print_dev_cap(struct amdvi_softc *softc); 70 71 MALLOC_DEFINE(M_AMDVI, "amdvi", "amdvi"); 72 73 extern device_t *ivhd_devs; 74 75 extern int ivhd_count; 76 SYSCTL_INT(_hw_vmm_amdvi, OID_AUTO, count, CTLFLAG_RDTUN, &ivhd_count, 77 0, NULL); 78 79 static int amdvi_enable_user = 0; 80 SYSCTL_INT(_hw_vmm_amdvi, OID_AUTO, enable, CTLFLAG_RDTUN, 81 &amdvi_enable_user, 0, NULL); 82 TUNABLE_INT("hw.vmm.amdvi_enable", &amdvi_enable_user); 83 84 #ifdef AMDVI_ATS_ENABLE 85 /* XXX: ATS is not tested. */ 86 static int amdvi_enable_iotlb = 1; 87 SYSCTL_INT(_hw_vmm_amdvi, OID_AUTO, iotlb_enabled, CTLFLAG_RDTUN, 88 &amdvi_enable_iotlb, 0, NULL); 89 TUNABLE_INT("hw.vmm.enable_iotlb", &amdvi_enable_iotlb); 90 #endif 91 92 static int amdvi_host_ptp = 1; /* Use page tables for host. */ 93 SYSCTL_INT(_hw_vmm_amdvi, OID_AUTO, host_ptp, CTLFLAG_RDTUN, 94 &amdvi_host_ptp, 0, NULL); 95 TUNABLE_INT("hw.vmm.amdvi.host_ptp", &amdvi_host_ptp); 96 97 /* Page table level used <= supported by h/w[v1=7]. */ 98 int amdvi_ptp_level = 4; 99 SYSCTL_INT(_hw_vmm_amdvi, OID_AUTO, ptp_level, CTLFLAG_RDTUN, 100 &amdvi_ptp_level, 0, NULL); 101 TUNABLE_INT("hw.vmm.amdvi.ptp_level", &amdvi_ptp_level); 102 103 /* Disable fault event reporting. */ 104 static int amdvi_disable_io_fault = 0; 105 SYSCTL_INT(_hw_vmm_amdvi, OID_AUTO, disable_io_fault, CTLFLAG_RDTUN, 106 &amdvi_disable_io_fault, 0, NULL); 107 TUNABLE_INT("hw.vmm.amdvi.disable_io_fault", &amdvi_disable_io_fault); 108 109 static uint32_t amdvi_dom_id = 0; /* 0 is reserved for host. */ 110 SYSCTL_UINT(_hw_vmm_amdvi, OID_AUTO, domain_id, CTLFLAG_RD, 111 &amdvi_dom_id, 0, NULL); 112 /* 113 * Device table entry. 114 * Bus(256) x Dev(32) x Fun(8) x DTE(256 bits or 32 bytes). 115 * = 256 * 2 * PAGE_SIZE. 116 */ 117 static struct amdvi_dte amdvi_dte[PCI_NUM_DEV_MAX] __aligned(PAGE_SIZE); 118 CTASSERT(PCI_NUM_DEV_MAX == 0x10000); 119 CTASSERT(sizeof(amdvi_dte) == 0x200000); 120 121 static SLIST_HEAD (, amdvi_domain) dom_head; 122 123 static inline uint32_t 124 amdvi_pci_read(struct amdvi_softc *softc, int off) 125 { 126 127 return (pci_cfgregread(softc->pci_seg, PCI_RID2BUS(softc->pci_rid), 128 PCI_RID2SLOT(softc->pci_rid), PCI_RID2FUNC(softc->pci_rid), 129 off, 4)); 130 } 131 132 #ifdef AMDVI_ATS_ENABLE 133 /* XXX: Should be in pci.c */ 134 /* 135 * Check if device has ATS capability and its enabled. 136 * If ATS is absent or disabled, return (-1), otherwise ATS 137 * queue length. 138 */ 139 static int 140 amdvi_find_ats_qlen(uint16_t devid) 141 { 142 device_t dev; 143 uint32_t off, cap; 144 int qlen = -1; 145 146 dev = pci_find_bsf(PCI_RID2BUS(devid), PCI_RID2SLOT(devid), 147 PCI_RID2FUNC(devid)); 148 149 if (!dev) { 150 return (-1); 151 } 152 #define PCIM_ATS_EN BIT(31) 153 154 if (pci_find_extcap(dev, PCIZ_ATS, &off) == 0) { 155 cap = pci_read_config(dev, off + 4, 4); 156 qlen = (cap & 0x1F); 157 qlen = qlen ? qlen : 32; 158 printf("AMD-Vi: PCI device %d.%d.%d ATS %s qlen=%d\n", 159 RID2PCI_STR(devid), 160 (cap & PCIM_ATS_EN) ? "enabled" : "Disabled", 161 qlen); 162 qlen = (cap & PCIM_ATS_EN) ? qlen : -1; 163 } 164 165 return (qlen); 166 } 167 168 /* 169 * Check if an endpoint device support device IOTLB or ATS. 170 */ 171 static inline bool 172 amdvi_dev_support_iotlb(struct amdvi_softc *softc, uint16_t devid) 173 { 174 struct ivhd_dev_cfg *cfg; 175 int qlen, i; 176 bool pci_ats, ivhd_ats; 177 178 qlen = amdvi_find_ats_qlen(devid); 179 if (qlen < 0) 180 return (false); 181 182 KASSERT(softc, ("softc is NULL")); 183 cfg = softc->dev_cfg; 184 185 ivhd_ats = false; 186 for (i = 0; i < softc->dev_cfg_cnt; i++) { 187 if ((cfg->start_id <= devid) && (cfg->end_id >= devid)) { 188 ivhd_ats = cfg->enable_ats; 189 break; 190 } 191 cfg++; 192 } 193 194 pci_ats = (qlen < 0) ? false : true; 195 if (pci_ats != ivhd_ats) 196 device_printf(softc->dev, 197 "BIOS bug: mismatch in ATS setting for %d.%d.%d," 198 "ATS inv qlen = %d\n", RID2PCI_STR(devid), qlen); 199 200 /* Ignore IVRS setting and respect PCI setting. */ 201 return (pci_ats); 202 } 203 #endif 204 205 /* Enable IOTLB support for IOMMU if its supported. */ 206 static inline void 207 amdvi_hw_enable_iotlb(struct amdvi_softc *softc) 208 { 209 #ifndef AMDVI_ATS_ENABLE 210 softc->iotlb = false; 211 #else 212 bool supported; 213 214 supported = (softc->ivhd_flag & IVHD_FLAG_IOTLB) ? true : false; 215 216 if (softc->pci_cap & AMDVI_PCI_CAP_IOTLB) { 217 if (!supported) 218 device_printf(softc->dev, "IOTLB disabled by BIOS.\n"); 219 220 if (supported && !amdvi_enable_iotlb) { 221 device_printf(softc->dev, "IOTLB disabled by user.\n"); 222 supported = false; 223 } 224 } else 225 supported = false; 226 227 softc->iotlb = supported; 228 229 #endif 230 } 231 232 static int 233 amdvi_init_cmd(struct amdvi_softc *softc) 234 { 235 struct amdvi_ctrl *ctrl = softc->ctrl; 236 237 ctrl->cmd.len = 8; /* Use 256 command buffer entries. */ 238 softc->cmd_max = 1 << ctrl->cmd.len; 239 240 softc->cmd = malloc(sizeof(struct amdvi_cmd) * 241 softc->cmd_max, M_AMDVI, M_WAITOK | M_ZERO); 242 243 if ((uintptr_t)softc->cmd & PAGE_MASK) 244 panic("AMDVi: Command buffer not aligned on page boundary."); 245 246 ctrl->cmd.base = vtophys(softc->cmd) / PAGE_SIZE; 247 /* 248 * XXX: Reset the h/w pointers in case IOMMU is restarting, 249 * h/w doesn't clear these pointers based on empirical data. 250 */ 251 ctrl->cmd_tail = 0; 252 ctrl->cmd_head = 0; 253 254 return (0); 255 } 256 257 /* 258 * Note: Update tail pointer after we have written the command since tail 259 * pointer update cause h/w to execute new commands, see section 3.3 260 * of AMD IOMMU spec ver 2.0. 261 */ 262 /* Get the command tail pointer w/o updating it. */ 263 static struct amdvi_cmd * 264 amdvi_get_cmd_tail(struct amdvi_softc *softc) 265 { 266 struct amdvi_ctrl *ctrl; 267 struct amdvi_cmd *tail; 268 269 KASSERT(softc, ("softc is NULL")); 270 KASSERT(softc->cmd != NULL, ("cmd is NULL")); 271 272 ctrl = softc->ctrl; 273 KASSERT(ctrl != NULL, ("ctrl is NULL")); 274 275 tail = (struct amdvi_cmd *)((uint8_t *)softc->cmd + 276 ctrl->cmd_tail); 277 278 return (tail); 279 } 280 281 /* 282 * Update the command tail pointer which will start command execution. 283 */ 284 static void 285 amdvi_update_cmd_tail(struct amdvi_softc *softc) 286 { 287 struct amdvi_ctrl *ctrl; 288 int size; 289 290 size = sizeof(struct amdvi_cmd); 291 KASSERT(softc->cmd != NULL, ("cmd is NULL")); 292 293 ctrl = softc->ctrl; 294 KASSERT(ctrl != NULL, ("ctrl is NULL")); 295 296 ctrl->cmd_tail = MOD_INC(ctrl->cmd_tail, size, softc->cmd_max); 297 softc->total_cmd++; 298 299 #ifdef AMDVI_DEBUG_CMD 300 device_printf(softc->dev, "cmd_tail: %s Tail:0x%x, Head:0x%x.\n", 301 ctrl->cmd_tail, 302 ctrl->cmd_head); 303 #endif 304 305 } 306 307 /* 308 * Various commands supported by IOMMU. 309 */ 310 311 /* Completion wait command. */ 312 static void 313 amdvi_cmd_cmp(struct amdvi_softc *softc, const uint64_t data) 314 { 315 struct amdvi_cmd *cmd; 316 uint64_t pa; 317 318 cmd = amdvi_get_cmd_tail(softc); 319 KASSERT(cmd != NULL, ("Cmd is NULL")); 320 321 pa = vtophys(&softc->cmp_data); 322 cmd->opcode = AMDVI_CMP_WAIT_OPCODE; 323 cmd->word0 = (pa & 0xFFFFFFF8) | AMDVI_CMP_WAIT_STORE; 324 cmd->word1 = (pa >> 32) & 0xFFFFF; 325 cmd->addr = data; 326 327 amdvi_update_cmd_tail(softc); 328 } 329 330 /* Invalidate device table entry. */ 331 static void 332 amdvi_cmd_inv_dte(struct amdvi_softc *softc, uint16_t devid) 333 { 334 struct amdvi_cmd *cmd; 335 336 cmd = amdvi_get_cmd_tail(softc); 337 KASSERT(cmd != NULL, ("Cmd is NULL")); 338 cmd->opcode = AMDVI_INVD_DTE_OPCODE; 339 cmd->word0 = devid; 340 amdvi_update_cmd_tail(softc); 341 #ifdef AMDVI_DEBUG_CMD 342 device_printf(softc->dev, "Invalidated DTE:0x%x\n", devid); 343 #endif 344 } 345 346 /* Invalidate IOMMU page, use for invalidation of domain. */ 347 static void 348 amdvi_cmd_inv_iommu_pages(struct amdvi_softc *softc, uint16_t domain_id, 349 uint64_t addr, bool guest_nested, 350 bool pde, bool page) 351 { 352 struct amdvi_cmd *cmd; 353 354 cmd = amdvi_get_cmd_tail(softc); 355 KASSERT(cmd != NULL, ("Cmd is NULL")); 356 357 cmd->opcode = AMDVI_INVD_PAGE_OPCODE; 358 cmd->word1 = domain_id; 359 /* 360 * Invalidate all addresses for this domain. 361 */ 362 cmd->addr = addr; 363 cmd->addr |= pde ? AMDVI_INVD_PAGE_PDE : 0; 364 cmd->addr |= page ? AMDVI_INVD_PAGE_S : 0; 365 366 amdvi_update_cmd_tail(softc); 367 } 368 369 #ifdef AMDVI_ATS_ENABLE 370 /* Invalidate device IOTLB. */ 371 static void 372 amdvi_cmd_inv_iotlb(struct amdvi_softc *softc, uint16_t devid) 373 { 374 struct amdvi_cmd *cmd; 375 int qlen; 376 377 if (!softc->iotlb) 378 return; 379 380 qlen = amdvi_find_ats_qlen(devid); 381 if (qlen < 0) { 382 panic("AMDVI: Invalid ATS qlen(%d) for device %d.%d.%d\n", 383 qlen, RID2PCI_STR(devid)); 384 } 385 cmd = amdvi_get_cmd_tail(softc); 386 KASSERT(cmd != NULL, ("Cmd is NULL")); 387 388 #ifdef AMDVI_DEBUG_CMD 389 device_printf(softc->dev, "Invalidate IOTLB devID 0x%x" 390 " Qlen:%d\n", devid, qlen); 391 #endif 392 cmd->opcode = AMDVI_INVD_IOTLB_OPCODE; 393 cmd->word0 = devid; 394 cmd->word1 = qlen; 395 cmd->addr = AMDVI_INVD_IOTLB_ALL_ADDR | 396 AMDVI_INVD_IOTLB_S; 397 amdvi_update_cmd_tail(softc); 398 } 399 #endif 400 401 #ifdef notyet /* For Interrupt Remap. */ 402 static void 403 amdvi_cmd_inv_intr_map(struct amdvi_softc *softc, 404 uint16_t devid) 405 { 406 struct amdvi_cmd *cmd; 407 408 cmd = amdvi_get_cmd_tail(softc); 409 KASSERT(cmd != NULL, ("Cmd is NULL")); 410 cmd->opcode = AMDVI_INVD_INTR_OPCODE; 411 cmd->word0 = devid; 412 amdvi_update_cmd_tail(softc); 413 #ifdef AMDVI_DEBUG_CMD 414 device_printf(softc->dev, "Invalidate INTR map of devID 0x%x\n", devid); 415 #endif 416 } 417 #endif 418 419 /* Invalidate domain using INVALIDATE_IOMMU_PAGES command. */ 420 static void 421 amdvi_inv_domain(struct amdvi_softc *softc, uint16_t domain_id) 422 { 423 struct amdvi_cmd *cmd __diagused; 424 425 cmd = amdvi_get_cmd_tail(softc); 426 KASSERT(cmd != NULL, ("Cmd is NULL")); 427 428 /* 429 * See section 3.3.3 of IOMMU spec rev 2.0, software note 430 * for invalidating domain. 431 */ 432 amdvi_cmd_inv_iommu_pages(softc, domain_id, AMDVI_INVD_PAGE_ALL_ADDR, 433 false, true, true); 434 435 #ifdef AMDVI_DEBUG_CMD 436 device_printf(softc->dev, "Invalidate domain:0x%x\n", domain_id); 437 438 #endif 439 } 440 441 static bool 442 amdvi_cmp_wait(struct amdvi_softc *softc) 443 { 444 #ifdef AMDVI_DEBUG_CMD 445 struct amdvi_ctrl *ctrl = softc->ctrl; 446 #endif 447 const uint64_t VERIFY = 0xA5A5; 448 volatile uint64_t *read; 449 int i; 450 bool status; 451 452 read = &softc->cmp_data; 453 *read = 0; 454 amdvi_cmd_cmp(softc, VERIFY); 455 /* Wait for h/w to update completion data. */ 456 for (i = 0; i < 100 && (*read != VERIFY); i++) { 457 DELAY(1000); /* 1 ms */ 458 } 459 status = (VERIFY == softc->cmp_data) ? true : false; 460 461 #ifdef AMDVI_DEBUG_CMD 462 if (status) 463 device_printf(softc->dev, "CMD completion DONE Tail:0x%x, " 464 "Head:0x%x, loop:%d.\n", ctrl->cmd_tail, 465 ctrl->cmd_head, loop); 466 #endif 467 return (status); 468 } 469 470 static void 471 amdvi_wait(struct amdvi_softc *softc) 472 { 473 struct amdvi_ctrl *ctrl; 474 int i; 475 476 KASSERT(softc, ("softc is NULL")); 477 478 ctrl = softc->ctrl; 479 KASSERT(ctrl != NULL, ("ctrl is NULL")); 480 /* Don't wait if h/w is not enabled. */ 481 if ((ctrl->control & AMDVI_CTRL_EN) == 0) 482 return; 483 484 for (i = 0; i < 10; i++) { 485 if (amdvi_cmp_wait(softc)) 486 return; 487 } 488 489 device_printf(softc->dev, "Error: completion failed" 490 " tail:0x%x, head:0x%x.\n", 491 ctrl->cmd_tail, ctrl->cmd_head); 492 /* Dump the last command. */ 493 amdvi_dump_cmds(softc, 1); 494 } 495 496 static void 497 amdvi_dump_cmds(struct amdvi_softc *softc, int count) 498 { 499 struct amdvi_ctrl *ctrl; 500 struct amdvi_cmd *cmd; 501 int off, i; 502 503 ctrl = softc->ctrl; 504 device_printf(softc->dev, "Dump last %d command(s):\n", count); 505 /* 506 * If h/w is stuck in completion, it is the previous command, 507 * start dumping from previous command onward. 508 */ 509 off = MOD_DEC(ctrl->cmd_head, sizeof(struct amdvi_cmd), 510 softc->cmd_max); 511 for (i = 0; off != ctrl->cmd_tail && i < count; i++) { 512 cmd = (struct amdvi_cmd *)((uint8_t *)softc->cmd + off); 513 printf(" [CMD%d, off:0x%x] opcode= 0x%x 0x%x" 514 " 0x%x 0x%lx\n", i, off, cmd->opcode, 515 cmd->word0, cmd->word1, cmd->addr); 516 off = MOD_INC(off, sizeof(struct amdvi_cmd), softc->cmd_max); 517 } 518 } 519 520 static int 521 amdvi_init_event(struct amdvi_softc *softc) 522 { 523 struct amdvi_ctrl *ctrl; 524 525 ctrl = softc->ctrl; 526 ctrl->event.len = 8; 527 softc->event_max = 1 << ctrl->event.len; 528 softc->event = malloc(sizeof(struct amdvi_event) * 529 softc->event_max, M_AMDVI, M_WAITOK | M_ZERO); 530 if ((uintptr_t)softc->event & PAGE_MASK) { 531 device_printf(softc->dev, "Event buffer not aligned on page."); 532 return (false); 533 } 534 ctrl->event.base = vtophys(softc->event) / PAGE_SIZE; 535 536 /* Reset the pointers. */ 537 ctrl->evt_head = 0; 538 ctrl->evt_tail = 0; 539 540 return (0); 541 } 542 543 static inline void 544 amdvi_decode_evt_flag(uint16_t flag) 545 { 546 547 flag &= AMDVI_EVENT_FLAG_MASK; 548 printf(" 0x%b]\n", flag, 549 "\020" 550 "\001GN" 551 "\002NX" 552 "\003US" 553 "\004I" 554 "\005PR" 555 "\006RW" 556 "\007PE" 557 "\010RZ" 558 "\011TR" 559 ); 560 } 561 562 /* See section 2.5.4 of AMD IOMMU spec ver 2.62.*/ 563 static inline void 564 amdvi_decode_evt_flag_type(uint8_t type) 565 { 566 567 switch (AMDVI_EVENT_FLAG_TYPE(type)) { 568 case 0: 569 printf("RSVD\n"); 570 break; 571 case 1: 572 printf("Master Abort\n"); 573 break; 574 case 2: 575 printf("Target Abort\n"); 576 break; 577 case 3: 578 printf("Data Err\n"); 579 break; 580 default: 581 break; 582 } 583 } 584 585 static void 586 amdvi_decode_inv_dte_evt(uint16_t devid, uint16_t domid, uint64_t addr, 587 uint16_t flag) 588 { 589 590 printf("\t[IO_PAGE_FAULT EVT: devId:0x%x DomId:0x%x" 591 " Addr:0x%lx", 592 devid, domid, addr); 593 amdvi_decode_evt_flag(flag); 594 } 595 596 static void 597 amdvi_decode_pf_evt(uint16_t devid, uint16_t domid, uint64_t addr, 598 uint16_t flag) 599 { 600 601 printf("\t[IO_PAGE_FAULT EVT: devId:0x%x DomId:0x%x" 602 " Addr:0x%lx", 603 devid, domid, addr); 604 amdvi_decode_evt_flag(flag); 605 } 606 607 static void 608 amdvi_decode_dte_hwerr_evt(uint16_t devid, uint16_t domid, 609 uint64_t addr, uint16_t flag) 610 { 611 612 printf("\t[DEV_TAB_HW_ERR EVT: devId:0x%x DomId:0x%x" 613 " Addr:0x%lx", devid, domid, addr); 614 amdvi_decode_evt_flag(flag); 615 amdvi_decode_evt_flag_type(flag); 616 } 617 618 static void 619 amdvi_decode_page_hwerr_evt(uint16_t devid, uint16_t domid, uint64_t addr, 620 uint16_t flag) 621 { 622 623 printf("\t[PAGE_TAB_HW_ERR EVT: devId:0x%x DomId:0x%x" 624 " Addr:0x%lx", devid, domid, addr); 625 amdvi_decode_evt_flag(flag); 626 amdvi_decode_evt_flag_type(AMDVI_EVENT_FLAG_TYPE(flag)); 627 } 628 629 static void 630 amdvi_decode_evt(struct amdvi_event *evt) 631 { 632 struct amdvi_cmd *cmd; 633 634 switch (evt->opcode) { 635 case AMDVI_EVENT_INVALID_DTE: 636 amdvi_decode_inv_dte_evt(evt->devid, evt->pasid_domid, 637 evt->addr, evt->flag); 638 break; 639 640 case AMDVI_EVENT_PFAULT: 641 amdvi_decode_pf_evt(evt->devid, evt->pasid_domid, 642 evt->addr, evt->flag); 643 break; 644 645 case AMDVI_EVENT_DTE_HW_ERROR: 646 amdvi_decode_dte_hwerr_evt(evt->devid, evt->pasid_domid, 647 evt->addr, evt->flag); 648 break; 649 650 case AMDVI_EVENT_PAGE_HW_ERROR: 651 amdvi_decode_page_hwerr_evt(evt->devid, evt->pasid_domid, 652 evt->addr, evt->flag); 653 break; 654 655 case AMDVI_EVENT_ILLEGAL_CMD: 656 /* FALL THROUGH */ 657 case AMDVI_EVENT_CMD_HW_ERROR: 658 printf("\t[%s EVT]\n", (evt->opcode == AMDVI_EVENT_ILLEGAL_CMD) ? 659 "ILLEGAL CMD" : "CMD HW ERR"); 660 cmd = (struct amdvi_cmd *)PHYS_TO_DMAP(evt->addr); 661 printf("\tCMD opcode= 0x%x 0x%x 0x%x 0x%lx\n", 662 cmd->opcode, cmd->word0, cmd->word1, cmd->addr); 663 break; 664 665 case AMDVI_EVENT_IOTLB_TIMEOUT: 666 printf("\t[IOTLB_INV_TIMEOUT devid:0x%x addr:0x%lx]\n", 667 evt->devid, evt->addr); 668 break; 669 670 case AMDVI_EVENT_INVALID_DTE_REQ: 671 printf("\t[INV_DTE devid:0x%x addr:0x%lx type:0x%x tr:%d]\n", 672 evt->devid, evt->addr, evt->flag >> 9, 673 (evt->flag >> 8) & 1); 674 break; 675 676 case AMDVI_EVENT_INVALID_PPR_REQ: 677 case AMDVI_EVENT_COUNTER_ZERO: 678 printf("AMD-Vi: v2 events.\n"); 679 break; 680 681 default: 682 printf("Unsupported AMD-Vi event:%d\n", evt->opcode); 683 } 684 } 685 686 static void 687 amdvi_print_events(struct amdvi_softc *softc) 688 { 689 struct amdvi_ctrl *ctrl; 690 struct amdvi_event *event; 691 int i, size; 692 693 ctrl = softc->ctrl; 694 size = sizeof(struct amdvi_event); 695 for (i = 0; i < softc->event_max; i++) { 696 event = &softc->event[ctrl->evt_head / size]; 697 if (!event->opcode) 698 break; 699 device_printf(softc->dev, "\t[Event%d: Head:0x%x Tail:0x%x]\n", 700 i, ctrl->evt_head, ctrl->evt_tail); 701 amdvi_decode_evt(event); 702 ctrl->evt_head = MOD_INC(ctrl->evt_head, size, 703 softc->event_max); 704 } 705 } 706 707 static int 708 amdvi_init_dte(struct amdvi_softc *softc) 709 { 710 struct amdvi_ctrl *ctrl; 711 712 ctrl = softc->ctrl; 713 ctrl->dte.base = vtophys(amdvi_dte) / PAGE_SIZE; 714 ctrl->dte.size = 0x1FF; /* 2MB device table. */ 715 716 return (0); 717 } 718 719 /* 720 * Not all capabilities of IOMMU are available in ACPI IVHD flag 721 * or EFR entry, read directly from device. 722 */ 723 static int 724 amdvi_print_pci_cap(device_t dev) 725 { 726 struct amdvi_softc *softc; 727 uint32_t off, cap; 728 729 softc = device_get_softc(dev); 730 off = softc->cap_off; 731 732 /* 733 * Section 3.7.1 of IOMMU sepc rev 2.0. 734 * Read capability from device. 735 */ 736 cap = amdvi_pci_read(softc, off); 737 738 /* Make sure capability type[18:16] is 3. */ 739 KASSERT((((cap >> 16) & 0x7) == 0x3), 740 ("Not a IOMMU capability 0x%x@0x%x", cap, off)); 741 742 softc->pci_cap = cap >> 24; 743 device_printf(softc->dev, "PCI cap 0x%x@0x%x feature:%b\n", 744 cap, off, softc->pci_cap, 745 "\20\1IOTLB\2HT\3NPCache\4EFR\5CapExt"); 746 747 return (0); 748 } 749 750 static void 751 amdvi_event_intr(void *arg) 752 { 753 struct amdvi_softc *softc; 754 struct amdvi_ctrl *ctrl; 755 756 softc = (struct amdvi_softc *)arg; 757 ctrl = softc->ctrl; 758 device_printf(softc->dev, "EVT INTR %ld Status:0x%x" 759 " EVT Head:0x%x Tail:0x%x]\n", softc->event_intr_cnt++, 760 ctrl->status, ctrl->evt_head, ctrl->evt_tail); 761 printf(" [CMD Total 0x%lx] Tail:0x%x, Head:0x%x.\n", 762 softc->total_cmd, ctrl->cmd_tail, ctrl->cmd_head); 763 764 amdvi_print_events(softc); 765 ctrl->status &= AMDVI_STATUS_EV_OF | AMDVI_STATUS_EV_INTR; 766 } 767 768 static void 769 amdvi_free_evt_intr_res(device_t dev) 770 { 771 772 struct amdvi_softc *softc; 773 device_t mmio_dev; 774 775 softc = device_get_softc(dev); 776 mmio_dev = softc->pci_dev; 777 778 IVHD_TEARDOWN_INTR(mmio_dev); 779 } 780 781 static bool 782 amdvi_alloc_intr_resources(struct amdvi_softc *softc) 783 { 784 struct amdvi_ctrl *ctrl; 785 device_t dev, mmio_dev; 786 int err; 787 788 dev = softc->dev; 789 mmio_dev = softc->pci_dev; 790 791 /* Clear interrupt status bits. */ 792 ctrl = softc->ctrl; 793 ctrl->status &= AMDVI_STATUS_EV_OF | AMDVI_STATUS_EV_INTR; 794 795 err = IVHD_SETUP_INTR(mmio_dev, amdvi_event_intr, softc, "fault"); 796 if (err) 797 device_printf(dev, "Interrupt setup failed on %s\n", 798 device_get_nameunit(mmio_dev)); 799 return (err); 800 } 801 802 static void 803 amdvi_print_dev_cap(struct amdvi_softc *softc) 804 { 805 struct ivhd_dev_cfg *cfg; 806 int i; 807 808 cfg = softc->dev_cfg; 809 for (i = 0; i < softc->dev_cfg_cnt; i++) { 810 device_printf(softc->dev, "device [0x%x - 0x%x] " 811 "config:%b%s\n", cfg->start_id, cfg->end_id, 812 cfg->data, 813 "\020\001INIT\002ExtInt\003NMI" 814 "\007LINT0\010LINT1", 815 cfg->enable_ats ? "ATS enabled" : ""); 816 cfg++; 817 } 818 } 819 820 static int 821 amdvi_handle_sysctl(SYSCTL_HANDLER_ARGS) 822 { 823 struct amdvi_softc *softc; 824 int result, type, error = 0; 825 826 softc = (struct amdvi_softc *)arg1; 827 type = arg2; 828 829 switch (type) { 830 case 0: 831 result = softc->ctrl->cmd_head; 832 error = sysctl_handle_int(oidp, &result, 0, 833 req); 834 break; 835 case 1: 836 result = softc->ctrl->cmd_tail; 837 error = sysctl_handle_int(oidp, &result, 0, 838 req); 839 break; 840 case 2: 841 result = softc->ctrl->evt_head; 842 error = sysctl_handle_int(oidp, &result, 0, 843 req); 844 break; 845 case 3: 846 result = softc->ctrl->evt_tail; 847 error = sysctl_handle_int(oidp, &result, 0, 848 req); 849 break; 850 851 default: 852 device_printf(softc->dev, "Unknown sysctl:%d\n", type); 853 } 854 855 return (error); 856 } 857 858 static void 859 amdvi_add_sysctl(struct amdvi_softc *softc) 860 { 861 struct sysctl_oid_list *child; 862 struct sysctl_ctx_list *ctx; 863 device_t dev; 864 865 dev = softc->dev; 866 ctx = device_get_sysctl_ctx(dev); 867 child = SYSCTL_CHILDREN(device_get_sysctl_tree(dev)); 868 869 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "event_intr_count", CTLFLAG_RD, 870 &softc->event_intr_cnt, "Event interrupt count"); 871 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "command_count", CTLFLAG_RD, 872 &softc->total_cmd, "Command submitted count"); 873 SYSCTL_ADD_U16(ctx, child, OID_AUTO, "pci_rid", CTLFLAG_RD, 874 &softc->pci_rid, 0, "IOMMU RID"); 875 SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "command_head", 876 CTLTYPE_UINT | CTLFLAG_RD | CTLFLAG_MPSAFE, softc, 0, 877 amdvi_handle_sysctl, "IU", "Command head"); 878 SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "command_tail", 879 CTLTYPE_UINT | CTLFLAG_RD | CTLFLAG_MPSAFE, softc, 1, 880 amdvi_handle_sysctl, "IU", "Command tail"); 881 SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "event_head", 882 CTLTYPE_UINT | CTLFLAG_RD | CTLFLAG_MPSAFE, softc, 2, 883 amdvi_handle_sysctl, "IU", "Command head"); 884 SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "event_tail", 885 CTLTYPE_UINT | CTLFLAG_RD | CTLFLAG_MPSAFE, softc, 3, 886 amdvi_handle_sysctl, "IU", "Command tail"); 887 } 888 889 int 890 amdvi_setup_hw(struct amdvi_softc *softc) 891 { 892 device_t dev; 893 int status; 894 895 dev = softc->dev; 896 897 amdvi_hw_enable_iotlb(softc); 898 899 amdvi_print_dev_cap(softc); 900 901 if ((status = amdvi_print_pci_cap(dev)) != 0) { 902 device_printf(dev, "PCI capability.\n"); 903 return (status); 904 } 905 if ((status = amdvi_init_cmd(softc)) != 0) { 906 device_printf(dev, "Couldn't configure command buffer.\n"); 907 return (status); 908 } 909 if ((status = amdvi_init_event(softc)) != 0) { 910 device_printf(dev, "Couldn't configure event buffer.\n"); 911 return (status); 912 } 913 if ((status = amdvi_init_dte(softc)) != 0) { 914 device_printf(dev, "Couldn't configure device table.\n"); 915 return (status); 916 } 917 if ((status = amdvi_alloc_intr_resources(softc)) != 0) { 918 return (status); 919 } 920 amdvi_add_sysctl(softc); 921 return (0); 922 } 923 924 int 925 amdvi_teardown_hw(struct amdvi_softc *softc) 926 { 927 device_t dev; 928 929 dev = softc->dev; 930 931 /* 932 * Called after disable, h/w is stopped by now, free all the resources. 933 */ 934 amdvi_free_evt_intr_res(dev); 935 936 if (softc->cmd) 937 free(softc->cmd, M_AMDVI); 938 939 if (softc->event) 940 free(softc->event, M_AMDVI); 941 942 return (0); 943 } 944 945 /*********** bhyve interfaces *********************/ 946 static int 947 amdvi_init(void) 948 { 949 if (!ivhd_count) { 950 return (EIO); 951 } 952 if (!amdvi_enable_user && ivhd_count) { 953 printf("bhyve: Found %d AMD-Vi/IOMMU device(s), " 954 "use hw.vmm.amdvi.enable=1 to enable pass-through.\n", 955 ivhd_count); 956 return (EINVAL); 957 } 958 return (0); 959 } 960 961 static void 962 amdvi_cleanup(void) 963 { 964 /* Nothing. */ 965 } 966 967 static uint16_t 968 amdvi_domainId(void) 969 { 970 971 /* 972 * If we hit maximum domain limit, rollover leaving host 973 * domain(0). 974 * XXX: make sure that this domain is not used. 975 */ 976 if (amdvi_dom_id == AMDVI_MAX_DOMAIN) 977 amdvi_dom_id = 1; 978 979 return ((uint16_t)amdvi_dom_id++); 980 } 981 982 static void 983 amdvi_do_inv_domain(uint16_t domain_id, bool create) 984 { 985 struct amdvi_softc *softc; 986 int i; 987 988 for (i = 0; i < ivhd_count; i++) { 989 softc = device_get_softc(ivhd_devs[i]); 990 KASSERT(softc, ("softc is NULL")); 991 /* 992 * If not present pages are cached, invalidate page after 993 * creating domain. 994 */ 995 #if 0 996 if (create && ((softc->pci_cap & AMDVI_PCI_CAP_NPCACHE) == 0)) 997 continue; 998 #endif 999 amdvi_inv_domain(softc, domain_id); 1000 amdvi_wait(softc); 1001 } 1002 } 1003 1004 static void * 1005 amdvi_create_domain(vm_paddr_t maxaddr) 1006 { 1007 struct amdvi_domain *dom; 1008 1009 dom = malloc(sizeof(struct amdvi_domain), M_AMDVI, M_ZERO | M_WAITOK); 1010 dom->id = amdvi_domainId(); 1011 //dom->maxaddr = maxaddr; 1012 #ifdef AMDVI_DEBUG_CMD 1013 printf("Created domain #%d\n", dom->id); 1014 #endif 1015 /* 1016 * Host domain(#0) don't create translation table. 1017 */ 1018 if (dom->id || amdvi_host_ptp) 1019 dom->ptp = malloc(PAGE_SIZE, M_AMDVI, M_WAITOK | M_ZERO); 1020 1021 dom->ptp_level = amdvi_ptp_level; 1022 1023 amdvi_do_inv_domain(dom->id, true); 1024 SLIST_INSERT_HEAD(&dom_head, dom, next); 1025 1026 return (dom); 1027 } 1028 1029 static void 1030 amdvi_free_ptp(uint64_t *ptp, int level) 1031 { 1032 int i; 1033 1034 if (level < 1) 1035 return; 1036 1037 for (i = 0; i < NPTEPG ; i++) { 1038 if ((ptp[i] & AMDVI_PT_PRESENT) == 0) 1039 continue; 1040 /* XXX: Add super-page or PTE mapping > 4KB. */ 1041 #ifdef notyet 1042 /* Super-page mapping. */ 1043 if (AMDVI_PD_SUPER(ptp[i])) 1044 continue; 1045 #endif 1046 1047 amdvi_free_ptp((uint64_t *)PHYS_TO_DMAP(ptp[i] 1048 & AMDVI_PT_MASK), level - 1); 1049 } 1050 1051 free(ptp, M_AMDVI); 1052 } 1053 1054 static void 1055 amdvi_destroy_domain(void *arg) 1056 { 1057 struct amdvi_domain *domain; 1058 1059 domain = (struct amdvi_domain *)arg; 1060 KASSERT(domain, ("domain is NULL")); 1061 #ifdef AMDVI_DEBUG_CMD 1062 printf("Destroying domain %d\n", domain->id); 1063 #endif 1064 if (domain->ptp) 1065 amdvi_free_ptp(domain->ptp, domain->ptp_level); 1066 1067 amdvi_do_inv_domain(domain->id, false); 1068 SLIST_REMOVE(&dom_head, domain, amdvi_domain, next); 1069 free(domain, M_AMDVI); 1070 } 1071 1072 static uint64_t 1073 amdvi_set_pt(uint64_t *pt, int level, vm_paddr_t gpa, 1074 vm_paddr_t hpa, uint64_t pg_size, bool create) 1075 { 1076 uint64_t *page, pa; 1077 int shift, index; 1078 const int PT_SHIFT = 9; 1079 const int PT_INDEX_MASK = (1 << PT_SHIFT) - 1; /* Based on PT_SHIFT */ 1080 1081 if (!pg_size) 1082 return (0); 1083 1084 if (hpa & (pg_size - 1)) { 1085 printf("HPA is not size aligned.\n"); 1086 return (0); 1087 } 1088 if (gpa & (pg_size - 1)) { 1089 printf("HPA is not size aligned.\n"); 1090 return (0); 1091 } 1092 shift = PML4SHIFT; 1093 while ((shift > PAGE_SHIFT) && (pg_size < (1UL << shift))) { 1094 index = (gpa >> shift) & PT_INDEX_MASK; 1095 1096 if ((pt[index] == 0) && create) { 1097 page = malloc(PAGE_SIZE, M_AMDVI, M_WAITOK | M_ZERO); 1098 pa = vtophys(page); 1099 pt[index] = pa | AMDVI_PT_PRESENT | AMDVI_PT_RW | 1100 ((level - 1) << AMDVI_PD_LEVEL_SHIFT); 1101 } 1102 #ifdef AMDVI_DEBUG_PTE 1103 if ((gpa % 0x1000000) == 0) 1104 printf("[level%d, shift = %d]PTE:0x%lx\n", 1105 level, shift, pt[index]); 1106 #endif 1107 #define PTE2PA(x) ((uint64_t)(x) & AMDVI_PT_MASK) 1108 pa = PTE2PA(pt[index]); 1109 pt = (uint64_t *)PHYS_TO_DMAP(pa); 1110 shift -= PT_SHIFT; 1111 level--; 1112 } 1113 1114 /* Leaf entry. */ 1115 index = (gpa >> shift) & PT_INDEX_MASK; 1116 1117 if (create) { 1118 pt[index] = hpa | AMDVI_PT_RW | AMDVI_PT_PRESENT; 1119 } else 1120 pt[index] = 0; 1121 1122 #ifdef AMDVI_DEBUG_PTE 1123 if ((gpa % 0x1000000) == 0) 1124 printf("[Last level%d, shift = %d]PTE:0x%lx\n", 1125 level, shift, pt[index]); 1126 #endif 1127 return (1ULL << shift); 1128 } 1129 1130 static uint64_t 1131 amdvi_update_mapping(struct amdvi_domain *domain, vm_paddr_t gpa, 1132 vm_paddr_t hpa, uint64_t size, bool create) 1133 { 1134 uint64_t mapped, *ptp, len; 1135 int level; 1136 1137 KASSERT(domain, ("domain is NULL")); 1138 level = domain->ptp_level; 1139 KASSERT(level, ("Page table level is 0")); 1140 1141 ptp = domain->ptp; 1142 KASSERT(ptp, ("PTP is NULL")); 1143 mapped = 0; 1144 while (mapped < size) { 1145 len = amdvi_set_pt(ptp, level, gpa + mapped, hpa + mapped, 1146 PAGE_SIZE, create); 1147 if (!len) { 1148 printf("Error: Couldn't map HPA:0x%lx GPA:0x%lx\n", 1149 hpa, gpa); 1150 return (0); 1151 } 1152 mapped += len; 1153 } 1154 1155 return (mapped); 1156 } 1157 1158 static uint64_t 1159 amdvi_create_mapping(void *arg, vm_paddr_t gpa, vm_paddr_t hpa, 1160 uint64_t len) 1161 { 1162 struct amdvi_domain *domain; 1163 1164 domain = (struct amdvi_domain *)arg; 1165 1166 if (domain->id && !domain->ptp) { 1167 printf("ptp is NULL"); 1168 return (-1); 1169 } 1170 1171 /* 1172 * If host domain is created w/o page table, skip IOMMU page 1173 * table set-up. 1174 */ 1175 if (domain->ptp) 1176 return (amdvi_update_mapping(domain, gpa, hpa, len, true)); 1177 else 1178 return (len); 1179 } 1180 1181 static uint64_t 1182 amdvi_remove_mapping(void *arg, vm_paddr_t gpa, uint64_t len) 1183 { 1184 struct amdvi_domain *domain; 1185 1186 domain = (struct amdvi_domain *)arg; 1187 /* 1188 * If host domain is created w/o page table, skip IOMMU page 1189 * table set-up. 1190 */ 1191 if (domain->ptp) 1192 return (amdvi_update_mapping(domain, gpa, 0, len, false)); 1193 return 1194 (len); 1195 } 1196 1197 static struct amdvi_softc * 1198 amdvi_find_iommu(uint16_t devid) 1199 { 1200 struct amdvi_softc *softc; 1201 int i, j; 1202 1203 for (i = 0; i < ivhd_count; i++) { 1204 softc = device_get_softc(ivhd_devs[i]); 1205 for (j = 0; j < softc->dev_cfg_cnt; j++) 1206 if ((devid >= softc->dev_cfg[j].start_id) && 1207 (devid <= softc->dev_cfg[j].end_id)) 1208 return (softc); 1209 } 1210 1211 return (NULL); 1212 } 1213 1214 /* 1215 * Set-up device table entry. 1216 * IOMMU spec Rev 2.0, section 3.2.2.2, some of the fields must 1217 * be set concurrently, e.g. read and write bits. 1218 */ 1219 static void 1220 amdvi_set_dte(struct amdvi_domain *domain, struct amdvi_softc *softc, 1221 uint16_t devid, bool enable) 1222 { 1223 struct amdvi_dte* temp; 1224 1225 KASSERT(domain, ("domain is NULL for pci_rid:0x%x\n", devid)); 1226 KASSERT(softc, ("softc is NULL for pci_rid:0x%x\n", devid)); 1227 1228 temp = &amdvi_dte[devid]; 1229 1230 #ifdef AMDVI_ATS_ENABLE 1231 /* If IOMMU and device support IOTLB, enable it. */ 1232 if (amdvi_dev_support_iotlb(softc, devid) && softc->iotlb) 1233 temp->iotlb_enable = 1; 1234 #endif 1235 1236 /* Avoid duplicate I/O faults. */ 1237 temp->sup_second_io_fault = 1; 1238 temp->sup_all_io_fault = amdvi_disable_io_fault; 1239 1240 temp->dt_valid = 1; 1241 temp->domain_id = domain->id; 1242 1243 if (enable) { 1244 if (domain->ptp) { 1245 temp->pt_base = vtophys(domain->ptp) >> 12; 1246 temp->pt_level = amdvi_ptp_level; 1247 } 1248 /* 1249 * XXX: Page table valid[TV] bit must be set even if host domain 1250 * page tables are not enabled. 1251 */ 1252 temp->pt_valid = 1; 1253 temp->read_allow = 1; 1254 temp->write_allow = 1; 1255 } 1256 } 1257 1258 static void 1259 amdvi_inv_device(struct amdvi_softc *softc, uint16_t devid) 1260 { 1261 KASSERT(softc, ("softc is NULL")); 1262 1263 amdvi_cmd_inv_dte(softc, devid); 1264 #ifdef AMDVI_ATS_ENABLE 1265 if (amdvi_dev_support_iotlb(softc, devid)) 1266 amdvi_cmd_inv_iotlb(softc, devid); 1267 #endif 1268 amdvi_wait(softc); 1269 } 1270 1271 static void 1272 amdvi_add_device(void *arg, uint16_t devid) 1273 { 1274 struct amdvi_domain *domain; 1275 struct amdvi_softc *softc; 1276 1277 domain = (struct amdvi_domain *)arg; 1278 KASSERT(domain != NULL, ("domain is NULL")); 1279 #ifdef AMDVI_DEBUG_CMD 1280 printf("Assigning device(%d.%d.%d) to domain:%d\n", 1281 RID2PCI_STR(devid), domain->id); 1282 #endif 1283 softc = amdvi_find_iommu(devid); 1284 if (softc == NULL) 1285 return; 1286 amdvi_set_dte(domain, softc, devid, true); 1287 amdvi_inv_device(softc, devid); 1288 } 1289 1290 static void 1291 amdvi_remove_device(void *arg, uint16_t devid) 1292 { 1293 struct amdvi_domain *domain; 1294 struct amdvi_softc *softc; 1295 1296 domain = (struct amdvi_domain *)arg; 1297 #ifdef AMDVI_DEBUG_CMD 1298 printf("Remove device(0x%x) from domain:%d\n", 1299 devid, domain->id); 1300 #endif 1301 softc = amdvi_find_iommu(devid); 1302 if (softc == NULL) 1303 return; 1304 amdvi_set_dte(domain, softc, devid, false); 1305 amdvi_inv_device(softc, devid); 1306 } 1307 1308 static void 1309 amdvi_enable(void) 1310 { 1311 struct amdvi_ctrl *ctrl; 1312 struct amdvi_softc *softc; 1313 uint64_t val; 1314 int i; 1315 1316 for (i = 0; i < ivhd_count; i++) { 1317 softc = device_get_softc(ivhd_devs[i]); 1318 KASSERT(softc, ("softc is NULL\n")); 1319 ctrl = softc->ctrl; 1320 KASSERT(ctrl, ("ctrl is NULL\n")); 1321 1322 val = ( AMDVI_CTRL_EN | 1323 AMDVI_CTRL_CMD | 1324 AMDVI_CTRL_ELOG | 1325 AMDVI_CTRL_ELOGINT | 1326 AMDVI_CTRL_INV_TO_1S); 1327 1328 if (softc->ivhd_flag & IVHD_FLAG_COH) 1329 val |= AMDVI_CTRL_COH; 1330 if (softc->ivhd_flag & IVHD_FLAG_HTT) 1331 val |= AMDVI_CTRL_HTT; 1332 if (softc->ivhd_flag & IVHD_FLAG_RPPW) 1333 val |= AMDVI_CTRL_RPPW; 1334 if (softc->ivhd_flag & IVHD_FLAG_PPW) 1335 val |= AMDVI_CTRL_PPW; 1336 if (softc->ivhd_flag & IVHD_FLAG_ISOC) 1337 val |= AMDVI_CTRL_ISOC; 1338 1339 ctrl->control = val; 1340 } 1341 } 1342 1343 static void 1344 amdvi_disable(void) 1345 { 1346 struct amdvi_ctrl *ctrl; 1347 struct amdvi_softc *softc; 1348 int i; 1349 1350 for (i = 0; i < ivhd_count; i++) { 1351 softc = device_get_softc(ivhd_devs[i]); 1352 KASSERT(softc, ("softc is NULL\n")); 1353 ctrl = softc->ctrl; 1354 KASSERT(ctrl, ("ctrl is NULL\n")); 1355 1356 ctrl->control = 0; 1357 } 1358 } 1359 1360 static void 1361 amdvi_invalidate_tlb(void *arg) 1362 { 1363 struct amdvi_domain *domain; 1364 1365 domain = (struct amdvi_domain *)arg; 1366 KASSERT(domain, ("domain is NULL")); 1367 amdvi_do_inv_domain(domain->id, false); 1368 } 1369 1370 const struct iommu_ops iommu_ops_amd = { 1371 .init = amdvi_init, 1372 .cleanup = amdvi_cleanup, 1373 .enable = amdvi_enable, 1374 .disable = amdvi_disable, 1375 .create_domain = amdvi_create_domain, 1376 .destroy_domain = amdvi_destroy_domain, 1377 .create_mapping = amdvi_create_mapping, 1378 .remove_mapping = amdvi_remove_mapping, 1379 .add_device = amdvi_add_device, 1380 .remove_device = amdvi_remove_device, 1381 .invalidate_tlb = amdvi_invalidate_tlb 1382 }; 1383