1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 22 /* 23 * Copyright (c) 2009, 2010, Oracle and/or its affiliates. All rights reserved. 24 */ 25 26 #include <sys/sunddi.h> 27 #include <sys/sunndi.h> 28 #include <sys/acpi/acpi.h> 29 #include <sys/acpica.h> 30 #include <sys/amd_iommu.h> 31 #include <sys/bootconf.h> 32 #include <sys/sysmacros.h> 33 #include <sys/ddidmareq.h> 34 35 #include "amd_iommu_impl.h" 36 #include "amd_iommu_acpi.h" 37 #include "amd_iommu_page_tables.h" 38 39 ddi_dma_attr_t amd_iommu_pgtable_dma_attr = { 40 DMA_ATTR_V0, 41 0U, /* dma_attr_addr_lo */ 42 0xffffffffffffffffULL, /* dma_attr_addr_hi */ 43 0xffffffffU, /* dma_attr_count_max */ 44 (uint64_t)4096, /* dma_attr_align */ 45 1, /* dma_attr_burstsizes */ 46 64, /* dma_attr_minxfer */ 47 0xffffffffU, /* dma_attr_maxxfer */ 48 0xffffffffU, /* dma_attr_seg */ 49 1, /* dma_attr_sgllen, variable */ 50 64, /* dma_attr_granular */ 51 0 /* dma_attr_flags */ 52 }; 53 54 static amd_iommu_domain_t **amd_iommu_domain_table; 55 56 static struct { 57 int f_count; 58 amd_iommu_page_table_t *f_list; 59 } amd_iommu_pgtable_freelist; 60 int amd_iommu_no_pgtable_freelist; 61 62 /*ARGSUSED*/ 63 static int 64 amd_iommu_get_src_bdf(amd_iommu_t *iommu, int32_t bdf, int32_t *src_bdfp) 65 { 66 amd_iommu_acpi_ivhd_t *hinfop; 67 68 hinfop = amd_iommu_lookup_ivhd(bdf); 69 if (hinfop == NULL) { 70 if (bdf == -1) { 71 *src_bdfp = bdf; 72 } else { 73 cmn_err(CE_WARN, "No IVHD entry for 0x%x", bdf); 74 return (DDI_FAILURE); 75 } 76 } else if (hinfop->ach_src_deviceid == -1) { 77 *src_bdfp = bdf; 78 } else { 79 *src_bdfp = hinfop->ach_src_deviceid; 80 } 81 82 return (DDI_SUCCESS); 83 } 84 85 /*ARGSUSED*/ 86 static int 87 amd_iommu_get_domain(amd_iommu_t *iommu, dev_info_t *rdip, int alias, 88 uint16_t deviceid, domain_id_t *domainid, const char *path) 89 { 90 const char *f = "amd_iommu_get_domain"; 91 92 *domainid = AMD_IOMMU_INVALID_DOMAIN; 93 94 ASSERT(strcmp(ddi_driver_name(rdip), "agpgart") != 0); 95 96 switch (deviceid) { 97 case AMD_IOMMU_INVALID_DOMAIN: 98 case AMD_IOMMU_IDENTITY_DOMAIN: 99 case AMD_IOMMU_PASSTHRU_DOMAIN: 100 case AMD_IOMMU_SYS_DOMAIN: 101 *domainid = AMD_IOMMU_SYS_DOMAIN; 102 break; 103 default: 104 *domainid = deviceid; 105 break; 106 } 107 108 if (amd_iommu_debug & AMD_IOMMU_DEBUG_DEVTBL) { 109 cmn_err(CE_NOTE, "%s: domainid for %s = %d", 110 f, path, *domainid); 111 } 112 113 return (DDI_SUCCESS); 114 } 115 116 static uint16_t 117 hash_domain(domain_id_t domainid) 118 { 119 return (domainid % AMD_IOMMU_DOMAIN_HASH_SZ); 120 } 121 122 /*ARGSUSED*/ 123 void 124 amd_iommu_init_page_tables(amd_iommu_t *iommu) 125 { 126 amd_iommu_domain_table = kmem_zalloc( 127 sizeof (amd_iommu_domain_t *) * AMD_IOMMU_DOMAIN_HASH_SZ, KM_SLEEP); 128 } 129 130 /*ARGSUSED*/ 131 void 132 amd_iommu_fini_page_tables(amd_iommu_t *iommu) 133 { 134 if (amd_iommu_domain_table) { 135 kmem_free(amd_iommu_domain_table, 136 sizeof (amd_iommu_domain_t *) * AMD_IOMMU_DOMAIN_HASH_SZ); 137 amd_iommu_domain_table = NULL; 138 } 139 } 140 141 static amd_iommu_domain_t * 142 amd_iommu_lookup_domain(amd_iommu_t *iommu, domain_id_t domainid, 143 map_type_t type, int km_flags) 144 { 145 uint16_t idx; 146 amd_iommu_domain_t *dp; 147 char name[AMD_IOMMU_VMEM_NAMELEN+1]; 148 149 ASSERT(amd_iommu_domain_table); 150 151 idx = hash_domain(domainid); 152 153 for (dp = amd_iommu_domain_table[idx]; dp; dp = dp->d_next) { 154 if (dp->d_domainid == domainid) 155 return (dp); 156 } 157 158 ASSERT(type != AMD_IOMMU_INVALID_MAP); 159 160 dp = kmem_zalloc(sizeof (*dp), km_flags); 161 if (dp == NULL) 162 return (NULL); 163 dp->d_domainid = domainid; 164 dp->d_pgtable_root_4K = 0; /* make this explicit */ 165 166 if (type == AMD_IOMMU_VMEM_MAP) { 167 uint64_t base; 168 uint64_t size; 169 (void) snprintf(name, sizeof (name), "dvma_idx%d_domain%d", 170 iommu->aiomt_idx, domainid); 171 base = MMU_PAGESIZE; 172 size = AMD_IOMMU_SIZE_4G - MMU_PAGESIZE; 173 dp->d_vmem = vmem_create(name, (void *)(uintptr_t)base, size, 174 MMU_PAGESIZE, NULL, NULL, NULL, 0, 175 km_flags == KM_SLEEP ? VM_SLEEP : VM_NOSLEEP); 176 if (dp->d_vmem == NULL) { 177 kmem_free(dp, sizeof (*dp)); 178 return (NULL); 179 } 180 } else { 181 dp->d_vmem = NULL; 182 } 183 184 dp->d_next = amd_iommu_domain_table[idx]; 185 dp->d_prev = NULL; 186 amd_iommu_domain_table[idx] = dp; 187 if (dp->d_next) 188 dp->d_next->d_prev = dp; 189 dp->d_ref = 0; 190 191 192 return (dp); 193 } 194 195 static void 196 amd_iommu_teardown_domain(amd_iommu_t *iommu, amd_iommu_domain_t *dp) 197 { 198 uint16_t idx; 199 int flags; 200 amd_iommu_cmdargs_t cmdargs = {0}; 201 domain_id_t domainid = dp->d_domainid; 202 const char *f = "amd_iommu_teardown_domain"; 203 204 ASSERT(dp->d_ref == 0); 205 206 idx = hash_domain(dp->d_domainid); 207 208 if (dp->d_prev == NULL) 209 amd_iommu_domain_table[idx] = dp->d_next; 210 else 211 dp->d_prev->d_next = dp->d_next; 212 213 if (dp->d_next) 214 dp->d_next->d_prev = dp->d_prev; 215 216 if (dp->d_vmem != NULL) { 217 vmem_destroy(dp->d_vmem); 218 dp->d_vmem = NULL; 219 } 220 221 kmem_free(dp, sizeof (*dp)); 222 223 cmdargs.ca_domainid = (uint16_t)domainid; 224 cmdargs.ca_addr = (uintptr_t)0x7FFFFFFFFFFFF000; 225 flags = AMD_IOMMU_CMD_FLAGS_PAGE_PDE_INVAL | 226 AMD_IOMMU_CMD_FLAGS_PAGE_INVAL_S; 227 228 if (amd_iommu_cmd(iommu, AMD_IOMMU_CMD_INVAL_IOMMU_PAGES, 229 &cmdargs, flags, 0) != DDI_SUCCESS) { 230 cmn_err(CE_WARN, "%s: idx=%d: domainid=%d" 231 "Failed to invalidate domain in IOMMU HW cache", 232 f, iommu->aiomt_idx, cmdargs.ca_domainid); 233 } 234 } 235 236 static int 237 amd_iommu_get_deviceid(amd_iommu_t *iommu, dev_info_t *rdip, int32_t *deviceid, 238 int *aliasp, const char *path) 239 { 240 int bus = -1; 241 int device = -1; 242 int func = -1; 243 uint16_t bdf; 244 int32_t src_bdf; 245 dev_info_t *idip = iommu->aiomt_dip; 246 const char *driver = ddi_driver_name(idip); 247 int instance = ddi_get_instance(idip); 248 dev_info_t *pci_dip; 249 const char *f = "amd_iommu_get_deviceid"; 250 251 /* be conservative. Always assume an alias */ 252 *aliasp = 1; 253 *deviceid = 0; 254 255 /* Check for special special devices (rdip == NULL) */ 256 if (rdip == NULL) { 257 if (amd_iommu_get_src_bdf(iommu, -1, &src_bdf) != DDI_SUCCESS) { 258 cmn_err(CE_WARN, 259 "%s: %s%d: idx=%d, failed to get SRC BDF " 260 "for special-device", 261 f, driver, instance, iommu->aiomt_idx); 262 return (DDI_DMA_NOMAPPING); 263 } 264 *deviceid = src_bdf; 265 *aliasp = 1; 266 return (DDI_SUCCESS); 267 } 268 269 if (amd_iommu_debug & AMD_IOMMU_DEBUG_DEVTBL) { 270 cmn_err(CE_NOTE, "%s: attempting to get deviceid for %s", 271 f, path); 272 } 273 274 pci_dip = amd_iommu_pci_dip(rdip, path); 275 if (pci_dip == NULL) { 276 cmn_err(CE_WARN, "%s: %s%d: idx = %d, failed to get PCI dip " 277 "for rdip=%p, path = %s", 278 f, driver, instance, iommu->aiomt_idx, (void *)rdip, 279 path); 280 return (DDI_DMA_NOMAPPING); 281 } 282 283 if (acpica_get_bdf(pci_dip, &bus, &device, &func) != DDI_SUCCESS) { 284 ndi_rele_devi(pci_dip); 285 cmn_err(CE_WARN, "%s: %s%d: idx=%d, failed to get BDF for " 286 "PCI dip (%p). rdip path = %s", 287 f, driver, instance, iommu->aiomt_idx, 288 (void *)pci_dip, path); 289 return (DDI_DMA_NOMAPPING); 290 } 291 292 ndi_rele_devi(pci_dip); 293 294 if (bus > UINT8_MAX || bus < 0 || 295 device > UINT8_MAX || device < 0 || 296 func > UINT8_MAX || func < 0) { 297 cmn_err(CE_WARN, "%s: %s%d: idx=%d, invalid BDF(%d,%d,%d) " 298 "for PCI dip (%p). rdip path = %s", f, driver, instance, 299 iommu->aiomt_idx, 300 bus, device, func, 301 (void *)pci_dip, path); 302 return (DDI_DMA_NOMAPPING); 303 } 304 305 bdf = ((uint8_t)bus << 8) | ((uint8_t)device << 3) | (uint8_t)func; 306 307 if (amd_iommu_get_src_bdf(iommu, bdf, &src_bdf) != DDI_SUCCESS) { 308 cmn_err(CE_WARN, "%s: %s%d: idx=%d, failed to get SRC BDF " 309 "for PCI dip (%p) rdip path = %s.", 310 f, driver, instance, iommu->aiomt_idx, (void *)pci_dip, 311 path); 312 return (DDI_DMA_NOMAPPING); 313 } 314 315 if (amd_iommu_debug & AMD_IOMMU_DEBUG_DEVTBL) { 316 cmn_err(CE_NOTE, "%s: Deviceid = %u for path = %s", 317 f, src_bdf, path); 318 } 319 320 *deviceid = src_bdf; 321 *aliasp = (src_bdf != bdf); 322 323 return (DDI_SUCCESS); 324 } 325 326 /*ARGSUSED*/ 327 static int 328 init_devtbl(amd_iommu_t *iommu, uint64_t *devtbl_entry, domain_id_t domainid, 329 amd_iommu_domain_t *dp) 330 { 331 uint64_t entry[4] = {0}; 332 int i; 333 334 /* If already passthru, don't touch */ 335 if (AMD_IOMMU_REG_GET64(&(devtbl_entry[0]), AMD_IOMMU_DEVTBL_V) == 0 && 336 AMD_IOMMU_REG_GET64(&(devtbl_entry[0]), AMD_IOMMU_DEVTBL_TV) == 0) { 337 return (0); 338 } 339 340 if (AMD_IOMMU_REG_GET64(&(devtbl_entry[0]), AMD_IOMMU_DEVTBL_V) == 1 && 341 AMD_IOMMU_REG_GET64(&(devtbl_entry[0]), AMD_IOMMU_DEVTBL_TV) == 1) { 342 343 ASSERT(dp->d_pgtable_root_4K == 344 AMD_IOMMU_REG_GET64(&(devtbl_entry[0]), 345 AMD_IOMMU_DEVTBL_ROOT_PGTBL)); 346 347 ASSERT(dp->d_domainid == AMD_IOMMU_REG_GET64(&(devtbl_entry[1]), 348 AMD_IOMMU_DEVTBL_DOMAINID)); 349 350 return (0); 351 } 352 353 /* New devtbl entry for this domain. Bump up the domain ref-count */ 354 dp->d_ref++; 355 356 entry[3] = 0; 357 entry[2] = 0; 358 AMD_IOMMU_REG_SET64(&(entry[1]), AMD_IOMMU_DEVTBL_EX, 1); 359 AMD_IOMMU_REG_SET64(&(entry[1]), AMD_IOMMU_DEVTBL_SD, 0); 360 AMD_IOMMU_REG_SET64(&(entry[1]), AMD_IOMMU_DEVTBL_CACHE, 0); 361 AMD_IOMMU_REG_SET64(&(entry[1]), AMD_IOMMU_DEVTBL_IOCTL, 1); 362 AMD_IOMMU_REG_SET64(&(entry[1]), AMD_IOMMU_DEVTBL_SA, 0); 363 AMD_IOMMU_REG_SET64(&(entry[1]), AMD_IOMMU_DEVTBL_SE, 1); 364 AMD_IOMMU_REG_SET64(&(entry[1]), AMD_IOMMU_DEVTBL_DOMAINID, 365 (uint16_t)domainid); 366 AMD_IOMMU_REG_SET64(&(entry[0]), AMD_IOMMU_DEVTBL_IW, 1); 367 AMD_IOMMU_REG_SET64(&(entry[0]), AMD_IOMMU_DEVTBL_IR, 1); 368 AMD_IOMMU_REG_SET64(&(entry[0]), AMD_IOMMU_DEVTBL_ROOT_PGTBL, 369 dp->d_pgtable_root_4K); 370 AMD_IOMMU_REG_SET64(&(entry[0]), AMD_IOMMU_DEVTBL_PG_MODE, 371 AMD_IOMMU_PGTABLE_MAXLEVEL); 372 AMD_IOMMU_REG_SET64(&(entry[0]), AMD_IOMMU_DEVTBL_TV, 373 domainid == AMD_IOMMU_PASSTHRU_DOMAIN ? 0 : 1); 374 AMD_IOMMU_REG_SET64(&(entry[0]), AMD_IOMMU_DEVTBL_V, 375 domainid == AMD_IOMMU_PASSTHRU_DOMAIN ? 0 : 1); 376 377 for (i = 1; i < 4; i++) { 378 devtbl_entry[i] = entry[i]; 379 } 380 devtbl_entry[0] = entry[0]; 381 382 /* we did an actual init */ 383 return (1); 384 } 385 386 void 387 amd_iommu_set_passthru(amd_iommu_t *iommu, dev_info_t *rdip) 388 { 389 int32_t deviceid; 390 int alias; 391 uint64_t *devtbl_entry; 392 amd_iommu_cmdargs_t cmdargs = {0}; 393 char *path; 394 int pathfree; 395 int V; 396 int TV; 397 int instance; 398 const char *driver; 399 const char *f = "amd_iommu_set_passthru"; 400 401 if (rdip) { 402 driver = ddi_driver_name(rdip); 403 instance = ddi_get_instance(rdip); 404 } else { 405 driver = "special-device"; 406 instance = 0; 407 } 408 409 path = kmem_alloc(MAXPATHLEN, KM_NOSLEEP); 410 if (path) { 411 if (rdip) 412 (void) ddi_pathname(rdip, path); 413 else 414 (void) strcpy(path, "special-device"); 415 pathfree = 1; 416 } else { 417 pathfree = 0; 418 path = "<path-mem-alloc-failed>"; 419 } 420 421 if (amd_iommu_get_deviceid(iommu, rdip, &deviceid, &alias, path) 422 != DDI_SUCCESS) { 423 cmn_err(CE_WARN, "%s: %s%d: idx=%d: rdip=%p. " 424 "Failed to get device ID for device %s.", f, driver, 425 instance, 426 iommu->aiomt_idx, (void *)rdip, path); 427 goto out; 428 } 429 430 /* No deviceid */ 431 if (deviceid == -1) { 432 goto out; 433 } 434 435 if ((deviceid + 1) * AMD_IOMMU_DEVTBL_ENTRY_SZ > 436 iommu->aiomt_devtbl_sz) { 437 cmn_err(CE_WARN, "%s: %s%d: IOMMU idx=%d, deviceid (%u) " 438 "for rdip (%p) exceeds device table size (%u), path=%s", 439 f, driver, 440 instance, iommu->aiomt_idx, deviceid, (void *)rdip, 441 iommu->aiomt_devtbl_sz, path); 442 goto out; 443 } 444 445 /*LINTED*/ 446 devtbl_entry = (uint64_t *)&iommu->aiomt_devtbl 447 [deviceid * AMD_IOMMU_DEVTBL_ENTRY_SZ]; 448 449 V = AMD_IOMMU_REG_GET64(&(devtbl_entry[0]), AMD_IOMMU_DEVTBL_V); 450 TV = AMD_IOMMU_REG_GET64(&(devtbl_entry[0]), AMD_IOMMU_DEVTBL_TV); 451 452 /* Already passthru */ 453 if (V == 0 && TV == 0) { 454 goto out; 455 } 456 457 /* Existing translations */ 458 if (V == 1 && TV == 1) { 459 goto out; 460 } 461 462 /* Invalid setting */ 463 if (V == 0 && TV == 1) { 464 goto out; 465 } 466 467 AMD_IOMMU_REG_SET64(&(devtbl_entry[0]), AMD_IOMMU_DEVTBL_V, 0); 468 469 cmdargs.ca_deviceid = (uint16_t)deviceid; 470 (void) amd_iommu_cmd(iommu, AMD_IOMMU_CMD_INVAL_DEVTAB_ENTRY, 471 &cmdargs, 0, 0); 472 473 out: 474 if (pathfree) 475 kmem_free(path, MAXPATHLEN); 476 } 477 478 static int 479 amd_iommu_set_devtbl_entry(amd_iommu_t *iommu, dev_info_t *rdip, 480 domain_id_t domainid, uint16_t deviceid, amd_iommu_domain_t *dp, 481 const char *path) 482 { 483 uint64_t *devtbl_entry; 484 amd_iommu_cmdargs_t cmdargs = {0}; 485 int error, flags; 486 dev_info_t *idip = iommu->aiomt_dip; 487 const char *driver = ddi_driver_name(idip); 488 int instance = ddi_get_instance(idip); 489 const char *f = "amd_iommu_set_devtbl_entry"; 490 491 if (amd_iommu_debug & AMD_IOMMU_DEBUG_DEVTBL) { 492 cmn_err(CE_NOTE, "%s: attempting to set devtbl entry for %s", 493 f, path); 494 } 495 496 if ((deviceid + 1) * AMD_IOMMU_DEVTBL_ENTRY_SZ > 497 iommu->aiomt_devtbl_sz) { 498 cmn_err(CE_WARN, "%s: %s%d: IOMMU idx=%d, deviceid (%u) " 499 "for rdip (%p) exceeds device table size (%u), path=%s", 500 f, driver, 501 instance, iommu->aiomt_idx, deviceid, (void *)rdip, 502 iommu->aiomt_devtbl_sz, path); 503 return (DDI_DMA_NOMAPPING); 504 } 505 506 /*LINTED*/ 507 devtbl_entry = (uint64_t *)&iommu->aiomt_devtbl 508 [deviceid * AMD_IOMMU_DEVTBL_ENTRY_SZ]; 509 510 if (amd_iommu_debug & AMD_IOMMU_DEBUG_DEVTBL) { 511 cmn_err(CE_NOTE, "%s: deviceid=%u devtbl entry (%p) for %s", 512 f, deviceid, (void *)(uintptr_t)(*devtbl_entry), path); 513 } 514 515 /* 516 * Flush internal caches, need to do this if we came up from 517 * fast boot 518 */ 519 cmdargs.ca_deviceid = deviceid; 520 error = amd_iommu_cmd(iommu, AMD_IOMMU_CMD_INVAL_DEVTAB_ENTRY, 521 &cmdargs, 0, 0); 522 if (error != DDI_SUCCESS) { 523 cmn_err(CE_WARN, "%s: idx=%d: deviceid=%d" 524 "Failed to invalidate domain in IOMMU HW cache", 525 f, iommu->aiomt_idx, deviceid); 526 return (error); 527 } 528 529 cmdargs.ca_domainid = (uint16_t)domainid; 530 cmdargs.ca_addr = (uintptr_t)0x7FFFFFFFFFFFF000; 531 flags = AMD_IOMMU_CMD_FLAGS_PAGE_PDE_INVAL | 532 AMD_IOMMU_CMD_FLAGS_PAGE_INVAL_S; 533 534 error = amd_iommu_cmd(iommu, AMD_IOMMU_CMD_INVAL_IOMMU_PAGES, 535 &cmdargs, flags, 0); 536 if (error != DDI_SUCCESS) { 537 cmn_err(CE_WARN, "%s: idx=%d: domainid=%d" 538 "Failed to invalidate translations in IOMMU HW cache", 539 f, iommu->aiomt_idx, cmdargs.ca_domainid); 540 return (error); 541 } 542 543 /* Initialize device table entry */ 544 if (init_devtbl(iommu, devtbl_entry, domainid, dp)) { 545 cmdargs.ca_deviceid = deviceid; 546 error = amd_iommu_cmd(iommu, AMD_IOMMU_CMD_INVAL_DEVTAB_ENTRY, 547 &cmdargs, 0, 0); 548 } 549 550 return (error); 551 } 552 553 int 554 amd_iommu_clear_devtbl_entry(amd_iommu_t *iommu, dev_info_t *rdip, 555 domain_id_t domainid, uint16_t deviceid, amd_iommu_domain_t *dp, 556 int *domain_freed, char *path) 557 { 558 uint64_t *devtbl_entry; 559 int error = DDI_SUCCESS; 560 amd_iommu_cmdargs_t cmdargs = {0}; 561 const char *driver = ddi_driver_name(iommu->aiomt_dip); 562 int instance = ddi_get_instance(iommu->aiomt_dip); 563 const char *f = "amd_iommu_clear_devtbl_entry"; 564 565 if (amd_iommu_debug & AMD_IOMMU_DEBUG_DEVTBL) { 566 cmn_err(CE_NOTE, "%s: attempting to clear devtbl entry for " 567 "domainid = %d, deviceid = %u, path = %s", 568 f, domainid, deviceid, path); 569 } 570 571 if ((deviceid + 1) * AMD_IOMMU_DEVTBL_ENTRY_SZ > 572 iommu->aiomt_devtbl_sz) { 573 cmn_err(CE_WARN, "%s: %s%d: IOMMU idx=%d, deviceid (%u) " 574 "for rdip (%p) exceeds device table size (%u), path = %s", 575 f, driver, instance, 576 iommu->aiomt_idx, deviceid, (void *)rdip, 577 iommu->aiomt_devtbl_sz, path); 578 return (DDI_FAILURE); 579 } 580 581 /*LINTED*/ 582 devtbl_entry = (uint64_t *)&iommu->aiomt_devtbl 583 [deviceid * AMD_IOMMU_DEVTBL_ENTRY_SZ]; 584 585 if (amd_iommu_debug & AMD_IOMMU_DEBUG_DEVTBL) { 586 cmn_err(CE_NOTE, "%s: deviceid=%u devtbl entry (%p) for %s", 587 f, deviceid, (void *)(uintptr_t)(*devtbl_entry), path); 588 } 589 590 if (AMD_IOMMU_REG_GET64(&(devtbl_entry[0]), AMD_IOMMU_DEVTBL_TV) == 0) { 591 /* Nothing to do */ 592 return (DDI_SUCCESS); 593 } 594 595 ASSERT(dp->d_pgtable_root_4K == AMD_IOMMU_REG_GET64(&(devtbl_entry[0]), 596 AMD_IOMMU_DEVTBL_ROOT_PGTBL)); 597 598 ASSERT(domainid == AMD_IOMMU_REG_GET64(&(devtbl_entry[1]), 599 AMD_IOMMU_DEVTBL_DOMAINID)); 600 601 AMD_IOMMU_REG_SET64(&(devtbl_entry[0]), AMD_IOMMU_DEVTBL_TV, 0); 602 AMD_IOMMU_REG_SET64(&(devtbl_entry[0]), AMD_IOMMU_DEVTBL_ROOT_PGTBL, 0); 603 AMD_IOMMU_REG_SET64(&(devtbl_entry[0]), AMD_IOMMU_DEVTBL_V, 1); 604 605 SYNC_FORDEV(iommu->aiomt_dmahdl); 606 607 dp->d_ref--; 608 ASSERT(dp->d_ref >= 0); 609 610 if (dp->d_ref == 0) { 611 *domain_freed = 1; 612 } 613 614 cmdargs.ca_deviceid = deviceid; 615 error = amd_iommu_cmd(iommu, AMD_IOMMU_CMD_INVAL_DEVTAB_ENTRY, 616 &cmdargs, 0, 0); 617 if (error != DDI_SUCCESS) 618 error = DDI_FAILURE; 619 620 return (error); 621 } 622 623 int 624 amd_iommu_page_table_hash_init(amd_iommu_page_table_hash_t *ampt) 625 { 626 ampt->ampt_hash = kmem_zalloc(sizeof (amd_iommu_page_table_t *) * 627 AMD_IOMMU_PGTABLE_HASH_SZ, KM_SLEEP); 628 return (DDI_SUCCESS); 629 } 630 631 void 632 amd_iommu_page_table_hash_fini(amd_iommu_page_table_hash_t *ampt) 633 { 634 kmem_free(ampt->ampt_hash, 635 sizeof (amd_iommu_page_table_t *) * AMD_IOMMU_PGTABLE_HASH_SZ); 636 ampt->ampt_hash = NULL; 637 } 638 639 static uint32_t 640 pt_hashfn(uint64_t pa_4K) 641 { 642 return (pa_4K % AMD_IOMMU_PGTABLE_HASH_SZ); 643 } 644 645 static void 646 amd_iommu_insert_pgtable_hash(amd_iommu_page_table_t *pt) 647 { 648 uint64_t pa_4K = ((uint64_t)pt->pt_cookie.dmac_cookie_addr) >> 12; 649 uint32_t idx = pt_hashfn(pa_4K); 650 651 ASSERT((pt->pt_cookie.dmac_cookie_addr & AMD_IOMMU_PGTABLE_ALIGN) == 0); 652 653 mutex_enter(&amd_iommu_page_table_hash.ampt_lock); 654 655 pt->pt_next = amd_iommu_page_table_hash.ampt_hash[idx]; 656 pt->pt_prev = NULL; 657 amd_iommu_page_table_hash.ampt_hash[idx] = pt; 658 if (pt->pt_next) 659 pt->pt_next->pt_prev = pt; 660 661 mutex_exit(&amd_iommu_page_table_hash.ampt_lock); 662 } 663 664 static void 665 amd_iommu_remove_pgtable_hash(amd_iommu_page_table_t *pt) 666 { 667 uint64_t pa_4K = (pt->pt_cookie.dmac_cookie_addr >> 12); 668 uint32_t idx = pt_hashfn(pa_4K); 669 670 ASSERT((pt->pt_cookie.dmac_cookie_addr & AMD_IOMMU_PGTABLE_ALIGN) == 0); 671 672 mutex_enter(&amd_iommu_page_table_hash.ampt_lock); 673 674 if (pt->pt_next) 675 pt->pt_next->pt_prev = pt->pt_prev; 676 677 if (pt->pt_prev) 678 pt->pt_prev->pt_next = pt->pt_next; 679 else 680 amd_iommu_page_table_hash.ampt_hash[idx] = pt->pt_next; 681 682 pt->pt_next = NULL; 683 pt->pt_prev = NULL; 684 685 mutex_exit(&amd_iommu_page_table_hash.ampt_lock); 686 } 687 688 static amd_iommu_page_table_t * 689 amd_iommu_lookup_pgtable_hash(domain_id_t domainid, uint64_t pgtable_pa_4K) 690 { 691 amd_iommu_page_table_t *pt; 692 uint32_t idx = pt_hashfn(pgtable_pa_4K); 693 694 mutex_enter(&amd_iommu_page_table_hash.ampt_lock); 695 pt = amd_iommu_page_table_hash.ampt_hash[idx]; 696 for (; pt; pt = pt->pt_next) { 697 if (domainid != pt->pt_domainid) 698 continue; 699 ASSERT((pt->pt_cookie.dmac_cookie_addr & 700 AMD_IOMMU_PGTABLE_ALIGN) == 0); 701 if ((pt->pt_cookie.dmac_cookie_addr >> 12) == pgtable_pa_4K) { 702 break; 703 } 704 } 705 mutex_exit(&amd_iommu_page_table_hash.ampt_lock); 706 707 return (pt); 708 } 709 710 /*ARGSUSED*/ 711 static amd_iommu_page_table_t * 712 amd_iommu_lookup_pgtable(amd_iommu_t *iommu, amd_iommu_page_table_t *ppt, 713 amd_iommu_domain_t *dp, int level, uint16_t index) 714 { 715 uint64_t *pdtep; 716 uint64_t pgtable_pa_4K; 717 718 ASSERT(level > 0 && level <= AMD_IOMMU_PGTABLE_MAXLEVEL); 719 ASSERT(dp); 720 721 if (level == AMD_IOMMU_PGTABLE_MAXLEVEL) { 722 ASSERT(ppt == NULL); 723 ASSERT(index == 0); 724 pgtable_pa_4K = dp->d_pgtable_root_4K; 725 } else { 726 ASSERT(ppt); 727 pdtep = &(ppt->pt_pgtblva[index]); 728 if (AMD_IOMMU_REG_GET64(pdtep, AMD_IOMMU_PTDE_PR) == 0) { 729 if (amd_iommu_debug & AMD_IOMMU_DEBUG_PAGE_TABLES) { 730 cmn_err(CE_NOTE, "Skipping PR=0 pdte: 0x%" 731 PRIx64, *pdtep); 732 } 733 return (NULL); 734 } 735 pgtable_pa_4K = AMD_IOMMU_REG_GET64(pdtep, AMD_IOMMU_PTDE_ADDR); 736 } 737 738 return (amd_iommu_lookup_pgtable_hash(dp->d_domainid, pgtable_pa_4K)); 739 } 740 741 static amd_iommu_page_table_t * 742 amd_iommu_alloc_from_freelist(void) 743 { 744 int i; 745 uint64_t *pte_array; 746 amd_iommu_page_table_t *pt; 747 748 if (amd_iommu_no_pgtable_freelist == 1) 749 return (NULL); 750 751 if (amd_iommu_pgtable_freelist.f_count == 0) 752 return (NULL); 753 754 pt = amd_iommu_pgtable_freelist.f_list; 755 amd_iommu_pgtable_freelist.f_list = pt->pt_next; 756 amd_iommu_pgtable_freelist.f_count--; 757 758 pte_array = pt->pt_pgtblva; 759 for (i = 0; i < AMD_IOMMU_PGTABLE_SZ / (sizeof (*pte_array)); i++) { 760 ASSERT(pt->pt_pte_ref[i] == 0); 761 ASSERT(AMD_IOMMU_REG_GET64(&(pte_array[i]), 762 AMD_IOMMU_PTDE_PR) == 0); 763 } 764 765 return (pt); 766 } 767 768 static int 769 amd_iommu_alloc_pgtable(amd_iommu_t *iommu, domain_id_t domainid, 770 const char *path, amd_iommu_page_table_t **ptp, int km_flags) 771 { 772 int err; 773 uint_t ncookies; 774 amd_iommu_page_table_t *pt; 775 dev_info_t *idip = iommu->aiomt_dip; 776 const char *driver = ddi_driver_name(idip); 777 int instance = ddi_get_instance(idip); 778 const char *f = "amd_iommu_alloc_pgtable"; 779 780 *ptp = NULL; 781 782 pt = amd_iommu_alloc_from_freelist(); 783 if (pt) 784 goto init_pgtable; 785 786 pt = kmem_zalloc(sizeof (amd_iommu_page_table_t), km_flags); 787 if (pt == NULL) 788 return (DDI_DMA_NORESOURCES); 789 790 /* 791 * Each page table is 4K in size 792 */ 793 pt->pt_mem_reqsz = AMD_IOMMU_PGTABLE_SZ; 794 795 /* 796 * Alloc a DMA handle. Use the IOMMU dip as we want this DMA 797 * to *not* enter the IOMMU - no recursive entrance. 798 */ 799 err = ddi_dma_alloc_handle(idip, &amd_iommu_pgtable_dma_attr, 800 km_flags == KM_SLEEP ? DDI_DMA_SLEEP : DDI_DMA_DONTWAIT, 801 NULL, &pt->pt_dma_hdl); 802 if (err != DDI_SUCCESS) { 803 cmn_err(CE_WARN, "%s: %s%d: domainid = %d, path = %s. " 804 "Cannot alloc DMA handle for IO Page Table", 805 f, driver, instance, domainid, path); 806 kmem_free(pt, sizeof (amd_iommu_page_table_t)); 807 return (err == DDI_DMA_NORESOURCES ? err : DDI_DMA_NOMAPPING); 808 } 809 810 /* 811 * Alloc memory for IO Page Table. 812 * XXX remove size_t cast kludge 813 */ 814 err = ddi_dma_mem_alloc(pt->pt_dma_hdl, pt->pt_mem_reqsz, 815 &amd_iommu_devacc, DDI_DMA_CONSISTENT|IOMEM_DATA_UNCACHED, 816 km_flags == KM_SLEEP ? DDI_DMA_SLEEP : DDI_DMA_DONTWAIT, 817 NULL, (caddr_t *)&pt->pt_pgtblva, 818 (size_t *)&pt->pt_mem_realsz, &pt->pt_mem_hdl); 819 if (err != DDI_SUCCESS) { 820 cmn_err(CE_WARN, "%s: %s%d: domainid=%d, path = %s. " 821 "Cannot allocate DMA memory for IO Page table", 822 f, driver, instance, domainid, path); 823 ddi_dma_free_handle(&pt->pt_dma_hdl); 824 kmem_free(pt, sizeof (amd_iommu_page_table_t)); 825 return (DDI_DMA_NORESOURCES); 826 } 827 828 /* 829 * The Page table DMA VA must be 4K aligned and 830 * size >= than requested memory. 831 * 832 */ 833 ASSERT(((uint64_t)(uintptr_t)pt->pt_pgtblva & AMD_IOMMU_PGTABLE_ALIGN) 834 == 0); 835 ASSERT(pt->pt_mem_realsz >= pt->pt_mem_reqsz); 836 837 /* 838 * Now bind the handle 839 */ 840 err = ddi_dma_addr_bind_handle(pt->pt_dma_hdl, NULL, 841 (caddr_t)pt->pt_pgtblva, pt->pt_mem_realsz, 842 DDI_DMA_READ | DDI_DMA_CONSISTENT, 843 km_flags == KM_SLEEP ? DDI_DMA_SLEEP : DDI_DMA_DONTWAIT, 844 NULL, &pt->pt_cookie, &ncookies); 845 if (err != DDI_DMA_MAPPED) { 846 cmn_err(CE_WARN, "%s: %s%d: domainid=%d, path = %s. " 847 "Cannot bind memory for DMA to IO Page Tables. " 848 "bufrealsz=%p", 849 f, driver, instance, domainid, path, 850 (void *)(uintptr_t)pt->pt_mem_realsz); 851 ddi_dma_mem_free(&pt->pt_mem_hdl); 852 ddi_dma_free_handle(&pt->pt_dma_hdl); 853 kmem_free(pt, sizeof (amd_iommu_page_table_t)); 854 return (err == DDI_DMA_PARTIAL_MAP ? DDI_DMA_NOMAPPING : 855 err); 856 } 857 858 /* 859 * We assume the DMA engine on the IOMMU is capable of handling the 860 * whole page table in a single cookie. If not and multiple cookies 861 * are needed we fail. 862 */ 863 if (ncookies != 1) { 864 cmn_err(CE_WARN, "%s: %s%d: domainid = %d, path=%s " 865 "Cannot handle multiple " 866 "cookies for DMA to IO page Table, #cookies=%u", 867 f, driver, instance, domainid, path, ncookies); 868 (void) ddi_dma_unbind_handle(pt->pt_dma_hdl); 869 ddi_dma_mem_free(&pt->pt_mem_hdl); 870 ddi_dma_free_handle(&pt->pt_dma_hdl); 871 kmem_free(pt, sizeof (amd_iommu_page_table_t)); 872 return (DDI_DMA_NOMAPPING); 873 } 874 875 init_pgtable: 876 /* 877 * The address in the cookie must be 4K aligned and >= table size 878 */ 879 ASSERT(pt->pt_cookie.dmac_cookie_addr != NULL); 880 ASSERT((pt->pt_cookie.dmac_cookie_addr & AMD_IOMMU_PGTABLE_ALIGN) == 0); 881 ASSERT(pt->pt_cookie.dmac_size >= pt->pt_mem_realsz); 882 ASSERT(pt->pt_cookie.dmac_size >= pt->pt_mem_reqsz); 883 ASSERT(pt->pt_mem_reqsz >= AMD_IOMMU_PGTABLE_SIZE); 884 ASSERT(pt->pt_mem_realsz >= pt->pt_mem_reqsz); 885 ASSERT(pt->pt_pgtblva); 886 887 pt->pt_domainid = AMD_IOMMU_INVALID_DOMAIN; 888 pt->pt_level = 0x7; 889 pt->pt_index = 0; 890 pt->pt_ref = 0; 891 pt->pt_next = NULL; 892 pt->pt_prev = NULL; 893 pt->pt_parent = NULL; 894 895 bzero(pt->pt_pgtblva, pt->pt_mem_realsz); 896 SYNC_FORDEV(pt->pt_dma_hdl); 897 898 amd_iommu_insert_pgtable_hash(pt); 899 900 *ptp = pt; 901 902 return (DDI_SUCCESS); 903 } 904 905 static int 906 amd_iommu_move_to_freelist(amd_iommu_page_table_t *pt) 907 { 908 if (amd_iommu_no_pgtable_freelist == 1) 909 return (DDI_FAILURE); 910 911 if (amd_iommu_pgtable_freelist.f_count == 912 AMD_IOMMU_PGTABLE_FREELIST_MAX) 913 return (DDI_FAILURE); 914 915 pt->pt_next = amd_iommu_pgtable_freelist.f_list; 916 amd_iommu_pgtable_freelist.f_list = pt; 917 amd_iommu_pgtable_freelist.f_count++; 918 919 return (DDI_SUCCESS); 920 } 921 922 static void 923 amd_iommu_free_pgtable(amd_iommu_t *iommu, amd_iommu_page_table_t *pt) 924 { 925 int i; 926 uint64_t *pte_array; 927 dev_info_t *dip = iommu->aiomt_dip; 928 int instance = ddi_get_instance(dip); 929 const char *driver = ddi_driver_name(dip); 930 const char *f = "amd_iommu_free_pgtable"; 931 932 ASSERT(pt->pt_ref == 0); 933 934 amd_iommu_remove_pgtable_hash(pt); 935 936 pte_array = pt->pt_pgtblva; 937 for (i = 0; i < AMD_IOMMU_PGTABLE_SZ / (sizeof (*pte_array)); i++) { 938 ASSERT(pt->pt_pte_ref[i] == 0); 939 ASSERT(AMD_IOMMU_REG_GET64(&(pte_array[i]), 940 AMD_IOMMU_PTDE_PR) == 0); 941 } 942 943 if (amd_iommu_move_to_freelist(pt) == DDI_SUCCESS) 944 return; 945 946 /* Unbind the handle */ 947 if (ddi_dma_unbind_handle(pt->pt_dma_hdl) != DDI_SUCCESS) { 948 cmn_err(CE_WARN, "%s: %s%d: idx=%d, domainid=%d. " 949 "Failed to unbind handle: %p for IOMMU Page Table", 950 f, driver, instance, iommu->aiomt_idx, pt->pt_domainid, 951 (void *)pt->pt_dma_hdl); 952 } 953 /* Free the table memory allocated for DMA */ 954 ddi_dma_mem_free(&pt->pt_mem_hdl); 955 956 /* Free the DMA handle */ 957 ddi_dma_free_handle(&pt->pt_dma_hdl); 958 959 kmem_free(pt, sizeof (amd_iommu_page_table_t)); 960 961 } 962 963 static int 964 init_pde(amd_iommu_page_table_t *ppt, amd_iommu_page_table_t *pt) 965 { 966 uint64_t *pdep = &(ppt->pt_pgtblva[pt->pt_index]); 967 uint64_t next_pgtable_pa_4K = (pt->pt_cookie.dmac_cookie_addr) >> 12; 968 969 /* nothing to set. PDE is already set */ 970 if (AMD_IOMMU_REG_GET64(pdep, AMD_IOMMU_PTDE_PR) == 1) { 971 ASSERT(PT_REF_VALID(ppt)); 972 ASSERT(PT_REF_VALID(pt)); 973 ASSERT(ppt->pt_pte_ref[pt->pt_index] == 0); 974 ASSERT(AMD_IOMMU_REG_GET64(pdep, AMD_IOMMU_PTDE_ADDR) 975 == next_pgtable_pa_4K); 976 return (DDI_SUCCESS); 977 } 978 979 ppt->pt_ref++; 980 ASSERT(PT_REF_VALID(ppt)); 981 982 /* Page Directories are always RW */ 983 AMD_IOMMU_REG_SET64(pdep, AMD_IOMMU_PTDE_IW, 1); 984 AMD_IOMMU_REG_SET64(pdep, AMD_IOMMU_PTDE_IR, 1); 985 AMD_IOMMU_REG_SET64(pdep, AMD_IOMMU_PTDE_ADDR, 986 next_pgtable_pa_4K); 987 pt->pt_parent = ppt; 988 AMD_IOMMU_REG_SET64(pdep, AMD_IOMMU_PTDE_NXT_LVL, 989 pt->pt_level); 990 ppt->pt_pte_ref[pt->pt_index] = 0; 991 AMD_IOMMU_REG_SET64(pdep, AMD_IOMMU_PTDE_PR, 1); 992 SYNC_FORDEV(ppt->pt_dma_hdl); 993 ASSERT(AMD_IOMMU_REG_GET64(pdep, AMD_IOMMU_PTDE_PR) == 1); 994 995 return (DDI_SUCCESS); 996 } 997 998 static int 999 init_pte(amd_iommu_page_table_t *pt, uint64_t pa, uint16_t index, 1000 struct ddi_dma_req *dmareq) 1001 { 1002 uint64_t *ptep = &(pt->pt_pgtblva[index]); 1003 uint64_t pa_4K = pa >> 12; 1004 int R; 1005 int W; 1006 1007 /* nothing to set if PTE is already set */ 1008 if (AMD_IOMMU_REG_GET64(ptep, AMD_IOMMU_PTDE_PR) == 1) { 1009 /* 1010 * Adjust current permissions 1011 * DDI_DMA_WRITE means direction of DMA is MEM -> I/O 1012 * so that requires Memory READ permissions i.e. sense 1013 * is inverted. 1014 * Note: either or both of DD_DMA_READ/WRITE may be set 1015 */ 1016 if (amd_iommu_no_RW_perms == 0) { 1017 R = AMD_IOMMU_REG_GET64(ptep, AMD_IOMMU_PTDE_IR); 1018 W = AMD_IOMMU_REG_GET64(ptep, AMD_IOMMU_PTDE_IW); 1019 if (R == 0 && ((dmareq->dmar_flags & DDI_DMA_WRITE) || 1020 (dmareq->dmar_flags & DDI_DMA_RDWR))) { 1021 AMD_IOMMU_REG_SET64(ptep, AMD_IOMMU_PTDE_IR, 1); 1022 } 1023 if (W == 0 && ((dmareq->dmar_flags & DDI_DMA_READ) || 1024 (dmareq->dmar_flags & DDI_DMA_RDWR))) { 1025 AMD_IOMMU_REG_SET64(ptep, AMD_IOMMU_PTDE_IW, 1); 1026 } 1027 } 1028 ASSERT(PT_REF_VALID(pt)); 1029 pt->pt_pte_ref[index]++; 1030 ASSERT(AMD_IOMMU_REG_GET64(ptep, AMD_IOMMU_PTDE_ADDR) 1031 == pa_4K); 1032 return (DDI_SUCCESS); 1033 } 1034 1035 pt->pt_ref++; 1036 ASSERT(PT_REF_VALID(pt)); 1037 1038 /* see comment above about inverting sense of RD/WR */ 1039 if (amd_iommu_no_RW_perms == 0) { 1040 AMD_IOMMU_REG_SET64(ptep, AMD_IOMMU_PTDE_IR, 0); 1041 AMD_IOMMU_REG_SET64(ptep, AMD_IOMMU_PTDE_IW, 0); 1042 if (dmareq->dmar_flags & DDI_DMA_RDWR) { 1043 AMD_IOMMU_REG_SET64(ptep, AMD_IOMMU_PTDE_IW, 1); 1044 AMD_IOMMU_REG_SET64(ptep, AMD_IOMMU_PTDE_IR, 1); 1045 } else { 1046 if (dmareq->dmar_flags & DDI_DMA_WRITE) { 1047 AMD_IOMMU_REG_SET64(ptep, AMD_IOMMU_PTDE_IR, 1); 1048 } 1049 if (dmareq->dmar_flags & DDI_DMA_READ) { 1050 AMD_IOMMU_REG_SET64(ptep, AMD_IOMMU_PTDE_IW, 1); 1051 } 1052 } 1053 } else { 1054 AMD_IOMMU_REG_SET64(ptep, AMD_IOMMU_PTDE_IR, 1); 1055 AMD_IOMMU_REG_SET64(ptep, AMD_IOMMU_PTDE_IW, 1); 1056 } 1057 1058 /* TODO what is correct for FC and U */ 1059 AMD_IOMMU_REG_SET64(ptep, AMD_IOMMU_PTE_FC, 0); 1060 AMD_IOMMU_REG_SET64(ptep, AMD_IOMMU_PTE_U, 0); 1061 AMD_IOMMU_REG_SET64(ptep, AMD_IOMMU_PTDE_ADDR, pa_4K); 1062 AMD_IOMMU_REG_SET64(ptep, AMD_IOMMU_PTDE_NXT_LVL, 0); 1063 ASSERT(pt->pt_pte_ref[index] == 0); 1064 pt->pt_pte_ref[index] = 1; 1065 AMD_IOMMU_REG_SET64(ptep, AMD_IOMMU_PTDE_PR, 1); 1066 SYNC_FORDEV(pt->pt_dma_hdl); 1067 ASSERT(AMD_IOMMU_REG_GET64(ptep, AMD_IOMMU_PTDE_PR) == 1); 1068 1069 return (DDI_SUCCESS); 1070 } 1071 1072 1073 static void 1074 init_pt(amd_iommu_page_table_t *pt, amd_iommu_domain_t *dp, 1075 int level, uint16_t index) 1076 { 1077 ASSERT(dp); 1078 1079 if (level == AMD_IOMMU_PGTABLE_MAXLEVEL) { 1080 dp->d_pgtable_root_4K = (pt->pt_cookie.dmac_cookie_addr) >> 12; 1081 } else { 1082 ASSERT(level >= 1 && level < AMD_IOMMU_PGTABLE_MAXLEVEL); 1083 } 1084 1085 pt->pt_domainid = dp->d_domainid; 1086 pt->pt_level = level; 1087 pt->pt_index = index; 1088 } 1089 1090 static int 1091 amd_iommu_setup_1_pgtable(amd_iommu_t *iommu, dev_info_t *rdip, 1092 struct ddi_dma_req *dmareq, 1093 domain_id_t domainid, amd_iommu_domain_t *dp, 1094 amd_iommu_page_table_t *ppt, 1095 uint16_t index, int level, uint64_t va, uint64_t pa, 1096 amd_iommu_page_table_t **ptp, uint16_t *next_idxp, const char *path, 1097 int km_flags) 1098 { 1099 int error; 1100 amd_iommu_page_table_t *pt; 1101 const char *driver = ddi_driver_name(rdip); 1102 int instance = ddi_get_instance(rdip); 1103 const char *f = "amd_iommu_setup_1_pgtable"; 1104 1105 *ptp = NULL; 1106 *next_idxp = 0; 1107 error = DDI_SUCCESS; 1108 1109 ASSERT(level > 0 && level <= AMD_IOMMU_PGTABLE_MAXLEVEL); 1110 1111 ASSERT(dp); 1112 if (level == AMD_IOMMU_PGTABLE_MAXLEVEL) { 1113 ASSERT(ppt == NULL); 1114 ASSERT(index == 0); 1115 } else { 1116 ASSERT(ppt); 1117 } 1118 1119 /* Check if page table is already allocated */ 1120 if (pt = amd_iommu_lookup_pgtable(iommu, ppt, dp, level, index)) { 1121 ASSERT(pt->pt_domainid == domainid); 1122 ASSERT(pt->pt_level == level); 1123 ASSERT(pt->pt_index == index); 1124 goto out; 1125 } 1126 1127 if ((error = amd_iommu_alloc_pgtable(iommu, domainid, path, &pt, 1128 km_flags)) != DDI_SUCCESS) { 1129 cmn_err(CE_WARN, "%s: %s%d: idx = %u, domainid = %d, va = %p " 1130 "path = %s", f, driver, instance, iommu->aiomt_idx, 1131 domainid, (void *)(uintptr_t)va, path); 1132 return (error); 1133 } 1134 1135 ASSERT(dp->d_domainid == domainid); 1136 1137 init_pt(pt, dp, level, index); 1138 1139 out: 1140 if (level != AMD_IOMMU_PGTABLE_MAXLEVEL) { 1141 error = init_pde(ppt, pt); 1142 } 1143 1144 if (level == 1) { 1145 ASSERT(error == DDI_SUCCESS); 1146 error = init_pte(pt, pa, AMD_IOMMU_VA_BITS(va, level), dmareq); 1147 } else { 1148 *next_idxp = AMD_IOMMU_VA_BITS(va, level); 1149 *ptp = pt; 1150 } 1151 1152 return (error); 1153 } 1154 1155 typedef enum { 1156 PDTE_NOT_TORN = 0x1, 1157 PDTE_TORN_DOWN = 0x2, 1158 PGTABLE_TORN_DOWN = 0x4 1159 } pdte_tear_t; 1160 1161 static pdte_tear_t 1162 amd_iommu_teardown_pdte(amd_iommu_t *iommu, 1163 amd_iommu_page_table_t *pt, int index) 1164 { 1165 uint8_t next_level; 1166 pdte_tear_t retval; 1167 uint64_t *ptdep = &(pt->pt_pgtblva[index]); 1168 1169 next_level = AMD_IOMMU_REG_GET64(ptdep, 1170 AMD_IOMMU_PTDE_NXT_LVL); 1171 1172 if (AMD_IOMMU_REG_GET64(ptdep, AMD_IOMMU_PTDE_PR) == 1) { 1173 if (pt->pt_level == 1) { 1174 ASSERT(next_level == 0); 1175 /* PTE */ 1176 pt->pt_pte_ref[index]--; 1177 if (pt->pt_pte_ref[index] != 0) { 1178 return (PDTE_NOT_TORN); 1179 } 1180 } else { 1181 ASSERT(next_level != 0 && next_level != 7); 1182 } 1183 ASSERT(pt->pt_pte_ref[index] == 0); 1184 ASSERT(PT_REF_VALID(pt)); 1185 1186 AMD_IOMMU_REG_SET64(ptdep, AMD_IOMMU_PTDE_PR, 0); 1187 SYNC_FORDEV(pt->pt_dma_hdl); 1188 ASSERT(AMD_IOMMU_REG_GET64(ptdep, 1189 AMD_IOMMU_PTDE_PR) == 0); 1190 pt->pt_ref--; 1191 ASSERT(PT_REF_VALID(pt)); 1192 retval = PDTE_TORN_DOWN; 1193 } else { 1194 ASSERT(0); 1195 ASSERT(pt->pt_pte_ref[index] == 0); 1196 ASSERT(PT_REF_VALID(pt)); 1197 retval = PDTE_NOT_TORN; 1198 } 1199 1200 if (pt->pt_ref == 0) { 1201 amd_iommu_free_pgtable(iommu, pt); 1202 return (PGTABLE_TORN_DOWN); 1203 } 1204 1205 return (retval); 1206 } 1207 1208 static int 1209 amd_iommu_create_pgtables(amd_iommu_t *iommu, dev_info_t *rdip, 1210 struct ddi_dma_req *dmareq, uint64_t va, 1211 uint64_t pa, uint16_t deviceid, domain_id_t domainid, 1212 amd_iommu_domain_t *dp, const char *path, int km_flags) 1213 { 1214 int level; 1215 uint16_t index; 1216 uint16_t next_idx; 1217 amd_iommu_page_table_t *pt; 1218 amd_iommu_page_table_t *ppt; 1219 int error; 1220 const char *driver = ddi_driver_name(rdip); 1221 int instance = ddi_get_instance(rdip); 1222 const char *f = "amd_iommu_create_pgtables"; 1223 1224 if (amd_iommu_debug & AMD_IOMMU_DEBUG_PAGE_TABLES) { 1225 cmn_err(CE_NOTE, "%s: %s%d: idx = %u, domainid = %d, " 1226 "deviceid = %u, va = %p, pa = %p, path = %s", 1227 f, driver, instance, 1228 iommu->aiomt_idx, domainid, deviceid, 1229 (void *)(uintptr_t)va, 1230 (void *)(uintptr_t)pa, path); 1231 } 1232 1233 if (domainid == AMD_IOMMU_PASSTHRU_DOMAIN) { 1234 /* No need for pagetables. Just set up device table entry */ 1235 goto passthru; 1236 } 1237 1238 index = 0; 1239 ppt = NULL; 1240 for (level = AMD_IOMMU_PGTABLE_MAXLEVEL; level > 0; 1241 level--, pt = NULL, next_idx = 0) { 1242 if ((error = amd_iommu_setup_1_pgtable(iommu, rdip, dmareq, 1243 domainid, dp, ppt, index, level, va, pa, &pt, 1244 &next_idx, path, km_flags)) != DDI_SUCCESS) { 1245 cmn_err(CE_WARN, "%s: %s%d: idx=%d: domainid=%d, " 1246 "deviceid=%u, va= %p, pa = %p, Failed to setup " 1247 "page table(s) at level = %d, path = %s.", 1248 f, driver, instance, iommu->aiomt_idx, 1249 domainid, deviceid, (void *)(uintptr_t)va, 1250 (void *)(uintptr_t)pa, level, path); 1251 return (error); 1252 } 1253 1254 if (level > 1) { 1255 ASSERT(pt); 1256 ASSERT(pt->pt_domainid == domainid); 1257 ppt = pt; 1258 index = next_idx; 1259 } else { 1260 ASSERT(level == 1); 1261 ASSERT(pt == NULL); 1262 ASSERT(next_idx == 0); 1263 ppt = NULL; 1264 index = 0; 1265 } 1266 } 1267 1268 passthru: 1269 if ((error = amd_iommu_set_devtbl_entry(iommu, rdip, domainid, deviceid, 1270 dp, path)) != DDI_SUCCESS) { 1271 cmn_err(CE_WARN, "%s: %s%d: idx=%d: rdip=%p, deviceid=%u, " 1272 "domainid=%d." 1273 "Failed to set device table entry for path %s.", 1274 f, driver, instance, 1275 iommu->aiomt_idx, (void *)rdip, deviceid, domainid, path); 1276 return (error); 1277 } 1278 1279 SYNC_FORDEV(iommu->aiomt_dmahdl); 1280 1281 return (DDI_SUCCESS); 1282 } 1283 1284 static int 1285 amd_iommu_destroy_pgtables(amd_iommu_t *iommu, dev_info_t *rdip, 1286 uint64_t pageva, uint16_t deviceid, domain_id_t domainid, 1287 amd_iommu_domain_t *dp, map_type_t type, int *domain_freed, char *path) 1288 { 1289 int level; 1290 int flags; 1291 amd_iommu_cmdargs_t cmdargs = {0}; 1292 uint16_t index; 1293 uint16_t prev_index; 1294 amd_iommu_page_table_t *pt; 1295 amd_iommu_page_table_t *ppt; 1296 pdte_tear_t retval; 1297 int tear_level; 1298 int invalidate_pte; 1299 int invalidate_pde; 1300 int error = DDI_FAILURE; 1301 const char *driver = ddi_driver_name(iommu->aiomt_dip); 1302 int instance = ddi_get_instance(iommu->aiomt_dip); 1303 const char *f = "amd_iommu_destroy_pgtables"; 1304 1305 if (amd_iommu_debug & AMD_IOMMU_DEBUG_PAGE_TABLES) { 1306 cmn_err(CE_NOTE, "%s: %s%d: idx = %u, domainid = %d, " 1307 "deviceid = %u, va = %p, path = %s", 1308 f, driver, instance, 1309 iommu->aiomt_idx, domainid, deviceid, 1310 (void *)(uintptr_t)pageva, path); 1311 } 1312 1313 if (domainid == AMD_IOMMU_PASSTHRU_DOMAIN) { 1314 /* 1315 * there are no pagetables for the passthru domain. 1316 * Just the device table entry 1317 */ 1318 error = DDI_SUCCESS; 1319 goto passthru; 1320 } 1321 1322 ppt = NULL; 1323 index = 0; 1324 for (level = AMD_IOMMU_PGTABLE_MAXLEVEL; level > 0; level--) { 1325 pt = amd_iommu_lookup_pgtable(iommu, ppt, dp, level, index); 1326 if (pt) { 1327 ppt = pt; 1328 index = AMD_IOMMU_VA_BITS(pageva, level); 1329 continue; 1330 } 1331 break; 1332 } 1333 1334 if (level == 0) { 1335 uint64_t *ptep; 1336 uint64_t pa_4K; 1337 1338 ASSERT(pt); 1339 ASSERT(pt == ppt); 1340 ASSERT(pt->pt_domainid == dp->d_domainid); 1341 1342 ptep = &(pt->pt_pgtblva[index]); 1343 1344 pa_4K = AMD_IOMMU_REG_GET64(ptep, AMD_IOMMU_PTDE_ADDR); 1345 if (amd_iommu_unity_map || type == AMD_IOMMU_UNITY_MAP) { 1346 ASSERT(pageva == (pa_4K << MMU_PAGESHIFT)); 1347 } 1348 } 1349 1350 tear_level = -1; 1351 invalidate_pde = 0; 1352 invalidate_pte = 0; 1353 for (++level; level <= AMD_IOMMU_PGTABLE_MAXLEVEL; level++) { 1354 prev_index = pt->pt_index; 1355 ppt = pt->pt_parent; 1356 retval = amd_iommu_teardown_pdte(iommu, pt, index); 1357 switch (retval) { 1358 case PDTE_NOT_TORN: 1359 goto invalidate; 1360 case PDTE_TORN_DOWN: 1361 invalidate_pte = 1; 1362 goto invalidate; 1363 case PGTABLE_TORN_DOWN: 1364 invalidate_pte = 1; 1365 invalidate_pde = 1; 1366 tear_level = level; 1367 break; 1368 } 1369 index = prev_index; 1370 pt = ppt; 1371 } 1372 1373 invalidate: 1374 /* 1375 * Now teardown the IOMMU HW caches if applicable 1376 */ 1377 if (invalidate_pte) { 1378 cmdargs.ca_domainid = (uint16_t)domainid; 1379 if (amd_iommu_pageva_inval_all) { 1380 cmdargs.ca_addr = (uintptr_t)0x7FFFFFFFFFFFF000; 1381 flags = AMD_IOMMU_CMD_FLAGS_PAGE_PDE_INVAL | 1382 AMD_IOMMU_CMD_FLAGS_PAGE_INVAL_S; 1383 } else if (invalidate_pde) { 1384 cmdargs.ca_addr = 1385 (uintptr_t)AMD_IOMMU_VA_INVAL(pageva, tear_level); 1386 flags = AMD_IOMMU_CMD_FLAGS_PAGE_PDE_INVAL | 1387 AMD_IOMMU_CMD_FLAGS_PAGE_INVAL_S; 1388 } else { 1389 cmdargs.ca_addr = (uintptr_t)pageva; 1390 flags = 0; 1391 } 1392 if (amd_iommu_cmd(iommu, AMD_IOMMU_CMD_INVAL_IOMMU_PAGES, 1393 &cmdargs, flags, 0) != DDI_SUCCESS) { 1394 cmn_err(CE_WARN, "%s: %s%d: idx=%d: domainid=%d, " 1395 "rdip=%p. Failed to invalidate IOMMU HW cache " 1396 "for %s", f, driver, instance, 1397 iommu->aiomt_idx, domainid, (void *)rdip, path); 1398 error = DDI_FAILURE; 1399 goto out; 1400 } 1401 } 1402 1403 passthru: 1404 if (tear_level == AMD_IOMMU_PGTABLE_MAXLEVEL) { 1405 error = amd_iommu_clear_devtbl_entry(iommu, rdip, domainid, 1406 deviceid, dp, domain_freed, path); 1407 } else { 1408 error = DDI_SUCCESS; 1409 } 1410 1411 out: 1412 SYNC_FORDEV(iommu->aiomt_dmahdl); 1413 1414 return (error); 1415 } 1416 1417 static int 1418 cvt_bind_error(int error) 1419 { 1420 switch (error) { 1421 case DDI_DMA_MAPPED: 1422 case DDI_DMA_PARTIAL_MAP: 1423 case DDI_DMA_NORESOURCES: 1424 case DDI_DMA_NOMAPPING: 1425 break; 1426 default: 1427 cmn_err(CE_PANIC, "Unsupported error code: %d", error); 1428 /*NOTREACHED*/ 1429 } 1430 return (error); 1431 } 1432 1433 int 1434 amd_iommu_map_pa2va(amd_iommu_t *iommu, dev_info_t *rdip, ddi_dma_attr_t *attrp, 1435 struct ddi_dma_req *dmareq, uint64_t start_pa, uint64_t pa_sz, 1436 map_type_t type, uint64_t *start_vap, int km_flags) 1437 { 1438 pfn_t pfn_start; 1439 pfn_t pfn_end; 1440 pfn_t pfn; 1441 int alias; 1442 int32_t deviceid; 1443 domain_id_t domainid; 1444 amd_iommu_domain_t *dp; 1445 uint64_t end_pa; 1446 uint64_t start_va; 1447 uint64_t end_va; 1448 uint64_t pg_start; 1449 uint64_t pg_end; 1450 uint64_t pg; 1451 uint64_t va_sz; 1452 char *path; 1453 int error = DDI_DMA_NOMAPPING; 1454 const char *driver = ddi_driver_name(iommu->aiomt_dip); 1455 int instance = ddi_get_instance(iommu->aiomt_dip); 1456 const char *f = "amd_iommu_map_pa2va"; 1457 1458 ASSERT(pa_sz != 0); 1459 1460 *start_vap = 0; 1461 1462 ASSERT(rdip); 1463 1464 path = kmem_alloc(MAXPATHLEN, km_flags); 1465 if (path == NULL) { 1466 error = DDI_DMA_NORESOURCES; 1467 goto out; 1468 } 1469 (void) ddi_pathname(rdip, path); 1470 1471 /* 1472 * First get deviceid 1473 */ 1474 if (amd_iommu_get_deviceid(iommu, rdip, &deviceid, &alias, path) 1475 != DDI_SUCCESS) { 1476 cmn_err(CE_WARN, "%s: %s%d: idx=%d: rdip=%p. " 1477 "Failed to get device ID for %s.", f, driver, instance, 1478 iommu->aiomt_idx, (void *)rdip, path); 1479 error = DDI_DMA_NOMAPPING; 1480 goto out; 1481 } 1482 1483 /* 1484 * Next get the domain for this rdip 1485 */ 1486 if (amd_iommu_get_domain(iommu, rdip, alias, deviceid, &domainid, path) 1487 != DDI_SUCCESS) { 1488 cmn_err(CE_WARN, "%s: %s%d: idx=%d: rdip=%p, path=%s. " 1489 "Failed to get domain.", f, driver, instance, 1490 iommu->aiomt_idx, (void *)rdip, path); 1491 error = DDI_DMA_NOMAPPING; 1492 goto out; 1493 } 1494 1495 dp = amd_iommu_lookup_domain(iommu, domainid, type, km_flags); 1496 if (dp == NULL) { 1497 cmn_err(CE_WARN, "%s: %s%d: idx=%d: domainid=%d, rdip=%p. " 1498 "Failed to get device ID for %s.", f, driver, instance, 1499 iommu->aiomt_idx, domainid, (void *)rdip, path); 1500 error = DDI_DMA_NORESOURCES; 1501 goto out; 1502 } 1503 1504 ASSERT(dp->d_domainid == domainid); 1505 1506 pfn_start = start_pa >> MMU_PAGESHIFT; 1507 1508 if (amd_iommu_debug & AMD_IOMMU_DEBUG_PAGE_TABLES) { 1509 cmn_err(CE_NOTE, "pa = %p, pfn_new = %p, pfn_start = %p, " 1510 "pgshift = %d", 1511 (void *)(uintptr_t)start_pa, 1512 (void *)(uintptr_t)(start_pa >> MMU_PAGESHIFT), 1513 (void *)(uintptr_t)pfn_start, MMU_PAGESHIFT); 1514 } 1515 1516 end_pa = start_pa + pa_sz - 1; 1517 pfn_end = end_pa >> MMU_PAGESHIFT; 1518 1519 if (amd_iommu_unity_map || type == AMD_IOMMU_UNITY_MAP) { 1520 start_va = start_pa; 1521 end_va = end_pa; 1522 va_sz = pa_sz; 1523 *start_vap = start_va; 1524 } else { 1525 va_sz = mmu_ptob(pfn_end - pfn_start + 1); 1526 start_va = (uintptr_t)vmem_xalloc(dp->d_vmem, va_sz, 1527 MAX(attrp->dma_attr_align, MMU_PAGESIZE), 1528 0, 1529 attrp->dma_attr_seg + 1, 1530 (void *)(uintptr_t)attrp->dma_attr_addr_lo, 1531 (void *)(uintptr_t)MIN((attrp->dma_attr_addr_hi + 1), 1532 AMD_IOMMU_SIZE_4G), /* XXX rollover */ 1533 km_flags == KM_SLEEP ? VM_SLEEP : VM_NOSLEEP); 1534 if (start_va == 0) { 1535 cmn_err(CE_WARN, "%s: No VA resources", 1536 amd_iommu_modname); 1537 error = DDI_DMA_NORESOURCES; 1538 goto out; 1539 } 1540 ASSERT((start_va & MMU_PAGEOFFSET) == 0); 1541 end_va = start_va + va_sz - 1; 1542 *start_vap = start_va + (start_pa & MMU_PAGEOFFSET); 1543 } 1544 1545 pg_start = start_va >> MMU_PAGESHIFT; 1546 pg_end = end_va >> MMU_PAGESHIFT; 1547 1548 pg = pg_start; 1549 for (pfn = pfn_start; pfn <= pfn_end; pfn++, pg++) { 1550 1551 if (amd_iommu_debug & AMD_IOMMU_DEBUG_PAGE_TABLES) { 1552 cmn_err(CE_NOTE, "%s: attempting to create page tables " 1553 "for pfn = %p, va = %p, path = %s", 1554 f, (void *)(uintptr_t)(pfn << MMU_PAGESHIFT), 1555 (void *)(uintptr_t)(pg << MMU_PAGESHIFT), path); 1556 1557 } 1558 1559 if (amd_iommu_unity_map || type == AMD_IOMMU_UNITY_MAP) { 1560 ASSERT(pfn == pg); 1561 } 1562 1563 if ((error = amd_iommu_create_pgtables(iommu, rdip, dmareq, 1564 pg << MMU_PAGESHIFT, 1565 pfn << MMU_PAGESHIFT, deviceid, domainid, dp, path, 1566 km_flags)) != DDI_SUCCESS) { 1567 cmn_err(CE_WARN, "Failed to create_pgtables"); 1568 goto out; 1569 } 1570 1571 if (amd_iommu_debug & AMD_IOMMU_DEBUG_PAGE_TABLES) { 1572 cmn_err(CE_NOTE, "%s: successfully created page tables " 1573 "for pfn = %p, vapg = %p, path = %s", 1574 f, (void *)(uintptr_t)pfn, 1575 (void *)(uintptr_t)pg, path); 1576 } 1577 1578 } 1579 ASSERT(pg == pg_end + 1); 1580 1581 1582 if (amd_iommu_debug & AMD_IOMMU_DEBUG_PA2VA) { 1583 cmn_err(CE_NOTE, "pa=%p, va=%p", 1584 (void *)(uintptr_t)start_pa, 1585 (void *)(uintptr_t)(*start_vap)); 1586 } 1587 error = DDI_DMA_MAPPED; 1588 1589 out: 1590 kmem_free(path, MAXPATHLEN); 1591 return (cvt_bind_error(error)); 1592 } 1593 1594 int 1595 amd_iommu_unmap_va(amd_iommu_t *iommu, dev_info_t *rdip, uint64_t start_va, 1596 uint64_t va_sz, map_type_t type) 1597 { 1598 uint64_t end_va; 1599 uint64_t pg_start; 1600 uint64_t pg_end; 1601 uint64_t pg; 1602 uint64_t actual_sz; 1603 char *path; 1604 int pathfree; 1605 int alias; 1606 int32_t deviceid; 1607 domain_id_t domainid; 1608 amd_iommu_domain_t *dp; 1609 int error; 1610 int domain_freed; 1611 const char *driver = ddi_driver_name(iommu->aiomt_dip); 1612 int instance = ddi_get_instance(iommu->aiomt_dip); 1613 const char *f = "amd_iommu_unmap_va"; 1614 1615 if (amd_iommu_no_unmap) 1616 return (DDI_SUCCESS); 1617 1618 path = kmem_alloc(MAXPATHLEN, KM_NOSLEEP); 1619 if (path) { 1620 (void) ddi_pathname(rdip, path); 1621 pathfree = 1; 1622 } else { 1623 pathfree = 0; 1624 path = "<path-mem-alloc-failed>"; 1625 } 1626 1627 /* 1628 * First get deviceid 1629 */ 1630 if (amd_iommu_get_deviceid(iommu, rdip, &deviceid, &alias, path) 1631 != DDI_SUCCESS) { 1632 cmn_err(CE_WARN, "%s: %s%d: idx=%d: rdip=%p. " 1633 "Failed to get device ID for %s.", f, driver, instance, 1634 iommu->aiomt_idx, (void *)rdip, path); 1635 error = DDI_FAILURE; 1636 goto out; 1637 } 1638 1639 /* 1640 * Next get the domain for this rdip 1641 */ 1642 if (amd_iommu_get_domain(iommu, rdip, alias, deviceid, &domainid, path) 1643 != DDI_SUCCESS) { 1644 cmn_err(CE_WARN, "%s: %s%d: idx=%d: rdip=%p, path=%s. " 1645 "Failed to get domain.", f, driver, instance, 1646 iommu->aiomt_idx, (void *)rdip, path); 1647 error = DDI_FAILURE; 1648 goto out; 1649 } 1650 1651 /* should never result in domain allocation/vmem_create */ 1652 dp = amd_iommu_lookup_domain(iommu, domainid, AMD_IOMMU_INVALID_MAP, 1653 KM_NOSLEEP); 1654 if (dp == NULL) { 1655 cmn_err(CE_WARN, "%s: %s%d: idx=%d: domainid=%d, rdip=%p. " 1656 "Failed to get device ID for %s.", f, driver, instance, 1657 iommu->aiomt_idx, domainid, (void *)rdip, path); 1658 error = DDI_FAILURE; 1659 goto out; 1660 } 1661 1662 ASSERT(dp->d_domainid == domainid); 1663 1664 pg_start = start_va >> MMU_PAGESHIFT; 1665 end_va = start_va + va_sz - 1; 1666 pg_end = end_va >> MMU_PAGESHIFT; 1667 actual_sz = (pg_end - pg_start + 1) << MMU_PAGESHIFT; 1668 1669 domain_freed = 0; 1670 for (pg = pg_start; pg <= pg_end; pg++) { 1671 domain_freed = 0; 1672 if (amd_iommu_destroy_pgtables(iommu, rdip, 1673 pg << MMU_PAGESHIFT, deviceid, domainid, dp, type, 1674 &domain_freed, path) != DDI_SUCCESS) { 1675 error = DDI_FAILURE; 1676 goto out; 1677 } 1678 if (domain_freed) { 1679 ASSERT(pg == pg_end); 1680 break; 1681 } 1682 } 1683 1684 /* 1685 * vmem_xalloc() must be paired with vmem_xfree 1686 */ 1687 if (type == AMD_IOMMU_VMEM_MAP && !amd_iommu_unity_map) { 1688 vmem_xfree(dp->d_vmem, 1689 (void *)(uintptr_t)(pg_start << MMU_PAGESHIFT), actual_sz); 1690 } 1691 1692 if (domain_freed) 1693 amd_iommu_teardown_domain(iommu, dp); 1694 1695 error = DDI_SUCCESS; 1696 out: 1697 if (pathfree) 1698 kmem_free(path, MAXPATHLEN); 1699 return (error); 1700 } 1701