1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause 3 * 4 * Copyright (c) 2013 The FreeBSD Foundation 5 * 6 * This software was developed by Konstantin Belousov <kib@FreeBSD.org> 7 * under sponsorship from the FreeBSD Foundation. 8 * 9 * Redistribution and use in source and binary forms, with or without 10 * modification, are permitted provided that the following conditions 11 * are met: 12 * 1. Redistributions of source code must retain the above copyright 13 * notice, this list of conditions and the following disclaimer. 14 * 2. Redistributions in binary form must reproduce the above copyright 15 * notice, this list of conditions and the following disclaimer in the 16 * documentation and/or other materials provided with the distribution. 17 * 18 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 19 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 20 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 21 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 22 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 23 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 24 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 25 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 26 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 27 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 28 * SUCH DAMAGE. 29 */ 30 31 #include <sys/cdefs.h> 32 #include <sys/param.h> 33 #include <sys/systm.h> 34 #include <sys/malloc.h> 35 #include <sys/bus.h> 36 #include <sys/interrupt.h> 37 #include <sys/kernel.h> 38 #include <sys/ktr.h> 39 #include <sys/limits.h> 40 #include <sys/lock.h> 41 #include <sys/memdesc.h> 42 #include <sys/mutex.h> 43 #include <sys/proc.h> 44 #include <sys/rwlock.h> 45 #include <sys/rman.h> 46 #include <sys/sysctl.h> 47 #include <sys/taskqueue.h> 48 #include <sys/tree.h> 49 #include <sys/uio.h> 50 #include <sys/vmem.h> 51 #include <vm/vm.h> 52 #include <vm/vm_extern.h> 53 #include <vm/vm_kern.h> 54 #include <vm/vm_object.h> 55 #include <vm/vm_page.h> 56 #include <vm/vm_pager.h> 57 #include <vm/vm_map.h> 58 #include <contrib/dev/acpica/include/acpi.h> 59 #include <contrib/dev/acpica/include/accommon.h> 60 #include <dev/pci/pcireg.h> 61 #include <dev/pci/pcivar.h> 62 #include <machine/atomic.h> 63 #include <machine/bus.h> 64 #include <machine/md_var.h> 65 #include <machine/specialreg.h> 66 #include <x86/include/busdma_impl.h> 67 #include <dev/iommu/busdma_iommu.h> 68 #include <x86/iommu/intel_reg.h> 69 #include <x86/iommu/intel_dmar.h> 70 71 static MALLOC_DEFINE(M_DMAR_CTX, "dmar_ctx", "Intel DMAR Context"); 72 static MALLOC_DEFINE(M_DMAR_DOMAIN, "dmar_dom", "Intel DMAR Domain"); 73 74 static void dmar_unref_domain_locked(struct dmar_unit *dmar, 75 struct dmar_domain *domain); 76 static void dmar_domain_destroy(struct dmar_domain *domain); 77 78 static void 79 dmar_ensure_ctx_page(struct dmar_unit *dmar, int bus) 80 { 81 struct sf_buf *sf; 82 dmar_root_entry_t *re; 83 vm_page_t ctxm; 84 85 /* 86 * Allocated context page must be linked. 87 */ 88 ctxm = dmar_pgalloc(dmar->ctx_obj, 1 + bus, IOMMU_PGF_NOALLOC); 89 if (ctxm != NULL) 90 return; 91 92 /* 93 * Page not present, allocate and link. Note that other 94 * thread might execute this sequence in parallel. This 95 * should be safe, because the context entries written by both 96 * threads are equal. 97 */ 98 TD_PREP_PINNED_ASSERT; 99 ctxm = dmar_pgalloc(dmar->ctx_obj, 1 + bus, IOMMU_PGF_ZERO | 100 IOMMU_PGF_WAITOK); 101 re = dmar_map_pgtbl(dmar->ctx_obj, 0, IOMMU_PGF_NOALLOC, &sf); 102 re += bus; 103 dmar_pte_store(&re->r1, DMAR_ROOT_R1_P | (DMAR_ROOT_R1_CTP_MASK & 104 VM_PAGE_TO_PHYS(ctxm))); 105 dmar_flush_root_to_ram(dmar, re); 106 dmar_unmap_pgtbl(sf); 107 TD_PINNED_ASSERT; 108 } 109 110 static dmar_ctx_entry_t * 111 dmar_map_ctx_entry(struct dmar_ctx *ctx, struct sf_buf **sfp) 112 { 113 struct dmar_unit *dmar; 114 dmar_ctx_entry_t *ctxp; 115 116 dmar = CTX2DMAR(ctx); 117 118 ctxp = dmar_map_pgtbl(dmar->ctx_obj, 1 + PCI_RID2BUS(ctx->context.rid), 119 IOMMU_PGF_NOALLOC | IOMMU_PGF_WAITOK, sfp); 120 ctxp += ctx->context.rid & 0xff; 121 return (ctxp); 122 } 123 124 static void 125 device_tag_init(struct dmar_ctx *ctx, device_t dev) 126 { 127 struct dmar_domain *domain; 128 bus_addr_t maxaddr; 129 130 domain = CTX2DOM(ctx); 131 maxaddr = MIN(domain->iodom.end, BUS_SPACE_MAXADDR); 132 ctx->context.tag->common.ref_count = 1; /* Prevent free */ 133 ctx->context.tag->common.impl = &bus_dma_iommu_impl; 134 ctx->context.tag->common.boundary = 0; 135 ctx->context.tag->common.lowaddr = maxaddr; 136 ctx->context.tag->common.highaddr = maxaddr; 137 ctx->context.tag->common.maxsize = maxaddr; 138 ctx->context.tag->common.nsegments = BUS_SPACE_UNRESTRICTED; 139 ctx->context.tag->common.maxsegsz = maxaddr; 140 ctx->context.tag->ctx = CTX2IOCTX(ctx); 141 ctx->context.tag->owner = dev; 142 } 143 144 static void 145 ctx_id_entry_init_one(dmar_ctx_entry_t *ctxp, struct dmar_domain *domain, 146 vm_page_t ctx_root) 147 { 148 /* 149 * For update due to move, the store is not atomic. It is 150 * possible that DMAR read upper doubleword, while low 151 * doubleword is not yet updated. The domain id is stored in 152 * the upper doubleword, while the table pointer in the lower. 153 * 154 * There is no good solution, for the same reason it is wrong 155 * to clear P bit in the ctx entry for update. 156 */ 157 dmar_pte_store1(&ctxp->ctx2, DMAR_CTX2_DID(domain->domain) | 158 domain->awlvl); 159 if (ctx_root == NULL) { 160 dmar_pte_store1(&ctxp->ctx1, DMAR_CTX1_T_PASS | DMAR_CTX1_P); 161 } else { 162 dmar_pte_store1(&ctxp->ctx1, DMAR_CTX1_T_UNTR | 163 (DMAR_CTX1_ASR_MASK & VM_PAGE_TO_PHYS(ctx_root)) | 164 DMAR_CTX1_P); 165 } 166 } 167 168 static void 169 ctx_id_entry_init(struct dmar_ctx *ctx, dmar_ctx_entry_t *ctxp, bool move, 170 int busno) 171 { 172 struct dmar_unit *unit; 173 struct dmar_domain *domain; 174 vm_page_t ctx_root; 175 int i; 176 177 domain = CTX2DOM(ctx); 178 unit = DOM2DMAR(domain); 179 KASSERT(move || (ctxp->ctx1 == 0 && ctxp->ctx2 == 0), 180 ("dmar%d: initialized ctx entry %d:%d:%d 0x%jx 0x%jx", 181 unit->iommu.unit, busno, pci_get_slot(ctx->context.tag->owner), 182 pci_get_function(ctx->context.tag->owner), 183 ctxp->ctx1, ctxp->ctx2)); 184 185 if ((domain->iodom.flags & IOMMU_DOMAIN_IDMAP) != 0 && 186 (unit->hw_ecap & DMAR_ECAP_PT) != 0) { 187 KASSERT(domain->pgtbl_obj == NULL, 188 ("ctx %p non-null pgtbl_obj", ctx)); 189 ctx_root = NULL; 190 } else { 191 ctx_root = dmar_pgalloc(domain->pgtbl_obj, 0, 192 IOMMU_PGF_NOALLOC); 193 } 194 195 if (iommu_is_buswide_ctx(DMAR2IOMMU(unit), busno)) { 196 MPASS(!move); 197 for (i = 0; i <= PCI_BUSMAX; i++) { 198 ctx_id_entry_init_one(&ctxp[i], domain, ctx_root); 199 } 200 } else { 201 ctx_id_entry_init_one(ctxp, domain, ctx_root); 202 } 203 dmar_flush_ctx_to_ram(unit, ctxp); 204 } 205 206 static int 207 dmar_flush_for_ctx_entry(struct dmar_unit *dmar, bool force) 208 { 209 int error; 210 211 /* 212 * If dmar declares Caching Mode as Set, follow 11.5 "Caching 213 * Mode Consideration" and do the (global) invalidation of the 214 * negative TLB entries. 215 */ 216 if ((dmar->hw_cap & DMAR_CAP_CM) == 0 && !force) 217 return (0); 218 if (dmar->qi_enabled) { 219 dmar_qi_invalidate_ctx_glob_locked(dmar); 220 if ((dmar->hw_ecap & DMAR_ECAP_DI) != 0 || force) 221 dmar_qi_invalidate_iotlb_glob_locked(dmar); 222 return (0); 223 } 224 error = dmar_inv_ctx_glob(dmar); 225 if (error == 0 && ((dmar->hw_ecap & DMAR_ECAP_DI) != 0 || force)) 226 error = dmar_inv_iotlb_glob(dmar); 227 return (error); 228 } 229 230 static int 231 domain_init_rmrr(struct dmar_domain *domain, device_t dev, int bus, 232 int slot, int func, int dev_domain, int dev_busno, 233 const void *dev_path, int dev_path_len) 234 { 235 struct iommu_map_entries_tailq rmrr_entries; 236 struct iommu_map_entry *entry, *entry1; 237 vm_page_t *ma; 238 iommu_gaddr_t start, end; 239 vm_pindex_t size, i; 240 int error, error1; 241 242 error = 0; 243 TAILQ_INIT(&rmrr_entries); 244 dmar_dev_parse_rmrr(domain, dev_domain, dev_busno, dev_path, 245 dev_path_len, &rmrr_entries); 246 TAILQ_FOREACH_SAFE(entry, &rmrr_entries, dmamap_link, entry1) { 247 /* 248 * VT-d specification requires that the start of an 249 * RMRR entry is 4k-aligned. Buggy BIOSes put 250 * anything into the start and end fields. Truncate 251 * and round as neccesary. 252 * 253 * We also allow the overlapping RMRR entries, see 254 * iommu_gas_alloc_region(). 255 */ 256 start = entry->start; 257 end = entry->end; 258 if (bootverbose) 259 printf("dmar%d ctx pci%d:%d:%d RMRR [%#jx, %#jx]\n", 260 domain->iodom.iommu->unit, bus, slot, func, 261 (uintmax_t)start, (uintmax_t)end); 262 entry->start = trunc_page(start); 263 entry->end = round_page(end); 264 if (entry->start == entry->end) { 265 /* Workaround for some AMI (?) BIOSes */ 266 if (bootverbose) { 267 if (dev != NULL) 268 device_printf(dev, ""); 269 printf("pci%d:%d:%d ", bus, slot, func); 270 printf("BIOS bug: dmar%d RMRR " 271 "region (%jx, %jx) corrected\n", 272 domain->iodom.iommu->unit, start, end); 273 } 274 entry->end += DMAR_PAGE_SIZE * 0x20; 275 } 276 size = OFF_TO_IDX(entry->end - entry->start); 277 ma = malloc(sizeof(vm_page_t) * size, M_TEMP, M_WAITOK); 278 for (i = 0; i < size; i++) { 279 ma[i] = vm_page_getfake(entry->start + PAGE_SIZE * i, 280 VM_MEMATTR_DEFAULT); 281 } 282 error1 = iommu_gas_map_region(DOM2IODOM(domain), entry, 283 IOMMU_MAP_ENTRY_READ | IOMMU_MAP_ENTRY_WRITE, 284 IOMMU_MF_CANWAIT | IOMMU_MF_RMRR, ma); 285 /* 286 * Non-failed RMRR entries are owned by context rb 287 * tree. Get rid of the failed entry, but do not stop 288 * the loop. Rest of the parsed RMRR entries are 289 * loaded and removed on the context destruction. 290 */ 291 if (error1 == 0 && entry->end != entry->start) { 292 IOMMU_LOCK(domain->iodom.iommu); 293 domain->refs++; /* XXXKIB prevent free */ 294 domain->iodom.flags |= IOMMU_DOMAIN_RMRR; 295 IOMMU_UNLOCK(domain->iodom.iommu); 296 } else { 297 if (error1 != 0) { 298 if (dev != NULL) 299 device_printf(dev, ""); 300 printf("pci%d:%d:%d ", bus, slot, func); 301 printf( 302 "dmar%d failed to map RMRR region (%jx, %jx) %d\n", 303 domain->iodom.iommu->unit, start, end, 304 error1); 305 error = error1; 306 } 307 TAILQ_REMOVE(&rmrr_entries, entry, dmamap_link); 308 iommu_gas_free_entry(entry); 309 } 310 for (i = 0; i < size; i++) 311 vm_page_putfake(ma[i]); 312 free(ma, M_TEMP); 313 } 314 return (error); 315 } 316 317 /* 318 * PCI memory address space is shared between memory-mapped devices (MMIO) and 319 * host memory (which may be remapped by an IOMMU). Device accesses to an 320 * address within a memory aperture in a PCIe root port will be treated as 321 * peer-to-peer and not forwarded to an IOMMU. To avoid this, reserve the 322 * address space of the root port's memory apertures in the address space used 323 * by the IOMMU for remapping. 324 */ 325 static int 326 dmar_reserve_pci_regions(struct dmar_domain *domain, device_t dev) 327 { 328 struct iommu_domain *iodom; 329 device_t root; 330 uint32_t val; 331 uint64_t base, limit; 332 int error; 333 334 iodom = DOM2IODOM(domain); 335 336 root = pci_find_pcie_root_port(dev); 337 if (root == NULL) 338 return (0); 339 340 /* Disable downstream memory */ 341 base = PCI_PPBMEMBASE(0, pci_read_config(root, PCIR_MEMBASE_1, 2)); 342 limit = PCI_PPBMEMLIMIT(0, pci_read_config(root, PCIR_MEMLIMIT_1, 2)); 343 error = iommu_gas_reserve_region_extend(iodom, base, limit + 1); 344 if (bootverbose || error != 0) 345 device_printf(dev, "DMAR reserve [%#jx-%#jx] (error %d)\n", 346 base, limit + 1, error); 347 if (error != 0) 348 return (error); 349 350 /* Disable downstream prefetchable memory */ 351 val = pci_read_config(root, PCIR_PMBASEL_1, 2); 352 if (val != 0 || pci_read_config(root, PCIR_PMLIMITL_1, 2) != 0) { 353 if ((val & PCIM_BRPM_MASK) == PCIM_BRPM_64) { 354 base = PCI_PPBMEMBASE( 355 pci_read_config(root, PCIR_PMBASEH_1, 4), 356 val); 357 limit = PCI_PPBMEMLIMIT( 358 pci_read_config(root, PCIR_PMLIMITH_1, 4), 359 pci_read_config(root, PCIR_PMLIMITL_1, 2)); 360 } else { 361 base = PCI_PPBMEMBASE(0, val); 362 limit = PCI_PPBMEMLIMIT(0, 363 pci_read_config(root, PCIR_PMLIMITL_1, 2)); 364 } 365 error = iommu_gas_reserve_region_extend(iodom, base, 366 limit + 1); 367 if (bootverbose || error != 0) 368 device_printf(dev, "DMAR reserve [%#jx-%#jx] " 369 "(error %d)\n", base, limit + 1, error); 370 if (error != 0) 371 return (error); 372 } 373 374 return (error); 375 } 376 377 static struct dmar_domain * 378 dmar_domain_alloc(struct dmar_unit *dmar, bool id_mapped) 379 { 380 struct iommu_domain *iodom; 381 struct iommu_unit *unit; 382 struct dmar_domain *domain; 383 int error, id, mgaw; 384 385 id = alloc_unr(dmar->domids); 386 if (id == -1) 387 return (NULL); 388 domain = malloc(sizeof(*domain), M_DMAR_DOMAIN, M_WAITOK | M_ZERO); 389 iodom = DOM2IODOM(domain); 390 unit = DMAR2IOMMU(dmar); 391 domain->domain = id; 392 LIST_INIT(&domain->contexts); 393 iommu_domain_init(unit, iodom, &dmar_domain_map_ops); 394 395 domain->dmar = dmar; 396 397 /* 398 * For now, use the maximal usable physical address of the 399 * installed memory to calculate the mgaw on id_mapped domain. 400 * It is useful for the identity mapping, and less so for the 401 * virtualized bus address space. 402 */ 403 domain->iodom.end = id_mapped ? ptoa(Maxmem) : BUS_SPACE_MAXADDR; 404 mgaw = dmar_maxaddr2mgaw(dmar, domain->iodom.end, !id_mapped); 405 error = domain_set_agaw(domain, mgaw); 406 if (error != 0) 407 goto fail; 408 if (!id_mapped) 409 /* Use all supported address space for remapping. */ 410 domain->iodom.end = 1ULL << (domain->agaw - 1); 411 412 iommu_gas_init_domain(DOM2IODOM(domain)); 413 414 if (id_mapped) { 415 if ((dmar->hw_ecap & DMAR_ECAP_PT) == 0) { 416 domain->pgtbl_obj = domain_get_idmap_pgtbl(domain, 417 domain->iodom.end); 418 } 419 domain->iodom.flags |= IOMMU_DOMAIN_IDMAP; 420 } else { 421 error = domain_alloc_pgtbl(domain); 422 if (error != 0) 423 goto fail; 424 /* Disable local apic region access */ 425 error = iommu_gas_reserve_region(iodom, 0xfee00000, 426 0xfeefffff + 1, &iodom->msi_entry); 427 if (error != 0) 428 goto fail; 429 } 430 return (domain); 431 432 fail: 433 dmar_domain_destroy(domain); 434 return (NULL); 435 } 436 437 static struct dmar_ctx * 438 dmar_ctx_alloc(struct dmar_domain *domain, uint16_t rid) 439 { 440 struct dmar_ctx *ctx; 441 442 ctx = malloc(sizeof(*ctx), M_DMAR_CTX, M_WAITOK | M_ZERO); 443 ctx->context.domain = DOM2IODOM(domain); 444 ctx->context.tag = malloc(sizeof(struct bus_dma_tag_iommu), 445 M_DMAR_CTX, M_WAITOK | M_ZERO); 446 ctx->context.rid = rid; 447 ctx->refs = 1; 448 return (ctx); 449 } 450 451 static void 452 dmar_ctx_link(struct dmar_ctx *ctx) 453 { 454 struct dmar_domain *domain; 455 456 domain = CTX2DOM(ctx); 457 IOMMU_ASSERT_LOCKED(domain->iodom.iommu); 458 KASSERT(domain->refs >= domain->ctx_cnt, 459 ("dom %p ref underflow %d %d", domain, domain->refs, 460 domain->ctx_cnt)); 461 domain->refs++; 462 domain->ctx_cnt++; 463 LIST_INSERT_HEAD(&domain->contexts, ctx, link); 464 } 465 466 static void 467 dmar_ctx_unlink(struct dmar_ctx *ctx) 468 { 469 struct dmar_domain *domain; 470 471 domain = CTX2DOM(ctx); 472 IOMMU_ASSERT_LOCKED(domain->iodom.iommu); 473 KASSERT(domain->refs > 0, 474 ("domain %p ctx dtr refs %d", domain, domain->refs)); 475 KASSERT(domain->ctx_cnt >= domain->refs, 476 ("domain %p ctx dtr refs %d ctx_cnt %d", domain, 477 domain->refs, domain->ctx_cnt)); 478 domain->refs--; 479 domain->ctx_cnt--; 480 LIST_REMOVE(ctx, link); 481 } 482 483 static void 484 dmar_domain_destroy(struct dmar_domain *domain) 485 { 486 struct iommu_domain *iodom; 487 struct dmar_unit *dmar; 488 489 iodom = DOM2IODOM(domain); 490 491 KASSERT(TAILQ_EMPTY(&domain->iodom.unload_entries), 492 ("unfinished unloads %p", domain)); 493 KASSERT(LIST_EMPTY(&domain->contexts), 494 ("destroying dom %p with contexts", domain)); 495 KASSERT(domain->ctx_cnt == 0, 496 ("destroying dom %p with ctx_cnt %d", domain, domain->ctx_cnt)); 497 KASSERT(domain->refs == 0, 498 ("destroying dom %p with refs %d", domain, domain->refs)); 499 if ((domain->iodom.flags & IOMMU_DOMAIN_GAS_INITED) != 0) { 500 DMAR_DOMAIN_LOCK(domain); 501 iommu_gas_fini_domain(iodom); 502 DMAR_DOMAIN_UNLOCK(domain); 503 } 504 if ((domain->iodom.flags & IOMMU_DOMAIN_PGTBL_INITED) != 0) { 505 if (domain->pgtbl_obj != NULL) 506 DMAR_DOMAIN_PGLOCK(domain); 507 domain_free_pgtbl(domain); 508 } 509 iommu_domain_fini(iodom); 510 dmar = DOM2DMAR(domain); 511 free_unr(dmar->domids, domain->domain); 512 free(domain, M_DMAR_DOMAIN); 513 } 514 515 static struct dmar_ctx * 516 dmar_get_ctx_for_dev1(struct dmar_unit *dmar, device_t dev, uint16_t rid, 517 int dev_domain, int dev_busno, const void *dev_path, int dev_path_len, 518 bool id_mapped, bool rmrr_init) 519 { 520 struct dmar_domain *domain, *domain1; 521 struct dmar_ctx *ctx, *ctx1; 522 struct iommu_unit *unit __diagused; 523 dmar_ctx_entry_t *ctxp; 524 struct sf_buf *sf; 525 int bus, slot, func, error; 526 bool enable; 527 528 if (dev != NULL) { 529 bus = pci_get_bus(dev); 530 slot = pci_get_slot(dev); 531 func = pci_get_function(dev); 532 } else { 533 bus = PCI_RID2BUS(rid); 534 slot = PCI_RID2SLOT(rid); 535 func = PCI_RID2FUNC(rid); 536 } 537 enable = false; 538 TD_PREP_PINNED_ASSERT; 539 unit = DMAR2IOMMU(dmar); 540 DMAR_LOCK(dmar); 541 KASSERT(!iommu_is_buswide_ctx(unit, bus) || (slot == 0 && func == 0), 542 ("iommu%d pci%d:%d:%d get_ctx for buswide", dmar->iommu.unit, bus, 543 slot, func)); 544 ctx = dmar_find_ctx_locked(dmar, rid); 545 error = 0; 546 if (ctx == NULL) { 547 /* 548 * Perform the allocations which require sleep or have 549 * higher chance to succeed if the sleep is allowed. 550 */ 551 DMAR_UNLOCK(dmar); 552 dmar_ensure_ctx_page(dmar, PCI_RID2BUS(rid)); 553 domain1 = dmar_domain_alloc(dmar, id_mapped); 554 if (domain1 == NULL) { 555 TD_PINNED_ASSERT; 556 return (NULL); 557 } 558 if (!id_mapped) { 559 error = domain_init_rmrr(domain1, dev, bus, 560 slot, func, dev_domain, dev_busno, dev_path, 561 dev_path_len); 562 if (error == 0 && dev != NULL) 563 error = dmar_reserve_pci_regions(domain1, dev); 564 if (error != 0) { 565 dmar_domain_destroy(domain1); 566 TD_PINNED_ASSERT; 567 return (NULL); 568 } 569 } 570 ctx1 = dmar_ctx_alloc(domain1, rid); 571 ctxp = dmar_map_ctx_entry(ctx1, &sf); 572 DMAR_LOCK(dmar); 573 574 /* 575 * Recheck the contexts, other thread might have 576 * already allocated needed one. 577 */ 578 ctx = dmar_find_ctx_locked(dmar, rid); 579 if (ctx == NULL) { 580 domain = domain1; 581 ctx = ctx1; 582 dmar_ctx_link(ctx); 583 ctx->context.tag->owner = dev; 584 device_tag_init(ctx, dev); 585 586 /* 587 * This is the first activated context for the 588 * DMAR unit. Enable the translation after 589 * everything is set up. 590 */ 591 if (LIST_EMPTY(&dmar->domains)) 592 enable = true; 593 LIST_INSERT_HEAD(&dmar->domains, domain, link); 594 ctx_id_entry_init(ctx, ctxp, false, bus); 595 if (dev != NULL) { 596 device_printf(dev, 597 "dmar%d pci%d:%d:%d:%d rid %x domain %d mgaw %d " 598 "agaw %d %s-mapped\n", 599 dmar->iommu.unit, dmar->segment, bus, slot, 600 func, rid, domain->domain, domain->mgaw, 601 domain->agaw, id_mapped ? "id" : "re"); 602 } 603 dmar_unmap_pgtbl(sf); 604 } else { 605 dmar_unmap_pgtbl(sf); 606 dmar_domain_destroy(domain1); 607 /* Nothing needs to be done to destroy ctx1. */ 608 free(ctx1, M_DMAR_CTX); 609 domain = CTX2DOM(ctx); 610 ctx->refs++; /* tag referenced us */ 611 } 612 } else { 613 domain = CTX2DOM(ctx); 614 if (ctx->context.tag->owner == NULL) 615 ctx->context.tag->owner = dev; 616 ctx->refs++; /* tag referenced us */ 617 } 618 619 error = dmar_flush_for_ctx_entry(dmar, enable); 620 if (error != 0) { 621 dmar_free_ctx_locked(dmar, ctx); 622 TD_PINNED_ASSERT; 623 return (NULL); 624 } 625 626 /* 627 * The dmar lock was potentially dropped between check for the 628 * empty context list and now. Recheck the state of GCMD_TE 629 * to avoid unneeded command. 630 */ 631 if (enable && !rmrr_init && (dmar->hw_gcmd & DMAR_GCMD_TE) == 0) { 632 error = dmar_disable_protected_regions(dmar); 633 if (error != 0) 634 printf("dmar%d: Failed to disable protected regions\n", 635 dmar->iommu.unit); 636 error = dmar_enable_translation(dmar); 637 if (error == 0) { 638 if (bootverbose) { 639 printf("dmar%d: enabled translation\n", 640 dmar->iommu.unit); 641 } 642 } else { 643 printf("dmar%d: enabling translation failed, " 644 "error %d\n", dmar->iommu.unit, error); 645 dmar_free_ctx_locked(dmar, ctx); 646 TD_PINNED_ASSERT; 647 return (NULL); 648 } 649 } 650 DMAR_UNLOCK(dmar); 651 TD_PINNED_ASSERT; 652 return (ctx); 653 } 654 655 struct dmar_ctx * 656 dmar_get_ctx_for_dev(struct dmar_unit *dmar, device_t dev, uint16_t rid, 657 bool id_mapped, bool rmrr_init) 658 { 659 int dev_domain, dev_path_len, dev_busno; 660 661 dev_domain = pci_get_domain(dev); 662 dev_path_len = dmar_dev_depth(dev); 663 ACPI_DMAR_PCI_PATH dev_path[dev_path_len]; 664 dmar_dev_path(dev, &dev_busno, dev_path, dev_path_len); 665 return (dmar_get_ctx_for_dev1(dmar, dev, rid, dev_domain, dev_busno, 666 dev_path, dev_path_len, id_mapped, rmrr_init)); 667 } 668 669 struct dmar_ctx * 670 dmar_get_ctx_for_devpath(struct dmar_unit *dmar, uint16_t rid, 671 int dev_domain, int dev_busno, 672 const void *dev_path, int dev_path_len, 673 bool id_mapped, bool rmrr_init) 674 { 675 676 return (dmar_get_ctx_for_dev1(dmar, NULL, rid, dev_domain, dev_busno, 677 dev_path, dev_path_len, id_mapped, rmrr_init)); 678 } 679 680 int 681 dmar_move_ctx_to_domain(struct dmar_domain *domain, struct dmar_ctx *ctx) 682 { 683 struct dmar_unit *dmar; 684 struct dmar_domain *old_domain; 685 dmar_ctx_entry_t *ctxp; 686 struct sf_buf *sf; 687 int error; 688 689 dmar = domain->dmar; 690 old_domain = CTX2DOM(ctx); 691 if (domain == old_domain) 692 return (0); 693 KASSERT(old_domain->iodom.iommu == domain->iodom.iommu, 694 ("domain %p %u moving between dmars %u %u", domain, 695 domain->domain, old_domain->iodom.iommu->unit, 696 domain->iodom.iommu->unit)); 697 TD_PREP_PINNED_ASSERT; 698 699 ctxp = dmar_map_ctx_entry(ctx, &sf); 700 DMAR_LOCK(dmar); 701 dmar_ctx_unlink(ctx); 702 ctx->context.domain = &domain->iodom; 703 dmar_ctx_link(ctx); 704 ctx_id_entry_init(ctx, ctxp, true, PCI_BUSMAX + 100); 705 dmar_unmap_pgtbl(sf); 706 error = dmar_flush_for_ctx_entry(dmar, true); 707 /* If flush failed, rolling back would not work as well. */ 708 printf("dmar%d rid %x domain %d->%d %s-mapped\n", 709 dmar->iommu.unit, ctx->context.rid, old_domain->domain, 710 domain->domain, (domain->iodom.flags & IOMMU_DOMAIN_IDMAP) != 0 ? 711 "id" : "re"); 712 dmar_unref_domain_locked(dmar, old_domain); 713 TD_PINNED_ASSERT; 714 return (error); 715 } 716 717 static void 718 dmar_unref_domain_locked(struct dmar_unit *dmar, struct dmar_domain *domain) 719 { 720 721 DMAR_ASSERT_LOCKED(dmar); 722 KASSERT(domain->refs >= 1, 723 ("dmar %d domain %p refs %u", dmar->iommu.unit, domain, 724 domain->refs)); 725 KASSERT(domain->refs > domain->ctx_cnt, 726 ("dmar %d domain %p refs %d ctx_cnt %d", dmar->iommu.unit, domain, 727 domain->refs, domain->ctx_cnt)); 728 729 if (domain->refs > 1) { 730 domain->refs--; 731 DMAR_UNLOCK(dmar); 732 return; 733 } 734 735 KASSERT((domain->iodom.flags & IOMMU_DOMAIN_RMRR) == 0, 736 ("lost ref on RMRR domain %p", domain)); 737 738 LIST_REMOVE(domain, link); 739 DMAR_UNLOCK(dmar); 740 741 taskqueue_drain(dmar->iommu.delayed_taskqueue, 742 &domain->iodom.unload_task); 743 dmar_domain_destroy(domain); 744 } 745 746 void 747 dmar_free_ctx_locked(struct dmar_unit *dmar, struct dmar_ctx *ctx) 748 { 749 struct sf_buf *sf; 750 dmar_ctx_entry_t *ctxp; 751 struct dmar_domain *domain; 752 753 DMAR_ASSERT_LOCKED(dmar); 754 KASSERT(ctx->refs >= 1, 755 ("dmar %p ctx %p refs %u", dmar, ctx, ctx->refs)); 756 757 /* 758 * If our reference is not last, only the dereference should 759 * be performed. 760 */ 761 if (ctx->refs > 1) { 762 ctx->refs--; 763 DMAR_UNLOCK(dmar); 764 return; 765 } 766 767 KASSERT((ctx->context.flags & IOMMU_CTX_DISABLED) == 0, 768 ("lost ref on disabled ctx %p", ctx)); 769 770 /* 771 * Otherwise, the context entry must be cleared before the 772 * page table is destroyed. The mapping of the context 773 * entries page could require sleep, unlock the dmar. 774 */ 775 DMAR_UNLOCK(dmar); 776 TD_PREP_PINNED_ASSERT; 777 ctxp = dmar_map_ctx_entry(ctx, &sf); 778 DMAR_LOCK(dmar); 779 KASSERT(ctx->refs >= 1, 780 ("dmar %p ctx %p refs %u", dmar, ctx, ctx->refs)); 781 782 /* 783 * Other thread might have referenced the context, in which 784 * case again only the dereference should be performed. 785 */ 786 if (ctx->refs > 1) { 787 ctx->refs--; 788 DMAR_UNLOCK(dmar); 789 dmar_unmap_pgtbl(sf); 790 TD_PINNED_ASSERT; 791 return; 792 } 793 794 KASSERT((ctx->context.flags & IOMMU_CTX_DISABLED) == 0, 795 ("lost ref on disabled ctx %p", ctx)); 796 797 /* 798 * Clear the context pointer and flush the caches. 799 * XXXKIB: cannot do this if any RMRR entries are still present. 800 */ 801 dmar_pte_clear(&ctxp->ctx1); 802 ctxp->ctx2 = 0; 803 dmar_flush_ctx_to_ram(dmar, ctxp); 804 dmar_inv_ctx_glob(dmar); 805 if ((dmar->hw_ecap & DMAR_ECAP_DI) != 0) { 806 if (dmar->qi_enabled) 807 dmar_qi_invalidate_iotlb_glob_locked(dmar); 808 else 809 dmar_inv_iotlb_glob(dmar); 810 } 811 dmar_unmap_pgtbl(sf); 812 domain = CTX2DOM(ctx); 813 dmar_ctx_unlink(ctx); 814 free(ctx->context.tag, M_DMAR_CTX); 815 free(ctx, M_DMAR_CTX); 816 dmar_unref_domain_locked(dmar, domain); 817 TD_PINNED_ASSERT; 818 } 819 820 void 821 dmar_free_ctx(struct dmar_ctx *ctx) 822 { 823 struct dmar_unit *dmar; 824 825 dmar = CTX2DMAR(ctx); 826 DMAR_LOCK(dmar); 827 dmar_free_ctx_locked(dmar, ctx); 828 } 829 830 /* 831 * Returns with the domain locked. 832 */ 833 struct dmar_ctx * 834 dmar_find_ctx_locked(struct dmar_unit *dmar, uint16_t rid) 835 { 836 struct dmar_domain *domain; 837 struct dmar_ctx *ctx; 838 839 DMAR_ASSERT_LOCKED(dmar); 840 841 LIST_FOREACH(domain, &dmar->domains, link) { 842 LIST_FOREACH(ctx, &domain->contexts, link) { 843 if (ctx->context.rid == rid) 844 return (ctx); 845 } 846 } 847 return (NULL); 848 } 849 850 void 851 dmar_domain_free_entry(struct iommu_map_entry *entry, bool free) 852 { 853 if ((entry->flags & IOMMU_MAP_ENTRY_RMRR) != 0) 854 iommu_gas_free_region(entry); 855 else 856 iommu_gas_free_space(entry); 857 if (free) 858 iommu_gas_free_entry(entry); 859 else 860 entry->flags = 0; 861 } 862 863 /* 864 * If the given value for "free" is true, then the caller must not be using 865 * the entry's dmamap_link field. 866 */ 867 void 868 iommu_domain_unload_entry(struct iommu_map_entry *entry, bool free, 869 bool cansleep) 870 { 871 struct dmar_domain *domain; 872 struct dmar_unit *unit; 873 874 domain = IODOM2DOM(entry->domain); 875 unit = DOM2DMAR(domain); 876 877 /* 878 * If "free" is false, then the IOTLB invalidation must be performed 879 * synchronously. Otherwise, the caller might free the entry before 880 * dmar_qi_task() is finished processing it. 881 */ 882 if (unit->qi_enabled) { 883 if (free) { 884 DMAR_LOCK(unit); 885 dmar_qi_invalidate_locked(domain, entry, true); 886 DMAR_UNLOCK(unit); 887 } else { 888 dmar_qi_invalidate_sync(domain, entry->start, 889 entry->end - entry->start, cansleep); 890 dmar_domain_free_entry(entry, false); 891 } 892 } else { 893 domain_flush_iotlb_sync(domain, entry->start, entry->end - 894 entry->start); 895 dmar_domain_free_entry(entry, free); 896 } 897 } 898 899 static bool 900 dmar_domain_unload_emit_wait(struct dmar_domain *domain, 901 struct iommu_map_entry *entry) 902 { 903 904 if (TAILQ_NEXT(entry, dmamap_link) == NULL) 905 return (true); 906 return (domain->batch_no++ % dmar_batch_coalesce == 0); 907 } 908 909 void 910 iommu_domain_unload(struct iommu_domain *iodom, 911 struct iommu_map_entries_tailq *entries, bool cansleep) 912 { 913 struct dmar_domain *domain; 914 struct dmar_unit *unit; 915 struct iommu_map_entry *entry, *entry1; 916 int error __diagused; 917 918 domain = IODOM2DOM(iodom); 919 unit = DOM2DMAR(domain); 920 921 TAILQ_FOREACH_SAFE(entry, entries, dmamap_link, entry1) { 922 KASSERT((entry->flags & IOMMU_MAP_ENTRY_MAP) != 0, 923 ("not mapped entry %p %p", domain, entry)); 924 error = iodom->ops->unmap(iodom, entry->start, entry->end - 925 entry->start, cansleep ? IOMMU_PGF_WAITOK : 0); 926 KASSERT(error == 0, ("unmap %p error %d", domain, error)); 927 if (!unit->qi_enabled) { 928 domain_flush_iotlb_sync(domain, entry->start, 929 entry->end - entry->start); 930 TAILQ_REMOVE(entries, entry, dmamap_link); 931 dmar_domain_free_entry(entry, true); 932 } 933 } 934 if (TAILQ_EMPTY(entries)) 935 return; 936 937 KASSERT(unit->qi_enabled, ("loaded entry left")); 938 DMAR_LOCK(unit); 939 while ((entry = TAILQ_FIRST(entries)) != NULL) { 940 TAILQ_REMOVE(entries, entry, dmamap_link); 941 dmar_qi_invalidate_locked(domain, entry, 942 dmar_domain_unload_emit_wait(domain, entry)); 943 } 944 DMAR_UNLOCK(unit); 945 } 946 947 struct iommu_ctx * 948 iommu_get_ctx(struct iommu_unit *iommu, device_t dev, uint16_t rid, 949 bool id_mapped, bool rmrr_init) 950 { 951 struct dmar_unit *dmar; 952 struct dmar_ctx *ret; 953 954 dmar = IOMMU2DMAR(iommu); 955 956 ret = dmar_get_ctx_for_dev(dmar, dev, rid, id_mapped, rmrr_init); 957 958 return (CTX2IOCTX(ret)); 959 } 960 961 void 962 iommu_free_ctx_locked(struct iommu_unit *iommu, struct iommu_ctx *context) 963 { 964 struct dmar_unit *dmar; 965 struct dmar_ctx *ctx; 966 967 dmar = IOMMU2DMAR(iommu); 968 ctx = IOCTX2CTX(context); 969 970 dmar_free_ctx_locked(dmar, ctx); 971 } 972 973 void 974 iommu_free_ctx(struct iommu_ctx *context) 975 { 976 struct dmar_ctx *ctx; 977 978 ctx = IOCTX2CTX(context); 979 980 dmar_free_ctx(ctx); 981 } 982