1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause 3 * 4 * Copyright (c) 2013 The FreeBSD Foundation 5 * 6 * This software was developed by Konstantin Belousov <kib@FreeBSD.org> 7 * under sponsorship from the FreeBSD Foundation. 8 * 9 * Redistribution and use in source and binary forms, with or without 10 * modification, are permitted provided that the following conditions 11 * are met: 12 * 1. Redistributions of source code must retain the above copyright 13 * notice, this list of conditions and the following disclaimer. 14 * 2. Redistributions in binary form must reproduce the above copyright 15 * notice, this list of conditions and the following disclaimer in the 16 * documentation and/or other materials provided with the distribution. 17 * 18 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 19 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 20 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 21 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 22 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 23 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 24 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 25 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 26 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 27 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 28 * SUCH DAMAGE. 29 */ 30 31 #include <sys/param.h> 32 #include <sys/systm.h> 33 #include <sys/malloc.h> 34 #include <sys/bus.h> 35 #include <sys/interrupt.h> 36 #include <sys/kernel.h> 37 #include <sys/ktr.h> 38 #include <sys/limits.h> 39 #include <sys/lock.h> 40 #include <sys/memdesc.h> 41 #include <sys/mutex.h> 42 #include <sys/proc.h> 43 #include <sys/rwlock.h> 44 #include <sys/rman.h> 45 #include <sys/sysctl.h> 46 #include <sys/taskqueue.h> 47 #include <sys/tree.h> 48 #include <sys/uio.h> 49 #include <sys/vmem.h> 50 #include <vm/vm.h> 51 #include <vm/vm_extern.h> 52 #include <vm/vm_kern.h> 53 #include <vm/vm_object.h> 54 #include <vm/vm_page.h> 55 #include <vm/vm_pager.h> 56 #include <vm/vm_map.h> 57 #include <contrib/dev/acpica/include/acpi.h> 58 #include <contrib/dev/acpica/include/accommon.h> 59 #include <dev/pci/pcireg.h> 60 #include <dev/pci/pcivar.h> 61 #include <machine/atomic.h> 62 #include <machine/bus.h> 63 #include <machine/md_var.h> 64 #include <machine/specialreg.h> 65 #include <x86/include/busdma_impl.h> 66 #include <dev/iommu/busdma_iommu.h> 67 #include <x86/iommu/intel_reg.h> 68 #include <x86/iommu/intel_dmar.h> 69 70 static MALLOC_DEFINE(M_DMAR_CTX, "dmar_ctx", "Intel DMAR Context"); 71 static MALLOC_DEFINE(M_DMAR_DOMAIN, "dmar_dom", "Intel DMAR Domain"); 72 73 static void dmar_unref_domain_locked(struct dmar_unit *dmar, 74 struct dmar_domain *domain); 75 static void dmar_domain_destroy(struct dmar_domain *domain); 76 77 static void 78 dmar_ensure_ctx_page(struct dmar_unit *dmar, int bus) 79 { 80 struct sf_buf *sf; 81 dmar_root_entry_t *re; 82 vm_page_t ctxm; 83 84 /* 85 * Allocated context page must be linked. 86 */ 87 ctxm = dmar_pgalloc(dmar->ctx_obj, 1 + bus, IOMMU_PGF_NOALLOC); 88 if (ctxm != NULL) 89 return; 90 91 /* 92 * Page not present, allocate and link. Note that other 93 * thread might execute this sequence in parallel. This 94 * should be safe, because the context entries written by both 95 * threads are equal. 96 */ 97 TD_PREP_PINNED_ASSERT; 98 ctxm = dmar_pgalloc(dmar->ctx_obj, 1 + bus, IOMMU_PGF_ZERO | 99 IOMMU_PGF_WAITOK); 100 re = dmar_map_pgtbl(dmar->ctx_obj, 0, IOMMU_PGF_NOALLOC, &sf); 101 re += bus; 102 dmar_pte_store(&re->r1, DMAR_ROOT_R1_P | (DMAR_ROOT_R1_CTP_MASK & 103 VM_PAGE_TO_PHYS(ctxm))); 104 dmar_flush_root_to_ram(dmar, re); 105 dmar_unmap_pgtbl(sf); 106 TD_PINNED_ASSERT; 107 } 108 109 static dmar_ctx_entry_t * 110 dmar_map_ctx_entry(struct dmar_ctx *ctx, struct sf_buf **sfp) 111 { 112 struct dmar_unit *dmar; 113 dmar_ctx_entry_t *ctxp; 114 115 dmar = CTX2DMAR(ctx); 116 117 ctxp = dmar_map_pgtbl(dmar->ctx_obj, 1 + PCI_RID2BUS(ctx->context.rid), 118 IOMMU_PGF_NOALLOC | IOMMU_PGF_WAITOK, sfp); 119 ctxp += ctx->context.rid & 0xff; 120 return (ctxp); 121 } 122 123 static void 124 device_tag_init(struct dmar_ctx *ctx, device_t dev) 125 { 126 struct dmar_domain *domain; 127 bus_addr_t maxaddr; 128 129 domain = CTX2DOM(ctx); 130 maxaddr = MIN(domain->iodom.end, BUS_SPACE_MAXADDR); 131 ctx->context.tag->common.ref_count = 1; /* Prevent free */ 132 ctx->context.tag->common.impl = &bus_dma_iommu_impl; 133 ctx->context.tag->common.boundary = 0; 134 ctx->context.tag->common.lowaddr = maxaddr; 135 ctx->context.tag->common.highaddr = maxaddr; 136 ctx->context.tag->common.maxsize = maxaddr; 137 ctx->context.tag->common.nsegments = BUS_SPACE_UNRESTRICTED; 138 ctx->context.tag->common.maxsegsz = maxaddr; 139 ctx->context.tag->ctx = CTX2IOCTX(ctx); 140 ctx->context.tag->owner = dev; 141 } 142 143 static void 144 ctx_id_entry_init_one(dmar_ctx_entry_t *ctxp, struct dmar_domain *domain, 145 vm_page_t ctx_root) 146 { 147 /* 148 * For update due to move, the store is not atomic. It is 149 * possible that DMAR read upper doubleword, while low 150 * doubleword is not yet updated. The domain id is stored in 151 * the upper doubleword, while the table pointer in the lower. 152 * 153 * There is no good solution, for the same reason it is wrong 154 * to clear P bit in the ctx entry for update. 155 */ 156 dmar_pte_store1(&ctxp->ctx2, DMAR_CTX2_DID(domain->domain) | 157 domain->awlvl); 158 if (ctx_root == NULL) { 159 dmar_pte_store1(&ctxp->ctx1, DMAR_CTX1_T_PASS | DMAR_CTX1_P); 160 } else { 161 dmar_pte_store1(&ctxp->ctx1, DMAR_CTX1_T_UNTR | 162 (DMAR_CTX1_ASR_MASK & VM_PAGE_TO_PHYS(ctx_root)) | 163 DMAR_CTX1_P); 164 } 165 } 166 167 static void 168 ctx_id_entry_init(struct dmar_ctx *ctx, dmar_ctx_entry_t *ctxp, bool move, 169 int busno) 170 { 171 struct dmar_unit *unit; 172 struct dmar_domain *domain; 173 vm_page_t ctx_root; 174 int i; 175 176 domain = CTX2DOM(ctx); 177 unit = DOM2DMAR(domain); 178 KASSERT(move || (ctxp->ctx1 == 0 && ctxp->ctx2 == 0), 179 ("dmar%d: initialized ctx entry %d:%d:%d 0x%jx 0x%jx", 180 unit->iommu.unit, busno, pci_get_slot(ctx->context.tag->owner), 181 pci_get_function(ctx->context.tag->owner), 182 ctxp->ctx1, ctxp->ctx2)); 183 184 if ((domain->iodom.flags & IOMMU_DOMAIN_IDMAP) != 0 && 185 (unit->hw_ecap & DMAR_ECAP_PT) != 0) { 186 KASSERT(domain->pgtbl_obj == NULL, 187 ("ctx %p non-null pgtbl_obj", ctx)); 188 ctx_root = NULL; 189 } else { 190 ctx_root = dmar_pgalloc(domain->pgtbl_obj, 0, 191 IOMMU_PGF_NOALLOC); 192 } 193 194 if (iommu_is_buswide_ctx(DMAR2IOMMU(unit), busno)) { 195 MPASS(!move); 196 for (i = 0; i <= PCI_BUSMAX; i++) { 197 ctx_id_entry_init_one(&ctxp[i], domain, ctx_root); 198 } 199 } else { 200 ctx_id_entry_init_one(ctxp, domain, ctx_root); 201 } 202 dmar_flush_ctx_to_ram(unit, ctxp); 203 } 204 205 static int 206 dmar_flush_for_ctx_entry(struct dmar_unit *dmar, bool force) 207 { 208 int error; 209 210 /* 211 * If dmar declares Caching Mode as Set, follow 11.5 "Caching 212 * Mode Consideration" and do the (global) invalidation of the 213 * negative TLB entries. 214 */ 215 if ((dmar->hw_cap & DMAR_CAP_CM) == 0 && !force) 216 return (0); 217 if (dmar->qi_enabled) { 218 dmar_qi_invalidate_ctx_glob_locked(dmar); 219 if ((dmar->hw_ecap & DMAR_ECAP_DI) != 0 || force) 220 dmar_qi_invalidate_iotlb_glob_locked(dmar); 221 return (0); 222 } 223 error = dmar_inv_ctx_glob(dmar); 224 if (error == 0 && ((dmar->hw_ecap & DMAR_ECAP_DI) != 0 || force)) 225 error = dmar_inv_iotlb_glob(dmar); 226 return (error); 227 } 228 229 static int 230 domain_init_rmrr(struct dmar_domain *domain, device_t dev, int bus, 231 int slot, int func, int dev_domain, int dev_busno, 232 const void *dev_path, int dev_path_len) 233 { 234 struct iommu_map_entries_tailq rmrr_entries; 235 struct iommu_map_entry *entry, *entry1; 236 vm_page_t *ma; 237 iommu_gaddr_t start, end; 238 vm_pindex_t size, i; 239 int error, error1; 240 241 error = 0; 242 TAILQ_INIT(&rmrr_entries); 243 dmar_dev_parse_rmrr(domain, dev_domain, dev_busno, dev_path, 244 dev_path_len, &rmrr_entries); 245 TAILQ_FOREACH_SAFE(entry, &rmrr_entries, dmamap_link, entry1) { 246 /* 247 * VT-d specification requires that the start of an 248 * RMRR entry is 4k-aligned. Buggy BIOSes put 249 * anything into the start and end fields. Truncate 250 * and round as neccesary. 251 * 252 * We also allow the overlapping RMRR entries, see 253 * iommu_gas_alloc_region(). 254 */ 255 start = entry->start; 256 end = entry->end; 257 if (bootverbose) 258 printf("dmar%d ctx pci%d:%d:%d RMRR [%#jx, %#jx]\n", 259 domain->iodom.iommu->unit, bus, slot, func, 260 (uintmax_t)start, (uintmax_t)end); 261 entry->start = trunc_page(start); 262 entry->end = round_page(end); 263 if (entry->start == entry->end) { 264 /* Workaround for some AMI (?) BIOSes */ 265 if (bootverbose) { 266 if (dev != NULL) 267 device_printf(dev, ""); 268 printf("pci%d:%d:%d ", bus, slot, func); 269 printf("BIOS bug: dmar%d RMRR " 270 "region (%jx, %jx) corrected\n", 271 domain->iodom.iommu->unit, start, end); 272 } 273 entry->end += DMAR_PAGE_SIZE * 0x20; 274 } 275 size = OFF_TO_IDX(entry->end - entry->start); 276 ma = malloc(sizeof(vm_page_t) * size, M_TEMP, M_WAITOK); 277 for (i = 0; i < size; i++) { 278 ma[i] = vm_page_getfake(entry->start + PAGE_SIZE * i, 279 VM_MEMATTR_DEFAULT); 280 } 281 error1 = iommu_gas_map_region(DOM2IODOM(domain), entry, 282 IOMMU_MAP_ENTRY_READ | IOMMU_MAP_ENTRY_WRITE, 283 IOMMU_MF_CANWAIT | IOMMU_MF_RMRR, ma); 284 /* 285 * Non-failed RMRR entries are owned by context rb 286 * tree. Get rid of the failed entry, but do not stop 287 * the loop. Rest of the parsed RMRR entries are 288 * loaded and removed on the context destruction. 289 */ 290 if (error1 == 0 && entry->end != entry->start) { 291 IOMMU_LOCK(domain->iodom.iommu); 292 domain->refs++; /* XXXKIB prevent free */ 293 domain->iodom.flags |= IOMMU_DOMAIN_RMRR; 294 IOMMU_UNLOCK(domain->iodom.iommu); 295 } else { 296 if (error1 != 0) { 297 if (dev != NULL) 298 device_printf(dev, ""); 299 printf("pci%d:%d:%d ", bus, slot, func); 300 printf( 301 "dmar%d failed to map RMRR region (%jx, %jx) %d\n", 302 domain->iodom.iommu->unit, start, end, 303 error1); 304 error = error1; 305 } 306 TAILQ_REMOVE(&rmrr_entries, entry, dmamap_link); 307 iommu_gas_free_entry(entry); 308 } 309 for (i = 0; i < size; i++) 310 vm_page_putfake(ma[i]); 311 free(ma, M_TEMP); 312 } 313 return (error); 314 } 315 316 /* 317 * PCI memory address space is shared between memory-mapped devices (MMIO) and 318 * host memory (which may be remapped by an IOMMU). Device accesses to an 319 * address within a memory aperture in a PCIe root port will be treated as 320 * peer-to-peer and not forwarded to an IOMMU. To avoid this, reserve the 321 * address space of the root port's memory apertures in the address space used 322 * by the IOMMU for remapping. 323 */ 324 static int 325 dmar_reserve_pci_regions(struct dmar_domain *domain, device_t dev) 326 { 327 struct iommu_domain *iodom; 328 device_t root; 329 uint32_t val; 330 uint64_t base, limit; 331 int error; 332 333 iodom = DOM2IODOM(domain); 334 335 root = pci_find_pcie_root_port(dev); 336 if (root == NULL) 337 return (0); 338 339 /* Disable downstream memory */ 340 base = PCI_PPBMEMBASE(0, pci_read_config(root, PCIR_MEMBASE_1, 2)); 341 limit = PCI_PPBMEMLIMIT(0, pci_read_config(root, PCIR_MEMLIMIT_1, 2)); 342 error = iommu_gas_reserve_region_extend(iodom, base, limit + 1); 343 if (bootverbose || error != 0) 344 device_printf(dev, "DMAR reserve [%#jx-%#jx] (error %d)\n", 345 base, limit + 1, error); 346 if (error != 0) 347 return (error); 348 349 /* Disable downstream prefetchable memory */ 350 val = pci_read_config(root, PCIR_PMBASEL_1, 2); 351 if (val != 0 || pci_read_config(root, PCIR_PMLIMITL_1, 2) != 0) { 352 if ((val & PCIM_BRPM_MASK) == PCIM_BRPM_64) { 353 base = PCI_PPBMEMBASE( 354 pci_read_config(root, PCIR_PMBASEH_1, 4), 355 val); 356 limit = PCI_PPBMEMLIMIT( 357 pci_read_config(root, PCIR_PMLIMITH_1, 4), 358 pci_read_config(root, PCIR_PMLIMITL_1, 2)); 359 } else { 360 base = PCI_PPBMEMBASE(0, val); 361 limit = PCI_PPBMEMLIMIT(0, 362 pci_read_config(root, PCIR_PMLIMITL_1, 2)); 363 } 364 error = iommu_gas_reserve_region_extend(iodom, base, 365 limit + 1); 366 if (bootverbose || error != 0) 367 device_printf(dev, "DMAR reserve [%#jx-%#jx] " 368 "(error %d)\n", base, limit + 1, error); 369 if (error != 0) 370 return (error); 371 } 372 373 return (error); 374 } 375 376 static struct dmar_domain * 377 dmar_domain_alloc(struct dmar_unit *dmar, bool id_mapped) 378 { 379 struct iommu_domain *iodom; 380 struct iommu_unit *unit; 381 struct dmar_domain *domain; 382 int error, id, mgaw; 383 384 id = alloc_unr(dmar->domids); 385 if (id == -1) 386 return (NULL); 387 domain = malloc(sizeof(*domain), M_DMAR_DOMAIN, M_WAITOK | M_ZERO); 388 iodom = DOM2IODOM(domain); 389 unit = DMAR2IOMMU(dmar); 390 domain->domain = id; 391 LIST_INIT(&domain->contexts); 392 iommu_domain_init(unit, iodom, &dmar_domain_map_ops); 393 394 domain->dmar = dmar; 395 396 /* 397 * For now, use the maximal usable physical address of the 398 * installed memory to calculate the mgaw on id_mapped domain. 399 * It is useful for the identity mapping, and less so for the 400 * virtualized bus address space. 401 */ 402 domain->iodom.end = id_mapped ? ptoa(Maxmem) : BUS_SPACE_MAXADDR; 403 mgaw = dmar_maxaddr2mgaw(dmar, domain->iodom.end, !id_mapped); 404 error = domain_set_agaw(domain, mgaw); 405 if (error != 0) 406 goto fail; 407 if (!id_mapped) 408 /* Use all supported address space for remapping. */ 409 domain->iodom.end = 1ULL << (domain->agaw - 1); 410 411 iommu_gas_init_domain(DOM2IODOM(domain)); 412 413 if (id_mapped) { 414 if ((dmar->hw_ecap & DMAR_ECAP_PT) == 0) { 415 domain->pgtbl_obj = domain_get_idmap_pgtbl(domain, 416 domain->iodom.end); 417 } 418 domain->iodom.flags |= IOMMU_DOMAIN_IDMAP; 419 } else { 420 error = domain_alloc_pgtbl(domain); 421 if (error != 0) 422 goto fail; 423 /* Disable local apic region access */ 424 error = iommu_gas_reserve_region(iodom, 0xfee00000, 425 0xfeefffff + 1, &iodom->msi_entry); 426 if (error != 0) 427 goto fail; 428 } 429 return (domain); 430 431 fail: 432 dmar_domain_destroy(domain); 433 return (NULL); 434 } 435 436 static struct dmar_ctx * 437 dmar_ctx_alloc(struct dmar_domain *domain, uint16_t rid) 438 { 439 struct dmar_ctx *ctx; 440 441 ctx = malloc(sizeof(*ctx), M_DMAR_CTX, M_WAITOK | M_ZERO); 442 ctx->context.domain = DOM2IODOM(domain); 443 ctx->context.tag = malloc(sizeof(struct bus_dma_tag_iommu), 444 M_DMAR_CTX, M_WAITOK | M_ZERO); 445 ctx->context.rid = rid; 446 ctx->refs = 1; 447 return (ctx); 448 } 449 450 static void 451 dmar_ctx_link(struct dmar_ctx *ctx) 452 { 453 struct dmar_domain *domain; 454 455 domain = CTX2DOM(ctx); 456 IOMMU_ASSERT_LOCKED(domain->iodom.iommu); 457 KASSERT(domain->refs >= domain->ctx_cnt, 458 ("dom %p ref underflow %d %d", domain, domain->refs, 459 domain->ctx_cnt)); 460 domain->refs++; 461 domain->ctx_cnt++; 462 LIST_INSERT_HEAD(&domain->contexts, ctx, link); 463 } 464 465 static void 466 dmar_ctx_unlink(struct dmar_ctx *ctx) 467 { 468 struct dmar_domain *domain; 469 470 domain = CTX2DOM(ctx); 471 IOMMU_ASSERT_LOCKED(domain->iodom.iommu); 472 KASSERT(domain->refs > 0, 473 ("domain %p ctx dtr refs %d", domain, domain->refs)); 474 KASSERT(domain->ctx_cnt >= domain->refs, 475 ("domain %p ctx dtr refs %d ctx_cnt %d", domain, 476 domain->refs, domain->ctx_cnt)); 477 domain->refs--; 478 domain->ctx_cnt--; 479 LIST_REMOVE(ctx, link); 480 } 481 482 static void 483 dmar_domain_destroy(struct dmar_domain *domain) 484 { 485 struct iommu_domain *iodom; 486 struct dmar_unit *dmar; 487 488 iodom = DOM2IODOM(domain); 489 490 KASSERT(TAILQ_EMPTY(&domain->iodom.unload_entries), 491 ("unfinished unloads %p", domain)); 492 KASSERT(LIST_EMPTY(&domain->contexts), 493 ("destroying dom %p with contexts", domain)); 494 KASSERT(domain->ctx_cnt == 0, 495 ("destroying dom %p with ctx_cnt %d", domain, domain->ctx_cnt)); 496 KASSERT(domain->refs == 0, 497 ("destroying dom %p with refs %d", domain, domain->refs)); 498 if ((domain->iodom.flags & IOMMU_DOMAIN_GAS_INITED) != 0) { 499 DMAR_DOMAIN_LOCK(domain); 500 iommu_gas_fini_domain(iodom); 501 DMAR_DOMAIN_UNLOCK(domain); 502 } 503 if ((domain->iodom.flags & IOMMU_DOMAIN_PGTBL_INITED) != 0) { 504 if (domain->pgtbl_obj != NULL) 505 DMAR_DOMAIN_PGLOCK(domain); 506 domain_free_pgtbl(domain); 507 } 508 iommu_domain_fini(iodom); 509 dmar = DOM2DMAR(domain); 510 free_unr(dmar->domids, domain->domain); 511 free(domain, M_DMAR_DOMAIN); 512 } 513 514 static struct dmar_ctx * 515 dmar_get_ctx_for_dev1(struct dmar_unit *dmar, device_t dev, uint16_t rid, 516 int dev_domain, int dev_busno, const void *dev_path, int dev_path_len, 517 bool id_mapped, bool rmrr_init) 518 { 519 struct dmar_domain *domain, *domain1; 520 struct dmar_ctx *ctx, *ctx1; 521 struct iommu_unit *unit __diagused; 522 dmar_ctx_entry_t *ctxp; 523 struct sf_buf *sf; 524 int bus, slot, func, error; 525 bool enable; 526 527 if (dev != NULL) { 528 bus = pci_get_bus(dev); 529 slot = pci_get_slot(dev); 530 func = pci_get_function(dev); 531 } else { 532 bus = PCI_RID2BUS(rid); 533 slot = PCI_RID2SLOT(rid); 534 func = PCI_RID2FUNC(rid); 535 } 536 enable = false; 537 TD_PREP_PINNED_ASSERT; 538 unit = DMAR2IOMMU(dmar); 539 DMAR_LOCK(dmar); 540 KASSERT(!iommu_is_buswide_ctx(unit, bus) || (slot == 0 && func == 0), 541 ("iommu%d pci%d:%d:%d get_ctx for buswide", dmar->iommu.unit, bus, 542 slot, func)); 543 ctx = dmar_find_ctx_locked(dmar, rid); 544 error = 0; 545 if (ctx == NULL) { 546 /* 547 * Perform the allocations which require sleep or have 548 * higher chance to succeed if the sleep is allowed. 549 */ 550 DMAR_UNLOCK(dmar); 551 dmar_ensure_ctx_page(dmar, PCI_RID2BUS(rid)); 552 domain1 = dmar_domain_alloc(dmar, id_mapped); 553 if (domain1 == NULL) { 554 TD_PINNED_ASSERT; 555 return (NULL); 556 } 557 if (!id_mapped) { 558 error = domain_init_rmrr(domain1, dev, bus, 559 slot, func, dev_domain, dev_busno, dev_path, 560 dev_path_len); 561 if (error == 0 && dev != NULL) 562 error = dmar_reserve_pci_regions(domain1, dev); 563 if (error != 0) { 564 dmar_domain_destroy(domain1); 565 TD_PINNED_ASSERT; 566 return (NULL); 567 } 568 } 569 ctx1 = dmar_ctx_alloc(domain1, rid); 570 ctxp = dmar_map_ctx_entry(ctx1, &sf); 571 DMAR_LOCK(dmar); 572 573 /* 574 * Recheck the contexts, other thread might have 575 * already allocated needed one. 576 */ 577 ctx = dmar_find_ctx_locked(dmar, rid); 578 if (ctx == NULL) { 579 domain = domain1; 580 ctx = ctx1; 581 dmar_ctx_link(ctx); 582 ctx->context.tag->owner = dev; 583 device_tag_init(ctx, dev); 584 585 /* 586 * This is the first activated context for the 587 * DMAR unit. Enable the translation after 588 * everything is set up. 589 */ 590 if (LIST_EMPTY(&dmar->domains)) 591 enable = true; 592 LIST_INSERT_HEAD(&dmar->domains, domain, link); 593 ctx_id_entry_init(ctx, ctxp, false, bus); 594 if (dev != NULL) { 595 device_printf(dev, 596 "dmar%d pci%d:%d:%d:%d rid %x domain %d mgaw %d " 597 "agaw %d %s-mapped\n", 598 dmar->iommu.unit, dmar->segment, bus, slot, 599 func, rid, domain->domain, domain->mgaw, 600 domain->agaw, id_mapped ? "id" : "re"); 601 } 602 dmar_unmap_pgtbl(sf); 603 } else { 604 dmar_unmap_pgtbl(sf); 605 dmar_domain_destroy(domain1); 606 /* Nothing needs to be done to destroy ctx1. */ 607 free(ctx1, M_DMAR_CTX); 608 domain = CTX2DOM(ctx); 609 ctx->refs++; /* tag referenced us */ 610 } 611 } else { 612 domain = CTX2DOM(ctx); 613 if (ctx->context.tag->owner == NULL) 614 ctx->context.tag->owner = dev; 615 ctx->refs++; /* tag referenced us */ 616 } 617 618 error = dmar_flush_for_ctx_entry(dmar, enable); 619 if (error != 0) { 620 dmar_free_ctx_locked(dmar, ctx); 621 TD_PINNED_ASSERT; 622 return (NULL); 623 } 624 625 /* 626 * The dmar lock was potentially dropped between check for the 627 * empty context list and now. Recheck the state of GCMD_TE 628 * to avoid unneeded command. 629 */ 630 if (enable && !rmrr_init && (dmar->hw_gcmd & DMAR_GCMD_TE) == 0) { 631 error = dmar_disable_protected_regions(dmar); 632 if (error != 0) 633 printf("dmar%d: Failed to disable protected regions\n", 634 dmar->iommu.unit); 635 error = dmar_enable_translation(dmar); 636 if (error == 0) { 637 if (bootverbose) { 638 printf("dmar%d: enabled translation\n", 639 dmar->iommu.unit); 640 } 641 } else { 642 printf("dmar%d: enabling translation failed, " 643 "error %d\n", dmar->iommu.unit, error); 644 dmar_free_ctx_locked(dmar, ctx); 645 TD_PINNED_ASSERT; 646 return (NULL); 647 } 648 } 649 DMAR_UNLOCK(dmar); 650 TD_PINNED_ASSERT; 651 return (ctx); 652 } 653 654 struct dmar_ctx * 655 dmar_get_ctx_for_dev(struct dmar_unit *dmar, device_t dev, uint16_t rid, 656 bool id_mapped, bool rmrr_init) 657 { 658 int dev_domain, dev_path_len, dev_busno; 659 660 dev_domain = pci_get_domain(dev); 661 dev_path_len = dmar_dev_depth(dev); 662 ACPI_DMAR_PCI_PATH dev_path[dev_path_len]; 663 dmar_dev_path(dev, &dev_busno, dev_path, dev_path_len); 664 return (dmar_get_ctx_for_dev1(dmar, dev, rid, dev_domain, dev_busno, 665 dev_path, dev_path_len, id_mapped, rmrr_init)); 666 } 667 668 struct dmar_ctx * 669 dmar_get_ctx_for_devpath(struct dmar_unit *dmar, uint16_t rid, 670 int dev_domain, int dev_busno, 671 const void *dev_path, int dev_path_len, 672 bool id_mapped, bool rmrr_init) 673 { 674 675 return (dmar_get_ctx_for_dev1(dmar, NULL, rid, dev_domain, dev_busno, 676 dev_path, dev_path_len, id_mapped, rmrr_init)); 677 } 678 679 int 680 dmar_move_ctx_to_domain(struct dmar_domain *domain, struct dmar_ctx *ctx) 681 { 682 struct dmar_unit *dmar; 683 struct dmar_domain *old_domain; 684 dmar_ctx_entry_t *ctxp; 685 struct sf_buf *sf; 686 int error; 687 688 dmar = domain->dmar; 689 old_domain = CTX2DOM(ctx); 690 if (domain == old_domain) 691 return (0); 692 KASSERT(old_domain->iodom.iommu == domain->iodom.iommu, 693 ("domain %p %u moving between dmars %u %u", domain, 694 domain->domain, old_domain->iodom.iommu->unit, 695 domain->iodom.iommu->unit)); 696 TD_PREP_PINNED_ASSERT; 697 698 ctxp = dmar_map_ctx_entry(ctx, &sf); 699 DMAR_LOCK(dmar); 700 dmar_ctx_unlink(ctx); 701 ctx->context.domain = &domain->iodom; 702 dmar_ctx_link(ctx); 703 ctx_id_entry_init(ctx, ctxp, true, PCI_BUSMAX + 100); 704 dmar_unmap_pgtbl(sf); 705 error = dmar_flush_for_ctx_entry(dmar, true); 706 /* If flush failed, rolling back would not work as well. */ 707 printf("dmar%d rid %x domain %d->%d %s-mapped\n", 708 dmar->iommu.unit, ctx->context.rid, old_domain->domain, 709 domain->domain, (domain->iodom.flags & IOMMU_DOMAIN_IDMAP) != 0 ? 710 "id" : "re"); 711 dmar_unref_domain_locked(dmar, old_domain); 712 TD_PINNED_ASSERT; 713 return (error); 714 } 715 716 static void 717 dmar_unref_domain_locked(struct dmar_unit *dmar, struct dmar_domain *domain) 718 { 719 720 DMAR_ASSERT_LOCKED(dmar); 721 KASSERT(domain->refs >= 1, 722 ("dmar %d domain %p refs %u", dmar->iommu.unit, domain, 723 domain->refs)); 724 KASSERT(domain->refs > domain->ctx_cnt, 725 ("dmar %d domain %p refs %d ctx_cnt %d", dmar->iommu.unit, domain, 726 domain->refs, domain->ctx_cnt)); 727 728 if (domain->refs > 1) { 729 domain->refs--; 730 DMAR_UNLOCK(dmar); 731 return; 732 } 733 734 KASSERT((domain->iodom.flags & IOMMU_DOMAIN_RMRR) == 0, 735 ("lost ref on RMRR domain %p", domain)); 736 737 LIST_REMOVE(domain, link); 738 DMAR_UNLOCK(dmar); 739 740 taskqueue_drain(dmar->iommu.delayed_taskqueue, 741 &domain->iodom.unload_task); 742 dmar_domain_destroy(domain); 743 } 744 745 void 746 dmar_free_ctx_locked(struct dmar_unit *dmar, struct dmar_ctx *ctx) 747 { 748 struct sf_buf *sf; 749 dmar_ctx_entry_t *ctxp; 750 struct dmar_domain *domain; 751 752 DMAR_ASSERT_LOCKED(dmar); 753 KASSERT(ctx->refs >= 1, 754 ("dmar %p ctx %p refs %u", dmar, ctx, ctx->refs)); 755 756 /* 757 * If our reference is not last, only the dereference should 758 * be performed. 759 */ 760 if (ctx->refs > 1) { 761 ctx->refs--; 762 DMAR_UNLOCK(dmar); 763 return; 764 } 765 766 KASSERT((ctx->context.flags & IOMMU_CTX_DISABLED) == 0, 767 ("lost ref on disabled ctx %p", ctx)); 768 769 /* 770 * Otherwise, the context entry must be cleared before the 771 * page table is destroyed. The mapping of the context 772 * entries page could require sleep, unlock the dmar. 773 */ 774 DMAR_UNLOCK(dmar); 775 TD_PREP_PINNED_ASSERT; 776 ctxp = dmar_map_ctx_entry(ctx, &sf); 777 DMAR_LOCK(dmar); 778 KASSERT(ctx->refs >= 1, 779 ("dmar %p ctx %p refs %u", dmar, ctx, ctx->refs)); 780 781 /* 782 * Other thread might have referenced the context, in which 783 * case again only the dereference should be performed. 784 */ 785 if (ctx->refs > 1) { 786 ctx->refs--; 787 DMAR_UNLOCK(dmar); 788 dmar_unmap_pgtbl(sf); 789 TD_PINNED_ASSERT; 790 return; 791 } 792 793 KASSERT((ctx->context.flags & IOMMU_CTX_DISABLED) == 0, 794 ("lost ref on disabled ctx %p", ctx)); 795 796 /* 797 * Clear the context pointer and flush the caches. 798 * XXXKIB: cannot do this if any RMRR entries are still present. 799 */ 800 dmar_pte_clear(&ctxp->ctx1); 801 ctxp->ctx2 = 0; 802 dmar_flush_ctx_to_ram(dmar, ctxp); 803 dmar_inv_ctx_glob(dmar); 804 if ((dmar->hw_ecap & DMAR_ECAP_DI) != 0) { 805 if (dmar->qi_enabled) 806 dmar_qi_invalidate_iotlb_glob_locked(dmar); 807 else 808 dmar_inv_iotlb_glob(dmar); 809 } 810 dmar_unmap_pgtbl(sf); 811 domain = CTX2DOM(ctx); 812 dmar_ctx_unlink(ctx); 813 free(ctx->context.tag, M_DMAR_CTX); 814 free(ctx, M_DMAR_CTX); 815 dmar_unref_domain_locked(dmar, domain); 816 TD_PINNED_ASSERT; 817 } 818 819 void 820 dmar_free_ctx(struct dmar_ctx *ctx) 821 { 822 struct dmar_unit *dmar; 823 824 dmar = CTX2DMAR(ctx); 825 DMAR_LOCK(dmar); 826 dmar_free_ctx_locked(dmar, ctx); 827 } 828 829 /* 830 * Returns with the domain locked. 831 */ 832 struct dmar_ctx * 833 dmar_find_ctx_locked(struct dmar_unit *dmar, uint16_t rid) 834 { 835 struct dmar_domain *domain; 836 struct dmar_ctx *ctx; 837 838 DMAR_ASSERT_LOCKED(dmar); 839 840 LIST_FOREACH(domain, &dmar->domains, link) { 841 LIST_FOREACH(ctx, &domain->contexts, link) { 842 if (ctx->context.rid == rid) 843 return (ctx); 844 } 845 } 846 return (NULL); 847 } 848 849 void 850 dmar_domain_free_entry(struct iommu_map_entry *entry, bool free) 851 { 852 if ((entry->flags & IOMMU_MAP_ENTRY_RMRR) != 0) 853 iommu_gas_free_region(entry); 854 else 855 iommu_gas_free_space(entry); 856 if (free) 857 iommu_gas_free_entry(entry); 858 else 859 entry->flags = 0; 860 } 861 862 /* 863 * If the given value for "free" is true, then the caller must not be using 864 * the entry's dmamap_link field. 865 */ 866 void 867 iommu_domain_unload_entry(struct iommu_map_entry *entry, bool free, 868 bool cansleep) 869 { 870 struct dmar_domain *domain; 871 struct dmar_unit *unit; 872 873 domain = IODOM2DOM(entry->domain); 874 unit = DOM2DMAR(domain); 875 876 /* 877 * If "free" is false, then the IOTLB invalidation must be performed 878 * synchronously. Otherwise, the caller might free the entry before 879 * dmar_qi_task() is finished processing it. 880 */ 881 if (unit->qi_enabled) { 882 if (free) { 883 DMAR_LOCK(unit); 884 dmar_qi_invalidate_locked(domain, entry, true); 885 DMAR_UNLOCK(unit); 886 } else { 887 dmar_qi_invalidate_sync(domain, entry->start, 888 entry->end - entry->start, cansleep); 889 dmar_domain_free_entry(entry, false); 890 } 891 } else { 892 domain_flush_iotlb_sync(domain, entry->start, entry->end - 893 entry->start); 894 dmar_domain_free_entry(entry, free); 895 } 896 } 897 898 static bool 899 dmar_domain_unload_emit_wait(struct dmar_domain *domain, 900 struct iommu_map_entry *entry) 901 { 902 903 if (TAILQ_NEXT(entry, dmamap_link) == NULL) 904 return (true); 905 return (domain->batch_no++ % dmar_batch_coalesce == 0); 906 } 907 908 void 909 iommu_domain_unload(struct iommu_domain *iodom, 910 struct iommu_map_entries_tailq *entries, bool cansleep) 911 { 912 struct dmar_domain *domain; 913 struct dmar_unit *unit; 914 struct iommu_map_entry *entry, *entry1; 915 int error __diagused; 916 917 domain = IODOM2DOM(iodom); 918 unit = DOM2DMAR(domain); 919 920 TAILQ_FOREACH_SAFE(entry, entries, dmamap_link, entry1) { 921 KASSERT((entry->flags & IOMMU_MAP_ENTRY_MAP) != 0, 922 ("not mapped entry %p %p", domain, entry)); 923 error = iodom->ops->unmap(iodom, entry->start, entry->end - 924 entry->start, cansleep ? IOMMU_PGF_WAITOK : 0); 925 KASSERT(error == 0, ("unmap %p error %d", domain, error)); 926 if (!unit->qi_enabled) { 927 domain_flush_iotlb_sync(domain, entry->start, 928 entry->end - entry->start); 929 TAILQ_REMOVE(entries, entry, dmamap_link); 930 dmar_domain_free_entry(entry, true); 931 } 932 } 933 if (TAILQ_EMPTY(entries)) 934 return; 935 936 KASSERT(unit->qi_enabled, ("loaded entry left")); 937 DMAR_LOCK(unit); 938 while ((entry = TAILQ_FIRST(entries)) != NULL) { 939 TAILQ_REMOVE(entries, entry, dmamap_link); 940 dmar_qi_invalidate_locked(domain, entry, 941 dmar_domain_unload_emit_wait(domain, entry)); 942 } 943 DMAR_UNLOCK(unit); 944 } 945 946 struct iommu_ctx * 947 iommu_get_ctx(struct iommu_unit *iommu, device_t dev, uint16_t rid, 948 bool id_mapped, bool rmrr_init) 949 { 950 struct dmar_unit *dmar; 951 struct dmar_ctx *ret; 952 953 dmar = IOMMU2DMAR(iommu); 954 955 ret = dmar_get_ctx_for_dev(dmar, dev, rid, id_mapped, rmrr_init); 956 957 return (CTX2IOCTX(ret)); 958 } 959 960 void 961 iommu_free_ctx_locked(struct iommu_unit *iommu, struct iommu_ctx *context) 962 { 963 struct dmar_unit *dmar; 964 struct dmar_ctx *ctx; 965 966 dmar = IOMMU2DMAR(iommu); 967 ctx = IOCTX2CTX(context); 968 969 dmar_free_ctx_locked(dmar, ctx); 970 } 971 972 void 973 iommu_free_ctx(struct iommu_ctx *context) 974 { 975 struct dmar_ctx *ctx; 976 977 ctx = IOCTX2CTX(context); 978 979 dmar_free_ctx(ctx); 980 } 981