1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 3 * 4 * Copyright (c) 2013 The FreeBSD Foundation 5 * All rights reserved. 6 * 7 * This software was developed by Konstantin Belousov <kib@FreeBSD.org> 8 * under sponsorship from the FreeBSD Foundation. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 20 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 21 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 22 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 23 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 24 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 25 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 26 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 27 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 28 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 29 * SUCH DAMAGE. 30 */ 31 32 #include <sys/cdefs.h> 33 __FBSDID("$FreeBSD$"); 34 35 #include <sys/param.h> 36 #include <sys/systm.h> 37 #include <sys/malloc.h> 38 #include <sys/bus.h> 39 #include <sys/interrupt.h> 40 #include <sys/kernel.h> 41 #include <sys/ktr.h> 42 #include <sys/limits.h> 43 #include <sys/lock.h> 44 #include <sys/memdesc.h> 45 #include <sys/mutex.h> 46 #include <sys/proc.h> 47 #include <sys/rwlock.h> 48 #include <sys/rman.h> 49 #include <sys/sysctl.h> 50 #include <sys/taskqueue.h> 51 #include <sys/tree.h> 52 #include <sys/uio.h> 53 #include <sys/vmem.h> 54 #include <vm/vm.h> 55 #include <vm/vm_extern.h> 56 #include <vm/vm_kern.h> 57 #include <vm/vm_object.h> 58 #include <vm/vm_page.h> 59 #include <vm/vm_pager.h> 60 #include <vm/vm_map.h> 61 #include <contrib/dev/acpica/include/acpi.h> 62 #include <contrib/dev/acpica/include/accommon.h> 63 #include <dev/pci/pcireg.h> 64 #include <dev/pci/pcivar.h> 65 #include <machine/atomic.h> 66 #include <machine/bus.h> 67 #include <machine/md_var.h> 68 #include <machine/specialreg.h> 69 #include <x86/include/busdma_impl.h> 70 #include <dev/iommu/busdma_iommu.h> 71 #include <x86/iommu/intel_reg.h> 72 #include <x86/iommu/intel_dmar.h> 73 74 static MALLOC_DEFINE(M_DMAR_CTX, "dmar_ctx", "Intel DMAR Context"); 75 static MALLOC_DEFINE(M_DMAR_DOMAIN, "dmar_dom", "Intel DMAR Domain"); 76 77 static void dmar_unref_domain_locked(struct dmar_unit *dmar, 78 struct dmar_domain *domain); 79 static void dmar_domain_destroy(struct dmar_domain *domain); 80 81 static void 82 dmar_ensure_ctx_page(struct dmar_unit *dmar, int bus) 83 { 84 struct sf_buf *sf; 85 dmar_root_entry_t *re; 86 vm_page_t ctxm; 87 88 /* 89 * Allocated context page must be linked. 90 */ 91 ctxm = dmar_pgalloc(dmar->ctx_obj, 1 + bus, IOMMU_PGF_NOALLOC); 92 if (ctxm != NULL) 93 return; 94 95 /* 96 * Page not present, allocate and link. Note that other 97 * thread might execute this sequence in parallel. This 98 * should be safe, because the context entries written by both 99 * threads are equal. 100 */ 101 TD_PREP_PINNED_ASSERT; 102 ctxm = dmar_pgalloc(dmar->ctx_obj, 1 + bus, IOMMU_PGF_ZERO | 103 IOMMU_PGF_WAITOK); 104 re = dmar_map_pgtbl(dmar->ctx_obj, 0, IOMMU_PGF_NOALLOC, &sf); 105 re += bus; 106 dmar_pte_store(&re->r1, DMAR_ROOT_R1_P | (DMAR_ROOT_R1_CTP_MASK & 107 VM_PAGE_TO_PHYS(ctxm))); 108 dmar_flush_root_to_ram(dmar, re); 109 dmar_unmap_pgtbl(sf); 110 TD_PINNED_ASSERT; 111 } 112 113 static dmar_ctx_entry_t * 114 dmar_map_ctx_entry(struct dmar_ctx *ctx, struct sf_buf **sfp) 115 { 116 struct dmar_unit *dmar; 117 dmar_ctx_entry_t *ctxp; 118 119 dmar = CTX2DMAR(ctx); 120 121 ctxp = dmar_map_pgtbl(dmar->ctx_obj, 1 + 122 PCI_RID2BUS(ctx->rid), IOMMU_PGF_NOALLOC | IOMMU_PGF_WAITOK, sfp); 123 ctxp += ctx->rid & 0xff; 124 return (ctxp); 125 } 126 127 static void 128 device_tag_init(struct dmar_ctx *ctx, device_t dev) 129 { 130 struct dmar_domain *domain; 131 bus_addr_t maxaddr; 132 133 domain = CTX2DOM(ctx); 134 maxaddr = MIN(domain->iodom.end, BUS_SPACE_MAXADDR); 135 ctx->context.tag->common.ref_count = 1; /* Prevent free */ 136 ctx->context.tag->common.impl = &bus_dma_iommu_impl; 137 ctx->context.tag->common.boundary = 0; 138 ctx->context.tag->common.lowaddr = maxaddr; 139 ctx->context.tag->common.highaddr = maxaddr; 140 ctx->context.tag->common.maxsize = maxaddr; 141 ctx->context.tag->common.nsegments = BUS_SPACE_UNRESTRICTED; 142 ctx->context.tag->common.maxsegsz = maxaddr; 143 ctx->context.tag->ctx = CTX2IOCTX(ctx); 144 ctx->context.tag->owner = dev; 145 } 146 147 static void 148 ctx_id_entry_init_one(dmar_ctx_entry_t *ctxp, struct dmar_domain *domain, 149 vm_page_t ctx_root) 150 { 151 /* 152 * For update due to move, the store is not atomic. It is 153 * possible that DMAR read upper doubleword, while low 154 * doubleword is not yet updated. The domain id is stored in 155 * the upper doubleword, while the table pointer in the lower. 156 * 157 * There is no good solution, for the same reason it is wrong 158 * to clear P bit in the ctx entry for update. 159 */ 160 dmar_pte_store1(&ctxp->ctx2, DMAR_CTX2_DID(domain->domain) | 161 domain->awlvl); 162 if (ctx_root == NULL) { 163 dmar_pte_store1(&ctxp->ctx1, DMAR_CTX1_T_PASS | DMAR_CTX1_P); 164 } else { 165 dmar_pte_store1(&ctxp->ctx1, DMAR_CTX1_T_UNTR | 166 (DMAR_CTX1_ASR_MASK & VM_PAGE_TO_PHYS(ctx_root)) | 167 DMAR_CTX1_P); 168 } 169 } 170 171 static void 172 ctx_id_entry_init(struct dmar_ctx *ctx, dmar_ctx_entry_t *ctxp, bool move, 173 int busno) 174 { 175 struct dmar_unit *unit; 176 struct dmar_domain *domain; 177 vm_page_t ctx_root; 178 int i; 179 180 domain = CTX2DOM(ctx); 181 unit = DOM2DMAR(domain); 182 KASSERT(move || (ctxp->ctx1 == 0 && ctxp->ctx2 == 0), 183 ("dmar%d: initialized ctx entry %d:%d:%d 0x%jx 0x%jx", 184 unit->iommu.unit, busno, pci_get_slot(ctx->context.tag->owner), 185 pci_get_function(ctx->context.tag->owner), 186 ctxp->ctx1, ctxp->ctx2)); 187 188 if ((domain->iodom.flags & IOMMU_DOMAIN_IDMAP) != 0 && 189 (unit->hw_ecap & DMAR_ECAP_PT) != 0) { 190 KASSERT(domain->pgtbl_obj == NULL, 191 ("ctx %p non-null pgtbl_obj", ctx)); 192 ctx_root = NULL; 193 } else { 194 ctx_root = dmar_pgalloc(domain->pgtbl_obj, 0, 195 IOMMU_PGF_NOALLOC); 196 } 197 198 if (iommu_is_buswide_ctx(DMAR2IOMMU(unit), busno)) { 199 MPASS(!move); 200 for (i = 0; i <= PCI_BUSMAX; i++) { 201 ctx_id_entry_init_one(&ctxp[i], domain, ctx_root); 202 } 203 } else { 204 ctx_id_entry_init_one(ctxp, domain, ctx_root); 205 } 206 dmar_flush_ctx_to_ram(unit, ctxp); 207 } 208 209 static int 210 dmar_flush_for_ctx_entry(struct dmar_unit *dmar, bool force) 211 { 212 int error; 213 214 /* 215 * If dmar declares Caching Mode as Set, follow 11.5 "Caching 216 * Mode Consideration" and do the (global) invalidation of the 217 * negative TLB entries. 218 */ 219 if ((dmar->hw_cap & DMAR_CAP_CM) == 0 && !force) 220 return (0); 221 if (dmar->qi_enabled) { 222 dmar_qi_invalidate_ctx_glob_locked(dmar); 223 if ((dmar->hw_ecap & DMAR_ECAP_DI) != 0 || force) 224 dmar_qi_invalidate_iotlb_glob_locked(dmar); 225 return (0); 226 } 227 error = dmar_inv_ctx_glob(dmar); 228 if (error == 0 && ((dmar->hw_ecap & DMAR_ECAP_DI) != 0 || force)) 229 error = dmar_inv_iotlb_glob(dmar); 230 return (error); 231 } 232 233 static int 234 domain_init_rmrr(struct dmar_domain *domain, device_t dev, int bus, 235 int slot, int func, int dev_domain, int dev_busno, 236 const void *dev_path, int dev_path_len) 237 { 238 struct iommu_map_entries_tailq rmrr_entries; 239 struct iommu_map_entry *entry, *entry1; 240 vm_page_t *ma; 241 iommu_gaddr_t start, end; 242 vm_pindex_t size, i; 243 int error, error1; 244 245 error = 0; 246 TAILQ_INIT(&rmrr_entries); 247 dmar_dev_parse_rmrr(domain, dev_domain, dev_busno, dev_path, 248 dev_path_len, &rmrr_entries); 249 TAILQ_FOREACH_SAFE(entry, &rmrr_entries, unroll_link, entry1) { 250 /* 251 * VT-d specification requires that the start of an 252 * RMRR entry is 4k-aligned. Buggy BIOSes put 253 * anything into the start and end fields. Truncate 254 * and round as neccesary. 255 * 256 * We also allow the overlapping RMRR entries, see 257 * iommu_gas_alloc_region(). 258 */ 259 start = entry->start; 260 end = entry->end; 261 if (bootverbose) 262 printf("dmar%d ctx pci%d:%d:%d RMRR [%#jx, %#jx]\n", 263 domain->iodom.iommu->unit, bus, slot, func, 264 (uintmax_t)start, (uintmax_t)end); 265 entry->start = trunc_page(start); 266 entry->end = round_page(end); 267 if (entry->start == entry->end) { 268 /* Workaround for some AMI (?) BIOSes */ 269 if (bootverbose) { 270 if (dev != NULL) 271 device_printf(dev, ""); 272 printf("pci%d:%d:%d ", bus, slot, func); 273 printf("BIOS bug: dmar%d RMRR " 274 "region (%jx, %jx) corrected\n", 275 domain->iodom.iommu->unit, start, end); 276 } 277 entry->end += DMAR_PAGE_SIZE * 0x20; 278 } 279 size = OFF_TO_IDX(entry->end - entry->start); 280 ma = malloc(sizeof(vm_page_t) * size, M_TEMP, M_WAITOK); 281 for (i = 0; i < size; i++) { 282 ma[i] = vm_page_getfake(entry->start + PAGE_SIZE * i, 283 VM_MEMATTR_DEFAULT); 284 } 285 error1 = iommu_gas_map_region(DOM2IODOM(domain), entry, 286 IOMMU_MAP_ENTRY_READ | IOMMU_MAP_ENTRY_WRITE, 287 IOMMU_MF_CANWAIT | IOMMU_MF_RMRR, ma); 288 /* 289 * Non-failed RMRR entries are owned by context rb 290 * tree. Get rid of the failed entry, but do not stop 291 * the loop. Rest of the parsed RMRR entries are 292 * loaded and removed on the context destruction. 293 */ 294 if (error1 == 0 && entry->end != entry->start) { 295 IOMMU_LOCK(domain->iodom.iommu); 296 domain->refs++; /* XXXKIB prevent free */ 297 domain->iodom.flags |= IOMMU_DOMAIN_RMRR; 298 IOMMU_UNLOCK(domain->iodom.iommu); 299 } else { 300 if (error1 != 0) { 301 if (dev != NULL) 302 device_printf(dev, ""); 303 printf("pci%d:%d:%d ", bus, slot, func); 304 printf( 305 "dmar%d failed to map RMRR region (%jx, %jx) %d\n", 306 domain->iodom.iommu->unit, start, end, 307 error1); 308 error = error1; 309 } 310 TAILQ_REMOVE(&rmrr_entries, entry, unroll_link); 311 iommu_gas_free_entry(DOM2IODOM(domain), entry); 312 } 313 for (i = 0; i < size; i++) 314 vm_page_putfake(ma[i]); 315 free(ma, M_TEMP); 316 } 317 return (error); 318 } 319 320 static struct dmar_domain * 321 dmar_domain_alloc(struct dmar_unit *dmar, bool id_mapped) 322 { 323 struct iommu_domain *iodom; 324 struct iommu_unit *unit; 325 struct dmar_domain *domain; 326 int error, id, mgaw; 327 328 id = alloc_unr(dmar->domids); 329 if (id == -1) 330 return (NULL); 331 domain = malloc(sizeof(*domain), M_DMAR_DOMAIN, M_WAITOK | M_ZERO); 332 iodom = DOM2IODOM(domain); 333 unit = DMAR2IOMMU(dmar); 334 domain->domain = id; 335 LIST_INIT(&domain->contexts); 336 iommu_domain_init(unit, iodom, &dmar_domain_map_ops); 337 338 domain->dmar = dmar; 339 340 /* 341 * For now, use the maximal usable physical address of the 342 * installed memory to calculate the mgaw on id_mapped domain. 343 * It is useful for the identity mapping, and less so for the 344 * virtualized bus address space. 345 */ 346 domain->iodom.end = id_mapped ? ptoa(Maxmem) : BUS_SPACE_MAXADDR; 347 mgaw = dmar_maxaddr2mgaw(dmar, domain->iodom.end, !id_mapped); 348 error = domain_set_agaw(domain, mgaw); 349 if (error != 0) 350 goto fail; 351 if (!id_mapped) 352 /* Use all supported address space for remapping. */ 353 domain->iodom.end = 1ULL << (domain->agaw - 1); 354 355 iommu_gas_init_domain(DOM2IODOM(domain)); 356 357 if (id_mapped) { 358 if ((dmar->hw_ecap & DMAR_ECAP_PT) == 0) { 359 domain->pgtbl_obj = domain_get_idmap_pgtbl(domain, 360 domain->iodom.end); 361 } 362 domain->iodom.flags |= IOMMU_DOMAIN_IDMAP; 363 } else { 364 error = domain_alloc_pgtbl(domain); 365 if (error != 0) 366 goto fail; 367 /* Disable local apic region access */ 368 error = iommu_gas_reserve_region(iodom, 0xfee00000, 369 0xfeefffff + 1); 370 if (error != 0) 371 goto fail; 372 } 373 return (domain); 374 375 fail: 376 dmar_domain_destroy(domain); 377 return (NULL); 378 } 379 380 static struct dmar_ctx * 381 dmar_ctx_alloc(struct dmar_domain *domain, uint16_t rid) 382 { 383 struct dmar_ctx *ctx; 384 385 ctx = malloc(sizeof(*ctx), M_DMAR_CTX, M_WAITOK | M_ZERO); 386 ctx->context.domain = DOM2IODOM(domain); 387 ctx->context.tag = malloc(sizeof(struct bus_dma_tag_iommu), 388 M_DMAR_CTX, M_WAITOK | M_ZERO); 389 ctx->rid = rid; 390 ctx->refs = 1; 391 return (ctx); 392 } 393 394 static void 395 dmar_ctx_link(struct dmar_ctx *ctx) 396 { 397 struct dmar_domain *domain; 398 399 domain = CTX2DOM(ctx); 400 IOMMU_ASSERT_LOCKED(domain->iodom.iommu); 401 KASSERT(domain->refs >= domain->ctx_cnt, 402 ("dom %p ref underflow %d %d", domain, domain->refs, 403 domain->ctx_cnt)); 404 domain->refs++; 405 domain->ctx_cnt++; 406 LIST_INSERT_HEAD(&domain->contexts, ctx, link); 407 } 408 409 static void 410 dmar_ctx_unlink(struct dmar_ctx *ctx) 411 { 412 struct dmar_domain *domain; 413 414 domain = CTX2DOM(ctx); 415 IOMMU_ASSERT_LOCKED(domain->iodom.iommu); 416 KASSERT(domain->refs > 0, 417 ("domain %p ctx dtr refs %d", domain, domain->refs)); 418 KASSERT(domain->ctx_cnt >= domain->refs, 419 ("domain %p ctx dtr refs %d ctx_cnt %d", domain, 420 domain->refs, domain->ctx_cnt)); 421 domain->refs--; 422 domain->ctx_cnt--; 423 LIST_REMOVE(ctx, link); 424 } 425 426 static void 427 dmar_domain_destroy(struct dmar_domain *domain) 428 { 429 struct iommu_domain *iodom; 430 struct dmar_unit *dmar; 431 432 iodom = DOM2IODOM(domain); 433 434 KASSERT(TAILQ_EMPTY(&domain->iodom.unload_entries), 435 ("unfinished unloads %p", domain)); 436 KASSERT(LIST_EMPTY(&domain->contexts), 437 ("destroying dom %p with contexts", domain)); 438 KASSERT(domain->ctx_cnt == 0, 439 ("destroying dom %p with ctx_cnt %d", domain, domain->ctx_cnt)); 440 KASSERT(domain->refs == 0, 441 ("destroying dom %p with refs %d", domain, domain->refs)); 442 if ((domain->iodom.flags & IOMMU_DOMAIN_GAS_INITED) != 0) { 443 DMAR_DOMAIN_LOCK(domain); 444 iommu_gas_fini_domain(iodom); 445 DMAR_DOMAIN_UNLOCK(domain); 446 } 447 if ((domain->iodom.flags & IOMMU_DOMAIN_PGTBL_INITED) != 0) { 448 if (domain->pgtbl_obj != NULL) 449 DMAR_DOMAIN_PGLOCK(domain); 450 domain_free_pgtbl(domain); 451 } 452 iommu_domain_fini(iodom); 453 dmar = DOM2DMAR(domain); 454 free_unr(dmar->domids, domain->domain); 455 free(domain, M_DMAR_DOMAIN); 456 } 457 458 static struct dmar_ctx * 459 dmar_get_ctx_for_dev1(struct dmar_unit *dmar, device_t dev, uint16_t rid, 460 int dev_domain, int dev_busno, const void *dev_path, int dev_path_len, 461 bool id_mapped, bool rmrr_init) 462 { 463 struct dmar_domain *domain, *domain1; 464 struct dmar_ctx *ctx, *ctx1; 465 struct iommu_unit *unit; 466 dmar_ctx_entry_t *ctxp; 467 struct sf_buf *sf; 468 int bus, slot, func, error; 469 bool enable; 470 471 if (dev != NULL) { 472 bus = pci_get_bus(dev); 473 slot = pci_get_slot(dev); 474 func = pci_get_function(dev); 475 } else { 476 bus = PCI_RID2BUS(rid); 477 slot = PCI_RID2SLOT(rid); 478 func = PCI_RID2FUNC(rid); 479 } 480 enable = false; 481 TD_PREP_PINNED_ASSERT; 482 unit = DMAR2IOMMU(dmar); 483 DMAR_LOCK(dmar); 484 KASSERT(!iommu_is_buswide_ctx(unit, bus) || (slot == 0 && func == 0), 485 ("iommu%d pci%d:%d:%d get_ctx for buswide", dmar->iommu.unit, bus, 486 slot, func)); 487 ctx = dmar_find_ctx_locked(dmar, rid); 488 error = 0; 489 if (ctx == NULL) { 490 /* 491 * Perform the allocations which require sleep or have 492 * higher chance to succeed if the sleep is allowed. 493 */ 494 DMAR_UNLOCK(dmar); 495 dmar_ensure_ctx_page(dmar, PCI_RID2BUS(rid)); 496 domain1 = dmar_domain_alloc(dmar, id_mapped); 497 if (domain1 == NULL) { 498 TD_PINNED_ASSERT; 499 return (NULL); 500 } 501 if (!id_mapped) { 502 error = domain_init_rmrr(domain1, dev, bus, 503 slot, func, dev_domain, dev_busno, dev_path, 504 dev_path_len); 505 if (error != 0) { 506 dmar_domain_destroy(domain1); 507 TD_PINNED_ASSERT; 508 return (NULL); 509 } 510 } 511 ctx1 = dmar_ctx_alloc(domain1, rid); 512 ctxp = dmar_map_ctx_entry(ctx1, &sf); 513 DMAR_LOCK(dmar); 514 515 /* 516 * Recheck the contexts, other thread might have 517 * already allocated needed one. 518 */ 519 ctx = dmar_find_ctx_locked(dmar, rid); 520 if (ctx == NULL) { 521 domain = domain1; 522 ctx = ctx1; 523 dmar_ctx_link(ctx); 524 ctx->context.tag->owner = dev; 525 device_tag_init(ctx, dev); 526 527 /* 528 * This is the first activated context for the 529 * DMAR unit. Enable the translation after 530 * everything is set up. 531 */ 532 if (LIST_EMPTY(&dmar->domains)) 533 enable = true; 534 LIST_INSERT_HEAD(&dmar->domains, domain, link); 535 ctx_id_entry_init(ctx, ctxp, false, bus); 536 if (dev != NULL) { 537 device_printf(dev, 538 "dmar%d pci%d:%d:%d:%d rid %x domain %d mgaw %d " 539 "agaw %d %s-mapped\n", 540 dmar->iommu.unit, dmar->segment, bus, slot, 541 func, rid, domain->domain, domain->mgaw, 542 domain->agaw, id_mapped ? "id" : "re"); 543 } 544 dmar_unmap_pgtbl(sf); 545 } else { 546 dmar_unmap_pgtbl(sf); 547 dmar_domain_destroy(domain1); 548 /* Nothing needs to be done to destroy ctx1. */ 549 free(ctx1, M_DMAR_CTX); 550 domain = CTX2DOM(ctx); 551 ctx->refs++; /* tag referenced us */ 552 } 553 } else { 554 domain = CTX2DOM(ctx); 555 if (ctx->context.tag->owner == NULL) 556 ctx->context.tag->owner = dev; 557 ctx->refs++; /* tag referenced us */ 558 } 559 560 error = dmar_flush_for_ctx_entry(dmar, enable); 561 if (error != 0) { 562 dmar_free_ctx_locked(dmar, ctx); 563 TD_PINNED_ASSERT; 564 return (NULL); 565 } 566 567 /* 568 * The dmar lock was potentially dropped between check for the 569 * empty context list and now. Recheck the state of GCMD_TE 570 * to avoid unneeded command. 571 */ 572 if (enable && !rmrr_init && (dmar->hw_gcmd & DMAR_GCMD_TE) == 0) { 573 error = dmar_enable_translation(dmar); 574 if (error == 0) { 575 if (bootverbose) { 576 printf("dmar%d: enabled translation\n", 577 dmar->iommu.unit); 578 } 579 } else { 580 printf("dmar%d: enabling translation failed, " 581 "error %d\n", dmar->iommu.unit, error); 582 dmar_free_ctx_locked(dmar, ctx); 583 TD_PINNED_ASSERT; 584 return (NULL); 585 } 586 } 587 DMAR_UNLOCK(dmar); 588 TD_PINNED_ASSERT; 589 return (ctx); 590 } 591 592 struct dmar_ctx * 593 dmar_get_ctx_for_dev(struct dmar_unit *dmar, device_t dev, uint16_t rid, 594 bool id_mapped, bool rmrr_init) 595 { 596 int dev_domain, dev_path_len, dev_busno; 597 598 dev_domain = pci_get_domain(dev); 599 dev_path_len = dmar_dev_depth(dev); 600 ACPI_DMAR_PCI_PATH dev_path[dev_path_len]; 601 dmar_dev_path(dev, &dev_busno, dev_path, dev_path_len); 602 return (dmar_get_ctx_for_dev1(dmar, dev, rid, dev_domain, dev_busno, 603 dev_path, dev_path_len, id_mapped, rmrr_init)); 604 } 605 606 struct dmar_ctx * 607 dmar_get_ctx_for_devpath(struct dmar_unit *dmar, uint16_t rid, 608 int dev_domain, int dev_busno, 609 const void *dev_path, int dev_path_len, 610 bool id_mapped, bool rmrr_init) 611 { 612 613 return (dmar_get_ctx_for_dev1(dmar, NULL, rid, dev_domain, dev_busno, 614 dev_path, dev_path_len, id_mapped, rmrr_init)); 615 } 616 617 int 618 dmar_move_ctx_to_domain(struct dmar_domain *domain, struct dmar_ctx *ctx) 619 { 620 struct dmar_unit *dmar; 621 struct dmar_domain *old_domain; 622 dmar_ctx_entry_t *ctxp; 623 struct sf_buf *sf; 624 int error; 625 626 dmar = domain->dmar; 627 old_domain = CTX2DOM(ctx); 628 if (domain == old_domain) 629 return (0); 630 KASSERT(old_domain->iodom.iommu == domain->iodom.iommu, 631 ("domain %p %u moving between dmars %u %u", domain, 632 domain->domain, old_domain->iodom.iommu->unit, 633 domain->iodom.iommu->unit)); 634 TD_PREP_PINNED_ASSERT; 635 636 ctxp = dmar_map_ctx_entry(ctx, &sf); 637 DMAR_LOCK(dmar); 638 dmar_ctx_unlink(ctx); 639 ctx->context.domain = &domain->iodom; 640 dmar_ctx_link(ctx); 641 ctx_id_entry_init(ctx, ctxp, true, PCI_BUSMAX + 100); 642 dmar_unmap_pgtbl(sf); 643 error = dmar_flush_for_ctx_entry(dmar, true); 644 /* If flush failed, rolling back would not work as well. */ 645 printf("dmar%d rid %x domain %d->%d %s-mapped\n", 646 dmar->iommu.unit, ctx->rid, old_domain->domain, domain->domain, 647 (domain->iodom.flags & IOMMU_DOMAIN_IDMAP) != 0 ? "id" : "re"); 648 dmar_unref_domain_locked(dmar, old_domain); 649 TD_PINNED_ASSERT; 650 return (error); 651 } 652 653 static void 654 dmar_unref_domain_locked(struct dmar_unit *dmar, struct dmar_domain *domain) 655 { 656 657 DMAR_ASSERT_LOCKED(dmar); 658 KASSERT(domain->refs >= 1, 659 ("dmar %d domain %p refs %u", dmar->iommu.unit, domain, 660 domain->refs)); 661 KASSERT(domain->refs > domain->ctx_cnt, 662 ("dmar %d domain %p refs %d ctx_cnt %d", dmar->iommu.unit, domain, 663 domain->refs, domain->ctx_cnt)); 664 665 if (domain->refs > 1) { 666 domain->refs--; 667 DMAR_UNLOCK(dmar); 668 return; 669 } 670 671 KASSERT((domain->iodom.flags & IOMMU_DOMAIN_RMRR) == 0, 672 ("lost ref on RMRR domain %p", domain)); 673 674 LIST_REMOVE(domain, link); 675 DMAR_UNLOCK(dmar); 676 677 taskqueue_drain(dmar->iommu.delayed_taskqueue, 678 &domain->iodom.unload_task); 679 dmar_domain_destroy(domain); 680 } 681 682 void 683 dmar_free_ctx_locked(struct dmar_unit *dmar, struct dmar_ctx *ctx) 684 { 685 struct sf_buf *sf; 686 dmar_ctx_entry_t *ctxp; 687 struct dmar_domain *domain; 688 689 DMAR_ASSERT_LOCKED(dmar); 690 KASSERT(ctx->refs >= 1, 691 ("dmar %p ctx %p refs %u", dmar, ctx, ctx->refs)); 692 693 /* 694 * If our reference is not last, only the dereference should 695 * be performed. 696 */ 697 if (ctx->refs > 1) { 698 ctx->refs--; 699 DMAR_UNLOCK(dmar); 700 return; 701 } 702 703 KASSERT((ctx->context.flags & IOMMU_CTX_DISABLED) == 0, 704 ("lost ref on disabled ctx %p", ctx)); 705 706 /* 707 * Otherwise, the context entry must be cleared before the 708 * page table is destroyed. The mapping of the context 709 * entries page could require sleep, unlock the dmar. 710 */ 711 DMAR_UNLOCK(dmar); 712 TD_PREP_PINNED_ASSERT; 713 ctxp = dmar_map_ctx_entry(ctx, &sf); 714 DMAR_LOCK(dmar); 715 KASSERT(ctx->refs >= 1, 716 ("dmar %p ctx %p refs %u", dmar, ctx, ctx->refs)); 717 718 /* 719 * Other thread might have referenced the context, in which 720 * case again only the dereference should be performed. 721 */ 722 if (ctx->refs > 1) { 723 ctx->refs--; 724 DMAR_UNLOCK(dmar); 725 dmar_unmap_pgtbl(sf); 726 TD_PINNED_ASSERT; 727 return; 728 } 729 730 KASSERT((ctx->context.flags & IOMMU_CTX_DISABLED) == 0, 731 ("lost ref on disabled ctx %p", ctx)); 732 733 /* 734 * Clear the context pointer and flush the caches. 735 * XXXKIB: cannot do this if any RMRR entries are still present. 736 */ 737 dmar_pte_clear(&ctxp->ctx1); 738 ctxp->ctx2 = 0; 739 dmar_flush_ctx_to_ram(dmar, ctxp); 740 dmar_inv_ctx_glob(dmar); 741 if ((dmar->hw_ecap & DMAR_ECAP_DI) != 0) { 742 if (dmar->qi_enabled) 743 dmar_qi_invalidate_iotlb_glob_locked(dmar); 744 else 745 dmar_inv_iotlb_glob(dmar); 746 } 747 dmar_unmap_pgtbl(sf); 748 domain = CTX2DOM(ctx); 749 dmar_ctx_unlink(ctx); 750 free(ctx->context.tag, M_DMAR_CTX); 751 free(ctx, M_DMAR_CTX); 752 dmar_unref_domain_locked(dmar, domain); 753 TD_PINNED_ASSERT; 754 } 755 756 void 757 dmar_free_ctx(struct dmar_ctx *ctx) 758 { 759 struct dmar_unit *dmar; 760 761 dmar = CTX2DMAR(ctx); 762 DMAR_LOCK(dmar); 763 dmar_free_ctx_locked(dmar, ctx); 764 } 765 766 /* 767 * Returns with the domain locked. 768 */ 769 struct dmar_ctx * 770 dmar_find_ctx_locked(struct dmar_unit *dmar, uint16_t rid) 771 { 772 struct dmar_domain *domain; 773 struct dmar_ctx *ctx; 774 775 DMAR_ASSERT_LOCKED(dmar); 776 777 LIST_FOREACH(domain, &dmar->domains, link) { 778 LIST_FOREACH(ctx, &domain->contexts, link) { 779 if (ctx->rid == rid) 780 return (ctx); 781 } 782 } 783 return (NULL); 784 } 785 786 void 787 dmar_domain_free_entry(struct iommu_map_entry *entry, bool free) 788 { 789 struct iommu_domain *domain; 790 791 domain = entry->domain; 792 IOMMU_DOMAIN_LOCK(domain); 793 if ((entry->flags & IOMMU_MAP_ENTRY_RMRR) != 0) 794 iommu_gas_free_region(domain, entry); 795 else 796 iommu_gas_free_space(domain, entry); 797 IOMMU_DOMAIN_UNLOCK(domain); 798 if (free) 799 iommu_gas_free_entry(domain, entry); 800 else 801 entry->flags = 0; 802 } 803 804 void 805 dmar_domain_unload_entry(struct iommu_map_entry *entry, bool free) 806 { 807 struct dmar_domain *domain; 808 struct dmar_unit *unit; 809 810 domain = IODOM2DOM(entry->domain); 811 unit = DOM2DMAR(domain); 812 if (unit->qi_enabled) { 813 DMAR_LOCK(unit); 814 dmar_qi_invalidate_locked(IODOM2DOM(entry->domain), 815 entry->start, entry->end - entry->start, &entry->gseq, 816 true); 817 if (!free) 818 entry->flags |= IOMMU_MAP_ENTRY_QI_NF; 819 TAILQ_INSERT_TAIL(&unit->tlb_flush_entries, entry, dmamap_link); 820 DMAR_UNLOCK(unit); 821 } else { 822 domain_flush_iotlb_sync(IODOM2DOM(entry->domain), 823 entry->start, entry->end - entry->start); 824 dmar_domain_free_entry(entry, free); 825 } 826 } 827 828 static bool 829 dmar_domain_unload_emit_wait(struct dmar_domain *domain, 830 struct iommu_map_entry *entry) 831 { 832 833 if (TAILQ_NEXT(entry, dmamap_link) == NULL) 834 return (true); 835 return (domain->batch_no++ % dmar_batch_coalesce == 0); 836 } 837 838 void 839 dmar_domain_unload(struct dmar_domain *domain, 840 struct iommu_map_entries_tailq *entries, bool cansleep) 841 { 842 struct dmar_unit *unit; 843 struct iommu_domain *iodom; 844 struct iommu_map_entry *entry, *entry1; 845 int error; 846 847 iodom = DOM2IODOM(domain); 848 unit = DOM2DMAR(domain); 849 850 TAILQ_FOREACH_SAFE(entry, entries, dmamap_link, entry1) { 851 KASSERT((entry->flags & IOMMU_MAP_ENTRY_MAP) != 0, 852 ("not mapped entry %p %p", domain, entry)); 853 error = iodom->ops->unmap(iodom, entry->start, entry->end - 854 entry->start, cansleep ? IOMMU_PGF_WAITOK : 0); 855 KASSERT(error == 0, ("unmap %p error %d", domain, error)); 856 if (!unit->qi_enabled) { 857 domain_flush_iotlb_sync(domain, entry->start, 858 entry->end - entry->start); 859 TAILQ_REMOVE(entries, entry, dmamap_link); 860 dmar_domain_free_entry(entry, true); 861 } 862 } 863 if (TAILQ_EMPTY(entries)) 864 return; 865 866 KASSERT(unit->qi_enabled, ("loaded entry left")); 867 DMAR_LOCK(unit); 868 TAILQ_FOREACH(entry, entries, dmamap_link) { 869 dmar_qi_invalidate_locked(domain, entry->start, entry->end - 870 entry->start, &entry->gseq, 871 dmar_domain_unload_emit_wait(domain, entry)); 872 } 873 TAILQ_CONCAT(&unit->tlb_flush_entries, entries, dmamap_link); 874 DMAR_UNLOCK(unit); 875 } 876 877 struct iommu_ctx * 878 iommu_get_ctx(struct iommu_unit *iommu, device_t dev, uint16_t rid, 879 bool id_mapped, bool rmrr_init) 880 { 881 struct dmar_unit *dmar; 882 struct dmar_ctx *ret; 883 884 dmar = IOMMU2DMAR(iommu); 885 886 ret = dmar_get_ctx_for_dev(dmar, dev, rid, id_mapped, rmrr_init); 887 888 return (CTX2IOCTX(ret)); 889 } 890 891 void 892 iommu_free_ctx_locked(struct iommu_unit *iommu, struct iommu_ctx *context) 893 { 894 struct dmar_unit *dmar; 895 struct dmar_ctx *ctx; 896 897 dmar = IOMMU2DMAR(iommu); 898 ctx = IOCTX2CTX(context); 899 900 dmar_free_ctx_locked(dmar, ctx); 901 } 902 903 void 904 iommu_free_ctx(struct iommu_ctx *context) 905 { 906 struct dmar_ctx *ctx; 907 908 ctx = IOCTX2CTX(context); 909 910 dmar_free_ctx(ctx); 911 } 912 913 void 914 iommu_domain_unload_entry(struct iommu_map_entry *entry, bool free) 915 { 916 917 dmar_domain_unload_entry(entry, free); 918 } 919 920 void 921 iommu_domain_unload(struct iommu_domain *iodom, 922 struct iommu_map_entries_tailq *entries, bool cansleep) 923 { 924 struct dmar_domain *domain; 925 926 domain = IODOM2DOM(iodom); 927 928 dmar_domain_unload(domain, entries, cansleep); 929 } 930