1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 3 * 4 * Copyright (c) 2013 The FreeBSD Foundation 5 * 6 * This software was developed by Konstantin Belousov <kib@FreeBSD.org> 7 * under sponsorship from the FreeBSD Foundation. 8 * 9 * Redistribution and use in source and binary forms, with or without 10 * modification, are permitted provided that the following conditions 11 * are met: 12 * 1. Redistributions of source code must retain the above copyright 13 * notice, this list of conditions and the following disclaimer. 14 * 2. Redistributions in binary form must reproduce the above copyright 15 * notice, this list of conditions and the following disclaimer in the 16 * documentation and/or other materials provided with the distribution. 17 * 18 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 19 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 20 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 21 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 22 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 23 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 24 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 25 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 26 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 27 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 28 * SUCH DAMAGE. 29 */ 30 31 #include <sys/cdefs.h> 32 __FBSDID("$FreeBSD$"); 33 34 #include "opt_acpi.h" 35 36 #include <sys/param.h> 37 #include <sys/bus.h> 38 #include <sys/kernel.h> 39 #include <sys/malloc.h> 40 #include <sys/memdesc.h> 41 #include <sys/module.h> 42 #include <sys/rman.h> 43 #include <sys/taskqueue.h> 44 #include <sys/time.h> 45 #include <sys/tree.h> 46 #include <sys/vmem.h> 47 #include <vm/vm.h> 48 #include <vm/vm_extern.h> 49 #include <vm/vm_kern.h> 50 #include <vm/vm_page.h> 51 #include <vm/vm_map.h> 52 #include <contrib/dev/acpica/include/acpi.h> 53 #include <contrib/dev/acpica/include/accommon.h> 54 #include <dev/acpica/acpivar.h> 55 #include <dev/pci/pcireg.h> 56 #include <machine/bus.h> 57 #include <machine/cpu.h> 58 #include <x86/include/busdma_impl.h> 59 #include <dev/iommu/busdma_iommu.h> 60 #include <x86/iommu/intel_reg.h> 61 #include <x86/iommu/intel_dmar.h> 62 63 static bool 64 dmar_qi_seq_processed(const struct dmar_unit *unit, 65 const struct iommu_qi_genseq *pseq) 66 { 67 u_int gen; 68 69 gen = unit->inv_waitd_gen; 70 return (pseq->gen < gen || 71 (pseq->gen == gen && pseq->seq <= unit->inv_waitd_seq_hw)); 72 } 73 74 static int 75 dmar_enable_qi(struct dmar_unit *unit) 76 { 77 int error; 78 79 DMAR_ASSERT_LOCKED(unit); 80 unit->hw_gcmd |= DMAR_GCMD_QIE; 81 dmar_write4(unit, DMAR_GCMD_REG, unit->hw_gcmd); 82 DMAR_WAIT_UNTIL(((dmar_read4(unit, DMAR_GSTS_REG) & DMAR_GSTS_QIES) 83 != 0)); 84 return (error); 85 } 86 87 static int 88 dmar_disable_qi(struct dmar_unit *unit) 89 { 90 int error; 91 92 DMAR_ASSERT_LOCKED(unit); 93 unit->hw_gcmd &= ~DMAR_GCMD_QIE; 94 dmar_write4(unit, DMAR_GCMD_REG, unit->hw_gcmd); 95 DMAR_WAIT_UNTIL(((dmar_read4(unit, DMAR_GSTS_REG) & DMAR_GSTS_QIES) 96 == 0)); 97 return (error); 98 } 99 100 static void 101 dmar_qi_advance_tail(struct dmar_unit *unit) 102 { 103 104 DMAR_ASSERT_LOCKED(unit); 105 dmar_write4(unit, DMAR_IQT_REG, unit->inv_queue_tail); 106 } 107 108 static void 109 dmar_qi_ensure(struct dmar_unit *unit, int descr_count) 110 { 111 uint32_t head; 112 int bytes; 113 114 DMAR_ASSERT_LOCKED(unit); 115 bytes = descr_count << DMAR_IQ_DESCR_SZ_SHIFT; 116 for (;;) { 117 if (bytes <= unit->inv_queue_avail) 118 break; 119 /* refill */ 120 head = dmar_read4(unit, DMAR_IQH_REG); 121 head &= DMAR_IQH_MASK; 122 unit->inv_queue_avail = head - unit->inv_queue_tail - 123 DMAR_IQ_DESCR_SZ; 124 if (head <= unit->inv_queue_tail) 125 unit->inv_queue_avail += unit->inv_queue_size; 126 if (bytes <= unit->inv_queue_avail) 127 break; 128 129 /* 130 * No space in the queue, do busy wait. Hardware must 131 * make a progress. But first advance the tail to 132 * inform the descriptor streamer about entries we 133 * might have already filled, otherwise they could 134 * clog the whole queue.. 135 * 136 * See dmar_qi_invalidate_locked() for a discussion 137 * about data race prevention. 138 */ 139 dmar_qi_advance_tail(unit); 140 unit->inv_queue_full++; 141 cpu_spinwait(); 142 } 143 unit->inv_queue_avail -= bytes; 144 } 145 146 static void 147 dmar_qi_emit(struct dmar_unit *unit, uint64_t data1, uint64_t data2) 148 { 149 150 DMAR_ASSERT_LOCKED(unit); 151 *(volatile uint64_t *)(unit->inv_queue + unit->inv_queue_tail) = data1; 152 unit->inv_queue_tail += DMAR_IQ_DESCR_SZ / 2; 153 KASSERT(unit->inv_queue_tail <= unit->inv_queue_size, 154 ("tail overflow 0x%x 0x%jx", unit->inv_queue_tail, 155 (uintmax_t)unit->inv_queue_size)); 156 unit->inv_queue_tail &= unit->inv_queue_size - 1; 157 *(volatile uint64_t *)(unit->inv_queue + unit->inv_queue_tail) = data2; 158 unit->inv_queue_tail += DMAR_IQ_DESCR_SZ / 2; 159 KASSERT(unit->inv_queue_tail <= unit->inv_queue_size, 160 ("tail overflow 0x%x 0x%jx", unit->inv_queue_tail, 161 (uintmax_t)unit->inv_queue_size)); 162 unit->inv_queue_tail &= unit->inv_queue_size - 1; 163 } 164 165 static void 166 dmar_qi_emit_wait_descr(struct dmar_unit *unit, uint32_t seq, bool intr, 167 bool memw, bool fence) 168 { 169 170 DMAR_ASSERT_LOCKED(unit); 171 dmar_qi_emit(unit, DMAR_IQ_DESCR_WAIT_ID | 172 (intr ? DMAR_IQ_DESCR_WAIT_IF : 0) | 173 (memw ? DMAR_IQ_DESCR_WAIT_SW : 0) | 174 (fence ? DMAR_IQ_DESCR_WAIT_FN : 0) | 175 (memw ? DMAR_IQ_DESCR_WAIT_SD(seq) : 0), 176 memw ? unit->inv_waitd_seq_hw_phys : 0); 177 } 178 179 static void 180 dmar_qi_emit_wait_seq(struct dmar_unit *unit, struct iommu_qi_genseq *pseq, 181 bool emit_wait) 182 { 183 struct iommu_qi_genseq gsec; 184 uint32_t seq; 185 186 KASSERT(pseq != NULL, ("wait descriptor with no place for seq")); 187 DMAR_ASSERT_LOCKED(unit); 188 if (unit->inv_waitd_seq == 0xffffffff) { 189 gsec.gen = unit->inv_waitd_gen; 190 gsec.seq = unit->inv_waitd_seq; 191 dmar_qi_ensure(unit, 1); 192 dmar_qi_emit_wait_descr(unit, gsec.seq, false, true, false); 193 dmar_qi_advance_tail(unit); 194 while (!dmar_qi_seq_processed(unit, &gsec)) 195 cpu_spinwait(); 196 unit->inv_waitd_gen++; 197 unit->inv_waitd_seq = 1; 198 } 199 seq = unit->inv_waitd_seq++; 200 pseq->gen = unit->inv_waitd_gen; 201 pseq->seq = seq; 202 if (emit_wait) { 203 dmar_qi_ensure(unit, 1); 204 dmar_qi_emit_wait_descr(unit, seq, true, true, false); 205 } 206 } 207 208 /* 209 * To avoid missed wakeups, callers must increment the unit's waiters count 210 * before advancing the tail past the wait descriptor. 211 */ 212 static void 213 dmar_qi_wait_for_seq(struct dmar_unit *unit, const struct iommu_qi_genseq *gseq, 214 bool nowait) 215 { 216 217 DMAR_ASSERT_LOCKED(unit); 218 KASSERT(unit->inv_seq_waiters > 0, ("%s: no waiters", __func__)); 219 while (!dmar_qi_seq_processed(unit, gseq)) { 220 if (cold || nowait) { 221 cpu_spinwait(); 222 } else { 223 msleep(&unit->inv_seq_waiters, &unit->iommu.lock, 0, 224 "dmarse", hz); 225 } 226 } 227 unit->inv_seq_waiters--; 228 } 229 230 static void 231 dmar_qi_invalidate_emit(struct dmar_domain *domain, iommu_gaddr_t base, 232 iommu_gaddr_t size, struct iommu_qi_genseq *pseq, bool emit_wait) 233 { 234 struct dmar_unit *unit; 235 iommu_gaddr_t isize; 236 int am; 237 238 unit = domain->dmar; 239 DMAR_ASSERT_LOCKED(unit); 240 for (; size > 0; base += isize, size -= isize) { 241 am = calc_am(unit, base, size, &isize); 242 dmar_qi_ensure(unit, 1); 243 dmar_qi_emit(unit, DMAR_IQ_DESCR_IOTLB_INV | 244 DMAR_IQ_DESCR_IOTLB_PAGE | DMAR_IQ_DESCR_IOTLB_DW | 245 DMAR_IQ_DESCR_IOTLB_DR | 246 DMAR_IQ_DESCR_IOTLB_DID(domain->domain), 247 base | am); 248 } 249 dmar_qi_emit_wait_seq(unit, pseq, emit_wait); 250 } 251 252 /* 253 * The caller must not be using the entry's dmamap_link field. 254 */ 255 void 256 dmar_qi_invalidate_locked(struct dmar_domain *domain, 257 struct iommu_map_entry *entry, bool emit_wait) 258 { 259 struct dmar_unit *unit; 260 261 unit = domain->dmar; 262 DMAR_ASSERT_LOCKED(unit); 263 dmar_qi_invalidate_emit(domain, entry->start, entry->end - 264 entry->start, &entry->gseq, emit_wait); 265 266 /* 267 * To avoid a data race in dmar_qi_task(), the entry's gseq must be 268 * initialized before the entry is added to the TLB flush list, and the 269 * entry must be added to that list before the tail is advanced. More 270 * precisely, the tail must not be advanced past the wait descriptor 271 * that will generate the interrupt that schedules dmar_qi_task() for 272 * execution before the entry is added to the list. While an earlier 273 * call to dmar_qi_ensure() might have advanced the tail, it will not 274 * advance it past the wait descriptor. 275 * 276 * See the definition of struct dmar_unit for more information on 277 * synchronization. 278 */ 279 entry->tlb_flush_next = NULL; 280 atomic_store_rel_ptr((uintptr_t *)&unit->tlb_flush_tail->tlb_flush_next, 281 (uintptr_t)entry); 282 unit->tlb_flush_tail = entry; 283 284 dmar_qi_advance_tail(unit); 285 } 286 287 void 288 dmar_qi_invalidate_sync(struct dmar_domain *domain, iommu_gaddr_t base, 289 iommu_gaddr_t size, bool cansleep) 290 { 291 struct dmar_unit *unit; 292 struct iommu_qi_genseq gseq; 293 294 unit = domain->dmar; 295 DMAR_LOCK(unit); 296 dmar_qi_invalidate_emit(domain, base, size, &gseq, true); 297 298 /* 299 * To avoid a missed wakeup in dmar_qi_task(), the unit's waiters count 300 * must be incremented before the tail is advanced. 301 */ 302 unit->inv_seq_waiters++; 303 304 dmar_qi_advance_tail(unit); 305 dmar_qi_wait_for_seq(unit, &gseq, !cansleep); 306 DMAR_UNLOCK(unit); 307 } 308 309 void 310 dmar_qi_invalidate_ctx_glob_locked(struct dmar_unit *unit) 311 { 312 struct iommu_qi_genseq gseq; 313 314 DMAR_ASSERT_LOCKED(unit); 315 dmar_qi_ensure(unit, 2); 316 dmar_qi_emit(unit, DMAR_IQ_DESCR_CTX_INV | DMAR_IQ_DESCR_CTX_GLOB, 0); 317 dmar_qi_emit_wait_seq(unit, &gseq, true); 318 /* See dmar_qi_invalidate_sync(). */ 319 unit->inv_seq_waiters++; 320 dmar_qi_advance_tail(unit); 321 dmar_qi_wait_for_seq(unit, &gseq, false); 322 } 323 324 void 325 dmar_qi_invalidate_iotlb_glob_locked(struct dmar_unit *unit) 326 { 327 struct iommu_qi_genseq gseq; 328 329 DMAR_ASSERT_LOCKED(unit); 330 dmar_qi_ensure(unit, 2); 331 dmar_qi_emit(unit, DMAR_IQ_DESCR_IOTLB_INV | DMAR_IQ_DESCR_IOTLB_GLOB | 332 DMAR_IQ_DESCR_IOTLB_DW | DMAR_IQ_DESCR_IOTLB_DR, 0); 333 dmar_qi_emit_wait_seq(unit, &gseq, true); 334 /* See dmar_qi_invalidate_sync(). */ 335 unit->inv_seq_waiters++; 336 dmar_qi_advance_tail(unit); 337 dmar_qi_wait_for_seq(unit, &gseq, false); 338 } 339 340 void 341 dmar_qi_invalidate_iec_glob(struct dmar_unit *unit) 342 { 343 struct iommu_qi_genseq gseq; 344 345 DMAR_ASSERT_LOCKED(unit); 346 dmar_qi_ensure(unit, 2); 347 dmar_qi_emit(unit, DMAR_IQ_DESCR_IEC_INV, 0); 348 dmar_qi_emit_wait_seq(unit, &gseq, true); 349 /* See dmar_qi_invalidate_sync(). */ 350 unit->inv_seq_waiters++; 351 dmar_qi_advance_tail(unit); 352 dmar_qi_wait_for_seq(unit, &gseq, false); 353 } 354 355 void 356 dmar_qi_invalidate_iec(struct dmar_unit *unit, u_int start, u_int cnt) 357 { 358 struct iommu_qi_genseq gseq; 359 u_int c, l; 360 361 DMAR_ASSERT_LOCKED(unit); 362 KASSERT(start < unit->irte_cnt && start < start + cnt && 363 start + cnt <= unit->irte_cnt, 364 ("inv iec overflow %d %d %d", unit->irte_cnt, start, cnt)); 365 for (; cnt > 0; cnt -= c, start += c) { 366 l = ffs(start | cnt) - 1; 367 c = 1 << l; 368 dmar_qi_ensure(unit, 1); 369 dmar_qi_emit(unit, DMAR_IQ_DESCR_IEC_INV | 370 DMAR_IQ_DESCR_IEC_IDX | DMAR_IQ_DESCR_IEC_IIDX(start) | 371 DMAR_IQ_DESCR_IEC_IM(l), 0); 372 } 373 dmar_qi_ensure(unit, 1); 374 dmar_qi_emit_wait_seq(unit, &gseq, true); 375 376 /* 377 * Since dmar_qi_wait_for_seq() will not sleep, this increment's 378 * placement relative to advancing the tail doesn't matter. 379 */ 380 unit->inv_seq_waiters++; 381 382 dmar_qi_advance_tail(unit); 383 384 /* 385 * The caller of the function, in particular, 386 * dmar_ir_program_irte(), may be called from the context 387 * where the sleeping is forbidden (in fact, the 388 * intr_table_lock mutex may be held, locked from 389 * intr_shuffle_irqs()). Wait for the invalidation completion 390 * using the busy wait. 391 * 392 * The impact on the interrupt input setup code is small, the 393 * expected overhead is comparable with the chipset register 394 * read. It is more harmful for the parallel DMA operations, 395 * since we own the dmar unit lock until whole invalidation 396 * queue is processed, which includes requests possibly issued 397 * before our request. 398 */ 399 dmar_qi_wait_for_seq(unit, &gseq, true); 400 } 401 402 int 403 dmar_qi_intr(void *arg) 404 { 405 struct dmar_unit *unit; 406 407 unit = arg; 408 KASSERT(unit->qi_enabled, ("dmar%d: QI is not enabled", 409 unit->iommu.unit)); 410 taskqueue_enqueue(unit->qi_taskqueue, &unit->qi_task); 411 return (FILTER_HANDLED); 412 } 413 414 static void 415 dmar_qi_drain_tlb_flush(struct dmar_unit *unit) 416 { 417 struct iommu_map_entry *entry, *head; 418 419 for (head = unit->tlb_flush_head;; head = entry) { 420 entry = (struct iommu_map_entry *) 421 atomic_load_acq_ptr((uintptr_t *)&head->tlb_flush_next); 422 if (entry == NULL || 423 !dmar_qi_seq_processed(unit, &entry->gseq)) 424 break; 425 unit->tlb_flush_head = entry; 426 iommu_gas_free_entry(head); 427 if ((entry->flags & IOMMU_MAP_ENTRY_RMRR) != 0) 428 iommu_gas_free_region(entry); 429 else 430 iommu_gas_free_space(entry); 431 } 432 } 433 434 static void 435 dmar_qi_task(void *arg, int pending __unused) 436 { 437 struct dmar_unit *unit; 438 uint32_t ics; 439 440 unit = arg; 441 dmar_qi_drain_tlb_flush(unit); 442 443 /* 444 * Request an interrupt on the completion of the next invalidation 445 * wait descriptor with the IF field set. 446 */ 447 ics = dmar_read4(unit, DMAR_ICS_REG); 448 if ((ics & DMAR_ICS_IWC) != 0) { 449 ics = DMAR_ICS_IWC; 450 dmar_write4(unit, DMAR_ICS_REG, ics); 451 452 /* 453 * Drain a second time in case the DMAR processes an entry 454 * after the first call and before clearing DMAR_ICS_IWC. 455 * Otherwise, such entries will linger until a later entry 456 * that requests an interrupt is processed. 457 */ 458 dmar_qi_drain_tlb_flush(unit); 459 } 460 461 if (unit->inv_seq_waiters > 0) { 462 /* 463 * Acquire the DMAR lock so that wakeup() is called only after 464 * the waiter is sleeping. 465 */ 466 DMAR_LOCK(unit); 467 wakeup(&unit->inv_seq_waiters); 468 DMAR_UNLOCK(unit); 469 } 470 } 471 472 int 473 dmar_init_qi(struct dmar_unit *unit) 474 { 475 uint64_t iqa; 476 uint32_t ics; 477 int qi_sz; 478 479 if (!DMAR_HAS_QI(unit) || (unit->hw_cap & DMAR_CAP_CM) != 0) 480 return (0); 481 unit->qi_enabled = 1; 482 TUNABLE_INT_FETCH("hw.dmar.qi", &unit->qi_enabled); 483 if (!unit->qi_enabled) 484 return (0); 485 486 unit->tlb_flush_head = unit->tlb_flush_tail = 487 iommu_gas_alloc_entry(NULL, 0); 488 TASK_INIT(&unit->qi_task, 0, dmar_qi_task, unit); 489 unit->qi_taskqueue = taskqueue_create_fast("dmarqf", M_WAITOK, 490 taskqueue_thread_enqueue, &unit->qi_taskqueue); 491 taskqueue_start_threads(&unit->qi_taskqueue, 1, PI_AV, 492 "dmar%d qi taskq", unit->iommu.unit); 493 494 unit->inv_waitd_gen = 0; 495 unit->inv_waitd_seq = 1; 496 497 qi_sz = DMAR_IQA_QS_DEF; 498 TUNABLE_INT_FETCH("hw.dmar.qi_size", &qi_sz); 499 if (qi_sz > DMAR_IQA_QS_MAX) 500 qi_sz = DMAR_IQA_QS_MAX; 501 unit->inv_queue_size = (1ULL << qi_sz) * PAGE_SIZE; 502 /* Reserve one descriptor to prevent wraparound. */ 503 unit->inv_queue_avail = unit->inv_queue_size - DMAR_IQ_DESCR_SZ; 504 505 /* The invalidation queue reads by DMARs are always coherent. */ 506 unit->inv_queue = kmem_alloc_contig(unit->inv_queue_size, M_WAITOK | 507 M_ZERO, 0, dmar_high, PAGE_SIZE, 0, VM_MEMATTR_DEFAULT); 508 unit->inv_waitd_seq_hw_phys = pmap_kextract( 509 (vm_offset_t)&unit->inv_waitd_seq_hw); 510 511 DMAR_LOCK(unit); 512 dmar_write8(unit, DMAR_IQT_REG, 0); 513 iqa = pmap_kextract((uintptr_t)unit->inv_queue); 514 iqa |= qi_sz; 515 dmar_write8(unit, DMAR_IQA_REG, iqa); 516 dmar_enable_qi(unit); 517 ics = dmar_read4(unit, DMAR_ICS_REG); 518 if ((ics & DMAR_ICS_IWC) != 0) { 519 ics = DMAR_ICS_IWC; 520 dmar_write4(unit, DMAR_ICS_REG, ics); 521 } 522 dmar_enable_qi_intr(unit); 523 DMAR_UNLOCK(unit); 524 525 return (0); 526 } 527 528 void 529 dmar_fini_qi(struct dmar_unit *unit) 530 { 531 struct iommu_qi_genseq gseq; 532 533 if (!unit->qi_enabled) 534 return; 535 taskqueue_drain(unit->qi_taskqueue, &unit->qi_task); 536 taskqueue_free(unit->qi_taskqueue); 537 unit->qi_taskqueue = NULL; 538 539 DMAR_LOCK(unit); 540 /* quisce */ 541 dmar_qi_ensure(unit, 1); 542 dmar_qi_emit_wait_seq(unit, &gseq, true); 543 /* See dmar_qi_invalidate_sync_locked(). */ 544 unit->inv_seq_waiters++; 545 dmar_qi_advance_tail(unit); 546 dmar_qi_wait_for_seq(unit, &gseq, false); 547 /* only after the quisce, disable queue */ 548 dmar_disable_qi_intr(unit); 549 dmar_disable_qi(unit); 550 KASSERT(unit->inv_seq_waiters == 0, 551 ("dmar%d: waiters on disabled queue", unit->iommu.unit)); 552 DMAR_UNLOCK(unit); 553 554 kmem_free(unit->inv_queue, unit->inv_queue_size); 555 unit->inv_queue = NULL; 556 unit->inv_queue_size = 0; 557 unit->qi_enabled = 0; 558 } 559 560 void 561 dmar_enable_qi_intr(struct dmar_unit *unit) 562 { 563 uint32_t iectl; 564 565 DMAR_ASSERT_LOCKED(unit); 566 KASSERT(DMAR_HAS_QI(unit), ("dmar%d: QI is not supported", 567 unit->iommu.unit)); 568 iectl = dmar_read4(unit, DMAR_IECTL_REG); 569 iectl &= ~DMAR_IECTL_IM; 570 dmar_write4(unit, DMAR_IECTL_REG, iectl); 571 } 572 573 void 574 dmar_disable_qi_intr(struct dmar_unit *unit) 575 { 576 uint32_t iectl; 577 578 DMAR_ASSERT_LOCKED(unit); 579 KASSERT(DMAR_HAS_QI(unit), ("dmar%d: QI is not supported", 580 unit->iommu.unit)); 581 iectl = dmar_read4(unit, DMAR_IECTL_REG); 582 dmar_write4(unit, DMAR_IECTL_REG, iectl | DMAR_IECTL_IM); 583 } 584