1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause 3 * 4 * Copyright (c) 2013 The FreeBSD Foundation 5 * 6 * This software was developed by Konstantin Belousov <kib@FreeBSD.org> 7 * under sponsorship from the FreeBSD Foundation. 8 * 9 * Redistribution and use in source and binary forms, with or without 10 * modification, are permitted provided that the following conditions 11 * are met: 12 * 1. Redistributions of source code must retain the above copyright 13 * notice, this list of conditions and the following disclaimer. 14 * 2. Redistributions in binary form must reproduce the above copyright 15 * notice, this list of conditions and the following disclaimer in the 16 * documentation and/or other materials provided with the distribution. 17 * 18 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 19 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 20 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 21 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 22 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 23 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 24 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 25 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 26 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 27 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 28 * SUCH DAMAGE. 29 */ 30 31 #include "opt_acpi.h" 32 33 #include <sys/param.h> 34 #include <sys/bus.h> 35 #include <sys/kernel.h> 36 #include <sys/malloc.h> 37 #include <sys/memdesc.h> 38 #include <sys/module.h> 39 #include <sys/rman.h> 40 #include <sys/taskqueue.h> 41 #include <sys/time.h> 42 #include <sys/tree.h> 43 #include <sys/vmem.h> 44 #include <vm/vm.h> 45 #include <vm/vm_extern.h> 46 #include <vm/vm_kern.h> 47 #include <vm/vm_page.h> 48 #include <vm/vm_map.h> 49 #include <contrib/dev/acpica/include/acpi.h> 50 #include <contrib/dev/acpica/include/accommon.h> 51 #include <dev/acpica/acpivar.h> 52 #include <dev/pci/pcireg.h> 53 #include <machine/bus.h> 54 #include <machine/cpu.h> 55 #include <x86/include/busdma_impl.h> 56 #include <dev/iommu/busdma_iommu.h> 57 #include <x86/iommu/intel_reg.h> 58 #include <x86/iommu/x86_iommu.h> 59 #include <x86/iommu/intel_dmar.h> 60 61 static int 62 dmar_enable_qi(struct dmar_unit *unit) 63 { 64 int error; 65 66 DMAR_ASSERT_LOCKED(unit); 67 unit->hw_gcmd |= DMAR_GCMD_QIE; 68 dmar_write4(unit, DMAR_GCMD_REG, unit->hw_gcmd); 69 DMAR_WAIT_UNTIL(((dmar_read4(unit, DMAR_GSTS_REG) & DMAR_GSTS_QIES) 70 != 0)); 71 return (error); 72 } 73 74 static int 75 dmar_disable_qi(struct dmar_unit *unit) 76 { 77 int error; 78 79 DMAR_ASSERT_LOCKED(unit); 80 unit->hw_gcmd &= ~DMAR_GCMD_QIE; 81 dmar_write4(unit, DMAR_GCMD_REG, unit->hw_gcmd); 82 DMAR_WAIT_UNTIL(((dmar_read4(unit, DMAR_GSTS_REG) & DMAR_GSTS_QIES) 83 == 0)); 84 return (error); 85 } 86 87 static void 88 dmar_qi_advance_tail(struct iommu_unit *iommu) 89 { 90 struct dmar_unit *unit; 91 92 unit = IOMMU2DMAR(iommu); 93 DMAR_ASSERT_LOCKED(unit); 94 dmar_write4(unit, DMAR_IQT_REG, unit->x86c.inv_queue_tail); 95 } 96 97 static void 98 dmar_qi_ensure(struct iommu_unit *iommu, int descr_count) 99 { 100 struct dmar_unit *unit; 101 uint32_t head; 102 int bytes; 103 104 unit = IOMMU2DMAR(iommu); 105 DMAR_ASSERT_LOCKED(unit); 106 bytes = descr_count << DMAR_IQ_DESCR_SZ_SHIFT; 107 for (;;) { 108 if (bytes <= unit->x86c.inv_queue_avail) 109 break; 110 /* refill */ 111 head = dmar_read4(unit, DMAR_IQH_REG); 112 head &= DMAR_IQH_MASK; 113 unit->x86c.inv_queue_avail = head - unit->x86c.inv_queue_tail - 114 DMAR_IQ_DESCR_SZ; 115 if (head <= unit->x86c.inv_queue_tail) 116 unit->x86c.inv_queue_avail += unit->x86c.inv_queue_size; 117 if (bytes <= unit->x86c.inv_queue_avail) 118 break; 119 120 /* 121 * No space in the queue, do busy wait. Hardware must 122 * make a progress. But first advance the tail to 123 * inform the descriptor streamer about entries we 124 * might have already filled, otherwise they could 125 * clog the whole queue.. 126 * 127 * See dmar_qi_invalidate_locked() for a discussion 128 * about data race prevention. 129 */ 130 dmar_qi_advance_tail(DMAR2IOMMU(unit)); 131 unit->x86c.inv_queue_full++; 132 cpu_spinwait(); 133 } 134 unit->x86c.inv_queue_avail -= bytes; 135 } 136 137 static void 138 dmar_qi_emit(struct dmar_unit *unit, uint64_t data1, uint64_t data2) 139 { 140 141 DMAR_ASSERT_LOCKED(unit); 142 #ifdef __LP64__ 143 atomic_store_64((uint64_t *)(unit->x86c.inv_queue + 144 unit->x86c.inv_queue_tail), data1); 145 #else 146 *(volatile uint64_t *)(unit->x86c.inv_queue + 147 unit->x86c.inv_queue_tail) = data1; 148 #endif 149 unit->x86c.inv_queue_tail += DMAR_IQ_DESCR_SZ / 2; 150 KASSERT(unit->x86c.inv_queue_tail <= unit->x86c.inv_queue_size, 151 ("tail overflow 0x%x 0x%jx", unit->x86c.inv_queue_tail, 152 (uintmax_t)unit->x86c.inv_queue_size)); 153 unit->x86c.inv_queue_tail &= unit->x86c.inv_queue_size - 1; 154 #ifdef __LP64__ 155 atomic_store_64((uint64_t *)(unit->x86c.inv_queue + 156 unit->x86c.inv_queue_tail), data2); 157 #else 158 *(volatile uint64_t *)(unit->x86c.inv_queue + 159 unit->x86c.inv_queue_tail) = data2; 160 #endif 161 unit->x86c.inv_queue_tail += DMAR_IQ_DESCR_SZ / 2; 162 KASSERT(unit->x86c.inv_queue_tail <= unit->x86c.inv_queue_size, 163 ("tail overflow 0x%x 0x%jx", unit->x86c.inv_queue_tail, 164 (uintmax_t)unit->x86c.inv_queue_size)); 165 unit->x86c.inv_queue_tail &= unit->x86c.inv_queue_size - 1; 166 } 167 168 static void 169 dmar_qi_emit_wait_descr(struct iommu_unit *iommu, uint32_t seq, bool intr, 170 bool memw, bool fence) 171 { 172 struct dmar_unit *unit; 173 174 unit = IOMMU2DMAR(iommu); 175 DMAR_ASSERT_LOCKED(unit); 176 dmar_qi_emit(unit, DMAR_IQ_DESCR_WAIT_ID | 177 (intr ? DMAR_IQ_DESCR_WAIT_IF : 0) | 178 (memw ? DMAR_IQ_DESCR_WAIT_SW : 0) | 179 (fence ? DMAR_IQ_DESCR_WAIT_FN : 0) | 180 (memw ? DMAR_IQ_DESCR_WAIT_SD(seq) : 0), 181 memw ? unit->x86c.inv_waitd_seq_hw_phys : 0); 182 } 183 184 static void 185 dmar_qi_invalidate_emit(struct iommu_domain *idomain, iommu_gaddr_t base, 186 iommu_gaddr_t size, struct iommu_qi_genseq *pseq, bool emit_wait) 187 { 188 struct dmar_unit *unit; 189 struct dmar_domain *domain; 190 iommu_gaddr_t isize; 191 int am; 192 193 domain = __containerof(idomain, struct dmar_domain, iodom); 194 unit = domain->dmar; 195 DMAR_ASSERT_LOCKED(unit); 196 for (; size > 0; base += isize, size -= isize) { 197 am = calc_am(unit, base, size, &isize); 198 dmar_qi_ensure(DMAR2IOMMU(unit), 1); 199 dmar_qi_emit(unit, DMAR_IQ_DESCR_IOTLB_INV | 200 DMAR_IQ_DESCR_IOTLB_PAGE | DMAR_IQ_DESCR_IOTLB_DW | 201 DMAR_IQ_DESCR_IOTLB_DR | 202 DMAR_IQ_DESCR_IOTLB_DID(domain->domain), 203 base | am); 204 } 205 iommu_qi_emit_wait_seq(DMAR2IOMMU(unit), pseq, emit_wait); 206 } 207 208 static void 209 dmar_qi_invalidate_glob_impl(struct dmar_unit *unit, uint64_t data1) 210 { 211 struct iommu_qi_genseq gseq; 212 213 DMAR_ASSERT_LOCKED(unit); 214 dmar_qi_ensure(DMAR2IOMMU(unit), 2); 215 dmar_qi_emit(unit, data1, 0); 216 iommu_qi_emit_wait_seq(DMAR2IOMMU(unit), &gseq, true); 217 /* See dmar_qi_invalidate_sync(). */ 218 unit->x86c.inv_seq_waiters++; 219 dmar_qi_advance_tail(DMAR2IOMMU(unit)); 220 iommu_qi_wait_for_seq(DMAR2IOMMU(unit), &gseq, false); 221 } 222 223 void 224 dmar_qi_invalidate_ctx_glob_locked(struct dmar_unit *unit) 225 { 226 dmar_qi_invalidate_glob_impl(unit, DMAR_IQ_DESCR_CTX_INV | 227 DMAR_IQ_DESCR_CTX_GLOB); 228 } 229 230 void 231 dmar_qi_invalidate_iotlb_glob_locked(struct dmar_unit *unit) 232 { 233 dmar_qi_invalidate_glob_impl(unit, DMAR_IQ_DESCR_IOTLB_INV | 234 DMAR_IQ_DESCR_IOTLB_GLOB | DMAR_IQ_DESCR_IOTLB_DW | 235 DMAR_IQ_DESCR_IOTLB_DR); 236 } 237 238 void 239 dmar_qi_invalidate_iec_glob(struct dmar_unit *unit) 240 { 241 dmar_qi_invalidate_glob_impl(unit, DMAR_IQ_DESCR_IEC_INV); 242 } 243 244 void 245 dmar_qi_invalidate_iec(struct dmar_unit *unit, u_int start, u_int cnt) 246 { 247 struct iommu_qi_genseq gseq; 248 u_int c, l; 249 250 DMAR_ASSERT_LOCKED(unit); 251 KASSERT(start < unit->irte_cnt && start < start + cnt && 252 start + cnt <= unit->irte_cnt, 253 ("inv iec overflow %d %d %d", unit->irte_cnt, start, cnt)); 254 for (; cnt > 0; cnt -= c, start += c) { 255 l = ffs(start | cnt) - 1; 256 c = 1 << l; 257 dmar_qi_ensure(DMAR2IOMMU(unit), 1); 258 dmar_qi_emit(unit, DMAR_IQ_DESCR_IEC_INV | 259 DMAR_IQ_DESCR_IEC_IDX | DMAR_IQ_DESCR_IEC_IIDX(start) | 260 DMAR_IQ_DESCR_IEC_IM(l), 0); 261 } 262 dmar_qi_ensure(DMAR2IOMMU(unit), 1); 263 iommu_qi_emit_wait_seq(DMAR2IOMMU(unit), &gseq, true); 264 265 /* 266 * Since iommu_qi_wait_for_seq() will not sleep, this increment's 267 * placement relative to advancing the tail doesn't matter. 268 */ 269 unit->x86c.inv_seq_waiters++; 270 271 dmar_qi_advance_tail(DMAR2IOMMU(unit)); 272 273 /* 274 * The caller of the function, in particular, 275 * dmar_ir_program_irte(), may be called from the context 276 * where the sleeping is forbidden (in fact, the 277 * intr_table_lock mutex may be held, locked from 278 * intr_shuffle_irqs()). Wait for the invalidation completion 279 * using the busy wait. 280 * 281 * The impact on the interrupt input setup code is small, the 282 * expected overhead is comparable with the chipset register 283 * read. It is more harmful for the parallel DMA operations, 284 * since we own the dmar unit lock until whole invalidation 285 * queue is processed, which includes requests possibly issued 286 * before our request. 287 */ 288 iommu_qi_wait_for_seq(DMAR2IOMMU(unit), &gseq, true); 289 } 290 291 int 292 dmar_qi_intr(void *arg) 293 { 294 struct dmar_unit *unit; 295 296 unit = IOMMU2DMAR((struct iommu_unit *)arg); 297 KASSERT(unit->qi_enabled, ("dmar%d: QI is not enabled", 298 unit->iommu.unit)); 299 taskqueue_enqueue(unit->x86c.qi_taskqueue, &unit->x86c.qi_task); 300 return (FILTER_HANDLED); 301 } 302 303 static void 304 dmar_qi_task(void *arg, int pending __unused) 305 { 306 struct dmar_unit *unit; 307 uint32_t ics; 308 309 unit = IOMMU2DMAR(arg); 310 iommu_qi_drain_tlb_flush(DMAR2IOMMU(unit)); 311 312 /* 313 * Request an interrupt on the completion of the next invalidation 314 * wait descriptor with the IF field set. 315 */ 316 ics = dmar_read4(unit, DMAR_ICS_REG); 317 if ((ics & DMAR_ICS_IWC) != 0) { 318 ics = DMAR_ICS_IWC; 319 dmar_write4(unit, DMAR_ICS_REG, ics); 320 321 /* 322 * Drain a second time in case the DMAR processes an entry 323 * after the first call and before clearing DMAR_ICS_IWC. 324 * Otherwise, such entries will linger until a later entry 325 * that requests an interrupt is processed. 326 */ 327 iommu_qi_drain_tlb_flush(DMAR2IOMMU(unit)); 328 } 329 330 if (unit->x86c.inv_seq_waiters > 0) { 331 /* 332 * Acquire the DMAR lock so that wakeup() is called only after 333 * the waiter is sleeping. 334 */ 335 DMAR_LOCK(unit); 336 wakeup(&unit->x86c.inv_seq_waiters); 337 DMAR_UNLOCK(unit); 338 } 339 } 340 341 int 342 dmar_init_qi(struct dmar_unit *unit) 343 { 344 uint64_t iqa; 345 uint32_t ics; 346 u_int qi_sz; 347 348 if (!DMAR_HAS_QI(unit) || (unit->hw_cap & DMAR_CAP_CM) != 0) 349 return (0); 350 unit->qi_enabled = 1; 351 TUNABLE_INT_FETCH("hw.dmar.qi", &unit->qi_enabled); 352 if (!unit->qi_enabled) 353 return (0); 354 355 unit->x86c.qi_buf_maxsz = DMAR_IQA_QS_MAX; 356 unit->x86c.qi_cmd_sz = DMAR_IQ_DESCR_SZ; 357 iommu_qi_common_init(DMAR2IOMMU(unit), dmar_qi_task); 358 get_x86_iommu()->qi_ensure = dmar_qi_ensure; 359 get_x86_iommu()->qi_emit_wait_descr = dmar_qi_emit_wait_descr; 360 get_x86_iommu()->qi_advance_tail = dmar_qi_advance_tail; 361 get_x86_iommu()->qi_invalidate_emit = dmar_qi_invalidate_emit; 362 363 qi_sz = ilog2(unit->x86c.inv_queue_size / PAGE_SIZE); 364 365 DMAR_LOCK(unit); 366 dmar_write8(unit, DMAR_IQT_REG, 0); 367 iqa = pmap_kextract((uintptr_t)unit->x86c.inv_queue); 368 iqa |= qi_sz; 369 dmar_write8(unit, DMAR_IQA_REG, iqa); 370 dmar_enable_qi(unit); 371 ics = dmar_read4(unit, DMAR_ICS_REG); 372 if ((ics & DMAR_ICS_IWC) != 0) { 373 ics = DMAR_ICS_IWC; 374 dmar_write4(unit, DMAR_ICS_REG, ics); 375 } 376 dmar_enable_qi_intr(DMAR2IOMMU(unit)); 377 DMAR_UNLOCK(unit); 378 379 return (0); 380 } 381 382 static void 383 dmar_fini_qi_helper(struct iommu_unit *iommu) 384 { 385 dmar_disable_qi_intr(iommu); 386 dmar_disable_qi(IOMMU2DMAR(iommu)); 387 } 388 389 void 390 dmar_fini_qi(struct dmar_unit *unit) 391 { 392 if (!unit->qi_enabled) 393 return; 394 iommu_qi_common_fini(DMAR2IOMMU(unit), dmar_fini_qi_helper); 395 unit->qi_enabled = 0; 396 } 397 398 void 399 dmar_enable_qi_intr(struct iommu_unit *iommu) 400 { 401 struct dmar_unit *unit; 402 uint32_t iectl; 403 404 unit = IOMMU2DMAR(iommu); 405 DMAR_ASSERT_LOCKED(unit); 406 KASSERT(DMAR_HAS_QI(unit), ("dmar%d: QI is not supported", 407 unit->iommu.unit)); 408 iectl = dmar_read4(unit, DMAR_IECTL_REG); 409 iectl &= ~DMAR_IECTL_IM; 410 dmar_write4(unit, DMAR_IECTL_REG, iectl); 411 } 412 413 void 414 dmar_disable_qi_intr(struct iommu_unit *iommu) 415 { 416 struct dmar_unit *unit; 417 uint32_t iectl; 418 419 unit = IOMMU2DMAR(iommu); 420 DMAR_ASSERT_LOCKED(unit); 421 KASSERT(DMAR_HAS_QI(unit), ("dmar%d: QI is not supported", 422 unit->iommu.unit)); 423 iectl = dmar_read4(unit, DMAR_IECTL_REG); 424 dmar_write4(unit, DMAR_IECTL_REG, iectl | DMAR_IECTL_IM); 425 } 426