1 /*- 2 * Copyright (c) 2013 The FreeBSD Foundation 3 * All rights reserved. 4 * 5 * This software was developed by Konstantin Belousov <kib@FreeBSD.org> 6 * under sponsorship from the FreeBSD Foundation. 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 27 * SUCH DAMAGE. 28 */ 29 30 #include <sys/cdefs.h> 31 __FBSDID("$FreeBSD$"); 32 33 #include "opt_acpi.h" 34 35 #include <sys/param.h> 36 #include <sys/bus.h> 37 #include <sys/kernel.h> 38 #include <sys/malloc.h> 39 #include <sys/memdesc.h> 40 #include <sys/module.h> 41 #include <sys/rman.h> 42 #include <sys/taskqueue.h> 43 #include <sys/time.h> 44 #include <sys/tree.h> 45 #include <sys/vmem.h> 46 #include <machine/bus.h> 47 #include <contrib/dev/acpica/include/acpi.h> 48 #include <contrib/dev/acpica/include/accommon.h> 49 #include <dev/acpica/acpivar.h> 50 #include <vm/vm.h> 51 #include <vm/vm_extern.h> 52 #include <vm/vm_kern.h> 53 #include <vm/vm_page.h> 54 #include <vm/vm_map.h> 55 #include <machine/cpu.h> 56 #include <x86/include/busdma_impl.h> 57 #include <x86/iommu/intel_reg.h> 58 #include <x86/iommu/busdma_dmar.h> 59 #include <x86/iommu/intel_dmar.h> 60 61 static bool 62 dmar_qi_seq_processed(const struct dmar_unit *unit, 63 const struct dmar_qi_genseq *pseq) 64 { 65 66 return (pseq->gen < unit->inv_waitd_gen || 67 (pseq->gen == unit->inv_waitd_gen && 68 pseq->seq <= unit->inv_waitd_seq_hw)); 69 } 70 71 static int 72 dmar_enable_qi(struct dmar_unit *unit) 73 { 74 int error; 75 76 DMAR_ASSERT_LOCKED(unit); 77 unit->hw_gcmd |= DMAR_GCMD_QIE; 78 dmar_write4(unit, DMAR_GCMD_REG, unit->hw_gcmd); 79 DMAR_WAIT_UNTIL(((dmar_read4(unit, DMAR_GSTS_REG) & DMAR_GSTS_QIES) 80 != 0)); 81 return (error); 82 } 83 84 static int 85 dmar_disable_qi(struct dmar_unit *unit) 86 { 87 int error; 88 89 DMAR_ASSERT_LOCKED(unit); 90 unit->hw_gcmd &= ~DMAR_GCMD_QIE; 91 dmar_write4(unit, DMAR_GCMD_REG, unit->hw_gcmd); 92 DMAR_WAIT_UNTIL(((dmar_read4(unit, DMAR_GSTS_REG) & DMAR_GSTS_QIES) 93 == 0)); 94 return (error); 95 } 96 97 static void 98 dmar_qi_advance_tail(struct dmar_unit *unit) 99 { 100 101 DMAR_ASSERT_LOCKED(unit); 102 dmar_write4(unit, DMAR_IQT_REG, unit->inv_queue_tail); 103 } 104 105 static void 106 dmar_qi_ensure(struct dmar_unit *unit, int descr_count) 107 { 108 uint32_t head; 109 int bytes; 110 111 DMAR_ASSERT_LOCKED(unit); 112 bytes = descr_count << DMAR_IQ_DESCR_SZ_SHIFT; 113 for (;;) { 114 if (bytes <= unit->inv_queue_avail) 115 break; 116 /* refill */ 117 head = dmar_read4(unit, DMAR_IQH_REG); 118 head &= DMAR_IQH_MASK; 119 unit->inv_queue_avail = head - unit->inv_queue_tail - 120 DMAR_IQ_DESCR_SZ; 121 if (head <= unit->inv_queue_tail) 122 unit->inv_queue_avail += unit->inv_queue_size; 123 if (bytes <= unit->inv_queue_avail) 124 break; 125 126 /* 127 * No space in the queue, do busy wait. Hardware must 128 * make a progress. But first advance the tail to 129 * inform the descriptor streamer about entries we 130 * might have already filled, otherwise they could 131 * clog the whole queue.. 132 */ 133 dmar_qi_advance_tail(unit); 134 unit->inv_queue_full++; 135 cpu_spinwait(); 136 } 137 unit->inv_queue_avail -= bytes; 138 } 139 140 static void 141 dmar_qi_emit(struct dmar_unit *unit, uint64_t data1, uint64_t data2) 142 { 143 144 DMAR_ASSERT_LOCKED(unit); 145 *(volatile uint64_t *)(unit->inv_queue + unit->inv_queue_tail) = data1; 146 unit->inv_queue_tail += DMAR_IQ_DESCR_SZ / 2; 147 KASSERT(unit->inv_queue_tail <= unit->inv_queue_size, 148 ("tail overflow 0x%x 0x%jx", unit->inv_queue_tail, 149 (uintmax_t)unit->inv_queue_size)); 150 unit->inv_queue_tail &= unit->inv_queue_size - 1; 151 *(volatile uint64_t *)(unit->inv_queue + unit->inv_queue_tail) = data2; 152 unit->inv_queue_tail += DMAR_IQ_DESCR_SZ / 2; 153 KASSERT(unit->inv_queue_tail <= unit->inv_queue_size, 154 ("tail overflow 0x%x 0x%jx", unit->inv_queue_tail, 155 (uintmax_t)unit->inv_queue_size)); 156 unit->inv_queue_tail &= unit->inv_queue_size - 1; 157 } 158 159 static void 160 dmar_qi_emit_wait_descr(struct dmar_unit *unit, uint32_t seq, bool intr, 161 bool memw, bool fence) 162 { 163 164 DMAR_ASSERT_LOCKED(unit); 165 dmar_qi_emit(unit, DMAR_IQ_DESCR_WAIT_ID | 166 (intr ? DMAR_IQ_DESCR_WAIT_IF : 0) | 167 (memw ? DMAR_IQ_DESCR_WAIT_SW : 0) | 168 (fence ? DMAR_IQ_DESCR_WAIT_FN : 0) | 169 (memw ? DMAR_IQ_DESCR_WAIT_SD(seq) : 0), 170 memw ? unit->inv_waitd_seq_hw_phys : 0); 171 } 172 173 static void 174 dmar_qi_emit_wait_seq(struct dmar_unit *unit, struct dmar_qi_genseq *pseq) 175 { 176 struct dmar_qi_genseq gsec; 177 uint32_t seq; 178 179 KASSERT(pseq != NULL, ("wait descriptor with no place for seq")); 180 DMAR_ASSERT_LOCKED(unit); 181 if (unit->inv_waitd_seq == 0xffffffff) { 182 gsec.gen = unit->inv_waitd_gen; 183 gsec.seq = unit->inv_waitd_seq; 184 dmar_qi_ensure(unit, 1); 185 dmar_qi_emit_wait_descr(unit, gsec.seq, false, true, false); 186 dmar_qi_advance_tail(unit); 187 while (!dmar_qi_seq_processed(unit, &gsec)) 188 cpu_spinwait(); 189 unit->inv_waitd_gen++; 190 unit->inv_waitd_seq = 1; 191 } 192 seq = unit->inv_waitd_seq++; 193 pseq->gen = unit->inv_waitd_gen; 194 pseq->seq = seq; 195 dmar_qi_emit_wait_descr(unit, seq, true, true, false); 196 } 197 198 static void 199 dmar_qi_wait_for_seq(struct dmar_unit *unit, const struct dmar_qi_genseq *gseq, 200 bool nowait) 201 { 202 203 DMAR_ASSERT_LOCKED(unit); 204 unit->inv_seq_waiters++; 205 while (!dmar_qi_seq_processed(unit, gseq)) { 206 if (cold || nowait) { 207 cpu_spinwait(); 208 } else { 209 msleep(&unit->inv_seq_waiters, &unit->lock, 0, 210 "dmarse", hz); 211 } 212 } 213 unit->inv_seq_waiters--; 214 } 215 216 void 217 dmar_qi_invalidate_locked(struct dmar_domain *domain, dmar_gaddr_t base, 218 dmar_gaddr_t size, struct dmar_qi_genseq *pseq) 219 { 220 struct dmar_unit *unit; 221 dmar_gaddr_t isize; 222 int am; 223 224 unit = domain->dmar; 225 DMAR_ASSERT_LOCKED(unit); 226 for (; size > 0; base += isize, size -= isize) { 227 am = calc_am(unit, base, size, &isize); 228 dmar_qi_ensure(unit, 1); 229 dmar_qi_emit(unit, DMAR_IQ_DESCR_IOTLB_INV | 230 DMAR_IQ_DESCR_IOTLB_PAGE | DMAR_IQ_DESCR_IOTLB_DW | 231 DMAR_IQ_DESCR_IOTLB_DR | 232 DMAR_IQ_DESCR_IOTLB_DID(domain->domain), 233 base | am); 234 } 235 if (pseq != NULL) { 236 dmar_qi_ensure(unit, 1); 237 dmar_qi_emit_wait_seq(unit, pseq); 238 } 239 dmar_qi_advance_tail(unit); 240 } 241 242 void 243 dmar_qi_invalidate_ctx_glob_locked(struct dmar_unit *unit) 244 { 245 struct dmar_qi_genseq gseq; 246 247 DMAR_ASSERT_LOCKED(unit); 248 dmar_qi_ensure(unit, 2); 249 dmar_qi_emit(unit, DMAR_IQ_DESCR_CTX_INV | DMAR_IQ_DESCR_CTX_GLOB, 0); 250 dmar_qi_emit_wait_seq(unit, &gseq); 251 dmar_qi_advance_tail(unit); 252 dmar_qi_wait_for_seq(unit, &gseq, false); 253 } 254 255 void 256 dmar_qi_invalidate_iotlb_glob_locked(struct dmar_unit *unit) 257 { 258 struct dmar_qi_genseq gseq; 259 260 DMAR_ASSERT_LOCKED(unit); 261 dmar_qi_ensure(unit, 2); 262 dmar_qi_emit(unit, DMAR_IQ_DESCR_IOTLB_INV | DMAR_IQ_DESCR_IOTLB_GLOB | 263 DMAR_IQ_DESCR_IOTLB_DW | DMAR_IQ_DESCR_IOTLB_DR, 0); 264 dmar_qi_emit_wait_seq(unit, &gseq); 265 dmar_qi_advance_tail(unit); 266 dmar_qi_wait_for_seq(unit, &gseq, false); 267 } 268 269 void 270 dmar_qi_invalidate_iec_glob(struct dmar_unit *unit) 271 { 272 struct dmar_qi_genseq gseq; 273 274 DMAR_ASSERT_LOCKED(unit); 275 dmar_qi_ensure(unit, 2); 276 dmar_qi_emit(unit, DMAR_IQ_DESCR_IEC_INV, 0); 277 dmar_qi_emit_wait_seq(unit, &gseq); 278 dmar_qi_advance_tail(unit); 279 dmar_qi_wait_for_seq(unit, &gseq, false); 280 } 281 282 void 283 dmar_qi_invalidate_iec(struct dmar_unit *unit, u_int start, u_int cnt) 284 { 285 struct dmar_qi_genseq gseq; 286 u_int c, l; 287 288 DMAR_ASSERT_LOCKED(unit); 289 KASSERT(start < unit->irte_cnt && start < start + cnt && 290 start + cnt <= unit->irte_cnt, 291 ("inv iec overflow %d %d %d", unit->irte_cnt, start, cnt)); 292 for (; cnt > 0; cnt -= c, start += c) { 293 l = ffs(start | cnt) - 1; 294 c = 1 << l; 295 dmar_qi_ensure(unit, 1); 296 dmar_qi_emit(unit, DMAR_IQ_DESCR_IEC_INV | 297 DMAR_IQ_DESCR_IEC_IDX | DMAR_IQ_DESCR_IEC_IIDX(start) | 298 DMAR_IQ_DESCR_IEC_IM(l), 0); 299 } 300 dmar_qi_ensure(unit, 1); 301 dmar_qi_emit_wait_seq(unit, &gseq); 302 dmar_qi_advance_tail(unit); 303 304 /* 305 * The caller of the function, in particular, 306 * dmar_ir_program_irte(), may be called from the context 307 * where the sleeping is forbidden (in fact, the 308 * intr_table_lock mutex may be held, locked from 309 * intr_shuffle_irqs()). Wait for the invalidation completion 310 * using the busy wait. 311 * 312 * The impact on the interrupt input setup code is small, the 313 * expected overhead is comparable with the chipset register 314 * read. It is more harmful for the parallel DMA operations, 315 * since we own the dmar unit lock until whole invalidation 316 * queue is processed, which includes requests possibly issued 317 * before our request. 318 */ 319 dmar_qi_wait_for_seq(unit, &gseq, true); 320 } 321 322 int 323 dmar_qi_intr(void *arg) 324 { 325 struct dmar_unit *unit; 326 327 unit = arg; 328 KASSERT(unit->qi_enabled, ("dmar%d: QI is not enabled", unit->unit)); 329 taskqueue_enqueue(unit->qi_taskqueue, &unit->qi_task); 330 return (FILTER_HANDLED); 331 } 332 333 static void 334 dmar_qi_task(void *arg, int pending __unused) 335 { 336 struct dmar_unit *unit; 337 struct dmar_map_entry *entry; 338 uint32_t ics; 339 340 unit = arg; 341 342 DMAR_LOCK(unit); 343 for (;;) { 344 entry = TAILQ_FIRST(&unit->tlb_flush_entries); 345 if (entry == NULL) 346 break; 347 if ((entry->gseq.gen == 0 && entry->gseq.seq == 0) || 348 !dmar_qi_seq_processed(unit, &entry->gseq)) 349 break; 350 TAILQ_REMOVE(&unit->tlb_flush_entries, entry, dmamap_link); 351 DMAR_UNLOCK(unit); 352 dmar_domain_free_entry(entry, (entry->flags & 353 DMAR_MAP_ENTRY_QI_NF) == 0); 354 DMAR_LOCK(unit); 355 } 356 ics = dmar_read4(unit, DMAR_ICS_REG); 357 if ((ics & DMAR_ICS_IWC) != 0) { 358 ics = DMAR_ICS_IWC; 359 dmar_write4(unit, DMAR_ICS_REG, ics); 360 } 361 if (unit->inv_seq_waiters > 0) 362 wakeup(&unit->inv_seq_waiters); 363 DMAR_UNLOCK(unit); 364 } 365 366 int 367 dmar_init_qi(struct dmar_unit *unit) 368 { 369 uint64_t iqa; 370 uint32_t ics; 371 int qi_sz; 372 373 if (!DMAR_HAS_QI(unit) || (unit->hw_cap & DMAR_CAP_CM) != 0) 374 return (0); 375 unit->qi_enabled = 1; 376 TUNABLE_INT_FETCH("hw.dmar.qi", &unit->qi_enabled); 377 if (!unit->qi_enabled) 378 return (0); 379 380 TAILQ_INIT(&unit->tlb_flush_entries); 381 TASK_INIT(&unit->qi_task, 0, dmar_qi_task, unit); 382 unit->qi_taskqueue = taskqueue_create_fast("dmarqf", M_WAITOK, 383 taskqueue_thread_enqueue, &unit->qi_taskqueue); 384 taskqueue_start_threads(&unit->qi_taskqueue, 1, PI_AV, 385 "dmar%d qi taskq", unit->unit); 386 387 unit->inv_waitd_gen = 0; 388 unit->inv_waitd_seq = 1; 389 390 qi_sz = DMAR_IQA_QS_DEF; 391 TUNABLE_INT_FETCH("hw.dmar.qi_size", &qi_sz); 392 if (qi_sz > DMAR_IQA_QS_MAX) 393 qi_sz = DMAR_IQA_QS_MAX; 394 unit->inv_queue_size = (1ULL << qi_sz) * PAGE_SIZE; 395 /* Reserve one descriptor to prevent wraparound. */ 396 unit->inv_queue_avail = unit->inv_queue_size - DMAR_IQ_DESCR_SZ; 397 398 /* The invalidation queue reads by DMARs are always coherent. */ 399 unit->inv_queue = kmem_alloc_contig(kernel_arena, unit->inv_queue_size, 400 M_WAITOK | M_ZERO, 0, dmar_high, PAGE_SIZE, 0, VM_MEMATTR_DEFAULT); 401 unit->inv_waitd_seq_hw_phys = pmap_kextract( 402 (vm_offset_t)&unit->inv_waitd_seq_hw); 403 404 DMAR_LOCK(unit); 405 dmar_write8(unit, DMAR_IQT_REG, 0); 406 iqa = pmap_kextract(unit->inv_queue); 407 iqa |= qi_sz; 408 dmar_write8(unit, DMAR_IQA_REG, iqa); 409 dmar_enable_qi(unit); 410 ics = dmar_read4(unit, DMAR_ICS_REG); 411 if ((ics & DMAR_ICS_IWC) != 0) { 412 ics = DMAR_ICS_IWC; 413 dmar_write4(unit, DMAR_ICS_REG, ics); 414 } 415 dmar_enable_qi_intr(unit); 416 DMAR_UNLOCK(unit); 417 418 return (0); 419 } 420 421 void 422 dmar_fini_qi(struct dmar_unit *unit) 423 { 424 struct dmar_qi_genseq gseq; 425 426 if (unit->qi_enabled) 427 return; 428 taskqueue_drain(unit->qi_taskqueue, &unit->qi_task); 429 taskqueue_free(unit->qi_taskqueue); 430 unit->qi_taskqueue = NULL; 431 432 DMAR_LOCK(unit); 433 /* quisce */ 434 dmar_qi_ensure(unit, 1); 435 dmar_qi_emit_wait_seq(unit, &gseq); 436 dmar_qi_advance_tail(unit); 437 dmar_qi_wait_for_seq(unit, &gseq, false); 438 /* only after the quisce, disable queue */ 439 dmar_disable_qi_intr(unit); 440 dmar_disable_qi(unit); 441 KASSERT(unit->inv_seq_waiters == 0, 442 ("dmar%d: waiters on disabled queue", unit->unit)); 443 DMAR_UNLOCK(unit); 444 445 kmem_free(kernel_arena, unit->inv_queue, unit->inv_queue_size); 446 unit->inv_queue = 0; 447 unit->inv_queue_size = 0; 448 unit->qi_enabled = 0; 449 } 450 451 void 452 dmar_enable_qi_intr(struct dmar_unit *unit) 453 { 454 uint32_t iectl; 455 456 DMAR_ASSERT_LOCKED(unit); 457 KASSERT(DMAR_HAS_QI(unit), ("dmar%d: QI is not supported", unit->unit)); 458 iectl = dmar_read4(unit, DMAR_IECTL_REG); 459 iectl &= ~DMAR_IECTL_IM; 460 dmar_write4(unit, DMAR_IECTL_REG, iectl); 461 } 462 463 void 464 dmar_disable_qi_intr(struct dmar_unit *unit) 465 { 466 uint32_t iectl; 467 468 DMAR_ASSERT_LOCKED(unit); 469 KASSERT(DMAR_HAS_QI(unit), ("dmar%d: QI is not supported", unit->unit)); 470 iectl = dmar_read4(unit, DMAR_IECTL_REG); 471 dmar_write4(unit, DMAR_IECTL_REG, iectl | DMAR_IECTL_IM); 472 } 473