1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 3 * 4 * Copyright (c) 2013 The FreeBSD Foundation 5 * All rights reserved. 6 * 7 * This software was developed by Konstantin Belousov <kib@FreeBSD.org> 8 * under sponsorship from the FreeBSD Foundation. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 20 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 21 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 22 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 23 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 24 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 25 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 26 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 27 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 28 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 29 * SUCH DAMAGE. 30 */ 31 32 #include <sys/cdefs.h> 33 __FBSDID("$FreeBSD$"); 34 35 #include "opt_acpi.h" 36 37 #include <sys/param.h> 38 #include <sys/bus.h> 39 #include <sys/kernel.h> 40 #include <sys/malloc.h> 41 #include <sys/memdesc.h> 42 #include <sys/module.h> 43 #include <sys/rman.h> 44 #include <sys/taskqueue.h> 45 #include <sys/tree.h> 46 #include <sys/vmem.h> 47 #include <machine/bus.h> 48 #include <contrib/dev/acpica/include/acpi.h> 49 #include <contrib/dev/acpica/include/accommon.h> 50 #include <dev/acpica/acpivar.h> 51 #include <dev/pci/pcireg.h> 52 #include <dev/pci/pcivar.h> 53 #include <vm/vm.h> 54 #include <vm/vm_extern.h> 55 #include <vm/vm_kern.h> 56 #include <vm/vm_page.h> 57 #include <vm/vm_map.h> 58 #include <x86/include/busdma_impl.h> 59 #include <x86/iommu/intel_reg.h> 60 #include <dev/iommu/busdma_iommu.h> 61 #include <x86/iommu/intel_dmar.h> 62 63 /* 64 * Fault interrupt handling for DMARs. If advanced fault logging is 65 * not implemented by hardware, the code emulates it. Fast interrupt 66 * handler flushes the fault registers into circular buffer at 67 * unit->fault_log, and schedules a task. 68 * 69 * The fast handler is used since faults usually come in bursts, and 70 * number of fault log registers is limited, e.g. down to one for 5400 71 * MCH. We are trying to reduce the latency for clearing the fault 72 * register file. The task is usually long-running, since printf() is 73 * slow, but this is not problematic because bursts are rare. 74 * 75 * For the same reason, each translation unit task is executed in its 76 * own thread. 77 * 78 * XXXKIB It seems there is no hardware available which implements 79 * advanced fault logging, so the code to handle AFL is not written. 80 */ 81 82 static int 83 dmar_fault_next(struct dmar_unit *unit, int faultp) 84 { 85 86 faultp += 2; 87 if (faultp == unit->fault_log_size) 88 faultp = 0; 89 return (faultp); 90 } 91 92 static void 93 dmar_fault_intr_clear(struct dmar_unit *unit, uint32_t fsts) 94 { 95 uint32_t clear; 96 97 clear = 0; 98 if ((fsts & DMAR_FSTS_ITE) != 0) { 99 printf("DMAR%d: Invalidation timed out\n", unit->iommu.unit); 100 clear |= DMAR_FSTS_ITE; 101 } 102 if ((fsts & DMAR_FSTS_ICE) != 0) { 103 printf("DMAR%d: Invalidation completion error\n", 104 unit->iommu.unit); 105 clear |= DMAR_FSTS_ICE; 106 } 107 if ((fsts & DMAR_FSTS_IQE) != 0) { 108 printf("DMAR%d: Invalidation queue error\n", 109 unit->iommu.unit); 110 clear |= DMAR_FSTS_IQE; 111 } 112 if ((fsts & DMAR_FSTS_APF) != 0) { 113 printf("DMAR%d: Advanced pending fault\n", unit->iommu.unit); 114 clear |= DMAR_FSTS_APF; 115 } 116 if ((fsts & DMAR_FSTS_AFO) != 0) { 117 printf("DMAR%d: Advanced fault overflow\n", unit->iommu.unit); 118 clear |= DMAR_FSTS_AFO; 119 } 120 if (clear != 0) 121 dmar_write4(unit, DMAR_FSTS_REG, clear); 122 } 123 124 int 125 dmar_fault_intr(void *arg) 126 { 127 struct dmar_unit *unit; 128 uint64_t fault_rec[2]; 129 uint32_t fsts; 130 int fri, frir, faultp; 131 bool enqueue; 132 133 unit = arg; 134 enqueue = false; 135 fsts = dmar_read4(unit, DMAR_FSTS_REG); 136 dmar_fault_intr_clear(unit, fsts); 137 138 if ((fsts & DMAR_FSTS_PPF) == 0) 139 goto done; 140 141 fri = DMAR_FSTS_FRI(fsts); 142 for (;;) { 143 frir = (DMAR_CAP_FRO(unit->hw_cap) + fri) * 16; 144 fault_rec[1] = dmar_read8(unit, frir + 8); 145 if ((fault_rec[1] & DMAR_FRCD2_F) == 0) 146 break; 147 fault_rec[0] = dmar_read8(unit, frir); 148 dmar_write4(unit, frir + 12, DMAR_FRCD2_F32); 149 DMAR_FAULT_LOCK(unit); 150 faultp = unit->fault_log_head; 151 if (dmar_fault_next(unit, faultp) == unit->fault_log_tail) { 152 /* XXXKIB log overflow */ 153 } else { 154 unit->fault_log[faultp] = fault_rec[0]; 155 unit->fault_log[faultp + 1] = fault_rec[1]; 156 unit->fault_log_head = dmar_fault_next(unit, faultp); 157 enqueue = true; 158 } 159 DMAR_FAULT_UNLOCK(unit); 160 fri += 1; 161 if (fri >= DMAR_CAP_NFR(unit->hw_cap)) 162 fri = 0; 163 } 164 165 done: 166 /* 167 * On SandyBridge, due to errata BJ124, IvyBridge errata 168 * BV100, and Haswell errata HSD40, "Spurious Intel VT-d 169 * Interrupts May Occur When the PFO Bit is Set". Handle the 170 * cases by clearing overflow bit even if no fault is 171 * reported. 172 * 173 * On IvyBridge, errata BV30 states that clearing clear 174 * DMAR_FRCD2_F bit in the fault register causes spurious 175 * interrupt. Do nothing. 176 * 177 */ 178 if ((fsts & DMAR_FSTS_PFO) != 0) { 179 printf("DMAR%d: Fault Overflow\n", unit->iommu.unit); 180 dmar_write4(unit, DMAR_FSTS_REG, DMAR_FSTS_PFO); 181 } 182 183 if (enqueue) { 184 taskqueue_enqueue(unit->fault_taskqueue, 185 &unit->fault_task); 186 } 187 return (FILTER_HANDLED); 188 } 189 190 static void 191 dmar_fault_task(void *arg, int pending __unused) 192 { 193 struct dmar_unit *unit; 194 struct dmar_ctx *ctx; 195 uint64_t fault_rec[2]; 196 int sid, bus, slot, func, faultp; 197 198 unit = arg; 199 DMAR_FAULT_LOCK(unit); 200 for (;;) { 201 faultp = unit->fault_log_tail; 202 if (faultp == unit->fault_log_head) 203 break; 204 205 fault_rec[0] = unit->fault_log[faultp]; 206 fault_rec[1] = unit->fault_log[faultp + 1]; 207 unit->fault_log_tail = dmar_fault_next(unit, faultp); 208 DMAR_FAULT_UNLOCK(unit); 209 210 sid = DMAR_FRCD2_SID(fault_rec[1]); 211 printf("DMAR%d: ", unit->iommu.unit); 212 DMAR_LOCK(unit); 213 ctx = dmar_find_ctx_locked(unit, sid); 214 if (ctx == NULL) { 215 printf("<unknown dev>:"); 216 217 /* 218 * Note that the slot and function will not be correct 219 * if ARI is in use, but without a ctx entry we have 220 * no way of knowing whether ARI is in use or not. 221 */ 222 bus = PCI_RID2BUS(sid); 223 slot = PCI_RID2SLOT(sid); 224 func = PCI_RID2FUNC(sid); 225 } else { 226 ctx->context.flags |= IOMMU_CTX_FAULTED; 227 ctx->last_fault_rec[0] = fault_rec[0]; 228 ctx->last_fault_rec[1] = fault_rec[1]; 229 device_print_prettyname(ctx->context.tag->owner); 230 bus = pci_get_bus(ctx->context.tag->owner); 231 slot = pci_get_slot(ctx->context.tag->owner); 232 func = pci_get_function(ctx->context.tag->owner); 233 } 234 DMAR_UNLOCK(unit); 235 printf( 236 "pci%d:%d:%d sid %x fault acc %x adt 0x%x reason 0x%x " 237 "addr %jx\n", 238 bus, slot, func, sid, DMAR_FRCD2_T(fault_rec[1]), 239 DMAR_FRCD2_AT(fault_rec[1]), DMAR_FRCD2_FR(fault_rec[1]), 240 (uintmax_t)fault_rec[0]); 241 DMAR_FAULT_LOCK(unit); 242 } 243 DMAR_FAULT_UNLOCK(unit); 244 } 245 246 static void 247 dmar_clear_faults(struct dmar_unit *unit) 248 { 249 uint32_t frec, frir, fsts; 250 int i; 251 252 for (i = 0; i < DMAR_CAP_NFR(unit->hw_cap); i++) { 253 frir = (DMAR_CAP_FRO(unit->hw_cap) + i) * 16; 254 frec = dmar_read4(unit, frir + 12); 255 if ((frec & DMAR_FRCD2_F32) == 0) 256 continue; 257 dmar_write4(unit, frir + 12, DMAR_FRCD2_F32); 258 } 259 fsts = dmar_read4(unit, DMAR_FSTS_REG); 260 dmar_write4(unit, DMAR_FSTS_REG, fsts); 261 } 262 263 int 264 dmar_init_fault_log(struct dmar_unit *unit) 265 { 266 267 mtx_init(&unit->fault_lock, "dmarflt", NULL, MTX_SPIN); 268 unit->fault_log_size = 256; /* 128 fault log entries */ 269 TUNABLE_INT_FETCH("hw.dmar.fault_log_size", &unit->fault_log_size); 270 if (unit->fault_log_size % 2 != 0) 271 panic("hw.dmar_fault_log_size must be even"); 272 unit->fault_log = malloc(sizeof(uint64_t) * unit->fault_log_size, 273 M_DEVBUF, M_WAITOK | M_ZERO); 274 275 TASK_INIT(&unit->fault_task, 0, dmar_fault_task, unit); 276 unit->fault_taskqueue = taskqueue_create_fast("dmarff", M_WAITOK, 277 taskqueue_thread_enqueue, &unit->fault_taskqueue); 278 taskqueue_start_threads(&unit->fault_taskqueue, 1, PI_AV, 279 "dmar%d fault taskq", unit->iommu.unit); 280 281 DMAR_LOCK(unit); 282 dmar_disable_fault_intr(unit); 283 dmar_clear_faults(unit); 284 dmar_enable_fault_intr(unit); 285 DMAR_UNLOCK(unit); 286 287 return (0); 288 } 289 290 void 291 dmar_fini_fault_log(struct dmar_unit *unit) 292 { 293 294 if (unit->fault_taskqueue == NULL) 295 return; 296 297 DMAR_LOCK(unit); 298 dmar_disable_fault_intr(unit); 299 DMAR_UNLOCK(unit); 300 301 taskqueue_drain(unit->fault_taskqueue, &unit->fault_task); 302 taskqueue_free(unit->fault_taskqueue); 303 unit->fault_taskqueue = NULL; 304 mtx_destroy(&unit->fault_lock); 305 306 free(unit->fault_log, M_DEVBUF); 307 unit->fault_log = NULL; 308 unit->fault_log_head = unit->fault_log_tail = 0; 309 } 310 311 void 312 dmar_enable_fault_intr(struct dmar_unit *unit) 313 { 314 uint32_t fectl; 315 316 DMAR_ASSERT_LOCKED(unit); 317 fectl = dmar_read4(unit, DMAR_FECTL_REG); 318 fectl &= ~DMAR_FECTL_IM; 319 dmar_write4(unit, DMAR_FECTL_REG, fectl); 320 } 321 322 void 323 dmar_disable_fault_intr(struct dmar_unit *unit) 324 { 325 uint32_t fectl; 326 327 DMAR_ASSERT_LOCKED(unit); 328 fectl = dmar_read4(unit, DMAR_FECTL_REG); 329 dmar_write4(unit, DMAR_FECTL_REG, fectl | DMAR_FECTL_IM); 330 } 331