1 /*- 2 * Copyright (c) 2013 The FreeBSD Foundation 3 * All rights reserved. 4 * 5 * This software was developed by Konstantin Belousov <kib@FreeBSD.org> 6 * under sponsorship from the FreeBSD Foundation. 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 27 * SUCH DAMAGE. 28 */ 29 30 #include <sys/cdefs.h> 31 __FBSDID("$FreeBSD$"); 32 33 #include "opt_acpi.h" 34 35 #include <sys/param.h> 36 #include <sys/bus.h> 37 #include <sys/kernel.h> 38 #include <sys/malloc.h> 39 #include <sys/memdesc.h> 40 #include <sys/module.h> 41 #include <sys/rman.h> 42 #include <sys/taskqueue.h> 43 #include <sys/tree.h> 44 #include <machine/bus.h> 45 #include <contrib/dev/acpica/include/acpi.h> 46 #include <contrib/dev/acpica/include/accommon.h> 47 #include <dev/acpica/acpivar.h> 48 #include <vm/vm.h> 49 #include <vm/vm_extern.h> 50 #include <vm/vm_kern.h> 51 #include <vm/vm_page.h> 52 #include <vm/vm_map.h> 53 #include <x86/include/busdma_impl.h> 54 #include <x86/iommu/intel_reg.h> 55 #include <x86/iommu/busdma_dmar.h> 56 #include <x86/iommu/intel_dmar.h> 57 58 /* 59 * Fault interrupt handling for DMARs. If advanced fault logging is 60 * not implemented by hardware, the code emulates it. Fast interrupt 61 * handler flushes the fault registers into circular buffer at 62 * unit->fault_log, and schedules a task. 63 * 64 * The fast handler is used since faults usually come in bursts, and 65 * number of fault log registers is limited, e.g. down to one for 5400 66 * MCH. We are trying to reduce the latency for clearing the fault 67 * register file. The task is usually long-running, since printf() is 68 * slow, but this is not problematic because bursts are rare. 69 * 70 * For the same reason, each translation unit task is executed in its 71 * own thread. 72 * 73 * XXXKIB It seems there is no hardware available which implements 74 * advanced fault logging, so the code to handle AFL is not written. 75 */ 76 77 static int 78 dmar_fault_next(struct dmar_unit *unit, int faultp) 79 { 80 81 faultp += 2; 82 if (faultp == unit->fault_log_size) 83 faultp = 0; 84 return (faultp); 85 } 86 87 static void 88 dmar_fault_intr_clear(struct dmar_unit *unit, uint32_t fsts) 89 { 90 uint32_t clear; 91 92 clear = 0; 93 if ((fsts & DMAR_FSTS_ITE) != 0) { 94 printf("DMAR%d: Invalidation timed out\n", unit->unit); 95 clear |= DMAR_FSTS_ITE; 96 } 97 if ((fsts & DMAR_FSTS_ICE) != 0) { 98 printf("DMAR%d: Invalidation completion error\n", 99 unit->unit); 100 clear |= DMAR_FSTS_ICE; 101 } 102 if ((fsts & DMAR_FSTS_IQE) != 0) { 103 printf("DMAR%d: Invalidation queue error\n", 104 unit->unit); 105 clear |= DMAR_FSTS_IQE; 106 } 107 if ((fsts & DMAR_FSTS_APF) != 0) { 108 printf("DMAR%d: Advanced pending fault\n", unit->unit); 109 clear |= DMAR_FSTS_APF; 110 } 111 if ((fsts & DMAR_FSTS_AFO) != 0) { 112 printf("DMAR%d: Advanced fault overflow\n", unit->unit); 113 clear |= DMAR_FSTS_AFO; 114 } 115 if (clear != 0) 116 dmar_write4(unit, DMAR_FSTS_REG, clear); 117 } 118 119 int 120 dmar_fault_intr(void *arg) 121 { 122 struct dmar_unit *unit; 123 uint64_t fault_rec[2]; 124 uint32_t fsts; 125 int fri, frir, faultp; 126 bool enqueue; 127 128 unit = arg; 129 enqueue = false; 130 fsts = dmar_read4(unit, DMAR_FSTS_REG); 131 dmar_fault_intr_clear(unit, fsts); 132 133 if ((fsts & DMAR_FSTS_PPF) == 0) 134 goto done; 135 136 fri = DMAR_FSTS_FRI(fsts); 137 for (;;) { 138 frir = (DMAR_CAP_FRO(unit->hw_cap) + fri) * 16; 139 fault_rec[1] = dmar_read8(unit, frir + 8); 140 if ((fault_rec[1] & DMAR_FRCD2_F) == 0) 141 break; 142 fault_rec[0] = dmar_read8(unit, frir); 143 dmar_write4(unit, frir + 12, DMAR_FRCD2_F32); 144 DMAR_FAULT_LOCK(unit); 145 faultp = unit->fault_log_head; 146 if (dmar_fault_next(unit, faultp) == unit->fault_log_tail) { 147 /* XXXKIB log overflow */ 148 } else { 149 unit->fault_log[faultp] = fault_rec[0]; 150 unit->fault_log[faultp + 1] = fault_rec[1]; 151 unit->fault_log_head = dmar_fault_next(unit, faultp); 152 enqueue = true; 153 } 154 DMAR_FAULT_UNLOCK(unit); 155 fri += 1; 156 if (fri >= DMAR_CAP_NFR(unit->hw_cap)) 157 fri = 0; 158 } 159 160 done: 161 /* 162 * On SandyBridge, due to errata BJ124, IvyBridge errata 163 * BV100, and Haswell errata HSD40, "Spurious Intel VT-d 164 * Interrupts May Occur When the PFO Bit is Set". Handle the 165 * cases by clearing overflow bit even if no fault is 166 * reported. 167 * 168 * On IvyBridge, errata BV30 states that clearing clear 169 * DMAR_FRCD2_F bit in the fault register causes spurious 170 * interrupt. Do nothing. 171 * 172 */ 173 if ((fsts & DMAR_FSTS_PFO) != 0) { 174 printf("DMAR%d: Fault Overflow\n", unit->unit); 175 dmar_write4(unit, DMAR_FSTS_REG, DMAR_FSTS_PFO); 176 } 177 178 if (enqueue) { 179 taskqueue_enqueue_fast(unit->fault_taskqueue, 180 &unit->fault_task); 181 } 182 return (FILTER_HANDLED); 183 } 184 185 static void 186 dmar_fault_task(void *arg, int pending __unused) 187 { 188 struct dmar_unit *unit; 189 struct dmar_ctx *ctx; 190 uint64_t fault_rec[2]; 191 int sid, bus, slot, func, faultp; 192 193 unit = arg; 194 DMAR_FAULT_LOCK(unit); 195 for (;;) { 196 faultp = unit->fault_log_tail; 197 if (faultp == unit->fault_log_head) 198 break; 199 200 fault_rec[0] = unit->fault_log[faultp]; 201 fault_rec[1] = unit->fault_log[faultp + 1]; 202 unit->fault_log_tail = dmar_fault_next(unit, faultp); 203 DMAR_FAULT_UNLOCK(unit); 204 205 sid = DMAR_FRCD2_SID(fault_rec[1]); 206 bus = (sid >> 8) & 0xf; 207 slot = (sid >> 3) & 0x1f; 208 func = sid & 0x7; 209 printf("DMAR%d: ", unit->unit); 210 DMAR_LOCK(unit); 211 ctx = dmar_find_ctx_locked(unit, bus, slot, func); 212 if (ctx == NULL) { 213 printf("<unknown dev>:"); 214 } else { 215 ctx->flags |= DMAR_CTX_FAULTED; 216 ctx->last_fault_rec[0] = fault_rec[0]; 217 ctx->last_fault_rec[1] = fault_rec[1]; 218 device_print_prettyname(ctx->ctx_tag.owner); 219 } 220 DMAR_UNLOCK(unit); 221 printf( 222 "pci%d:%d:%d fault acc %x adt 0x%x reason 0x%x addr %jx\n", 223 bus, slot, func, DMAR_FRCD2_T(fault_rec[1]), 224 DMAR_FRCD2_AT(fault_rec[1]), DMAR_FRCD2_FR(fault_rec[1]), 225 (uintmax_t)fault_rec[0]); 226 DMAR_FAULT_LOCK(unit); 227 } 228 DMAR_FAULT_UNLOCK(unit); 229 } 230 231 static void 232 dmar_clear_faults(struct dmar_unit *unit) 233 { 234 uint32_t frec, frir, fsts; 235 int i; 236 237 for (i = 0; i < DMAR_CAP_NFR(unit->hw_cap); i++) { 238 frir = (DMAR_CAP_FRO(unit->hw_cap) + i) * 16; 239 frec = dmar_read4(unit, frir + 12); 240 if ((frec & DMAR_FRCD2_F32) == 0) 241 continue; 242 dmar_write4(unit, frir + 12, DMAR_FRCD2_F32); 243 } 244 fsts = dmar_read4(unit, DMAR_FSTS_REG); 245 dmar_write4(unit, DMAR_FSTS_REG, fsts); 246 } 247 248 int 249 dmar_init_fault_log(struct dmar_unit *unit) 250 { 251 252 mtx_init(&unit->fault_lock, "dmarflt", NULL, MTX_SPIN); 253 unit->fault_log_size = 256; /* 128 fault log entries */ 254 TUNABLE_INT_FETCH("hw.dmar.fault_log_size", &unit->fault_log_size); 255 if (unit->fault_log_size % 2 != 0) 256 panic("hw.dmar_fault_log_size must be even"); 257 unit->fault_log = malloc(sizeof(uint64_t) * unit->fault_log_size, 258 M_DEVBUF, M_WAITOK | M_ZERO); 259 260 TASK_INIT(&unit->fault_task, 0, dmar_fault_task, unit); 261 unit->fault_taskqueue = taskqueue_create_fast("dmar", M_WAITOK, 262 taskqueue_thread_enqueue, &unit->fault_taskqueue); 263 taskqueue_start_threads(&unit->fault_taskqueue, 1, PI_AV, 264 "dmar%d fault taskq", unit->unit); 265 266 DMAR_LOCK(unit); 267 dmar_disable_fault_intr(unit); 268 dmar_clear_faults(unit); 269 dmar_enable_fault_intr(unit); 270 DMAR_UNLOCK(unit); 271 272 return (0); 273 } 274 275 void 276 dmar_fini_fault_log(struct dmar_unit *unit) 277 { 278 279 DMAR_LOCK(unit); 280 dmar_disable_fault_intr(unit); 281 DMAR_UNLOCK(unit); 282 283 if (unit->fault_taskqueue == NULL) 284 return; 285 286 taskqueue_drain(unit->fault_taskqueue, &unit->fault_task); 287 taskqueue_free(unit->fault_taskqueue); 288 unit->fault_taskqueue = NULL; 289 mtx_destroy(&unit->fault_lock); 290 291 free(unit->fault_log, M_DEVBUF); 292 unit->fault_log = NULL; 293 unit->fault_log_head = unit->fault_log_tail = 0; 294 } 295 296 void 297 dmar_enable_fault_intr(struct dmar_unit *unit) 298 { 299 uint32_t fectl; 300 301 DMAR_ASSERT_LOCKED(unit); 302 fectl = dmar_read4(unit, DMAR_FECTL_REG); 303 fectl &= ~DMAR_FECTL_IM; 304 dmar_write4(unit, DMAR_FECTL_REG, fectl); 305 } 306 307 void 308 dmar_disable_fault_intr(struct dmar_unit *unit) 309 { 310 uint32_t fectl; 311 312 DMAR_ASSERT_LOCKED(unit); 313 fectl = dmar_read4(unit, DMAR_FECTL_REG); 314 dmar_write4(unit, DMAR_FECTL_REG, fectl | DMAR_FECTL_IM); 315 } 316