1 /*- 2 * Copyright (c) 2013 The FreeBSD Foundation 3 * All rights reserved. 4 * 5 * This software was developed by Konstantin Belousov <kib@FreeBSD.org> 6 * under sponsorship from the FreeBSD Foundation. 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 27 * SUCH DAMAGE. 28 */ 29 30 #include <sys/cdefs.h> 31 __FBSDID("$FreeBSD$"); 32 33 #include "opt_acpi.h" 34 35 #include <sys/param.h> 36 #include <sys/bus.h> 37 #include <sys/kernel.h> 38 #include <sys/malloc.h> 39 #include <sys/memdesc.h> 40 #include <sys/module.h> 41 #include <sys/rman.h> 42 #include <sys/taskqueue.h> 43 #include <sys/tree.h> 44 #include <sys/vmem.h> 45 #include <machine/bus.h> 46 #include <contrib/dev/acpica/include/acpi.h> 47 #include <contrib/dev/acpica/include/accommon.h> 48 #include <dev/acpica/acpivar.h> 49 #include <dev/pci/pcireg.h> 50 #include <dev/pci/pcivar.h> 51 #include <vm/vm.h> 52 #include <vm/vm_extern.h> 53 #include <vm/vm_kern.h> 54 #include <vm/vm_page.h> 55 #include <vm/vm_map.h> 56 #include <x86/include/busdma_impl.h> 57 #include <x86/iommu/intel_reg.h> 58 #include <x86/iommu/busdma_dmar.h> 59 #include <x86/iommu/intel_dmar.h> 60 61 /* 62 * Fault interrupt handling for DMARs. If advanced fault logging is 63 * not implemented by hardware, the code emulates it. Fast interrupt 64 * handler flushes the fault registers into circular buffer at 65 * unit->fault_log, and schedules a task. 66 * 67 * The fast handler is used since faults usually come in bursts, and 68 * number of fault log registers is limited, e.g. down to one for 5400 69 * MCH. We are trying to reduce the latency for clearing the fault 70 * register file. The task is usually long-running, since printf() is 71 * slow, but this is not problematic because bursts are rare. 72 * 73 * For the same reason, each translation unit task is executed in its 74 * own thread. 75 * 76 * XXXKIB It seems there is no hardware available which implements 77 * advanced fault logging, so the code to handle AFL is not written. 78 */ 79 80 static int 81 dmar_fault_next(struct dmar_unit *unit, int faultp) 82 { 83 84 faultp += 2; 85 if (faultp == unit->fault_log_size) 86 faultp = 0; 87 return (faultp); 88 } 89 90 static void 91 dmar_fault_intr_clear(struct dmar_unit *unit, uint32_t fsts) 92 { 93 uint32_t clear; 94 95 clear = 0; 96 if ((fsts & DMAR_FSTS_ITE) != 0) { 97 printf("DMAR%d: Invalidation timed out\n", unit->unit); 98 clear |= DMAR_FSTS_ITE; 99 } 100 if ((fsts & DMAR_FSTS_ICE) != 0) { 101 printf("DMAR%d: Invalidation completion error\n", 102 unit->unit); 103 clear |= DMAR_FSTS_ICE; 104 } 105 if ((fsts & DMAR_FSTS_IQE) != 0) { 106 printf("DMAR%d: Invalidation queue error\n", 107 unit->unit); 108 clear |= DMAR_FSTS_IQE; 109 } 110 if ((fsts & DMAR_FSTS_APF) != 0) { 111 printf("DMAR%d: Advanced pending fault\n", unit->unit); 112 clear |= DMAR_FSTS_APF; 113 } 114 if ((fsts & DMAR_FSTS_AFO) != 0) { 115 printf("DMAR%d: Advanced fault overflow\n", unit->unit); 116 clear |= DMAR_FSTS_AFO; 117 } 118 if (clear != 0) 119 dmar_write4(unit, DMAR_FSTS_REG, clear); 120 } 121 122 int 123 dmar_fault_intr(void *arg) 124 { 125 struct dmar_unit *unit; 126 uint64_t fault_rec[2]; 127 uint32_t fsts; 128 int fri, frir, faultp; 129 bool enqueue; 130 131 unit = arg; 132 enqueue = false; 133 fsts = dmar_read4(unit, DMAR_FSTS_REG); 134 dmar_fault_intr_clear(unit, fsts); 135 136 if ((fsts & DMAR_FSTS_PPF) == 0) 137 goto done; 138 139 fri = DMAR_FSTS_FRI(fsts); 140 for (;;) { 141 frir = (DMAR_CAP_FRO(unit->hw_cap) + fri) * 16; 142 fault_rec[1] = dmar_read8(unit, frir + 8); 143 if ((fault_rec[1] & DMAR_FRCD2_F) == 0) 144 break; 145 fault_rec[0] = dmar_read8(unit, frir); 146 dmar_write4(unit, frir + 12, DMAR_FRCD2_F32); 147 DMAR_FAULT_LOCK(unit); 148 faultp = unit->fault_log_head; 149 if (dmar_fault_next(unit, faultp) == unit->fault_log_tail) { 150 /* XXXKIB log overflow */ 151 } else { 152 unit->fault_log[faultp] = fault_rec[0]; 153 unit->fault_log[faultp + 1] = fault_rec[1]; 154 unit->fault_log_head = dmar_fault_next(unit, faultp); 155 enqueue = true; 156 } 157 DMAR_FAULT_UNLOCK(unit); 158 fri += 1; 159 if (fri >= DMAR_CAP_NFR(unit->hw_cap)) 160 fri = 0; 161 } 162 163 done: 164 /* 165 * On SandyBridge, due to errata BJ124, IvyBridge errata 166 * BV100, and Haswell errata HSD40, "Spurious Intel VT-d 167 * Interrupts May Occur When the PFO Bit is Set". Handle the 168 * cases by clearing overflow bit even if no fault is 169 * reported. 170 * 171 * On IvyBridge, errata BV30 states that clearing clear 172 * DMAR_FRCD2_F bit in the fault register causes spurious 173 * interrupt. Do nothing. 174 * 175 */ 176 if ((fsts & DMAR_FSTS_PFO) != 0) { 177 printf("DMAR%d: Fault Overflow\n", unit->unit); 178 dmar_write4(unit, DMAR_FSTS_REG, DMAR_FSTS_PFO); 179 } 180 181 if (enqueue) { 182 taskqueue_enqueue_fast(unit->fault_taskqueue, 183 &unit->fault_task); 184 } 185 return (FILTER_HANDLED); 186 } 187 188 static void 189 dmar_fault_task(void *arg, int pending __unused) 190 { 191 struct dmar_unit *unit; 192 struct dmar_ctx *ctx; 193 uint64_t fault_rec[2]; 194 int sid, bus, slot, func, faultp; 195 196 unit = arg; 197 DMAR_FAULT_LOCK(unit); 198 for (;;) { 199 faultp = unit->fault_log_tail; 200 if (faultp == unit->fault_log_head) 201 break; 202 203 fault_rec[0] = unit->fault_log[faultp]; 204 fault_rec[1] = unit->fault_log[faultp + 1]; 205 unit->fault_log_tail = dmar_fault_next(unit, faultp); 206 DMAR_FAULT_UNLOCK(unit); 207 208 sid = DMAR_FRCD2_SID(fault_rec[1]); 209 printf("DMAR%d: ", unit->unit); 210 DMAR_LOCK(unit); 211 ctx = dmar_find_ctx_locked(unit, sid); 212 if (ctx == NULL) { 213 printf("<unknown dev>:"); 214 215 /* 216 * Note that the slot and function will not be correct 217 * if ARI is in use, but without a ctx entry we have 218 * no way of knowing whether ARI is in use or not. 219 */ 220 bus = PCI_RID2BUS(sid); 221 slot = PCI_RID2SLOT(sid); 222 func = PCI_RID2FUNC(sid); 223 } else { 224 ctx->flags |= DMAR_CTX_FAULTED; 225 ctx->last_fault_rec[0] = fault_rec[0]; 226 ctx->last_fault_rec[1] = fault_rec[1]; 227 device_print_prettyname(ctx->ctx_tag.owner); 228 bus = pci_get_bus(ctx->ctx_tag.owner); 229 slot = pci_get_slot(ctx->ctx_tag.owner); 230 func = pci_get_function(ctx->ctx_tag.owner); 231 } 232 DMAR_UNLOCK(unit); 233 printf( 234 "pci%d:%d:%d sid %x fault acc %x adt 0x%x reason 0x%x " 235 "addr %jx\n", 236 bus, slot, func, sid, DMAR_FRCD2_T(fault_rec[1]), 237 DMAR_FRCD2_AT(fault_rec[1]), DMAR_FRCD2_FR(fault_rec[1]), 238 (uintmax_t)fault_rec[0]); 239 DMAR_FAULT_LOCK(unit); 240 } 241 DMAR_FAULT_UNLOCK(unit); 242 } 243 244 static void 245 dmar_clear_faults(struct dmar_unit *unit) 246 { 247 uint32_t frec, frir, fsts; 248 int i; 249 250 for (i = 0; i < DMAR_CAP_NFR(unit->hw_cap); i++) { 251 frir = (DMAR_CAP_FRO(unit->hw_cap) + i) * 16; 252 frec = dmar_read4(unit, frir + 12); 253 if ((frec & DMAR_FRCD2_F32) == 0) 254 continue; 255 dmar_write4(unit, frir + 12, DMAR_FRCD2_F32); 256 } 257 fsts = dmar_read4(unit, DMAR_FSTS_REG); 258 dmar_write4(unit, DMAR_FSTS_REG, fsts); 259 } 260 261 int 262 dmar_init_fault_log(struct dmar_unit *unit) 263 { 264 265 mtx_init(&unit->fault_lock, "dmarflt", NULL, MTX_SPIN); 266 unit->fault_log_size = 256; /* 128 fault log entries */ 267 TUNABLE_INT_FETCH("hw.dmar.fault_log_size", &unit->fault_log_size); 268 if (unit->fault_log_size % 2 != 0) 269 panic("hw.dmar_fault_log_size must be even"); 270 unit->fault_log = malloc(sizeof(uint64_t) * unit->fault_log_size, 271 M_DEVBUF, M_WAITOK | M_ZERO); 272 273 TASK_INIT(&unit->fault_task, 0, dmar_fault_task, unit); 274 unit->fault_taskqueue = taskqueue_create_fast("dmar", M_WAITOK, 275 taskqueue_thread_enqueue, &unit->fault_taskqueue); 276 taskqueue_start_threads(&unit->fault_taskqueue, 1, PI_AV, 277 "dmar%d fault taskq", unit->unit); 278 279 DMAR_LOCK(unit); 280 dmar_disable_fault_intr(unit); 281 dmar_clear_faults(unit); 282 dmar_enable_fault_intr(unit); 283 DMAR_UNLOCK(unit); 284 285 return (0); 286 } 287 288 void 289 dmar_fini_fault_log(struct dmar_unit *unit) 290 { 291 292 DMAR_LOCK(unit); 293 dmar_disable_fault_intr(unit); 294 DMAR_UNLOCK(unit); 295 296 if (unit->fault_taskqueue == NULL) 297 return; 298 299 taskqueue_drain(unit->fault_taskqueue, &unit->fault_task); 300 taskqueue_free(unit->fault_taskqueue); 301 unit->fault_taskqueue = NULL; 302 mtx_destroy(&unit->fault_lock); 303 304 free(unit->fault_log, M_DEVBUF); 305 unit->fault_log = NULL; 306 unit->fault_log_head = unit->fault_log_tail = 0; 307 } 308 309 void 310 dmar_enable_fault_intr(struct dmar_unit *unit) 311 { 312 uint32_t fectl; 313 314 DMAR_ASSERT_LOCKED(unit); 315 fectl = dmar_read4(unit, DMAR_FECTL_REG); 316 fectl &= ~DMAR_FECTL_IM; 317 dmar_write4(unit, DMAR_FECTL_REG, fectl); 318 } 319 320 void 321 dmar_disable_fault_intr(struct dmar_unit *unit) 322 { 323 uint32_t fectl; 324 325 DMAR_ASSERT_LOCKED(unit); 326 fectl = dmar_read4(unit, DMAR_FECTL_REG); 327 dmar_write4(unit, DMAR_FECTL_REG, fectl | DMAR_FECTL_IM); 328 } 329