1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause 3 * 4 * Copyright (c) 2024 The FreeBSD Foundation 5 * 6 * This software was developed by Konstantin Belousov <kib@FreeBSD.org> 7 * under sponsorship from the FreeBSD Foundation. 8 * 9 * Redistribution and use in source and binary forms, with or without 10 * modification, are permitted provided that the following conditions 11 * are met: 12 * 1. Redistributions of source code must retain the above copyright 13 * notice, this list of conditions and the following disclaimer. 14 * 2. Redistributions in binary form must reproduce the above copyright 15 * notice, this list of conditions and the following disclaimer in the 16 * documentation and/or other materials provided with the distribution. 17 * 18 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 19 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 20 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 21 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 22 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 23 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 24 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 25 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 26 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 27 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 28 * SUCH DAMAGE. 29 */ 30 31 #include "opt_acpi.h" 32 33 #include <sys/param.h> 34 #include <sys/bus.h> 35 #include <sys/kernel.h> 36 #include <sys/lock.h> 37 #include <sys/malloc.h> 38 #include <sys/memdesc.h> 39 #include <sys/module.h> 40 #include <sys/mutex.h> 41 #include <sys/rman.h> 42 #include <sys/rwlock.h> 43 #include <sys/smp.h> 44 #include <sys/taskqueue.h> 45 #include <sys/tree.h> 46 #include <sys/vmem.h> 47 #include <vm/vm.h> 48 #include <vm/vm_extern.h> 49 #include <vm/vm_object.h> 50 #include <vm/vm_page.h> 51 #include <vm/vm_pageout.h> 52 #include <vm/vm_pager.h> 53 #include <contrib/dev/acpica/include/acpi.h> 54 #include <contrib/dev/acpica/include/accommon.h> 55 #include <dev/acpica/acpivar.h> 56 #include <dev/pci/pcireg.h> 57 #include <dev/pci/pcivar.h> 58 #include <machine/bus.h> 59 #include <machine/pci_cfgreg.h> 60 #include "pcib_if.h" 61 #include <machine/intr_machdep.h> 62 #include <machine/md_var.h> 63 #include <machine/cputypes.h> 64 #include <x86/apicreg.h> 65 #include <x86/apicvar.h> 66 #include <dev/iommu/iommu.h> 67 #include <dev/iommu/busdma_iommu.h> 68 #include <x86/iommu/amd_reg.h> 69 #include <x86/iommu/x86_iommu.h> 70 #include <x86/iommu/amd_iommu.h> 71 72 static void 73 amdiommu_event_rearm_intr(struct amdiommu_unit *unit) 74 { 75 amdiommu_write8(unit, AMDIOMMU_CMDEV_STATUS, 76 AMDIOMMU_CMDEVS_EVLOGINT); 77 } 78 79 static void 80 amdiommu_event_log_inc_head(struct amdiommu_unit *unit) 81 { 82 unit->event_log_head++; 83 if (unit->event_log_head >= unit->event_log_size) 84 unit->event_log_head = 0; 85 } 86 87 static void 88 amdiommu_event_log_print(struct amdiommu_unit *unit, 89 const struct amdiommu_event_generic *evp, bool fancy) 90 { 91 printf("amdiommu%d: event type 0x%x 0x%08x 0x%08x 0x%08x 0x%08x\n", 92 unit->iommu.unit, evp->code, evp->w0, evp->ww1, evp->w2, evp->w3); 93 if (!fancy) 94 return; 95 96 AMDIOMMU_ASSERT_LOCKED(unit); 97 if (evp->code == AMDIOMMU_EV_ILL_DEV_TABLE_ENTRY) { 98 const struct amdiommu_event_ill_dev_table_entry *ev_dte_p; 99 const struct amdiommu_dte *dte; 100 const uint32_t *x; 101 int i; 102 103 ev_dte_p = (const struct 104 amdiommu_event_ill_dev_table_entry *)evp; 105 dte = &unit->dev_tbl[ev_dte_p->devid]; 106 107 printf("\tIllegal Dev Tab Entry dte@%p:", dte); 108 for (i = 0, x = (const uint32_t *)dte; i < sizeof(*dte) / 109 sizeof(uint32_t); i++, x++) 110 printf(" 0x%08x", *x); 111 printf("\n"); 112 } else if (evp->code == AMDIOMMU_EV_IO_PAGE_FAULT) { 113 const struct amdiommu_event_io_page_fault_entry *ev_iopf_p; 114 struct amdiommu_ctx *ctx; 115 device_t dev; 116 117 ev_iopf_p = (const struct 118 amdiommu_event_io_page_fault_entry *)evp; 119 printf("\tPage Fault rid %#x dom %d", 120 ev_iopf_p->devid, ev_iopf_p->pasid); 121 ctx = amdiommu_find_ctx_locked(unit, ev_iopf_p->devid); 122 if (ctx != NULL) { 123 dev = ctx->context.tag->owner; 124 if (dev != NULL) 125 printf(" %s", device_get_nameunit(dev)); 126 } 127 printf("\n\t" 128 "gn %d nx %d us %d i %d pr %d rw %d pe %d rz %d tr %d" 129 "\n\tgaddr %#jx\n", 130 ev_iopf_p->gn, ev_iopf_p->nx, ev_iopf_p->us, ev_iopf_p->i, 131 ev_iopf_p->pr, ev_iopf_p->rw, ev_iopf_p->pe, ev_iopf_p->rz, 132 ev_iopf_p->tr, 133 (((uintmax_t)(ev_iopf_p->addr2)) << 32) | 134 ev_iopf_p->addr1); 135 } 136 } 137 138 static u_int 139 amdiommu_event_log_tail(struct amdiommu_unit *unit) 140 { 141 return (amdiommu_read8(unit, AMDIOMMU_EVNTLOG_TAIL) >> 142 AMDIOMMU_EV_SZ_SHIFT); 143 } 144 145 static u_int 146 amdiommu_event_copy_log_inc(u_int idx) 147 { 148 idx++; 149 if (idx == nitems(((struct amdiommu_unit *)NULL)->event_copy_log)) 150 idx = 0; 151 return (idx); 152 } 153 154 static bool 155 amdiommu_event_copy_log_hasspace(struct amdiommu_unit *unit) 156 { 157 return (unit->event_copy_tail != amdiommu_event_copy_log_inc( 158 unit->event_copy_head)); 159 } 160 161 void 162 amdiommu_event_intr(struct amdiommu_unit *unit, uint64_t status) 163 { 164 struct amdiommu_event_generic *evp; 165 u_int hw_tail, hw_tail1; 166 bool enqueue; 167 168 enqueue = (status & AMDIOMMU_CMDEVS_EVOVRFLW) != 0; 169 170 hw_tail1 = amdiommu_event_log_tail(unit); 171 do { 172 hw_tail = hw_tail1; 173 for (; hw_tail != unit->event_log_head; 174 amdiommu_event_log_inc_head(unit)) { 175 evp = &unit->event_log[unit->event_log_head]; 176 mtx_lock_spin(&unit->event_lock); 177 if (amdiommu_event_copy_log_hasspace(unit)) { 178 unit->event_copy_log[unit->event_copy_head] = 179 *evp; 180 unit->event_copy_head = 181 amdiommu_event_copy_log_inc(unit-> 182 event_copy_head); 183 enqueue = true; 184 } else { 185 amdiommu_event_log_print(unit, evp, false); 186 } 187 mtx_unlock_spin(&unit->event_lock); 188 } 189 amdiommu_write8(unit, AMDIOMMU_EVNTLOG_HEAD, 190 unit->event_log_head << AMDIOMMU_EV_SZ_SHIFT); 191 hw_tail1 = amdiommu_event_log_tail(unit); 192 } while (hw_tail1 != hw_tail); 193 amdiommu_event_rearm_intr(unit); 194 195 if (enqueue) 196 taskqueue_enqueue(unit->event_taskqueue, &unit->event_task); 197 } 198 199 static void 200 amdiommu_event_task(void *arg, int pending __unused) 201 { 202 struct amdiommu_unit *unit; 203 uint64_t hwev_status, status; 204 struct amdiommu_event_generic hwev; 205 206 unit = arg; 207 AMDIOMMU_LOCK(unit); 208 209 if ((unit->efr & AMDIOMMU_EFR_HWEV_SUP) != 0) { 210 hwev_status = amdiommu_read8(unit, AMDIOMMU_HWEV_STATUS); 211 if ((hwev_status & AMDIOMMU_HWEVS_HEV) != 0) { 212 *(uint64_t *)&hwev = amdiommu_read8(unit, 213 AMDIOMMU_HWEV_LOWER); 214 *((uint64_t *)&hwev + 1) = amdiommu_read8(unit, 215 AMDIOMMU_HWEV_UPPER); 216 printf("amdiommu%d: hw event%s\n", unit->iommu.unit, 217 (hwev_status & AMDIOMMU_HWEVS_HEO) != 0 ? 218 " (overflown)" : ""); 219 amdiommu_event_log_print(unit, &hwev, true); 220 amdiommu_write8(unit, AMDIOMMU_HWEV_STATUS, 221 hwev_status); 222 } 223 } 224 225 status = amdiommu_read8(unit, AMDIOMMU_CMDEV_STATUS); 226 if ((status & AMDIOMMU_CMDEVS_EVOVRFLW) != 0) { 227 printf("amdiommu%d: event log overflow\n", unit->iommu.unit); 228 229 while ((status & AMDIOMMU_CMDEVS_EVLOGRUN) != 0) { 230 DELAY(1); 231 status = amdiommu_read8(unit, AMDIOMMU_CMDEV_STATUS); 232 } 233 234 unit->hw_ctrl &= ~AMDIOMMU_CTRL_EVNTLOG_EN; 235 amdiommu_write8(unit, AMDIOMMU_CTRL, unit->hw_ctrl); 236 237 unit->event_log_head = 0; 238 amdiommu_write8(unit, AMDIOMMU_EVNTLOG_HEAD, 0); 239 240 amdiommu_write8(unit, AMDIOMMU_CMDEV_STATUS, 241 AMDIOMMU_CMDEVS_EVOVRFLW); /* RW1C */ 242 243 unit->hw_ctrl |= AMDIOMMU_CTRL_EVNTLOG_EN; 244 amdiommu_write8(unit, AMDIOMMU_CTRL, unit->hw_ctrl); 245 246 amdiommu_event_rearm_intr(unit); 247 } 248 249 mtx_lock_spin(&unit->event_lock); 250 while (unit->event_copy_head != unit->event_copy_tail) { 251 mtx_unlock_spin(&unit->event_lock); 252 amdiommu_event_log_print(unit, &unit->event_copy_log[ 253 unit->event_copy_tail], true); 254 mtx_lock_spin(&unit->event_lock); 255 unit->event_copy_tail = amdiommu_event_copy_log_inc(unit-> 256 event_copy_tail); 257 } 258 mtx_unlock_spin(&unit->event_lock); 259 260 AMDIOMMU_UNLOCK(unit); 261 } 262 263 int 264 amdiommu_init_event(struct amdiommu_unit *unit) 265 { 266 uint64_t base_reg; 267 268 mtx_init(&unit->event_lock, "amdevl", NULL, MTX_SPIN); 269 270 /* event log entries */ 271 unit->event_log_size = AMDIOMMU_EVNTLOG_MIN; 272 TUNABLE_INT_FETCH("hw.amdiommu.event_log_size", &unit->event_log_size); 273 if (unit->event_log_size < AMDIOMMU_EVNTLOG_MIN || 274 unit->event_log_size > AMDIOMMU_EVNTLOG_MAX || 275 !powerof2(unit->event_log_size)) 276 panic("invalid hw.amdiommu.event_log_size"); 277 unit->event_log = kmem_alloc_contig(AMDIOMMU_EV_SZ * 278 unit->event_log_size, M_WAITOK | M_ZERO, 0, ~0ull, PAGE_SIZE, 279 0, VM_MEMATTR_DEFAULT); 280 281 TASK_INIT(&unit->event_task, 0, amdiommu_event_task, unit); 282 unit->event_taskqueue = taskqueue_create_fast("amdiommuff", M_WAITOK, 283 taskqueue_thread_enqueue, &unit->event_taskqueue); 284 taskqueue_start_threads(&unit->event_taskqueue, 1, PI_AV, 285 "amdiommu%d event taskq", unit->iommu.unit); 286 287 base_reg = pmap_kextract((vm_offset_t)unit->event_log) | 288 (((uint64_t)0x8 + ilog2(unit->event_log_size / 289 AMDIOMMU_EVNTLOG_MIN)) << AMDIOMMU_EVNTLOG_BASE_SZSHIFT); 290 AMDIOMMU_LOCK(unit); 291 /* 292 * Re-arm before enabling interrupt, to not loose it when 293 * re-arming in the interrupt handler. 294 */ 295 amdiommu_event_rearm_intr(unit); 296 amdiommu_write8(unit, AMDIOMMU_EVNTLOG_BASE, base_reg); 297 unit->hw_ctrl |= AMDIOMMU_CTRL_EVNTLOG_EN | AMDIOMMU_CTRL_EVENTINT_EN; 298 amdiommu_write8(unit, AMDIOMMU_CTRL, unit->hw_ctrl); 299 AMDIOMMU_UNLOCK(unit); 300 301 return (0); 302 } 303 304 void 305 amdiommu_fini_event(struct amdiommu_unit *unit) 306 { 307 AMDIOMMU_LOCK(unit); 308 unit->hw_ctrl &= ~(AMDIOMMU_CTRL_EVNTLOG_EN | 309 AMDIOMMU_CTRL_EVENTINT_EN); 310 amdiommu_write8(unit, AMDIOMMU_CTRL, unit->hw_ctrl); 311 amdiommu_write8(unit, AMDIOMMU_EVNTLOG_BASE, 0); 312 AMDIOMMU_UNLOCK(unit); 313 314 taskqueue_drain(unit->event_taskqueue, &unit->event_task); 315 taskqueue_free(unit->event_taskqueue); 316 unit->event_taskqueue = NULL; 317 318 kmem_free(unit->event_log, unit->event_log_size * AMDIOMMU_EV_SZ); 319 unit->event_log = NULL; 320 unit->event_log_head = unit->event_log_tail = 0; 321 322 mtx_destroy(&unit->event_lock); 323 } 324