1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause 3 * 4 * Copyright (c) 2024 The FreeBSD Foundation 5 * 6 * This software was developed by Konstantin Belousov <kib@FreeBSD.org> 7 * under sponsorship from the FreeBSD Foundation. 8 * 9 * Redistribution and use in source and binary forms, with or without 10 * modification, are permitted provided that the following conditions 11 * are met: 12 * 1. Redistributions of source code must retain the above copyright 13 * notice, this list of conditions and the following disclaimer. 14 * 2. Redistributions in binary form must reproduce the above copyright 15 * notice, this list of conditions and the following disclaimer in the 16 * documentation and/or other materials provided with the distribution. 17 * 18 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 19 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 20 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 21 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 22 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 23 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 24 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 25 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 26 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 27 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 28 * SUCH DAMAGE. 29 */ 30 31 #include <sys/param.h> 32 #include <sys/systm.h> 33 #include <sys/malloc.h> 34 #include <sys/bus.h> 35 #include <sys/interrupt.h> 36 #include <sys/domainset.h> 37 #include <sys/kernel.h> 38 #include <sys/ktr.h> 39 #include <sys/limits.h> 40 #include <sys/lock.h> 41 #include <sys/memdesc.h> 42 #include <sys/mutex.h> 43 #include <sys/proc.h> 44 #include <sys/rwlock.h> 45 #include <sys/rman.h> 46 #include <sys/sysctl.h> 47 #include <sys/taskqueue.h> 48 #include <sys/tree.h> 49 #include <sys/uio.h> 50 #include <sys/vmem.h> 51 #include <vm/vm.h> 52 #include <vm/vm_extern.h> 53 #include <vm/vm_kern.h> 54 #include <vm/vm_object.h> 55 #include <vm/vm_page.h> 56 #include <vm/vm_pager.h> 57 #include <vm/vm_map.h> 58 #include <contrib/dev/acpica/include/acpi.h> 59 #include <contrib/dev/acpica/include/accommon.h> 60 #include <dev/acpica/acpivar.h> 61 #include <dev/pci/pcireg.h> 62 #include <dev/pci/pcivar.h> 63 #include <machine/atomic.h> 64 #include <machine/bus.h> 65 #include <machine/md_var.h> 66 #include <machine/intr_machdep.h> 67 #include <x86/include/apicreg.h> 68 #include <x86/include/apicvar.h> 69 #include <machine/specialreg.h> 70 #include <x86/include/busdma_impl.h> 71 #include <dev/iommu/busdma_iommu.h> 72 #include <x86/iommu/amd_reg.h> 73 #include <x86/iommu/x86_iommu.h> 74 #include <x86/iommu/amd_iommu.h> 75 76 static struct amdiommu_ctx *amdiommu_ir_find(device_t src, uint16_t *rid, 77 bool *is_iommu); 78 static void amdiommu_ir_free_irte(struct amdiommu_ctx *ctx, device_t src, 79 u_int cookie); 80 81 int 82 amdiommu_alloc_msi_intr(device_t src, u_int *cookies, u_int count) 83 { 84 struct amdiommu_ctx *ctx; 85 vmem_addr_t vmem_res; 86 u_int idx, i; 87 int error; 88 89 ctx = amdiommu_ir_find(src, NULL, NULL); 90 if (ctx == NULL || !CTX2AMD(ctx)->irte_enabled) { 91 for (i = 0; i < count; i++) 92 cookies[i] = -1; 93 return (EOPNOTSUPP); 94 } 95 96 error = vmem_alloc(ctx->irtids, count, M_FIRSTFIT | M_NOWAIT, 97 &vmem_res); 98 if (error != 0) { 99 KASSERT(error != EOPNOTSUPP, 100 ("impossible EOPNOTSUPP from vmem")); 101 return (error); 102 } 103 idx = vmem_res; 104 for (i = 0; i < count; i++) 105 cookies[i] = idx + i; 106 return (0); 107 } 108 109 int 110 amdiommu_map_msi_intr(device_t src, u_int cpu, u_int vector, 111 u_int cookie, uint64_t *addr, uint32_t *data) 112 { 113 struct amdiommu_ctx *ctx; 114 struct amdiommu_unit *unit; 115 device_t requester; 116 int error __diagused; 117 uint16_t rid; 118 bool is_iommu; 119 120 ctx = amdiommu_ir_find(src, &rid, &is_iommu); 121 if (is_iommu) { 122 if (addr != NULL) { 123 *data = vector; 124 *addr = MSI_INTEL_ADDR_BASE | ((cpu & 0xff) << 12); 125 if (x2apic_mode) 126 *addr |= ((uint64_t)cpu & 0xffffff00) << 32; 127 else 128 KASSERT(cpu <= 0xff, 129 ("cpu id too big %d", cpu)); 130 } 131 return (0); 132 } 133 134 if (ctx == NULL) 135 return (EOPNOTSUPP); 136 unit = CTX2AMD(ctx); 137 if (!unit->irte_enabled || cookie == -1) 138 return (EOPNOTSUPP); 139 if (cookie >= unit->irte_nentries) { 140 device_printf(src, "amdiommu%d: cookie %u irte max %u\n", 141 unit->iommu.unit, cookie, unit->irte_nentries); 142 return (EINVAL); 143 } 144 145 if (unit->irte_x2apic) { 146 struct amdiommu_irte_basic_vapic_x2 *irte; 147 148 irte = &ctx->irtx2[cookie]; 149 irte->supiopf = 0; 150 irte->inttype = 0; 151 irte->rqeoi = 0; 152 irte->dm = 0; 153 irte->guestmode = 0; 154 irte->dest0 = cpu; 155 irte->rsrv0 = 0; 156 irte->vector = vector; 157 irte->rsrv1 = 0; 158 irte->rsrv2 = 0; 159 irte->dest1 = cpu >> 24; 160 atomic_thread_fence_rel(); 161 irte->remapen = 1; 162 } else { 163 struct amdiommu_irte_basic_novapic *irte; 164 165 irte = &ctx->irtb[cookie]; 166 irte->supiopf = 0; 167 irte->inttype = 0; /* fixed */ 168 irte->rqeoi = 0; 169 irte->dm = 0; /* phys */ 170 irte->guestmode = 0; 171 irte->dest = cpu; 172 irte->vector = vector; 173 irte->rsrv = 0; 174 atomic_thread_fence_rel(); 175 irte->remapen = 1; 176 } 177 178 if (addr != NULL) { 179 *data = cookie; 180 *addr = MSI_INTEL_ADDR_BASE | ((cpu & 0xff) << 12); 181 if (unit->irte_x2apic) 182 *addr |= ((uint64_t)cpu & 0xffffff00) << 32; 183 } 184 185 error = iommu_get_requester(src, &requester, &rid); 186 MPASS(error == 0); 187 AMDIOMMU_LOCK(unit); 188 amdiommu_qi_invalidate_ir_locked(unit, rid); 189 AMDIOMMU_UNLOCK(unit); 190 191 return (0); 192 } 193 194 int 195 amdiommu_unmap_msi_intr(device_t src, u_int cookie) 196 { 197 struct amdiommu_ctx *ctx; 198 199 if (cookie == -1) 200 return (0); 201 ctx = amdiommu_ir_find(src, NULL, NULL); 202 amdiommu_ir_free_irte(ctx, src, cookie); 203 return (0); 204 } 205 206 int 207 amdiommu_map_ioapic_intr(u_int ioapic_id, u_int cpu, u_int vector, 208 bool edge, bool activehi, int irq, u_int *cookie, uint32_t *hi, 209 uint32_t *lo) 210 { 211 /* XXXKIB for early call from ioapic_create() */ 212 return (EOPNOTSUPP); 213 } 214 215 int 216 amdiommu_unmap_ioapic_intr(u_int ioapic_id, u_int *cookie) 217 { 218 /* XXXKIB */ 219 return (0); 220 } 221 222 static struct amdiommu_ctx * 223 amdiommu_ir_find(device_t src, uint16_t *ridp, bool *is_iommu) 224 { 225 devclass_t src_class; 226 device_t requester; 227 struct amdiommu_unit *unit; 228 struct amdiommu_ctx *ctx; 229 uint32_t edte; 230 uint16_t rid; 231 uint8_t dte; 232 int error; 233 234 /* 235 * We need to determine if the interrupt source generates FSB 236 * interrupts. If yes, it is either IOMMU, in which case 237 * interrupts are not remapped. Or it is HPET, and interrupts 238 * are remapped. For HPET, source id is reported by HPET 239 * record in IVHD ACPI table. 240 */ 241 if (is_iommu != NULL) 242 *is_iommu = false; 243 244 ctx = NULL; 245 246 src_class = device_get_devclass(src); 247 if (src_class == devclass_find("amdiommu")) { 248 if (is_iommu != NULL) 249 *is_iommu = true; 250 } else if (src_class == devclass_find("hpet")) { 251 error = amdiommu_find_unit_for_hpet(src, &unit, &rid, &dte, 252 &edte, bootverbose); 253 ctx = NULL; // XXXKIB allocate ctx 254 } else { 255 error = amdiommu_find_unit(src, &unit, &rid, &dte, &edte, 256 bootverbose); 257 if (error == 0) { 258 error = iommu_get_requester(src, &requester, &rid); 259 MPASS(error == 0); 260 ctx = amdiommu_get_ctx_for_dev(unit, src, 261 rid, 0, false /* XXXKIB */, false, dte, edte); 262 } 263 } 264 if (ridp != NULL) 265 *ridp = rid; 266 return (ctx); 267 } 268 269 static void 270 amdiommu_ir_free_irte(struct amdiommu_ctx *ctx, device_t src, 271 u_int cookie) 272 { 273 struct amdiommu_unit *unit; 274 device_t requester; 275 int error __diagused; 276 uint16_t rid; 277 278 MPASS(ctx != NULL); 279 unit = CTX2AMD(ctx); 280 281 KASSERT(unit->irte_enabled, 282 ("unmap: cookie %d ctx %p unit %p", cookie, ctx, unit)); 283 KASSERT(cookie < unit->irte_nentries, 284 ("bad cookie %u %u", cookie, unit->irte_nentries)); 285 286 if (unit->irte_x2apic) { 287 struct amdiommu_irte_basic_vapic_x2 *irte; 288 289 irte = &ctx->irtx2[cookie]; 290 irte->remapen = 0; 291 atomic_thread_fence_rel(); 292 bzero(irte, sizeof(*irte)); 293 } else { 294 struct amdiommu_irte_basic_novapic *irte; 295 296 irte = &ctx->irtb[cookie]; 297 irte->remapen = 0; 298 atomic_thread_fence_rel(); 299 bzero(irte, sizeof(*irte)); 300 } 301 error = iommu_get_requester(src, &requester, &rid); 302 MPASS(error == 0); 303 AMDIOMMU_LOCK(unit); 304 amdiommu_qi_invalidate_ir_locked(unit, rid); 305 AMDIOMMU_UNLOCK(unit); 306 } 307 308 int 309 amdiommu_ctx_init_irte(struct amdiommu_ctx *ctx) 310 { 311 struct amdiommu_unit *unit; 312 void *ptr; 313 unsigned long sz; 314 int dom; 315 316 unit = CTX2AMD(ctx); 317 if (!unit->irte_enabled) 318 return (0); 319 320 KASSERT(unit->irte_nentries > 0 && 321 unit->irte_nentries <= 2048 && 322 powerof2(unit->irte_nentries), 323 ("amdiommu%d: unit %p irte_nentries %u", unit->iommu.unit, 324 unit, unit->irte_nentries)); 325 326 if (bus_get_domain(unit->iommu.dev, &dom) != 0) 327 dom = -1; 328 sz = unit->irte_nentries; 329 sz *= unit->irte_x2apic ? sizeof(struct amdiommu_irte_basic_vapic_x2) : 330 sizeof(struct amdiommu_irte_basic_novapic); 331 332 if (dom != -1) { 333 ptr = contigmalloc_domainset(sz, M_DEVBUF, DOMAINSET_PREF(dom), 334 M_WAITOK | M_ZERO, 0, ~0ull, 128, 0); 335 } else { 336 ptr = contigmalloc(sz, M_DEVBUF, M_WAITOK | M_ZERO, 337 0, ~0ull, 128, 0); 338 } 339 if (unit->irte_x2apic) 340 ctx->irtx2 = ptr; 341 else 342 ctx->irtb = ptr; 343 ctx->irtids = vmem_create("amdirt", 0, unit->irte_nentries, 1, 0, 344 M_FIRSTFIT | M_NOWAIT); 345 346 intr_reprogram(); // XXXKIB 347 348 return (0); 349 } 350 351 void 352 amdiommu_ctx_fini_irte(struct amdiommu_ctx *ctx) 353 { 354 struct amdiommu_unit *unit; 355 356 unit = CTX2AMD(ctx); 357 if (!unit->irte_enabled) 358 return; 359 if (unit->irte_x2apic) 360 free(ctx->irtx2, M_DEVBUF); 361 else 362 free(ctx->irtb, M_DEVBUF); 363 vmem_destroy(ctx->irtids); 364 } 365 366 int 367 amdiommu_init_irt(struct amdiommu_unit *unit) 368 { 369 int enabled, nentries; 370 371 SYSCTL_ADD_INT(&unit->iommu.sysctl_ctx, 372 SYSCTL_CHILDREN(device_get_sysctl_tree(unit->iommu.dev)), 373 OID_AUTO, "ir", CTLFLAG_RD, &unit->irte_enabled, 0, 374 "Interrupt remapping ops enabled"); 375 376 enabled = 1; 377 TUNABLE_INT_FETCH("hw.iommu.ir", &enabled); 378 379 unit->irte_enabled = enabled != 0; 380 if (!unit->irte_enabled) 381 return (0); 382 383 nentries = 32; 384 TUNABLE_INT_FETCH("hw.iommu.amd.ir_num", &nentries); 385 nentries = roundup_pow_of_two(nentries); 386 if (nentries < 1) 387 nentries = 1; 388 if (nentries > 2048) 389 nentries = 2048; 390 unit->irte_nentries = nentries; 391 392 unit->irte_x2apic = x2apic_mode; 393 return (0); 394 } 395 396 void 397 amdiommu_fini_irt(struct amdiommu_unit *unit) 398 { 399 } 400