1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause 3 * 4 * Copyright (c) 2024 The FreeBSD Foundation 5 * 6 * This software was developed by Konstantin Belousov <kib@FreeBSD.org> 7 * under sponsorship from the FreeBSD Foundation. 8 * 9 * Redistribution and use in source and binary forms, with or without 10 * modification, are permitted provided that the following conditions 11 * are met: 12 * 1. Redistributions of source code must retain the above copyright 13 * notice, this list of conditions and the following disclaimer. 14 * 2. Redistributions in binary form must reproduce the above copyright 15 * notice, this list of conditions and the following disclaimer in the 16 * documentation and/or other materials provided with the distribution. 17 * 18 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 19 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 20 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 21 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 22 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 23 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 24 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 25 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 26 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 27 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 28 * SUCH DAMAGE. 29 */ 30 31 #include <sys/param.h> 32 #include <sys/systm.h> 33 #include <sys/malloc.h> 34 #include <sys/bus.h> 35 #include <sys/interrupt.h> 36 #include <sys/domainset.h> 37 #include <sys/kernel.h> 38 #include <sys/ktr.h> 39 #include <sys/limits.h> 40 #include <sys/lock.h> 41 #include <sys/memdesc.h> 42 #include <sys/mutex.h> 43 #include <sys/proc.h> 44 #include <sys/rwlock.h> 45 #include <sys/rman.h> 46 #include <sys/sysctl.h> 47 #include <sys/taskqueue.h> 48 #include <sys/tree.h> 49 #include <sys/uio.h> 50 #include <sys/vmem.h> 51 #include <vm/vm.h> 52 #include <vm/vm_extern.h> 53 #include <vm/vm_kern.h> 54 #include <vm/vm_object.h> 55 #include <vm/vm_page.h> 56 #include <vm/vm_pager.h> 57 #include <vm/vm_map.h> 58 #include <contrib/dev/acpica/include/acpi.h> 59 #include <contrib/dev/acpica/include/accommon.h> 60 #include <dev/acpica/acpivar.h> 61 #include <dev/pci/pcireg.h> 62 #include <dev/pci/pcivar.h> 63 #include <machine/atomic.h> 64 #include <machine/bus.h> 65 #include <machine/md_var.h> 66 #include <machine/intr_machdep.h> 67 #include <x86/include/apicreg.h> 68 #include <x86/include/apicvar.h> 69 #include <machine/specialreg.h> 70 #include <x86/include/busdma_impl.h> 71 #include <dev/iommu/busdma_iommu.h> 72 #include <x86/iommu/amd_reg.h> 73 #include <x86/iommu/x86_iommu.h> 74 #include <x86/iommu/amd_iommu.h> 75 76 static struct amdiommu_ctx *amdiommu_ir_find(device_t src, uint16_t *rid, 77 bool *is_iommu); 78 static void amdiommu_ir_free_irte(struct amdiommu_ctx *ctx, device_t src, 79 u_int cookie); 80 81 int 82 amdiommu_alloc_msi_intr(device_t src, u_int *cookies, u_int count) 83 { 84 struct amdiommu_ctx *ctx; 85 vmem_addr_t vmem_res; 86 u_int idx, i; 87 int error; 88 89 ctx = amdiommu_ir_find(src, NULL, NULL); 90 if (ctx == NULL || !CTX2AMD(ctx)->irte_enabled) { 91 for (i = 0; i < count; i++) 92 cookies[i] = -1; 93 return (EOPNOTSUPP); 94 } 95 96 error = vmem_alloc(ctx->irtids, count, M_FIRSTFIT | M_NOWAIT, 97 &vmem_res); 98 if (error != 0) { 99 KASSERT(error != EOPNOTSUPP, 100 ("impossible EOPNOTSUPP from vmem")); 101 return (error); 102 } 103 idx = vmem_res; 104 for (i = 0; i < count; i++) 105 cookies[i] = idx + i; 106 return (0); 107 } 108 109 int 110 amdiommu_map_msi_intr(device_t src, u_int cpu, u_int vector, 111 u_int cookie, uint64_t *addr, uint32_t *data) 112 { 113 struct amdiommu_ctx *ctx; 114 struct amdiommu_unit *unit; 115 uint16_t rid; 116 bool is_iommu; 117 118 ctx = amdiommu_ir_find(src, &rid, &is_iommu); 119 if (is_iommu) { 120 if (addr != NULL) { 121 *data = vector; 122 *addr = MSI_INTEL_ADDR_BASE | ((cpu & 0xff) << 12); 123 if (x2apic_mode) 124 *addr |= ((uint64_t)cpu & 0xffffff00) << 32; 125 else 126 KASSERT(cpu <= 0xff, 127 ("cpu id too big %d", cpu)); 128 } 129 return (0); 130 } 131 132 if (ctx == NULL) 133 return (EOPNOTSUPP); 134 unit = CTX2AMD(ctx); 135 if (!unit->irte_enabled || cookie == -1) 136 return (EOPNOTSUPP); 137 if (cookie >= unit->irte_nentries) { 138 device_printf(src, "amdiommu%d: cookie %u irte max %u\n", 139 unit->iommu.unit, cookie, unit->irte_nentries); 140 return (EINVAL); 141 } 142 143 if (unit->irte_x2apic) { 144 struct amdiommu_irte_basic_vapic_x2 *irte; 145 146 irte = &ctx->irtx2[cookie]; 147 irte->supiopf = 0; 148 irte->inttype = 0; 149 irte->rqeoi = 0; 150 irte->dm = 0; 151 irte->guestmode = 0; 152 irte->dest0 = cpu; 153 irte->rsrv0 = 0; 154 irte->vector = vector; 155 irte->rsrv1 = 0; 156 irte->rsrv2 = 0; 157 irte->dest1 = cpu >> 24; 158 atomic_thread_fence_rel(); 159 irte->remapen = 1; 160 } else { 161 struct amdiommu_irte_basic_novapic *irte; 162 163 irte = &ctx->irtb[cookie]; 164 irte->supiopf = 0; 165 irte->inttype = 0; /* fixed */ 166 irte->rqeoi = 0; 167 irte->dm = 0; /* phys */ 168 irte->guestmode = 0; 169 irte->dest = cpu; 170 irte->vector = vector; 171 irte->rsrv = 0; 172 atomic_thread_fence_rel(); 173 irte->remapen = 1; 174 } 175 176 if (addr != NULL) { 177 *data = cookie; 178 *addr = MSI_INTEL_ADDR_BASE | ((cpu & 0xff) << 12); 179 if (unit->irte_x2apic) 180 *addr |= ((uint64_t)cpu & 0xffffff00) << 32; 181 } 182 183 iommu_get_requester(src, &rid); 184 AMDIOMMU_LOCK(unit); 185 amdiommu_qi_invalidate_ir_locked(unit, rid); 186 AMDIOMMU_UNLOCK(unit); 187 188 return (0); 189 } 190 191 int 192 amdiommu_unmap_msi_intr(device_t src, u_int cookie) 193 { 194 struct amdiommu_ctx *ctx; 195 196 if (cookie == -1) 197 return (0); 198 ctx = amdiommu_ir_find(src, NULL, NULL); 199 amdiommu_ir_free_irte(ctx, src, cookie); 200 return (0); 201 } 202 203 int 204 amdiommu_map_ioapic_intr(u_int ioapic_id, u_int cpu, u_int vector, 205 bool edge, bool activehi, int irq, u_int *cookie, uint32_t *hi, 206 uint32_t *lo) 207 { 208 /* XXXKIB for early call from ioapic_create() */ 209 return (EOPNOTSUPP); 210 } 211 212 int 213 amdiommu_unmap_ioapic_intr(u_int ioapic_id, u_int *cookie) 214 { 215 /* XXXKIB */ 216 return (0); 217 } 218 219 static struct amdiommu_ctx * 220 amdiommu_ir_find(device_t src, uint16_t *ridp, bool *is_iommu) 221 { 222 devclass_t src_class; 223 struct amdiommu_unit *unit; 224 struct amdiommu_ctx *ctx; 225 uint32_t edte; 226 uint16_t rid; 227 uint8_t dte; 228 int error; 229 230 /* 231 * We need to determine if the interrupt source generates FSB 232 * interrupts. If yes, it is either IOMMU, in which case 233 * interrupts are not remapped. Or it is HPET, and interrupts 234 * are remapped. For HPET, source id is reported by HPET 235 * record in IVHD ACPI table. 236 */ 237 if (is_iommu != NULL) 238 *is_iommu = false; 239 240 ctx = NULL; 241 242 src_class = device_get_devclass(src); 243 if (src_class == devclass_find("amdiommu")) { 244 if (is_iommu != NULL) 245 *is_iommu = true; 246 } else if (src_class == devclass_find("hpet")) { 247 error = amdiommu_find_unit_for_hpet(src, &unit, &rid, &dte, 248 &edte, bootverbose); 249 ctx = NULL; // XXXKIB allocate ctx 250 } else { 251 error = amdiommu_find_unit(src, &unit, &rid, &dte, &edte, 252 bootverbose); 253 if (error == 0) { 254 iommu_get_requester(src, &rid); 255 ctx = amdiommu_get_ctx_for_dev(unit, src, 256 rid, 0, false /* XXXKIB */, false, dte, edte); 257 } 258 } 259 if (ridp != NULL) 260 *ridp = rid; 261 return (ctx); 262 } 263 264 static void 265 amdiommu_ir_free_irte(struct amdiommu_ctx *ctx, device_t src, 266 u_int cookie) 267 { 268 struct amdiommu_unit *unit; 269 uint16_t rid; 270 271 MPASS(ctx != NULL); 272 unit = CTX2AMD(ctx); 273 274 KASSERT(unit->irte_enabled, 275 ("unmap: cookie %d ctx %p unit %p", cookie, ctx, unit)); 276 KASSERT(cookie < unit->irte_nentries, 277 ("bad cookie %u %u", cookie, unit->irte_nentries)); 278 279 if (unit->irte_x2apic) { 280 struct amdiommu_irte_basic_vapic_x2 *irte; 281 282 irte = &ctx->irtx2[cookie]; 283 irte->remapen = 0; 284 atomic_thread_fence_rel(); 285 bzero(irte, sizeof(*irte)); 286 } else { 287 struct amdiommu_irte_basic_novapic *irte; 288 289 irte = &ctx->irtb[cookie]; 290 irte->remapen = 0; 291 atomic_thread_fence_rel(); 292 bzero(irte, sizeof(*irte)); 293 } 294 iommu_get_requester(src, &rid); 295 AMDIOMMU_LOCK(unit); 296 amdiommu_qi_invalidate_ir_locked(unit, rid); 297 AMDIOMMU_UNLOCK(unit); 298 } 299 300 int 301 amdiommu_ctx_init_irte(struct amdiommu_ctx *ctx) 302 { 303 struct amdiommu_unit *unit; 304 void *ptr; 305 unsigned long sz; 306 int dom; 307 308 unit = CTX2AMD(ctx); 309 if (!unit->irte_enabled) 310 return (0); 311 312 KASSERT(unit->irte_nentries > 0 && 313 unit->irte_nentries <= 2048 && 314 powerof2(unit->irte_nentries), 315 ("amdiommu%d: unit %p irte_nentries %u", unit->iommu.unit, 316 unit, unit->irte_nentries)); 317 318 if (bus_get_domain(unit->iommu.dev, &dom) != 0) 319 dom = -1; 320 sz = unit->irte_nentries; 321 sz *= unit->irte_x2apic ? sizeof(struct amdiommu_irte_basic_vapic_x2) : 322 sizeof(struct amdiommu_irte_basic_novapic); 323 324 if (dom != -1) { 325 ptr = contigmalloc_domainset(sz, M_DEVBUF, DOMAINSET_PREF(dom), 326 M_WAITOK | M_ZERO, 0, ~0ull, 128, 0); 327 } else { 328 ptr = contigmalloc(sz, M_DEVBUF, M_WAITOK | M_ZERO, 329 0, ~0ull, 128, 0); 330 } 331 if (unit->irte_x2apic) 332 ctx->irtx2 = ptr; 333 else 334 ctx->irtb = ptr; 335 ctx->irtids = vmem_create("amdirt", 0, unit->irte_nentries, 1, 0, 336 M_FIRSTFIT | M_NOWAIT); 337 338 intr_reprogram(); // XXXKIB 339 340 return (0); 341 } 342 343 void 344 amdiommu_ctx_fini_irte(struct amdiommu_ctx *ctx) 345 { 346 struct amdiommu_unit *unit; 347 348 unit = CTX2AMD(ctx); 349 if (!unit->irte_enabled) 350 return; 351 if (unit->irte_x2apic) 352 free(ctx->irtx2, M_DEVBUF); 353 else 354 free(ctx->irtb, M_DEVBUF); 355 vmem_destroy(ctx->irtids); 356 } 357 358 int 359 amdiommu_init_irt(struct amdiommu_unit *unit) 360 { 361 int enabled, nentries; 362 363 SYSCTL_ADD_INT(&unit->iommu.sysctl_ctx, 364 SYSCTL_CHILDREN(device_get_sysctl_tree(unit->iommu.dev)), 365 OID_AUTO, "ir", CTLFLAG_RD, &unit->irte_enabled, 0, 366 "Interrupt remapping ops enabled"); 367 368 enabled = 1; 369 TUNABLE_INT_FETCH("hw.iommu.ir", &enabled); 370 371 unit->irte_enabled = enabled != 0; 372 if (!unit->irte_enabled) 373 return (0); 374 375 nentries = 32; 376 TUNABLE_INT_FETCH("hw.iommu.amd.ir_num", &nentries); 377 nentries = roundup_pow_of_two(nentries); 378 if (nentries < 1) 379 nentries = 1; 380 if (nentries > 2048) 381 nentries = 2048; 382 unit->irte_nentries = nentries; 383 384 unit->irte_x2apic = x2apic_mode; 385 return (0); 386 } 387 388 void 389 amdiommu_fini_irt(struct amdiommu_unit *unit) 390 { 391 } 392