1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause 3 * 4 * Copyright (c) 2024 The FreeBSD Foundation 5 * 6 * This software was developed by Konstantin Belousov <kib@FreeBSD.org> 7 * under sponsorship from the FreeBSD Foundation. 8 * 9 * Redistribution and use in source and binary forms, with or without 10 * modification, are permitted provided that the following conditions 11 * are met: 12 * 1. Redistributions of source code must retain the above copyright 13 * notice, this list of conditions and the following disclaimer. 14 * 2. Redistributions in binary form must reproduce the above copyright 15 * notice, this list of conditions and the following disclaimer in the 16 * documentation and/or other materials provided with the distribution. 17 * 18 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 19 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 20 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 21 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 22 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 23 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 24 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 25 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 26 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 27 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 28 * SUCH DAMAGE. 29 */ 30 31 #include <sys/param.h> 32 #include <sys/systm.h> 33 #include <sys/malloc.h> 34 #include <sys/bus.h> 35 #include <sys/interrupt.h> 36 #include <sys/domainset.h> 37 #include <sys/kernel.h> 38 #include <sys/ktr.h> 39 #include <sys/limits.h> 40 #include <sys/lock.h> 41 #include <sys/memdesc.h> 42 #include <sys/mutex.h> 43 #include <sys/proc.h> 44 #include <sys/rwlock.h> 45 #include <sys/rman.h> 46 #include <sys/sysctl.h> 47 #include <sys/taskqueue.h> 48 #include <sys/tree.h> 49 #include <sys/uio.h> 50 #include <sys/vmem.h> 51 #include <vm/vm.h> 52 #include <vm/vm_extern.h> 53 #include <vm/vm_kern.h> 54 #include <vm/vm_object.h> 55 #include <vm/vm_page.h> 56 #include <vm/vm_pager.h> 57 #include <vm/vm_map.h> 58 #include <contrib/dev/acpica/include/acpi.h> 59 #include <contrib/dev/acpica/include/accommon.h> 60 #include <dev/acpica/acpivar.h> 61 #include <dev/pci/pcireg.h> 62 #include <dev/pci/pcivar.h> 63 #include <machine/atomic.h> 64 #include <machine/bus.h> 65 #include <machine/md_var.h> 66 #include <machine/intr_machdep.h> 67 #include <x86/include/apicreg.h> 68 #include <x86/include/apicvar.h> 69 #include <machine/specialreg.h> 70 #include <x86/include/busdma_impl.h> 71 #include <dev/iommu/busdma_iommu.h> 72 #include <x86/iommu/amd_reg.h> 73 #include <x86/iommu/x86_iommu.h> 74 #include <x86/iommu/amd_iommu.h> 75 76 static struct amdiommu_ctx *amdiommu_ir_find(device_t src, uint16_t *rid, 77 bool *is_iommu); 78 static void amdiommu_ir_free_irte(struct amdiommu_ctx *ctx, device_t src, 79 u_int cookie); 80 81 int 82 amdiommu_alloc_msi_intr(device_t src, u_int *cookies, u_int count) 83 { 84 struct amdiommu_ctx *ctx; 85 vmem_addr_t vmem_res; 86 u_int idx, i; 87 int error; 88 89 ctx = amdiommu_ir_find(src, NULL, NULL); 90 if (ctx == NULL || !CTX2AMD(ctx)->irte_enabled) { 91 for (i = 0; i < count; i++) 92 cookies[i] = -1; 93 return (EOPNOTSUPP); 94 } 95 96 error = vmem_alloc(ctx->irtids, count, M_FIRSTFIT | M_NOWAIT, 97 &vmem_res); 98 if (error != 0) { 99 KASSERT(error != EOPNOTSUPP, 100 ("impossible EOPNOTSUPP from vmem")); 101 return (error); 102 } 103 idx = vmem_res; 104 for (i = 0; i < count; i++) 105 cookies[i] = idx + i; 106 return (0); 107 } 108 109 int 110 amdiommu_map_msi_intr(device_t src, u_int cpu, u_int vector, 111 u_int cookie, uint64_t *addr, uint32_t *data) 112 { 113 struct amdiommu_ctx *ctx; 114 struct amdiommu_unit *unit; 115 device_t requester; 116 int error __diagused; 117 uint16_t rid; 118 bool is_iommu; 119 120 ctx = amdiommu_ir_find(src, &rid, &is_iommu); 121 if (is_iommu) { 122 if (addr != NULL) { 123 *data = vector; 124 *addr = MSI_INTEL_ADDR_BASE | ((cpu & 0xff) << 12); 125 if (x2apic_mode) 126 *addr |= ((uint64_t)cpu & 0xffffff00) << 32; 127 else 128 KASSERT(cpu <= 0xff, 129 ("cpu id too big %d", cpu)); 130 } 131 return (0); 132 } 133 134 if (ctx == NULL) 135 return (EOPNOTSUPP); 136 unit = CTX2AMD(ctx); 137 if (!unit->irte_enabled || cookie == -1) 138 return (EOPNOTSUPP); 139 if (cookie >= unit->irte_nentries) { 140 device_printf(src, "amdiommu%d: cookie %u irte max %u\n", 141 unit->iommu.unit, cookie, unit->irte_nentries); 142 return (EINVAL); 143 } 144 145 if (unit->irte_x2apic) { 146 struct amdiommu_irte_basic_vapic_x2 *irte; 147 148 irte = &ctx->irtx2[cookie]; 149 irte->supiopf = 0; 150 irte->inttype = 0; 151 irte->rqeoi = 0; 152 irte->dm = 0; 153 irte->guestmode = 0; 154 irte->dest0 = cpu; 155 irte->rsrv0 = 0; 156 irte->vector = vector; 157 irte->rsrv1 = 0; 158 irte->rsrv2 = 0; 159 irte->dest1 = cpu >> 24; 160 atomic_thread_fence_rel(); 161 irte->remapen = 1; 162 } else { 163 struct amdiommu_irte_basic_novapic *irte; 164 165 irte = &ctx->irtb[cookie]; 166 irte->supiopf = 0; 167 irte->inttype = 0; /* fixed */ 168 irte->rqeoi = 0; 169 irte->dm = 0; /* phys */ 170 irte->guestmode = 0; 171 irte->dest = cpu; 172 irte->vector = vector; 173 irte->rsrv = 0; 174 atomic_thread_fence_rel(); 175 irte->remapen = 1; 176 } 177 178 if (addr != NULL) { 179 *data = cookie; 180 *addr = MSI_INTEL_ADDR_BASE | ((cpu & 0xff) << 12); 181 if (unit->irte_x2apic) 182 *addr |= ((uint64_t)cpu & 0xffffff00) << 32; 183 } 184 185 error = iommu_get_requester(src, &requester, &rid); 186 MPASS(error == 0); 187 AMDIOMMU_LOCK(unit); 188 amdiommu_qi_invalidate_ir_locked(unit, rid); 189 AMDIOMMU_UNLOCK(unit); 190 191 return (0); 192 } 193 194 int 195 amdiommu_unmap_msi_intr(device_t src, u_int cookie) 196 { 197 struct amdiommu_ctx *ctx; 198 199 if (cookie == -1) 200 return (0); 201 ctx = amdiommu_ir_find(src, NULL, NULL); 202 amdiommu_ir_free_irte(ctx, src, cookie); 203 return (0); 204 } 205 206 int 207 amdiommu_map_ioapic_intr(u_int ioapic_id, u_int cpu, u_int vector, 208 bool edge, bool activehi, int irq, u_int *cookie, uint32_t *hi, 209 uint32_t *lo) 210 { 211 /* XXXKIB for early call from ioapic_create() */ 212 return (EOPNOTSUPP); 213 } 214 215 int 216 amdiommu_unmap_ioapic_intr(u_int ioapic_id, u_int *cookie) 217 { 218 /* XXXKIB */ 219 return (0); 220 } 221 222 static struct amdiommu_ctx * 223 amdiommu_ir_find(device_t src, uint16_t *ridp, bool *is_iommu) 224 { 225 devclass_t src_class; 226 struct amdiommu_unit *unit; 227 struct amdiommu_ctx *ctx; 228 struct iommu_ctx *ioctx; 229 uint32_t edte; 230 uint16_t rid; 231 uint8_t dte; 232 int error; 233 234 /* 235 * We need to determine if the interrupt source generates FSB 236 * interrupts. If yes, it is either IOMMU, in which case 237 * interrupts are not remapped. Or it is HPET, and interrupts 238 * are remapped. For HPET, source id is reported by HPET 239 * record in IVHD ACPI table. 240 */ 241 if (is_iommu != NULL) 242 *is_iommu = false; 243 244 ctx = NULL; 245 246 src_class = device_get_devclass(src); 247 if (src_class == devclass_find("amdiommu")) { 248 if (is_iommu != NULL) 249 *is_iommu = true; 250 } else if (src_class == devclass_find("hpet")) { 251 error = amdiommu_find_unit_for_hpet(src, &unit, &rid, &dte, 252 &edte, bootverbose); 253 ctx = NULL; // XXXKIB allocate ctx 254 } else { 255 error = amdiommu_find_unit(src, &unit, &rid, &dte, &edte, 256 bootverbose); 257 if (error == 0) { 258 ioctx = iommu_instantiate_ctx(AMD2IOMMU(unit), src, false); 259 if (ioctx != NULL) 260 ctx = IOCTX2CTX(ioctx); 261 } 262 } 263 if (ridp != NULL) 264 *ridp = rid; 265 return (ctx); 266 } 267 268 static void 269 amdiommu_ir_free_irte(struct amdiommu_ctx *ctx, device_t src, 270 u_int cookie) 271 { 272 struct amdiommu_unit *unit; 273 device_t requester; 274 int error __diagused; 275 uint16_t rid; 276 277 MPASS(ctx != NULL); 278 unit = CTX2AMD(ctx); 279 280 KASSERT(unit->irte_enabled, 281 ("unmap: cookie %d ctx %p unit %p", cookie, ctx, unit)); 282 KASSERT(cookie < unit->irte_nentries, 283 ("bad cookie %u %u", cookie, unit->irte_nentries)); 284 285 if (unit->irte_x2apic) { 286 struct amdiommu_irte_basic_vapic_x2 *irte; 287 288 irte = &ctx->irtx2[cookie]; 289 irte->remapen = 0; 290 atomic_thread_fence_rel(); 291 bzero(irte, sizeof(*irte)); 292 } else { 293 struct amdiommu_irte_basic_novapic *irte; 294 295 irte = &ctx->irtb[cookie]; 296 irte->remapen = 0; 297 atomic_thread_fence_rel(); 298 bzero(irte, sizeof(*irte)); 299 } 300 error = iommu_get_requester(src, &requester, &rid); 301 MPASS(error == 0); 302 AMDIOMMU_LOCK(unit); 303 amdiommu_qi_invalidate_ir_locked(unit, rid); 304 AMDIOMMU_UNLOCK(unit); 305 } 306 307 int 308 amdiommu_ctx_init_irte(struct amdiommu_ctx *ctx) 309 { 310 struct amdiommu_unit *unit; 311 void *ptr; 312 unsigned long sz; 313 int dom; 314 315 unit = CTX2AMD(ctx); 316 if (!unit->irte_enabled) 317 return (0); 318 319 KASSERT(unit->irte_nentries > 0 && 320 unit->irte_nentries <= 2048 && 321 powerof2(unit->irte_nentries), 322 ("amdiommu%d: unit %p irte_nentries %u", unit->iommu.unit, 323 unit, unit->irte_nentries)); 324 325 if (bus_get_domain(unit->iommu.dev, &dom) != 0) 326 dom = -1; 327 sz = unit->irte_nentries; 328 sz *= unit->irte_x2apic ? sizeof(struct amdiommu_irte_basic_vapic_x2) : 329 sizeof(struct amdiommu_irte_basic_novapic); 330 331 if (dom != -1) { 332 ptr = contigmalloc_domainset(sz, M_DEVBUF, DOMAINSET_PREF(dom), 333 M_WAITOK | M_ZERO, 0, ~0ull, 128, 0); 334 } else { 335 ptr = contigmalloc(sz, M_DEVBUF, M_WAITOK | M_ZERO, 336 0, ~0ull, 128, 0); 337 } 338 if (unit->irte_x2apic) 339 ctx->irtx2 = ptr; 340 else 341 ctx->irtb = ptr; 342 ctx->irtids = vmem_create("amdirt", 0, unit->irte_nentries, 1, 0, 343 M_FIRSTFIT | M_NOWAIT); 344 345 intr_reprogram(); // XXXKIB 346 347 return (0); 348 } 349 350 void 351 amdiommu_ctx_fini_irte(struct amdiommu_ctx *ctx) 352 { 353 struct amdiommu_unit *unit; 354 355 unit = CTX2AMD(ctx); 356 if (!unit->irte_enabled) 357 return; 358 if (unit->irte_x2apic) 359 free(ctx->irtx2, M_DEVBUF); 360 else 361 free(ctx->irtb, M_DEVBUF); 362 vmem_destroy(ctx->irtids); 363 } 364 365 int 366 amdiommu_init_irt(struct amdiommu_unit *unit) 367 { 368 int enabled, nentries; 369 370 SYSCTL_ADD_INT(&unit->iommu.sysctl_ctx, 371 SYSCTL_CHILDREN(device_get_sysctl_tree(unit->iommu.dev)), 372 OID_AUTO, "ir", CTLFLAG_RD, &unit->irte_enabled, 0, 373 "Interrupt remapping ops enabled"); 374 375 enabled = 1; 376 TUNABLE_INT_FETCH("hw.iommu.ir", &enabled); 377 378 unit->irte_enabled = enabled != 0; 379 if (!unit->irte_enabled) 380 return (0); 381 382 nentries = 32; 383 TUNABLE_INT_FETCH("hw.iommu.amd.ir_num", &nentries); 384 nentries = roundup_pow_of_two(nentries); 385 if (nentries < 1) 386 nentries = 1; 387 if (nentries > 2048) 388 nentries = 2048; 389 unit->irte_nentries = nentries; 390 391 unit->irte_x2apic = x2apic_mode; 392 return (0); 393 } 394 395 void 396 amdiommu_fini_irt(struct amdiommu_unit *unit) 397 { 398 } 399