1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause 3 * 4 * Copyright (c) 2024 The FreeBSD Foundation 5 * 6 * This software was developed by Konstantin Belousov <kib@FreeBSD.org> 7 * under sponsorship from the FreeBSD Foundation. 8 * 9 * Redistribution and use in source and binary forms, with or without 10 * modification, are permitted provided that the following conditions 11 * are met: 12 * 1. Redistributions of source code must retain the above copyright 13 * notice, this list of conditions and the following disclaimer. 14 * 2. Redistributions in binary form must reproduce the above copyright 15 * notice, this list of conditions and the following disclaimer in the 16 * documentation and/or other materials provided with the distribution. 17 * 18 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 19 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 20 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 21 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 22 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 23 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 24 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 25 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 26 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 27 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 28 * SUCH DAMAGE. 29 */ 30 31 #include "opt_acpi.h" 32 33 #include <sys/param.h> 34 #include <sys/bus.h> 35 #include <sys/kernel.h> 36 #include <sys/malloc.h> 37 #include <sys/memdesc.h> 38 #include <sys/module.h> 39 #include <sys/rman.h> 40 #include <sys/taskqueue.h> 41 #include <sys/time.h> 42 #include <sys/tree.h> 43 #include <sys/vmem.h> 44 #include <vm/vm.h> 45 #include <vm/vm_extern.h> 46 #include <vm/vm_kern.h> 47 #include <vm/vm_page.h> 48 #include <vm/vm_map.h> 49 #include <contrib/dev/acpica/include/acpi.h> 50 #include <contrib/dev/acpica/include/accommon.h> 51 #include <dev/acpica/acpivar.h> 52 #include <dev/pci/pcireg.h> 53 #include <machine/bus.h> 54 #include <machine/cpu.h> 55 #include <x86/include/busdma_impl.h> 56 #include <dev/iommu/busdma_iommu.h> 57 #include <x86/iommu/amd_reg.h> 58 #include <x86/iommu/x86_iommu.h> 59 #include <x86/iommu/amd_iommu.h> 60 61 static void 62 amdiommu_enable_cmdbuf(struct amdiommu_unit *unit) 63 { 64 AMDIOMMU_ASSERT_LOCKED(unit); 65 66 unit->hw_ctrl |= AMDIOMMU_CTRL_CMDBUF_EN; 67 amdiommu_write8(unit, AMDIOMMU_CTRL, unit->hw_ctrl); 68 } 69 70 static void 71 amdiommu_disable_cmdbuf(struct amdiommu_unit *unit) 72 { 73 AMDIOMMU_ASSERT_LOCKED(unit); 74 75 unit->hw_ctrl &= ~AMDIOMMU_CTRL_CMDBUF_EN; 76 amdiommu_write8(unit, AMDIOMMU_CTRL, unit->hw_ctrl); 77 } 78 79 80 static void 81 amdiommu_enable_qi_intr(struct iommu_unit *iommu) 82 { 83 struct amdiommu_unit *unit; 84 85 unit = IOMMU2AMD(iommu); 86 AMDIOMMU_ASSERT_LOCKED(unit); 87 unit->hw_ctrl |= AMDIOMMU_CTRL_COMWINT_EN; 88 amdiommu_write8(unit, AMDIOMMU_CTRL, unit->hw_ctrl); 89 amdiommu_write8(unit, AMDIOMMU_CMDEV_STATUS, 90 AMDIOMMU_CMDEVS_COMWAITINT); 91 } 92 93 static void 94 amdiommu_disable_qi_intr(struct iommu_unit *iommu) 95 { 96 struct amdiommu_unit *unit; 97 98 unit = IOMMU2AMD(iommu); 99 AMDIOMMU_ASSERT_LOCKED(unit); 100 unit->hw_ctrl &= ~AMDIOMMU_CTRL_COMWINT_EN; 101 amdiommu_write8(unit, AMDIOMMU_CTRL, unit->hw_ctrl); 102 } 103 104 static void 105 amdiommu_cmd_advance_tail(struct iommu_unit *iommu) 106 { 107 struct amdiommu_unit *unit; 108 109 unit = IOMMU2AMD(iommu); 110 AMDIOMMU_ASSERT_LOCKED(unit); 111 amdiommu_write8(unit, AMDIOMMU_CMDBUF_TAIL, unit->x86c.inv_queue_tail); 112 } 113 114 static void 115 amdiommu_cmd_ensure(struct iommu_unit *iommu, int descr_count) 116 { 117 struct amdiommu_unit *unit; 118 uint64_t head; 119 int bytes; 120 121 unit = IOMMU2AMD(iommu); 122 AMDIOMMU_ASSERT_LOCKED(unit); 123 bytes = descr_count << AMDIOMMU_CMD_SZ_SHIFT; 124 for (;;) { 125 if (bytes <= unit->x86c.inv_queue_avail) 126 break; 127 /* refill */ 128 head = amdiommu_read8(unit, AMDIOMMU_CMDBUF_HEAD); 129 head &= AMDIOMMU_CMDPTR_MASK; 130 unit->x86c.inv_queue_avail = head - unit->x86c.inv_queue_tail - 131 AMDIOMMU_CMD_SZ; 132 if (head <= unit->x86c.inv_queue_tail) 133 unit->x86c.inv_queue_avail += unit->x86c.inv_queue_size; 134 if (bytes <= unit->x86c.inv_queue_avail) 135 break; 136 137 /* 138 * No space in the queue, do busy wait. Hardware must 139 * make a progress. But first advance the tail to 140 * inform the descriptor streamer about entries we 141 * might have already filled, otherwise they could 142 * clog the whole queue.. 143 * 144 * See dmar_qi_invalidate_locked() for a discussion 145 * about data race prevention. 146 */ 147 amdiommu_cmd_advance_tail(iommu); 148 unit->x86c.inv_queue_full++; 149 cpu_spinwait(); 150 } 151 unit->x86c.inv_queue_avail -= bytes; 152 } 153 154 static void 155 amdiommu_cmd_emit(struct amdiommu_unit *unit, const struct 156 amdiommu_cmd_generic *cmd) 157 { 158 AMDIOMMU_ASSERT_LOCKED(unit); 159 160 memcpy(unit->x86c.inv_queue + unit->x86c.inv_queue_tail, cmd, 161 sizeof(*cmd)); 162 unit->x86c.inv_queue_tail += AMDIOMMU_CMD_SZ; 163 KASSERT(unit->x86c.inv_queue_tail <= unit->x86c.inv_queue_size, 164 ("tail overflow 0x%x 0x%jx", unit->x86c.inv_queue_tail, 165 (uintmax_t)unit->x86c.inv_queue_size)); 166 unit->x86c.inv_queue_tail &= unit->x86c.inv_queue_size - 1; 167 } 168 169 static void 170 amdiommu_cmd_emit_wait_descr(struct iommu_unit *iommu, uint32_t seq, 171 bool intr, bool memw, bool fence) 172 { 173 struct amdiommu_unit *unit; 174 struct amdiommu_cmd_completion_wait c; 175 176 unit = IOMMU2AMD(iommu); 177 AMDIOMMU_ASSERT_LOCKED(unit); 178 179 bzero(&c, sizeof(c)); 180 c.op = AMDIOMMU_CMD_COMPLETION_WAIT; 181 if (memw) { 182 uint32_t x; 183 184 c.s = 1; 185 x = unit->x86c.inv_waitd_seq_hw_phys; 186 x >>= 3; 187 c.address0 = x; 188 x = unit->x86c.inv_waitd_seq_hw_phys >> 32; 189 c.address1 = x; 190 c.data0 = seq; 191 } 192 if (fence) 193 c.f = 1; 194 if (intr) 195 c.i = 1; 196 amdiommu_cmd_emit(unit, (struct amdiommu_cmd_generic *)&c); 197 } 198 199 static void 200 amdiommu_qi_invalidate_emit(struct iommu_domain *adomain, iommu_gaddr_t base, 201 iommu_gaddr_t size, struct iommu_qi_genseq *pseq, bool emit_wait) 202 { 203 struct amdiommu_domain *domain; 204 struct amdiommu_unit *unit; 205 struct amdiommu_cmd_invalidate_iommu_pages c; 206 u_int isize; 207 208 domain = IODOM2DOM(adomain); 209 unit = domain->unit; 210 AMDIOMMU_ASSERT_LOCKED(unit); 211 bzero(&c, sizeof(c)); 212 c.op = AMDIOMMU_CMD_INVALIDATE_IOMMU_PAGES; 213 c.domainid = domain->domain; 214 isize = IOMMU_PAGE_SIZE; /* XXXKIB handle superpages */ 215 216 for (; size > 0; base += isize, size -= isize) { 217 amdiommu_cmd_ensure(AMD2IOMMU(unit), 1); 218 c.s = 0; 219 c.pde = 1; 220 c.address = base >> IOMMU_PAGE_SHIFT; 221 amdiommu_cmd_emit(unit, (struct amdiommu_cmd_generic *)&c); 222 } 223 iommu_qi_emit_wait_seq(AMD2IOMMU(unit), pseq, emit_wait); 224 } 225 226 void 227 amdiommu_qi_invalidate_all_pages_locked_nowait(struct amdiommu_domain *domain) 228 { 229 struct amdiommu_unit *unit; 230 struct amdiommu_cmd_invalidate_iommu_pages c; 231 232 unit = domain->unit; 233 AMDIOMMU_ASSERT_LOCKED(unit); 234 bzero(&c, sizeof(c)); 235 c.op = AMDIOMMU_CMD_INVALIDATE_IOMMU_PAGES; 236 c.domainid = domain->domain; 237 238 /* 239 * The magic specified in the note for INVALIDATE_IOMMU_PAGES 240 * description. 241 */ 242 c.s = 1; 243 c.pde = 1; 244 c.address = 0x7ffffffffffff; 245 246 amdiommu_cmd_ensure(AMD2IOMMU(unit), 1); 247 amdiommu_cmd_emit(unit, (struct amdiommu_cmd_generic *)&c); 248 } 249 250 void 251 amdiommu_qi_invalidate_wait_sync(struct iommu_unit *iommu) 252 { 253 struct iommu_qi_genseq gseq; 254 255 amdiommu_cmd_ensure(iommu, 1); 256 iommu_qi_emit_wait_seq(iommu, &gseq, true); 257 IOMMU2AMD(iommu)->x86c.inv_seq_waiters++; 258 amdiommu_cmd_advance_tail(iommu); 259 iommu_qi_wait_for_seq(iommu, &gseq, true); 260 } 261 262 void 263 amdiommu_qi_invalidate_ctx_locked_nowait(struct amdiommu_ctx *ctx) 264 { 265 struct amdiommu_cmd_invalidate_devtab_entry c; 266 267 amdiommu_cmd_ensure(AMD2IOMMU(CTX2AMD(ctx)), 1); 268 bzero(&c, sizeof(c)); 269 c.op = AMDIOMMU_CMD_INVALIDATE_DEVTAB_ENTRY; 270 c.devid = ctx->context.rid; 271 amdiommu_cmd_emit(CTX2AMD(ctx), (struct amdiommu_cmd_generic *)&c); 272 } 273 274 275 void 276 amdiommu_qi_invalidate_ctx_locked(struct amdiommu_ctx *ctx) 277 { 278 amdiommu_qi_invalidate_ctx_locked_nowait(ctx); 279 amdiommu_qi_invalidate_wait_sync(AMD2IOMMU(CTX2AMD(ctx))); 280 } 281 282 void 283 amdiommu_qi_invalidate_ir_locked_nowait(struct amdiommu_unit *unit, 284 uint16_t devid) 285 { 286 struct amdiommu_cmd_invalidate_interrupt_table c; 287 288 AMDIOMMU_ASSERT_LOCKED(unit); 289 290 amdiommu_cmd_ensure(AMD2IOMMU(unit), 1); 291 bzero(&c, sizeof(c)); 292 c.op = AMDIOMMU_CMD_INVALIDATE_INTERRUPT_TABLE; 293 c.devid = devid; 294 amdiommu_cmd_emit(unit, (struct amdiommu_cmd_generic *)&c); 295 } 296 297 void 298 amdiommu_qi_invalidate_ir_locked(struct amdiommu_unit *unit, uint16_t devid) 299 { 300 amdiommu_qi_invalidate_ir_locked_nowait(unit, devid); 301 amdiommu_qi_invalidate_wait_sync(AMD2IOMMU(unit)); 302 } 303 304 static void 305 amdiommu_qi_task(void *arg, int pending __unused) 306 { 307 struct amdiommu_unit *unit; 308 309 unit = IOMMU2AMD(arg); 310 iommu_qi_drain_tlb_flush(AMD2IOMMU(unit)); 311 312 AMDIOMMU_LOCK(unit); 313 if (unit->x86c.inv_seq_waiters > 0) 314 wakeup(&unit->x86c.inv_seq_waiters); 315 AMDIOMMU_UNLOCK(unit); 316 } 317 318 int 319 amdiommu_init_cmd(struct amdiommu_unit *unit) 320 { 321 uint64_t qi_sz, rv; 322 323 unit->x86c.qi_buf_maxsz = ilog2(AMDIOMMU_CMDBUF_MAX / PAGE_SIZE); 324 unit->x86c.qi_cmd_sz = AMDIOMMU_CMD_SZ; 325 iommu_qi_common_init(AMD2IOMMU(unit), amdiommu_qi_task); 326 get_x86_iommu()->qi_ensure = amdiommu_cmd_ensure; 327 get_x86_iommu()->qi_emit_wait_descr = amdiommu_cmd_emit_wait_descr; 328 get_x86_iommu()->qi_advance_tail = amdiommu_cmd_advance_tail; 329 get_x86_iommu()->qi_invalidate_emit = amdiommu_qi_invalidate_emit; 330 331 rv = pmap_kextract((uintptr_t)unit->x86c.inv_queue); 332 333 /* 334 * See the description of the ComLen encoding for Command 335 * buffer Base Address Register. 336 */ 337 qi_sz = ilog2(unit->x86c.inv_queue_size / PAGE_SIZE) + 8; 338 rv |= qi_sz << AMDIOMMU_CMDBUF_BASE_SZSHIFT; 339 340 AMDIOMMU_LOCK(unit); 341 amdiommu_write8(unit, AMDIOMMU_CMDBUF_BASE, rv); 342 amdiommu_enable_cmdbuf(unit); 343 amdiommu_enable_qi_intr(AMD2IOMMU(unit)); 344 AMDIOMMU_UNLOCK(unit); 345 346 return (0); 347 } 348 349 static void 350 amdiommu_fini_cmd_helper(struct iommu_unit *iommu) 351 { 352 amdiommu_disable_cmdbuf(IOMMU2AMD(iommu)); 353 amdiommu_disable_qi_intr(iommu); 354 } 355 356 void 357 amdiommu_fini_cmd(struct amdiommu_unit *unit) 358 { 359 iommu_qi_common_fini(AMD2IOMMU(unit), amdiommu_fini_cmd_helper); 360 } 361