1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause
3 *
4 * Copyright (c) 2024 The FreeBSD Foundation
5 *
6 * This software was developed by Konstantin Belousov <kib@FreeBSD.org>
7 * under sponsorship from the FreeBSD Foundation.
8 *
9 * Redistribution and use in source and binary forms, with or without
10 * modification, are permitted provided that the following conditions
11 * are met:
12 * 1. Redistributions of source code must retain the above copyright
13 * notice, this list of conditions and the following disclaimer.
14 * 2. Redistributions in binary form must reproduce the above copyright
15 * notice, this list of conditions and the following disclaimer in the
16 * documentation and/or other materials provided with the distribution.
17 *
18 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
19 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
20 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
21 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
22 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
23 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
24 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
25 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
26 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
27 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
28 * SUCH DAMAGE.
29 */
30
31 #include <sys/param.h>
32 #include <sys/systm.h>
33 #include <sys/malloc.h>
34 #include <sys/bus.h>
35 #include <sys/interrupt.h>
36 #include <sys/domainset.h>
37 #include <sys/kernel.h>
38 #include <sys/ktr.h>
39 #include <sys/limits.h>
40 #include <sys/lock.h>
41 #include <sys/memdesc.h>
42 #include <sys/mutex.h>
43 #include <sys/proc.h>
44 #include <sys/rwlock.h>
45 #include <sys/rman.h>
46 #include <sys/sysctl.h>
47 #include <sys/taskqueue.h>
48 #include <sys/tree.h>
49 #include <sys/uio.h>
50 #include <sys/vmem.h>
51 #include <vm/vm.h>
52 #include <vm/vm_extern.h>
53 #include <vm/vm_kern.h>
54 #include <vm/vm_object.h>
55 #include <vm/vm_page.h>
56 #include <vm/vm_pager.h>
57 #include <vm/vm_map.h>
58 #include <contrib/dev/acpica/include/acpi.h>
59 #include <contrib/dev/acpica/include/accommon.h>
60 #include <dev/acpica/acpivar.h>
61 #include <dev/pci/pcireg.h>
62 #include <dev/pci/pcivar.h>
63 #include <machine/atomic.h>
64 #include <machine/bus.h>
65 #include <machine/md_var.h>
66 #include <machine/intr_machdep.h>
67 #include <x86/include/apicreg.h>
68 #include <x86/include/apicvar.h>
69 #include <machine/specialreg.h>
70 #include <x86/include/busdma_impl.h>
71 #include <dev/iommu/busdma_iommu.h>
72 #include <x86/iommu/amd_reg.h>
73 #include <x86/iommu/x86_iommu.h>
74 #include <x86/iommu/amd_iommu.h>
75
76 static struct amdiommu_ctx *amdiommu_ir_find(device_t src, uint16_t *rid,
77 bool *is_iommu);
78 static void amdiommu_ir_free_irte(struct amdiommu_ctx *ctx, device_t src,
79 u_int cookie);
80
81 int
amdiommu_alloc_msi_intr(device_t src,u_int * cookies,u_int count)82 amdiommu_alloc_msi_intr(device_t src, u_int *cookies, u_int count)
83 {
84 struct amdiommu_ctx *ctx;
85 vmem_addr_t vmem_res;
86 u_int idx, i;
87 int error;
88
89 ctx = amdiommu_ir_find(src, NULL, NULL);
90 if (ctx == NULL || !CTX2AMD(ctx)->irte_enabled) {
91 for (i = 0; i < count; i++)
92 cookies[i] = -1;
93 return (EOPNOTSUPP);
94 }
95
96 error = vmem_alloc(ctx->irtids, count, M_FIRSTFIT | M_NOWAIT,
97 &vmem_res);
98 if (error != 0) {
99 KASSERT(error != EOPNOTSUPP,
100 ("impossible EOPNOTSUPP from vmem"));
101 return (error);
102 }
103 idx = vmem_res;
104 for (i = 0; i < count; i++)
105 cookies[i] = idx + i;
106 return (0);
107 }
108
109 int
amdiommu_map_msi_intr(device_t src,u_int cpu,u_int vector,u_int cookie,uint64_t * addr,uint32_t * data)110 amdiommu_map_msi_intr(device_t src, u_int cpu, u_int vector,
111 u_int cookie, uint64_t *addr, uint32_t *data)
112 {
113 struct amdiommu_ctx *ctx;
114 struct amdiommu_unit *unit;
115 device_t requester;
116 int error __diagused;
117 uint16_t rid;
118 bool is_iommu;
119
120 ctx = amdiommu_ir_find(src, &rid, &is_iommu);
121 if (is_iommu) {
122 if (addr != NULL) {
123 *data = vector;
124 *addr = MSI_INTEL_ADDR_BASE | ((cpu & 0xff) << 12);
125 if (x2apic_mode)
126 *addr |= ((uint64_t)cpu & 0xffffff00) << 32;
127 else
128 KASSERT(cpu <= 0xff,
129 ("cpu id too big %d", cpu));
130 }
131 return (0);
132 }
133
134 if (ctx == NULL)
135 return (EOPNOTSUPP);
136 unit = CTX2AMD(ctx);
137 if (!unit->irte_enabled || cookie == -1)
138 return (EOPNOTSUPP);
139 if (cookie >= unit->irte_nentries) {
140 device_printf(src, "amdiommu%d: cookie %u irte max %u\n",
141 unit->iommu.unit, cookie, unit->irte_nentries);
142 return (EINVAL);
143 }
144
145 if (unit->irte_x2apic) {
146 struct amdiommu_irte_basic_vapic_x2 *irte;
147
148 irte = &ctx->irtx2[cookie];
149 irte->supiopf = 0;
150 irte->inttype = 0;
151 irte->rqeoi = 0;
152 irte->dm = 0;
153 irte->guestmode = 0;
154 irte->dest0 = cpu;
155 irte->rsrv0 = 0;
156 irte->vector = vector;
157 irte->rsrv1 = 0;
158 irte->rsrv2 = 0;
159 irte->dest1 = cpu >> 24;
160 atomic_thread_fence_rel();
161 irte->remapen = 1;
162 } else {
163 struct amdiommu_irte_basic_novapic *irte;
164
165 irte = &ctx->irtb[cookie];
166 irte->supiopf = 0;
167 irte->inttype = 0; /* fixed */
168 irte->rqeoi = 0;
169 irte->dm = 0; /* phys */
170 irte->guestmode = 0;
171 irte->dest = cpu;
172 irte->vector = vector;
173 irte->rsrv = 0;
174 atomic_thread_fence_rel();
175 irte->remapen = 1;
176 }
177
178 if (addr != NULL) {
179 *data = cookie;
180 *addr = MSI_INTEL_ADDR_BASE | ((cpu & 0xff) << 12);
181 if (unit->irte_x2apic)
182 *addr |= ((uint64_t)cpu & 0xffffff00) << 32;
183 }
184
185 error = iommu_get_requester(src, &requester, &rid);
186 MPASS(error == 0);
187 AMDIOMMU_LOCK(unit);
188 amdiommu_qi_invalidate_ir_locked(unit, rid);
189 AMDIOMMU_UNLOCK(unit);
190
191 return (0);
192 }
193
194 int
amdiommu_unmap_msi_intr(device_t src,u_int cookie)195 amdiommu_unmap_msi_intr(device_t src, u_int cookie)
196 {
197 struct amdiommu_ctx *ctx;
198
199 if (cookie == -1)
200 return (0);
201 ctx = amdiommu_ir_find(src, NULL, NULL);
202 amdiommu_ir_free_irte(ctx, src, cookie);
203 return (0);
204 }
205
206 int
amdiommu_map_ioapic_intr(u_int ioapic_id,u_int cpu,u_int vector,bool edge,bool activehi,int irq,u_int * cookie,uint32_t * hi,uint32_t * lo)207 amdiommu_map_ioapic_intr(u_int ioapic_id, u_int cpu, u_int vector,
208 bool edge, bool activehi, int irq, u_int *cookie, uint32_t *hi,
209 uint32_t *lo)
210 {
211 /* XXXKIB for early call from ioapic_create() */
212 return (EOPNOTSUPP);
213 }
214
215 int
amdiommu_unmap_ioapic_intr(u_int ioapic_id,u_int * cookie)216 amdiommu_unmap_ioapic_intr(u_int ioapic_id, u_int *cookie)
217 {
218 /* XXXKIB */
219 return (0);
220 }
221
222 static struct amdiommu_ctx *
amdiommu_ir_find(device_t src,uint16_t * ridp,bool * is_iommu)223 amdiommu_ir_find(device_t src, uint16_t *ridp, bool *is_iommu)
224 {
225 devclass_t src_class;
226 device_t requester;
227 struct amdiommu_unit *unit;
228 struct amdiommu_ctx *ctx;
229 uint32_t edte;
230 uint16_t rid;
231 uint8_t dte;
232 int error;
233
234 /*
235 * We need to determine if the interrupt source generates FSB
236 * interrupts. If yes, it is either IOMMU, in which case
237 * interrupts are not remapped. Or it is HPET, and interrupts
238 * are remapped. For HPET, source id is reported by HPET
239 * record in IVHD ACPI table.
240 */
241 if (is_iommu != NULL)
242 *is_iommu = false;
243
244 ctx = NULL;
245
246 src_class = device_get_devclass(src);
247 if (src_class == devclass_find("amdiommu")) {
248 if (is_iommu != NULL)
249 *is_iommu = true;
250 } else if (src_class == devclass_find("hpet")) {
251 error = amdiommu_find_unit_for_hpet(src, &unit, &rid, &dte,
252 &edte, bootverbose);
253 ctx = NULL; // XXXKIB allocate ctx
254 } else {
255 error = amdiommu_find_unit(src, &unit, &rid, &dte, &edte,
256 bootverbose);
257 if (error == 0) {
258 error = iommu_get_requester(src, &requester, &rid);
259 MPASS(error == 0);
260 ctx = amdiommu_get_ctx_for_dev(unit, src,
261 rid, 0, false /* XXXKIB */, false, dte, edte);
262 }
263 }
264 if (ridp != NULL)
265 *ridp = rid;
266 return (ctx);
267 }
268
269 static void
amdiommu_ir_free_irte(struct amdiommu_ctx * ctx,device_t src,u_int cookie)270 amdiommu_ir_free_irte(struct amdiommu_ctx *ctx, device_t src,
271 u_int cookie)
272 {
273 struct amdiommu_unit *unit;
274 device_t requester;
275 int error __diagused;
276 uint16_t rid;
277
278 MPASS(ctx != NULL);
279 unit = CTX2AMD(ctx);
280
281 KASSERT(unit->irte_enabled,
282 ("unmap: cookie %d ctx %p unit %p", cookie, ctx, unit));
283 KASSERT(cookie < unit->irte_nentries,
284 ("bad cookie %u %u", cookie, unit->irte_nentries));
285
286 if (unit->irte_x2apic) {
287 struct amdiommu_irte_basic_vapic_x2 *irte;
288
289 irte = &ctx->irtx2[cookie];
290 irte->remapen = 0;
291 atomic_thread_fence_rel();
292 bzero(irte, sizeof(*irte));
293 } else {
294 struct amdiommu_irte_basic_novapic *irte;
295
296 irte = &ctx->irtb[cookie];
297 irte->remapen = 0;
298 atomic_thread_fence_rel();
299 bzero(irte, sizeof(*irte));
300 }
301 error = iommu_get_requester(src, &requester, &rid);
302 MPASS(error == 0);
303 AMDIOMMU_LOCK(unit);
304 amdiommu_qi_invalidate_ir_locked(unit, rid);
305 AMDIOMMU_UNLOCK(unit);
306 }
307
308 int
amdiommu_ctx_init_irte(struct amdiommu_ctx * ctx)309 amdiommu_ctx_init_irte(struct amdiommu_ctx *ctx)
310 {
311 struct amdiommu_unit *unit;
312 void *ptr;
313 unsigned long sz;
314 int dom;
315
316 unit = CTX2AMD(ctx);
317 if (!unit->irte_enabled)
318 return (0);
319
320 KASSERT(unit->irte_nentries > 0 &&
321 unit->irte_nentries <= 2048 &&
322 powerof2(unit->irte_nentries),
323 ("amdiommu%d: unit %p irte_nentries %u", unit->iommu.unit,
324 unit, unit->irte_nentries));
325
326 if (bus_get_domain(unit->iommu.dev, &dom) != 0)
327 dom = -1;
328 sz = unit->irte_nentries;
329 sz *= unit->irte_x2apic ? sizeof(struct amdiommu_irte_basic_vapic_x2) :
330 sizeof(struct amdiommu_irte_basic_novapic);
331
332 if (dom != -1) {
333 ptr = contigmalloc_domainset(sz, M_DEVBUF, DOMAINSET_PREF(dom),
334 M_WAITOK | M_ZERO, 0, ~0ull, 128, 0);
335 } else {
336 ptr = contigmalloc(sz, M_DEVBUF, M_WAITOK | M_ZERO,
337 0, ~0ull, 128, 0);
338 }
339 if (unit->irte_x2apic)
340 ctx->irtx2 = ptr;
341 else
342 ctx->irtb = ptr;
343 ctx->irtids = vmem_create("amdirt", 0, unit->irte_nentries, 1, 0,
344 M_FIRSTFIT | M_NOWAIT);
345
346 intr_reprogram(); // XXXKIB
347
348 return (0);
349 }
350
351 void
amdiommu_ctx_fini_irte(struct amdiommu_ctx * ctx)352 amdiommu_ctx_fini_irte(struct amdiommu_ctx *ctx)
353 {
354 struct amdiommu_unit *unit;
355
356 unit = CTX2AMD(ctx);
357 if (!unit->irte_enabled)
358 return;
359 if (unit->irte_x2apic)
360 free(ctx->irtx2, M_DEVBUF);
361 else
362 free(ctx->irtb, M_DEVBUF);
363 vmem_destroy(ctx->irtids);
364 }
365
366 int
amdiommu_init_irt(struct amdiommu_unit * unit)367 amdiommu_init_irt(struct amdiommu_unit *unit)
368 {
369 int enabled, nentries;
370
371 SYSCTL_ADD_INT(&unit->iommu.sysctl_ctx,
372 SYSCTL_CHILDREN(device_get_sysctl_tree(unit->iommu.dev)),
373 OID_AUTO, "ir", CTLFLAG_RD, &unit->irte_enabled, 0,
374 "Interrupt remapping ops enabled");
375
376 enabled = 1;
377 TUNABLE_INT_FETCH("hw.iommu.ir", &enabled);
378
379 unit->irte_enabled = enabled != 0;
380 if (!unit->irte_enabled)
381 return (0);
382
383 nentries = 32;
384 TUNABLE_INT_FETCH("hw.iommu.amd.ir_num", &nentries);
385 nentries = roundup_pow_of_two(nentries);
386 if (nentries < 1)
387 nentries = 1;
388 if (nentries > 2048)
389 nentries = 2048;
390 unit->irte_nentries = nentries;
391
392 unit->irte_x2apic = x2apic_mode;
393 return (0);
394 }
395
396 void
amdiommu_fini_irt(struct amdiommu_unit * unit)397 amdiommu_fini_irt(struct amdiommu_unit *unit)
398 {
399 }
400