1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause
3 *
4 * Copyright (c) 2024 The FreeBSD Foundation
5 *
6 * This software was developed by Konstantin Belousov <kib@FreeBSD.org>
7 * under sponsorship from the FreeBSD Foundation.
8 *
9 * Redistribution and use in source and binary forms, with or without
10 * modification, are permitted provided that the following conditions
11 * are met:
12 * 1. Redistributions of source code must retain the above copyright
13 * notice, this list of conditions and the following disclaimer.
14 * 2. Redistributions in binary form must reproduce the above copyright
15 * notice, this list of conditions and the following disclaimer in the
16 * documentation and/or other materials provided with the distribution.
17 *
18 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
19 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
20 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
21 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
22 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
23 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
24 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
25 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
26 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
27 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
28 * SUCH DAMAGE.
29 */
30
31 #include <sys/param.h>
32 #include <sys/systm.h>
33 #include <sys/malloc.h>
34 #include <sys/bus.h>
35 #include <sys/interrupt.h>
36 #include <sys/domainset.h>
37 #include <sys/kernel.h>
38 #include <sys/ktr.h>
39 #include <sys/limits.h>
40 #include <sys/lock.h>
41 #include <sys/memdesc.h>
42 #include <sys/mutex.h>
43 #include <sys/proc.h>
44 #include <sys/rwlock.h>
45 #include <sys/rman.h>
46 #include <sys/sysctl.h>
47 #include <sys/taskqueue.h>
48 #include <sys/tree.h>
49 #include <sys/uio.h>
50 #include <sys/vmem.h>
51 #include <vm/vm.h>
52 #include <vm/vm_extern.h>
53 #include <vm/vm_kern.h>
54 #include <vm/vm_object.h>
55 #include <vm/vm_page.h>
56 #include <vm/vm_pager.h>
57 #include <vm/vm_map.h>
58 #include <contrib/dev/acpica/include/acpi.h>
59 #include <contrib/dev/acpica/include/accommon.h>
60 #include <dev/acpica/acpivar.h>
61 #include <dev/pci/pcireg.h>
62 #include <dev/pci/pcivar.h>
63 #include <machine/atomic.h>
64 #include <machine/bus.h>
65 #include <machine/md_var.h>
66 #include <machine/intr_machdep.h>
67 #include <x86/include/apicreg.h>
68 #include <x86/include/apicvar.h>
69 #include <machine/specialreg.h>
70 #include <x86/include/busdma_impl.h>
71 #include <dev/iommu/busdma_iommu.h>
72 #include <x86/iommu/amd_reg.h>
73 #include <x86/iommu/x86_iommu.h>
74 #include <x86/iommu/amd_iommu.h>
75
76 static struct amdiommu_ctx *amdiommu_ir_find(device_t src, uint16_t *rid,
77 bool *is_iommu);
78 static void amdiommu_ir_free_irte(struct amdiommu_ctx *ctx, device_t src,
79 u_int cookie);
80
81 int
amdiommu_alloc_msi_intr(device_t src,u_int * cookies,u_int count)82 amdiommu_alloc_msi_intr(device_t src, u_int *cookies, u_int count)
83 {
84 struct amdiommu_ctx *ctx;
85 vmem_addr_t vmem_res;
86 u_int idx, i;
87 int error;
88
89 ctx = amdiommu_ir_find(src, NULL, NULL);
90 if (ctx == NULL || !CTX2AMD(ctx)->irte_enabled) {
91 for (i = 0; i < count; i++)
92 cookies[i] = -1;
93 return (EOPNOTSUPP);
94 }
95
96 error = vmem_alloc(ctx->irtids, count, M_FIRSTFIT | M_NOWAIT,
97 &vmem_res);
98 if (error != 0) {
99 KASSERT(error != EOPNOTSUPP,
100 ("impossible EOPNOTSUPP from vmem"));
101 return (error);
102 }
103 idx = vmem_res;
104 for (i = 0; i < count; i++)
105 cookies[i] = idx + i;
106 return (0);
107 }
108
109 int
amdiommu_map_msi_intr(device_t src,u_int cpu,u_int vector,u_int cookie,uint64_t * addr,uint32_t * data)110 amdiommu_map_msi_intr(device_t src, u_int cpu, u_int vector,
111 u_int cookie, uint64_t *addr, uint32_t *data)
112 {
113 struct amdiommu_ctx *ctx;
114 struct amdiommu_unit *unit;
115 device_t requester;
116 int error __diagused;
117 uint16_t rid;
118 bool is_iommu;
119
120 ctx = amdiommu_ir_find(src, &rid, &is_iommu);
121 if (is_iommu) {
122 if (addr != NULL) {
123 *data = vector;
124 *addr = MSI_INTEL_ADDR_BASE | ((cpu & 0xff) << 12);
125 if (x2apic_mode)
126 *addr |= ((uint64_t)cpu & 0xffffff00) << 32;
127 else
128 KASSERT(cpu <= 0xff,
129 ("cpu id too big %d", cpu));
130 }
131 return (0);
132 }
133
134 if (ctx == NULL)
135 return (EOPNOTSUPP);
136 unit = CTX2AMD(ctx);
137 if (!unit->irte_enabled || cookie == -1)
138 return (EOPNOTSUPP);
139 if (cookie >= unit->irte_nentries) {
140 device_printf(src, "amdiommu%d: cookie %u irte max %u\n",
141 unit->iommu.unit, cookie, unit->irte_nentries);
142 return (EINVAL);
143 }
144
145 if (unit->irte_x2apic) {
146 struct amdiommu_irte_basic_vapic_x2 *irte;
147
148 irte = &ctx->irtx2[cookie];
149 irte->supiopf = 0;
150 irte->inttype = 0;
151 irte->rqeoi = 0;
152 irte->dm = 0;
153 irte->guestmode = 0;
154 irte->dest0 = cpu;
155 irte->rsrv0 = 0;
156 irte->vector = vector;
157 irte->rsrv1 = 0;
158 irte->rsrv2 = 0;
159 irte->dest1 = cpu >> 24;
160 atomic_thread_fence_rel();
161 irte->remapen = 1;
162 } else {
163 struct amdiommu_irte_basic_novapic *irte;
164
165 irte = &ctx->irtb[cookie];
166 irte->supiopf = 0;
167 irte->inttype = 0; /* fixed */
168 irte->rqeoi = 0;
169 irte->dm = 0; /* phys */
170 irte->guestmode = 0;
171 irte->dest = cpu;
172 irte->vector = vector;
173 irte->rsrv = 0;
174 atomic_thread_fence_rel();
175 irte->remapen = 1;
176 }
177
178 if (addr != NULL) {
179 *data = cookie;
180 *addr = MSI_INTEL_ADDR_BASE | ((cpu & 0xff) << 12);
181 if (unit->irte_x2apic)
182 *addr |= ((uint64_t)cpu & 0xffffff00) << 32;
183 }
184
185 error = iommu_get_requester(src, &requester, &rid);
186 MPASS(error == 0);
187 AMDIOMMU_LOCK(unit);
188 amdiommu_qi_invalidate_ir_locked(unit, rid);
189 AMDIOMMU_UNLOCK(unit);
190
191 return (0);
192 }
193
194 int
amdiommu_unmap_msi_intr(device_t src,u_int cookie)195 amdiommu_unmap_msi_intr(device_t src, u_int cookie)
196 {
197 struct amdiommu_ctx *ctx;
198
199 if (cookie == -1)
200 return (0);
201 ctx = amdiommu_ir_find(src, NULL, NULL);
202 amdiommu_ir_free_irte(ctx, src, cookie);
203 return (0);
204 }
205
206 int
amdiommu_map_ioapic_intr(u_int ioapic_id,u_int cpu,u_int vector,bool edge,bool activehi,int irq,u_int * cookie,uint32_t * hi,uint32_t * lo)207 amdiommu_map_ioapic_intr(u_int ioapic_id, u_int cpu, u_int vector,
208 bool edge, bool activehi, int irq, u_int *cookie, uint32_t *hi,
209 uint32_t *lo)
210 {
211 /* XXXKIB for early call from ioapic_create() */
212 return (EOPNOTSUPP);
213 }
214
215 int
amdiommu_unmap_ioapic_intr(u_int ioapic_id,u_int * cookie)216 amdiommu_unmap_ioapic_intr(u_int ioapic_id, u_int *cookie)
217 {
218 /* XXXKIB */
219 return (0);
220 }
221
222 static struct amdiommu_ctx *
amdiommu_ir_find(device_t src,uint16_t * ridp,bool * is_iommu)223 amdiommu_ir_find(device_t src, uint16_t *ridp, bool *is_iommu)
224 {
225 devclass_t src_class;
226 struct amdiommu_unit *unit;
227 struct amdiommu_ctx *ctx;
228 struct iommu_ctx *ioctx;
229 uint32_t edte;
230 uint16_t rid;
231 uint8_t dte;
232 int error;
233
234 /*
235 * We need to determine if the interrupt source generates FSB
236 * interrupts. If yes, it is either IOMMU, in which case
237 * interrupts are not remapped. Or it is HPET, and interrupts
238 * are remapped. For HPET, source id is reported by HPET
239 * record in IVHD ACPI table.
240 */
241 if (is_iommu != NULL)
242 *is_iommu = false;
243
244 ctx = NULL;
245
246 src_class = device_get_devclass(src);
247 if (src_class == devclass_find("amdiommu")) {
248 if (is_iommu != NULL)
249 *is_iommu = true;
250 } else if (src_class == devclass_find("hpet")) {
251 error = amdiommu_find_unit_for_hpet(src, &unit, &rid, &dte,
252 &edte, bootverbose);
253 ctx = NULL; // XXXKIB allocate ctx
254 } else {
255 error = amdiommu_find_unit(src, &unit, &rid, &dte, &edte,
256 bootverbose);
257 if (error == 0) {
258 ioctx = iommu_instantiate_ctx(AMD2IOMMU(unit), src, false);
259 if (ioctx != NULL)
260 ctx = IOCTX2CTX(ioctx);
261 }
262 }
263 if (ridp != NULL)
264 *ridp = rid;
265 return (ctx);
266 }
267
268 static void
amdiommu_ir_free_irte(struct amdiommu_ctx * ctx,device_t src,u_int cookie)269 amdiommu_ir_free_irte(struct amdiommu_ctx *ctx, device_t src,
270 u_int cookie)
271 {
272 struct amdiommu_unit *unit;
273 device_t requester;
274 int error __diagused;
275 uint16_t rid;
276
277 MPASS(ctx != NULL);
278 unit = CTX2AMD(ctx);
279
280 KASSERT(unit->irte_enabled,
281 ("unmap: cookie %d ctx %p unit %p", cookie, ctx, unit));
282 KASSERT(cookie < unit->irte_nentries,
283 ("bad cookie %u %u", cookie, unit->irte_nentries));
284
285 if (unit->irte_x2apic) {
286 struct amdiommu_irte_basic_vapic_x2 *irte;
287
288 irte = &ctx->irtx2[cookie];
289 irte->remapen = 0;
290 atomic_thread_fence_rel();
291 bzero(irte, sizeof(*irte));
292 } else {
293 struct amdiommu_irte_basic_novapic *irte;
294
295 irte = &ctx->irtb[cookie];
296 irte->remapen = 0;
297 atomic_thread_fence_rel();
298 bzero(irte, sizeof(*irte));
299 }
300 error = iommu_get_requester(src, &requester, &rid);
301 MPASS(error == 0);
302 AMDIOMMU_LOCK(unit);
303 amdiommu_qi_invalidate_ir_locked(unit, rid);
304 AMDIOMMU_UNLOCK(unit);
305 }
306
307 int
amdiommu_ctx_init_irte(struct amdiommu_ctx * ctx)308 amdiommu_ctx_init_irte(struct amdiommu_ctx *ctx)
309 {
310 struct amdiommu_unit *unit;
311 void *ptr;
312 unsigned long sz;
313 int dom;
314
315 unit = CTX2AMD(ctx);
316 if (!unit->irte_enabled)
317 return (0);
318
319 KASSERT(unit->irte_nentries > 0 &&
320 unit->irte_nentries <= 2048 &&
321 powerof2(unit->irte_nentries),
322 ("amdiommu%d: unit %p irte_nentries %u", unit->iommu.unit,
323 unit, unit->irte_nentries));
324
325 if (bus_get_domain(unit->iommu.dev, &dom) != 0)
326 dom = -1;
327 sz = unit->irte_nentries;
328 sz *= unit->irte_x2apic ? sizeof(struct amdiommu_irte_basic_vapic_x2) :
329 sizeof(struct amdiommu_irte_basic_novapic);
330
331 if (dom != -1) {
332 ptr = contigmalloc_domainset(sz, M_DEVBUF, DOMAINSET_PREF(dom),
333 M_WAITOK | M_ZERO, 0, ~0ull, 128, 0);
334 } else {
335 ptr = contigmalloc(sz, M_DEVBUF, M_WAITOK | M_ZERO,
336 0, ~0ull, 128, 0);
337 }
338 if (unit->irte_x2apic)
339 ctx->irtx2 = ptr;
340 else
341 ctx->irtb = ptr;
342 ctx->irtids = vmem_create("amdirt", 0, unit->irte_nentries, 1, 0,
343 M_FIRSTFIT | M_NOWAIT);
344
345 intr_reprogram(); // XXXKIB
346
347 return (0);
348 }
349
350 void
amdiommu_ctx_fini_irte(struct amdiommu_ctx * ctx)351 amdiommu_ctx_fini_irte(struct amdiommu_ctx *ctx)
352 {
353 struct amdiommu_unit *unit;
354
355 unit = CTX2AMD(ctx);
356 if (!unit->irte_enabled)
357 return;
358 if (unit->irte_x2apic)
359 free(ctx->irtx2, M_DEVBUF);
360 else
361 free(ctx->irtb, M_DEVBUF);
362 vmem_destroy(ctx->irtids);
363 }
364
365 int
amdiommu_init_irt(struct amdiommu_unit * unit)366 amdiommu_init_irt(struct amdiommu_unit *unit)
367 {
368 int enabled, nentries;
369
370 SYSCTL_ADD_INT(&unit->iommu.sysctl_ctx,
371 SYSCTL_CHILDREN(device_get_sysctl_tree(unit->iommu.dev)),
372 OID_AUTO, "ir", CTLFLAG_RD, &unit->irte_enabled, 0,
373 "Interrupt remapping ops enabled");
374
375 enabled = 1;
376 TUNABLE_INT_FETCH("hw.iommu.ir", &enabled);
377
378 unit->irte_enabled = enabled != 0;
379 if (!unit->irte_enabled)
380 return (0);
381
382 nentries = 32;
383 TUNABLE_INT_FETCH("hw.iommu.amd.ir_num", &nentries);
384 nentries = roundup_pow_of_two(nentries);
385 if (nentries < 1)
386 nentries = 1;
387 if (nentries > 2048)
388 nentries = 2048;
389 unit->irte_nentries = nentries;
390
391 unit->irte_x2apic = x2apic_mode;
392 return (0);
393 }
394
395 void
amdiommu_fini_irt(struct amdiommu_unit * unit)396 amdiommu_fini_irt(struct amdiommu_unit *unit)
397 {
398 }
399