1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause
3 *
4 * Copyright (c) 2024 The FreeBSD Foundation
5 *
6 * This software was developed by Konstantin Belousov <kib@FreeBSD.org>
7 * under sponsorship from the FreeBSD Foundation.
8 *
9 * Redistribution and use in source and binary forms, with or without
10 * modification, are permitted provided that the following conditions
11 * are met:
12 * 1. Redistributions of source code must retain the above copyright
13 * notice, this list of conditions and the following disclaimer.
14 * 2. Redistributions in binary form must reproduce the above copyright
15 * notice, this list of conditions and the following disclaimer in the
16 * documentation and/or other materials provided with the distribution.
17 *
18 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
19 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
20 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
21 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
22 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
23 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
24 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
25 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
26 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
27 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
28 * SUCH DAMAGE.
29 */
30
31 #include <sys/param.h>
32 #include <sys/systm.h>
33 #include <sys/malloc.h>
34 #include <sys/bus.h>
35 #include <sys/interrupt.h>
36 #include <sys/domainset.h>
37 #include <sys/kernel.h>
38 #include <sys/ktr.h>
39 #include <sys/limits.h>
40 #include <sys/lock.h>
41 #include <sys/memdesc.h>
42 #include <sys/mutex.h>
43 #include <sys/proc.h>
44 #include <sys/rwlock.h>
45 #include <sys/rman.h>
46 #include <sys/sysctl.h>
47 #include <sys/taskqueue.h>
48 #include <sys/tree.h>
49 #include <sys/uio.h>
50 #include <sys/vmem.h>
51 #include <vm/vm.h>
52 #include <vm/vm_extern.h>
53 #include <vm/vm_kern.h>
54 #include <vm/vm_object.h>
55 #include <vm/vm_page.h>
56 #include <vm/vm_pager.h>
57 #include <vm/vm_map.h>
58 #include <contrib/dev/acpica/include/acpi.h>
59 #include <contrib/dev/acpica/include/accommon.h>
60 #include <dev/acpica/acpivar.h>
61 #include <dev/pci/pcireg.h>
62 #include <dev/pci/pcivar.h>
63 #include <machine/atomic.h>
64 #include <machine/bus.h>
65 #include <machine/md_var.h>
66 #include <machine/intr_machdep.h>
67 #include <x86/include/apicreg.h>
68 #include <x86/include/apicvar.h>
69 #include <machine/specialreg.h>
70 #include <x86/include/busdma_impl.h>
71 #include <dev/iommu/busdma_iommu.h>
72 #include <x86/iommu/amd_reg.h>
73 #include <x86/iommu/x86_iommu.h>
74 #include <x86/iommu/amd_iommu.h>
75
76 static struct amdiommu_ctx *amdiommu_ir_find(device_t src, uint16_t *rid,
77 bool *is_iommu);
78 static void amdiommu_ir_free_irte(struct amdiommu_ctx *ctx, device_t src,
79 u_int cookie);
80
81 int
amdiommu_alloc_msi_intr(device_t src,u_int * cookies,u_int count)82 amdiommu_alloc_msi_intr(device_t src, u_int *cookies, u_int count)
83 {
84 struct amdiommu_ctx *ctx;
85 vmem_addr_t vmem_res;
86 u_int idx, i;
87 int error;
88
89 ctx = amdiommu_ir_find(src, NULL, NULL);
90 if (ctx == NULL || !CTX2AMD(ctx)->irte_enabled) {
91 for (i = 0; i < count; i++)
92 cookies[i] = -1;
93 return (EOPNOTSUPP);
94 }
95
96 error = vmem_alloc(ctx->irtids, count, M_FIRSTFIT | M_NOWAIT,
97 &vmem_res);
98 if (error != 0) {
99 KASSERT(error != EOPNOTSUPP,
100 ("impossible EOPNOTSUPP from vmem"));
101 return (error);
102 }
103 idx = vmem_res;
104 for (i = 0; i < count; i++)
105 cookies[i] = idx + i;
106 return (0);
107 }
108
109 int
amdiommu_map_msi_intr(device_t src,u_int cpu,u_int vector,u_int cookie,uint64_t * addr,uint32_t * data)110 amdiommu_map_msi_intr(device_t src, u_int cpu, u_int vector,
111 u_int cookie, uint64_t *addr, uint32_t *data)
112 {
113 struct amdiommu_ctx *ctx;
114 struct amdiommu_unit *unit;
115 uint16_t rid;
116 bool is_iommu;
117
118 ctx = amdiommu_ir_find(src, &rid, &is_iommu);
119 if (is_iommu) {
120 if (addr != NULL) {
121 *data = vector;
122 *addr = MSI_INTEL_ADDR_BASE | ((cpu & 0xff) << 12);
123 if (x2apic_mode)
124 *addr |= ((uint64_t)cpu & 0xffffff00) << 32;
125 else
126 KASSERT(cpu <= 0xff,
127 ("cpu id too big %d", cpu));
128 }
129 return (0);
130 }
131
132 if (ctx == NULL)
133 return (EOPNOTSUPP);
134 unit = CTX2AMD(ctx);
135 if (!unit->irte_enabled || cookie == -1)
136 return (EOPNOTSUPP);
137 if (cookie >= unit->irte_nentries) {
138 device_printf(src, "amdiommu%d: cookie %u irte max %u\n",
139 unit->iommu.unit, cookie, unit->irte_nentries);
140 return (EINVAL);
141 }
142
143 if (unit->irte_x2apic) {
144 struct amdiommu_irte_basic_vapic_x2 *irte;
145
146 irte = &ctx->irtx2[cookie];
147 irte->supiopf = 0;
148 irte->inttype = 0;
149 irte->rqeoi = 0;
150 irte->dm = 0;
151 irte->guestmode = 0;
152 irte->dest0 = cpu;
153 irte->rsrv0 = 0;
154 irte->vector = vector;
155 irte->rsrv1 = 0;
156 irte->rsrv2 = 0;
157 irte->dest1 = cpu >> 24;
158 atomic_thread_fence_rel();
159 irte->remapen = 1;
160 } else {
161 struct amdiommu_irte_basic_novapic *irte;
162
163 irte = &ctx->irtb[cookie];
164 irte->supiopf = 0;
165 irte->inttype = 0; /* fixed */
166 irte->rqeoi = 0;
167 irte->dm = 0; /* phys */
168 irte->guestmode = 0;
169 irte->dest = cpu;
170 irte->vector = vector;
171 irte->rsrv = 0;
172 atomic_thread_fence_rel();
173 irte->remapen = 1;
174 }
175
176 if (addr != NULL) {
177 *data = cookie;
178 *addr = MSI_INTEL_ADDR_BASE | ((cpu & 0xff) << 12);
179 if (unit->irte_x2apic)
180 *addr |= ((uint64_t)cpu & 0xffffff00) << 32;
181 }
182
183 iommu_get_requester(src, &rid);
184 AMDIOMMU_LOCK(unit);
185 amdiommu_qi_invalidate_ir_locked(unit, rid);
186 AMDIOMMU_UNLOCK(unit);
187
188 return (0);
189 }
190
191 int
amdiommu_unmap_msi_intr(device_t src,u_int cookie)192 amdiommu_unmap_msi_intr(device_t src, u_int cookie)
193 {
194 struct amdiommu_ctx *ctx;
195
196 if (cookie == -1)
197 return (0);
198 ctx = amdiommu_ir_find(src, NULL, NULL);
199 amdiommu_ir_free_irte(ctx, src, cookie);
200 return (0);
201 }
202
203 int
amdiommu_map_ioapic_intr(u_int ioapic_id,u_int cpu,u_int vector,bool edge,bool activehi,int irq,u_int * cookie,uint32_t * hi,uint32_t * lo)204 amdiommu_map_ioapic_intr(u_int ioapic_id, u_int cpu, u_int vector,
205 bool edge, bool activehi, int irq, u_int *cookie, uint32_t *hi,
206 uint32_t *lo)
207 {
208 /* XXXKIB for early call from ioapic_create() */
209 return (EOPNOTSUPP);
210 }
211
212 int
amdiommu_unmap_ioapic_intr(u_int ioapic_id,u_int * cookie)213 amdiommu_unmap_ioapic_intr(u_int ioapic_id, u_int *cookie)
214 {
215 /* XXXKIB */
216 return (0);
217 }
218
219 static struct amdiommu_ctx *
amdiommu_ir_find(device_t src,uint16_t * ridp,bool * is_iommu)220 amdiommu_ir_find(device_t src, uint16_t *ridp, bool *is_iommu)
221 {
222 devclass_t src_class;
223 struct amdiommu_unit *unit;
224 struct amdiommu_ctx *ctx;
225 uint32_t edte;
226 uint16_t rid;
227 uint8_t dte;
228 int error;
229
230 /*
231 * We need to determine if the interrupt source generates FSB
232 * interrupts. If yes, it is either IOMMU, in which case
233 * interrupts are not remapped. Or it is HPET, and interrupts
234 * are remapped. For HPET, source id is reported by HPET
235 * record in IVHD ACPI table.
236 */
237 if (is_iommu != NULL)
238 *is_iommu = false;
239
240 ctx = NULL;
241
242 src_class = device_get_devclass(src);
243 if (src_class == devclass_find("amdiommu")) {
244 if (is_iommu != NULL)
245 *is_iommu = true;
246 } else if (src_class == devclass_find("hpet")) {
247 error = amdiommu_find_unit_for_hpet(src, &unit, &rid, &dte,
248 &edte, bootverbose);
249 ctx = NULL; // XXXKIB allocate ctx
250 } else {
251 error = amdiommu_find_unit(src, &unit, &rid, &dte, &edte,
252 bootverbose);
253 if (error == 0) {
254 iommu_get_requester(src, &rid);
255 ctx = amdiommu_get_ctx_for_dev(unit, src,
256 rid, 0, false /* XXXKIB */, false, dte, edte);
257 }
258 }
259 if (ridp != NULL)
260 *ridp = rid;
261 return (ctx);
262 }
263
264 static void
amdiommu_ir_free_irte(struct amdiommu_ctx * ctx,device_t src,u_int cookie)265 amdiommu_ir_free_irte(struct amdiommu_ctx *ctx, device_t src,
266 u_int cookie)
267 {
268 struct amdiommu_unit *unit;
269 uint16_t rid;
270
271 MPASS(ctx != NULL);
272 unit = CTX2AMD(ctx);
273
274 KASSERT(unit->irte_enabled,
275 ("unmap: cookie %d ctx %p unit %p", cookie, ctx, unit));
276 KASSERT(cookie < unit->irte_nentries,
277 ("bad cookie %u %u", cookie, unit->irte_nentries));
278
279 if (unit->irte_x2apic) {
280 struct amdiommu_irte_basic_vapic_x2 *irte;
281
282 irte = &ctx->irtx2[cookie];
283 irte->remapen = 0;
284 atomic_thread_fence_rel();
285 bzero(irte, sizeof(*irte));
286 } else {
287 struct amdiommu_irte_basic_novapic *irte;
288
289 irte = &ctx->irtb[cookie];
290 irte->remapen = 0;
291 atomic_thread_fence_rel();
292 bzero(irte, sizeof(*irte));
293 }
294 iommu_get_requester(src, &rid);
295 AMDIOMMU_LOCK(unit);
296 amdiommu_qi_invalidate_ir_locked(unit, rid);
297 AMDIOMMU_UNLOCK(unit);
298 }
299
300 int
amdiommu_ctx_init_irte(struct amdiommu_ctx * ctx)301 amdiommu_ctx_init_irte(struct amdiommu_ctx *ctx)
302 {
303 struct amdiommu_unit *unit;
304 void *ptr;
305 unsigned long sz;
306 int dom;
307
308 unit = CTX2AMD(ctx);
309 if (!unit->irte_enabled)
310 return (0);
311
312 KASSERT(unit->irte_nentries > 0 &&
313 unit->irte_nentries <= 2048 &&
314 powerof2(unit->irte_nentries),
315 ("amdiommu%d: unit %p irte_nentries %u", unit->iommu.unit,
316 unit, unit->irte_nentries));
317
318 if (bus_get_domain(unit->iommu.dev, &dom) != 0)
319 dom = -1;
320 sz = unit->irte_nentries;
321 sz *= unit->irte_x2apic ? sizeof(struct amdiommu_irte_basic_vapic_x2) :
322 sizeof(struct amdiommu_irte_basic_novapic);
323
324 if (dom != -1) {
325 ptr = contigmalloc_domainset(sz, M_DEVBUF, DOMAINSET_PREF(dom),
326 M_WAITOK | M_ZERO, 0, ~0ull, 128, 0);
327 } else {
328 ptr = contigmalloc(sz, M_DEVBUF, M_WAITOK | M_ZERO,
329 0, ~0ull, 128, 0);
330 }
331 if (unit->irte_x2apic)
332 ctx->irtx2 = ptr;
333 else
334 ctx->irtb = ptr;
335 ctx->irtids = vmem_create("amdirt", 0, unit->irte_nentries, 1, 0,
336 M_FIRSTFIT | M_NOWAIT);
337
338 intr_reprogram(); // XXXKIB
339
340 return (0);
341 }
342
343 void
amdiommu_ctx_fini_irte(struct amdiommu_ctx * ctx)344 amdiommu_ctx_fini_irte(struct amdiommu_ctx *ctx)
345 {
346 struct amdiommu_unit *unit;
347
348 unit = CTX2AMD(ctx);
349 if (!unit->irte_enabled)
350 return;
351 if (unit->irte_x2apic)
352 free(ctx->irtx2, M_DEVBUF);
353 else
354 free(ctx->irtb, M_DEVBUF);
355 vmem_destroy(ctx->irtids);
356 }
357
358 int
amdiommu_init_irt(struct amdiommu_unit * unit)359 amdiommu_init_irt(struct amdiommu_unit *unit)
360 {
361 int enabled, nentries;
362
363 SYSCTL_ADD_INT(&unit->iommu.sysctl_ctx,
364 SYSCTL_CHILDREN(device_get_sysctl_tree(unit->iommu.dev)),
365 OID_AUTO, "ir", CTLFLAG_RD, &unit->irte_enabled, 0,
366 "Interrupt remapping ops enabled");
367
368 enabled = 1;
369 TUNABLE_INT_FETCH("hw.iommu.ir", &enabled);
370
371 unit->irte_enabled = enabled != 0;
372 if (!unit->irte_enabled)
373 return (0);
374
375 nentries = 32;
376 TUNABLE_INT_FETCH("hw.iommu.amd.ir_num", &nentries);
377 nentries = roundup_pow_of_two(nentries);
378 if (nentries < 1)
379 nentries = 1;
380 if (nentries > 2048)
381 nentries = 2048;
382 unit->irte_nentries = nentries;
383
384 unit->irte_x2apic = x2apic_mode;
385 return (0);
386 }
387
388 void
amdiommu_fini_irt(struct amdiommu_unit * unit)389 amdiommu_fini_irt(struct amdiommu_unit *unit)
390 {
391 }
392