1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause
3 *
4 * Copyright (c) 2024 The FreeBSD Foundation
5 *
6 * This software was developed by Konstantin Belousov <kib@FreeBSD.org>
7 * under sponsorship from the FreeBSD Foundation.
8 *
9 * Redistribution and use in source and binary forms, with or without
10 * modification, are permitted provided that the following conditions
11 * are met:
12 * 1. Redistributions of source code must retain the above copyright
13 * notice, this list of conditions and the following disclaimer.
14 * 2. Redistributions in binary form must reproduce the above copyright
15 * notice, this list of conditions and the following disclaimer in the
16 * documentation and/or other materials provided with the distribution.
17 *
18 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
19 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
20 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
21 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
22 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
23 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
24 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
25 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
26 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
27 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
28 * SUCH DAMAGE.
29 */
30
31 #include "opt_acpi.h"
32
33 #include <sys/param.h>
34 #include <sys/bus.h>
35 #include <sys/kernel.h>
36 #include <sys/malloc.h>
37 #include <sys/memdesc.h>
38 #include <sys/module.h>
39 #include <sys/rman.h>
40 #include <sys/taskqueue.h>
41 #include <sys/time.h>
42 #include <sys/tree.h>
43 #include <sys/vmem.h>
44 #include <vm/vm.h>
45 #include <vm/vm_extern.h>
46 #include <vm/vm_kern.h>
47 #include <vm/vm_page.h>
48 #include <vm/vm_map.h>
49 #include <contrib/dev/acpica/include/acpi.h>
50 #include <contrib/dev/acpica/include/accommon.h>
51 #include <dev/acpica/acpivar.h>
52 #include <dev/pci/pcireg.h>
53 #include <machine/bus.h>
54 #include <machine/cpu.h>
55 #include <x86/include/busdma_impl.h>
56 #include <dev/iommu/busdma_iommu.h>
57 #include <x86/iommu/amd_reg.h>
58 #include <x86/iommu/x86_iommu.h>
59 #include <x86/iommu/amd_iommu.h>
60
61 static void
amdiommu_enable_cmdbuf(struct amdiommu_unit * unit)62 amdiommu_enable_cmdbuf(struct amdiommu_unit *unit)
63 {
64 AMDIOMMU_ASSERT_LOCKED(unit);
65
66 unit->hw_ctrl |= AMDIOMMU_CTRL_CMDBUF_EN;
67 amdiommu_write8(unit, AMDIOMMU_CTRL, unit->hw_ctrl);
68 }
69
70 static void
amdiommu_disable_cmdbuf(struct amdiommu_unit * unit)71 amdiommu_disable_cmdbuf(struct amdiommu_unit *unit)
72 {
73 AMDIOMMU_ASSERT_LOCKED(unit);
74
75 unit->hw_ctrl &= ~AMDIOMMU_CTRL_CMDBUF_EN;
76 amdiommu_write8(unit, AMDIOMMU_CTRL, unit->hw_ctrl);
77 }
78
79
80 static void
amdiommu_enable_qi_intr(struct iommu_unit * iommu)81 amdiommu_enable_qi_intr(struct iommu_unit *iommu)
82 {
83 struct amdiommu_unit *unit;
84
85 unit = IOMMU2AMD(iommu);
86 AMDIOMMU_ASSERT_LOCKED(unit);
87 unit->hw_ctrl |= AMDIOMMU_CTRL_COMWINT_EN;
88 amdiommu_write8(unit, AMDIOMMU_CTRL, unit->hw_ctrl);
89 amdiommu_write8(unit, AMDIOMMU_CMDEV_STATUS,
90 AMDIOMMU_CMDEVS_COMWAITINT);
91 }
92
93 static void
amdiommu_disable_qi_intr(struct iommu_unit * iommu)94 amdiommu_disable_qi_intr(struct iommu_unit *iommu)
95 {
96 struct amdiommu_unit *unit;
97
98 unit = IOMMU2AMD(iommu);
99 AMDIOMMU_ASSERT_LOCKED(unit);
100 unit->hw_ctrl &= ~AMDIOMMU_CTRL_COMWINT_EN;
101 amdiommu_write8(unit, AMDIOMMU_CTRL, unit->hw_ctrl);
102 }
103
104 static void
amdiommu_cmd_advance_tail(struct iommu_unit * iommu)105 amdiommu_cmd_advance_tail(struct iommu_unit *iommu)
106 {
107 struct amdiommu_unit *unit;
108
109 unit = IOMMU2AMD(iommu);
110 AMDIOMMU_ASSERT_LOCKED(unit);
111 amdiommu_write8(unit, AMDIOMMU_CMDBUF_TAIL, unit->x86c.inv_queue_tail);
112 }
113
114 static void
amdiommu_cmd_ensure(struct iommu_unit * iommu,int descr_count)115 amdiommu_cmd_ensure(struct iommu_unit *iommu, int descr_count)
116 {
117 struct amdiommu_unit *unit;
118 uint64_t head;
119 int bytes;
120
121 unit = IOMMU2AMD(iommu);
122 AMDIOMMU_ASSERT_LOCKED(unit);
123 bytes = descr_count << AMDIOMMU_CMD_SZ_SHIFT;
124 for (;;) {
125 if (bytes <= unit->x86c.inv_queue_avail)
126 break;
127 /* refill */
128 head = amdiommu_read8(unit, AMDIOMMU_CMDBUF_HEAD);
129 head &= AMDIOMMU_CMDPTR_MASK;
130 unit->x86c.inv_queue_avail = head - unit->x86c.inv_queue_tail -
131 AMDIOMMU_CMD_SZ;
132 if (head <= unit->x86c.inv_queue_tail)
133 unit->x86c.inv_queue_avail += unit->x86c.inv_queue_size;
134 if (bytes <= unit->x86c.inv_queue_avail)
135 break;
136
137 /*
138 * No space in the queue, do busy wait. Hardware must
139 * make a progress. But first advance the tail to
140 * inform the descriptor streamer about entries we
141 * might have already filled, otherwise they could
142 * clog the whole queue..
143 *
144 * See dmar_qi_invalidate_locked() for a discussion
145 * about data race prevention.
146 */
147 amdiommu_cmd_advance_tail(iommu);
148 unit->x86c.inv_queue_full++;
149 cpu_spinwait();
150 }
151 unit->x86c.inv_queue_avail -= bytes;
152 }
153
154 static void
amdiommu_cmd_emit(struct amdiommu_unit * unit,const struct amdiommu_cmd_generic * cmd)155 amdiommu_cmd_emit(struct amdiommu_unit *unit, const struct
156 amdiommu_cmd_generic *cmd)
157 {
158 AMDIOMMU_ASSERT_LOCKED(unit);
159
160 memcpy(unit->x86c.inv_queue + unit->x86c.inv_queue_tail, cmd,
161 sizeof(*cmd));
162 unit->x86c.inv_queue_tail += AMDIOMMU_CMD_SZ;
163 KASSERT(unit->x86c.inv_queue_tail <= unit->x86c.inv_queue_size,
164 ("tail overflow 0x%x 0x%jx", unit->x86c.inv_queue_tail,
165 (uintmax_t)unit->x86c.inv_queue_size));
166 unit->x86c.inv_queue_tail &= unit->x86c.inv_queue_size - 1;
167 }
168
169 static void
amdiommu_cmd_emit_wait_descr(struct iommu_unit * iommu,uint32_t seq,bool intr,bool memw,bool fence)170 amdiommu_cmd_emit_wait_descr(struct iommu_unit *iommu, uint32_t seq,
171 bool intr, bool memw, bool fence)
172 {
173 struct amdiommu_unit *unit;
174 struct amdiommu_cmd_completion_wait c;
175
176 unit = IOMMU2AMD(iommu);
177 AMDIOMMU_ASSERT_LOCKED(unit);
178
179 bzero(&c, sizeof(c));
180 c.op = AMDIOMMU_CMD_COMPLETION_WAIT;
181 if (memw) {
182 uint32_t x;
183
184 c.s = 1;
185 x = unit->x86c.inv_waitd_seq_hw_phys;
186 x >>= 3;
187 c.address0 = x;
188 x = unit->x86c.inv_waitd_seq_hw_phys >> 32;
189 c.address1 = x;
190 c.data0 = seq;
191 }
192 if (fence)
193 c.f = 1;
194 if (intr)
195 c.i = 1;
196 amdiommu_cmd_emit(unit, (struct amdiommu_cmd_generic *)&c);
197 }
198
199 static void
amdiommu_qi_invalidate_emit(struct iommu_domain * adomain,iommu_gaddr_t base,iommu_gaddr_t size,struct iommu_qi_genseq * pseq,bool emit_wait)200 amdiommu_qi_invalidate_emit(struct iommu_domain *adomain, iommu_gaddr_t base,
201 iommu_gaddr_t size, struct iommu_qi_genseq *pseq, bool emit_wait)
202 {
203 struct amdiommu_domain *domain;
204 struct amdiommu_unit *unit;
205 struct amdiommu_cmd_invalidate_iommu_pages c;
206 u_int isize;
207
208 domain = IODOM2DOM(adomain);
209 unit = domain->unit;
210 AMDIOMMU_ASSERT_LOCKED(unit);
211 bzero(&c, sizeof(c));
212 c.op = AMDIOMMU_CMD_INVALIDATE_IOMMU_PAGES;
213 c.domainid = domain->domain;
214 isize = IOMMU_PAGE_SIZE; /* XXXKIB handle superpages */
215
216 for (; size > 0; base += isize, size -= isize) {
217 amdiommu_cmd_ensure(AMD2IOMMU(unit), 1);
218 c.s = 0;
219 c.pde = 1;
220 c.address = base >> IOMMU_PAGE_SHIFT;
221 amdiommu_cmd_emit(unit, (struct amdiommu_cmd_generic *)&c);
222 }
223 iommu_qi_emit_wait_seq(AMD2IOMMU(unit), pseq, emit_wait);
224 }
225
226 void
amdiommu_qi_invalidate_all_pages_locked_nowait(struct amdiommu_domain * domain)227 amdiommu_qi_invalidate_all_pages_locked_nowait(struct amdiommu_domain *domain)
228 {
229 struct amdiommu_unit *unit;
230 struct amdiommu_cmd_invalidate_iommu_pages c;
231
232 unit = domain->unit;
233 AMDIOMMU_ASSERT_LOCKED(unit);
234 bzero(&c, sizeof(c));
235 c.op = AMDIOMMU_CMD_INVALIDATE_IOMMU_PAGES;
236 c.domainid = domain->domain;
237
238 /*
239 * The magic specified in the note for INVALIDATE_IOMMU_PAGES
240 * description.
241 */
242 c.s = 1;
243 c.pde = 1;
244 c.address = 0x7ffffffffffff;
245
246 amdiommu_cmd_ensure(AMD2IOMMU(unit), 1);
247 amdiommu_cmd_emit(unit, (struct amdiommu_cmd_generic *)&c);
248 }
249
250 void
amdiommu_qi_invalidate_wait_sync(struct iommu_unit * iommu)251 amdiommu_qi_invalidate_wait_sync(struct iommu_unit *iommu)
252 {
253 struct iommu_qi_genseq gseq;
254
255 amdiommu_cmd_ensure(iommu, 1);
256 iommu_qi_emit_wait_seq(iommu, &gseq, true);
257 IOMMU2AMD(iommu)->x86c.inv_seq_waiters++;
258 amdiommu_cmd_advance_tail(iommu);
259 iommu_qi_wait_for_seq(iommu, &gseq, true);
260 }
261
262 void
amdiommu_qi_invalidate_ctx_locked_nowait(struct amdiommu_ctx * ctx)263 amdiommu_qi_invalidate_ctx_locked_nowait(struct amdiommu_ctx *ctx)
264 {
265 struct amdiommu_cmd_invalidate_devtab_entry c;
266
267 amdiommu_cmd_ensure(AMD2IOMMU(CTX2AMD(ctx)), 1);
268 bzero(&c, sizeof(c));
269 c.op = AMDIOMMU_CMD_INVALIDATE_DEVTAB_ENTRY;
270 c.devid = ctx->context.rid;
271 amdiommu_cmd_emit(CTX2AMD(ctx), (struct amdiommu_cmd_generic *)&c);
272 }
273
274
275 void
amdiommu_qi_invalidate_ctx_locked(struct amdiommu_ctx * ctx)276 amdiommu_qi_invalidate_ctx_locked(struct amdiommu_ctx *ctx)
277 {
278 amdiommu_qi_invalidate_ctx_locked_nowait(ctx);
279 amdiommu_qi_invalidate_wait_sync(AMD2IOMMU(CTX2AMD(ctx)));
280 }
281
282 void
amdiommu_qi_invalidate_ir_locked_nowait(struct amdiommu_unit * unit,uint16_t devid)283 amdiommu_qi_invalidate_ir_locked_nowait(struct amdiommu_unit *unit,
284 uint16_t devid)
285 {
286 struct amdiommu_cmd_invalidate_interrupt_table c;
287
288 AMDIOMMU_ASSERT_LOCKED(unit);
289
290 amdiommu_cmd_ensure(AMD2IOMMU(unit), 1);
291 bzero(&c, sizeof(c));
292 c.op = AMDIOMMU_CMD_INVALIDATE_INTERRUPT_TABLE;
293 c.devid = devid;
294 amdiommu_cmd_emit(unit, (struct amdiommu_cmd_generic *)&c);
295 }
296
297 void
amdiommu_qi_invalidate_ir_locked(struct amdiommu_unit * unit,uint16_t devid)298 amdiommu_qi_invalidate_ir_locked(struct amdiommu_unit *unit, uint16_t devid)
299 {
300 amdiommu_qi_invalidate_ir_locked_nowait(unit, devid);
301 amdiommu_qi_invalidate_wait_sync(AMD2IOMMU(unit));
302 }
303
304 static void
amdiommu_qi_task(void * arg,int pending __unused)305 amdiommu_qi_task(void *arg, int pending __unused)
306 {
307 struct amdiommu_unit *unit;
308
309 unit = IOMMU2AMD(arg);
310 iommu_qi_drain_tlb_flush(AMD2IOMMU(unit));
311
312 AMDIOMMU_LOCK(unit);
313 if (unit->x86c.inv_seq_waiters > 0)
314 wakeup(&unit->x86c.inv_seq_waiters);
315 AMDIOMMU_UNLOCK(unit);
316 }
317
318 int
amdiommu_init_cmd(struct amdiommu_unit * unit)319 amdiommu_init_cmd(struct amdiommu_unit *unit)
320 {
321 uint64_t qi_sz, rv;
322
323 unit->x86c.qi_buf_maxsz = ilog2(AMDIOMMU_CMDBUF_MAX / PAGE_SIZE);
324 unit->x86c.qi_cmd_sz = AMDIOMMU_CMD_SZ;
325 iommu_qi_common_init(AMD2IOMMU(unit), amdiommu_qi_task);
326 get_x86_iommu()->qi_ensure = amdiommu_cmd_ensure;
327 get_x86_iommu()->qi_emit_wait_descr = amdiommu_cmd_emit_wait_descr;
328 get_x86_iommu()->qi_advance_tail = amdiommu_cmd_advance_tail;
329 get_x86_iommu()->qi_invalidate_emit = amdiommu_qi_invalidate_emit;
330
331 rv = pmap_kextract((uintptr_t)unit->x86c.inv_queue);
332
333 /*
334 * See the description of the ComLen encoding for Command
335 * buffer Base Address Register.
336 */
337 qi_sz = ilog2(unit->x86c.inv_queue_size / PAGE_SIZE) + 8;
338 rv |= qi_sz << AMDIOMMU_CMDBUF_BASE_SZSHIFT;
339
340 AMDIOMMU_LOCK(unit);
341 amdiommu_write8(unit, AMDIOMMU_CMDBUF_BASE, rv);
342 amdiommu_enable_cmdbuf(unit);
343 amdiommu_enable_qi_intr(AMD2IOMMU(unit));
344 AMDIOMMU_UNLOCK(unit);
345
346 return (0);
347 }
348
349 static void
amdiommu_fini_cmd_helper(struct iommu_unit * iommu)350 amdiommu_fini_cmd_helper(struct iommu_unit *iommu)
351 {
352 amdiommu_disable_cmdbuf(IOMMU2AMD(iommu));
353 amdiommu_disable_qi_intr(iommu);
354 }
355
356 void
amdiommu_fini_cmd(struct amdiommu_unit * unit)357 amdiommu_fini_cmd(struct amdiommu_unit *unit)
358 {
359 iommu_qi_common_fini(AMD2IOMMU(unit), amdiommu_fini_cmd_helper);
360 }
361