1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause
3 *
4 * Copyright (c) 2013 The FreeBSD Foundation
5 *
6 * This software was developed by Konstantin Belousov <kib@FreeBSD.org>
7 * under sponsorship from the FreeBSD Foundation.
8 *
9 * Redistribution and use in source and binary forms, with or without
10 * modification, are permitted provided that the following conditions
11 * are met:
12 * 1. Redistributions of source code must retain the above copyright
13 * notice, this list of conditions and the following disclaimer.
14 * 2. Redistributions in binary form must reproduce the above copyright
15 * notice, this list of conditions and the following disclaimer in the
16 * documentation and/or other materials provided with the distribution.
17 *
18 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
19 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
20 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
21 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
22 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
23 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
24 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
25 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
26 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
27 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
28 * SUCH DAMAGE.
29 */
30
31 #include "opt_acpi.h"
32
33 #include <sys/param.h>
34 #include <sys/bus.h>
35 #include <sys/kernel.h>
36 #include <sys/malloc.h>
37 #include <sys/memdesc.h>
38 #include <sys/module.h>
39 #include <sys/rman.h>
40 #include <sys/taskqueue.h>
41 #include <sys/time.h>
42 #include <sys/tree.h>
43 #include <sys/vmem.h>
44 #include <vm/vm.h>
45 #include <vm/vm_extern.h>
46 #include <vm/vm_kern.h>
47 #include <vm/vm_page.h>
48 #include <vm/vm_map.h>
49 #include <contrib/dev/acpica/include/acpi.h>
50 #include <contrib/dev/acpica/include/accommon.h>
51 #include <dev/acpica/acpivar.h>
52 #include <dev/pci/pcireg.h>
53 #include <machine/bus.h>
54 #include <machine/cpu.h>
55 #include <x86/include/busdma_impl.h>
56 #include <dev/iommu/busdma_iommu.h>
57 #include <x86/iommu/intel_reg.h>
58 #include <x86/iommu/x86_iommu.h>
59 #include <x86/iommu/intel_dmar.h>
60
61 static int
dmar_enable_qi(struct dmar_unit * unit)62 dmar_enable_qi(struct dmar_unit *unit)
63 {
64 int error;
65
66 DMAR_ASSERT_LOCKED(unit);
67 unit->hw_gcmd |= DMAR_GCMD_QIE;
68 dmar_write4(unit, DMAR_GCMD_REG, unit->hw_gcmd);
69 DMAR_WAIT_UNTIL(((dmar_read4(unit, DMAR_GSTS_REG) & DMAR_GSTS_QIES)
70 != 0));
71 return (error);
72 }
73
74 static int
dmar_disable_qi(struct dmar_unit * unit)75 dmar_disable_qi(struct dmar_unit *unit)
76 {
77 int error;
78
79 DMAR_ASSERT_LOCKED(unit);
80 unit->hw_gcmd &= ~DMAR_GCMD_QIE;
81 dmar_write4(unit, DMAR_GCMD_REG, unit->hw_gcmd);
82 DMAR_WAIT_UNTIL(((dmar_read4(unit, DMAR_GSTS_REG) & DMAR_GSTS_QIES)
83 == 0));
84 return (error);
85 }
86
87 static void
dmar_qi_advance_tail(struct iommu_unit * iommu)88 dmar_qi_advance_tail(struct iommu_unit *iommu)
89 {
90 struct dmar_unit *unit;
91
92 unit = IOMMU2DMAR(iommu);
93 DMAR_ASSERT_LOCKED(unit);
94 dmar_write4(unit, DMAR_IQT_REG, unit->x86c.inv_queue_tail);
95 }
96
97 static void
dmar_qi_ensure(struct iommu_unit * iommu,int descr_count)98 dmar_qi_ensure(struct iommu_unit *iommu, int descr_count)
99 {
100 struct dmar_unit *unit;
101 uint32_t head;
102 int bytes;
103
104 unit = IOMMU2DMAR(iommu);
105 DMAR_ASSERT_LOCKED(unit);
106 bytes = descr_count << DMAR_IQ_DESCR_SZ_SHIFT;
107 for (;;) {
108 if (bytes <= unit->x86c.inv_queue_avail)
109 break;
110 /* refill */
111 head = dmar_read4(unit, DMAR_IQH_REG);
112 head &= DMAR_IQH_MASK;
113 unit->x86c.inv_queue_avail = head - unit->x86c.inv_queue_tail -
114 DMAR_IQ_DESCR_SZ;
115 if (head <= unit->x86c.inv_queue_tail)
116 unit->x86c.inv_queue_avail += unit->x86c.inv_queue_size;
117 if (bytes <= unit->x86c.inv_queue_avail)
118 break;
119
120 /*
121 * No space in the queue, do busy wait. Hardware must
122 * make a progress. But first advance the tail to
123 * inform the descriptor streamer about entries we
124 * might have already filled, otherwise they could
125 * clog the whole queue..
126 *
127 * See dmar_qi_invalidate_locked() for a discussion
128 * about data race prevention.
129 */
130 dmar_qi_advance_tail(DMAR2IOMMU(unit));
131 unit->x86c.inv_queue_full++;
132 cpu_spinwait();
133 }
134 unit->x86c.inv_queue_avail -= bytes;
135 }
136
137 static void
dmar_qi_emit(struct dmar_unit * unit,uint64_t data1,uint64_t data2)138 dmar_qi_emit(struct dmar_unit *unit, uint64_t data1, uint64_t data2)
139 {
140
141 DMAR_ASSERT_LOCKED(unit);
142 #ifdef __LP64__
143 atomic_store_64((uint64_t *)(unit->x86c.inv_queue +
144 unit->x86c.inv_queue_tail), data1);
145 #else
146 *(volatile uint64_t *)(unit->x86c.inv_queue +
147 unit->x86c.inv_queue_tail) = data1;
148 #endif
149 unit->x86c.inv_queue_tail += DMAR_IQ_DESCR_SZ / 2;
150 KASSERT(unit->x86c.inv_queue_tail <= unit->x86c.inv_queue_size,
151 ("tail overflow 0x%x 0x%jx", unit->x86c.inv_queue_tail,
152 (uintmax_t)unit->x86c.inv_queue_size));
153 unit->x86c.inv_queue_tail &= unit->x86c.inv_queue_size - 1;
154 #ifdef __LP64__
155 atomic_store_64((uint64_t *)(unit->x86c.inv_queue +
156 unit->x86c.inv_queue_tail), data2);
157 #else
158 *(volatile uint64_t *)(unit->x86c.inv_queue +
159 unit->x86c.inv_queue_tail) = data2;
160 #endif
161 unit->x86c.inv_queue_tail += DMAR_IQ_DESCR_SZ / 2;
162 KASSERT(unit->x86c.inv_queue_tail <= unit->x86c.inv_queue_size,
163 ("tail overflow 0x%x 0x%jx", unit->x86c.inv_queue_tail,
164 (uintmax_t)unit->x86c.inv_queue_size));
165 unit->x86c.inv_queue_tail &= unit->x86c.inv_queue_size - 1;
166 }
167
168 static void
dmar_qi_emit_wait_descr(struct iommu_unit * iommu,uint32_t seq,bool intr,bool memw,bool fence)169 dmar_qi_emit_wait_descr(struct iommu_unit *iommu, uint32_t seq, bool intr,
170 bool memw, bool fence)
171 {
172 struct dmar_unit *unit;
173
174 unit = IOMMU2DMAR(iommu);
175 DMAR_ASSERT_LOCKED(unit);
176 dmar_qi_emit(unit, DMAR_IQ_DESCR_WAIT_ID |
177 (intr ? DMAR_IQ_DESCR_WAIT_IF : 0) |
178 (memw ? DMAR_IQ_DESCR_WAIT_SW : 0) |
179 (fence ? DMAR_IQ_DESCR_WAIT_FN : 0) |
180 (memw ? DMAR_IQ_DESCR_WAIT_SD(seq) : 0),
181 memw ? unit->x86c.inv_waitd_seq_hw_phys : 0);
182 }
183
184 static void
dmar_qi_invalidate_emit(struct iommu_domain * idomain,iommu_gaddr_t base,iommu_gaddr_t size,struct iommu_qi_genseq * pseq,bool emit_wait)185 dmar_qi_invalidate_emit(struct iommu_domain *idomain, iommu_gaddr_t base,
186 iommu_gaddr_t size, struct iommu_qi_genseq *pseq, bool emit_wait)
187 {
188 struct dmar_unit *unit;
189 struct dmar_domain *domain;
190 iommu_gaddr_t isize;
191 int am;
192
193 domain = __containerof(idomain, struct dmar_domain, iodom);
194 unit = domain->dmar;
195 DMAR_ASSERT_LOCKED(unit);
196 for (; size > 0; base += isize, size -= isize) {
197 am = calc_am(unit, base, size, &isize);
198 dmar_qi_ensure(DMAR2IOMMU(unit), 1);
199 dmar_qi_emit(unit, DMAR_IQ_DESCR_IOTLB_INV |
200 DMAR_IQ_DESCR_IOTLB_PAGE | DMAR_IQ_DESCR_IOTLB_DW |
201 DMAR_IQ_DESCR_IOTLB_DR |
202 DMAR_IQ_DESCR_IOTLB_DID(domain->domain),
203 base | am);
204 }
205 iommu_qi_emit_wait_seq(DMAR2IOMMU(unit), pseq, emit_wait);
206 }
207
208 static void
dmar_qi_invalidate_glob_impl(struct dmar_unit * unit,uint64_t data1)209 dmar_qi_invalidate_glob_impl(struct dmar_unit *unit, uint64_t data1)
210 {
211 struct iommu_qi_genseq gseq;
212
213 DMAR_ASSERT_LOCKED(unit);
214 dmar_qi_ensure(DMAR2IOMMU(unit), 2);
215 dmar_qi_emit(unit, data1, 0);
216 iommu_qi_emit_wait_seq(DMAR2IOMMU(unit), &gseq, true);
217 /* See dmar_qi_invalidate_sync(). */
218 unit->x86c.inv_seq_waiters++;
219 dmar_qi_advance_tail(DMAR2IOMMU(unit));
220 iommu_qi_wait_for_seq(DMAR2IOMMU(unit), &gseq, false);
221 }
222
223 void
dmar_qi_invalidate_ctx_glob_locked(struct dmar_unit * unit)224 dmar_qi_invalidate_ctx_glob_locked(struct dmar_unit *unit)
225 {
226 dmar_qi_invalidate_glob_impl(unit, DMAR_IQ_DESCR_CTX_INV |
227 DMAR_IQ_DESCR_CTX_GLOB);
228 }
229
230 void
dmar_qi_invalidate_iotlb_glob_locked(struct dmar_unit * unit)231 dmar_qi_invalidate_iotlb_glob_locked(struct dmar_unit *unit)
232 {
233 dmar_qi_invalidate_glob_impl(unit, DMAR_IQ_DESCR_IOTLB_INV |
234 DMAR_IQ_DESCR_IOTLB_GLOB | DMAR_IQ_DESCR_IOTLB_DW |
235 DMAR_IQ_DESCR_IOTLB_DR);
236 }
237
238 void
dmar_qi_invalidate_iec_glob(struct dmar_unit * unit)239 dmar_qi_invalidate_iec_glob(struct dmar_unit *unit)
240 {
241 dmar_qi_invalidate_glob_impl(unit, DMAR_IQ_DESCR_IEC_INV);
242 }
243
244 void
dmar_qi_invalidate_iec(struct dmar_unit * unit,u_int start,u_int cnt)245 dmar_qi_invalidate_iec(struct dmar_unit *unit, u_int start, u_int cnt)
246 {
247 struct iommu_qi_genseq gseq;
248 u_int c, l;
249
250 DMAR_ASSERT_LOCKED(unit);
251 KASSERT(start < unit->irte_cnt && start < start + cnt &&
252 start + cnt <= unit->irte_cnt,
253 ("inv iec overflow %d %d %d", unit->irte_cnt, start, cnt));
254 for (; cnt > 0; cnt -= c, start += c) {
255 l = ffs(start | cnt) - 1;
256 c = 1 << l;
257 dmar_qi_ensure(DMAR2IOMMU(unit), 1);
258 dmar_qi_emit(unit, DMAR_IQ_DESCR_IEC_INV |
259 DMAR_IQ_DESCR_IEC_IDX | DMAR_IQ_DESCR_IEC_IIDX(start) |
260 DMAR_IQ_DESCR_IEC_IM(l), 0);
261 }
262 dmar_qi_ensure(DMAR2IOMMU(unit), 1);
263 iommu_qi_emit_wait_seq(DMAR2IOMMU(unit), &gseq, true);
264
265 /*
266 * Since iommu_qi_wait_for_seq() will not sleep, this increment's
267 * placement relative to advancing the tail doesn't matter.
268 */
269 unit->x86c.inv_seq_waiters++;
270
271 dmar_qi_advance_tail(DMAR2IOMMU(unit));
272
273 /*
274 * The caller of the function, in particular,
275 * dmar_ir_program_irte(), may be called from the context
276 * where the sleeping is forbidden (in fact, the
277 * intr_table_lock mutex may be held, locked from
278 * intr_shuffle_irqs()). Wait for the invalidation completion
279 * using the busy wait.
280 *
281 * The impact on the interrupt input setup code is small, the
282 * expected overhead is comparable with the chipset register
283 * read. It is more harmful for the parallel DMA operations,
284 * since we own the dmar unit lock until whole invalidation
285 * queue is processed, which includes requests possibly issued
286 * before our request.
287 */
288 iommu_qi_wait_for_seq(DMAR2IOMMU(unit), &gseq, true);
289 }
290
291 int
dmar_qi_intr(void * arg)292 dmar_qi_intr(void *arg)
293 {
294 struct dmar_unit *unit;
295
296 unit = IOMMU2DMAR((struct iommu_unit *)arg);
297 KASSERT(unit->qi_enabled, ("dmar%d: QI is not enabled",
298 unit->iommu.unit));
299 taskqueue_enqueue(unit->x86c.qi_taskqueue, &unit->x86c.qi_task);
300 return (FILTER_HANDLED);
301 }
302
303 static void
dmar_qi_task(void * arg,int pending __unused)304 dmar_qi_task(void *arg, int pending __unused)
305 {
306 struct dmar_unit *unit;
307 uint32_t ics;
308
309 unit = IOMMU2DMAR(arg);
310 iommu_qi_drain_tlb_flush(DMAR2IOMMU(unit));
311
312 /*
313 * Request an interrupt on the completion of the next invalidation
314 * wait descriptor with the IF field set.
315 */
316 ics = dmar_read4(unit, DMAR_ICS_REG);
317 if ((ics & DMAR_ICS_IWC) != 0) {
318 ics = DMAR_ICS_IWC;
319 dmar_write4(unit, DMAR_ICS_REG, ics);
320
321 /*
322 * Drain a second time in case the DMAR processes an entry
323 * after the first call and before clearing DMAR_ICS_IWC.
324 * Otherwise, such entries will linger until a later entry
325 * that requests an interrupt is processed.
326 */
327 iommu_qi_drain_tlb_flush(DMAR2IOMMU(unit));
328 }
329
330 if (unit->x86c.inv_seq_waiters > 0) {
331 /*
332 * Acquire the DMAR lock so that wakeup() is called only after
333 * the waiter is sleeping.
334 */
335 DMAR_LOCK(unit);
336 wakeup(&unit->x86c.inv_seq_waiters);
337 DMAR_UNLOCK(unit);
338 }
339 }
340
341 int
dmar_init_qi(struct dmar_unit * unit)342 dmar_init_qi(struct dmar_unit *unit)
343 {
344 uint64_t iqa;
345 uint32_t ics;
346 u_int qi_sz;
347
348 if (!DMAR_HAS_QI(unit) || (unit->hw_cap & DMAR_CAP_CM) != 0)
349 return (0);
350 unit->qi_enabled = 1;
351 TUNABLE_INT_FETCH("hw.dmar.qi", &unit->qi_enabled);
352 if (!unit->qi_enabled)
353 return (0);
354
355 unit->x86c.qi_buf_maxsz = DMAR_IQA_QS_MAX;
356 unit->x86c.qi_cmd_sz = DMAR_IQ_DESCR_SZ;
357 iommu_qi_common_init(DMAR2IOMMU(unit), dmar_qi_task);
358 get_x86_iommu()->qi_ensure = dmar_qi_ensure;
359 get_x86_iommu()->qi_emit_wait_descr = dmar_qi_emit_wait_descr;
360 get_x86_iommu()->qi_advance_tail = dmar_qi_advance_tail;
361 get_x86_iommu()->qi_invalidate_emit = dmar_qi_invalidate_emit;
362
363 qi_sz = ilog2(unit->x86c.inv_queue_size / PAGE_SIZE);
364
365 DMAR_LOCK(unit);
366 dmar_write8(unit, DMAR_IQT_REG, 0);
367 iqa = pmap_kextract((uintptr_t)unit->x86c.inv_queue);
368 iqa |= qi_sz;
369 dmar_write8(unit, DMAR_IQA_REG, iqa);
370 dmar_enable_qi(unit);
371 ics = dmar_read4(unit, DMAR_ICS_REG);
372 if ((ics & DMAR_ICS_IWC) != 0) {
373 ics = DMAR_ICS_IWC;
374 dmar_write4(unit, DMAR_ICS_REG, ics);
375 }
376 dmar_enable_qi_intr(DMAR2IOMMU(unit));
377 DMAR_UNLOCK(unit);
378
379 return (0);
380 }
381
382 static void
dmar_fini_qi_helper(struct iommu_unit * iommu)383 dmar_fini_qi_helper(struct iommu_unit *iommu)
384 {
385 dmar_disable_qi_intr(iommu);
386 dmar_disable_qi(IOMMU2DMAR(iommu));
387 }
388
389 void
dmar_fini_qi(struct dmar_unit * unit)390 dmar_fini_qi(struct dmar_unit *unit)
391 {
392 if (!unit->qi_enabled)
393 return;
394 iommu_qi_common_fini(DMAR2IOMMU(unit), dmar_fini_qi_helper);
395 unit->qi_enabled = 0;
396 }
397
398 void
dmar_enable_qi_intr(struct iommu_unit * iommu)399 dmar_enable_qi_intr(struct iommu_unit *iommu)
400 {
401 struct dmar_unit *unit;
402 uint32_t iectl;
403
404 unit = IOMMU2DMAR(iommu);
405 DMAR_ASSERT_LOCKED(unit);
406 KASSERT(DMAR_HAS_QI(unit), ("dmar%d: QI is not supported",
407 unit->iommu.unit));
408 iectl = dmar_read4(unit, DMAR_IECTL_REG);
409 iectl &= ~DMAR_IECTL_IM;
410 dmar_write4(unit, DMAR_IECTL_REG, iectl);
411 }
412
413 void
dmar_disable_qi_intr(struct iommu_unit * iommu)414 dmar_disable_qi_intr(struct iommu_unit *iommu)
415 {
416 struct dmar_unit *unit;
417 uint32_t iectl;
418
419 unit = IOMMU2DMAR(iommu);
420 DMAR_ASSERT_LOCKED(unit);
421 KASSERT(DMAR_HAS_QI(unit), ("dmar%d: QI is not supported",
422 unit->iommu.unit));
423 iectl = dmar_read4(unit, DMAR_IECTL_REG);
424 dmar_write4(unit, DMAR_IECTL_REG, iectl | DMAR_IECTL_IM);
425 }
426