xref: /freebsd/sys/x86/iommu/intel_qi.c (revision 8554754c7bbe3cda171378ad6305bbe8a1a94cb6)
1 /*-
2  * Copyright (c) 2013 The FreeBSD Foundation
3  * All rights reserved.
4  *
5  * This software was developed by Konstantin Belousov <kib@FreeBSD.org>
6  * under sponsorship from the FreeBSD Foundation.
7  *
8  * Redistribution and use in source and binary forms, with or without
9  * modification, are permitted provided that the following conditions
10  * are met:
11  * 1. Redistributions of source code must retain the above copyright
12  *    notice, this list of conditions and the following disclaimer.
13  * 2. Redistributions in binary form must reproduce the above copyright
14  *    notice, this list of conditions and the following disclaimer in the
15  *    documentation and/or other materials provided with the distribution.
16  *
17  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
18  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
19  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
20  * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
21  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
22  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
23  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
24  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
26  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
27  * SUCH DAMAGE.
28  */
29 
30 #include <sys/cdefs.h>
31 __FBSDID("$FreeBSD$");
32 
33 #include "opt_acpi.h"
34 
35 #include <sys/param.h>
36 #include <sys/bus.h>
37 #include <sys/kernel.h>
38 #include <sys/malloc.h>
39 #include <sys/memdesc.h>
40 #include <sys/module.h>
41 #include <sys/rman.h>
42 #include <sys/taskqueue.h>
43 #include <sys/time.h>
44 #include <sys/tree.h>
45 #include <sys/vmem.h>
46 #include <machine/bus.h>
47 #include <contrib/dev/acpica/include/acpi.h>
48 #include <contrib/dev/acpica/include/accommon.h>
49 #include <dev/acpica/acpivar.h>
50 #include <vm/vm.h>
51 #include <vm/vm_extern.h>
52 #include <vm/vm_kern.h>
53 #include <vm/vm_page.h>
54 #include <vm/vm_map.h>
55 #include <machine/cpu.h>
56 #include <x86/include/busdma_impl.h>
57 #include <x86/iommu/intel_reg.h>
58 #include <x86/iommu/busdma_dmar.h>
59 #include <x86/iommu/intel_dmar.h>
60 
61 static bool
62 dmar_qi_seq_processed(const struct dmar_unit *unit,
63     const struct dmar_qi_genseq *pseq)
64 {
65 
66 	return (pseq->gen < unit->inv_waitd_gen ||
67 	    (pseq->gen == unit->inv_waitd_gen &&
68 	     pseq->seq <= unit->inv_waitd_seq_hw));
69 }
70 
71 static int
72 dmar_enable_qi(struct dmar_unit *unit)
73 {
74 	int error;
75 
76 	DMAR_ASSERT_LOCKED(unit);
77 	unit->hw_gcmd |= DMAR_GCMD_QIE;
78 	dmar_write4(unit, DMAR_GCMD_REG, unit->hw_gcmd);
79 	DMAR_WAIT_UNTIL(((dmar_read4(unit, DMAR_GSTS_REG) & DMAR_GSTS_QIES)
80 	    != 0));
81 	return (error);
82 }
83 
84 static int
85 dmar_disable_qi(struct dmar_unit *unit)
86 {
87 	int error;
88 
89 	DMAR_ASSERT_LOCKED(unit);
90 	unit->hw_gcmd &= ~DMAR_GCMD_QIE;
91 	dmar_write4(unit, DMAR_GCMD_REG, unit->hw_gcmd);
92 	DMAR_WAIT_UNTIL(((dmar_read4(unit, DMAR_GSTS_REG) & DMAR_GSTS_QIES)
93 	    == 0));
94 	return (error);
95 }
96 
97 static void
98 dmar_qi_advance_tail(struct dmar_unit *unit)
99 {
100 
101 	DMAR_ASSERT_LOCKED(unit);
102 	dmar_write4(unit, DMAR_IQT_REG, unit->inv_queue_tail);
103 }
104 
105 static void
106 dmar_qi_ensure(struct dmar_unit *unit, int descr_count)
107 {
108 	uint32_t head;
109 	int bytes;
110 
111 	DMAR_ASSERT_LOCKED(unit);
112 	bytes = descr_count << DMAR_IQ_DESCR_SZ_SHIFT;
113 	for (;;) {
114 		if (bytes <= unit->inv_queue_avail)
115 			break;
116 		/* refill */
117 		head = dmar_read4(unit, DMAR_IQH_REG);
118 		head &= DMAR_IQH_MASK;
119 		unit->inv_queue_avail = head - unit->inv_queue_tail -
120 		    DMAR_IQ_DESCR_SZ;
121 		if (head <= unit->inv_queue_tail)
122 			unit->inv_queue_avail += unit->inv_queue_size;
123 		if (bytes <= unit->inv_queue_avail)
124 			break;
125 
126 		/*
127 		 * No space in the queue, do busy wait.  Hardware must
128 		 * make a progress.  But first advance the tail to
129 		 * inform the descriptor streamer about entries we
130 		 * might have already filled, otherwise they could
131 		 * clog the whole queue..
132 		 */
133 		dmar_qi_advance_tail(unit);
134 		unit->inv_queue_full++;
135 		cpu_spinwait();
136 	}
137 	unit->inv_queue_avail -= bytes;
138 }
139 
140 static void
141 dmar_qi_emit(struct dmar_unit *unit, uint64_t data1, uint64_t data2)
142 {
143 
144 	DMAR_ASSERT_LOCKED(unit);
145 	*(volatile uint64_t *)(unit->inv_queue + unit->inv_queue_tail) = data1;
146 	unit->inv_queue_tail += DMAR_IQ_DESCR_SZ / 2;
147 	KASSERT(unit->inv_queue_tail <= unit->inv_queue_size,
148 	    ("tail overflow 0x%x 0x%jx", unit->inv_queue_tail,
149 	    (uintmax_t)unit->inv_queue_size));
150 	unit->inv_queue_tail &= unit->inv_queue_size - 1;
151 	*(volatile uint64_t *)(unit->inv_queue + unit->inv_queue_tail) = data2;
152 	unit->inv_queue_tail += DMAR_IQ_DESCR_SZ / 2;
153 	KASSERT(unit->inv_queue_tail <= unit->inv_queue_size,
154 	    ("tail overflow 0x%x 0x%jx", unit->inv_queue_tail,
155 	    (uintmax_t)unit->inv_queue_size));
156 	unit->inv_queue_tail &= unit->inv_queue_size - 1;
157 }
158 
159 static void
160 dmar_qi_emit_wait_descr(struct dmar_unit *unit, uint32_t seq, bool intr,
161     bool memw, bool fence)
162 {
163 
164 	DMAR_ASSERT_LOCKED(unit);
165 	dmar_qi_emit(unit, DMAR_IQ_DESCR_WAIT_ID |
166 	    (intr ? DMAR_IQ_DESCR_WAIT_IF : 0) |
167 	    (memw ? DMAR_IQ_DESCR_WAIT_SW : 0) |
168 	    (fence ? DMAR_IQ_DESCR_WAIT_FN : 0) |
169 	    (memw ? DMAR_IQ_DESCR_WAIT_SD(seq) : 0),
170 	    memw ? unit->inv_waitd_seq_hw_phys : 0);
171 }
172 
173 static void
174 dmar_qi_emit_wait_seq(struct dmar_unit *unit, struct dmar_qi_genseq *pseq)
175 {
176 	struct dmar_qi_genseq gsec;
177 	uint32_t seq;
178 
179 	KASSERT(pseq != NULL, ("wait descriptor with no place for seq"));
180 	DMAR_ASSERT_LOCKED(unit);
181 	if (unit->inv_waitd_seq == 0xffffffff) {
182 		gsec.gen = unit->inv_waitd_gen;
183 		gsec.seq = unit->inv_waitd_seq;
184 		dmar_qi_ensure(unit, 1);
185 		dmar_qi_emit_wait_descr(unit, gsec.seq, false, true, false);
186 		dmar_qi_advance_tail(unit);
187 		while (!dmar_qi_seq_processed(unit, &gsec))
188 			cpu_spinwait();
189 		unit->inv_waitd_gen++;
190 		unit->inv_waitd_seq = 1;
191 	}
192 	seq = unit->inv_waitd_seq++;
193 	pseq->gen = unit->inv_waitd_gen;
194 	pseq->seq = seq;
195 	dmar_qi_emit_wait_descr(unit, seq, true, true, false);
196 }
197 
198 static void
199 dmar_qi_wait_for_seq(struct dmar_unit *unit, const struct dmar_qi_genseq *gseq,
200     bool nowait)
201 {
202 
203 	DMAR_ASSERT_LOCKED(unit);
204 	unit->inv_seq_waiters++;
205 	while (!dmar_qi_seq_processed(unit, gseq)) {
206 		if (cold || nowait) {
207 			cpu_spinwait();
208 		} else {
209 			msleep(&unit->inv_seq_waiters, &unit->lock, 0,
210 			    "dmarse", hz);
211 		}
212 	}
213 	unit->inv_seq_waiters--;
214 }
215 
216 void
217 dmar_qi_invalidate_locked(struct dmar_domain *domain, dmar_gaddr_t base,
218     dmar_gaddr_t size, struct dmar_qi_genseq *pseq)
219 {
220 	struct dmar_unit *unit;
221 	dmar_gaddr_t isize;
222 	int am;
223 
224 	unit = domain->dmar;
225 	DMAR_ASSERT_LOCKED(unit);
226 	for (; size > 0; base += isize, size -= isize) {
227 		am = calc_am(unit, base, size, &isize);
228 		dmar_qi_ensure(unit, 1);
229 		dmar_qi_emit(unit, DMAR_IQ_DESCR_IOTLB_INV |
230 		    DMAR_IQ_DESCR_IOTLB_PAGE | DMAR_IQ_DESCR_IOTLB_DW |
231 		    DMAR_IQ_DESCR_IOTLB_DR |
232 		    DMAR_IQ_DESCR_IOTLB_DID(domain->domain),
233 		    base | am);
234 	}
235 	if (pseq != NULL) {
236 		dmar_qi_ensure(unit, 1);
237 		dmar_qi_emit_wait_seq(unit, pseq);
238 	}
239 	dmar_qi_advance_tail(unit);
240 }
241 
242 void
243 dmar_qi_invalidate_ctx_glob_locked(struct dmar_unit *unit)
244 {
245 	struct dmar_qi_genseq gseq;
246 
247 	DMAR_ASSERT_LOCKED(unit);
248 	dmar_qi_ensure(unit, 2);
249 	dmar_qi_emit(unit, DMAR_IQ_DESCR_CTX_INV | DMAR_IQ_DESCR_CTX_GLOB, 0);
250 	dmar_qi_emit_wait_seq(unit, &gseq);
251 	dmar_qi_advance_tail(unit);
252 	dmar_qi_wait_for_seq(unit, &gseq, false);
253 }
254 
255 void
256 dmar_qi_invalidate_iotlb_glob_locked(struct dmar_unit *unit)
257 {
258 	struct dmar_qi_genseq gseq;
259 
260 	DMAR_ASSERT_LOCKED(unit);
261 	dmar_qi_ensure(unit, 2);
262 	dmar_qi_emit(unit, DMAR_IQ_DESCR_IOTLB_INV | DMAR_IQ_DESCR_IOTLB_GLOB |
263 	    DMAR_IQ_DESCR_IOTLB_DW | DMAR_IQ_DESCR_IOTLB_DR, 0);
264 	dmar_qi_emit_wait_seq(unit, &gseq);
265 	dmar_qi_advance_tail(unit);
266 	dmar_qi_wait_for_seq(unit, &gseq, false);
267 }
268 
269 void
270 dmar_qi_invalidate_iec_glob(struct dmar_unit *unit)
271 {
272 	struct dmar_qi_genseq gseq;
273 
274 	DMAR_ASSERT_LOCKED(unit);
275 	dmar_qi_ensure(unit, 2);
276 	dmar_qi_emit(unit, DMAR_IQ_DESCR_IEC_INV, 0);
277 	dmar_qi_emit_wait_seq(unit, &gseq);
278 	dmar_qi_advance_tail(unit);
279 	dmar_qi_wait_for_seq(unit, &gseq, false);
280 }
281 
282 void
283 dmar_qi_invalidate_iec(struct dmar_unit *unit, u_int start, u_int cnt)
284 {
285 	struct dmar_qi_genseq gseq;
286 	u_int c, l;
287 
288 	DMAR_ASSERT_LOCKED(unit);
289 	KASSERT(start < unit->irte_cnt && start < start + cnt &&
290 	    start + cnt <= unit->irte_cnt,
291 	    ("inv iec overflow %d %d %d", unit->irte_cnt, start, cnt));
292 	for (; cnt > 0; cnt -= c, start += c) {
293 		l = ffs(start | cnt) - 1;
294 		c = 1 << l;
295 		dmar_qi_ensure(unit, 1);
296 		dmar_qi_emit(unit, DMAR_IQ_DESCR_IEC_INV |
297 		    DMAR_IQ_DESCR_IEC_IDX | DMAR_IQ_DESCR_IEC_IIDX(start) |
298 		    DMAR_IQ_DESCR_IEC_IM(l), 0);
299 	}
300 	dmar_qi_ensure(unit, 1);
301 	dmar_qi_emit_wait_seq(unit, &gseq);
302 	dmar_qi_advance_tail(unit);
303 
304 	/*
305 	 * The caller of the function, in particular,
306 	 * dmar_ir_program_irte(), may be called from the context
307 	 * where the sleeping is forbidden (in fact, the
308 	 * intr_table_lock mutex may be held, locked from
309 	 * intr_shuffle_irqs()).  Wait for the invalidation completion
310 	 * using the busy wait.
311 	 *
312 	 * The impact on the interrupt input setup code is small, the
313 	 * expected overhead is comparable with the chipset register
314 	 * read.  It is more harmful for the parallel DMA operations,
315 	 * since we own the dmar unit lock until whole invalidation
316 	 * queue is processed, which includes requests possibly issued
317 	 * before our request.
318 	 */
319 	dmar_qi_wait_for_seq(unit, &gseq, true);
320 }
321 
322 int
323 dmar_qi_intr(void *arg)
324 {
325 	struct dmar_unit *unit;
326 
327 	unit = arg;
328 	KASSERT(unit->qi_enabled, ("dmar%d: QI is not enabled", unit->unit));
329 	taskqueue_enqueue(unit->qi_taskqueue, &unit->qi_task);
330 	return (FILTER_HANDLED);
331 }
332 
333 static void
334 dmar_qi_task(void *arg, int pending __unused)
335 {
336 	struct dmar_unit *unit;
337 	struct dmar_map_entry *entry;
338 	uint32_t ics;
339 
340 	unit = arg;
341 
342 	DMAR_LOCK(unit);
343 	for (;;) {
344 		entry = TAILQ_FIRST(&unit->tlb_flush_entries);
345 		if (entry == NULL)
346 			break;
347 		if ((entry->gseq.gen == 0 && entry->gseq.seq == 0) ||
348 		    !dmar_qi_seq_processed(unit, &entry->gseq))
349 			break;
350 		TAILQ_REMOVE(&unit->tlb_flush_entries, entry, dmamap_link);
351 		DMAR_UNLOCK(unit);
352 		dmar_domain_free_entry(entry, (entry->flags &
353 		    DMAR_MAP_ENTRY_QI_NF) == 0);
354 		DMAR_LOCK(unit);
355 	}
356 	ics = dmar_read4(unit, DMAR_ICS_REG);
357 	if ((ics & DMAR_ICS_IWC) != 0) {
358 		ics = DMAR_ICS_IWC;
359 		dmar_write4(unit, DMAR_ICS_REG, ics);
360 	}
361 	if (unit->inv_seq_waiters > 0)
362 		wakeup(&unit->inv_seq_waiters);
363 	DMAR_UNLOCK(unit);
364 }
365 
366 int
367 dmar_init_qi(struct dmar_unit *unit)
368 {
369 	uint64_t iqa;
370 	uint32_t ics;
371 	int qi_sz;
372 
373 	if (!DMAR_HAS_QI(unit) || (unit->hw_cap & DMAR_CAP_CM) != 0)
374 		return (0);
375 	unit->qi_enabled = 1;
376 	TUNABLE_INT_FETCH("hw.dmar.qi", &unit->qi_enabled);
377 	if (!unit->qi_enabled)
378 		return (0);
379 
380 	TAILQ_INIT(&unit->tlb_flush_entries);
381 	TASK_INIT(&unit->qi_task, 0, dmar_qi_task, unit);
382 	unit->qi_taskqueue = taskqueue_create_fast("dmarqf", M_WAITOK,
383 	    taskqueue_thread_enqueue, &unit->qi_taskqueue);
384 	taskqueue_start_threads(&unit->qi_taskqueue, 1, PI_AV,
385 	    "dmar%d qi taskq", unit->unit);
386 
387 	unit->inv_waitd_gen = 0;
388 	unit->inv_waitd_seq = 1;
389 
390 	qi_sz = DMAR_IQA_QS_DEF;
391 	TUNABLE_INT_FETCH("hw.dmar.qi_size", &qi_sz);
392 	if (qi_sz > DMAR_IQA_QS_MAX)
393 		qi_sz = DMAR_IQA_QS_MAX;
394 	unit->inv_queue_size = (1ULL << qi_sz) * PAGE_SIZE;
395 	/* Reserve one descriptor to prevent wraparound. */
396 	unit->inv_queue_avail = unit->inv_queue_size - DMAR_IQ_DESCR_SZ;
397 
398 	/* The invalidation queue reads by DMARs are always coherent. */
399 	unit->inv_queue = kmem_alloc_contig(kernel_arena, unit->inv_queue_size,
400 	    M_WAITOK | M_ZERO, 0, dmar_high, PAGE_SIZE, 0, VM_MEMATTR_DEFAULT);
401 	unit->inv_waitd_seq_hw_phys = pmap_kextract(
402 	    (vm_offset_t)&unit->inv_waitd_seq_hw);
403 
404 	DMAR_LOCK(unit);
405 	dmar_write8(unit, DMAR_IQT_REG, 0);
406 	iqa = pmap_kextract(unit->inv_queue);
407 	iqa |= qi_sz;
408 	dmar_write8(unit, DMAR_IQA_REG, iqa);
409 	dmar_enable_qi(unit);
410 	ics = dmar_read4(unit, DMAR_ICS_REG);
411 	if ((ics & DMAR_ICS_IWC) != 0) {
412 		ics = DMAR_ICS_IWC;
413 		dmar_write4(unit, DMAR_ICS_REG, ics);
414 	}
415 	dmar_enable_qi_intr(unit);
416 	DMAR_UNLOCK(unit);
417 
418 	return (0);
419 }
420 
421 void
422 dmar_fini_qi(struct dmar_unit *unit)
423 {
424 	struct dmar_qi_genseq gseq;
425 
426 	if (unit->qi_enabled)
427 		return;
428 	taskqueue_drain(unit->qi_taskqueue, &unit->qi_task);
429 	taskqueue_free(unit->qi_taskqueue);
430 	unit->qi_taskqueue = NULL;
431 
432 	DMAR_LOCK(unit);
433 	/* quisce */
434 	dmar_qi_ensure(unit, 1);
435 	dmar_qi_emit_wait_seq(unit, &gseq);
436 	dmar_qi_advance_tail(unit);
437 	dmar_qi_wait_for_seq(unit, &gseq, false);
438 	/* only after the quisce, disable queue */
439 	dmar_disable_qi_intr(unit);
440 	dmar_disable_qi(unit);
441 	KASSERT(unit->inv_seq_waiters == 0,
442 	    ("dmar%d: waiters on disabled queue", unit->unit));
443 	DMAR_UNLOCK(unit);
444 
445 	kmem_free(kernel_arena, unit->inv_queue, unit->inv_queue_size);
446 	unit->inv_queue = 0;
447 	unit->inv_queue_size = 0;
448 	unit->qi_enabled = 0;
449 }
450 
451 void
452 dmar_enable_qi_intr(struct dmar_unit *unit)
453 {
454 	uint32_t iectl;
455 
456 	DMAR_ASSERT_LOCKED(unit);
457 	KASSERT(DMAR_HAS_QI(unit), ("dmar%d: QI is not supported", unit->unit));
458 	iectl = dmar_read4(unit, DMAR_IECTL_REG);
459 	iectl &= ~DMAR_IECTL_IM;
460 	dmar_write4(unit, DMAR_IECTL_REG, iectl);
461 }
462 
463 void
464 dmar_disable_qi_intr(struct dmar_unit *unit)
465 {
466 	uint32_t iectl;
467 
468 	DMAR_ASSERT_LOCKED(unit);
469 	KASSERT(DMAR_HAS_QI(unit), ("dmar%d: QI is not supported", unit->unit));
470 	iectl = dmar_read4(unit, DMAR_IECTL_REG);
471 	dmar_write4(unit, DMAR_IECTL_REG, iectl | DMAR_IECTL_IM);
472 }
473