xref: /illumos-gate/usr/src/uts/i86pc/io/immu_qinv.c (revision 6a634c9dca3093f3922e4b7ab826d7bdf17bf78e)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 /*
22  * Portions Copyright (c) 2010, Oracle and/or its affiliates.
23  * All rights reserved.
24  */
25 
26 /*
27  * Copyright (c) 2009, Intel Corporation.
28  * All rights reserved.
29  */
30 
31 #include <sys/ddi.h>
32 #include <sys/archsystm.h>
33 #include <vm/hat_i86.h>
34 #include <sys/types.h>
35 #include <sys/cpu.h>
36 #include <sys/sysmacros.h>
37 #include <sys/immu.h>
38 
39 /* invalidation queue table entry size */
40 #define	QINV_ENTRY_SIZE		0x10
41 
42 /* max value of Queue Size field of Invalidation Queue Address Register */
43 #define	QINV_MAX_QUEUE_SIZE	0x7
44 
45 /* status data size of invalidation wait descriptor */
46 #define	QINV_SYNC_DATA_SIZE	0x4
47 
48 /* invalidation queue head and tail */
49 #define	QINV_IQA_HEAD(QH)	BITX((QH), 18, 4)
50 #define	QINV_IQA_TAIL_SHIFT	4
51 
52 /* invalidation queue entry structure */
53 typedef struct qinv_inv_dsc {
54 	uint64_t	lo;
55 	uint64_t	hi;
56 } qinv_dsc_t;
57 
58 /* physical contigous pages for invalidation queue */
59 typedef struct qinv_mem {
60 	kmutex_t	   qinv_mem_lock;
61 	ddi_dma_handle_t   qinv_mem_dma_hdl;
62 	ddi_acc_handle_t   qinv_mem_acc_hdl;
63 	caddr_t		   qinv_mem_vaddr;
64 	paddr_t		   qinv_mem_paddr;
65 	uint_t		   qinv_mem_size;
66 	uint16_t	   qinv_mem_head;
67 	uint16_t	   qinv_mem_tail;
68 } qinv_mem_t;
69 
70 
71 /*
72  * invalidation queue state
73  *   This structure describes the state information of the
74  *   invalidation queue table and related status memeory for
75  *   invalidation wait descriptor
76  *
77  * qinv_table		- invalidation queue table
78  * qinv_sync		- sync status memory for invalidation wait descriptor
79  */
80 typedef struct qinv {
81 	qinv_mem_t		qinv_table;
82 	qinv_mem_t		qinv_sync;
83 } qinv_t;
84 
85 static void immu_qinv_inv_wait(immu_inv_wait_t *iwp);
86 
87 static struct immu_flushops immu_qinv_flushops = {
88 	immu_qinv_context_fsi,
89 	immu_qinv_context_dsi,
90 	immu_qinv_context_gbl,
91 	immu_qinv_iotlb_psi,
92 	immu_qinv_iotlb_dsi,
93 	immu_qinv_iotlb_gbl,
94 	immu_qinv_inv_wait
95 };
96 
97 /* helper macro for making queue invalidation descriptor */
98 #define	INV_DSC_TYPE(dsc)	((dsc)->lo & 0xF)
99 #define	CC_INV_DSC_HIGH		(0)
100 #define	CC_INV_DSC_LOW(fm, sid, did, g)	(((uint64_t)(fm) << 48) | \
101 	((uint64_t)(sid) << 32) | \
102 	((uint64_t)(did) << 16) | \
103 	((uint64_t)(g) << 4) | \
104 	1)
105 
106 #define	IOTLB_INV_DSC_HIGH(addr, ih, am) (((uint64_t)(addr)) | \
107 	((uint64_t)(ih) << 6) |	\
108 	((uint64_t)(am)))
109 
110 #define	IOTLB_INV_DSC_LOW(did, dr, dw, g) (((uint64_t)(did) << 16) | \
111 	((uint64_t)(dr) << 7) | \
112 	((uint64_t)(dw) << 6) | \
113 	((uint64_t)(g) << 4) | \
114 	2)
115 
116 #define	DEV_IOTLB_INV_DSC_HIGH(addr, s) (((uint64_t)(addr)) | (s))
117 
118 #define	DEV_IOTLB_INV_DSC_LOW(sid, max_invs_pd) ( \
119 	((uint64_t)(sid) << 32) | \
120 	((uint64_t)(max_invs_pd) << 16) | \
121 	3)
122 
123 #define	IEC_INV_DSC_HIGH (0)
124 #define	IEC_INV_DSC_LOW(idx, im, g) (((uint64_t)(idx) << 32) | \
125 	((uint64_t)(im) << 27) | \
126 	((uint64_t)(g) << 4) | \
127 	4)
128 
129 #define	INV_WAIT_DSC_HIGH(saddr) ((uint64_t)(saddr))
130 
131 #define	INV_WAIT_DSC_LOW(sdata, fn, sw, iflag) (((uint64_t)(sdata) << 32) | \
132 	((uint64_t)(fn) << 6) | \
133 	((uint64_t)(sw) << 5) | \
134 	((uint64_t)(iflag) << 4) | \
135 	5)
136 
137 /*
138  * QS field of Invalidation Queue Address Register
139  * the size of invalidation queue is 1 << (qinv_iqa_qs + 8)
140  */
141 static uint_t qinv_iqa_qs = 6;
142 
143 /*
144  * the invalidate desctiptor type of queued invalidation interface
145  */
146 static char *qinv_dsc_type[] = {
147 	"Reserved",
148 	"Context Cache Invalidate Descriptor",
149 	"IOTLB Invalidate Descriptor",
150 	"Device-IOTLB Invalidate Descriptor",
151 	"Interrupt Entry Cache Invalidate Descriptor",
152 	"Invalidation Wait Descriptor",
153 	"Incorrect queue invalidation type"
154 };
155 
156 #define	QINV_MAX_DSC_TYPE	(sizeof (qinv_dsc_type) / sizeof (char *))
157 
158 /*
159  * the queued invalidation interface functions
160  */
161 static void qinv_submit_inv_dsc(immu_t *immu, qinv_dsc_t *dsc);
162 static void qinv_context_common(immu_t *immu, uint8_t function_mask,
163     uint16_t source_id, uint_t domain_id, ctt_inv_g_t type);
164 static void qinv_iotlb_common(immu_t *immu, uint_t domain_id,
165     uint64_t addr, uint_t am, uint_t hint, tlb_inv_g_t type);
166 static void qinv_iec_common(immu_t *immu, uint_t iidx,
167     uint_t im, uint_t g);
168 static void immu_qinv_inv_wait(immu_inv_wait_t *iwp);
169 static void qinv_wait_sync(immu_t *immu, immu_inv_wait_t *iwp);
170 /*LINTED*/
171 static void qinv_dev_iotlb_common(immu_t *immu, uint16_t sid,
172     uint64_t addr, uint_t size, uint_t max_invs_pd);
173 
174 
175 /* submit invalidation request descriptor to invalidation queue */
176 static void
qinv_submit_inv_dsc(immu_t * immu,qinv_dsc_t * dsc)177 qinv_submit_inv_dsc(immu_t *immu, qinv_dsc_t *dsc)
178 {
179 	qinv_t *qinv;
180 	qinv_mem_t *qinv_table;
181 	uint_t tail;
182 #ifdef DEBUG
183 	uint_t count = 0;
184 #endif
185 
186 	qinv = (qinv_t *)immu->immu_qinv;
187 	qinv_table = &(qinv->qinv_table);
188 
189 	mutex_enter(&qinv_table->qinv_mem_lock);
190 	tail = qinv_table->qinv_mem_tail;
191 	qinv_table->qinv_mem_tail++;
192 
193 	if (qinv_table->qinv_mem_tail == qinv_table->qinv_mem_size)
194 		qinv_table->qinv_mem_tail = 0;
195 
196 	while (qinv_table->qinv_mem_head == qinv_table->qinv_mem_tail) {
197 #ifdef DEBUG
198 		count++;
199 #endif
200 		/*
201 		 * inv queue table exhausted, wait hardware to fetch
202 		 * next descriptor
203 		 */
204 		qinv_table->qinv_mem_head = QINV_IQA_HEAD(
205 		    immu_regs_get64(immu, IMMU_REG_INVAL_QH));
206 	}
207 
208 	IMMU_DPROBE3(immu__qinv__sub, uint64_t, dsc->lo, uint64_t, dsc->hi,
209 	    uint_t, count);
210 
211 	bcopy(dsc, qinv_table->qinv_mem_vaddr + tail * QINV_ENTRY_SIZE,
212 	    QINV_ENTRY_SIZE);
213 
214 	immu_regs_put64(immu, IMMU_REG_INVAL_QT,
215 	    qinv_table->qinv_mem_tail << QINV_IQA_TAIL_SHIFT);
216 
217 	mutex_exit(&qinv_table->qinv_mem_lock);
218 }
219 
220 /* queued invalidation interface -- invalidate context cache */
221 static void
qinv_context_common(immu_t * immu,uint8_t function_mask,uint16_t source_id,uint_t domain_id,ctt_inv_g_t type)222 qinv_context_common(immu_t *immu, uint8_t function_mask,
223     uint16_t source_id, uint_t domain_id, ctt_inv_g_t type)
224 {
225 	qinv_dsc_t dsc;
226 
227 	dsc.lo = CC_INV_DSC_LOW(function_mask, source_id, domain_id, type);
228 	dsc.hi = CC_INV_DSC_HIGH;
229 
230 	qinv_submit_inv_dsc(immu, &dsc);
231 }
232 
233 /* queued invalidation interface -- invalidate iotlb */
234 static void
qinv_iotlb_common(immu_t * immu,uint_t domain_id,uint64_t addr,uint_t am,uint_t hint,tlb_inv_g_t type)235 qinv_iotlb_common(immu_t *immu, uint_t domain_id,
236     uint64_t addr, uint_t am, uint_t hint, tlb_inv_g_t type)
237 {
238 	qinv_dsc_t dsc;
239 	uint8_t dr = 0;
240 	uint8_t dw = 0;
241 
242 	if (IMMU_CAP_GET_DRD(immu->immu_regs_cap))
243 		dr = 1;
244 	if (IMMU_CAP_GET_DWD(immu->immu_regs_cap))
245 		dw = 1;
246 
247 	switch (type) {
248 	case TLB_INV_G_PAGE:
249 		if (!IMMU_CAP_GET_PSI(immu->immu_regs_cap) ||
250 		    am > IMMU_CAP_GET_MAMV(immu->immu_regs_cap) ||
251 		    addr & IMMU_PAGEOFFSET) {
252 			type = TLB_INV_G_DOMAIN;
253 			goto qinv_ignore_psi;
254 		}
255 		dsc.lo = IOTLB_INV_DSC_LOW(domain_id, dr, dw, type);
256 		dsc.hi = IOTLB_INV_DSC_HIGH(addr, hint, am);
257 		break;
258 
259 	qinv_ignore_psi:
260 	case TLB_INV_G_DOMAIN:
261 		dsc.lo = IOTLB_INV_DSC_LOW(domain_id, dr, dw, type);
262 		dsc.hi = 0;
263 		break;
264 
265 	case TLB_INV_G_GLOBAL:
266 		dsc.lo = IOTLB_INV_DSC_LOW(0, dr, dw, type);
267 		dsc.hi = 0;
268 		break;
269 	default:
270 		ddi_err(DER_WARN, NULL, "incorrect iotlb flush type");
271 		return;
272 	}
273 
274 	qinv_submit_inv_dsc(immu, &dsc);
275 }
276 
277 /* queued invalidation interface -- invalidate dev_iotlb */
278 static void
qinv_dev_iotlb_common(immu_t * immu,uint16_t sid,uint64_t addr,uint_t size,uint_t max_invs_pd)279 qinv_dev_iotlb_common(immu_t *immu, uint16_t sid,
280     uint64_t addr, uint_t size, uint_t max_invs_pd)
281 {
282 	qinv_dsc_t dsc;
283 
284 	dsc.lo = DEV_IOTLB_INV_DSC_LOW(sid, max_invs_pd);
285 	dsc.hi = DEV_IOTLB_INV_DSC_HIGH(addr, size);
286 
287 	qinv_submit_inv_dsc(immu, &dsc);
288 }
289 
290 /* queued invalidation interface -- invalidate interrupt entry cache */
291 static void
qinv_iec_common(immu_t * immu,uint_t iidx,uint_t im,uint_t g)292 qinv_iec_common(immu_t *immu, uint_t iidx, uint_t im, uint_t g)
293 {
294 	qinv_dsc_t dsc;
295 
296 	dsc.lo = IEC_INV_DSC_LOW(iidx, im, g);
297 	dsc.hi = IEC_INV_DSC_HIGH;
298 
299 	qinv_submit_inv_dsc(immu, &dsc);
300 }
301 
302 /*
303  * queued invalidation interface -- invalidation wait descriptor
304  *   wait until the invalidation request finished
305  */
306 static void
qinv_wait_sync(immu_t * immu,immu_inv_wait_t * iwp)307 qinv_wait_sync(immu_t *immu, immu_inv_wait_t *iwp)
308 {
309 	qinv_dsc_t dsc;
310 	volatile uint32_t *status;
311 	uint64_t paddr;
312 #ifdef DEBUG
313 	uint_t count;
314 #endif
315 
316 	status = &iwp->iwp_vstatus;
317 	paddr = iwp->iwp_pstatus;
318 
319 	*status = IMMU_INV_DATA_PENDING;
320 	membar_producer();
321 
322 	/*
323 	 * sdata = IMMU_INV_DATA_DONE, fence = 1, sw = 1, if = 0
324 	 * indicate the invalidation wait descriptor completion by
325 	 * performing a coherent DWORD write to the status address,
326 	 * not by generating an invalidation completion event
327 	 */
328 	dsc.lo = INV_WAIT_DSC_LOW(IMMU_INV_DATA_DONE, 1, 1, 0);
329 	dsc.hi = INV_WAIT_DSC_HIGH(paddr);
330 
331 	qinv_submit_inv_dsc(immu, &dsc);
332 
333 	if (iwp->iwp_sync) {
334 #ifdef DEBUG
335 		count = 0;
336 		while (*status != IMMU_INV_DATA_DONE) {
337 			count++;
338 			ht_pause();
339 		}
340 		DTRACE_PROBE2(immu__wait__sync, const char *, iwp->iwp_name,
341 		    uint_t, count);
342 #else
343 		while (*status != IMMU_INV_DATA_DONE)
344 			ht_pause();
345 #endif
346 	}
347 }
348 
349 static void
immu_qinv_inv_wait(immu_inv_wait_t * iwp)350 immu_qinv_inv_wait(immu_inv_wait_t *iwp)
351 {
352 	volatile uint32_t *status = &iwp->iwp_vstatus;
353 #ifdef DEBUG
354 	uint_t count;
355 
356 	count = 0;
357 	while (*status != IMMU_INV_DATA_DONE) {
358 		count++;
359 		ht_pause();
360 	}
361 	DTRACE_PROBE2(immu__wait__async, const char *, iwp->iwp_name,
362 	    uint_t, count);
363 #else
364 
365 	while (*status != IMMU_INV_DATA_DONE)
366 		ht_pause();
367 #endif
368 }
369 
370 /*
371  * call ddi_dma_mem_alloc to allocate physical contigous
372  * pages for invalidation queue table
373  */
374 static int
qinv_setup(immu_t * immu)375 qinv_setup(immu_t *immu)
376 {
377 	qinv_t *qinv;
378 	size_t size;
379 
380 	ddi_dma_attr_t qinv_dma_attr = {
381 		DMA_ATTR_V0,
382 		0U,
383 		0xffffffffffffffffULL,
384 		0xffffffffU,
385 		MMU_PAGESIZE, /* page aligned */
386 		0x1,
387 		0x1,
388 		0xffffffffU,
389 		0xffffffffffffffffULL,
390 		1,
391 		4,
392 		0
393 	};
394 
395 	ddi_device_acc_attr_t qinv_acc_attr = {
396 		DDI_DEVICE_ATTR_V0,
397 		DDI_NEVERSWAP_ACC,
398 		DDI_STRICTORDER_ACC
399 	};
400 
401 	mutex_init(&(immu->immu_qinv_lock), NULL, MUTEX_DRIVER, NULL);
402 
403 
404 	mutex_enter(&(immu->immu_qinv_lock));
405 
406 	immu->immu_qinv = NULL;
407 	if (!IMMU_ECAP_GET_QI(immu->immu_regs_excap) ||
408 	    immu_qinv_enable == B_FALSE) {
409 		mutex_exit(&(immu->immu_qinv_lock));
410 		return (DDI_SUCCESS);
411 	}
412 
413 	if (qinv_iqa_qs > QINV_MAX_QUEUE_SIZE)
414 		qinv_iqa_qs = QINV_MAX_QUEUE_SIZE;
415 
416 	qinv = kmem_zalloc(sizeof (qinv_t), KM_SLEEP);
417 
418 	if (ddi_dma_alloc_handle(root_devinfo,
419 	    &qinv_dma_attr, DDI_DMA_SLEEP, NULL,
420 	    &(qinv->qinv_table.qinv_mem_dma_hdl)) != DDI_SUCCESS) {
421 		ddi_err(DER_WARN, root_devinfo,
422 		    "alloc invalidation queue table handler failed");
423 		goto queue_table_handle_failed;
424 	}
425 
426 	if (ddi_dma_alloc_handle(root_devinfo,
427 	    &qinv_dma_attr, DDI_DMA_SLEEP, NULL,
428 	    &(qinv->qinv_sync.qinv_mem_dma_hdl)) != DDI_SUCCESS) {
429 		ddi_err(DER_WARN, root_devinfo,
430 		    "alloc invalidation queue sync mem handler failed");
431 		goto sync_table_handle_failed;
432 	}
433 
434 	qinv->qinv_table.qinv_mem_size = (1 << (qinv_iqa_qs + 8));
435 	size = qinv->qinv_table.qinv_mem_size * QINV_ENTRY_SIZE;
436 
437 	/* alloc physical contiguous pages for invalidation queue */
438 	if (ddi_dma_mem_alloc(qinv->qinv_table.qinv_mem_dma_hdl,
439 	    size,
440 	    &qinv_acc_attr,
441 	    DDI_DMA_CONSISTENT | IOMEM_DATA_UNCACHED,
442 	    DDI_DMA_SLEEP,
443 	    NULL,
444 	    &(qinv->qinv_table.qinv_mem_vaddr),
445 	    &size,
446 	    &(qinv->qinv_table.qinv_mem_acc_hdl)) != DDI_SUCCESS) {
447 		ddi_err(DER_WARN, root_devinfo,
448 		    "alloc invalidation queue table failed");
449 		goto queue_table_mem_failed;
450 	}
451 
452 	ASSERT(!((uintptr_t)qinv->qinv_table.qinv_mem_vaddr & MMU_PAGEOFFSET));
453 	bzero(qinv->qinv_table.qinv_mem_vaddr, size);
454 
455 	/* get the base physical address of invalidation request queue */
456 	qinv->qinv_table.qinv_mem_paddr = pfn_to_pa(
457 	    hat_getpfnum(kas.a_hat, qinv->qinv_table.qinv_mem_vaddr));
458 
459 	qinv->qinv_table.qinv_mem_head = qinv->qinv_table.qinv_mem_tail = 0;
460 
461 	qinv->qinv_sync.qinv_mem_size = qinv->qinv_table.qinv_mem_size;
462 	size = qinv->qinv_sync.qinv_mem_size * QINV_SYNC_DATA_SIZE;
463 
464 	/* alloc status memory for invalidation wait descriptor */
465 	if (ddi_dma_mem_alloc(qinv->qinv_sync.qinv_mem_dma_hdl,
466 	    size,
467 	    &qinv_acc_attr,
468 	    DDI_DMA_CONSISTENT | IOMEM_DATA_UNCACHED,
469 	    DDI_DMA_SLEEP,
470 	    NULL,
471 	    &(qinv->qinv_sync.qinv_mem_vaddr),
472 	    &size,
473 	    &(qinv->qinv_sync.qinv_mem_acc_hdl)) != DDI_SUCCESS) {
474 		ddi_err(DER_WARN, root_devinfo,
475 		    "alloc invalidation queue sync mem failed");
476 		goto sync_table_mem_failed;
477 	}
478 
479 	ASSERT(!((uintptr_t)qinv->qinv_sync.qinv_mem_vaddr & MMU_PAGEOFFSET));
480 	bzero(qinv->qinv_sync.qinv_mem_vaddr, size);
481 	qinv->qinv_sync.qinv_mem_paddr = pfn_to_pa(
482 	    hat_getpfnum(kas.a_hat, qinv->qinv_sync.qinv_mem_vaddr));
483 
484 	qinv->qinv_sync.qinv_mem_head = qinv->qinv_sync.qinv_mem_tail = 0;
485 
486 	mutex_init(&(qinv->qinv_table.qinv_mem_lock), NULL, MUTEX_DRIVER, NULL);
487 	mutex_init(&(qinv->qinv_sync.qinv_mem_lock), NULL, MUTEX_DRIVER, NULL);
488 
489 	immu->immu_qinv = qinv;
490 
491 	mutex_exit(&(immu->immu_qinv_lock));
492 
493 	return (DDI_SUCCESS);
494 
495 sync_table_mem_failed:
496 	ddi_dma_mem_free(&(qinv->qinv_table.qinv_mem_acc_hdl));
497 
498 queue_table_mem_failed:
499 	ddi_dma_free_handle(&(qinv->qinv_sync.qinv_mem_dma_hdl));
500 
501 sync_table_handle_failed:
502 	ddi_dma_free_handle(&(qinv->qinv_table.qinv_mem_dma_hdl));
503 
504 queue_table_handle_failed:
505 	kmem_free(qinv, sizeof (qinv_t));
506 
507 	mutex_exit(&(immu->immu_qinv_lock));
508 
509 	return (DDI_FAILURE);
510 }
511 
512 /*
513  * ###########################################################################
514  *
515  * Functions exported by immu_qinv.c
516  *
517  * ###########################################################################
518  */
519 
520 /*
521  * initialize invalidation request queue structure.
522  */
523 int
immu_qinv_setup(list_t * listp)524 immu_qinv_setup(list_t *listp)
525 {
526 	immu_t *immu;
527 	int nerr;
528 
529 	if (immu_qinv_enable == B_FALSE) {
530 		return (DDI_FAILURE);
531 	}
532 
533 	nerr = 0;
534 	immu = list_head(listp);
535 	for (; immu; immu = list_next(listp, immu)) {
536 		if (qinv_setup(immu) == DDI_SUCCESS) {
537 			immu->immu_qinv_setup = B_TRUE;
538 		} else {
539 			nerr++;
540 			break;
541 		}
542 	}
543 
544 	return (nerr > 0 ? DDI_FAILURE : DDI_SUCCESS);
545 }
546 
547 void
immu_qinv_startup(immu_t * immu)548 immu_qinv_startup(immu_t *immu)
549 {
550 	qinv_t *qinv;
551 	uint64_t qinv_reg_value;
552 
553 	if (immu->immu_qinv_setup == B_FALSE) {
554 		return;
555 	}
556 
557 	qinv = (qinv_t *)immu->immu_qinv;
558 	qinv_reg_value = qinv->qinv_table.qinv_mem_paddr | qinv_iqa_qs;
559 	immu_regs_qinv_enable(immu, qinv_reg_value);
560 	immu->immu_flushops = &immu_qinv_flushops;
561 	immu->immu_qinv_running = B_TRUE;
562 }
563 
564 /*
565  * queued invalidation interface
566  *   function based context cache invalidation
567  */
568 void
immu_qinv_context_fsi(immu_t * immu,uint8_t function_mask,uint16_t source_id,uint_t domain_id,immu_inv_wait_t * iwp)569 immu_qinv_context_fsi(immu_t *immu, uint8_t function_mask,
570     uint16_t source_id, uint_t domain_id, immu_inv_wait_t *iwp)
571 {
572 	qinv_context_common(immu, function_mask, source_id,
573 	    domain_id, CTT_INV_G_DEVICE);
574 	qinv_wait_sync(immu, iwp);
575 }
576 
577 /*
578  * queued invalidation interface
579  *   domain based context cache invalidation
580  */
581 void
immu_qinv_context_dsi(immu_t * immu,uint_t domain_id,immu_inv_wait_t * iwp)582 immu_qinv_context_dsi(immu_t *immu, uint_t domain_id, immu_inv_wait_t *iwp)
583 {
584 	qinv_context_common(immu, 0, 0, domain_id, CTT_INV_G_DOMAIN);
585 	qinv_wait_sync(immu, iwp);
586 }
587 
588 /*
589  * queued invalidation interface
590  *   invalidation global context cache
591  */
592 void
immu_qinv_context_gbl(immu_t * immu,immu_inv_wait_t * iwp)593 immu_qinv_context_gbl(immu_t *immu, immu_inv_wait_t *iwp)
594 {
595 	qinv_context_common(immu, 0, 0, 0, CTT_INV_G_GLOBAL);
596 	qinv_wait_sync(immu, iwp);
597 }
598 
599 /*
600  * queued invalidation interface
601  *   paged based iotlb invalidation
602  */
603 void
immu_qinv_iotlb_psi(immu_t * immu,uint_t domain_id,uint64_t dvma,uint_t count,uint_t hint,immu_inv_wait_t * iwp)604 immu_qinv_iotlb_psi(immu_t *immu, uint_t domain_id,
605 	uint64_t dvma, uint_t count, uint_t hint, immu_inv_wait_t *iwp)
606 {
607 	uint_t am = 0;
608 	uint_t max_am;
609 
610 	max_am = IMMU_CAP_GET_MAMV(immu->immu_regs_cap);
611 
612 	/* choose page specified invalidation */
613 	if (IMMU_CAP_GET_PSI(immu->immu_regs_cap)) {
614 		while (am <= max_am) {
615 			if ((ADDR_AM_OFFSET(IMMU_BTOP(dvma), am) + count)
616 			    <= ADDR_AM_MAX(am)) {
617 				qinv_iotlb_common(immu, domain_id,
618 				    dvma, am, hint, TLB_INV_G_PAGE);
619 				break;
620 			}
621 			am++;
622 		}
623 		if (am > max_am) {
624 			qinv_iotlb_common(immu, domain_id,
625 			    dvma, 0, hint, TLB_INV_G_DOMAIN);
626 		}
627 
628 	/* choose domain invalidation */
629 	} else {
630 		qinv_iotlb_common(immu, domain_id, dvma,
631 		    0, hint, TLB_INV_G_DOMAIN);
632 	}
633 
634 	qinv_wait_sync(immu, iwp);
635 }
636 
637 /*
638  * queued invalidation interface
639  *   domain based iotlb invalidation
640  */
641 void
immu_qinv_iotlb_dsi(immu_t * immu,uint_t domain_id,immu_inv_wait_t * iwp)642 immu_qinv_iotlb_dsi(immu_t *immu, uint_t domain_id, immu_inv_wait_t *iwp)
643 {
644 	qinv_iotlb_common(immu, domain_id, 0, 0, 0, TLB_INV_G_DOMAIN);
645 	qinv_wait_sync(immu, iwp);
646 }
647 
648 /*
649  * queued invalidation interface
650  *    global iotlb invalidation
651  */
652 void
immu_qinv_iotlb_gbl(immu_t * immu,immu_inv_wait_t * iwp)653 immu_qinv_iotlb_gbl(immu_t *immu, immu_inv_wait_t *iwp)
654 {
655 	qinv_iotlb_common(immu, 0, 0, 0, 0, TLB_INV_G_GLOBAL);
656 	qinv_wait_sync(immu, iwp);
657 }
658 
659 /* queued invalidation interface -- global invalidate interrupt entry cache */
660 void
immu_qinv_intr_global(immu_t * immu,immu_inv_wait_t * iwp)661 immu_qinv_intr_global(immu_t *immu, immu_inv_wait_t *iwp)
662 {
663 	qinv_iec_common(immu, 0, 0, IEC_INV_GLOBAL);
664 	qinv_wait_sync(immu, iwp);
665 }
666 
667 /* queued invalidation interface -- invalidate single interrupt entry cache */
668 void
immu_qinv_intr_one_cache(immu_t * immu,uint_t iidx,immu_inv_wait_t * iwp)669 immu_qinv_intr_one_cache(immu_t *immu, uint_t iidx, immu_inv_wait_t *iwp)
670 {
671 	qinv_iec_common(immu, iidx, 0, IEC_INV_INDEX);
672 	qinv_wait_sync(immu, iwp);
673 }
674 
675 /* queued invalidation interface -- invalidate interrupt entry caches */
676 void
immu_qinv_intr_caches(immu_t * immu,uint_t iidx,uint_t cnt,immu_inv_wait_t * iwp)677 immu_qinv_intr_caches(immu_t *immu, uint_t iidx, uint_t cnt,
678     immu_inv_wait_t *iwp)
679 {
680 	uint_t	i, mask = 0;
681 
682 	ASSERT(cnt != 0);
683 
684 	/* requested interrupt count is not a power of 2 */
685 	if (!ISP2(cnt)) {
686 		for (i = 0; i < cnt; i++) {
687 			qinv_iec_common(immu, iidx + cnt, 0, IEC_INV_INDEX);
688 		}
689 		qinv_wait_sync(immu, iwp);
690 		return;
691 	}
692 
693 	while ((2 << mask) < cnt) {
694 		mask++;
695 	}
696 
697 	if (mask > IMMU_ECAP_GET_MHMV(immu->immu_regs_excap)) {
698 		for (i = 0; i < cnt; i++) {
699 			qinv_iec_common(immu, iidx + cnt, 0, IEC_INV_INDEX);
700 		}
701 		qinv_wait_sync(immu, iwp);
702 		return;
703 	}
704 
705 	qinv_iec_common(immu, iidx, mask, IEC_INV_INDEX);
706 
707 	qinv_wait_sync(immu, iwp);
708 }
709 
710 void
immu_qinv_report_fault(immu_t * immu)711 immu_qinv_report_fault(immu_t *immu)
712 {
713 	uint16_t head;
714 	qinv_dsc_t *dsc;
715 	qinv_t *qinv;
716 
717 	/* access qinv data */
718 	mutex_enter(&(immu->immu_qinv_lock));
719 
720 	qinv = (qinv_t *)(immu->immu_qinv);
721 
722 	head = QINV_IQA_HEAD(
723 	    immu_regs_get64(immu, IMMU_REG_INVAL_QH));
724 
725 	dsc = (qinv_dsc_t *)(qinv->qinv_table.qinv_mem_vaddr
726 	    + (head * QINV_ENTRY_SIZE));
727 
728 	/* report the error */
729 	ddi_err(DER_WARN, immu->immu_dip,
730 	    "generated a fault when fetching a descriptor from the"
731 	    "\tinvalidation queue, or detects that the fetched"
732 	    "\tdescriptor is invalid. The head register is "
733 	    "0x%" PRIx64
734 	    "\tthe type is %s",
735 	    head,
736 	    qinv_dsc_type[MIN(INV_DSC_TYPE(dsc), QINV_MAX_DSC_TYPE)]);
737 
738 	mutex_exit(&(immu->immu_qinv_lock));
739 }
740