xref: /titanic_44/usr/src/uts/i86pc/io/immu_intrmap.c (revision 56b2bdd1f04d465cfe4a95b88ae5cba5884154e4)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 
22 /*
23  * Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved.
24  */
25 
26 /*
27  * Copyright (c) 2009, Intel Corporation.
28  * All rights reserved.
29  */
30 
31 
32 #include <sys/apic.h>
33 #include <vm/hat_i86.h>
34 #include <sys/sysmacros.h>
35 #include <sys/smp_impldefs.h>
36 #include <sys/immu.h>
37 
38 
39 typedef struct intrmap_private {
40 	immu_t		*ir_immu;
41 	immu_inv_wait_t	ir_inv_wait;
42 	uint16_t	ir_idx;
43 	uint32_t	ir_sid_svt_sq;
44 } intrmap_private_t;
45 
46 #define	INTRMAP_PRIVATE(intrmap) ((intrmap_private_t *)intrmap)
47 
48 /* interrupt remapping table entry */
49 typedef struct intrmap_rte {
50 	uint64_t	lo;
51 	uint64_t	hi;
52 } intrmap_rte_t;
53 
54 #define	IRTE_HIGH(sid_svt_sq) (sid_svt_sq)
55 #define	IRTE_LOW(dst, vector, dlm, tm, rh, dm, fpd, p)	\
56 	    (((uint64_t)(dst) << 32) |  \
57 	    ((uint64_t)(vector) << 16) | \
58 	    ((uint64_t)(dlm) << 5) | \
59 	    ((uint64_t)(tm) << 4) | \
60 	    ((uint64_t)(rh) << 3) | \
61 	    ((uint64_t)(dm) << 2) | \
62 	    ((uint64_t)(fpd) << 1) | \
63 	    (p))
64 
65 typedef enum {
66 	SVT_NO_VERIFY = 0, 	/* no verification */
67 	SVT_ALL_VERIFY,		/* using sid and sq to verify */
68 	SVT_BUS_VERIFY,		/* verify #startbus and #endbus */
69 	SVT_RSVD
70 } intrmap_svt_t;
71 
72 typedef enum {
73 	SQ_VERIFY_ALL = 0,	/* verify all 16 bits */
74 	SQ_VERIFY_IGR_1,	/* ignore bit 3 */
75 	SQ_VERIFY_IGR_2,	/* ignore bit 2-3 */
76 	SQ_VERIFY_IGR_3		/* ignore bit 1-3 */
77 } intrmap_sq_t;
78 
79 /*
80  * S field of the Interrupt Remapping Table Address Register
81  * the size of the interrupt remapping table is 1 << (immu_intrmap_irta_s + 1)
82  */
83 static uint_t intrmap_irta_s = INTRMAP_MAX_IRTA_SIZE;
84 
85 /*
86  * If true, arrange to suppress broadcast EOI by setting edge-triggered mode
87  * even for level-triggered interrupts in the interrupt-remapping engine.
88  * If false, broadcast EOI can still be suppressed if the CPU supports the
89  * APIC_SVR_SUPPRESS_BROADCAST_EOI bit.  In both cases, the IOAPIC is still
90  * programmed with the correct trigger mode, and pcplusmp must send an EOI
91  * to the IOAPIC by writing to the IOAPIC's EOI register to make up for the
92  * missing broadcast EOI.
93  */
94 static int intrmap_suppress_brdcst_eoi = 0;
95 
96 /*
97  * whether verify the source id of interrupt request
98  */
99 static int intrmap_enable_sid_verify = 0;
100 
101 /* fault types for DVMA remapping */
102 static char *immu_dvma_faults[] = {
103 	"Reserved",
104 	"The present field in root-entry is Clear",
105 	"The present field in context-entry is Clear",
106 	"Hardware detected invalid programming of a context-entry",
107 	"The DMA request attempted to access an address beyond max support",
108 	"The Write field in a page-table entry is Clear when DMA write",
109 	"The Read field in a page-table entry is Clear when DMA read",
110 	"Access the next level page table resulted in error",
111 	"Access the root-entry table resulted in error",
112 	"Access the context-entry table resulted in error",
113 	"Reserved field not initialized to zero in a present root-entry",
114 	"Reserved field not initialized to zero in a present context-entry",
115 	"Reserved field not initialized to zero in a present page-table entry",
116 	"DMA blocked due to the Translation Type field in context-entry",
117 	"Incorrect fault event reason number",
118 };
119 #define	DVMA_MAX_FAULTS (sizeof (immu_dvma_faults)/(sizeof (char *))) - 1
120 
121 /* fault types for interrupt remapping */
122 static char *immu_intrmap_faults[] = {
123 	"reserved field set in IRTE",
124 	"interrupt_index exceed the intr-remap table size",
125 	"present field in IRTE is clear",
126 	"hardware access intr-remap table address resulted in error",
127 	"reserved field set in IRTE, include various conditional",
128 	"hardware blocked an interrupt request in Compatibility format",
129 	"remappable interrupt request blocked due to verification failure"
130 };
131 #define	INTRMAP_MAX_FAULTS \
132 	(sizeof (immu_intrmap_faults) / (sizeof (char *))) - 1
133 
134 /* Function prototypes */
135 static int immu_intrmap_init(int apic_mode);
136 static void immu_intrmap_switchon(int suppress_brdcst_eoi);
137 static void immu_intrmap_alloc(void **intrmap_private_tbl, dev_info_t *dip,
138     uint16_t type, int count, uchar_t ioapic_index);
139 static void immu_intrmap_map(void *intrmap_private, void *intrmap_data,
140     uint16_t type, int count);
141 static void immu_intrmap_free(void **intrmap_privatep);
142 static void immu_intrmap_rdt(void *intrmap_private, ioapic_rdt_t *irdt);
143 static void immu_intrmap_msi(void *intrmap_private, msi_regs_t *mregs);
144 
145 static struct apic_intrmap_ops intrmap_ops = {
146 	immu_intrmap_init,
147 	immu_intrmap_switchon,
148 	immu_intrmap_alloc,
149 	immu_intrmap_map,
150 	immu_intrmap_free,
151 	immu_intrmap_rdt,
152 	immu_intrmap_msi,
153 };
154 
155 /* apic mode, APIC/X2APIC */
156 static int intrmap_apic_mode = LOCAL_APIC;
157 
158 
159 /*
160  * helper functions
161  */
162 static uint_t
163 bitset_find_free(bitset_t *b, uint_t post)
164 {
165 	uint_t	i;
166 	uint_t	cap = bitset_capacity(b);
167 
168 	if (post == cap)
169 		post = 0;
170 
171 	ASSERT(post < cap);
172 
173 	for (i = post; i < cap; i++) {
174 		if (!bitset_in_set(b, i))
175 			return (i);
176 	}
177 
178 	for (i = 0; i < post; i++) {
179 		if (!bitset_in_set(b, i))
180 			return (i);
181 	}
182 
183 	return (INTRMAP_IDX_FULL);	/* no free index */
184 }
185 
186 /*
187  * helper function to find 'count' contigous free
188  * interrupt remapping table entries
189  */
190 static uint_t
191 bitset_find_multi_free(bitset_t *b, uint_t post, uint_t count)
192 {
193 	uint_t  i, j;
194 	uint_t	cap = bitset_capacity(b);
195 
196 	if (post == INTRMAP_IDX_FULL) {
197 		return (INTRMAP_IDX_FULL);
198 	}
199 
200 	if (count > cap)
201 		return (INTRMAP_IDX_FULL);
202 
203 	ASSERT(post < cap);
204 
205 	for (i = post; (i + count) <= cap; i++) {
206 		for (j = 0; j < count; j++) {
207 			if (bitset_in_set(b, (i + j))) {
208 				i = i + j;
209 				break;
210 			}
211 			if (j == count - 1)
212 				return (i);
213 		}
214 	}
215 
216 	for (i = 0; (i < post) && ((i + count) <= cap); i++) {
217 		for (j = 0; j < count; j++) {
218 			if (bitset_in_set(b, (i + j))) {
219 				i = i + j;
220 				break;
221 			}
222 			if (j == count - 1)
223 				return (i);
224 		}
225 	}
226 
227 	return (INTRMAP_IDX_FULL);  		/* no free index */
228 }
229 
230 /* alloc one interrupt remapping table entry */
231 static int
232 alloc_tbl_entry(intrmap_t *intrmap)
233 {
234 	uint32_t idx;
235 
236 	for (;;) {
237 		mutex_enter(&intrmap->intrmap_lock);
238 		idx = intrmap->intrmap_free;
239 		if (idx != INTRMAP_IDX_FULL) {
240 			bitset_add(&intrmap->intrmap_map, idx);
241 			intrmap->intrmap_free =
242 			    bitset_find_free(&intrmap->intrmap_map, idx + 1);
243 			mutex_exit(&intrmap->intrmap_lock);
244 			break;
245 		}
246 
247 		/* no free intr entry, use compatible format intr */
248 		mutex_exit(&intrmap->intrmap_lock);
249 
250 		if (intrmap_apic_mode != LOCAL_X2APIC) {
251 			break;
252 		}
253 
254 		/*
255 		 * x2apic mode not allowed compatible
256 		 * interrupt
257 		 */
258 		delay(IMMU_ALLOC_RESOURCE_DELAY);
259 	}
260 
261 	return (idx);
262 }
263 
264 /* alloc 'cnt' contigous interrupt remapping table entries */
265 static int
266 alloc_tbl_multi_entries(intrmap_t *intrmap, uint_t cnt)
267 {
268 	uint_t idx, pos, i;
269 
270 	for (; ; ) {
271 		mutex_enter(&intrmap->intrmap_lock);
272 		pos = intrmap->intrmap_free;
273 		idx = bitset_find_multi_free(&intrmap->intrmap_map, pos, cnt);
274 
275 		if (idx != INTRMAP_IDX_FULL) {
276 			if (idx <= pos && pos < (idx + cnt)) {
277 				intrmap->intrmap_free = bitset_find_free(
278 				    &intrmap->intrmap_map, idx + cnt);
279 			}
280 			for (i = 0; i < cnt; i++) {
281 				bitset_add(&intrmap->intrmap_map, idx + i);
282 			}
283 			mutex_exit(&intrmap->intrmap_lock);
284 			break;
285 		}
286 
287 		mutex_exit(&intrmap->intrmap_lock);
288 
289 		if (intrmap_apic_mode != LOCAL_X2APIC) {
290 			break;
291 		}
292 
293 		/* x2apic mode not allowed comapitible interrupt */
294 		delay(IMMU_ALLOC_RESOURCE_DELAY);
295 	}
296 
297 	return (idx);
298 }
299 
300 /* init interrupt remapping table */
301 static int
302 init_unit(immu_t *immu)
303 {
304 	intrmap_t *intrmap;
305 	size_t size;
306 
307 	ddi_dma_attr_t intrmap_dma_attr = {
308 		DMA_ATTR_V0,
309 		0U,
310 		0xffffffffffffffffULL,
311 		0xffffffffU,
312 		MMU_PAGESIZE,	/* page aligned */
313 		0x1,
314 		0x1,
315 		0xffffffffU,
316 		0xffffffffffffffffULL,
317 		1,
318 		4,
319 		0
320 	};
321 
322 	ddi_device_acc_attr_t intrmap_acc_attr = {
323 		DDI_DEVICE_ATTR_V0,
324 		DDI_NEVERSWAP_ACC,
325 		DDI_STRICTORDER_ACC
326 	};
327 
328 	/*
329 	 * Using interrupt remapping implies using the queue
330 	 * invalidation interface. According to Intel,
331 	 * hardware that supports interrupt remapping should
332 	 * also support QI.
333 	 */
334 	ASSERT(IMMU_ECAP_GET_QI(immu->immu_regs_excap));
335 
336 	if (intrmap_apic_mode == LOCAL_X2APIC) {
337 		if (!IMMU_ECAP_GET_EIM(immu->immu_regs_excap)) {
338 			return (DDI_FAILURE);
339 		}
340 	}
341 
342 	if (intrmap_irta_s > INTRMAP_MAX_IRTA_SIZE) {
343 		intrmap_irta_s = INTRMAP_MAX_IRTA_SIZE;
344 	}
345 
346 	intrmap =  kmem_zalloc(sizeof (intrmap_t), KM_SLEEP);
347 
348 	if (ddi_dma_alloc_handle(immu->immu_dip,
349 	    &intrmap_dma_attr,
350 	    DDI_DMA_SLEEP,
351 	    NULL,
352 	    &(intrmap->intrmap_dma_hdl)) != DDI_SUCCESS) {
353 		kmem_free(intrmap, sizeof (intrmap_t));
354 		return (DDI_FAILURE);
355 	}
356 
357 	intrmap->intrmap_size = 1 << (intrmap_irta_s + 1);
358 	size = intrmap->intrmap_size * INTRMAP_RTE_SIZE;
359 	if (ddi_dma_mem_alloc(intrmap->intrmap_dma_hdl,
360 	    size,
361 	    &intrmap_acc_attr,
362 	    DDI_DMA_CONSISTENT | IOMEM_DATA_UNCACHED,
363 	    DDI_DMA_SLEEP,
364 	    NULL,
365 	    &(intrmap->intrmap_vaddr),
366 	    &size,
367 	    &(intrmap->intrmap_acc_hdl)) != DDI_SUCCESS) {
368 		ddi_dma_free_handle(&(intrmap->intrmap_dma_hdl));
369 		kmem_free(intrmap, sizeof (intrmap_t));
370 		return (DDI_FAILURE);
371 	}
372 
373 	ASSERT(!((uintptr_t)intrmap->intrmap_vaddr & MMU_PAGEOFFSET));
374 	bzero(intrmap->intrmap_vaddr, size);
375 	intrmap->intrmap_paddr = pfn_to_pa(
376 	    hat_getpfnum(kas.a_hat, intrmap->intrmap_vaddr));
377 
378 	mutex_init(&(intrmap->intrmap_lock), NULL, MUTEX_DRIVER, NULL);
379 	bitset_init(&intrmap->intrmap_map);
380 	bitset_resize(&intrmap->intrmap_map, intrmap->intrmap_size);
381 	intrmap->intrmap_free = 0;
382 
383 	immu->immu_intrmap = intrmap;
384 
385 	return (DDI_SUCCESS);
386 }
387 
388 static immu_t *
389 get_immu(dev_info_t *dip, uint16_t type, uchar_t ioapic_index)
390 {
391 	immu_t	*immu = NULL;
392 
393 	if (!DDI_INTR_IS_MSI_OR_MSIX(type)) {
394 		immu = immu_dmar_ioapic_immu(ioapic_index);
395 	} else {
396 		if (dip != NULL)
397 			immu = immu_dmar_get_immu(dip);
398 	}
399 
400 	return (immu);
401 }
402 
403 static int
404 get_top_pcibridge(dev_info_t *dip, void *arg)
405 {
406 	dev_info_t **topdipp = arg;
407 	immu_devi_t *immu_devi;
408 
409 	mutex_enter(&(DEVI(dip)->devi_lock));
410 	immu_devi = DEVI(dip)->devi_iommu;
411 	mutex_exit(&(DEVI(dip)->devi_lock));
412 
413 	if (immu_devi == NULL || immu_devi->imd_pcib_type == IMMU_PCIB_BAD ||
414 	    immu_devi->imd_pcib_type == IMMU_PCIB_ENDPOINT) {
415 		return (DDI_WALK_CONTINUE);
416 	}
417 
418 	*topdipp = dip;
419 
420 	return (DDI_WALK_CONTINUE);
421 }
422 
423 static dev_info_t *
424 intrmap_top_pcibridge(dev_info_t *rdip)
425 {
426 	dev_info_t *top_pcibridge = NULL;
427 
428 	if (immu_walk_ancestor(rdip, NULL, get_top_pcibridge,
429 	    &top_pcibridge, NULL, 0) != DDI_SUCCESS) {
430 		return (NULL);
431 	}
432 
433 	return (top_pcibridge);
434 }
435 
436 /* function to get interrupt request source id */
437 static uint32_t
438 get_sid(dev_info_t *dip, uint16_t type, uchar_t ioapic_index)
439 {
440 	dev_info_t	*pdip;
441 	immu_devi_t	*immu_devi;
442 	uint16_t	sid;
443 	uchar_t		svt, sq;
444 
445 	if (!intrmap_enable_sid_verify) {
446 		return (0);
447 	}
448 
449 	if (!DDI_INTR_IS_MSI_OR_MSIX(type)) {
450 		/* for interrupt through I/O APIC */
451 		sid = immu_dmar_ioapic_sid(ioapic_index);
452 		svt = SVT_ALL_VERIFY;
453 		sq = SQ_VERIFY_ALL;
454 	} else {
455 		/* MSI/MSI-X interrupt */
456 		ASSERT(dip);
457 		pdip = intrmap_top_pcibridge(dip);
458 		ASSERT(pdip);
459 		immu_devi = DEVI(pdip)->devi_iommu;
460 		ASSERT(immu_devi);
461 		if (immu_devi->imd_pcib_type == IMMU_PCIB_PCIE_PCI) {
462 			/* device behind pcie to pci bridge */
463 			sid = (immu_devi->imd_bus << 8) | immu_devi->imd_sec;
464 			svt = SVT_BUS_VERIFY;
465 			sq = SQ_VERIFY_ALL;
466 		} else {
467 			/* pcie device or device behind pci to pci bridge */
468 			sid = (immu_devi->imd_bus << 8) |
469 			    immu_devi->imd_devfunc;
470 			svt = SVT_ALL_VERIFY;
471 			sq = SQ_VERIFY_ALL;
472 		}
473 	}
474 
475 	return (sid | (svt << 18) | (sq << 16));
476 }
477 
478 static void
479 intrmap_enable(immu_t *immu)
480 {
481 	intrmap_t *intrmap;
482 	uint64_t irta_reg;
483 
484 	intrmap = immu->immu_intrmap;
485 
486 	irta_reg = intrmap->intrmap_paddr | intrmap_irta_s;
487 	if (intrmap_apic_mode == LOCAL_X2APIC) {
488 		irta_reg |= (0x1 << 11);
489 	}
490 
491 	immu_regs_intrmap_enable(immu, irta_reg);
492 }
493 
494 /* ####################################################################### */
495 
496 /*
497  * immu_intr_handler()
498  * 	the fault event handler for a single immu unit
499  */
500 int
501 immu_intr_handler(immu_t *immu)
502 {
503 	uint32_t status;
504 	int index, fault_reg_offset;
505 	int max_fault_index;
506 	boolean_t found_fault;
507 	dev_info_t *idip;
508 
509 	mutex_enter(&(immu->immu_intr_lock));
510 	mutex_enter(&(immu->immu_regs_lock));
511 
512 	/* read the fault status */
513 	status = immu_regs_get32(immu, IMMU_REG_FAULT_STS);
514 
515 	idip = immu->immu_dip;
516 	ASSERT(idip);
517 
518 	/* check if we have a pending fault for this immu unit */
519 	if ((status & IMMU_FAULT_STS_PPF) == 0) {
520 		mutex_exit(&(immu->immu_regs_lock));
521 		mutex_exit(&(immu->immu_intr_lock));
522 		return (DDI_INTR_UNCLAIMED);
523 	}
524 
525 	/*
526 	 * handle all primary pending faults
527 	 */
528 	index = IMMU_FAULT_GET_INDEX(status);
529 	max_fault_index =  IMMU_CAP_GET_NFR(immu->immu_regs_cap) - 1;
530 	fault_reg_offset = IMMU_CAP_GET_FRO(immu->immu_regs_cap);
531 
532 	found_fault = B_FALSE;
533 	_NOTE(CONSTCOND)
534 	while (1) {
535 		uint64_t val;
536 		uint8_t fault_reason;
537 		uint8_t fault_type;
538 		uint16_t sid;
539 		uint64_t pg_addr;
540 		uint64_t idx;
541 
542 		/* read the higher 64bits */
543 		val = immu_regs_get64(immu, fault_reg_offset + index * 16 + 8);
544 
545 		/* check if this fault register has pending fault */
546 		if (!IMMU_FRR_GET_F(val)) {
547 			break;
548 		}
549 
550 		found_fault = B_TRUE;
551 
552 		/* get the fault reason, fault type and sid */
553 		fault_reason = IMMU_FRR_GET_FR(val);
554 		fault_type = IMMU_FRR_GET_FT(val);
555 		sid = IMMU_FRR_GET_SID(val);
556 
557 		/* read the first 64bits */
558 		val = immu_regs_get64(immu, fault_reg_offset + index * 16);
559 		pg_addr = val & IMMU_PAGEMASK;
560 		idx = val >> 48;
561 
562 		/* clear the fault */
563 		immu_regs_put32(immu, fault_reg_offset + index * 16 + 12,
564 		    (((uint32_t)1) << 31));
565 
566 		/* report the fault info */
567 		if (fault_reason < 0x20) {
568 			/* immu-remapping fault */
569 			ddi_err(DER_WARN, idip,
570 			    "generated a fault event when translating DMA %s\n"
571 			    "\t on address 0x%" PRIx64 " for PCI(%d, %d, %d), "
572 			    "the reason is:\n\t %s",
573 			    fault_type ? "read" : "write", pg_addr,
574 			    (sid >> 8) & 0xff, (sid >> 3) & 0x1f, sid & 0x7,
575 			    immu_dvma_faults[MIN(fault_reason,
576 			    DVMA_MAX_FAULTS)]);
577 			immu_print_fault_info(sid, pg_addr);
578 		} else if (fault_reason < 0x27) {
579 			/* intr-remapping fault */
580 			ddi_err(DER_WARN, idip,
581 			    "generated a fault event when translating "
582 			    "interrupt request\n"
583 			    "\t on index 0x%" PRIx64 " for PCI(%d, %d, %d), "
584 			    "the reason is:\n\t %s",
585 			    idx,
586 			    (sid >> 8) & 0xff, (sid >> 3) & 0x1f, sid & 0x7,
587 			    immu_intrmap_faults[MIN((fault_reason - 0x20),
588 			    INTRMAP_MAX_FAULTS)]);
589 		} else {
590 			ddi_err(DER_WARN, idip, "Unknown fault reason: 0x%x",
591 			    fault_reason);
592 		}
593 
594 		index++;
595 		if (index > max_fault_index)
596 			index = 0;
597 	}
598 
599 	/* Clear the fault */
600 	if (!found_fault) {
601 		ddi_err(DER_MODE, idip,
602 		    "Fault register set but no fault present");
603 	}
604 	immu_regs_put32(immu, IMMU_REG_FAULT_STS, 1);
605 	mutex_exit(&(immu->immu_regs_lock));
606 	mutex_exit(&(immu->immu_intr_lock));
607 	return (DDI_INTR_CLAIMED);
608 }
609 /* ######################################################################### */
610 
611 /*
612  * Interrupt remap entry points
613  */
614 
615 /* initialize interrupt remapping */
616 static int
617 immu_intrmap_init(int apic_mode)
618 {
619 	immu_t *immu;
620 	int error = DDI_FAILURE;
621 
622 	if (immu_intrmap_enable == B_FALSE) {
623 		return (DDI_SUCCESS);
624 	}
625 
626 	intrmap_apic_mode = apic_mode;
627 
628 	immu = list_head(&immu_list);
629 	for (; immu; immu = list_next(&immu_list, immu)) {
630 		if ((immu->immu_intrmap_running == B_TRUE) &&
631 		    IMMU_ECAP_GET_IR(immu->immu_regs_excap)) {
632 			if (init_unit(immu) == DDI_SUCCESS) {
633 				error = DDI_SUCCESS;
634 			}
635 		}
636 	}
637 
638 	/*
639 	 * if all IOMMU units disable intr remapping,
640 	 * return FAILURE
641 	 */
642 	return (error);
643 }
644 
645 
646 
647 /* enable interrupt remapping */
648 static void
649 immu_intrmap_switchon(int suppress_brdcst_eoi)
650 {
651 	immu_t *immu;
652 
653 
654 	intrmap_suppress_brdcst_eoi = suppress_brdcst_eoi;
655 
656 	immu = list_head(&immu_list);
657 	for (; immu; immu = list_next(&immu_list, immu)) {
658 		if (immu->immu_intrmap_setup == B_TRUE) {
659 			intrmap_enable(immu);
660 		}
661 	}
662 }
663 
664 /* alloc remapping entry for the interrupt */
665 static void
666 immu_intrmap_alloc(void **intrmap_private_tbl, dev_info_t *dip,
667     uint16_t type, int count, uchar_t ioapic_index)
668 {
669 	immu_t	*immu;
670 	intrmap_t *intrmap;
671 	immu_inv_wait_t *iwp;
672 	uint32_t		idx, i;
673 	uint32_t		sid_svt_sq;
674 	intrmap_private_t	*intrmap_private;
675 
676 	if (intrmap_private_tbl[0] == INTRMAP_DISABLE ||
677 	    intrmap_private_tbl[0] != NULL) {
678 		return;
679 	}
680 
681 	intrmap_private_tbl[0] =
682 	    kmem_zalloc(sizeof (intrmap_private_t), KM_SLEEP);
683 	intrmap_private = INTRMAP_PRIVATE(intrmap_private_tbl[0]);
684 
685 	immu = get_immu(dip, type, ioapic_index);
686 	if ((immu != NULL) && (immu->immu_intrmap_running == B_TRUE)) {
687 		intrmap_private->ir_immu = immu;
688 	} else {
689 		goto intrmap_disable;
690 	}
691 
692 	intrmap = immu->immu_intrmap;
693 
694 	if (count == 1) {
695 		idx = alloc_tbl_entry(intrmap);
696 	} else {
697 		idx = alloc_tbl_multi_entries(intrmap, count);
698 	}
699 
700 	if (idx == INTRMAP_IDX_FULL) {
701 		goto intrmap_disable;
702 	}
703 
704 	intrmap_private->ir_idx = idx;
705 
706 	sid_svt_sq = intrmap_private->ir_sid_svt_sq =
707 	    get_sid(dip, type, ioapic_index);
708 	iwp = &intrmap_private->ir_inv_wait;
709 	immu_init_inv_wait(iwp, "intrmaplocal", B_TRUE);
710 
711 	if (count == 1) {
712 		if (IMMU_CAP_GET_CM(immu->immu_regs_cap)) {
713 			immu_qinv_intr_one_cache(immu, idx, iwp);
714 		} else {
715 			immu_regs_wbf_flush(immu);
716 		}
717 		return;
718 	}
719 
720 	for (i = 1; i < count; i++) {
721 		intrmap_private_tbl[i] =
722 		    kmem_zalloc(sizeof (intrmap_private_t), KM_SLEEP);
723 
724 		INTRMAP_PRIVATE(intrmap_private_tbl[i])->ir_immu = immu;
725 		INTRMAP_PRIVATE(intrmap_private_tbl[i])->ir_sid_svt_sq =
726 		    sid_svt_sq;
727 		INTRMAP_PRIVATE(intrmap_private_tbl[i])->ir_idx = idx + i;
728 	}
729 
730 	if (IMMU_CAP_GET_CM(immu->immu_regs_cap)) {
731 		immu_qinv_intr_caches(immu, idx, count, iwp);
732 	} else {
733 		immu_regs_wbf_flush(immu);
734 	}
735 
736 	return;
737 
738 intrmap_disable:
739 	kmem_free(intrmap_private_tbl[0], sizeof (intrmap_private_t));
740 	intrmap_private_tbl[0] = INTRMAP_DISABLE;
741 }
742 
743 
744 /* remapping the interrupt */
745 static void
746 immu_intrmap_map(void *intrmap_private, void *intrmap_data, uint16_t type,
747     int count)
748 {
749 	immu_t	*immu;
750 	immu_inv_wait_t	*iwp;
751 	intrmap_t	*intrmap;
752 	ioapic_rdt_t	*irdt = (ioapic_rdt_t *)intrmap_data;
753 	msi_regs_t	*mregs = (msi_regs_t *)intrmap_data;
754 	intrmap_rte_t	irte;
755 	uint_t		idx, i;
756 	uint32_t	dst, sid_svt_sq;
757 	uchar_t		vector, dlm, tm, rh, dm;
758 
759 	if (intrmap_private == INTRMAP_DISABLE)
760 		return;
761 
762 	idx = INTRMAP_PRIVATE(intrmap_private)->ir_idx;
763 	immu = INTRMAP_PRIVATE(intrmap_private)->ir_immu;
764 	iwp = &INTRMAP_PRIVATE(intrmap_private)->ir_inv_wait;
765 	intrmap = immu->immu_intrmap;
766 	sid_svt_sq = INTRMAP_PRIVATE(intrmap_private)->ir_sid_svt_sq;
767 
768 	if (!DDI_INTR_IS_MSI_OR_MSIX(type)) {
769 		dm = RDT_DM(irdt->ir_lo);
770 		rh = 0;
771 		tm = RDT_TM(irdt->ir_lo);
772 		dlm = RDT_DLM(irdt->ir_lo);
773 		dst = irdt->ir_hi;
774 
775 		/*
776 		 * Mark the IRTE's TM as Edge to suppress broadcast EOI.
777 		 */
778 		if (intrmap_suppress_brdcst_eoi) {
779 			tm = TRIGGER_MODE_EDGE;
780 		}
781 
782 		vector = RDT_VECTOR(irdt->ir_lo);
783 	} else {
784 		dm = MSI_ADDR_DM_PHYSICAL;
785 		rh = MSI_ADDR_RH_FIXED;
786 		tm = TRIGGER_MODE_EDGE;
787 		dlm = 0;
788 		dst = mregs->mr_addr;
789 
790 		vector = mregs->mr_data & 0xff;
791 	}
792 
793 	if (intrmap_apic_mode == LOCAL_APIC)
794 		dst = (dst & 0xFF) << 8;
795 
796 	if (count == 1) {
797 		irte.lo = IRTE_LOW(dst, vector, dlm, tm, rh, dm, 0, 1);
798 		irte.hi = IRTE_HIGH(sid_svt_sq);
799 
800 		/* set interrupt remapping table entry */
801 		bcopy(&irte, intrmap->intrmap_vaddr +
802 		    idx * INTRMAP_RTE_SIZE,
803 		    INTRMAP_RTE_SIZE);
804 
805 		immu_qinv_intr_one_cache(immu, idx, iwp);
806 
807 	} else {
808 		for (i = 0; i < count; i++) {
809 			irte.lo = IRTE_LOW(dst, vector, dlm, tm, rh, dm, 0, 1);
810 			irte.hi = IRTE_HIGH(sid_svt_sq);
811 
812 			/* set interrupt remapping table entry */
813 			bcopy(&irte, intrmap->intrmap_vaddr +
814 			    idx * INTRMAP_RTE_SIZE,
815 			    INTRMAP_RTE_SIZE);
816 			vector++;
817 			idx++;
818 		}
819 
820 		immu_qinv_intr_caches(immu, idx, count, iwp);
821 	}
822 }
823 
824 /* free the remapping entry */
825 static void
826 immu_intrmap_free(void **intrmap_privatep)
827 {
828 	immu_t *immu;
829 	immu_inv_wait_t *iwp;
830 	intrmap_t *intrmap;
831 	uint32_t idx;
832 
833 	if (*intrmap_privatep == INTRMAP_DISABLE || *intrmap_privatep == NULL) {
834 		*intrmap_privatep = NULL;
835 		return;
836 	}
837 
838 	immu = INTRMAP_PRIVATE(*intrmap_privatep)->ir_immu;
839 	iwp = &INTRMAP_PRIVATE(*intrmap_privatep)->ir_inv_wait;
840 	intrmap = immu->immu_intrmap;
841 	idx = INTRMAP_PRIVATE(*intrmap_privatep)->ir_idx;
842 
843 	bzero(intrmap->intrmap_vaddr + idx * INTRMAP_RTE_SIZE,
844 	    INTRMAP_RTE_SIZE);
845 
846 	immu_qinv_intr_one_cache(immu, idx, iwp);
847 
848 	mutex_enter(&intrmap->intrmap_lock);
849 	bitset_del(&intrmap->intrmap_map, idx);
850 	if (intrmap->intrmap_free == INTRMAP_IDX_FULL) {
851 		intrmap->intrmap_free = idx;
852 	}
853 	mutex_exit(&intrmap->intrmap_lock);
854 
855 	kmem_free(*intrmap_privatep, sizeof (intrmap_private_t));
856 	*intrmap_privatep = NULL;
857 }
858 
859 /* record the ioapic rdt entry */
860 static void
861 immu_intrmap_rdt(void *intrmap_private, ioapic_rdt_t *irdt)
862 {
863 	uint32_t rdt_entry, tm, pol, idx, vector;
864 
865 	rdt_entry = irdt->ir_lo;
866 
867 	if (intrmap_private != INTRMAP_DISABLE && intrmap_private != NULL) {
868 		idx = INTRMAP_PRIVATE(intrmap_private)->ir_idx;
869 		tm = RDT_TM(rdt_entry);
870 		pol = RDT_POL(rdt_entry);
871 		vector = RDT_VECTOR(rdt_entry);
872 		irdt->ir_lo = (tm << INTRMAP_IOAPIC_TM_SHIFT) |
873 		    (pol << INTRMAP_IOAPIC_POL_SHIFT) |
874 		    ((idx >> 15) << INTRMAP_IOAPIC_IDX15_SHIFT) |
875 		    vector;
876 		irdt->ir_hi = (idx << INTRMAP_IOAPIC_IDX_SHIFT) |
877 		    (1 << INTRMAP_IOAPIC_FORMAT_SHIFT);
878 	} else {
879 		irdt->ir_hi <<= APIC_ID_BIT_OFFSET;
880 	}
881 }
882 
883 /* record the msi interrupt structure */
884 /*ARGSUSED*/
885 static void
886 immu_intrmap_msi(void *intrmap_private, msi_regs_t *mregs)
887 {
888 	uint_t	idx;
889 
890 	if (intrmap_private != INTRMAP_DISABLE && intrmap_private != NULL) {
891 		idx = INTRMAP_PRIVATE(intrmap_private)->ir_idx;
892 
893 		mregs->mr_data = 0;
894 		mregs->mr_addr = MSI_ADDR_HDR |
895 		    ((idx & 0x7fff) << INTRMAP_MSI_IDX_SHIFT) |
896 		    (1 << INTRMAP_MSI_FORMAT_SHIFT) |
897 		    (1 << INTRMAP_MSI_SHV_SHIFT) |
898 		    ((idx >> 15) << INTRMAP_MSI_IDX15_SHIFT);
899 	} else {
900 		mregs->mr_addr = MSI_ADDR_HDR |
901 		    (MSI_ADDR_RH_FIXED << MSI_ADDR_RH_SHIFT) |
902 		    (MSI_ADDR_DM_PHYSICAL << MSI_ADDR_DM_SHIFT) |
903 		    (mregs->mr_addr << MSI_ADDR_DEST_SHIFT);
904 		mregs->mr_data = (MSI_DATA_TM_EDGE << MSI_DATA_TM_SHIFT) |
905 		    mregs->mr_data;
906 	}
907 }
908 
909 /* ######################################################################### */
910 /*
911  * Functions exported by immu_intr.c
912  */
913 void
914 immu_intrmap_setup(list_t *listp)
915 {
916 	immu_t *immu;
917 
918 	/*
919 	 * Check if ACPI DMAR tables say that
920 	 * interrupt remapping is supported
921 	 */
922 	if (immu_dmar_intrmap_supported() == B_FALSE) {
923 		return;
924 	}
925 
926 	/*
927 	 * Check if interrupt remapping is disabled.
928 	 */
929 	if (immu_intrmap_enable == B_FALSE) {
930 		return;
931 	}
932 
933 	psm_vt_ops = &intrmap_ops;
934 
935 	immu = list_head(listp);
936 	for (; immu; immu = list_next(listp, immu)) {
937 		mutex_init(&(immu->immu_intrmap_lock), NULL,
938 		    MUTEX_DEFAULT, NULL);
939 		mutex_enter(&(immu->immu_intrmap_lock));
940 		immu_init_inv_wait(&immu->immu_intrmap_inv_wait,
941 		    "intrmapglobal", B_TRUE);
942 		immu->immu_intrmap_setup = B_TRUE;
943 		mutex_exit(&(immu->immu_intrmap_lock));
944 	}
945 }
946 
947 void
948 immu_intrmap_startup(immu_t *immu)
949 {
950 	/* do nothing */
951 	mutex_enter(&(immu->immu_intrmap_lock));
952 	if (immu->immu_intrmap_setup == B_TRUE) {
953 		immu->immu_intrmap_running = B_TRUE;
954 	}
955 	mutex_exit(&(immu->immu_intrmap_lock));
956 }
957 
958 /*
959  * Register a Intel IOMMU unit (i.e. DMAR unit's)
960  * interrupt handler
961  */
962 void
963 immu_intr_register(immu_t *immu)
964 {
965 	int irq, vect;
966 	char intr_handler_name[IMMU_MAXNAMELEN];
967 	uint32_t msi_data;
968 	uint32_t uaddr;
969 	uint32_t msi_addr;
970 	uint32_t localapic_id = 0;
971 
972 	if (psm_get_localapicid)
973 		localapic_id = psm_get_localapicid(0);
974 
975 	msi_addr = (MSI_ADDR_HDR |
976 	    ((localapic_id & 0xFF) << MSI_ADDR_DEST_SHIFT) |
977 	    (MSI_ADDR_RH_FIXED << MSI_ADDR_RH_SHIFT) |
978 	    (MSI_ADDR_DM_PHYSICAL << MSI_ADDR_DM_SHIFT));
979 
980 	if (intrmap_apic_mode == LOCAL_X2APIC) {
981 		uaddr = localapic_id & 0xFFFFFF00;
982 	} else {
983 		uaddr = 0;
984 	}
985 
986 	/* Dont need to hold immu_intr_lock since we are in boot */
987 	irq = vect = psm_get_ipivect(IMMU_INTR_IPL, -1);
988 	if (psm_xlate_vector_by_irq != NULL)
989 		vect = psm_xlate_vector_by_irq(irq);
990 
991 	msi_data = ((MSI_DATA_DELIVERY_FIXED <<
992 	    MSI_DATA_DELIVERY_SHIFT) | vect);
993 
994 	(void) snprintf(intr_handler_name, sizeof (intr_handler_name),
995 	    "%s-intr-handler", immu->immu_name);
996 
997 	(void) add_avintr((void *)NULL, IMMU_INTR_IPL,
998 	    (avfunc)(immu_intr_handler), intr_handler_name, irq,
999 	    (caddr_t)immu, NULL, NULL, NULL);
1000 
1001 	immu_regs_intr_enable(immu, msi_addr, msi_data, uaddr);
1002 
1003 	(void) immu_intr_handler(immu);
1004 }
1005