1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21
22 /*
23 * Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved.
24 */
25
26 /*
27 * Copyright (c) 2009, Intel Corporation.
28 * All rights reserved.
29 */
30
31
32 #include <sys/apic.h>
33 #include <vm/hat_i86.h>
34 #include <sys/sysmacros.h>
35 #include <sys/smp_impldefs.h>
36 #include <sys/immu.h>
37
38
39 typedef struct intrmap_private {
40 immu_t *ir_immu;
41 immu_inv_wait_t ir_inv_wait;
42 uint16_t ir_idx;
43 uint32_t ir_sid_svt_sq;
44 } intrmap_private_t;
45
46 #define INTRMAP_PRIVATE(intrmap) ((intrmap_private_t *)intrmap)
47
48 /* interrupt remapping table entry */
49 typedef struct intrmap_rte {
50 uint64_t lo;
51 uint64_t hi;
52 } intrmap_rte_t;
53
54 #define IRTE_HIGH(sid_svt_sq) (sid_svt_sq)
55 #define IRTE_LOW(dst, vector, dlm, tm, rh, dm, fpd, p) \
56 (((uint64_t)(dst) << 32) | \
57 ((uint64_t)(vector) << 16) | \
58 ((uint64_t)(dlm) << 5) | \
59 ((uint64_t)(tm) << 4) | \
60 ((uint64_t)(rh) << 3) | \
61 ((uint64_t)(dm) << 2) | \
62 ((uint64_t)(fpd) << 1) | \
63 (p))
64
65 typedef enum {
66 SVT_NO_VERIFY = 0, /* no verification */
67 SVT_ALL_VERIFY, /* using sid and sq to verify */
68 SVT_BUS_VERIFY, /* verify #startbus and #endbus */
69 SVT_RSVD
70 } intrmap_svt_t;
71
72 typedef enum {
73 SQ_VERIFY_ALL = 0, /* verify all 16 bits */
74 SQ_VERIFY_IGR_1, /* ignore bit 3 */
75 SQ_VERIFY_IGR_2, /* ignore bit 2-3 */
76 SQ_VERIFY_IGR_3 /* ignore bit 1-3 */
77 } intrmap_sq_t;
78
79 /*
80 * S field of the Interrupt Remapping Table Address Register
81 * the size of the interrupt remapping table is 1 << (immu_intrmap_irta_s + 1)
82 */
83 static uint_t intrmap_irta_s = INTRMAP_MAX_IRTA_SIZE;
84
85 /*
86 * If true, arrange to suppress broadcast EOI by setting edge-triggered mode
87 * even for level-triggered interrupts in the interrupt-remapping engine.
88 * If false, broadcast EOI can still be suppressed if the CPU supports the
89 * APIC_SVR_SUPPRESS_BROADCAST_EOI bit. In both cases, the IOAPIC is still
90 * programmed with the correct trigger mode, and pcplusmp must send an EOI
91 * to the IOAPIC by writing to the IOAPIC's EOI register to make up for the
92 * missing broadcast EOI.
93 */
94 static int intrmap_suppress_brdcst_eoi = 0;
95
96 /*
97 * whether verify the source id of interrupt request
98 */
99 static int intrmap_enable_sid_verify = 0;
100
101 /* fault types for DVMA remapping */
102 static char *immu_dvma_faults[] = {
103 "Reserved",
104 "The present field in root-entry is Clear",
105 "The present field in context-entry is Clear",
106 "Hardware detected invalid programming of a context-entry",
107 "The DMA request attempted to access an address beyond max support",
108 "The Write field in a page-table entry is Clear when DMA write",
109 "The Read field in a page-table entry is Clear when DMA read",
110 "Access the next level page table resulted in error",
111 "Access the root-entry table resulted in error",
112 "Access the context-entry table resulted in error",
113 "Reserved field not initialized to zero in a present root-entry",
114 "Reserved field not initialized to zero in a present context-entry",
115 "Reserved field not initialized to zero in a present page-table entry",
116 "DMA blocked due to the Translation Type field in context-entry",
117 "Incorrect fault event reason number",
118 };
119 #define DVMA_MAX_FAULTS (sizeof (immu_dvma_faults)/(sizeof (char *))) - 1
120
121 /* fault types for interrupt remapping */
122 static char *immu_intrmap_faults[] = {
123 "reserved field set in IRTE",
124 "interrupt_index exceed the intr-remap table size",
125 "present field in IRTE is clear",
126 "hardware access intr-remap table address resulted in error",
127 "reserved field set in IRTE, include various conditional",
128 "hardware blocked an interrupt request in Compatibility format",
129 "remappable interrupt request blocked due to verification failure"
130 };
131 #define INTRMAP_MAX_FAULTS \
132 (sizeof (immu_intrmap_faults) / (sizeof (char *))) - 1
133
134 /* Function prototypes */
135 static int immu_intrmap_init(int apic_mode);
136 static void immu_intrmap_switchon(int suppress_brdcst_eoi);
137 static void immu_intrmap_alloc(void **intrmap_private_tbl, dev_info_t *dip,
138 uint16_t type, int count, uchar_t ioapic_index);
139 static void immu_intrmap_map(void *intrmap_private, void *intrmap_data,
140 uint16_t type, int count);
141 static void immu_intrmap_free(void **intrmap_privatep);
142 static void immu_intrmap_rdt(void *intrmap_private, ioapic_rdt_t *irdt);
143 static void immu_intrmap_msi(void *intrmap_private, msi_regs_t *mregs);
144
145 static struct apic_intrmap_ops intrmap_ops = {
146 immu_intrmap_init,
147 immu_intrmap_switchon,
148 immu_intrmap_alloc,
149 immu_intrmap_map,
150 immu_intrmap_free,
151 immu_intrmap_rdt,
152 immu_intrmap_msi,
153 };
154
155 /* apic mode, APIC/X2APIC */
156 static int intrmap_apic_mode = LOCAL_APIC;
157
158
159 /*
160 * helper functions
161 */
162 static uint_t
bitset_find_free(bitset_t * b,uint_t post)163 bitset_find_free(bitset_t *b, uint_t post)
164 {
165 uint_t i;
166 uint_t cap = bitset_capacity(b);
167
168 if (post == cap)
169 post = 0;
170
171 ASSERT(post < cap);
172
173 for (i = post; i < cap; i++) {
174 if (!bitset_in_set(b, i))
175 return (i);
176 }
177
178 for (i = 0; i < post; i++) {
179 if (!bitset_in_set(b, i))
180 return (i);
181 }
182
183 return (INTRMAP_IDX_FULL); /* no free index */
184 }
185
186 /*
187 * helper function to find 'count' contigous free
188 * interrupt remapping table entries
189 */
190 static uint_t
bitset_find_multi_free(bitset_t * b,uint_t post,uint_t count)191 bitset_find_multi_free(bitset_t *b, uint_t post, uint_t count)
192 {
193 uint_t i, j;
194 uint_t cap = bitset_capacity(b);
195
196 if (post == INTRMAP_IDX_FULL) {
197 return (INTRMAP_IDX_FULL);
198 }
199
200 if (count > cap)
201 return (INTRMAP_IDX_FULL);
202
203 ASSERT(post < cap);
204
205 for (i = post; (i + count) <= cap; i++) {
206 for (j = 0; j < count; j++) {
207 if (bitset_in_set(b, (i + j))) {
208 i = i + j;
209 break;
210 }
211 if (j == count - 1)
212 return (i);
213 }
214 }
215
216 for (i = 0; (i < post) && ((i + count) <= cap); i++) {
217 for (j = 0; j < count; j++) {
218 if (bitset_in_set(b, (i + j))) {
219 i = i + j;
220 break;
221 }
222 if (j == count - 1)
223 return (i);
224 }
225 }
226
227 return (INTRMAP_IDX_FULL); /* no free index */
228 }
229
230 /* alloc one interrupt remapping table entry */
231 static int
alloc_tbl_entry(intrmap_t * intrmap)232 alloc_tbl_entry(intrmap_t *intrmap)
233 {
234 uint32_t idx;
235
236 for (;;) {
237 mutex_enter(&intrmap->intrmap_lock);
238 idx = intrmap->intrmap_free;
239 if (idx != INTRMAP_IDX_FULL) {
240 bitset_add(&intrmap->intrmap_map, idx);
241 intrmap->intrmap_free =
242 bitset_find_free(&intrmap->intrmap_map, idx + 1);
243 mutex_exit(&intrmap->intrmap_lock);
244 break;
245 }
246
247 /* no free intr entry, use compatible format intr */
248 mutex_exit(&intrmap->intrmap_lock);
249
250 if (intrmap_apic_mode != LOCAL_X2APIC) {
251 break;
252 }
253
254 /*
255 * x2apic mode not allowed compatible
256 * interrupt
257 */
258 delay(IMMU_ALLOC_RESOURCE_DELAY);
259 }
260
261 return (idx);
262 }
263
264 /* alloc 'cnt' contigous interrupt remapping table entries */
265 static int
alloc_tbl_multi_entries(intrmap_t * intrmap,uint_t cnt)266 alloc_tbl_multi_entries(intrmap_t *intrmap, uint_t cnt)
267 {
268 uint_t idx, pos, i;
269
270 for (; ; ) {
271 mutex_enter(&intrmap->intrmap_lock);
272 pos = intrmap->intrmap_free;
273 idx = bitset_find_multi_free(&intrmap->intrmap_map, pos, cnt);
274
275 if (idx != INTRMAP_IDX_FULL) {
276 if (idx <= pos && pos < (idx + cnt)) {
277 intrmap->intrmap_free = bitset_find_free(
278 &intrmap->intrmap_map, idx + cnt);
279 }
280 for (i = 0; i < cnt; i++) {
281 bitset_add(&intrmap->intrmap_map, idx + i);
282 }
283 mutex_exit(&intrmap->intrmap_lock);
284 break;
285 }
286
287 mutex_exit(&intrmap->intrmap_lock);
288
289 if (intrmap_apic_mode != LOCAL_X2APIC) {
290 break;
291 }
292
293 /* x2apic mode not allowed comapitible interrupt */
294 delay(IMMU_ALLOC_RESOURCE_DELAY);
295 }
296
297 return (idx);
298 }
299
300 /* init interrupt remapping table */
301 static int
init_unit(immu_t * immu)302 init_unit(immu_t *immu)
303 {
304 intrmap_t *intrmap;
305 size_t size;
306
307 ddi_dma_attr_t intrmap_dma_attr = {
308 DMA_ATTR_V0,
309 0U,
310 0xffffffffffffffffULL,
311 0xffffffffU,
312 MMU_PAGESIZE, /* page aligned */
313 0x1,
314 0x1,
315 0xffffffffU,
316 0xffffffffffffffffULL,
317 1,
318 4,
319 0
320 };
321
322 ddi_device_acc_attr_t intrmap_acc_attr = {
323 DDI_DEVICE_ATTR_V0,
324 DDI_NEVERSWAP_ACC,
325 DDI_STRICTORDER_ACC
326 };
327
328 /*
329 * Using interrupt remapping implies using the queue
330 * invalidation interface. According to Intel,
331 * hardware that supports interrupt remapping should
332 * also support QI.
333 */
334 ASSERT(IMMU_ECAP_GET_QI(immu->immu_regs_excap));
335
336 if (intrmap_apic_mode == LOCAL_X2APIC) {
337 if (!IMMU_ECAP_GET_EIM(immu->immu_regs_excap)) {
338 return (DDI_FAILURE);
339 }
340 }
341
342 if (intrmap_irta_s > INTRMAP_MAX_IRTA_SIZE) {
343 intrmap_irta_s = INTRMAP_MAX_IRTA_SIZE;
344 }
345
346 intrmap = kmem_zalloc(sizeof (intrmap_t), KM_SLEEP);
347
348 if (ddi_dma_alloc_handle(immu->immu_dip,
349 &intrmap_dma_attr,
350 DDI_DMA_SLEEP,
351 NULL,
352 &(intrmap->intrmap_dma_hdl)) != DDI_SUCCESS) {
353 kmem_free(intrmap, sizeof (intrmap_t));
354 return (DDI_FAILURE);
355 }
356
357 intrmap->intrmap_size = 1 << (intrmap_irta_s + 1);
358 size = intrmap->intrmap_size * INTRMAP_RTE_SIZE;
359 if (ddi_dma_mem_alloc(intrmap->intrmap_dma_hdl,
360 size,
361 &intrmap_acc_attr,
362 DDI_DMA_CONSISTENT | IOMEM_DATA_UNCACHED,
363 DDI_DMA_SLEEP,
364 NULL,
365 &(intrmap->intrmap_vaddr),
366 &size,
367 &(intrmap->intrmap_acc_hdl)) != DDI_SUCCESS) {
368 ddi_dma_free_handle(&(intrmap->intrmap_dma_hdl));
369 kmem_free(intrmap, sizeof (intrmap_t));
370 return (DDI_FAILURE);
371 }
372
373 ASSERT(!((uintptr_t)intrmap->intrmap_vaddr & MMU_PAGEOFFSET));
374 bzero(intrmap->intrmap_vaddr, size);
375 intrmap->intrmap_paddr = pfn_to_pa(
376 hat_getpfnum(kas.a_hat, intrmap->intrmap_vaddr));
377
378 mutex_init(&(intrmap->intrmap_lock), NULL, MUTEX_DRIVER, NULL);
379 bitset_init(&intrmap->intrmap_map);
380 bitset_resize(&intrmap->intrmap_map, intrmap->intrmap_size);
381 intrmap->intrmap_free = 0;
382
383 immu->immu_intrmap = intrmap;
384
385 return (DDI_SUCCESS);
386 }
387
388 static immu_t *
get_immu(dev_info_t * dip,uint16_t type,uchar_t ioapic_index)389 get_immu(dev_info_t *dip, uint16_t type, uchar_t ioapic_index)
390 {
391 immu_t *immu = NULL;
392
393 if (!DDI_INTR_IS_MSI_OR_MSIX(type)) {
394 immu = immu_dmar_ioapic_immu(ioapic_index);
395 } else {
396 if (dip != NULL)
397 immu = immu_dmar_get_immu(dip);
398 }
399
400 return (immu);
401 }
402
403 static int
get_top_pcibridge(dev_info_t * dip,void * arg)404 get_top_pcibridge(dev_info_t *dip, void *arg)
405 {
406 dev_info_t **topdipp = arg;
407 immu_devi_t *immu_devi;
408
409 mutex_enter(&(DEVI(dip)->devi_lock));
410 immu_devi = DEVI(dip)->devi_iommu;
411 mutex_exit(&(DEVI(dip)->devi_lock));
412
413 if (immu_devi == NULL || immu_devi->imd_pcib_type == IMMU_PCIB_BAD ||
414 immu_devi->imd_pcib_type == IMMU_PCIB_ENDPOINT) {
415 return (DDI_WALK_CONTINUE);
416 }
417
418 *topdipp = dip;
419
420 return (DDI_WALK_CONTINUE);
421 }
422
423 static dev_info_t *
intrmap_top_pcibridge(dev_info_t * rdip)424 intrmap_top_pcibridge(dev_info_t *rdip)
425 {
426 dev_info_t *top_pcibridge = NULL;
427
428 if (immu_walk_ancestor(rdip, NULL, get_top_pcibridge,
429 &top_pcibridge, NULL, 0) != DDI_SUCCESS) {
430 return (NULL);
431 }
432
433 return (top_pcibridge);
434 }
435
436 /* function to get interrupt request source id */
437 static uint32_t
get_sid(dev_info_t * dip,uint16_t type,uchar_t ioapic_index)438 get_sid(dev_info_t *dip, uint16_t type, uchar_t ioapic_index)
439 {
440 dev_info_t *pdip;
441 immu_devi_t *immu_devi;
442 uint16_t sid;
443 uchar_t svt, sq;
444
445 if (!intrmap_enable_sid_verify) {
446 return (0);
447 }
448
449 if (!DDI_INTR_IS_MSI_OR_MSIX(type)) {
450 /* for interrupt through I/O APIC */
451 sid = immu_dmar_ioapic_sid(ioapic_index);
452 svt = SVT_ALL_VERIFY;
453 sq = SQ_VERIFY_ALL;
454 } else {
455 /* MSI/MSI-X interrupt */
456 ASSERT(dip);
457 pdip = intrmap_top_pcibridge(dip);
458 ASSERT(pdip);
459 immu_devi = DEVI(pdip)->devi_iommu;
460 ASSERT(immu_devi);
461 if (immu_devi->imd_pcib_type == IMMU_PCIB_PCIE_PCI) {
462 /* device behind pcie to pci bridge */
463 sid = (immu_devi->imd_bus << 8) | immu_devi->imd_sec;
464 svt = SVT_BUS_VERIFY;
465 sq = SQ_VERIFY_ALL;
466 } else {
467 /* pcie device or device behind pci to pci bridge */
468 sid = (immu_devi->imd_bus << 8) |
469 immu_devi->imd_devfunc;
470 svt = SVT_ALL_VERIFY;
471 sq = SQ_VERIFY_ALL;
472 }
473 }
474
475 return (sid | (svt << 18) | (sq << 16));
476 }
477
478 static void
intrmap_enable(immu_t * immu)479 intrmap_enable(immu_t *immu)
480 {
481 intrmap_t *intrmap;
482 uint64_t irta_reg;
483
484 intrmap = immu->immu_intrmap;
485
486 irta_reg = intrmap->intrmap_paddr | intrmap_irta_s;
487 if (intrmap_apic_mode == LOCAL_X2APIC) {
488 irta_reg |= (0x1 << 11);
489 }
490
491 immu_regs_intrmap_enable(immu, irta_reg);
492 }
493
494 /* ####################################################################### */
495
496 /*
497 * immu_intr_handler()
498 * the fault event handler for a single immu unit
499 */
500 uint_t
immu_intr_handler(caddr_t arg,caddr_t arg1 __unused)501 immu_intr_handler(caddr_t arg, caddr_t arg1 __unused)
502 {
503 immu_t *immu = (immu_t *)arg;
504 uint32_t status;
505 int index, fault_reg_offset;
506 int max_fault_index;
507 boolean_t found_fault;
508 dev_info_t *idip;
509
510 mutex_enter(&(immu->immu_intr_lock));
511 mutex_enter(&(immu->immu_regs_lock));
512
513 /* read the fault status */
514 status = immu_regs_get32(immu, IMMU_REG_FAULT_STS);
515
516 idip = immu->immu_dip;
517 ASSERT(idip);
518
519 /* check if we have a pending fault for this immu unit */
520 if ((status & IMMU_FAULT_STS_PPF) == 0) {
521 mutex_exit(&(immu->immu_regs_lock));
522 mutex_exit(&(immu->immu_intr_lock));
523 return (DDI_INTR_UNCLAIMED);
524 }
525
526 /*
527 * handle all primary pending faults
528 */
529 index = IMMU_FAULT_GET_INDEX(status);
530 max_fault_index = IMMU_CAP_GET_NFR(immu->immu_regs_cap) - 1;
531 fault_reg_offset = IMMU_CAP_GET_FRO(immu->immu_regs_cap);
532
533 found_fault = B_FALSE;
534 _NOTE(CONSTCOND)
535 while (1) {
536 uint64_t val;
537 uint8_t fault_reason;
538 uint8_t fault_type;
539 uint16_t sid;
540 uint64_t pg_addr;
541 uint64_t idx;
542
543 /* read the higher 64bits */
544 val = immu_regs_get64(immu, fault_reg_offset + index * 16 + 8);
545
546 /* check if this fault register has pending fault */
547 if (!IMMU_FRR_GET_F(val)) {
548 break;
549 }
550
551 found_fault = B_TRUE;
552
553 /* get the fault reason, fault type and sid */
554 fault_reason = IMMU_FRR_GET_FR(val);
555 fault_type = IMMU_FRR_GET_FT(val);
556 sid = IMMU_FRR_GET_SID(val);
557
558 /* read the first 64bits */
559 val = immu_regs_get64(immu, fault_reg_offset + index * 16);
560 pg_addr = val & IMMU_PAGEMASK;
561 idx = val >> 48;
562
563 /* clear the fault */
564 immu_regs_put32(immu, fault_reg_offset + index * 16 + 12,
565 (((uint32_t)1) << 31));
566
567 /* report the fault info */
568 if (fault_reason < 0x20) {
569 /* immu-remapping fault */
570 ddi_err(DER_WARN, idip,
571 "generated a fault event when translating DMA %s\n"
572 "\t on address 0x%" PRIx64 " for PCI(%d, %d, %d), "
573 "the reason is:\n\t %s",
574 fault_type ? "read" : "write", pg_addr,
575 (sid >> 8) & 0xff, (sid >> 3) & 0x1f, sid & 0x7,
576 immu_dvma_faults[MIN(fault_reason,
577 DVMA_MAX_FAULTS)]);
578 immu_print_fault_info(sid, pg_addr);
579 } else if (fault_reason < 0x27) {
580 /* intr-remapping fault */
581 ddi_err(DER_WARN, idip,
582 "generated a fault event when translating "
583 "interrupt request\n"
584 "\t on index 0x%" PRIx64 " for PCI(%d, %d, %d), "
585 "the reason is:\n\t %s",
586 idx,
587 (sid >> 8) & 0xff, (sid >> 3) & 0x1f, sid & 0x7,
588 immu_intrmap_faults[MIN((fault_reason - 0x20),
589 INTRMAP_MAX_FAULTS)]);
590 } else {
591 ddi_err(DER_WARN, idip, "Unknown fault reason: 0x%x",
592 fault_reason);
593 }
594
595 index++;
596 if (index > max_fault_index)
597 index = 0;
598 }
599
600 /* Clear the fault */
601 if (!found_fault) {
602 ddi_err(DER_MODE, idip,
603 "Fault register set but no fault present");
604 }
605 immu_regs_put32(immu, IMMU_REG_FAULT_STS, 1);
606 mutex_exit(&(immu->immu_regs_lock));
607 mutex_exit(&(immu->immu_intr_lock));
608 return (DDI_INTR_CLAIMED);
609 }
610 /* ######################################################################### */
611
612 /*
613 * Interrupt remap entry points
614 */
615
616 /* initialize interrupt remapping */
617 static int
immu_intrmap_init(int apic_mode)618 immu_intrmap_init(int apic_mode)
619 {
620 immu_t *immu;
621 int error = DDI_FAILURE;
622
623 if (immu_intrmap_enable == B_FALSE) {
624 return (DDI_SUCCESS);
625 }
626
627 intrmap_apic_mode = apic_mode;
628
629 immu = list_head(&immu_list);
630 for (; immu; immu = list_next(&immu_list, immu)) {
631 if ((immu->immu_intrmap_running == B_TRUE) &&
632 IMMU_ECAP_GET_IR(immu->immu_regs_excap)) {
633 if (init_unit(immu) == DDI_SUCCESS) {
634 error = DDI_SUCCESS;
635 }
636 }
637 }
638
639 /*
640 * if all IOMMU units disable intr remapping,
641 * return FAILURE
642 */
643 return (error);
644 }
645
646
647
648 /* enable interrupt remapping */
649 static void
immu_intrmap_switchon(int suppress_brdcst_eoi)650 immu_intrmap_switchon(int suppress_brdcst_eoi)
651 {
652 immu_t *immu;
653
654
655 intrmap_suppress_brdcst_eoi = suppress_brdcst_eoi;
656
657 immu = list_head(&immu_list);
658 for (; immu; immu = list_next(&immu_list, immu)) {
659 if (immu->immu_intrmap_setup == B_TRUE) {
660 intrmap_enable(immu);
661 }
662 }
663 }
664
665 /* alloc remapping entry for the interrupt */
666 static void
immu_intrmap_alloc(void ** intrmap_private_tbl,dev_info_t * dip,uint16_t type,int count,uchar_t ioapic_index)667 immu_intrmap_alloc(void **intrmap_private_tbl, dev_info_t *dip,
668 uint16_t type, int count, uchar_t ioapic_index)
669 {
670 immu_t *immu;
671 intrmap_t *intrmap;
672 immu_inv_wait_t *iwp;
673 uint32_t idx, i;
674 uint32_t sid_svt_sq;
675 intrmap_private_t *intrmap_private;
676
677 if (intrmap_private_tbl[0] == INTRMAP_DISABLE ||
678 intrmap_private_tbl[0] != NULL) {
679 return;
680 }
681
682 intrmap_private_tbl[0] =
683 kmem_zalloc(sizeof (intrmap_private_t), KM_SLEEP);
684 intrmap_private = INTRMAP_PRIVATE(intrmap_private_tbl[0]);
685
686 immu = get_immu(dip, type, ioapic_index);
687 if ((immu != NULL) && (immu->immu_intrmap_running == B_TRUE)) {
688 intrmap_private->ir_immu = immu;
689 } else {
690 goto intrmap_disable;
691 }
692
693 intrmap = immu->immu_intrmap;
694
695 if (count == 1) {
696 idx = alloc_tbl_entry(intrmap);
697 } else {
698 idx = alloc_tbl_multi_entries(intrmap, count);
699 }
700
701 if (idx == INTRMAP_IDX_FULL) {
702 goto intrmap_disable;
703 }
704
705 intrmap_private->ir_idx = idx;
706
707 sid_svt_sq = intrmap_private->ir_sid_svt_sq =
708 get_sid(dip, type, ioapic_index);
709 iwp = &intrmap_private->ir_inv_wait;
710 immu_init_inv_wait(iwp, "intrmaplocal", B_TRUE);
711
712 if (count == 1) {
713 if (IMMU_CAP_GET_CM(immu->immu_regs_cap)) {
714 immu_qinv_intr_one_cache(immu, idx, iwp);
715 } else {
716 immu_regs_wbf_flush(immu);
717 }
718 return;
719 }
720
721 for (i = 1; i < count; i++) {
722 intrmap_private_tbl[i] =
723 kmem_zalloc(sizeof (intrmap_private_t), KM_SLEEP);
724
725 INTRMAP_PRIVATE(intrmap_private_tbl[i])->ir_immu = immu;
726 INTRMAP_PRIVATE(intrmap_private_tbl[i])->ir_sid_svt_sq =
727 sid_svt_sq;
728 INTRMAP_PRIVATE(intrmap_private_tbl[i])->ir_idx = idx + i;
729 }
730
731 if (IMMU_CAP_GET_CM(immu->immu_regs_cap)) {
732 immu_qinv_intr_caches(immu, idx, count, iwp);
733 } else {
734 immu_regs_wbf_flush(immu);
735 }
736
737 return;
738
739 intrmap_disable:
740 kmem_free(intrmap_private_tbl[0], sizeof (intrmap_private_t));
741 intrmap_private_tbl[0] = INTRMAP_DISABLE;
742 }
743
744
745 /* remapping the interrupt */
746 static void
immu_intrmap_map(void * intrmap_private,void * intrmap_data,uint16_t type,int count)747 immu_intrmap_map(void *intrmap_private, void *intrmap_data, uint16_t type,
748 int count)
749 {
750 immu_t *immu;
751 immu_inv_wait_t *iwp;
752 intrmap_t *intrmap;
753 ioapic_rdt_t *irdt = (ioapic_rdt_t *)intrmap_data;
754 msi_regs_t *mregs = (msi_regs_t *)intrmap_data;
755 intrmap_rte_t irte;
756 uint_t idx, i;
757 uint32_t dst, sid_svt_sq;
758 uchar_t vector, dlm, tm, rh, dm;
759
760 if (intrmap_private == INTRMAP_DISABLE)
761 return;
762
763 idx = INTRMAP_PRIVATE(intrmap_private)->ir_idx;
764 immu = INTRMAP_PRIVATE(intrmap_private)->ir_immu;
765 iwp = &INTRMAP_PRIVATE(intrmap_private)->ir_inv_wait;
766 intrmap = immu->immu_intrmap;
767 sid_svt_sq = INTRMAP_PRIVATE(intrmap_private)->ir_sid_svt_sq;
768
769 if (!DDI_INTR_IS_MSI_OR_MSIX(type)) {
770 dm = RDT_DM(irdt->ir_lo);
771 rh = 0;
772 tm = RDT_TM(irdt->ir_lo);
773 dlm = RDT_DLM(irdt->ir_lo);
774 dst = irdt->ir_hi;
775
776 /*
777 * Mark the IRTE's TM as Edge to suppress broadcast EOI.
778 */
779 if (intrmap_suppress_brdcst_eoi) {
780 tm = TRIGGER_MODE_EDGE;
781 }
782
783 vector = RDT_VECTOR(irdt->ir_lo);
784 } else {
785 dm = MSI_ADDR_DM_PHYSICAL;
786 rh = MSI_ADDR_RH_FIXED;
787 tm = TRIGGER_MODE_EDGE;
788 dlm = 0;
789 dst = mregs->mr_addr;
790
791 vector = mregs->mr_data & 0xff;
792 }
793
794 if (intrmap_apic_mode == LOCAL_APIC)
795 dst = (dst & 0xFF) << 8;
796
797 if (count == 1) {
798 irte.lo = IRTE_LOW(dst, vector, dlm, tm, rh, dm, 0, 1);
799 irte.hi = IRTE_HIGH(sid_svt_sq);
800
801 /* set interrupt remapping table entry */
802 bcopy(&irte, intrmap->intrmap_vaddr +
803 idx * INTRMAP_RTE_SIZE,
804 INTRMAP_RTE_SIZE);
805
806 immu_qinv_intr_one_cache(immu, idx, iwp);
807
808 } else {
809 for (i = 0; i < count; i++) {
810 irte.lo = IRTE_LOW(dst, vector, dlm, tm, rh, dm, 0, 1);
811 irte.hi = IRTE_HIGH(sid_svt_sq);
812
813 /* set interrupt remapping table entry */
814 bcopy(&irte, intrmap->intrmap_vaddr +
815 idx * INTRMAP_RTE_SIZE,
816 INTRMAP_RTE_SIZE);
817 vector++;
818 idx++;
819 }
820
821 immu_qinv_intr_caches(immu, idx, count, iwp);
822 }
823 }
824
825 /* free the remapping entry */
826 static void
immu_intrmap_free(void ** intrmap_privatep)827 immu_intrmap_free(void **intrmap_privatep)
828 {
829 immu_t *immu;
830 immu_inv_wait_t *iwp;
831 intrmap_t *intrmap;
832 uint32_t idx;
833
834 if (*intrmap_privatep == INTRMAP_DISABLE || *intrmap_privatep == NULL) {
835 *intrmap_privatep = NULL;
836 return;
837 }
838
839 immu = INTRMAP_PRIVATE(*intrmap_privatep)->ir_immu;
840 iwp = &INTRMAP_PRIVATE(*intrmap_privatep)->ir_inv_wait;
841 intrmap = immu->immu_intrmap;
842 idx = INTRMAP_PRIVATE(*intrmap_privatep)->ir_idx;
843
844 bzero(intrmap->intrmap_vaddr + idx * INTRMAP_RTE_SIZE,
845 INTRMAP_RTE_SIZE);
846
847 immu_qinv_intr_one_cache(immu, idx, iwp);
848
849 mutex_enter(&intrmap->intrmap_lock);
850 bitset_del(&intrmap->intrmap_map, idx);
851 if (intrmap->intrmap_free == INTRMAP_IDX_FULL) {
852 intrmap->intrmap_free = idx;
853 }
854 mutex_exit(&intrmap->intrmap_lock);
855
856 kmem_free(*intrmap_privatep, sizeof (intrmap_private_t));
857 *intrmap_privatep = NULL;
858 }
859
860 /* record the ioapic rdt entry */
861 static void
immu_intrmap_rdt(void * intrmap_private,ioapic_rdt_t * irdt)862 immu_intrmap_rdt(void *intrmap_private, ioapic_rdt_t *irdt)
863 {
864 uint32_t rdt_entry, tm, pol, idx, vector;
865
866 rdt_entry = irdt->ir_lo;
867
868 if (intrmap_private != INTRMAP_DISABLE && intrmap_private != NULL) {
869 idx = INTRMAP_PRIVATE(intrmap_private)->ir_idx;
870 tm = RDT_TM(rdt_entry);
871 pol = RDT_POL(rdt_entry);
872 vector = RDT_VECTOR(rdt_entry);
873 irdt->ir_lo = (tm << INTRMAP_IOAPIC_TM_SHIFT) |
874 (pol << INTRMAP_IOAPIC_POL_SHIFT) |
875 ((idx >> 15) << INTRMAP_IOAPIC_IDX15_SHIFT) |
876 vector;
877 irdt->ir_hi = (idx << INTRMAP_IOAPIC_IDX_SHIFT) |
878 (1 << INTRMAP_IOAPIC_FORMAT_SHIFT);
879 } else {
880 irdt->ir_hi <<= APIC_ID_BIT_OFFSET;
881 }
882 }
883
884 /* record the msi interrupt structure */
885 /*ARGSUSED*/
886 static void
immu_intrmap_msi(void * intrmap_private,msi_regs_t * mregs)887 immu_intrmap_msi(void *intrmap_private, msi_regs_t *mregs)
888 {
889 uint_t idx;
890
891 if (intrmap_private != INTRMAP_DISABLE && intrmap_private != NULL) {
892 idx = INTRMAP_PRIVATE(intrmap_private)->ir_idx;
893
894 mregs->mr_data = 0;
895 mregs->mr_addr = MSI_ADDR_HDR |
896 ((idx & 0x7fff) << INTRMAP_MSI_IDX_SHIFT) |
897 (1 << INTRMAP_MSI_FORMAT_SHIFT) |
898 (1 << INTRMAP_MSI_SHV_SHIFT) |
899 ((idx >> 15) << INTRMAP_MSI_IDX15_SHIFT);
900 } else {
901 mregs->mr_addr = MSI_ADDR_HDR |
902 (MSI_ADDR_RH_FIXED << MSI_ADDR_RH_SHIFT) |
903 (MSI_ADDR_DM_PHYSICAL << MSI_ADDR_DM_SHIFT) |
904 (mregs->mr_addr << MSI_ADDR_DEST_SHIFT);
905 mregs->mr_data = (MSI_DATA_TM_EDGE << MSI_DATA_TM_SHIFT) |
906 mregs->mr_data;
907 }
908 }
909
910 /* ######################################################################### */
911 /*
912 * Functions exported by immu_intr.c
913 */
914 void
immu_intrmap_setup(list_t * listp)915 immu_intrmap_setup(list_t *listp)
916 {
917 immu_t *immu;
918
919 /*
920 * Check if ACPI DMAR tables say that
921 * interrupt remapping is supported
922 */
923 if (immu_dmar_intrmap_supported() == B_FALSE) {
924 return;
925 }
926
927 /*
928 * Check if interrupt remapping is disabled.
929 */
930 if (immu_intrmap_enable == B_FALSE) {
931 return;
932 }
933
934 psm_vt_ops = &intrmap_ops;
935
936 immu = list_head(listp);
937 for (; immu; immu = list_next(listp, immu)) {
938 mutex_init(&(immu->immu_intrmap_lock), NULL,
939 MUTEX_DEFAULT, NULL);
940 mutex_enter(&(immu->immu_intrmap_lock));
941 immu_init_inv_wait(&immu->immu_intrmap_inv_wait,
942 "intrmapglobal", B_TRUE);
943 immu->immu_intrmap_setup = B_TRUE;
944 mutex_exit(&(immu->immu_intrmap_lock));
945 }
946 }
947
948 void
immu_intrmap_startup(immu_t * immu)949 immu_intrmap_startup(immu_t *immu)
950 {
951 /* do nothing */
952 mutex_enter(&(immu->immu_intrmap_lock));
953 if (immu->immu_intrmap_setup == B_TRUE) {
954 immu->immu_intrmap_running = B_TRUE;
955 }
956 mutex_exit(&(immu->immu_intrmap_lock));
957 }
958
959 /*
960 * Register a Intel IOMMU unit (i.e. DMAR unit's)
961 * interrupt handler
962 */
963 void
immu_intr_register(immu_t * immu)964 immu_intr_register(immu_t *immu)
965 {
966 int irq, vect;
967 char intr_handler_name[IMMU_MAXNAMELEN];
968 uint32_t msi_data;
969 uint32_t uaddr;
970 uint32_t msi_addr;
971 uint32_t localapic_id = 0;
972
973 if (psm_get_localapicid)
974 localapic_id = psm_get_localapicid(0);
975
976 msi_addr = (MSI_ADDR_HDR |
977 ((localapic_id & 0xFF) << MSI_ADDR_DEST_SHIFT) |
978 (MSI_ADDR_RH_FIXED << MSI_ADDR_RH_SHIFT) |
979 (MSI_ADDR_DM_PHYSICAL << MSI_ADDR_DM_SHIFT));
980
981 if (intrmap_apic_mode == LOCAL_X2APIC) {
982 uaddr = localapic_id & 0xFFFFFF00;
983 } else {
984 uaddr = 0;
985 }
986
987 /* Dont need to hold immu_intr_lock since we are in boot */
988 irq = vect = psm_get_ipivect(IMMU_INTR_IPL, -1);
989 if (psm_xlate_vector_by_irq != NULL)
990 vect = psm_xlate_vector_by_irq(irq);
991
992 msi_data = ((MSI_DATA_DELIVERY_FIXED <<
993 MSI_DATA_DELIVERY_SHIFT) | vect);
994
995 (void) snprintf(intr_handler_name, sizeof (intr_handler_name),
996 "%s-intr-handler", immu->immu_name);
997
998 (void) add_avintr((void *)NULL, IMMU_INTR_IPL,
999 immu_intr_handler, intr_handler_name, irq,
1000 (caddr_t)immu, NULL, NULL, NULL);
1001
1002 immu_regs_intr_enable(immu, msi_addr, msi_data, uaddr);
1003
1004 (void) immu_intr_handler((caddr_t)immu, NULL);
1005 }
1006