1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21 /*
22 * Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved.
23 */
24
25 /*
26 * Copyright (c) 2009, Intel Corporation.
27 * All rights reserved.
28 */
29
30 /*
31 * Copyright 2023 Oxide Computer Company
32 * Copyright 2025 RackTop Systems, Inc.
33 */
34
35 #include <sys/debug.h>
36 #include <sys/sysmacros.h>
37 #include <sys/types.h>
38 #include <sys/kmem.h>
39 #include <sys/sunddi.h>
40 #include <sys/list.h>
41 #include <sys/pci.h>
42 #include <sys/pci_cfgspace.h>
43 #include <sys/pci_impl.h>
44 #include <sys/sunndi.h>
45 #include <sys/ksynch.h>
46 #include <sys/cmn_err.h>
47 #include <sys/bootconf.h>
48 #include <sys/int_fmtio.h>
49 #include <sys/smbios.h>
50 #include <sys/apic.h>
51 #include <sys/acpi/acpi.h>
52 #include <sys/acpica.h>
53 #include <sys/immu.h>
54 #include <sys/smp_impldefs.h>
55
56 static void dmar_table_destroy(dmar_table_t *tbl);
57
58 /*
59 * internal global variables
60 */
61 static char *dmar_raw; /* raw DMAR ACPI table */
62 static dmar_table_t *dmar_table; /* converted form of DMAR table */
63
64 /*
65 * global variables exported outside this file
66 */
67 kmutex_t ioapic_drhd_lock;
68 list_t ioapic_drhd_list;
69
70 /* ######################################################################### */
71
72 /*
73 * helper functions to read the "raw" DMAR table
74 */
75
76 static uint8_t
get_uint8(char * cp)77 get_uint8(char *cp)
78 {
79 uint8_t val = *((uint8_t *)cp);
80 return (val);
81 }
82
83 static uint16_t
get_uint16(char * cp)84 get_uint16(char *cp)
85 {
86 uint16_t val = *((uint16_t *)cp);
87 return (val);
88 }
89
90 static uint32_t
get_uint32(char * cp)91 get_uint32(char *cp)
92 {
93 uint32_t val = *((uint32_t *)cp);
94 return (val);
95 }
96
97 static uint64_t
get_uint64(char * cp)98 get_uint64(char *cp)
99 {
100 uint64_t val = *((uint64_t *)cp);
101 return (val);
102 }
103
104 static char *
get_str(char * cp,uint_t len)105 get_str(char *cp, uint_t len)
106 {
107 char *str = kmem_alloc(len + 1, KM_SLEEP);
108
109 (void) strlcpy(str, cp, len + 1);
110
111 return (str);
112 }
113
114 static void
scope_list_free(list_t * scope_list)115 scope_list_free(list_t *scope_list)
116 {
117 scope_t *scope;
118
119 if (list_is_empty(scope_list)) {
120 list_destroy(scope_list);
121 return;
122 }
123
124 while ((scope = list_remove_head(scope_list)) != NULL) {
125 kmem_free(scope, sizeof (scope_t));
126 }
127
128 ASSERT(list_is_empty(scope_list));
129 list_destroy(scope_list);
130 }
131
132 static void
drhd_list_destroy(list_t * drhd_list)133 drhd_list_destroy(list_t *drhd_list)
134 {
135 drhd_t *drhd;
136
137 ASSERT(drhd_list);
138
139 if (list_is_empty(drhd_list)) {
140 list_destroy(drhd_list);
141 return;
142 }
143
144 while ((drhd = list_remove_head(drhd_list)) != NULL) {
145 scope_list_free(&(drhd->dr_scope_list));
146 kmem_free(drhd, sizeof (drhd_t));
147 }
148
149 ASSERT(list_is_empty(drhd_list));
150 list_destroy(drhd_list);
151 }
152
153 static void
rmrr_list_destroy(list_t * rmrr_list)154 rmrr_list_destroy(list_t *rmrr_list)
155 {
156 rmrr_t *rmrr;
157
158 ASSERT(rmrr_list);
159
160 if (list_is_empty(rmrr_list)) {
161 list_destroy(rmrr_list);
162 return;
163 }
164
165 while ((rmrr = list_remove_head(rmrr_list)) != NULL) {
166 scope_list_free(&(rmrr->rm_scope_list));
167 kmem_free(rmrr, sizeof (rmrr_t));
168 }
169
170 ASSERT(list_is_empty(rmrr_list));
171 list_destroy(rmrr_list);
172 }
173
174 /*
175 * parse_scope()
176 * parse a scope structure in the "raw" table
177 */
178 static scope_t *
parse_scope(char * shead)179 parse_scope(char *shead)
180 {
181 scope_t *scope;
182 char *phead;
183 int bus, dev, func;
184 uint8_t startbus;
185 uint8_t len;
186 int depth;
187
188 ASSERT(shead);
189
190 scope = kmem_zalloc(sizeof (scope_t), KM_SLEEP);
191 scope->scp_type = get_uint8(&shead[0]);
192 scope->scp_enumid = get_uint8(&shead[4]);
193
194 len = get_uint8(&shead[1]);
195 startbus = get_uint8(&shead[5]);
196 depth = (len - 6)/2;
197 ASSERT(depth >= 1);
198
199 phead = &shead[6];
200
201 bus = startbus;
202 dev = get_uint8(phead++);
203 func = get_uint8(phead++);
204
205 for (depth--; depth > 0; depth--) {
206 bus = pci_getb_func(bus, dev, func, PCI_BCNF_SECBUS);
207 dev = get_uint8(phead++);
208 func = get_uint8(phead++);
209 }
210
211 ASSERT(bus >= 0 && bus < 256);
212 ASSERT(dev >= 0 && dev < 32);
213 ASSERT(func >= 0 && func < 8);
214
215 /* ok we got the device BDF */
216 scope->scp_bus = bus;
217 scope->scp_dev = dev;
218 scope->scp_func = func;
219
220 return (scope);
221 }
222
223
224 /* setup the ioapic_drhd structure */
225 static void
ioapic_drhd_setup(void)226 ioapic_drhd_setup(void)
227 {
228 mutex_init(&(ioapic_drhd_lock), NULL, MUTEX_DEFAULT, NULL);
229
230 mutex_enter(&(ioapic_drhd_lock));
231 list_create(&(ioapic_drhd_list), sizeof (ioapic_drhd_t),
232 offsetof(ioapic_drhd_t, ioapic_node));
233 mutex_exit(&(ioapic_drhd_lock));
234 }
235
236 /* get ioapic source id for interrupt remapping */
237 static void
ioapic_drhd_insert(scope_t * scope,drhd_t * drhd)238 ioapic_drhd_insert(scope_t *scope, drhd_t *drhd)
239 {
240 ioapic_drhd_t *idt;
241
242 idt = kmem_zalloc(sizeof (ioapic_drhd_t), KM_SLEEP);
243 idt->ioapic_ioapicid = scope->scp_enumid;
244 idt->ioapic_sid = ((scope->scp_bus << 8) | (scope->scp_dev << 3) |
245 (scope->scp_func));
246 idt->ioapic_drhd = drhd;
247
248 mutex_enter(&ioapic_drhd_lock);
249 list_insert_tail(&ioapic_drhd_list, idt);
250 mutex_exit(&ioapic_drhd_lock);
251 }
252
253 static ioapic_drhd_t *
ioapic_drhd_lookup(int ioapicid)254 ioapic_drhd_lookup(int ioapicid)
255 {
256 ioapic_drhd_t *idt;
257
258 mutex_enter(&ioapic_drhd_lock);
259 idt = list_head(&ioapic_drhd_list);
260 for (; idt; idt = list_next(&ioapic_drhd_list, idt)) {
261 if (idt->ioapic_ioapicid == ioapicid) {
262 break;
263 }
264 }
265 mutex_exit(&ioapic_drhd_lock);
266
267 return (idt);
268 }
269
270 static void
ioapic_drhd_destroy(void)271 ioapic_drhd_destroy(void)
272 {
273 ioapic_drhd_t *idt;
274
275 mutex_enter(&ioapic_drhd_lock);
276 while (idt = list_remove_head(&ioapic_drhd_list)) {
277 kmem_free(idt, sizeof (ioapic_drhd_t));
278 }
279 list_destroy(&ioapic_drhd_list);
280 mutex_exit(&(ioapic_drhd_lock));
281
282 mutex_destroy(&(ioapic_drhd_lock));
283 }
284
285 /*
286 * parse_drhd()
287 * parse the drhd uints in dmar table
288 */
289 static int
parse_drhd(char * uhead,dmar_table_t * tbl)290 parse_drhd(char *uhead, dmar_table_t *tbl)
291 {
292 drhd_t *drhd;
293 int seg;
294 int len;
295 char *shead;
296 scope_t *scope;
297
298 ASSERT(uhead);
299 ASSERT(tbl);
300 ASSERT(get_uint16(&uhead[0]) == DMAR_DRHD);
301
302 seg = get_uint16(&uhead[6]);
303 if (seg < 0 || seg >= IMMU_MAXSEG) {
304 ddi_err(DER_WARN, NULL, "invalid segment# <%d>"
305 "in DRHD unit in ACPI DMAR table", seg);
306 return (DDI_FAILURE);
307 }
308
309 drhd = kmem_zalloc(sizeof (drhd_t), KM_SLEEP);
310 mutex_init(&(drhd->dr_lock), NULL, MUTEX_DEFAULT, NULL);
311 list_create(&(drhd->dr_scope_list), sizeof (scope_t),
312 offsetof(scope_t, scp_node));
313
314 len = get_uint16(&uhead[2]);
315 drhd->dr_include_all =
316 (get_uint8(&uhead[4]) & DMAR_INCLUDE_ALL) ? B_TRUE : B_FALSE;
317 drhd->dr_seg = seg;
318 drhd->dr_regs = get_uint64(&uhead[8]);
319
320 /*
321 * parse each scope.
322 */
323 shead = &uhead[16];
324 while (shead < &uhead[len - 1]) {
325 scope = parse_scope(shead);
326 if (scope == NULL) {
327 return (DDI_FAILURE);
328 }
329
330 if (scope->scp_type == DMAR_IOAPIC) {
331 ioapic_drhd_insert(scope, drhd);
332 }
333
334 list_insert_tail(&(drhd->dr_scope_list), scope);
335 shead += get_uint8(&shead[1]);
336 }
337
338 list_insert_tail(&(tbl->tbl_drhd_list[drhd->dr_seg]), drhd);
339
340 return (DDI_SUCCESS);
341 }
342
343 /*
344 * parse_rmrr()
345 * parse the rmrr units in dmar table
346 */
347 static int
parse_rmrr(char * uhead,dmar_table_t * tbl)348 parse_rmrr(char *uhead, dmar_table_t *tbl)
349 {
350 rmrr_t *rmrr;
351 int seg;
352 int len;
353 char *shead;
354 scope_t *scope;
355
356 ASSERT(uhead);
357 ASSERT(tbl);
358 ASSERT(get_uint16(&uhead[0]) == DMAR_RMRR);
359
360 seg = get_uint16(&uhead[6]);
361 if (seg < 0 || seg >= IMMU_MAXSEG) {
362 ddi_err(DER_WARN, NULL, "invalid segment# <%d>"
363 "in RMRR unit in ACPI DMAR table", seg);
364 return (DDI_FAILURE);
365 }
366
367 rmrr = kmem_zalloc(sizeof (rmrr_t), KM_SLEEP);
368 mutex_init(&(rmrr->rm_lock), NULL, MUTEX_DEFAULT, NULL);
369 list_create(&(rmrr->rm_scope_list), sizeof (scope_t),
370 offsetof(scope_t, scp_node));
371
372 /* RMRR region is [base,limit] */
373 len = get_uint16(&uhead[2]);
374 rmrr->rm_seg = get_uint16(&uhead[6]);
375 rmrr->rm_base = get_uint64(&uhead[8]);
376 rmrr->rm_limit = get_uint64(&uhead[16]);
377
378 if (rmrr->rm_base > rmrr->rm_limit) {
379 ddi_err(DER_WARN, NULL, "IMMU: BIOS bug detected: "
380 "RMRR: base (%lx) > limit (%lx)",
381 rmrr->rm_base, rmrr->rm_limit);
382 list_destroy(&(rmrr->rm_scope_list));
383 mutex_destroy(&(rmrr->rm_lock));
384 kmem_free(rmrr, sizeof (rmrr_t));
385 return (DDI_SUCCESS);
386 }
387
388 /*
389 * parse each scope in RMRR
390 */
391 shead = &uhead[24];
392 while (shead < &uhead[len - 1]) {
393 scope = parse_scope(shead);
394 if (scope == NULL) {
395 return (DDI_FAILURE);
396 }
397 list_insert_tail(&(rmrr->rm_scope_list), scope);
398 shead += get_uint8(&shead[1]);
399 }
400
401 list_insert_tail(&(tbl->tbl_rmrr_list[rmrr->rm_seg]), rmrr);
402
403 return (DDI_SUCCESS);
404 }
405
406 #define TBL_OEM_ID_SZ (6)
407 #define TBL_OEM_TBLID_SZ (8)
408
409 /*
410 * parse the "raw" DMAR table and convert it
411 * into a useful form.
412 */
413 static int
dmar_parse(dmar_table_t ** tblpp,char * raw)414 dmar_parse(dmar_table_t **tblpp, char *raw)
415 {
416 char *uhead;
417 dmar_table_t *tbl;
418 int i;
419 char *unmstr;
420
421 ASSERT(raw);
422 ASSERT(tblpp);
423
424 *tblpp = NULL;
425
426 /*
427 * do a sanity check. make sure the raw table
428 * has the right signature
429 */
430 if (raw[0] != 'D' || raw[1] != 'M' ||
431 raw[2] != 'A' || raw[3] != 'R') {
432 ddi_err(DER_WARN, NULL, "IOMMU ACPI "
433 "signature != \"DMAR\"");
434 return (DDI_FAILURE);
435 }
436
437 /*
438 * the platform has intel iommu, create processed ACPI struct
439 */
440 tbl = kmem_zalloc(sizeof (dmar_table_t), KM_SLEEP);
441 mutex_init(&(tbl->tbl_lock), NULL, MUTEX_DEFAULT, NULL);
442
443 tbl->tbl_raw = raw;
444
445 /*
446 * Note we explicitly show offsets for clarity
447 */
448 tbl->tbl_rawlen = get_uint32(&raw[4]);
449
450 /* XXX TO DO verify checksum of table */
451 tbl->tbl_oem_id = get_str(&raw[10], TBL_OEM_ID_SZ);
452 tbl->tbl_oem_tblid = get_str(&raw[16], TBL_OEM_TBLID_SZ);
453 tbl->tbl_oem_rev = get_uint32(&raw[24]);
454 tbl->tbl_haw = get_uint8(&raw[36]) + 1;
455 tbl->tbl_intrmap = (get_uint8(&raw[37]) & DMAR_INTRMAP_SUPPORT)
456 ? B_TRUE : B_FALSE;
457
458 /* create lists for DRHD and RMRR */
459 for (i = 0; i < IMMU_MAXSEG; i++) {
460 list_create(&(tbl->tbl_drhd_list[i]), sizeof (drhd_t),
461 offsetof(drhd_t, dr_node));
462 list_create(&(tbl->tbl_rmrr_list[i]), sizeof (rmrr_t),
463 offsetof(rmrr_t, rm_node));
464 }
465
466 ioapic_drhd_setup();
467
468 /*
469 * parse each unit. Currently only DRHD and RMRR types
470 * are parsed. We ignore all other types of units.
471 */
472 uhead = &raw[48];
473 while (uhead < &raw[tbl->tbl_rawlen - 1]) {
474 unmstr = NULL;
475 switch (get_uint16(uhead)) {
476 case DMAR_DRHD:
477 if (parse_drhd(uhead, tbl) != DDI_SUCCESS) {
478 goto failed;
479 }
480 break;
481 case DMAR_RMRR:
482 if (parse_rmrr(uhead, tbl) != DDI_SUCCESS) {
483 goto failed;
484 }
485 break;
486 case DMAR_ATSR:
487 unmstr = "ATSR";
488 break;
489 case DMAR_RHSA:
490 unmstr = "RHSA";
491 break;
492 case DMAR_ANDD:
493 unmstr = "ANDD";
494 break;
495 case DMAR_SATC:
496 unmstr = "SATC";
497 break;
498 case DMAR_SIDP:
499 unmstr = "SIDP";
500 break;
501 default:
502 unmstr = "unknown unit type";
503 break;
504 }
505 if (unmstr) {
506 ddi_err(DER_NOTE, NULL, "DMAR ACPI table: "
507 "skipping unsupported unit type %u: %s",
508 get_uint16(uhead), unmstr);
509 }
510 uhead += get_uint16(&uhead[2]);
511 }
512
513 *tblpp = tbl;
514 return (DDI_SUCCESS);
515
516 failed:
517 dmar_table_destroy(tbl);
518 return (DDI_FAILURE);
519 }
520
521 static char *
scope_type(int devtype)522 scope_type(int devtype)
523 {
524 char *typestr;
525
526 switch (devtype) {
527 case DMAR_ENDPOINT:
528 typestr = "endpoint-device";
529 break;
530 case DMAR_SUBTREE:
531 typestr = "subtree-device";
532 break;
533 case DMAR_IOAPIC:
534 typestr = "IOAPIC";
535 break;
536 case DMAR_HPET:
537 typestr = "HPET";
538 break;
539 default:
540 typestr = "Unknown device";
541 break;
542 }
543
544 return (typestr);
545 }
546
547 static void
print_scope_list(list_t * scope_list)548 print_scope_list(list_t *scope_list)
549 {
550 scope_t *scope;
551
552 if (list_is_empty(scope_list))
553 return;
554
555 ddi_err(DER_CONT, NULL, "\tdevice list:\n");
556
557 for (scope = list_head(scope_list); scope;
558 scope = list_next(scope_list, scope)) {
559 ddi_err(DER_CONT, NULL, "\t\ttype = %s\n",
560 scope_type(scope->scp_type));
561 ddi_err(DER_CONT, NULL, "\n\t\tbus = %d\n",
562 scope->scp_bus);
563 ddi_err(DER_CONT, NULL, "\t\tdev = %d\n",
564 scope->scp_dev);
565 ddi_err(DER_CONT, NULL, "\t\tfunc = %d\n",
566 scope->scp_func);
567 }
568 }
569
570 static void
print_drhd_list(list_t * drhd_list)571 print_drhd_list(list_t *drhd_list)
572 {
573 drhd_t *drhd;
574
575 if (list_is_empty(drhd_list))
576 return;
577
578 ddi_err(DER_CONT, NULL, "\ndrhd list:\n");
579
580 for (drhd = list_head(drhd_list); drhd;
581 drhd = list_next(drhd_list, drhd)) {
582
583 ddi_err(DER_CONT, NULL, "\n\tsegment = %d\n",
584 drhd->dr_seg);
585 ddi_err(DER_CONT, NULL, "\treg_base = 0x%" PRIx64 "\n",
586 drhd->dr_regs);
587 ddi_err(DER_CONT, NULL, "\tinclude_all = %s\n",
588 drhd->dr_include_all == B_TRUE ? "TRUE" : "FALSE");
589 ddi_err(DER_CONT, NULL, "\tdip = 0x%p\n",
590 (void *)drhd->dr_dip);
591
592 print_scope_list(&(drhd->dr_scope_list));
593 }
594 }
595
596
597 static void
print_rmrr_list(list_t * rmrr_list)598 print_rmrr_list(list_t *rmrr_list)
599 {
600 rmrr_t *rmrr;
601
602 if (list_is_empty(rmrr_list))
603 return;
604
605 ddi_err(DER_CONT, NULL, "\nrmrr list:\n");
606
607 for (rmrr = list_head(rmrr_list); rmrr;
608 rmrr = list_next(rmrr_list, rmrr)) {
609
610 ddi_err(DER_CONT, NULL, "\n\tsegment = %d\n",
611 rmrr->rm_seg);
612 ddi_err(DER_CONT, NULL, "\tbase = 0x%lx\n",
613 rmrr->rm_base);
614 ddi_err(DER_CONT, NULL, "\tlimit = 0x%lx\n",
615 rmrr->rm_limit);
616
617 print_scope_list(&(rmrr->rm_scope_list));
618 }
619 }
620
621 /*
622 * print DMAR table
623 */
624 static void
dmar_table_print(dmar_table_t * tbl)625 dmar_table_print(dmar_table_t *tbl)
626 {
627 int i;
628
629 if (immu_dmar_print == B_FALSE) {
630 return;
631 }
632
633 /* print the title */
634 ddi_err(DER_CONT, NULL, "#### Start of dmar_table ####\n");
635 ddi_err(DER_CONT, NULL, "\thaw = %d\n", tbl->tbl_haw);
636 ddi_err(DER_CONT, NULL, "\tintr_remap = %s\n",
637 tbl->tbl_intrmap == B_TRUE ? "<true>" : "<false>");
638
639 /* print drhd list */
640 for (i = 0; i < IMMU_MAXSEG; i++) {
641 print_drhd_list(&(tbl->tbl_drhd_list[i]));
642 }
643
644
645 /* print rmrr list */
646 for (i = 0; i < IMMU_MAXSEG; i++) {
647 print_rmrr_list(&(tbl->tbl_rmrr_list[i]));
648 }
649
650 ddi_err(DER_CONT, NULL, "#### END of dmar_table ####\n");
651 }
652
653 static void
drhd_devi_create(drhd_t * drhd,int unit)654 drhd_devi_create(drhd_t *drhd, int unit)
655 {
656 struct ddi_parent_private_data *pdptr;
657 struct regspec reg;
658 dev_info_t *dip;
659
660 dip = ddi_add_child(root_devinfo, IMMU_UNIT_NAME,
661 DEVI_SID_NODEID, unit);
662
663 drhd->dr_dip = dip;
664
665 reg.regspec_bustype = 0;
666 reg.regspec_addr = drhd->dr_regs;
667 reg.regspec_size = IMMU_REGSZ;
668
669 /*
670 * update the reg properties
671 *
672 * reg property will be used for register
673 * set access
674 *
675 * refer to the bus_map of root nexus driver
676 * I/O or memory mapping:
677 *
678 * <bustype=0, addr=x, len=x>: memory
679 * <bustype=1, addr=x, len=x>: i/o
680 * <bustype>1, addr=0, len=x>: x86-compatibility i/o
681 */
682 (void) ndi_prop_update_int_array(DDI_DEV_T_NONE,
683 dip, "reg", (int *)®,
684 sizeof (struct regspec) / sizeof (int));
685
686 /*
687 * This is an artificially constructed dev_info, and we
688 * need to set a few more things to be able to use it
689 * for ddi_dma_alloc_handle/free_handle.
690 */
691 ddi_set_driver(dip, ddi_get_driver(ddi_root_node()));
692 DEVI(dip)->devi_bus_dma_allochdl =
693 DEVI(ddi_get_driver((ddi_root_node())));
694
695 pdptr = kmem_zalloc(sizeof (struct ddi_parent_private_data)
696 + sizeof (struct regspec), KM_SLEEP);
697 pdptr->par_nreg = 1;
698 pdptr->par_reg = (struct regspec *)(pdptr + 1);
699 pdptr->par_reg->regspec_bustype = 0;
700 pdptr->par_reg->regspec_addr = drhd->dr_regs;
701 pdptr->par_reg->regspec_size = IMMU_REGSZ;
702 ddi_set_parent_data(dip, pdptr);
703 }
704
705 /*
706 * dmar_devinfos_create()
707 *
708 * create the dev_info node in the device tree,
709 * the info node is a nuxus child of the root
710 * nexus
711 */
712 static void
dmar_devinfos_create(dmar_table_t * tbl)713 dmar_devinfos_create(dmar_table_t *tbl)
714 {
715 list_t *drhd_list;
716 drhd_t *drhd;
717 int i, unit;
718
719 for (i = 0; i < IMMU_MAXSEG; i++) {
720
721 drhd_list = &(tbl->tbl_drhd_list[i]);
722
723 if (list_is_empty(drhd_list))
724 continue;
725
726 drhd = list_head(drhd_list);
727 for (unit = 0; drhd;
728 drhd = list_next(drhd_list, drhd), unit++) {
729 drhd_devi_create(drhd, unit);
730 }
731 }
732 }
733
734 static void
drhd_devi_destroy(drhd_t * drhd)735 drhd_devi_destroy(drhd_t *drhd)
736 {
737 dev_info_t *dip;
738
739 dip = drhd->dr_dip;
740 ASSERT(dip);
741
742 ndi_devi_enter(root_devinfo);
743 if (ndi_devi_offline(dip, NDI_DEVI_REMOVE) != DDI_SUCCESS) {
744 ddi_err(DER_WARN, dip, "Failed to destroy");
745 }
746 ndi_devi_exit(root_devinfo);
747 drhd->dr_dip = NULL;
748 }
749
750 /*
751 * dmar_devi_destroy()
752 *
753 * destroy dev_info nodes for all drhd units
754 */
755 static void
dmar_devi_destroy(dmar_table_t * tbl)756 dmar_devi_destroy(dmar_table_t *tbl)
757 {
758 drhd_t *drhd;
759 list_t *drhd_list;
760 int i;
761
762 for (i = 0; i < IMMU_MAXSEG; i++) {
763 drhd_list = &(tbl->tbl_drhd_list[i]);
764 if (list_is_empty(drhd_list))
765 continue;
766
767 drhd = list_head(drhd_list);
768 for (; drhd; drhd = list_next(drhd_list, drhd)) {
769 drhd_devi_destroy(drhd);
770 }
771 }
772 }
773
774 static int
match_bdf(dev_info_t * ddip,void * arg)775 match_bdf(dev_info_t *ddip, void *arg)
776 {
777 immu_arg_t *imarg = (immu_arg_t *)arg;
778 immu_devi_t *immu_devi;
779
780 ASSERT(ddip);
781 ASSERT(imarg);
782 ASSERT(imarg->ima_seg == 0);
783 ASSERT(imarg->ima_bus >= 0);
784 ASSERT(imarg->ima_devfunc >= 0);
785 ASSERT(imarg->ima_ddip == NULL);
786
787 /* rdip can be NULL */
788
789 mutex_enter(&(DEVI(ddip)->devi_lock));
790
791 immu_devi = IMMU_DEVI(ddip);
792 ASSERT(immu_devi);
793
794 if (immu_devi->imd_seg == imarg->ima_seg &&
795 immu_devi->imd_bus == imarg->ima_bus &&
796 immu_devi->imd_devfunc == imarg->ima_devfunc) {
797 imarg->ima_ddip = ddip;
798 }
799
800 mutex_exit(&(DEVI(ddip)->devi_lock));
801
802 return (imarg->ima_ddip ? DDI_WALK_TERMINATE : DDI_WALK_CONTINUE);
803 }
804 static void
dmar_table_destroy(dmar_table_t * tbl)805 dmar_table_destroy(dmar_table_t *tbl)
806 {
807 int i;
808
809 ASSERT(tbl);
810
811 /* destroy lists for DRHD and RMRR */
812 for (i = 0; i < IMMU_MAXSEG; i++) {
813 rmrr_list_destroy(&(tbl->tbl_rmrr_list[i]));
814 drhd_list_destroy(&(tbl->tbl_drhd_list[i]));
815 }
816
817 /* free strings */
818 kmem_free(tbl->tbl_oem_tblid, TBL_OEM_TBLID_SZ + 1);
819 kmem_free(tbl->tbl_oem_id, TBL_OEM_ID_SZ + 1);
820 tbl->tbl_raw = NULL; /* raw ACPI table doesn't have to be freed */
821 mutex_destroy(&(tbl->tbl_lock));
822 kmem_free(tbl, sizeof (dmar_table_t));
823 }
824
825 /*
826 * #########################################################################
827 * Functions exported by dmar.c
828 * This file deals with reading and processing the DMAR ACPI table
829 * #########################################################################
830 */
831
832 /*
833 * immu_dmar_setup()
834 * Check if the system has a DMAR ACPI table. If yes, the system
835 * has Intel IOMMU hardware
836 */
837 int
immu_dmar_setup(void)838 immu_dmar_setup(void)
839 {
840 if (AcpiGetTable("DMAR", 1, (ACPI_TABLE_HEADER **)&dmar_raw) != AE_OK) {
841 ddi_err(DER_LOG, NULL,
842 "No DMAR ACPI table. No Intel IOMMU present\n");
843 dmar_raw = NULL;
844 return (DDI_FAILURE);
845 }
846 ASSERT(dmar_raw);
847 return (DDI_SUCCESS);
848 }
849
850 /*
851 * immu_dmar_parse()
852 * Called by immu.c to parse and convert "raw" ACPI DMAR table
853 */
854 int
immu_dmar_parse(void)855 immu_dmar_parse(void)
856 {
857 dmar_table_t *tbl = NULL;
858
859 /* we should already have found the "raw" table */
860 ASSERT(dmar_raw);
861
862 ddi_err(DER_CONT, NULL, "?Processing DMAR ACPI table\n");
863
864 dmar_table = NULL;
865
866 /*
867 * parse DMAR ACPI table
868 */
869 if (dmar_parse(&tbl, dmar_raw) != DDI_SUCCESS) {
870 ASSERT(tbl == NULL);
871 return (DDI_FAILURE);
872 }
873
874 ASSERT(tbl);
875
876 /*
877 * create one devinfo for every drhd unit
878 * in the DMAR table
879 */
880 dmar_devinfos_create(tbl);
881
882 /*
883 * print the dmar table if the debug option is set
884 */
885 dmar_table_print(tbl);
886
887 dmar_table = tbl;
888
889 return (DDI_SUCCESS);
890 }
891
892 void
immu_dmar_startup(void)893 immu_dmar_startup(void)
894 {
895 /* nothing to do */
896 }
897
898 void
immu_dmar_shutdown(void)899 immu_dmar_shutdown(void)
900 {
901 /* nothing to do */
902 }
903
904 void
immu_dmar_destroy(void)905 immu_dmar_destroy(void)
906 {
907 dmar_devi_destroy(dmar_table);
908 dmar_table_destroy(dmar_table);
909 ioapic_drhd_destroy();
910 dmar_table = NULL;
911 dmar_raw = NULL;
912 }
913
914 boolean_t
immu_dmar_blacklisted(char ** strptr,uint_t nstrs)915 immu_dmar_blacklisted(char **strptr, uint_t nstrs)
916 {
917 dmar_table_t *tbl = dmar_table;
918 int i;
919 char oem_rev[IMMU_MAXNAMELEN];
920
921 ASSERT(tbl);
922
923 ASSERT((strptr == NULL) ^ (nstrs != 0));
924
925 /*
926 * Must be a minimum of 4
927 */
928 if (nstrs < 4) {
929 return (B_FALSE);
930 }
931
932 ddi_err(DER_CONT, NULL, "?System DMAR ACPI table information:\n");
933 ddi_err(DER_CONT, NULL, "?OEM-ID = <%s>\n", tbl->tbl_oem_id);
934 ddi_err(DER_CONT, NULL, "?Table-ID = <%s>\n", tbl->tbl_oem_tblid);
935 (void) snprintf(oem_rev, sizeof (oem_rev), "%d", tbl->tbl_oem_rev);
936 ddi_err(DER_CONT, NULL, "?Revision = <%s>\n", oem_rev);
937
938 for (i = 0; nstrs - i >= 4; i++) {
939 if (strcmp(*strptr++, "DMAR") == 0) {
940 if (strcmp(*strptr++, tbl->tbl_oem_id) == 0 &&
941 (*strptr[0] == '\0' ||
942 strcmp(*strptr++, tbl->tbl_oem_tblid) == 0) &&
943 (*strptr[0] == '\0' ||
944 strcmp(*strptr++, oem_rev) == 0)) {
945 return (B_TRUE);
946 }
947 i += 3; /* for loops adds 1 as well, so only 3 here */
948 }
949 }
950 return (B_FALSE);
951 }
952
953 void
immu_dmar_rmrr_map(void)954 immu_dmar_rmrr_map(void)
955 {
956 int seg;
957 dev_info_t *rdip;
958 scope_t *scope;
959 rmrr_t *rmrr;
960 dmar_table_t *tbl;
961
962 ASSERT(dmar_table);
963
964 tbl = dmar_table;
965
966 /* called during boot, when kernel is single threaded. No lock */
967
968 /*
969 * for each segment, walk the rmrr list looking for an exact match
970 */
971 for (seg = 0; seg < IMMU_MAXSEG; seg++) {
972 rmrr = list_head(&(tbl->tbl_rmrr_list)[seg]);
973 for (; rmrr; rmrr = list_next(&(tbl->tbl_rmrr_list)[seg],
974 rmrr)) {
975
976 /*
977 * try to match BDF *exactly* to a device scope.
978 */
979 scope = list_head(&(rmrr->rm_scope_list));
980 for (; scope;
981 scope = list_next(&(rmrr->rm_scope_list), scope)) {
982 immu_arg_t imarg = {0};
983 memrng_t mrng = {0};
984
985 /* PCI endpoint devices only */
986 if (scope->scp_type != DMAR_ENDPOINT)
987 continue;
988
989 imarg.ima_seg = seg;
990 imarg.ima_bus = scope->scp_bus;
991 imarg.ima_devfunc =
992 IMMU_PCI_DEVFUNC(scope->scp_dev,
993 scope->scp_func);
994 imarg.ima_ddip = NULL;
995 imarg.ima_rdip = NULL;
996
997 ASSERT(root_devinfo);
998 /* XXX should be optimized */
999 ndi_devi_enter(root_devinfo);
1000 ddi_walk_devs(ddi_get_child(root_devinfo),
1001 match_bdf, &imarg);
1002 ndi_devi_exit(root_devinfo);
1003
1004 if (imarg.ima_ddip == NULL) {
1005 ddi_err(DER_WARN, NULL,
1006 "No dip found for "
1007 "bus=0x%x, dev=0x%x, func= 0x%x",
1008 scope->scp_bus, scope->scp_dev,
1009 scope->scp_func);
1010 continue;
1011 }
1012
1013 rdip = imarg.ima_ddip;
1014 /*
1015 * This address must be in the BIOS reserved
1016 * map
1017 */
1018 if (!address_in_memlist(bios_rsvd,
1019 (uint64_t)rmrr->rm_base, rmrr->rm_limit -
1020 rmrr->rm_base + 1)) {
1021 ddi_err(DER_WARN, rdip, "RMRR range "
1022 " [0x%" PRIx64 " - 0x%" PRIx64 "]"
1023 " not in BIOS reserved map",
1024 rmrr->rm_base, rmrr->rm_limit);
1025 }
1026
1027 /* XXX could be more efficient */
1028 memlist_read_lock();
1029 if (address_in_memlist(phys_install,
1030 (uint64_t)rmrr->rm_base, rmrr->rm_limit -
1031 rmrr->rm_base + 1)) {
1032 ddi_err(DER_WARN, rdip, "RMRR range "
1033 " [0x%" PRIx64 " - 0x%" PRIx64 "]"
1034 " is in physinstall map",
1035 rmrr->rm_base, rmrr->rm_limit);
1036 }
1037 memlist_read_unlock();
1038
1039 (void) immu_dvma_device_setup(rdip, 0);
1040
1041 ddi_err(DER_LOG, rdip,
1042 "IMMU: Mapping RMRR range "
1043 "[0x%" PRIx64 " - 0x%"PRIx64 "]",
1044 rmrr->rm_base, rmrr->rm_limit);
1045
1046 mrng.mrng_start =
1047 IMMU_ROUNDOWN((uintptr_t)rmrr->rm_base);
1048 mrng.mrng_npages =
1049 IMMU_ROUNDUP((uintptr_t)rmrr->rm_limit -
1050 (uintptr_t)rmrr->rm_base + 1) /
1051 IMMU_PAGESIZE;
1052
1053 (void) immu_map_memrange(rdip, &mrng);
1054 }
1055 }
1056 }
1057
1058 }
1059
1060 immu_t *
immu_dmar_get_immu(dev_info_t * rdip)1061 immu_dmar_get_immu(dev_info_t *rdip)
1062 {
1063 int seg;
1064 int tlevel;
1065 int level;
1066 drhd_t *drhd;
1067 drhd_t *tdrhd;
1068 scope_t *scope;
1069 dmar_table_t *tbl;
1070
1071 ASSERT(dmar_table);
1072
1073 tbl = dmar_table;
1074
1075 mutex_enter(&(tbl->tbl_lock));
1076
1077 /*
1078 * for each segment, walk the drhd list looking for an exact match
1079 */
1080 for (seg = 0; seg < IMMU_MAXSEG; seg++) {
1081 drhd = list_head(&(tbl->tbl_drhd_list)[seg]);
1082 for (; drhd; drhd = list_next(&(tbl->tbl_drhd_list)[seg],
1083 drhd)) {
1084
1085 /*
1086 * we are currently searching for exact matches so
1087 * skip "include all" (catchall) and subtree matches
1088 */
1089 if (drhd->dr_include_all == B_TRUE)
1090 continue;
1091
1092 /*
1093 * try to match BDF *exactly* to a device scope.
1094 */
1095 scope = list_head(&(drhd->dr_scope_list));
1096 for (; scope;
1097 scope = list_next(&(drhd->dr_scope_list), scope)) {
1098 immu_arg_t imarg = {0};
1099
1100 /* PCI endpoint devices only */
1101 if (scope->scp_type != DMAR_ENDPOINT)
1102 continue;
1103
1104 imarg.ima_seg = seg;
1105 imarg.ima_bus = scope->scp_bus;
1106 imarg.ima_devfunc =
1107 IMMU_PCI_DEVFUNC(scope->scp_dev,
1108 scope->scp_func);
1109 imarg.ima_ddip = NULL;
1110 imarg.ima_rdip = rdip;
1111 level = 0;
1112 if (immu_walk_ancestor(rdip, NULL, match_bdf,
1113 &imarg, &level, IMMU_FLAGS_DONTPASS)
1114 != DDI_SUCCESS) {
1115 /* skip - nothing else we can do */
1116 continue;
1117 }
1118
1119 /* Should have walked only 1 level i.e. rdip */
1120 ASSERT(level == 1);
1121
1122 if (imarg.ima_ddip) {
1123 ASSERT(imarg.ima_ddip == rdip);
1124 goto found;
1125 }
1126 }
1127 }
1128 }
1129
1130 /*
1131 * walk the drhd list looking for subtree match
1132 * i.e. is the device a descendant of a devscope BDF.
1133 * We want the lowest subtree.
1134 */
1135 tdrhd = NULL;
1136 tlevel = 0;
1137 for (seg = 0; seg < IMMU_MAXSEG; seg++) {
1138 drhd = list_head(&(tbl->tbl_drhd_list)[seg]);
1139 for (; drhd; drhd = list_next(&(tbl->tbl_drhd_list)[seg],
1140 drhd)) {
1141
1142 /* looking for subtree match */
1143 if (drhd->dr_include_all == B_TRUE)
1144 continue;
1145
1146 /*
1147 * try to match the device scope
1148 */
1149 scope = list_head(&(drhd->dr_scope_list));
1150 for (; scope;
1151 scope = list_next(&(drhd->dr_scope_list), scope)) {
1152 immu_arg_t imarg = {0};
1153
1154 /* PCI subtree only */
1155 if (scope->scp_type != DMAR_SUBTREE)
1156 continue;
1157
1158 imarg.ima_seg = seg;
1159 imarg.ima_bus = scope->scp_bus;
1160 imarg.ima_devfunc =
1161 IMMU_PCI_DEVFUNC(scope->scp_dev,
1162 scope->scp_func);
1163
1164 imarg.ima_ddip = NULL;
1165 imarg.ima_rdip = rdip;
1166 level = 0;
1167 if (immu_walk_ancestor(rdip, NULL, match_bdf,
1168 &imarg, &level, 0) != DDI_SUCCESS) {
1169 /* skip - nothing else we can do */
1170 continue;
1171 }
1172
1173 /* should have walked 1 level i.e. rdip */
1174 ASSERT(level > 0);
1175
1176 /* look for lowest ancestor matching drhd */
1177 if (imarg.ima_ddip && (tdrhd == NULL ||
1178 level < tlevel)) {
1179 tdrhd = drhd;
1180 tlevel = level;
1181 }
1182 }
1183 }
1184 }
1185
1186 if ((drhd = tdrhd) != NULL) {
1187 goto found;
1188 }
1189
1190 for (seg = 0; seg < IMMU_MAXSEG; seg++) {
1191 drhd = list_head(&(tbl->tbl_drhd_list[seg]));
1192 for (; drhd; drhd = list_next(&(tbl->tbl_drhd_list)[seg],
1193 drhd)) {
1194 /* Look for include all */
1195 if (drhd->dr_include_all == B_TRUE) {
1196 break;
1197 }
1198 }
1199 }
1200
1201 /*FALLTHRU*/
1202
1203 found:
1204 mutex_exit(&(tbl->tbl_lock));
1205
1206 /*
1207 * No drhd (dmar unit) found for this device in the ACPI DMAR tables.
1208 * This may happen with buggy versions of BIOSes. Just warn instead
1209 * of panic as we don't want whole system to go down because of one
1210 * device.
1211 */
1212 if (drhd == NULL) {
1213 ddi_err(DER_WARN, rdip, "can't find Intel IOMMU unit for "
1214 "device in ACPI DMAR table.");
1215 return (NULL);
1216 }
1217
1218 return (drhd->dr_immu);
1219 }
1220
1221 dev_info_t *
immu_dmar_unit_dip(void * dmar_unit)1222 immu_dmar_unit_dip(void *dmar_unit)
1223 {
1224 drhd_t *drhd = (drhd_t *)dmar_unit;
1225 return (drhd->dr_dip);
1226 }
1227
1228 void *
immu_dmar_walk_units(int seg,void * dmar_unit)1229 immu_dmar_walk_units(int seg, void *dmar_unit)
1230 {
1231 list_t *drhd_list;
1232 drhd_t *drhd = (drhd_t *)dmar_unit;
1233
1234 drhd_list = &(dmar_table->tbl_drhd_list[seg]);
1235
1236 if (drhd == NULL) {
1237 return ((void *)list_head(drhd_list));
1238 } else {
1239 return ((void *)list_next(drhd_list, drhd));
1240 }
1241 }
1242
1243 void
immu_dmar_set_immu(void * dmar_unit,immu_t * immu)1244 immu_dmar_set_immu(void *dmar_unit, immu_t *immu)
1245 {
1246 drhd_t *drhd = (drhd_t *)dmar_unit;
1247
1248 ASSERT(drhd);
1249 ASSERT(immu);
1250
1251 drhd->dr_immu = immu;
1252 }
1253
1254 boolean_t
immu_dmar_intrmap_supported(void)1255 immu_dmar_intrmap_supported(void)
1256 {
1257 ASSERT(dmar_table);
1258 return (dmar_table->tbl_intrmap);
1259 }
1260
1261 /* for a given ioapicid, find the source id and immu */
1262 uint16_t
immu_dmar_ioapic_sid(int ioapic_ix)1263 immu_dmar_ioapic_sid(int ioapic_ix)
1264 {
1265 ioapic_drhd_t *idt;
1266
1267 idt = ioapic_drhd_lookup(psm_get_ioapicid(ioapic_ix));
1268 if (idt == NULL) {
1269 ddi_err(DER_PANIC, NULL, "cannot determine source-id for "
1270 "IOAPIC (index = %d)", ioapic_ix);
1271 /*NOTREACHED*/
1272 }
1273
1274 return (idt->ioapic_sid);
1275 }
1276
1277 /* for a given ioapicid, find the source id and immu */
1278 immu_t *
immu_dmar_ioapic_immu(int ioapic_ix)1279 immu_dmar_ioapic_immu(int ioapic_ix)
1280 {
1281 ioapic_drhd_t *idt;
1282
1283 idt = ioapic_drhd_lookup(psm_get_ioapicid(ioapic_ix));
1284 if (idt) {
1285 return (idt->ioapic_drhd ? idt->ioapic_drhd->dr_immu : NULL);
1286 }
1287 return (NULL);
1288 }
1289