xref: /illumos-gate/usr/src/uts/i86pc/io/immu_dmar.c (revision 3fe80ca4a1f8a033d672a9a2e6e4babac651205a)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 /*
22  * Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved.
23  */
24 
25 /*
26  * Copyright (c) 2009, Intel Corporation.
27  * All rights reserved.
28  */
29 
30 /*
31  * Copyright 2023 Oxide Computer Company
32  */
33 
34 #include <sys/debug.h>
35 #include <sys/sysmacros.h>
36 #include <sys/types.h>
37 #include <sys/kmem.h>
38 #include <sys/sunddi.h>
39 #include <sys/list.h>
40 #include <sys/pci.h>
41 #include <sys/pci_cfgspace.h>
42 #include <sys/pci_impl.h>
43 #include <sys/sunndi.h>
44 #include <sys/ksynch.h>
45 #include <sys/cmn_err.h>
46 #include <sys/bootconf.h>
47 #include <sys/int_fmtio.h>
48 #include <sys/smbios.h>
49 #include <sys/apic.h>
50 #include <sys/acpi/acpi.h>
51 #include <sys/acpica.h>
52 #include <sys/immu.h>
53 #include <sys/smp_impldefs.h>
54 
55 static void dmar_table_destroy(dmar_table_t *tbl);
56 
57 /*
58  * internal global variables
59  */
60 static char	*dmar_raw;		/* raw DMAR ACPI table */
61 static dmar_table_t *dmar_table;	/* converted form of DMAR table */
62 
63 /*
64  * global variables exported outside this file
65  */
66 boolean_t dmar_print = B_FALSE;
67 kmutex_t ioapic_drhd_lock;
68 list_t ioapic_drhd_list;
69 
70 /* ######################################################################### */
71 
72 /*
73  * helper functions to read the "raw" DMAR table
74  */
75 
76 static uint8_t
get_uint8(char * cp)77 get_uint8(char *cp)
78 {
79 	uint8_t val = *((uint8_t *)cp);
80 	return (val);
81 }
82 
83 static uint16_t
get_uint16(char * cp)84 get_uint16(char *cp)
85 {
86 	uint16_t val = *((uint16_t *)cp);
87 	return (val);
88 }
89 
90 static uint32_t
get_uint32(char * cp)91 get_uint32(char *cp)
92 {
93 	uint32_t val = *((uint32_t *)cp);
94 	return (val);
95 }
96 
97 static uint64_t
get_uint64(char * cp)98 get_uint64(char *cp)
99 {
100 	uint64_t val = *((uint64_t *)cp);
101 	return (val);
102 }
103 
104 static char *
get_str(char * cp,uint_t len)105 get_str(char *cp, uint_t len)
106 {
107 	char *str = kmem_alloc(len + 1, KM_SLEEP);
108 
109 	(void) strlcpy(str, cp, len + 1);
110 
111 	return (str);
112 }
113 
114 static void
scope_list_free(list_t * scope_list)115 scope_list_free(list_t *scope_list)
116 {
117 	scope_t *scope;
118 
119 	if (list_is_empty(scope_list)) {
120 		list_destroy(scope_list);
121 		return;
122 	}
123 
124 	while ((scope = list_remove_head(scope_list)) != NULL) {
125 		kmem_free(scope, sizeof (scope_t));
126 	}
127 
128 	ASSERT(list_is_empty(scope_list));
129 	list_destroy(scope_list);
130 }
131 
132 static void
drhd_list_destroy(list_t * drhd_list)133 drhd_list_destroy(list_t *drhd_list)
134 {
135 	drhd_t *drhd;
136 
137 	ASSERT(drhd_list);
138 
139 	if (list_is_empty(drhd_list)) {
140 		list_destroy(drhd_list);
141 		return;
142 	}
143 
144 	while ((drhd = list_remove_head(drhd_list)) != NULL) {
145 		scope_list_free(&(drhd->dr_scope_list));
146 		kmem_free(drhd, sizeof (drhd_t));
147 	}
148 
149 	ASSERT(list_is_empty(drhd_list));
150 	list_destroy(drhd_list);
151 }
152 
153 static void
rmrr_list_destroy(list_t * rmrr_list)154 rmrr_list_destroy(list_t *rmrr_list)
155 {
156 	rmrr_t *rmrr;
157 
158 	ASSERT(rmrr_list);
159 
160 	if (list_is_empty(rmrr_list)) {
161 		list_destroy(rmrr_list);
162 		return;
163 	}
164 
165 	while ((rmrr = list_remove_head(rmrr_list)) != NULL) {
166 		scope_list_free(&(rmrr->rm_scope_list));
167 		kmem_free(rmrr, sizeof (rmrr_t));
168 	}
169 
170 	ASSERT(list_is_empty(rmrr_list));
171 	list_destroy(rmrr_list);
172 }
173 
174 /*
175  * parse_scope()
176  *      parse a scope structure in the "raw" table
177  */
178 static scope_t *
parse_scope(char * shead)179 parse_scope(char *shead)
180 {
181 	scope_t *scope;
182 	char *phead;
183 	int bus, dev, func;
184 	uint8_t startbus;
185 	uint8_t len;
186 	int depth;
187 
188 	ASSERT(shead);
189 
190 	scope = kmem_zalloc(sizeof (scope_t), KM_SLEEP);
191 	scope->scp_type = get_uint8(&shead[0]);
192 	scope->scp_enumid = get_uint8(&shead[4]);
193 
194 	len = get_uint8(&shead[1]);
195 	startbus = get_uint8(&shead[5]);
196 	depth = (len - 6)/2;
197 	ASSERT(depth >= 1);
198 
199 	phead = &shead[6];
200 
201 	bus = startbus;
202 	dev = get_uint8(phead++);
203 	func = get_uint8(phead++);
204 
205 	for (depth--; depth > 0; depth--) {
206 		bus = pci_getb_func(bus, dev, func, PCI_BCNF_SECBUS);
207 		dev = get_uint8(phead++);
208 		func = get_uint8(phead++);
209 	}
210 
211 	ASSERT(bus >= 0 && bus < 256);
212 	ASSERT(dev >= 0 && dev < 32);
213 	ASSERT(func >= 0 && func < 8);
214 
215 	/* ok we got the device BDF */
216 	scope->scp_bus = bus;
217 	scope->scp_dev = dev;
218 	scope->scp_func = func;
219 
220 	return (scope);
221 }
222 
223 
224 /* setup the ioapic_drhd structure */
225 static void
ioapic_drhd_setup(void)226 ioapic_drhd_setup(void)
227 {
228 	mutex_init(&(ioapic_drhd_lock), NULL, MUTEX_DEFAULT, NULL);
229 
230 	mutex_enter(&(ioapic_drhd_lock));
231 	list_create(&(ioapic_drhd_list), sizeof (ioapic_drhd_t),
232 	    offsetof(ioapic_drhd_t, ioapic_node));
233 	mutex_exit(&(ioapic_drhd_lock));
234 }
235 
236 /* get ioapic source id for interrupt remapping */
237 static void
ioapic_drhd_insert(scope_t * scope,drhd_t * drhd)238 ioapic_drhd_insert(scope_t *scope, drhd_t *drhd)
239 {
240 	ioapic_drhd_t *idt;
241 
242 	idt = kmem_zalloc(sizeof (ioapic_drhd_t), KM_SLEEP);
243 	idt->ioapic_ioapicid = scope->scp_enumid;
244 	idt->ioapic_sid = ((scope->scp_bus << 8) | (scope->scp_dev << 3) |
245 	    (scope->scp_func));
246 	idt->ioapic_drhd = drhd;
247 
248 	mutex_enter(&ioapic_drhd_lock);
249 	list_insert_tail(&ioapic_drhd_list, idt);
250 	mutex_exit(&ioapic_drhd_lock);
251 }
252 
253 static ioapic_drhd_t *
ioapic_drhd_lookup(int ioapicid)254 ioapic_drhd_lookup(int ioapicid)
255 {
256 	ioapic_drhd_t *idt;
257 
258 	mutex_enter(&ioapic_drhd_lock);
259 	idt = list_head(&ioapic_drhd_list);
260 	for (; idt; idt = list_next(&ioapic_drhd_list, idt)) {
261 		if (idt->ioapic_ioapicid == ioapicid) {
262 			break;
263 		}
264 	}
265 	mutex_exit(&ioapic_drhd_lock);
266 
267 	return (idt);
268 }
269 
270 static void
ioapic_drhd_destroy(void)271 ioapic_drhd_destroy(void)
272 {
273 	ioapic_drhd_t *idt;
274 
275 	mutex_enter(&ioapic_drhd_lock);
276 	while (idt = list_remove_head(&ioapic_drhd_list)) {
277 		kmem_free(idt, sizeof (ioapic_drhd_t));
278 	}
279 	list_destroy(&ioapic_drhd_list);
280 	mutex_exit(&(ioapic_drhd_lock));
281 
282 	mutex_destroy(&(ioapic_drhd_lock));
283 }
284 
285 /*
286  * parse_drhd()
287  *   parse the drhd uints in dmar table
288  */
289 static int
parse_drhd(char * uhead,dmar_table_t * tbl)290 parse_drhd(char *uhead, dmar_table_t *tbl)
291 {
292 	drhd_t *drhd;
293 	int seg;
294 	int len;
295 	char *shead;
296 	scope_t *scope;
297 
298 	ASSERT(uhead);
299 	ASSERT(tbl);
300 	ASSERT(get_uint16(&uhead[0]) == DMAR_DRHD);
301 
302 	seg = get_uint16(&uhead[6]);
303 	if (seg < 0 || seg >= IMMU_MAXSEG) {
304 		ddi_err(DER_WARN, NULL, "invalid segment# <%d>"
305 		    "in DRHD unit in ACPI DMAR table", seg);
306 		return (DDI_FAILURE);
307 	}
308 
309 	drhd = kmem_zalloc(sizeof (drhd_t), KM_SLEEP);
310 	mutex_init(&(drhd->dr_lock), NULL, MUTEX_DEFAULT, NULL);
311 	list_create(&(drhd->dr_scope_list), sizeof (scope_t),
312 	    offsetof(scope_t, scp_node));
313 
314 	len = get_uint16(&uhead[2]);
315 	drhd->dr_include_all =
316 	    (get_uint8(&uhead[4]) & DMAR_INCLUDE_ALL) ? B_TRUE : B_FALSE;
317 	drhd->dr_seg = seg;
318 	drhd->dr_regs = get_uint64(&uhead[8]);
319 
320 	/*
321 	 * parse each scope.
322 	 */
323 	shead = &uhead[16];
324 	while (shead < &uhead[len - 1]) {
325 		scope = parse_scope(shead);
326 		if (scope == NULL) {
327 			return (DDI_FAILURE);
328 		}
329 
330 		if (scope->scp_type == DMAR_IOAPIC)  {
331 			ioapic_drhd_insert(scope, drhd);
332 		}
333 
334 		list_insert_tail(&(drhd->dr_scope_list), scope);
335 		shead += get_uint8(&shead[1]);
336 	}
337 
338 	list_insert_tail(&(tbl->tbl_drhd_list[drhd->dr_seg]), drhd);
339 
340 	return (DDI_SUCCESS);
341 }
342 
343 /*
344  * parse_rmrr()
345  *   parse the rmrr units in dmar table
346  */
347 static int
parse_rmrr(char * uhead,dmar_table_t * tbl)348 parse_rmrr(char *uhead, dmar_table_t *tbl)
349 {
350 	rmrr_t *rmrr;
351 	int seg;
352 	int len;
353 	char *shead;
354 	scope_t *scope;
355 
356 	ASSERT(uhead);
357 	ASSERT(tbl);
358 	ASSERT(get_uint16(&uhead[0]) == DMAR_RMRR);
359 
360 	seg = get_uint16(&uhead[6]);
361 	if (seg < 0 || seg >= IMMU_MAXSEG) {
362 		ddi_err(DER_WARN, NULL, "invalid segment# <%d>"
363 		    "in RMRR unit in ACPI DMAR table", seg);
364 		return (DDI_FAILURE);
365 	}
366 
367 	rmrr = kmem_zalloc(sizeof (rmrr_t), KM_SLEEP);
368 	mutex_init(&(rmrr->rm_lock), NULL, MUTEX_DEFAULT, NULL);
369 	list_create(&(rmrr->rm_scope_list), sizeof (scope_t),
370 	    offsetof(scope_t, scp_node));
371 
372 	/* RMRR region is [base,limit] */
373 	len = get_uint16(&uhead[2]);
374 	rmrr->rm_seg = get_uint16(&uhead[6]);
375 	rmrr->rm_base = get_uint64(&uhead[8]);
376 	rmrr->rm_limit = get_uint64(&uhead[16]);
377 
378 	if (rmrr->rm_base > rmrr->rm_limit) {
379 		ddi_err(DER_WARN, NULL, "IMMU: BIOS bug detected: "
380 		    "RMRR: base (%lx) > limit (%lx)",
381 		    rmrr->rm_base, rmrr->rm_limit);
382 		list_destroy(&(rmrr->rm_scope_list));
383 		mutex_destroy(&(rmrr->rm_lock));
384 		kmem_free(rmrr, sizeof (rmrr_t));
385 		return (DDI_SUCCESS);
386 	}
387 
388 	/*
389 	 * parse each scope in RMRR
390 	 */
391 	shead = &uhead[24];
392 	while (shead < &uhead[len - 1]) {
393 		scope = parse_scope(shead);
394 		if (scope == NULL) {
395 			return (DDI_FAILURE);
396 		}
397 		list_insert_tail(&(rmrr->rm_scope_list), scope);
398 		shead += get_uint8(&shead[1]);
399 	}
400 
401 	list_insert_tail(&(tbl->tbl_rmrr_list[rmrr->rm_seg]), rmrr);
402 
403 	return (DDI_SUCCESS);
404 }
405 
406 #define	TBL_OEM_ID_SZ		(6)
407 #define	TBL_OEM_TBLID_SZ	(8)
408 
409 /*
410  * parse the "raw" DMAR table and convert it
411  * into a useful form.
412  */
413 static int
dmar_parse(dmar_table_t ** tblpp,char * raw)414 dmar_parse(dmar_table_t **tblpp, char *raw)
415 {
416 	char *uhead;
417 	dmar_table_t *tbl;
418 	int i;
419 	char *unmstr;
420 
421 	ASSERT(raw);
422 	ASSERT(tblpp);
423 
424 	*tblpp = NULL;
425 
426 	/*
427 	 * do a sanity check. make sure the raw table
428 	 * has the right signature
429 	 */
430 	if (raw[0] != 'D' || raw[1] != 'M' ||
431 	    raw[2] != 'A' || raw[3] != 'R') {
432 		ddi_err(DER_WARN, NULL, "IOMMU ACPI "
433 		    "signature != \"DMAR\"");
434 		return (DDI_FAILURE);
435 	}
436 
437 	/*
438 	 * the platform has intel iommu, create processed ACPI struct
439 	 */
440 	tbl = kmem_zalloc(sizeof (dmar_table_t), KM_SLEEP);
441 	mutex_init(&(tbl->tbl_lock), NULL, MUTEX_DEFAULT, NULL);
442 
443 	tbl->tbl_raw = raw;
444 
445 	/*
446 	 * Note we explicitly show offsets for clarity
447 	 */
448 	tbl->tbl_rawlen = get_uint32(&raw[4]);
449 
450 	/* XXX TO DO verify checksum of table */
451 	tbl->tbl_oem_id = get_str(&raw[10], TBL_OEM_ID_SZ);
452 	tbl->tbl_oem_tblid = get_str(&raw[16], TBL_OEM_TBLID_SZ);
453 	tbl->tbl_oem_rev = get_uint32(&raw[24]);
454 	tbl->tbl_haw = get_uint8(&raw[36]) + 1;
455 	tbl->tbl_intrmap = (get_uint8(&raw[37]) & DMAR_INTRMAP_SUPPORT)
456 	    ? B_TRUE : B_FALSE;
457 
458 	/* create lists for DRHD and RMRR */
459 	for (i = 0; i < IMMU_MAXSEG; i++) {
460 		list_create(&(tbl->tbl_drhd_list[i]), sizeof (drhd_t),
461 		    offsetof(drhd_t, dr_node));
462 		list_create(&(tbl->tbl_rmrr_list[i]), sizeof (rmrr_t),
463 		    offsetof(rmrr_t, rm_node));
464 	}
465 
466 	ioapic_drhd_setup();
467 
468 	/*
469 	 * parse each unit. Currently only DRHD and RMRR types
470 	 * are parsed. We ignore all other types of units.
471 	 */
472 	uhead = &raw[48];
473 	while (uhead < &raw[tbl->tbl_rawlen - 1]) {
474 		unmstr = NULL;
475 		switch (get_uint16(uhead)) {
476 		case DMAR_DRHD:
477 			if (parse_drhd(uhead, tbl) != DDI_SUCCESS) {
478 				goto failed;
479 			}
480 			break;
481 		case DMAR_RMRR:
482 			if (parse_rmrr(uhead, tbl) != DDI_SUCCESS) {
483 				goto failed;
484 			}
485 			break;
486 		case DMAR_ATSR:
487 			unmstr = "ATSR";
488 			break;
489 		case DMAR_RHSA:
490 			unmstr = "RHSA";
491 			break;
492 		default:
493 			unmstr = "unknown unity type";
494 			break;
495 		}
496 		if (unmstr) {
497 			ddi_err(DER_NOTE, NULL, "DMAR ACPI table: "
498 			    "skipping unsupported unit type %s", unmstr);
499 		}
500 		uhead += get_uint16(&uhead[2]);
501 	}
502 
503 	*tblpp = tbl;
504 	return (DDI_SUCCESS);
505 
506 failed:
507 	dmar_table_destroy(tbl);
508 	return (DDI_FAILURE);
509 }
510 
511 static char *
scope_type(int devtype)512 scope_type(int devtype)
513 {
514 	char *typestr;
515 
516 	switch (devtype) {
517 	case DMAR_ENDPOINT:
518 		typestr = "endpoint-device";
519 		break;
520 	case DMAR_SUBTREE:
521 		typestr = "subtree-device";
522 		break;
523 	case DMAR_IOAPIC:
524 		typestr = "IOAPIC";
525 		break;
526 	case DMAR_HPET:
527 		typestr = "HPET";
528 		break;
529 	default:
530 		typestr = "Unknown device";
531 		break;
532 	}
533 
534 	return (typestr);
535 }
536 
537 static void
print_scope_list(list_t * scope_list)538 print_scope_list(list_t *scope_list)
539 {
540 	scope_t *scope;
541 
542 	if (list_is_empty(scope_list))
543 		return;
544 
545 	ddi_err(DER_CONT, NULL, "\tdevice list:\n");
546 
547 	for (scope = list_head(scope_list); scope;
548 	    scope = list_next(scope_list, scope)) {
549 		ddi_err(DER_CONT, NULL, "\t\ttype = %s\n",
550 		    scope_type(scope->scp_type));
551 		ddi_err(DER_CONT, NULL, "\n\t\tbus = %d\n",
552 		    scope->scp_bus);
553 		ddi_err(DER_CONT, NULL, "\t\tdev = %d\n",
554 		    scope->scp_dev);
555 		ddi_err(DER_CONT, NULL, "\t\tfunc = %d\n",
556 		    scope->scp_func);
557 	}
558 }
559 
560 static void
print_drhd_list(list_t * drhd_list)561 print_drhd_list(list_t *drhd_list)
562 {
563 	drhd_t *drhd;
564 
565 	if (list_is_empty(drhd_list))
566 		return;
567 
568 	ddi_err(DER_CONT, NULL, "\ndrhd list:\n");
569 
570 	for (drhd = list_head(drhd_list); drhd;
571 	    drhd = list_next(drhd_list, drhd)) {
572 
573 		ddi_err(DER_CONT, NULL, "\n\tsegment = %d\n",
574 		    drhd->dr_seg);
575 		ddi_err(DER_CONT, NULL, "\treg_base = 0x%" PRIx64 "\n",
576 		    drhd->dr_regs);
577 		ddi_err(DER_CONT, NULL, "\tinclude_all = %s\n",
578 		    drhd->dr_include_all == B_TRUE ? "TRUE" : "FALSE");
579 		ddi_err(DER_CONT, NULL, "\tdip = 0x%p\n",
580 		    (void *)drhd->dr_dip);
581 
582 		print_scope_list(&(drhd->dr_scope_list));
583 	}
584 }
585 
586 
587 static void
print_rmrr_list(list_t * rmrr_list)588 print_rmrr_list(list_t *rmrr_list)
589 {
590 	rmrr_t *rmrr;
591 
592 	if (list_is_empty(rmrr_list))
593 		return;
594 
595 	ddi_err(DER_CONT, NULL, "\nrmrr list:\n");
596 
597 	for (rmrr = list_head(rmrr_list); rmrr;
598 	    rmrr = list_next(rmrr_list, rmrr)) {
599 
600 		ddi_err(DER_CONT, NULL, "\n\tsegment = %d\n",
601 		    rmrr->rm_seg);
602 		ddi_err(DER_CONT, NULL, "\tbase = 0x%lx\n",
603 		    rmrr->rm_base);
604 		ddi_err(DER_CONT, NULL, "\tlimit = 0x%lx\n",
605 		    rmrr->rm_limit);
606 
607 		print_scope_list(&(rmrr->rm_scope_list));
608 	}
609 }
610 
611 /*
612  * print DMAR table
613  */
614 static void
dmar_table_print(dmar_table_t * tbl)615 dmar_table_print(dmar_table_t *tbl)
616 {
617 	int i;
618 
619 	if (dmar_print == B_FALSE) {
620 		return;
621 	}
622 
623 	/* print the title */
624 	ddi_err(DER_CONT, NULL, "#### Start of dmar_table ####\n");
625 	ddi_err(DER_CONT, NULL, "\thaw = %d\n", tbl->tbl_haw);
626 	ddi_err(DER_CONT, NULL, "\tintr_remap = %s\n",
627 	    tbl->tbl_intrmap == B_TRUE ? "<true>" : "<false>");
628 
629 	/* print drhd list */
630 	for (i = 0; i < IMMU_MAXSEG; i++) {
631 		print_drhd_list(&(tbl->tbl_drhd_list[i]));
632 	}
633 
634 
635 	/* print rmrr list */
636 	for (i = 0; i < IMMU_MAXSEG; i++) {
637 		print_rmrr_list(&(tbl->tbl_rmrr_list[i]));
638 	}
639 
640 	ddi_err(DER_CONT, NULL, "#### END of dmar_table ####\n");
641 }
642 
643 static void
drhd_devi_create(drhd_t * drhd,int unit)644 drhd_devi_create(drhd_t *drhd, int unit)
645 {
646 	struct ddi_parent_private_data *pdptr;
647 	struct regspec reg;
648 	dev_info_t *dip;
649 
650 	dip = ddi_add_child(root_devinfo, IMMU_UNIT_NAME,
651 	    DEVI_SID_NODEID, unit);
652 
653 	drhd->dr_dip = dip;
654 
655 	reg.regspec_bustype = 0;
656 	reg.regspec_addr = drhd->dr_regs;
657 	reg.regspec_size = IMMU_REGSZ;
658 
659 	/*
660 	 * update the reg properties
661 	 *
662 	 *   reg property will be used for register
663 	 *   set access
664 	 *
665 	 * refer to the bus_map of root nexus driver
666 	 * I/O or memory mapping:
667 	 *
668 	 * <bustype=0, addr=x, len=x>: memory
669 	 * <bustype=1, addr=x, len=x>: i/o
670 	 * <bustype>1, addr=0, len=x>: x86-compatibility i/o
671 	 */
672 	(void) ndi_prop_update_int_array(DDI_DEV_T_NONE,
673 	    dip, "reg", (int *)&reg,
674 	    sizeof (struct regspec) / sizeof (int));
675 
676 	/*
677 	 * This is an artificially constructed dev_info, and we
678 	 * need to set a few more things to be able to use it
679 	 * for ddi_dma_alloc_handle/free_handle.
680 	 */
681 	ddi_set_driver(dip, ddi_get_driver(ddi_root_node()));
682 	DEVI(dip)->devi_bus_dma_allochdl =
683 	    DEVI(ddi_get_driver((ddi_root_node())));
684 
685 	pdptr = kmem_zalloc(sizeof (struct ddi_parent_private_data)
686 	    + sizeof (struct regspec), KM_SLEEP);
687 	pdptr->par_nreg = 1;
688 	pdptr->par_reg = (struct regspec *)(pdptr + 1);
689 	pdptr->par_reg->regspec_bustype = 0;
690 	pdptr->par_reg->regspec_addr = drhd->dr_regs;
691 	pdptr->par_reg->regspec_size = IMMU_REGSZ;
692 	ddi_set_parent_data(dip, pdptr);
693 }
694 
695 /*
696  * dmar_devinfos_create()
697  *
698  *   create the dev_info node in the device tree,
699  *   the info node is a nuxus child of the root
700  *   nexus
701  */
702 static void
dmar_devinfos_create(dmar_table_t * tbl)703 dmar_devinfos_create(dmar_table_t *tbl)
704 {
705 	list_t *drhd_list;
706 	drhd_t *drhd;
707 	int i, unit;
708 
709 	for (i = 0; i < IMMU_MAXSEG; i++) {
710 
711 		drhd_list = &(tbl->tbl_drhd_list[i]);
712 
713 		if (list_is_empty(drhd_list))
714 			continue;
715 
716 		drhd = list_head(drhd_list);
717 		for (unit = 0; drhd;
718 		    drhd = list_next(drhd_list, drhd), unit++) {
719 			drhd_devi_create(drhd, unit);
720 		}
721 	}
722 }
723 
724 static void
drhd_devi_destroy(drhd_t * drhd)725 drhd_devi_destroy(drhd_t *drhd)
726 {
727 	dev_info_t *dip;
728 
729 	dip = drhd->dr_dip;
730 	ASSERT(dip);
731 
732 	ndi_devi_enter(root_devinfo);
733 	if (ndi_devi_offline(dip, NDI_DEVI_REMOVE) != DDI_SUCCESS) {
734 		ddi_err(DER_WARN, dip, "Failed to destroy");
735 	}
736 	ndi_devi_exit(root_devinfo);
737 	drhd->dr_dip = NULL;
738 }
739 
740 /*
741  * dmar_devi_destroy()
742  *
743  * destroy dev_info nodes for all drhd units
744  */
745 static void
dmar_devi_destroy(dmar_table_t * tbl)746 dmar_devi_destroy(dmar_table_t *tbl)
747 {
748 	drhd_t *drhd;
749 	list_t *drhd_list;
750 	int i;
751 
752 	for (i = 0; i < IMMU_MAXSEG; i++) {
753 		drhd_list = &(tbl->tbl_drhd_list[i]);
754 		if (list_is_empty(drhd_list))
755 			continue;
756 
757 		drhd = list_head(drhd_list);
758 		for (; drhd; drhd = list_next(drhd_list, drhd)) {
759 			drhd_devi_destroy(drhd);
760 		}
761 	}
762 }
763 
764 static int
match_bdf(dev_info_t * ddip,void * arg)765 match_bdf(dev_info_t *ddip, void *arg)
766 {
767 	immu_arg_t *imarg = (immu_arg_t *)arg;
768 	immu_devi_t *immu_devi;
769 
770 	ASSERT(ddip);
771 	ASSERT(imarg);
772 	ASSERT(imarg->ima_seg == 0);
773 	ASSERT(imarg->ima_bus >= 0);
774 	ASSERT(imarg->ima_devfunc >= 0);
775 	ASSERT(imarg->ima_ddip == NULL);
776 
777 	/* rdip can be NULL */
778 
779 	mutex_enter(&(DEVI(ddip)->devi_lock));
780 
781 	immu_devi = IMMU_DEVI(ddip);
782 	ASSERT(immu_devi);
783 
784 	if (immu_devi->imd_seg == imarg->ima_seg &&
785 	    immu_devi->imd_bus == imarg->ima_bus &&
786 	    immu_devi->imd_devfunc == imarg->ima_devfunc) {
787 		imarg->ima_ddip = ddip;
788 	}
789 
790 	mutex_exit(&(DEVI(ddip)->devi_lock));
791 
792 	return (imarg->ima_ddip ? DDI_WALK_TERMINATE : DDI_WALK_CONTINUE);
793 }
794 static void
dmar_table_destroy(dmar_table_t * tbl)795 dmar_table_destroy(dmar_table_t *tbl)
796 {
797 	int i;
798 
799 	ASSERT(tbl);
800 
801 	/* destroy lists for DRHD and RMRR */
802 	for (i = 0; i < IMMU_MAXSEG; i++) {
803 		rmrr_list_destroy(&(tbl->tbl_rmrr_list[i]));
804 		drhd_list_destroy(&(tbl->tbl_drhd_list[i]));
805 	}
806 
807 	/* free strings */
808 	kmem_free(tbl->tbl_oem_tblid, TBL_OEM_TBLID_SZ + 1);
809 	kmem_free(tbl->tbl_oem_id, TBL_OEM_ID_SZ + 1);
810 	tbl->tbl_raw = NULL; /* raw ACPI table doesn't have to be freed */
811 	mutex_destroy(&(tbl->tbl_lock));
812 	kmem_free(tbl, sizeof (dmar_table_t));
813 }
814 
815 /*
816  * #########################################################################
817  * Functions exported by dmar.c
818  * This file deals with reading and processing the DMAR ACPI table
819  * #########################################################################
820  */
821 
822 /*
823  * immu_dmar_setup()
824  *	Check if the system has a DMAR ACPI table. If yes, the system
825  *	has Intel IOMMU hardware
826  */
827 int
immu_dmar_setup(void)828 immu_dmar_setup(void)
829 {
830 	if (AcpiGetTable("DMAR", 1, (ACPI_TABLE_HEADER **)&dmar_raw) != AE_OK) {
831 		ddi_err(DER_LOG, NULL,
832 		    "No DMAR ACPI table. No Intel IOMMU present\n");
833 		dmar_raw = NULL;
834 		return (DDI_FAILURE);
835 	}
836 	ASSERT(dmar_raw);
837 	return (DDI_SUCCESS);
838 }
839 
840 /*
841  * immu_dmar_parse()
842  *  Called by immu.c to parse and convert "raw" ACPI DMAR table
843  */
844 int
immu_dmar_parse(void)845 immu_dmar_parse(void)
846 {
847 	dmar_table_t *tbl = NULL;
848 
849 	/* we should already have found the "raw" table */
850 	ASSERT(dmar_raw);
851 
852 	ddi_err(DER_CONT, NULL, "?Processing DMAR ACPI table\n");
853 
854 	dmar_table = NULL;
855 
856 	/*
857 	 * parse DMAR ACPI table
858 	 */
859 	if (dmar_parse(&tbl, dmar_raw) != DDI_SUCCESS) {
860 		ASSERT(tbl == NULL);
861 		return (DDI_FAILURE);
862 	}
863 
864 	ASSERT(tbl);
865 
866 	/*
867 	 * create one devinfo for every drhd unit
868 	 * in the DMAR table
869 	 */
870 	dmar_devinfos_create(tbl);
871 
872 	/*
873 	 * print the dmar table if the debug option is set
874 	 */
875 	dmar_table_print(tbl);
876 
877 	dmar_table = tbl;
878 
879 	return (DDI_SUCCESS);
880 }
881 
882 void
immu_dmar_startup(void)883 immu_dmar_startup(void)
884 {
885 	/* nothing to do */
886 }
887 
888 void
immu_dmar_shutdown(void)889 immu_dmar_shutdown(void)
890 {
891 	/* nothing to do */
892 }
893 
894 void
immu_dmar_destroy(void)895 immu_dmar_destroy(void)
896 {
897 	dmar_devi_destroy(dmar_table);
898 	dmar_table_destroy(dmar_table);
899 	ioapic_drhd_destroy();
900 	dmar_table = NULL;
901 	dmar_raw = NULL;
902 }
903 
904 boolean_t
immu_dmar_blacklisted(char ** strptr,uint_t nstrs)905 immu_dmar_blacklisted(char **strptr, uint_t nstrs)
906 {
907 	dmar_table_t *tbl = dmar_table;
908 	int i;
909 	char oem_rev[IMMU_MAXNAMELEN];
910 
911 	ASSERT(tbl);
912 
913 	ASSERT((strptr == NULL) ^ (nstrs != 0));
914 
915 	/*
916 	 * Must be a minimum of 4
917 	 */
918 	if (nstrs < 4) {
919 		return (B_FALSE);
920 	}
921 
922 	ddi_err(DER_CONT, NULL, "?System DMAR ACPI table information:\n");
923 	ddi_err(DER_CONT, NULL, "?OEM-ID = <%s>\n", tbl->tbl_oem_id);
924 	ddi_err(DER_CONT, NULL, "?Table-ID = <%s>\n", tbl->tbl_oem_tblid);
925 	(void) snprintf(oem_rev, sizeof (oem_rev), "%d", tbl->tbl_oem_rev);
926 	ddi_err(DER_CONT, NULL, "?Revision = <%s>\n", oem_rev);
927 
928 	for (i = 0; nstrs - i >= 4; i++) {
929 		if (strcmp(*strptr++, "DMAR") == 0) {
930 			if (strcmp(*strptr++, tbl->tbl_oem_id) == 0 &&
931 			    (*strptr[0] == '\0' ||
932 			    strcmp(*strptr++, tbl->tbl_oem_tblid) == 0) &&
933 			    (*strptr[0] == '\0' ||
934 			    strcmp(*strptr++, oem_rev) == 0)) {
935 				return (B_TRUE);
936 			}
937 			i += 3; /* for loops adds 1 as well, so only 3 here */
938 		}
939 	}
940 	return (B_FALSE);
941 }
942 
943 void
immu_dmar_rmrr_map(void)944 immu_dmar_rmrr_map(void)
945 {
946 	int seg;
947 	dev_info_t *rdip;
948 	scope_t *scope;
949 	rmrr_t *rmrr;
950 	dmar_table_t *tbl;
951 
952 	ASSERT(dmar_table);
953 
954 	tbl = dmar_table;
955 
956 	/* called during boot, when kernel is single threaded. No lock */
957 
958 	/*
959 	 * for each segment, walk the rmrr list looking for an exact match
960 	 */
961 	for (seg = 0; seg < IMMU_MAXSEG; seg++) {
962 		rmrr = list_head(&(tbl->tbl_rmrr_list)[seg]);
963 		for (; rmrr; rmrr = list_next(&(tbl->tbl_rmrr_list)[seg],
964 		    rmrr)) {
965 
966 			/*
967 			 * try to match BDF *exactly* to a device scope.
968 			 */
969 			scope = list_head(&(rmrr->rm_scope_list));
970 			for (; scope;
971 			    scope = list_next(&(rmrr->rm_scope_list), scope)) {
972 				immu_arg_t imarg = {0};
973 				memrng_t mrng = {0};
974 
975 				/* PCI endpoint devices only */
976 				if (scope->scp_type != DMAR_ENDPOINT)
977 					continue;
978 
979 				imarg.ima_seg = seg;
980 				imarg.ima_bus = scope->scp_bus;
981 				imarg.ima_devfunc =
982 				    IMMU_PCI_DEVFUNC(scope->scp_dev,
983 				    scope->scp_func);
984 				imarg.ima_ddip = NULL;
985 				imarg.ima_rdip = NULL;
986 
987 				ASSERT(root_devinfo);
988 				/* XXX should be optimized */
989 				ndi_devi_enter(root_devinfo);
990 				ddi_walk_devs(ddi_get_child(root_devinfo),
991 				    match_bdf, &imarg);
992 				ndi_devi_exit(root_devinfo);
993 
994 				if (imarg.ima_ddip == NULL) {
995 					ddi_err(DER_WARN, NULL,
996 					    "No dip found for "
997 					    "bus=0x%x, dev=0x%x, func= 0x%x",
998 					    scope->scp_bus, scope->scp_dev,
999 					    scope->scp_func);
1000 					continue;
1001 				}
1002 
1003 				rdip = imarg.ima_ddip;
1004 				/*
1005 				 * This address must be in the BIOS reserved
1006 				 * map
1007 				 */
1008 				if (!address_in_memlist(bios_rsvd,
1009 				    (uint64_t)rmrr->rm_base, rmrr->rm_limit -
1010 				    rmrr->rm_base + 1)) {
1011 					ddi_err(DER_WARN, rdip, "RMRR range "
1012 					    " [0x%" PRIx64 " - 0x%" PRIx64 "]"
1013 					    " not in BIOS reserved map",
1014 					    rmrr->rm_base, rmrr->rm_limit);
1015 				}
1016 
1017 				/* XXX could be more efficient */
1018 				memlist_read_lock();
1019 				if (address_in_memlist(phys_install,
1020 				    (uint64_t)rmrr->rm_base, rmrr->rm_limit -
1021 				    rmrr->rm_base + 1)) {
1022 					ddi_err(DER_WARN, rdip, "RMRR range "
1023 					    " [0x%" PRIx64 " - 0x%" PRIx64 "]"
1024 					    " is in physinstall map",
1025 					    rmrr->rm_base, rmrr->rm_limit);
1026 				}
1027 				memlist_read_unlock();
1028 
1029 				(void) immu_dvma_device_setup(rdip, 0);
1030 
1031 				ddi_err(DER_LOG, rdip,
1032 				    "IMMU: Mapping RMRR range "
1033 				    "[0x%" PRIx64 " - 0x%"PRIx64 "]",
1034 				    rmrr->rm_base, rmrr->rm_limit);
1035 
1036 				mrng.mrng_start =
1037 				    IMMU_ROUNDOWN((uintptr_t)rmrr->rm_base);
1038 				mrng.mrng_npages =
1039 				    IMMU_ROUNDUP((uintptr_t)rmrr->rm_limit -
1040 				    (uintptr_t)rmrr->rm_base + 1) /
1041 				    IMMU_PAGESIZE;
1042 
1043 				(void) immu_map_memrange(rdip, &mrng);
1044 			}
1045 		}
1046 	}
1047 
1048 }
1049 
1050 immu_t *
immu_dmar_get_immu(dev_info_t * rdip)1051 immu_dmar_get_immu(dev_info_t *rdip)
1052 {
1053 	int seg;
1054 	int tlevel;
1055 	int level;
1056 	drhd_t *drhd;
1057 	drhd_t *tdrhd;
1058 	scope_t *scope;
1059 	dmar_table_t *tbl;
1060 
1061 	ASSERT(dmar_table);
1062 
1063 	tbl = dmar_table;
1064 
1065 	mutex_enter(&(tbl->tbl_lock));
1066 
1067 	/*
1068 	 * for each segment, walk the drhd list looking for an exact match
1069 	 */
1070 	for (seg = 0; seg < IMMU_MAXSEG; seg++) {
1071 		drhd = list_head(&(tbl->tbl_drhd_list)[seg]);
1072 		for (; drhd; drhd = list_next(&(tbl->tbl_drhd_list)[seg],
1073 		    drhd)) {
1074 
1075 			/*
1076 			 * we are currently searching for exact matches so
1077 			 * skip "include all" (catchall) and subtree matches
1078 			 */
1079 			if (drhd->dr_include_all == B_TRUE)
1080 				continue;
1081 
1082 			/*
1083 			 * try to match BDF *exactly* to a device scope.
1084 			 */
1085 			scope = list_head(&(drhd->dr_scope_list));
1086 			for (; scope;
1087 			    scope = list_next(&(drhd->dr_scope_list), scope)) {
1088 				immu_arg_t imarg = {0};
1089 
1090 				/* PCI endpoint devices only */
1091 				if (scope->scp_type != DMAR_ENDPOINT)
1092 					continue;
1093 
1094 				imarg.ima_seg = seg;
1095 				imarg.ima_bus = scope->scp_bus;
1096 				imarg.ima_devfunc =
1097 				    IMMU_PCI_DEVFUNC(scope->scp_dev,
1098 				    scope->scp_func);
1099 				imarg.ima_ddip = NULL;
1100 				imarg.ima_rdip = rdip;
1101 				level = 0;
1102 				if (immu_walk_ancestor(rdip, NULL, match_bdf,
1103 				    &imarg, &level, IMMU_FLAGS_DONTPASS)
1104 				    != DDI_SUCCESS) {
1105 					/* skip - nothing else we can do */
1106 					continue;
1107 				}
1108 
1109 				/* Should have walked only 1 level i.e. rdip */
1110 				ASSERT(level == 1);
1111 
1112 				if (imarg.ima_ddip) {
1113 					ASSERT(imarg.ima_ddip == rdip);
1114 					goto found;
1115 				}
1116 			}
1117 		}
1118 	}
1119 
1120 	/*
1121 	 * walk the drhd list looking for subtree match
1122 	 * i.e. is the device a descendant of a devscope BDF.
1123 	 * We want the lowest subtree.
1124 	 */
1125 	tdrhd = NULL;
1126 	tlevel = 0;
1127 	for (seg = 0; seg < IMMU_MAXSEG; seg++) {
1128 		drhd = list_head(&(tbl->tbl_drhd_list)[seg]);
1129 		for (; drhd; drhd = list_next(&(tbl->tbl_drhd_list)[seg],
1130 		    drhd)) {
1131 
1132 			/* looking for subtree match */
1133 			if (drhd->dr_include_all == B_TRUE)
1134 				continue;
1135 
1136 			/*
1137 			 * try to match the device scope
1138 			 */
1139 			scope = list_head(&(drhd->dr_scope_list));
1140 			for (; scope;
1141 			    scope = list_next(&(drhd->dr_scope_list), scope)) {
1142 				immu_arg_t imarg = {0};
1143 
1144 				/* PCI subtree only */
1145 				if (scope->scp_type != DMAR_SUBTREE)
1146 					continue;
1147 
1148 				imarg.ima_seg = seg;
1149 				imarg.ima_bus = scope->scp_bus;
1150 				imarg.ima_devfunc =
1151 				    IMMU_PCI_DEVFUNC(scope->scp_dev,
1152 				    scope->scp_func);
1153 
1154 				imarg.ima_ddip = NULL;
1155 				imarg.ima_rdip = rdip;
1156 				level = 0;
1157 				if (immu_walk_ancestor(rdip, NULL, match_bdf,
1158 				    &imarg, &level, 0) != DDI_SUCCESS) {
1159 					/* skip - nothing else we can do */
1160 					continue;
1161 				}
1162 
1163 				/* should have walked 1 level i.e. rdip */
1164 				ASSERT(level > 0);
1165 
1166 				/* look for lowest ancestor matching drhd */
1167 				if (imarg.ima_ddip && (tdrhd == NULL ||
1168 				    level < tlevel)) {
1169 					tdrhd = drhd;
1170 					tlevel = level;
1171 				}
1172 			}
1173 		}
1174 	}
1175 
1176 	if ((drhd = tdrhd) != NULL) {
1177 		goto found;
1178 	}
1179 
1180 	for (seg = 0; seg < IMMU_MAXSEG; seg++) {
1181 		drhd = list_head(&(tbl->tbl_drhd_list[seg]));
1182 		for (; drhd; drhd = list_next(&(tbl->tbl_drhd_list)[seg],
1183 		    drhd)) {
1184 			/* Look for include all */
1185 			if (drhd->dr_include_all == B_TRUE) {
1186 				break;
1187 			}
1188 		}
1189 	}
1190 
1191 	/*FALLTHRU*/
1192 
1193 found:
1194 	mutex_exit(&(tbl->tbl_lock));
1195 
1196 	/*
1197 	 * No drhd (dmar unit) found for this device in the ACPI DMAR tables.
1198 	 * This may happen with buggy versions of BIOSes. Just warn instead
1199 	 * of panic as we don't want whole system to go down because of one
1200 	 * device.
1201 	 */
1202 	if (drhd == NULL) {
1203 		ddi_err(DER_WARN, rdip, "can't find Intel IOMMU unit for "
1204 		    "device in ACPI DMAR table.");
1205 		return (NULL);
1206 	}
1207 
1208 	return (drhd->dr_immu);
1209 }
1210 
1211 dev_info_t *
immu_dmar_unit_dip(void * dmar_unit)1212 immu_dmar_unit_dip(void *dmar_unit)
1213 {
1214 	drhd_t *drhd = (drhd_t *)dmar_unit;
1215 	return (drhd->dr_dip);
1216 }
1217 
1218 void *
immu_dmar_walk_units(int seg,void * dmar_unit)1219 immu_dmar_walk_units(int seg, void *dmar_unit)
1220 {
1221 	list_t *drhd_list;
1222 	drhd_t *drhd = (drhd_t *)dmar_unit;
1223 
1224 	drhd_list = &(dmar_table->tbl_drhd_list[seg]);
1225 
1226 	if (drhd == NULL) {
1227 		return ((void *)list_head(drhd_list));
1228 	} else {
1229 		return ((void *)list_next(drhd_list, drhd));
1230 	}
1231 }
1232 
1233 void
immu_dmar_set_immu(void * dmar_unit,immu_t * immu)1234 immu_dmar_set_immu(void *dmar_unit, immu_t *immu)
1235 {
1236 	drhd_t *drhd = (drhd_t *)dmar_unit;
1237 
1238 	ASSERT(drhd);
1239 	ASSERT(immu);
1240 
1241 	drhd->dr_immu = immu;
1242 }
1243 
1244 boolean_t
immu_dmar_intrmap_supported(void)1245 immu_dmar_intrmap_supported(void)
1246 {
1247 	ASSERT(dmar_table);
1248 	return (dmar_table->tbl_intrmap);
1249 }
1250 
1251 /* for a given ioapicid, find the source id and immu */
1252 uint16_t
immu_dmar_ioapic_sid(int ioapic_ix)1253 immu_dmar_ioapic_sid(int ioapic_ix)
1254 {
1255 	ioapic_drhd_t *idt;
1256 
1257 	idt = ioapic_drhd_lookup(psm_get_ioapicid(ioapic_ix));
1258 	if (idt == NULL) {
1259 		ddi_err(DER_PANIC, NULL, "cannot determine source-id for "
1260 		    "IOAPIC (index = %d)", ioapic_ix);
1261 		/*NOTREACHED*/
1262 	}
1263 
1264 	return (idt->ioapic_sid);
1265 }
1266 
1267 /* for a given ioapicid, find the source id and immu */
1268 immu_t *
immu_dmar_ioapic_immu(int ioapic_ix)1269 immu_dmar_ioapic_immu(int ioapic_ix)
1270 {
1271 	ioapic_drhd_t *idt;
1272 
1273 	idt = ioapic_drhd_lookup(psm_get_ioapicid(ioapic_ix));
1274 	if (idt) {
1275 		return (idt->ioapic_drhd ? idt->ioapic_drhd->dr_immu : NULL);
1276 	}
1277 	return (NULL);
1278 }
1279