xref: /titanic_51/usr/src/uts/i86pc/os/ddi_impl.c (revision 92ba710950bf6ee35de67e1e0b7f20ec9e528106)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 
22 /*
23  * Copyright 2007 Sun Microsystems, Inc.  All rights reserved.
24  * Use is subject to license terms.
25  */
26 
27 #pragma ident	"%Z%%M%	%I%	%E% SMI"
28 
29 /*
30  * PC specific DDI implementation
31  */
32 #include <sys/types.h>
33 #include <sys/autoconf.h>
34 #include <sys/avintr.h>
35 #include <sys/bootconf.h>
36 #include <sys/conf.h>
37 #include <sys/cpuvar.h>
38 #include <sys/ddi_impldefs.h>
39 #include <sys/ddi_subrdefs.h>
40 #include <sys/ethernet.h>
41 #include <sys/fp.h>
42 #include <sys/instance.h>
43 #include <sys/kmem.h>
44 #include <sys/machsystm.h>
45 #include <sys/modctl.h>
46 #include <sys/promif.h>
47 #include <sys/prom_plat.h>
48 #include <sys/sunndi.h>
49 #include <sys/ndi_impldefs.h>
50 #include <sys/ddi_impldefs.h>
51 #include <sys/sysmacros.h>
52 #include <sys/systeminfo.h>
53 #include <sys/utsname.h>
54 #include <sys/atomic.h>
55 #include <sys/spl.h>
56 #include <sys/archsystm.h>
57 #include <vm/seg_kmem.h>
58 #include <sys/ontrap.h>
59 #include <sys/fm/protocol.h>
60 #include <sys/ramdisk.h>
61 #include <sys/sunndi.h>
62 #include <sys/vmem.h>
63 #include <sys/pci_impl.h>
64 #include <sys/mach_intr.h>
65 #include <vm/hat_i86.h>
66 #include <sys/x86_archext.h>
67 
68 /*
69  * DDI Boot Configuration
70  */
71 
72 /*
73  * No platform drivers on this platform
74  */
75 char *platform_module_list[] = {
76 	"ppm",
77 	(char *)0
78 };
79 
80 /* pci bus resource maps */
81 struct pci_bus_resource *pci_bus_res;
82 
83 extern int root_is_svm;
84 uint64_t ramdisk_start, ramdisk_end;
85 
86 /*
87  * Forward declarations
88  */
89 static int getlongprop_buf();
90 static void get_boot_properties(void);
91 static void impl_bus_initialprobe(void);
92 static void impl_bus_reprobe(void);
93 
94 static int poke_mem(peekpoke_ctlops_t *in_args);
95 static int peek_mem(peekpoke_ctlops_t *in_args);
96 
97 static int kmem_override_cache_attrs(caddr_t, size_t, uint_t);
98 
99 #define	CTGENTRIES	15
100 
101 static struct ctgas {
102 	struct ctgas	*ctg_next;
103 	int		ctg_index;
104 	void		*ctg_addr[CTGENTRIES];
105 	size_t		ctg_size[CTGENTRIES];
106 } ctglist;
107 
108 static kmutex_t		ctgmutex;
109 #define	CTGLOCK()	mutex_enter(&ctgmutex)
110 #define	CTGUNLOCK()	mutex_exit(&ctgmutex)
111 
112 /*
113  * Minimum pfn value of page_t's put on the free list.  This is to simplify
114  * support of ddi dma memory requests which specify small, non-zero addr_lo
115  * values.
116  *
117  * The default value of 2, which corresponds to the only known non-zero addr_lo
118  * value used, means a single page will be sacrificed (pfn typically starts
119  * at 1).  ddiphysmin can be set to 0 to disable. It cannot be set above 0x100
120  * otherwise mp startup panics.
121  */
122 pfn_t	ddiphysmin = 2;
123 
124 static void
125 check_driver_disable(void)
126 {
127 	int proplen = 128;
128 	char *prop_name;
129 	char *drv_name, *propval;
130 	major_t major;
131 
132 	prop_name = kmem_alloc(proplen, KM_SLEEP);
133 	for (major = 0; major < devcnt; major++) {
134 		drv_name = ddi_major_to_name(major);
135 		if (drv_name == NULL)
136 			continue;
137 		(void) snprintf(prop_name, proplen, "disable-%s", drv_name);
138 		if (ddi_prop_lookup_string(DDI_DEV_T_ANY, ddi_root_node(),
139 		    DDI_PROP_DONTPASS, prop_name, &propval) == DDI_SUCCESS) {
140 			if (strcmp(propval, "true") == 0) {
141 				devnamesp[major].dn_flags |= DN_DRIVER_REMOVED;
142 				cmn_err(CE_NOTE, "driver %s disabled",
143 				    drv_name);
144 			}
145 			ddi_prop_free(propval);
146 		}
147 	}
148 	kmem_free(prop_name, proplen);
149 }
150 
151 
152 /*
153  * Configure the hardware on the system.
154  * Called before the rootfs is mounted
155  */
156 void
157 configure(void)
158 {
159 	extern void i_ddi_init_root();
160 
161 #if defined(__i386)
162 	extern int fpu_pentium_fdivbug;
163 #endif	/* __i386 */
164 	extern int fpu_ignored;
165 
166 	/*
167 	 * Determine if an FPU is attached
168 	 */
169 
170 	fpu_probe();
171 
172 #if defined(__i386)
173 	if (fpu_pentium_fdivbug) {
174 		printf("\
175 FP hardware exhibits Pentium floating point divide problem\n");
176 	}
177 #endif	/* __i386 */
178 
179 	if (fpu_ignored) {
180 		printf("FP hardware will not be used\n");
181 	} else if (!fpu_exists) {
182 		printf("No FPU in configuration\n");
183 	}
184 
185 	/*
186 	 * Initialize devices on the machine.
187 	 * Uses configuration tree built by the PROMs to determine what
188 	 * is present, and builds a tree of prototype dev_info nodes
189 	 * corresponding to the hardware which identified itself.
190 	 */
191 #if !defined(SAS) && !defined(MPSAS)
192 	/*
193 	 * Check for disabled drivers and initialize root node.
194 	 */
195 	check_driver_disable();
196 	i_ddi_init_root();
197 
198 	/*
199 	 * attach the isa nexus to get ACPI resource usage
200 	 * isa is "kind of" a pseudo node
201 	 */
202 	(void) i_ddi_attach_pseudo_node("isa");
203 
204 	/* reprogram devices not set up by firmware (BIOS) */
205 	impl_bus_reprobe();
206 #endif	/* !SAS && !MPSAS */
207 }
208 
209 /*
210  * The "status" property indicates the operational status of a device.
211  * If this property is present, the value is a string indicating the
212  * status of the device as follows:
213  *
214  *	"okay"		operational.
215  *	"disabled"	not operational, but might become operational.
216  *	"fail"		not operational because a fault has been detected,
217  *			and it is unlikely that the device will become
218  *			operational without repair. no additional details
219  *			are available.
220  *	"fail-xxx"	not operational because a fault has been detected,
221  *			and it is unlikely that the device will become
222  *			operational without repair. "xxx" is additional
223  *			human-readable information about the particular
224  *			fault condition that was detected.
225  *
226  * The absence of this property means that the operational status is
227  * unknown or okay.
228  *
229  * This routine checks the status property of the specified device node
230  * and returns 0 if the operational status indicates failure, and 1 otherwise.
231  *
232  * The property may exist on plug-in cards the existed before IEEE 1275-1994.
233  * And, in that case, the property may not even be a string. So we carefully
234  * check for the value "fail", in the beginning of the string, noting
235  * the property length.
236  */
237 int
238 status_okay(int id, char *buf, int buflen)
239 {
240 	char status_buf[OBP_MAXPROPNAME];
241 	char *bufp = buf;
242 	int len = buflen;
243 	int proplen;
244 	static const char *status = "status";
245 	static const char *fail = "fail";
246 	int fail_len = (int)strlen(fail);
247 
248 	/*
249 	 * Get the proplen ... if it's smaller than "fail",
250 	 * or doesn't exist ... then we don't care, since
251 	 * the value can't begin with the char string "fail".
252 	 *
253 	 * NB: proplen, if it's a string, includes the NULL in the
254 	 * the size of the property, and fail_len does not.
255 	 */
256 	proplen = prom_getproplen((pnode_t)id, (caddr_t)status);
257 	if (proplen <= fail_len)	/* nonexistant or uninteresting len */
258 		return (1);
259 
260 	/*
261 	 * if a buffer was provided, use it
262 	 */
263 	if ((buf == (char *)NULL) || (buflen <= 0)) {
264 		bufp = status_buf;
265 		len = sizeof (status_buf);
266 	}
267 	*bufp = (char)0;
268 
269 	/*
270 	 * Get the property into the buffer, to the extent of the buffer,
271 	 * and in case the buffer is smaller than the property size,
272 	 * NULL terminate the buffer. (This handles the case where
273 	 * a buffer was passed in and the caller wants to print the
274 	 * value, but the buffer was too small).
275 	 */
276 	(void) prom_bounded_getprop((pnode_t)id, (caddr_t)status,
277 	    (caddr_t)bufp, len);
278 	*(bufp + len - 1) = (char)0;
279 
280 	/*
281 	 * If the value begins with the char string "fail",
282 	 * then it means the node is failed. We don't care
283 	 * about any other values. We assume the node is ok
284 	 * although it might be 'disabled'.
285 	 */
286 	if (strncmp(bufp, fail, fail_len) == 0)
287 		return (0);
288 
289 	return (1);
290 }
291 
292 /*
293  * Check the status of the device node passed as an argument.
294  *
295  *	if ((status is OKAY) || (status is DISABLED))
296  *		return DDI_SUCCESS
297  *	else
298  *		print a warning and return DDI_FAILURE
299  */
300 /*ARGSUSED1*/
301 int
302 check_status(int id, char *name, dev_info_t *parent)
303 {
304 	char status_buf[64];
305 	char devtype_buf[OBP_MAXPROPNAME];
306 	int retval = DDI_FAILURE;
307 
308 	/*
309 	 * is the status okay?
310 	 */
311 	if (status_okay(id, status_buf, sizeof (status_buf)))
312 		return (DDI_SUCCESS);
313 
314 	/*
315 	 * a status property indicating bad memory will be associated
316 	 * with a node which has a "device_type" property with a value of
317 	 * "memory-controller". in this situation, return DDI_SUCCESS
318 	 */
319 	if (getlongprop_buf(id, OBP_DEVICETYPE, devtype_buf,
320 	    sizeof (devtype_buf)) > 0) {
321 		if (strcmp(devtype_buf, "memory-controller") == 0)
322 			retval = DDI_SUCCESS;
323 	}
324 
325 	/*
326 	 * print the status property information
327 	 */
328 	cmn_err(CE_WARN, "status '%s' for '%s'", status_buf, name);
329 	return (retval);
330 }
331 
332 /*ARGSUSED*/
333 uint_t
334 softlevel1(caddr_t arg1, caddr_t arg2)
335 {
336 	softint();
337 	return (1);
338 }
339 
340 /*
341  * Allow for implementation specific correction of PROM property values.
342  */
343 
344 /*ARGSUSED*/
345 void
346 impl_fix_props(dev_info_t *dip, dev_info_t *ch_dip, char *name, int len,
347     caddr_t buffer)
348 {
349 	/*
350 	 * There are no adjustments needed in this implementation.
351 	 */
352 }
353 
354 static int
355 getlongprop_buf(int id, char *name, char *buf, int maxlen)
356 {
357 	int size;
358 
359 	size = prom_getproplen((pnode_t)id, name);
360 	if (size <= 0 || (size > maxlen - 1))
361 		return (-1);
362 
363 	if (-1 == prom_getprop((pnode_t)id, name, buf))
364 		return (-1);
365 
366 	if (strcmp("name", name) == 0) {
367 		if (buf[size - 1] != '\0') {
368 			buf[size] = '\0';
369 			size += 1;
370 		}
371 	}
372 
373 	return (size);
374 }
375 
376 static int
377 get_prop_int_array(dev_info_t *di, char *pname, int **pval, uint_t *plen)
378 {
379 	int ret;
380 
381 	if ((ret = ddi_prop_lookup_int_array(DDI_DEV_T_ANY, di,
382 	    DDI_PROP_DONTPASS, pname, pval, plen))
383 	    == DDI_PROP_SUCCESS) {
384 		*plen = (*plen) * (sizeof (int));
385 	}
386 	return (ret);
387 }
388 
389 
390 /*
391  * Node Configuration
392  */
393 
394 struct prop_ispec {
395 	uint_t	pri, vec;
396 };
397 
398 /*
399  * For the x86, we're prepared to claim that the interrupt string
400  * is in the form of a list of <ipl,vec> specifications.
401  */
402 
403 #define	VEC_MIN	1
404 #define	VEC_MAX	255
405 
406 static int
407 impl_xlate_intrs(dev_info_t *child, int *in,
408     struct ddi_parent_private_data *pdptr)
409 {
410 	size_t size;
411 	int n;
412 	struct intrspec *new;
413 	caddr_t got_prop;
414 	int *inpri;
415 	int got_len;
416 	extern int ignore_hardware_nodes;	/* force flag from ddi_impl.c */
417 
418 	static char bad_intr_fmt[] =
419 	    "bad interrupt spec from %s%d - ipl %d, irq %d\n";
420 
421 	/*
422 	 * determine if the driver is expecting the new style "interrupts"
423 	 * property which just contains the IRQ, or the old style which
424 	 * contains pairs of <IPL,IRQ>.  if it is the new style, we always
425 	 * assign IPL 5 unless an "interrupt-priorities" property exists.
426 	 * in that case, the "interrupt-priorities" property contains the
427 	 * IPL values that match, one for one, the IRQ values in the
428 	 * "interrupts" property.
429 	 */
430 	inpri = NULL;
431 	if ((ddi_getprop(DDI_DEV_T_ANY, child, DDI_PROP_DONTPASS,
432 	    "ignore-hardware-nodes", -1) != -1) || ignore_hardware_nodes) {
433 		/* the old style "interrupts" property... */
434 
435 		/*
436 		 * The list consists of <ipl,vec> elements
437 		 */
438 		if ((n = (*in++ >> 1)) < 1)
439 			return (DDI_FAILURE);
440 
441 		pdptr->par_nintr = n;
442 		size = n * sizeof (struct intrspec);
443 		new = pdptr->par_intr = kmem_zalloc(size, KM_SLEEP);
444 
445 		while (n--) {
446 			int level = *in++;
447 			int vec = *in++;
448 
449 			if (level < 1 || level > MAXIPL ||
450 			    vec < VEC_MIN || vec > VEC_MAX) {
451 				cmn_err(CE_CONT, bad_intr_fmt,
452 				    DEVI(child)->devi_name,
453 				    DEVI(child)->devi_instance, level, vec);
454 				goto broken;
455 			}
456 			new->intrspec_pri = level;
457 			if (vec != 2)
458 				new->intrspec_vec = vec;
459 			else
460 				/*
461 				 * irq 2 on the PC bus is tied to irq 9
462 				 * on ISA, EISA and MicroChannel
463 				 */
464 				new->intrspec_vec = 9;
465 			new++;
466 		}
467 
468 		return (DDI_SUCCESS);
469 	} else {
470 		/* the new style "interrupts" property... */
471 
472 		/*
473 		 * The list consists of <vec> elements
474 		 */
475 		if ((n = (*in++)) < 1)
476 			return (DDI_FAILURE);
477 
478 		pdptr->par_nintr = n;
479 		size = n * sizeof (struct intrspec);
480 		new = pdptr->par_intr = kmem_zalloc(size, KM_SLEEP);
481 
482 		/* XXX check for "interrupt-priorities" property... */
483 		if (ddi_getlongprop(DDI_DEV_T_ANY, child, DDI_PROP_DONTPASS,
484 		    "interrupt-priorities", (caddr_t)&got_prop, &got_len)
485 		    == DDI_PROP_SUCCESS) {
486 			if (n != (got_len / sizeof (int))) {
487 				cmn_err(CE_CONT,
488 				    "bad interrupt-priorities length"
489 				    " from %s%d: expected %d, got %d\n",
490 				    DEVI(child)->devi_name,
491 				    DEVI(child)->devi_instance, n,
492 				    (int)(got_len / sizeof (int)));
493 				goto broken;
494 			}
495 			inpri = (int *)got_prop;
496 		}
497 
498 		while (n--) {
499 			int level;
500 			int vec = *in++;
501 
502 			if (inpri == NULL)
503 				level = 5;
504 			else
505 				level = *inpri++;
506 
507 			if (level < 1 || level > MAXIPL ||
508 			    vec < VEC_MIN || vec > VEC_MAX) {
509 				cmn_err(CE_CONT, bad_intr_fmt,
510 				    DEVI(child)->devi_name,
511 				    DEVI(child)->devi_instance, level, vec);
512 				goto broken;
513 			}
514 			new->intrspec_pri = level;
515 			if (vec != 2)
516 				new->intrspec_vec = vec;
517 			else
518 				/*
519 				 * irq 2 on the PC bus is tied to irq 9
520 				 * on ISA, EISA and MicroChannel
521 				 */
522 				new->intrspec_vec = 9;
523 			new++;
524 		}
525 
526 		if (inpri != NULL)
527 			kmem_free(got_prop, got_len);
528 		return (DDI_SUCCESS);
529 	}
530 
531 broken:
532 	kmem_free(pdptr->par_intr, size);
533 	pdptr->par_intr = NULL;
534 	pdptr->par_nintr = 0;
535 	if (inpri != NULL)
536 		kmem_free(got_prop, got_len);
537 
538 	return (DDI_FAILURE);
539 }
540 
541 /*
542  * Create a ddi_parent_private_data structure from the ddi properties of
543  * the dev_info node.
544  *
545  * The "reg" and either an "intr" or "interrupts" properties are required
546  * if the driver wishes to create mappings or field interrupts on behalf
547  * of the device.
548  *
549  * The "reg" property is assumed to be a list of at least one triple
550  *
551  *	<bustype, address, size>*1
552  *
553  * The "intr" property is assumed to be a list of at least one duple
554  *
555  *	<SPARC ipl, vector#>*1
556  *
557  * The "interrupts" property is assumed to be a list of at least one
558  * n-tuples that describes the interrupt capabilities of the bus the device
559  * is connected to.  For SBus, this looks like
560  *
561  *	<SBus-level>*1
562  *
563  * (This property obsoletes the 'intr' property).
564  *
565  * The "ranges" property is optional.
566  */
567 void
568 make_ddi_ppd(dev_info_t *child, struct ddi_parent_private_data **ppd)
569 {
570 	struct ddi_parent_private_data *pdptr;
571 	int n;
572 	int *reg_prop, *rng_prop, *intr_prop, *irupts_prop;
573 	uint_t reg_len, rng_len, intr_len, irupts_len;
574 
575 	*ppd = pdptr = kmem_zalloc(sizeof (*pdptr), KM_SLEEP);
576 
577 	/*
578 	 * Handle the 'reg' property.
579 	 */
580 	if ((get_prop_int_array(child, "reg", &reg_prop, &reg_len) ==
581 	    DDI_PROP_SUCCESS) && (reg_len != 0)) {
582 		pdptr->par_nreg = reg_len / (int)sizeof (struct regspec);
583 		pdptr->par_reg = (struct regspec *)reg_prop;
584 	}
585 
586 	/*
587 	 * See if I have a range (adding one where needed - this
588 	 * means to add one for sbus node in sun4c, when romvec > 0,
589 	 * if no range is already defined in the PROM node.
590 	 * (Currently no sun4c PROMS define range properties,
591 	 * but they should and may in the future.)  For the SBus
592 	 * node, the range is defined by the SBus reg property.
593 	 */
594 	if (get_prop_int_array(child, "ranges", &rng_prop, &rng_len)
595 	    == DDI_PROP_SUCCESS) {
596 		pdptr->par_nrng = rng_len / (int)(sizeof (struct rangespec));
597 		pdptr->par_rng = (struct rangespec *)rng_prop;
598 	}
599 
600 	/*
601 	 * Handle the 'intr' and 'interrupts' properties
602 	 */
603 
604 	/*
605 	 * For backwards compatibility
606 	 * we first look for the 'intr' property for the device.
607 	 */
608 	if (get_prop_int_array(child, "intr", &intr_prop, &intr_len)
609 	    != DDI_PROP_SUCCESS) {
610 		intr_len = 0;
611 	}
612 
613 	/*
614 	 * If we're to support bus adapters and future platforms cleanly,
615 	 * we need to support the generalized 'interrupts' property.
616 	 */
617 	if (get_prop_int_array(child, "interrupts", &irupts_prop,
618 	    &irupts_len) != DDI_PROP_SUCCESS) {
619 		irupts_len = 0;
620 	} else if (intr_len != 0) {
621 		/*
622 		 * If both 'intr' and 'interrupts' are defined,
623 		 * then 'interrupts' wins and we toss the 'intr' away.
624 		 */
625 		ddi_prop_free((void *)intr_prop);
626 		intr_len = 0;
627 	}
628 
629 	if (intr_len != 0) {
630 
631 		/*
632 		 * Translate the 'intr' property into an array
633 		 * an array of struct intrspec's.  There's not really
634 		 * very much to do here except copy what's out there.
635 		 */
636 
637 		struct intrspec *new;
638 		struct prop_ispec *l;
639 
640 		n = pdptr->par_nintr =
641 			intr_len / sizeof (struct prop_ispec);
642 		l = (struct prop_ispec *)intr_prop;
643 		pdptr->par_intr =
644 		    new = kmem_zalloc(n * sizeof (struct intrspec), KM_SLEEP);
645 		while (n--) {
646 			new->intrspec_pri = l->pri;
647 			new->intrspec_vec = l->vec;
648 			new++;
649 			l++;
650 		}
651 		ddi_prop_free((void *)intr_prop);
652 
653 	} else if ((n = irupts_len) != 0) {
654 		size_t size;
655 		int *out;
656 
657 		/*
658 		 * Translate the 'interrupts' property into an array
659 		 * of intrspecs for the rest of the DDI framework to
660 		 * toy with.  Only our ancestors really know how to
661 		 * do this, so ask 'em.  We massage the 'interrupts'
662 		 * property so that it is pre-pended by a count of
663 		 * the number of integers in the argument.
664 		 */
665 		size = sizeof (int) + n;
666 		out = kmem_alloc(size, KM_SLEEP);
667 		*out = n / sizeof (int);
668 		bcopy(irupts_prop, out + 1, (size_t)n);
669 		ddi_prop_free((void *)irupts_prop);
670 		if (impl_xlate_intrs(child, out, pdptr) != DDI_SUCCESS) {
671 			cmn_err(CE_CONT,
672 			    "Unable to translate 'interrupts' for %s%d\n",
673 			    DEVI(child)->devi_binding_name,
674 			    DEVI(child)->devi_instance);
675 		}
676 		kmem_free(out, size);
677 	}
678 }
679 
680 /*
681  * Name a child
682  */
683 static int
684 impl_sunbus_name_child(dev_info_t *child, char *name, int namelen)
685 {
686 	/*
687 	 * Fill in parent-private data and this function returns to us
688 	 * an indication if it used "registers" to fill in the data.
689 	 */
690 	if (ddi_get_parent_data(child) == NULL) {
691 		struct ddi_parent_private_data *pdptr;
692 		make_ddi_ppd(child, &pdptr);
693 		ddi_set_parent_data(child, pdptr);
694 	}
695 
696 	name[0] = '\0';
697 	if (sparc_pd_getnreg(child) > 0) {
698 		(void) snprintf(name, namelen, "%x,%x",
699 		    (uint_t)sparc_pd_getreg(child, 0)->regspec_bustype,
700 		    (uint_t)sparc_pd_getreg(child, 0)->regspec_addr);
701 	}
702 
703 	return (DDI_SUCCESS);
704 }
705 
706 /*
707  * Called from the bus_ctl op of sunbus (sbus, obio, etc) nexus drivers
708  * to implement the DDI_CTLOPS_INITCHILD operation.  That is, it names
709  * the children of sun busses based on the reg spec.
710  *
711  * Handles the following properties (in make_ddi_ppd):
712  *	Property		value
713  *	  Name			type
714  *	reg		register spec
715  *	intr		old-form interrupt spec
716  *	interrupts	new (bus-oriented) interrupt spec
717  *	ranges		range spec
718  */
719 int
720 impl_ddi_sunbus_initchild(dev_info_t *child)
721 {
722 	char name[MAXNAMELEN];
723 	void impl_ddi_sunbus_removechild(dev_info_t *);
724 
725 	/*
726 	 * Name the child, also makes parent private data
727 	 */
728 	(void) impl_sunbus_name_child(child, name, MAXNAMELEN);
729 	ddi_set_name_addr(child, name);
730 
731 	/*
732 	 * Attempt to merge a .conf node; if successful, remove the
733 	 * .conf node.
734 	 */
735 	if ((ndi_dev_is_persistent_node(child) == 0) &&
736 	    (ndi_merge_node(child, impl_sunbus_name_child) == DDI_SUCCESS)) {
737 		/*
738 		 * Return failure to remove node
739 		 */
740 		impl_ddi_sunbus_removechild(child);
741 		return (DDI_FAILURE);
742 	}
743 	return (DDI_SUCCESS);
744 }
745 
746 void
747 impl_free_ddi_ppd(dev_info_t *dip)
748 {
749 	struct ddi_parent_private_data *pdptr;
750 	size_t n;
751 
752 	if ((pdptr = ddi_get_parent_data(dip)) == NULL)
753 		return;
754 
755 	if ((n = (size_t)pdptr->par_nintr) != 0)
756 		/*
757 		 * Note that kmem_free is used here (instead of
758 		 * ddi_prop_free) because the contents of the
759 		 * property were placed into a separate buffer and
760 		 * mucked with a bit before being stored in par_intr.
761 		 * The actual return value from the prop lookup
762 		 * was freed with ddi_prop_free previously.
763 		 */
764 		kmem_free(pdptr->par_intr, n * sizeof (struct intrspec));
765 
766 	if ((n = (size_t)pdptr->par_nrng) != 0)
767 		ddi_prop_free((void *)pdptr->par_rng);
768 
769 	if ((n = pdptr->par_nreg) != 0)
770 		ddi_prop_free((void *)pdptr->par_reg);
771 
772 	kmem_free(pdptr, sizeof (*pdptr));
773 	ddi_set_parent_data(dip, NULL);
774 }
775 
776 void
777 impl_ddi_sunbus_removechild(dev_info_t *dip)
778 {
779 	impl_free_ddi_ppd(dip);
780 	ddi_set_name_addr(dip, NULL);
781 	/*
782 	 * Strip the node to properly convert it back to prototype form
783 	 */
784 	impl_rem_dev_props(dip);
785 }
786 
787 /*
788  * DDI Interrupt
789  */
790 
791 /*
792  * turn this on to force isa, eisa, and mca device to ignore the new
793  * hardware nodes in the device tree (normally turned on only for
794  * drivers that need it by setting the property "ignore-hardware-nodes"
795  * in their driver.conf file).
796  *
797  * 7/31/96 -- Turned off globally.  Leaving variable in for the moment
798  *		as safety valve.
799  */
800 int ignore_hardware_nodes = 0;
801 
802 /*
803  * Local data
804  */
805 static struct impl_bus_promops *impl_busp;
806 
807 
808 /*
809  * New DDI interrupt framework
810  */
811 
812 /*
813  * i_ddi_intr_ops:
814  *
815  * This is the interrupt operator function wrapper for the bus function
816  * bus_intr_op.
817  */
818 int
819 i_ddi_intr_ops(dev_info_t *dip, dev_info_t *rdip, ddi_intr_op_t op,
820     ddi_intr_handle_impl_t *hdlp, void * result)
821 {
822 	dev_info_t	*pdip = (dev_info_t *)DEVI(dip)->devi_parent;
823 	int		ret = DDI_FAILURE;
824 
825 	/* request parent to process this interrupt op */
826 	if (NEXUS_HAS_INTR_OP(pdip))
827 		ret = (*(DEVI(pdip)->devi_ops->devo_bus_ops->bus_intr_op))(
828 		    pdip, rdip, op, hdlp, result);
829 	else
830 		cmn_err(CE_WARN, "Failed to process interrupt "
831 		    "for %s%d due to down-rev nexus driver %s%d",
832 		    ddi_get_name(rdip), ddi_get_instance(rdip),
833 		    ddi_get_name(pdip), ddi_get_instance(pdip));
834 	return (ret);
835 }
836 
837 /*
838  * i_ddi_add_softint - allocate and add a soft interrupt to the system
839  */
840 int
841 i_ddi_add_softint(ddi_softint_hdl_impl_t *hdlp)
842 {
843 	int ret;
844 
845 	/* add soft interrupt handler */
846 	ret = add_avsoftintr((void *)hdlp, hdlp->ih_pri, hdlp->ih_cb_func,
847 	    DEVI(hdlp->ih_dip)->devi_name, hdlp->ih_cb_arg1, hdlp->ih_cb_arg2);
848 	return (ret ? DDI_SUCCESS : DDI_FAILURE);
849 }
850 
851 
852 void
853 i_ddi_remove_softint(ddi_softint_hdl_impl_t *hdlp)
854 {
855 	(void) rem_avsoftintr((void *)hdlp, hdlp->ih_pri, hdlp->ih_cb_func);
856 }
857 
858 
859 extern void (*setsoftint)(int, struct av_softinfo *);
860 extern boolean_t av_check_softint_pending(struct av_softinfo *, boolean_t);
861 
862 int
863 i_ddi_trigger_softint(ddi_softint_hdl_impl_t *hdlp, void *arg2)
864 {
865 	if (av_check_softint_pending(hdlp->ih_pending, B_FALSE))
866 		return (DDI_EPENDING);
867 
868 	update_avsoftintr_args((void *)hdlp, hdlp->ih_pri, arg2);
869 
870 	(*setsoftint)(hdlp->ih_pri, hdlp->ih_pending);
871 	return (DDI_SUCCESS);
872 }
873 
874 /*
875  * i_ddi_set_softint_pri:
876  *
877  * The way this works is that it first tries to add a softint vector
878  * at the new priority in hdlp. If that succeeds; then it removes the
879  * existing softint vector at the old priority.
880  */
881 int
882 i_ddi_set_softint_pri(ddi_softint_hdl_impl_t *hdlp, uint_t old_pri)
883 {
884 	int ret;
885 
886 	/*
887 	 * If a softint is pending at the old priority then fail the request.
888 	 */
889 	if (av_check_softint_pending(hdlp->ih_pending, B_TRUE))
890 		return (DDI_FAILURE);
891 
892 	ret = av_softint_movepri((void *)hdlp, old_pri);
893 	return (ret ? DDI_SUCCESS : DDI_FAILURE);
894 }
895 
896 void
897 i_ddi_alloc_intr_phdl(ddi_intr_handle_impl_t *hdlp)
898 {
899 	hdlp->ih_private = (void *)kmem_zalloc(sizeof (ihdl_plat_t), KM_SLEEP);
900 }
901 
902 void
903 i_ddi_free_intr_phdl(ddi_intr_handle_impl_t *hdlp)
904 {
905 	kmem_free(hdlp->ih_private, sizeof (ihdl_plat_t));
906 	hdlp->ih_private = NULL;
907 }
908 
909 int
910 i_ddi_get_intx_nintrs(dev_info_t *dip)
911 {
912 	struct ddi_parent_private_data *pdp;
913 
914 	if ((pdp = ddi_get_parent_data(dip)) == NULL)
915 		return (0);
916 
917 	return (pdp->par_nintr);
918 }
919 
920 /*
921  * DDI Memory/DMA
922  */
923 
924 /*
925  * Support for allocating DMAable memory to implement
926  * ddi_dma_mem_alloc(9F) interface.
927  */
928 
929 #define	KA_ALIGN_SHIFT	7
930 #define	KA_ALIGN	(1 << KA_ALIGN_SHIFT)
931 #define	KA_NCACHE	(PAGESHIFT + 1 - KA_ALIGN_SHIFT)
932 
933 /*
934  * Dummy DMA attribute template for kmem_io[].kmem_io_attr.  We only
935  * care about addr_lo, addr_hi, and align.  addr_hi will be dynamically set.
936  */
937 
938 static ddi_dma_attr_t kmem_io_attr = {
939 	DMA_ATTR_V0,
940 	0x0000000000000000ULL,		/* dma_attr_addr_lo */
941 	0x0000000000000000ULL,		/* dma_attr_addr_hi */
942 	0x00ffffff,
943 	0x1000,				/* dma_attr_align */
944 	1, 1, 0xffffffffULL, 0xffffffffULL, 0x1, 1, 0
945 };
946 
947 /* kmem io memory ranges and indices */
948 enum {
949 	IO_4P, IO_64G, IO_4G, IO_2G, IO_1G, IO_512M,
950 	IO_256M, IO_128M, IO_64M, IO_32M, IO_16M, MAX_MEM_RANGES
951 };
952 
953 static struct {
954 	vmem_t		*kmem_io_arena;
955 	kmem_cache_t	*kmem_io_cache[KA_NCACHE];
956 	ddi_dma_attr_t	kmem_io_attr;
957 } kmem_io[MAX_MEM_RANGES];
958 
959 static int kmem_io_idx;		/* index of first populated kmem_io[] */
960 
961 static page_t *
962 page_create_io_wrapper(void *addr, size_t len, int vmflag, void *arg)
963 {
964 	extern page_t *page_create_io(vnode_t *, u_offset_t, uint_t,
965 	    uint_t, struct as *, caddr_t, ddi_dma_attr_t *);
966 
967 	return (page_create_io(&kvp, (u_offset_t)(uintptr_t)addr, len,
968 	    PG_EXCL | ((vmflag & VM_NOSLEEP) ? 0 : PG_WAIT), &kas, addr, arg));
969 }
970 
971 static void *
972 segkmem_alloc_io_4P(vmem_t *vmp, size_t size, int vmflag)
973 {
974 	return (segkmem_xalloc(vmp, NULL, size, vmflag, 0,
975 	    page_create_io_wrapper, &kmem_io[IO_4P].kmem_io_attr));
976 }
977 
978 static void *
979 segkmem_alloc_io_64G(vmem_t *vmp, size_t size, int vmflag)
980 {
981 	return (segkmem_xalloc(vmp, NULL, size, vmflag, 0,
982 	    page_create_io_wrapper, &kmem_io[IO_64G].kmem_io_attr));
983 }
984 
985 static void *
986 segkmem_alloc_io_4G(vmem_t *vmp, size_t size, int vmflag)
987 {
988 	return (segkmem_xalloc(vmp, NULL, size, vmflag, 0,
989 	    page_create_io_wrapper, &kmem_io[IO_4G].kmem_io_attr));
990 }
991 
992 static void *
993 segkmem_alloc_io_2G(vmem_t *vmp, size_t size, int vmflag)
994 {
995 	return (segkmem_xalloc(vmp, NULL, size, vmflag, 0,
996 	    page_create_io_wrapper, &kmem_io[IO_2G].kmem_io_attr));
997 }
998 
999 static void *
1000 segkmem_alloc_io_1G(vmem_t *vmp, size_t size, int vmflag)
1001 {
1002 	return (segkmem_xalloc(vmp, NULL, size, vmflag, 0,
1003 	    page_create_io_wrapper, &kmem_io[IO_1G].kmem_io_attr));
1004 }
1005 
1006 static void *
1007 segkmem_alloc_io_512M(vmem_t *vmp, size_t size, int vmflag)
1008 {
1009 	return (segkmem_xalloc(vmp, NULL, size, vmflag, 0,
1010 	    page_create_io_wrapper, &kmem_io[IO_512M].kmem_io_attr));
1011 }
1012 
1013 static void *
1014 segkmem_alloc_io_256M(vmem_t *vmp, size_t size, int vmflag)
1015 {
1016 	return (segkmem_xalloc(vmp, NULL, size, vmflag, 0,
1017 	    page_create_io_wrapper, &kmem_io[IO_256M].kmem_io_attr));
1018 }
1019 
1020 static void *
1021 segkmem_alloc_io_128M(vmem_t *vmp, size_t size, int vmflag)
1022 {
1023 	return (segkmem_xalloc(vmp, NULL, size, vmflag, 0,
1024 	    page_create_io_wrapper, &kmem_io[IO_128M].kmem_io_attr));
1025 }
1026 
1027 static void *
1028 segkmem_alloc_io_64M(vmem_t *vmp, size_t size, int vmflag)
1029 {
1030 	return (segkmem_xalloc(vmp, NULL, size, vmflag, 0,
1031 	    page_create_io_wrapper, &kmem_io[IO_64M].kmem_io_attr));
1032 }
1033 
1034 static void *
1035 segkmem_alloc_io_32M(vmem_t *vmp, size_t size, int vmflag)
1036 {
1037 	return (segkmem_xalloc(vmp, NULL, size, vmflag, 0,
1038 	    page_create_io_wrapper, &kmem_io[IO_32M].kmem_io_attr));
1039 }
1040 
1041 static void *
1042 segkmem_alloc_io_16M(vmem_t *vmp, size_t size, int vmflag)
1043 {
1044 	return (segkmem_xalloc(vmp, NULL, size, vmflag, 0,
1045 	    page_create_io_wrapper, &kmem_io[IO_16M].kmem_io_attr));
1046 }
1047 
1048 struct {
1049 	uint64_t	io_limit;
1050 	char		*io_name;
1051 	void		*(*io_alloc)(vmem_t *, size_t, int);
1052 	int		io_initial;	/* kmem_io_init during startup */
1053 } io_arena_params[MAX_MEM_RANGES] = {
1054 	{0x000fffffffffffffULL,	"kmem_io_4P",	segkmem_alloc_io_4P,	1},
1055 	{0x0000000fffffffffULL,	"kmem_io_64G",	segkmem_alloc_io_64G,	0},
1056 	{0x00000000ffffffffULL,	"kmem_io_4G",	segkmem_alloc_io_4G,	1},
1057 	{0x000000007fffffffULL,	"kmem_io_2G",	segkmem_alloc_io_2G,	1},
1058 	{0x000000003fffffffULL,	"kmem_io_1G",	segkmem_alloc_io_1G,	0},
1059 	{0x000000001fffffffULL,	"kmem_io_512M",	segkmem_alloc_io_512M,	0},
1060 	{0x000000000fffffffULL,	"kmem_io_256M",	segkmem_alloc_io_256M,	0},
1061 	{0x0000000007ffffffULL,	"kmem_io_128M",	segkmem_alloc_io_128M,	0},
1062 	{0x0000000003ffffffULL,	"kmem_io_64M",	segkmem_alloc_io_64M,	0},
1063 	{0x0000000001ffffffULL,	"kmem_io_32M",	segkmem_alloc_io_32M,	0},
1064 	{0x0000000000ffffffULL,	"kmem_io_16M",	segkmem_alloc_io_16M,	1}
1065 };
1066 
1067 void
1068 kmem_io_init(int a)
1069 {
1070 	int	c;
1071 	char name[40];
1072 
1073 	kmem_io[a].kmem_io_arena = vmem_create(io_arena_params[a].io_name,
1074 	    NULL, 0, PAGESIZE, io_arena_params[a].io_alloc,
1075 	    segkmem_free, heap_arena, 0, VM_SLEEP);
1076 	for (c = 0; c < KA_NCACHE; c++) {
1077 		size_t size = KA_ALIGN << c;
1078 		(void) sprintf(name, "%s_%lu",
1079 		    io_arena_params[a].io_name, size);
1080 		kmem_io[a].kmem_io_cache[c] = kmem_cache_create(name,
1081 		    size, size, NULL, NULL, NULL, NULL,
1082 		    kmem_io[a].kmem_io_arena, 0);
1083 	}
1084 }
1085 
1086 /*
1087  * Return the index of the highest memory range for addr.
1088  */
1089 static int
1090 kmem_io_index(uint64_t addr)
1091 {
1092 	int n;
1093 
1094 	for (n = kmem_io_idx; n < MAX_MEM_RANGES; n++) {
1095 		if (kmem_io[n].kmem_io_attr.dma_attr_addr_hi <= addr) {
1096 			if (kmem_io[n].kmem_io_arena == NULL)
1097 				kmem_io_init(n);
1098 			return (n);
1099 		}
1100 	}
1101 	panic("kmem_io_index: invalid addr - must be at least 16m");
1102 
1103 	/*NOTREACHED*/
1104 }
1105 
1106 /*
1107  * Return the index of the next kmem_io populated memory range
1108  * after curindex.
1109  */
1110 static int
1111 kmem_io_index_next(int curindex)
1112 {
1113 	int n;
1114 
1115 	for (n = curindex + 1; n < MAX_MEM_RANGES; n++) {
1116 		if (kmem_io[n].kmem_io_arena)
1117 			return (n);
1118 	}
1119 	return (-1);
1120 }
1121 
1122 /*
1123  * allow kmem to be mapped in with different PTE cache attribute settings.
1124  * Used by i_ddi_mem_alloc()
1125  */
1126 int
1127 kmem_override_cache_attrs(caddr_t kva, size_t size, uint_t order)
1128 {
1129 	uint_t hat_flags;
1130 	caddr_t kva_end;
1131 	uint_t hat_attr;
1132 	pfn_t pfn;
1133 
1134 	if (hat_getattr(kas.a_hat, kva, &hat_attr) == -1) {
1135 		return (-1);
1136 	}
1137 
1138 	hat_attr &= ~HAT_ORDER_MASK;
1139 	hat_attr |= order | HAT_NOSYNC;
1140 	hat_flags = HAT_LOAD_LOCK;
1141 
1142 	kva_end = (caddr_t)(((uintptr_t)kva + size + PAGEOFFSET) &
1143 	    (uintptr_t)PAGEMASK);
1144 	kva = (caddr_t)((uintptr_t)kva & (uintptr_t)PAGEMASK);
1145 
1146 	while (kva < kva_end) {
1147 		pfn = hat_getpfnum(kas.a_hat, kva);
1148 		hat_unload(kas.a_hat, kva, PAGESIZE, HAT_UNLOAD_UNLOCK);
1149 		hat_devload(kas.a_hat, kva, PAGESIZE, pfn, hat_attr, hat_flags);
1150 		kva += MMU_PAGESIZE;
1151 	}
1152 
1153 	return (0);
1154 }
1155 
1156 void
1157 ka_init(void)
1158 {
1159 	int a;
1160 	extern pfn_t physmax;
1161 	uint64_t maxphysaddr = mmu_ptob((uint64_t)physmax + 1) - 1;
1162 
1163 	ASSERT(maxphysaddr <= io_arena_params[0].io_limit);
1164 
1165 	for (a = 0; a < MAX_MEM_RANGES; a++) {
1166 		if (maxphysaddr >= io_arena_params[a + 1].io_limit) {
1167 			if (maxphysaddr > io_arena_params[a + 1].io_limit)
1168 				io_arena_params[a].io_limit = maxphysaddr;
1169 			else
1170 				a++;
1171 			break;
1172 		}
1173 	}
1174 	kmem_io_idx = a;
1175 
1176 	for (; a < MAX_MEM_RANGES; a++) {
1177 		kmem_io[a].kmem_io_attr = kmem_io_attr;
1178 		kmem_io[a].kmem_io_attr.dma_attr_addr_hi =
1179 		    io_arena_params[a].io_limit;
1180 		/*
1181 		 * initialize kmem_io[] arena/cache corresponding to
1182 		 * maxphysaddr and to the "common" io memory ranges that
1183 		 * have io_initial set to a non-zero value.
1184 		 */
1185 		if (io_arena_params[a].io_initial || a == kmem_io_idx)
1186 			kmem_io_init(a);
1187 	}
1188 }
1189 
1190 /*
1191  * put contig address/size
1192  */
1193 static void *
1194 putctgas(void *addr, size_t size)
1195 {
1196 	struct ctgas	*ctgp = &ctglist;
1197 	int		i;
1198 
1199 	CTGLOCK();
1200 	do {
1201 		if ((i = ctgp->ctg_index) < CTGENTRIES) {
1202 			ctgp->ctg_addr[i] = addr;
1203 			ctgp->ctg_size[i] = size;
1204 			ctgp->ctg_index++;
1205 			break;
1206 		}
1207 		if (!ctgp->ctg_next)
1208 			ctgp->ctg_next = kmem_zalloc(sizeof (struct ctgas),
1209 			    KM_NOSLEEP);
1210 		ctgp = ctgp->ctg_next;
1211 	} while (ctgp);
1212 
1213 	CTGUNLOCK();
1214 	return (ctgp);
1215 }
1216 
1217 /*
1218  * get contig size by addr
1219  */
1220 static size_t
1221 getctgsz(void *addr)
1222 {
1223 	struct ctgas	*ctgp = &ctglist;
1224 	int		i, j;
1225 	size_t		sz;
1226 
1227 	ASSERT(addr);
1228 	CTGLOCK();
1229 
1230 	while (ctgp) {
1231 		for (i = 0; i < ctgp->ctg_index; i++) {
1232 			if (addr != ctgp->ctg_addr[i])
1233 				continue;
1234 
1235 			sz = ctgp->ctg_size[i];
1236 			j = --ctgp->ctg_index;
1237 			if (i != j) {
1238 				ctgp->ctg_size[i] = ctgp->ctg_size[j];
1239 				ctgp->ctg_addr[i] = ctgp->ctg_addr[j];
1240 			}
1241 			CTGUNLOCK();
1242 			return (sz);
1243 		}
1244 		ctgp = ctgp->ctg_next;
1245 	}
1246 
1247 	CTGUNLOCK();
1248 	return (0);
1249 }
1250 
1251 /*
1252  * contig_alloc:
1253  *
1254  *	allocates contiguous memory to satisfy the 'size' and dma attributes
1255  *	specified in 'attr'.
1256  *
1257  *	Not all of memory need to be physically contiguous if the
1258  *	scatter-gather list length is greater than 1.
1259  */
1260 
1261 /*ARGSUSED*/
1262 void *
1263 contig_alloc(size_t size, ddi_dma_attr_t *attr, uintptr_t align, int cansleep)
1264 {
1265 	pgcnt_t		pgcnt = btopr(size);
1266 	size_t		asize = pgcnt * PAGESIZE;
1267 	page_t		*ppl;
1268 	int		pflag;
1269 	void		*addr;
1270 
1271 	extern page_t *page_create_io(vnode_t *, u_offset_t, uint_t,
1272 	    uint_t, struct as *, caddr_t, ddi_dma_attr_t *);
1273 
1274 	/* segkmem_xalloc */
1275 
1276 	if (align <= PAGESIZE)
1277 		addr = vmem_alloc(heap_arena, asize,
1278 		    (cansleep) ? VM_SLEEP : VM_NOSLEEP);
1279 	else
1280 		addr = vmem_xalloc(heap_arena, asize, align, 0, 0, NULL, NULL,
1281 		    (cansleep) ? VM_SLEEP : VM_NOSLEEP);
1282 	if (addr) {
1283 		ASSERT(!((uintptr_t)addr & (align - 1)));
1284 
1285 		if (page_resv(pgcnt,
1286 			(cansleep) ? KM_SLEEP : KM_NOSLEEP) == 0) {
1287 
1288 			vmem_free(heap_arena, addr, asize);
1289 			return (NULL);
1290 		}
1291 		pflag = PG_EXCL;
1292 
1293 		if (cansleep)
1294 			pflag |= PG_WAIT;
1295 
1296 		/* 4k req gets from freelists rather than pfn search */
1297 		if (pgcnt > 1 || align > PAGESIZE)
1298 			pflag |= PG_PHYSCONTIG;
1299 
1300 		ppl = page_create_io(&kvp, (u_offset_t)(uintptr_t)addr,
1301 			asize, pflag, &kas, (caddr_t)addr, attr);
1302 
1303 		if (!ppl) {
1304 			vmem_free(heap_arena, addr, asize);
1305 			page_unresv(pgcnt);
1306 			return (NULL);
1307 		}
1308 
1309 		while (ppl != NULL) {
1310 			page_t	*pp = ppl;
1311 			page_sub(&ppl, pp);
1312 			ASSERT(page_iolock_assert(pp));
1313 			page_io_unlock(pp);
1314 			page_downgrade(pp);
1315 			hat_memload(kas.a_hat, (caddr_t)(uintptr_t)pp->p_offset,
1316 				pp, (PROT_ALL & ~PROT_USER) |
1317 				HAT_NOSYNC, HAT_LOAD_LOCK);
1318 		}
1319 	}
1320 	return (addr);
1321 }
1322 
1323 static void
1324 contig_free(void *addr, size_t size)
1325 {
1326 	pgcnt_t	pgcnt = btopr(size);
1327 	size_t	asize = pgcnt * PAGESIZE;
1328 	caddr_t	a, ea;
1329 	page_t	*pp;
1330 
1331 	hat_unload(kas.a_hat, addr, asize, HAT_UNLOAD_UNLOCK);
1332 
1333 	for (a = addr, ea = a + asize; a < ea; a += PAGESIZE) {
1334 		pp = page_find(&kvp,
1335 				(u_offset_t)(uintptr_t)a);
1336 		if (!pp)
1337 			panic("contig_free: contig pp not found");
1338 
1339 		if (!page_tryupgrade(pp)) {
1340 			page_unlock(pp);
1341 			pp = page_lookup(&kvp,
1342 				(u_offset_t)(uintptr_t)a, SE_EXCL);
1343 			if (pp == NULL)
1344 				panic("contig_free: page freed");
1345 		}
1346 		page_destroy(pp, 0);
1347 	}
1348 
1349 	page_unresv(pgcnt);
1350 	vmem_free(heap_arena, addr, asize);
1351 }
1352 
1353 /*
1354  * Allocate from the system, aligned on a specific boundary.
1355  * The alignment, if non-zero, must be a power of 2.
1356  */
1357 static void *
1358 kalloca(size_t size, size_t align, int cansleep, int physcontig,
1359 	ddi_dma_attr_t *attr)
1360 {
1361 	size_t *addr, *raddr, rsize;
1362 	size_t hdrsize = 4 * sizeof (size_t);	/* must be power of 2 */
1363 	int a, i, c;
1364 	vmem_t *vmp;
1365 	kmem_cache_t *cp = NULL;
1366 
1367 	if (attr->dma_attr_addr_lo > mmu_ptob((uint64_t)ddiphysmin))
1368 		return (NULL);
1369 
1370 	align = MAX(align, hdrsize);
1371 	ASSERT((align & (align - 1)) == 0);
1372 
1373 	/*
1374 	 * All of our allocators guarantee 16-byte alignment, so we don't
1375 	 * need to reserve additional space for the header.
1376 	 * To simplify picking the correct kmem_io_cache, we round up to
1377 	 * a multiple of KA_ALIGN.
1378 	 */
1379 	rsize = P2ROUNDUP_TYPED(size + align, KA_ALIGN, size_t);
1380 
1381 	if (physcontig && rsize > PAGESIZE) {
1382 		if (addr = contig_alloc(size, attr, align, cansleep)) {
1383 			if (!putctgas(addr, size))
1384 				contig_free(addr, size);
1385 			else
1386 				return (addr);
1387 		}
1388 		return (NULL);
1389 	}
1390 
1391 	a = kmem_io_index(attr->dma_attr_addr_hi);
1392 
1393 	if (rsize > PAGESIZE) {
1394 		vmp = kmem_io[a].kmem_io_arena;
1395 		raddr = vmem_alloc(vmp, rsize,
1396 		    (cansleep) ? VM_SLEEP : VM_NOSLEEP);
1397 	} else {
1398 		c = highbit((rsize >> KA_ALIGN_SHIFT) - 1);
1399 		cp = kmem_io[a].kmem_io_cache[c];
1400 		raddr = kmem_cache_alloc(cp, (cansleep) ? KM_SLEEP :
1401 		    KM_NOSLEEP);
1402 	}
1403 
1404 	if (raddr == NULL) {
1405 		int	na;
1406 
1407 		ASSERT(cansleep == 0);
1408 		if (rsize > PAGESIZE)
1409 			return (NULL);
1410 		/*
1411 		 * System does not have memory in the requested range.
1412 		 * Try smaller kmem io ranges and larger cache sizes
1413 		 * to see if there might be memory available in
1414 		 * these other caches.
1415 		 */
1416 
1417 		for (na = kmem_io_index_next(a); na >= 0;
1418 		    na = kmem_io_index_next(na)) {
1419 			ASSERT(kmem_io[na].kmem_io_arena);
1420 			cp = kmem_io[na].kmem_io_cache[c];
1421 			raddr = kmem_cache_alloc(cp, KM_NOSLEEP);
1422 			if (raddr)
1423 				goto kallocdone;
1424 		}
1425 		/* now try the larger kmem io cache sizes */
1426 		for (na = a; na >= 0; na = kmem_io_index_next(na)) {
1427 			for (i = c + 1; i < KA_NCACHE; i++) {
1428 				cp = kmem_io[na].kmem_io_cache[i];
1429 				raddr = kmem_cache_alloc(cp, KM_NOSLEEP);
1430 				if (raddr)
1431 					goto kallocdone;
1432 			}
1433 		}
1434 		return (NULL);
1435 	}
1436 
1437 kallocdone:
1438 	ASSERT(!P2CROSS((uintptr_t)raddr, (uintptr_t)raddr + rsize - 1,
1439 	    PAGESIZE) || rsize > PAGESIZE);
1440 
1441 	addr = (size_t *)P2ROUNDUP((uintptr_t)raddr + hdrsize, align);
1442 	ASSERT((uintptr_t)addr + size - (uintptr_t)raddr <= rsize);
1443 
1444 	addr[-4] = (size_t)cp;
1445 	addr[-3] = (size_t)vmp;
1446 	addr[-2] = (size_t)raddr;
1447 	addr[-1] = rsize;
1448 
1449 	return (addr);
1450 }
1451 
1452 static void
1453 kfreea(void *addr)
1454 {
1455 	size_t		size;
1456 
1457 	if (!((uintptr_t)addr & PAGEOFFSET) && (size = getctgsz(addr))) {
1458 		contig_free(addr, size);
1459 	} else {
1460 		size_t	*saddr = addr;
1461 		if (saddr[-4] == 0)
1462 			vmem_free((vmem_t *)saddr[-3], (void *)saddr[-2],
1463 				saddr[-1]);
1464 		else
1465 			kmem_cache_free((kmem_cache_t *)saddr[-4],
1466 				(void *)saddr[-2]);
1467 	}
1468 }
1469 
1470 /*ARGSUSED*/
1471 void
1472 i_ddi_devacc_to_hatacc(ddi_device_acc_attr_t *devaccp, uint_t *hataccp)
1473 {
1474 }
1475 
1476 /*
1477  * Check if the specified cache attribute is supported on the platform.
1478  * This function must be called before i_ddi_cacheattr_to_hatacc().
1479  */
1480 boolean_t
1481 i_ddi_check_cache_attr(uint_t flags)
1482 {
1483 	/*
1484 	 * The cache attributes are mutually exclusive. Any combination of
1485 	 * the attributes leads to a failure.
1486 	 */
1487 	uint_t cache_attr = IOMEM_CACHE_ATTR(flags);
1488 	if ((cache_attr != 0) && ((cache_attr & (cache_attr - 1)) != 0))
1489 		return (B_FALSE);
1490 
1491 	/* All cache attributes are supported on X86/X64 */
1492 	if (cache_attr & (IOMEM_DATA_UNCACHED | IOMEM_DATA_CACHED |
1493 	    IOMEM_DATA_UC_WR_COMBINE))
1494 		return (B_TRUE);
1495 
1496 	/* undefined attributes */
1497 	return (B_FALSE);
1498 }
1499 
1500 /* set HAT cache attributes from the cache attributes */
1501 void
1502 i_ddi_cacheattr_to_hatacc(uint_t flags, uint_t *hataccp)
1503 {
1504 	uint_t cache_attr = IOMEM_CACHE_ATTR(flags);
1505 	static char *fname = "i_ddi_cacheattr_to_hatacc";
1506 
1507 	/*
1508 	 * If write-combining is not supported, then it falls back
1509 	 * to uncacheable.
1510 	 */
1511 	if (cache_attr == IOMEM_DATA_UC_WR_COMBINE && !(x86_feature & X86_PAT))
1512 		cache_attr = IOMEM_DATA_UNCACHED;
1513 
1514 	/*
1515 	 * set HAT attrs according to the cache attrs.
1516 	 */
1517 	switch (cache_attr) {
1518 	case IOMEM_DATA_UNCACHED:
1519 		*hataccp &= ~HAT_ORDER_MASK;
1520 		*hataccp |= (HAT_STRICTORDER | HAT_PLAT_NOCACHE);
1521 		break;
1522 	case IOMEM_DATA_UC_WR_COMBINE:
1523 		*hataccp &= ~HAT_ORDER_MASK;
1524 		*hataccp |= (HAT_MERGING_OK | HAT_PLAT_NOCACHE);
1525 		break;
1526 	case IOMEM_DATA_CACHED:
1527 		*hataccp &= ~HAT_ORDER_MASK;
1528 		*hataccp |= HAT_UNORDERED_OK;
1529 		break;
1530 	/*
1531 	 * This case must not occur because the cache attribute is scrutinized
1532 	 * before this function is called.
1533 	 */
1534 	default:
1535 		/*
1536 		 * set cacheable to hat attrs.
1537 		 */
1538 		*hataccp &= ~HAT_ORDER_MASK;
1539 		*hataccp |= HAT_UNORDERED_OK;
1540 		cmn_err(CE_WARN, "%s: cache_attr=0x%x is ignored.",
1541 		    fname, cache_attr);
1542 	}
1543 }
1544 
1545 /*
1546  * This should actually be called i_ddi_dma_mem_alloc. There should
1547  * also be an i_ddi_pio_mem_alloc. i_ddi_dma_mem_alloc should call
1548  * through the device tree with the DDI_CTLOPS_DMA_ALIGN ctl ops to
1549  * get alignment requirements for DMA memory. i_ddi_pio_mem_alloc
1550  * should use DDI_CTLOPS_PIO_ALIGN. Since we only have i_ddi_mem_alloc
1551  * so far which is used for both, DMA and PIO, we have to use the DMA
1552  * ctl ops to make everybody happy.
1553  */
1554 /*ARGSUSED*/
1555 int
1556 i_ddi_mem_alloc(dev_info_t *dip, ddi_dma_attr_t *attr,
1557 	size_t length, int cansleep, int flags,
1558 	ddi_device_acc_attr_t *accattrp, caddr_t *kaddrp,
1559 	size_t *real_length, ddi_acc_hdl_t *ap)
1560 {
1561 	caddr_t a;
1562 	int iomin;
1563 	ddi_acc_impl_t *iap;
1564 	int physcontig = 0;
1565 	pgcnt_t npages;
1566 	pgcnt_t minctg;
1567 	uint_t order;
1568 	int e;
1569 
1570 	/*
1571 	 * Check legality of arguments
1572 	 */
1573 	if (length == 0 || kaddrp == NULL || attr == NULL) {
1574 		return (DDI_FAILURE);
1575 	}
1576 
1577 	if (attr->dma_attr_minxfer == 0 || attr->dma_attr_align == 0 ||
1578 		(attr->dma_attr_align & (attr->dma_attr_align - 1)) ||
1579 		(attr->dma_attr_minxfer & (attr->dma_attr_minxfer - 1))) {
1580 			return (DDI_FAILURE);
1581 	}
1582 
1583 	/*
1584 	 * figure out most restrictive alignment requirement
1585 	 */
1586 	iomin = attr->dma_attr_minxfer;
1587 	iomin = maxbit(iomin, attr->dma_attr_align);
1588 	if (iomin == 0)
1589 		return (DDI_FAILURE);
1590 
1591 	ASSERT((iomin & (iomin - 1)) == 0);
1592 
1593 	/*
1594 	 * if we allocate memory with IOMEM_DATA_UNCACHED or
1595 	 * IOMEM_DATA_UC_WR_COMBINE, make sure we allocate a page aligned
1596 	 * memory that ends on a page boundry.
1597 	 * Don't want to have to different cache mappings to the same
1598 	 * physical page.
1599 	 */
1600 	if (OVERRIDE_CACHE_ATTR(flags)) {
1601 		iomin = (iomin + MMU_PAGEOFFSET) & MMU_PAGEMASK;
1602 		length = (length + MMU_PAGEOFFSET) & (size_t)MMU_PAGEMASK;
1603 	}
1604 
1605 	/*
1606 	 * Determine if we need to satisfy the request for physically
1607 	 * contiguous memory or alignments larger than pagesize.
1608 	 */
1609 	npages = btopr(length + attr->dma_attr_align);
1610 	minctg = howmany(npages, attr->dma_attr_sgllen);
1611 
1612 	if (minctg > 1) {
1613 		uint64_t pfnseg = attr->dma_attr_seg >> PAGESHIFT;
1614 		/*
1615 		 * verify that the minimum contig requirement for the
1616 		 * actual length does not cross segment boundary.
1617 		 */
1618 		length = P2ROUNDUP_TYPED(length, attr->dma_attr_minxfer,
1619 		    size_t);
1620 		npages = btopr(length);
1621 		minctg = howmany(npages, attr->dma_attr_sgllen);
1622 		if (minctg > pfnseg + 1)
1623 			return (DDI_FAILURE);
1624 		physcontig = 1;
1625 	} else {
1626 		length = P2ROUNDUP_TYPED(length, iomin, size_t);
1627 	}
1628 
1629 	/*
1630 	 * Allocate the requested amount from the system.
1631 	 */
1632 	a = kalloca(length, iomin, cansleep, physcontig, attr);
1633 
1634 	if ((*kaddrp = a) == NULL)
1635 		return (DDI_FAILURE);
1636 
1637 	/*
1638 	 * if we to modify the cache attributes, go back and muck with the
1639 	 * mappings.
1640 	 */
1641 	if (OVERRIDE_CACHE_ATTR(flags)) {
1642 		order = 0;
1643 		i_ddi_cacheattr_to_hatacc(flags, &order);
1644 		e = kmem_override_cache_attrs(a, length, order);
1645 		if (e != 0) {
1646 			kfreea(a);
1647 			return (DDI_FAILURE);
1648 		}
1649 	}
1650 
1651 	if (real_length) {
1652 		*real_length = length;
1653 	}
1654 	if (ap) {
1655 		/*
1656 		 * initialize access handle
1657 		 */
1658 		iap = (ddi_acc_impl_t *)ap->ah_platform_private;
1659 		iap->ahi_acc_attr |= DDI_ACCATTR_CPU_VADDR;
1660 		impl_acc_hdl_init(ap);
1661 	}
1662 
1663 	return (DDI_SUCCESS);
1664 }
1665 
1666 /*
1667  * covert old DMA limits structure to DMA attribute structure
1668  * and continue
1669  */
1670 int
1671 i_ddi_mem_alloc_lim(dev_info_t *dip, ddi_dma_lim_t *limits,
1672 	size_t length, int cansleep, int streaming,
1673 	ddi_device_acc_attr_t *accattrp, caddr_t *kaddrp,
1674 	uint_t *real_length, ddi_acc_hdl_t *ap)
1675 {
1676 	ddi_dma_attr_t dma_attr, *attrp;
1677 	size_t rlen;
1678 	int ret;
1679 
1680 	if (limits == NULL) {
1681 		return (DDI_FAILURE);
1682 	}
1683 
1684 	/*
1685 	 * set up DMA attribute structure to pass to i_ddi_mem_alloc()
1686 	 */
1687 	attrp = &dma_attr;
1688 	attrp->dma_attr_version = DMA_ATTR_V0;
1689 	attrp->dma_attr_addr_lo = (uint64_t)limits->dlim_addr_lo;
1690 	attrp->dma_attr_addr_hi = (uint64_t)limits->dlim_addr_hi;
1691 	attrp->dma_attr_count_max = (uint64_t)limits->dlim_ctreg_max;
1692 	attrp->dma_attr_align = 1;
1693 	attrp->dma_attr_burstsizes = (uint_t)limits->dlim_burstsizes;
1694 	attrp->dma_attr_minxfer = (uint32_t)limits->dlim_minxfer;
1695 	attrp->dma_attr_maxxfer = (uint64_t)limits->dlim_reqsize;
1696 	attrp->dma_attr_seg = (uint64_t)limits->dlim_adreg_max;
1697 	attrp->dma_attr_sgllen = limits->dlim_sgllen;
1698 	attrp->dma_attr_granular = (uint32_t)limits->dlim_granular;
1699 	attrp->dma_attr_flags = 0;
1700 
1701 	ret = i_ddi_mem_alloc(dip, attrp, length, cansleep, streaming,
1702 			accattrp, kaddrp, &rlen, ap);
1703 	if (ret == DDI_SUCCESS) {
1704 		if (real_length)
1705 			*real_length = (uint_t)rlen;
1706 	}
1707 	return (ret);
1708 }
1709 
1710 /* ARGSUSED */
1711 void
1712 i_ddi_mem_free(caddr_t kaddr, ddi_acc_hdl_t *ap)
1713 {
1714 	if (ap != NULL) {
1715 		/*
1716 		 * if we modified the cache attributes on alloc, go back and
1717 		 * fix them since this memory could be returned to the
1718 		 * general pool.
1719 		 */
1720 		if (OVERRIDE_CACHE_ATTR(ap->ah_xfermodes)) {
1721 			uint_t order = 0;
1722 			int e;
1723 			i_ddi_cacheattr_to_hatacc(IOMEM_DATA_CACHED, &order);
1724 			e = kmem_override_cache_attrs(kaddr, ap->ah_len, order);
1725 			if (e != 0) {
1726 				cmn_err(CE_WARN, "i_ddi_mem_free() failed to "
1727 				    "override cache attrs, memory leaked\n");
1728 				return;
1729 			}
1730 		}
1731 	}
1732 	kfreea(kaddr);
1733 }
1734 
1735 /*
1736  * Access Barriers
1737  *
1738  */
1739 /*ARGSUSED*/
1740 int
1741 i_ddi_ontrap(ddi_acc_handle_t hp)
1742 {
1743 	return (DDI_FAILURE);
1744 }
1745 
1746 /*ARGSUSED*/
1747 void
1748 i_ddi_notrap(ddi_acc_handle_t hp)
1749 {
1750 }
1751 
1752 
1753 /*
1754  * Misc Functions
1755  */
1756 
1757 /*
1758  * Implementation instance override functions
1759  *
1760  * No override on i86pc
1761  */
1762 /*ARGSUSED*/
1763 uint_t
1764 impl_assign_instance(dev_info_t *dip)
1765 {
1766 	return ((uint_t)-1);
1767 }
1768 
1769 /*ARGSUSED*/
1770 int
1771 impl_keep_instance(dev_info_t *dip)
1772 {
1773 	return (DDI_FAILURE);
1774 }
1775 
1776 /*ARGSUSED*/
1777 int
1778 impl_free_instance(dev_info_t *dip)
1779 {
1780 	return (DDI_FAILURE);
1781 }
1782 
1783 /*ARGSUSED*/
1784 int
1785 impl_check_cpu(dev_info_t *devi)
1786 {
1787 	return (DDI_SUCCESS);
1788 }
1789 
1790 /*
1791  * Referenced in common/cpr_driver.c: Power off machine.
1792  * Don't know how to power off i86pc.
1793  */
1794 void
1795 arch_power_down()
1796 {}
1797 
1798 /*
1799  * Copy name to property_name, since name
1800  * is in the low address range below kernelbase.
1801  */
1802 static void
1803 copy_boot_str(const char *boot_str, char *kern_str, int len)
1804 {
1805 	int i = 0;
1806 
1807 	while (i < len - 1 && boot_str[i] != '\0') {
1808 		kern_str[i] = boot_str[i];
1809 		i++;
1810 	}
1811 
1812 	kern_str[i] = 0;	/* null terminate */
1813 	if (boot_str[i] != '\0')
1814 		cmn_err(CE_WARN,
1815 		    "boot property string is truncated to %s", kern_str);
1816 }
1817 
1818 static void
1819 get_boot_properties(void)
1820 {
1821 	extern char hw_provider[];
1822 	dev_info_t *devi;
1823 	char *name;
1824 	int length;
1825 	char property_name[50], property_val[50];
1826 	void *bop_staging_area;
1827 
1828 	bop_staging_area = kmem_zalloc(MMU_PAGESIZE, KM_NOSLEEP);
1829 
1830 	/*
1831 	 * Import "root" properties from the boot.
1832 	 *
1833 	 * We do this by invoking BOP_NEXTPROP until the list
1834 	 * is completely copied in.
1835 	 */
1836 
1837 	devi = ddi_root_node();
1838 	for (name = BOP_NEXTPROP(bootops, "");		/* get first */
1839 	    name;					/* NULL => DONE */
1840 	    name = BOP_NEXTPROP(bootops, name)) {	/* get next */
1841 
1842 		/* copy string to memory above kernelbase */
1843 		copy_boot_str(name, property_name, 50);
1844 
1845 		/*
1846 		 * Skip vga properties. They will be picked up later
1847 		 * by get_vga_properties.
1848 		 */
1849 		if (strcmp(property_name, "display-edif-block") == 0 ||
1850 		    strcmp(property_name, "display-edif-id") == 0) {
1851 			continue;
1852 		}
1853 
1854 		length = BOP_GETPROPLEN(bootops, property_name);
1855 		if (length == 0)
1856 			continue;
1857 		if (length > MMU_PAGESIZE) {
1858 			cmn_err(CE_NOTE,
1859 			    "boot property %s longer than 0x%x, ignored\n",
1860 			    property_name, MMU_PAGESIZE);
1861 			continue;
1862 		}
1863 		BOP_GETPROP(bootops, property_name, bop_staging_area);
1864 
1865 		/*
1866 		 * special properties:
1867 		 * si-machine, si-hw-provider
1868 		 *	goes to kernel data structures.
1869 		 * bios-boot-device and stdout
1870 		 *	goes to hardware property list so it may show up
1871 		 *	in the prtconf -vp output. This is needed by
1872 		 *	Install/Upgrade. Once we fix install upgrade,
1873 		 *	this can be taken out.
1874 		 */
1875 		if (strcmp(name, "si-machine") == 0) {
1876 			(void) strncpy(utsname.machine, bop_staging_area,
1877 			    SYS_NMLN);
1878 			utsname.machine[SYS_NMLN - 1] = (char)NULL;
1879 		} else if (strcmp(name, "si-hw-provider") == 0) {
1880 			(void) strncpy(hw_provider, bop_staging_area, SYS_NMLN);
1881 			hw_provider[SYS_NMLN - 1] = (char)NULL;
1882 		} else if (strcmp(name, "bios-boot-device") == 0) {
1883 			copy_boot_str(bop_staging_area, property_val, 50);
1884 			(void) ndi_prop_update_string(DDI_DEV_T_NONE, devi,
1885 			    property_name, property_val);
1886 		} else if (strcmp(name, "stdout") == 0) {
1887 			(void) ndi_prop_update_int(DDI_DEV_T_NONE, devi,
1888 			    property_name, *((int *)bop_staging_area));
1889 		} else {
1890 			/* Property type unknown, use old prop interface */
1891 			(void) e_ddi_prop_create(DDI_DEV_T_NONE, devi,
1892 			    DDI_PROP_CANSLEEP, property_name, bop_staging_area,
1893 			    length);
1894 		}
1895 	}
1896 
1897 	kmem_free(bop_staging_area, MMU_PAGESIZE);
1898 }
1899 
1900 static void
1901 get_vga_properties(void)
1902 {
1903 	dev_info_t *devi;
1904 	major_t major;
1905 	char *name;
1906 	int length;
1907 	char property_val[50];
1908 	void *bop_staging_area;
1909 
1910 	major = ddi_name_to_major("vgatext");
1911 	if (major == (major_t)-1)
1912 		return;
1913 	devi = devnamesp[major].dn_head;
1914 	if (devi == NULL)
1915 		return;
1916 
1917 	bop_staging_area = kmem_zalloc(MMU_PAGESIZE, KM_SLEEP);
1918 
1919 	/*
1920 	 * Import "vga" properties from the boot.
1921 	 */
1922 	name = "display-edif-block";
1923 	length = BOP_GETPROPLEN(bootops, name);
1924 	if (length > 0 && length < MMU_PAGESIZE) {
1925 		BOP_GETPROP(bootops, name, bop_staging_area);
1926 		(void) ndi_prop_update_byte_array(DDI_DEV_T_NONE,
1927 		    devi, name, bop_staging_area, length);
1928 	}
1929 
1930 	/*
1931 	 * kdmconfig is also looking for display-type and
1932 	 * video-adapter-type. We default to color and svga.
1933 	 *
1934 	 * Could it be "monochrome", "vga"?
1935 	 * Nah, you've got to come to the 21st century...
1936 	 * And you can set monitor type manually in kdmconfig
1937 	 * if you are really an old junky.
1938 	 */
1939 	(void) ndi_prop_update_string(DDI_DEV_T_NONE,
1940 	    devi, "display-type", "color");
1941 	(void) ndi_prop_update_string(DDI_DEV_T_NONE,
1942 	    devi, "video-adapter-type", "svga");
1943 
1944 	name = "display-edif-id";
1945 	length = BOP_GETPROPLEN(bootops, name);
1946 	if (length > 0 && length < MMU_PAGESIZE) {
1947 		BOP_GETPROP(bootops, name, bop_staging_area);
1948 		copy_boot_str(bop_staging_area, property_val, length);
1949 		(void) ndi_prop_update_string(DDI_DEV_T_NONE,
1950 		    devi, name, property_val);
1951 	}
1952 
1953 	kmem_free(bop_staging_area, MMU_PAGESIZE);
1954 }
1955 
1956 
1957 /*
1958  * This is temporary, but absolutely necessary.  If we are being
1959  * booted with a device tree created by the DevConf project's bootconf
1960  * program, then we have device information nodes that reflect
1961  * reality.  At this point in time in the Solaris release schedule, the
1962  * kernel drivers aren't prepared for reality.  They still depend on their
1963  * own ad-hoc interpretations of the properties created when their .conf
1964  * files were interpreted. These drivers use an "ignore-hardware-nodes"
1965  * property to prevent them from using the nodes passed up from the bootconf
1966  * device tree.
1967  *
1968  * Trying to assemble root file system drivers as we are booting from
1969  * devconf will fail if the kernel driver is basing its name_addr's on the
1970  * psuedo-node device info while the bootpath passed up from bootconf is using
1971  * reality-based name_addrs.  We help the boot along in this case by
1972  * looking at the pre-bootconf bootpath and determining if we would have
1973  * successfully matched if that had been the bootpath we had chosen.
1974  *
1975  * Note that we only even perform this extra check if we've booted
1976  * using bootconf's 1275 compliant bootpath, this is the boot device, and
1977  * we're trying to match the name_addr specified in the 1275 bootpath.
1978  */
1979 
1980 #define	MAXCOMPONENTLEN	32
1981 
1982 int
1983 x86_old_bootpath_name_addr_match(dev_info_t *cdip, char *caddr, char *naddr)
1984 {
1985 	/*
1986 	 *  There are multiple criteria to be met before we can even
1987 	 *  consider allowing a name_addr match here.
1988 	 *
1989 	 *  1) We must have been booted such that the bootconf program
1990 	 *	created device tree nodes and properties.  This can be
1991 	 *	determined by examining the 'bootpath' property.  This
1992 	 *	property will be a non-null string iff bootconf was
1993 	 *	involved in the boot.
1994 	 *
1995 	 *  2) The module that we want to match must be the boot device.
1996 	 *
1997 	 *  3) The instance of the module we are thinking of letting be
1998 	 *	our match must be ignoring hardware nodes.
1999 	 *
2000 	 *  4) The name_addr we want to match must be the name_addr
2001 	 *	specified in the 1275 bootpath.
2002 	 */
2003 	static char bootdev_module[MAXCOMPONENTLEN];
2004 	static char bootdev_oldmod[MAXCOMPONENTLEN];
2005 	static char bootdev_newaddr[MAXCOMPONENTLEN];
2006 	static char bootdev_oldaddr[MAXCOMPONENTLEN];
2007 	static int  quickexit;
2008 
2009 	char *daddr;
2010 	int dlen;
2011 
2012 	char	*lkupname;
2013 	int	rv = DDI_FAILURE;
2014 
2015 	if ((ddi_getlongprop(DDI_DEV_T_ANY, cdip, DDI_PROP_DONTPASS,
2016 		"devconf-addr", (caddr_t)&daddr, &dlen) == DDI_PROP_SUCCESS) &&
2017 	    (ddi_getprop(DDI_DEV_T_ANY, cdip, DDI_PROP_DONTPASS,
2018 		"ignore-hardware-nodes", -1) != -1)) {
2019 		if (strcmp(daddr, caddr) == 0) {
2020 			return (DDI_SUCCESS);
2021 		}
2022 	}
2023 
2024 	if (quickexit)
2025 		return (rv);
2026 
2027 	if (bootdev_module[0] == '\0') {
2028 		char *addrp, *eoaddrp;
2029 		char *busp, *modp, *atp;
2030 		char *bp1275, *bp;
2031 		int  bp1275len, bplen;
2032 
2033 		bp1275 = bp = addrp = eoaddrp = busp = modp = atp = NULL;
2034 
2035 		if (ddi_getlongprop(DDI_DEV_T_ANY,
2036 		    ddi_root_node(), 0, "bootpath",
2037 		    (caddr_t)&bp1275, &bp1275len) != DDI_PROP_SUCCESS ||
2038 		    bp1275len <= 1) {
2039 			/*
2040 			 * We didn't boot from bootconf so we never need to
2041 			 * do any special matches.
2042 			 */
2043 			quickexit = 1;
2044 			if (bp1275)
2045 				kmem_free(bp1275, bp1275len);
2046 			return (rv);
2047 		}
2048 
2049 		if (ddi_getlongprop(DDI_DEV_T_ANY,
2050 		    ddi_root_node(), 0, "boot-path",
2051 		    (caddr_t)&bp, &bplen) != DDI_PROP_SUCCESS || bplen <= 1) {
2052 			/*
2053 			 * No fallback position for matching. This is
2054 			 * certainly unexpected, but we'll handle it
2055 			 * just in case.
2056 			 */
2057 			quickexit = 1;
2058 			kmem_free(bp1275, bp1275len);
2059 			if (bp)
2060 				kmem_free(bp, bplen);
2061 			return (rv);
2062 		}
2063 
2064 		/*
2065 		 *  Determine boot device module and 1275 name_addr
2066 		 *
2067 		 *  bootpath assumed to be of the form /bus/module@name_addr
2068 		 */
2069 		if (busp = strchr(bp1275, '/')) {
2070 			if (modp = strchr(busp + 1, '/')) {
2071 				if (atp = strchr(modp + 1, '@')) {
2072 					*atp = '\0';
2073 					addrp = atp + 1;
2074 					if (eoaddrp = strchr(addrp, '/'))
2075 						*eoaddrp = '\0';
2076 				}
2077 			}
2078 		}
2079 
2080 		if (modp && addrp) {
2081 			(void) strncpy(bootdev_module, modp + 1,
2082 			    MAXCOMPONENTLEN);
2083 			bootdev_module[MAXCOMPONENTLEN - 1] = '\0';
2084 
2085 			(void) strncpy(bootdev_newaddr, addrp, MAXCOMPONENTLEN);
2086 			bootdev_newaddr[MAXCOMPONENTLEN - 1] = '\0';
2087 		} else {
2088 			quickexit = 1;
2089 			kmem_free(bp1275, bp1275len);
2090 			kmem_free(bp, bplen);
2091 			return (rv);
2092 		}
2093 
2094 		/*
2095 		 *  Determine fallback name_addr
2096 		 *
2097 		 *  10/3/96 - Also save fallback module name because it
2098 		 *  might actually be different than the current module
2099 		 *  name.  E.G., ISA pnp drivers have new names.
2100 		 *
2101 		 *  bootpath assumed to be of the form /bus/module@name_addr
2102 		 */
2103 		addrp = NULL;
2104 		if (busp = strchr(bp, '/')) {
2105 			if (modp = strchr(busp + 1, '/')) {
2106 				if (atp = strchr(modp + 1, '@')) {
2107 					*atp = '\0';
2108 					addrp = atp + 1;
2109 					if (eoaddrp = strchr(addrp, '/'))
2110 						*eoaddrp = '\0';
2111 				}
2112 			}
2113 		}
2114 
2115 		if (modp && addrp) {
2116 			(void) strncpy(bootdev_oldmod, modp + 1,
2117 			    MAXCOMPONENTLEN);
2118 			bootdev_module[MAXCOMPONENTLEN - 1] = '\0';
2119 
2120 			(void) strncpy(bootdev_oldaddr, addrp, MAXCOMPONENTLEN);
2121 			bootdev_oldaddr[MAXCOMPONENTLEN - 1] = '\0';
2122 		}
2123 
2124 		/* Free up the bootpath storage now that we're done with it. */
2125 		kmem_free(bp1275, bp1275len);
2126 		kmem_free(bp, bplen);
2127 
2128 		if (bootdev_oldaddr[0] == '\0') {
2129 			quickexit = 1;
2130 			return (rv);
2131 		}
2132 	}
2133 
2134 	if (((lkupname = ddi_get_name(cdip)) != NULL) &&
2135 	    (strcmp(bootdev_module, lkupname) == 0 ||
2136 	    strcmp(bootdev_oldmod, lkupname) == 0) &&
2137 	    ((ddi_getprop(DDI_DEV_T_ANY, cdip, DDI_PROP_DONTPASS,
2138 		"ignore-hardware-nodes", -1) != -1) ||
2139 		ignore_hardware_nodes) &&
2140 	    strcmp(bootdev_newaddr, caddr) == 0 &&
2141 	    strcmp(bootdev_oldaddr, naddr) == 0) {
2142 		rv = DDI_SUCCESS;
2143 	}
2144 
2145 	return (rv);
2146 }
2147 
2148 /*
2149  * Perform a copy from a memory mapped device (whose devinfo pointer is devi)
2150  * separately mapped at devaddr in the kernel to a kernel buffer at kaddr.
2151  */
2152 /*ARGSUSED*/
2153 int
2154 e_ddi_copyfromdev(dev_info_t *devi,
2155     off_t off, const void *devaddr, void *kaddr, size_t len)
2156 {
2157 	bcopy(devaddr, kaddr, len);
2158 	return (0);
2159 }
2160 
2161 /*
2162  * Perform a copy to a memory mapped device (whose devinfo pointer is devi)
2163  * separately mapped at devaddr in the kernel from a kernel buffer at kaddr.
2164  */
2165 /*ARGSUSED*/
2166 int
2167 e_ddi_copytodev(dev_info_t *devi,
2168     off_t off, const void *kaddr, void *devaddr, size_t len)
2169 {
2170 	bcopy(kaddr, devaddr, len);
2171 	return (0);
2172 }
2173 
2174 
2175 static int
2176 poke_mem(peekpoke_ctlops_t *in_args)
2177 {
2178 	int err = DDI_SUCCESS;
2179 	on_trap_data_t otd;
2180 
2181 	/* Set up protected environment. */
2182 	if (!on_trap(&otd, OT_DATA_ACCESS)) {
2183 		switch (in_args->size) {
2184 		case sizeof (uint8_t):
2185 			*(uint8_t *)(in_args->dev_addr) =
2186 			    *(uint8_t *)in_args->host_addr;
2187 			break;
2188 
2189 		case sizeof (uint16_t):
2190 			*(uint16_t *)(in_args->dev_addr) =
2191 			    *(uint16_t *)in_args->host_addr;
2192 			break;
2193 
2194 		case sizeof (uint32_t):
2195 			*(uint32_t *)(in_args->dev_addr) =
2196 			    *(uint32_t *)in_args->host_addr;
2197 			break;
2198 
2199 		case sizeof (uint64_t):
2200 			*(uint64_t *)(in_args->dev_addr) =
2201 			    *(uint64_t *)in_args->host_addr;
2202 			break;
2203 
2204 		default:
2205 			err = DDI_FAILURE;
2206 			break;
2207 		}
2208 	} else
2209 		err = DDI_FAILURE;
2210 
2211 	/* Take down protected environment. */
2212 	no_trap();
2213 
2214 	return (err);
2215 }
2216 
2217 
2218 static int
2219 peek_mem(peekpoke_ctlops_t *in_args)
2220 {
2221 	int err = DDI_SUCCESS;
2222 	on_trap_data_t otd;
2223 
2224 	if (!on_trap(&otd, OT_DATA_ACCESS)) {
2225 		switch (in_args->size) {
2226 		case sizeof (uint8_t):
2227 			*(uint8_t *)in_args->host_addr =
2228 			    *(uint8_t *)in_args->dev_addr;
2229 			break;
2230 
2231 		case sizeof (uint16_t):
2232 			*(uint16_t *)in_args->host_addr =
2233 			    *(uint16_t *)in_args->dev_addr;
2234 			break;
2235 
2236 		case sizeof (uint32_t):
2237 			*(uint32_t *)in_args->host_addr =
2238 			    *(uint32_t *)in_args->dev_addr;
2239 			break;
2240 
2241 		case sizeof (uint64_t):
2242 			*(uint64_t *)in_args->host_addr =
2243 			    *(uint64_t *)in_args->dev_addr;
2244 			break;
2245 
2246 		default:
2247 			err = DDI_FAILURE;
2248 			break;
2249 		}
2250 	} else
2251 		err = DDI_FAILURE;
2252 
2253 	no_trap();
2254 	return (err);
2255 }
2256 
2257 
2258 /*
2259  * This is called only to process peek/poke when the DIP is NULL.
2260  * Assume that this is for memory, as nexi take care of device safe accesses.
2261  */
2262 int
2263 peekpoke_mem(ddi_ctl_enum_t cmd, peekpoke_ctlops_t *in_args)
2264 {
2265 	return (cmd == DDI_CTLOPS_PEEK ? peek_mem(in_args) : poke_mem(in_args));
2266 }
2267 
2268 /*
2269  * we've just done a cautious put/get. Check if it was successful by
2270  * calling pci_ereport_post() on all puts and for any gets that return -1
2271  */
2272 static int
2273 pci_peekpoke_check_fma(dev_info_t *dip, void *arg, ddi_ctl_enum_t ctlop)
2274 {
2275 	int	rval = DDI_SUCCESS;
2276 	peekpoke_ctlops_t *in_args = (peekpoke_ctlops_t *)arg;
2277 	ddi_fm_error_t de;
2278 	ddi_acc_impl_t *hp = (ddi_acc_impl_t *)in_args->handle;
2279 	ddi_acc_hdl_t *hdlp = (ddi_acc_hdl_t *)in_args->handle;
2280 	int check_err = 0;
2281 	int repcount = in_args->repcount;
2282 
2283 	if (ctlop == DDI_CTLOPS_POKE &&
2284 	    hdlp->ah_acc.devacc_attr_access != DDI_CAUTIOUS_ACC)
2285 		return (DDI_SUCCESS);
2286 
2287 	if (ctlop == DDI_CTLOPS_PEEK &&
2288 	    hdlp->ah_acc.devacc_attr_access != DDI_CAUTIOUS_ACC) {
2289 		for (; repcount; repcount--) {
2290 			switch (in_args->size) {
2291 			case sizeof (uint8_t):
2292 				if (*(uint8_t *)in_args->host_addr == 0xff)
2293 					check_err = 1;
2294 				break;
2295 			case sizeof (uint16_t):
2296 				if (*(uint16_t *)in_args->host_addr == 0xffff)
2297 					check_err = 1;
2298 				break;
2299 			case sizeof (uint32_t):
2300 				if (*(uint32_t *)in_args->host_addr ==
2301 				    0xffffffff)
2302 					check_err = 1;
2303 				break;
2304 			case sizeof (uint64_t):
2305 				if (*(uint64_t *)in_args->host_addr ==
2306 				    0xffffffffffffffff)
2307 					check_err = 1;
2308 				break;
2309 			}
2310 		}
2311 		if (check_err == 0)
2312 			return (DDI_SUCCESS);
2313 	}
2314 	/*
2315 	 * for a cautious put or get or a non-cautious get that returned -1 call
2316 	 * io framework to see if there really was an error
2317 	 */
2318 	bzero(&de, sizeof (ddi_fm_error_t));
2319 	de.fme_version = DDI_FME_VERSION;
2320 	de.fme_ena = fm_ena_generate(0, FM_ENA_FMT1);
2321 	if (hdlp->ah_acc.devacc_attr_access == DDI_CAUTIOUS_ACC) {
2322 		de.fme_flag = DDI_FM_ERR_EXPECTED;
2323 		de.fme_acc_handle = in_args->handle;
2324 	} else if (hdlp->ah_acc.devacc_attr_access == DDI_DEFAULT_ACC) {
2325 		/*
2326 		 * We only get here with DDI_DEFAULT_ACC for config space gets.
2327 		 * Non-hardened drivers may be probing the hardware and
2328 		 * expecting -1 returned. So need to treat errors on
2329 		 * DDI_DEFAULT_ACC as DDI_FM_ERR_EXPECTED.
2330 		 */
2331 		de.fme_flag = DDI_FM_ERR_EXPECTED;
2332 		de.fme_acc_handle = in_args->handle;
2333 	} else {
2334 		/*
2335 		 * Hardened driver doing protected accesses shouldn't
2336 		 * get errors unless there's a hardware problem. Treat
2337 		 * as nonfatal if there's an error, but set UNEXPECTED
2338 		 * so we raise ereports on any errors and potentially
2339 		 * fault the device
2340 		 */
2341 		de.fme_flag = DDI_FM_ERR_UNEXPECTED;
2342 	}
2343 	pci_ereport_post(dip, &de, NULL);
2344 	if (hdlp->ah_acc.devacc_attr_access != DDI_DEFAULT_ACC &&
2345 	    de.fme_status != DDI_FM_OK) {
2346 		ndi_err_t *errp = (ndi_err_t *)hp->ahi_err;
2347 		rval = DDI_FAILURE;
2348 		errp->err_ena = de.fme_ena;
2349 		errp->err_expected = de.fme_flag;
2350 		errp->err_status = DDI_FM_NONFATAL;
2351 	}
2352 	return (rval);
2353 }
2354 
2355 /*
2356  * pci_peekpoke_check_nofma() is for when an error occurs on a register access
2357  * during pci_ereport_post(). We can't call pci_ereport_post() again or we'd
2358  * recurse, so assume all puts are OK and gets have failed if they return -1
2359  */
2360 static int
2361 pci_peekpoke_check_nofma(void *arg, ddi_ctl_enum_t ctlop)
2362 {
2363 	int rval = DDI_SUCCESS;
2364 	peekpoke_ctlops_t *in_args = (peekpoke_ctlops_t *)arg;
2365 	ddi_acc_impl_t *hp = (ddi_acc_impl_t *)in_args->handle;
2366 	ddi_acc_hdl_t *hdlp = (ddi_acc_hdl_t *)in_args->handle;
2367 	int repcount = in_args->repcount;
2368 
2369 	if (ctlop == DDI_CTLOPS_POKE)
2370 		return (rval);
2371 
2372 	for (; repcount; repcount--) {
2373 		switch (in_args->size) {
2374 		case sizeof (uint8_t):
2375 			if (*(uint8_t *)in_args->host_addr == 0xff)
2376 				rval = DDI_FAILURE;
2377 			break;
2378 		case sizeof (uint16_t):
2379 			if (*(uint16_t *)in_args->host_addr == 0xffff)
2380 				rval = DDI_FAILURE;
2381 			break;
2382 		case sizeof (uint32_t):
2383 			if (*(uint32_t *)in_args->host_addr == 0xffffffff)
2384 				rval = DDI_FAILURE;
2385 			break;
2386 		case sizeof (uint64_t):
2387 			if (*(uint64_t *)in_args->host_addr ==
2388 			    0xffffffffffffffff)
2389 				rval = DDI_FAILURE;
2390 			break;
2391 		}
2392 	}
2393 	if (hdlp->ah_acc.devacc_attr_access != DDI_DEFAULT_ACC &&
2394 	    rval == DDI_FAILURE) {
2395 		ndi_err_t *errp = (ndi_err_t *)hp->ahi_err;
2396 		errp->err_ena = fm_ena_generate(0, FM_ENA_FMT1);
2397 		errp->err_expected = DDI_FM_ERR_UNEXPECTED;
2398 		errp->err_status = DDI_FM_NONFATAL;
2399 	}
2400 	return (rval);
2401 }
2402 
2403 int
2404 pci_peekpoke_check(dev_info_t *dip, dev_info_t *rdip,
2405 	ddi_ctl_enum_t ctlop, void *arg, void *result,
2406 	int (*handler)(dev_info_t *, dev_info_t *, ddi_ctl_enum_t, void *,
2407 	void *), kmutex_t *err_mutexp, kmutex_t *peek_poke_mutexp)
2408 {
2409 	int rval;
2410 	peekpoke_ctlops_t *in_args = (peekpoke_ctlops_t *)arg;
2411 	ddi_acc_impl_t *hp = (ddi_acc_impl_t *)in_args->handle;
2412 
2413 	/*
2414 	 * this function only supports cautious accesses, not peeks/pokes
2415 	 * which don't have a handle
2416 	 */
2417 	if (hp == NULL)
2418 		return (DDI_FAILURE);
2419 
2420 	if (hp->ahi_acc_attr & DDI_ACCATTR_CONFIG_SPACE) {
2421 		if (!mutex_tryenter(err_mutexp)) {
2422 			/*
2423 			 * As this may be a recursive call from within
2424 			 * pci_ereport_post() we can't wait for the mutexes.
2425 			 * Fortunately we know someone is already calling
2426 			 * pci_ereport_post() which will handle the error bits
2427 			 * for us, and as this is a config space access we can
2428 			 * just do the access and check return value for -1
2429 			 * using pci_peekpoke_check_nofma().
2430 			 */
2431 			rval = handler(dip, rdip, ctlop, arg, result);
2432 			if (rval == DDI_SUCCESS)
2433 				rval = pci_peekpoke_check_nofma(arg, ctlop);
2434 			return (rval);
2435 		}
2436 		/*
2437 		 * This can't be a recursive call. Drop the err_mutex and get
2438 		 * both mutexes in the right order. If an error hasn't already
2439 		 * been detected by the ontrap code, use pci_peekpoke_check_fma
2440 		 * which will call pci_ereport_post() to check error status.
2441 		 */
2442 		mutex_exit(err_mutexp);
2443 	}
2444 	mutex_enter(peek_poke_mutexp);
2445 	rval = handler(dip, rdip, ctlop, arg, result);
2446 	if (rval == DDI_SUCCESS) {
2447 		mutex_enter(err_mutexp);
2448 		rval = pci_peekpoke_check_fma(dip, arg, ctlop);
2449 		mutex_exit(err_mutexp);
2450 	}
2451 	mutex_exit(peek_poke_mutexp);
2452 	return (rval);
2453 }
2454 
2455 void
2456 impl_setup_ddi(void)
2457 {
2458 	dev_info_t *xdip, *isa_dip;
2459 	rd_existing_t rd_mem_prop;
2460 	int err;
2461 
2462 	ndi_devi_alloc_sleep(ddi_root_node(), "ramdisk",
2463 	    (pnode_t)DEVI_SID_NODEID, &xdip);
2464 
2465 	(void) BOP_GETPROP(bootops,
2466 	    "ramdisk_start", (void *)&ramdisk_start);
2467 	(void) BOP_GETPROP(bootops,
2468 	    "ramdisk_end", (void *)&ramdisk_end);
2469 
2470 	rd_mem_prop.phys = ramdisk_start;
2471 	rd_mem_prop.size = ramdisk_end - ramdisk_start + 1;
2472 
2473 	(void) ndi_prop_update_byte_array(DDI_DEV_T_NONE, xdip,
2474 	    RD_EXISTING_PROP_NAME, (uchar_t *)&rd_mem_prop,
2475 	    sizeof (rd_mem_prop));
2476 	err = ndi_devi_bind_driver(xdip, 0);
2477 	ASSERT(err == 0);
2478 
2479 	/* isa node */
2480 	ndi_devi_alloc_sleep(ddi_root_node(), "isa",
2481 	    (pnode_t)DEVI_SID_NODEID, &isa_dip);
2482 	(void) ndi_prop_update_string(DDI_DEV_T_NONE, isa_dip,
2483 	    "device_type", "isa");
2484 	(void) ndi_prop_update_string(DDI_DEV_T_NONE, isa_dip,
2485 	    "bus-type", "isa");
2486 	(void) ndi_devi_bind_driver(isa_dip, 0);
2487 
2488 	/*
2489 	 * Read in the properties from the boot.
2490 	 */
2491 	get_boot_properties();
2492 
2493 	/* do bus dependent probes. */
2494 	impl_bus_initialprobe();
2495 
2496 	/* not framebuffer should be enumerated, if present */
2497 	get_vga_properties();
2498 }
2499 
2500 dev_t
2501 getrootdev(void)
2502 {
2503 	/*
2504 	 * Precedence given to rootdev if set in /etc/system
2505 	 */
2506 	if (root_is_svm) {
2507 		return (ddi_pathname_to_dev_t(svm_bootpath));
2508 	}
2509 
2510 	/*
2511 	 * Usually rootfs.bo_name is initialized by the
2512 	 * the bootpath property from bootenv.rc, but
2513 	 * defaults to "/ramdisk:a" otherwise.
2514 	 */
2515 	return (ddi_pathname_to_dev_t(rootfs.bo_name));
2516 }
2517 
2518 static struct bus_probe {
2519 	struct bus_probe *next;
2520 	void (*probe)(int);
2521 } *bus_probes;
2522 
2523 void
2524 impl_bus_add_probe(void (*func)(int))
2525 {
2526 	struct bus_probe *probe;
2527 
2528 	probe = kmem_alloc(sizeof (*probe), KM_SLEEP);
2529 	probe->next = bus_probes;
2530 	probe->probe = func;
2531 	bus_probes = probe;
2532 }
2533 
2534 /*ARGSUSED*/
2535 void
2536 impl_bus_delete_probe(void (*func)(int))
2537 {
2538 	struct bus_probe *prev = NULL;
2539 	struct bus_probe *probe = bus_probes;
2540 
2541 	while (probe) {
2542 		if (probe->probe == func)
2543 			break;
2544 		prev = probe;
2545 		probe = probe->next;
2546 	}
2547 
2548 	if (probe == NULL)
2549 		return;
2550 
2551 	if (prev)
2552 		prev->next = probe->next;
2553 	else
2554 		bus_probes = probe->next;
2555 
2556 	kmem_free(probe, sizeof (struct bus_probe));
2557 }
2558 
2559 /*
2560  * impl_bus_initialprobe
2561  *	Modload the prom simulator, then let it probe to verify existence
2562  *	and type of PCI support.
2563  */
2564 static void
2565 impl_bus_initialprobe(void)
2566 {
2567 	struct bus_probe *probe;
2568 
2569 	/* load modules to install bus probes */
2570 	if (modload("misc", "pci_autoconfig") < 0) {
2571 		cmn_err(CE_PANIC, "failed to load misc/pci_autoconfig");
2572 	}
2573 
2574 	probe = bus_probes;
2575 	while (probe) {
2576 		/* run the probe function */
2577 		(*probe->probe)(0);
2578 		probe = probe->next;
2579 	}
2580 }
2581 
2582 /*
2583  * impl_bus_reprobe
2584  *	Reprogram devices not set up by firmware.
2585  */
2586 static void
2587 impl_bus_reprobe(void)
2588 {
2589 	struct bus_probe *probe;
2590 
2591 	probe = bus_probes;
2592 	while (probe) {
2593 		/* run the probe function */
2594 		(*probe->probe)(1);
2595 		probe = probe->next;
2596 	}
2597 }
2598 
2599 
2600 /*
2601  * The following functions ready a cautious request to go up to the nexus
2602  * driver.  It is up to the nexus driver to decide how to process the request.
2603  * It may choose to call i_ddi_do_caut_get/put in this file, or do it
2604  * differently.
2605  */
2606 
2607 static void
2608 i_ddi_caut_getput_ctlops(ddi_acc_impl_t *hp, uint64_t host_addr,
2609     uint64_t dev_addr, size_t size, size_t repcount, uint_t flags,
2610     ddi_ctl_enum_t cmd)
2611 {
2612 	peekpoke_ctlops_t	cautacc_ctlops_arg;
2613 
2614 	cautacc_ctlops_arg.size = size;
2615 	cautacc_ctlops_arg.dev_addr = dev_addr;
2616 	cautacc_ctlops_arg.host_addr = host_addr;
2617 	cautacc_ctlops_arg.handle = (ddi_acc_handle_t)hp;
2618 	cautacc_ctlops_arg.repcount = repcount;
2619 	cautacc_ctlops_arg.flags = flags;
2620 
2621 	(void) ddi_ctlops(hp->ahi_common.ah_dip, hp->ahi_common.ah_dip, cmd,
2622 	    &cautacc_ctlops_arg, NULL);
2623 }
2624 
2625 uint8_t
2626 i_ddi_caut_get8(ddi_acc_impl_t *hp, uint8_t *addr)
2627 {
2628 	uint8_t value;
2629 	i_ddi_caut_getput_ctlops(hp, (uintptr_t)&value, (uintptr_t)addr,
2630 	    sizeof (uint8_t), 1, 0, DDI_CTLOPS_PEEK);
2631 
2632 	return (value);
2633 }
2634 
2635 uint16_t
2636 i_ddi_caut_get16(ddi_acc_impl_t *hp, uint16_t *addr)
2637 {
2638 	uint16_t value;
2639 	i_ddi_caut_getput_ctlops(hp, (uintptr_t)&value, (uintptr_t)addr,
2640 	    sizeof (uint16_t), 1, 0, DDI_CTLOPS_PEEK);
2641 
2642 	return (value);
2643 }
2644 
2645 uint32_t
2646 i_ddi_caut_get32(ddi_acc_impl_t *hp, uint32_t *addr)
2647 {
2648 	uint32_t value;
2649 	i_ddi_caut_getput_ctlops(hp, (uintptr_t)&value, (uintptr_t)addr,
2650 	    sizeof (uint32_t), 1, 0, DDI_CTLOPS_PEEK);
2651 
2652 	return (value);
2653 }
2654 
2655 uint64_t
2656 i_ddi_caut_get64(ddi_acc_impl_t *hp, uint64_t *addr)
2657 {
2658 	uint64_t value;
2659 	i_ddi_caut_getput_ctlops(hp, (uintptr_t)&value, (uintptr_t)addr,
2660 	    sizeof (uint64_t), 1, 0, DDI_CTLOPS_PEEK);
2661 
2662 	return (value);
2663 }
2664 
2665 void
2666 i_ddi_caut_put8(ddi_acc_impl_t *hp, uint8_t *addr, uint8_t value)
2667 {
2668 	i_ddi_caut_getput_ctlops(hp, (uintptr_t)&value, (uintptr_t)addr,
2669 	    sizeof (uint8_t), 1, 0, DDI_CTLOPS_POKE);
2670 }
2671 
2672 void
2673 i_ddi_caut_put16(ddi_acc_impl_t *hp, uint16_t *addr, uint16_t value)
2674 {
2675 	i_ddi_caut_getput_ctlops(hp, (uintptr_t)&value, (uintptr_t)addr,
2676 	    sizeof (uint16_t), 1, 0, DDI_CTLOPS_POKE);
2677 }
2678 
2679 void
2680 i_ddi_caut_put32(ddi_acc_impl_t *hp, uint32_t *addr, uint32_t value)
2681 {
2682 	i_ddi_caut_getput_ctlops(hp, (uintptr_t)&value, (uintptr_t)addr,
2683 	    sizeof (uint32_t), 1, 0, DDI_CTLOPS_POKE);
2684 }
2685 
2686 void
2687 i_ddi_caut_put64(ddi_acc_impl_t *hp, uint64_t *addr, uint64_t value)
2688 {
2689 	i_ddi_caut_getput_ctlops(hp, (uintptr_t)&value, (uintptr_t)addr,
2690 	    sizeof (uint64_t), 1, 0, DDI_CTLOPS_POKE);
2691 }
2692 
2693 void
2694 i_ddi_caut_rep_get8(ddi_acc_impl_t *hp, uint8_t *host_addr, uint8_t *dev_addr,
2695 	size_t repcount, uint_t flags)
2696 {
2697 	i_ddi_caut_getput_ctlops(hp, (uintptr_t)host_addr, (uintptr_t)dev_addr,
2698 	    sizeof (uint8_t), repcount, flags, DDI_CTLOPS_PEEK);
2699 }
2700 
2701 void
2702 i_ddi_caut_rep_get16(ddi_acc_impl_t *hp, uint16_t *host_addr,
2703     uint16_t *dev_addr, size_t repcount, uint_t flags)
2704 {
2705 	i_ddi_caut_getput_ctlops(hp, (uintptr_t)host_addr, (uintptr_t)dev_addr,
2706 	    sizeof (uint16_t), repcount, flags, DDI_CTLOPS_PEEK);
2707 }
2708 
2709 void
2710 i_ddi_caut_rep_get32(ddi_acc_impl_t *hp, uint32_t *host_addr,
2711     uint32_t *dev_addr, size_t repcount, uint_t flags)
2712 {
2713 	i_ddi_caut_getput_ctlops(hp, (uintptr_t)host_addr, (uintptr_t)dev_addr,
2714 	    sizeof (uint32_t), repcount, flags, DDI_CTLOPS_PEEK);
2715 }
2716 
2717 void
2718 i_ddi_caut_rep_get64(ddi_acc_impl_t *hp, uint64_t *host_addr,
2719     uint64_t *dev_addr, size_t repcount, uint_t flags)
2720 {
2721 	i_ddi_caut_getput_ctlops(hp, (uintptr_t)host_addr, (uintptr_t)dev_addr,
2722 	    sizeof (uint64_t), repcount, flags, DDI_CTLOPS_PEEK);
2723 }
2724 
2725 void
2726 i_ddi_caut_rep_put8(ddi_acc_impl_t *hp, uint8_t *host_addr, uint8_t *dev_addr,
2727 	size_t repcount, uint_t flags)
2728 {
2729 	i_ddi_caut_getput_ctlops(hp, (uintptr_t)host_addr, (uintptr_t)dev_addr,
2730 	    sizeof (uint8_t), repcount, flags, DDI_CTLOPS_POKE);
2731 }
2732 
2733 void
2734 i_ddi_caut_rep_put16(ddi_acc_impl_t *hp, uint16_t *host_addr,
2735     uint16_t *dev_addr, size_t repcount, uint_t flags)
2736 {
2737 	i_ddi_caut_getput_ctlops(hp, (uintptr_t)host_addr, (uintptr_t)dev_addr,
2738 	    sizeof (uint16_t), repcount, flags, DDI_CTLOPS_POKE);
2739 }
2740 
2741 void
2742 i_ddi_caut_rep_put32(ddi_acc_impl_t *hp, uint32_t *host_addr,
2743     uint32_t *dev_addr, size_t repcount, uint_t flags)
2744 {
2745 	i_ddi_caut_getput_ctlops(hp, (uintptr_t)host_addr, (uintptr_t)dev_addr,
2746 	    sizeof (uint32_t), repcount, flags, DDI_CTLOPS_POKE);
2747 }
2748 
2749 void
2750 i_ddi_caut_rep_put64(ddi_acc_impl_t *hp, uint64_t *host_addr,
2751     uint64_t *dev_addr, size_t repcount, uint_t flags)
2752 {
2753 	i_ddi_caut_getput_ctlops(hp, (uintptr_t)host_addr, (uintptr_t)dev_addr,
2754 	    sizeof (uint64_t), repcount, flags, DDI_CTLOPS_POKE);
2755 }
2756