xref: /titanic_51/usr/src/uts/sun4/io/px/px.c (revision b6c3f7863936abeae522e48a13887dddeb691a45)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 /*
22  * Copyright 2007 Sun Microsystems, Inc.  All rights reserved.
23  * Use is subject to license terms.
24  */
25 
26 #pragma ident	"%Z%%M%	%I%	%E% SMI"
27 
28 /*
29  * PCI Express nexus driver interface
30  */
31 
32 #include <sys/types.h>
33 #include <sys/conf.h>		/* nulldev */
34 #include <sys/stat.h>		/* devctl */
35 #include <sys/kmem.h>
36 #include <sys/sunddi.h>
37 #include <sys/sunndi.h>
38 #include <sys/ddi_impldefs.h>
39 #include <sys/ddi_subrdefs.h>
40 #include <sys/spl.h>
41 #include <sys/epm.h>
42 #include <sys/iommutsb.h>
43 #include <sys/hotplug/pci/pcihp.h>
44 #include <sys/hotplug/pci/pciehpc.h>
45 #include "px_obj.h"
46 #include <sys/pci_tools.h>
47 #include "px_tools_ext.h"
48 #include "pcie_pwr.h"
49 
50 /*LINTLIBRARY*/
51 
52 /*
53  * function prototypes for dev ops routines:
54  */
55 static int px_attach(dev_info_t *dip, ddi_attach_cmd_t cmd);
56 static int px_detach(dev_info_t *dip, ddi_detach_cmd_t cmd);
57 static int px_info(dev_info_t *dip, ddi_info_cmd_t infocmd,
58 	void *arg, void **result);
59 static int px_cb_attach(px_t *);
60 static void px_cb_detach(px_t *);
61 static int px_pwr_setup(dev_info_t *dip);
62 static void px_pwr_teardown(dev_info_t *dip);
63 
64 extern errorq_t *pci_target_queue;
65 
66 /*
67  * function prototypes for hotplug routines:
68  */
69 static int px_init_hotplug(px_t *px_p);
70 static int px_uninit_hotplug(dev_info_t *dip);
71 
72 /*
73  * bus ops and dev ops structures:
74  */
75 static struct bus_ops px_bus_ops = {
76 	BUSO_REV,
77 	px_map,
78 	0,
79 	0,
80 	0,
81 	i_ddi_map_fault,
82 	px_dma_setup,
83 	px_dma_allochdl,
84 	px_dma_freehdl,
85 	px_dma_bindhdl,
86 	px_dma_unbindhdl,
87 	px_lib_dma_sync,
88 	px_dma_win,
89 	px_dma_ctlops,
90 	px_ctlops,
91 	ddi_bus_prop_op,
92 	ndi_busop_get_eventcookie,
93 	ndi_busop_add_eventcall,
94 	ndi_busop_remove_eventcall,
95 	ndi_post_event,
96 	NULL,
97 	NULL,			/* (*bus_config)(); */
98 	NULL,			/* (*bus_unconfig)(); */
99 	px_fm_init_child,	/* (*bus_fm_init)(); */
100 	NULL,			/* (*bus_fm_fini)(); */
101 	px_bus_enter,		/* (*bus_fm_access_enter)(); */
102 	px_bus_exit,		/* (*bus_fm_access_fini)(); */
103 	pcie_bus_power,		/* (*bus_power)(); */
104 	px_intr_ops		/* (*bus_intr_op)(); */
105 };
106 
107 extern struct cb_ops px_cb_ops;
108 
109 static struct dev_ops px_ops = {
110 	DEVO_REV,
111 	0,
112 	px_info,
113 	nulldev,
114 	0,
115 	px_attach,
116 	px_detach,
117 	nodev,
118 	&px_cb_ops,
119 	&px_bus_ops,
120 	nulldev
121 };
122 
123 /*
124  * module definitions:
125  */
126 #include <sys/modctl.h>
127 extern struct mod_ops mod_driverops;
128 
129 static struct modldrv modldrv = {
130 	&mod_driverops, 		/* Type of module - driver */
131 	"PCI Express nexus driver %I%",	/* Name of module. */
132 	&px_ops,			/* driver ops */
133 };
134 
135 static struct modlinkage modlinkage = {
136 	MODREV_1, (void *)&modldrv, NULL
137 };
138 
139 /* driver soft state */
140 void *px_state_p;
141 
142 int
143 _init(void)
144 {
145 	int e;
146 
147 	/*
148 	 * Initialize per-px bus soft state pointer.
149 	 */
150 	e = ddi_soft_state_init(&px_state_p, sizeof (px_t), 1);
151 	if (e != DDI_SUCCESS)
152 		return (e);
153 
154 	/*
155 	 * Install the module.
156 	 */
157 	e = mod_install(&modlinkage);
158 	if (e != DDI_SUCCESS)
159 		ddi_soft_state_fini(&px_state_p);
160 	return (e);
161 }
162 
163 int
164 _fini(void)
165 {
166 	int e;
167 
168 	/*
169 	 * Remove the module.
170 	 */
171 	e = mod_remove(&modlinkage);
172 	if (e != DDI_SUCCESS)
173 		return (e);
174 	/*
175 	 * Destroy pci_target_queue, and set it to NULL.
176 	 */
177 	if (pci_target_queue)
178 		errorq_destroy(pci_target_queue);
179 
180 	pci_target_queue = NULL;
181 
182 	/* Free px soft state */
183 	ddi_soft_state_fini(&px_state_p);
184 
185 	return (e);
186 }
187 
188 int
189 _info(struct modinfo *modinfop)
190 {
191 	return (mod_info(&modlinkage, modinfop));
192 }
193 
194 /* ARGSUSED */
195 static int
196 px_info(dev_info_t *dip, ddi_info_cmd_t infocmd, void *arg, void **result)
197 {
198 	int	instance = getminor((dev_t)arg);
199 	px_t	*px_p = INST_TO_STATE(instance);
200 
201 	/*
202 	 * Allow hotplug to deal with ones it manages
203 	 * Hot Plug will be done later.
204 	 */
205 	if (px_p && (px_p->px_dev_caps & PX_HOTPLUG_CAPABLE))
206 		return (pcihp_info(dip, infocmd, arg, result));
207 
208 	/* non-hotplug or not attached */
209 	switch (infocmd) {
210 	case DDI_INFO_DEVT2INSTANCE:
211 		*result = (void *)(intptr_t)instance;
212 		return (DDI_SUCCESS);
213 
214 	case DDI_INFO_DEVT2DEVINFO:
215 		if (px_p == NULL)
216 			return (DDI_FAILURE);
217 		*result = (void *)px_p->px_dip;
218 		return (DDI_SUCCESS);
219 
220 	default:
221 		return (DDI_FAILURE);
222 	}
223 }
224 
225 /* device driver entry points */
226 /*
227  * attach entry point:
228  */
229 /*ARGSUSED*/
230 static int
231 px_attach(dev_info_t *dip, ddi_attach_cmd_t cmd)
232 {
233 	px_t		*px_p;	/* per bus state pointer */
234 	int		instance = DIP_TO_INST(dip);
235 	int		ret = DDI_SUCCESS;
236 	devhandle_t	dev_hdl = NULL;
237 
238 	switch (cmd) {
239 	case DDI_ATTACH:
240 		DBG(DBG_ATTACH, dip, "DDI_ATTACH\n");
241 
242 		/*
243 		 * Allocate and get the per-px soft state structure.
244 		 */
245 		if (ddi_soft_state_zalloc(px_state_p, instance)
246 		    != DDI_SUCCESS) {
247 			cmn_err(CE_WARN, "%s%d: can't allocate px state",
248 				ddi_driver_name(dip), instance);
249 			goto err_bad_px_softstate;
250 		}
251 		px_p = INST_TO_STATE(instance);
252 		px_p->px_dip = dip;
253 		mutex_init(&px_p->px_mutex, NULL, MUTEX_DRIVER, NULL);
254 		px_p->px_soft_state = PX_SOFT_STATE_CLOSED;
255 		px_p->px_open_count = 0;
256 
257 		(void) ddi_prop_update_string(DDI_DEV_T_NONE, dip,
258 				"device_type", "pciex");
259 
260 		/* Initialize px_dbg for high pil printing */
261 		px_dbg_attach(dip, &px_p->px_dbg_hdl);
262 
263 		/*
264 		 * Get key properties of the pci bridge node and
265 		 * determine it's type (psycho, schizo, etc ...).
266 		 */
267 		if (px_get_props(px_p, dip) == DDI_FAILURE)
268 			goto err_bad_px_prop;
269 
270 		if (px_lib_dev_init(dip, &dev_hdl) != DDI_SUCCESS)
271 			goto err_bad_dev_init;
272 
273 		/* Initialize device handle */
274 		px_p->px_dev_hdl = dev_hdl;
275 
276 		/* Cache the BDF of the root port nexus */
277 		px_p->px_bdf = px_lib_get_bdf(px_p);
278 
279 		px_p->px_dq_p = (pf_data_t *)
280 		    kmem_zalloc(sizeof (pf_data_t) * pf_get_dq_size(),
281 		    KM_SLEEP);
282 
283 		px_p->px_dq_tail = -1;
284 
285 		/*
286 		 * Initialize interrupt block.  Note that this
287 		 * initialize error handling for the PEC as well.
288 		 */
289 		if ((ret = px_ib_attach(px_p)) != DDI_SUCCESS)
290 			goto err_bad_ib;
291 
292 		if (px_cb_attach(px_p) != DDI_SUCCESS)
293 			goto err_bad_cb;
294 
295 		/*
296 		 * Start creating the modules.
297 		 * Note that attach() routines should
298 		 * register and enable their own interrupts.
299 		 */
300 
301 		if ((px_mmu_attach(px_p)) != DDI_SUCCESS)
302 			goto err_bad_mmu;
303 
304 		if ((px_msiq_attach(px_p)) != DDI_SUCCESS)
305 			goto err_bad_msiq;
306 
307 		if ((px_msi_attach(px_p)) != DDI_SUCCESS)
308 			goto err_bad_msi;
309 
310 		if ((px_pec_attach(px_p)) != DDI_SUCCESS)
311 			goto err_bad_pec;
312 
313 		if ((px_dma_attach(px_p)) != DDI_SUCCESS)
314 			goto err_bad_dma; /* nothing to uninitialize on DMA */
315 
316 		if ((px_fm_attach(px_p)) != DDI_SUCCESS)
317 			goto err_bad_dma;
318 
319 		/*
320 		 * All of the error handlers have been registered
321 		 * by now so it's time to activate the interrupt.
322 		 */
323 		if ((ret = px_err_add_intr(&px_p->px_fault)) != DDI_SUCCESS)
324 			goto err_bad_intr;
325 
326 		(void) px_init_hotplug(px_p);
327 
328 		/*
329 		 * Create the "devctl" node for hotplug and pcitool support.
330 		 * For non-hotplug bus, we still need ":devctl" to
331 		 * support DEVCTL_DEVICE_* and DEVCTL_BUS_* ioctls.
332 		 */
333 		if (ddi_create_minor_node(dip, "devctl", S_IFCHR,
334 		    PCIHP_AP_MINOR_NUM(instance, PCIHP_DEVCTL_MINOR),
335 		    DDI_NT_NEXUS, 0) != DDI_SUCCESS) {
336 			goto err_bad_devctl_node;
337 		}
338 
339 		if (pxtool_init(dip) != DDI_SUCCESS)
340 			goto err_bad_pcitool_node;
341 
342 		/*
343 		 * power management setup. Even if it fails, attach will
344 		 * succeed as this is a optional feature. Since we are
345 		 * always at full power, this is not critical.
346 		 */
347 		if (pwr_common_setup(dip) != DDI_SUCCESS) {
348 			DBG(DBG_PWR, dip, "pwr_common_setup failed\n");
349 		} else if (px_pwr_setup(dip) != DDI_SUCCESS) {
350 			DBG(DBG_PWR, dip, "px_pwr_setup failed \n");
351 			pwr_common_teardown(dip);
352 		}
353 
354 		/*
355 		 * add cpr callback
356 		 */
357 		px_cpr_add_callb(px_p);
358 
359 		ddi_report_dev(dip);
360 
361 		px_p->px_state = PX_ATTACHED;
362 		DBG(DBG_ATTACH, dip, "attach success\n");
363 		break;
364 
365 err_bad_pcitool_node:
366 		ddi_remove_minor_node(dip, "devctl");
367 err_bad_devctl_node:
368 		px_err_rem_intr(&px_p->px_fault);
369 err_bad_intr:
370 		px_fm_detach(px_p);
371 err_bad_dma:
372 		px_pec_detach(px_p);
373 err_bad_pec:
374 		px_msi_detach(px_p);
375 err_bad_msi:
376 		px_msiq_detach(px_p);
377 err_bad_msiq:
378 		px_mmu_detach(px_p);
379 err_bad_mmu:
380 		px_cb_detach(px_p);
381 err_bad_cb:
382 		px_ib_detach(px_p);
383 err_bad_ib:
384 		(void) px_lib_dev_fini(dip);
385 err_bad_dev_init:
386 		px_free_props(px_p);
387 err_bad_px_prop:
388 		px_dbg_detach(dip, &px_p->px_dbg_hdl);
389 		mutex_destroy(&px_p->px_mutex);
390 		ddi_soft_state_free(px_state_p, instance);
391 err_bad_px_softstate:
392 		ret = DDI_FAILURE;
393 		break;
394 
395 	case DDI_RESUME:
396 		DBG(DBG_ATTACH, dip, "DDI_RESUME\n");
397 
398 		px_p = INST_TO_STATE(instance);
399 
400 		mutex_enter(&px_p->px_mutex);
401 
402 		/* suspend might have not succeeded */
403 		if (px_p->px_state != PX_SUSPENDED) {
404 			DBG(DBG_ATTACH, px_p->px_dip,
405 			    "instance NOT suspended\n");
406 			ret = DDI_FAILURE;
407 			break;
408 		}
409 
410 		px_msiq_resume(px_p);
411 		px_lib_resume(dip);
412 		(void) pcie_pwr_resume(dip);
413 		px_p->px_state = PX_ATTACHED;
414 
415 		mutex_exit(&px_p->px_mutex);
416 
417 		break;
418 	default:
419 		DBG(DBG_ATTACH, dip, "unsupported attach op\n");
420 		ret = DDI_FAILURE;
421 		break;
422 	}
423 
424 	return (ret);
425 }
426 
427 /*
428  * detach entry point:
429  */
430 /*ARGSUSED*/
431 static int
432 px_detach(dev_info_t *dip, ddi_detach_cmd_t cmd)
433 {
434 	int instance = ddi_get_instance(dip);
435 	px_t *px_p = INST_TO_STATE(instance);
436 	int ret;
437 
438 	/*
439 	 * Make sure we are currently attached
440 	 */
441 	if (px_p->px_state != PX_ATTACHED) {
442 		DBG(DBG_DETACH, dip, "Instance not attached\n");
443 		return (DDI_FAILURE);
444 	}
445 
446 	mutex_enter(&px_p->px_mutex);
447 
448 	switch (cmd) {
449 	case DDI_DETACH:
450 		DBG(DBG_DETACH, dip, "DDI_DETACH\n");
451 
452 		/*
453 		 * remove cpr callback
454 		 */
455 		px_cpr_rem_callb(px_p);
456 
457 		if (px_p->px_dev_caps & PX_HOTPLUG_CAPABLE)
458 			if (px_uninit_hotplug(dip) != DDI_SUCCESS) {
459 				mutex_exit(&px_p->px_mutex);
460 				return (DDI_FAILURE);
461 			}
462 
463 		/*
464 		 * things which used to be done in obj_destroy
465 		 * are now in-lined here.
466 		 */
467 
468 		px_p->px_state = PX_DETACHED;
469 
470 		pxtool_uninit(dip);
471 
472 		ddi_remove_minor_node(dip, "devctl");
473 		px_err_rem_intr(&px_p->px_fault);
474 		px_fm_detach(px_p);
475 		px_pec_detach(px_p);
476 		px_pwr_teardown(dip);
477 		pwr_common_teardown(dip);
478 		px_msi_detach(px_p);
479 		px_msiq_detach(px_p);
480 		px_mmu_detach(px_p);
481 		px_cb_detach(px_p);
482 		px_ib_detach(px_p);
483 		(void) px_lib_dev_fini(dip);
484 
485 		kmem_free(px_p->px_dq_p, sizeof (pf_data_t) *
486 		    pf_get_dq_size());
487 
488 		/*
489 		 * Free the px soft state structure and the rest of the
490 		 * resources it's using.
491 		 */
492 		px_free_props(px_p);
493 		px_dbg_detach(dip, &px_p->px_dbg_hdl);
494 		mutex_exit(&px_p->px_mutex);
495 		mutex_destroy(&px_p->px_mutex);
496 
497 		/* Free the interrupt-priorities prop if we created it. */ {
498 			int len;
499 
500 			if (ddi_getproplen(DDI_DEV_T_ANY, dip,
501 			    DDI_PROP_NOTPROM | DDI_PROP_DONTPASS,
502 			    "interrupt-priorities", &len) == DDI_PROP_SUCCESS)
503 				(void) ddi_prop_remove(DDI_DEV_T_NONE, dip,
504 				    "interrupt-priorities");
505 		}
506 
507 		px_p->px_dev_hdl = NULL;
508 		ddi_soft_state_free(px_state_p, instance);
509 
510 		return (DDI_SUCCESS);
511 
512 	case DDI_SUSPEND:
513 		if (pcie_pwr_suspend(dip) != DDI_SUCCESS) {
514 			mutex_exit(&px_p->px_mutex);
515 			return (DDI_FAILURE);
516 		}
517 		if ((ret = px_lib_suspend(dip)) == DDI_SUCCESS)
518 			px_p->px_state = PX_SUSPENDED;
519 		mutex_exit(&px_p->px_mutex);
520 
521 		return (ret);
522 
523 	default:
524 		DBG(DBG_DETACH, dip, "unsupported detach op\n");
525 		mutex_exit(&px_p->px_mutex);
526 		return (DDI_FAILURE);
527 	}
528 }
529 
530 int
531 px_cb_attach(px_t *px_p)
532 {
533 	px_fault_t	*fault_p = &px_p->px_cb_fault;
534 	dev_info_t	*dip = px_p->px_dip;
535 	sysino_t	sysino;
536 
537 	if (px_lib_intr_devino_to_sysino(dip,
538 	    px_p->px_inos[PX_INTR_XBC], &sysino) != DDI_SUCCESS)
539 		return (DDI_FAILURE);
540 
541 	fault_p->px_fh_dip = dip;
542 	fault_p->px_fh_sysino = sysino;
543 	fault_p->px_err_func = px_err_cb_intr;
544 	fault_p->px_intr_ino = px_p->px_inos[PX_INTR_XBC];
545 
546 	return (px_cb_add_intr(fault_p));
547 }
548 
549 void
550 px_cb_detach(px_t *px_p)
551 {
552 	px_cb_rem_intr(&px_p->px_cb_fault);
553 }
554 
555 /*
556  * power management related initialization specific to px
557  * called by px_attach()
558  */
559 static int
560 px_pwr_setup(dev_info_t *dip)
561 {
562 	pcie_pwr_t *pwr_p;
563 	int instance = ddi_get_instance(dip);
564 	px_t *px_p = INST_TO_STATE(instance);
565 	ddi_intr_handle_impl_t hdl;
566 
567 	ASSERT(PCIE_PMINFO(dip));
568 	pwr_p = PCIE_NEXUS_PMINFO(dip);
569 	ASSERT(pwr_p);
570 
571 	/*
572 	 * indicate support LDI (Layered Driver Interface)
573 	 * Create the property, if it is not already there
574 	 */
575 	if (!ddi_prop_exists(DDI_DEV_T_NONE, dip, DDI_PROP_DONTPASS,
576 	    DDI_KERNEL_IOCTL)) {
577 		if (ddi_prop_create(DDI_DEV_T_NONE, dip, DDI_PROP_CANSLEEP,
578 		    DDI_KERNEL_IOCTL, NULL, 0) != DDI_PROP_SUCCESS) {
579 			DBG(DBG_PWR, dip, "can't create kernel ioctl prop\n");
580 			return (DDI_FAILURE);
581 		}
582 	}
583 	/* No support for device PM. We are always at full power */
584 	pwr_p->pwr_func_lvl = PM_LEVEL_D0;
585 
586 	mutex_init(&px_p->px_l23ready_lock, NULL, MUTEX_DRIVER,
587 	    DDI_INTR_PRI(px_pwr_pil));
588 	cv_init(&px_p->px_l23ready_cv, NULL, CV_DRIVER, NULL);
589 
590 	/* Initialize handle */
591 	bzero(&hdl, sizeof (ddi_intr_handle_impl_t));
592 	hdl.ih_cb_arg1 = px_p;
593 	hdl.ih_ver = DDI_INTR_VERSION;
594 	hdl.ih_state = DDI_IHDL_STATE_ALLOC;
595 	hdl.ih_dip = dip;
596 	hdl.ih_pri = px_pwr_pil;
597 
598 	/* Add PME_TO_ACK message handler */
599 	hdl.ih_cb_func = (ddi_intr_handler_t *)px_pmeq_intr;
600 	if (px_add_msiq_intr(dip, dip, &hdl, MSG_REC,
601 	    (msgcode_t)PCIE_PME_ACK_MSG, &px_p->px_pm_msiq_id) != DDI_SUCCESS) {
602 		DBG(DBG_PWR, dip, "px_pwr_setup: couldn't add "
603 		    " PME_TO_ACK intr\n");
604 		goto pwr_setup_err1;
605 	}
606 	px_lib_msg_setmsiq(dip, PCIE_PME_ACK_MSG, px_p->px_pm_msiq_id);
607 	px_lib_msg_setvalid(dip, PCIE_PME_ACK_MSG, PCIE_MSG_VALID);
608 
609 	if (px_ib_update_intr_state(px_p, px_p->px_dip, hdl.ih_inum,
610 	    px_msiqid_to_devino(px_p, px_p->px_pm_msiq_id), px_pwr_pil,
611 	    PX_INTR_STATE_ENABLE, MSG_REC, PCIE_PME_ACK_MSG) != DDI_SUCCESS) {
612 		DBG(DBG_PWR, dip, "px_pwr_setup: PME_TO_ACK update interrupt"
613 		    " state failed\n");
614 		goto px_pwrsetup_err_state;
615 	}
616 
617 	return (DDI_SUCCESS);
618 
619 px_pwrsetup_err_state:
620 	px_lib_msg_setvalid(dip, PCIE_PME_ACK_MSG, PCIE_MSG_INVALID);
621 	(void) px_rem_msiq_intr(dip, dip, &hdl, MSG_REC, PCIE_PME_ACK_MSG,
622 	    px_p->px_pm_msiq_id);
623 pwr_setup_err1:
624 	mutex_destroy(&px_p->px_l23ready_lock);
625 	cv_destroy(&px_p->px_l23ready_cv);
626 
627 	return (DDI_FAILURE);
628 }
629 
630 /*
631  * undo whatever is done in px_pwr_setup. called by px_detach()
632  */
633 static void
634 px_pwr_teardown(dev_info_t *dip)
635 {
636 	int instance = ddi_get_instance(dip);
637 	px_t *px_p = INST_TO_STATE(instance);
638 	ddi_intr_handle_impl_t	hdl;
639 
640 	if (!PCIE_PMINFO(dip) || !PCIE_NEXUS_PMINFO(dip))
641 		return;
642 
643 	/* Initialize handle */
644 	bzero(&hdl, sizeof (ddi_intr_handle_impl_t));
645 	hdl.ih_ver = DDI_INTR_VERSION;
646 	hdl.ih_state = DDI_IHDL_STATE_ALLOC;
647 	hdl.ih_dip = dip;
648 	hdl.ih_pri = px_pwr_pil;
649 
650 	px_lib_msg_setvalid(dip, PCIE_PME_ACK_MSG, PCIE_MSG_INVALID);
651 	(void) px_rem_msiq_intr(dip, dip, &hdl, MSG_REC, PCIE_PME_ACK_MSG,
652 	    px_p->px_pm_msiq_id);
653 
654 	(void) px_ib_update_intr_state(px_p, px_p->px_dip, hdl.ih_inum,
655 	    px_msiqid_to_devino(px_p, px_p->px_pm_msiq_id), px_pwr_pil,
656 	    PX_INTR_STATE_DISABLE, MSG_REC, PCIE_PME_ACK_MSG);
657 
658 	px_p->px_pm_msiq_id = (msiqid_t)-1;
659 
660 	cv_destroy(&px_p->px_l23ready_cv);
661 	mutex_destroy(&px_p->px_l23ready_lock);
662 }
663 
664 /* bus driver entry points */
665 
666 /*
667  * bus map entry point:
668  *
669  * 	if map request is for an rnumber
670  *		get the corresponding regspec from device node
671  * 	build a new regspec in our parent's format
672  *	build a new map_req with the new regspec
673  *	call up the tree to complete the mapping
674  */
675 int
676 px_map(dev_info_t *dip, dev_info_t *rdip, ddi_map_req_t *mp,
677 	off_t off, off_t len, caddr_t *addrp)
678 {
679 	px_t *px_p = DIP_TO_STATE(dip);
680 	struct regspec p_regspec;
681 	ddi_map_req_t p_mapreq;
682 	int reglen, rval, r_no;
683 	pci_regspec_t reloc_reg, *rp = &reloc_reg;
684 
685 	DBG(DBG_MAP, dip, "rdip=%s%d:",
686 		ddi_driver_name(rdip), ddi_get_instance(rdip));
687 
688 	if (mp->map_flags & DDI_MF_USER_MAPPING)
689 		return (DDI_ME_UNIMPLEMENTED);
690 
691 	switch (mp->map_type) {
692 	case DDI_MT_REGSPEC:
693 		reloc_reg = *(pci_regspec_t *)mp->map_obj.rp;	/* dup whole */
694 		break;
695 
696 	case DDI_MT_RNUMBER:
697 		r_no = mp->map_obj.rnumber;
698 		DBG(DBG_MAP | DBG_CONT, dip, " r#=%x", r_no);
699 
700 		if (ddi_getlongprop(DDI_DEV_T_ANY, rdip, DDI_PROP_DONTPASS,
701 			"reg", (caddr_t)&rp, &reglen) != DDI_SUCCESS)
702 				return (DDI_ME_RNUMBER_RANGE);
703 
704 		if (r_no < 0 || r_no >= reglen / sizeof (pci_regspec_t)) {
705 			kmem_free(rp, reglen);
706 			return (DDI_ME_RNUMBER_RANGE);
707 		}
708 		rp += r_no;
709 		break;
710 
711 	default:
712 		return (DDI_ME_INVAL);
713 	}
714 	DBG(DBG_MAP | DBG_CONT, dip, "\n");
715 
716 	if ((rp->pci_phys_hi & PCI_REG_ADDR_M) == PCI_ADDR_CONFIG) {
717 		/*
718 		 * There may be a need to differentiate between PCI
719 		 * and PCI-Ex devices so the following range check is
720 		 * done correctly, depending on the implementation of
721 		 * px_pci bridge nexus driver.
722 		 */
723 		if ((off >= PCIE_CONF_HDR_SIZE) ||
724 				(len > PCIE_CONF_HDR_SIZE) ||
725 				(off + len > PCIE_CONF_HDR_SIZE))
726 			return (DDI_ME_INVAL);
727 		/*
728 		 * the following function returning a DDI_FAILURE assumes
729 		 * that there are no virtual config space access services
730 		 * defined in this layer. Otherwise it is availed right
731 		 * here and we return.
732 		 */
733 		rval = px_lib_map_vconfig(dip, mp, off, rp, addrp);
734 		if (rval == DDI_SUCCESS)
735 			goto done;
736 	}
737 
738 	/*
739 	 * No virtual config space services or we are mapping
740 	 * a region of memory mapped config/IO/memory space, so proceed
741 	 * to the parent.
742 	 */
743 
744 	/* relocate within 64-bit pci space through "assigned-addresses" */
745 	if (rval = px_reloc_reg(dip, rdip, px_p, rp))
746 		goto done;
747 
748 	if (len)	/* adjust regspec according to mapping request */
749 		rp->pci_size_low = len;	/* MIN ? */
750 	rp->pci_phys_low += off;
751 
752 	/* translate relocated pci regspec into parent space through "ranges" */
753 	if (rval = px_xlate_reg(px_p, rp, &p_regspec))
754 		goto done;
755 
756 	p_mapreq = *mp;		/* dup the whole structure */
757 	p_mapreq.map_type = DDI_MT_REGSPEC;
758 	p_mapreq.map_obj.rp = &p_regspec;
759 	px_lib_map_attr_check(&p_mapreq);
760 	rval = ddi_map(dip, &p_mapreq, 0, 0, addrp);
761 
762 	if (rval == DDI_SUCCESS) {
763 		/*
764 		 * Set-up access functions for FM access error capable drivers.
765 		 */
766 		if (DDI_FM_ACC_ERR_CAP(ddi_fm_capable(rdip)) &&
767 		    mp->map_handlep->ah_acc.devacc_attr_access !=
768 		    DDI_DEFAULT_ACC)
769 			px_fm_acc_setup(mp, rdip);
770 	}
771 
772 done:
773 	if (mp->map_type == DDI_MT_RNUMBER)
774 		kmem_free(rp - r_no, reglen);
775 
776 	return (rval);
777 }
778 
779 /*
780  * bus dma map entry point
781  * return value:
782  *	DDI_DMA_PARTIAL_MAP	 1
783  *	DDI_DMA_MAPOK		 0
784  *	DDI_DMA_MAPPED		 0
785  *	DDI_DMA_NORESOURCES	-1
786  *	DDI_DMA_NOMAPPING	-2
787  *	DDI_DMA_TOOBIG		-3
788  */
789 int
790 px_dma_setup(dev_info_t *dip, dev_info_t *rdip, ddi_dma_req_t *dmareq,
791 	ddi_dma_handle_t *handlep)
792 {
793 	px_t *px_p = DIP_TO_STATE(dip);
794 	px_mmu_t *mmu_p = px_p->px_mmu_p;
795 	ddi_dma_impl_t *mp;
796 	int ret;
797 
798 	DBG(DBG_DMA_MAP, dip, "mapping - rdip=%s%d type=%s\n",
799 		ddi_driver_name(rdip), ddi_get_instance(rdip),
800 		handlep ? "alloc" : "advisory");
801 
802 	if (!(mp = px_dma_lmts2hdl(dip, rdip, mmu_p, dmareq)))
803 		return (DDI_DMA_NORESOURCES);
804 	if (mp == (ddi_dma_impl_t *)DDI_DMA_NOMAPPING)
805 		return (DDI_DMA_NOMAPPING);
806 	if (ret = px_dma_type(px_p, dmareq, mp))
807 		goto freehandle;
808 	if (ret = px_dma_pfn(px_p, dmareq, mp))
809 		goto freehandle;
810 
811 	switch (PX_DMA_TYPE(mp)) {
812 	case PX_DMAI_FLAGS_DVMA:	/* LINTED E_EQUALITY_NOT_ASSIGNMENT */
813 		if ((ret = px_dvma_win(px_p, dmareq, mp)) || !handlep)
814 			goto freehandle;
815 		if (!PX_DMA_CANCACHE(mp)) {	/* try fast track */
816 			if (PX_DMA_CANFAST(mp)) {
817 				if (!px_dvma_map_fast(mmu_p, mp))
818 					break;
819 			/* LINTED E_NOP_ELSE_STMT */
820 			} else {
821 				PX_DVMA_FASTTRAK_PROF(mp);
822 			}
823 		}
824 		if (ret = px_dvma_map(mp, dmareq, mmu_p))
825 			goto freehandle;
826 		break;
827 	case PX_DMAI_FLAGS_PTP:	/* LINTED E_EQUALITY_NOT_ASSIGNMENT */
828 		if ((ret = px_dma_physwin(px_p, dmareq, mp)) || !handlep)
829 			goto freehandle;
830 		break;
831 	case PX_DMAI_FLAGS_BYPASS:
832 	default:
833 		cmn_err(CE_PANIC, "%s%d: px_dma_setup: bad dma type 0x%x",
834 			ddi_driver_name(rdip), ddi_get_instance(rdip),
835 			PX_DMA_TYPE(mp));
836 		/*NOTREACHED*/
837 	}
838 	*handlep = (ddi_dma_handle_t)mp;
839 	mp->dmai_flags |= PX_DMAI_FLAGS_INUSE;
840 	px_dump_dma_handle(DBG_DMA_MAP, dip, mp);
841 
842 	return ((mp->dmai_nwin == 1) ? DDI_DMA_MAPPED : DDI_DMA_PARTIAL_MAP);
843 freehandle:
844 	if (ret == DDI_DMA_NORESOURCES)
845 		px_dma_freemp(mp); /* don't run_callback() */
846 	else
847 		(void) px_dma_freehdl(dip, rdip, (ddi_dma_handle_t)mp);
848 	return (ret);
849 }
850 
851 
852 /*
853  * bus dma alloc handle entry point:
854  */
855 int
856 px_dma_allochdl(dev_info_t *dip, dev_info_t *rdip, ddi_dma_attr_t *attrp,
857 	int (*waitfp)(caddr_t), caddr_t arg, ddi_dma_handle_t *handlep)
858 {
859 	px_t *px_p = DIP_TO_STATE(dip);
860 	ddi_dma_impl_t *mp;
861 	int rval;
862 
863 	DBG(DBG_DMA_ALLOCH, dip, "rdip=%s%d\n",
864 		ddi_driver_name(rdip), ddi_get_instance(rdip));
865 
866 	if (attrp->dma_attr_version != DMA_ATTR_V0)
867 		return (DDI_DMA_BADATTR);
868 
869 	if (!(mp = px_dma_allocmp(dip, rdip, waitfp, arg)))
870 		return (DDI_DMA_NORESOURCES);
871 
872 	/*
873 	 * Save requestor's information
874 	 */
875 	mp->dmai_attr	= *attrp; /* whole object - augmented later  */
876 	*PX_DEV_ATTR(mp)	= *attrp; /* whole object - device orig attr */
877 	DBG(DBG_DMA_ALLOCH, dip, "mp=%p\n", mp);
878 
879 	/* check and convert dma attributes to handle parameters */
880 	if (rval = px_dma_attr2hdl(px_p, mp)) {
881 		px_dma_freehdl(dip, rdip, (ddi_dma_handle_t)mp);
882 		*handlep = NULL;
883 		return (rval);
884 	}
885 	*handlep = (ddi_dma_handle_t)mp;
886 	return (DDI_SUCCESS);
887 }
888 
889 
890 /*
891  * bus dma free handle entry point:
892  */
893 /*ARGSUSED*/
894 int
895 px_dma_freehdl(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle)
896 {
897 	DBG(DBG_DMA_FREEH, dip, "rdip=%s%d mp=%p\n",
898 		ddi_driver_name(rdip), ddi_get_instance(rdip), handle);
899 	px_dma_freemp((ddi_dma_impl_t *)handle);
900 
901 	if (px_kmem_clid) {
902 		DBG(DBG_DMA_FREEH, dip, "run handle callback\n");
903 		ddi_run_callback(&px_kmem_clid);
904 	}
905 	return (DDI_SUCCESS);
906 }
907 
908 
909 /*
910  * bus dma bind handle entry point:
911  */
912 int
913 px_dma_bindhdl(dev_info_t *dip, dev_info_t *rdip,
914 	ddi_dma_handle_t handle, ddi_dma_req_t *dmareq,
915 	ddi_dma_cookie_t *cookiep, uint_t *ccountp)
916 {
917 	px_t *px_p = DIP_TO_STATE(dip);
918 	px_mmu_t *mmu_p = px_p->px_mmu_p;
919 	ddi_dma_impl_t *mp = (ddi_dma_impl_t *)handle;
920 	int ret;
921 
922 	DBG(DBG_DMA_BINDH, dip, "rdip=%s%d mp=%p dmareq=%p\n",
923 		ddi_driver_name(rdip), ddi_get_instance(rdip), mp, dmareq);
924 
925 	if (mp->dmai_flags & PX_DMAI_FLAGS_INUSE)
926 		return (DDI_DMA_INUSE);
927 
928 	ASSERT((mp->dmai_flags & ~PX_DMAI_FLAGS_PRESERVE) == 0);
929 	mp->dmai_flags |= PX_DMAI_FLAGS_INUSE;
930 
931 	if (ret = px_dma_type(px_p, dmareq, mp))
932 		goto err;
933 	if (ret = px_dma_pfn(px_p, dmareq, mp))
934 		goto err;
935 
936 	switch (PX_DMA_TYPE(mp)) {
937 	case PX_DMAI_FLAGS_DVMA:
938 		if (ret = px_dvma_win(px_p, dmareq, mp))
939 			goto map_err;
940 		if (!PX_DMA_CANCACHE(mp)) {	/* try fast track */
941 			if (PX_DMA_CANFAST(mp)) {
942 				if (!px_dvma_map_fast(mmu_p, mp))
943 					goto mapped; /*LINTED E_NOP_ELSE_STMT*/
944 			} else {
945 				PX_DVMA_FASTTRAK_PROF(mp);
946 			}
947 		}
948 		if (ret = px_dvma_map(mp, dmareq, mmu_p))
949 			goto map_err;
950 mapped:
951 		*ccountp = 1;
952 		MAKE_DMA_COOKIE(cookiep, mp->dmai_mapping, mp->dmai_size);
953 		break;
954 	case PX_DMAI_FLAGS_BYPASS:
955 	case PX_DMAI_FLAGS_PTP:
956 		if (ret = px_dma_physwin(px_p, dmareq, mp))
957 			goto map_err;
958 		*ccountp = PX_WINLST(mp)->win_ncookies;
959 		*cookiep =
960 		    *(ddi_dma_cookie_t *)(PX_WINLST(mp) + 1); /* wholeobj */
961 		break;
962 	default:
963 		cmn_err(CE_PANIC, "%s%d: px_dma_bindhdl(%p): bad dma type",
964 			ddi_driver_name(rdip), ddi_get_instance(rdip), mp);
965 		/*NOTREACHED*/
966 	}
967 	DBG(DBG_DMA_BINDH, dip, "cookie %" PRIx64 "+%x\n",
968 		cookiep->dmac_address, cookiep->dmac_size);
969 	px_dump_dma_handle(DBG_DMA_MAP, dip, mp);
970 
971 	/* insert dma handle into FMA cache */
972 	if (mp->dmai_attr.dma_attr_flags & DDI_DMA_FLAGERR) {
973 		(void) ndi_fmc_insert(rdip, DMA_HANDLE, mp, NULL);
974 		mp->dmai_error.err_cf = impl_dma_check;
975 	}
976 
977 	return (mp->dmai_nwin == 1 ? DDI_DMA_MAPPED : DDI_DMA_PARTIAL_MAP);
978 map_err:
979 	px_dma_freepfn(mp);
980 err:
981 	mp->dmai_flags &= PX_DMAI_FLAGS_PRESERVE;
982 	return (ret);
983 }
984 
985 
986 /*
987  * bus dma unbind handle entry point:
988  */
989 /*ARGSUSED*/
990 int
991 px_dma_unbindhdl(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle)
992 {
993 	ddi_dma_impl_t *mp = (ddi_dma_impl_t *)handle;
994 	px_t *px_p = DIP_TO_STATE(dip);
995 	px_mmu_t *mmu_p = px_p->px_mmu_p;
996 
997 	DBG(DBG_DMA_UNBINDH, dip, "rdip=%s%d, mp=%p\n",
998 		ddi_driver_name(rdip), ddi_get_instance(rdip), handle);
999 	if ((mp->dmai_flags & PX_DMAI_FLAGS_INUSE) == 0) {
1000 		DBG(DBG_DMA_UNBINDH, dip, "handle not inuse\n");
1001 		return (DDI_FAILURE);
1002 	}
1003 
1004 	/* remove dma handle from FMA cache */
1005 	if (mp->dmai_attr.dma_attr_flags & DDI_DMA_FLAGERR) {
1006 		if (DEVI(rdip)->devi_fmhdl != NULL &&
1007 		    DDI_FM_DMA_ERR_CAP(DEVI(rdip)->devi_fmhdl->fh_cap)) {
1008 			(void) ndi_fmc_remove(rdip, DMA_HANDLE, mp);
1009 		}
1010 	}
1011 
1012 	/*
1013 	 * Here if the handle is using the iommu.  Unload all the iommu
1014 	 * translations.
1015 	 */
1016 	switch (PX_DMA_TYPE(mp)) {
1017 	case PX_DMAI_FLAGS_DVMA:
1018 		px_mmu_unmap_window(mmu_p, mp);
1019 		px_dvma_unmap(mmu_p, mp);
1020 		px_dma_freepfn(mp);
1021 		break;
1022 	case PX_DMAI_FLAGS_BYPASS:
1023 	case PX_DMAI_FLAGS_PTP:
1024 		px_dma_freewin(mp);
1025 		break;
1026 	default:
1027 		cmn_err(CE_PANIC, "%s%d: px_dma_unbindhdl:bad dma type %p",
1028 			ddi_driver_name(rdip), ddi_get_instance(rdip), mp);
1029 		/*NOTREACHED*/
1030 	}
1031 	if (mmu_p->mmu_dvma_clid != 0) {
1032 		DBG(DBG_DMA_UNBINDH, dip, "run dvma callback\n");
1033 		ddi_run_callback(&mmu_p->mmu_dvma_clid);
1034 	}
1035 	if (px_kmem_clid) {
1036 		DBG(DBG_DMA_UNBINDH, dip, "run handle callback\n");
1037 		ddi_run_callback(&px_kmem_clid);
1038 	}
1039 	mp->dmai_flags &= PX_DMAI_FLAGS_PRESERVE;
1040 
1041 	return (DDI_SUCCESS);
1042 }
1043 
1044 /*
1045  * bus dma win entry point:
1046  */
1047 int
1048 px_dma_win(dev_info_t *dip, dev_info_t *rdip,
1049 	ddi_dma_handle_t handle, uint_t win, off_t *offp,
1050 	size_t *lenp, ddi_dma_cookie_t *cookiep, uint_t *ccountp)
1051 {
1052 	ddi_dma_impl_t	*mp = (ddi_dma_impl_t *)handle;
1053 	int		ret;
1054 
1055 	DBG(DBG_DMA_WIN, dip, "rdip=%s%d\n",
1056 		ddi_driver_name(rdip), ddi_get_instance(rdip));
1057 
1058 	px_dump_dma_handle(DBG_DMA_WIN, dip, mp);
1059 	if (win >= mp->dmai_nwin) {
1060 		DBG(DBG_DMA_WIN, dip, "%x out of range\n", win);
1061 		return (DDI_FAILURE);
1062 	}
1063 
1064 	switch (PX_DMA_TYPE(mp)) {
1065 	case PX_DMAI_FLAGS_DVMA:
1066 		if (win != PX_DMA_CURWIN(mp)) {
1067 			px_t *px_p = DIP_TO_STATE(dip);
1068 			px_mmu_t *mmu_p = px_p->px_mmu_p;
1069 			px_mmu_unmap_window(mmu_p, mp);
1070 
1071 			/* map_window sets dmai_mapping/size/offset */
1072 			px_mmu_map_window(mmu_p, mp, win);
1073 			if ((ret = px_mmu_map_window(mmu_p,
1074 			    mp, win)) != DDI_SUCCESS)
1075 				return (ret);
1076 		}
1077 		if (cookiep)
1078 			MAKE_DMA_COOKIE(cookiep, mp->dmai_mapping,
1079 				mp->dmai_size);
1080 		if (ccountp)
1081 			*ccountp = 1;
1082 		break;
1083 	case PX_DMAI_FLAGS_PTP:
1084 	case PX_DMAI_FLAGS_BYPASS: {
1085 		int i;
1086 		ddi_dma_cookie_t *ck_p;
1087 		px_dma_win_t *win_p = mp->dmai_winlst;
1088 
1089 		for (i = 0; i < win; win_p = win_p->win_next, i++);
1090 		ck_p = (ddi_dma_cookie_t *)(win_p + 1);
1091 		*cookiep = *ck_p;
1092 		mp->dmai_offset = win_p->win_offset;
1093 		mp->dmai_size   = win_p->win_size;
1094 		mp->dmai_mapping = ck_p->dmac_laddress;
1095 		mp->dmai_cookie = ck_p + 1;
1096 		win_p->win_curseg = 0;
1097 		if (ccountp)
1098 			*ccountp = win_p->win_ncookies;
1099 		}
1100 		break;
1101 	default:
1102 		cmn_err(CE_WARN, "%s%d: px_dma_win:bad dma type 0x%x",
1103 			ddi_driver_name(rdip), ddi_get_instance(rdip),
1104 			PX_DMA_TYPE(mp));
1105 		return (DDI_FAILURE);
1106 	}
1107 	if (cookiep)
1108 		DBG(DBG_DMA_WIN, dip,
1109 			"cookie - dmac_address=%x dmac_size=%x\n",
1110 			cookiep->dmac_address, cookiep->dmac_size);
1111 	if (offp)
1112 		*offp = (off_t)mp->dmai_offset;
1113 	if (lenp)
1114 		*lenp = mp->dmai_size;
1115 	return (DDI_SUCCESS);
1116 }
1117 
1118 #ifdef	DEBUG
1119 static char *px_dmactl_str[] = {
1120 	"DDI_DMA_FREE",
1121 	"DDI_DMA_SYNC",
1122 	"DDI_DMA_HTOC",
1123 	"DDI_DMA_KVADDR",
1124 	"DDI_DMA_MOVWIN",
1125 	"DDI_DMA_REPWIN",
1126 	"DDI_DMA_GETERR",
1127 	"DDI_DMA_COFF",
1128 	"DDI_DMA_NEXTWIN",
1129 	"DDI_DMA_NEXTSEG",
1130 	"DDI_DMA_SEGTOC",
1131 	"DDI_DMA_RESERVE",
1132 	"DDI_DMA_RELEASE",
1133 	"DDI_DMA_RESETH",
1134 	"DDI_DMA_CKSYNC",
1135 	"DDI_DMA_IOPB_ALLOC",
1136 	"DDI_DMA_IOPB_FREE",
1137 	"DDI_DMA_SMEM_ALLOC",
1138 	"DDI_DMA_SMEM_FREE",
1139 	"DDI_DMA_SET_SBUS64"
1140 };
1141 #endif	/* DEBUG */
1142 
1143 /*
1144  * bus dma control entry point:
1145  */
1146 /*ARGSUSED*/
1147 int
1148 px_dma_ctlops(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle,
1149 	enum ddi_dma_ctlops cmd, off_t *offp, size_t *lenp, caddr_t *objp,
1150 	uint_t cache_flags)
1151 {
1152 	ddi_dma_impl_t *mp = (ddi_dma_impl_t *)handle;
1153 
1154 #ifdef	DEBUG
1155 	DBG(DBG_DMA_CTL, dip, "%s: rdip=%s%d\n", px_dmactl_str[cmd],
1156 		ddi_driver_name(rdip), ddi_get_instance(rdip));
1157 #endif	/* DEBUG */
1158 
1159 	switch (cmd) {
1160 	case DDI_DMA_FREE:
1161 		(void) px_dma_unbindhdl(dip, rdip, handle);
1162 		(void) px_dma_freehdl(dip, rdip, handle);
1163 		return (DDI_SUCCESS);
1164 	case DDI_DMA_RESERVE: {
1165 		px_t *px_p = DIP_TO_STATE(dip);
1166 		return (px_fdvma_reserve(dip, rdip, px_p,
1167 			(ddi_dma_req_t *)offp, (ddi_dma_handle_t *)objp));
1168 		}
1169 	case DDI_DMA_RELEASE: {
1170 		px_t *px_p = DIP_TO_STATE(dip);
1171 		return (px_fdvma_release(dip, px_p, mp));
1172 		}
1173 	default:
1174 		break;
1175 	}
1176 
1177 	switch (PX_DMA_TYPE(mp)) {
1178 	case PX_DMAI_FLAGS_DVMA:
1179 		return (px_dvma_ctl(dip, rdip, mp, cmd, offp, lenp, objp,
1180 			cache_flags));
1181 	case PX_DMAI_FLAGS_PTP:
1182 	case PX_DMAI_FLAGS_BYPASS:
1183 		return (px_dma_ctl(dip, rdip, mp, cmd, offp, lenp, objp,
1184 			cache_flags));
1185 	default:
1186 		cmn_err(CE_PANIC, "%s%d: px_dma_ctlops(%x):bad dma type %x",
1187 			ddi_driver_name(rdip), ddi_get_instance(rdip), cmd,
1188 			mp->dmai_flags);
1189 		/*NOTREACHED*/
1190 	}
1191 	return (0);
1192 }
1193 
1194 /*
1195  * control ops entry point:
1196  *
1197  * Requests handled completely:
1198  *	DDI_CTLOPS_INITCHILD	see init_child() for details
1199  *	DDI_CTLOPS_UNINITCHILD
1200  *	DDI_CTLOPS_REPORTDEV	see report_dev() for details
1201  *	DDI_CTLOPS_IOMIN	cache line size if streaming otherwise 1
1202  *	DDI_CTLOPS_REGSIZE
1203  *	DDI_CTLOPS_NREGS
1204  *	DDI_CTLOPS_DVMAPAGESIZE
1205  *	DDI_CTLOPS_POKE
1206  *	DDI_CTLOPS_PEEK
1207  *
1208  * All others passed to parent.
1209  */
1210 int
1211 px_ctlops(dev_info_t *dip, dev_info_t *rdip,
1212 	ddi_ctl_enum_t op, void *arg, void *result)
1213 {
1214 	px_t *px_p = DIP_TO_STATE(dip);
1215 	struct detachspec *ds;
1216 	struct attachspec *as;
1217 
1218 	switch (op) {
1219 	case DDI_CTLOPS_INITCHILD:
1220 		return (px_init_child(px_p, (dev_info_t *)arg));
1221 
1222 	case DDI_CTLOPS_UNINITCHILD:
1223 		return (px_uninit_child(px_p, (dev_info_t *)arg));
1224 
1225 	case DDI_CTLOPS_ATTACH:
1226 		if (!pcie_is_child(dip, rdip))
1227 			return (DDI_SUCCESS);
1228 
1229 		as = (struct attachspec *)arg;
1230 		switch (as->when) {
1231 		case DDI_PRE:
1232 			if (as->cmd == DDI_ATTACH) {
1233 				DBG(DBG_PWR, dip, "PRE_ATTACH for %s@%d\n",
1234 				    ddi_driver_name(rdip),
1235 				    ddi_get_instance(rdip));
1236 				return (pcie_pm_hold(dip));
1237 			}
1238 			if (as->cmd == DDI_RESUME) {
1239 				ddi_acc_handle_t	config_handle;
1240 				DBG(DBG_PWR, dip, "PRE_RESUME for %s@%d\n",
1241 				    ddi_driver_name(rdip),
1242 				    ddi_get_instance(rdip));
1243 
1244 				if (pci_config_setup(rdip, &config_handle) ==
1245 				    DDI_SUCCESS) {
1246 					pcie_clear_errors(rdip, config_handle);
1247 					pci_config_teardown(&config_handle);
1248 				}
1249 			}
1250 			return (DDI_SUCCESS);
1251 
1252 		case DDI_POST:
1253 			DBG(DBG_PWR, dip, "POST_ATTACH for %s@%d\n",
1254 			    ddi_driver_name(rdip), ddi_get_instance(rdip));
1255 			if (as->cmd == DDI_ATTACH && as->result != DDI_SUCCESS)
1256 				pcie_pm_release(dip);
1257 
1258 			if (as->result == DDI_SUCCESS)
1259 				pf_init(rdip, (void *)px_p->px_fm_ibc, as->cmd);
1260 
1261 			(void) pcie_postattach_child(rdip);
1262 
1263 			return (DDI_SUCCESS);
1264 		default:
1265 			break;
1266 		}
1267 		break;
1268 
1269 	case DDI_CTLOPS_DETACH:
1270 		if (!pcie_is_child(dip, rdip))
1271 			return (DDI_SUCCESS);
1272 
1273 		ds = (struct detachspec *)arg;
1274 		switch (ds->when) {
1275 		case DDI_POST:
1276 			if (ds->cmd == DDI_DETACH &&
1277 			    ds->result == DDI_SUCCESS) {
1278 				DBG(DBG_PWR, dip, "POST_DETACH for %s@%d\n",
1279 				    ddi_driver_name(rdip),
1280 				    ddi_get_instance(rdip));
1281 				return (pcie_pm_remove_child(dip, rdip));
1282 			}
1283 			return (DDI_SUCCESS);
1284 		case DDI_PRE:
1285 			pf_fini(rdip, ds->cmd);
1286 			return (DDI_SUCCESS);
1287 		default:
1288 			break;
1289 		}
1290 		break;
1291 
1292 	case DDI_CTLOPS_REPORTDEV:
1293 		return (px_report_dev(rdip));
1294 
1295 	case DDI_CTLOPS_IOMIN:
1296 		return (DDI_SUCCESS);
1297 
1298 	case DDI_CTLOPS_REGSIZE:
1299 		*((off_t *)result) = px_get_reg_set_size(rdip, *((int *)arg));
1300 		return (*((off_t *)result) == 0 ? DDI_FAILURE : DDI_SUCCESS);
1301 
1302 	case DDI_CTLOPS_NREGS:
1303 		*((uint_t *)result) = px_get_nreg_set(rdip);
1304 		return (DDI_SUCCESS);
1305 
1306 	case DDI_CTLOPS_DVMAPAGESIZE:
1307 		*((ulong_t *)result) = MMU_PAGE_SIZE;
1308 		return (DDI_SUCCESS);
1309 
1310 	case DDI_CTLOPS_POKE:	/* platform dependent implementation. */
1311 		return (px_lib_ctlops_poke(dip, rdip,
1312 		    (peekpoke_ctlops_t *)arg));
1313 
1314 	case DDI_CTLOPS_PEEK:	/* platform dependent implementation. */
1315 		return (px_lib_ctlops_peek(dip, rdip,
1316 		    (peekpoke_ctlops_t *)arg, result));
1317 
1318 	case DDI_CTLOPS_POWER:
1319 	default:
1320 		break;
1321 	}
1322 
1323 	/*
1324 	 * Now pass the request up to our parent.
1325 	 */
1326 	DBG(DBG_CTLOPS, dip, "passing request to parent: rdip=%s%d\n",
1327 		ddi_driver_name(rdip), ddi_get_instance(rdip));
1328 	return (ddi_ctlops(dip, rdip, op, arg, result));
1329 }
1330 
1331 /* ARGSUSED */
1332 int
1333 px_intr_ops(dev_info_t *dip, dev_info_t *rdip, ddi_intr_op_t intr_op,
1334     ddi_intr_handle_impl_t *hdlp, void *result)
1335 {
1336 	int	intr_types, ret = DDI_SUCCESS;
1337 
1338 	DBG(DBG_INTROPS, dip, "px_intr_ops: rdip=%s%d\n",
1339 	    ddi_driver_name(rdip), ddi_get_instance(rdip));
1340 
1341 	/* Process DDI_INTROP_SUPPORTED_TYPES request here */
1342 	if (intr_op == DDI_INTROP_SUPPORTED_TYPES) {
1343 		*(int *)result = i_ddi_get_intx_nintrs(rdip) ?
1344 		    DDI_INTR_TYPE_FIXED : 0;
1345 
1346 		if ((pci_msi_get_supported_type(rdip,
1347 		    &intr_types)) == DDI_SUCCESS) {
1348 			/*
1349 			 * Double check supported interrupt types vs.
1350 			 * what the host bridge supports.
1351 			 */
1352 			*(int *)result |= intr_types;
1353 		}
1354 
1355 		return (ret);
1356 	}
1357 
1358 	/*
1359 	 * PCI-E nexus driver supports fixed, MSI and MSI-X interrupts.
1360 	 * Return failure if interrupt type is not supported.
1361 	 */
1362 	switch (hdlp->ih_type) {
1363 	case DDI_INTR_TYPE_FIXED:
1364 		ret = px_intx_ops(dip, rdip, intr_op, hdlp, result);
1365 		break;
1366 	case DDI_INTR_TYPE_MSI:
1367 	case DDI_INTR_TYPE_MSIX:
1368 		ret = px_msix_ops(dip, rdip, intr_op, hdlp, result);
1369 		break;
1370 	default:
1371 		ret = DDI_ENOTSUP;
1372 		break;
1373 	}
1374 
1375 	return (ret);
1376 }
1377 
1378 static int
1379 px_init_hotplug(px_t *px_p)
1380 {
1381 	px_bus_range_t bus_range;
1382 	dev_info_t *dip;
1383 	pciehpc_regops_t regops;
1384 
1385 	dip = px_p->px_dip;
1386 
1387 	if (ddi_prop_exists(DDI_DEV_T_ANY, dip, DDI_PROP_DONTPASS,
1388 	    "hotplug-capable") == 0)
1389 		return (DDI_FAILURE);
1390 
1391 	/*
1392 	 * Before initializing hotplug - open up bus range.  The busra
1393 	 * module will initialize its pool of bus numbers from this.
1394 	 * "busra" will be the agent that keeps track of them during
1395 	 * hotplug.  Also, note, that busra will remove any bus numbers
1396 	 * already in use from boot time.
1397 	 */
1398 	if (ddi_prop_exists(DDI_DEV_T_ANY, dip, DDI_PROP_DONTPASS,
1399 	    "bus-range") == 0) {
1400 		cmn_err(CE_WARN, "%s%d: bus-range not found\n",
1401 		    ddi_driver_name(dip), ddi_get_instance(dip));
1402 #ifdef	DEBUG
1403 		bus_range.lo = 0x0;
1404 		bus_range.hi = 0xff;
1405 
1406 		if (ndi_prop_update_int_array(DDI_DEV_T_NONE,
1407 		    dip, "bus-range", (int *)&bus_range, 2)
1408 		    != DDI_PROP_SUCCESS) {
1409 			return (DDI_FAILURE);
1410 		}
1411 #else
1412 		return (DDI_FAILURE);
1413 #endif
1414 	}
1415 
1416 	if (px_lib_hotplug_init(dip, (void *)&regops) != DDI_SUCCESS)
1417 		return (DDI_FAILURE);
1418 
1419 	if (pciehpc_init(dip, &regops) != DDI_SUCCESS) {
1420 		px_lib_hotplug_uninit(dip);
1421 		return (DDI_FAILURE);
1422 	}
1423 
1424 	if (pcihp_init(dip) != DDI_SUCCESS) {
1425 		(void) pciehpc_uninit(dip);
1426 		px_lib_hotplug_uninit(dip);
1427 		return (DDI_FAILURE);
1428 	}
1429 
1430 	if (pcihp_get_cb_ops() != NULL) {
1431 		DBG(DBG_ATTACH, dip, "%s%d hotplug enabled",
1432 		    ddi_driver_name(dip), ddi_get_instance(dip));
1433 		px_p->px_dev_caps |= PX_HOTPLUG_CAPABLE;
1434 	}
1435 
1436 	return (DDI_SUCCESS);
1437 }
1438 
1439 static int
1440 px_uninit_hotplug(dev_info_t *dip)
1441 {
1442 	if (pcihp_uninit(dip) != DDI_SUCCESS)
1443 		return (DDI_FAILURE);
1444 
1445 	if (pciehpc_uninit(dip) != DDI_SUCCESS)
1446 		return (DDI_FAILURE);
1447 
1448 	px_lib_hotplug_uninit(dip);
1449 
1450 	return (DDI_SUCCESS);
1451 }
1452