xref: /titanic_51/usr/src/uts/i86pc/io/pciex/npe.c (revision 88447a05f537aabe9a1bc3d5313f22581ec992a7)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 
22 /*
23  * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
24  * Use is subject to license terms.
25  */
26 
27 /*
28  *	Host to PCI-Express local bus driver
29  */
30 
31 #include <sys/conf.h>
32 #include <sys/modctl.h>
33 #include <sys/pci_impl.h>
34 #include <sys/pcie_impl.h>
35 #include <sys/sysmacros.h>
36 #include <sys/ddi_intr.h>
37 #include <sys/sunndi.h>
38 #include <sys/sunddi.h>
39 #include <sys/ddifm.h>
40 #include <sys/ndifm.h>
41 #include <sys/fm/util.h>
42 #include <sys/hotplug/pci/pcihp.h>
43 #include <io/pci/pci_tools_ext.h>
44 #include <io/pci/pci_common.h>
45 #include <io/pciex/pcie_nvidia.h>
46 
47 /*
48  * Helper Macros
49  */
50 #define	NPE_IS_HANDLE_FOR_STDCFG_ACC(hp) \
51 	((hp) != NULL &&						\
52 	((ddi_acc_hdl_t *)(hp))->ah_platform_private != NULL &&		\
53 	(((ddi_acc_impl_t *)((ddi_acc_hdl_t *)(hp))->			\
54 	ah_platform_private)->						\
55 	    ahi_acc_attr &(DDI_ACCATTR_CPU_VADDR|DDI_ACCATTR_CONFIG_SPACE)) \
56 		== DDI_ACCATTR_CONFIG_SPACE)
57 
58 /*
59  * Bus Operation functions
60  */
61 static int	npe_bus_map(dev_info_t *, dev_info_t *, ddi_map_req_t *,
62 		    off_t, off_t, caddr_t *);
63 static int	npe_ctlops(dev_info_t *, dev_info_t *, ddi_ctl_enum_t,
64 		    void *, void *);
65 static int	npe_intr_ops(dev_info_t *, dev_info_t *, ddi_intr_op_t,
66 		    ddi_intr_handle_impl_t *, void *);
67 static int	npe_fm_init(dev_info_t *, dev_info_t *, int,
68 		    ddi_iblock_cookie_t *);
69 
70 static int	npe_fm_callback(dev_info_t *, ddi_fm_error_t *, const void *);
71 
72 /*
73  * Disable URs and Received MA for all PCIe devices.  Until x86 SW is changed so
74  * that random drivers do not do PIO accesses on devices that it does not own,
75  * these error bits must be disabled.  SERR must also be disabled if URs have
76  * been masked.
77  */
78 uint32_t	npe_aer_uce_mask = PCIE_AER_UCE_UR;
79 uint32_t	npe_aer_ce_mask = 0;
80 uint32_t	npe_aer_suce_mask = PCIE_AER_SUCE_RCVD_MA;
81 
82 struct bus_ops npe_bus_ops = {
83 	BUSO_REV,
84 	npe_bus_map,
85 	NULL,
86 	NULL,
87 	NULL,
88 	i_ddi_map_fault,
89 	ddi_dma_map,
90 	ddi_dma_allochdl,
91 	ddi_dma_freehdl,
92 	ddi_dma_bindhdl,
93 	ddi_dma_unbindhdl,
94 	ddi_dma_flush,
95 	ddi_dma_win,
96 	ddi_dma_mctl,
97 	npe_ctlops,
98 	ddi_bus_prop_op,
99 	0,		/* (*bus_get_eventcookie)();	*/
100 	0,		/* (*bus_add_eventcall)();	*/
101 	0,		/* (*bus_remove_eventcall)();	*/
102 	0,		/* (*bus_post_event)();		*/
103 	0,		/* (*bus_intr_ctl)(); */
104 	0,		/* (*bus_config)(); */
105 	0,		/* (*bus_unconfig)(); */
106 	npe_fm_init,	/* (*bus_fm_init)(); */
107 	NULL,		/* (*bus_fm_fini)(); */
108 	NULL,		/* (*bus_fm_access_enter)(); */
109 	NULL,		/* (*bus_fm_access_exit)(); */
110 	NULL,		/* (*bus_power)(); */
111 	npe_intr_ops	/* (*bus_intr_op)(); */
112 };
113 
114 /*
115  * One goal here is to leverage off of the pcihp.c source without making
116  * changes to it.  Call into it's cb_ops directly if needed, piggybacking
117  * anything else needed by the pci_tools.c module.  Only pci_tools and pcihp
118  * will be using the PCI devctl node.
119  */
120 static int	npe_open(dev_t *, int, int, cred_t *);
121 static int	npe_close(dev_t, int, int, cred_t *);
122 static int	npe_ioctl(dev_t, int, intptr_t, int, cred_t *, int *);
123 static int	npe_prop_op(dev_t, dev_info_t *, ddi_prop_op_t, int, char *,
124 		    caddr_t, int *);
125 static int	npe_info(dev_info_t *, ddi_info_cmd_t, void *, void **);
126 
127 struct cb_ops npe_cb_ops = {
128 	npe_open,			/* open */
129 	npe_close,			/* close */
130 	nodev,				/* strategy */
131 	nodev,				/* print */
132 	nodev,				/* dump */
133 	nodev,				/* read */
134 	nodev,				/* write */
135 	npe_ioctl,			/* ioctl */
136 	nodev,				/* devmap */
137 	nodev,				/* mmap */
138 	nodev,				/* segmap */
139 	nochpoll,			/* poll */
140 	npe_prop_op,			/* cb_prop_op */
141 	NULL,				/* streamtab */
142 	D_NEW | D_MP | D_HOTPLUG,	/* Driver compatibility flag */
143 	CB_REV,				/* rev */
144 	nodev,				/* int (*cb_aread)() */
145 	nodev				/* int (*cb_awrite)() */
146 };
147 
148 
149 /*
150  * Device Node Operation functions
151  */
152 static int	npe_attach(dev_info_t *devi, ddi_attach_cmd_t cmd);
153 static int	npe_detach(dev_info_t *devi, ddi_detach_cmd_t cmd);
154 
155 struct dev_ops npe_ops = {
156 	DEVO_REV,		/* devo_rev */
157 	0,			/* refcnt  */
158 	npe_info,		/* info */
159 	nulldev,		/* identify */
160 	nulldev,		/* probe */
161 	npe_attach,		/* attach */
162 	npe_detach,		/* detach */
163 	nulldev,		/* reset */
164 	&npe_cb_ops,		/* driver operations */
165 	&npe_bus_ops,		/* bus operations */
166 	NULL,			/* power */
167 	ddi_quiesce_not_needed,		/* quiesce */
168 };
169 
170 /*
171  * Internal routines in support of particular npe_ctlops.
172  */
173 static int npe_removechild(dev_info_t *child);
174 static int npe_initchild(dev_info_t *child);
175 
176 /*
177  * External support routine
178  */
179 extern void	npe_query_acpi_mcfg(dev_info_t *dip);
180 extern void	npe_ck804_fix_aer_ptr(ddi_acc_handle_t cfg_hdl);
181 extern int	npe_disable_empty_bridges_workaround(dev_info_t *child);
182 extern void	npe_nvidia_error_mask(ddi_acc_handle_t cfg_hdl);
183 extern void	npe_intel_error_mask(ddi_acc_handle_t cfg_hdl);
184 extern boolean_t npe_is_mmcfg_supported(dev_info_t *dip);
185 
186 /*
187  * Module linkage information for the kernel.
188  */
189 static struct modldrv modldrv = {
190 	&mod_driverops, /* Type of module */
191 	"Host to PCIe nexus driver",
192 	&npe_ops,	/* driver ops */
193 };
194 
195 static struct modlinkage modlinkage = {
196 	MODREV_1,
197 	(void *)&modldrv,
198 	NULL
199 };
200 
201 /* Save minimal state. */
202 void *npe_statep;
203 
204 int
205 _init(void)
206 {
207 	int e;
208 
209 	/*
210 	 * Initialize per-pci bus soft state pointer.
211 	 */
212 	e = ddi_soft_state_init(&npe_statep, sizeof (pci_state_t), 1);
213 	if (e != 0)
214 		return (e);
215 
216 	if ((e = mod_install(&modlinkage)) != 0)
217 		ddi_soft_state_fini(&npe_statep);
218 
219 	return (e);
220 }
221 
222 
223 int
224 _fini(void)
225 {
226 	int rc;
227 
228 	rc = mod_remove(&modlinkage);
229 	if (rc != 0)
230 		return (rc);
231 
232 	ddi_soft_state_fini(&npe_statep);
233 	return (rc);
234 }
235 
236 
237 int
238 _info(struct modinfo *modinfop)
239 {
240 	return (mod_info(&modlinkage, modinfop));
241 }
242 
243 /*ARGSUSED*/
244 static int
245 npe_attach(dev_info_t *devi, ddi_attach_cmd_t cmd)
246 {
247 	/*
248 	 * Use the minor number as constructed by pcihp, as the index value to
249 	 * ddi_soft_state_zalloc.
250 	 */
251 	int instance = ddi_get_instance(devi);
252 	pci_state_t *pcip = NULL;
253 
254 	if (cmd == DDI_RESUME)
255 		return (DDI_SUCCESS);
256 
257 	if (ddi_prop_update_string(DDI_DEV_T_NONE, devi, "device_type",
258 	    "pciex") != DDI_PROP_SUCCESS) {
259 		cmn_err(CE_WARN, "npe:  'device_type' prop create failed");
260 	}
261 
262 	if (ddi_soft_state_zalloc(npe_statep, instance) == DDI_SUCCESS)
263 		pcip = ddi_get_soft_state(npe_statep, instance);
264 
265 	if (pcip == NULL)
266 		return (DDI_FAILURE);
267 
268 	pcip->pci_dip = devi;
269 
270 	pcie_rc_init_bus(devi);
271 
272 	/*
273 	 * Initialize hotplug support on this bus. At minimum
274 	 * (for non hotplug bus) this would create ":devctl" minor
275 	 * node to support DEVCTL_DEVICE_* and DEVCTL_BUS_* ioctls
276 	 * to this bus.
277 	 */
278 	if (pcihp_init(devi) != DDI_SUCCESS) {
279 		cmn_err(CE_WARN, "npe: Failed to setup hotplug framework");
280 		ddi_soft_state_free(npe_statep, instance);
281 		return (DDI_FAILURE);
282 	}
283 
284 	/* Second arg: initialize for pci_express root nexus */
285 	if (pcitool_init(devi, B_TRUE) != DDI_SUCCESS) {
286 		(void) pcihp_uninit(devi);
287 		ddi_soft_state_free(npe_statep, instance);
288 		return (DDI_FAILURE);
289 	}
290 
291 	pcip->pci_fmcap = DDI_FM_EREPORT_CAPABLE | DDI_FM_ERRCB_CAPABLE |
292 	    DDI_FM_ACCCHK_CAPABLE | DDI_FM_DMACHK_CAPABLE;
293 	ddi_fm_init(devi, &pcip->pci_fmcap, &pcip->pci_fm_ibc);
294 
295 	if (pcip->pci_fmcap & DDI_FM_ERRCB_CAPABLE) {
296 		ddi_fm_handler_register(devi, npe_fm_callback, NULL);
297 	}
298 
299 	PCIE_DIP2PFD(devi) = kmem_zalloc(sizeof (pf_data_t), KM_SLEEP);
300 	pcie_rc_init_pfd(devi, PCIE_DIP2PFD(devi));
301 
302 	npe_query_acpi_mcfg(devi);
303 	ddi_report_dev(devi);
304 	return (DDI_SUCCESS);
305 
306 }
307 
308 /*ARGSUSED*/
309 static int
310 npe_detach(dev_info_t *devi, ddi_detach_cmd_t cmd)
311 {
312 	int instance = ddi_get_instance(devi);
313 	pci_state_t *pcip;
314 
315 	pcip = ddi_get_soft_state(npe_statep, ddi_get_instance(devi));
316 
317 	switch (cmd) {
318 	case DDI_DETACH:
319 
320 		/* Uninitialize pcitool support. */
321 		pcitool_uninit(devi);
322 
323 		/*
324 		 * Uninitialize hotplug support on this bus.
325 		 */
326 		(void) pcihp_uninit(devi);
327 
328 		if (pcip->pci_fmcap & DDI_FM_ERRCB_CAPABLE)
329 			ddi_fm_handler_unregister(devi);
330 
331 		pcie_rc_fini_bus(devi);
332 		pcie_rc_fini_pfd(PCIE_DIP2PFD(devi));
333 		kmem_free(PCIE_DIP2PFD(devi), sizeof (pf_data_t));
334 
335 		ddi_fm_fini(devi);
336 		ddi_soft_state_free(npe_statep, instance);
337 		return (DDI_SUCCESS);
338 
339 	case DDI_SUSPEND:
340 		return (DDI_SUCCESS);
341 	default:
342 		return (DDI_FAILURE);
343 	}
344 }
345 
346 /*
347  * Configure the access handle for standard configuration space
348  * access (see pci_fm_acc_setup for code that initializes the
349  * access-function pointers).
350  */
351 static int
352 npe_setup_std_pcicfg_acc(dev_info_t *rdip, ddi_map_req_t *mp,
353     ddi_acc_hdl_t *hp, off_t offset, off_t len)
354 {
355 	int ret;
356 
357 	if ((ret = pci_fm_acc_setup(hp, offset, len)) ==
358 	    DDI_SUCCESS) {
359 		if (DDI_FM_ACC_ERR_CAP(ddi_fm_capable(rdip)) &&
360 		    mp->map_handlep->ah_acc.devacc_attr_access
361 		    != DDI_DEFAULT_ACC) {
362 			ndi_fmc_insert(rdip, ACC_HANDLE,
363 			    (void *)mp->map_handlep, NULL);
364 		}
365 	}
366 	return (ret);
367 }
368 
369 static int
370 npe_bus_map(dev_info_t *dip, dev_info_t *rdip, ddi_map_req_t *mp,
371     off_t offset, off_t len, caddr_t *vaddrp)
372 {
373 	int 		rnumber;
374 	int		length;
375 	int		space;
376 	ddi_acc_impl_t	*ap;
377 	ddi_acc_hdl_t	*hp;
378 	ddi_map_req_t	mr;
379 	pci_regspec_t	pci_reg;
380 	pci_regspec_t	*pci_rp;
381 	struct regspec	reg;
382 	pci_acc_cfblk_t	*cfp;
383 	int		retval;
384 	int64_t		*ecfginfo;
385 	uint_t		nelem;
386 
387 	mr = *mp; /* Get private copy of request */
388 	mp = &mr;
389 
390 	/*
391 	 * check for register number
392 	 */
393 	switch (mp->map_type) {
394 	case DDI_MT_REGSPEC:
395 		pci_reg = *(pci_regspec_t *)(mp->map_obj.rp);
396 		pci_rp = &pci_reg;
397 		if (pci_common_get_reg_prop(rdip, pci_rp) != DDI_SUCCESS)
398 			return (DDI_FAILURE);
399 		break;
400 	case DDI_MT_RNUMBER:
401 		rnumber = mp->map_obj.rnumber;
402 		/*
403 		 * get ALL "reg" properties for dip, select the one of
404 		 * of interest. In x86, "assigned-addresses" property
405 		 * is identical to the "reg" property, so there is no
406 		 * need to cross check the two to determine the physical
407 		 * address of the registers.
408 		 * This routine still performs some validity checks to
409 		 * make sure that everything is okay.
410 		 */
411 		if (ddi_prop_lookup_int_array(DDI_DEV_T_ANY, rdip,
412 		    DDI_PROP_DONTPASS, "reg", (int **)&pci_rp,
413 		    (uint_t *)&length) != DDI_PROP_SUCCESS)
414 			return (DDI_FAILURE);
415 
416 		/*
417 		 * validate the register number.
418 		 */
419 		length /= (sizeof (pci_regspec_t) / sizeof (int));
420 		if (rnumber >= length) {
421 			ddi_prop_free(pci_rp);
422 			return (DDI_FAILURE);
423 		}
424 
425 		/*
426 		 * copy the required entry.
427 		 */
428 		pci_reg = pci_rp[rnumber];
429 
430 		/*
431 		 * free the memory allocated by ddi_prop_lookup_int_array
432 		 */
433 		ddi_prop_free(pci_rp);
434 
435 		pci_rp = &pci_reg;
436 		if (pci_common_get_reg_prop(rdip, pci_rp) != DDI_SUCCESS)
437 			return (DDI_FAILURE);
438 		mp->map_type = DDI_MT_REGSPEC;
439 		break;
440 	default:
441 		return (DDI_ME_INVAL);
442 	}
443 
444 	space = pci_rp->pci_phys_hi & PCI_REG_ADDR_M;
445 
446 	/*
447 	 * check for unmap and unlock of address space
448 	 */
449 	if ((mp->map_op == DDI_MO_UNMAP) || (mp->map_op == DDI_MO_UNLOCK)) {
450 		switch (space) {
451 		case PCI_ADDR_IO:
452 			reg.regspec_bustype = 1;
453 			break;
454 
455 		case PCI_ADDR_CONFIG:
456 			/*
457 			 * If this is an unmap/unlock of a standard config
458 			 * space mapping (memory-mapped config space mappings
459 			 * would have the DDI_ACCATTR_CPU_VADDR bit set in the
460 			 * acc_attr), undo that setup here.
461 			 */
462 			if (NPE_IS_HANDLE_FOR_STDCFG_ACC(mp->map_handlep)) {
463 
464 				if (DDI_FM_ACC_ERR_CAP(ddi_fm_capable(rdip)) &&
465 				    mp->map_handlep->ah_acc.devacc_attr_access
466 				    != DDI_DEFAULT_ACC) {
467 					ndi_fmc_remove(rdip, ACC_HANDLE,
468 					    (void *)mp->map_handlep);
469 				}
470 				return (DDI_SUCCESS);
471 			}
472 
473 			pci_rp->pci_size_low = PCIE_CONF_HDR_SIZE;
474 
475 			/* FALLTHROUGH */
476 		case PCI_ADDR_MEM64:
477 			/*
478 			 * MEM64 requires special treatment on map, to check
479 			 * that the device is below 4G.  On unmap, however,
480 			 * we can assume that everything is OK... the map
481 			 * must have succeeded.
482 			 */
483 			/* FALLTHROUGH */
484 		case PCI_ADDR_MEM32:
485 			reg.regspec_bustype = 0;
486 			break;
487 
488 		default:
489 			return (DDI_FAILURE);
490 		}
491 
492 		/*
493 		 * Adjust offset and length
494 		 * A non-zero length means override the one in the regspec.
495 		 */
496 		pci_rp->pci_phys_low += (uint_t)offset;
497 		if (len != 0)
498 			pci_rp->pci_size_low = len;
499 
500 		reg.regspec_addr = pci_rp->pci_phys_low;
501 		reg.regspec_size = pci_rp->pci_size_low;
502 
503 		mp->map_obj.rp = &reg;
504 		retval = ddi_map(dip, mp, (off_t)0, (off_t)0, vaddrp);
505 		if (DDI_FM_ACC_ERR_CAP(ddi_fm_capable(rdip)) &&
506 		    mp->map_handlep->ah_acc.devacc_attr_access !=
507 		    DDI_DEFAULT_ACC) {
508 			ndi_fmc_remove(rdip, ACC_HANDLE,
509 			    (void *)mp->map_handlep);
510 		}
511 		return (retval);
512 
513 	}
514 
515 	/* check for user mapping request - not legal for Config */
516 	if (mp->map_op == DDI_MO_MAP_HANDLE && space == PCI_ADDR_CONFIG) {
517 		cmn_err(CE_NOTE, "npe: Config mapping request from user\n");
518 		return (DDI_FAILURE);
519 	}
520 
521 
522 	/*
523 	 * Note that pci_fm_acc_setup() is called to serve two purposes
524 	 * i) enable legacy PCI I/O style config space access
525 	 * ii) register with FMA
526 	 */
527 	if (space == PCI_ADDR_CONFIG) {
528 
529 		/* Can't map config space without a handle */
530 		hp = (ddi_acc_hdl_t *)mp->map_handlep;
531 		if (hp == NULL)
532 			return (DDI_FAILURE);
533 
534 		/* record the device address for future reference */
535 		cfp = (pci_acc_cfblk_t *)&hp->ah_bus_private;
536 		cfp->c_busnum = PCI_REG_BUS_G(pci_rp->pci_phys_hi);
537 		cfp->c_devnum = PCI_REG_DEV_G(pci_rp->pci_phys_hi);
538 		cfp->c_funcnum = PCI_REG_FUNC_G(pci_rp->pci_phys_hi);
539 
540 		*vaddrp = (caddr_t)offset;
541 
542 		/* Check if MMCFG is supported */
543 		if (!npe_is_mmcfg_supported(rdip)) {
544 			return (npe_setup_std_pcicfg_acc(rdip, mp, hp,
545 			    offset, len));
546 		}
547 
548 
549 		if (ddi_prop_lookup_int64_array(DDI_DEV_T_ANY, rdip, 0,
550 		    "ecfg", &ecfginfo, &nelem) == DDI_PROP_SUCCESS) {
551 
552 			if (nelem != 4 ||
553 			    cfp->c_busnum < ecfginfo[2] ||
554 			    cfp->c_busnum > ecfginfo[3]) {
555 				/*
556 				 * Invalid property or Doesn't contain the
557 				 * requested bus; fall back to standard
558 				 * (I/O-based) config access.
559 				 */
560 				ddi_prop_free(ecfginfo);
561 				return (npe_setup_std_pcicfg_acc(rdip, mp, hp,
562 				    offset, len));
563 			} else {
564 				pci_rp->pci_phys_low = ecfginfo[0];
565 
566 				ddi_prop_free(ecfginfo);
567 
568 				pci_rp->pci_phys_low += ((cfp->c_busnum << 20) |
569 				    (cfp->c_devnum) << 15 |
570 				    (cfp->c_funcnum << 12));
571 
572 				pci_rp->pci_size_low = PCIE_CONF_HDR_SIZE;
573 			}
574 		} else {
575 			/*
576 			 * Couldn't find the MMCFG property -- fall back to
577 			 * standard config access
578 			 */
579 			return (npe_setup_std_pcicfg_acc(rdip, mp, hp,
580 			    offset, len));
581 		}
582 	}
583 
584 	length = pci_rp->pci_size_low;
585 
586 	/*
587 	 * range check
588 	 */
589 	if ((offset >= length) || (len > length) || (offset + len > length))
590 		return (DDI_FAILURE);
591 
592 	/*
593 	 * Adjust offset and length
594 	 * A non-zero length means override the one in the regspec.
595 	 */
596 	pci_rp->pci_phys_low += (uint_t)offset;
597 	if (len != 0)
598 		pci_rp->pci_size_low = len;
599 
600 	/*
601 	 * convert the pci regsec into the generic regspec used by the
602 	 * parent root nexus driver.
603 	 */
604 	switch (space) {
605 	case PCI_ADDR_IO:
606 		reg.regspec_bustype = 1;
607 		break;
608 	case PCI_ADDR_CONFIG:
609 	case PCI_ADDR_MEM64:
610 		/*
611 		 * We can't handle 64-bit devices that are mapped above
612 		 * 4G or that are larger than 4G.
613 		 */
614 		if (pci_rp->pci_phys_mid != 0 || pci_rp->pci_size_hi != 0)
615 			return (DDI_FAILURE);
616 		/*
617 		 * Other than that, we can treat them as 32-bit mappings
618 		 */
619 		/* FALLTHROUGH */
620 	case PCI_ADDR_MEM32:
621 		reg.regspec_bustype = 0;
622 		break;
623 	default:
624 		return (DDI_FAILURE);
625 	}
626 
627 	reg.regspec_addr = pci_rp->pci_phys_low;
628 	reg.regspec_size = pci_rp->pci_size_low;
629 
630 	mp->map_obj.rp = &reg;
631 	retval = ddi_map(dip, mp, (off_t)0, (off_t)0, vaddrp);
632 	if (retval == DDI_SUCCESS) {
633 		/*
634 		 * For config space gets force use of cautious access routines.
635 		 * These will handle default and protected mode accesses too.
636 		 */
637 		if (space == PCI_ADDR_CONFIG) {
638 			ap = (ddi_acc_impl_t *)mp->map_handlep;
639 			ap->ahi_acc_attr &= ~DDI_ACCATTR_DIRECT;
640 			ap->ahi_acc_attr |= DDI_ACCATTR_CONFIG_SPACE;
641 			ap->ahi_get8 = i_ddi_caut_get8;
642 			ap->ahi_get16 = i_ddi_caut_get16;
643 			ap->ahi_get32 = i_ddi_caut_get32;
644 			ap->ahi_get64 = i_ddi_caut_get64;
645 			ap->ahi_rep_get8 = i_ddi_caut_rep_get8;
646 			ap->ahi_rep_get16 = i_ddi_caut_rep_get16;
647 			ap->ahi_rep_get32 = i_ddi_caut_rep_get32;
648 			ap->ahi_rep_get64 = i_ddi_caut_rep_get64;
649 		}
650 		if (DDI_FM_ACC_ERR_CAP(ddi_fm_capable(rdip)) &&
651 		    mp->map_handlep->ah_acc.devacc_attr_access !=
652 		    DDI_DEFAULT_ACC) {
653 			ndi_fmc_insert(rdip, ACC_HANDLE,
654 			    (void *)mp->map_handlep, NULL);
655 		}
656 	}
657 	return (retval);
658 }
659 
660 
661 
662 /*ARGSUSED*/
663 static int
664 npe_ctlops(dev_info_t *dip, dev_info_t *rdip,
665 	ddi_ctl_enum_t ctlop, void *arg, void *result)
666 {
667 	int		rn;
668 	int		totreg;
669 	uint_t		reglen;
670 	pci_regspec_t	*drv_regp;
671 	struct attachspec *asp;
672 	struct detachspec *dsp;
673 	pci_state_t	*pci_p = ddi_get_soft_state(npe_statep,
674 	    ddi_get_instance(dip));
675 
676 	switch (ctlop) {
677 	case DDI_CTLOPS_REPORTDEV:
678 		if (rdip == (dev_info_t *)0)
679 			return (DDI_FAILURE);
680 		cmn_err(CE_CONT, "?PCI Express-device: %s@%s, %s%d\n",
681 		    ddi_node_name(rdip), ddi_get_name_addr(rdip),
682 		    ddi_driver_name(rdip), ddi_get_instance(rdip));
683 		return (DDI_SUCCESS);
684 
685 	case DDI_CTLOPS_INITCHILD:
686 		return (npe_initchild((dev_info_t *)arg));
687 
688 	case DDI_CTLOPS_UNINITCHILD:
689 		return (npe_removechild((dev_info_t *)arg));
690 
691 	case DDI_CTLOPS_SIDDEV:
692 		return (DDI_SUCCESS);
693 
694 	case DDI_CTLOPS_REGSIZE:
695 	case DDI_CTLOPS_NREGS:
696 		if (rdip == (dev_info_t *)0)
697 			return (DDI_FAILURE);
698 
699 		*(int *)result = 0;
700 		if (ddi_prop_lookup_int_array(DDI_DEV_T_ANY, rdip,
701 		    DDI_PROP_DONTPASS, "reg", (int **)&drv_regp,
702 		    &reglen) != DDI_PROP_SUCCESS) {
703 			return (DDI_FAILURE);
704 		}
705 
706 		totreg = (reglen * sizeof (int)) / sizeof (pci_regspec_t);
707 		if (ctlop == DDI_CTLOPS_NREGS)
708 			*(int *)result = totreg;
709 		else if (ctlop == DDI_CTLOPS_REGSIZE) {
710 			rn = *(int *)arg;
711 			if (rn >= totreg) {
712 				ddi_prop_free(drv_regp);
713 				return (DDI_FAILURE);
714 			}
715 			*(off_t *)result = drv_regp[rn].pci_size_low;
716 		}
717 		ddi_prop_free(drv_regp);
718 
719 		return (DDI_SUCCESS);
720 
721 	case DDI_CTLOPS_POWER:
722 	{
723 		power_req_t	*reqp = (power_req_t *)arg;
724 		/*
725 		 * We currently understand reporting of PCI_PM_IDLESPEED
726 		 * capability. Everything else is passed up.
727 		 */
728 		if ((reqp->request_type == PMR_REPORT_PMCAP) &&
729 		    (reqp->req.report_pmcap_req.cap ==  PCI_PM_IDLESPEED))
730 			return (DDI_SUCCESS);
731 
732 		break;
733 	}
734 
735 	case DDI_CTLOPS_PEEK:
736 	case DDI_CTLOPS_POKE:
737 		return (pci_common_peekpoke(dip, rdip, ctlop, arg, result));
738 
739 	/* X86 systems support PME wakeup from suspended state */
740 	case DDI_CTLOPS_ATTACH:
741 		if (!pcie_is_child(dip, rdip))
742 			return (DDI_SUCCESS);
743 
744 		asp = (struct attachspec *)arg;
745 		if ((asp->when == DDI_POST) && (asp->result == DDI_SUCCESS)) {
746 			pf_init(rdip, (void *)pci_p->pci_fm_ibc, asp->cmd);
747 			(void) pcie_postattach_child(rdip);
748 		}
749 
750 		/* only do this for immediate children */
751 		if (asp->cmd == DDI_RESUME && asp->when == DDI_PRE &&
752 		    ddi_get_parent(rdip) == dip)
753 			if (pci_pre_resume(rdip) != DDI_SUCCESS) {
754 				/* Not good, better stop now. */
755 				cmn_err(CE_PANIC,
756 				    "Couldn't pre-resume device %p",
757 				    (void *) dip);
758 				/* NOTREACHED */
759 			}
760 
761 		return (DDI_SUCCESS);
762 
763 	case DDI_CTLOPS_DETACH:
764 		if (!pcie_is_child(dip, rdip))
765 			return (DDI_SUCCESS);
766 
767 		dsp = (struct detachspec *)arg;
768 
769 		if (dsp->when == DDI_PRE)
770 			pf_fini(rdip, dsp->cmd);
771 
772 		/* only do this for immediate children */
773 		if (dsp->cmd == DDI_SUSPEND && dsp->when == DDI_POST &&
774 		    ddi_get_parent(rdip) == dip)
775 			if (pci_post_suspend(rdip) != DDI_SUCCESS)
776 				return (DDI_FAILURE);
777 
778 		return (DDI_SUCCESS);
779 
780 	default:
781 		break;
782 	}
783 
784 	return (ddi_ctlops(dip, rdip, ctlop, arg, result));
785 
786 }
787 
788 
789 /*
790  * npe_intr_ops
791  */
792 static int
793 npe_intr_ops(dev_info_t *pdip, dev_info_t *rdip, ddi_intr_op_t intr_op,
794     ddi_intr_handle_impl_t *hdlp, void *result)
795 {
796 	return (pci_common_intr_ops(pdip, rdip, intr_op, hdlp, result));
797 }
798 
799 
800 static int
801 npe_initchild(dev_info_t *child)
802 {
803 	char		name[80];
804 	pcie_bus_t	*bus_p;
805 	uint32_t	regs;
806 	ddi_acc_handle_t	cfg_hdl;
807 
808 	/*
809 	 * Do not bind drivers to empty bridges.
810 	 * Fail above, if the bridge is found to be hotplug capable
811 	 */
812 	if (npe_disable_empty_bridges_workaround(child) == 1)
813 		return (DDI_FAILURE);
814 
815 	if (pci_common_name_child(child, name, 80) != DDI_SUCCESS)
816 		return (DDI_FAILURE);
817 
818 	ddi_set_name_addr(child, name);
819 
820 	/*
821 	 * Pseudo nodes indicate a prototype node with per-instance
822 	 * properties to be merged into the real h/w device node.
823 	 * The interpretation of the unit-address is DD[,F]
824 	 * where DD is the device id and F is the function.
825 	 */
826 	if (ndi_dev_is_persistent_node(child) == 0) {
827 		extern int pci_allow_pseudo_children;
828 
829 		ddi_set_parent_data(child, NULL);
830 
831 		/*
832 		 * Try to merge the properties from this prototype
833 		 * node into real h/w nodes.
834 		 */
835 		if (ndi_merge_node(child, pci_common_name_child) ==
836 		    DDI_SUCCESS) {
837 			/*
838 			 * Merged ok - return failure to remove the node.
839 			 */
840 			ddi_set_name_addr(child, NULL);
841 			return (DDI_FAILURE);
842 		}
843 
844 		/* workaround for DDIVS to run under PCI Express */
845 		if (pci_allow_pseudo_children) {
846 			/*
847 			 * If the "interrupts" property doesn't exist,
848 			 * this must be the ddivs no-intr case, and it returns
849 			 * DDI_SUCCESS instead of DDI_FAILURE.
850 			 */
851 			if (ddi_prop_get_int(DDI_DEV_T_ANY, child,
852 			    DDI_PROP_DONTPASS, "interrupts", -1) == -1)
853 				return (DDI_SUCCESS);
854 			/*
855 			 * Create the ddi_parent_private_data for a pseudo
856 			 * child.
857 			 */
858 			pci_common_set_parent_private_data(child);
859 			return (DDI_SUCCESS);
860 		}
861 
862 		/*
863 		 * The child was not merged into a h/w node,
864 		 * but there's not much we can do with it other
865 		 * than return failure to cause the node to be removed.
866 		 */
867 		cmn_err(CE_WARN, "!%s@%s: %s.conf properties not merged",
868 		    ddi_get_name(child), ddi_get_name_addr(child),
869 		    ddi_get_name(child));
870 		ddi_set_name_addr(child, NULL);
871 		return (DDI_NOT_WELL_FORMED);
872 	}
873 
874 	if (ddi_prop_get_int(DDI_DEV_T_ANY, child, DDI_PROP_DONTPASS,
875 	    "interrupts", -1) != -1)
876 		pci_common_set_parent_private_data(child);
877 	else
878 		ddi_set_parent_data(child, NULL);
879 
880 	/* Disable certain errors on PCIe drivers for x86 platforms */
881 	regs = pcie_get_aer_uce_mask() | npe_aer_uce_mask;
882 	pcie_set_aer_uce_mask(regs);
883 	regs = pcie_get_aer_ce_mask() | npe_aer_ce_mask;
884 	pcie_set_aer_ce_mask(regs);
885 	regs = pcie_get_aer_suce_mask() | npe_aer_suce_mask;
886 	pcie_set_aer_suce_mask(regs);
887 
888 	/*
889 	 * If URs are disabled, mask SERRs as well, otherwise the system will
890 	 * still be notified of URs
891 	 */
892 	if (npe_aer_uce_mask & PCIE_AER_UCE_UR)
893 		pcie_set_serr_mask(1);
894 
895 	if (pci_config_setup(child, &cfg_hdl) == DDI_SUCCESS) {
896 		npe_ck804_fix_aer_ptr(cfg_hdl);
897 		npe_nvidia_error_mask(cfg_hdl);
898 		npe_intel_error_mask(cfg_hdl);
899 		pci_config_teardown(&cfg_hdl);
900 	}
901 
902 	bus_p = pcie_init_bus(child);
903 	if (bus_p) {
904 		uint16_t device_id = (uint16_t)(bus_p->bus_dev_ven_id >> 16);
905 		uint16_t vendor_id = (uint16_t)(bus_p->bus_dev_ven_id & 0xFFFF);
906 		uint16_t rev_id = bus_p->bus_rev_id;
907 
908 		/* Disable AER for certain NVIDIA Chipsets */
909 		if ((vendor_id == NVIDIA_VENDOR_ID) &&
910 		    (device_id == NVIDIA_CK804_DEVICE_ID) &&
911 		    (rev_id < NVIDIA_CK804_AER_VALID_REVID))
912 			bus_p->bus_aer_off = 0;
913 
914 		(void) pcie_initchild(child);
915 	}
916 
917 	return (DDI_SUCCESS);
918 }
919 
920 
921 static int
922 npe_removechild(dev_info_t *dip)
923 {
924 	pcie_uninitchild(dip);
925 
926 	ddi_set_name_addr(dip, NULL);
927 
928 	/*
929 	 * Strip the node to properly convert it back to prototype form
930 	 */
931 	ddi_remove_minor_node(dip, NULL);
932 
933 	ddi_prop_remove_all(dip);
934 
935 	return (DDI_SUCCESS);
936 }
937 
938 
939 /*
940  * When retrofitting this module for pci_tools, functions such as open, close,
941  * and ioctl are now pulled into this module.  Before this, the functions in
942  * the pcihp module were referenced directly.  Now they are called or
943  * referenced through the pcihp cb_ops structure from functions in this module.
944  */
945 static int
946 npe_open(dev_t *devp, int flags, int otyp, cred_t *credp)
947 {
948 	return ((pcihp_get_cb_ops())->cb_open(devp, flags, otyp, credp));
949 }
950 
951 static int
952 npe_close(dev_t dev, int flags, int otyp, cred_t *credp)
953 {
954 	return ((pcihp_get_cb_ops())->cb_close(dev, flags, otyp, credp));
955 }
956 
957 static int
958 npe_ioctl(dev_t dev, int cmd, intptr_t arg, int mode, cred_t *credp, int *rvalp)
959 {
960 	minor_t		minor = getminor(dev);
961 	int		instance = PCIHP_AP_MINOR_NUM_TO_INSTANCE(minor);
962 	pci_state_t	*pci_p = ddi_get_soft_state(npe_statep, instance);
963 	dev_info_t	*dip;
964 
965 	if (pci_p == NULL)
966 		return (ENXIO);
967 
968 	dip = pci_p->pci_dip;
969 
970 	return (pci_common_ioctl(dip, dev, cmd, arg, mode, credp, rvalp));
971 }
972 
973 static int
974 npe_prop_op(dev_t dev, dev_info_t *dip, ddi_prop_op_t prop_op,
975 	int flags, char *name, caddr_t valuep, int *lengthp)
976 {
977 	return ((pcihp_get_cb_ops())->cb_prop_op(dev, dip, prop_op, flags,
978 	    name, valuep, lengthp));
979 }
980 
981 static int
982 npe_info(dev_info_t *dip, ddi_info_cmd_t cmd, void *arg, void **result)
983 {
984 	return (pcihp_info(dip, cmd, arg, result));
985 }
986 
987 /*ARGSUSED*/
988 static int
989 npe_fm_init(dev_info_t *dip, dev_info_t *tdip, int cap,
990     ddi_iblock_cookie_t *ibc)
991 {
992 	pci_state_t  *pcip = ddi_get_soft_state(npe_statep,
993 	    ddi_get_instance(dip));
994 
995 	ASSERT(ibc != NULL);
996 	*ibc = pcip->pci_fm_ibc;
997 
998 	return (pcip->pci_fmcap);
999 }
1000 
1001 /*ARGSUSED*/
1002 static int
1003 npe_fm_callback(dev_info_t *dip, ddi_fm_error_t *derr, const void *no_used)
1004 {
1005 	/*
1006 	 * On current x86 systems, npe's callback does not get called for failed
1007 	 * loads.  If in the future this feature is used, the fault PA should be
1008 	 * logged in the derr->fme_bus_specific field.  The appropriate PCIe
1009 	 * error handling code should be called and needs to be coordinated with
1010 	 * safe access handling.
1011 	 */
1012 
1013 	return (DDI_FM_OK);
1014 }
1015