xref: /titanic_52/usr/src/uts/sun4u/io/px/px_lib4u.c (revision 88df2d76721d60b8b7cad14f9380446d06569f7c)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 /*
22  * Copyright 2006 Sun Microsystems, Inc.  All rights reserved.
23  * Use is subject to license terms.
24  */
25 
26 #pragma ident	"%Z%%M%	%I%	%E% SMI"
27 
28 #include <sys/types.h>
29 #include <sys/kmem.h>
30 #include <sys/conf.h>
31 #include <sys/ddi.h>
32 #include <sys/sunddi.h>
33 #include <sys/fm/protocol.h>
34 #include <sys/fm/util.h>
35 #include <sys/modctl.h>
36 #include <sys/disp.h>
37 #include <sys/stat.h>
38 #include <sys/ddi_impldefs.h>
39 #include <sys/vmem.h>
40 #include <sys/iommutsb.h>
41 #include <sys/cpuvar.h>
42 #include <sys/ivintr.h>
43 #include <sys/byteorder.h>
44 #include <sys/hotplug/pci/pciehpc.h>
45 #include <px_obj.h>
46 #include <pcie_pwr.h>
47 #include "px_tools_var.h"
48 #include <px_regs.h>
49 #include <px_csr.h>
50 #include <sys/machsystm.h>
51 #include "px_lib4u.h"
52 #include "px_err.h"
53 #include "oberon_regs.h"
54 
55 #pragma weak jbus_stst_order
56 
57 extern void jbus_stst_order();
58 
59 ulong_t px_mmu_dvma_end = 0xfffffffful;
60 uint_t px_ranges_phi_mask = 0xfffffffful;
61 uint64_t *px_oberon_ubc_scratch_regs;
62 
63 static int px_goto_l23ready(px_t *px_p);
64 static int px_goto_l0(px_t *px_p);
65 static int px_pre_pwron_check(px_t *px_p);
66 static uint32_t px_identity_chip(px_t *px_p);
67 static boolean_t px_cpr_callb(void *arg, int code);
68 static uint_t px_cb_intr(caddr_t arg);
69 
70 /*
71  * px_lib_map_registers
72  *
73  * This function is called from the attach routine to map the registers
74  * accessed by this driver.
75  *
76  * used by: px_attach()
77  *
78  * return value: DDI_FAILURE on failure
79  */
80 int
81 px_lib_map_regs(pxu_t *pxu_p, dev_info_t *dip)
82 {
83 	ddi_device_acc_attr_t	attr;
84 	px_reg_bank_t		reg_bank = PX_REG_CSR;
85 
86 	DBG(DBG_ATTACH, dip, "px_lib_map_regs: pxu_p:0x%p, dip 0x%p\n",
87 		pxu_p, dip);
88 
89 	attr.devacc_attr_version = DDI_DEVICE_ATTR_V0;
90 	attr.devacc_attr_dataorder = DDI_STRICTORDER_ACC;
91 	attr.devacc_attr_endian_flags = DDI_NEVERSWAP_ACC;
92 
93 	/*
94 	 * PCI CSR Base
95 	 */
96 	if (ddi_regs_map_setup(dip, reg_bank, &pxu_p->px_address[reg_bank],
97 	    0, 0, &attr, &pxu_p->px_ac[reg_bank]) != DDI_SUCCESS) {
98 		goto fail;
99 	}
100 
101 	reg_bank++;
102 
103 	/*
104 	 * XBUS CSR Base
105 	 */
106 	if (ddi_regs_map_setup(dip, reg_bank, &pxu_p->px_address[reg_bank],
107 	    0, 0, &attr, &pxu_p->px_ac[reg_bank]) != DDI_SUCCESS) {
108 		goto fail;
109 	}
110 
111 	pxu_p->px_address[reg_bank] -= FIRE_CONTROL_STATUS;
112 
113 done:
114 	for (; reg_bank >= PX_REG_CSR; reg_bank--) {
115 		DBG(DBG_ATTACH, dip, "reg_bank 0x%x address 0x%p\n",
116 		    reg_bank, pxu_p->px_address[reg_bank]);
117 	}
118 
119 	return (DDI_SUCCESS);
120 
121 fail:
122 	cmn_err(CE_WARN, "%s%d: unable to map reg entry %d\n",
123 	    ddi_driver_name(dip), ddi_get_instance(dip), reg_bank);
124 
125 	for (reg_bank--; reg_bank >= PX_REG_CSR; reg_bank--) {
126 		pxu_p->px_address[reg_bank] = NULL;
127 		ddi_regs_map_free(&pxu_p->px_ac[reg_bank]);
128 	}
129 
130 	return (DDI_FAILURE);
131 }
132 
133 /*
134  * px_lib_unmap_regs:
135  *
136  * This routine unmaps the registers mapped by map_px_registers.
137  *
138  * used by: px_detach(), and error conditions in px_attach()
139  *
140  * return value: none
141  */
142 void
143 px_lib_unmap_regs(pxu_t *pxu_p)
144 {
145 	int i;
146 
147 	for (i = 0; i < PX_REG_MAX; i++) {
148 		if (pxu_p->px_ac[i])
149 			ddi_regs_map_free(&pxu_p->px_ac[i]);
150 	}
151 }
152 
153 int
154 px_lib_dev_init(dev_info_t *dip, devhandle_t *dev_hdl)
155 {
156 	px_t		*px_p = DIP_TO_STATE(dip);
157 	caddr_t		xbc_csr_base, csr_base;
158 	px_dvma_range_prop_t	px_dvma_range;
159 	uint32_t	chip_id;
160 	pxu_t		*pxu_p;
161 
162 	DBG(DBG_ATTACH, dip, "px_lib_dev_init: dip 0x%p\n", dip);
163 
164 	if ((chip_id = px_identity_chip(px_p)) == PX_CHIP_UNIDENTIFIED)
165 		return (DDI_FAILURE);
166 
167 	switch (chip_id) {
168 	case FIRE_VER_10:
169 		cmn_err(CE_WARN, "FIRE Hardware Version 1.0 is not supported");
170 		return (DDI_FAILURE);
171 	case FIRE_VER_20:
172 		DBG(DBG_ATTACH, dip, "FIRE Hardware Version 2.0\n");
173 		break;
174 	case OBERON_VER_10:
175 		DBG(DBG_ATTACH, dip, "Oberon Hardware Version 1.0\n");
176 		break;
177 	default:
178 		cmn_err(CE_WARN, "%s%d: PX Hardware Version Unknown\n",
179 		    ddi_driver_name(dip), ddi_get_instance(dip));
180 		return (DDI_FAILURE);
181 	}
182 
183 	/*
184 	 * Allocate platform specific structure and link it to
185 	 * the px state structure.
186 	 */
187 	pxu_p = kmem_zalloc(sizeof (pxu_t), KM_SLEEP);
188 	pxu_p->chip_id = chip_id;
189 	pxu_p->portid  = ddi_getprop(DDI_DEV_T_ANY, dip, DDI_PROP_DONTPASS,
190 	    "portid", -1);
191 
192 	/* Map in the registers */
193 	if (px_lib_map_regs(pxu_p, dip) == DDI_FAILURE) {
194 		kmem_free(pxu_p, sizeof (pxu_t));
195 
196 		return (DDI_FAILURE);
197 	}
198 
199 	xbc_csr_base = (caddr_t)pxu_p->px_address[PX_REG_XBC];
200 	csr_base = (caddr_t)pxu_p->px_address[PX_REG_CSR];
201 
202 	pxu_p->tsb_cookie = iommu_tsb_alloc(pxu_p->portid);
203 	pxu_p->tsb_size = iommu_tsb_cookie_to_size(pxu_p->tsb_cookie);
204 	pxu_p->tsb_vaddr = iommu_tsb_cookie_to_va(pxu_p->tsb_cookie);
205 
206 	pxu_p->tsb_paddr = va_to_pa(pxu_p->tsb_vaddr);
207 
208 	/*
209 	 * Create "virtual-dma" property to support child devices
210 	 * needing to know DVMA range.
211 	 */
212 	px_dvma_range.dvma_base = (uint32_t)px_mmu_dvma_end + 1
213 	    - ((pxu_p->tsb_size >> 3) << MMU_PAGE_SHIFT);
214 	px_dvma_range.dvma_len = (uint32_t)
215 	    px_mmu_dvma_end - px_dvma_range.dvma_base + 1;
216 
217 	(void) ddi_prop_create(DDI_DEV_T_NONE, dip, DDI_PROP_CANSLEEP,
218 		"virtual-dma", (caddr_t)&px_dvma_range,
219 		sizeof (px_dvma_range_prop_t));
220 	/*
221 	 * Initilize all fire hardware specific blocks.
222 	 */
223 	hvio_cb_init(xbc_csr_base, pxu_p);
224 	hvio_ib_init(csr_base, pxu_p);
225 	hvio_pec_init(csr_base, pxu_p);
226 	hvio_mmu_init(csr_base, pxu_p);
227 
228 	px_p->px_plat_p = (void *)pxu_p;
229 
230 	/*
231 	 * Initialize all the interrupt handlers
232 	 */
233 	switch (PX_CHIP_TYPE(pxu_p)) {
234 	case PX_CHIP_OBERON:
235 		px_err_reg_enable(px_p, PX_ERR_UBC);
236 		px_err_reg_enable(px_p, PX_ERR_MMU);
237 		px_err_reg_enable(px_p, PX_ERR_IMU);
238 		px_err_reg_enable(px_p, PX_ERR_TLU_UE);
239 		px_err_reg_enable(px_p, PX_ERR_TLU_CE);
240 		px_err_reg_enable(px_p, PX_ERR_TLU_OE);
241 		px_err_reg_enable(px_p, PX_ERR_ILU);
242 
243 		px_fabric_die_rc_ue |= PCIE_AER_UCE_UC;
244 		break;
245 
246 	case PX_CHIP_FIRE:
247 		px_err_reg_enable(px_p, PX_ERR_JBC);
248 		px_err_reg_enable(px_p, PX_ERR_MMU);
249 		px_err_reg_enable(px_p, PX_ERR_IMU);
250 		px_err_reg_enable(px_p, PX_ERR_TLU_UE);
251 		px_err_reg_enable(px_p, PX_ERR_TLU_CE);
252 		px_err_reg_enable(px_p, PX_ERR_TLU_OE);
253 		px_err_reg_enable(px_p, PX_ERR_ILU);
254 		px_err_reg_enable(px_p, PX_ERR_LPU_LINK);
255 		px_err_reg_enable(px_p, PX_ERR_LPU_PHY);
256 		px_err_reg_enable(px_p, PX_ERR_LPU_RX);
257 		px_err_reg_enable(px_p, PX_ERR_LPU_TX);
258 		px_err_reg_enable(px_p, PX_ERR_LPU_LTSSM);
259 		px_err_reg_enable(px_p, PX_ERR_LPU_GIGABLZ);
260 		break;
261 	default:
262 		cmn_err(CE_WARN, "%s%d: PX primary bus Unknown\n",
263 		    ddi_driver_name(dip), ddi_get_instance(dip));
264 		return (DDI_FAILURE);
265 	}
266 
267 	/* Initilize device handle */
268 	*dev_hdl = (devhandle_t)csr_base;
269 
270 	DBG(DBG_ATTACH, dip, "px_lib_dev_init: dev_hdl 0x%llx\n", *dev_hdl);
271 
272 	return (DDI_SUCCESS);
273 }
274 
275 int
276 px_lib_dev_fini(dev_info_t *dip)
277 {
278 	px_t	*px_p = DIP_TO_STATE(dip);
279 	pxu_t	*pxu_p = (pxu_t *)px_p->px_plat_p;
280 
281 	DBG(DBG_DETACH, dip, "px_lib_dev_fini: dip 0x%p\n", dip);
282 
283 	/*
284 	 * Deinitialize all the interrupt handlers
285 	 */
286 	switch (PX_CHIP_TYPE(pxu_p)) {
287 	case PX_CHIP_OBERON:
288 		px_err_reg_disable(px_p, PX_ERR_UBC);
289 		px_err_reg_disable(px_p, PX_ERR_MMU);
290 		px_err_reg_disable(px_p, PX_ERR_IMU);
291 		px_err_reg_disable(px_p, PX_ERR_TLU_UE);
292 		px_err_reg_disable(px_p, PX_ERR_TLU_CE);
293 		px_err_reg_disable(px_p, PX_ERR_TLU_OE);
294 		px_err_reg_disable(px_p, PX_ERR_ILU);
295 		break;
296 	case PX_CHIP_FIRE:
297 		px_err_reg_disable(px_p, PX_ERR_JBC);
298 		px_err_reg_disable(px_p, PX_ERR_MMU);
299 		px_err_reg_disable(px_p, PX_ERR_IMU);
300 		px_err_reg_disable(px_p, PX_ERR_TLU_UE);
301 		px_err_reg_disable(px_p, PX_ERR_TLU_CE);
302 		px_err_reg_disable(px_p, PX_ERR_TLU_OE);
303 		px_err_reg_disable(px_p, PX_ERR_ILU);
304 		px_err_reg_disable(px_p, PX_ERR_LPU_LINK);
305 		px_err_reg_disable(px_p, PX_ERR_LPU_PHY);
306 		px_err_reg_disable(px_p, PX_ERR_LPU_RX);
307 		px_err_reg_disable(px_p, PX_ERR_LPU_TX);
308 		px_err_reg_disable(px_p, PX_ERR_LPU_LTSSM);
309 		px_err_reg_disable(px_p, PX_ERR_LPU_GIGABLZ);
310 		break;
311 	default:
312 		cmn_err(CE_WARN, "%s%d: PX primary bus Unknown\n",
313 		    ddi_driver_name(dip), ddi_get_instance(dip));
314 		return (DDI_FAILURE);
315 	}
316 
317 	iommu_tsb_free(pxu_p->tsb_cookie);
318 
319 	px_lib_unmap_regs((pxu_t *)px_p->px_plat_p);
320 	kmem_free(px_p->px_plat_p, sizeof (pxu_t));
321 	px_p->px_plat_p = NULL;
322 
323 	return (DDI_SUCCESS);
324 }
325 
326 /*ARGSUSED*/
327 int
328 px_lib_intr_devino_to_sysino(dev_info_t *dip, devino_t devino,
329     sysino_t *sysino)
330 {
331 	px_t	*px_p = DIP_TO_STATE(dip);
332 	pxu_t	*pxu_p = (pxu_t *)px_p->px_plat_p;
333 	uint64_t	ret;
334 
335 	DBG(DBG_LIB_INT, dip, "px_lib_intr_devino_to_sysino: dip 0x%p "
336 	    "devino 0x%x\n", dip, devino);
337 
338 	if ((ret = hvio_intr_devino_to_sysino(DIP_TO_HANDLE(dip),
339 	    pxu_p, devino, sysino)) != H_EOK) {
340 		DBG(DBG_LIB_INT, dip,
341 		    "hvio_intr_devino_to_sysino failed, ret 0x%lx\n", ret);
342 		return (DDI_FAILURE);
343 	}
344 
345 	DBG(DBG_LIB_INT, dip, "px_lib_intr_devino_to_sysino: sysino 0x%llx\n",
346 	    *sysino);
347 
348 	return (DDI_SUCCESS);
349 }
350 
351 /*ARGSUSED*/
352 int
353 px_lib_intr_getvalid(dev_info_t *dip, sysino_t sysino,
354     intr_valid_state_t *intr_valid_state)
355 {
356 	uint64_t	ret;
357 
358 	DBG(DBG_LIB_INT, dip, "px_lib_intr_getvalid: dip 0x%p sysino 0x%llx\n",
359 	    dip, sysino);
360 
361 	if ((ret = hvio_intr_getvalid(DIP_TO_HANDLE(dip),
362 	    sysino, intr_valid_state)) != H_EOK) {
363 		DBG(DBG_LIB_INT, dip, "hvio_intr_getvalid failed, ret 0x%lx\n",
364 		    ret);
365 		return (DDI_FAILURE);
366 	}
367 
368 	DBG(DBG_LIB_INT, dip, "px_lib_intr_getvalid: intr_valid_state 0x%x\n",
369 	    *intr_valid_state);
370 
371 	return (DDI_SUCCESS);
372 }
373 
374 /*ARGSUSED*/
375 int
376 px_lib_intr_setvalid(dev_info_t *dip, sysino_t sysino,
377     intr_valid_state_t intr_valid_state)
378 {
379 	uint64_t	ret;
380 
381 	DBG(DBG_LIB_INT, dip, "px_lib_intr_setvalid: dip 0x%p sysino 0x%llx "
382 	    "intr_valid_state 0x%x\n", dip, sysino, intr_valid_state);
383 
384 	if ((ret = hvio_intr_setvalid(DIP_TO_HANDLE(dip),
385 	    sysino, intr_valid_state)) != H_EOK) {
386 		DBG(DBG_LIB_INT, dip, "hvio_intr_setvalid failed, ret 0x%lx\n",
387 		    ret);
388 		return (DDI_FAILURE);
389 	}
390 
391 	return (DDI_SUCCESS);
392 }
393 
394 /*ARGSUSED*/
395 int
396 px_lib_intr_getstate(dev_info_t *dip, sysino_t sysino,
397     intr_state_t *intr_state)
398 {
399 	uint64_t	ret;
400 
401 	DBG(DBG_LIB_INT, dip, "px_lib_intr_getstate: dip 0x%p sysino 0x%llx\n",
402 	    dip, sysino);
403 
404 	if ((ret = hvio_intr_getstate(DIP_TO_HANDLE(dip),
405 	    sysino, intr_state)) != H_EOK) {
406 		DBG(DBG_LIB_INT, dip, "hvio_intr_getstate failed, ret 0x%lx\n",
407 		    ret);
408 		return (DDI_FAILURE);
409 	}
410 
411 	DBG(DBG_LIB_INT, dip, "px_lib_intr_getstate: intr_state 0x%x\n",
412 	    *intr_state);
413 
414 	return (DDI_SUCCESS);
415 }
416 
417 /*ARGSUSED*/
418 int
419 px_lib_intr_setstate(dev_info_t *dip, sysino_t sysino,
420     intr_state_t intr_state)
421 {
422 	uint64_t	ret;
423 
424 	DBG(DBG_LIB_INT, dip, "px_lib_intr_setstate: dip 0x%p sysino 0x%llx "
425 	    "intr_state 0x%x\n", dip, sysino, intr_state);
426 
427 	if ((ret = hvio_intr_setstate(DIP_TO_HANDLE(dip),
428 	    sysino, intr_state)) != H_EOK) {
429 		DBG(DBG_LIB_INT, dip, "hvio_intr_setstate failed, ret 0x%lx\n",
430 		    ret);
431 		return (DDI_FAILURE);
432 	}
433 
434 	return (DDI_SUCCESS);
435 }
436 
437 /*ARGSUSED*/
438 int
439 px_lib_intr_gettarget(dev_info_t *dip, sysino_t sysino, cpuid_t *cpuid)
440 {
441 	px_t		*px_p = DIP_TO_STATE(dip);
442 	pxu_t		*pxu_p = (pxu_t *)px_p->px_plat_p;
443 	uint64_t	ret;
444 
445 	DBG(DBG_LIB_INT, dip, "px_lib_intr_gettarget: dip 0x%p sysino 0x%llx\n",
446 	    dip, sysino);
447 
448 	if ((ret = hvio_intr_gettarget(DIP_TO_HANDLE(dip), pxu_p,
449 	    sysino, cpuid)) != H_EOK) {
450 		DBG(DBG_LIB_INT, dip, "hvio_intr_gettarget failed, ret 0x%lx\n",
451 		    ret);
452 		return (DDI_FAILURE);
453 	}
454 
455 	DBG(DBG_LIB_INT, dip, "px_lib_intr_gettarget: cpuid 0x%x\n", cpuid);
456 
457 	return (DDI_SUCCESS);
458 }
459 
460 /*ARGSUSED*/
461 int
462 px_lib_intr_settarget(dev_info_t *dip, sysino_t sysino, cpuid_t cpuid)
463 {
464 	px_t		*px_p = DIP_TO_STATE(dip);
465 	pxu_t		*pxu_p = (pxu_t *)px_p->px_plat_p;
466 	uint64_t	ret;
467 
468 	DBG(DBG_LIB_INT, dip, "px_lib_intr_settarget: dip 0x%p sysino 0x%llx "
469 	    "cpuid 0x%x\n", dip, sysino, cpuid);
470 
471 	if ((ret = hvio_intr_settarget(DIP_TO_HANDLE(dip), pxu_p,
472 	    sysino, cpuid)) != H_EOK) {
473 		DBG(DBG_LIB_INT, dip, "hvio_intr_settarget failed, ret 0x%lx\n",
474 		    ret);
475 		return (DDI_FAILURE);
476 	}
477 
478 	return (DDI_SUCCESS);
479 }
480 
481 /*ARGSUSED*/
482 int
483 px_lib_intr_reset(dev_info_t *dip)
484 {
485 	devino_t	ino;
486 	sysino_t	sysino;
487 
488 	DBG(DBG_LIB_INT, dip, "px_lib_intr_reset: dip 0x%p\n", dip);
489 
490 	/* Reset all Interrupts */
491 	for (ino = 0; ino < INTERRUPT_MAPPING_ENTRIES; ino++) {
492 		if (px_lib_intr_devino_to_sysino(dip, ino,
493 		    &sysino) != DDI_SUCCESS)
494 			return (BF_FATAL);
495 
496 		if (px_lib_intr_setstate(dip, sysino,
497 		    INTR_IDLE_STATE) != DDI_SUCCESS)
498 			return (BF_FATAL);
499 	}
500 
501 	return (BF_NONE);
502 }
503 
504 /*ARGSUSED*/
505 int
506 px_lib_iommu_map(dev_info_t *dip, tsbid_t tsbid, pages_t pages,
507     io_attributes_t attr, void *addr, size_t pfn_index, int flags)
508 {
509 	px_t		*px_p = DIP_TO_STATE(dip);
510 	pxu_t		*pxu_p = (pxu_t *)px_p->px_plat_p;
511 	uint64_t	ret;
512 
513 	DBG(DBG_LIB_DMA, dip, "px_lib_iommu_map: dip 0x%p tsbid 0x%llx "
514 	    "pages 0x%x attr 0x%x addr 0x%p pfn_index 0x%llx flags 0x%x\n",
515 	    dip, tsbid, pages, attr, addr, pfn_index, flags);
516 
517 	if ((ret = hvio_iommu_map(px_p->px_dev_hdl, pxu_p, tsbid, pages,
518 	    attr, addr, pfn_index, flags)) != H_EOK) {
519 		DBG(DBG_LIB_DMA, dip,
520 		    "px_lib_iommu_map failed, ret 0x%lx\n", ret);
521 		return (DDI_FAILURE);
522 	}
523 
524 	return (DDI_SUCCESS);
525 }
526 
527 /*ARGSUSED*/
528 int
529 px_lib_iommu_demap(dev_info_t *dip, tsbid_t tsbid, pages_t pages)
530 {
531 	px_t		*px_p = DIP_TO_STATE(dip);
532 	pxu_t		*pxu_p = (pxu_t *)px_p->px_plat_p;
533 	uint64_t	ret;
534 
535 	DBG(DBG_LIB_DMA, dip, "px_lib_iommu_demap: dip 0x%p tsbid 0x%llx "
536 	    "pages 0x%x\n", dip, tsbid, pages);
537 
538 	if ((ret = hvio_iommu_demap(px_p->px_dev_hdl, pxu_p, tsbid, pages))
539 	    != H_EOK) {
540 		DBG(DBG_LIB_DMA, dip,
541 		    "px_lib_iommu_demap failed, ret 0x%lx\n", ret);
542 
543 		return (DDI_FAILURE);
544 	}
545 
546 	return (DDI_SUCCESS);
547 }
548 
549 /*ARGSUSED*/
550 int
551 px_lib_iommu_getmap(dev_info_t *dip, tsbid_t tsbid, io_attributes_t *attr_p,
552     r_addr_t *r_addr_p)
553 {
554 	px_t	*px_p = DIP_TO_STATE(dip);
555 	pxu_t	*pxu_p = (pxu_t *)px_p->px_plat_p;
556 	uint64_t	ret;
557 
558 	DBG(DBG_LIB_DMA, dip, "px_lib_iommu_getmap: dip 0x%p tsbid 0x%llx\n",
559 	    dip, tsbid);
560 
561 	if ((ret = hvio_iommu_getmap(DIP_TO_HANDLE(dip), pxu_p, tsbid,
562 	    attr_p, r_addr_p)) != H_EOK) {
563 		DBG(DBG_LIB_DMA, dip,
564 		    "hvio_iommu_getmap failed, ret 0x%lx\n", ret);
565 
566 		return ((ret == H_ENOMAP) ? DDI_DMA_NOMAPPING:DDI_FAILURE);
567 	}
568 
569 	DBG(DBG_LIB_DMA, dip, "px_lib_iommu_getmap: attr 0x%x r_addr 0x%llx\n",
570 	    *attr_p, *r_addr_p);
571 
572 	return (DDI_SUCCESS);
573 }
574 
575 
576 /*
577  * Checks dma attributes against system bypass ranges
578  * The bypass range is determined by the hardware. Return them so the
579  * common code can do generic checking against them.
580  */
581 /*ARGSUSED*/
582 int
583 px_lib_dma_bypass_rngchk(dev_info_t *dip, ddi_dma_attr_t *attr_p,
584     uint64_t *lo_p, uint64_t *hi_p)
585 {
586 	px_t	*px_p = DIP_TO_STATE(dip);
587 	pxu_t	*pxu_p = (pxu_t *)px_p->px_plat_p;
588 
589 	*lo_p = hvio_get_bypass_base(pxu_p);
590 	*hi_p = hvio_get_bypass_end(pxu_p);
591 
592 	return (DDI_SUCCESS);
593 }
594 
595 
596 /*ARGSUSED*/
597 int
598 px_lib_iommu_getbypass(dev_info_t *dip, r_addr_t ra, io_attributes_t attr,
599     io_addr_t *io_addr_p)
600 {
601 	uint64_t	ret;
602 	px_t	*px_p = DIP_TO_STATE(dip);
603 	pxu_t	*pxu_p = (pxu_t *)px_p->px_plat_p;
604 
605 	DBG(DBG_LIB_DMA, dip, "px_lib_iommu_getbypass: dip 0x%p ra 0x%llx "
606 	    "attr 0x%x\n", dip, ra, attr);
607 
608 	if ((ret = hvio_iommu_getbypass(DIP_TO_HANDLE(dip), pxu_p, ra,
609 	    attr, io_addr_p)) != H_EOK) {
610 		DBG(DBG_LIB_DMA, dip,
611 		    "hvio_iommu_getbypass failed, ret 0x%lx\n", ret);
612 		return (DDI_FAILURE);
613 	}
614 
615 	DBG(DBG_LIB_DMA, dip, "px_lib_iommu_getbypass: io_addr 0x%llx\n",
616 	    *io_addr_p);
617 
618 	return (DDI_SUCCESS);
619 }
620 
621 /*
622  * bus dma sync entry point.
623  */
624 /*ARGSUSED*/
625 int
626 px_lib_dma_sync(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle,
627     off_t off, size_t len, uint_t cache_flags)
628 {
629 	ddi_dma_impl_t *mp = (ddi_dma_impl_t *)handle;
630 	px_t	*px_p = DIP_TO_STATE(dip);
631 	pxu_t	*pxu_p = (pxu_t *)px_p->px_plat_p;
632 
633 	DBG(DBG_LIB_DMA, dip, "px_lib_dma_sync: dip 0x%p rdip 0x%p "
634 	    "handle 0x%llx off 0x%x len 0x%x flags 0x%x\n",
635 	    dip, rdip, handle, off, len, cache_flags);
636 
637 	/*
638 	 * No flush needed for Oberon
639 	 */
640 	if (PX_CHIP_TYPE(pxu_p) == PX_CHIP_OBERON)
641 		return (DDI_SUCCESS);
642 
643 	/*
644 	 * jbus_stst_order is found only in certain cpu modules.
645 	 * Just return success if not present.
646 	 */
647 	if (&jbus_stst_order == NULL)
648 		return (DDI_SUCCESS);
649 
650 	if (!(mp->dmai_flags & PX_DMAI_FLAGS_INUSE)) {
651 		cmn_err(CE_WARN, "%s%d: Unbound dma handle %p.",
652 		    ddi_driver_name(rdip), ddi_get_instance(rdip), (void *)mp);
653 
654 		return (DDI_FAILURE);
655 	}
656 
657 	if (mp->dmai_flags & PX_DMAI_FLAGS_NOSYNC)
658 		return (DDI_SUCCESS);
659 
660 	/*
661 	 * No flush needed when sending data from memory to device.
662 	 * Nothing to do to "sync" memory to what device would already see.
663 	 */
664 	if (!(mp->dmai_rflags & DDI_DMA_READ) ||
665 	    ((cache_flags & PX_DMA_SYNC_DDI_FLAGS) == DDI_DMA_SYNC_FORDEV))
666 		return (DDI_SUCCESS);
667 
668 	/*
669 	 * Perform necessary cpu workaround to ensure jbus ordering.
670 	 * CPU's internal "invalidate FIFOs" are flushed.
671 	 */
672 
673 #if !defined(lint)
674 	kpreempt_disable();
675 #endif
676 	jbus_stst_order();
677 #if !defined(lint)
678 	kpreempt_enable();
679 #endif
680 	return (DDI_SUCCESS);
681 }
682 
683 /*
684  * MSIQ Functions:
685  */
686 /*ARGSUSED*/
687 int
688 px_lib_msiq_init(dev_info_t *dip)
689 {
690 	px_t		*px_p = DIP_TO_STATE(dip);
691 	pxu_t		*pxu_p = (pxu_t *)px_p->px_plat_p;
692 	px_msiq_state_t	*msiq_state_p = &px_p->px_ib_p->ib_msiq_state;
693 	caddr_t		msiq_addr;
694 	px_dvma_addr_t	pg_index;
695 	size_t		size;
696 	int		ret;
697 
698 	DBG(DBG_LIB_MSIQ, dip, "px_lib_msiq_init: dip 0x%p\n", dip);
699 
700 	/*
701 	 * Map the EQ memory into the Fire MMU (has to be 512KB aligned)
702 	 * and then initialize the base address register.
703 	 *
704 	 * Allocate entries from Fire IOMMU so that the resulting address
705 	 * is properly aligned.  Calculate the index of the first allocated
706 	 * entry.  Note: The size of the mapping is assumed to be a multiple
707 	 * of the page size.
708 	 */
709 	msiq_addr = (caddr_t)(((uint64_t)msiq_state_p->msiq_buf_p +
710 	    (MMU_PAGE_SIZE - 1)) >> MMU_PAGE_SHIFT << MMU_PAGE_SHIFT);
711 
712 	size = msiq_state_p->msiq_cnt *
713 	    msiq_state_p->msiq_rec_cnt * sizeof (msiq_rec_t);
714 
715 	pxu_p->msiq_mapped_p = vmem_xalloc(px_p->px_mmu_p->mmu_dvma_map,
716 	    size, (512 * 1024), 0, 0, NULL, NULL, VM_NOSLEEP | VM_BESTFIT);
717 
718 	if (pxu_p->msiq_mapped_p == NULL)
719 		return (DDI_FAILURE);
720 
721 	pg_index = MMU_PAGE_INDEX(px_p->px_mmu_p,
722 	    MMU_BTOP((ulong_t)pxu_p->msiq_mapped_p));
723 
724 	if ((ret = px_lib_iommu_map(px_p->px_dip, PCI_TSBID(0, pg_index),
725 	    MMU_BTOP(size), PCI_MAP_ATTR_WRITE, (void *)msiq_addr, 0,
726 	    MMU_MAP_BUF)) != DDI_SUCCESS) {
727 		DBG(DBG_LIB_MSIQ, dip,
728 		    "hvio_msiq_init failed, ret 0x%lx\n", ret);
729 
730 		(void) px_lib_msiq_fini(dip);
731 		return (DDI_FAILURE);
732 	}
733 
734 	(void) hvio_msiq_init(DIP_TO_HANDLE(dip), pxu_p);
735 
736 	return (DDI_SUCCESS);
737 }
738 
739 /*ARGSUSED*/
740 int
741 px_lib_msiq_fini(dev_info_t *dip)
742 {
743 	px_t		*px_p = DIP_TO_STATE(dip);
744 	pxu_t		*pxu_p = (pxu_t *)px_p->px_plat_p;
745 	px_msiq_state_t	*msiq_state_p = &px_p->px_ib_p->ib_msiq_state;
746 	px_dvma_addr_t	pg_index;
747 	size_t		size;
748 
749 	DBG(DBG_LIB_MSIQ, dip, "px_lib_msiq_fini: dip 0x%p\n", dip);
750 
751 	/*
752 	 * Unmap and free the EQ memory that had been mapped
753 	 * into the Fire IOMMU.
754 	 */
755 	size = msiq_state_p->msiq_cnt *
756 	    msiq_state_p->msiq_rec_cnt * sizeof (msiq_rec_t);
757 
758 	pg_index = MMU_PAGE_INDEX(px_p->px_mmu_p,
759 	    MMU_BTOP((ulong_t)pxu_p->msiq_mapped_p));
760 
761 	(void) px_lib_iommu_demap(px_p->px_dip,
762 	    PCI_TSBID(0, pg_index), MMU_BTOP(size));
763 
764 	/* Free the entries from the Fire MMU */
765 	vmem_xfree(px_p->px_mmu_p->mmu_dvma_map,
766 	    (void *)pxu_p->msiq_mapped_p, size);
767 
768 	return (DDI_SUCCESS);
769 }
770 
771 /*ARGSUSED*/
772 int
773 px_lib_msiq_info(dev_info_t *dip, msiqid_t msiq_id, r_addr_t *ra_p,
774     uint_t *msiq_rec_cnt_p)
775 {
776 	px_t		*px_p = DIP_TO_STATE(dip);
777 	px_msiq_state_t	*msiq_state_p = &px_p->px_ib_p->ib_msiq_state;
778 	uint64_t	*msiq_addr;
779 	size_t		msiq_size;
780 
781 	DBG(DBG_LIB_MSIQ, dip, "px_msiq_info: dip 0x%p msiq_id 0x%x\n",
782 	    dip, msiq_id);
783 
784 	msiq_addr = (uint64_t *)(((uint64_t)msiq_state_p->msiq_buf_p +
785 	    (MMU_PAGE_SIZE - 1)) >> MMU_PAGE_SHIFT << MMU_PAGE_SHIFT);
786 	msiq_size = msiq_state_p->msiq_rec_cnt * sizeof (msiq_rec_t);
787 	ra_p = (r_addr_t *)((caddr_t)msiq_addr + (msiq_id * msiq_size));
788 
789 	*msiq_rec_cnt_p = msiq_state_p->msiq_rec_cnt;
790 
791 	DBG(DBG_LIB_MSIQ, dip, "px_msiq_info: ra_p 0x%p msiq_rec_cnt 0x%x\n",
792 	    ra_p, *msiq_rec_cnt_p);
793 
794 	return (DDI_SUCCESS);
795 }
796 
797 /*ARGSUSED*/
798 int
799 px_lib_msiq_getvalid(dev_info_t *dip, msiqid_t msiq_id,
800     pci_msiq_valid_state_t *msiq_valid_state)
801 {
802 	uint64_t	ret;
803 
804 	DBG(DBG_LIB_MSIQ, dip, "px_lib_msiq_getvalid: dip 0x%p msiq_id 0x%x\n",
805 	    dip, msiq_id);
806 
807 	if ((ret = hvio_msiq_getvalid(DIP_TO_HANDLE(dip),
808 	    msiq_id, msiq_valid_state)) != H_EOK) {
809 		DBG(DBG_LIB_MSIQ, dip,
810 		    "hvio_msiq_getvalid failed, ret 0x%lx\n", ret);
811 		return (DDI_FAILURE);
812 	}
813 
814 	DBG(DBG_LIB_MSIQ, dip, "px_lib_msiq_getvalid: msiq_valid_state 0x%x\n",
815 	    *msiq_valid_state);
816 
817 	return (DDI_SUCCESS);
818 }
819 
820 /*ARGSUSED*/
821 int
822 px_lib_msiq_setvalid(dev_info_t *dip, msiqid_t msiq_id,
823     pci_msiq_valid_state_t msiq_valid_state)
824 {
825 	uint64_t	ret;
826 
827 	DBG(DBG_LIB_MSIQ, dip, "px_lib_msiq_setvalid: dip 0x%p msiq_id 0x%x "
828 	    "msiq_valid_state 0x%x\n", dip, msiq_id, msiq_valid_state);
829 
830 	if ((ret = hvio_msiq_setvalid(DIP_TO_HANDLE(dip),
831 	    msiq_id, msiq_valid_state)) != H_EOK) {
832 		DBG(DBG_LIB_MSIQ, dip,
833 		    "hvio_msiq_setvalid failed, ret 0x%lx\n", ret);
834 		return (DDI_FAILURE);
835 	}
836 
837 	return (DDI_SUCCESS);
838 }
839 
840 /*ARGSUSED*/
841 int
842 px_lib_msiq_getstate(dev_info_t *dip, msiqid_t msiq_id,
843     pci_msiq_state_t *msiq_state)
844 {
845 	uint64_t	ret;
846 
847 	DBG(DBG_LIB_MSIQ, dip, "px_lib_msiq_getstate: dip 0x%p msiq_id 0x%x\n",
848 	    dip, msiq_id);
849 
850 	if ((ret = hvio_msiq_getstate(DIP_TO_HANDLE(dip),
851 	    msiq_id, msiq_state)) != H_EOK) {
852 		DBG(DBG_LIB_MSIQ, dip,
853 		    "hvio_msiq_getstate failed, ret 0x%lx\n", ret);
854 		return (DDI_FAILURE);
855 	}
856 
857 	DBG(DBG_LIB_MSIQ, dip, "px_lib_msiq_getstate: msiq_state 0x%x\n",
858 	    *msiq_state);
859 
860 	return (DDI_SUCCESS);
861 }
862 
863 /*ARGSUSED*/
864 int
865 px_lib_msiq_setstate(dev_info_t *dip, msiqid_t msiq_id,
866     pci_msiq_state_t msiq_state)
867 {
868 	uint64_t	ret;
869 
870 	DBG(DBG_LIB_MSIQ, dip, "px_lib_msiq_setstate: dip 0x%p msiq_id 0x%x "
871 	    "msiq_state 0x%x\n", dip, msiq_id, msiq_state);
872 
873 	if ((ret = hvio_msiq_setstate(DIP_TO_HANDLE(dip),
874 	    msiq_id, msiq_state)) != H_EOK) {
875 		DBG(DBG_LIB_MSIQ, dip,
876 		    "hvio_msiq_setstate failed, ret 0x%lx\n", ret);
877 		return (DDI_FAILURE);
878 	}
879 
880 	return (DDI_SUCCESS);
881 }
882 
883 /*ARGSUSED*/
884 int
885 px_lib_msiq_gethead(dev_info_t *dip, msiqid_t msiq_id,
886     msiqhead_t *msiq_head)
887 {
888 	uint64_t	ret;
889 
890 	DBG(DBG_LIB_MSIQ, dip, "px_lib_msiq_gethead: dip 0x%p msiq_id 0x%x\n",
891 	    dip, msiq_id);
892 
893 	if ((ret = hvio_msiq_gethead(DIP_TO_HANDLE(dip),
894 	    msiq_id, msiq_head)) != H_EOK) {
895 		DBG(DBG_LIB_MSIQ, dip,
896 		    "hvio_msiq_gethead failed, ret 0x%lx\n", ret);
897 		return (DDI_FAILURE);
898 	}
899 
900 	DBG(DBG_LIB_MSIQ, dip, "px_lib_msiq_gethead: msiq_head 0x%x\n",
901 	    *msiq_head);
902 
903 	return (DDI_SUCCESS);
904 }
905 
906 /*ARGSUSED*/
907 int
908 px_lib_msiq_sethead(dev_info_t *dip, msiqid_t msiq_id,
909     msiqhead_t msiq_head)
910 {
911 	uint64_t	ret;
912 
913 	DBG(DBG_LIB_MSIQ, dip, "px_lib_msiq_sethead: dip 0x%p msiq_id 0x%x "
914 	    "msiq_head 0x%x\n", dip, msiq_id, msiq_head);
915 
916 	if ((ret = hvio_msiq_sethead(DIP_TO_HANDLE(dip),
917 	    msiq_id, msiq_head)) != H_EOK) {
918 		DBG(DBG_LIB_MSIQ, dip,
919 		    "hvio_msiq_sethead failed, ret 0x%lx\n", ret);
920 		return (DDI_FAILURE);
921 	}
922 
923 	return (DDI_SUCCESS);
924 }
925 
926 /*ARGSUSED*/
927 int
928 px_lib_msiq_gettail(dev_info_t *dip, msiqid_t msiq_id,
929     msiqtail_t *msiq_tail)
930 {
931 	uint64_t	ret;
932 
933 	DBG(DBG_LIB_MSIQ, dip, "px_lib_msiq_gettail: dip 0x%p msiq_id 0x%x\n",
934 	    dip, msiq_id);
935 
936 	if ((ret = hvio_msiq_gettail(DIP_TO_HANDLE(dip),
937 	    msiq_id, msiq_tail)) != H_EOK) {
938 		DBG(DBG_LIB_MSIQ, dip,
939 		    "hvio_msiq_gettail failed, ret 0x%lx\n", ret);
940 		return (DDI_FAILURE);
941 	}
942 
943 	DBG(DBG_LIB_MSIQ, dip, "px_lib_msiq_gettail: msiq_tail 0x%x\n",
944 	    *msiq_tail);
945 
946 	return (DDI_SUCCESS);
947 }
948 
949 /*ARGSUSED*/
950 void
951 px_lib_get_msiq_rec(dev_info_t *dip, px_msiq_t *msiq_p, msiq_rec_t *msiq_rec_p)
952 {
953 	eq_rec_t	*eq_rec_p = (eq_rec_t *)msiq_p->msiq_curr;
954 
955 	DBG(DBG_LIB_MSIQ, dip, "px_lib_get_msiq_rec: dip 0x%p eq_rec_p 0x%p\n",
956 	    dip, eq_rec_p);
957 
958 	if (!eq_rec_p->eq_rec_fmt_type) {
959 		/* Set msiq_rec_type to zero */
960 		msiq_rec_p->msiq_rec_type = 0;
961 
962 		return;
963 	}
964 
965 	DBG(DBG_LIB_MSIQ, dip, "px_lib_get_msiq_rec: EQ RECORD, "
966 	    "eq_rec_rid 0x%llx eq_rec_fmt_type 0x%llx "
967 	    "eq_rec_len 0x%llx eq_rec_addr0 0x%llx "
968 	    "eq_rec_addr1 0x%llx eq_rec_data0 0x%llx "
969 	    "eq_rec_data1 0x%llx\n", eq_rec_p->eq_rec_rid,
970 	    eq_rec_p->eq_rec_fmt_type, eq_rec_p->eq_rec_len,
971 	    eq_rec_p->eq_rec_addr0, eq_rec_p->eq_rec_addr1,
972 	    eq_rec_p->eq_rec_data0, eq_rec_p->eq_rec_data1);
973 
974 	/*
975 	 * Only upper 4 bits of eq_rec_fmt_type is used
976 	 * to identify the EQ record type.
977 	 */
978 	switch (eq_rec_p->eq_rec_fmt_type >> 3) {
979 	case EQ_REC_MSI32:
980 		msiq_rec_p->msiq_rec_type = MSI32_REC;
981 
982 		msiq_rec_p->msiq_rec_data.msi.msi_data =
983 		    eq_rec_p->eq_rec_data0;
984 		break;
985 	case EQ_REC_MSI64:
986 		msiq_rec_p->msiq_rec_type = MSI64_REC;
987 
988 		msiq_rec_p->msiq_rec_data.msi.msi_data =
989 		    eq_rec_p->eq_rec_data0;
990 		break;
991 	case EQ_REC_MSG:
992 		msiq_rec_p->msiq_rec_type = MSG_REC;
993 
994 		msiq_rec_p->msiq_rec_data.msg.msg_route =
995 		    eq_rec_p->eq_rec_fmt_type & 7;
996 		msiq_rec_p->msiq_rec_data.msg.msg_targ = eq_rec_p->eq_rec_rid;
997 		msiq_rec_p->msiq_rec_data.msg.msg_code = eq_rec_p->eq_rec_data0;
998 		break;
999 	default:
1000 		cmn_err(CE_WARN, "%s%d: px_lib_get_msiq_rec: "
1001 		    "0x%x is an unknown EQ record type",
1002 		    ddi_driver_name(dip), ddi_get_instance(dip),
1003 		    (int)eq_rec_p->eq_rec_fmt_type);
1004 		break;
1005 	}
1006 
1007 	msiq_rec_p->msiq_rec_rid = eq_rec_p->eq_rec_rid;
1008 	msiq_rec_p->msiq_rec_msi_addr = ((eq_rec_p->eq_rec_addr1 << 16) |
1009 	    (eq_rec_p->eq_rec_addr0 << 2));
1010 
1011 	/* Zero out eq_rec_fmt_type field */
1012 	eq_rec_p->eq_rec_fmt_type = 0;
1013 }
1014 
1015 /*
1016  * MSI Functions:
1017  */
1018 /*ARGSUSED*/
1019 int
1020 px_lib_msi_init(dev_info_t *dip)
1021 {
1022 	px_t		*px_p = DIP_TO_STATE(dip);
1023 	px_msi_state_t	*msi_state_p = &px_p->px_ib_p->ib_msi_state;
1024 	uint64_t	ret;
1025 
1026 	DBG(DBG_LIB_MSI, dip, "px_lib_msi_init: dip 0x%p\n", dip);
1027 
1028 	if ((ret = hvio_msi_init(DIP_TO_HANDLE(dip),
1029 	    msi_state_p->msi_addr32, msi_state_p->msi_addr64)) != H_EOK) {
1030 		DBG(DBG_LIB_MSIQ, dip, "px_lib_msi_init failed, ret 0x%lx\n",
1031 		    ret);
1032 		return (DDI_FAILURE);
1033 	}
1034 
1035 	return (DDI_SUCCESS);
1036 }
1037 
1038 /*ARGSUSED*/
1039 int
1040 px_lib_msi_getmsiq(dev_info_t *dip, msinum_t msi_num,
1041     msiqid_t *msiq_id)
1042 {
1043 	uint64_t	ret;
1044 
1045 	DBG(DBG_LIB_MSI, dip, "px_lib_msi_getmsiq: dip 0x%p msi_num 0x%x\n",
1046 	    dip, msi_num);
1047 
1048 	if ((ret = hvio_msi_getmsiq(DIP_TO_HANDLE(dip),
1049 	    msi_num, msiq_id)) != H_EOK) {
1050 		DBG(DBG_LIB_MSI, dip,
1051 		    "hvio_msi_getmsiq failed, ret 0x%lx\n", ret);
1052 		return (DDI_FAILURE);
1053 	}
1054 
1055 	DBG(DBG_LIB_MSI, dip, "px_lib_msi_getmsiq: msiq_id 0x%x\n",
1056 	    *msiq_id);
1057 
1058 	return (DDI_SUCCESS);
1059 }
1060 
1061 /*ARGSUSED*/
1062 int
1063 px_lib_msi_setmsiq(dev_info_t *dip, msinum_t msi_num,
1064     msiqid_t msiq_id, msi_type_t msitype)
1065 {
1066 	uint64_t	ret;
1067 
1068 	DBG(DBG_LIB_MSI, dip, "px_lib_msi_setmsiq: dip 0x%p msi_num 0x%x "
1069 	    "msq_id 0x%x\n", dip, msi_num, msiq_id);
1070 
1071 	if ((ret = hvio_msi_setmsiq(DIP_TO_HANDLE(dip),
1072 	    msi_num, msiq_id)) != H_EOK) {
1073 		DBG(DBG_LIB_MSI, dip,
1074 		    "hvio_msi_setmsiq failed, ret 0x%lx\n", ret);
1075 		return (DDI_FAILURE);
1076 	}
1077 
1078 	return (DDI_SUCCESS);
1079 }
1080 
1081 /*ARGSUSED*/
1082 int
1083 px_lib_msi_getvalid(dev_info_t *dip, msinum_t msi_num,
1084     pci_msi_valid_state_t *msi_valid_state)
1085 {
1086 	uint64_t	ret;
1087 
1088 	DBG(DBG_LIB_MSI, dip, "px_lib_msi_getvalid: dip 0x%p msi_num 0x%x\n",
1089 	    dip, msi_num);
1090 
1091 	if ((ret = hvio_msi_getvalid(DIP_TO_HANDLE(dip),
1092 	    msi_num, msi_valid_state)) != H_EOK) {
1093 		DBG(DBG_LIB_MSI, dip,
1094 		    "hvio_msi_getvalid failed, ret 0x%lx\n", ret);
1095 		return (DDI_FAILURE);
1096 	}
1097 
1098 	DBG(DBG_LIB_MSI, dip, "px_lib_msi_getvalid: msiq_id 0x%x\n",
1099 	    *msi_valid_state);
1100 
1101 	return (DDI_SUCCESS);
1102 }
1103 
1104 /*ARGSUSED*/
1105 int
1106 px_lib_msi_setvalid(dev_info_t *dip, msinum_t msi_num,
1107     pci_msi_valid_state_t msi_valid_state)
1108 {
1109 	uint64_t	ret;
1110 
1111 	DBG(DBG_LIB_MSI, dip, "px_lib_msi_setvalid: dip 0x%p msi_num 0x%x "
1112 	    "msi_valid_state 0x%x\n", dip, msi_num, msi_valid_state);
1113 
1114 	if ((ret = hvio_msi_setvalid(DIP_TO_HANDLE(dip),
1115 	    msi_num, msi_valid_state)) != H_EOK) {
1116 		DBG(DBG_LIB_MSI, dip,
1117 		    "hvio_msi_setvalid failed, ret 0x%lx\n", ret);
1118 		return (DDI_FAILURE);
1119 	}
1120 
1121 	return (DDI_SUCCESS);
1122 }
1123 
1124 /*ARGSUSED*/
1125 int
1126 px_lib_msi_getstate(dev_info_t *dip, msinum_t msi_num,
1127     pci_msi_state_t *msi_state)
1128 {
1129 	uint64_t	ret;
1130 
1131 	DBG(DBG_LIB_MSI, dip, "px_lib_msi_getstate: dip 0x%p msi_num 0x%x\n",
1132 	    dip, msi_num);
1133 
1134 	if ((ret = hvio_msi_getstate(DIP_TO_HANDLE(dip),
1135 	    msi_num, msi_state)) != H_EOK) {
1136 		DBG(DBG_LIB_MSI, dip,
1137 		    "hvio_msi_getstate failed, ret 0x%lx\n", ret);
1138 		return (DDI_FAILURE);
1139 	}
1140 
1141 	DBG(DBG_LIB_MSI, dip, "px_lib_msi_getstate: msi_state 0x%x\n",
1142 	    *msi_state);
1143 
1144 	return (DDI_SUCCESS);
1145 }
1146 
1147 /*ARGSUSED*/
1148 int
1149 px_lib_msi_setstate(dev_info_t *dip, msinum_t msi_num,
1150     pci_msi_state_t msi_state)
1151 {
1152 	uint64_t	ret;
1153 
1154 	DBG(DBG_LIB_MSI, dip, "px_lib_msi_setstate: dip 0x%p msi_num 0x%x "
1155 	    "msi_state 0x%x\n", dip, msi_num, msi_state);
1156 
1157 	if ((ret = hvio_msi_setstate(DIP_TO_HANDLE(dip),
1158 	    msi_num, msi_state)) != H_EOK) {
1159 		DBG(DBG_LIB_MSI, dip,
1160 		    "hvio_msi_setstate failed, ret 0x%lx\n", ret);
1161 		return (DDI_FAILURE);
1162 	}
1163 
1164 	return (DDI_SUCCESS);
1165 }
1166 
1167 /*
1168  * MSG Functions:
1169  */
1170 /*ARGSUSED*/
1171 int
1172 px_lib_msg_getmsiq(dev_info_t *dip, pcie_msg_type_t msg_type,
1173     msiqid_t *msiq_id)
1174 {
1175 	uint64_t	ret;
1176 
1177 	DBG(DBG_LIB_MSG, dip, "px_lib_msg_getmsiq: dip 0x%p msg_type 0x%x\n",
1178 	    dip, msg_type);
1179 
1180 	if ((ret = hvio_msg_getmsiq(DIP_TO_HANDLE(dip),
1181 	    msg_type, msiq_id)) != H_EOK) {
1182 		DBG(DBG_LIB_MSG, dip,
1183 		    "hvio_msg_getmsiq failed, ret 0x%lx\n", ret);
1184 		return (DDI_FAILURE);
1185 	}
1186 
1187 	DBG(DBG_LIB_MSI, dip, "px_lib_msg_getmsiq: msiq_id 0x%x\n",
1188 	    *msiq_id);
1189 
1190 	return (DDI_SUCCESS);
1191 }
1192 
1193 /*ARGSUSED*/
1194 int
1195 px_lib_msg_setmsiq(dev_info_t *dip, pcie_msg_type_t msg_type,
1196     msiqid_t msiq_id)
1197 {
1198 	uint64_t	ret;
1199 
1200 	DBG(DBG_LIB_MSG, dip, "px_lib_msi_setstate: dip 0x%p msg_type 0x%x "
1201 	    "msiq_id 0x%x\n", dip, msg_type, msiq_id);
1202 
1203 	if ((ret = hvio_msg_setmsiq(DIP_TO_HANDLE(dip),
1204 	    msg_type, msiq_id)) != H_EOK) {
1205 		DBG(DBG_LIB_MSG, dip,
1206 		    "hvio_msg_setmsiq failed, ret 0x%lx\n", ret);
1207 		return (DDI_FAILURE);
1208 	}
1209 
1210 	return (DDI_SUCCESS);
1211 }
1212 
1213 /*ARGSUSED*/
1214 int
1215 px_lib_msg_getvalid(dev_info_t *dip, pcie_msg_type_t msg_type,
1216     pcie_msg_valid_state_t *msg_valid_state)
1217 {
1218 	uint64_t	ret;
1219 
1220 	DBG(DBG_LIB_MSG, dip, "px_lib_msg_getvalid: dip 0x%p msg_type 0x%x\n",
1221 	    dip, msg_type);
1222 
1223 	if ((ret = hvio_msg_getvalid(DIP_TO_HANDLE(dip), msg_type,
1224 	    msg_valid_state)) != H_EOK) {
1225 		DBG(DBG_LIB_MSG, dip,
1226 		    "hvio_msg_getvalid failed, ret 0x%lx\n", ret);
1227 		return (DDI_FAILURE);
1228 	}
1229 
1230 	DBG(DBG_LIB_MSI, dip, "px_lib_msg_getvalid: msg_valid_state 0x%x\n",
1231 	    *msg_valid_state);
1232 
1233 	return (DDI_SUCCESS);
1234 }
1235 
1236 /*ARGSUSED*/
1237 int
1238 px_lib_msg_setvalid(dev_info_t *dip, pcie_msg_type_t msg_type,
1239     pcie_msg_valid_state_t msg_valid_state)
1240 {
1241 	uint64_t	ret;
1242 
1243 	DBG(DBG_LIB_MSG, dip, "px_lib_msg_setvalid: dip 0x%p msg_type 0x%x "
1244 	    "msg_valid_state 0x%x\n", dip, msg_type, msg_valid_state);
1245 
1246 	if ((ret = hvio_msg_setvalid(DIP_TO_HANDLE(dip), msg_type,
1247 	    msg_valid_state)) != H_EOK) {
1248 		DBG(DBG_LIB_MSG, dip,
1249 		    "hvio_msg_setvalid failed, ret 0x%lx\n", ret);
1250 		return (DDI_FAILURE);
1251 	}
1252 
1253 	return (DDI_SUCCESS);
1254 }
1255 
1256 /*
1257  * Suspend/Resume Functions:
1258  * Currently unsupported by hypervisor
1259  */
1260 int
1261 px_lib_suspend(dev_info_t *dip)
1262 {
1263 	px_t		*px_p = DIP_TO_STATE(dip);
1264 	pxu_t		*pxu_p = (pxu_t *)px_p->px_plat_p;
1265 	px_cb_t		*cb_p = PX2CB(px_p);
1266 	devhandle_t	dev_hdl, xbus_dev_hdl;
1267 	uint64_t	ret = H_EOK;
1268 
1269 	DBG(DBG_DETACH, dip, "px_lib_suspend: dip 0x%p\n", dip);
1270 
1271 	dev_hdl = (devhandle_t)pxu_p->px_address[PX_REG_CSR];
1272 	xbus_dev_hdl = (devhandle_t)pxu_p->px_address[PX_REG_XBC];
1273 
1274 	if ((ret = hvio_suspend(dev_hdl, pxu_p)) != H_EOK)
1275 		goto fail;
1276 
1277 	if (--cb_p->attachcnt == 0) {
1278 		ret = hvio_cb_suspend(xbus_dev_hdl, pxu_p);
1279 		if (ret != H_EOK)
1280 			cb_p->attachcnt++;
1281 	}
1282 
1283 fail:
1284 	return ((ret != H_EOK) ? DDI_FAILURE: DDI_SUCCESS);
1285 }
1286 
1287 void
1288 px_lib_resume(dev_info_t *dip)
1289 {
1290 	px_t		*px_p = DIP_TO_STATE(dip);
1291 	pxu_t		*pxu_p = (pxu_t *)px_p->px_plat_p;
1292 	px_cb_t		*cb_p = PX2CB(px_p);
1293 	devhandle_t	dev_hdl, xbus_dev_hdl;
1294 	devino_t	pec_ino = px_p->px_inos[PX_INTR_PEC];
1295 	devino_t	xbc_ino = px_p->px_inos[PX_INTR_XBC];
1296 
1297 	DBG(DBG_ATTACH, dip, "px_lib_resume: dip 0x%p\n", dip);
1298 
1299 	dev_hdl = (devhandle_t)pxu_p->px_address[PX_REG_CSR];
1300 	xbus_dev_hdl = (devhandle_t)pxu_p->px_address[PX_REG_XBC];
1301 
1302 	if (++cb_p->attachcnt == 1)
1303 		hvio_cb_resume(dev_hdl, xbus_dev_hdl, xbc_ino, pxu_p);
1304 
1305 	hvio_resume(dev_hdl, pec_ino, pxu_p);
1306 }
1307 
1308 /*
1309  * Generate a unique Oberon UBC ID based on the Logicial System Board and
1310  * the IO Channel from the portid property field.
1311  */
1312 static uint64_t
1313 oberon_get_ubc_id(dev_info_t *dip)
1314 {
1315 	px_t	*px_p = DIP_TO_STATE(dip);
1316 	pxu_t	*pxu_p = (pxu_t *)px_p->px_plat_p;
1317 	uint64_t	ubc_id;
1318 
1319 	/*
1320 	 * Generate a unique 6 bit UBC ID using the 2 IO_Channel#[1:0] bits and
1321 	 * the 4 LSB_ID[3:0] bits from the Oberon's portid property.
1322 	 */
1323 	ubc_id = (((pxu_p->portid >> OBERON_PORT_ID_IOC) &
1324 	    OBERON_PORT_ID_IOC_MASK) | (((pxu_p->portid >>
1325 	    OBERON_PORT_ID_LSB) & OBERON_PORT_ID_LSB_MASK)
1326 	    << OBERON_UBC_ID_LSB));
1327 
1328 	return (ubc_id);
1329 }
1330 
1331 /*
1332  * Oberon does not have a UBC scratch register, so alloc an array of scratch
1333  * registers when needed and use a unique UBC ID as an index. This code
1334  * can be simplified if we use a pre-allocated array. They are currently
1335  * being dynamically allocated because it's only needed by the Oberon.
1336  */
1337 static void
1338 oberon_set_cb(dev_info_t *dip, uint64_t val)
1339 {
1340 	uint64_t	ubc_id;
1341 
1342 	if (px_oberon_ubc_scratch_regs == NULL)
1343 		px_oberon_ubc_scratch_regs =
1344 		    (uint64_t *)kmem_zalloc(sizeof (uint64_t)*
1345 		    OBERON_UBC_ID_MAX, KM_SLEEP);
1346 
1347 	ubc_id = oberon_get_ubc_id(dip);
1348 
1349 	px_oberon_ubc_scratch_regs[ubc_id] = val;
1350 
1351 	/*
1352 	 * Check if any scratch registers are still in use. If all scratch
1353 	 * registers are currently set to zero, then deallocate the scratch
1354 	 * register array.
1355 	 */
1356 	for (ubc_id = 0; ubc_id < OBERON_UBC_ID_MAX; ubc_id++) {
1357 		if (px_oberon_ubc_scratch_regs[ubc_id] != NULL)
1358 			return;
1359 	}
1360 
1361 	/*
1362 	 * All scratch registers are set to zero so deallocate the scratch
1363 	 * register array and set the pointer to NULL.
1364 	 */
1365 	kmem_free(px_oberon_ubc_scratch_regs,
1366 	    (sizeof (uint64_t)*OBERON_UBC_ID_MAX));
1367 
1368 	px_oberon_ubc_scratch_regs = NULL;
1369 }
1370 
1371 /*
1372  * Oberon does not have a UBC scratch register, so use an allocated array of
1373  * scratch registers and use the unique UBC ID as an index into that array.
1374  */
1375 static uint64_t
1376 oberon_get_cb(dev_info_t *dip)
1377 {
1378 	uint64_t	ubc_id;
1379 
1380 	if (px_oberon_ubc_scratch_regs == NULL)
1381 		return (0);
1382 
1383 	ubc_id = oberon_get_ubc_id(dip);
1384 
1385 	return (px_oberon_ubc_scratch_regs[ubc_id]);
1386 }
1387 
1388 /*
1389  * Misc Functions:
1390  * Currently unsupported by hypervisor
1391  */
1392 static uint64_t
1393 px_get_cb(dev_info_t *dip)
1394 {
1395 	px_t	*px_p = DIP_TO_STATE(dip);
1396 	pxu_t	*pxu_p = (pxu_t *)px_p->px_plat_p;
1397 
1398 	/*
1399 	 * Oberon does not currently have Scratchpad registers.
1400 	 */
1401 	if (PX_CHIP_TYPE(pxu_p) == PX_CHIP_OBERON)
1402 		return (oberon_get_cb(dip));
1403 
1404 	return (CSR_XR((caddr_t)pxu_p->px_address[PX_REG_XBC], JBUS_SCRATCH_1));
1405 }
1406 
1407 static void
1408 px_set_cb(dev_info_t *dip, uint64_t val)
1409 {
1410 	px_t	*px_p = DIP_TO_STATE(dip);
1411 	pxu_t	*pxu_p = (pxu_t *)px_p->px_plat_p;
1412 
1413 	/*
1414 	 * Oberon does not currently have Scratchpad registers.
1415 	 */
1416 	if (PX_CHIP_TYPE(pxu_p) == PX_CHIP_OBERON) {
1417 		oberon_set_cb(dip, val);
1418 		return;
1419 	}
1420 
1421 	CSR_XS((caddr_t)pxu_p->px_address[PX_REG_XBC], JBUS_SCRATCH_1, val);
1422 }
1423 
1424 /*ARGSUSED*/
1425 int
1426 px_lib_map_vconfig(dev_info_t *dip,
1427 	ddi_map_req_t *mp, pci_config_offset_t off,
1428 		pci_regspec_t *rp, caddr_t *addrp)
1429 {
1430 	/*
1431 	 * No special config space access services in this layer.
1432 	 */
1433 	return (DDI_FAILURE);
1434 }
1435 
1436 void
1437 px_lib_map_attr_check(ddi_map_req_t *mp)
1438 {
1439 	ddi_acc_hdl_t *hp = mp->map_handlep;
1440 
1441 	/* fire does not accept byte masks from PIO store merge */
1442 	if (hp->ah_acc.devacc_attr_dataorder == DDI_STORECACHING_OK_ACC)
1443 		hp->ah_acc.devacc_attr_dataorder = DDI_STRICTORDER_ACC;
1444 }
1445 
1446 void
1447 px_lib_clr_errs(px_t *px_p)
1448 {
1449 	px_pec_t	*pec_p = px_p->px_pec_p;
1450 	dev_info_t	*rpdip = px_p->px_dip;
1451 	int		err = PX_OK, ret;
1452 	int		acctype = pec_p->pec_safeacc_type;
1453 	ddi_fm_error_t	derr;
1454 
1455 	/* Create the derr */
1456 	bzero(&derr, sizeof (ddi_fm_error_t));
1457 	derr.fme_version = DDI_FME_VERSION;
1458 	derr.fme_ena = fm_ena_generate(0, FM_ENA_FMT1);
1459 	derr.fme_flag = acctype;
1460 
1461 	if (acctype == DDI_FM_ERR_EXPECTED) {
1462 		derr.fme_status = DDI_FM_NONFATAL;
1463 		ndi_fm_acc_err_set(pec_p->pec_acc_hdl, &derr);
1464 	}
1465 
1466 	mutex_enter(&px_p->px_fm_mutex);
1467 
1468 	/* send ereport/handle/clear fire registers */
1469 	err = px_err_handle(px_p, &derr, PX_LIB_CALL, B_TRUE);
1470 
1471 	/* Check all child devices for errors */
1472 	ret = ndi_fm_handler_dispatch(rpdip, NULL, &derr);
1473 
1474 	mutex_exit(&px_p->px_fm_mutex);
1475 
1476 	/*
1477 	 * PX_FATAL_HW indicates a condition recovered from Fatal-Reset,
1478 	 * therefore it does not cause panic.
1479 	 */
1480 	if ((err & (PX_FATAL_GOS | PX_FATAL_SW)) || (ret == DDI_FM_FATAL))
1481 		PX_FM_PANIC("Fatal System Port Error has occurred\n");
1482 }
1483 
1484 #ifdef  DEBUG
1485 int	px_peekfault_cnt = 0;
1486 int	px_pokefault_cnt = 0;
1487 #endif  /* DEBUG */
1488 
1489 /*ARGSUSED*/
1490 static int
1491 px_lib_do_poke(dev_info_t *dip, dev_info_t *rdip,
1492     peekpoke_ctlops_t *in_args)
1493 {
1494 	px_t *px_p = DIP_TO_STATE(dip);
1495 	px_pec_t *pec_p = px_p->px_pec_p;
1496 	int err = DDI_SUCCESS;
1497 	on_trap_data_t otd;
1498 
1499 	mutex_enter(&pec_p->pec_pokefault_mutex);
1500 	pec_p->pec_ontrap_data = &otd;
1501 	pec_p->pec_safeacc_type = DDI_FM_ERR_POKE;
1502 
1503 	/* Set up protected environment. */
1504 	if (!on_trap(&otd, OT_DATA_ACCESS)) {
1505 		uintptr_t tramp = otd.ot_trampoline;
1506 
1507 		otd.ot_trampoline = (uintptr_t)&poke_fault;
1508 		err = do_poke(in_args->size, (void *)in_args->dev_addr,
1509 		    (void *)in_args->host_addr);
1510 		otd.ot_trampoline = tramp;
1511 	} else
1512 		err = DDI_FAILURE;
1513 
1514 	px_lib_clr_errs(px_p);
1515 
1516 	if (otd.ot_trap & OT_DATA_ACCESS)
1517 		err = DDI_FAILURE;
1518 
1519 	/* Take down protected environment. */
1520 	no_trap();
1521 
1522 	pec_p->pec_ontrap_data = NULL;
1523 	pec_p->pec_safeacc_type = DDI_FM_ERR_UNEXPECTED;
1524 	mutex_exit(&pec_p->pec_pokefault_mutex);
1525 
1526 #ifdef  DEBUG
1527 	if (err == DDI_FAILURE)
1528 		px_pokefault_cnt++;
1529 #endif
1530 	return (err);
1531 }
1532 
1533 /*ARGSUSED*/
1534 static int
1535 px_lib_do_caut_put(dev_info_t *dip, dev_info_t *rdip,
1536     peekpoke_ctlops_t *cautacc_ctlops_arg)
1537 {
1538 	size_t size = cautacc_ctlops_arg->size;
1539 	uintptr_t dev_addr = cautacc_ctlops_arg->dev_addr;
1540 	uintptr_t host_addr = cautacc_ctlops_arg->host_addr;
1541 	ddi_acc_impl_t *hp = (ddi_acc_impl_t *)cautacc_ctlops_arg->handle;
1542 	size_t repcount = cautacc_ctlops_arg->repcount;
1543 	uint_t flags = cautacc_ctlops_arg->flags;
1544 
1545 	px_t *px_p = DIP_TO_STATE(dip);
1546 	px_pec_t *pec_p = px_p->px_pec_p;
1547 	int err = DDI_SUCCESS;
1548 
1549 	/*
1550 	 * Note that i_ndi_busop_access_enter ends up grabbing the pokefault
1551 	 * mutex.
1552 	 */
1553 	i_ndi_busop_access_enter(hp->ahi_common.ah_dip, (ddi_acc_handle_t)hp);
1554 
1555 	pec_p->pec_ontrap_data = (on_trap_data_t *)hp->ahi_err->err_ontrap;
1556 	pec_p->pec_safeacc_type = DDI_FM_ERR_EXPECTED;
1557 	hp->ahi_err->err_expected = DDI_FM_ERR_EXPECTED;
1558 
1559 	if (!i_ddi_ontrap((ddi_acc_handle_t)hp)) {
1560 		for (; repcount; repcount--) {
1561 			switch (size) {
1562 
1563 			case sizeof (uint8_t):
1564 				i_ddi_put8(hp, (uint8_t *)dev_addr,
1565 				    *(uint8_t *)host_addr);
1566 				break;
1567 
1568 			case sizeof (uint16_t):
1569 				i_ddi_put16(hp, (uint16_t *)dev_addr,
1570 				    *(uint16_t *)host_addr);
1571 				break;
1572 
1573 			case sizeof (uint32_t):
1574 				i_ddi_put32(hp, (uint32_t *)dev_addr,
1575 				    *(uint32_t *)host_addr);
1576 				break;
1577 
1578 			case sizeof (uint64_t):
1579 				i_ddi_put64(hp, (uint64_t *)dev_addr,
1580 				    *(uint64_t *)host_addr);
1581 				break;
1582 			}
1583 
1584 			host_addr += size;
1585 
1586 			if (flags == DDI_DEV_AUTOINCR)
1587 				dev_addr += size;
1588 
1589 			px_lib_clr_errs(px_p);
1590 
1591 			if (pec_p->pec_ontrap_data->ot_trap & OT_DATA_ACCESS) {
1592 				err = DDI_FAILURE;
1593 #ifdef  DEBUG
1594 				px_pokefault_cnt++;
1595 #endif
1596 				break;
1597 			}
1598 		}
1599 	}
1600 
1601 	i_ddi_notrap((ddi_acc_handle_t)hp);
1602 	pec_p->pec_ontrap_data = NULL;
1603 	pec_p->pec_safeacc_type = DDI_FM_ERR_UNEXPECTED;
1604 	i_ndi_busop_access_exit(hp->ahi_common.ah_dip, (ddi_acc_handle_t)hp);
1605 	hp->ahi_err->err_expected = DDI_FM_ERR_UNEXPECTED;
1606 
1607 	return (err);
1608 }
1609 
1610 
1611 int
1612 px_lib_ctlops_poke(dev_info_t *dip, dev_info_t *rdip,
1613     peekpoke_ctlops_t *in_args)
1614 {
1615 	return (in_args->handle ? px_lib_do_caut_put(dip, rdip, in_args) :
1616 	    px_lib_do_poke(dip, rdip, in_args));
1617 }
1618 
1619 
1620 /*ARGSUSED*/
1621 static int
1622 px_lib_do_peek(dev_info_t *dip, peekpoke_ctlops_t *in_args)
1623 {
1624 	px_t *px_p = DIP_TO_STATE(dip);
1625 	px_pec_t *pec_p = px_p->px_pec_p;
1626 	int err = DDI_SUCCESS;
1627 	on_trap_data_t otd;
1628 
1629 	mutex_enter(&pec_p->pec_pokefault_mutex);
1630 	pec_p->pec_safeacc_type = DDI_FM_ERR_PEEK;
1631 
1632 	if (!on_trap(&otd, OT_DATA_ACCESS)) {
1633 		uintptr_t tramp = otd.ot_trampoline;
1634 
1635 		otd.ot_trampoline = (uintptr_t)&peek_fault;
1636 		err = do_peek(in_args->size, (void *)in_args->dev_addr,
1637 		    (void *)in_args->host_addr);
1638 		otd.ot_trampoline = tramp;
1639 	} else
1640 		err = DDI_FAILURE;
1641 
1642 	no_trap();
1643 	pec_p->pec_safeacc_type = DDI_FM_ERR_UNEXPECTED;
1644 	mutex_exit(&pec_p->pec_pokefault_mutex);
1645 
1646 #ifdef  DEBUG
1647 	if (err == DDI_FAILURE)
1648 		px_peekfault_cnt++;
1649 #endif
1650 	return (err);
1651 }
1652 
1653 
1654 static int
1655 px_lib_do_caut_get(dev_info_t *dip, peekpoke_ctlops_t *cautacc_ctlops_arg)
1656 {
1657 	size_t size = cautacc_ctlops_arg->size;
1658 	uintptr_t dev_addr = cautacc_ctlops_arg->dev_addr;
1659 	uintptr_t host_addr = cautacc_ctlops_arg->host_addr;
1660 	ddi_acc_impl_t *hp = (ddi_acc_impl_t *)cautacc_ctlops_arg->handle;
1661 	size_t repcount = cautacc_ctlops_arg->repcount;
1662 	uint_t flags = cautacc_ctlops_arg->flags;
1663 
1664 	px_t *px_p = DIP_TO_STATE(dip);
1665 	px_pec_t *pec_p = px_p->px_pec_p;
1666 	int err = DDI_SUCCESS;
1667 
1668 	/*
1669 	 * Note that i_ndi_busop_access_enter ends up grabbing the pokefault
1670 	 * mutex.
1671 	 */
1672 	i_ndi_busop_access_enter(hp->ahi_common.ah_dip, (ddi_acc_handle_t)hp);
1673 
1674 	pec_p->pec_ontrap_data = (on_trap_data_t *)hp->ahi_err->err_ontrap;
1675 	pec_p->pec_safeacc_type = DDI_FM_ERR_EXPECTED;
1676 	hp->ahi_err->err_expected = DDI_FM_ERR_EXPECTED;
1677 
1678 	if (repcount == 1) {
1679 		if (!i_ddi_ontrap((ddi_acc_handle_t)hp)) {
1680 			i_ddi_caut_get(size, (void *)dev_addr,
1681 			    (void *)host_addr);
1682 		} else {
1683 			int i;
1684 			uint8_t *ff_addr = (uint8_t *)host_addr;
1685 			for (i = 0; i < size; i++)
1686 				*ff_addr++ = 0xff;
1687 
1688 			err = DDI_FAILURE;
1689 #ifdef  DEBUG
1690 			px_peekfault_cnt++;
1691 #endif
1692 		}
1693 	} else {
1694 		if (!i_ddi_ontrap((ddi_acc_handle_t)hp)) {
1695 			for (; repcount; repcount--) {
1696 				i_ddi_caut_get(size, (void *)dev_addr,
1697 				    (void *)host_addr);
1698 
1699 				host_addr += size;
1700 
1701 				if (flags == DDI_DEV_AUTOINCR)
1702 					dev_addr += size;
1703 			}
1704 		} else {
1705 			err = DDI_FAILURE;
1706 #ifdef  DEBUG
1707 			px_peekfault_cnt++;
1708 #endif
1709 		}
1710 	}
1711 
1712 	i_ddi_notrap((ddi_acc_handle_t)hp);
1713 	pec_p->pec_ontrap_data = NULL;
1714 	pec_p->pec_safeacc_type = DDI_FM_ERR_UNEXPECTED;
1715 	i_ndi_busop_access_exit(hp->ahi_common.ah_dip, (ddi_acc_handle_t)hp);
1716 	hp->ahi_err->err_expected = DDI_FM_ERR_UNEXPECTED;
1717 
1718 	return (err);
1719 }
1720 
1721 /*ARGSUSED*/
1722 int
1723 px_lib_ctlops_peek(dev_info_t *dip, dev_info_t *rdip,
1724     peekpoke_ctlops_t *in_args, void *result)
1725 {
1726 	result = (void *)in_args->host_addr;
1727 	return (in_args->handle ? px_lib_do_caut_get(dip, in_args) :
1728 	    px_lib_do_peek(dip, in_args));
1729 }
1730 
1731 /*
1732  * implements PPM interface
1733  */
1734 int
1735 px_lib_pmctl(int cmd, px_t *px_p)
1736 {
1737 	ASSERT((cmd & ~PPMREQ_MASK) == PPMREQ);
1738 	switch (cmd) {
1739 	case PPMREQ_PRE_PWR_OFF:
1740 		/*
1741 		 * Currently there is no device power management for
1742 		 * the root complex (fire). When there is we need to make
1743 		 * sure that it is at full power before trying to send the
1744 		 * PME_Turn_Off message.
1745 		 */
1746 		DBG(DBG_PWR, px_p->px_dip,
1747 		    "ioctl: request to send PME_Turn_Off\n");
1748 		return (px_goto_l23ready(px_p));
1749 
1750 	case PPMREQ_PRE_PWR_ON:
1751 		DBG(DBG_PWR, px_p->px_dip, "ioctl: PRE_PWR_ON request\n");
1752 		return (px_pre_pwron_check(px_p));
1753 
1754 	case PPMREQ_POST_PWR_ON:
1755 		DBG(DBG_PWR, px_p->px_dip, "ioctl: POST_PWR_ON request\n");
1756 		return (px_goto_l0(px_p));
1757 
1758 	default:
1759 		return (DDI_FAILURE);
1760 	}
1761 }
1762 
1763 /*
1764  * sends PME_Turn_Off message to put the link in L2/L3 ready state.
1765  * called by px_ioctl.
1766  * returns DDI_SUCCESS or DDI_FAILURE
1767  * 1. Wait for link to be in L1 state (link status reg)
1768  * 2. write to PME_Turn_off reg to boradcast
1769  * 3. set timeout
1770  * 4. If timeout, return failure.
1771  * 5. If PM_TO_Ack, wait till link is in L2/L3 ready
1772  */
1773 static int
1774 px_goto_l23ready(px_t *px_p)
1775 {
1776 	pcie_pwr_t	*pwr_p;
1777 	pxu_t		*pxu_p = (pxu_t *)px_p->px_plat_p;
1778 	caddr_t	csr_base = (caddr_t)pxu_p->px_address[PX_REG_CSR];
1779 	int		ret = DDI_SUCCESS;
1780 	clock_t		end, timeleft;
1781 	int		mutex_held = 1;
1782 
1783 	/* If no PM info, return failure */
1784 	if (!PCIE_PMINFO(px_p->px_dip) ||
1785 	    !(pwr_p = PCIE_NEXUS_PMINFO(px_p->px_dip)))
1786 		return (DDI_FAILURE);
1787 
1788 	mutex_enter(&pwr_p->pwr_lock);
1789 	mutex_enter(&px_p->px_l23ready_lock);
1790 	/* Clear the PME_To_ACK receieved flag */
1791 	px_p->px_pm_flags &= ~PX_PMETOACK_RECVD;
1792 	/*
1793 	 * When P25 is the downstream device, after receiving
1794 	 * PME_To_ACK, fire will go to Detect state, which causes
1795 	 * the link down event. Inform FMA that this is expected.
1796 	 * In case of all other cards complaint with the pci express
1797 	 * spec, this will happen when the power is re-applied. FMA
1798 	 * code will clear this flag after one instance of LDN. Since
1799 	 * there will not be a LDN event for the spec compliant cards,
1800 	 * we need to clear the flag after receiving PME_To_ACK.
1801 	 */
1802 	px_p->px_pm_flags |= PX_LDN_EXPECTED;
1803 	if (px_send_pme_turnoff(csr_base) != DDI_SUCCESS) {
1804 		ret = DDI_FAILURE;
1805 		goto l23ready_done;
1806 	}
1807 	px_p->px_pm_flags |= PX_PME_TURNOFF_PENDING;
1808 
1809 	end = ddi_get_lbolt() + drv_usectohz(px_pme_to_ack_timeout);
1810 	while (!(px_p->px_pm_flags & PX_PMETOACK_RECVD)) {
1811 		timeleft = cv_timedwait(&px_p->px_l23ready_cv,
1812 		    &px_p->px_l23ready_lock, end);
1813 		/*
1814 		 * if cv_timedwait returns -1, it is either
1815 		 * 1) timed out or
1816 		 * 2) there was a pre-mature wakeup but by the time
1817 		 * cv_timedwait is called again end < lbolt i.e.
1818 		 * end is in the past.
1819 		 * 3) By the time we make first cv_timedwait call,
1820 		 * end < lbolt is true.
1821 		 */
1822 		if (timeleft == -1)
1823 			break;
1824 	}
1825 	if (!(px_p->px_pm_flags & PX_PMETOACK_RECVD)) {
1826 		/*
1827 		 * Either timedout or interrupt didn't get a
1828 		 * chance to grab the mutex and set the flag.
1829 		 * release the mutex and delay for sometime.
1830 		 * This will 1) give a chance for interrupt to
1831 		 * set the flag 2) creates a delay between two
1832 		 * consequetive requests.
1833 		 */
1834 		mutex_exit(&px_p->px_l23ready_lock);
1835 		delay(drv_usectohz(50 * PX_MSEC_TO_USEC));
1836 		mutex_held = 0;
1837 		if (!(px_p->px_pm_flags & PX_PMETOACK_RECVD)) {
1838 			ret = DDI_FAILURE;
1839 			DBG(DBG_PWR, px_p->px_dip, " Timed out while waiting"
1840 			    " for PME_TO_ACK\n");
1841 		}
1842 	}
1843 	px_p->px_pm_flags &=
1844 	    ~(PX_PME_TURNOFF_PENDING | PX_PMETOACK_RECVD | PX_LDN_EXPECTED);
1845 
1846 l23ready_done:
1847 	if (mutex_held)
1848 		mutex_exit(&px_p->px_l23ready_lock);
1849 	/*
1850 	 * Wait till link is in L1 idle, if sending PME_Turn_Off
1851 	 * was succesful.
1852 	 */
1853 	if (ret == DDI_SUCCESS) {
1854 		if (px_link_wait4l1idle(csr_base) != DDI_SUCCESS) {
1855 			DBG(DBG_PWR, px_p->px_dip, " Link is not at L1"
1856 			    " even though we received PME_To_ACK.\n");
1857 			/*
1858 			 * Workaround for hardware bug with P25.
1859 			 * Due to a hardware bug with P25, link state
1860 			 * will be Detect state rather than L1 after
1861 			 * link is transitioned to L23Ready state. Since
1862 			 * we don't know whether link is L23ready state
1863 			 * without Fire's state being L1_idle, we delay
1864 			 * here just to make sure that we wait till link
1865 			 * is transitioned to L23Ready state.
1866 			 */
1867 			delay(drv_usectohz(100 * PX_MSEC_TO_USEC));
1868 		}
1869 		pwr_p->pwr_link_lvl = PM_LEVEL_L3;
1870 
1871 	}
1872 	mutex_exit(&pwr_p->pwr_lock);
1873 	return (ret);
1874 }
1875 
1876 /*
1877  * Message interrupt handler intended to be shared for both
1878  * PME and PME_TO_ACK msg handling, currently only handles
1879  * PME_To_ACK message.
1880  */
1881 uint_t
1882 px_pmeq_intr(caddr_t arg)
1883 {
1884 	px_t	*px_p = (px_t *)arg;
1885 
1886 	DBG(DBG_PWR, px_p->px_dip, " PME_To_ACK received \n");
1887 	mutex_enter(&px_p->px_l23ready_lock);
1888 	cv_broadcast(&px_p->px_l23ready_cv);
1889 	if (px_p->px_pm_flags & PX_PME_TURNOFF_PENDING) {
1890 		px_p->px_pm_flags |= PX_PMETOACK_RECVD;
1891 	} else {
1892 		/*
1893 		 * This maybe the second ack received. If so then,
1894 		 * we should be receiving it during wait4L1 stage.
1895 		 */
1896 		px_p->px_pmetoack_ignored++;
1897 	}
1898 	mutex_exit(&px_p->px_l23ready_lock);
1899 	return (DDI_INTR_CLAIMED);
1900 }
1901 
1902 static int
1903 px_pre_pwron_check(px_t *px_p)
1904 {
1905 	pcie_pwr_t	*pwr_p;
1906 
1907 	/* If no PM info, return failure */
1908 	if (!PCIE_PMINFO(px_p->px_dip) ||
1909 	    !(pwr_p = PCIE_NEXUS_PMINFO(px_p->px_dip)))
1910 		return (DDI_FAILURE);
1911 
1912 	/*
1913 	 * For the spec compliant downstream cards link down
1914 	 * is expected when the device is powered on.
1915 	 */
1916 	px_p->px_pm_flags |= PX_LDN_EXPECTED;
1917 	return (pwr_p->pwr_link_lvl == PM_LEVEL_L3 ? DDI_SUCCESS : DDI_FAILURE);
1918 }
1919 
1920 static int
1921 px_goto_l0(px_t *px_p)
1922 {
1923 	pcie_pwr_t	*pwr_p;
1924 	pxu_t		*pxu_p = (pxu_t *)px_p->px_plat_p;
1925 	caddr_t csr_base = (caddr_t)pxu_p->px_address[PX_REG_CSR];
1926 	int		ret = DDI_SUCCESS;
1927 	uint64_t	time_spent = 0;
1928 
1929 	/* If no PM info, return failure */
1930 	if (!PCIE_PMINFO(px_p->px_dip) ||
1931 	    !(pwr_p = PCIE_NEXUS_PMINFO(px_p->px_dip)))
1932 		return (DDI_FAILURE);
1933 
1934 	mutex_enter(&pwr_p->pwr_lock);
1935 	/*
1936 	 * The following link retrain activity will cause LDN and LUP event.
1937 	 * Receiving LDN prior to receiving LUP is expected, not an error in
1938 	 * this case.  Receiving LUP indicates link is fully up to support
1939 	 * powering up down stream device, and of course any further LDN and
1940 	 * LUP outside this context will be error.
1941 	 */
1942 	px_p->px_lup_pending = 1;
1943 	if (px_link_retrain(csr_base) != DDI_SUCCESS) {
1944 		ret = DDI_FAILURE;
1945 		goto l0_done;
1946 	}
1947 
1948 	/* LUP event takes the order of 15ms amount of time to occur */
1949 	for (; px_p->px_lup_pending && (time_spent < px_lup_poll_to);
1950 	    time_spent += px_lup_poll_interval)
1951 		drv_usecwait(px_lup_poll_interval);
1952 	if (px_p->px_lup_pending)
1953 		ret = DDI_FAILURE;
1954 l0_done:
1955 	px_enable_detect_quiet(csr_base);
1956 	if (ret == DDI_SUCCESS)
1957 		pwr_p->pwr_link_lvl = PM_LEVEL_L0;
1958 	mutex_exit(&pwr_p->pwr_lock);
1959 	return (ret);
1960 }
1961 
1962 /*
1963  * Extract the drivers binding name to identify which chip we're binding to.
1964  * Whenever a new bus bridge is created, the driver alias entry should be
1965  * added here to identify the device if needed.  If a device isn't added,
1966  * the identity defaults to PX_CHIP_UNIDENTIFIED.
1967  */
1968 static uint32_t
1969 px_identity_chip(px_t *px_p)
1970 {
1971 	dev_info_t	*dip = px_p->px_dip;
1972 	char		*name = ddi_binding_name(dip);
1973 	uint32_t	revision = 0;
1974 
1975 	revision = ddi_prop_get_int(DDI_DEV_T_ANY, dip, DDI_PROP_DONTPASS,
1976 	    "module-revision#", 0);
1977 
1978 	/* Check for Fire driver binding name */
1979 	if ((strcmp(name, "pci108e,80f0") == 0) ||
1980 	    (strcmp(name, "pciex108e,80f0") == 0)) {
1981 		DBG(DBG_ATTACH, dip, "px_identity_chip: %s%d: "
1982 		    "name %s module-revision %d\n", ddi_driver_name(dip),
1983 		    ddi_get_instance(dip), name, revision);
1984 
1985 		return (PX_CHIP_ID(PX_CHIP_FIRE, revision, 0x00));
1986 	}
1987 
1988 	/* Check for Oberon driver binding name */
1989 	if (strcmp(name, "pciex108e,80f8") == 0) {
1990 		DBG(DBG_ATTACH, dip, "px_identity_chip: %s%d: "
1991 		    "name %s module-revision %d\n", ddi_driver_name(dip),
1992 		    ddi_get_instance(dip), name, revision);
1993 
1994 		return (PX_CHIP_ID(PX_CHIP_OBERON, revision, 0x00));
1995 	}
1996 
1997 	DBG(DBG_ATTACH, dip, "%s%d: Unknown PCI Express Host bridge %s %x\n",
1998 	    ddi_driver_name(dip), ddi_get_instance(dip), name, revision);
1999 
2000 	return (PX_CHIP_UNIDENTIFIED);
2001 }
2002 
2003 int
2004 px_err_add_intr(px_fault_t *px_fault_p)
2005 {
2006 	dev_info_t	*dip = px_fault_p->px_fh_dip;
2007 	px_t		*px_p = DIP_TO_STATE(dip);
2008 
2009 	VERIFY(add_ivintr(px_fault_p->px_fh_sysino, PX_ERR_PIL,
2010 		px_fault_p->px_err_func, (caddr_t)px_fault_p, NULL) == 0);
2011 
2012 	px_ib_intr_enable(px_p, intr_dist_cpuid(), px_fault_p->px_intr_ino);
2013 
2014 	return (DDI_SUCCESS);
2015 }
2016 
2017 void
2018 px_err_rem_intr(px_fault_t *px_fault_p)
2019 {
2020 	dev_info_t	*dip = px_fault_p->px_fh_dip;
2021 	px_t		*px_p = DIP_TO_STATE(dip);
2022 
2023 	px_ib_intr_disable(px_p->px_ib_p, px_fault_p->px_intr_ino,
2024 		IB_INTR_WAIT);
2025 
2026 	rem_ivintr(px_fault_p->px_fh_sysino, NULL);
2027 }
2028 
2029 /*
2030  * px_cb_add_intr() - Called from attach(9E) to create CB if not yet
2031  * created, to add CB interrupt vector always, but enable only once.
2032  */
2033 int
2034 px_cb_add_intr(px_fault_t *fault_p)
2035 {
2036 	px_t		*px_p = DIP_TO_STATE(fault_p->px_fh_dip);
2037 	pxu_t		*pxu_p = (pxu_t *)px_p->px_plat_p;
2038 	px_cb_t		*cb_p = (px_cb_t *)px_get_cb(fault_p->px_fh_dip);
2039 	px_cb_list_t	*pxl, *pxl_new;
2040 	cpuid_t		cpuid;
2041 
2042 
2043 	if (cb_p == NULL) {
2044 		cb_p = kmem_zalloc(sizeof (px_cb_t), KM_SLEEP);
2045 		mutex_init(&cb_p->cb_mutex, NULL, MUTEX_DRIVER, NULL);
2046 		cb_p->px_cb_func = px_cb_intr;
2047 		pxu_p->px_cb_p = cb_p;
2048 		px_set_cb(fault_p->px_fh_dip, (uint64_t)cb_p);
2049 	} else
2050 		pxu_p->px_cb_p = cb_p;
2051 
2052 	mutex_enter(&cb_p->cb_mutex);
2053 
2054 	VERIFY(add_ivintr(fault_p->px_fh_sysino, PX_ERR_PIL,
2055 	    cb_p->px_cb_func, (caddr_t)cb_p, NULL) == 0);
2056 
2057 	if (cb_p->pxl == NULL) {
2058 
2059 		cpuid = intr_dist_cpuid(),
2060 		px_ib_intr_enable(px_p, cpuid, fault_p->px_intr_ino);
2061 
2062 		pxl = kmem_zalloc(sizeof (px_cb_list_t), KM_SLEEP);
2063 		pxl->pxp = px_p;
2064 
2065 		cb_p->pxl = pxl;
2066 		cb_p->sysino = fault_p->px_fh_sysino;
2067 		cb_p->cpuid = cpuid;
2068 
2069 	} else {
2070 		/*
2071 		 * Find the last pxl or
2072 		 * stop short at encoutering a redundent, or
2073 		 * both.
2074 		 */
2075 		pxl = cb_p->pxl;
2076 		for (; !(pxl->pxp == px_p) && pxl->next; pxl = pxl->next);
2077 		if (pxl->pxp == px_p) {
2078 			cmn_err(CE_WARN, "px_cb_add_intr: reregister sysino "
2079 			    "%lx by px_p 0x%p\n", cb_p->sysino, (void *)px_p);
2080 			return (DDI_FAILURE);
2081 		}
2082 
2083 		/* add to linked list */
2084 		pxl_new = kmem_zalloc(sizeof (px_cb_list_t), KM_SLEEP);
2085 		pxl_new->pxp = px_p;
2086 		pxl->next = pxl_new;
2087 	}
2088 	cb_p->attachcnt++;
2089 
2090 	mutex_exit(&cb_p->cb_mutex);
2091 
2092 	return (DDI_SUCCESS);
2093 }
2094 
2095 /*
2096  * px_cb_rem_intr() - Called from detach(9E) to remove its CB
2097  * interrupt vector, to shift proxy to the next available px,
2098  * or disable CB interrupt when itself is the last.
2099  */
2100 void
2101 px_cb_rem_intr(px_fault_t *fault_p)
2102 {
2103 	px_t		*px_p = DIP_TO_STATE(fault_p->px_fh_dip), *pxp;
2104 	pxu_t		*pxu_p = (pxu_t *)px_p->px_plat_p;
2105 	px_cb_t		*cb_p = PX2CB(px_p);
2106 	px_cb_list_t	*pxl, *prev;
2107 	px_fault_t	*f_p;
2108 
2109 	ASSERT(cb_p->pxl);
2110 
2111 	/* De-list the target px, move the next px up */
2112 
2113 	mutex_enter(&cb_p->cb_mutex);
2114 
2115 	pxl = cb_p->pxl;
2116 	if (pxl->pxp == px_p) {
2117 		cb_p->pxl = pxl->next;
2118 	} else {
2119 		prev = pxl;
2120 		pxl = pxl->next;
2121 		for (; pxl && (pxl->pxp != px_p); prev = pxl, pxl = pxl->next);
2122 		if (!pxl) {
2123 			cmn_err(CE_WARN, "px_cb_rem_intr: can't find px_p 0x%p "
2124 			    "in registered CB list.", (void *)px_p);
2125 			return;
2126 		}
2127 		prev->next = pxl->next;
2128 	}
2129 	kmem_free(pxl, sizeof (px_cb_list_t));
2130 
2131 	if (fault_p->px_fh_sysino == cb_p->sysino) {
2132 		px_ib_intr_disable(px_p->px_ib_p, fault_p->px_intr_ino,
2133 		    IB_INTR_WAIT);
2134 
2135 		if (cb_p->pxl) {
2136 			pxp = cb_p->pxl->pxp;
2137 			f_p = &pxp->px_cb_fault;
2138 			cb_p->sysino = f_p->px_fh_sysino;
2139 
2140 			PX_INTR_ENABLE(pxp->px_dip, cb_p->sysino, cb_p->cpuid);
2141 			(void) px_lib_intr_setstate(pxp->px_dip, cb_p->sysino,
2142 			    INTR_IDLE_STATE);
2143 		}
2144 	}
2145 
2146 	rem_ivintr(fault_p->px_fh_sysino, NULL);
2147 	pxu_p->px_cb_p = NULL;
2148 	cb_p->attachcnt--;
2149 	if (cb_p->pxl) {
2150 		mutex_exit(&cb_p->cb_mutex);
2151 		return;
2152 	}
2153 	mutex_exit(&cb_p->cb_mutex);
2154 
2155 	mutex_destroy(&cb_p->cb_mutex);
2156 	px_set_cb(fault_p->px_fh_dip, 0ull);
2157 	kmem_free(cb_p, sizeof (px_cb_t));
2158 }
2159 
2160 /*
2161  * px_cb_intr() - sun4u only,  CB interrupt dispatcher
2162  */
2163 uint_t
2164 px_cb_intr(caddr_t arg)
2165 {
2166 	px_cb_t		*cb_p = (px_cb_t *)arg;
2167 	px_cb_list_t	*pxl = cb_p->pxl;
2168 	px_t		*pxp = pxl ? pxl->pxp : NULL;
2169 	px_fault_t	*fault_p;
2170 
2171 	while (pxl && pxp && (pxp->px_state != PX_ATTACHED)) {
2172 		pxl = pxl->next;
2173 		pxp = (pxl) ? pxl->pxp : NULL;
2174 	}
2175 
2176 	if (pxp) {
2177 		fault_p = &pxp->px_cb_fault;
2178 		return (fault_p->px_err_func((caddr_t)fault_p));
2179 	} else
2180 		return (DDI_INTR_UNCLAIMED);
2181 }
2182 
2183 /*
2184  * px_cb_intr_redist() - sun4u only, CB interrupt redistribution
2185  */
2186 void
2187 px_cb_intr_redist(px_t	*px_p)
2188 {
2189 	px_fault_t	*f_p = &px_p->px_cb_fault;
2190 	px_cb_t		*cb_p = PX2CB(px_p);
2191 	devino_t	ino = px_p->px_inos[PX_INTR_XBC];
2192 	cpuid_t		cpuid;
2193 
2194 	mutex_enter(&cb_p->cb_mutex);
2195 
2196 	if (cb_p->sysino != f_p->px_fh_sysino) {
2197 		mutex_exit(&cb_p->cb_mutex);
2198 		return;
2199 	}
2200 
2201 	cb_p->cpuid = cpuid = intr_dist_cpuid();
2202 	px_ib_intr_dist_en(px_p->px_dip, cpuid, ino, B_FALSE);
2203 
2204 	mutex_exit(&cb_p->cb_mutex);
2205 }
2206 
2207 #ifdef FMA
2208 void
2209 px_fill_rc_status(px_fault_t *px_fault_p, pciex_rc_error_regs_t *rc_status)
2210 {
2211 	/* populate the rc_status by reading the registers - TBD */
2212 }
2213 #endif /* FMA */
2214 
2215 /*
2216  * Unprotected raw reads/writes of fabric device's config space.
2217  * Only used for temporary PCI-E Fabric Error Handling.
2218  */
2219 uint32_t
2220 px_fab_get(px_t *px_p, pcie_req_id_t bdf, uint16_t offset)
2221 {
2222 	px_ranges_t	*rp = px_p->px_ranges_p;
2223 	uint64_t	range_prop, base_addr;
2224 	int		bank = PCI_REG_ADDR_G(PCI_ADDR_CONFIG);
2225 	uint32_t	val;
2226 
2227 	/* Get Fire's Physical Base Address */
2228 	range_prop = px_get_range_prop(px_p, rp, bank);
2229 
2230 	/* Get config space first. */
2231 	base_addr = range_prop + PX_BDF_TO_CFGADDR(bdf, offset);
2232 
2233 	val = ldphysio(base_addr);
2234 
2235 	return (LE_32(val));
2236 }
2237 
2238 void
2239 px_fab_set(px_t *px_p, pcie_req_id_t bdf, uint16_t offset,
2240     uint32_t val) {
2241 	px_ranges_t	*rp = px_p->px_ranges_p;
2242 	uint64_t	range_prop, base_addr;
2243 	int		bank = PCI_REG_ADDR_G(PCI_ADDR_CONFIG);
2244 
2245 	/* Get Fire's Physical Base Address */
2246 	range_prop = px_get_range_prop(px_p, rp, bank);
2247 
2248 	/* Get config space first. */
2249 	base_addr = range_prop + PX_BDF_TO_CFGADDR(bdf, offset);
2250 
2251 	stphysio(base_addr, LE_32(val));
2252 }
2253 
2254 /*
2255  * cpr callback
2256  *
2257  * disable fabric error msg interrupt prior to suspending
2258  * all device drivers; re-enable fabric error msg interrupt
2259  * after all devices are resumed.
2260  */
2261 static boolean_t
2262 px_cpr_callb(void *arg, int code)
2263 {
2264 	px_t		*px_p = (px_t *)arg;
2265 	px_ib_t		*ib_p = px_p->px_ib_p;
2266 	px_pec_t	*pec_p = px_p->px_pec_p;
2267 	pxu_t		*pxu_p = (pxu_t *)px_p->px_plat_p;
2268 	caddr_t		csr_base;
2269 	devino_t	ce_ino, nf_ino, f_ino;
2270 	px_ib_ino_info_t	*ce_ino_p, *nf_ino_p, *f_ino_p;
2271 	uint64_t	imu_log_enable, imu_intr_enable;
2272 	uint64_t	imu_log_mask, imu_intr_mask;
2273 
2274 	ce_ino = px_msiqid_to_devino(px_p, pec_p->pec_corr_msg_msiq_id);
2275 	nf_ino = px_msiqid_to_devino(px_p, pec_p->pec_non_fatal_msg_msiq_id);
2276 	f_ino = px_msiqid_to_devino(px_p, pec_p->pec_fatal_msg_msiq_id);
2277 	csr_base = (caddr_t)pxu_p->px_address[PX_REG_CSR];
2278 
2279 	imu_log_enable = CSR_XR(csr_base, IMU_ERROR_LOG_ENABLE);
2280 	imu_intr_enable = CSR_XR(csr_base, IMU_INTERRUPT_ENABLE);
2281 
2282 	imu_log_mask = BITMASK(IMU_ERROR_LOG_ENABLE_FATAL_MES_NOT_EN_LOG_EN) |
2283 	    BITMASK(IMU_ERROR_LOG_ENABLE_NONFATAL_MES_NOT_EN_LOG_EN) |
2284 	    BITMASK(IMU_ERROR_LOG_ENABLE_COR_MES_NOT_EN_LOG_EN);
2285 
2286 	imu_intr_mask =
2287 	    BITMASK(IMU_INTERRUPT_ENABLE_FATAL_MES_NOT_EN_S_INT_EN) |
2288 	    BITMASK(IMU_INTERRUPT_ENABLE_NONFATAL_MES_NOT_EN_S_INT_EN) |
2289 	    BITMASK(IMU_INTERRUPT_ENABLE_COR_MES_NOT_EN_S_INT_EN) |
2290 	    BITMASK(IMU_INTERRUPT_ENABLE_FATAL_MES_NOT_EN_P_INT_EN) |
2291 	    BITMASK(IMU_INTERRUPT_ENABLE_NONFATAL_MES_NOT_EN_P_INT_EN) |
2292 	    BITMASK(IMU_INTERRUPT_ENABLE_COR_MES_NOT_EN_P_INT_EN);
2293 
2294 	switch (code) {
2295 	case CB_CODE_CPR_CHKPT:
2296 		/* disable imu rbne on corr/nonfatal/fatal errors */
2297 		CSR_XS(csr_base, IMU_ERROR_LOG_ENABLE,
2298 		    imu_log_enable & (~imu_log_mask));
2299 
2300 		CSR_XS(csr_base, IMU_INTERRUPT_ENABLE,
2301 		    imu_intr_enable & (~imu_intr_mask));
2302 
2303 		/* disable CORR intr mapping */
2304 		px_ib_intr_disable(ib_p, ce_ino, IB_INTR_NOWAIT);
2305 
2306 		/* disable NON FATAL intr mapping */
2307 		px_ib_intr_disable(ib_p, nf_ino, IB_INTR_NOWAIT);
2308 
2309 		/* disable FATAL intr mapping */
2310 		px_ib_intr_disable(ib_p, f_ino, IB_INTR_NOWAIT);
2311 
2312 		break;
2313 
2314 	case CB_CODE_CPR_RESUME:
2315 		mutex_enter(&ib_p->ib_ino_lst_mutex);
2316 
2317 		ce_ino_p = px_ib_locate_ino(ib_p, ce_ino);
2318 		nf_ino_p = px_ib_locate_ino(ib_p, nf_ino);
2319 		f_ino_p = px_ib_locate_ino(ib_p, f_ino);
2320 
2321 		/* enable CORR intr mapping */
2322 		if (ce_ino_p)
2323 			px_ib_intr_enable(px_p, ce_ino_p->ino_cpuid, ce_ino);
2324 		else
2325 			cmn_err(CE_WARN, "px_cpr_callb: RESUME unable to "
2326 			    "reenable PCIe Correctable msg intr.\n");
2327 
2328 		/* enable NON FATAL intr mapping */
2329 		if (nf_ino_p)
2330 			px_ib_intr_enable(px_p, nf_ino_p->ino_cpuid, nf_ino);
2331 		else
2332 			cmn_err(CE_WARN, "px_cpr_callb: RESUME unable to "
2333 			    "reenable PCIe Non Fatal msg intr.\n");
2334 
2335 		/* enable FATAL intr mapping */
2336 		if (f_ino_p)
2337 			px_ib_intr_enable(px_p, f_ino_p->ino_cpuid, f_ino);
2338 		else
2339 			cmn_err(CE_WARN, "px_cpr_callb: RESUME unable to "
2340 			    "reenable PCIe Fatal msg intr.\n");
2341 
2342 		mutex_exit(&ib_p->ib_ino_lst_mutex);
2343 
2344 		/* enable corr/nonfatal/fatal not enable error */
2345 		CSR_XS(csr_base, IMU_ERROR_LOG_ENABLE, (imu_log_enable |
2346 		    (imu_log_mask & px_imu_log_mask)));
2347 		CSR_XS(csr_base, IMU_INTERRUPT_ENABLE, (imu_intr_enable |
2348 		    (imu_intr_mask & px_imu_intr_mask)));
2349 
2350 		break;
2351 	}
2352 
2353 	return (B_TRUE);
2354 }
2355 
2356 /*
2357  * fetch chip's range propery's value
2358  */
2359 uint64_t
2360 px_get_range_prop(px_t *px_p, px_ranges_t *rp, int bank)
2361 {
2362 	pxu_t *pxu_p = (pxu_t *)px_p->px_plat_p;
2363 	uint64_t mask, range_prop;
2364 
2365 	switch (PX_CHIP_TYPE(pxu_p)) {
2366 	case PX_CHIP_OBERON:
2367 		mask = OBERON_RANGE_PROP_MASK;
2368 		break;
2369 	case PX_CHIP_FIRE:
2370 		mask = FIRE_RANGE_PROP_MASK;
2371 		break;
2372 	default:
2373 		mask = FIRE_RANGE_PROP_MASK;
2374 	}
2375 	range_prop = (((uint64_t)(rp[bank].parent_high & mask)) << 32) |
2376 		rp[bank].parent_low;
2377 
2378 	return (range_prop);
2379 }
2380 
2381 /*
2382  * add cpr callback
2383  */
2384 void
2385 px_cpr_add_callb(px_t *px_p)
2386 {
2387 	px_p->px_cprcb_id = callb_add(px_cpr_callb, (void *)px_p,
2388 	CB_CL_CPR_POST_USER, "px_cpr");
2389 }
2390 
2391 /*
2392  * remove cpr callback
2393  */
2394 void
2395 px_cpr_rem_callb(px_t *px_p)
2396 {
2397 	(void) callb_delete(px_p->px_cprcb_id);
2398 }
2399 
2400 /*ARGSUSED*/
2401 static uint_t
2402 px_hp_intr(caddr_t arg1, caddr_t arg2)
2403 {
2404 	px_t *px_p = (px_t *)arg1;
2405 	int rval;
2406 
2407 	rval = pciehpc_intr(px_p->px_dip);
2408 
2409 #ifdef  DEBUG
2410 	if (rval == DDI_INTR_UNCLAIMED)
2411 	    cmn_err(CE_WARN, "%s%d: UNCLAIMED intr\n",
2412 		ddi_driver_name(px_p->px_dip),
2413 		ddi_get_instance(px_p->px_dip));
2414 #endif
2415 
2416 	return (rval);
2417 }
2418 
2419 int
2420 px_lib_hotplug_init(dev_info_t *dip, void *arg)
2421 {
2422 	px_t	*px_p = DIP_TO_STATE(dip);
2423 	uint64_t ret;
2424 
2425 	if ((ret = hvio_hotplug_init(dip, arg)) == DDI_SUCCESS) {
2426 		sysino_t sysino;
2427 
2428 		if (px_lib_intr_devino_to_sysino(px_p->px_dip,
2429 		    px_p->px_inos[PX_INTR_HOTPLUG], &sysino) !=
2430 		    DDI_SUCCESS) {
2431 #ifdef	DEBUG
2432 			cmn_err(CE_WARN, "%s%d: devino_to_sysino fails\n",
2433 			    ddi_driver_name(px_p->px_dip),
2434 			    ddi_get_instance(px_p->px_dip));
2435 #endif
2436 			return (DDI_FAILURE);
2437 		}
2438 
2439 		VERIFY(add_ivintr(sysino, PX_PCIEHP_PIL,
2440 		    (intrfunc)px_hp_intr, (caddr_t)px_p, NULL) == 0);
2441 	}
2442 
2443 	return (ret);
2444 }
2445 
2446 void
2447 px_lib_hotplug_uninit(dev_info_t *dip)
2448 {
2449 	if (hvio_hotplug_uninit(dip) == DDI_SUCCESS) {
2450 		px_t	*px_p = DIP_TO_STATE(dip);
2451 		sysino_t sysino;
2452 
2453 		if (px_lib_intr_devino_to_sysino(px_p->px_dip,
2454 		    px_p->px_inos[PX_INTR_HOTPLUG], &sysino) !=
2455 		    DDI_SUCCESS) {
2456 #ifdef	DEBUG
2457 			cmn_err(CE_WARN, "%s%d: devino_to_sysino fails\n",
2458 			    ddi_driver_name(px_p->px_dip),
2459 			    ddi_get_instance(px_p->px_dip));
2460 #endif
2461 			return;
2462 		}
2463 
2464 		rem_ivintr(sysino, NULL);
2465 	}
2466 }
2467