xref: /titanic_44/usr/src/uts/sun4u/io/px/px_lib4u.c (revision 767b0abf70408797bf5ca4a8dac501bb1a90003d)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 /*
22  * Copyright 2007 Sun Microsystems, Inc.  All rights reserved.
23  * Use is subject to license terms.
24  */
25 
26 #pragma ident	"%Z%%M%	%I%	%E% SMI"
27 
28 #include <sys/types.h>
29 #include <sys/kmem.h>
30 #include <sys/conf.h>
31 #include <sys/ddi.h>
32 #include <sys/sunddi.h>
33 #include <sys/fm/protocol.h>
34 #include <sys/fm/util.h>
35 #include <sys/modctl.h>
36 #include <sys/disp.h>
37 #include <sys/stat.h>
38 #include <sys/ddi_impldefs.h>
39 #include <sys/vmem.h>
40 #include <sys/iommutsb.h>
41 #include <sys/cpuvar.h>
42 #include <sys/ivintr.h>
43 #include <sys/byteorder.h>
44 #include <sys/hotplug/pci/pciehpc.h>
45 #include <px_obj.h>
46 #include <pcie_pwr.h>
47 #include "px_tools_var.h"
48 #include <px_regs.h>
49 #include <px_csr.h>
50 #include <sys/machsystm.h>
51 #include "px_lib4u.h"
52 #include "px_err.h"
53 #include "oberon_regs.h"
54 
55 #pragma weak jbus_stst_order
56 
57 extern void jbus_stst_order();
58 
59 ulong_t px_mmu_dvma_end = 0xfffffffful;
60 uint_t px_ranges_phi_mask = 0xfffffffful;
61 uint64_t *px_oberon_ubc_scratch_regs;
62 uint64_t px_paddr_mask;
63 
64 static int px_goto_l23ready(px_t *px_p);
65 static int px_goto_l0(px_t *px_p);
66 static int px_pre_pwron_check(px_t *px_p);
67 static uint32_t px_identity_init(px_t *px_p);
68 static boolean_t px_cpr_callb(void *arg, int code);
69 static uint_t px_cb_intr(caddr_t arg);
70 
71 /*
72  * px_lib_map_registers
73  *
74  * This function is called from the attach routine to map the registers
75  * accessed by this driver.
76  *
77  * used by: px_attach()
78  *
79  * return value: DDI_FAILURE on failure
80  */
81 int
82 px_lib_map_regs(pxu_t *pxu_p, dev_info_t *dip)
83 {
84 	ddi_device_acc_attr_t	attr;
85 	px_reg_bank_t		reg_bank = PX_REG_CSR;
86 
87 	DBG(DBG_ATTACH, dip, "px_lib_map_regs: pxu_p:0x%p, dip 0x%p\n",
88 		pxu_p, dip);
89 
90 	attr.devacc_attr_version = DDI_DEVICE_ATTR_V0;
91 	attr.devacc_attr_dataorder = DDI_STRICTORDER_ACC;
92 	attr.devacc_attr_endian_flags = DDI_NEVERSWAP_ACC;
93 
94 	/*
95 	 * PCI CSR Base
96 	 */
97 	if (ddi_regs_map_setup(dip, reg_bank, &pxu_p->px_address[reg_bank],
98 	    0, 0, &attr, &pxu_p->px_ac[reg_bank]) != DDI_SUCCESS) {
99 		goto fail;
100 	}
101 
102 	reg_bank++;
103 
104 	/*
105 	 * XBUS CSR Base
106 	 */
107 	if (ddi_regs_map_setup(dip, reg_bank, &pxu_p->px_address[reg_bank],
108 	    0, 0, &attr, &pxu_p->px_ac[reg_bank]) != DDI_SUCCESS) {
109 		goto fail;
110 	}
111 
112 	pxu_p->px_address[reg_bank] -= FIRE_CONTROL_STATUS;
113 
114 done:
115 	for (; reg_bank >= PX_REG_CSR; reg_bank--) {
116 		DBG(DBG_ATTACH, dip, "reg_bank 0x%x address 0x%p\n",
117 		    reg_bank, pxu_p->px_address[reg_bank]);
118 	}
119 
120 	return (DDI_SUCCESS);
121 
122 fail:
123 	cmn_err(CE_WARN, "%s%d: unable to map reg entry %d\n",
124 	    ddi_driver_name(dip), ddi_get_instance(dip), reg_bank);
125 
126 	for (reg_bank--; reg_bank >= PX_REG_CSR; reg_bank--) {
127 		pxu_p->px_address[reg_bank] = NULL;
128 		ddi_regs_map_free(&pxu_p->px_ac[reg_bank]);
129 	}
130 
131 	return (DDI_FAILURE);
132 }
133 
134 /*
135  * px_lib_unmap_regs:
136  *
137  * This routine unmaps the registers mapped by map_px_registers.
138  *
139  * used by: px_detach(), and error conditions in px_attach()
140  *
141  * return value: none
142  */
143 void
144 px_lib_unmap_regs(pxu_t *pxu_p)
145 {
146 	int i;
147 
148 	for (i = 0; i < PX_REG_MAX; i++) {
149 		if (pxu_p->px_ac[i])
150 			ddi_regs_map_free(&pxu_p->px_ac[i]);
151 	}
152 }
153 
154 int
155 px_lib_dev_init(dev_info_t *dip, devhandle_t *dev_hdl)
156 {
157 
158 	caddr_t			xbc_csr_base, csr_base;
159 	px_dvma_range_prop_t	px_dvma_range;
160 	pxu_t			*pxu_p;
161 	uint8_t			chip_mask;
162 	px_t			*px_p = DIP_TO_STATE(dip);
163 	px_chip_type_t		chip_type = px_identity_init(px_p);
164 
165 	DBG(DBG_ATTACH, dip, "px_lib_dev_init: dip 0x%p", dip);
166 
167 	if (chip_type == PX_CHIP_UNIDENTIFIED) {
168 		cmn_err(CE_WARN, "%s%d: Unrecognized Hardware Version\n",
169 		    NAMEINST(dip));
170 		return (DDI_FAILURE);
171 	}
172 
173 	chip_mask = BITMASK(chip_type);
174 	px_paddr_mask = (chip_type == PX_CHIP_FIRE) ? MMU_FIRE_PADDR_MASK :
175 	    MMU_OBERON_PADDR_MASK;
176 
177 	/*
178 	 * Allocate platform specific structure and link it to
179 	 * the px state structure.
180 	 */
181 	pxu_p = kmem_zalloc(sizeof (pxu_t), KM_SLEEP);
182 	pxu_p->chip_type = chip_type;
183 	pxu_p->portid  = ddi_getprop(DDI_DEV_T_ANY, dip, DDI_PROP_DONTPASS,
184 	    "portid", -1);
185 
186 	/* Map in the registers */
187 	if (px_lib_map_regs(pxu_p, dip) == DDI_FAILURE) {
188 		kmem_free(pxu_p, sizeof (pxu_t));
189 
190 		return (DDI_FAILURE);
191 	}
192 
193 	xbc_csr_base = (caddr_t)pxu_p->px_address[PX_REG_XBC];
194 	csr_base = (caddr_t)pxu_p->px_address[PX_REG_CSR];
195 
196 	pxu_p->tsb_cookie = iommu_tsb_alloc(pxu_p->portid);
197 	pxu_p->tsb_size = iommu_tsb_cookie_to_size(pxu_p->tsb_cookie);
198 	pxu_p->tsb_vaddr = iommu_tsb_cookie_to_va(pxu_p->tsb_cookie);
199 
200 	pxu_p->tsb_paddr = va_to_pa(pxu_p->tsb_vaddr);
201 
202 	/*
203 	 * Create "virtual-dma" property to support child devices
204 	 * needing to know DVMA range.
205 	 */
206 	px_dvma_range.dvma_base = (uint32_t)px_mmu_dvma_end + 1
207 	    - ((pxu_p->tsb_size >> 3) << MMU_PAGE_SHIFT);
208 	px_dvma_range.dvma_len = (uint32_t)
209 	    px_mmu_dvma_end - px_dvma_range.dvma_base + 1;
210 
211 	(void) ddi_prop_create(DDI_DEV_T_NONE, dip, DDI_PROP_CANSLEEP,
212 		"virtual-dma", (caddr_t)&px_dvma_range,
213 		sizeof (px_dvma_range_prop_t));
214 	/*
215 	 * Initilize all fire hardware specific blocks.
216 	 */
217 	hvio_cb_init(xbc_csr_base, pxu_p);
218 	hvio_ib_init(csr_base, pxu_p);
219 	hvio_pec_init(csr_base, pxu_p);
220 	hvio_mmu_init(csr_base, pxu_p);
221 
222 	px_p->px_plat_p = (void *)pxu_p;
223 
224 	/*
225 	 * Initialize all the interrupt handlers
226 	 */
227 	switch (PX_CHIP_TYPE(pxu_p)) {
228 	case PX_CHIP_OBERON:
229 		/*
230 		 * Oberon hotplug uses SPARE3 field in ILU Error Log Enable
231 		 * register to indicate the status of leaf reset,
232 		 * we need to preserve the value of this bit, and keep it in
233 		 * px_ilu_log_mask to reflect the state of the bit
234 		 */
235 		if (CSR_BR(csr_base, ILU_ERROR_LOG_ENABLE, SPARE3))
236 			px_ilu_log_mask |= (1ull <<
237 			    ILU_ERROR_LOG_ENABLE_SPARE3);
238 		else
239 			px_ilu_log_mask &= ~(1ull <<
240 			    ILU_ERROR_LOG_ENABLE_SPARE3);
241 
242 		px_err_reg_setup_pcie(chip_mask, csr_base, PX_ERR_ENABLE);
243 		px_fabric_die_rc_ue |= PCIE_AER_UCE_UC;
244 		break;
245 
246 	case PX_CHIP_FIRE:
247 		px_err_reg_setup_pcie(chip_mask, csr_base, PX_ERR_ENABLE);
248 		break;
249 
250 	default:
251 		cmn_err(CE_WARN, "%s%d: PX primary bus Unknown\n",
252 		    ddi_driver_name(dip), ddi_get_instance(dip));
253 		return (DDI_FAILURE);
254 	}
255 
256 	/* Initilize device handle */
257 	*dev_hdl = (devhandle_t)csr_base;
258 
259 	DBG(DBG_ATTACH, dip, "px_lib_dev_init: dev_hdl 0x%llx\n", *dev_hdl);
260 
261 	return (DDI_SUCCESS);
262 }
263 
264 int
265 px_lib_dev_fini(dev_info_t *dip)
266 {
267 	caddr_t			csr_base;
268 	uint8_t			chip_mask;
269 	px_t			*px_p = DIP_TO_STATE(dip);
270 	pxu_t			*pxu_p = (pxu_t *)px_p->px_plat_p;
271 
272 	DBG(DBG_DETACH, dip, "px_lib_dev_fini: dip 0x%p\n", dip);
273 
274 	/*
275 	 * Deinitialize all the interrupt handlers
276 	 */
277 	switch (PX_CHIP_TYPE(pxu_p)) {
278 	case PX_CHIP_OBERON:
279 	case PX_CHIP_FIRE:
280 		chip_mask = BITMASK(PX_CHIP_TYPE(pxu_p));
281 		csr_base = (caddr_t)pxu_p->px_address[PX_REG_CSR];
282 		px_err_reg_setup_pcie(chip_mask, csr_base, PX_ERR_DISABLE);
283 		break;
284 
285 	default:
286 		cmn_err(CE_WARN, "%s%d: PX primary bus Unknown\n",
287 		    ddi_driver_name(dip), ddi_get_instance(dip));
288 		return (DDI_FAILURE);
289 	}
290 
291 	iommu_tsb_free(pxu_p->tsb_cookie);
292 
293 	px_lib_unmap_regs((pxu_t *)px_p->px_plat_p);
294 	kmem_free(px_p->px_plat_p, sizeof (pxu_t));
295 	px_p->px_plat_p = NULL;
296 
297 	return (DDI_SUCCESS);
298 }
299 
300 /*ARGSUSED*/
301 int
302 px_lib_intr_devino_to_sysino(dev_info_t *dip, devino_t devino,
303     sysino_t *sysino)
304 {
305 	px_t	*px_p = DIP_TO_STATE(dip);
306 	pxu_t	*pxu_p = (pxu_t *)px_p->px_plat_p;
307 	uint64_t	ret;
308 
309 	DBG(DBG_LIB_INT, dip, "px_lib_intr_devino_to_sysino: dip 0x%p "
310 	    "devino 0x%x\n", dip, devino);
311 
312 	if ((ret = hvio_intr_devino_to_sysino(DIP_TO_HANDLE(dip),
313 	    pxu_p, devino, sysino)) != H_EOK) {
314 		DBG(DBG_LIB_INT, dip,
315 		    "hvio_intr_devino_to_sysino failed, ret 0x%lx\n", ret);
316 		return (DDI_FAILURE);
317 	}
318 
319 	DBG(DBG_LIB_INT, dip, "px_lib_intr_devino_to_sysino: sysino 0x%llx\n",
320 	    *sysino);
321 
322 	return (DDI_SUCCESS);
323 }
324 
325 /*ARGSUSED*/
326 int
327 px_lib_intr_getvalid(dev_info_t *dip, sysino_t sysino,
328     intr_valid_state_t *intr_valid_state)
329 {
330 	uint64_t	ret;
331 
332 	DBG(DBG_LIB_INT, dip, "px_lib_intr_getvalid: dip 0x%p sysino 0x%llx\n",
333 	    dip, sysino);
334 
335 	if ((ret = hvio_intr_getvalid(DIP_TO_HANDLE(dip),
336 	    sysino, intr_valid_state)) != H_EOK) {
337 		DBG(DBG_LIB_INT, dip, "hvio_intr_getvalid failed, ret 0x%lx\n",
338 		    ret);
339 		return (DDI_FAILURE);
340 	}
341 
342 	DBG(DBG_LIB_INT, dip, "px_lib_intr_getvalid: intr_valid_state 0x%x\n",
343 	    *intr_valid_state);
344 
345 	return (DDI_SUCCESS);
346 }
347 
348 /*ARGSUSED*/
349 int
350 px_lib_intr_setvalid(dev_info_t *dip, sysino_t sysino,
351     intr_valid_state_t intr_valid_state)
352 {
353 	uint64_t	ret;
354 
355 	DBG(DBG_LIB_INT, dip, "px_lib_intr_setvalid: dip 0x%p sysino 0x%llx "
356 	    "intr_valid_state 0x%x\n", dip, sysino, intr_valid_state);
357 
358 	if ((ret = hvio_intr_setvalid(DIP_TO_HANDLE(dip),
359 	    sysino, intr_valid_state)) != H_EOK) {
360 		DBG(DBG_LIB_INT, dip, "hvio_intr_setvalid failed, ret 0x%lx\n",
361 		    ret);
362 		return (DDI_FAILURE);
363 	}
364 
365 	return (DDI_SUCCESS);
366 }
367 
368 /*ARGSUSED*/
369 int
370 px_lib_intr_getstate(dev_info_t *dip, sysino_t sysino,
371     intr_state_t *intr_state)
372 {
373 	uint64_t	ret;
374 
375 	DBG(DBG_LIB_INT, dip, "px_lib_intr_getstate: dip 0x%p sysino 0x%llx\n",
376 	    dip, sysino);
377 
378 	if ((ret = hvio_intr_getstate(DIP_TO_HANDLE(dip),
379 	    sysino, intr_state)) != H_EOK) {
380 		DBG(DBG_LIB_INT, dip, "hvio_intr_getstate failed, ret 0x%lx\n",
381 		    ret);
382 		return (DDI_FAILURE);
383 	}
384 
385 	DBG(DBG_LIB_INT, dip, "px_lib_intr_getstate: intr_state 0x%x\n",
386 	    *intr_state);
387 
388 	return (DDI_SUCCESS);
389 }
390 
391 /*ARGSUSED*/
392 int
393 px_lib_intr_setstate(dev_info_t *dip, sysino_t sysino,
394     intr_state_t intr_state)
395 {
396 	uint64_t	ret;
397 
398 	DBG(DBG_LIB_INT, dip, "px_lib_intr_setstate: dip 0x%p sysino 0x%llx "
399 	    "intr_state 0x%x\n", dip, sysino, intr_state);
400 
401 	if ((ret = hvio_intr_setstate(DIP_TO_HANDLE(dip),
402 	    sysino, intr_state)) != H_EOK) {
403 		DBG(DBG_LIB_INT, dip, "hvio_intr_setstate failed, ret 0x%lx\n",
404 		    ret);
405 		return (DDI_FAILURE);
406 	}
407 
408 	return (DDI_SUCCESS);
409 }
410 
411 /*ARGSUSED*/
412 int
413 px_lib_intr_gettarget(dev_info_t *dip, sysino_t sysino, cpuid_t *cpuid)
414 {
415 	px_t		*px_p = DIP_TO_STATE(dip);
416 	pxu_t		*pxu_p = (pxu_t *)px_p->px_plat_p;
417 	uint64_t	ret;
418 
419 	DBG(DBG_LIB_INT, dip, "px_lib_intr_gettarget: dip 0x%p sysino 0x%llx\n",
420 	    dip, sysino);
421 
422 	if ((ret = hvio_intr_gettarget(DIP_TO_HANDLE(dip), pxu_p,
423 	    sysino, cpuid)) != H_EOK) {
424 		DBG(DBG_LIB_INT, dip, "hvio_intr_gettarget failed, ret 0x%lx\n",
425 		    ret);
426 		return (DDI_FAILURE);
427 	}
428 
429 	DBG(DBG_LIB_INT, dip, "px_lib_intr_gettarget: cpuid 0x%x\n", cpuid);
430 
431 	return (DDI_SUCCESS);
432 }
433 
434 /*ARGSUSED*/
435 int
436 px_lib_intr_settarget(dev_info_t *dip, sysino_t sysino, cpuid_t cpuid)
437 {
438 	px_t		*px_p = DIP_TO_STATE(dip);
439 	pxu_t		*pxu_p = (pxu_t *)px_p->px_plat_p;
440 	uint64_t	ret;
441 
442 	DBG(DBG_LIB_INT, dip, "px_lib_intr_settarget: dip 0x%p sysino 0x%llx "
443 	    "cpuid 0x%x\n", dip, sysino, cpuid);
444 
445 	if ((ret = hvio_intr_settarget(DIP_TO_HANDLE(dip), pxu_p,
446 	    sysino, cpuid)) != H_EOK) {
447 		DBG(DBG_LIB_INT, dip, "hvio_intr_settarget failed, ret 0x%lx\n",
448 		    ret);
449 		return (DDI_FAILURE);
450 	}
451 
452 	return (DDI_SUCCESS);
453 }
454 
455 /*ARGSUSED*/
456 int
457 px_lib_intr_reset(dev_info_t *dip)
458 {
459 	devino_t	ino;
460 	sysino_t	sysino;
461 
462 	DBG(DBG_LIB_INT, dip, "px_lib_intr_reset: dip 0x%p\n", dip);
463 
464 	/* Reset all Interrupts */
465 	for (ino = 0; ino < INTERRUPT_MAPPING_ENTRIES; ino++) {
466 		if (px_lib_intr_devino_to_sysino(dip, ino,
467 		    &sysino) != DDI_SUCCESS)
468 			return (BF_FATAL);
469 
470 		if (px_lib_intr_setstate(dip, sysino,
471 		    INTR_IDLE_STATE) != DDI_SUCCESS)
472 			return (BF_FATAL);
473 	}
474 
475 	return (BF_NONE);
476 }
477 
478 /*ARGSUSED*/
479 int
480 px_lib_iommu_map(dev_info_t *dip, tsbid_t tsbid, pages_t pages,
481     io_attributes_t attr, void *addr, size_t pfn_index, int flags)
482 {
483 	px_t		*px_p = DIP_TO_STATE(dip);
484 	pxu_t		*pxu_p = (pxu_t *)px_p->px_plat_p;
485 	uint64_t	ret;
486 
487 	DBG(DBG_LIB_DMA, dip, "px_lib_iommu_map: dip 0x%p tsbid 0x%llx "
488 	    "pages 0x%x attr 0x%x addr 0x%p pfn_index 0x%llx flags 0x%x\n",
489 	    dip, tsbid, pages, attr, addr, pfn_index, flags);
490 
491 	if ((ret = hvio_iommu_map(px_p->px_dev_hdl, pxu_p, tsbid, pages,
492 	    attr, addr, pfn_index, flags)) != H_EOK) {
493 		DBG(DBG_LIB_DMA, dip,
494 		    "px_lib_iommu_map failed, ret 0x%lx\n", ret);
495 		return (DDI_FAILURE);
496 	}
497 
498 	return (DDI_SUCCESS);
499 }
500 
501 /*ARGSUSED*/
502 int
503 px_lib_iommu_demap(dev_info_t *dip, tsbid_t tsbid, pages_t pages)
504 {
505 	px_t		*px_p = DIP_TO_STATE(dip);
506 	pxu_t		*pxu_p = (pxu_t *)px_p->px_plat_p;
507 	uint64_t	ret;
508 
509 	DBG(DBG_LIB_DMA, dip, "px_lib_iommu_demap: dip 0x%p tsbid 0x%llx "
510 	    "pages 0x%x\n", dip, tsbid, pages);
511 
512 	if ((ret = hvio_iommu_demap(px_p->px_dev_hdl, pxu_p, tsbid, pages))
513 	    != H_EOK) {
514 		DBG(DBG_LIB_DMA, dip,
515 		    "px_lib_iommu_demap failed, ret 0x%lx\n", ret);
516 
517 		return (DDI_FAILURE);
518 	}
519 
520 	return (DDI_SUCCESS);
521 }
522 
523 /*ARGSUSED*/
524 int
525 px_lib_iommu_getmap(dev_info_t *dip, tsbid_t tsbid, io_attributes_t *attr_p,
526     r_addr_t *r_addr_p)
527 {
528 	px_t	*px_p = DIP_TO_STATE(dip);
529 	pxu_t	*pxu_p = (pxu_t *)px_p->px_plat_p;
530 	uint64_t	ret;
531 
532 	DBG(DBG_LIB_DMA, dip, "px_lib_iommu_getmap: dip 0x%p tsbid 0x%llx\n",
533 	    dip, tsbid);
534 
535 	if ((ret = hvio_iommu_getmap(DIP_TO_HANDLE(dip), pxu_p, tsbid,
536 	    attr_p, r_addr_p)) != H_EOK) {
537 		DBG(DBG_LIB_DMA, dip,
538 		    "hvio_iommu_getmap failed, ret 0x%lx\n", ret);
539 
540 		return ((ret == H_ENOMAP) ? DDI_DMA_NOMAPPING:DDI_FAILURE);
541 	}
542 
543 	DBG(DBG_LIB_DMA, dip, "px_lib_iommu_getmap: attr 0x%x r_addr 0x%llx\n",
544 	    *attr_p, *r_addr_p);
545 
546 	return (DDI_SUCCESS);
547 }
548 
549 
550 /*
551  * Checks dma attributes against system bypass ranges
552  * The bypass range is determined by the hardware. Return them so the
553  * common code can do generic checking against them.
554  */
555 /*ARGSUSED*/
556 int
557 px_lib_dma_bypass_rngchk(dev_info_t *dip, ddi_dma_attr_t *attr_p,
558     uint64_t *lo_p, uint64_t *hi_p)
559 {
560 	px_t	*px_p = DIP_TO_STATE(dip);
561 	pxu_t	*pxu_p = (pxu_t *)px_p->px_plat_p;
562 
563 	*lo_p = hvio_get_bypass_base(pxu_p);
564 	*hi_p = hvio_get_bypass_end(pxu_p);
565 
566 	return (DDI_SUCCESS);
567 }
568 
569 
570 /*ARGSUSED*/
571 int
572 px_lib_iommu_getbypass(dev_info_t *dip, r_addr_t ra, io_attributes_t attr,
573     io_addr_t *io_addr_p)
574 {
575 	uint64_t	ret;
576 	px_t	*px_p = DIP_TO_STATE(dip);
577 	pxu_t	*pxu_p = (pxu_t *)px_p->px_plat_p;
578 
579 	DBG(DBG_LIB_DMA, dip, "px_lib_iommu_getbypass: dip 0x%p ra 0x%llx "
580 	    "attr 0x%x\n", dip, ra, attr);
581 
582 	if ((ret = hvio_iommu_getbypass(DIP_TO_HANDLE(dip), pxu_p, ra,
583 	    attr, io_addr_p)) != H_EOK) {
584 		DBG(DBG_LIB_DMA, dip,
585 		    "hvio_iommu_getbypass failed, ret 0x%lx\n", ret);
586 		return (DDI_FAILURE);
587 	}
588 
589 	DBG(DBG_LIB_DMA, dip, "px_lib_iommu_getbypass: io_addr 0x%llx\n",
590 	    *io_addr_p);
591 
592 	return (DDI_SUCCESS);
593 }
594 
595 /*
596  * bus dma sync entry point.
597  */
598 /*ARGSUSED*/
599 int
600 px_lib_dma_sync(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle,
601     off_t off, size_t len, uint_t cache_flags)
602 {
603 	ddi_dma_impl_t *mp = (ddi_dma_impl_t *)handle;
604 	px_t	*px_p = DIP_TO_STATE(dip);
605 	pxu_t	*pxu_p = (pxu_t *)px_p->px_plat_p;
606 
607 	DBG(DBG_LIB_DMA, dip, "px_lib_dma_sync: dip 0x%p rdip 0x%p "
608 	    "handle 0x%llx off 0x%x len 0x%x flags 0x%x\n",
609 	    dip, rdip, handle, off, len, cache_flags);
610 
611 	/*
612 	 * No flush needed for Oberon
613 	 */
614 	if (PX_CHIP_TYPE(pxu_p) == PX_CHIP_OBERON)
615 		return (DDI_SUCCESS);
616 
617 	/*
618 	 * jbus_stst_order is found only in certain cpu modules.
619 	 * Just return success if not present.
620 	 */
621 	if (&jbus_stst_order == NULL)
622 		return (DDI_SUCCESS);
623 
624 	if (!(mp->dmai_flags & PX_DMAI_FLAGS_INUSE)) {
625 		cmn_err(CE_WARN, "%s%d: Unbound dma handle %p.",
626 		    ddi_driver_name(rdip), ddi_get_instance(rdip), (void *)mp);
627 
628 		return (DDI_FAILURE);
629 	}
630 
631 	if (mp->dmai_flags & PX_DMAI_FLAGS_NOSYNC)
632 		return (DDI_SUCCESS);
633 
634 	/*
635 	 * No flush needed when sending data from memory to device.
636 	 * Nothing to do to "sync" memory to what device would already see.
637 	 */
638 	if (!(mp->dmai_rflags & DDI_DMA_READ) ||
639 	    ((cache_flags & PX_DMA_SYNC_DDI_FLAGS) == DDI_DMA_SYNC_FORDEV))
640 		return (DDI_SUCCESS);
641 
642 	/*
643 	 * Perform necessary cpu workaround to ensure jbus ordering.
644 	 * CPU's internal "invalidate FIFOs" are flushed.
645 	 */
646 
647 #if !defined(lint)
648 	kpreempt_disable();
649 #endif
650 	jbus_stst_order();
651 #if !defined(lint)
652 	kpreempt_enable();
653 #endif
654 	return (DDI_SUCCESS);
655 }
656 
657 /*
658  * MSIQ Functions:
659  */
660 /*ARGSUSED*/
661 int
662 px_lib_msiq_init(dev_info_t *dip)
663 {
664 	px_t		*px_p = DIP_TO_STATE(dip);
665 	pxu_t		*pxu_p = (pxu_t *)px_p->px_plat_p;
666 	px_msiq_state_t	*msiq_state_p = &px_p->px_ib_p->ib_msiq_state;
667 	px_dvma_addr_t	pg_index;
668 	size_t		size;
669 	int		ret;
670 
671 	DBG(DBG_LIB_MSIQ, dip, "px_lib_msiq_init: dip 0x%p\n", dip);
672 
673 	/*
674 	 * Map the EQ memory into the Fire MMU (has to be 512KB aligned)
675 	 * and then initialize the base address register.
676 	 *
677 	 * Allocate entries from Fire IOMMU so that the resulting address
678 	 * is properly aligned.  Calculate the index of the first allocated
679 	 * entry.  Note: The size of the mapping is assumed to be a multiple
680 	 * of the page size.
681 	 */
682 	size = msiq_state_p->msiq_cnt *
683 	    msiq_state_p->msiq_rec_cnt * sizeof (msiq_rec_t);
684 
685 	pxu_p->msiq_mapped_p = vmem_xalloc(px_p->px_mmu_p->mmu_dvma_map,
686 	    size, (512 * 1024), 0, 0, NULL, NULL, VM_NOSLEEP | VM_BESTFIT);
687 
688 	if (pxu_p->msiq_mapped_p == NULL)
689 		return (DDI_FAILURE);
690 
691 	pg_index = MMU_PAGE_INDEX(px_p->px_mmu_p,
692 	    MMU_BTOP((ulong_t)pxu_p->msiq_mapped_p));
693 
694 	if ((ret = px_lib_iommu_map(px_p->px_dip, PCI_TSBID(0, pg_index),
695 	    MMU_BTOP(size), PCI_MAP_ATTR_WRITE, msiq_state_p->msiq_buf_p,
696 	    0, MMU_MAP_BUF)) != DDI_SUCCESS) {
697 		DBG(DBG_LIB_MSIQ, dip,
698 		    "hvio_msiq_init failed, ret 0x%lx\n", ret);
699 
700 		(void) px_lib_msiq_fini(dip);
701 		return (DDI_FAILURE);
702 	}
703 
704 	(void) hvio_msiq_init(DIP_TO_HANDLE(dip), pxu_p);
705 
706 	return (DDI_SUCCESS);
707 }
708 
709 /*ARGSUSED*/
710 int
711 px_lib_msiq_fini(dev_info_t *dip)
712 {
713 	px_t		*px_p = DIP_TO_STATE(dip);
714 	pxu_t		*pxu_p = (pxu_t *)px_p->px_plat_p;
715 	px_msiq_state_t	*msiq_state_p = &px_p->px_ib_p->ib_msiq_state;
716 	px_dvma_addr_t	pg_index;
717 	size_t		size;
718 
719 	DBG(DBG_LIB_MSIQ, dip, "px_lib_msiq_fini: dip 0x%p\n", dip);
720 
721 	/*
722 	 * Unmap and free the EQ memory that had been mapped
723 	 * into the Fire IOMMU.
724 	 */
725 	size = msiq_state_p->msiq_cnt *
726 	    msiq_state_p->msiq_rec_cnt * sizeof (msiq_rec_t);
727 
728 	pg_index = MMU_PAGE_INDEX(px_p->px_mmu_p,
729 	    MMU_BTOP((ulong_t)pxu_p->msiq_mapped_p));
730 
731 	(void) px_lib_iommu_demap(px_p->px_dip,
732 	    PCI_TSBID(0, pg_index), MMU_BTOP(size));
733 
734 	/* Free the entries from the Fire MMU */
735 	vmem_xfree(px_p->px_mmu_p->mmu_dvma_map,
736 	    (void *)pxu_p->msiq_mapped_p, size);
737 
738 	return (DDI_SUCCESS);
739 }
740 
741 /*ARGSUSED*/
742 int
743 px_lib_msiq_info(dev_info_t *dip, msiqid_t msiq_id, r_addr_t *ra_p,
744     uint_t *msiq_rec_cnt_p)
745 {
746 	px_t		*px_p = DIP_TO_STATE(dip);
747 	px_msiq_state_t	*msiq_state_p = &px_p->px_ib_p->ib_msiq_state;
748 	size_t		msiq_size;
749 
750 	DBG(DBG_LIB_MSIQ, dip, "px_msiq_info: dip 0x%p msiq_id 0x%x\n",
751 	    dip, msiq_id);
752 
753 	msiq_size = msiq_state_p->msiq_rec_cnt * sizeof (msiq_rec_t);
754 	ra_p = (r_addr_t *)((caddr_t)msiq_state_p->msiq_buf_p +
755 	    (msiq_id * msiq_size));
756 
757 	*msiq_rec_cnt_p = msiq_state_p->msiq_rec_cnt;
758 
759 	DBG(DBG_LIB_MSIQ, dip, "px_msiq_info: ra_p 0x%p msiq_rec_cnt 0x%x\n",
760 	    ra_p, *msiq_rec_cnt_p);
761 
762 	return (DDI_SUCCESS);
763 }
764 
765 /*ARGSUSED*/
766 int
767 px_lib_msiq_getvalid(dev_info_t *dip, msiqid_t msiq_id,
768     pci_msiq_valid_state_t *msiq_valid_state)
769 {
770 	uint64_t	ret;
771 
772 	DBG(DBG_LIB_MSIQ, dip, "px_lib_msiq_getvalid: dip 0x%p msiq_id 0x%x\n",
773 	    dip, msiq_id);
774 
775 	if ((ret = hvio_msiq_getvalid(DIP_TO_HANDLE(dip),
776 	    msiq_id, msiq_valid_state)) != H_EOK) {
777 		DBG(DBG_LIB_MSIQ, dip,
778 		    "hvio_msiq_getvalid failed, ret 0x%lx\n", ret);
779 		return (DDI_FAILURE);
780 	}
781 
782 	DBG(DBG_LIB_MSIQ, dip, "px_lib_msiq_getvalid: msiq_valid_state 0x%x\n",
783 	    *msiq_valid_state);
784 
785 	return (DDI_SUCCESS);
786 }
787 
788 /*ARGSUSED*/
789 int
790 px_lib_msiq_setvalid(dev_info_t *dip, msiqid_t msiq_id,
791     pci_msiq_valid_state_t msiq_valid_state)
792 {
793 	uint64_t	ret;
794 
795 	DBG(DBG_LIB_MSIQ, dip, "px_lib_msiq_setvalid: dip 0x%p msiq_id 0x%x "
796 	    "msiq_valid_state 0x%x\n", dip, msiq_id, msiq_valid_state);
797 
798 	if ((ret = hvio_msiq_setvalid(DIP_TO_HANDLE(dip),
799 	    msiq_id, msiq_valid_state)) != H_EOK) {
800 		DBG(DBG_LIB_MSIQ, dip,
801 		    "hvio_msiq_setvalid failed, ret 0x%lx\n", ret);
802 		return (DDI_FAILURE);
803 	}
804 
805 	return (DDI_SUCCESS);
806 }
807 
808 /*ARGSUSED*/
809 int
810 px_lib_msiq_getstate(dev_info_t *dip, msiqid_t msiq_id,
811     pci_msiq_state_t *msiq_state)
812 {
813 	uint64_t	ret;
814 
815 	DBG(DBG_LIB_MSIQ, dip, "px_lib_msiq_getstate: dip 0x%p msiq_id 0x%x\n",
816 	    dip, msiq_id);
817 
818 	if ((ret = hvio_msiq_getstate(DIP_TO_HANDLE(dip),
819 	    msiq_id, msiq_state)) != H_EOK) {
820 		DBG(DBG_LIB_MSIQ, dip,
821 		    "hvio_msiq_getstate failed, ret 0x%lx\n", ret);
822 		return (DDI_FAILURE);
823 	}
824 
825 	DBG(DBG_LIB_MSIQ, dip, "px_lib_msiq_getstate: msiq_state 0x%x\n",
826 	    *msiq_state);
827 
828 	return (DDI_SUCCESS);
829 }
830 
831 /*ARGSUSED*/
832 int
833 px_lib_msiq_setstate(dev_info_t *dip, msiqid_t msiq_id,
834     pci_msiq_state_t msiq_state)
835 {
836 	uint64_t	ret;
837 
838 	DBG(DBG_LIB_MSIQ, dip, "px_lib_msiq_setstate: dip 0x%p msiq_id 0x%x "
839 	    "msiq_state 0x%x\n", dip, msiq_id, msiq_state);
840 
841 	if ((ret = hvio_msiq_setstate(DIP_TO_HANDLE(dip),
842 	    msiq_id, msiq_state)) != H_EOK) {
843 		DBG(DBG_LIB_MSIQ, dip,
844 		    "hvio_msiq_setstate failed, ret 0x%lx\n", ret);
845 		return (DDI_FAILURE);
846 	}
847 
848 	return (DDI_SUCCESS);
849 }
850 
851 /*ARGSUSED*/
852 int
853 px_lib_msiq_gethead(dev_info_t *dip, msiqid_t msiq_id,
854     msiqhead_t *msiq_head)
855 {
856 	uint64_t	ret;
857 
858 	DBG(DBG_LIB_MSIQ, dip, "px_lib_msiq_gethead: dip 0x%p msiq_id 0x%x\n",
859 	    dip, msiq_id);
860 
861 	if ((ret = hvio_msiq_gethead(DIP_TO_HANDLE(dip),
862 	    msiq_id, msiq_head)) != H_EOK) {
863 		DBG(DBG_LIB_MSIQ, dip,
864 		    "hvio_msiq_gethead failed, ret 0x%lx\n", ret);
865 		return (DDI_FAILURE);
866 	}
867 
868 	DBG(DBG_LIB_MSIQ, dip, "px_lib_msiq_gethead: msiq_head 0x%x\n",
869 	    *msiq_head);
870 
871 	return (DDI_SUCCESS);
872 }
873 
874 /*ARGSUSED*/
875 int
876 px_lib_msiq_sethead(dev_info_t *dip, msiqid_t msiq_id,
877     msiqhead_t msiq_head)
878 {
879 	uint64_t	ret;
880 
881 	DBG(DBG_LIB_MSIQ, dip, "px_lib_msiq_sethead: dip 0x%p msiq_id 0x%x "
882 	    "msiq_head 0x%x\n", dip, msiq_id, msiq_head);
883 
884 	if ((ret = hvio_msiq_sethead(DIP_TO_HANDLE(dip),
885 	    msiq_id, msiq_head)) != H_EOK) {
886 		DBG(DBG_LIB_MSIQ, dip,
887 		    "hvio_msiq_sethead failed, ret 0x%lx\n", ret);
888 		return (DDI_FAILURE);
889 	}
890 
891 	return (DDI_SUCCESS);
892 }
893 
894 /*ARGSUSED*/
895 int
896 px_lib_msiq_gettail(dev_info_t *dip, msiqid_t msiq_id,
897     msiqtail_t *msiq_tail)
898 {
899 	uint64_t	ret;
900 
901 	DBG(DBG_LIB_MSIQ, dip, "px_lib_msiq_gettail: dip 0x%p msiq_id 0x%x\n",
902 	    dip, msiq_id);
903 
904 	if ((ret = hvio_msiq_gettail(DIP_TO_HANDLE(dip),
905 	    msiq_id, msiq_tail)) != H_EOK) {
906 		DBG(DBG_LIB_MSIQ, dip,
907 		    "hvio_msiq_gettail failed, ret 0x%lx\n", ret);
908 		return (DDI_FAILURE);
909 	}
910 
911 	DBG(DBG_LIB_MSIQ, dip, "px_lib_msiq_gettail: msiq_tail 0x%x\n",
912 	    *msiq_tail);
913 
914 	return (DDI_SUCCESS);
915 }
916 
917 /*ARGSUSED*/
918 void
919 px_lib_get_msiq_rec(dev_info_t *dip, msiqhead_t *msiq_head_p,
920     msiq_rec_t *msiq_rec_p)
921 {
922 	eq_rec_t	*eq_rec_p = (eq_rec_t *)msiq_head_p;
923 
924 	DBG(DBG_LIB_MSIQ, dip, "px_lib_get_msiq_rec: dip 0x%p eq_rec_p 0x%p\n",
925 	    dip, eq_rec_p);
926 
927 	if (!eq_rec_p->eq_rec_fmt_type) {
928 		/* Set msiq_rec_type to zero */
929 		msiq_rec_p->msiq_rec_type = 0;
930 
931 		return;
932 	}
933 
934 	DBG(DBG_LIB_MSIQ, dip, "px_lib_get_msiq_rec: EQ RECORD, "
935 	    "eq_rec_rid 0x%llx eq_rec_fmt_type 0x%llx "
936 	    "eq_rec_len 0x%llx eq_rec_addr0 0x%llx "
937 	    "eq_rec_addr1 0x%llx eq_rec_data0 0x%llx "
938 	    "eq_rec_data1 0x%llx\n", eq_rec_p->eq_rec_rid,
939 	    eq_rec_p->eq_rec_fmt_type, eq_rec_p->eq_rec_len,
940 	    eq_rec_p->eq_rec_addr0, eq_rec_p->eq_rec_addr1,
941 	    eq_rec_p->eq_rec_data0, eq_rec_p->eq_rec_data1);
942 
943 	/*
944 	 * Only upper 4 bits of eq_rec_fmt_type is used
945 	 * to identify the EQ record type.
946 	 */
947 	switch (eq_rec_p->eq_rec_fmt_type >> 3) {
948 	case EQ_REC_MSI32:
949 		msiq_rec_p->msiq_rec_type = MSI32_REC;
950 
951 		msiq_rec_p->msiq_rec_data.msi.msi_data =
952 		    eq_rec_p->eq_rec_data0;
953 		break;
954 	case EQ_REC_MSI64:
955 		msiq_rec_p->msiq_rec_type = MSI64_REC;
956 
957 		msiq_rec_p->msiq_rec_data.msi.msi_data =
958 		    eq_rec_p->eq_rec_data0;
959 		break;
960 	case EQ_REC_MSG:
961 		msiq_rec_p->msiq_rec_type = MSG_REC;
962 
963 		msiq_rec_p->msiq_rec_data.msg.msg_route =
964 		    eq_rec_p->eq_rec_fmt_type & 7;
965 		msiq_rec_p->msiq_rec_data.msg.msg_targ = eq_rec_p->eq_rec_rid;
966 		msiq_rec_p->msiq_rec_data.msg.msg_code = eq_rec_p->eq_rec_data0;
967 		break;
968 	default:
969 		cmn_err(CE_WARN, "%s%d: px_lib_get_msiq_rec: "
970 		    "0x%x is an unknown EQ record type",
971 		    ddi_driver_name(dip), ddi_get_instance(dip),
972 		    (int)eq_rec_p->eq_rec_fmt_type);
973 		break;
974 	}
975 
976 	msiq_rec_p->msiq_rec_rid = eq_rec_p->eq_rec_rid;
977 	msiq_rec_p->msiq_rec_msi_addr = ((eq_rec_p->eq_rec_addr1 << 16) |
978 	    (eq_rec_p->eq_rec_addr0 << 2));
979 }
980 
981 /*ARGSUSED*/
982 void
983 px_lib_clr_msiq_rec(dev_info_t *dip, msiqhead_t *msiq_head_p)
984 {
985 	eq_rec_t	*eq_rec_p = (eq_rec_t *)msiq_head_p;
986 
987 	DBG(DBG_LIB_MSIQ, dip, "px_lib_clr_msiq_rec: dip 0x%p eq_rec_p 0x%p\n",
988 	    dip, eq_rec_p);
989 
990 	if (eq_rec_p->eq_rec_fmt_type) {
991 		/* Zero out eq_rec_fmt_type field */
992 		eq_rec_p->eq_rec_fmt_type = 0;
993 	}
994 }
995 
996 /*
997  * MSI Functions:
998  */
999 /*ARGSUSED*/
1000 int
1001 px_lib_msi_init(dev_info_t *dip)
1002 {
1003 	px_t		*px_p = DIP_TO_STATE(dip);
1004 	px_msi_state_t	*msi_state_p = &px_p->px_ib_p->ib_msi_state;
1005 	uint64_t	ret;
1006 
1007 	DBG(DBG_LIB_MSI, dip, "px_lib_msi_init: dip 0x%p\n", dip);
1008 
1009 	if ((ret = hvio_msi_init(DIP_TO_HANDLE(dip),
1010 	    msi_state_p->msi_addr32, msi_state_p->msi_addr64)) != H_EOK) {
1011 		DBG(DBG_LIB_MSIQ, dip, "px_lib_msi_init failed, ret 0x%lx\n",
1012 		    ret);
1013 		return (DDI_FAILURE);
1014 	}
1015 
1016 	return (DDI_SUCCESS);
1017 }
1018 
1019 /*ARGSUSED*/
1020 int
1021 px_lib_msi_getmsiq(dev_info_t *dip, msinum_t msi_num,
1022     msiqid_t *msiq_id)
1023 {
1024 	uint64_t	ret;
1025 
1026 	DBG(DBG_LIB_MSI, dip, "px_lib_msi_getmsiq: dip 0x%p msi_num 0x%x\n",
1027 	    dip, msi_num);
1028 
1029 	if ((ret = hvio_msi_getmsiq(DIP_TO_HANDLE(dip),
1030 	    msi_num, msiq_id)) != H_EOK) {
1031 		DBG(DBG_LIB_MSI, dip,
1032 		    "hvio_msi_getmsiq failed, ret 0x%lx\n", ret);
1033 		return (DDI_FAILURE);
1034 	}
1035 
1036 	DBG(DBG_LIB_MSI, dip, "px_lib_msi_getmsiq: msiq_id 0x%x\n",
1037 	    *msiq_id);
1038 
1039 	return (DDI_SUCCESS);
1040 }
1041 
1042 /*ARGSUSED*/
1043 int
1044 px_lib_msi_setmsiq(dev_info_t *dip, msinum_t msi_num,
1045     msiqid_t msiq_id, msi_type_t msitype)
1046 {
1047 	uint64_t	ret;
1048 
1049 	DBG(DBG_LIB_MSI, dip, "px_lib_msi_setmsiq: dip 0x%p msi_num 0x%x "
1050 	    "msq_id 0x%x\n", dip, msi_num, msiq_id);
1051 
1052 	if ((ret = hvio_msi_setmsiq(DIP_TO_HANDLE(dip),
1053 	    msi_num, msiq_id)) != H_EOK) {
1054 		DBG(DBG_LIB_MSI, dip,
1055 		    "hvio_msi_setmsiq failed, ret 0x%lx\n", ret);
1056 		return (DDI_FAILURE);
1057 	}
1058 
1059 	return (DDI_SUCCESS);
1060 }
1061 
1062 /*ARGSUSED*/
1063 int
1064 px_lib_msi_getvalid(dev_info_t *dip, msinum_t msi_num,
1065     pci_msi_valid_state_t *msi_valid_state)
1066 {
1067 	uint64_t	ret;
1068 
1069 	DBG(DBG_LIB_MSI, dip, "px_lib_msi_getvalid: dip 0x%p msi_num 0x%x\n",
1070 	    dip, msi_num);
1071 
1072 	if ((ret = hvio_msi_getvalid(DIP_TO_HANDLE(dip),
1073 	    msi_num, msi_valid_state)) != H_EOK) {
1074 		DBG(DBG_LIB_MSI, dip,
1075 		    "hvio_msi_getvalid failed, ret 0x%lx\n", ret);
1076 		return (DDI_FAILURE);
1077 	}
1078 
1079 	DBG(DBG_LIB_MSI, dip, "px_lib_msi_getvalid: msiq_id 0x%x\n",
1080 	    *msi_valid_state);
1081 
1082 	return (DDI_SUCCESS);
1083 }
1084 
1085 /*ARGSUSED*/
1086 int
1087 px_lib_msi_setvalid(dev_info_t *dip, msinum_t msi_num,
1088     pci_msi_valid_state_t msi_valid_state)
1089 {
1090 	uint64_t	ret;
1091 
1092 	DBG(DBG_LIB_MSI, dip, "px_lib_msi_setvalid: dip 0x%p msi_num 0x%x "
1093 	    "msi_valid_state 0x%x\n", dip, msi_num, msi_valid_state);
1094 
1095 	if ((ret = hvio_msi_setvalid(DIP_TO_HANDLE(dip),
1096 	    msi_num, msi_valid_state)) != H_EOK) {
1097 		DBG(DBG_LIB_MSI, dip,
1098 		    "hvio_msi_setvalid failed, ret 0x%lx\n", ret);
1099 		return (DDI_FAILURE);
1100 	}
1101 
1102 	return (DDI_SUCCESS);
1103 }
1104 
1105 /*ARGSUSED*/
1106 int
1107 px_lib_msi_getstate(dev_info_t *dip, msinum_t msi_num,
1108     pci_msi_state_t *msi_state)
1109 {
1110 	uint64_t	ret;
1111 
1112 	DBG(DBG_LIB_MSI, dip, "px_lib_msi_getstate: dip 0x%p msi_num 0x%x\n",
1113 	    dip, msi_num);
1114 
1115 	if ((ret = hvio_msi_getstate(DIP_TO_HANDLE(dip),
1116 	    msi_num, msi_state)) != H_EOK) {
1117 		DBG(DBG_LIB_MSI, dip,
1118 		    "hvio_msi_getstate failed, ret 0x%lx\n", ret);
1119 		return (DDI_FAILURE);
1120 	}
1121 
1122 	DBG(DBG_LIB_MSI, dip, "px_lib_msi_getstate: msi_state 0x%x\n",
1123 	    *msi_state);
1124 
1125 	return (DDI_SUCCESS);
1126 }
1127 
1128 /*ARGSUSED*/
1129 int
1130 px_lib_msi_setstate(dev_info_t *dip, msinum_t msi_num,
1131     pci_msi_state_t msi_state)
1132 {
1133 	uint64_t	ret;
1134 
1135 	DBG(DBG_LIB_MSI, dip, "px_lib_msi_setstate: dip 0x%p msi_num 0x%x "
1136 	    "msi_state 0x%x\n", dip, msi_num, msi_state);
1137 
1138 	if ((ret = hvio_msi_setstate(DIP_TO_HANDLE(dip),
1139 	    msi_num, msi_state)) != H_EOK) {
1140 		DBG(DBG_LIB_MSI, dip,
1141 		    "hvio_msi_setstate failed, ret 0x%lx\n", ret);
1142 		return (DDI_FAILURE);
1143 	}
1144 
1145 	return (DDI_SUCCESS);
1146 }
1147 
1148 /*
1149  * MSG Functions:
1150  */
1151 /*ARGSUSED*/
1152 int
1153 px_lib_msg_getmsiq(dev_info_t *dip, pcie_msg_type_t msg_type,
1154     msiqid_t *msiq_id)
1155 {
1156 	uint64_t	ret;
1157 
1158 	DBG(DBG_LIB_MSG, dip, "px_lib_msg_getmsiq: dip 0x%p msg_type 0x%x\n",
1159 	    dip, msg_type);
1160 
1161 	if ((ret = hvio_msg_getmsiq(DIP_TO_HANDLE(dip),
1162 	    msg_type, msiq_id)) != H_EOK) {
1163 		DBG(DBG_LIB_MSG, dip,
1164 		    "hvio_msg_getmsiq failed, ret 0x%lx\n", ret);
1165 		return (DDI_FAILURE);
1166 	}
1167 
1168 	DBG(DBG_LIB_MSI, dip, "px_lib_msg_getmsiq: msiq_id 0x%x\n",
1169 	    *msiq_id);
1170 
1171 	return (DDI_SUCCESS);
1172 }
1173 
1174 /*ARGSUSED*/
1175 int
1176 px_lib_msg_setmsiq(dev_info_t *dip, pcie_msg_type_t msg_type,
1177     msiqid_t msiq_id)
1178 {
1179 	uint64_t	ret;
1180 
1181 	DBG(DBG_LIB_MSG, dip, "px_lib_msi_setstate: dip 0x%p msg_type 0x%x "
1182 	    "msiq_id 0x%x\n", dip, msg_type, msiq_id);
1183 
1184 	if ((ret = hvio_msg_setmsiq(DIP_TO_HANDLE(dip),
1185 	    msg_type, msiq_id)) != H_EOK) {
1186 		DBG(DBG_LIB_MSG, dip,
1187 		    "hvio_msg_setmsiq failed, ret 0x%lx\n", ret);
1188 		return (DDI_FAILURE);
1189 	}
1190 
1191 	return (DDI_SUCCESS);
1192 }
1193 
1194 /*ARGSUSED*/
1195 int
1196 px_lib_msg_getvalid(dev_info_t *dip, pcie_msg_type_t msg_type,
1197     pcie_msg_valid_state_t *msg_valid_state)
1198 {
1199 	uint64_t	ret;
1200 
1201 	DBG(DBG_LIB_MSG, dip, "px_lib_msg_getvalid: dip 0x%p msg_type 0x%x\n",
1202 	    dip, msg_type);
1203 
1204 	if ((ret = hvio_msg_getvalid(DIP_TO_HANDLE(dip), msg_type,
1205 	    msg_valid_state)) != H_EOK) {
1206 		DBG(DBG_LIB_MSG, dip,
1207 		    "hvio_msg_getvalid failed, ret 0x%lx\n", ret);
1208 		return (DDI_FAILURE);
1209 	}
1210 
1211 	DBG(DBG_LIB_MSI, dip, "px_lib_msg_getvalid: msg_valid_state 0x%x\n",
1212 	    *msg_valid_state);
1213 
1214 	return (DDI_SUCCESS);
1215 }
1216 
1217 /*ARGSUSED*/
1218 int
1219 px_lib_msg_setvalid(dev_info_t *dip, pcie_msg_type_t msg_type,
1220     pcie_msg_valid_state_t msg_valid_state)
1221 {
1222 	uint64_t	ret;
1223 
1224 	DBG(DBG_LIB_MSG, dip, "px_lib_msg_setvalid: dip 0x%p msg_type 0x%x "
1225 	    "msg_valid_state 0x%x\n", dip, msg_type, msg_valid_state);
1226 
1227 	if ((ret = hvio_msg_setvalid(DIP_TO_HANDLE(dip), msg_type,
1228 	    msg_valid_state)) != H_EOK) {
1229 		DBG(DBG_LIB_MSG, dip,
1230 		    "hvio_msg_setvalid failed, ret 0x%lx\n", ret);
1231 		return (DDI_FAILURE);
1232 	}
1233 
1234 	return (DDI_SUCCESS);
1235 }
1236 
1237 /*
1238  * Suspend/Resume Functions:
1239  * Currently unsupported by hypervisor
1240  */
1241 int
1242 px_lib_suspend(dev_info_t *dip)
1243 {
1244 	px_t		*px_p = DIP_TO_STATE(dip);
1245 	pxu_t		*pxu_p = (pxu_t *)px_p->px_plat_p;
1246 	px_cb_t		*cb_p = PX2CB(px_p);
1247 	devhandle_t	dev_hdl, xbus_dev_hdl;
1248 	uint64_t	ret = H_EOK;
1249 
1250 	DBG(DBG_DETACH, dip, "px_lib_suspend: dip 0x%p\n", dip);
1251 
1252 	dev_hdl = (devhandle_t)pxu_p->px_address[PX_REG_CSR];
1253 	xbus_dev_hdl = (devhandle_t)pxu_p->px_address[PX_REG_XBC];
1254 
1255 	if ((ret = hvio_suspend(dev_hdl, pxu_p)) != H_EOK)
1256 		goto fail;
1257 
1258 	if (--cb_p->attachcnt == 0) {
1259 		ret = hvio_cb_suspend(xbus_dev_hdl, pxu_p);
1260 		if (ret != H_EOK)
1261 			cb_p->attachcnt++;
1262 	}
1263 	pxu_p->cpr_flag = PX_ENTERED_CPR;
1264 
1265 fail:
1266 	return ((ret != H_EOK) ? DDI_FAILURE: DDI_SUCCESS);
1267 }
1268 
1269 void
1270 px_lib_resume(dev_info_t *dip)
1271 {
1272 	px_t		*px_p = DIP_TO_STATE(dip);
1273 	pxu_t		*pxu_p = (pxu_t *)px_p->px_plat_p;
1274 	px_cb_t		*cb_p = PX2CB(px_p);
1275 	devhandle_t	dev_hdl, xbus_dev_hdl;
1276 	devino_t	pec_ino = px_p->px_inos[PX_INTR_PEC];
1277 	devino_t	xbc_ino = px_p->px_inos[PX_INTR_XBC];
1278 
1279 	DBG(DBG_ATTACH, dip, "px_lib_resume: dip 0x%p\n", dip);
1280 
1281 	dev_hdl = (devhandle_t)pxu_p->px_address[PX_REG_CSR];
1282 	xbus_dev_hdl = (devhandle_t)pxu_p->px_address[PX_REG_XBC];
1283 
1284 	if (++cb_p->attachcnt == 1)
1285 		hvio_cb_resume(dev_hdl, xbus_dev_hdl, xbc_ino, pxu_p);
1286 
1287 	hvio_resume(dev_hdl, pec_ino, pxu_p);
1288 }
1289 
1290 /*
1291  * Generate a unique Oberon UBC ID based on the Logicial System Board and
1292  * the IO Channel from the portid property field.
1293  */
1294 static uint64_t
1295 oberon_get_ubc_id(dev_info_t *dip)
1296 {
1297 	px_t	*px_p = DIP_TO_STATE(dip);
1298 	pxu_t	*pxu_p = (pxu_t *)px_p->px_plat_p;
1299 	uint64_t	ubc_id;
1300 
1301 	/*
1302 	 * Generate a unique 6 bit UBC ID using the 2 IO_Channel#[1:0] bits and
1303 	 * the 4 LSB_ID[3:0] bits from the Oberon's portid property.
1304 	 */
1305 	ubc_id = (((pxu_p->portid >> OBERON_PORT_ID_IOC) &
1306 	    OBERON_PORT_ID_IOC_MASK) | (((pxu_p->portid >>
1307 	    OBERON_PORT_ID_LSB) & OBERON_PORT_ID_LSB_MASK)
1308 	    << OBERON_UBC_ID_LSB));
1309 
1310 	return (ubc_id);
1311 }
1312 
1313 /*
1314  * Oberon does not have a UBC scratch register, so alloc an array of scratch
1315  * registers when needed and use a unique UBC ID as an index. This code
1316  * can be simplified if we use a pre-allocated array. They are currently
1317  * being dynamically allocated because it's only needed by the Oberon.
1318  */
1319 static void
1320 oberon_set_cb(dev_info_t *dip, uint64_t val)
1321 {
1322 	uint64_t	ubc_id;
1323 
1324 	if (px_oberon_ubc_scratch_regs == NULL)
1325 		px_oberon_ubc_scratch_regs =
1326 		    (uint64_t *)kmem_zalloc(sizeof (uint64_t)*
1327 		    OBERON_UBC_ID_MAX, KM_SLEEP);
1328 
1329 	ubc_id = oberon_get_ubc_id(dip);
1330 
1331 	px_oberon_ubc_scratch_regs[ubc_id] = val;
1332 
1333 	/*
1334 	 * Check if any scratch registers are still in use. If all scratch
1335 	 * registers are currently set to zero, then deallocate the scratch
1336 	 * register array.
1337 	 */
1338 	for (ubc_id = 0; ubc_id < OBERON_UBC_ID_MAX; ubc_id++) {
1339 		if (px_oberon_ubc_scratch_regs[ubc_id] != NULL)
1340 			return;
1341 	}
1342 
1343 	/*
1344 	 * All scratch registers are set to zero so deallocate the scratch
1345 	 * register array and set the pointer to NULL.
1346 	 */
1347 	kmem_free(px_oberon_ubc_scratch_regs,
1348 	    (sizeof (uint64_t)*OBERON_UBC_ID_MAX));
1349 
1350 	px_oberon_ubc_scratch_regs = NULL;
1351 }
1352 
1353 /*
1354  * Oberon does not have a UBC scratch register, so use an allocated array of
1355  * scratch registers and use the unique UBC ID as an index into that array.
1356  */
1357 static uint64_t
1358 oberon_get_cb(dev_info_t *dip)
1359 {
1360 	uint64_t	ubc_id;
1361 
1362 	if (px_oberon_ubc_scratch_regs == NULL)
1363 		return (0);
1364 
1365 	ubc_id = oberon_get_ubc_id(dip);
1366 
1367 	return (px_oberon_ubc_scratch_regs[ubc_id]);
1368 }
1369 
1370 /*
1371  * Misc Functions:
1372  * Currently unsupported by hypervisor
1373  */
1374 static uint64_t
1375 px_get_cb(dev_info_t *dip)
1376 {
1377 	px_t	*px_p = DIP_TO_STATE(dip);
1378 	pxu_t	*pxu_p = (pxu_t *)px_p->px_plat_p;
1379 
1380 	/*
1381 	 * Oberon does not currently have Scratchpad registers.
1382 	 */
1383 	if (PX_CHIP_TYPE(pxu_p) == PX_CHIP_OBERON)
1384 		return (oberon_get_cb(dip));
1385 
1386 	return (CSR_XR((caddr_t)pxu_p->px_address[PX_REG_XBC], JBUS_SCRATCH_1));
1387 }
1388 
1389 static void
1390 px_set_cb(dev_info_t *dip, uint64_t val)
1391 {
1392 	px_t	*px_p = DIP_TO_STATE(dip);
1393 	pxu_t	*pxu_p = (pxu_t *)px_p->px_plat_p;
1394 
1395 	/*
1396 	 * Oberon does not currently have Scratchpad registers.
1397 	 */
1398 	if (PX_CHIP_TYPE(pxu_p) == PX_CHIP_OBERON) {
1399 		oberon_set_cb(dip, val);
1400 		return;
1401 	}
1402 
1403 	CSR_XS((caddr_t)pxu_p->px_address[PX_REG_XBC], JBUS_SCRATCH_1, val);
1404 }
1405 
1406 /*ARGSUSED*/
1407 int
1408 px_lib_map_vconfig(dev_info_t *dip,
1409 	ddi_map_req_t *mp, pci_config_offset_t off,
1410 		pci_regspec_t *rp, caddr_t *addrp)
1411 {
1412 	/*
1413 	 * No special config space access services in this layer.
1414 	 */
1415 	return (DDI_FAILURE);
1416 }
1417 
1418 void
1419 px_lib_map_attr_check(ddi_map_req_t *mp)
1420 {
1421 	ddi_acc_hdl_t *hp = mp->map_handlep;
1422 
1423 	/* fire does not accept byte masks from PIO store merge */
1424 	if (hp->ah_acc.devacc_attr_dataorder == DDI_STORECACHING_OK_ACC)
1425 		hp->ah_acc.devacc_attr_dataorder = DDI_STRICTORDER_ACC;
1426 }
1427 
1428 /* This function is called only by poke, caut put and pxtool poke. */
1429 void
1430 px_lib_clr_errs(px_t *px_p, dev_info_t *rdip, uint64_t addr)
1431 {
1432 	px_pec_t	*pec_p = px_p->px_pec_p;
1433 	dev_info_t	*rpdip = px_p->px_dip;
1434 	int		rc_err, fab_err, i;
1435 	int		acctype = pec_p->pec_safeacc_type;
1436 	ddi_fm_error_t	derr;
1437 	px_ranges_t	*ranges_p;
1438 	int		range_len;
1439 	uint32_t	addr_high, addr_low;
1440 	pcie_req_id_t	bdf = 0;
1441 
1442 	/* Create the derr */
1443 	bzero(&derr, sizeof (ddi_fm_error_t));
1444 	derr.fme_version = DDI_FME_VERSION;
1445 	derr.fme_ena = fm_ena_generate(0, FM_ENA_FMT1);
1446 	derr.fme_flag = acctype;
1447 
1448 	if (acctype == DDI_FM_ERR_EXPECTED) {
1449 		derr.fme_status = DDI_FM_NONFATAL;
1450 		ndi_fm_acc_err_set(pec_p->pec_acc_hdl, &derr);
1451 	}
1452 
1453 	mutex_enter(&px_p->px_fm_mutex);
1454 
1455 	/* send ereport/handle/clear fire registers */
1456 	rc_err = px_err_cmn_intr(px_p, &derr, PX_LIB_CALL, PX_FM_BLOCK_ALL);
1457 
1458 	/* Figure out if this is a cfg or mem32 access */
1459 	addr_high = (uint32_t)(addr >> 32);
1460 	addr_low = (uint32_t)addr;
1461 	range_len = px_p->px_ranges_length / sizeof (px_ranges_t);
1462 	i = 0;
1463 	for (ranges_p = px_p->px_ranges_p; i < range_len; i++, ranges_p++) {
1464 		if (ranges_p->parent_high == addr_high) {
1465 			switch (ranges_p->child_high & PCI_ADDR_MASK) {
1466 			case PCI_ADDR_CONFIG:
1467 				bdf = (pcie_req_id_t)(addr_low >> 12);
1468 				addr_low = 0;
1469 				break;
1470 			case PCI_ADDR_MEM32:
1471 				if (rdip)
1472 					(void) pcie_get_bdf_from_dip(rdip,
1473 					    &bdf);
1474 				else
1475 					bdf = NULL;
1476 				break;
1477 			}
1478 			break;
1479 		}
1480 	}
1481 
1482 	px_rp_en_q(px_p, bdf, addr_low, NULL);
1483 
1484 	/*
1485 	 * XXX - Current code scans the fabric for all px_tool accesses.
1486 	 * In future, do not scan fabric for px_tool access to IO Root Nexus
1487 	 */
1488 	fab_err = pf_scan_fabric(rpdip, &derr, px_p->px_dq_p,
1489 	    &px_p->px_dq_tail);
1490 
1491 	mutex_exit(&px_p->px_fm_mutex);
1492 
1493 	px_err_panic(rc_err, PX_RC, fab_err);
1494 }
1495 
1496 #ifdef  DEBUG
1497 int	px_peekfault_cnt = 0;
1498 int	px_pokefault_cnt = 0;
1499 #endif  /* DEBUG */
1500 
1501 /*ARGSUSED*/
1502 static int
1503 px_lib_do_poke(dev_info_t *dip, dev_info_t *rdip,
1504     peekpoke_ctlops_t *in_args)
1505 {
1506 	px_t *px_p = DIP_TO_STATE(dip);
1507 	px_pec_t *pec_p = px_p->px_pec_p;
1508 	int err = DDI_SUCCESS;
1509 	on_trap_data_t otd;
1510 
1511 	mutex_enter(&pec_p->pec_pokefault_mutex);
1512 	pec_p->pec_ontrap_data = &otd;
1513 	pec_p->pec_safeacc_type = DDI_FM_ERR_POKE;
1514 
1515 	/* Set up protected environment. */
1516 	if (!on_trap(&otd, OT_DATA_ACCESS)) {
1517 		uintptr_t tramp = otd.ot_trampoline;
1518 
1519 		otd.ot_trampoline = (uintptr_t)&poke_fault;
1520 		err = do_poke(in_args->size, (void *)in_args->dev_addr,
1521 		    (void *)in_args->host_addr);
1522 		otd.ot_trampoline = tramp;
1523 	} else
1524 		err = DDI_FAILURE;
1525 
1526 	px_lib_clr_errs(px_p, rdip, in_args->dev_addr);
1527 
1528 	if (otd.ot_trap & OT_DATA_ACCESS)
1529 		err = DDI_FAILURE;
1530 
1531 	/* Take down protected environment. */
1532 	no_trap();
1533 
1534 	pec_p->pec_ontrap_data = NULL;
1535 	pec_p->pec_safeacc_type = DDI_FM_ERR_UNEXPECTED;
1536 	mutex_exit(&pec_p->pec_pokefault_mutex);
1537 
1538 #ifdef  DEBUG
1539 	if (err == DDI_FAILURE)
1540 		px_pokefault_cnt++;
1541 #endif
1542 	return (err);
1543 }
1544 
1545 /*ARGSUSED*/
1546 static int
1547 px_lib_do_caut_put(dev_info_t *dip, dev_info_t *rdip,
1548     peekpoke_ctlops_t *cautacc_ctlops_arg)
1549 {
1550 	size_t size = cautacc_ctlops_arg->size;
1551 	uintptr_t dev_addr = cautacc_ctlops_arg->dev_addr;
1552 	uintptr_t host_addr = cautacc_ctlops_arg->host_addr;
1553 	ddi_acc_impl_t *hp = (ddi_acc_impl_t *)cautacc_ctlops_arg->handle;
1554 	size_t repcount = cautacc_ctlops_arg->repcount;
1555 	uint_t flags = cautacc_ctlops_arg->flags;
1556 
1557 	px_t *px_p = DIP_TO_STATE(dip);
1558 	px_pec_t *pec_p = px_p->px_pec_p;
1559 	int err = DDI_SUCCESS;
1560 
1561 	/*
1562 	 * Note that i_ndi_busop_access_enter ends up grabbing the pokefault
1563 	 * mutex.
1564 	 */
1565 	i_ndi_busop_access_enter(hp->ahi_common.ah_dip, (ddi_acc_handle_t)hp);
1566 
1567 	pec_p->pec_ontrap_data = (on_trap_data_t *)hp->ahi_err->err_ontrap;
1568 	pec_p->pec_safeacc_type = DDI_FM_ERR_EXPECTED;
1569 	hp->ahi_err->err_expected = DDI_FM_ERR_EXPECTED;
1570 
1571 	if (!i_ddi_ontrap((ddi_acc_handle_t)hp)) {
1572 		for (; repcount; repcount--) {
1573 			switch (size) {
1574 
1575 			case sizeof (uint8_t):
1576 				i_ddi_put8(hp, (uint8_t *)dev_addr,
1577 				    *(uint8_t *)host_addr);
1578 				break;
1579 
1580 			case sizeof (uint16_t):
1581 				i_ddi_put16(hp, (uint16_t *)dev_addr,
1582 				    *(uint16_t *)host_addr);
1583 				break;
1584 
1585 			case sizeof (uint32_t):
1586 				i_ddi_put32(hp, (uint32_t *)dev_addr,
1587 				    *(uint32_t *)host_addr);
1588 				break;
1589 
1590 			case sizeof (uint64_t):
1591 				i_ddi_put64(hp, (uint64_t *)dev_addr,
1592 				    *(uint64_t *)host_addr);
1593 				break;
1594 			}
1595 
1596 			host_addr += size;
1597 
1598 			if (flags == DDI_DEV_AUTOINCR)
1599 				dev_addr += size;
1600 
1601 			px_lib_clr_errs(px_p, rdip, dev_addr);
1602 
1603 			if (pec_p->pec_ontrap_data->ot_trap & OT_DATA_ACCESS) {
1604 				err = DDI_FAILURE;
1605 #ifdef  DEBUG
1606 				px_pokefault_cnt++;
1607 #endif
1608 				break;
1609 			}
1610 		}
1611 	}
1612 
1613 	i_ddi_notrap((ddi_acc_handle_t)hp);
1614 	pec_p->pec_ontrap_data = NULL;
1615 	pec_p->pec_safeacc_type = DDI_FM_ERR_UNEXPECTED;
1616 	i_ndi_busop_access_exit(hp->ahi_common.ah_dip, (ddi_acc_handle_t)hp);
1617 	hp->ahi_err->err_expected = DDI_FM_ERR_UNEXPECTED;
1618 
1619 	return (err);
1620 }
1621 
1622 
1623 int
1624 px_lib_ctlops_poke(dev_info_t *dip, dev_info_t *rdip,
1625     peekpoke_ctlops_t *in_args)
1626 {
1627 	return (in_args->handle ? px_lib_do_caut_put(dip, rdip, in_args) :
1628 	    px_lib_do_poke(dip, rdip, in_args));
1629 }
1630 
1631 
1632 /*ARGSUSED*/
1633 static int
1634 px_lib_do_peek(dev_info_t *dip, peekpoke_ctlops_t *in_args)
1635 {
1636 	px_t *px_p = DIP_TO_STATE(dip);
1637 	px_pec_t *pec_p = px_p->px_pec_p;
1638 	int err = DDI_SUCCESS;
1639 	on_trap_data_t otd;
1640 
1641 	mutex_enter(&pec_p->pec_pokefault_mutex);
1642 	pec_p->pec_safeacc_type = DDI_FM_ERR_PEEK;
1643 
1644 	if (!on_trap(&otd, OT_DATA_ACCESS)) {
1645 		uintptr_t tramp = otd.ot_trampoline;
1646 
1647 		otd.ot_trampoline = (uintptr_t)&peek_fault;
1648 		err = do_peek(in_args->size, (void *)in_args->dev_addr,
1649 		    (void *)in_args->host_addr);
1650 		otd.ot_trampoline = tramp;
1651 	} else
1652 		err = DDI_FAILURE;
1653 
1654 	no_trap();
1655 	pec_p->pec_safeacc_type = DDI_FM_ERR_UNEXPECTED;
1656 	mutex_exit(&pec_p->pec_pokefault_mutex);
1657 
1658 #ifdef  DEBUG
1659 	if (err == DDI_FAILURE)
1660 		px_peekfault_cnt++;
1661 #endif
1662 	return (err);
1663 }
1664 
1665 
1666 static int
1667 px_lib_do_caut_get(dev_info_t *dip, peekpoke_ctlops_t *cautacc_ctlops_arg)
1668 {
1669 	size_t size = cautacc_ctlops_arg->size;
1670 	uintptr_t dev_addr = cautacc_ctlops_arg->dev_addr;
1671 	uintptr_t host_addr = cautacc_ctlops_arg->host_addr;
1672 	ddi_acc_impl_t *hp = (ddi_acc_impl_t *)cautacc_ctlops_arg->handle;
1673 	size_t repcount = cautacc_ctlops_arg->repcount;
1674 	uint_t flags = cautacc_ctlops_arg->flags;
1675 
1676 	px_t *px_p = DIP_TO_STATE(dip);
1677 	px_pec_t *pec_p = px_p->px_pec_p;
1678 	int err = DDI_SUCCESS;
1679 
1680 	/*
1681 	 * Note that i_ndi_busop_access_enter ends up grabbing the pokefault
1682 	 * mutex.
1683 	 */
1684 	i_ndi_busop_access_enter(hp->ahi_common.ah_dip, (ddi_acc_handle_t)hp);
1685 
1686 	pec_p->pec_ontrap_data = (on_trap_data_t *)hp->ahi_err->err_ontrap;
1687 	pec_p->pec_safeacc_type = DDI_FM_ERR_EXPECTED;
1688 	hp->ahi_err->err_expected = DDI_FM_ERR_EXPECTED;
1689 
1690 	if (repcount == 1) {
1691 		if (!i_ddi_ontrap((ddi_acc_handle_t)hp)) {
1692 			i_ddi_caut_get(size, (void *)dev_addr,
1693 			    (void *)host_addr);
1694 		} else {
1695 			int i;
1696 			uint8_t *ff_addr = (uint8_t *)host_addr;
1697 			for (i = 0; i < size; i++)
1698 				*ff_addr++ = 0xff;
1699 
1700 			err = DDI_FAILURE;
1701 #ifdef  DEBUG
1702 			px_peekfault_cnt++;
1703 #endif
1704 		}
1705 	} else {
1706 		if (!i_ddi_ontrap((ddi_acc_handle_t)hp)) {
1707 			for (; repcount; repcount--) {
1708 				i_ddi_caut_get(size, (void *)dev_addr,
1709 				    (void *)host_addr);
1710 
1711 				host_addr += size;
1712 
1713 				if (flags == DDI_DEV_AUTOINCR)
1714 					dev_addr += size;
1715 			}
1716 		} else {
1717 			err = DDI_FAILURE;
1718 #ifdef  DEBUG
1719 			px_peekfault_cnt++;
1720 #endif
1721 		}
1722 	}
1723 
1724 	i_ddi_notrap((ddi_acc_handle_t)hp);
1725 	pec_p->pec_ontrap_data = NULL;
1726 	pec_p->pec_safeacc_type = DDI_FM_ERR_UNEXPECTED;
1727 	i_ndi_busop_access_exit(hp->ahi_common.ah_dip, (ddi_acc_handle_t)hp);
1728 	hp->ahi_err->err_expected = DDI_FM_ERR_UNEXPECTED;
1729 
1730 	return (err);
1731 }
1732 
1733 /*ARGSUSED*/
1734 int
1735 px_lib_ctlops_peek(dev_info_t *dip, dev_info_t *rdip,
1736     peekpoke_ctlops_t *in_args, void *result)
1737 {
1738 	result = (void *)in_args->host_addr;
1739 	return (in_args->handle ? px_lib_do_caut_get(dip, in_args) :
1740 	    px_lib_do_peek(dip, in_args));
1741 }
1742 
1743 /*
1744  * implements PPM interface
1745  */
1746 int
1747 px_lib_pmctl(int cmd, px_t *px_p)
1748 {
1749 	ASSERT((cmd & ~PPMREQ_MASK) == PPMREQ);
1750 	switch (cmd) {
1751 	case PPMREQ_PRE_PWR_OFF:
1752 		/*
1753 		 * Currently there is no device power management for
1754 		 * the root complex (fire). When there is we need to make
1755 		 * sure that it is at full power before trying to send the
1756 		 * PME_Turn_Off message.
1757 		 */
1758 		DBG(DBG_PWR, px_p->px_dip,
1759 		    "ioctl: request to send PME_Turn_Off\n");
1760 		return (px_goto_l23ready(px_p));
1761 
1762 	case PPMREQ_PRE_PWR_ON:
1763 		DBG(DBG_PWR, px_p->px_dip, "ioctl: PRE_PWR_ON request\n");
1764 		return (px_pre_pwron_check(px_p));
1765 
1766 	case PPMREQ_POST_PWR_ON:
1767 		DBG(DBG_PWR, px_p->px_dip, "ioctl: POST_PWR_ON request\n");
1768 		return (px_goto_l0(px_p));
1769 
1770 	default:
1771 		return (DDI_FAILURE);
1772 	}
1773 }
1774 
1775 /*
1776  * sends PME_Turn_Off message to put the link in L2/L3 ready state.
1777  * called by px_ioctl.
1778  * returns DDI_SUCCESS or DDI_FAILURE
1779  * 1. Wait for link to be in L1 state (link status reg)
1780  * 2. write to PME_Turn_off reg to boradcast
1781  * 3. set timeout
1782  * 4. If timeout, return failure.
1783  * 5. If PM_TO_Ack, wait till link is in L2/L3 ready
1784  */
1785 static int
1786 px_goto_l23ready(px_t *px_p)
1787 {
1788 	pcie_pwr_t	*pwr_p;
1789 	pxu_t		*pxu_p = (pxu_t *)px_p->px_plat_p;
1790 	caddr_t	csr_base = (caddr_t)pxu_p->px_address[PX_REG_CSR];
1791 	int		ret = DDI_SUCCESS;
1792 	clock_t		end, timeleft;
1793 	int		mutex_held = 1;
1794 
1795 	/* If no PM info, return failure */
1796 	if (!PCIE_PMINFO(px_p->px_dip) ||
1797 	    !(pwr_p = PCIE_NEXUS_PMINFO(px_p->px_dip)))
1798 		return (DDI_FAILURE);
1799 
1800 	mutex_enter(&pwr_p->pwr_lock);
1801 	mutex_enter(&px_p->px_l23ready_lock);
1802 	/* Clear the PME_To_ACK receieved flag */
1803 	px_p->px_pm_flags &= ~PX_PMETOACK_RECVD;
1804 	/*
1805 	 * When P25 is the downstream device, after receiving
1806 	 * PME_To_ACK, fire will go to Detect state, which causes
1807 	 * the link down event. Inform FMA that this is expected.
1808 	 * In case of all other cards complaint with the pci express
1809 	 * spec, this will happen when the power is re-applied. FMA
1810 	 * code will clear this flag after one instance of LDN. Since
1811 	 * there will not be a LDN event for the spec compliant cards,
1812 	 * we need to clear the flag after receiving PME_To_ACK.
1813 	 */
1814 	px_p->px_pm_flags |= PX_LDN_EXPECTED;
1815 	if (px_send_pme_turnoff(csr_base) != DDI_SUCCESS) {
1816 		ret = DDI_FAILURE;
1817 		goto l23ready_done;
1818 	}
1819 	px_p->px_pm_flags |= PX_PME_TURNOFF_PENDING;
1820 
1821 	end = ddi_get_lbolt() + drv_usectohz(px_pme_to_ack_timeout);
1822 	while (!(px_p->px_pm_flags & PX_PMETOACK_RECVD)) {
1823 		timeleft = cv_timedwait(&px_p->px_l23ready_cv,
1824 		    &px_p->px_l23ready_lock, end);
1825 		/*
1826 		 * if cv_timedwait returns -1, it is either
1827 		 * 1) timed out or
1828 		 * 2) there was a pre-mature wakeup but by the time
1829 		 * cv_timedwait is called again end < lbolt i.e.
1830 		 * end is in the past.
1831 		 * 3) By the time we make first cv_timedwait call,
1832 		 * end < lbolt is true.
1833 		 */
1834 		if (timeleft == -1)
1835 			break;
1836 	}
1837 	if (!(px_p->px_pm_flags & PX_PMETOACK_RECVD)) {
1838 		/*
1839 		 * Either timedout or interrupt didn't get a
1840 		 * chance to grab the mutex and set the flag.
1841 		 * release the mutex and delay for sometime.
1842 		 * This will 1) give a chance for interrupt to
1843 		 * set the flag 2) creates a delay between two
1844 		 * consequetive requests.
1845 		 */
1846 		mutex_exit(&px_p->px_l23ready_lock);
1847 		delay(drv_usectohz(50 * PX_MSEC_TO_USEC));
1848 		mutex_held = 0;
1849 		if (!(px_p->px_pm_flags & PX_PMETOACK_RECVD)) {
1850 			ret = DDI_FAILURE;
1851 			DBG(DBG_PWR, px_p->px_dip, " Timed out while waiting"
1852 			    " for PME_TO_ACK\n");
1853 		}
1854 	}
1855 	px_p->px_pm_flags &=
1856 	    ~(PX_PME_TURNOFF_PENDING | PX_PMETOACK_RECVD | PX_LDN_EXPECTED);
1857 
1858 l23ready_done:
1859 	if (mutex_held)
1860 		mutex_exit(&px_p->px_l23ready_lock);
1861 	/*
1862 	 * Wait till link is in L1 idle, if sending PME_Turn_Off
1863 	 * was succesful.
1864 	 */
1865 	if (ret == DDI_SUCCESS) {
1866 		if (px_link_wait4l1idle(csr_base) != DDI_SUCCESS) {
1867 			DBG(DBG_PWR, px_p->px_dip, " Link is not at L1"
1868 			    " even though we received PME_To_ACK.\n");
1869 			/*
1870 			 * Workaround for hardware bug with P25.
1871 			 * Due to a hardware bug with P25, link state
1872 			 * will be Detect state rather than L1 after
1873 			 * link is transitioned to L23Ready state. Since
1874 			 * we don't know whether link is L23ready state
1875 			 * without Fire's state being L1_idle, we delay
1876 			 * here just to make sure that we wait till link
1877 			 * is transitioned to L23Ready state.
1878 			 */
1879 			delay(drv_usectohz(100 * PX_MSEC_TO_USEC));
1880 		}
1881 		pwr_p->pwr_link_lvl = PM_LEVEL_L3;
1882 
1883 	}
1884 	mutex_exit(&pwr_p->pwr_lock);
1885 	return (ret);
1886 }
1887 
1888 /*
1889  * Message interrupt handler intended to be shared for both
1890  * PME and PME_TO_ACK msg handling, currently only handles
1891  * PME_To_ACK message.
1892  */
1893 uint_t
1894 px_pmeq_intr(caddr_t arg)
1895 {
1896 	px_t	*px_p = (px_t *)arg;
1897 
1898 	DBG(DBG_PWR, px_p->px_dip, " PME_To_ACK received \n");
1899 	mutex_enter(&px_p->px_l23ready_lock);
1900 	cv_broadcast(&px_p->px_l23ready_cv);
1901 	if (px_p->px_pm_flags & PX_PME_TURNOFF_PENDING) {
1902 		px_p->px_pm_flags |= PX_PMETOACK_RECVD;
1903 	} else {
1904 		/*
1905 		 * This maybe the second ack received. If so then,
1906 		 * we should be receiving it during wait4L1 stage.
1907 		 */
1908 		px_p->px_pmetoack_ignored++;
1909 	}
1910 	mutex_exit(&px_p->px_l23ready_lock);
1911 	return (DDI_INTR_CLAIMED);
1912 }
1913 
1914 static int
1915 px_pre_pwron_check(px_t *px_p)
1916 {
1917 	pcie_pwr_t	*pwr_p;
1918 
1919 	/* If no PM info, return failure */
1920 	if (!PCIE_PMINFO(px_p->px_dip) ||
1921 	    !(pwr_p = PCIE_NEXUS_PMINFO(px_p->px_dip)))
1922 		return (DDI_FAILURE);
1923 
1924 	/*
1925 	 * For the spec compliant downstream cards link down
1926 	 * is expected when the device is powered on.
1927 	 */
1928 	px_p->px_pm_flags |= PX_LDN_EXPECTED;
1929 	return (pwr_p->pwr_link_lvl == PM_LEVEL_L3 ? DDI_SUCCESS : DDI_FAILURE);
1930 }
1931 
1932 static int
1933 px_goto_l0(px_t *px_p)
1934 {
1935 	pcie_pwr_t	*pwr_p;
1936 	pxu_t		*pxu_p = (pxu_t *)px_p->px_plat_p;
1937 	caddr_t csr_base = (caddr_t)pxu_p->px_address[PX_REG_CSR];
1938 	int		ret = DDI_SUCCESS;
1939 	uint64_t	time_spent = 0;
1940 
1941 	/* If no PM info, return failure */
1942 	if (!PCIE_PMINFO(px_p->px_dip) ||
1943 	    !(pwr_p = PCIE_NEXUS_PMINFO(px_p->px_dip)))
1944 		return (DDI_FAILURE);
1945 
1946 	mutex_enter(&pwr_p->pwr_lock);
1947 	/*
1948 	 * The following link retrain activity will cause LDN and LUP event.
1949 	 * Receiving LDN prior to receiving LUP is expected, not an error in
1950 	 * this case.  Receiving LUP indicates link is fully up to support
1951 	 * powering up down stream device, and of course any further LDN and
1952 	 * LUP outside this context will be error.
1953 	 */
1954 	px_p->px_lup_pending = 1;
1955 	if (px_link_retrain(csr_base) != DDI_SUCCESS) {
1956 		ret = DDI_FAILURE;
1957 		goto l0_done;
1958 	}
1959 
1960 	/* LUP event takes the order of 15ms amount of time to occur */
1961 	for (; px_p->px_lup_pending && (time_spent < px_lup_poll_to);
1962 	    time_spent += px_lup_poll_interval)
1963 		drv_usecwait(px_lup_poll_interval);
1964 	if (px_p->px_lup_pending)
1965 		ret = DDI_FAILURE;
1966 l0_done:
1967 	px_enable_detect_quiet(csr_base);
1968 	if (ret == DDI_SUCCESS)
1969 		pwr_p->pwr_link_lvl = PM_LEVEL_L0;
1970 	mutex_exit(&pwr_p->pwr_lock);
1971 	return (ret);
1972 }
1973 
1974 /*
1975  * Extract the drivers binding name to identify which chip we're binding to.
1976  * Whenever a new bus bridge is created, the driver alias entry should be
1977  * added here to identify the device if needed.  If a device isn't added,
1978  * the identity defaults to PX_CHIP_UNIDENTIFIED.
1979  */
1980 static uint32_t
1981 px_identity_init(px_t *px_p)
1982 {
1983 	dev_info_t	*dip = px_p->px_dip;
1984 	char		*name = ddi_binding_name(dip);
1985 	uint32_t	revision = 0;
1986 
1987 	revision = ddi_prop_get_int(DDI_DEV_T_ANY, dip, DDI_PROP_DONTPASS,
1988 	    "module-revision#", 0);
1989 
1990 	/* Check for Fire driver binding name */
1991 	if (strcmp(name, "pciex108e,80f0") == 0) {
1992 		DBG(DBG_ATTACH, dip, "px_identity_init: %s%d: "
1993 		    "(FIRE), module-revision %d\n", NAMEINST(dip),
1994 		    revision);
1995 
1996 		return ((revision >= FIRE_MOD_REV_20) ?
1997 		    PX_CHIP_FIRE : PX_CHIP_UNIDENTIFIED);
1998 	}
1999 
2000 	/* Check for Oberon driver binding name */
2001 	if (strcmp(name, "pciex108e,80f8") == 0) {
2002 		DBG(DBG_ATTACH, dip, "px_identity_init: %s%d: "
2003 		    "(OBERON), module-revision %d\n", NAMEINST(dip),
2004 		    revision);
2005 
2006 		return (PX_CHIP_OBERON);
2007 	}
2008 
2009 	DBG(DBG_ATTACH, dip, "%s%d: Unknown PCI Express Host bridge %s %x\n",
2010 	    ddi_driver_name(dip), ddi_get_instance(dip), name, revision);
2011 
2012 	return (PX_CHIP_UNIDENTIFIED);
2013 }
2014 
2015 int
2016 px_err_add_intr(px_fault_t *px_fault_p)
2017 {
2018 	dev_info_t	*dip = px_fault_p->px_fh_dip;
2019 	px_t		*px_p = DIP_TO_STATE(dip);
2020 
2021 	VERIFY(add_ivintr(px_fault_p->px_fh_sysino, PX_ERR_PIL,
2022 	    (intrfunc)px_fault_p->px_err_func, (caddr_t)px_fault_p,
2023 	    NULL, NULL) == 0);
2024 
2025 	px_ib_intr_enable(px_p, intr_dist_cpuid(), px_fault_p->px_intr_ino);
2026 
2027 	return (DDI_SUCCESS);
2028 }
2029 
2030 void
2031 px_err_rem_intr(px_fault_t *px_fault_p)
2032 {
2033 	dev_info_t	*dip = px_fault_p->px_fh_dip;
2034 	px_t		*px_p = DIP_TO_STATE(dip);
2035 
2036 	px_ib_intr_disable(px_p->px_ib_p, px_fault_p->px_intr_ino,
2037 		IB_INTR_WAIT);
2038 
2039 	VERIFY(rem_ivintr(px_fault_p->px_fh_sysino, PX_ERR_PIL) == 0);
2040 }
2041 
2042 /*
2043  * px_cb_add_intr() - Called from attach(9E) to create CB if not yet
2044  * created, to add CB interrupt vector always, but enable only once.
2045  */
2046 int
2047 px_cb_add_intr(px_fault_t *fault_p)
2048 {
2049 	px_t		*px_p = DIP_TO_STATE(fault_p->px_fh_dip);
2050 	pxu_t		*pxu_p = (pxu_t *)px_p->px_plat_p;
2051 	px_cb_t		*cb_p = (px_cb_t *)px_get_cb(fault_p->px_fh_dip);
2052 	px_cb_list_t	*pxl, *pxl_new;
2053 	cpuid_t		cpuid;
2054 
2055 
2056 	if (cb_p == NULL) {
2057 		cb_p = kmem_zalloc(sizeof (px_cb_t), KM_SLEEP);
2058 		mutex_init(&cb_p->cb_mutex, NULL, MUTEX_DRIVER, NULL);
2059 		cb_p->px_cb_func = px_cb_intr;
2060 		pxu_p->px_cb_p = cb_p;
2061 		px_set_cb(fault_p->px_fh_dip, (uint64_t)cb_p);
2062 
2063 		/* px_lib_dev_init allows only FIRE and OBERON */
2064 		px_err_reg_enable(
2065 		    (pxu_p->chip_type == PX_CHIP_FIRE) ?
2066 			PX_ERR_JBC : PX_ERR_UBC,
2067 		    pxu_p->px_address[PX_REG_XBC]);
2068 	} else
2069 		pxu_p->px_cb_p = cb_p;
2070 
2071 	mutex_enter(&cb_p->cb_mutex);
2072 
2073 	VERIFY(add_ivintr(fault_p->px_fh_sysino, PX_ERR_PIL,
2074 	    (intrfunc)cb_p->px_cb_func, (caddr_t)cb_p, NULL, NULL) == 0);
2075 
2076 	if (cb_p->pxl == NULL) {
2077 
2078 		cpuid = intr_dist_cpuid(),
2079 		px_ib_intr_enable(px_p, cpuid, fault_p->px_intr_ino);
2080 
2081 		pxl = kmem_zalloc(sizeof (px_cb_list_t), KM_SLEEP);
2082 		pxl->pxp = px_p;
2083 
2084 		cb_p->pxl = pxl;
2085 		cb_p->sysino = fault_p->px_fh_sysino;
2086 		cb_p->cpuid = cpuid;
2087 
2088 	} else {
2089 		/*
2090 		 * Find the last pxl or
2091 		 * stop short at encoutering a redundent, or
2092 		 * both.
2093 		 */
2094 		pxl = cb_p->pxl;
2095 		for (; !(pxl->pxp == px_p) && pxl->next; pxl = pxl->next);
2096 		if (pxl->pxp == px_p) {
2097 			cmn_err(CE_WARN, "px_cb_add_intr: reregister sysino "
2098 			    "%lx by px_p 0x%p\n", cb_p->sysino, (void *)px_p);
2099 			return (DDI_FAILURE);
2100 		}
2101 
2102 		/* add to linked list */
2103 		pxl_new = kmem_zalloc(sizeof (px_cb_list_t), KM_SLEEP);
2104 		pxl_new->pxp = px_p;
2105 		pxl->next = pxl_new;
2106 	}
2107 	cb_p->attachcnt++;
2108 
2109 	mutex_exit(&cb_p->cb_mutex);
2110 
2111 	return (DDI_SUCCESS);
2112 }
2113 
2114 /*
2115  * px_cb_rem_intr() - Called from detach(9E) to remove its CB
2116  * interrupt vector, to shift proxy to the next available px,
2117  * or disable CB interrupt when itself is the last.
2118  */
2119 void
2120 px_cb_rem_intr(px_fault_t *fault_p)
2121 {
2122 	px_t		*px_p = DIP_TO_STATE(fault_p->px_fh_dip), *pxp;
2123 	pxu_t		*pxu_p = (pxu_t *)px_p->px_plat_p;
2124 	px_cb_t		*cb_p = PX2CB(px_p);
2125 	px_cb_list_t	*pxl, *prev;
2126 	px_fault_t	*f_p;
2127 
2128 	ASSERT(cb_p->pxl);
2129 
2130 	/* De-list the target px, move the next px up */
2131 
2132 	mutex_enter(&cb_p->cb_mutex);
2133 
2134 	pxl = cb_p->pxl;
2135 	if (pxl->pxp == px_p) {
2136 		cb_p->pxl = pxl->next;
2137 	} else {
2138 		prev = pxl;
2139 		pxl = pxl->next;
2140 		for (; pxl && (pxl->pxp != px_p); prev = pxl, pxl = pxl->next);
2141 		if (!pxl) {
2142 			cmn_err(CE_WARN, "px_cb_rem_intr: can't find px_p 0x%p "
2143 			    "in registered CB list.", (void *)px_p);
2144 			return;
2145 		}
2146 		prev->next = pxl->next;
2147 	}
2148 	kmem_free(pxl, sizeof (px_cb_list_t));
2149 
2150 	if (fault_p->px_fh_sysino == cb_p->sysino) {
2151 		px_ib_intr_disable(px_p->px_ib_p, fault_p->px_intr_ino,
2152 		    IB_INTR_WAIT);
2153 
2154 		if (cb_p->pxl) {
2155 			pxp = cb_p->pxl->pxp;
2156 			f_p = &pxp->px_cb_fault;
2157 			cb_p->sysino = f_p->px_fh_sysino;
2158 
2159 			PX_INTR_ENABLE(pxp->px_dip, cb_p->sysino, cb_p->cpuid);
2160 			(void) px_lib_intr_setstate(pxp->px_dip, cb_p->sysino,
2161 			    INTR_IDLE_STATE);
2162 		}
2163 	}
2164 
2165 	VERIFY(rem_ivintr(fault_p->px_fh_sysino, PX_ERR_PIL) == 0);
2166 	pxu_p->px_cb_p = NULL;
2167 	cb_p->attachcnt--;
2168 	if (cb_p->pxl) {
2169 		mutex_exit(&cb_p->cb_mutex);
2170 		return;
2171 	}
2172 	mutex_exit(&cb_p->cb_mutex);
2173 
2174 	/* px_lib_dev_init allows only FIRE and OBERON */
2175 	px_err_reg_disable(
2176 	    (pxu_p->chip_type == PX_CHIP_FIRE) ? PX_ERR_JBC : PX_ERR_UBC,
2177 	    pxu_p->px_address[PX_REG_XBC]);
2178 
2179 	mutex_destroy(&cb_p->cb_mutex);
2180 	px_set_cb(fault_p->px_fh_dip, 0ull);
2181 	kmem_free(cb_p, sizeof (px_cb_t));
2182 }
2183 
2184 /*
2185  * px_cb_intr() - sun4u only,  CB interrupt dispatcher
2186  */
2187 uint_t
2188 px_cb_intr(caddr_t arg)
2189 {
2190 	px_cb_t		*cb_p = (px_cb_t *)arg;
2191 	px_cb_list_t	*pxl = cb_p->pxl;
2192 	px_t		*pxp = pxl ? pxl->pxp : NULL;
2193 	px_fault_t	*fault_p;
2194 
2195 	while (pxl && pxp && (pxp->px_state != PX_ATTACHED)) {
2196 		pxl = pxl->next;
2197 		pxp = (pxl) ? pxl->pxp : NULL;
2198 	}
2199 
2200 	if (pxp) {
2201 		fault_p = &pxp->px_cb_fault;
2202 		return (fault_p->px_err_func((caddr_t)fault_p));
2203 	} else
2204 		return (DDI_INTR_UNCLAIMED);
2205 }
2206 
2207 /*
2208  * px_cb_intr_redist() - sun4u only, CB interrupt redistribution
2209  */
2210 void
2211 px_cb_intr_redist(px_t	*px_p)
2212 {
2213 	px_fault_t	*f_p = &px_p->px_cb_fault;
2214 	px_cb_t		*cb_p = PX2CB(px_p);
2215 	devino_t	ino = px_p->px_inos[PX_INTR_XBC];
2216 	cpuid_t		cpuid;
2217 
2218 	/* Make sure there is an interrupt control block setup. */
2219 	if (!cb_p)
2220 		return;
2221 
2222 	mutex_enter(&cb_p->cb_mutex);
2223 
2224 	if (cb_p->sysino != f_p->px_fh_sysino) {
2225 		mutex_exit(&cb_p->cb_mutex);
2226 		return;
2227 	}
2228 
2229 	cb_p->cpuid = cpuid = intr_dist_cpuid();
2230 	px_ib_intr_dist_en(px_p->px_dip, cpuid, ino, B_FALSE);
2231 
2232 	mutex_exit(&cb_p->cb_mutex);
2233 }
2234 
2235 #ifdef FMA
2236 void
2237 px_fill_rc_status(px_fault_t *px_fault_p, pciex_rc_error_regs_t *rc_status)
2238 {
2239 	/* populate the rc_status by reading the registers - TBD */
2240 }
2241 #endif /* FMA */
2242 
2243 /*
2244  * Unprotected raw reads/writes of fabric device's config space.
2245  * Only used for temporary PCI-E Fabric Error Handling.
2246  */
2247 uint32_t
2248 px_fab_get(px_t *px_p, pcie_req_id_t bdf, uint16_t offset)
2249 {
2250 	px_ranges_t	*rp = px_p->px_ranges_p;
2251 	uint64_t	range_prop, base_addr;
2252 	int		bank = PCI_REG_ADDR_G(PCI_ADDR_CONFIG);
2253 	uint32_t	val;
2254 
2255 	/* Get Fire's Physical Base Address */
2256 	range_prop = px_get_range_prop(px_p, rp, bank);
2257 
2258 	/* Get config space first. */
2259 	base_addr = range_prop + PX_BDF_TO_CFGADDR(bdf, offset);
2260 
2261 	val = ldphysio(base_addr);
2262 
2263 	return (LE_32(val));
2264 }
2265 
2266 void
2267 px_fab_set(px_t *px_p, pcie_req_id_t bdf, uint16_t offset,
2268     uint32_t val) {
2269 	px_ranges_t	*rp = px_p->px_ranges_p;
2270 	uint64_t	range_prop, base_addr;
2271 	int		bank = PCI_REG_ADDR_G(PCI_ADDR_CONFIG);
2272 
2273 	/* Get Fire's Physical Base Address */
2274 	range_prop = px_get_range_prop(px_p, rp, bank);
2275 
2276 	/* Get config space first. */
2277 	base_addr = range_prop + PX_BDF_TO_CFGADDR(bdf, offset);
2278 
2279 	stphysio(base_addr, LE_32(val));
2280 }
2281 
2282 /*
2283  * cpr callback
2284  *
2285  * disable fabric error msg interrupt prior to suspending
2286  * all device drivers; re-enable fabric error msg interrupt
2287  * after all devices are resumed.
2288  */
2289 static boolean_t
2290 px_cpr_callb(void *arg, int code)
2291 {
2292 	px_t		*px_p = (px_t *)arg;
2293 	px_ib_t		*ib_p = px_p->px_ib_p;
2294 	px_pec_t	*pec_p = px_p->px_pec_p;
2295 	pxu_t		*pxu_p = (pxu_t *)px_p->px_plat_p;
2296 	caddr_t		csr_base;
2297 	devino_t	ce_ino, nf_ino, f_ino;
2298 	px_ino_t	*ce_ino_p, *nf_ino_p, *f_ino_p;
2299 	uint64_t	imu_log_enable, imu_intr_enable;
2300 	uint64_t	imu_log_mask, imu_intr_mask;
2301 
2302 	ce_ino = px_msiqid_to_devino(px_p, pec_p->pec_corr_msg_msiq_id);
2303 	nf_ino = px_msiqid_to_devino(px_p, pec_p->pec_non_fatal_msg_msiq_id);
2304 	f_ino = px_msiqid_to_devino(px_p, pec_p->pec_fatal_msg_msiq_id);
2305 	csr_base = (caddr_t)pxu_p->px_address[PX_REG_CSR];
2306 
2307 	imu_log_enable = CSR_XR(csr_base, IMU_ERROR_LOG_ENABLE);
2308 	imu_intr_enable = CSR_XR(csr_base, IMU_INTERRUPT_ENABLE);
2309 
2310 	imu_log_mask = BITMASK(IMU_ERROR_LOG_ENABLE_FATAL_MES_NOT_EN_LOG_EN) |
2311 	    BITMASK(IMU_ERROR_LOG_ENABLE_NONFATAL_MES_NOT_EN_LOG_EN) |
2312 	    BITMASK(IMU_ERROR_LOG_ENABLE_COR_MES_NOT_EN_LOG_EN);
2313 
2314 	imu_intr_mask =
2315 	    BITMASK(IMU_INTERRUPT_ENABLE_FATAL_MES_NOT_EN_S_INT_EN) |
2316 	    BITMASK(IMU_INTERRUPT_ENABLE_NONFATAL_MES_NOT_EN_S_INT_EN) |
2317 	    BITMASK(IMU_INTERRUPT_ENABLE_COR_MES_NOT_EN_S_INT_EN) |
2318 	    BITMASK(IMU_INTERRUPT_ENABLE_FATAL_MES_NOT_EN_P_INT_EN) |
2319 	    BITMASK(IMU_INTERRUPT_ENABLE_NONFATAL_MES_NOT_EN_P_INT_EN) |
2320 	    BITMASK(IMU_INTERRUPT_ENABLE_COR_MES_NOT_EN_P_INT_EN);
2321 
2322 	switch (code) {
2323 	case CB_CODE_CPR_CHKPT:
2324 		/* disable imu rbne on corr/nonfatal/fatal errors */
2325 		CSR_XS(csr_base, IMU_ERROR_LOG_ENABLE,
2326 		    imu_log_enable & (~imu_log_mask));
2327 
2328 		CSR_XS(csr_base, IMU_INTERRUPT_ENABLE,
2329 		    imu_intr_enable & (~imu_intr_mask));
2330 
2331 		/* disable CORR intr mapping */
2332 		px_ib_intr_disable(ib_p, ce_ino, IB_INTR_NOWAIT);
2333 
2334 		/* disable NON FATAL intr mapping */
2335 		px_ib_intr_disable(ib_p, nf_ino, IB_INTR_NOWAIT);
2336 
2337 		/* disable FATAL intr mapping */
2338 		px_ib_intr_disable(ib_p, f_ino, IB_INTR_NOWAIT);
2339 
2340 		break;
2341 
2342 	case CB_CODE_CPR_RESUME:
2343 		pxu_p->cpr_flag = PX_NOT_CPR;
2344 		mutex_enter(&ib_p->ib_ino_lst_mutex);
2345 
2346 		ce_ino_p = px_ib_locate_ino(ib_p, ce_ino);
2347 		nf_ino_p = px_ib_locate_ino(ib_p, nf_ino);
2348 		f_ino_p = px_ib_locate_ino(ib_p, f_ino);
2349 
2350 		/* enable CORR intr mapping */
2351 		if (ce_ino_p)
2352 			px_ib_intr_enable(px_p, ce_ino_p->ino_cpuid, ce_ino);
2353 		else
2354 			cmn_err(CE_WARN, "px_cpr_callb: RESUME unable to "
2355 			    "reenable PCIe Correctable msg intr.\n");
2356 
2357 		/* enable NON FATAL intr mapping */
2358 		if (nf_ino_p)
2359 			px_ib_intr_enable(px_p, nf_ino_p->ino_cpuid, nf_ino);
2360 		else
2361 			cmn_err(CE_WARN, "px_cpr_callb: RESUME unable to "
2362 			    "reenable PCIe Non Fatal msg intr.\n");
2363 
2364 		/* enable FATAL intr mapping */
2365 		if (f_ino_p)
2366 			px_ib_intr_enable(px_p, f_ino_p->ino_cpuid, f_ino);
2367 		else
2368 			cmn_err(CE_WARN, "px_cpr_callb: RESUME unable to "
2369 			    "reenable PCIe Fatal msg intr.\n");
2370 
2371 		mutex_exit(&ib_p->ib_ino_lst_mutex);
2372 
2373 		/* enable corr/nonfatal/fatal not enable error */
2374 		CSR_XS(csr_base, IMU_ERROR_LOG_ENABLE, (imu_log_enable |
2375 		    (imu_log_mask & px_imu_log_mask)));
2376 		CSR_XS(csr_base, IMU_INTERRUPT_ENABLE, (imu_intr_enable |
2377 		    (imu_intr_mask & px_imu_intr_mask)));
2378 
2379 		break;
2380 	}
2381 
2382 	return (B_TRUE);
2383 }
2384 
2385 uint64_t
2386 px_get_rng_parent_hi_mask(px_t *px_p)
2387 {
2388 	pxu_t *pxu_p = (pxu_t *)px_p->px_plat_p;
2389 	uint64_t mask;
2390 
2391 	switch (PX_CHIP_TYPE(pxu_p)) {
2392 	case PX_CHIP_OBERON:
2393 		mask = OBERON_RANGE_PROP_MASK;
2394 		break;
2395 	case PX_CHIP_FIRE:
2396 		mask = PX_RANGE_PROP_MASK;
2397 		break;
2398 	default:
2399 		mask = PX_RANGE_PROP_MASK;
2400 	}
2401 
2402 	return (mask);
2403 }
2404 
2405 /*
2406  * fetch chip's range propery's value
2407  */
2408 uint64_t
2409 px_get_range_prop(px_t *px_p, px_ranges_t *rp, int bank)
2410 {
2411 	uint64_t mask, range_prop;
2412 
2413 	mask = px_get_rng_parent_hi_mask(px_p);
2414 	range_prop = (((uint64_t)(rp[bank].parent_high & mask)) << 32) |
2415 		rp[bank].parent_low;
2416 
2417 	return (range_prop);
2418 }
2419 
2420 /*
2421  * add cpr callback
2422  */
2423 void
2424 px_cpr_add_callb(px_t *px_p)
2425 {
2426 	px_p->px_cprcb_id = callb_add(px_cpr_callb, (void *)px_p,
2427 	CB_CL_CPR_POST_USER, "px_cpr");
2428 }
2429 
2430 /*
2431  * remove cpr callback
2432  */
2433 void
2434 px_cpr_rem_callb(px_t *px_p)
2435 {
2436 	(void) callb_delete(px_p->px_cprcb_id);
2437 }
2438 
2439 /*ARGSUSED*/
2440 static uint_t
2441 px_hp_intr(caddr_t arg1, caddr_t arg2)
2442 {
2443 	px_t *px_p = (px_t *)arg1;
2444 	int rval;
2445 
2446 	rval = pciehpc_intr(px_p->px_dip);
2447 
2448 #ifdef  DEBUG
2449 	if (rval == DDI_INTR_UNCLAIMED)
2450 	    cmn_err(CE_WARN, "%s%d: UNCLAIMED intr\n",
2451 		ddi_driver_name(px_p->px_dip),
2452 		ddi_get_instance(px_p->px_dip));
2453 #endif
2454 
2455 	return (rval);
2456 }
2457 
2458 int
2459 px_lib_hotplug_init(dev_info_t *dip, void *arg)
2460 {
2461 	px_t	*px_p = DIP_TO_STATE(dip);
2462 	uint64_t ret;
2463 
2464 	if ((ret = hvio_hotplug_init(dip, arg)) == DDI_SUCCESS) {
2465 		sysino_t sysino;
2466 
2467 		if (px_lib_intr_devino_to_sysino(px_p->px_dip,
2468 		    px_p->px_inos[PX_INTR_HOTPLUG], &sysino) !=
2469 		    DDI_SUCCESS) {
2470 #ifdef	DEBUG
2471 			cmn_err(CE_WARN, "%s%d: devino_to_sysino fails\n",
2472 			    ddi_driver_name(px_p->px_dip),
2473 			    ddi_get_instance(px_p->px_dip));
2474 #endif
2475 			return (DDI_FAILURE);
2476 		}
2477 
2478 		VERIFY(add_ivintr(sysino, PX_PCIEHP_PIL,
2479 		    (intrfunc)px_hp_intr, (caddr_t)px_p, NULL, NULL) == 0);
2480 	}
2481 
2482 	return (ret);
2483 }
2484 
2485 void
2486 px_lib_hotplug_uninit(dev_info_t *dip)
2487 {
2488 	if (hvio_hotplug_uninit(dip) == DDI_SUCCESS) {
2489 		px_t	*px_p = DIP_TO_STATE(dip);
2490 		sysino_t sysino;
2491 
2492 		if (px_lib_intr_devino_to_sysino(px_p->px_dip,
2493 		    px_p->px_inos[PX_INTR_HOTPLUG], &sysino) !=
2494 		    DDI_SUCCESS) {
2495 #ifdef	DEBUG
2496 			cmn_err(CE_WARN, "%s%d: devino_to_sysino fails\n",
2497 			    ddi_driver_name(px_p->px_dip),
2498 			    ddi_get_instance(px_p->px_dip));
2499 #endif
2500 			return;
2501 		}
2502 
2503 		VERIFY(rem_ivintr(sysino, PX_PCIEHP_PIL) == 0);
2504 	}
2505 }
2506 
2507 boolean_t
2508 px_lib_is_in_drain_state(px_t *px_p)
2509 {
2510 	pxu_t 	*pxu_p = (pxu_t *)px_p->px_plat_p;
2511 	caddr_t csr_base = (caddr_t)pxu_p->px_address[PX_REG_CSR];
2512 	uint64_t drain_status;
2513 
2514 	if (PX_CHIP_TYPE(pxu_p) == PX_CHIP_OBERON) {
2515 		drain_status = CSR_BR(csr_base, DRAIN_CONTROL_STATUS, DRAIN);
2516 	} else {
2517 		drain_status = CSR_BR(csr_base, TLU_STATUS, DRAIN);
2518 	}
2519 
2520 	return (drain_status);
2521 }
2522