xref: /titanic_44/usr/src/uts/i86pc/io/pci/pci_common.c (revision 82629e3015252bf18319ba3815c773df23e21436)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 
22 /*
23  * Copyright 2010 Sun Microsystems, Inc.  All rights reserved.
24  * Use is subject to license terms.
25  */
26 
27 /*
28  *	File that has code which is common between pci(7d) and npe(7d)
29  *	It shares the following:
30  *	- interrupt code
31  *	- pci_tools ioctl code
32  *	- name_child code
33  *	- set_parent_private_data code
34  */
35 
36 #include <sys/conf.h>
37 #include <sys/pci.h>
38 #include <sys/sunndi.h>
39 #include <sys/mach_intr.h>
40 #include <sys/pci_intr_lib.h>
41 #include <sys/psm.h>
42 #include <sys/policy.h>
43 #include <sys/sysmacros.h>
44 #include <sys/clock.h>
45 #include <sys/apic.h>
46 #include <sys/pci_tools.h>
47 #include <io/pci/pci_var.h>
48 #include <io/pci/pci_tools_ext.h>
49 #include <io/pci/pci_common.h>
50 #include <sys/pci_cfgspace.h>
51 #include <sys/pci_impl.h>
52 #include <sys/pci_cap.h>
53 
54 /*
55  * Function prototypes
56  */
57 static int	pci_get_priority(dev_info_t *, ddi_intr_handle_impl_t *, int *);
58 static int	pci_enable_intr(dev_info_t *, dev_info_t *,
59 		    ddi_intr_handle_impl_t *, uint32_t);
60 static void	pci_disable_intr(dev_info_t *, dev_info_t *,
61 		    ddi_intr_handle_impl_t *, uint32_t);
62 
63 /* Extern decalration for pcplusmp module */
64 extern int	(*psm_intr_ops)(dev_info_t *, ddi_intr_handle_impl_t *,
65 		    psm_intr_op_t, int *);
66 
67 /*
68  * pci_name_child:
69  *
70  *	Assign the address portion of the node name
71  */
72 int
73 pci_common_name_child(dev_info_t *child, char *name, int namelen)
74 {
75 	int		dev, func, length;
76 	char		**unit_addr;
77 	uint_t		n;
78 	pci_regspec_t	*pci_rp;
79 
80 	if (ndi_dev_is_persistent_node(child) == 0) {
81 		/*
82 		 * For .conf node, use "unit-address" property
83 		 */
84 		if (ddi_prop_lookup_string_array(DDI_DEV_T_ANY, child,
85 		    DDI_PROP_DONTPASS, "unit-address", &unit_addr, &n) !=
86 		    DDI_PROP_SUCCESS) {
87 			cmn_err(CE_WARN, "cannot find unit-address in %s.conf",
88 			    ddi_get_name(child));
89 			return (DDI_FAILURE);
90 		}
91 		if (n != 1 || *unit_addr == NULL || **unit_addr == 0) {
92 			cmn_err(CE_WARN, "unit-address property in %s.conf"
93 			    " not well-formed", ddi_get_name(child));
94 			ddi_prop_free(unit_addr);
95 			return (DDI_FAILURE);
96 		}
97 		(void) snprintf(name, namelen, "%s", *unit_addr);
98 		ddi_prop_free(unit_addr);
99 		return (DDI_SUCCESS);
100 	}
101 
102 	if (ddi_prop_lookup_int_array(DDI_DEV_T_ANY, child, DDI_PROP_DONTPASS,
103 	    "reg", (int **)&pci_rp, (uint_t *)&length) != DDI_PROP_SUCCESS) {
104 		cmn_err(CE_WARN, "cannot find reg property in %s",
105 		    ddi_get_name(child));
106 		return (DDI_FAILURE);
107 	}
108 
109 	/* copy the device identifications */
110 	dev = PCI_REG_DEV_G(pci_rp->pci_phys_hi);
111 	func = PCI_REG_FUNC_G(pci_rp->pci_phys_hi);
112 
113 	/*
114 	 * free the memory allocated by ddi_prop_lookup_int_array
115 	 */
116 	ddi_prop_free(pci_rp);
117 
118 	if (func != 0) {
119 		(void) snprintf(name, namelen, "%x,%x", dev, func);
120 	} else {
121 		(void) snprintf(name, namelen, "%x", dev);
122 	}
123 
124 	return (DDI_SUCCESS);
125 }
126 
127 /*
128  * Interrupt related code:
129  *
130  * The following busop is common to npe and pci drivers
131  *	bus_introp
132  */
133 
134 /*
135  * Create the ddi_parent_private_data for a pseudo child.
136  */
137 void
138 pci_common_set_parent_private_data(dev_info_t *dip)
139 {
140 	struct ddi_parent_private_data *pdptr;
141 
142 	pdptr = (struct ddi_parent_private_data *)kmem_zalloc(
143 	    (sizeof (struct ddi_parent_private_data) +
144 	    sizeof (struct intrspec)), KM_SLEEP);
145 	pdptr->par_intr = (struct intrspec *)(pdptr + 1);
146 	pdptr->par_nintr = 1;
147 	ddi_set_parent_data(dip, pdptr);
148 }
149 
150 /*
151  * pci_get_priority:
152  *	Figure out the priority of the device
153  */
154 static int
155 pci_get_priority(dev_info_t *dip, ddi_intr_handle_impl_t *hdlp, int *pri)
156 {
157 	struct intrspec *ispec;
158 
159 	DDI_INTR_NEXDBG((CE_CONT, "pci_get_priority: dip = 0x%p, hdlp = %p\n",
160 	    (void *)dip, (void *)hdlp));
161 
162 	if ((ispec = (struct intrspec *)pci_intx_get_ispec(dip, dip,
163 	    hdlp->ih_inum)) == NULL) {
164 		if (DDI_INTR_IS_MSI_OR_MSIX(hdlp->ih_type)) {
165 			*pri = pci_class_to_pil(dip);
166 			pci_common_set_parent_private_data(hdlp->ih_dip);
167 			ispec = (struct intrspec *)pci_intx_get_ispec(dip, dip,
168 			    hdlp->ih_inum);
169 			return (DDI_SUCCESS);
170 		}
171 		return (DDI_FAILURE);
172 	}
173 
174 	*pri = ispec->intrspec_pri;
175 	return (DDI_SUCCESS);
176 }
177 
178 
179 
180 static int pcieb_intr_pri_counter = 0;
181 
182 /*
183  * pci_common_intr_ops: bus_intr_op() function for interrupt support
184  */
185 int
186 pci_common_intr_ops(dev_info_t *pdip, dev_info_t *rdip, ddi_intr_op_t intr_op,
187     ddi_intr_handle_impl_t *hdlp, void *result)
188 {
189 	int			priority = 0;
190 	int			psm_status = 0;
191 	int			pci_status = 0;
192 	int			pci_rval, psm_rval = PSM_FAILURE;
193 	int			types = 0;
194 	int			pciepci = 0;
195 	int			i, j, count;
196 	int			rv;
197 	int			behavior;
198 	int			cap_ptr;
199 	uint16_t		msi_cap_base, msix_cap_base, cap_ctrl;
200 	char			*prop;
201 	ddi_intrspec_t		isp;
202 	struct intrspec		*ispec;
203 	ddi_intr_handle_impl_t	tmp_hdl;
204 	ddi_intr_msix_t		*msix_p;
205 	ihdl_plat_t		*ihdl_plat_datap;
206 	ddi_intr_handle_t	*h_array;
207 	ddi_acc_handle_t	handle;
208 
209 	DDI_INTR_NEXDBG((CE_CONT,
210 	    "pci_common_intr_ops: pdip 0x%p, rdip 0x%p, op %x handle 0x%p\n",
211 	    (void *)pdip, (void *)rdip, intr_op, (void *)hdlp));
212 
213 	/* Process the request */
214 	switch (intr_op) {
215 	case DDI_INTROP_SUPPORTED_TYPES:
216 		/*
217 		 * First we determine the interrupt types supported by the
218 		 * device itself, then we filter them through what the OS
219 		 * and system supports.  We determine system-level
220 		 * interrupt type support for anything other than fixed intrs
221 		 * through the psm_intr_ops vector
222 		 */
223 		rv = DDI_FAILURE;
224 
225 		/* Fixed supported by default */
226 		types = DDI_INTR_TYPE_FIXED;
227 
228 		if (psm_intr_ops == NULL) {
229 			*(int *)result = types;
230 			return (DDI_SUCCESS);
231 		}
232 		if (pci_config_setup(rdip, &handle) != DDI_SUCCESS)
233 			return (DDI_FAILURE);
234 
235 		/* Sanity test cap control values if found */
236 
237 		if (PCI_CAP_LOCATE(handle, PCI_CAP_ID_MSI, &msi_cap_base) ==
238 		    DDI_SUCCESS) {
239 			cap_ctrl = PCI_CAP_GET16(handle, 0, msi_cap_base,
240 			    PCI_MSI_CTRL);
241 			if (cap_ctrl == PCI_CAP_EINVAL16)
242 				goto SUPPORTED_TYPES_OUT;
243 
244 			types |= DDI_INTR_TYPE_MSI;
245 		}
246 
247 		if (PCI_CAP_LOCATE(handle, PCI_CAP_ID_MSI_X, &msix_cap_base) ==
248 		    DDI_SUCCESS) {
249 			cap_ctrl = PCI_CAP_GET16(handle, 0, msix_cap_base,
250 			    PCI_MSIX_CTRL);
251 			if (cap_ctrl == PCI_CAP_EINVAL16)
252 				goto SUPPORTED_TYPES_OUT;
253 
254 			types |= DDI_INTR_TYPE_MSIX;
255 		}
256 
257 		/*
258 		 * Filter device-level types through system-level support
259 		 */
260 		tmp_hdl.ih_type = types;
261 		if ((*psm_intr_ops)(rdip, &tmp_hdl, PSM_INTR_OP_CHECK_MSI,
262 		    &types) != PSM_SUCCESS)
263 			goto SUPPORTED_TYPES_OUT;
264 
265 		DDI_INTR_NEXDBG((CE_CONT, "pci_common_intr_ops: "
266 		    "rdip: 0x%p supported types: 0x%x\n", (void *)rdip,
267 		    *(int *)result));
268 
269 		/*
270 		 * Export any MSI/MSI-X cap locations via properties
271 		 */
272 		if (types & DDI_INTR_TYPE_MSI) {
273 			if (ndi_prop_update_int(DDI_DEV_T_NONE, rdip,
274 			    "pci-msi-capid-pointer", (int)msi_cap_base) !=
275 			    DDI_PROP_SUCCESS)
276 				goto SUPPORTED_TYPES_OUT;
277 		}
278 		if (types & DDI_INTR_TYPE_MSIX) {
279 			if (ndi_prop_update_int(DDI_DEV_T_NONE, rdip,
280 			    "pci-msix-capid-pointer", (int)msix_cap_base) !=
281 			    DDI_PROP_SUCCESS)
282 				goto SUPPORTED_TYPES_OUT;
283 		}
284 
285 		rv = DDI_SUCCESS;
286 
287 SUPPORTED_TYPES_OUT:
288 		*(int *)result = types;
289 		pci_config_teardown(&handle);
290 		return (rv);
291 
292 	case DDI_INTROP_NAVAIL:
293 	case DDI_INTROP_NINTRS:
294 		if (DDI_INTR_IS_MSI_OR_MSIX(hdlp->ih_type)) {
295 			if (pci_msi_get_nintrs(hdlp->ih_dip, hdlp->ih_type,
296 			    result) != DDI_SUCCESS)
297 				return (DDI_FAILURE);
298 		} else {
299 			*(int *)result = i_ddi_get_intx_nintrs(hdlp->ih_dip);
300 			if (*(int *)result == 0)
301 				return (DDI_FAILURE);
302 		}
303 		break;
304 	case DDI_INTROP_ALLOC:
305 		/*
306 		 * MSI or MSIX (figure out number of vectors available)
307 		 * FIXED interrupts: just return available interrupts
308 		 */
309 		if (DDI_INTR_IS_MSI_OR_MSIX(hdlp->ih_type) &&
310 		    (psm_intr_ops != NULL) &&
311 		    (pci_get_priority(rdip, hdlp, &priority) == DDI_SUCCESS)) {
312 			/*
313 			 * Following check is a special case for 'pcieb'.
314 			 * This makes sure vectors with the right priority
315 			 * are allocated for pcieb during ALLOC time.
316 			 */
317 			if (strcmp(ddi_driver_name(rdip), "pcieb") == 0) {
318 				hdlp->ih_pri =
319 				    (pcieb_intr_pri_counter % 2) ? 4 : 7;
320 				pciepci = 1;
321 			} else
322 				hdlp->ih_pri = priority;
323 			behavior = (int)(uintptr_t)hdlp->ih_scratch2;
324 
325 			/*
326 			 * Cache in the config handle and cap_ptr
327 			 */
328 			if (i_ddi_get_pci_config_handle(rdip) == NULL) {
329 				if (pci_config_setup(rdip, &handle) !=
330 				    DDI_SUCCESS)
331 					return (DDI_FAILURE);
332 				i_ddi_set_pci_config_handle(rdip, handle);
333 			}
334 
335 			prop = NULL;
336 			cap_ptr = 0;
337 			if (hdlp->ih_type == DDI_INTR_TYPE_MSI)
338 				prop = "pci-msi-capid-pointer";
339 			else if (hdlp->ih_type == DDI_INTR_TYPE_MSIX)
340 				prop = "pci-msix-capid-pointer";
341 
342 			/*
343 			 * Enforce the calling of DDI_INTROP_SUPPORTED_TYPES
344 			 * for MSI(X) before allocation
345 			 */
346 			if (prop != NULL) {
347 				cap_ptr = ddi_prop_get_int(DDI_DEV_T_ANY, rdip,
348 				    DDI_PROP_DONTPASS, prop, 0);
349 				if (cap_ptr == 0) {
350 					DDI_INTR_NEXDBG((CE_CONT,
351 					    "pci_common_intr_ops: rdip: 0x%p "
352 					    "attempted MSI(X) alloc without "
353 					    "cap property\n", (void *)rdip));
354 					return (DDI_FAILURE);
355 				}
356 			}
357 			i_ddi_set_msi_msix_cap_ptr(rdip, cap_ptr);
358 
359 			/*
360 			 * Allocate interrupt vectors
361 			 */
362 			(void) (*psm_intr_ops)(rdip, hdlp,
363 			    PSM_INTR_OP_ALLOC_VECTORS, result);
364 
365 			if (*(int *)result == 0)
366 				return (DDI_INTR_NOTFOUND);
367 
368 			/* verify behavior flag and take appropriate action */
369 			if ((behavior == DDI_INTR_ALLOC_STRICT) &&
370 			    (*(int *)result < hdlp->ih_scratch1)) {
371 				DDI_INTR_NEXDBG((CE_CONT,
372 				    "pci_common_intr_ops: behavior %x, "
373 				    "couldn't get enough intrs\n", behavior));
374 				hdlp->ih_scratch1 = *(int *)result;
375 				(void) (*psm_intr_ops)(rdip, hdlp,
376 				    PSM_INTR_OP_FREE_VECTORS, NULL);
377 				return (DDI_EAGAIN);
378 			}
379 
380 			if (hdlp->ih_type == DDI_INTR_TYPE_MSIX) {
381 				if (!(msix_p = i_ddi_get_msix(hdlp->ih_dip))) {
382 					msix_p = pci_msix_init(hdlp->ih_dip);
383 					if (msix_p) {
384 						i_ddi_set_msix(hdlp->ih_dip,
385 						    msix_p);
386 					} else {
387 						DDI_INTR_NEXDBG((CE_CONT,
388 						    "pci_common_intr_ops: MSI-X"
389 						    "table initilization failed"
390 						    ", rdip 0x%p inum 0x%x\n",
391 						    (void *)rdip,
392 						    hdlp->ih_inum));
393 
394 						(void) (*psm_intr_ops)(rdip,
395 						    hdlp,
396 						    PSM_INTR_OP_FREE_VECTORS,
397 						    NULL);
398 
399 						return (DDI_FAILURE);
400 					}
401 				}
402 			}
403 
404 			if (pciepci) {
405 				/* update priority in ispec */
406 				isp = pci_intx_get_ispec(pdip, rdip,
407 				    (int)hdlp->ih_inum);
408 				ispec = (struct intrspec *)isp;
409 				if (ispec)
410 					ispec->intrspec_pri = hdlp->ih_pri;
411 				++pcieb_intr_pri_counter;
412 			}
413 
414 		} else if (hdlp->ih_type == DDI_INTR_TYPE_FIXED) {
415 			/* Figure out if this device supports MASKING */
416 			pci_rval = pci_intx_get_cap(rdip, &pci_status);
417 			if (pci_rval == DDI_SUCCESS && pci_status)
418 				hdlp->ih_cap |= pci_status;
419 			*(int *)result = 1;	/* DDI_INTR_TYPE_FIXED */
420 		} else
421 			return (DDI_FAILURE);
422 		break;
423 	case DDI_INTROP_FREE:
424 		if (DDI_INTR_IS_MSI_OR_MSIX(hdlp->ih_type) &&
425 		    (psm_intr_ops != NULL)) {
426 			if (i_ddi_intr_get_current_nintrs(hdlp->ih_dip) - 1 ==
427 			    0) {
428 				if (handle = i_ddi_get_pci_config_handle(
429 				    rdip)) {
430 					(void) pci_config_teardown(&handle);
431 					i_ddi_set_pci_config_handle(rdip, NULL);
432 				}
433 				if (cap_ptr = i_ddi_get_msi_msix_cap_ptr(rdip))
434 					i_ddi_set_msi_msix_cap_ptr(rdip, 0);
435 			}
436 
437 			(void) (*psm_intr_ops)(rdip, hdlp,
438 			    PSM_INTR_OP_FREE_VECTORS, NULL);
439 
440 			if (hdlp->ih_type == DDI_INTR_TYPE_MSIX) {
441 				msix_p = i_ddi_get_msix(hdlp->ih_dip);
442 				if (msix_p &&
443 				    (i_ddi_intr_get_current_nintrs(
444 				    hdlp->ih_dip) - 1) == 0) {
445 					pci_msix_fini(msix_p);
446 					i_ddi_set_msix(hdlp->ih_dip, NULL);
447 				}
448 			}
449 		}
450 		break;
451 	case DDI_INTROP_GETPRI:
452 		/* Get the priority */
453 		if (pci_get_priority(rdip, hdlp, &priority) != DDI_SUCCESS)
454 			return (DDI_FAILURE);
455 		DDI_INTR_NEXDBG((CE_CONT, "pci_common_intr_ops: "
456 		    "priority = 0x%x\n", priority));
457 		*(int *)result = priority;
458 		break;
459 	case DDI_INTROP_SETPRI:
460 		/* Validate the interrupt priority passed */
461 		if (*(int *)result > LOCK_LEVEL)
462 			return (DDI_FAILURE);
463 
464 		/* Ensure that PSM is all initialized */
465 		if (psm_intr_ops == NULL)
466 			return (DDI_FAILURE);
467 
468 		isp = pci_intx_get_ispec(pdip, rdip, (int)hdlp->ih_inum);
469 		ispec = (struct intrspec *)isp;
470 		if (ispec == NULL)
471 			return (DDI_FAILURE);
472 
473 		/* For fixed interrupts */
474 		if (hdlp->ih_type == DDI_INTR_TYPE_FIXED) {
475 			/* if interrupt is shared, return failure */
476 			((ihdl_plat_t *)hdlp->ih_private)->ip_ispecp = ispec;
477 			psm_rval = (*psm_intr_ops)(rdip, hdlp,
478 			    PSM_INTR_OP_GET_SHARED, &psm_status);
479 			/*
480 			 * For fixed interrupts, the irq may not have been
481 			 * allocated when SET_PRI is called, and the above
482 			 * GET_SHARED op may return PSM_FAILURE. This is not
483 			 * a real error and is ignored below.
484 			 */
485 			if ((psm_rval != PSM_FAILURE) && (psm_status == 1)) {
486 				DDI_INTR_NEXDBG((CE_CONT,
487 				    "pci_common_intr_ops: "
488 				    "dip 0x%p cannot setpri, psm_rval=%d,"
489 				    "psm_status=%d\n", (void *)rdip, psm_rval,
490 				    psm_status));
491 				return (DDI_FAILURE);
492 			}
493 		}
494 
495 		/* Change the priority */
496 		if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_SET_PRI, result) ==
497 		    PSM_FAILURE)
498 			return (DDI_FAILURE);
499 
500 		/* update ispec */
501 		ispec->intrspec_pri = *(int *)result;
502 		break;
503 	case DDI_INTROP_ADDISR:
504 		/* update ispec */
505 		isp = pci_intx_get_ispec(pdip, rdip, (int)hdlp->ih_inum);
506 		ispec = (struct intrspec *)isp;
507 		if (ispec) {
508 			ispec->intrspec_func = hdlp->ih_cb_func;
509 			ihdl_plat_datap = (ihdl_plat_t *)hdlp->ih_private;
510 			pci_kstat_create(&ihdl_plat_datap->ip_ksp, pdip, hdlp);
511 		}
512 		break;
513 	case DDI_INTROP_REMISR:
514 		/* Get the interrupt structure pointer */
515 		isp = pci_intx_get_ispec(pdip, rdip, (int)hdlp->ih_inum);
516 		ispec = (struct intrspec *)isp;
517 		if (ispec) {
518 			ispec->intrspec_func = (uint_t (*)()) 0;
519 			ihdl_plat_datap = (ihdl_plat_t *)hdlp->ih_private;
520 			if (ihdl_plat_datap->ip_ksp != NULL)
521 				pci_kstat_delete(ihdl_plat_datap->ip_ksp);
522 		}
523 		break;
524 	case DDI_INTROP_GETCAP:
525 		/*
526 		 * First check the config space and/or
527 		 * MSI capability register(s)
528 		 */
529 		if (DDI_INTR_IS_MSI_OR_MSIX(hdlp->ih_type))
530 			pci_rval = pci_msi_get_cap(rdip, hdlp->ih_type,
531 			    &pci_status);
532 		else if (hdlp->ih_type == DDI_INTR_TYPE_FIXED)
533 			pci_rval = pci_intx_get_cap(rdip, &pci_status);
534 
535 		/* next check with pcplusmp */
536 		if (psm_intr_ops != NULL)
537 			psm_rval = (*psm_intr_ops)(rdip, hdlp,
538 			    PSM_INTR_OP_GET_CAP, &psm_status);
539 
540 		DDI_INTR_NEXDBG((CE_CONT, "pci: GETCAP returned psm_rval = %x, "
541 		    "psm_status = %x, pci_rval = %x, pci_status = %x\n",
542 		    psm_rval, psm_status, pci_rval, pci_status));
543 
544 		if (psm_rval == PSM_FAILURE && pci_rval == DDI_FAILURE) {
545 			*(int *)result = 0;
546 			return (DDI_FAILURE);
547 		}
548 
549 		if (psm_rval == PSM_SUCCESS)
550 			*(int *)result = psm_status;
551 
552 		if (pci_rval == DDI_SUCCESS)
553 			*(int *)result |= pci_status;
554 
555 		DDI_INTR_NEXDBG((CE_CONT, "pci: GETCAP returned = %x\n",
556 		    *(int *)result));
557 		break;
558 	case DDI_INTROP_SETCAP:
559 		DDI_INTR_NEXDBG((CE_CONT, "pci_common_intr_ops: "
560 		    "SETCAP cap=0x%x\n", *(int *)result));
561 		if (psm_intr_ops == NULL)
562 			return (DDI_FAILURE);
563 
564 		if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_SET_CAP, result)) {
565 			DDI_INTR_NEXDBG((CE_CONT, "GETCAP: psm_intr_ops"
566 			    " returned failure\n"));
567 			return (DDI_FAILURE);
568 		}
569 		break;
570 	case DDI_INTROP_ENABLE:
571 		DDI_INTR_NEXDBG((CE_CONT, "pci_common_intr_ops: ENABLE\n"));
572 		if (psm_intr_ops == NULL)
573 			return (DDI_FAILURE);
574 
575 		if (pci_enable_intr(pdip, rdip, hdlp, hdlp->ih_inum) !=
576 		    DDI_SUCCESS)
577 			return (DDI_FAILURE);
578 
579 		DDI_INTR_NEXDBG((CE_CONT, "pci_common_intr_ops: ENABLE "
580 		    "vector=0x%x\n", hdlp->ih_vector));
581 		break;
582 	case DDI_INTROP_DISABLE:
583 		DDI_INTR_NEXDBG((CE_CONT, "pci_common_intr_ops: DISABLE\n"));
584 		if (psm_intr_ops == NULL)
585 			return (DDI_FAILURE);
586 
587 		pci_disable_intr(pdip, rdip, hdlp, hdlp->ih_inum);
588 		DDI_INTR_NEXDBG((CE_CONT, "pci_common_intr_ops: DISABLE "
589 		    "vector = %x\n", hdlp->ih_vector));
590 		break;
591 	case DDI_INTROP_BLOCKENABLE:
592 		DDI_INTR_NEXDBG((CE_CONT, "pci_common_intr_ops: "
593 		    "BLOCKENABLE\n"));
594 		if (hdlp->ih_type != DDI_INTR_TYPE_MSI) {
595 			DDI_INTR_NEXDBG((CE_CONT, "BLOCKENABLE: not MSI\n"));
596 			return (DDI_FAILURE);
597 		}
598 
599 		/* Check if psm_intr_ops is NULL? */
600 		if (psm_intr_ops == NULL)
601 			return (DDI_FAILURE);
602 
603 		count = hdlp->ih_scratch1;
604 		h_array = (ddi_intr_handle_t *)hdlp->ih_scratch2;
605 		for (i = 0; i < count; i++) {
606 			hdlp = (ddi_intr_handle_impl_t *)h_array[i];
607 			if (pci_enable_intr(pdip, rdip, hdlp,
608 			    hdlp->ih_inum) != DDI_SUCCESS) {
609 				DDI_INTR_NEXDBG((CE_CONT, "BLOCKENABLE: "
610 				    "pci_enable_intr failed for %d\n", i));
611 				for (j = 0; j < i; j++) {
612 					hdlp = (ddi_intr_handle_impl_t *)
613 					    h_array[j];
614 					pci_disable_intr(pdip, rdip, hdlp,
615 					    hdlp->ih_inum);
616 				}
617 				return (DDI_FAILURE);
618 			}
619 			DDI_INTR_NEXDBG((CE_CONT, "pci_common_intr_ops: "
620 			    "BLOCKENABLE inum %x done\n", hdlp->ih_inum));
621 		}
622 		break;
623 	case DDI_INTROP_BLOCKDISABLE:
624 		DDI_INTR_NEXDBG((CE_CONT, "pci_common_intr_ops: "
625 		    "BLOCKDISABLE\n"));
626 		if (hdlp->ih_type != DDI_INTR_TYPE_MSI) {
627 			DDI_INTR_NEXDBG((CE_CONT, "BLOCKDISABLE: not MSI\n"));
628 			return (DDI_FAILURE);
629 		}
630 
631 		/* Check if psm_intr_ops is present */
632 		if (psm_intr_ops == NULL)
633 			return (DDI_FAILURE);
634 
635 		count = hdlp->ih_scratch1;
636 		h_array = (ddi_intr_handle_t *)hdlp->ih_scratch2;
637 		for (i = 0; i < count; i++) {
638 			hdlp = (ddi_intr_handle_impl_t *)h_array[i];
639 			pci_disable_intr(pdip, rdip, hdlp, hdlp->ih_inum);
640 			DDI_INTR_NEXDBG((CE_CONT, "pci_common_intr_ops: "
641 			    "BLOCKDISABLE inum %x done\n", hdlp->ih_inum));
642 		}
643 		break;
644 	case DDI_INTROP_SETMASK:
645 	case DDI_INTROP_CLRMASK:
646 		/*
647 		 * First handle in the config space
648 		 */
649 		if (intr_op == DDI_INTROP_SETMASK) {
650 			if (DDI_INTR_IS_MSI_OR_MSIX(hdlp->ih_type))
651 				pci_status = pci_msi_set_mask(rdip,
652 				    hdlp->ih_type, hdlp->ih_inum);
653 			else if (hdlp->ih_type == DDI_INTR_TYPE_FIXED)
654 				pci_status = pci_intx_set_mask(rdip);
655 		} else {
656 			if (DDI_INTR_IS_MSI_OR_MSIX(hdlp->ih_type))
657 				pci_status = pci_msi_clr_mask(rdip,
658 				    hdlp->ih_type, hdlp->ih_inum);
659 			else if (hdlp->ih_type == DDI_INTR_TYPE_FIXED)
660 				pci_status = pci_intx_clr_mask(rdip);
661 		}
662 
663 		/* For MSI/X; no need to check with pcplusmp */
664 		if (hdlp->ih_type != DDI_INTR_TYPE_FIXED)
665 			return (pci_status);
666 
667 		/* For fixed interrupts only: handle config space first */
668 		if (hdlp->ih_type == DDI_INTR_TYPE_FIXED &&
669 		    pci_status == DDI_SUCCESS)
670 			break;
671 
672 		/* For fixed interrupts only: confer with pcplusmp next */
673 		if (psm_intr_ops != NULL) {
674 			/* If interrupt is shared; do nothing */
675 			psm_rval = (*psm_intr_ops)(rdip, hdlp,
676 			    PSM_INTR_OP_GET_SHARED, &psm_status);
677 
678 			if (psm_rval == PSM_FAILURE || psm_status == 1)
679 				return (pci_status);
680 
681 			/* Now, pcplusmp should try to set/clear the mask */
682 			if (intr_op == DDI_INTROP_SETMASK)
683 				psm_rval = (*psm_intr_ops)(rdip, hdlp,
684 				    PSM_INTR_OP_SET_MASK, NULL);
685 			else
686 				psm_rval = (*psm_intr_ops)(rdip, hdlp,
687 				    PSM_INTR_OP_CLEAR_MASK, NULL);
688 		}
689 		return ((psm_rval == PSM_FAILURE) ? DDI_FAILURE : DDI_SUCCESS);
690 	case DDI_INTROP_GETPENDING:
691 		/*
692 		 * First check the config space and/or
693 		 * MSI capability register(s)
694 		 */
695 		if (DDI_INTR_IS_MSI_OR_MSIX(hdlp->ih_type))
696 			pci_rval = pci_msi_get_pending(rdip, hdlp->ih_type,
697 			    hdlp->ih_inum, &pci_status);
698 		else if (hdlp->ih_type == DDI_INTR_TYPE_FIXED)
699 			pci_rval = pci_intx_get_pending(rdip, &pci_status);
700 
701 		/* On failure; next try with pcplusmp */
702 		if (pci_rval != DDI_SUCCESS && psm_intr_ops != NULL)
703 			psm_rval = (*psm_intr_ops)(rdip, hdlp,
704 			    PSM_INTR_OP_GET_PENDING, &psm_status);
705 
706 		DDI_INTR_NEXDBG((CE_CONT, "pci: GETPENDING returned "
707 		    "psm_rval = %x, psm_status = %x, pci_rval = %x, "
708 		    "pci_status = %x\n", psm_rval, psm_status, pci_rval,
709 		    pci_status));
710 		if (psm_rval == PSM_FAILURE && pci_rval == DDI_FAILURE) {
711 			*(int *)result = 0;
712 			return (DDI_FAILURE);
713 		}
714 
715 		if (psm_rval != PSM_FAILURE)
716 			*(int *)result = psm_status;
717 		else if (pci_rval != DDI_FAILURE)
718 			*(int *)result = pci_status;
719 		DDI_INTR_NEXDBG((CE_CONT, "pci: GETPENDING returned = %x\n",
720 		    *(int *)result));
721 		break;
722 	case DDI_INTROP_GETTARGET:
723 		DDI_INTR_NEXDBG((CE_CONT, "pci_common_intr_ops: GETTARGET\n"));
724 
725 		/* Note hdlp->ih_vector is actually an irq */
726 		if ((rv = pci_get_cpu_from_vecirq(hdlp->ih_vector, IS_IRQ)) ==
727 		    -1)
728 			return (DDI_FAILURE);
729 		*(int *)result = rv;
730 		DDI_INTR_NEXDBG((CE_CONT, "pci_common_intr_ops: GETTARGET "
731 		    "vector = 0x%x, cpu = 0x%x\n", hdlp->ih_vector, rv));
732 		break;
733 	case DDI_INTROP_SETTARGET:
734 		DDI_INTR_NEXDBG((CE_CONT, "pci_common_intr_ops: SETTARGET\n"));
735 
736 		/* hdlp->ih_vector is actually an irq */
737 		tmp_hdl.ih_vector = hdlp->ih_vector;
738 		tmp_hdl.ih_flags = PSMGI_INTRBY_IRQ;
739 		tmp_hdl.ih_private = (void *)(uintptr_t)*(int *)result;
740 		psm_rval = (*psm_intr_ops)(rdip, &tmp_hdl, PSM_INTR_OP_SET_CPU,
741 		    &psm_status);
742 
743 		if (psm_rval != PSM_SUCCESS)
744 			return (DDI_FAILURE);
745 		break;
746 	default:
747 		return (i_ddi_intr_ops(pdip, rdip, intr_op, hdlp, result));
748 	}
749 
750 	return (DDI_SUCCESS);
751 }
752 
753 int
754 pci_get_intr_from_vecirq(apic_get_intr_t *intrinfo_p,
755     int vecirq, boolean_t is_irq)
756 {
757 	ddi_intr_handle_impl_t	get_info_ii_hdl;
758 
759 	if (is_irq)
760 		intrinfo_p->avgi_req_flags |= PSMGI_INTRBY_IRQ;
761 
762 	/*
763 	 * For this locally-declared and used handle, ih_private will contain a
764 	 * pointer to apic_get_intr_t, not an ihdl_plat_t as used for
765 	 * global interrupt handling.
766 	 */
767 	get_info_ii_hdl.ih_private = intrinfo_p;
768 	get_info_ii_hdl.ih_vector = (ushort_t)vecirq;
769 
770 	if ((*psm_intr_ops)(NULL, &get_info_ii_hdl,
771 	    PSM_INTR_OP_GET_INTR, NULL) == PSM_FAILURE)
772 		return (DDI_FAILURE);
773 
774 	return (DDI_SUCCESS);
775 }
776 
777 
778 int
779 pci_get_cpu_from_vecirq(int vecirq, boolean_t is_irq)
780 {
781 	int rval;
782 
783 	apic_get_intr_t	intrinfo;
784 	intrinfo.avgi_req_flags = PSMGI_REQ_CPUID;
785 	rval = pci_get_intr_from_vecirq(&intrinfo, vecirq, is_irq);
786 
787 	if (rval == DDI_SUCCESS)
788 		return (intrinfo.avgi_cpu_id);
789 	else
790 		return (-1);
791 }
792 
793 
794 static int
795 pci_enable_intr(dev_info_t *pdip, dev_info_t *rdip,
796     ddi_intr_handle_impl_t *hdlp, uint32_t inum)
797 {
798 	struct intrspec	*ispec;
799 	int		irq;
800 	ihdl_plat_t	*ihdl_plat_datap = (ihdl_plat_t *)hdlp->ih_private;
801 
802 	DDI_INTR_NEXDBG((CE_CONT, "pci_enable_intr: hdlp %p inum %x\n",
803 	    (void *)hdlp, inum));
804 
805 	/* Translate the interrupt if needed */
806 	ispec = (struct intrspec *)pci_intx_get_ispec(pdip, rdip, (int)inum);
807 	if (ispec == NULL)
808 		return (DDI_FAILURE);
809 	if (DDI_INTR_IS_MSI_OR_MSIX(hdlp->ih_type)) {
810 		ispec->intrspec_vec = inum;
811 		ispec->intrspec_pri = hdlp->ih_pri;
812 	}
813 	ihdl_plat_datap->ip_ispecp = ispec;
814 
815 	/* translate the interrupt if needed */
816 	if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_XLATE_VECTOR, &irq) ==
817 	    PSM_FAILURE)
818 		return (DDI_FAILURE);
819 	DDI_INTR_NEXDBG((CE_CONT, "pci_enable_intr: priority=%x irq=%x\n",
820 	    hdlp->ih_pri, irq));
821 
822 	/* Add the interrupt handler */
823 	if (!add_avintr((void *)hdlp, hdlp->ih_pri, hdlp->ih_cb_func,
824 	    DEVI(rdip)->devi_name, irq, hdlp->ih_cb_arg1,
825 	    hdlp->ih_cb_arg2, &ihdl_plat_datap->ip_ticks, rdip))
826 		return (DDI_FAILURE);
827 
828 	/* Note this really is an irq. */
829 	hdlp->ih_vector = (ushort_t)irq;
830 
831 	return (DDI_SUCCESS);
832 }
833 
834 
835 static void
836 pci_disable_intr(dev_info_t *pdip, dev_info_t *rdip,
837     ddi_intr_handle_impl_t *hdlp, uint32_t inum)
838 {
839 	int		irq;
840 	struct intrspec	*ispec;
841 	ihdl_plat_t	*ihdl_plat_datap = (ihdl_plat_t *)hdlp->ih_private;
842 
843 	DDI_INTR_NEXDBG((CE_CONT, "pci_disable_intr: \n"));
844 	ispec = (struct intrspec *)pci_intx_get_ispec(pdip, rdip, (int)inum);
845 	if (ispec == NULL)
846 		return;
847 	if (DDI_INTR_IS_MSI_OR_MSIX(hdlp->ih_type)) {
848 		ispec->intrspec_vec = inum;
849 		ispec->intrspec_pri = hdlp->ih_pri;
850 	}
851 	ihdl_plat_datap->ip_ispecp = ispec;
852 
853 	/* translate the interrupt if needed */
854 	(void) (*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_XLATE_VECTOR, &irq);
855 
856 	/* Disable the interrupt handler */
857 	rem_avintr((void *)hdlp, hdlp->ih_pri, hdlp->ih_cb_func, irq);
858 	ihdl_plat_datap->ip_ispecp = NULL;
859 }
860 
861 /*
862  * Miscellaneous library function
863  */
864 int
865 pci_common_get_reg_prop(dev_info_t *dip, pci_regspec_t *pci_rp)
866 {
867 	int		i;
868 	int 		number;
869 	int		assigned_addr_len;
870 	uint_t		phys_hi = pci_rp->pci_phys_hi;
871 	pci_regspec_t	*assigned_addr;
872 
873 	if (((phys_hi & PCI_REG_ADDR_M) == PCI_ADDR_CONFIG) ||
874 	    (phys_hi & PCI_RELOCAT_B))
875 		return (DDI_SUCCESS);
876 
877 	/*
878 	 * the "reg" property specifies relocatable, get and interpret the
879 	 * "assigned-addresses" property.
880 	 */
881 	if (ddi_prop_lookup_int_array(DDI_DEV_T_ANY, dip, DDI_PROP_DONTPASS,
882 	    "assigned-addresses", (int **)&assigned_addr,
883 	    (uint_t *)&assigned_addr_len) != DDI_PROP_SUCCESS)
884 		return (DDI_FAILURE);
885 
886 	/*
887 	 * Scan the "assigned-addresses" for one that matches the specified
888 	 * "reg" property entry.
889 	 */
890 	phys_hi &= PCI_CONF_ADDR_MASK;
891 	number = assigned_addr_len / (sizeof (pci_regspec_t) / sizeof (int));
892 	for (i = 0; i < number; i++) {
893 		if ((assigned_addr[i].pci_phys_hi & PCI_CONF_ADDR_MASK) ==
894 		    phys_hi) {
895 			pci_rp->pci_phys_mid = assigned_addr[i].pci_phys_mid;
896 			pci_rp->pci_phys_low = assigned_addr[i].pci_phys_low;
897 			ddi_prop_free(assigned_addr);
898 			return (DDI_SUCCESS);
899 		}
900 	}
901 
902 	ddi_prop_free(assigned_addr);
903 	return (DDI_FAILURE);
904 }
905 
906 
907 /*
908  * To handle PCI tool ioctls
909  */
910 
911 /*ARGSUSED*/
912 int
913 pci_common_ioctl(dev_info_t *dip, dev_t dev, int cmd, intptr_t arg,
914     int mode, cred_t *credp, int *rvalp)
915 {
916 	minor_t	minor = getminor(dev);
917 	int	rv = ENOTTY;
918 
919 	switch (PCI_MINOR_NUM_TO_PCI_DEVNUM(minor)) {
920 	case PCI_TOOL_REG_MINOR_NUM:
921 
922 		switch (cmd) {
923 		case PCITOOL_DEVICE_SET_REG:
924 		case PCITOOL_DEVICE_GET_REG:
925 
926 			/* Require full privileges. */
927 			if (secpolicy_kmdb(credp))
928 				rv = EPERM;
929 			else
930 				rv = pcitool_dev_reg_ops(dip, (void *)arg,
931 				    cmd, mode);
932 			break;
933 
934 		case PCITOOL_NEXUS_SET_REG:
935 		case PCITOOL_NEXUS_GET_REG:
936 
937 			/* Require full privileges. */
938 			if (secpolicy_kmdb(credp))
939 				rv = EPERM;
940 			else
941 				rv = pcitool_bus_reg_ops(dip, (void *)arg,
942 				    cmd, mode);
943 			break;
944 		}
945 		break;
946 
947 	case PCI_TOOL_INTR_MINOR_NUM:
948 
949 		switch (cmd) {
950 		case PCITOOL_DEVICE_SET_INTR:
951 
952 			/* Require PRIV_SYS_RES_CONFIG, same as psradm */
953 			if (secpolicy_ponline(credp)) {
954 				rv = EPERM;
955 				break;
956 			}
957 
958 		/*FALLTHRU*/
959 		/* These require no special privileges. */
960 		case PCITOOL_DEVICE_GET_INTR:
961 		case PCITOOL_SYSTEM_INTR_INFO:
962 			rv = pcitool_intr_admn(dip, (void *)arg, cmd, mode);
963 			break;
964 		}
965 		break;
966 
967 	default:
968 		break;
969 	}
970 
971 	return (rv);
972 }
973 
974 
975 int
976 pci_common_ctlops_poke(peekpoke_ctlops_t *in_args)
977 {
978 	size_t size = in_args->size;
979 	uintptr_t dev_addr = in_args->dev_addr;
980 	uintptr_t host_addr = in_args->host_addr;
981 	ddi_acc_impl_t *hp = (ddi_acc_impl_t *)in_args->handle;
982 	ddi_acc_hdl_t *hdlp = (ddi_acc_hdl_t *)in_args->handle;
983 	size_t repcount = in_args->repcount;
984 	uint_t flags = in_args->flags;
985 	int err = DDI_SUCCESS;
986 
987 	/*
988 	 * if no handle then this is a poke. We have to return failure here
989 	 * as we have no way of knowing whether this is a MEM or IO space access
990 	 */
991 	if (in_args->handle == NULL)
992 		return (DDI_FAILURE);
993 
994 	/*
995 	 * rest of this function is actually for cautious puts
996 	 */
997 	for (; repcount; repcount--) {
998 		if (hp->ahi_acc_attr == DDI_ACCATTR_CONFIG_SPACE) {
999 			switch (size) {
1000 			case sizeof (uint8_t):
1001 				pci_config_wr8(hp, (uint8_t *)dev_addr,
1002 				    *(uint8_t *)host_addr);
1003 				break;
1004 			case sizeof (uint16_t):
1005 				pci_config_wr16(hp, (uint16_t *)dev_addr,
1006 				    *(uint16_t *)host_addr);
1007 				break;
1008 			case sizeof (uint32_t):
1009 				pci_config_wr32(hp, (uint32_t *)dev_addr,
1010 				    *(uint32_t *)host_addr);
1011 				break;
1012 			case sizeof (uint64_t):
1013 				pci_config_wr64(hp, (uint64_t *)dev_addr,
1014 				    *(uint64_t *)host_addr);
1015 				break;
1016 			default:
1017 				err = DDI_FAILURE;
1018 				break;
1019 			}
1020 		} else if (hp->ahi_acc_attr & DDI_ACCATTR_IO_SPACE) {
1021 			if (hdlp->ah_acc.devacc_attr_endian_flags ==
1022 			    DDI_STRUCTURE_BE_ACC) {
1023 				switch (size) {
1024 				case sizeof (uint8_t):
1025 					i_ddi_io_put8(hp,
1026 					    (uint8_t *)dev_addr,
1027 					    *(uint8_t *)host_addr);
1028 					break;
1029 				case sizeof (uint16_t):
1030 					i_ddi_io_swap_put16(hp,
1031 					    (uint16_t *)dev_addr,
1032 					    *(uint16_t *)host_addr);
1033 					break;
1034 				case sizeof (uint32_t):
1035 					i_ddi_io_swap_put32(hp,
1036 					    (uint32_t *)dev_addr,
1037 					    *(uint32_t *)host_addr);
1038 					break;
1039 				/*
1040 				 * note the 64-bit case is a dummy
1041 				 * function - so no need to swap
1042 				 */
1043 				case sizeof (uint64_t):
1044 					i_ddi_io_put64(hp,
1045 					    (uint64_t *)dev_addr,
1046 					    *(uint64_t *)host_addr);
1047 					break;
1048 				default:
1049 					err = DDI_FAILURE;
1050 					break;
1051 				}
1052 			} else {
1053 				switch (size) {
1054 				case sizeof (uint8_t):
1055 					i_ddi_io_put8(hp,
1056 					    (uint8_t *)dev_addr,
1057 					    *(uint8_t *)host_addr);
1058 					break;
1059 				case sizeof (uint16_t):
1060 					i_ddi_io_put16(hp,
1061 					    (uint16_t *)dev_addr,
1062 					    *(uint16_t *)host_addr);
1063 					break;
1064 				case sizeof (uint32_t):
1065 					i_ddi_io_put32(hp,
1066 					    (uint32_t *)dev_addr,
1067 					    *(uint32_t *)host_addr);
1068 					break;
1069 				case sizeof (uint64_t):
1070 					i_ddi_io_put64(hp,
1071 					    (uint64_t *)dev_addr,
1072 					    *(uint64_t *)host_addr);
1073 					break;
1074 				default:
1075 					err = DDI_FAILURE;
1076 					break;
1077 				}
1078 			}
1079 		} else {
1080 			if (hdlp->ah_acc.devacc_attr_endian_flags ==
1081 			    DDI_STRUCTURE_BE_ACC) {
1082 				switch (size) {
1083 				case sizeof (uint8_t):
1084 					*(uint8_t *)dev_addr =
1085 					    *(uint8_t *)host_addr;
1086 					break;
1087 				case sizeof (uint16_t):
1088 					*(uint16_t *)dev_addr =
1089 					    ddi_swap16(*(uint16_t *)host_addr);
1090 					break;
1091 				case sizeof (uint32_t):
1092 					*(uint32_t *)dev_addr =
1093 					    ddi_swap32(*(uint32_t *)host_addr);
1094 					break;
1095 				case sizeof (uint64_t):
1096 					*(uint64_t *)dev_addr =
1097 					    ddi_swap64(*(uint64_t *)host_addr);
1098 					break;
1099 				default:
1100 					err = DDI_FAILURE;
1101 					break;
1102 				}
1103 			} else {
1104 				switch (size) {
1105 				case sizeof (uint8_t):
1106 					*(uint8_t *)dev_addr =
1107 					    *(uint8_t *)host_addr;
1108 					break;
1109 				case sizeof (uint16_t):
1110 					*(uint16_t *)dev_addr =
1111 					    *(uint16_t *)host_addr;
1112 					break;
1113 				case sizeof (uint32_t):
1114 					*(uint32_t *)dev_addr =
1115 					    *(uint32_t *)host_addr;
1116 					break;
1117 				case sizeof (uint64_t):
1118 					*(uint64_t *)dev_addr =
1119 					    *(uint64_t *)host_addr;
1120 					break;
1121 				default:
1122 					err = DDI_FAILURE;
1123 					break;
1124 				}
1125 			}
1126 		}
1127 		host_addr += size;
1128 		if (flags == DDI_DEV_AUTOINCR)
1129 			dev_addr += size;
1130 	}
1131 	return (err);
1132 }
1133 
1134 
1135 int
1136 pci_fm_acc_setup(ddi_acc_hdl_t *hp, off_t offset, off_t len)
1137 {
1138 	ddi_acc_impl_t	*ap = (ddi_acc_impl_t *)hp->ah_platform_private;
1139 
1140 	/* endian-ness check */
1141 	if (hp->ah_acc.devacc_attr_endian_flags == DDI_STRUCTURE_BE_ACC)
1142 		return (DDI_FAILURE);
1143 
1144 	/*
1145 	 * range check
1146 	 */
1147 	if ((offset >= PCI_CONF_HDR_SIZE) ||
1148 	    (len > PCI_CONF_HDR_SIZE) ||
1149 	    (offset + len > PCI_CONF_HDR_SIZE))
1150 		return (DDI_FAILURE);
1151 
1152 	ap->ahi_acc_attr |= DDI_ACCATTR_CONFIG_SPACE;
1153 	/*
1154 	 * always use cautious mechanism for config space gets
1155 	 */
1156 	ap->ahi_get8 = i_ddi_caut_get8;
1157 	ap->ahi_get16 = i_ddi_caut_get16;
1158 	ap->ahi_get32 = i_ddi_caut_get32;
1159 	ap->ahi_get64 = i_ddi_caut_get64;
1160 	ap->ahi_rep_get8 = i_ddi_caut_rep_get8;
1161 	ap->ahi_rep_get16 = i_ddi_caut_rep_get16;
1162 	ap->ahi_rep_get32 = i_ddi_caut_rep_get32;
1163 	ap->ahi_rep_get64 = i_ddi_caut_rep_get64;
1164 	if (hp->ah_acc.devacc_attr_access == DDI_CAUTIOUS_ACC) {
1165 		ap->ahi_put8 = i_ddi_caut_put8;
1166 		ap->ahi_put16 = i_ddi_caut_put16;
1167 		ap->ahi_put32 = i_ddi_caut_put32;
1168 		ap->ahi_put64 = i_ddi_caut_put64;
1169 		ap->ahi_rep_put8 = i_ddi_caut_rep_put8;
1170 		ap->ahi_rep_put16 = i_ddi_caut_rep_put16;
1171 		ap->ahi_rep_put32 = i_ddi_caut_rep_put32;
1172 		ap->ahi_rep_put64 = i_ddi_caut_rep_put64;
1173 	} else {
1174 		ap->ahi_put8 = pci_config_wr8;
1175 		ap->ahi_put16 = pci_config_wr16;
1176 		ap->ahi_put32 = pci_config_wr32;
1177 		ap->ahi_put64 = pci_config_wr64;
1178 		ap->ahi_rep_put8 = pci_config_rep_wr8;
1179 		ap->ahi_rep_put16 = pci_config_rep_wr16;
1180 		ap->ahi_rep_put32 = pci_config_rep_wr32;
1181 		ap->ahi_rep_put64 = pci_config_rep_wr64;
1182 	}
1183 
1184 	/* Initialize to default check/notify functions */
1185 	ap->ahi_fault_check = i_ddi_acc_fault_check;
1186 	ap->ahi_fault_notify = i_ddi_acc_fault_notify;
1187 	ap->ahi_fault = 0;
1188 	impl_acc_err_init(hp);
1189 	return (DDI_SUCCESS);
1190 }
1191 
1192 
1193 int
1194 pci_common_ctlops_peek(peekpoke_ctlops_t *in_args)
1195 {
1196 	size_t size = in_args->size;
1197 	uintptr_t dev_addr = in_args->dev_addr;
1198 	uintptr_t host_addr = in_args->host_addr;
1199 	ddi_acc_impl_t *hp = (ddi_acc_impl_t *)in_args->handle;
1200 	ddi_acc_hdl_t *hdlp = (ddi_acc_hdl_t *)in_args->handle;
1201 	size_t repcount = in_args->repcount;
1202 	uint_t flags = in_args->flags;
1203 	int err = DDI_SUCCESS;
1204 
1205 	/*
1206 	 * if no handle then this is a peek. We have to return failure here
1207 	 * as we have no way of knowing whether this is a MEM or IO space access
1208 	 */
1209 	if (in_args->handle == NULL)
1210 		return (DDI_FAILURE);
1211 
1212 	for (; repcount; repcount--) {
1213 		if (hp->ahi_acc_attr == DDI_ACCATTR_CONFIG_SPACE) {
1214 			switch (size) {
1215 			case sizeof (uint8_t):
1216 				*(uint8_t *)host_addr = pci_config_rd8(hp,
1217 				    (uint8_t *)dev_addr);
1218 				break;
1219 			case sizeof (uint16_t):
1220 				*(uint16_t *)host_addr = pci_config_rd16(hp,
1221 				    (uint16_t *)dev_addr);
1222 				break;
1223 			case sizeof (uint32_t):
1224 				*(uint32_t *)host_addr = pci_config_rd32(hp,
1225 				    (uint32_t *)dev_addr);
1226 				break;
1227 			case sizeof (uint64_t):
1228 				*(uint64_t *)host_addr = pci_config_rd64(hp,
1229 				    (uint64_t *)dev_addr);
1230 				break;
1231 			default:
1232 				err = DDI_FAILURE;
1233 				break;
1234 			}
1235 		} else if (hp->ahi_acc_attr & DDI_ACCATTR_IO_SPACE) {
1236 			if (hdlp->ah_acc.devacc_attr_endian_flags ==
1237 			    DDI_STRUCTURE_BE_ACC) {
1238 				switch (size) {
1239 				case sizeof (uint8_t):
1240 					*(uint8_t *)host_addr =
1241 					    i_ddi_io_get8(hp,
1242 					    (uint8_t *)dev_addr);
1243 					break;
1244 				case sizeof (uint16_t):
1245 					*(uint16_t *)host_addr =
1246 					    i_ddi_io_swap_get16(hp,
1247 					    (uint16_t *)dev_addr);
1248 					break;
1249 				case sizeof (uint32_t):
1250 					*(uint32_t *)host_addr =
1251 					    i_ddi_io_swap_get32(hp,
1252 					    (uint32_t *)dev_addr);
1253 					break;
1254 				/*
1255 				 * note the 64-bit case is a dummy
1256 				 * function - so no need to swap
1257 				 */
1258 				case sizeof (uint64_t):
1259 					*(uint64_t *)host_addr =
1260 					    i_ddi_io_get64(hp,
1261 					    (uint64_t *)dev_addr);
1262 					break;
1263 				default:
1264 					err = DDI_FAILURE;
1265 					break;
1266 				}
1267 			} else {
1268 				switch (size) {
1269 				case sizeof (uint8_t):
1270 					*(uint8_t *)host_addr =
1271 					    i_ddi_io_get8(hp,
1272 					    (uint8_t *)dev_addr);
1273 					break;
1274 				case sizeof (uint16_t):
1275 					*(uint16_t *)host_addr =
1276 					    i_ddi_io_get16(hp,
1277 					    (uint16_t *)dev_addr);
1278 					break;
1279 				case sizeof (uint32_t):
1280 					*(uint32_t *)host_addr =
1281 					    i_ddi_io_get32(hp,
1282 					    (uint32_t *)dev_addr);
1283 					break;
1284 				case sizeof (uint64_t):
1285 					*(uint64_t *)host_addr =
1286 					    i_ddi_io_get64(hp,
1287 					    (uint64_t *)dev_addr);
1288 					break;
1289 				default:
1290 					err = DDI_FAILURE;
1291 					break;
1292 				}
1293 			}
1294 		} else {
1295 			if (hdlp->ah_acc.devacc_attr_endian_flags ==
1296 			    DDI_STRUCTURE_BE_ACC) {
1297 				switch (in_args->size) {
1298 				case sizeof (uint8_t):
1299 					*(uint8_t *)host_addr =
1300 					    *(uint8_t *)dev_addr;
1301 					break;
1302 				case sizeof (uint16_t):
1303 					*(uint16_t *)host_addr =
1304 					    ddi_swap16(*(uint16_t *)dev_addr);
1305 					break;
1306 				case sizeof (uint32_t):
1307 					*(uint32_t *)host_addr =
1308 					    ddi_swap32(*(uint32_t *)dev_addr);
1309 					break;
1310 				case sizeof (uint64_t):
1311 					*(uint64_t *)host_addr =
1312 					    ddi_swap64(*(uint64_t *)dev_addr);
1313 					break;
1314 				default:
1315 					err = DDI_FAILURE;
1316 					break;
1317 				}
1318 			} else {
1319 				switch (in_args->size) {
1320 				case sizeof (uint8_t):
1321 					*(uint8_t *)host_addr =
1322 					    *(uint8_t *)dev_addr;
1323 					break;
1324 				case sizeof (uint16_t):
1325 					*(uint16_t *)host_addr =
1326 					    *(uint16_t *)dev_addr;
1327 					break;
1328 				case sizeof (uint32_t):
1329 					*(uint32_t *)host_addr =
1330 					    *(uint32_t *)dev_addr;
1331 					break;
1332 				case sizeof (uint64_t):
1333 					*(uint64_t *)host_addr =
1334 					    *(uint64_t *)dev_addr;
1335 					break;
1336 				default:
1337 					err = DDI_FAILURE;
1338 					break;
1339 				}
1340 			}
1341 		}
1342 		host_addr += size;
1343 		if (flags == DDI_DEV_AUTOINCR)
1344 			dev_addr += size;
1345 	}
1346 	return (err);
1347 }
1348 
1349 /*ARGSUSED*/
1350 int
1351 pci_common_peekpoke(dev_info_t *dip, dev_info_t *rdip,
1352 	ddi_ctl_enum_t ctlop, void *arg, void *result)
1353 {
1354 	if (ctlop == DDI_CTLOPS_PEEK)
1355 		return (pci_common_ctlops_peek((peekpoke_ctlops_t *)arg));
1356 	else
1357 		return (pci_common_ctlops_poke((peekpoke_ctlops_t *)arg));
1358 }
1359 
1360 /*
1361  * These are the get and put functions to be shared with drivers. The
1362  * mutex locking is done inside the functions referenced, rather than
1363  * here, and is thus shared across PCI child drivers and any other
1364  * consumers of PCI config space (such as the ACPI subsystem).
1365  *
1366  * The configuration space addresses come in as pointers.  This is fine on
1367  * a 32-bit system, where the VM space and configuration space are the same
1368  * size.  It's not such a good idea on a 64-bit system, where memory
1369  * addresses are twice as large as configuration space addresses.  At some
1370  * point in the call tree we need to take a stand and say "you are 32-bit
1371  * from this time forth", and this seems like a nice self-contained place.
1372  */
1373 
1374 uint8_t
1375 pci_config_rd8(ddi_acc_impl_t *hdlp, uint8_t *addr)
1376 {
1377 	pci_acc_cfblk_t *cfp;
1378 	uint8_t	rval;
1379 	int reg;
1380 
1381 	ASSERT64(((uintptr_t)addr >> 32) == 0);
1382 
1383 	reg = (int)(uintptr_t)addr;
1384 
1385 	cfp = (pci_acc_cfblk_t *)&hdlp->ahi_common.ah_bus_private;
1386 
1387 	rval = (*pci_getb_func)(cfp->c_busnum, cfp->c_devnum, cfp->c_funcnum,
1388 	    reg);
1389 
1390 	return (rval);
1391 }
1392 
1393 void
1394 pci_config_rep_rd8(ddi_acc_impl_t *hdlp, uint8_t *host_addr,
1395 	uint8_t *dev_addr, size_t repcount, uint_t flags)
1396 {
1397 	uint8_t *h, *d;
1398 
1399 	h = host_addr;
1400 	d = dev_addr;
1401 
1402 	if (flags == DDI_DEV_AUTOINCR)
1403 		for (; repcount; repcount--)
1404 			*h++ = pci_config_rd8(hdlp, d++);
1405 	else
1406 		for (; repcount; repcount--)
1407 			*h++ = pci_config_rd8(hdlp, d);
1408 }
1409 
1410 uint16_t
1411 pci_config_rd16(ddi_acc_impl_t *hdlp, uint16_t *addr)
1412 {
1413 	pci_acc_cfblk_t *cfp;
1414 	uint16_t rval;
1415 	int reg;
1416 
1417 	ASSERT64(((uintptr_t)addr >> 32) == 0);
1418 
1419 	reg = (int)(uintptr_t)addr;
1420 
1421 	cfp = (pci_acc_cfblk_t *)&hdlp->ahi_common.ah_bus_private;
1422 
1423 	rval = (*pci_getw_func)(cfp->c_busnum, cfp->c_devnum, cfp->c_funcnum,
1424 	    reg);
1425 
1426 	return (rval);
1427 }
1428 
1429 void
1430 pci_config_rep_rd16(ddi_acc_impl_t *hdlp, uint16_t *host_addr,
1431 	uint16_t *dev_addr, size_t repcount, uint_t flags)
1432 {
1433 	uint16_t *h, *d;
1434 
1435 	h = host_addr;
1436 	d = dev_addr;
1437 
1438 	if (flags == DDI_DEV_AUTOINCR)
1439 		for (; repcount; repcount--)
1440 			*h++ = pci_config_rd16(hdlp, d++);
1441 	else
1442 		for (; repcount; repcount--)
1443 			*h++ = pci_config_rd16(hdlp, d);
1444 }
1445 
1446 uint32_t
1447 pci_config_rd32(ddi_acc_impl_t *hdlp, uint32_t *addr)
1448 {
1449 	pci_acc_cfblk_t *cfp;
1450 	uint32_t rval;
1451 	int reg;
1452 
1453 	ASSERT64(((uintptr_t)addr >> 32) == 0);
1454 
1455 	reg = (int)(uintptr_t)addr;
1456 
1457 	cfp = (pci_acc_cfblk_t *)&hdlp->ahi_common.ah_bus_private;
1458 
1459 	rval = (*pci_getl_func)(cfp->c_busnum, cfp->c_devnum,
1460 	    cfp->c_funcnum, reg);
1461 
1462 	return (rval);
1463 }
1464 
1465 void
1466 pci_config_rep_rd32(ddi_acc_impl_t *hdlp, uint32_t *host_addr,
1467 	uint32_t *dev_addr, size_t repcount, uint_t flags)
1468 {
1469 	uint32_t *h, *d;
1470 
1471 	h = host_addr;
1472 	d = dev_addr;
1473 
1474 	if (flags == DDI_DEV_AUTOINCR)
1475 		for (; repcount; repcount--)
1476 			*h++ = pci_config_rd32(hdlp, d++);
1477 	else
1478 		for (; repcount; repcount--)
1479 			*h++ = pci_config_rd32(hdlp, d);
1480 }
1481 
1482 
1483 void
1484 pci_config_wr8(ddi_acc_impl_t *hdlp, uint8_t *addr, uint8_t value)
1485 {
1486 	pci_acc_cfblk_t *cfp;
1487 	int reg;
1488 
1489 	ASSERT64(((uintptr_t)addr >> 32) == 0);
1490 
1491 	reg = (int)(uintptr_t)addr;
1492 
1493 	cfp = (pci_acc_cfblk_t *)&hdlp->ahi_common.ah_bus_private;
1494 
1495 	(*pci_putb_func)(cfp->c_busnum, cfp->c_devnum,
1496 	    cfp->c_funcnum, reg, value);
1497 }
1498 
1499 void
1500 pci_config_rep_wr8(ddi_acc_impl_t *hdlp, uint8_t *host_addr,
1501 	uint8_t *dev_addr, size_t repcount, uint_t flags)
1502 {
1503 	uint8_t *h, *d;
1504 
1505 	h = host_addr;
1506 	d = dev_addr;
1507 
1508 	if (flags == DDI_DEV_AUTOINCR)
1509 		for (; repcount; repcount--)
1510 			pci_config_wr8(hdlp, d++, *h++);
1511 	else
1512 		for (; repcount; repcount--)
1513 			pci_config_wr8(hdlp, d, *h++);
1514 }
1515 
1516 void
1517 pci_config_wr16(ddi_acc_impl_t *hdlp, uint16_t *addr, uint16_t value)
1518 {
1519 	pci_acc_cfblk_t *cfp;
1520 	int reg;
1521 
1522 	ASSERT64(((uintptr_t)addr >> 32) == 0);
1523 
1524 	reg = (int)(uintptr_t)addr;
1525 
1526 	cfp = (pci_acc_cfblk_t *)&hdlp->ahi_common.ah_bus_private;
1527 
1528 	(*pci_putw_func)(cfp->c_busnum, cfp->c_devnum,
1529 	    cfp->c_funcnum, reg, value);
1530 }
1531 
1532 void
1533 pci_config_rep_wr16(ddi_acc_impl_t *hdlp, uint16_t *host_addr,
1534 	uint16_t *dev_addr, size_t repcount, uint_t flags)
1535 {
1536 	uint16_t *h, *d;
1537 
1538 	h = host_addr;
1539 	d = dev_addr;
1540 
1541 	if (flags == DDI_DEV_AUTOINCR)
1542 		for (; repcount; repcount--)
1543 			pci_config_wr16(hdlp, d++, *h++);
1544 	else
1545 		for (; repcount; repcount--)
1546 			pci_config_wr16(hdlp, d, *h++);
1547 }
1548 
1549 void
1550 pci_config_wr32(ddi_acc_impl_t *hdlp, uint32_t *addr, uint32_t value)
1551 {
1552 	pci_acc_cfblk_t *cfp;
1553 	int reg;
1554 
1555 	ASSERT64(((uintptr_t)addr >> 32) == 0);
1556 
1557 	reg = (int)(uintptr_t)addr;
1558 
1559 	cfp = (pci_acc_cfblk_t *)&hdlp->ahi_common.ah_bus_private;
1560 
1561 	(*pci_putl_func)(cfp->c_busnum, cfp->c_devnum,
1562 	    cfp->c_funcnum, reg, value);
1563 }
1564 
1565 void
1566 pci_config_rep_wr32(ddi_acc_impl_t *hdlp, uint32_t *host_addr,
1567 	uint32_t *dev_addr, size_t repcount, uint_t flags)
1568 {
1569 	uint32_t *h, *d;
1570 
1571 	h = host_addr;
1572 	d = dev_addr;
1573 
1574 	if (flags == DDI_DEV_AUTOINCR)
1575 		for (; repcount; repcount--)
1576 			pci_config_wr32(hdlp, d++, *h++);
1577 	else
1578 		for (; repcount; repcount--)
1579 			pci_config_wr32(hdlp, d, *h++);
1580 }
1581 
1582 uint64_t
1583 pci_config_rd64(ddi_acc_impl_t *hdlp, uint64_t *addr)
1584 {
1585 	uint32_t lw_val;
1586 	uint32_t hi_val;
1587 	uint32_t *dp;
1588 	uint64_t val;
1589 
1590 	dp = (uint32_t *)addr;
1591 	lw_val = pci_config_rd32(hdlp, dp);
1592 	dp++;
1593 	hi_val = pci_config_rd32(hdlp, dp);
1594 	val = ((uint64_t)hi_val << 32) | lw_val;
1595 	return (val);
1596 }
1597 
1598 void
1599 pci_config_wr64(ddi_acc_impl_t *hdlp, uint64_t *addr, uint64_t value)
1600 {
1601 	uint32_t lw_val;
1602 	uint32_t hi_val;
1603 	uint32_t *dp;
1604 
1605 	dp = (uint32_t *)addr;
1606 	lw_val = (uint32_t)(value & 0xffffffff);
1607 	hi_val = (uint32_t)(value >> 32);
1608 	pci_config_wr32(hdlp, dp, lw_val);
1609 	dp++;
1610 	pci_config_wr32(hdlp, dp, hi_val);
1611 }
1612 
1613 void
1614 pci_config_rep_rd64(ddi_acc_impl_t *hdlp, uint64_t *host_addr,
1615 	uint64_t *dev_addr, size_t repcount, uint_t flags)
1616 {
1617 	if (flags == DDI_DEV_AUTOINCR) {
1618 		for (; repcount; repcount--)
1619 			*host_addr++ = pci_config_rd64(hdlp, dev_addr++);
1620 	} else {
1621 		for (; repcount; repcount--)
1622 			*host_addr++ = pci_config_rd64(hdlp, dev_addr);
1623 	}
1624 }
1625 
1626 void
1627 pci_config_rep_wr64(ddi_acc_impl_t *hdlp, uint64_t *host_addr,
1628 	uint64_t *dev_addr, size_t repcount, uint_t flags)
1629 {
1630 	if (flags == DDI_DEV_AUTOINCR) {
1631 		for (; repcount; repcount--)
1632 			pci_config_wr64(hdlp, host_addr++, *dev_addr++);
1633 	} else {
1634 		for (; repcount; repcount--)
1635 			pci_config_wr64(hdlp, host_addr++, *dev_addr);
1636 	}
1637 }
1638