xref: /titanic_52/usr/src/uts/i86pc/io/pci/pci_tools.c (revision de710d24d2fae4468e64da999e1d952a247f142c)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 /*
22  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
23  */
24 
25 #include <sys/sysmacros.h>
26 #include <sys/types.h>
27 #include <sys/mkdev.h>
28 #include <sys/stat.h>
29 #include <sys/sunddi.h>
30 #include <vm/seg_kmem.h>
31 #include <sys/machparam.h>
32 #include <sys/sunndi.h>
33 #include <sys/ontrap.h>
34 #include <sys/psm.h>
35 #include <sys/pcie.h>
36 #include <sys/pci_cfgspace.h>
37 #include <sys/pci_tools.h>
38 #include <io/pci/pci_tools_ext.h>
39 #include <sys/apic.h>
40 #include <sys/apix.h>
41 #include <io/pci/pci_var.h>
42 #include <sys/pci_impl.h>
43 #include <sys/promif.h>
44 #include <sys/x86_archext.h>
45 #include <sys/cpuvar.h>
46 #include <sys/pci_cfgacc.h>
47 
48 #ifdef __xpv
49 #include <sys/hypervisor.h>
50 #endif
51 
52 #define	PCIEX_BDF_OFFSET_DELTA	4
53 #define	PCIEX_REG_FUNC_SHIFT	(PCI_REG_FUNC_SHIFT + PCIEX_BDF_OFFSET_DELTA)
54 #define	PCIEX_REG_DEV_SHIFT	(PCI_REG_DEV_SHIFT + PCIEX_BDF_OFFSET_DELTA)
55 #define	PCIEX_REG_BUS_SHIFT	(PCI_REG_BUS_SHIFT + PCIEX_BDF_OFFSET_DELTA)
56 
57 #define	SUCCESS	0
58 
59 extern uint64_t mcfg_mem_base;
60 int pcitool_debug = 0;
61 
62 /*
63  * Offsets of BARS in config space.  First entry of 0 means config space.
64  * Entries here correlate to pcitool_bars_t enumerated type.
65  */
66 static uint8_t pci_bars[] = {
67 	0x0,
68 	PCI_CONF_BASE0,
69 	PCI_CONF_BASE1,
70 	PCI_CONF_BASE2,
71 	PCI_CONF_BASE3,
72 	PCI_CONF_BASE4,
73 	PCI_CONF_BASE5,
74 	PCI_CONF_ROM
75 };
76 
77 /* Max offset allowed into config space for a particular device. */
78 static uint64_t max_cfg_size = PCI_CONF_HDR_SIZE;
79 
80 static uint64_t pcitool_swap_endian(uint64_t data, int size);
81 static int pcitool_cfg_access(pcitool_reg_t *prg, boolean_t write_flag,
82     boolean_t io_access);
83 static int pcitool_io_access(pcitool_reg_t *prg, boolean_t write_flag);
84 static int pcitool_mem_access(pcitool_reg_t *prg, uint64_t virt_addr,
85     boolean_t write_flag);
86 static uint64_t pcitool_map(uint64_t phys_addr, size_t size, size_t *num_pages);
87 static void pcitool_unmap(uint64_t virt_addr, size_t num_pages);
88 
89 /* Extern declarations */
90 extern int	(*psm_intr_ops)(dev_info_t *, ddi_intr_handle_impl_t *,
91 		    psm_intr_op_t, int *);
92 
93 int
94 pcitool_init(dev_info_t *dip, boolean_t is_pciex)
95 {
96 	int instance = ddi_get_instance(dip);
97 
98 	/* Create pcitool nodes for register access and interrupt routing. */
99 
100 	if (ddi_create_minor_node(dip, PCI_MINOR_REG, S_IFCHR,
101 	    PCI_MINOR_NUM(instance, PCI_TOOL_REG_MINOR_NUM),
102 	    DDI_NT_REGACC, 0) != DDI_SUCCESS) {
103 		return (DDI_FAILURE);
104 	}
105 
106 	if (ddi_create_minor_node(dip, PCI_MINOR_INTR, S_IFCHR,
107 	    PCI_MINOR_NUM(instance, PCI_TOOL_INTR_MINOR_NUM),
108 	    DDI_NT_INTRCTL, 0) != DDI_SUCCESS) {
109 		ddi_remove_minor_node(dip, PCI_MINOR_REG);
110 		return (DDI_FAILURE);
111 	}
112 
113 	if (is_pciex)
114 		max_cfg_size = PCIE_CONF_HDR_SIZE;
115 
116 	return (DDI_SUCCESS);
117 }
118 
119 void
120 pcitool_uninit(dev_info_t *dip)
121 {
122 	ddi_remove_minor_node(dip, PCI_MINOR_INTR);
123 	ddi_remove_minor_node(dip, PCI_MINOR_REG);
124 }
125 
126 /*ARGSUSED*/
127 static int
128 pcitool_set_intr(dev_info_t *dip, void *arg, int mode)
129 {
130 	ddi_intr_handle_impl_t info_hdl;
131 	pcitool_intr_set_t iset;
132 	uint32_t old_cpu;
133 	int ret, result;
134 	size_t copyinout_size;
135 	int rval = SUCCESS;
136 	apic_get_type_t type_info;
137 
138 	/* Version 1 of pcitool_intr_set_t doesn't have flags. */
139 	copyinout_size = (size_t)&iset.flags - (size_t)&iset;
140 
141 	if (ddi_copyin(arg, &iset, copyinout_size, mode) != DDI_SUCCESS)
142 		return (EFAULT);
143 
144 	switch (iset.user_version) {
145 	case PCITOOL_V1:
146 		break;
147 
148 	case PCITOOL_V2:
149 		copyinout_size = sizeof (pcitool_intr_set_t);
150 		if (ddi_copyin(arg, &iset, copyinout_size, mode) != DDI_SUCCESS)
151 			return (EFAULT);
152 		break;
153 
154 	default:
155 		iset.status = PCITOOL_OUT_OF_RANGE;
156 		rval = ENOTSUP;
157 		goto done_set_intr;
158 	}
159 
160 	if (iset.flags & PCITOOL_INTR_FLAG_SET_MSI) {
161 		rval = ENOTSUP;
162 		iset.status = PCITOOL_IO_ERROR;
163 		goto done_set_intr;
164 	}
165 
166 	info_hdl.ih_private = &type_info;
167 
168 	if ((*psm_intr_ops)(NULL, &info_hdl,
169 	    PSM_INTR_OP_APIC_TYPE, NULL) != PSM_SUCCESS) {
170 		rval = ENOTSUP;
171 		iset.status = PCITOOL_IO_ERROR;
172 		goto done_set_intr;
173 	}
174 
175 	if (strcmp(type_info.avgi_type, APIC_APIX_NAME) == 0) {
176 		if (iset.old_cpu > type_info.avgi_num_cpu) {
177 			rval = EINVAL;
178 			iset.status = PCITOOL_INVALID_CPUID;
179 			goto done_set_intr;
180 		}
181 		old_cpu = iset.old_cpu;
182 	} else {
183 		if ((old_cpu =
184 		    pci_get_cpu_from_vecirq(iset.ino, IS_VEC)) == -1) {
185 			iset.status = PCITOOL_IO_ERROR;
186 			rval = EINVAL;
187 			goto done_set_intr;
188 		}
189 	}
190 
191 	if (iset.ino > type_info.avgi_num_intr) {
192 		rval = EINVAL;
193 		iset.status = PCITOOL_INVALID_INO;
194 		goto done_set_intr;
195 	}
196 
197 	iset.status = PCITOOL_SUCCESS;
198 
199 	old_cpu &= ~PSMGI_CPU_USER_BOUND;
200 
201 	/*
202 	 * For this locally-declared and used handle, ih_private will contain a
203 	 * CPU value, not an ihdl_plat_t as used for global interrupt handling.
204 	 */
205 	if (strcmp(type_info.avgi_type, APIC_APIX_NAME) == 0) {
206 		info_hdl.ih_vector = APIX_VIRTVECTOR(old_cpu, iset.ino);
207 	} else {
208 		info_hdl.ih_vector = iset.ino;
209 	}
210 	info_hdl.ih_private = (void *)(uintptr_t)iset.cpu_id;
211 	info_hdl.ih_flags = PSMGI_INTRBY_VEC;
212 	if (pcitool_debug)
213 		prom_printf("user version:%d, flags:0x%x\n",
214 		    iset.user_version, iset.flags);
215 
216 	result = ENOTSUP;
217 	if ((iset.user_version >= PCITOOL_V2) &&
218 	    (iset.flags & PCITOOL_INTR_FLAG_SET_GROUP)) {
219 		ret = (*psm_intr_ops)(NULL, &info_hdl, PSM_INTR_OP_GRP_SET_CPU,
220 		    &result);
221 	} else {
222 		ret = (*psm_intr_ops)(NULL, &info_hdl, PSM_INTR_OP_SET_CPU,
223 		    &result);
224 	}
225 
226 	if (ret != PSM_SUCCESS) {
227 		switch (result) {
228 		case EIO:		/* Error making the change */
229 			rval = EIO;
230 			iset.status = PCITOOL_IO_ERROR;
231 			break;
232 		case ENXIO:		/* Couldn't convert vector to irq */
233 			rval = EINVAL;
234 			iset.status = PCITOOL_INVALID_INO;
235 			break;
236 		case EINVAL:		/* CPU out of range */
237 			rval = EINVAL;
238 			iset.status = PCITOOL_INVALID_CPUID;
239 			break;
240 		case ENOTSUP:		/* Requested PSM intr ops missing */
241 			rval = ENOTSUP;
242 			iset.status = PCITOOL_IO_ERROR;
243 			break;
244 		}
245 	}
246 
247 	/* Return original CPU. */
248 	iset.cpu_id = old_cpu;
249 
250 	/* Return new vector */
251 	if (strcmp(type_info.avgi_type, APIC_APIX_NAME) == 0) {
252 		iset.ino = APIX_VIRTVEC_VECTOR(info_hdl.ih_vector);
253 	}
254 
255 done_set_intr:
256 	iset.drvr_version = PCITOOL_VERSION;
257 	if (ddi_copyout(&iset, arg, copyinout_size, mode) != DDI_SUCCESS)
258 		rval = EFAULT;
259 	return (rval);
260 }
261 
262 
263 /* It is assumed that dip != NULL */
264 static void
265 pcitool_get_intr_dev_info(dev_info_t *dip, pcitool_intr_dev_t *devs)
266 {
267 	(void) strncpy(devs->driver_name,
268 	    ddi_driver_name(dip), MAXMODCONFNAME-2);
269 	devs->driver_name[MAXMODCONFNAME-1] = '\0';
270 	(void) ddi_pathname(dip, devs->path);
271 	devs->dev_inst = ddi_get_instance(dip);
272 }
273 
274 static int
275 pcitool_get_intr(dev_info_t *dip, void *arg, int mode)
276 {
277 	/* Array part isn't used here, but oh well... */
278 	pcitool_intr_get_t partial_iget;
279 	pcitool_intr_get_t *iget = &partial_iget;
280 	size_t	iget_kmem_alloc_size = 0;
281 	uint8_t num_devs_ret;
282 	int copyout_rval;
283 	int rval = SUCCESS;
284 	int circ;
285 	int i;
286 
287 	ddi_intr_handle_impl_t info_hdl;
288 	apic_get_intr_t intr_info;
289 	apic_get_type_t type_info;
290 
291 	/* Read in just the header part, no array section. */
292 	if (ddi_copyin(arg, &partial_iget, PCITOOL_IGET_SIZE(0), mode) !=
293 	    DDI_SUCCESS)
294 		return (EFAULT);
295 
296 	if (partial_iget.flags & PCITOOL_INTR_FLAG_GET_MSI) {
297 		partial_iget.status = PCITOOL_IO_ERROR;
298 		partial_iget.num_devs_ret = 0;
299 		rval = ENOTSUP;
300 		goto done_get_intr;
301 	}
302 
303 	info_hdl.ih_private = &type_info;
304 
305 	if ((*psm_intr_ops)(NULL, &info_hdl,
306 	    PSM_INTR_OP_APIC_TYPE, NULL) != PSM_SUCCESS) {
307 		iget->status = PCITOOL_IO_ERROR;
308 		iget->num_devs_ret = 0;
309 		rval = EINVAL;
310 		goto done_get_intr;
311 	}
312 
313 	if (strcmp(type_info.avgi_type, APIC_APIX_NAME) == 0) {
314 		if (partial_iget.cpu_id > type_info.avgi_num_cpu) {
315 			partial_iget.status = PCITOOL_INVALID_CPUID;
316 			partial_iget.num_devs_ret = 0;
317 			rval = EINVAL;
318 			goto done_get_intr;
319 		}
320 	}
321 
322 	/* Validate argument. */
323 	if ((partial_iget.ino & APIX_VIRTVEC_VECMASK) >
324 	    type_info.avgi_num_intr) {
325 		partial_iget.status = PCITOOL_INVALID_INO;
326 		partial_iget.num_devs_ret = 0;
327 		rval = EINVAL;
328 		goto done_get_intr;
329 	}
330 
331 	num_devs_ret = partial_iget.num_devs_ret;
332 	intr_info.avgi_dip_list = NULL;
333 	intr_info.avgi_req_flags =
334 	    PSMGI_REQ_CPUID | PSMGI_REQ_NUM_DEVS | PSMGI_INTRBY_VEC;
335 	/*
336 	 * For this locally-declared and used handle, ih_private will contain a
337 	 * pointer to apic_get_intr_t, not an ihdl_plat_t as used for
338 	 * global interrupt handling.
339 	 */
340 	info_hdl.ih_private = &intr_info;
341 
342 	if (strcmp(type_info.avgi_type, APIC_APIX_NAME) == 0) {
343 		info_hdl.ih_vector =
344 		    APIX_VIRTVECTOR(partial_iget.cpu_id, partial_iget.ino);
345 	} else {
346 		info_hdl.ih_vector = partial_iget.ino;
347 	}
348 
349 	/* Caller wants device information returned. */
350 	if (num_devs_ret > 0) {
351 
352 		intr_info.avgi_req_flags |= PSMGI_REQ_GET_DEVS;
353 
354 		/*
355 		 * Allocate room.
356 		 * If num_devs_ret == 0 iget remains pointing to partial_iget.
357 		 */
358 		iget_kmem_alloc_size = PCITOOL_IGET_SIZE(num_devs_ret);
359 		iget = kmem_alloc(iget_kmem_alloc_size, KM_SLEEP);
360 
361 		/* Read in whole structure to verify there's room. */
362 		if (ddi_copyin(arg, iget, iget_kmem_alloc_size, mode) !=
363 		    SUCCESS) {
364 
365 			/* Be consistent and just return EFAULT here. */
366 			kmem_free(iget, iget_kmem_alloc_size);
367 
368 			return (EFAULT);
369 		}
370 	}
371 
372 	bzero(iget, PCITOOL_IGET_SIZE(num_devs_ret));
373 	iget->ino = info_hdl.ih_vector;
374 
375 	/*
376 	 * Lock device tree branch from the pci root nexus on down if info will
377 	 * be extracted from dips returned from the tree.
378 	 */
379 	if (intr_info.avgi_req_flags & PSMGI_REQ_GET_DEVS) {
380 		ndi_devi_enter(dip, &circ);
381 	}
382 
383 	/* Call psm_intr_ops(PSM_INTR_OP_GET_INTR) to get information. */
384 	if ((rval = (*psm_intr_ops)(NULL, &info_hdl,
385 	    PSM_INTR_OP_GET_INTR, NULL)) != PSM_SUCCESS) {
386 		iget->status = PCITOOL_IO_ERROR;
387 		iget->num_devs_ret = 0;
388 		rval = EINVAL;
389 		goto done_get_intr;
390 	}
391 
392 	/*
393 	 * Fill in the pcitool_intr_get_t to be returned,
394 	 * with the CPU, num_devs_ret and num_devs.
395 	 */
396 	if (intr_info.avgi_cpu_id == IRQ_UNBOUND ||
397 	    intr_info.avgi_cpu_id == IRQ_UNINIT)
398 		iget->cpu_id = 0;
399 	else
400 		iget->cpu_id = intr_info.avgi_cpu_id & ~PSMGI_CPU_USER_BOUND;
401 
402 	/* Number of devices returned by apic. */
403 	iget->num_devs = intr_info.avgi_num_devs;
404 
405 	/* Device info was returned. */
406 	if (intr_info.avgi_req_flags & PSMGI_REQ_GET_DEVS) {
407 
408 		/*
409 		 * num devs returned is num devs ret by apic,
410 		 * space permitting.
411 		 */
412 		iget->num_devs_ret = min(num_devs_ret, intr_info.avgi_num_devs);
413 
414 		/*
415 		 * Loop thru list of dips and extract driver, name and instance.
416 		 * Fill in the pcitool_intr_dev_t's with this info.
417 		 */
418 		for (i = 0; i < iget->num_devs_ret; i++)
419 			pcitool_get_intr_dev_info(intr_info.avgi_dip_list[i],
420 			    &iget->dev[i]);
421 
422 		/* Free kmem_alloc'ed memory of the apic_get_intr_t */
423 		kmem_free(intr_info.avgi_dip_list,
424 		    intr_info.avgi_num_devs * sizeof (dev_info_t *));
425 	}
426 
427 done_get_intr:
428 
429 	if (intr_info.avgi_req_flags & PSMGI_REQ_GET_DEVS) {
430 		ndi_devi_exit(dip, circ);
431 	}
432 
433 	iget->drvr_version = PCITOOL_VERSION;
434 	copyout_rval = ddi_copyout(iget, arg,
435 	    PCITOOL_IGET_SIZE(num_devs_ret), mode);
436 
437 	if (iget_kmem_alloc_size > 0)
438 		kmem_free(iget, iget_kmem_alloc_size);
439 
440 	if (copyout_rval != DDI_SUCCESS)
441 		rval = EFAULT;
442 
443 	return (rval);
444 }
445 
446 /*ARGSUSED*/
447 static int
448 pcitool_intr_info(dev_info_t *dip, void *arg, int mode)
449 {
450 	pcitool_intr_info_t intr_info;
451 	ddi_intr_handle_impl_t info_hdl;
452 	int rval = SUCCESS;
453 	apic_get_type_t type_info;
454 
455 	/* If we need user_version, and to ret same user version as passed in */
456 	if (ddi_copyin(arg, &intr_info, sizeof (pcitool_intr_info_t), mode) !=
457 	    DDI_SUCCESS) {
458 		if (pcitool_debug)
459 			prom_printf("Error reading arguments\n");
460 		return (EFAULT);
461 	}
462 
463 	if (intr_info.flags & PCITOOL_INTR_FLAG_GET_MSI)
464 		return (ENOTSUP);
465 
466 	info_hdl.ih_private = &type_info;
467 
468 	/* For UPPC systems, psm_intr_ops has no entry for APIC_TYPE. */
469 	if ((rval = (*psm_intr_ops)(NULL, &info_hdl,
470 	    PSM_INTR_OP_APIC_TYPE, NULL)) != PSM_SUCCESS) {
471 		intr_info.ctlr_type = PCITOOL_CTLR_TYPE_UPPC;
472 		intr_info.ctlr_version = 0;
473 		intr_info.num_intr = APIC_MAX_VECTOR;
474 	} else {
475 		intr_info.ctlr_version = (uint32_t)info_hdl.ih_ver;
476 		intr_info.num_cpu = type_info.avgi_num_cpu;
477 		if (strcmp(type_info.avgi_type,
478 		    APIC_PCPLUSMP_NAME) == 0) {
479 			intr_info.ctlr_type = PCITOOL_CTLR_TYPE_PCPLUSMP;
480 			intr_info.num_intr = type_info.avgi_num_intr;
481 		} else if (strcmp(type_info.avgi_type,
482 		    APIC_APIX_NAME) == 0) {
483 			intr_info.ctlr_type = PCITOOL_CTLR_TYPE_APIX;
484 			intr_info.num_intr = type_info.avgi_num_intr;
485 		} else {
486 			intr_info.ctlr_type = PCITOOL_CTLR_TYPE_UNKNOWN;
487 			intr_info.num_intr = APIC_MAX_VECTOR;
488 		}
489 	}
490 
491 	intr_info.drvr_version = PCITOOL_VERSION;
492 	if (ddi_copyout(&intr_info, arg, sizeof (pcitool_intr_info_t), mode) !=
493 	    DDI_SUCCESS) {
494 		if (pcitool_debug)
495 			prom_printf("Error returning arguments.\n");
496 		rval = EFAULT;
497 	}
498 
499 	return (rval);
500 }
501 
502 
503 
504 /*
505  * Main function for handling interrupt CPU binding requests and queries.
506  * Need to implement later
507  */
508 int
509 pcitool_intr_admn(dev_info_t *dip, void *arg, int cmd, int mode)
510 {
511 	int rval;
512 
513 	switch (cmd) {
514 
515 	/* Associate a new CPU with a given vector */
516 	case PCITOOL_DEVICE_SET_INTR:
517 		rval = pcitool_set_intr(dip, arg, mode);
518 		break;
519 
520 	case PCITOOL_DEVICE_GET_INTR:
521 		rval = pcitool_get_intr(dip, arg, mode);
522 		break;
523 
524 	case PCITOOL_SYSTEM_INTR_INFO:
525 		rval = pcitool_intr_info(dip, arg, mode);
526 		break;
527 
528 	default:
529 		rval = ENOTSUP;
530 	}
531 
532 	return (rval);
533 }
534 
535 /*
536  * Perform register accesses on the nexus device itself.
537  * No explicit PCI nexus device for X86, so not applicable.
538  */
539 
540 /*ARGSUSED*/
541 int
542 pcitool_bus_reg_ops(dev_info_t *dip, void *arg, int cmd, int mode)
543 {
544 	return (ENOTSUP);
545 }
546 
547 /* Swap endianness. */
548 static uint64_t
549 pcitool_swap_endian(uint64_t data, int size)
550 {
551 	typedef union {
552 		uint64_t data64;
553 		uint8_t data8[8];
554 	} data_split_t;
555 
556 	data_split_t orig_data;
557 	data_split_t returned_data;
558 	int i;
559 
560 	orig_data.data64 = data;
561 	returned_data.data64 = 0;
562 
563 	for (i = 0; i < size; i++) {
564 		returned_data.data8[i] = orig_data.data8[size - 1 - i];
565 	}
566 
567 	return (returned_data.data64);
568 }
569 
570 /*
571  * A note about ontrap handling:
572  *
573  * X86 systems on which this module was tested return FFs instead of bus errors
574  * when accessing devices with invalid addresses.  Ontrap handling, which
575  * gracefully handles kernel bus errors, is installed anyway for I/O and mem
576  * space accessing (not for pci config space), in case future X86 platforms
577  * require it.
578  */
579 
580 /* Access device.  prg is modified. */
581 static int
582 pcitool_cfg_access(pcitool_reg_t *prg, boolean_t write_flag,
583     boolean_t io_access)
584 {
585 	int size = PCITOOL_ACC_ATTR_SIZE(prg->acc_attr);
586 	boolean_t big_endian = PCITOOL_ACC_IS_BIG_ENDIAN(prg->acc_attr);
587 	int rval = SUCCESS;
588 	uint64_t local_data;
589 	pci_cfgacc_req_t req;
590 	uint32_t max_offset;
591 
592 	if ((size <= 0) || (size > 8) || !ISP2(size)) {
593 		prg->status = PCITOOL_INVALID_SIZE;
594 		return (ENOTSUP);
595 	}
596 
597 	/*
598 	 * NOTE: there is no way to verify whether or not the address is
599 	 * valid other than that it is within the maximum offset.  The
600 	 * put functions return void and the get functions return -1 on error.
601 	 */
602 
603 	if (io_access)
604 		max_offset = 0xFF;
605 	else
606 		max_offset = 0xFFF;
607 	if (prg->offset + size - 1 > max_offset) {
608 		prg->status = PCITOOL_INVALID_ADDRESS;
609 		return (ENOTSUP);
610 	}
611 
612 	prg->status = PCITOOL_SUCCESS;
613 
614 	req.rcdip = NULL;
615 	req.bdf = PCI_GETBDF(prg->bus_no, prg->dev_no, prg->func_no);
616 	req.offset = prg->offset;
617 	req.size = size;
618 	req.write = write_flag;
619 	req.ioacc = io_access;
620 	if (write_flag) {
621 		if (big_endian) {
622 			local_data = pcitool_swap_endian(prg->data, size);
623 		} else {
624 			local_data = prg->data;
625 		}
626 		VAL64(&req) = local_data;
627 		pci_cfgacc_acc(&req);
628 	} else {
629 		pci_cfgacc_acc(&req);
630 		switch (size) {
631 		case 1:
632 			local_data = VAL8(&req);
633 			break;
634 		case 2:
635 			local_data = VAL16(&req);
636 			break;
637 		case 4:
638 			local_data = VAL32(&req);
639 			break;
640 		case 8:
641 			local_data = VAL64(&req);
642 			break;
643 		}
644 		if (big_endian) {
645 			prg->data =
646 			    pcitool_swap_endian(local_data, size);
647 		} else {
648 			prg->data = local_data;
649 		}
650 	}
651 	/*
652 	 * Check if legacy IO config access is used, in which case
653 	 * only first 256 bytes are valid.
654 	 */
655 	if (req.ioacc && (prg->offset + size - 1 > 0xFF)) {
656 		prg->status = PCITOOL_INVALID_ADDRESS;
657 		return (ENOTSUP);
658 	}
659 
660 	/* Set phys_addr only if MMIO is used */
661 	prg->phys_addr = 0;
662 	if (!req.ioacc && mcfg_mem_base != 0) {
663 		prg->phys_addr = mcfg_mem_base + prg->offset +
664 		    ((prg->bus_no << PCIEX_REG_BUS_SHIFT) |
665 		    (prg->dev_no << PCIEX_REG_DEV_SHIFT) |
666 		    (prg->func_no << PCIEX_REG_FUNC_SHIFT));
667 	}
668 
669 	return (rval);
670 }
671 
672 static int
673 pcitool_io_access(pcitool_reg_t *prg, boolean_t write_flag)
674 {
675 	int port = (int)prg->phys_addr;
676 	size_t size = PCITOOL_ACC_ATTR_SIZE(prg->acc_attr);
677 	boolean_t big_endian = PCITOOL_ACC_IS_BIG_ENDIAN(prg->acc_attr);
678 	int rval = SUCCESS;
679 	on_trap_data_t otd;
680 	uint64_t local_data;
681 
682 
683 	/*
684 	 * on_trap works like setjmp.
685 	 *
686 	 * A non-zero return here means on_trap has returned from an error.
687 	 *
688 	 * A zero return here means that on_trap has just returned from setup.
689 	 */
690 	if (on_trap(&otd, OT_DATA_ACCESS)) {
691 		no_trap();
692 		if (pcitool_debug)
693 			prom_printf(
694 			    "pcitool_io_access: on_trap caught an error...\n");
695 		prg->status = PCITOOL_INVALID_ADDRESS;
696 		return (EFAULT);
697 	}
698 
699 	if (write_flag) {
700 
701 		if (big_endian) {
702 			local_data = pcitool_swap_endian(prg->data, size);
703 		} else {
704 			local_data = prg->data;
705 		}
706 
707 		if (pcitool_debug)
708 			prom_printf("Writing %ld byte(s) to port 0x%x\n",
709 			    size, port);
710 
711 		switch (size) {
712 		case 1:
713 			outb(port, (uint8_t)local_data);
714 			break;
715 		case 2:
716 			outw(port, (uint16_t)local_data);
717 			break;
718 		case 4:
719 			outl(port, (uint32_t)local_data);
720 			break;
721 		default:
722 			rval = ENOTSUP;
723 			prg->status = PCITOOL_INVALID_SIZE;
724 			break;
725 		}
726 	} else {
727 		if (pcitool_debug)
728 			prom_printf("Reading %ld byte(s) from port 0x%x\n",
729 			    size, port);
730 
731 		switch (size) {
732 		case 1:
733 			local_data = inb(port);
734 			break;
735 		case 2:
736 			local_data = inw(port);
737 			break;
738 		case 4:
739 			local_data = inl(port);
740 			break;
741 		default:
742 			rval = ENOTSUP;
743 			prg->status = PCITOOL_INVALID_SIZE;
744 			break;
745 		}
746 
747 		if (rval == SUCCESS) {
748 			if (big_endian) {
749 				prg->data =
750 				    pcitool_swap_endian(local_data, size);
751 			} else {
752 				prg->data = local_data;
753 			}
754 		}
755 	}
756 
757 	no_trap();
758 	return (rval);
759 }
760 
761 static int
762 pcitool_mem_access(pcitool_reg_t *prg, uint64_t virt_addr, boolean_t write_flag)
763 {
764 	size_t size = PCITOOL_ACC_ATTR_SIZE(prg->acc_attr);
765 	boolean_t big_endian = PCITOOL_ACC_IS_BIG_ENDIAN(prg->acc_attr);
766 	int rval = DDI_SUCCESS;
767 	on_trap_data_t otd;
768 	uint64_t local_data;
769 
770 	/*
771 	 * on_trap works like setjmp.
772 	 *
773 	 * A non-zero return here means on_trap has returned from an error.
774 	 *
775 	 * A zero return here means that on_trap has just returned from setup.
776 	 */
777 	if (on_trap(&otd, OT_DATA_ACCESS)) {
778 		no_trap();
779 		if (pcitool_debug)
780 			prom_printf(
781 			    "pcitool_mem_access: on_trap caught an error...\n");
782 		prg->status = PCITOOL_INVALID_ADDRESS;
783 		return (EFAULT);
784 	}
785 
786 	if (write_flag) {
787 
788 		if (big_endian) {
789 			local_data = pcitool_swap_endian(prg->data, size);
790 		} else {
791 			local_data = prg->data;
792 		}
793 
794 		switch (size) {
795 		case 1:
796 			*((uint8_t *)(uintptr_t)virt_addr) = local_data;
797 			break;
798 		case 2:
799 			*((uint16_t *)(uintptr_t)virt_addr) = local_data;
800 			break;
801 		case 4:
802 			*((uint32_t *)(uintptr_t)virt_addr) = local_data;
803 			break;
804 		case 8:
805 			*((uint64_t *)(uintptr_t)virt_addr) = local_data;
806 			break;
807 		default:
808 			rval = ENOTSUP;
809 			prg->status = PCITOOL_INVALID_SIZE;
810 			break;
811 		}
812 	} else {
813 		switch (size) {
814 		case 1:
815 			local_data = *((uint8_t *)(uintptr_t)virt_addr);
816 			break;
817 		case 2:
818 			local_data = *((uint16_t *)(uintptr_t)virt_addr);
819 			break;
820 		case 4:
821 			local_data = *((uint32_t *)(uintptr_t)virt_addr);
822 			break;
823 		case 8:
824 			local_data = *((uint64_t *)(uintptr_t)virt_addr);
825 			break;
826 		default:
827 			rval = ENOTSUP;
828 			prg->status = PCITOOL_INVALID_SIZE;
829 			break;
830 		}
831 
832 		if (rval == SUCCESS) {
833 			if (big_endian) {
834 				prg->data =
835 				    pcitool_swap_endian(local_data, size);
836 			} else {
837 				prg->data = local_data;
838 			}
839 		}
840 	}
841 
842 	no_trap();
843 	return (rval);
844 }
845 
846 /*
847  * Map up to 2 pages which contain the address we want to access.
848  *
849  * Mapping should span no more than 8 bytes.  With X86 it is possible for an
850  * 8 byte value to start on a 4 byte boundary, so it can cross a page boundary.
851  * We'll never have to map more than two pages.
852  */
853 
854 static uint64_t
855 pcitool_map(uint64_t phys_addr, size_t size, size_t *num_pages)
856 {
857 
858 	uint64_t page_base = phys_addr & ~MMU_PAGEOFFSET;
859 	uint64_t offset = phys_addr & MMU_PAGEOFFSET;
860 	void *virt_base;
861 	uint64_t returned_addr;
862 	pfn_t pfn;
863 
864 	if (pcitool_debug)
865 		prom_printf("pcitool_map: Called with PA:0x%p\n",
866 		    (void *)(uintptr_t)phys_addr);
867 
868 	*num_pages = 1;
869 
870 	/* Desired mapping would span more than two pages. */
871 	if ((offset + size) > (MMU_PAGESIZE * 2)) {
872 		if (pcitool_debug)
873 			prom_printf("boundary violation: "
874 			    "offset:0x%" PRIx64 ", size:%ld, pagesize:0x%lx\n",
875 			    offset, (uintptr_t)size, (uintptr_t)MMU_PAGESIZE);
876 		return (NULL);
877 
878 	} else if ((offset + size) > MMU_PAGESIZE) {
879 		(*num_pages)++;
880 	}
881 
882 	/* Get page(s) of virtual space. */
883 	virt_base = vmem_alloc(heap_arena, ptob(*num_pages), VM_NOSLEEP);
884 	if (virt_base == NULL) {
885 		if (pcitool_debug)
886 			prom_printf("Couldn't get virtual base address.\n");
887 		return (NULL);
888 	}
889 
890 	if (pcitool_debug)
891 		prom_printf("Got base virtual address:0x%p\n", virt_base);
892 
893 #ifdef __xpv
894 	/*
895 	 * We should only get here if we are dom0.
896 	 * We're using a real device so we need to translate the MA to a PFN.
897 	 */
898 	ASSERT(DOMAIN_IS_INITDOMAIN(xen_info));
899 	pfn = xen_assign_pfn(mmu_btop(page_base));
900 #else
901 	pfn = btop(page_base);
902 #endif
903 
904 	/* Now map the allocated virtual space to the physical address. */
905 	hat_devload(kas.a_hat, virt_base, mmu_ptob(*num_pages), pfn,
906 	    PROT_READ | PROT_WRITE | HAT_STRICTORDER,
907 	    HAT_LOAD_LOCK);
908 
909 	returned_addr = ((uintptr_t)(virt_base)) + offset;
910 
911 	if (pcitool_debug)
912 		prom_printf("pcitool_map: returning VA:0x%p\n",
913 		    (void *)(uintptr_t)returned_addr);
914 
915 	return (returned_addr);
916 }
917 
918 /* Unmap the mapped page(s). */
919 static void
920 pcitool_unmap(uint64_t virt_addr, size_t num_pages)
921 {
922 	void *base_virt_addr = (void *)(uintptr_t)(virt_addr & ~MMU_PAGEOFFSET);
923 
924 	hat_unload(kas.a_hat, base_virt_addr, ptob(num_pages),
925 	    HAT_UNLOAD_UNLOCK);
926 	vmem_free(heap_arena, base_virt_addr, ptob(num_pages));
927 }
928 
929 
930 /* Perform register accesses on PCI leaf devices. */
931 /*ARGSUSED*/
932 int
933 pcitool_dev_reg_ops(dev_info_t *dip, void *arg, int cmd, int mode)
934 {
935 	boolean_t	write_flag = B_FALSE;
936 	boolean_t	io_access = B_TRUE;
937 	int		rval = 0;
938 	pcitool_reg_t	prg;
939 	uint8_t		size;
940 
941 	uint64_t	base_addr;
942 	uint64_t	virt_addr;
943 	size_t		num_virt_pages;
944 
945 	switch (cmd) {
946 	case (PCITOOL_DEVICE_SET_REG):
947 		write_flag = B_TRUE;
948 
949 	/*FALLTHRU*/
950 	case (PCITOOL_DEVICE_GET_REG):
951 		if (pcitool_debug)
952 			prom_printf("pci_dev_reg_ops set/get reg\n");
953 		if (ddi_copyin(arg, &prg, sizeof (pcitool_reg_t), mode) !=
954 		    DDI_SUCCESS) {
955 			if (pcitool_debug)
956 				prom_printf("Error reading arguments\n");
957 			return (EFAULT);
958 		}
959 
960 		if (prg.barnum >= (sizeof (pci_bars) / sizeof (pci_bars[0]))) {
961 			prg.status = PCITOOL_OUT_OF_RANGE;
962 			rval = EINVAL;
963 			goto done_reg;
964 		}
965 
966 		if (pcitool_debug)
967 			prom_printf("raw bus:0x%x, dev:0x%x, func:0x%x\n",
968 			    prg.bus_no, prg.dev_no, prg.func_no);
969 		/* Validate address arguments of bus / dev / func */
970 		if (((prg.bus_no &
971 		    (PCI_REG_BUS_M >> PCI_REG_BUS_SHIFT)) !=
972 		    prg.bus_no) ||
973 		    ((prg.dev_no &
974 		    (PCI_REG_DEV_M >> PCI_REG_DEV_SHIFT)) !=
975 		    prg.dev_no) ||
976 		    ((prg.func_no &
977 		    (PCI_REG_FUNC_M >> PCI_REG_FUNC_SHIFT)) !=
978 		    prg.func_no)) {
979 			prg.status = PCITOOL_INVALID_ADDRESS;
980 			rval = EINVAL;
981 			goto done_reg;
982 		}
983 
984 		size = PCITOOL_ACC_ATTR_SIZE(prg.acc_attr);
985 
986 		/* Proper config space desired. */
987 		if (prg.barnum == 0) {
988 
989 			if (pcitool_debug)
990 				prom_printf(
991 				    "config access: offset:0x%" PRIx64 ", "
992 				    "phys_addr:0x%" PRIx64 "\n",
993 				    prg.offset, prg.phys_addr);
994 
995 			if (prg.offset >= max_cfg_size) {
996 				prg.status = PCITOOL_OUT_OF_RANGE;
997 				rval = EINVAL;
998 				goto done_reg;
999 			}
1000 			if (max_cfg_size == PCIE_CONF_HDR_SIZE)
1001 				io_access = B_FALSE;
1002 
1003 			rval = pcitool_cfg_access(&prg, write_flag, io_access);
1004 			if (pcitool_debug)
1005 				prom_printf(
1006 				    "config access: data:0x%" PRIx64 "\n",
1007 				    prg.data);
1008 
1009 		/* IO/ MEM/ MEM64 space. */
1010 		} else {
1011 
1012 			pcitool_reg_t	prg2;
1013 			bcopy(&prg, &prg2, sizeof (pcitool_reg_t));
1014 
1015 			/*
1016 			 * Translate BAR number into offset of the BAR in
1017 			 * the device's config space.
1018 			 */
1019 			prg2.offset = pci_bars[prg2.barnum];
1020 			prg2.acc_attr =
1021 			    PCITOOL_ACC_ATTR_SIZE_4 | PCITOOL_ACC_ATTR_ENDN_LTL;
1022 
1023 			if (pcitool_debug)
1024 				prom_printf(
1025 				    "barnum:%d, bar_offset:0x%" PRIx64 "\n",
1026 				    prg2.barnum, prg2.offset);
1027 			/*
1028 			 * Get Bus Address Register (BAR) from config space.
1029 			 * prg2.offset is the offset into config space of the
1030 			 * BAR desired.  prg.status is modified on error.
1031 			 */
1032 			rval = pcitool_cfg_access(&prg2, B_FALSE, B_TRUE);
1033 			if (rval != SUCCESS) {
1034 				if (pcitool_debug)
1035 					prom_printf("BAR access failed\n");
1036 				prg.status = prg2.status;
1037 				goto done_reg;
1038 			}
1039 			/*
1040 			 * Reference proper PCI space based on the BAR.
1041 			 * If 64 bit MEM space, need to load other half of the
1042 			 * BAR first.
1043 			 */
1044 
1045 			if (pcitool_debug)
1046 				prom_printf("bar returned is 0x%" PRIx64 "\n",
1047 				    prg2.data);
1048 			if (!prg2.data) {
1049 				if (pcitool_debug)
1050 					prom_printf("BAR data == 0\n");
1051 				rval = EINVAL;
1052 				prg.status = PCITOOL_INVALID_ADDRESS;
1053 				goto done_reg;
1054 			}
1055 			if (prg2.data == 0xffffffff) {
1056 				if (pcitool_debug)
1057 					prom_printf("BAR data == -1\n");
1058 				rval = EINVAL;
1059 				prg.status = PCITOOL_INVALID_ADDRESS;
1060 				goto done_reg;
1061 			}
1062 
1063 			/*
1064 			 * BAR has bits saying this space is IO space, unless
1065 			 * this is the ROM address register.
1066 			 */
1067 			if (((PCI_BASE_SPACE_M & prg2.data) ==
1068 			    PCI_BASE_SPACE_IO) &&
1069 			    (prg2.offset != PCI_CONF_ROM)) {
1070 				if (pcitool_debug)
1071 					prom_printf("IO space\n");
1072 
1073 				prg2.data &= PCI_BASE_IO_ADDR_M;
1074 				prg.phys_addr = prg2.data + prg.offset;
1075 
1076 				rval = pcitool_io_access(&prg, write_flag);
1077 				if ((rval != SUCCESS) && (pcitool_debug))
1078 					prom_printf("IO access failed\n");
1079 
1080 				goto done_reg;
1081 
1082 
1083 			/*
1084 			 * BAR has bits saying this space is 64 bit memory
1085 			 * space, unless this is the ROM address register.
1086 			 *
1087 			 * The 64 bit address stored in two BAR cells is not
1088 			 * necessarily aligned on an 8-byte boundary.
1089 			 * Need to keep the first 4 bytes read,
1090 			 * and do a separate read of the high 4 bytes.
1091 			 */
1092 
1093 			} else if ((PCI_BASE_TYPE_ALL & prg2.data) &&
1094 			    (prg2.offset != PCI_CONF_ROM)) {
1095 
1096 				uint32_t low_bytes =
1097 				    (uint32_t)(prg2.data & ~PCI_BASE_TYPE_ALL);
1098 
1099 				/*
1100 				 * Don't try to read the next 4 bytes
1101 				 * past the end of BARs.
1102 				 */
1103 				if (prg2.offset >= PCI_CONF_BASE5) {
1104 					prg.status = PCITOOL_OUT_OF_RANGE;
1105 					rval = EIO;
1106 					goto done_reg;
1107 				}
1108 
1109 				/*
1110 				 * Access device.
1111 				 * prg2.status is modified on error.
1112 				 */
1113 				prg2.offset += 4;
1114 				rval = pcitool_cfg_access(&prg2,
1115 				    B_FALSE, B_TRUE);
1116 				if (rval != SUCCESS) {
1117 					prg.status = prg2.status;
1118 					goto done_reg;
1119 				}
1120 
1121 				if (prg2.data == 0xffffffff) {
1122 					prg.status = PCITOOL_INVALID_ADDRESS;
1123 					prg.status = EFAULT;
1124 					goto done_reg;
1125 				}
1126 
1127 				prg2.data = (prg2.data << 32) + low_bytes;
1128 				if (pcitool_debug)
1129 					prom_printf(
1130 					    "64 bit mem space.  "
1131 					    "64-bit bar is 0x%" PRIx64 "\n",
1132 					    prg2.data);
1133 
1134 			/* Mem32 space, including ROM */
1135 			} else {
1136 
1137 				if (prg2.offset == PCI_CONF_ROM) {
1138 					if (pcitool_debug)
1139 						prom_printf(
1140 						    "Additional ROM "
1141 						    "checking\n");
1142 					/* Can't write to ROM */
1143 					if (write_flag) {
1144 						prg.status = PCITOOL_ROM_WRITE;
1145 						rval = EIO;
1146 						goto done_reg;
1147 
1148 					/* ROM disabled for reading */
1149 					} else if (!(prg2.data & 0x00000001)) {
1150 						prg.status =
1151 						    PCITOOL_ROM_DISABLED;
1152 						rval = EIO;
1153 						goto done_reg;
1154 					}
1155 				}
1156 
1157 				if (pcitool_debug)
1158 					prom_printf("32 bit mem space\n");
1159 			}
1160 
1161 			/* Common code for all IO/MEM range spaces. */
1162 
1163 			base_addr = prg2.data;
1164 			if (pcitool_debug)
1165 				prom_printf(
1166 				    "addr portion of bar is 0x%" PRIx64 ", "
1167 				    "base=0x%" PRIx64 ", "
1168 				    "offset:0x%" PRIx64 "\n",
1169 				    prg2.data, base_addr, prg.offset);
1170 			/*
1171 			 * Use offset provided by caller to index into
1172 			 * desired space, then access.
1173 			 * Note that prg.status is modified on error.
1174 			 */
1175 			prg.phys_addr = base_addr + prg.offset;
1176 
1177 			virt_addr = pcitool_map(prg.phys_addr, size,
1178 			    &num_virt_pages);
1179 			if (virt_addr == NULL) {
1180 				prg.status = PCITOOL_IO_ERROR;
1181 				rval = EIO;
1182 				goto done_reg;
1183 			}
1184 
1185 			rval = pcitool_mem_access(&prg, virt_addr, write_flag);
1186 			pcitool_unmap(virt_addr, num_virt_pages);
1187 		}
1188 done_reg:
1189 		prg.drvr_version = PCITOOL_VERSION;
1190 		if (ddi_copyout(&prg, arg, sizeof (pcitool_reg_t), mode) !=
1191 		    DDI_SUCCESS) {
1192 			if (pcitool_debug)
1193 				prom_printf("Error returning arguments.\n");
1194 			rval = EFAULT;
1195 		}
1196 		break;
1197 	default:
1198 		rval = ENOTTY;
1199 		break;
1200 	}
1201 	return (rval);
1202 }
1203