xref: /titanic_51/usr/src/uts/i86pc/io/rootnex.c (revision b6917abefc343244b784f0cc34bc65b01469c3bf)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 /*
22  * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
23  * Use is subject to license terms.
24  */
25 
26 #pragma ident	"%Z%%M%	%I%	%E% SMI"
27 
28 /*
29  * x86 root nexus driver
30  */
31 
32 #include <sys/sysmacros.h>
33 #include <sys/conf.h>
34 #include <sys/autoconf.h>
35 #include <sys/sysmacros.h>
36 #include <sys/debug.h>
37 #include <sys/psw.h>
38 #include <sys/ddidmareq.h>
39 #include <sys/promif.h>
40 #include <sys/devops.h>
41 #include <sys/kmem.h>
42 #include <sys/cmn_err.h>
43 #include <vm/seg.h>
44 #include <vm/seg_kmem.h>
45 #include <vm/seg_dev.h>
46 #include <sys/vmem.h>
47 #include <sys/mman.h>
48 #include <vm/hat.h>
49 #include <vm/as.h>
50 #include <vm/page.h>
51 #include <sys/avintr.h>
52 #include <sys/errno.h>
53 #include <sys/modctl.h>
54 #include <sys/ddi_impldefs.h>
55 #include <sys/sunddi.h>
56 #include <sys/sunndi.h>
57 #include <sys/mach_intr.h>
58 #include <sys/psm.h>
59 #include <sys/ontrap.h>
60 #include <sys/atomic.h>
61 #include <sys/sdt.h>
62 #include <sys/rootnex.h>
63 #include <vm/hat_i86.h>
64 #include <sys/ddifm.h>
65 #include <sys/ddi_isa.h>
66 
67 #ifdef __xpv
68 #include <sys/bootinfo.h>
69 #include <sys/hypervisor.h>
70 #include <sys/bootconf.h>
71 #include <vm/kboot_mmu.h>
72 #endif
73 
74 /*
75  * enable/disable extra checking of function parameters. Useful for debugging
76  * drivers.
77  */
78 #ifdef	DEBUG
79 int rootnex_alloc_check_parms = 1;
80 int rootnex_bind_check_parms = 1;
81 int rootnex_bind_check_inuse = 1;
82 int rootnex_unbind_verify_buffer = 0;
83 int rootnex_sync_check_parms = 1;
84 #else
85 int rootnex_alloc_check_parms = 0;
86 int rootnex_bind_check_parms = 0;
87 int rootnex_bind_check_inuse = 0;
88 int rootnex_unbind_verify_buffer = 0;
89 int rootnex_sync_check_parms = 0;
90 #endif
91 
92 /* Master Abort and Target Abort panic flag */
93 int rootnex_fm_ma_ta_panic_flag = 0;
94 
95 /* Semi-temporary patchables to phase in bug fixes, test drivers, etc. */
96 int rootnex_bind_fail = 1;
97 int rootnex_bind_warn = 1;
98 uint8_t *rootnex_warn_list;
99 /* bitmasks for rootnex_warn_list. Up to 8 different warnings with uint8_t */
100 #define	ROOTNEX_BIND_WARNING	(0x1 << 0)
101 
102 /*
103  * revert back to old broken behavior of always sync'ing entire copy buffer.
104  * This is useful if be have a buggy driver which doesn't correctly pass in
105  * the offset and size into ddi_dma_sync().
106  */
107 int rootnex_sync_ignore_params = 0;
108 
109 /*
110  * For the 64-bit kernel, pre-alloc enough cookies for a 256K buffer plus 1
111  * page for alignment. For the 32-bit kernel, pre-alloc enough cookies for a
112  * 64K buffer plus 1 page for alignment (we have less kernel space in a 32-bit
113  * kernel). Allocate enough windows to handle a 256K buffer w/ at least 65
114  * sgllen DMA engine, and enough copybuf buffer state pages to handle 2 pages
115  * (< 8K). We will still need to allocate the copy buffer during bind though
116  * (if we need one). These can only be modified in /etc/system before rootnex
117  * attach.
118  */
119 #if defined(__amd64)
120 int rootnex_prealloc_cookies = 65;
121 int rootnex_prealloc_windows = 4;
122 int rootnex_prealloc_copybuf = 2;
123 #else
124 int rootnex_prealloc_cookies = 33;
125 int rootnex_prealloc_windows = 4;
126 int rootnex_prealloc_copybuf = 2;
127 #endif
128 
129 /* driver global state */
130 static rootnex_state_t *rootnex_state;
131 
132 /* shortcut to rootnex counters */
133 static uint64_t *rootnex_cnt;
134 
135 /*
136  * XXX - does x86 even need these or are they left over from the SPARC days?
137  */
138 /* statically defined integer/boolean properties for the root node */
139 static rootnex_intprop_t rootnex_intprp[] = {
140 	{ "PAGESIZE",			PAGESIZE },
141 	{ "MMU_PAGESIZE",		MMU_PAGESIZE },
142 	{ "MMU_PAGEOFFSET",		MMU_PAGEOFFSET },
143 	{ DDI_RELATIVE_ADDRESSING,	1 },
144 };
145 #define	NROOT_INTPROPS	(sizeof (rootnex_intprp) / sizeof (rootnex_intprop_t))
146 
147 #ifdef __xpv
148 typedef maddr_t rootnex_addr_t;
149 #define	ROOTNEX_PADDR_TO_RBASE(xinfo, pa)	\
150 	(DOMAIN_IS_INITDOMAIN(xinfo) ? pa_to_ma(pa) : (pa))
151 #else
152 typedef paddr_t rootnex_addr_t;
153 #endif
154 
155 
156 static struct cb_ops rootnex_cb_ops = {
157 	nodev,		/* open */
158 	nodev,		/* close */
159 	nodev,		/* strategy */
160 	nodev,		/* print */
161 	nodev,		/* dump */
162 	nodev,		/* read */
163 	nodev,		/* write */
164 	nodev,		/* ioctl */
165 	nodev,		/* devmap */
166 	nodev,		/* mmap */
167 	nodev,		/* segmap */
168 	nochpoll,	/* chpoll */
169 	ddi_prop_op,	/* cb_prop_op */
170 	NULL,		/* struct streamtab */
171 	D_NEW | D_MP | D_HOTPLUG, /* compatibility flags */
172 	CB_REV,		/* Rev */
173 	nodev,		/* cb_aread */
174 	nodev		/* cb_awrite */
175 };
176 
177 static int rootnex_map(dev_info_t *dip, dev_info_t *rdip, ddi_map_req_t *mp,
178     off_t offset, off_t len, caddr_t *vaddrp);
179 static int rootnex_map_fault(dev_info_t *dip, dev_info_t *rdip,
180     struct hat *hat, struct seg *seg, caddr_t addr,
181     struct devpage *dp, pfn_t pfn, uint_t prot, uint_t lock);
182 static int rootnex_dma_map(dev_info_t *dip, dev_info_t *rdip,
183     struct ddi_dma_req *dmareq, ddi_dma_handle_t *handlep);
184 static int rootnex_dma_allochdl(dev_info_t *dip, dev_info_t *rdip,
185     ddi_dma_attr_t *attr, int (*waitfp)(caddr_t), caddr_t arg,
186     ddi_dma_handle_t *handlep);
187 static int rootnex_dma_freehdl(dev_info_t *dip, dev_info_t *rdip,
188     ddi_dma_handle_t handle);
189 static int rootnex_dma_bindhdl(dev_info_t *dip, dev_info_t *rdip,
190     ddi_dma_handle_t handle, struct ddi_dma_req *dmareq,
191     ddi_dma_cookie_t *cookiep, uint_t *ccountp);
192 static int rootnex_dma_unbindhdl(dev_info_t *dip, dev_info_t *rdip,
193     ddi_dma_handle_t handle);
194 static int rootnex_dma_sync(dev_info_t *dip, dev_info_t *rdip,
195     ddi_dma_handle_t handle, off_t off, size_t len, uint_t cache_flags);
196 static int rootnex_dma_win(dev_info_t *dip, dev_info_t *rdip,
197     ddi_dma_handle_t handle, uint_t win, off_t *offp, size_t *lenp,
198     ddi_dma_cookie_t *cookiep, uint_t *ccountp);
199 static int rootnex_dma_mctl(dev_info_t *dip, dev_info_t *rdip,
200     ddi_dma_handle_t handle, enum ddi_dma_ctlops request,
201     off_t *offp, size_t *lenp, caddr_t *objp, uint_t cache_flags);
202 static int rootnex_ctlops(dev_info_t *dip, dev_info_t *rdip,
203     ddi_ctl_enum_t ctlop, void *arg, void *result);
204 static int rootnex_fm_init(dev_info_t *dip, dev_info_t *tdip, int tcap,
205     ddi_iblock_cookie_t *ibc);
206 static int rootnex_intr_ops(dev_info_t *pdip, dev_info_t *rdip,
207     ddi_intr_op_t intr_op, ddi_intr_handle_impl_t *hdlp, void *result);
208 
209 
210 static struct bus_ops rootnex_bus_ops = {
211 	BUSO_REV,
212 	rootnex_map,
213 	NULL,
214 	NULL,
215 	NULL,
216 	rootnex_map_fault,
217 	rootnex_dma_map,
218 	rootnex_dma_allochdl,
219 	rootnex_dma_freehdl,
220 	rootnex_dma_bindhdl,
221 	rootnex_dma_unbindhdl,
222 	rootnex_dma_sync,
223 	rootnex_dma_win,
224 	rootnex_dma_mctl,
225 	rootnex_ctlops,
226 	ddi_bus_prop_op,
227 	i_ddi_rootnex_get_eventcookie,
228 	i_ddi_rootnex_add_eventcall,
229 	i_ddi_rootnex_remove_eventcall,
230 	i_ddi_rootnex_post_event,
231 	0,			/* bus_intr_ctl */
232 	0,			/* bus_config */
233 	0,			/* bus_unconfig */
234 	rootnex_fm_init,	/* bus_fm_init */
235 	NULL,			/* bus_fm_fini */
236 	NULL,			/* bus_fm_access_enter */
237 	NULL,			/* bus_fm_access_exit */
238 	NULL,			/* bus_powr */
239 	rootnex_intr_ops	/* bus_intr_op */
240 };
241 
242 static int rootnex_attach(dev_info_t *dip, ddi_attach_cmd_t cmd);
243 static int rootnex_detach(dev_info_t *dip, ddi_detach_cmd_t cmd);
244 
245 static struct dev_ops rootnex_ops = {
246 	DEVO_REV,
247 	0,
248 	ddi_no_info,
249 	nulldev,
250 	nulldev,
251 	rootnex_attach,
252 	rootnex_detach,
253 	nulldev,
254 	&rootnex_cb_ops,
255 	&rootnex_bus_ops
256 };
257 
258 static struct modldrv rootnex_modldrv = {
259 	&mod_driverops,
260 	"i86pc root nexus %I%",
261 	&rootnex_ops
262 };
263 
264 static struct modlinkage rootnex_modlinkage = {
265 	MODREV_1,
266 	(void *)&rootnex_modldrv,
267 	NULL
268 };
269 
270 
271 /*
272  *  extern hacks
273  */
274 extern struct seg_ops segdev_ops;
275 extern int ignore_hardware_nodes;	/* force flag from ddi_impl.c */
276 #ifdef	DDI_MAP_DEBUG
277 extern int ddi_map_debug_flag;
278 #define	ddi_map_debug	if (ddi_map_debug_flag) prom_printf
279 #endif
280 extern void i86_pp_map(page_t *pp, caddr_t kaddr);
281 extern void i86_va_map(caddr_t vaddr, struct as *asp, caddr_t kaddr);
282 extern int (*psm_intr_ops)(dev_info_t *, ddi_intr_handle_impl_t *,
283     psm_intr_op_t, int *);
284 extern int impl_ddi_sunbus_initchild(dev_info_t *dip);
285 extern void impl_ddi_sunbus_removechild(dev_info_t *dip);
286 
287 /*
288  * Use device arena to use for device control register mappings.
289  * Various kernel memory walkers (debugger, dtrace) need to know
290  * to avoid this address range to prevent undesired device activity.
291  */
292 extern void *device_arena_alloc(size_t size, int vm_flag);
293 extern void device_arena_free(void * vaddr, size_t size);
294 
295 
296 /*
297  *  Internal functions
298  */
299 static int rootnex_dma_init();
300 static void rootnex_add_props(dev_info_t *);
301 static int rootnex_ctl_reportdev(dev_info_t *dip);
302 static struct intrspec *rootnex_get_ispec(dev_info_t *rdip, int inum);
303 static int rootnex_map_regspec(ddi_map_req_t *mp, caddr_t *vaddrp);
304 static int rootnex_unmap_regspec(ddi_map_req_t *mp, caddr_t *vaddrp);
305 static int rootnex_map_handle(ddi_map_req_t *mp);
306 static void rootnex_clean_dmahdl(ddi_dma_impl_t *hp);
307 static int rootnex_valid_alloc_parms(ddi_dma_attr_t *attr, uint_t maxsegsize);
308 static int rootnex_valid_bind_parms(ddi_dma_req_t *dmareq,
309     ddi_dma_attr_t *attr);
310 static void rootnex_get_sgl(ddi_dma_obj_t *dmar_object, ddi_dma_cookie_t *sgl,
311     rootnex_sglinfo_t *sglinfo);
312 static int rootnex_bind_slowpath(ddi_dma_impl_t *hp, struct ddi_dma_req *dmareq,
313     rootnex_dma_t *dma, ddi_dma_attr_t *attr, int kmflag);
314 static int rootnex_setup_copybuf(ddi_dma_impl_t *hp, struct ddi_dma_req *dmareq,
315     rootnex_dma_t *dma, ddi_dma_attr_t *attr);
316 static void rootnex_teardown_copybuf(rootnex_dma_t *dma);
317 static int rootnex_setup_windows(ddi_dma_impl_t *hp, rootnex_dma_t *dma,
318     ddi_dma_attr_t *attr, int kmflag);
319 static void rootnex_teardown_windows(rootnex_dma_t *dma);
320 static void rootnex_init_win(ddi_dma_impl_t *hp, rootnex_dma_t *dma,
321     rootnex_window_t *window, ddi_dma_cookie_t *cookie, off_t cur_offset);
322 static void rootnex_setup_cookie(ddi_dma_obj_t *dmar_object,
323     rootnex_dma_t *dma, ddi_dma_cookie_t *cookie, off_t cur_offset,
324     size_t *copybuf_used, page_t **cur_pp);
325 static int rootnex_sgllen_window_boundary(ddi_dma_impl_t *hp,
326     rootnex_dma_t *dma, rootnex_window_t **windowp, ddi_dma_cookie_t *cookie,
327     ddi_dma_attr_t *attr, off_t cur_offset);
328 static int rootnex_copybuf_window_boundary(ddi_dma_impl_t *hp,
329     rootnex_dma_t *dma, rootnex_window_t **windowp,
330     ddi_dma_cookie_t *cookie, off_t cur_offset, size_t *copybuf_used);
331 static int rootnex_maxxfer_window_boundary(ddi_dma_impl_t *hp,
332     rootnex_dma_t *dma, rootnex_window_t **windowp, ddi_dma_cookie_t *cookie);
333 static int rootnex_valid_sync_parms(ddi_dma_impl_t *hp, rootnex_window_t *win,
334     off_t offset, size_t size, uint_t cache_flags);
335 static int rootnex_verify_buffer(rootnex_dma_t *dma);
336 static int rootnex_dma_check(dev_info_t *dip, const void *handle,
337     const void *comp_addr, const void *not_used);
338 
339 /*
340  * _init()
341  *
342  */
343 int
344 _init(void)
345 {
346 
347 	rootnex_state = NULL;
348 	return (mod_install(&rootnex_modlinkage));
349 }
350 
351 
352 /*
353  * _info()
354  *
355  */
356 int
357 _info(struct modinfo *modinfop)
358 {
359 	return (mod_info(&rootnex_modlinkage, modinfop));
360 }
361 
362 
363 /*
364  * _fini()
365  *
366  */
367 int
368 _fini(void)
369 {
370 	return (EBUSY);
371 }
372 
373 
374 /*
375  * rootnex_attach()
376  *
377  */
378 static int
379 rootnex_attach(dev_info_t *dip, ddi_attach_cmd_t cmd)
380 {
381 	int fmcap;
382 	int e;
383 
384 	switch (cmd) {
385 	case DDI_ATTACH:
386 		break;
387 	case DDI_RESUME:
388 		return (DDI_SUCCESS);
389 	default:
390 		return (DDI_FAILURE);
391 	}
392 
393 	/*
394 	 * We should only have one instance of rootnex. Save it away since we
395 	 * don't have an easy way to get it back later.
396 	 */
397 	ASSERT(rootnex_state == NULL);
398 	rootnex_state = kmem_zalloc(sizeof (rootnex_state_t), KM_SLEEP);
399 
400 	rootnex_state->r_dip = dip;
401 	rootnex_state->r_err_ibc = (ddi_iblock_cookie_t)ipltospl(15);
402 	rootnex_state->r_reserved_msg_printed = B_FALSE;
403 	rootnex_cnt = &rootnex_state->r_counters[0];
404 
405 	/*
406 	 * Set minimum fm capability level for i86pc platforms and then
407 	 * initialize error handling. Since we're the rootnex, we don't
408 	 * care what's returned in the fmcap field.
409 	 */
410 	ddi_system_fmcap = DDI_FM_EREPORT_CAPABLE | DDI_FM_ERRCB_CAPABLE |
411 	    DDI_FM_ACCCHK_CAPABLE | DDI_FM_DMACHK_CAPABLE;
412 	fmcap = ddi_system_fmcap;
413 	ddi_fm_init(dip, &fmcap, &rootnex_state->r_err_ibc);
414 
415 	/* initialize DMA related state */
416 	e = rootnex_dma_init();
417 	if (e != DDI_SUCCESS) {
418 		kmem_free(rootnex_state, sizeof (rootnex_state_t));
419 		return (DDI_FAILURE);
420 	}
421 
422 	/* Add static root node properties */
423 	rootnex_add_props(dip);
424 
425 	/* since we can't call ddi_report_dev() */
426 	cmn_err(CE_CONT, "?root nexus = %s\n", ddi_get_name(dip));
427 
428 	/* Initialize rootnex event handle */
429 	i_ddi_rootnex_init_events(dip);
430 
431 	return (DDI_SUCCESS);
432 }
433 
434 
435 /*
436  * rootnex_detach()
437  *
438  */
439 /*ARGSUSED*/
440 static int
441 rootnex_detach(dev_info_t *dip, ddi_detach_cmd_t cmd)
442 {
443 	switch (cmd) {
444 	case DDI_SUSPEND:
445 		break;
446 	default:
447 		return (DDI_FAILURE);
448 	}
449 
450 	return (DDI_SUCCESS);
451 }
452 
453 
454 /*
455  * rootnex_dma_init()
456  *
457  */
458 /*ARGSUSED*/
459 static int
460 rootnex_dma_init()
461 {
462 	size_t bufsize;
463 
464 
465 	/*
466 	 * size of our cookie/window/copybuf state needed in dma bind that we
467 	 * pre-alloc in dma_alloc_handle
468 	 */
469 	rootnex_state->r_prealloc_cookies = rootnex_prealloc_cookies;
470 	rootnex_state->r_prealloc_size =
471 	    (rootnex_state->r_prealloc_cookies * sizeof (ddi_dma_cookie_t)) +
472 	    (rootnex_prealloc_windows * sizeof (rootnex_window_t)) +
473 	    (rootnex_prealloc_copybuf * sizeof (rootnex_pgmap_t));
474 
475 	/*
476 	 * setup DDI DMA handle kmem cache, align each handle on 64 bytes,
477 	 * allocate 16 extra bytes for struct pointer alignment
478 	 * (p->dmai_private & dma->dp_prealloc_buffer)
479 	 */
480 	bufsize = sizeof (ddi_dma_impl_t) + sizeof (rootnex_dma_t) +
481 	    rootnex_state->r_prealloc_size + 0x10;
482 	rootnex_state->r_dmahdl_cache = kmem_cache_create("rootnex_dmahdl",
483 	    bufsize, 64, NULL, NULL, NULL, NULL, NULL, 0);
484 	if (rootnex_state->r_dmahdl_cache == NULL) {
485 		return (DDI_FAILURE);
486 	}
487 
488 	/*
489 	 * allocate array to track which major numbers we have printed warnings
490 	 * for.
491 	 */
492 	rootnex_warn_list = kmem_zalloc(devcnt * sizeof (*rootnex_warn_list),
493 	    KM_SLEEP);
494 
495 	return (DDI_SUCCESS);
496 }
497 
498 
499 /*
500  * rootnex_add_props()
501  *
502  */
503 static void
504 rootnex_add_props(dev_info_t *dip)
505 {
506 	rootnex_intprop_t *rpp;
507 	int i;
508 
509 	/* Add static integer/boolean properties to the root node */
510 	rpp = rootnex_intprp;
511 	for (i = 0; i < NROOT_INTPROPS; i++) {
512 		(void) e_ddi_prop_update_int(DDI_DEV_T_NONE, dip,
513 		    rpp[i].prop_name, rpp[i].prop_value);
514 	}
515 }
516 
517 
518 
519 /*
520  * *************************
521  *  ctlops related routines
522  * *************************
523  */
524 
525 /*
526  * rootnex_ctlops()
527  *
528  */
529 /*ARGSUSED*/
530 static int
531 rootnex_ctlops(dev_info_t *dip, dev_info_t *rdip, ddi_ctl_enum_t ctlop,
532     void *arg, void *result)
533 {
534 	int n, *ptr;
535 	struct ddi_parent_private_data *pdp;
536 
537 	switch (ctlop) {
538 	case DDI_CTLOPS_DMAPMAPC:
539 		/*
540 		 * Return 'partial' to indicate that dma mapping
541 		 * has to be done in the main MMU.
542 		 */
543 		return (DDI_DMA_PARTIAL);
544 
545 	case DDI_CTLOPS_BTOP:
546 		/*
547 		 * Convert byte count input to physical page units.
548 		 * (byte counts that are not a page-size multiple
549 		 * are rounded down)
550 		 */
551 		*(ulong_t *)result = btop(*(ulong_t *)arg);
552 		return (DDI_SUCCESS);
553 
554 	case DDI_CTLOPS_PTOB:
555 		/*
556 		 * Convert size in physical pages to bytes
557 		 */
558 		*(ulong_t *)result = ptob(*(ulong_t *)arg);
559 		return (DDI_SUCCESS);
560 
561 	case DDI_CTLOPS_BTOPR:
562 		/*
563 		 * Convert byte count input to physical page units
564 		 * (byte counts that are not a page-size multiple
565 		 * are rounded up)
566 		 */
567 		*(ulong_t *)result = btopr(*(ulong_t *)arg);
568 		return (DDI_SUCCESS);
569 
570 	case DDI_CTLOPS_INITCHILD:
571 		return (impl_ddi_sunbus_initchild(arg));
572 
573 	case DDI_CTLOPS_UNINITCHILD:
574 		impl_ddi_sunbus_removechild(arg);
575 		return (DDI_SUCCESS);
576 
577 	case DDI_CTLOPS_REPORTDEV:
578 		return (rootnex_ctl_reportdev(rdip));
579 
580 	case DDI_CTLOPS_IOMIN:
581 		/*
582 		 * Nothing to do here but reflect back..
583 		 */
584 		return (DDI_SUCCESS);
585 
586 	case DDI_CTLOPS_REGSIZE:
587 	case DDI_CTLOPS_NREGS:
588 		break;
589 
590 	case DDI_CTLOPS_SIDDEV:
591 		if (ndi_dev_is_prom_node(rdip))
592 			return (DDI_SUCCESS);
593 		if (ndi_dev_is_persistent_node(rdip))
594 			return (DDI_SUCCESS);
595 		return (DDI_FAILURE);
596 
597 	case DDI_CTLOPS_POWER:
598 		return ((*pm_platform_power)((power_req_t *)arg));
599 
600 	case DDI_CTLOPS_RESERVED0: /* Was DDI_CTLOPS_NINTRS, obsolete */
601 	case DDI_CTLOPS_RESERVED1: /* Was DDI_CTLOPS_POKE_INIT, obsolete */
602 	case DDI_CTLOPS_RESERVED2: /* Was DDI_CTLOPS_POKE_FLUSH, obsolete */
603 	case DDI_CTLOPS_RESERVED3: /* Was DDI_CTLOPS_POKE_FINI, obsolete */
604 	case DDI_CTLOPS_RESERVED4: /* Was DDI_CTLOPS_INTR_HILEVEL, obsolete */
605 	case DDI_CTLOPS_RESERVED5: /* Was DDI_CTLOPS_XLATE_INTRS, obsolete */
606 		if (!rootnex_state->r_reserved_msg_printed) {
607 			rootnex_state->r_reserved_msg_printed = B_TRUE;
608 			cmn_err(CE_WARN, "Failing ddi_ctlops call(s) for "
609 			    "1 or more reserved/obsolete operations.");
610 		}
611 		return (DDI_FAILURE);
612 
613 	default:
614 		return (DDI_FAILURE);
615 	}
616 	/*
617 	 * The rest are for "hardware" properties
618 	 */
619 	if ((pdp = ddi_get_parent_data(rdip)) == NULL)
620 		return (DDI_FAILURE);
621 
622 	if (ctlop == DDI_CTLOPS_NREGS) {
623 		ptr = (int *)result;
624 		*ptr = pdp->par_nreg;
625 	} else {
626 		off_t *size = (off_t *)result;
627 
628 		ptr = (int *)arg;
629 		n = *ptr;
630 		if (n >= pdp->par_nreg) {
631 			return (DDI_FAILURE);
632 		}
633 		*size = (off_t)pdp->par_reg[n].regspec_size;
634 	}
635 	return (DDI_SUCCESS);
636 }
637 
638 
639 /*
640  * rootnex_ctl_reportdev()
641  *
642  */
643 static int
644 rootnex_ctl_reportdev(dev_info_t *dev)
645 {
646 	int i, n, len, f_len = 0;
647 	char *buf;
648 
649 	buf = kmem_alloc(REPORTDEV_BUFSIZE, KM_SLEEP);
650 	f_len += snprintf(buf, REPORTDEV_BUFSIZE,
651 	    "%s%d at root", ddi_driver_name(dev), ddi_get_instance(dev));
652 	len = strlen(buf);
653 
654 	for (i = 0; i < sparc_pd_getnreg(dev); i++) {
655 
656 		struct regspec *rp = sparc_pd_getreg(dev, i);
657 
658 		if (i == 0)
659 			f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len,
660 			    ": ");
661 		else
662 			f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len,
663 			    " and ");
664 		len = strlen(buf);
665 
666 		switch (rp->regspec_bustype) {
667 
668 		case BTEISA:
669 			f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len,
670 			    "%s 0x%x", DEVI_EISA_NEXNAME, rp->regspec_addr);
671 			break;
672 
673 		case BTISA:
674 			f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len,
675 			    "%s 0x%x", DEVI_ISA_NEXNAME, rp->regspec_addr);
676 			break;
677 
678 		default:
679 			f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len,
680 			    "space %x offset %x",
681 			    rp->regspec_bustype, rp->regspec_addr);
682 			break;
683 		}
684 		len = strlen(buf);
685 	}
686 	for (i = 0, n = sparc_pd_getnintr(dev); i < n; i++) {
687 		int pri;
688 
689 		if (i != 0) {
690 			f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len,
691 			    ",");
692 			len = strlen(buf);
693 		}
694 		pri = INT_IPL(sparc_pd_getintr(dev, i)->intrspec_pri);
695 		f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len,
696 		    " sparc ipl %d", pri);
697 		len = strlen(buf);
698 	}
699 #ifdef DEBUG
700 	if (f_len + 1 >= REPORTDEV_BUFSIZE) {
701 		cmn_err(CE_NOTE, "next message is truncated: "
702 		    "printed length 1024, real length %d", f_len);
703 	}
704 #endif /* DEBUG */
705 	cmn_err(CE_CONT, "?%s\n", buf);
706 	kmem_free(buf, REPORTDEV_BUFSIZE);
707 	return (DDI_SUCCESS);
708 }
709 
710 
711 /*
712  * ******************
713  *  map related code
714  * ******************
715  */
716 
717 /*
718  * rootnex_map()
719  *
720  */
721 static int
722 rootnex_map(dev_info_t *dip, dev_info_t *rdip, ddi_map_req_t *mp, off_t offset,
723     off_t len, caddr_t *vaddrp)
724 {
725 	struct regspec *rp, tmp_reg;
726 	ddi_map_req_t mr = *mp;		/* Get private copy of request */
727 	int error;
728 
729 	mp = &mr;
730 
731 	switch (mp->map_op)  {
732 	case DDI_MO_MAP_LOCKED:
733 	case DDI_MO_UNMAP:
734 	case DDI_MO_MAP_HANDLE:
735 		break;
736 	default:
737 #ifdef	DDI_MAP_DEBUG
738 		cmn_err(CE_WARN, "rootnex_map: unimplemented map op %d.",
739 		    mp->map_op);
740 #endif	/* DDI_MAP_DEBUG */
741 		return (DDI_ME_UNIMPLEMENTED);
742 	}
743 
744 	if (mp->map_flags & DDI_MF_USER_MAPPING)  {
745 #ifdef	DDI_MAP_DEBUG
746 		cmn_err(CE_WARN, "rootnex_map: unimplemented map type: user.");
747 #endif	/* DDI_MAP_DEBUG */
748 		return (DDI_ME_UNIMPLEMENTED);
749 	}
750 
751 	/*
752 	 * First, if given an rnumber, convert it to a regspec...
753 	 * (Presumably, this is on behalf of a child of the root node?)
754 	 */
755 
756 	if (mp->map_type == DDI_MT_RNUMBER)  {
757 
758 		int rnumber = mp->map_obj.rnumber;
759 #ifdef	DDI_MAP_DEBUG
760 		static char *out_of_range =
761 		    "rootnex_map: Out of range rnumber <%d>, device <%s>";
762 #endif	/* DDI_MAP_DEBUG */
763 
764 		rp = i_ddi_rnumber_to_regspec(rdip, rnumber);
765 		if (rp == NULL)  {
766 #ifdef	DDI_MAP_DEBUG
767 			cmn_err(CE_WARN, out_of_range, rnumber,
768 			    ddi_get_name(rdip));
769 #endif	/* DDI_MAP_DEBUG */
770 			return (DDI_ME_RNUMBER_RANGE);
771 		}
772 
773 		/*
774 		 * Convert the given ddi_map_req_t from rnumber to regspec...
775 		 */
776 
777 		mp->map_type = DDI_MT_REGSPEC;
778 		mp->map_obj.rp = rp;
779 	}
780 
781 	/*
782 	 * Adjust offset and length correspnding to called values...
783 	 * XXX: A non-zero length means override the one in the regspec
784 	 * XXX: (regardless of what's in the parent's range?)
785 	 */
786 
787 	tmp_reg = *(mp->map_obj.rp);		/* Preserve underlying data */
788 	rp = mp->map_obj.rp = &tmp_reg;		/* Use tmp_reg in request */
789 
790 #ifdef	DDI_MAP_DEBUG
791 	cmn_err(CE_CONT, "rootnex: <%s,%s> <0x%x, 0x%x, 0x%d> offset %d len %d "
792 	    "handle 0x%x\n", ddi_get_name(dip), ddi_get_name(rdip),
793 	    rp->regspec_bustype, rp->regspec_addr, rp->regspec_size, offset,
794 	    len, mp->map_handlep);
795 #endif	/* DDI_MAP_DEBUG */
796 
797 	/*
798 	 * I/O or memory mapping:
799 	 *
800 	 *	<bustype=0, addr=x, len=x>: memory
801 	 *	<bustype=1, addr=x, len=x>: i/o
802 	 *	<bustype>1, addr=0, len=x>: x86-compatibility i/o
803 	 */
804 
805 	if (rp->regspec_bustype > 1 && rp->regspec_addr != 0) {
806 		cmn_err(CE_WARN, "<%s,%s> invalid register spec"
807 		    " <0x%x, 0x%x, 0x%x>", ddi_get_name(dip),
808 		    ddi_get_name(rdip), rp->regspec_bustype,
809 		    rp->regspec_addr, rp->regspec_size);
810 		return (DDI_ME_INVAL);
811 	}
812 
813 	if (rp->regspec_bustype > 1 && rp->regspec_addr == 0) {
814 		/*
815 		 * compatibility i/o mapping
816 		 */
817 		rp->regspec_bustype += (uint_t)offset;
818 	} else {
819 		/*
820 		 * Normal memory or i/o mapping
821 		 */
822 		rp->regspec_addr += (uint_t)offset;
823 	}
824 
825 	if (len != 0)
826 		rp->regspec_size = (uint_t)len;
827 
828 #ifdef	DDI_MAP_DEBUG
829 	cmn_err(CE_CONT, "             <%s,%s> <0x%x, 0x%x, 0x%d> offset %d "
830 	    "len %d handle 0x%x\n", ddi_get_name(dip), ddi_get_name(rdip),
831 	    rp->regspec_bustype, rp->regspec_addr, rp->regspec_size,
832 	    offset, len, mp->map_handlep);
833 #endif	/* DDI_MAP_DEBUG */
834 
835 	/*
836 	 * Apply any parent ranges at this level, if applicable.
837 	 * (This is where nexus specific regspec translation takes place.
838 	 * Use of this function is implicit agreement that translation is
839 	 * provided via ddi_apply_range.)
840 	 */
841 
842 #ifdef	DDI_MAP_DEBUG
843 	ddi_map_debug("applying range of parent <%s> to child <%s>...\n",
844 	    ddi_get_name(dip), ddi_get_name(rdip));
845 #endif	/* DDI_MAP_DEBUG */
846 
847 	if ((error = i_ddi_apply_range(dip, rdip, mp->map_obj.rp)) != 0)
848 		return (error);
849 
850 	switch (mp->map_op)  {
851 	case DDI_MO_MAP_LOCKED:
852 
853 		/*
854 		 * Set up the locked down kernel mapping to the regspec...
855 		 */
856 
857 		return (rootnex_map_regspec(mp, vaddrp));
858 
859 	case DDI_MO_UNMAP:
860 
861 		/*
862 		 * Release mapping...
863 		 */
864 
865 		return (rootnex_unmap_regspec(mp, vaddrp));
866 
867 	case DDI_MO_MAP_HANDLE:
868 
869 		return (rootnex_map_handle(mp));
870 
871 	default:
872 		return (DDI_ME_UNIMPLEMENTED);
873 	}
874 }
875 
876 
877 /*
878  * rootnex_map_fault()
879  *
880  *	fault in mappings for requestors
881  */
882 /*ARGSUSED*/
883 static int
884 rootnex_map_fault(dev_info_t *dip, dev_info_t *rdip, struct hat *hat,
885     struct seg *seg, caddr_t addr, struct devpage *dp, pfn_t pfn, uint_t prot,
886     uint_t lock)
887 {
888 
889 #ifdef	DDI_MAP_DEBUG
890 	ddi_map_debug("rootnex_map_fault: address <%x> pfn <%x>", addr, pfn);
891 	ddi_map_debug(" Seg <%s>\n",
892 	    seg->s_ops == &segdev_ops ? "segdev" :
893 	    seg == &kvseg ? "segkmem" : "NONE!");
894 #endif	/* DDI_MAP_DEBUG */
895 
896 	/*
897 	 * This is all terribly broken, but it is a start
898 	 *
899 	 * XXX	Note that this test means that segdev_ops
900 	 *	must be exported from seg_dev.c.
901 	 * XXX	What about devices with their own segment drivers?
902 	 */
903 	if (seg->s_ops == &segdev_ops) {
904 		struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
905 
906 		if (hat == NULL) {
907 			/*
908 			 * This is one plausible interpretation of
909 			 * a null hat i.e. use the first hat on the
910 			 * address space hat list which by convention is
911 			 * the hat of the system MMU.  At alternative
912 			 * would be to panic .. this might well be better ..
913 			 */
914 			ASSERT(AS_READ_HELD(seg->s_as, &seg->s_as->a_lock));
915 			hat = seg->s_as->a_hat;
916 			cmn_err(CE_NOTE, "rootnex_map_fault: nil hat");
917 		}
918 		hat_devload(hat, addr, MMU_PAGESIZE, pfn, prot | sdp->hat_attr,
919 		    (lock ? HAT_LOAD_LOCK : HAT_LOAD));
920 	} else if (seg == &kvseg && dp == NULL) {
921 		hat_devload(kas.a_hat, addr, MMU_PAGESIZE, pfn, prot,
922 		    HAT_LOAD_LOCK);
923 	} else
924 		return (DDI_FAILURE);
925 	return (DDI_SUCCESS);
926 }
927 
928 
929 /*
930  * rootnex_map_regspec()
931  *     we don't support mapping of I/O cards above 4Gb
932  */
933 static int
934 rootnex_map_regspec(ddi_map_req_t *mp, caddr_t *vaddrp)
935 {
936 	rootnex_addr_t rbase;
937 	void *cvaddr;
938 	uint_t npages, pgoffset;
939 	struct regspec *rp;
940 	ddi_acc_hdl_t *hp;
941 	ddi_acc_impl_t *ap;
942 	uint_t	hat_acc_flags;
943 	paddr_t pbase;
944 
945 	rp = mp->map_obj.rp;
946 	hp = mp->map_handlep;
947 
948 #ifdef	DDI_MAP_DEBUG
949 	ddi_map_debug(
950 	    "rootnex_map_regspec: <0x%x 0x%x 0x%x> handle 0x%x\n",
951 	    rp->regspec_bustype, rp->regspec_addr,
952 	    rp->regspec_size, mp->map_handlep);
953 #endif	/* DDI_MAP_DEBUG */
954 
955 	/*
956 	 * I/O or memory mapping
957 	 *
958 	 *	<bustype=0, addr=x, len=x>: memory
959 	 *	<bustype=1, addr=x, len=x>: i/o
960 	 *	<bustype>1, addr=0, len=x>: x86-compatibility i/o
961 	 */
962 
963 	if (rp->regspec_bustype > 1 && rp->regspec_addr != 0) {
964 		cmn_err(CE_WARN, "rootnex: invalid register spec"
965 		    " <0x%x, 0x%x, 0x%x>", rp->regspec_bustype,
966 		    rp->regspec_addr, rp->regspec_size);
967 		return (DDI_FAILURE);
968 	}
969 
970 	if (rp->regspec_bustype != 0) {
971 		/*
972 		 * I/O space - needs a handle.
973 		 */
974 		if (hp == NULL) {
975 			return (DDI_FAILURE);
976 		}
977 		ap = (ddi_acc_impl_t *)hp->ah_platform_private;
978 		ap->ahi_acc_attr |= DDI_ACCATTR_IO_SPACE;
979 		impl_acc_hdl_init(hp);
980 
981 		if (mp->map_flags & DDI_MF_DEVICE_MAPPING) {
982 #ifdef  DDI_MAP_DEBUG
983 			ddi_map_debug("rootnex_map_regspec: mmap() "
984 			    "to I/O space is not supported.\n");
985 #endif  /* DDI_MAP_DEBUG */
986 			return (DDI_ME_INVAL);
987 		} else {
988 			/*
989 			 * 1275-compliant vs. compatibility i/o mapping
990 			 */
991 			*vaddrp =
992 			    (rp->regspec_bustype > 1 && rp->regspec_addr == 0) ?
993 			    ((caddr_t)(uintptr_t)rp->regspec_bustype) :
994 			    ((caddr_t)(uintptr_t)rp->regspec_addr);
995 #ifdef __xpv
996 			if (DOMAIN_IS_INITDOMAIN(xen_info)) {
997 				hp->ah_pfn = xen_assign_pfn(
998 				    mmu_btop((ulong_t)rp->regspec_addr &
999 				    MMU_PAGEMASK));
1000 			} else {
1001 				hp->ah_pfn = mmu_btop(
1002 				    (ulong_t)rp->regspec_addr & MMU_PAGEMASK);
1003 			}
1004 #else
1005 			hp->ah_pfn = mmu_btop((ulong_t)rp->regspec_addr &
1006 			    MMU_PAGEMASK);
1007 #endif
1008 			hp->ah_pnum = mmu_btopr(rp->regspec_size +
1009 			    (ulong_t)rp->regspec_addr & MMU_PAGEOFFSET);
1010 		}
1011 
1012 #ifdef	DDI_MAP_DEBUG
1013 		ddi_map_debug(
1014 	    "rootnex_map_regspec: \"Mapping\" %d bytes I/O space at 0x%x\n",
1015 		    rp->regspec_size, *vaddrp);
1016 #endif	/* DDI_MAP_DEBUG */
1017 		return (DDI_SUCCESS);
1018 	}
1019 
1020 	/*
1021 	 * Memory space
1022 	 */
1023 
1024 	if (hp != NULL) {
1025 		/*
1026 		 * hat layer ignores
1027 		 * hp->ah_acc.devacc_attr_endian_flags.
1028 		 */
1029 		switch (hp->ah_acc.devacc_attr_dataorder) {
1030 		case DDI_STRICTORDER_ACC:
1031 			hat_acc_flags = HAT_STRICTORDER;
1032 			break;
1033 		case DDI_UNORDERED_OK_ACC:
1034 			hat_acc_flags = HAT_UNORDERED_OK;
1035 			break;
1036 		case DDI_MERGING_OK_ACC:
1037 			hat_acc_flags = HAT_MERGING_OK;
1038 			break;
1039 		case DDI_LOADCACHING_OK_ACC:
1040 			hat_acc_flags = HAT_LOADCACHING_OK;
1041 			break;
1042 		case DDI_STORECACHING_OK_ACC:
1043 			hat_acc_flags = HAT_STORECACHING_OK;
1044 			break;
1045 		}
1046 		ap = (ddi_acc_impl_t *)hp->ah_platform_private;
1047 		ap->ahi_acc_attr |= DDI_ACCATTR_CPU_VADDR;
1048 		impl_acc_hdl_init(hp);
1049 		hp->ah_hat_flags = hat_acc_flags;
1050 	} else {
1051 		hat_acc_flags = HAT_STRICTORDER;
1052 	}
1053 
1054 	rbase = (rootnex_addr_t)(rp->regspec_addr & MMU_PAGEMASK);
1055 #ifdef __xpv
1056 	/*
1057 	 * If we're dom0, we're using a real device so we need to translate
1058 	 * the MA to a PA.
1059 	 */
1060 	if (DOMAIN_IS_INITDOMAIN(xen_info)) {
1061 		pbase = pfn_to_pa(xen_assign_pfn(mmu_btop(rbase)));
1062 	} else {
1063 		pbase = rbase;
1064 	}
1065 #else
1066 	pbase = rbase;
1067 #endif
1068 	pgoffset = (ulong_t)rp->regspec_addr & MMU_PAGEOFFSET;
1069 
1070 	if (rp->regspec_size == 0) {
1071 #ifdef  DDI_MAP_DEBUG
1072 		ddi_map_debug("rootnex_map_regspec: zero regspec_size\n");
1073 #endif  /* DDI_MAP_DEBUG */
1074 		return (DDI_ME_INVAL);
1075 	}
1076 
1077 	if (mp->map_flags & DDI_MF_DEVICE_MAPPING) {
1078 		/* extra cast to make gcc happy */
1079 		*vaddrp = (caddr_t)((uintptr_t)mmu_btop(pbase));
1080 	} else {
1081 		npages = mmu_btopr(rp->regspec_size + pgoffset);
1082 
1083 #ifdef	DDI_MAP_DEBUG
1084 		ddi_map_debug("rootnex_map_regspec: Mapping %d pages "
1085 		    "physical %llx", npages, pbase);
1086 #endif	/* DDI_MAP_DEBUG */
1087 
1088 		cvaddr = device_arena_alloc(ptob(npages), VM_NOSLEEP);
1089 		if (cvaddr == NULL)
1090 			return (DDI_ME_NORESOURCES);
1091 
1092 		/*
1093 		 * Now map in the pages we've allocated...
1094 		 */
1095 		hat_devload(kas.a_hat, cvaddr, mmu_ptob(npages),
1096 		    mmu_btop(pbase), mp->map_prot | hat_acc_flags,
1097 		    HAT_LOAD_LOCK);
1098 		*vaddrp = (caddr_t)cvaddr + pgoffset;
1099 
1100 		/* save away pfn and npages for FMA */
1101 		hp = mp->map_handlep;
1102 		if (hp) {
1103 			hp->ah_pfn = mmu_btop(pbase);
1104 			hp->ah_pnum = npages;
1105 		}
1106 	}
1107 
1108 #ifdef	DDI_MAP_DEBUG
1109 	ddi_map_debug("at virtual 0x%x\n", *vaddrp);
1110 #endif	/* DDI_MAP_DEBUG */
1111 	return (DDI_SUCCESS);
1112 }
1113 
1114 
1115 /*
1116  * rootnex_unmap_regspec()
1117  *
1118  */
1119 static int
1120 rootnex_unmap_regspec(ddi_map_req_t *mp, caddr_t *vaddrp)
1121 {
1122 	caddr_t addr = (caddr_t)*vaddrp;
1123 	uint_t npages, pgoffset;
1124 	struct regspec *rp;
1125 
1126 	if (mp->map_flags & DDI_MF_DEVICE_MAPPING)
1127 		return (0);
1128 
1129 	rp = mp->map_obj.rp;
1130 
1131 	if (rp->regspec_size == 0) {
1132 #ifdef  DDI_MAP_DEBUG
1133 		ddi_map_debug("rootnex_unmap_regspec: zero regspec_size\n");
1134 #endif  /* DDI_MAP_DEBUG */
1135 		return (DDI_ME_INVAL);
1136 	}
1137 
1138 	/*
1139 	 * I/O or memory mapping:
1140 	 *
1141 	 *	<bustype=0, addr=x, len=x>: memory
1142 	 *	<bustype=1, addr=x, len=x>: i/o
1143 	 *	<bustype>1, addr=0, len=x>: x86-compatibility i/o
1144 	 */
1145 	if (rp->regspec_bustype != 0) {
1146 		/*
1147 		 * This is I/O space, which requires no particular
1148 		 * processing on unmap since it isn't mapped in the
1149 		 * first place.
1150 		 */
1151 		return (DDI_SUCCESS);
1152 	}
1153 
1154 	/*
1155 	 * Memory space
1156 	 */
1157 	pgoffset = (uintptr_t)addr & MMU_PAGEOFFSET;
1158 	npages = mmu_btopr(rp->regspec_size + pgoffset);
1159 	hat_unload(kas.a_hat, addr - pgoffset, ptob(npages), HAT_UNLOAD_UNLOCK);
1160 	device_arena_free(addr - pgoffset, ptob(npages));
1161 
1162 	/*
1163 	 * Destroy the pointer - the mapping has logically gone
1164 	 */
1165 	*vaddrp = NULL;
1166 
1167 	return (DDI_SUCCESS);
1168 }
1169 
1170 
1171 /*
1172  * rootnex_map_handle()
1173  *
1174  */
1175 static int
1176 rootnex_map_handle(ddi_map_req_t *mp)
1177 {
1178 	rootnex_addr_t rbase;
1179 	ddi_acc_hdl_t *hp;
1180 	uint_t pgoffset;
1181 	struct regspec *rp;
1182 	paddr_t pbase;
1183 
1184 	rp = mp->map_obj.rp;
1185 
1186 #ifdef	DDI_MAP_DEBUG
1187 	ddi_map_debug(
1188 	    "rootnex_map_handle: <0x%x 0x%x 0x%x> handle 0x%x\n",
1189 	    rp->regspec_bustype, rp->regspec_addr,
1190 	    rp->regspec_size, mp->map_handlep);
1191 #endif	/* DDI_MAP_DEBUG */
1192 
1193 	/*
1194 	 * I/O or memory mapping:
1195 	 *
1196 	 *	<bustype=0, addr=x, len=x>: memory
1197 	 *	<bustype=1, addr=x, len=x>: i/o
1198 	 *	<bustype>1, addr=0, len=x>: x86-compatibility i/o
1199 	 */
1200 	if (rp->regspec_bustype != 0) {
1201 		/*
1202 		 * This refers to I/O space, and we don't support "mapping"
1203 		 * I/O space to a user.
1204 		 */
1205 		return (DDI_FAILURE);
1206 	}
1207 
1208 	/*
1209 	 * Set up the hat_flags for the mapping.
1210 	 */
1211 	hp = mp->map_handlep;
1212 
1213 	switch (hp->ah_acc.devacc_attr_endian_flags) {
1214 	case DDI_NEVERSWAP_ACC:
1215 		hp->ah_hat_flags = HAT_NEVERSWAP | HAT_STRICTORDER;
1216 		break;
1217 	case DDI_STRUCTURE_LE_ACC:
1218 		hp->ah_hat_flags = HAT_STRUCTURE_LE;
1219 		break;
1220 	case DDI_STRUCTURE_BE_ACC:
1221 		return (DDI_FAILURE);
1222 	default:
1223 		return (DDI_REGS_ACC_CONFLICT);
1224 	}
1225 
1226 	switch (hp->ah_acc.devacc_attr_dataorder) {
1227 	case DDI_STRICTORDER_ACC:
1228 		break;
1229 	case DDI_UNORDERED_OK_ACC:
1230 		hp->ah_hat_flags |= HAT_UNORDERED_OK;
1231 		break;
1232 	case DDI_MERGING_OK_ACC:
1233 		hp->ah_hat_flags |= HAT_MERGING_OK;
1234 		break;
1235 	case DDI_LOADCACHING_OK_ACC:
1236 		hp->ah_hat_flags |= HAT_LOADCACHING_OK;
1237 		break;
1238 	case DDI_STORECACHING_OK_ACC:
1239 		hp->ah_hat_flags |= HAT_STORECACHING_OK;
1240 		break;
1241 	default:
1242 		return (DDI_FAILURE);
1243 	}
1244 
1245 	rbase = (rootnex_addr_t)rp->regspec_addr &
1246 	    (~(rootnex_addr_t)MMU_PAGEOFFSET);
1247 	pgoffset = (ulong_t)rp->regspec_addr & MMU_PAGEOFFSET;
1248 
1249 	if (rp->regspec_size == 0)
1250 		return (DDI_ME_INVAL);
1251 
1252 #ifdef __xpv
1253 	/*
1254 	 * If we're dom0, we're using a real device so we need to translate
1255 	 * the MA to a PA.
1256 	 */
1257 	if (DOMAIN_IS_INITDOMAIN(xen_info)) {
1258 		pbase = pfn_to_pa(xen_assign_pfn(mmu_btop(rbase))) |
1259 		    (rbase & MMU_PAGEOFFSET);
1260 	} else {
1261 		pbase = rbase;
1262 	}
1263 #else
1264 	pbase = rbase;
1265 #endif
1266 
1267 	hp->ah_pfn = mmu_btop(pbase);
1268 	hp->ah_pnum = mmu_btopr(rp->regspec_size + pgoffset);
1269 
1270 	return (DDI_SUCCESS);
1271 }
1272 
1273 
1274 
1275 /*
1276  * ************************
1277  *  interrupt related code
1278  * ************************
1279  */
1280 
1281 /*
1282  * rootnex_intr_ops()
1283  *	bus_intr_op() function for interrupt support
1284  */
1285 /* ARGSUSED */
1286 static int
1287 rootnex_intr_ops(dev_info_t *pdip, dev_info_t *rdip, ddi_intr_op_t intr_op,
1288     ddi_intr_handle_impl_t *hdlp, void *result)
1289 {
1290 	struct intrspec			*ispec;
1291 	struct ddi_parent_private_data	*pdp;
1292 
1293 	DDI_INTR_NEXDBG((CE_CONT,
1294 	    "rootnex_intr_ops: pdip = %p, rdip = %p, intr_op = %x, hdlp = %p\n",
1295 	    (void *)pdip, (void *)rdip, intr_op, (void *)hdlp));
1296 
1297 	/* Process the interrupt operation */
1298 	switch (intr_op) {
1299 	case DDI_INTROP_GETCAP:
1300 		/* First check with pcplusmp */
1301 		if (psm_intr_ops == NULL)
1302 			return (DDI_FAILURE);
1303 
1304 		if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_GET_CAP, result)) {
1305 			*(int *)result = 0;
1306 			return (DDI_FAILURE);
1307 		}
1308 		break;
1309 	case DDI_INTROP_SETCAP:
1310 		if (psm_intr_ops == NULL)
1311 			return (DDI_FAILURE);
1312 
1313 		if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_SET_CAP, result))
1314 			return (DDI_FAILURE);
1315 		break;
1316 	case DDI_INTROP_ALLOC:
1317 		if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL)
1318 			return (DDI_FAILURE);
1319 		hdlp->ih_pri = ispec->intrspec_pri;
1320 		*(int *)result = hdlp->ih_scratch1;
1321 		break;
1322 	case DDI_INTROP_FREE:
1323 		pdp = ddi_get_parent_data(rdip);
1324 		/*
1325 		 * Special case for 'pcic' driver' only.
1326 		 * If an intrspec was created for it, clean it up here
1327 		 * See detailed comments on this in the function
1328 		 * rootnex_get_ispec().
1329 		 */
1330 		if (pdp->par_intr && strcmp(ddi_get_name(rdip), "pcic") == 0) {
1331 			kmem_free(pdp->par_intr, sizeof (struct intrspec) *
1332 			    pdp->par_nintr);
1333 			/*
1334 			 * Set it to zero; so that
1335 			 * DDI framework doesn't free it again
1336 			 */
1337 			pdp->par_intr = NULL;
1338 			pdp->par_nintr = 0;
1339 		}
1340 		break;
1341 	case DDI_INTROP_GETPRI:
1342 		if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL)
1343 			return (DDI_FAILURE);
1344 		*(int *)result = ispec->intrspec_pri;
1345 		break;
1346 	case DDI_INTROP_SETPRI:
1347 		/* Validate the interrupt priority passed to us */
1348 		if (*(int *)result > LOCK_LEVEL)
1349 			return (DDI_FAILURE);
1350 
1351 		/* Ensure that PSM is all initialized and ispec is ok */
1352 		if ((psm_intr_ops == NULL) ||
1353 		    ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL))
1354 			return (DDI_FAILURE);
1355 
1356 		/* Change the priority */
1357 		if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_SET_PRI, result) ==
1358 		    PSM_FAILURE)
1359 			return (DDI_FAILURE);
1360 
1361 		/* update the ispec with the new priority */
1362 		ispec->intrspec_pri =  *(int *)result;
1363 		break;
1364 	case DDI_INTROP_ADDISR:
1365 		if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL)
1366 			return (DDI_FAILURE);
1367 		ispec->intrspec_func = hdlp->ih_cb_func;
1368 		break;
1369 	case DDI_INTROP_REMISR:
1370 		if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL)
1371 			return (DDI_FAILURE);
1372 		ispec->intrspec_func = (uint_t (*)()) 0;
1373 		break;
1374 	case DDI_INTROP_ENABLE:
1375 		if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL)
1376 			return (DDI_FAILURE);
1377 
1378 		/* Call psmi to translate irq with the dip */
1379 		if (psm_intr_ops == NULL)
1380 			return (DDI_FAILURE);
1381 
1382 		((ihdl_plat_t *)hdlp->ih_private)->ip_ispecp = ispec;
1383 		(void) (*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_XLATE_VECTOR,
1384 		    (int *)&hdlp->ih_vector);
1385 
1386 		/* Add the interrupt handler */
1387 		if (!add_avintr((void *)hdlp, ispec->intrspec_pri,
1388 		    hdlp->ih_cb_func, DEVI(rdip)->devi_name, hdlp->ih_vector,
1389 		    hdlp->ih_cb_arg1, hdlp->ih_cb_arg2, NULL, rdip))
1390 			return (DDI_FAILURE);
1391 		break;
1392 	case DDI_INTROP_DISABLE:
1393 		if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL)
1394 			return (DDI_FAILURE);
1395 
1396 		/* Call psm_ops() to translate irq with the dip */
1397 		if (psm_intr_ops == NULL)
1398 			return (DDI_FAILURE);
1399 
1400 		((ihdl_plat_t *)hdlp->ih_private)->ip_ispecp = ispec;
1401 		(void) (*psm_intr_ops)(rdip, hdlp,
1402 		    PSM_INTR_OP_XLATE_VECTOR, (int *)&hdlp->ih_vector);
1403 
1404 		/* Remove the interrupt handler */
1405 		rem_avintr((void *)hdlp, ispec->intrspec_pri,
1406 		    hdlp->ih_cb_func, hdlp->ih_vector);
1407 		break;
1408 	case DDI_INTROP_SETMASK:
1409 		if (psm_intr_ops == NULL)
1410 			return (DDI_FAILURE);
1411 
1412 		if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_SET_MASK, NULL))
1413 			return (DDI_FAILURE);
1414 		break;
1415 	case DDI_INTROP_CLRMASK:
1416 		if (psm_intr_ops == NULL)
1417 			return (DDI_FAILURE);
1418 
1419 		if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_CLEAR_MASK, NULL))
1420 			return (DDI_FAILURE);
1421 		break;
1422 	case DDI_INTROP_GETPENDING:
1423 		if (psm_intr_ops == NULL)
1424 			return (DDI_FAILURE);
1425 
1426 		if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_GET_PENDING,
1427 		    result)) {
1428 			*(int *)result = 0;
1429 			return (DDI_FAILURE);
1430 		}
1431 		break;
1432 	case DDI_INTROP_NAVAIL:
1433 	case DDI_INTROP_NINTRS:
1434 		*(int *)result = i_ddi_get_intx_nintrs(rdip);
1435 		if (*(int *)result == 0) {
1436 			/*
1437 			 * Special case for 'pcic' driver' only. This driver
1438 			 * driver is a child of 'isa' and 'rootnex' drivers.
1439 			 *
1440 			 * See detailed comments on this in the function
1441 			 * rootnex_get_ispec().
1442 			 *
1443 			 * Children of 'pcic' send 'NINITR' request all the
1444 			 * way to rootnex driver. But, the 'pdp->par_nintr'
1445 			 * field may not initialized. So, we fake it here
1446 			 * to return 1 (a la what PCMCIA nexus does).
1447 			 */
1448 			if (strcmp(ddi_get_name(rdip), "pcic") == 0)
1449 				*(int *)result = 1;
1450 			else
1451 				return (DDI_FAILURE);
1452 		}
1453 		break;
1454 	case DDI_INTROP_SUPPORTED_TYPES:
1455 		*(int *)result = DDI_INTR_TYPE_FIXED;	/* Always ... */
1456 		break;
1457 	default:
1458 		return (DDI_FAILURE);
1459 	}
1460 
1461 	return (DDI_SUCCESS);
1462 }
1463 
1464 
1465 /*
1466  * rootnex_get_ispec()
1467  *	convert an interrupt number to an interrupt specification.
1468  *	The interrupt number determines which interrupt spec will be
1469  *	returned if more than one exists.
1470  *
1471  *	Look into the parent private data area of the 'rdip' to find out
1472  *	the interrupt specification.  First check to make sure there is
1473  *	one that matchs "inumber" and then return a pointer to it.
1474  *
1475  *	Return NULL if one could not be found.
1476  *
1477  *	NOTE: This is needed for rootnex_intr_ops()
1478  */
1479 static struct intrspec *
1480 rootnex_get_ispec(dev_info_t *rdip, int inum)
1481 {
1482 	struct ddi_parent_private_data *pdp = ddi_get_parent_data(rdip);
1483 
1484 	/*
1485 	 * Special case handling for drivers that provide their own
1486 	 * intrspec structures instead of relying on the DDI framework.
1487 	 *
1488 	 * A broken hardware driver in ON could potentially provide its
1489 	 * own intrspec structure, instead of relying on the hardware.
1490 	 * If these drivers are children of 'rootnex' then we need to
1491 	 * continue to provide backward compatibility to them here.
1492 	 *
1493 	 * Following check is a special case for 'pcic' driver which
1494 	 * was found to have broken hardwre andby provides its own intrspec.
1495 	 *
1496 	 * Verbatim comments from this driver are shown here:
1497 	 * "Don't use the ddi_add_intr since we don't have a
1498 	 * default intrspec in all cases."
1499 	 *
1500 	 * Since an 'ispec' may not be always created for it,
1501 	 * check for that and create one if so.
1502 	 *
1503 	 * NOTE: Currently 'pcic' is the only driver found to do this.
1504 	 */
1505 	if (!pdp->par_intr && strcmp(ddi_get_name(rdip), "pcic") == 0) {
1506 		pdp->par_nintr = 1;
1507 		pdp->par_intr = kmem_zalloc(sizeof (struct intrspec) *
1508 		    pdp->par_nintr, KM_SLEEP);
1509 	}
1510 
1511 	/* Validate the interrupt number */
1512 	if (inum >= pdp->par_nintr)
1513 		return (NULL);
1514 
1515 	/* Get the interrupt structure pointer and return that */
1516 	return ((struct intrspec *)&pdp->par_intr[inum]);
1517 }
1518 
1519 
1520 /*
1521  * ******************
1522  *  dma related code
1523  * ******************
1524  */
1525 
1526 /*
1527  * rootnex_dma_allochdl()
1528  *    called from ddi_dma_alloc_handle().
1529  */
1530 /*ARGSUSED*/
1531 static int
1532 rootnex_dma_allochdl(dev_info_t *dip, dev_info_t *rdip, ddi_dma_attr_t *attr,
1533     int (*waitfp)(caddr_t), caddr_t arg, ddi_dma_handle_t *handlep)
1534 {
1535 	uint64_t maxsegmentsize_ll;
1536 	uint_t maxsegmentsize;
1537 	ddi_dma_impl_t *hp;
1538 	rootnex_dma_t *dma;
1539 	uint64_t count_max;
1540 	uint64_t seg;
1541 	int kmflag;
1542 	int e;
1543 
1544 
1545 	/* convert our sleep flags */
1546 	if (waitfp == DDI_DMA_SLEEP) {
1547 		kmflag = KM_SLEEP;
1548 	} else {
1549 		kmflag = KM_NOSLEEP;
1550 	}
1551 
1552 	/*
1553 	 * We try to do only one memory allocation here. We'll do a little
1554 	 * pointer manipulation later. If the bind ends up taking more than
1555 	 * our prealloc's space, we'll have to allocate more memory in the
1556 	 * bind operation. Not great, but much better than before and the
1557 	 * best we can do with the current bind interfaces.
1558 	 */
1559 	hp = kmem_cache_alloc(rootnex_state->r_dmahdl_cache, kmflag);
1560 	if (hp == NULL) {
1561 		if (waitfp != DDI_DMA_DONTWAIT) {
1562 			ddi_set_callback(waitfp, arg,
1563 			    &rootnex_state->r_dvma_call_list_id);
1564 		}
1565 		return (DDI_DMA_NORESOURCES);
1566 	}
1567 
1568 	/* Do our pointer manipulation now, align the structures */
1569 	hp->dmai_private = (void *)(((uintptr_t)hp +
1570 	    (uintptr_t)sizeof (ddi_dma_impl_t) + 0x7) & ~0x7);
1571 	dma = (rootnex_dma_t *)hp->dmai_private;
1572 	dma->dp_prealloc_buffer = (uchar_t *)(((uintptr_t)dma +
1573 	    sizeof (rootnex_dma_t) + 0x7) & ~0x7);
1574 
1575 	/* setup the handle */
1576 	rootnex_clean_dmahdl(hp);
1577 	dma->dp_dip = rdip;
1578 	dma->dp_sglinfo.si_min_addr = attr->dma_attr_addr_lo;
1579 	dma->dp_sglinfo.si_max_addr = attr->dma_attr_addr_hi;
1580 	hp->dmai_minxfer = attr->dma_attr_minxfer;
1581 	hp->dmai_burstsizes = attr->dma_attr_burstsizes;
1582 	hp->dmai_rdip = rdip;
1583 	hp->dmai_attr = *attr;
1584 
1585 	/* we don't need to worry about the SPL since we do a tryenter */
1586 	mutex_init(&dma->dp_mutex, NULL, MUTEX_DRIVER, NULL);
1587 
1588 	/*
1589 	 * Figure out our maximum segment size. If the segment size is greater
1590 	 * than 4G, we will limit it to (4G - 1) since the max size of a dma
1591 	 * object (ddi_dma_obj_t.dmao_size) is 32 bits. dma_attr_seg and
1592 	 * dma_attr_count_max are size-1 type values.
1593 	 *
1594 	 * Maximum segment size is the largest physically contiguous chunk of
1595 	 * memory that we can return from a bind (i.e. the maximum size of a
1596 	 * single cookie).
1597 	 */
1598 
1599 	/* handle the rollover cases */
1600 	seg = attr->dma_attr_seg + 1;
1601 	if (seg < attr->dma_attr_seg) {
1602 		seg = attr->dma_attr_seg;
1603 	}
1604 	count_max = attr->dma_attr_count_max + 1;
1605 	if (count_max < attr->dma_attr_count_max) {
1606 		count_max = attr->dma_attr_count_max;
1607 	}
1608 
1609 	/*
1610 	 * granularity may or may not be a power of two. If it isn't, we can't
1611 	 * use a simple mask.
1612 	 */
1613 	if (attr->dma_attr_granular & (attr->dma_attr_granular - 1)) {
1614 		dma->dp_granularity_power_2 = B_FALSE;
1615 	} else {
1616 		dma->dp_granularity_power_2 = B_TRUE;
1617 	}
1618 
1619 	/*
1620 	 * maxxfer should be a whole multiple of granularity. If we're going to
1621 	 * break up a window because we're greater than maxxfer, we might as
1622 	 * well make sure it's maxxfer is a whole multiple so we don't have to
1623 	 * worry about triming the window later on for this case.
1624 	 */
1625 	if (attr->dma_attr_granular > 1) {
1626 		if (dma->dp_granularity_power_2) {
1627 			dma->dp_maxxfer = attr->dma_attr_maxxfer -
1628 			    (attr->dma_attr_maxxfer &
1629 			    (attr->dma_attr_granular - 1));
1630 		} else {
1631 			dma->dp_maxxfer = attr->dma_attr_maxxfer -
1632 			    (attr->dma_attr_maxxfer % attr->dma_attr_granular);
1633 		}
1634 	} else {
1635 		dma->dp_maxxfer = attr->dma_attr_maxxfer;
1636 	}
1637 
1638 	maxsegmentsize_ll = MIN(seg, dma->dp_maxxfer);
1639 	maxsegmentsize_ll = MIN(maxsegmentsize_ll, count_max);
1640 	if (maxsegmentsize_ll == 0 || (maxsegmentsize_ll > 0xFFFFFFFF)) {
1641 		maxsegmentsize = 0xFFFFFFFF;
1642 	} else {
1643 		maxsegmentsize = maxsegmentsize_ll;
1644 	}
1645 	dma->dp_sglinfo.si_max_cookie_size = maxsegmentsize;
1646 	dma->dp_sglinfo.si_segmask = attr->dma_attr_seg;
1647 
1648 	/* check the ddi_dma_attr arg to make sure it makes a little sense */
1649 	if (rootnex_alloc_check_parms) {
1650 		e = rootnex_valid_alloc_parms(attr, maxsegmentsize);
1651 		if (e != DDI_SUCCESS) {
1652 			ROOTNEX_PROF_INC(&rootnex_cnt[ROOTNEX_CNT_ALLOC_FAIL]);
1653 			(void) rootnex_dma_freehdl(dip, rdip,
1654 			    (ddi_dma_handle_t)hp);
1655 			return (e);
1656 		}
1657 	}
1658 
1659 	*handlep = (ddi_dma_handle_t)hp;
1660 
1661 	ROOTNEX_PROF_INC(&rootnex_cnt[ROOTNEX_CNT_ACTIVE_HDLS]);
1662 	DTRACE_PROBE1(rootnex__alloc__handle, uint64_t,
1663 	    rootnex_cnt[ROOTNEX_CNT_ACTIVE_HDLS]);
1664 
1665 	return (DDI_SUCCESS);
1666 }
1667 
1668 
1669 /*
1670  * rootnex_dma_freehdl()
1671  *    called from ddi_dma_free_handle().
1672  */
1673 /*ARGSUSED*/
1674 static int
1675 rootnex_dma_freehdl(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle)
1676 {
1677 	ddi_dma_impl_t *hp;
1678 	rootnex_dma_t *dma;
1679 
1680 
1681 	hp = (ddi_dma_impl_t *)handle;
1682 	dma = (rootnex_dma_t *)hp->dmai_private;
1683 
1684 	/* unbind should have been called first */
1685 	ASSERT(!dma->dp_inuse);
1686 
1687 	mutex_destroy(&dma->dp_mutex);
1688 	kmem_cache_free(rootnex_state->r_dmahdl_cache, hp);
1689 
1690 	ROOTNEX_PROF_DEC(&rootnex_cnt[ROOTNEX_CNT_ACTIVE_HDLS]);
1691 	DTRACE_PROBE1(rootnex__free__handle, uint64_t,
1692 	    rootnex_cnt[ROOTNEX_CNT_ACTIVE_HDLS]);
1693 
1694 	if (rootnex_state->r_dvma_call_list_id)
1695 		ddi_run_callback(&rootnex_state->r_dvma_call_list_id);
1696 
1697 	return (DDI_SUCCESS);
1698 }
1699 
1700 
1701 /*
1702  * rootnex_dma_bindhdl()
1703  *    called from ddi_dma_addr_bind_handle() and ddi_dma_buf_bind_handle().
1704  */
1705 /*ARGSUSED*/
1706 static int
1707 rootnex_dma_bindhdl(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle,
1708     struct ddi_dma_req *dmareq, ddi_dma_cookie_t *cookiep, uint_t *ccountp)
1709 {
1710 	rootnex_sglinfo_t *sinfo;
1711 	ddi_dma_attr_t *attr;
1712 	ddi_dma_impl_t *hp;
1713 	rootnex_dma_t *dma;
1714 	int kmflag;
1715 	int e;
1716 
1717 
1718 	hp = (ddi_dma_impl_t *)handle;
1719 	dma = (rootnex_dma_t *)hp->dmai_private;
1720 	sinfo = &dma->dp_sglinfo;
1721 	attr = &hp->dmai_attr;
1722 
1723 	hp->dmai_rflags = dmareq->dmar_flags & DMP_DDIFLAGS;
1724 
1725 	/*
1726 	 * This is useful for debugging a driver. Not as useful in a production
1727 	 * system. The only time this will fail is if you have a driver bug.
1728 	 */
1729 	if (rootnex_bind_check_inuse) {
1730 		/*
1731 		 * No one else should ever have this lock unless someone else
1732 		 * is trying to use this handle. So contention on the lock
1733 		 * is the same as inuse being set.
1734 		 */
1735 		e = mutex_tryenter(&dma->dp_mutex);
1736 		if (e == 0) {
1737 			ROOTNEX_PROF_INC(&rootnex_cnt[ROOTNEX_CNT_BIND_FAIL]);
1738 			return (DDI_DMA_INUSE);
1739 		}
1740 		if (dma->dp_inuse) {
1741 			mutex_exit(&dma->dp_mutex);
1742 			ROOTNEX_PROF_INC(&rootnex_cnt[ROOTNEX_CNT_BIND_FAIL]);
1743 			return (DDI_DMA_INUSE);
1744 		}
1745 		dma->dp_inuse = B_TRUE;
1746 		mutex_exit(&dma->dp_mutex);
1747 	}
1748 
1749 	/* check the ddi_dma_attr arg to make sure it makes a little sense */
1750 	if (rootnex_bind_check_parms) {
1751 		e = rootnex_valid_bind_parms(dmareq, attr);
1752 		if (e != DDI_SUCCESS) {
1753 			ROOTNEX_PROF_INC(&rootnex_cnt[ROOTNEX_CNT_BIND_FAIL]);
1754 			rootnex_clean_dmahdl(hp);
1755 			return (e);
1756 		}
1757 	}
1758 
1759 	/* save away the original bind info */
1760 	dma->dp_dma = dmareq->dmar_object;
1761 
1762 	/*
1763 	 * Figure out a rough estimate of what maximum number of pages this
1764 	 * buffer could use (a high estimate of course).
1765 	 */
1766 	sinfo->si_max_pages = mmu_btopr(dma->dp_dma.dmao_size) + 1;
1767 
1768 	/*
1769 	 * We'll use the pre-allocated cookies for any bind that will *always*
1770 	 * fit (more important to be consistent, we don't want to create
1771 	 * additional degenerate cases).
1772 	 */
1773 	if (sinfo->si_max_pages <= rootnex_state->r_prealloc_cookies) {
1774 		dma->dp_cookies = (ddi_dma_cookie_t *)dma->dp_prealloc_buffer;
1775 		dma->dp_need_to_free_cookie = B_FALSE;
1776 		DTRACE_PROBE2(rootnex__bind__prealloc, dev_info_t *, rdip,
1777 		    uint_t, sinfo->si_max_pages);
1778 
1779 	/*
1780 	 * For anything larger than that, we'll go ahead and allocate the
1781 	 * maximum number of pages we expect to see. Hopefuly, we won't be
1782 	 * seeing this path in the fast path for high performance devices very
1783 	 * frequently.
1784 	 *
1785 	 * a ddi bind interface that allowed the driver to provide storage to
1786 	 * the bind interface would speed this case up.
1787 	 */
1788 	} else {
1789 		/* convert the sleep flags */
1790 		if (dmareq->dmar_fp == DDI_DMA_SLEEP) {
1791 			kmflag =  KM_SLEEP;
1792 		} else {
1793 			kmflag =  KM_NOSLEEP;
1794 		}
1795 
1796 		/*
1797 		 * Save away how much memory we allocated. If we're doing a
1798 		 * nosleep, the alloc could fail...
1799 		 */
1800 		dma->dp_cookie_size = sinfo->si_max_pages *
1801 		    sizeof (ddi_dma_cookie_t);
1802 		dma->dp_cookies = kmem_alloc(dma->dp_cookie_size, kmflag);
1803 		if (dma->dp_cookies == NULL) {
1804 			ROOTNEX_PROF_INC(&rootnex_cnt[ROOTNEX_CNT_BIND_FAIL]);
1805 			rootnex_clean_dmahdl(hp);
1806 			return (DDI_DMA_NORESOURCES);
1807 		}
1808 		dma->dp_need_to_free_cookie = B_TRUE;
1809 		DTRACE_PROBE2(rootnex__bind__alloc, dev_info_t *, rdip, uint_t,
1810 		    sinfo->si_max_pages);
1811 	}
1812 	hp->dmai_cookie = dma->dp_cookies;
1813 
1814 	/*
1815 	 * Get the real sgl. rootnex_get_sgl will fill in cookie array while
1816 	 * looking at the contraints in the dma structure. It will then put some
1817 	 * additional state about the sgl in the dma struct (i.e. is the sgl
1818 	 * clean, or do we need to do some munging; how many pages need to be
1819 	 * copied, etc.)
1820 	 */
1821 	rootnex_get_sgl(&dmareq->dmar_object, dma->dp_cookies,
1822 	    &dma->dp_sglinfo);
1823 	ASSERT(sinfo->si_sgl_size <= sinfo->si_max_pages);
1824 
1825 	/* if we don't need a copy buffer, we don't need to sync */
1826 	if (sinfo->si_copybuf_req == 0) {
1827 		hp->dmai_rflags |= DMP_NOSYNC;
1828 	}
1829 
1830 	/*
1831 	 * if we don't need the copybuf and we don't need to do a partial,  we
1832 	 * hit the fast path. All the high performance devices should be trying
1833 	 * to hit this path. To hit this path, a device should be able to reach
1834 	 * all of memory, shouldn't try to bind more than it can transfer, and
1835 	 * the buffer shouldn't require more cookies than the driver/device can
1836 	 * handle [sgllen]).
1837 	 */
1838 	if ((sinfo->si_copybuf_req == 0) &&
1839 	    (sinfo->si_sgl_size <= attr->dma_attr_sgllen) &&
1840 	    (dma->dp_dma.dmao_size < dma->dp_maxxfer)) {
1841 		/*
1842 		 * If the driver supports FMA, insert the handle in the FMA DMA
1843 		 * handle cache.
1844 		 */
1845 		if (attr->dma_attr_flags & DDI_DMA_FLAGERR) {
1846 			hp->dmai_error.err_cf = rootnex_dma_check;
1847 			(void) ndi_fmc_insert(rdip, DMA_HANDLE, hp, NULL);
1848 		}
1849 
1850 		/*
1851 		 * copy out the first cookie and ccountp, set the cookie
1852 		 * pointer to the second cookie. The first cookie is passed
1853 		 * back on the stack. Additional cookies are accessed via
1854 		 * ddi_dma_nextcookie()
1855 		 */
1856 		*cookiep = dma->dp_cookies[0];
1857 		*ccountp = sinfo->si_sgl_size;
1858 		hp->dmai_cookie++;
1859 		hp->dmai_rflags &= ~DDI_DMA_PARTIAL;
1860 		hp->dmai_nwin = 1;
1861 		ROOTNEX_PROF_INC(&rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS]);
1862 		DTRACE_PROBE3(rootnex__bind__fast, dev_info_t *, rdip, uint64_t,
1863 		    rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS], uint_t,
1864 		    dma->dp_dma.dmao_size);
1865 		return (DDI_DMA_MAPPED);
1866 	}
1867 
1868 	/*
1869 	 * go to the slow path, we may need to alloc more memory, create
1870 	 * multiple windows, and munge up a sgl to make the device happy.
1871 	 */
1872 	e = rootnex_bind_slowpath(hp, dmareq, dma, attr, kmflag);
1873 	if ((e != DDI_DMA_MAPPED) && (e != DDI_DMA_PARTIAL_MAP)) {
1874 		if (dma->dp_need_to_free_cookie) {
1875 			kmem_free(dma->dp_cookies, dma->dp_cookie_size);
1876 		}
1877 		ROOTNEX_PROF_INC(&rootnex_cnt[ROOTNEX_CNT_BIND_FAIL]);
1878 		rootnex_clean_dmahdl(hp); /* must be after free cookie */
1879 		return (e);
1880 	}
1881 
1882 	/*
1883 	 * If the driver supports FMA, insert the handle in the FMA DMA handle
1884 	 * cache.
1885 	 */
1886 	if (attr->dma_attr_flags & DDI_DMA_FLAGERR) {
1887 		hp->dmai_error.err_cf = rootnex_dma_check;
1888 		(void) ndi_fmc_insert(rdip, DMA_HANDLE, hp, NULL);
1889 	}
1890 
1891 	/* if the first window uses the copy buffer, sync it for the device */
1892 	if ((dma->dp_window[dma->dp_current_win].wd_dosync) &&
1893 	    (hp->dmai_rflags & DDI_DMA_WRITE)) {
1894 		(void) rootnex_dma_sync(dip, rdip, handle, 0, 0,
1895 		    DDI_DMA_SYNC_FORDEV);
1896 	}
1897 
1898 	/*
1899 	 * copy out the first cookie and ccountp, set the cookie pointer to the
1900 	 * second cookie. Make sure the partial flag is set/cleared correctly.
1901 	 * If we have a partial map (i.e. multiple windows), the number of
1902 	 * cookies we return is the number of cookies in the first window.
1903 	 */
1904 	if (e == DDI_DMA_MAPPED) {
1905 		hp->dmai_rflags &= ~DDI_DMA_PARTIAL;
1906 		*ccountp = sinfo->si_sgl_size;
1907 	} else {
1908 		hp->dmai_rflags |= DDI_DMA_PARTIAL;
1909 		*ccountp = dma->dp_window[dma->dp_current_win].wd_cookie_cnt;
1910 		ASSERT(hp->dmai_nwin <= dma->dp_max_win);
1911 	}
1912 	*cookiep = dma->dp_cookies[0];
1913 	hp->dmai_cookie++;
1914 
1915 	ROOTNEX_PROF_INC(&rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS]);
1916 	DTRACE_PROBE3(rootnex__bind__slow, dev_info_t *, rdip, uint64_t,
1917 	    rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS], uint_t,
1918 	    dma->dp_dma.dmao_size);
1919 	return (e);
1920 }
1921 
1922 
1923 /*
1924  * rootnex_dma_unbindhdl()
1925  *    called from ddi_dma_unbind_handle()
1926  */
1927 /*ARGSUSED*/
1928 static int
1929 rootnex_dma_unbindhdl(dev_info_t *dip, dev_info_t *rdip,
1930     ddi_dma_handle_t handle)
1931 {
1932 	ddi_dma_impl_t *hp;
1933 	rootnex_dma_t *dma;
1934 	int e;
1935 
1936 
1937 	hp = (ddi_dma_impl_t *)handle;
1938 	dma = (rootnex_dma_t *)hp->dmai_private;
1939 
1940 	/* make sure the buffer wasn't free'd before calling unbind */
1941 	if (rootnex_unbind_verify_buffer) {
1942 		e = rootnex_verify_buffer(dma);
1943 		if (e != DDI_SUCCESS) {
1944 			ASSERT(0);
1945 			return (DDI_FAILURE);
1946 		}
1947 	}
1948 
1949 	/* sync the current window before unbinding the buffer */
1950 	if (dma->dp_window && dma->dp_window[dma->dp_current_win].wd_dosync &&
1951 	    (hp->dmai_rflags & DDI_DMA_READ)) {
1952 		(void) rootnex_dma_sync(dip, rdip, handle, 0, 0,
1953 		    DDI_DMA_SYNC_FORCPU);
1954 	}
1955 
1956 	/*
1957 	 * If the driver supports FMA, remove the handle in the FMA DMA handle
1958 	 * cache.
1959 	 */
1960 	if (hp->dmai_attr.dma_attr_flags & DDI_DMA_FLAGERR) {
1961 		if ((DEVI(rdip)->devi_fmhdl != NULL) &&
1962 		    (DDI_FM_DMA_ERR_CAP(DEVI(rdip)->devi_fmhdl->fh_cap))) {
1963 			(void) ndi_fmc_remove(rdip, DMA_HANDLE, hp);
1964 		}
1965 	}
1966 
1967 	/*
1968 	 * cleanup and copy buffer or window state. if we didn't use the copy
1969 	 * buffer or windows, there won't be much to do :-)
1970 	 */
1971 	rootnex_teardown_copybuf(dma);
1972 	rootnex_teardown_windows(dma);
1973 
1974 	/*
1975 	 * If we had to allocate space to for the worse case sgl (it didn't
1976 	 * fit into our pre-allocate buffer), free that up now
1977 	 */
1978 	if (dma->dp_need_to_free_cookie) {
1979 		kmem_free(dma->dp_cookies, dma->dp_cookie_size);
1980 	}
1981 
1982 	/*
1983 	 * clean up the handle so it's ready for the next bind (i.e. if the
1984 	 * handle is reused).
1985 	 */
1986 	rootnex_clean_dmahdl(hp);
1987 
1988 	if (rootnex_state->r_dvma_call_list_id)
1989 		ddi_run_callback(&rootnex_state->r_dvma_call_list_id);
1990 
1991 	ROOTNEX_PROF_DEC(&rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS]);
1992 	DTRACE_PROBE1(rootnex__unbind, uint64_t,
1993 	    rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS]);
1994 
1995 	return (DDI_SUCCESS);
1996 }
1997 
1998 
1999 /*
2000  * rootnex_verify_buffer()
2001  *   verify buffer wasn't free'd
2002  */
2003 static int
2004 rootnex_verify_buffer(rootnex_dma_t *dma)
2005 {
2006 	page_t **pplist;
2007 	caddr_t vaddr;
2008 	uint_t pcnt;
2009 	uint_t poff;
2010 	page_t *pp;
2011 	char b;
2012 	int i;
2013 
2014 	/* Figure out how many pages this buffer occupies */
2015 	if (dma->dp_dma.dmao_type == DMA_OTYP_PAGES) {
2016 		poff = dma->dp_dma.dmao_obj.pp_obj.pp_offset & MMU_PAGEOFFSET;
2017 	} else {
2018 		vaddr = dma->dp_dma.dmao_obj.virt_obj.v_addr;
2019 		poff = (uintptr_t)vaddr & MMU_PAGEOFFSET;
2020 	}
2021 	pcnt = mmu_btopr(dma->dp_dma.dmao_size + poff);
2022 
2023 	switch (dma->dp_dma.dmao_type) {
2024 	case DMA_OTYP_PAGES:
2025 		/*
2026 		 * for a linked list of pp's walk through them to make sure
2027 		 * they're locked and not free.
2028 		 */
2029 		pp = dma->dp_dma.dmao_obj.pp_obj.pp_pp;
2030 		for (i = 0; i < pcnt; i++) {
2031 			if (PP_ISFREE(pp) || !PAGE_LOCKED(pp)) {
2032 				return (DDI_FAILURE);
2033 			}
2034 			pp = pp->p_next;
2035 		}
2036 		break;
2037 
2038 	case DMA_OTYP_VADDR:
2039 	case DMA_OTYP_BUFVADDR:
2040 		pplist = dma->dp_dma.dmao_obj.virt_obj.v_priv;
2041 		/*
2042 		 * for an array of pp's walk through them to make sure they're
2043 		 * not free. It's possible that they may not be locked.
2044 		 */
2045 		if (pplist) {
2046 			for (i = 0; i < pcnt; i++) {
2047 				if (PP_ISFREE(pplist[i])) {
2048 					return (DDI_FAILURE);
2049 				}
2050 			}
2051 
2052 		/* For a virtual address, try to peek at each page */
2053 		} else {
2054 			if (dma->dp_sglinfo.si_asp == &kas) {
2055 				for (i = 0; i < pcnt; i++) {
2056 					if (ddi_peek8(NULL, vaddr, &b) ==
2057 					    DDI_FAILURE)
2058 						return (DDI_FAILURE);
2059 					vaddr += MMU_PAGESIZE;
2060 				}
2061 			}
2062 		}
2063 		break;
2064 
2065 	default:
2066 		ASSERT(0);
2067 		break;
2068 	}
2069 
2070 	return (DDI_SUCCESS);
2071 }
2072 
2073 
2074 /*
2075  * rootnex_clean_dmahdl()
2076  *    Clean the dma handle. This should be called on a handle alloc and an
2077  *    unbind handle. Set the handle state to the default settings.
2078  */
2079 static void
2080 rootnex_clean_dmahdl(ddi_dma_impl_t *hp)
2081 {
2082 	rootnex_dma_t *dma;
2083 
2084 
2085 	dma = (rootnex_dma_t *)hp->dmai_private;
2086 
2087 	hp->dmai_nwin = 0;
2088 	dma->dp_current_cookie = 0;
2089 	dma->dp_copybuf_size = 0;
2090 	dma->dp_window = NULL;
2091 	dma->dp_cbaddr = NULL;
2092 	dma->dp_inuse = B_FALSE;
2093 	dma->dp_need_to_free_cookie = B_FALSE;
2094 	dma->dp_need_to_free_window = B_FALSE;
2095 	dma->dp_partial_required = B_FALSE;
2096 	dma->dp_trim_required = B_FALSE;
2097 	dma->dp_sglinfo.si_copybuf_req = 0;
2098 #if !defined(__amd64)
2099 	dma->dp_cb_remaping = B_FALSE;
2100 	dma->dp_kva = NULL;
2101 #endif
2102 
2103 	/* FMA related initialization */
2104 	hp->dmai_fault = 0;
2105 	hp->dmai_fault_check = NULL;
2106 	hp->dmai_fault_notify = NULL;
2107 	hp->dmai_error.err_ena = 0;
2108 	hp->dmai_error.err_status = DDI_FM_OK;
2109 	hp->dmai_error.err_expected = DDI_FM_ERR_UNEXPECTED;
2110 	hp->dmai_error.err_ontrap = NULL;
2111 	hp->dmai_error.err_fep = NULL;
2112 	hp->dmai_error.err_cf = NULL;
2113 }
2114 
2115 
2116 /*
2117  * rootnex_valid_alloc_parms()
2118  *    Called in ddi_dma_alloc_handle path to validate its parameters.
2119  */
2120 static int
2121 rootnex_valid_alloc_parms(ddi_dma_attr_t *attr, uint_t maxsegmentsize)
2122 {
2123 	if ((attr->dma_attr_seg < MMU_PAGEOFFSET) ||
2124 	    (attr->dma_attr_count_max < MMU_PAGEOFFSET) ||
2125 	    (attr->dma_attr_granular > MMU_PAGESIZE) ||
2126 	    (attr->dma_attr_maxxfer < MMU_PAGESIZE)) {
2127 		return (DDI_DMA_BADATTR);
2128 	}
2129 
2130 	if (attr->dma_attr_addr_hi <= attr->dma_attr_addr_lo) {
2131 		return (DDI_DMA_BADATTR);
2132 	}
2133 
2134 	if ((attr->dma_attr_seg & MMU_PAGEOFFSET) != MMU_PAGEOFFSET ||
2135 	    MMU_PAGESIZE & (attr->dma_attr_granular - 1) ||
2136 	    attr->dma_attr_sgllen <= 0) {
2137 		return (DDI_DMA_BADATTR);
2138 	}
2139 
2140 	/* We should be able to DMA into every byte offset in a page */
2141 	if (maxsegmentsize < MMU_PAGESIZE) {
2142 		return (DDI_DMA_BADATTR);
2143 	}
2144 
2145 	return (DDI_SUCCESS);
2146 }
2147 
2148 
2149 /*
2150  * rootnex_valid_bind_parms()
2151  *    Called in ddi_dma_*_bind_handle path to validate its parameters.
2152  */
2153 /* ARGSUSED */
2154 static int
2155 rootnex_valid_bind_parms(ddi_dma_req_t *dmareq, ddi_dma_attr_t *attr)
2156 {
2157 #if !defined(__amd64)
2158 	/*
2159 	 * we only support up to a 2G-1 transfer size on 32-bit kernels so
2160 	 * we can track the offset for the obsoleted interfaces.
2161 	 */
2162 	if (dmareq->dmar_object.dmao_size > 0x7FFFFFFF) {
2163 		return (DDI_DMA_TOOBIG);
2164 	}
2165 #endif
2166 
2167 	return (DDI_SUCCESS);
2168 }
2169 
2170 
2171 /*
2172  * rootnex_get_sgl()
2173  *    Called in bind fastpath to get the sgl. Most of this will be replaced
2174  *    with a call to the vm layer when vm2.0 comes around...
2175  */
2176 static void
2177 rootnex_get_sgl(ddi_dma_obj_t *dmar_object, ddi_dma_cookie_t *sgl,
2178     rootnex_sglinfo_t *sglinfo)
2179 {
2180 	ddi_dma_atyp_t buftype;
2181 	rootnex_addr_t raddr;
2182 	uint64_t last_page;
2183 	uint64_t offset;
2184 	uint64_t addrhi;
2185 	uint64_t addrlo;
2186 	uint64_t maxseg;
2187 	page_t **pplist;
2188 	uint64_t paddr;
2189 	uint32_t psize;
2190 	uint32_t size;
2191 	caddr_t vaddr;
2192 	uint_t pcnt;
2193 	page_t *pp;
2194 	uint_t cnt;
2195 
2196 
2197 	/* shortcuts */
2198 	pplist = dmar_object->dmao_obj.virt_obj.v_priv;
2199 	vaddr = dmar_object->dmao_obj.virt_obj.v_addr;
2200 	maxseg = sglinfo->si_max_cookie_size;
2201 	buftype = dmar_object->dmao_type;
2202 	addrhi = sglinfo->si_max_addr;
2203 	addrlo = sglinfo->si_min_addr;
2204 	size = dmar_object->dmao_size;
2205 
2206 	pcnt = 0;
2207 	cnt = 0;
2208 
2209 	/*
2210 	 * if we were passed down a linked list of pages, i.e. pointer to
2211 	 * page_t, use this to get our physical address and buf offset.
2212 	 */
2213 	if (buftype == DMA_OTYP_PAGES) {
2214 		pp = dmar_object->dmao_obj.pp_obj.pp_pp;
2215 		ASSERT(!PP_ISFREE(pp) && PAGE_LOCKED(pp));
2216 		offset =  dmar_object->dmao_obj.pp_obj.pp_offset &
2217 		    MMU_PAGEOFFSET;
2218 		paddr = pfn_to_pa(pp->p_pagenum) + offset;
2219 		psize = MIN(size, (MMU_PAGESIZE - offset));
2220 		pp = pp->p_next;
2221 		sglinfo->si_asp = NULL;
2222 
2223 	/*
2224 	 * We weren't passed down a linked list of pages, but if we were passed
2225 	 * down an array of pages, use this to get our physical address and buf
2226 	 * offset.
2227 	 */
2228 	} else if (pplist != NULL) {
2229 		ASSERT((buftype == DMA_OTYP_VADDR) ||
2230 		    (buftype == DMA_OTYP_BUFVADDR));
2231 
2232 		offset = (uintptr_t)vaddr & MMU_PAGEOFFSET;
2233 		sglinfo->si_asp = dmar_object->dmao_obj.virt_obj.v_as;
2234 		if (sglinfo->si_asp == NULL) {
2235 			sglinfo->si_asp = &kas;
2236 		}
2237 
2238 		ASSERT(!PP_ISFREE(pplist[pcnt]));
2239 		paddr = pfn_to_pa(pplist[pcnt]->p_pagenum);
2240 		paddr += offset;
2241 		psize = MIN(size, (MMU_PAGESIZE - offset));
2242 		pcnt++;
2243 
2244 	/*
2245 	 * All we have is a virtual address, we'll need to call into the VM
2246 	 * to get the physical address.
2247 	 */
2248 	} else {
2249 		ASSERT((buftype == DMA_OTYP_VADDR) ||
2250 		    (buftype == DMA_OTYP_BUFVADDR));
2251 
2252 		offset = (uintptr_t)vaddr & MMU_PAGEOFFSET;
2253 		sglinfo->si_asp = dmar_object->dmao_obj.virt_obj.v_as;
2254 		if (sglinfo->si_asp == NULL) {
2255 			sglinfo->si_asp = &kas;
2256 		}
2257 
2258 		paddr = pfn_to_pa(hat_getpfnum(sglinfo->si_asp->a_hat, vaddr));
2259 		paddr += offset;
2260 		psize = MIN(size, (MMU_PAGESIZE - offset));
2261 		vaddr += psize;
2262 	}
2263 
2264 #ifdef __xpv
2265 	/*
2266 	 * If we're dom0, we're using a real device so we need to load
2267 	 * the cookies with MFNs instead of PFNs.
2268 	 */
2269 	raddr = ROOTNEX_PADDR_TO_RBASE(xen_info, paddr);
2270 #else
2271 	raddr = paddr;
2272 #endif
2273 
2274 	/*
2275 	 * Setup the first cookie with the physical address of the page and the
2276 	 * size of the page (which takes into account the initial offset into
2277 	 * the page.
2278 	 */
2279 	sgl[cnt].dmac_laddress = raddr;
2280 	sgl[cnt].dmac_size = psize;
2281 	sgl[cnt].dmac_type = 0;
2282 
2283 	/*
2284 	 * Save away the buffer offset into the page. We'll need this later in
2285 	 * the copy buffer code to help figure out the page index within the
2286 	 * buffer and the offset into the current page.
2287 	 */
2288 	sglinfo->si_buf_offset = offset;
2289 
2290 	/*
2291 	 * If the DMA engine can't reach the physical address, increase how
2292 	 * much copy buffer we need. We always increase by pagesize so we don't
2293 	 * have to worry about converting offsets. Set a flag in the cookies
2294 	 * dmac_type to indicate that it uses the copy buffer. If this isn't the
2295 	 * last cookie, go to the next cookie (since we separate each page which
2296 	 * uses the copy buffer in case the copy buffer is not physically
2297 	 * contiguous.
2298 	 */
2299 	if ((raddr < addrlo) || ((raddr + psize) > addrhi)) {
2300 		sglinfo->si_copybuf_req += MMU_PAGESIZE;
2301 		sgl[cnt].dmac_type = ROOTNEX_USES_COPYBUF;
2302 		if ((cnt + 1) < sglinfo->si_max_pages) {
2303 			cnt++;
2304 			sgl[cnt].dmac_laddress = 0;
2305 			sgl[cnt].dmac_size = 0;
2306 			sgl[cnt].dmac_type = 0;
2307 		}
2308 	}
2309 
2310 	/*
2311 	 * save this page's physical address so we can figure out if the next
2312 	 * page is physically contiguous. Keep decrementing size until we are
2313 	 * done with the buffer.
2314 	 */
2315 	last_page = raddr & MMU_PAGEMASK;
2316 	size -= psize;
2317 
2318 	while (size > 0) {
2319 		/* Get the size for this page (i.e. partial or full page) */
2320 		psize = MIN(size, MMU_PAGESIZE);
2321 
2322 		if (buftype == DMA_OTYP_PAGES) {
2323 			/* get the paddr from the page_t */
2324 			ASSERT(!PP_ISFREE(pp) && PAGE_LOCKED(pp));
2325 			paddr = pfn_to_pa(pp->p_pagenum);
2326 			pp = pp->p_next;
2327 		} else if (pplist != NULL) {
2328 			/* index into the array of page_t's to get the paddr */
2329 			ASSERT(!PP_ISFREE(pplist[pcnt]));
2330 			paddr = pfn_to_pa(pplist[pcnt]->p_pagenum);
2331 			pcnt++;
2332 		} else {
2333 			/* call into the VM to get the paddr */
2334 			paddr =  pfn_to_pa(hat_getpfnum(sglinfo->si_asp->a_hat,
2335 			    vaddr));
2336 			vaddr += psize;
2337 		}
2338 
2339 #ifdef __xpv
2340 		/*
2341 		 * If we're dom0, we're using a real device so we need to load
2342 		 * the cookies with MFNs instead of PFNs.
2343 		 */
2344 		raddr = ROOTNEX_PADDR_TO_RBASE(xen_info, paddr);
2345 #else
2346 		raddr = paddr;
2347 #endif
2348 
2349 		/* check to see if this page needs the copy buffer */
2350 		if ((raddr < addrlo) || ((raddr + psize) > addrhi)) {
2351 			sglinfo->si_copybuf_req += MMU_PAGESIZE;
2352 
2353 			/*
2354 			 * if there is something in the current cookie, go to
2355 			 * the next one. We only want one page in a cookie which
2356 			 * uses the copybuf since the copybuf doesn't have to
2357 			 * be physically contiguous.
2358 			 */
2359 			if (sgl[cnt].dmac_size != 0) {
2360 				cnt++;
2361 			}
2362 			sgl[cnt].dmac_laddress = raddr;
2363 			sgl[cnt].dmac_size = psize;
2364 #if defined(__amd64)
2365 			sgl[cnt].dmac_type = ROOTNEX_USES_COPYBUF;
2366 #else
2367 			/*
2368 			 * save the buf offset for 32-bit kernel. used in the
2369 			 * obsoleted interfaces.
2370 			 */
2371 			sgl[cnt].dmac_type = ROOTNEX_USES_COPYBUF |
2372 			    (dmar_object->dmao_size - size);
2373 #endif
2374 			/* if this isn't the last cookie, go to the next one */
2375 			if ((cnt + 1) < sglinfo->si_max_pages) {
2376 				cnt++;
2377 				sgl[cnt].dmac_laddress = 0;
2378 				sgl[cnt].dmac_size = 0;
2379 				sgl[cnt].dmac_type = 0;
2380 			}
2381 
2382 		/*
2383 		 * this page didn't need the copy buffer, if it's not physically
2384 		 * contiguous, or it would put us over a segment boundary, or it
2385 		 * puts us over the max cookie size, or the current sgl doesn't
2386 		 * have anything in it.
2387 		 */
2388 		} else if (((last_page + MMU_PAGESIZE) != raddr) ||
2389 		    !(raddr & sglinfo->si_segmask) ||
2390 		    ((sgl[cnt].dmac_size + psize) > maxseg) ||
2391 		    (sgl[cnt].dmac_size == 0)) {
2392 			/*
2393 			 * if we're not already in a new cookie, go to the next
2394 			 * cookie.
2395 			 */
2396 			if (sgl[cnt].dmac_size != 0) {
2397 				cnt++;
2398 			}
2399 
2400 			/* save the cookie information */
2401 			sgl[cnt].dmac_laddress = raddr;
2402 			sgl[cnt].dmac_size = psize;
2403 #if defined(__amd64)
2404 			sgl[cnt].dmac_type = 0;
2405 #else
2406 			/*
2407 			 * save the buf offset for 32-bit kernel. used in the
2408 			 * obsoleted interfaces.
2409 			 */
2410 			sgl[cnt].dmac_type = dmar_object->dmao_size - size;
2411 #endif
2412 
2413 		/*
2414 		 * this page didn't need the copy buffer, it is physically
2415 		 * contiguous with the last page, and it's <= the max cookie
2416 		 * size.
2417 		 */
2418 		} else {
2419 			sgl[cnt].dmac_size += psize;
2420 
2421 			/*
2422 			 * if this exactly ==  the maximum cookie size, and
2423 			 * it isn't the last cookie, go to the next cookie.
2424 			 */
2425 			if (((sgl[cnt].dmac_size + psize) == maxseg) &&
2426 			    ((cnt + 1) < sglinfo->si_max_pages)) {
2427 				cnt++;
2428 				sgl[cnt].dmac_laddress = 0;
2429 				sgl[cnt].dmac_size = 0;
2430 				sgl[cnt].dmac_type = 0;
2431 			}
2432 		}
2433 
2434 		/*
2435 		 * save this page's physical address so we can figure out if the
2436 		 * next page is physically contiguous. Keep decrementing size
2437 		 * until we are done with the buffer.
2438 		 */
2439 		last_page = raddr;
2440 		size -= psize;
2441 	}
2442 
2443 	/* we're done, save away how many cookies the sgl has */
2444 	if (sgl[cnt].dmac_size == 0) {
2445 		ASSERT(cnt < sglinfo->si_max_pages);
2446 		sglinfo->si_sgl_size = cnt;
2447 	} else {
2448 		sglinfo->si_sgl_size = cnt + 1;
2449 	}
2450 }
2451 
2452 
2453 /*
2454  * rootnex_bind_slowpath()
2455  *    Call in the bind path if the calling driver can't use the sgl without
2456  *    modifying it. We either need to use the copy buffer and/or we will end up
2457  *    with a partial bind.
2458  */
2459 static int
2460 rootnex_bind_slowpath(ddi_dma_impl_t *hp, struct ddi_dma_req *dmareq,
2461     rootnex_dma_t *dma, ddi_dma_attr_t *attr, int kmflag)
2462 {
2463 	rootnex_sglinfo_t *sinfo;
2464 	rootnex_window_t *window;
2465 	ddi_dma_cookie_t *cookie;
2466 	size_t copybuf_used;
2467 	size_t dmac_size;
2468 	boolean_t partial;
2469 	off_t cur_offset;
2470 	page_t *cur_pp;
2471 	major_t mnum;
2472 	int e;
2473 	int i;
2474 
2475 
2476 	sinfo = &dma->dp_sglinfo;
2477 	copybuf_used = 0;
2478 	partial = B_FALSE;
2479 
2480 	/*
2481 	 * If we're using the copybuf, set the copybuf state in dma struct.
2482 	 * Needs to be first since it sets the copy buffer size.
2483 	 */
2484 	if (sinfo->si_copybuf_req != 0) {
2485 		e = rootnex_setup_copybuf(hp, dmareq, dma, attr);
2486 		if (e != DDI_SUCCESS) {
2487 			return (e);
2488 		}
2489 	} else {
2490 		dma->dp_copybuf_size = 0;
2491 	}
2492 
2493 	/*
2494 	 * Figure out if we need to do a partial mapping. If so, figure out
2495 	 * if we need to trim the buffers when we munge the sgl.
2496 	 */
2497 	if ((dma->dp_copybuf_size < sinfo->si_copybuf_req) ||
2498 	    (dma->dp_dma.dmao_size > dma->dp_maxxfer) ||
2499 	    (attr->dma_attr_sgllen < sinfo->si_sgl_size)) {
2500 		dma->dp_partial_required = B_TRUE;
2501 		if (attr->dma_attr_granular != 1) {
2502 			dma->dp_trim_required = B_TRUE;
2503 		}
2504 	} else {
2505 		dma->dp_partial_required = B_FALSE;
2506 		dma->dp_trim_required = B_FALSE;
2507 	}
2508 
2509 	/* If we need to do a partial bind, make sure the driver supports it */
2510 	if (dma->dp_partial_required &&
2511 	    !(dmareq->dmar_flags & DDI_DMA_PARTIAL)) {
2512 
2513 		mnum = ddi_driver_major(dma->dp_dip);
2514 		/*
2515 		 * patchable which allows us to print one warning per major
2516 		 * number.
2517 		 */
2518 		if ((rootnex_bind_warn) &&
2519 		    ((rootnex_warn_list[mnum] & ROOTNEX_BIND_WARNING) == 0)) {
2520 			rootnex_warn_list[mnum] |= ROOTNEX_BIND_WARNING;
2521 			cmn_err(CE_WARN, "!%s: coding error detected, the "
2522 			    "driver is using ddi_dma_attr(9S) incorrectly. "
2523 			    "There is a small risk of data corruption in "
2524 			    "particular with large I/Os. The driver should be "
2525 			    "replaced with a corrected version for proper "
2526 			    "system operation. To disable this warning, add "
2527 			    "'set rootnex:rootnex_bind_warn=0' to "
2528 			    "/etc/system(4).", ddi_driver_name(dma->dp_dip));
2529 		}
2530 		return (DDI_DMA_TOOBIG);
2531 	}
2532 
2533 	/*
2534 	 * we might need multiple windows, setup state to handle them. In this
2535 	 * code path, we will have at least one window.
2536 	 */
2537 	e = rootnex_setup_windows(hp, dma, attr, kmflag);
2538 	if (e != DDI_SUCCESS) {
2539 		rootnex_teardown_copybuf(dma);
2540 		return (e);
2541 	}
2542 
2543 	window = &dma->dp_window[0];
2544 	cookie = &dma->dp_cookies[0];
2545 	cur_offset = 0;
2546 	rootnex_init_win(hp, dma, window, cookie, cur_offset);
2547 	if (dmareq->dmar_object.dmao_type == DMA_OTYP_PAGES) {
2548 		cur_pp = dmareq->dmar_object.dmao_obj.pp_obj.pp_pp;
2549 	}
2550 
2551 	/* loop though all the cookies we got back from get_sgl() */
2552 	for (i = 0; i < sinfo->si_sgl_size; i++) {
2553 		/*
2554 		 * If we're using the copy buffer, check this cookie and setup
2555 		 * its associated copy buffer state. If this cookie uses the
2556 		 * copy buffer, make sure we sync this window during dma_sync.
2557 		 */
2558 		if (dma->dp_copybuf_size > 0) {
2559 			rootnex_setup_cookie(&dmareq->dmar_object, dma, cookie,
2560 			    cur_offset, &copybuf_used, &cur_pp);
2561 			if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) {
2562 				window->wd_dosync = B_TRUE;
2563 			}
2564 		}
2565 
2566 		/*
2567 		 * save away the cookie size, since it could be modified in
2568 		 * the windowing code.
2569 		 */
2570 		dmac_size = cookie->dmac_size;
2571 
2572 		/* if we went over max copybuf size */
2573 		if (dma->dp_copybuf_size &&
2574 		    (copybuf_used > dma->dp_copybuf_size)) {
2575 			partial = B_TRUE;
2576 			e = rootnex_copybuf_window_boundary(hp, dma, &window,
2577 			    cookie, cur_offset, &copybuf_used);
2578 			if (e != DDI_SUCCESS) {
2579 				rootnex_teardown_copybuf(dma);
2580 				rootnex_teardown_windows(dma);
2581 				return (e);
2582 			}
2583 
2584 			/*
2585 			 * if the coookie uses the copy buffer, make sure the
2586 			 * new window we just moved to is set to sync.
2587 			 */
2588 			if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) {
2589 				window->wd_dosync = B_TRUE;
2590 			}
2591 			DTRACE_PROBE1(rootnex__copybuf__window, dev_info_t *,
2592 			    dma->dp_dip);
2593 
2594 		/* if the cookie cnt == max sgllen, move to the next window */
2595 		} else if (window->wd_cookie_cnt >= attr->dma_attr_sgllen) {
2596 			partial = B_TRUE;
2597 			ASSERT(window->wd_cookie_cnt == attr->dma_attr_sgllen);
2598 			e = rootnex_sgllen_window_boundary(hp, dma, &window,
2599 			    cookie, attr, cur_offset);
2600 			if (e != DDI_SUCCESS) {
2601 				rootnex_teardown_copybuf(dma);
2602 				rootnex_teardown_windows(dma);
2603 				return (e);
2604 			}
2605 
2606 			/*
2607 			 * if the coookie uses the copy buffer, make sure the
2608 			 * new window we just moved to is set to sync.
2609 			 */
2610 			if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) {
2611 				window->wd_dosync = B_TRUE;
2612 			}
2613 			DTRACE_PROBE1(rootnex__sgllen__window, dev_info_t *,
2614 			    dma->dp_dip);
2615 
2616 		/* else if we will be over maxxfer */
2617 		} else if ((window->wd_size + dmac_size) >
2618 		    dma->dp_maxxfer) {
2619 			partial = B_TRUE;
2620 			e = rootnex_maxxfer_window_boundary(hp, dma, &window,
2621 			    cookie);
2622 			if (e != DDI_SUCCESS) {
2623 				rootnex_teardown_copybuf(dma);
2624 				rootnex_teardown_windows(dma);
2625 				return (e);
2626 			}
2627 
2628 			/*
2629 			 * if the coookie uses the copy buffer, make sure the
2630 			 * new window we just moved to is set to sync.
2631 			 */
2632 			if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) {
2633 				window->wd_dosync = B_TRUE;
2634 			}
2635 			DTRACE_PROBE1(rootnex__maxxfer__window, dev_info_t *,
2636 			    dma->dp_dip);
2637 
2638 		/* else this cookie fits in the current window */
2639 		} else {
2640 			window->wd_cookie_cnt++;
2641 			window->wd_size += dmac_size;
2642 		}
2643 
2644 		/* track our offset into the buffer, go to the next cookie */
2645 		ASSERT(dmac_size <= dma->dp_dma.dmao_size);
2646 		ASSERT(cookie->dmac_size <= dmac_size);
2647 		cur_offset += dmac_size;
2648 		cookie++;
2649 	}
2650 
2651 	/* if we ended up with a zero sized window in the end, clean it up */
2652 	if (window->wd_size == 0) {
2653 		hp->dmai_nwin--;
2654 		window--;
2655 	}
2656 
2657 	ASSERT(window->wd_trim.tr_trim_last == B_FALSE);
2658 
2659 	if (!partial) {
2660 		return (DDI_DMA_MAPPED);
2661 	}
2662 
2663 	ASSERT(dma->dp_partial_required);
2664 	return (DDI_DMA_PARTIAL_MAP);
2665 }
2666 
2667 
2668 /*
2669  * rootnex_setup_copybuf()
2670  *    Called in bind slowpath. Figures out if we're going to use the copy
2671  *    buffer, and if we do, sets up the basic state to handle it.
2672  */
2673 static int
2674 rootnex_setup_copybuf(ddi_dma_impl_t *hp, struct ddi_dma_req *dmareq,
2675     rootnex_dma_t *dma, ddi_dma_attr_t *attr)
2676 {
2677 	rootnex_sglinfo_t *sinfo;
2678 	ddi_dma_attr_t lattr;
2679 	size_t max_copybuf;
2680 	int cansleep;
2681 	int e;
2682 #if !defined(__amd64)
2683 	int vmflag;
2684 #endif
2685 
2686 
2687 	sinfo = &dma->dp_sglinfo;
2688 
2689 	/* read this first so it's consistent through the routine  */
2690 	max_copybuf = i_ddi_copybuf_size() & MMU_PAGEMASK;
2691 
2692 	/* We need to call into the rootnex on ddi_dma_sync() */
2693 	hp->dmai_rflags &= ~DMP_NOSYNC;
2694 
2695 	/* make sure the copybuf size <= the max size */
2696 	dma->dp_copybuf_size = MIN(sinfo->si_copybuf_req, max_copybuf);
2697 	ASSERT((dma->dp_copybuf_size & MMU_PAGEOFFSET) == 0);
2698 
2699 #if !defined(__amd64)
2700 	/*
2701 	 * if we don't have kva space to copy to/from, allocate the KVA space
2702 	 * now. We only do this for the 32-bit kernel. We use seg kpm space for
2703 	 * the 64-bit kernel.
2704 	 */
2705 	if ((dmareq->dmar_object.dmao_type == DMA_OTYP_PAGES) ||
2706 	    (dmareq->dmar_object.dmao_obj.virt_obj.v_as != NULL)) {
2707 
2708 		/* convert the sleep flags */
2709 		if (dmareq->dmar_fp == DDI_DMA_SLEEP) {
2710 			vmflag = VM_SLEEP;
2711 		} else {
2712 			vmflag = VM_NOSLEEP;
2713 		}
2714 
2715 		/* allocate Kernel VA space that we can bcopy to/from */
2716 		dma->dp_kva = vmem_alloc(heap_arena, dma->dp_copybuf_size,
2717 		    vmflag);
2718 		if (dma->dp_kva == NULL) {
2719 			return (DDI_DMA_NORESOURCES);
2720 		}
2721 	}
2722 #endif
2723 
2724 	/* convert the sleep flags */
2725 	if (dmareq->dmar_fp == DDI_DMA_SLEEP) {
2726 		cansleep = 1;
2727 	} else {
2728 		cansleep = 0;
2729 	}
2730 
2731 	/*
2732 	 * Allocate the actual copy buffer. This needs to fit within the DMA
2733 	 * engine limits, so we can't use kmem_alloc... We don't need
2734 	 * contiguous memory (sgllen) since we will be forcing windows on
2735 	 * sgllen anyway.
2736 	 */
2737 	lattr = *attr;
2738 	lattr.dma_attr_align = MMU_PAGESIZE;
2739 	/*
2740 	 * this should be < 0 to indicate no limit, but due to a bug in
2741 	 * the rootnex, we'll set it to the maximum positive int.
2742 	 */
2743 	lattr.dma_attr_sgllen = 0x7fffffff;
2744 	e = i_ddi_mem_alloc(dma->dp_dip, &lattr, dma->dp_copybuf_size, cansleep,
2745 	    0, NULL, &dma->dp_cbaddr, &dma->dp_cbsize, NULL);
2746 	if (e != DDI_SUCCESS) {
2747 #if !defined(__amd64)
2748 		if (dma->dp_kva != NULL) {
2749 			vmem_free(heap_arena, dma->dp_kva,
2750 			    dma->dp_copybuf_size);
2751 		}
2752 #endif
2753 		return (DDI_DMA_NORESOURCES);
2754 	}
2755 
2756 	DTRACE_PROBE2(rootnex__alloc__copybuf, dev_info_t *, dma->dp_dip,
2757 	    size_t, dma->dp_copybuf_size);
2758 
2759 	return (DDI_SUCCESS);
2760 }
2761 
2762 
2763 /*
2764  * rootnex_setup_windows()
2765  *    Called in bind slowpath to setup the window state. We always have windows
2766  *    in the slowpath. Even if the window count = 1.
2767  */
2768 static int
2769 rootnex_setup_windows(ddi_dma_impl_t *hp, rootnex_dma_t *dma,
2770     ddi_dma_attr_t *attr, int kmflag)
2771 {
2772 	rootnex_window_t *windowp;
2773 	rootnex_sglinfo_t *sinfo;
2774 	size_t copy_state_size;
2775 	size_t win_state_size;
2776 	size_t state_available;
2777 	size_t space_needed;
2778 	uint_t copybuf_win;
2779 	uint_t maxxfer_win;
2780 	size_t space_used;
2781 	uint_t sglwin;
2782 
2783 
2784 	sinfo = &dma->dp_sglinfo;
2785 
2786 	dma->dp_current_win = 0;
2787 	hp->dmai_nwin = 0;
2788 
2789 	/* If we don't need to do a partial, we only have one window */
2790 	if (!dma->dp_partial_required) {
2791 		dma->dp_max_win = 1;
2792 
2793 	/*
2794 	 * we need multiple windows, need to figure out the worse case number
2795 	 * of windows.
2796 	 */
2797 	} else {
2798 		/*
2799 		 * if we need windows because we need more copy buffer that
2800 		 * we allow, the worse case number of windows we could need
2801 		 * here would be (copybuf space required / copybuf space that
2802 		 * we have) plus one for remainder, and plus 2 to handle the
2803 		 * extra pages on the trim for the first and last pages of the
2804 		 * buffer (a page is the minimum window size so under the right
2805 		 * attr settings, you could have a window for each page).
2806 		 * The last page will only be hit here if the size is not a
2807 		 * multiple of the granularity (which theoretically shouldn't
2808 		 * be the case but never has been enforced, so we could have
2809 		 * broken things without it).
2810 		 */
2811 		if (sinfo->si_copybuf_req > dma->dp_copybuf_size) {
2812 			ASSERT(dma->dp_copybuf_size > 0);
2813 			copybuf_win = (sinfo->si_copybuf_req /
2814 			    dma->dp_copybuf_size) + 1 + 2;
2815 		} else {
2816 			copybuf_win = 0;
2817 		}
2818 
2819 		/*
2820 		 * if we need windows because we have more cookies than the H/W
2821 		 * can handle, the number of windows we would need here would
2822 		 * be (cookie count / cookies count H/W supports) plus one for
2823 		 * remainder, and plus 2 to handle the extra pages on the trim
2824 		 * (see above comment about trim)
2825 		 */
2826 		if (attr->dma_attr_sgllen < sinfo->si_sgl_size) {
2827 			sglwin = ((sinfo->si_sgl_size / attr->dma_attr_sgllen)
2828 			    + 1) + 2;
2829 		} else {
2830 			sglwin = 0;
2831 		}
2832 
2833 		/*
2834 		 * if we need windows because we're binding more memory than the
2835 		 * H/W can transfer at once, the number of windows we would need
2836 		 * here would be (xfer count / max xfer H/W supports) plus one
2837 		 * for remainder, and plus 2 to handle the extra pages on the
2838 		 * trim (see above comment about trim)
2839 		 */
2840 		if (dma->dp_dma.dmao_size > dma->dp_maxxfer) {
2841 			maxxfer_win = (dma->dp_dma.dmao_size /
2842 			    dma->dp_maxxfer) + 1 + 2;
2843 		} else {
2844 			maxxfer_win = 0;
2845 		}
2846 		dma->dp_max_win =  copybuf_win + sglwin + maxxfer_win;
2847 		ASSERT(dma->dp_max_win > 0);
2848 	}
2849 	win_state_size = dma->dp_max_win * sizeof (rootnex_window_t);
2850 
2851 	/*
2852 	 * Get space for window and potential copy buffer state. Before we
2853 	 * go and allocate memory, see if we can get away with using what's
2854 	 * left in the pre-allocted state or the dynamically allocated sgl.
2855 	 */
2856 	space_used = (uintptr_t)(sinfo->si_sgl_size *
2857 	    sizeof (ddi_dma_cookie_t));
2858 
2859 	/* if we dynamically allocated space for the cookies */
2860 	if (dma->dp_need_to_free_cookie) {
2861 		/* if we have more space in the pre-allocted buffer, use it */
2862 		ASSERT(space_used <= dma->dp_cookie_size);
2863 		if ((dma->dp_cookie_size - space_used) <=
2864 		    rootnex_state->r_prealloc_size) {
2865 			state_available = rootnex_state->r_prealloc_size;
2866 			windowp = (rootnex_window_t *)dma->dp_prealloc_buffer;
2867 
2868 		/*
2869 		 * else, we have more free space in the dynamically allocated
2870 		 * buffer, i.e. the buffer wasn't worse case fragmented so we
2871 		 * didn't need a lot of cookies.
2872 		 */
2873 		} else {
2874 			state_available = dma->dp_cookie_size - space_used;
2875 			windowp = (rootnex_window_t *)
2876 			    &dma->dp_cookies[sinfo->si_sgl_size];
2877 		}
2878 
2879 	/* we used the pre-alloced buffer */
2880 	} else {
2881 		ASSERT(space_used <= rootnex_state->r_prealloc_size);
2882 		state_available = rootnex_state->r_prealloc_size - space_used;
2883 		windowp = (rootnex_window_t *)
2884 		    &dma->dp_cookies[sinfo->si_sgl_size];
2885 	}
2886 
2887 	/*
2888 	 * figure out how much state we need to track the copy buffer. Add an
2889 	 * addition 8 bytes for pointer alignemnt later.
2890 	 */
2891 	if (dma->dp_copybuf_size > 0) {
2892 		copy_state_size = sinfo->si_max_pages *
2893 		    sizeof (rootnex_pgmap_t);
2894 	} else {
2895 		copy_state_size = 0;
2896 	}
2897 	/* add an additional 8 bytes for pointer alignment */
2898 	space_needed = win_state_size + copy_state_size + 0x8;
2899 
2900 	/* if we have enough space already, use it */
2901 	if (state_available >= space_needed) {
2902 		dma->dp_window = windowp;
2903 		dma->dp_need_to_free_window = B_FALSE;
2904 
2905 	/* not enough space, need to allocate more. */
2906 	} else {
2907 		dma->dp_window = kmem_alloc(space_needed, kmflag);
2908 		if (dma->dp_window == NULL) {
2909 			return (DDI_DMA_NORESOURCES);
2910 		}
2911 		dma->dp_need_to_free_window = B_TRUE;
2912 		dma->dp_window_size = space_needed;
2913 		DTRACE_PROBE2(rootnex__bind__sp__alloc, dev_info_t *,
2914 		    dma->dp_dip, size_t, space_needed);
2915 	}
2916 
2917 	/*
2918 	 * we allocate copy buffer state and window state at the same time.
2919 	 * setup our copy buffer state pointers. Make sure it's aligned.
2920 	 */
2921 	if (dma->dp_copybuf_size > 0) {
2922 		dma->dp_pgmap = (rootnex_pgmap_t *)(((uintptr_t)
2923 		    &dma->dp_window[dma->dp_max_win] + 0x7) & ~0x7);
2924 
2925 #if !defined(__amd64)
2926 		/*
2927 		 * make sure all pm_mapped, pm_vaddr, and pm_pp are set to
2928 		 * false/NULL. Should be quicker to bzero vs loop and set.
2929 		 */
2930 		bzero(dma->dp_pgmap, copy_state_size);
2931 #endif
2932 	} else {
2933 		dma->dp_pgmap = NULL;
2934 	}
2935 
2936 	return (DDI_SUCCESS);
2937 }
2938 
2939 
2940 /*
2941  * rootnex_teardown_copybuf()
2942  *    cleans up after rootnex_setup_copybuf()
2943  */
2944 static void
2945 rootnex_teardown_copybuf(rootnex_dma_t *dma)
2946 {
2947 #if !defined(__amd64)
2948 	int i;
2949 
2950 	/*
2951 	 * if we allocated kernel heap VMEM space, go through all the pages and
2952 	 * map out any of the ones that we're mapped into the kernel heap VMEM
2953 	 * arena. Then free the VMEM space.
2954 	 */
2955 	if (dma->dp_kva != NULL) {
2956 		for (i = 0; i < dma->dp_sglinfo.si_max_pages; i++) {
2957 			if (dma->dp_pgmap[i].pm_mapped) {
2958 				hat_unload(kas.a_hat, dma->dp_pgmap[i].pm_kaddr,
2959 				    MMU_PAGESIZE, HAT_UNLOAD);
2960 				dma->dp_pgmap[i].pm_mapped = B_FALSE;
2961 			}
2962 		}
2963 
2964 		vmem_free(heap_arena, dma->dp_kva, dma->dp_copybuf_size);
2965 	}
2966 
2967 #endif
2968 
2969 	/* if we allocated a copy buffer, free it */
2970 	if (dma->dp_cbaddr != NULL) {
2971 		i_ddi_mem_free(dma->dp_cbaddr, NULL);
2972 	}
2973 }
2974 
2975 
2976 /*
2977  * rootnex_teardown_windows()
2978  *    cleans up after rootnex_setup_windows()
2979  */
2980 static void
2981 rootnex_teardown_windows(rootnex_dma_t *dma)
2982 {
2983 	/*
2984 	 * if we had to allocate window state on the last bind (because we
2985 	 * didn't have enough pre-allocated space in the handle), free it.
2986 	 */
2987 	if (dma->dp_need_to_free_window) {
2988 		kmem_free(dma->dp_window, dma->dp_window_size);
2989 	}
2990 }
2991 
2992 
2993 /*
2994  * rootnex_init_win()
2995  *    Called in bind slow path during creation of a new window. Initializes
2996  *    window state to default values.
2997  */
2998 /*ARGSUSED*/
2999 static void
3000 rootnex_init_win(ddi_dma_impl_t *hp, rootnex_dma_t *dma,
3001     rootnex_window_t *window, ddi_dma_cookie_t *cookie, off_t cur_offset)
3002 {
3003 	hp->dmai_nwin++;
3004 	window->wd_dosync = B_FALSE;
3005 	window->wd_offset = cur_offset;
3006 	window->wd_size = 0;
3007 	window->wd_first_cookie = cookie;
3008 	window->wd_cookie_cnt = 0;
3009 	window->wd_trim.tr_trim_first = B_FALSE;
3010 	window->wd_trim.tr_trim_last = B_FALSE;
3011 	window->wd_trim.tr_first_copybuf_win = B_FALSE;
3012 	window->wd_trim.tr_last_copybuf_win = B_FALSE;
3013 #if !defined(__amd64)
3014 	window->wd_remap_copybuf = dma->dp_cb_remaping;
3015 #endif
3016 }
3017 
3018 
3019 /*
3020  * rootnex_setup_cookie()
3021  *    Called in the bind slow path when the sgl uses the copy buffer. If any of
3022  *    the sgl uses the copy buffer, we need to go through each cookie, figure
3023  *    out if it uses the copy buffer, and if it does, save away everything we'll
3024  *    need during sync.
3025  */
3026 static void
3027 rootnex_setup_cookie(ddi_dma_obj_t *dmar_object, rootnex_dma_t *dma,
3028     ddi_dma_cookie_t *cookie, off_t cur_offset, size_t *copybuf_used,
3029     page_t **cur_pp)
3030 {
3031 	boolean_t copybuf_sz_power_2;
3032 	rootnex_sglinfo_t *sinfo;
3033 	paddr_t paddr;
3034 	uint_t pidx;
3035 	uint_t pcnt;
3036 	off_t poff;
3037 #if defined(__amd64)
3038 	pfn_t pfn;
3039 #else
3040 	page_t **pplist;
3041 #endif
3042 
3043 	sinfo = &dma->dp_sglinfo;
3044 
3045 	/*
3046 	 * Calculate the page index relative to the start of the buffer. The
3047 	 * index to the current page for our buffer is the offset into the
3048 	 * first page of the buffer plus our current offset into the buffer
3049 	 * itself, shifted of course...
3050 	 */
3051 	pidx = (sinfo->si_buf_offset + cur_offset) >> MMU_PAGESHIFT;
3052 	ASSERT(pidx < sinfo->si_max_pages);
3053 
3054 	/* if this cookie uses the copy buffer */
3055 	if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) {
3056 		/*
3057 		 * NOTE: we know that since this cookie uses the copy buffer, it
3058 		 * is <= MMU_PAGESIZE.
3059 		 */
3060 
3061 		/*
3062 		 * get the offset into the page. For the 64-bit kernel, get the
3063 		 * pfn which we'll use with seg kpm.
3064 		 */
3065 		poff = cookie->dmac_laddress & MMU_PAGEOFFSET;
3066 #if defined(__amd64)
3067 		/* mfn_to_pfn() is a NOP on i86pc */
3068 		pfn = mfn_to_pfn(cookie->dmac_laddress >> MMU_PAGESHIFT);
3069 #endif /* __amd64 */
3070 
3071 		/* figure out if the copybuf size is a power of 2 */
3072 		if (dma->dp_copybuf_size & (dma->dp_copybuf_size - 1)) {
3073 			copybuf_sz_power_2 = B_FALSE;
3074 		} else {
3075 			copybuf_sz_power_2 = B_TRUE;
3076 		}
3077 
3078 		/* This page uses the copy buffer */
3079 		dma->dp_pgmap[pidx].pm_uses_copybuf = B_TRUE;
3080 
3081 		/*
3082 		 * save the copy buffer KVA that we'll use with this page.
3083 		 * if we still fit within the copybuf, it's a simple add.
3084 		 * otherwise, we need to wrap over using & or % accordingly.
3085 		 */
3086 		if ((*copybuf_used + MMU_PAGESIZE) <= dma->dp_copybuf_size) {
3087 			dma->dp_pgmap[pidx].pm_cbaddr = dma->dp_cbaddr +
3088 			    *copybuf_used;
3089 		} else {
3090 			if (copybuf_sz_power_2) {
3091 				dma->dp_pgmap[pidx].pm_cbaddr = (caddr_t)(
3092 				    (uintptr_t)dma->dp_cbaddr +
3093 				    (*copybuf_used &
3094 				    (dma->dp_copybuf_size - 1)));
3095 			} else {
3096 				dma->dp_pgmap[pidx].pm_cbaddr = (caddr_t)(
3097 				    (uintptr_t)dma->dp_cbaddr +
3098 				    (*copybuf_used % dma->dp_copybuf_size));
3099 			}
3100 		}
3101 
3102 		/*
3103 		 * over write the cookie physical address with the address of
3104 		 * the physical address of the copy buffer page that we will
3105 		 * use.
3106 		 */
3107 		paddr = pfn_to_pa(hat_getpfnum(kas.a_hat,
3108 		    dma->dp_pgmap[pidx].pm_cbaddr)) + poff;
3109 
3110 #ifdef __xpv
3111 		/*
3112 		 * If we're dom0, we're using a real device so we need to load
3113 		 * the cookies with MAs instead of PAs.
3114 		 */
3115 		cookie->dmac_laddress = ROOTNEX_PADDR_TO_RBASE(xen_info, paddr);
3116 #else
3117 		cookie->dmac_laddress = paddr;
3118 #endif
3119 
3120 		/* if we have a kernel VA, it's easy, just save that address */
3121 		if ((dmar_object->dmao_type != DMA_OTYP_PAGES) &&
3122 		    (sinfo->si_asp == &kas)) {
3123 			/*
3124 			 * save away the page aligned virtual address of the
3125 			 * driver buffer. Offsets are handled in the sync code.
3126 			 */
3127 			dma->dp_pgmap[pidx].pm_kaddr = (caddr_t)(((uintptr_t)
3128 			    dmar_object->dmao_obj.virt_obj.v_addr + cur_offset)
3129 			    & MMU_PAGEMASK);
3130 #if !defined(__amd64)
3131 			/*
3132 			 * we didn't need to, and will never need to map this
3133 			 * page.
3134 			 */
3135 			dma->dp_pgmap[pidx].pm_mapped = B_FALSE;
3136 #endif
3137 
3138 		/* we don't have a kernel VA. We need one for the bcopy. */
3139 		} else {
3140 #if defined(__amd64)
3141 			/*
3142 			 * for the 64-bit kernel, it's easy. We use seg kpm to
3143 			 * get a Kernel VA for the corresponding pfn.
3144 			 */
3145 			dma->dp_pgmap[pidx].pm_kaddr = hat_kpm_pfn2va(pfn);
3146 #else
3147 			/*
3148 			 * for the 32-bit kernel, this is a pain. First we'll
3149 			 * save away the page_t or user VA for this page. This
3150 			 * is needed in rootnex_dma_win() when we switch to a
3151 			 * new window which requires us to re-map the copy
3152 			 * buffer.
3153 			 */
3154 			pplist = dmar_object->dmao_obj.virt_obj.v_priv;
3155 			if (dmar_object->dmao_type == DMA_OTYP_PAGES) {
3156 				dma->dp_pgmap[pidx].pm_pp = *cur_pp;
3157 				dma->dp_pgmap[pidx].pm_vaddr = NULL;
3158 			} else if (pplist != NULL) {
3159 				dma->dp_pgmap[pidx].pm_pp = pplist[pidx];
3160 				dma->dp_pgmap[pidx].pm_vaddr = NULL;
3161 			} else {
3162 				dma->dp_pgmap[pidx].pm_pp = NULL;
3163 				dma->dp_pgmap[pidx].pm_vaddr = (caddr_t)
3164 				    (((uintptr_t)
3165 				    dmar_object->dmao_obj.virt_obj.v_addr +
3166 				    cur_offset) & MMU_PAGEMASK);
3167 			}
3168 
3169 			/*
3170 			 * save away the page aligned virtual address which was
3171 			 * allocated from the kernel heap arena (taking into
3172 			 * account if we need more copy buffer than we alloced
3173 			 * and use multiple windows to handle this, i.e. &,%).
3174 			 * NOTE: there isn't and physical memory backing up this
3175 			 * virtual address space currently.
3176 			 */
3177 			if ((*copybuf_used + MMU_PAGESIZE) <=
3178 			    dma->dp_copybuf_size) {
3179 				dma->dp_pgmap[pidx].pm_kaddr = (caddr_t)
3180 				    (((uintptr_t)dma->dp_kva + *copybuf_used) &
3181 				    MMU_PAGEMASK);
3182 			} else {
3183 				if (copybuf_sz_power_2) {
3184 					dma->dp_pgmap[pidx].pm_kaddr = (caddr_t)
3185 					    (((uintptr_t)dma->dp_kva +
3186 					    (*copybuf_used &
3187 					    (dma->dp_copybuf_size - 1))) &
3188 					    MMU_PAGEMASK);
3189 				} else {
3190 					dma->dp_pgmap[pidx].pm_kaddr = (caddr_t)
3191 					    (((uintptr_t)dma->dp_kva +
3192 					    (*copybuf_used %
3193 					    dma->dp_copybuf_size)) &
3194 					    MMU_PAGEMASK);
3195 				}
3196 			}
3197 
3198 			/*
3199 			 * if we haven't used up the available copy buffer yet,
3200 			 * map the kva to the physical page.
3201 			 */
3202 			if (!dma->dp_cb_remaping && ((*copybuf_used +
3203 			    MMU_PAGESIZE) <= dma->dp_copybuf_size)) {
3204 				dma->dp_pgmap[pidx].pm_mapped = B_TRUE;
3205 				if (dma->dp_pgmap[pidx].pm_pp != NULL) {
3206 					i86_pp_map(dma->dp_pgmap[pidx].pm_pp,
3207 					    dma->dp_pgmap[pidx].pm_kaddr);
3208 				} else {
3209 					i86_va_map(dma->dp_pgmap[pidx].pm_vaddr,
3210 					    sinfo->si_asp,
3211 					    dma->dp_pgmap[pidx].pm_kaddr);
3212 				}
3213 
3214 			/*
3215 			 * we've used up the available copy buffer, this page
3216 			 * will have to be mapped during rootnex_dma_win() when
3217 			 * we switch to a new window which requires a re-map
3218 			 * the copy buffer. (32-bit kernel only)
3219 			 */
3220 			} else {
3221 				dma->dp_pgmap[pidx].pm_mapped = B_FALSE;
3222 			}
3223 #endif
3224 			/* go to the next page_t */
3225 			if (dmar_object->dmao_type == DMA_OTYP_PAGES) {
3226 				*cur_pp = (*cur_pp)->p_next;
3227 			}
3228 		}
3229 
3230 		/* add to the copy buffer count */
3231 		*copybuf_used += MMU_PAGESIZE;
3232 
3233 	/*
3234 	 * This cookie doesn't use the copy buffer. Walk through the pages this
3235 	 * cookie occupies to reflect this.
3236 	 */
3237 	} else {
3238 		/*
3239 		 * figure out how many pages the cookie occupies. We need to
3240 		 * use the original page offset of the buffer and the cookies
3241 		 * offset in the buffer to do this.
3242 		 */
3243 		poff = (sinfo->si_buf_offset + cur_offset) & MMU_PAGEOFFSET;
3244 		pcnt = mmu_btopr(cookie->dmac_size + poff);
3245 
3246 		while (pcnt > 0) {
3247 #if !defined(__amd64)
3248 			/*
3249 			 * the 32-bit kernel doesn't have seg kpm, so we need
3250 			 * to map in the driver buffer (if it didn't come down
3251 			 * with a kernel VA) on the fly. Since this page doesn't
3252 			 * use the copy buffer, it's not, or will it ever, have
3253 			 * to be mapped in.
3254 			 */
3255 			dma->dp_pgmap[pidx].pm_mapped = B_FALSE;
3256 #endif
3257 			dma->dp_pgmap[pidx].pm_uses_copybuf = B_FALSE;
3258 
3259 			/*
3260 			 * we need to update pidx and cur_pp or we'll loose
3261 			 * track of where we are.
3262 			 */
3263 			if (dmar_object->dmao_type == DMA_OTYP_PAGES) {
3264 				*cur_pp = (*cur_pp)->p_next;
3265 			}
3266 			pidx++;
3267 			pcnt--;
3268 		}
3269 	}
3270 }
3271 
3272 
3273 /*
3274  * rootnex_sgllen_window_boundary()
3275  *    Called in the bind slow path when the next cookie causes us to exceed (in
3276  *    this case == since we start at 0 and sgllen starts at 1) the maximum sgl
3277  *    length supported by the DMA H/W.
3278  */
3279 static int
3280 rootnex_sgllen_window_boundary(ddi_dma_impl_t *hp, rootnex_dma_t *dma,
3281     rootnex_window_t **windowp, ddi_dma_cookie_t *cookie, ddi_dma_attr_t *attr,
3282     off_t cur_offset)
3283 {
3284 	off_t new_offset;
3285 	size_t trim_sz;
3286 	off_t coffset;
3287 
3288 
3289 	/*
3290 	 * if we know we'll never have to trim, it's pretty easy. Just move to
3291 	 * the next window and init it. We're done.
3292 	 */
3293 	if (!dma->dp_trim_required) {
3294 		(*windowp)++;
3295 		rootnex_init_win(hp, dma, *windowp, cookie, cur_offset);
3296 		(*windowp)->wd_cookie_cnt++;
3297 		(*windowp)->wd_size = cookie->dmac_size;
3298 		return (DDI_SUCCESS);
3299 	}
3300 
3301 	/* figure out how much we need to trim from the window */
3302 	ASSERT(attr->dma_attr_granular != 0);
3303 	if (dma->dp_granularity_power_2) {
3304 		trim_sz = (*windowp)->wd_size & (attr->dma_attr_granular - 1);
3305 	} else {
3306 		trim_sz = (*windowp)->wd_size % attr->dma_attr_granular;
3307 	}
3308 
3309 	/* The window's a whole multiple of granularity. We're done */
3310 	if (trim_sz == 0) {
3311 		(*windowp)++;
3312 		rootnex_init_win(hp, dma, *windowp, cookie, cur_offset);
3313 		(*windowp)->wd_cookie_cnt++;
3314 		(*windowp)->wd_size = cookie->dmac_size;
3315 		return (DDI_SUCCESS);
3316 	}
3317 
3318 	/*
3319 	 * The window's not a whole multiple of granularity, since we know this
3320 	 * is due to the sgllen, we need to go back to the last cookie and trim
3321 	 * that one, add the left over part of the old cookie into the new
3322 	 * window, and then add in the new cookie into the new window.
3323 	 */
3324 
3325 	/*
3326 	 * make sure the driver isn't making us do something bad... Trimming and
3327 	 * sgllen == 1 don't go together.
3328 	 */
3329 	if (attr->dma_attr_sgllen == 1) {
3330 		return (DDI_DMA_NOMAPPING);
3331 	}
3332 
3333 	/*
3334 	 * first, setup the current window to account for the trim. Need to go
3335 	 * back to the last cookie for this.
3336 	 */
3337 	cookie--;
3338 	(*windowp)->wd_trim.tr_trim_last = B_TRUE;
3339 	(*windowp)->wd_trim.tr_last_cookie = cookie;
3340 	(*windowp)->wd_trim.tr_last_paddr = cookie->dmac_laddress;
3341 	ASSERT(cookie->dmac_size > trim_sz);
3342 	(*windowp)->wd_trim.tr_last_size = cookie->dmac_size - trim_sz;
3343 	(*windowp)->wd_size -= trim_sz;
3344 
3345 	/* save the buffer offsets for the next window */
3346 	coffset = cookie->dmac_size - trim_sz;
3347 	new_offset = (*windowp)->wd_offset + (*windowp)->wd_size;
3348 
3349 	/*
3350 	 * set this now in case this is the first window. all other cases are
3351 	 * set in dma_win()
3352 	 */
3353 	cookie->dmac_size = (*windowp)->wd_trim.tr_last_size;
3354 
3355 	/*
3356 	 * initialize the next window using what's left over in the previous
3357 	 * cookie.
3358 	 */
3359 	(*windowp)++;
3360 	rootnex_init_win(hp, dma, *windowp, cookie, new_offset);
3361 	(*windowp)->wd_cookie_cnt++;
3362 	(*windowp)->wd_trim.tr_trim_first = B_TRUE;
3363 	(*windowp)->wd_trim.tr_first_paddr = cookie->dmac_laddress + coffset;
3364 	(*windowp)->wd_trim.tr_first_size = trim_sz;
3365 	if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) {
3366 		(*windowp)->wd_dosync = B_TRUE;
3367 	}
3368 
3369 	/*
3370 	 * now go back to the current cookie and add it to the new window. set
3371 	 * the new window size to the what was left over from the previous
3372 	 * cookie and what's in the current cookie.
3373 	 */
3374 	cookie++;
3375 	(*windowp)->wd_cookie_cnt++;
3376 	(*windowp)->wd_size = trim_sz + cookie->dmac_size;
3377 
3378 	/*
3379 	 * trim plus the next cookie could put us over maxxfer (a cookie can be
3380 	 * a max size of maxxfer). Handle that case.
3381 	 */
3382 	if ((*windowp)->wd_size > dma->dp_maxxfer) {
3383 		/*
3384 		 * maxxfer is already a whole multiple of granularity, and this
3385 		 * trim will be <= the previous trim (since a cookie can't be
3386 		 * larger than maxxfer). Make things simple here.
3387 		 */
3388 		trim_sz = (*windowp)->wd_size - dma->dp_maxxfer;
3389 		(*windowp)->wd_trim.tr_trim_last = B_TRUE;
3390 		(*windowp)->wd_trim.tr_last_cookie = cookie;
3391 		(*windowp)->wd_trim.tr_last_paddr = cookie->dmac_laddress;
3392 		(*windowp)->wd_trim.tr_last_size = cookie->dmac_size - trim_sz;
3393 		(*windowp)->wd_size -= trim_sz;
3394 		ASSERT((*windowp)->wd_size == dma->dp_maxxfer);
3395 
3396 		/* save the buffer offsets for the next window */
3397 		coffset = cookie->dmac_size - trim_sz;
3398 		new_offset = (*windowp)->wd_offset + (*windowp)->wd_size;
3399 
3400 		/* setup the next window */
3401 		(*windowp)++;
3402 		rootnex_init_win(hp, dma, *windowp, cookie, new_offset);
3403 		(*windowp)->wd_cookie_cnt++;
3404 		(*windowp)->wd_trim.tr_trim_first = B_TRUE;
3405 		(*windowp)->wd_trim.tr_first_paddr = cookie->dmac_laddress +
3406 		    coffset;
3407 		(*windowp)->wd_trim.tr_first_size = trim_sz;
3408 	}
3409 
3410 	return (DDI_SUCCESS);
3411 }
3412 
3413 
3414 /*
3415  * rootnex_copybuf_window_boundary()
3416  *    Called in bind slowpath when we get to a window boundary because we used
3417  *    up all the copy buffer that we have.
3418  */
3419 static int
3420 rootnex_copybuf_window_boundary(ddi_dma_impl_t *hp, rootnex_dma_t *dma,
3421     rootnex_window_t **windowp, ddi_dma_cookie_t *cookie, off_t cur_offset,
3422     size_t *copybuf_used)
3423 {
3424 	rootnex_sglinfo_t *sinfo;
3425 	off_t new_offset;
3426 	size_t trim_sz;
3427 	paddr_t paddr;
3428 	off_t coffset;
3429 	uint_t pidx;
3430 	off_t poff;
3431 
3432 
3433 	sinfo = &dma->dp_sglinfo;
3434 
3435 	/*
3436 	 * the copy buffer should be a whole multiple of page size. We know that
3437 	 * this cookie is <= MMU_PAGESIZE.
3438 	 */
3439 	ASSERT(cookie->dmac_size <= MMU_PAGESIZE);
3440 
3441 	/*
3442 	 * from now on, all new windows in this bind need to be re-mapped during
3443 	 * ddi_dma_getwin() (32-bit kernel only). i.e. we ran out out copybuf
3444 	 * space...
3445 	 */
3446 #if !defined(__amd64)
3447 	dma->dp_cb_remaping = B_TRUE;
3448 #endif
3449 
3450 	/* reset copybuf used */
3451 	*copybuf_used = 0;
3452 
3453 	/*
3454 	 * if we don't have to trim (since granularity is set to 1), go to the
3455 	 * next window and add the current cookie to it. We know the current
3456 	 * cookie uses the copy buffer since we're in this code path.
3457 	 */
3458 	if (!dma->dp_trim_required) {
3459 		(*windowp)++;
3460 		rootnex_init_win(hp, dma, *windowp, cookie, cur_offset);
3461 
3462 		/* Add this cookie to the new window */
3463 		(*windowp)->wd_cookie_cnt++;
3464 		(*windowp)->wd_size += cookie->dmac_size;
3465 		*copybuf_used += MMU_PAGESIZE;
3466 		return (DDI_SUCCESS);
3467 	}
3468 
3469 	/*
3470 	 * *** may need to trim, figure it out.
3471 	 */
3472 
3473 	/* figure out how much we need to trim from the window */
3474 	if (dma->dp_granularity_power_2) {
3475 		trim_sz = (*windowp)->wd_size &
3476 		    (hp->dmai_attr.dma_attr_granular - 1);
3477 	} else {
3478 		trim_sz = (*windowp)->wd_size % hp->dmai_attr.dma_attr_granular;
3479 	}
3480 
3481 	/*
3482 	 * if the window's a whole multiple of granularity, go to the next
3483 	 * window, init it, then add in the current cookie. We know the current
3484 	 * cookie uses the copy buffer since we're in this code path.
3485 	 */
3486 	if (trim_sz == 0) {
3487 		(*windowp)++;
3488 		rootnex_init_win(hp, dma, *windowp, cookie, cur_offset);
3489 
3490 		/* Add this cookie to the new window */
3491 		(*windowp)->wd_cookie_cnt++;
3492 		(*windowp)->wd_size += cookie->dmac_size;
3493 		*copybuf_used += MMU_PAGESIZE;
3494 		return (DDI_SUCCESS);
3495 	}
3496 
3497 	/*
3498 	 * *** We figured it out, we definitly need to trim
3499 	 */
3500 
3501 	/*
3502 	 * make sure the driver isn't making us do something bad...
3503 	 * Trimming and sgllen == 1 don't go together.
3504 	 */
3505 	if (hp->dmai_attr.dma_attr_sgllen == 1) {
3506 		return (DDI_DMA_NOMAPPING);
3507 	}
3508 
3509 	/*
3510 	 * first, setup the current window to account for the trim. Need to go
3511 	 * back to the last cookie for this. Some of the last cookie will be in
3512 	 * the current window, and some of the last cookie will be in the new
3513 	 * window. All of the current cookie will be in the new window.
3514 	 */
3515 	cookie--;
3516 	(*windowp)->wd_trim.tr_trim_last = B_TRUE;
3517 	(*windowp)->wd_trim.tr_last_cookie = cookie;
3518 	(*windowp)->wd_trim.tr_last_paddr = cookie->dmac_laddress;
3519 	ASSERT(cookie->dmac_size > trim_sz);
3520 	(*windowp)->wd_trim.tr_last_size = cookie->dmac_size - trim_sz;
3521 	(*windowp)->wd_size -= trim_sz;
3522 
3523 	/*
3524 	 * we're trimming the last cookie (not the current cookie). So that
3525 	 * last cookie may have or may not have been using the copy buffer (
3526 	 * we know the cookie passed in uses the copy buffer since we're in
3527 	 * this code path).
3528 	 *
3529 	 * If the last cookie doesn't use the copy buffer, nothing special to
3530 	 * do. However, if it does uses the copy buffer, it will be both the
3531 	 * last page in the current window and the first page in the next
3532 	 * window. Since we are reusing the copy buffer (and KVA space on the
3533 	 * 32-bit kernel), this page will use the end of the copy buffer in the
3534 	 * current window, and the start of the copy buffer in the next window.
3535 	 * Track that info... The cookie physical address was already set to
3536 	 * the copy buffer physical address in setup_cookie..
3537 	 */
3538 	if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) {
3539 		pidx = (sinfo->si_buf_offset + (*windowp)->wd_offset +
3540 		    (*windowp)->wd_size) >> MMU_PAGESHIFT;
3541 		(*windowp)->wd_trim.tr_last_copybuf_win = B_TRUE;
3542 		(*windowp)->wd_trim.tr_last_pidx = pidx;
3543 		(*windowp)->wd_trim.tr_last_cbaddr =
3544 		    dma->dp_pgmap[pidx].pm_cbaddr;
3545 #if !defined(__amd64)
3546 		(*windowp)->wd_trim.tr_last_kaddr =
3547 		    dma->dp_pgmap[pidx].pm_kaddr;
3548 #endif
3549 	}
3550 
3551 	/* save the buffer offsets for the next window */
3552 	coffset = cookie->dmac_size - trim_sz;
3553 	new_offset = (*windowp)->wd_offset + (*windowp)->wd_size;
3554 
3555 	/*
3556 	 * set this now in case this is the first window. all other cases are
3557 	 * set in dma_win()
3558 	 */
3559 	cookie->dmac_size = (*windowp)->wd_trim.tr_last_size;
3560 
3561 	/*
3562 	 * initialize the next window using what's left over in the previous
3563 	 * cookie.
3564 	 */
3565 	(*windowp)++;
3566 	rootnex_init_win(hp, dma, *windowp, cookie, new_offset);
3567 	(*windowp)->wd_cookie_cnt++;
3568 	(*windowp)->wd_trim.tr_trim_first = B_TRUE;
3569 	(*windowp)->wd_trim.tr_first_paddr = cookie->dmac_laddress + coffset;
3570 	(*windowp)->wd_trim.tr_first_size = trim_sz;
3571 
3572 	/*
3573 	 * again, we're tracking if the last cookie uses the copy buffer.
3574 	 * read the comment above for more info on why we need to track
3575 	 * additional state.
3576 	 *
3577 	 * For the first cookie in the new window, we need reset the physical
3578 	 * address to DMA into to the start of the copy buffer plus any
3579 	 * initial page offset which may be present.
3580 	 */
3581 	if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) {
3582 		(*windowp)->wd_dosync = B_TRUE;
3583 		(*windowp)->wd_trim.tr_first_copybuf_win = B_TRUE;
3584 		(*windowp)->wd_trim.tr_first_pidx = pidx;
3585 		(*windowp)->wd_trim.tr_first_cbaddr = dma->dp_cbaddr;
3586 		poff = (*windowp)->wd_trim.tr_first_paddr & MMU_PAGEOFFSET;
3587 
3588 		paddr = pfn_to_pa(hat_getpfnum(kas.a_hat, dma->dp_cbaddr)) +
3589 		    poff;
3590 #ifdef __xpv
3591 		/*
3592 		 * If we're dom0, we're using a real device so we need to load
3593 		 * the cookies with MAs instead of PAs.
3594 		 */
3595 		(*windowp)->wd_trim.tr_first_paddr =
3596 		    ROOTNEX_PADDR_TO_RBASE(xen_info, paddr);
3597 #else
3598 		(*windowp)->wd_trim.tr_first_paddr = paddr;
3599 #endif
3600 
3601 #if !defined(__amd64)
3602 		(*windowp)->wd_trim.tr_first_kaddr = dma->dp_kva;
3603 #endif
3604 		/* account for the cookie copybuf usage in the new window */
3605 		*copybuf_used += MMU_PAGESIZE;
3606 
3607 		/*
3608 		 * every piece of code has to have a hack, and here is this
3609 		 * ones :-)
3610 		 *
3611 		 * There is a complex interaction between setup_cookie and the
3612 		 * copybuf window boundary. The complexity had to be in either
3613 		 * the maxxfer window, or the copybuf window, and I chose the
3614 		 * copybuf code.
3615 		 *
3616 		 * So in this code path, we have taken the last cookie,
3617 		 * virtually broken it in half due to the trim, and it happens
3618 		 * to use the copybuf which further complicates life. At the
3619 		 * same time, we have already setup the current cookie, which
3620 		 * is now wrong. More background info: the current cookie uses
3621 		 * the copybuf, so it is only a page long max. So we need to
3622 		 * fix the current cookies copy buffer address, physical
3623 		 * address, and kva for the 32-bit kernel. We due this by
3624 		 * bumping them by page size (of course, we can't due this on
3625 		 * the physical address since the copy buffer may not be
3626 		 * physically contiguous).
3627 		 */
3628 		cookie++;
3629 		dma->dp_pgmap[pidx + 1].pm_cbaddr += MMU_PAGESIZE;
3630 		poff = cookie->dmac_laddress & MMU_PAGEOFFSET;
3631 
3632 		paddr = pfn_to_pa(hat_getpfnum(kas.a_hat,
3633 		    dma->dp_pgmap[pidx + 1].pm_cbaddr)) + poff;
3634 #ifdef __xpv
3635 		/*
3636 		 * If we're dom0, we're using a real device so we need to load
3637 		 * the cookies with MAs instead of PAs.
3638 		 */
3639 		cookie->dmac_laddress = ROOTNEX_PADDR_TO_RBASE(xen_info, paddr);
3640 #else
3641 		cookie->dmac_laddress = paddr;
3642 #endif
3643 
3644 #if !defined(__amd64)
3645 		ASSERT(dma->dp_pgmap[pidx + 1].pm_mapped == B_FALSE);
3646 		dma->dp_pgmap[pidx + 1].pm_kaddr += MMU_PAGESIZE;
3647 #endif
3648 	} else {
3649 		/* go back to the current cookie */
3650 		cookie++;
3651 	}
3652 
3653 	/*
3654 	 * add the current cookie to the new window. set the new window size to
3655 	 * the what was left over from the previous cookie and what's in the
3656 	 * current cookie.
3657 	 */
3658 	(*windowp)->wd_cookie_cnt++;
3659 	(*windowp)->wd_size = trim_sz + cookie->dmac_size;
3660 	ASSERT((*windowp)->wd_size < dma->dp_maxxfer);
3661 
3662 	/*
3663 	 * we know that the cookie passed in always uses the copy buffer. We
3664 	 * wouldn't be here if it didn't.
3665 	 */
3666 	*copybuf_used += MMU_PAGESIZE;
3667 
3668 	return (DDI_SUCCESS);
3669 }
3670 
3671 
3672 /*
3673  * rootnex_maxxfer_window_boundary()
3674  *    Called in bind slowpath when we get to a window boundary because we will
3675  *    go over maxxfer.
3676  */
3677 static int
3678 rootnex_maxxfer_window_boundary(ddi_dma_impl_t *hp, rootnex_dma_t *dma,
3679     rootnex_window_t **windowp, ddi_dma_cookie_t *cookie)
3680 {
3681 	size_t dmac_size;
3682 	off_t new_offset;
3683 	size_t trim_sz;
3684 	off_t coffset;
3685 
3686 
3687 	/*
3688 	 * calculate how much we have to trim off of the current cookie to equal
3689 	 * maxxfer. We don't have to account for granularity here since our
3690 	 * maxxfer already takes that into account.
3691 	 */
3692 	trim_sz = ((*windowp)->wd_size + cookie->dmac_size) - dma->dp_maxxfer;
3693 	ASSERT(trim_sz <= cookie->dmac_size);
3694 	ASSERT(trim_sz <= dma->dp_maxxfer);
3695 
3696 	/* save cookie size since we need it later and we might change it */
3697 	dmac_size = cookie->dmac_size;
3698 
3699 	/*
3700 	 * if we're not trimming the entire cookie, setup the current window to
3701 	 * account for the trim.
3702 	 */
3703 	if (trim_sz < cookie->dmac_size) {
3704 		(*windowp)->wd_cookie_cnt++;
3705 		(*windowp)->wd_trim.tr_trim_last = B_TRUE;
3706 		(*windowp)->wd_trim.tr_last_cookie = cookie;
3707 		(*windowp)->wd_trim.tr_last_paddr = cookie->dmac_laddress;
3708 		(*windowp)->wd_trim.tr_last_size = cookie->dmac_size - trim_sz;
3709 		(*windowp)->wd_size = dma->dp_maxxfer;
3710 
3711 		/*
3712 		 * set the adjusted cookie size now in case this is the first
3713 		 * window. All other windows are taken care of in get win
3714 		 */
3715 		cookie->dmac_size = (*windowp)->wd_trim.tr_last_size;
3716 	}
3717 
3718 	/*
3719 	 * coffset is the current offset within the cookie, new_offset is the
3720 	 * current offset with the entire buffer.
3721 	 */
3722 	coffset = dmac_size - trim_sz;
3723 	new_offset = (*windowp)->wd_offset + (*windowp)->wd_size;
3724 
3725 	/* initialize the next window */
3726 	(*windowp)++;
3727 	rootnex_init_win(hp, dma, *windowp, cookie, new_offset);
3728 	(*windowp)->wd_cookie_cnt++;
3729 	(*windowp)->wd_size = trim_sz;
3730 	if (trim_sz < dmac_size) {
3731 		(*windowp)->wd_trim.tr_trim_first = B_TRUE;
3732 		(*windowp)->wd_trim.tr_first_paddr = cookie->dmac_laddress +
3733 		    coffset;
3734 		(*windowp)->wd_trim.tr_first_size = trim_sz;
3735 	}
3736 
3737 	return (DDI_SUCCESS);
3738 }
3739 
3740 
3741 /*
3742  * rootnex_dma_sync()
3743  *    called from ddi_dma_sync() if DMP_NOSYNC is not set in hp->dmai_rflags.
3744  *    We set DMP_NOSYNC if we're not using the copy buffer. If DMP_NOSYNC
3745  *    is set, ddi_dma_sync() returns immediately passing back success.
3746  */
3747 /*ARGSUSED*/
3748 static int
3749 rootnex_dma_sync(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle,
3750     off_t off, size_t len, uint_t cache_flags)
3751 {
3752 	rootnex_sglinfo_t *sinfo;
3753 	rootnex_pgmap_t *cbpage;
3754 	rootnex_window_t *win;
3755 	ddi_dma_impl_t *hp;
3756 	rootnex_dma_t *dma;
3757 	caddr_t fromaddr;
3758 	caddr_t toaddr;
3759 	uint_t psize;
3760 	off_t offset;
3761 	uint_t pidx;
3762 	size_t size;
3763 	off_t poff;
3764 	int e;
3765 
3766 
3767 	hp = (ddi_dma_impl_t *)handle;
3768 	dma = (rootnex_dma_t *)hp->dmai_private;
3769 	sinfo = &dma->dp_sglinfo;
3770 
3771 	/*
3772 	 * if we don't have any windows, we don't need to sync. A copybuf
3773 	 * will cause us to have at least one window.
3774 	 */
3775 	if (dma->dp_window == NULL) {
3776 		return (DDI_SUCCESS);
3777 	}
3778 
3779 	/* This window may not need to be sync'd */
3780 	win = &dma->dp_window[dma->dp_current_win];
3781 	if (!win->wd_dosync) {
3782 		return (DDI_SUCCESS);
3783 	}
3784 
3785 	/* handle off and len special cases */
3786 	if ((off == 0) || (rootnex_sync_ignore_params)) {
3787 		offset = win->wd_offset;
3788 	} else {
3789 		offset = off;
3790 	}
3791 	if ((len == 0) || (rootnex_sync_ignore_params)) {
3792 		size = win->wd_size;
3793 	} else {
3794 		size = len;
3795 	}
3796 
3797 	/* check the sync args to make sure they make a little sense */
3798 	if (rootnex_sync_check_parms) {
3799 		e = rootnex_valid_sync_parms(hp, win, offset, size,
3800 		    cache_flags);
3801 		if (e != DDI_SUCCESS) {
3802 			ROOTNEX_PROF_INC(&rootnex_cnt[ROOTNEX_CNT_SYNC_FAIL]);
3803 			return (DDI_FAILURE);
3804 		}
3805 	}
3806 
3807 	/*
3808 	 * special case the first page to handle the offset into the page. The
3809 	 * offset to the current page for our buffer is the offset into the
3810 	 * first page of the buffer plus our current offset into the buffer
3811 	 * itself, masked of course.
3812 	 */
3813 	poff = (sinfo->si_buf_offset + offset) & MMU_PAGEOFFSET;
3814 	psize = MIN((MMU_PAGESIZE - poff), size);
3815 
3816 	/* go through all the pages that we want to sync */
3817 	while (size > 0) {
3818 		/*
3819 		 * Calculate the page index relative to the start of the buffer.
3820 		 * The index to the current page for our buffer is the offset
3821 		 * into the first page of the buffer plus our current offset
3822 		 * into the buffer itself, shifted of course...
3823 		 */
3824 		pidx = (sinfo->si_buf_offset + offset) >> MMU_PAGESHIFT;
3825 		ASSERT(pidx < sinfo->si_max_pages);
3826 
3827 		/*
3828 		 * if this page uses the copy buffer, we need to sync it,
3829 		 * otherwise, go on to the next page.
3830 		 */
3831 		cbpage = &dma->dp_pgmap[pidx];
3832 		ASSERT((cbpage->pm_uses_copybuf == B_TRUE) ||
3833 		    (cbpage->pm_uses_copybuf == B_FALSE));
3834 		if (cbpage->pm_uses_copybuf) {
3835 			/* cbaddr and kaddr should be page aligned */
3836 			ASSERT(((uintptr_t)cbpage->pm_cbaddr &
3837 			    MMU_PAGEOFFSET) == 0);
3838 			ASSERT(((uintptr_t)cbpage->pm_kaddr &
3839 			    MMU_PAGEOFFSET) == 0);
3840 
3841 			/*
3842 			 * if we're copying for the device, we are going to
3843 			 * copy from the drivers buffer and to the rootnex
3844 			 * allocated copy buffer.
3845 			 */
3846 			if (cache_flags == DDI_DMA_SYNC_FORDEV) {
3847 				fromaddr = cbpage->pm_kaddr + poff;
3848 				toaddr = cbpage->pm_cbaddr + poff;
3849 				DTRACE_PROBE2(rootnex__sync__dev,
3850 				    dev_info_t *, dma->dp_dip, size_t, psize);
3851 
3852 			/*
3853 			 * if we're copying for the cpu/kernel, we are going to
3854 			 * copy from the rootnex allocated copy buffer to the
3855 			 * drivers buffer.
3856 			 */
3857 			} else {
3858 				fromaddr = cbpage->pm_cbaddr + poff;
3859 				toaddr = cbpage->pm_kaddr + poff;
3860 				DTRACE_PROBE2(rootnex__sync__cpu,
3861 				    dev_info_t *, dma->dp_dip, size_t, psize);
3862 			}
3863 
3864 			bcopy(fromaddr, toaddr, psize);
3865 		}
3866 
3867 		/*
3868 		 * decrement size until we're done, update our offset into the
3869 		 * buffer, and get the next page size.
3870 		 */
3871 		size -= psize;
3872 		offset += psize;
3873 		psize = MIN(MMU_PAGESIZE, size);
3874 
3875 		/* page offset is zero for the rest of this loop */
3876 		poff = 0;
3877 	}
3878 
3879 	return (DDI_SUCCESS);
3880 }
3881 
3882 
3883 /*
3884  * rootnex_valid_sync_parms()
3885  *    checks the parameters passed to sync to verify they are correct.
3886  */
3887 static int
3888 rootnex_valid_sync_parms(ddi_dma_impl_t *hp, rootnex_window_t *win,
3889     off_t offset, size_t size, uint_t cache_flags)
3890 {
3891 	off_t woffset;
3892 
3893 
3894 	/*
3895 	 * the first part of the test to make sure the offset passed in is
3896 	 * within the window.
3897 	 */
3898 	if (offset < win->wd_offset) {
3899 		return (DDI_FAILURE);
3900 	}
3901 
3902 	/*
3903 	 * second and last part of the test to make sure the offset and length
3904 	 * passed in is within the window.
3905 	 */
3906 	woffset = offset - win->wd_offset;
3907 	if ((woffset + size) > win->wd_size) {
3908 		return (DDI_FAILURE);
3909 	}
3910 
3911 	/*
3912 	 * if we are sync'ing for the device, the DDI_DMA_WRITE flag should
3913 	 * be set too.
3914 	 */
3915 	if ((cache_flags == DDI_DMA_SYNC_FORDEV) &&
3916 	    (hp->dmai_rflags & DDI_DMA_WRITE)) {
3917 		return (DDI_SUCCESS);
3918 	}
3919 
3920 	/*
3921 	 * at this point, either DDI_DMA_SYNC_FORCPU or DDI_DMA_SYNC_FORKERNEL
3922 	 * should be set. Also DDI_DMA_READ should be set in the flags.
3923 	 */
3924 	if (((cache_flags == DDI_DMA_SYNC_FORCPU) ||
3925 	    (cache_flags == DDI_DMA_SYNC_FORKERNEL)) &&
3926 	    (hp->dmai_rflags & DDI_DMA_READ)) {
3927 		return (DDI_SUCCESS);
3928 	}
3929 
3930 	return (DDI_FAILURE);
3931 }
3932 
3933 
3934 /*
3935  * rootnex_dma_win()
3936  *    called from ddi_dma_getwin()
3937  */
3938 /*ARGSUSED*/
3939 static int
3940 rootnex_dma_win(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle,
3941     uint_t win, off_t *offp, size_t *lenp, ddi_dma_cookie_t *cookiep,
3942     uint_t *ccountp)
3943 {
3944 	rootnex_window_t *window;
3945 	rootnex_trim_t *trim;
3946 	ddi_dma_impl_t *hp;
3947 	rootnex_dma_t *dma;
3948 #if !defined(__amd64)
3949 	rootnex_sglinfo_t *sinfo;
3950 	rootnex_pgmap_t *pmap;
3951 	uint_t pidx;
3952 	uint_t pcnt;
3953 	off_t poff;
3954 	int i;
3955 #endif
3956 
3957 
3958 	hp = (ddi_dma_impl_t *)handle;
3959 	dma = (rootnex_dma_t *)hp->dmai_private;
3960 #if !defined(__amd64)
3961 	sinfo = &dma->dp_sglinfo;
3962 #endif
3963 
3964 	/* If we try and get a window which doesn't exist, return failure */
3965 	if (win >= hp->dmai_nwin) {
3966 		ROOTNEX_PROF_INC(&rootnex_cnt[ROOTNEX_CNT_GETWIN_FAIL]);
3967 		return (DDI_FAILURE);
3968 	}
3969 
3970 	/*
3971 	 * if we don't have any windows, and they're asking for the first
3972 	 * window, setup the cookie pointer to the first cookie in the bind.
3973 	 * setup our return values, then increment the cookie since we return
3974 	 * the first cookie on the stack.
3975 	 */
3976 	if (dma->dp_window == NULL) {
3977 		if (win != 0) {
3978 			ROOTNEX_PROF_INC(&rootnex_cnt[ROOTNEX_CNT_GETWIN_FAIL]);
3979 			return (DDI_FAILURE);
3980 		}
3981 		hp->dmai_cookie = dma->dp_cookies;
3982 		*offp = 0;
3983 		*lenp = dma->dp_dma.dmao_size;
3984 		*ccountp = dma->dp_sglinfo.si_sgl_size;
3985 		*cookiep = hp->dmai_cookie[0];
3986 		hp->dmai_cookie++;
3987 		return (DDI_SUCCESS);
3988 	}
3989 
3990 	/* sync the old window before moving on to the new one */
3991 	window = &dma->dp_window[dma->dp_current_win];
3992 	if ((window->wd_dosync) && (hp->dmai_rflags & DDI_DMA_READ)) {
3993 		(void) rootnex_dma_sync(dip, rdip, handle, 0, 0,
3994 		    DDI_DMA_SYNC_FORCPU);
3995 	}
3996 
3997 #if !defined(__amd64)
3998 	/*
3999 	 * before we move to the next window, if we need to re-map, unmap all
4000 	 * the pages in this window.
4001 	 */
4002 	if (dma->dp_cb_remaping) {
4003 		/*
4004 		 * If we switch to this window again, we'll need to map in
4005 		 * on the fly next time.
4006 		 */
4007 		window->wd_remap_copybuf = B_TRUE;
4008 
4009 		/*
4010 		 * calculate the page index into the buffer where this window
4011 		 * starts, and the number of pages this window takes up.
4012 		 */
4013 		pidx = (sinfo->si_buf_offset + window->wd_offset) >>
4014 		    MMU_PAGESHIFT;
4015 		poff = (sinfo->si_buf_offset + window->wd_offset) &
4016 		    MMU_PAGEOFFSET;
4017 		pcnt = mmu_btopr(window->wd_size + poff);
4018 		ASSERT((pidx + pcnt) <= sinfo->si_max_pages);
4019 
4020 		/* unmap pages which are currently mapped in this window */
4021 		for (i = 0; i < pcnt; i++) {
4022 			if (dma->dp_pgmap[pidx].pm_mapped) {
4023 				hat_unload(kas.a_hat,
4024 				    dma->dp_pgmap[pidx].pm_kaddr, MMU_PAGESIZE,
4025 				    HAT_UNLOAD);
4026 				dma->dp_pgmap[pidx].pm_mapped = B_FALSE;
4027 			}
4028 			pidx++;
4029 		}
4030 	}
4031 #endif
4032 
4033 	/*
4034 	 * Move to the new window.
4035 	 * NOTE: current_win must be set for sync to work right
4036 	 */
4037 	dma->dp_current_win = win;
4038 	window = &dma->dp_window[win];
4039 
4040 	/* if needed, adjust the first and/or last cookies for trim */
4041 	trim = &window->wd_trim;
4042 	if (trim->tr_trim_first) {
4043 		window->wd_first_cookie->dmac_laddress = trim->tr_first_paddr;
4044 		window->wd_first_cookie->dmac_size = trim->tr_first_size;
4045 #if !defined(__amd64)
4046 		window->wd_first_cookie->dmac_type =
4047 		    (window->wd_first_cookie->dmac_type &
4048 		    ROOTNEX_USES_COPYBUF) + window->wd_offset;
4049 #endif
4050 		if (trim->tr_first_copybuf_win) {
4051 			dma->dp_pgmap[trim->tr_first_pidx].pm_cbaddr =
4052 			    trim->tr_first_cbaddr;
4053 #if !defined(__amd64)
4054 			dma->dp_pgmap[trim->tr_first_pidx].pm_kaddr =
4055 			    trim->tr_first_kaddr;
4056 #endif
4057 		}
4058 	}
4059 	if (trim->tr_trim_last) {
4060 		trim->tr_last_cookie->dmac_laddress = trim->tr_last_paddr;
4061 		trim->tr_last_cookie->dmac_size = trim->tr_last_size;
4062 		if (trim->tr_last_copybuf_win) {
4063 			dma->dp_pgmap[trim->tr_last_pidx].pm_cbaddr =
4064 			    trim->tr_last_cbaddr;
4065 #if !defined(__amd64)
4066 			dma->dp_pgmap[trim->tr_last_pidx].pm_kaddr =
4067 			    trim->tr_last_kaddr;
4068 #endif
4069 		}
4070 	}
4071 
4072 	/*
4073 	 * setup the cookie pointer to the first cookie in the window. setup
4074 	 * our return values, then increment the cookie since we return the
4075 	 * first cookie on the stack.
4076 	 */
4077 	hp->dmai_cookie = window->wd_first_cookie;
4078 	*offp = window->wd_offset;
4079 	*lenp = window->wd_size;
4080 	*ccountp = window->wd_cookie_cnt;
4081 	*cookiep = hp->dmai_cookie[0];
4082 	hp->dmai_cookie++;
4083 
4084 #if !defined(__amd64)
4085 	/* re-map copybuf if required for this window */
4086 	if (dma->dp_cb_remaping) {
4087 		/*
4088 		 * calculate the page index into the buffer where this
4089 		 * window starts.
4090 		 */
4091 		pidx = (sinfo->si_buf_offset + window->wd_offset) >>
4092 		    MMU_PAGESHIFT;
4093 		ASSERT(pidx < sinfo->si_max_pages);
4094 
4095 		/*
4096 		 * the first page can get unmapped if it's shared with the
4097 		 * previous window. Even if the rest of this window is already
4098 		 * mapped in, we need to still check this one.
4099 		 */
4100 		pmap = &dma->dp_pgmap[pidx];
4101 		if ((pmap->pm_uses_copybuf) && (pmap->pm_mapped == B_FALSE)) {
4102 			if (pmap->pm_pp != NULL) {
4103 				pmap->pm_mapped = B_TRUE;
4104 				i86_pp_map(pmap->pm_pp, pmap->pm_kaddr);
4105 			} else if (pmap->pm_vaddr != NULL) {
4106 				pmap->pm_mapped = B_TRUE;
4107 				i86_va_map(pmap->pm_vaddr, sinfo->si_asp,
4108 				    pmap->pm_kaddr);
4109 			}
4110 		}
4111 		pidx++;
4112 
4113 		/* map in the rest of the pages if required */
4114 		if (window->wd_remap_copybuf) {
4115 			window->wd_remap_copybuf = B_FALSE;
4116 
4117 			/* figure out many pages this window takes up */
4118 			poff = (sinfo->si_buf_offset + window->wd_offset) &
4119 			    MMU_PAGEOFFSET;
4120 			pcnt = mmu_btopr(window->wd_size + poff);
4121 			ASSERT(((pidx - 1) + pcnt) <= sinfo->si_max_pages);
4122 
4123 			/* map pages which require it */
4124 			for (i = 1; i < pcnt; i++) {
4125 				pmap = &dma->dp_pgmap[pidx];
4126 				if (pmap->pm_uses_copybuf) {
4127 					ASSERT(pmap->pm_mapped == B_FALSE);
4128 					if (pmap->pm_pp != NULL) {
4129 						pmap->pm_mapped = B_TRUE;
4130 						i86_pp_map(pmap->pm_pp,
4131 						    pmap->pm_kaddr);
4132 					} else if (pmap->pm_vaddr != NULL) {
4133 						pmap->pm_mapped = B_TRUE;
4134 						i86_va_map(pmap->pm_vaddr,
4135 						    sinfo->si_asp,
4136 						    pmap->pm_kaddr);
4137 					}
4138 				}
4139 				pidx++;
4140 			}
4141 		}
4142 	}
4143 #endif
4144 
4145 	/* if the new window uses the copy buffer, sync it for the device */
4146 	if ((window->wd_dosync) && (hp->dmai_rflags & DDI_DMA_WRITE)) {
4147 		(void) rootnex_dma_sync(dip, rdip, handle, 0, 0,
4148 		    DDI_DMA_SYNC_FORDEV);
4149 	}
4150 
4151 	return (DDI_SUCCESS);
4152 }
4153 
4154 
4155 
4156 /*
4157  * ************************
4158  *  obsoleted dma routines
4159  * ************************
4160  */
4161 
4162 /*
4163  * rootnex_dma_map()
4164  *    called from ddi_dma_setup()
4165  */
4166 /* ARGSUSED */
4167 static int
4168 rootnex_dma_map(dev_info_t *dip, dev_info_t *rdip, struct ddi_dma_req *dmareq,
4169     ddi_dma_handle_t *handlep)
4170 {
4171 #if defined(__amd64)
4172 	/*
4173 	 * this interface is not supported in 64-bit x86 kernel. See comment in
4174 	 * rootnex_dma_mctl()
4175 	 */
4176 	return (DDI_DMA_NORESOURCES);
4177 
4178 #else /* 32-bit x86 kernel */
4179 	ddi_dma_handle_t *lhandlep;
4180 	ddi_dma_handle_t lhandle;
4181 	ddi_dma_cookie_t cookie;
4182 	ddi_dma_attr_t dma_attr;
4183 	ddi_dma_lim_t *dma_lim;
4184 	uint_t ccnt;
4185 	int e;
4186 
4187 
4188 	/*
4189 	 * if the driver is just testing to see if it's possible to do the bind,
4190 	 * we'll use local state. Otherwise, use the handle pointer passed in.
4191 	 */
4192 	if (handlep == NULL) {
4193 		lhandlep = &lhandle;
4194 	} else {
4195 		lhandlep = handlep;
4196 	}
4197 
4198 	/* convert the limit structure to a dma_attr one */
4199 	dma_lim = dmareq->dmar_limits;
4200 	dma_attr.dma_attr_version = DMA_ATTR_V0;
4201 	dma_attr.dma_attr_addr_lo = dma_lim->dlim_addr_lo;
4202 	dma_attr.dma_attr_addr_hi = dma_lim->dlim_addr_hi;
4203 	dma_attr.dma_attr_minxfer = dma_lim->dlim_minxfer;
4204 	dma_attr.dma_attr_seg = dma_lim->dlim_adreg_max;
4205 	dma_attr.dma_attr_count_max = dma_lim->dlim_ctreg_max;
4206 	dma_attr.dma_attr_granular = dma_lim->dlim_granular;
4207 	dma_attr.dma_attr_sgllen = dma_lim->dlim_sgllen;
4208 	dma_attr.dma_attr_maxxfer = dma_lim->dlim_reqsize;
4209 	dma_attr.dma_attr_burstsizes = dma_lim->dlim_burstsizes;
4210 	dma_attr.dma_attr_align = MMU_PAGESIZE;
4211 	dma_attr.dma_attr_flags = 0;
4212 
4213 	e = rootnex_dma_allochdl(dip, rdip, &dma_attr, dmareq->dmar_fp,
4214 	    dmareq->dmar_arg, lhandlep);
4215 	if (e != DDI_SUCCESS) {
4216 		return (e);
4217 	}
4218 
4219 	e = rootnex_dma_bindhdl(dip, rdip, *lhandlep, dmareq, &cookie, &ccnt);
4220 	if ((e != DDI_DMA_MAPPED) && (e != DDI_DMA_PARTIAL_MAP)) {
4221 		(void) rootnex_dma_freehdl(dip, rdip, *lhandlep);
4222 		return (e);
4223 	}
4224 
4225 	/*
4226 	 * if the driver is just testing to see if it's possible to do the bind,
4227 	 * free up the local state and return the result.
4228 	 */
4229 	if (handlep == NULL) {
4230 		(void) rootnex_dma_unbindhdl(dip, rdip, *lhandlep);
4231 		(void) rootnex_dma_freehdl(dip, rdip, *lhandlep);
4232 		if (e == DDI_DMA_MAPPED) {
4233 			return (DDI_DMA_MAPOK);
4234 		} else {
4235 			return (DDI_DMA_NOMAPPING);
4236 		}
4237 	}
4238 
4239 	return (e);
4240 #endif /* defined(__amd64) */
4241 }
4242 
4243 
4244 /*
4245  * rootnex_dma_mctl()
4246  *
4247  */
4248 /* ARGSUSED */
4249 static int
4250 rootnex_dma_mctl(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle,
4251     enum ddi_dma_ctlops request, off_t *offp, size_t *lenp, caddr_t *objpp,
4252     uint_t cache_flags)
4253 {
4254 #if defined(__amd64)
4255 	/*
4256 	 * DDI_DMA_SMEM_ALLOC & DDI_DMA_IOPB_ALLOC we're changed to have a
4257 	 * common implementation in genunix, so they no longer have x86
4258 	 * specific functionality which called into dma_ctl.
4259 	 *
4260 	 * The rest of the obsoleted interfaces were never supported in the
4261 	 * 64-bit x86 kernel. For s10, the obsoleted DDI_DMA_SEGTOC interface
4262 	 * was not ported to the x86 64-bit kernel do to serious x86 rootnex
4263 	 * implementation issues.
4264 	 *
4265 	 * If you can't use DDI_DMA_SEGTOC; DDI_DMA_NEXTSEG, DDI_DMA_FREE, and
4266 	 * DDI_DMA_NEXTWIN are useless since you can get to the cookie, so we
4267 	 * reflect that now too...
4268 	 *
4269 	 * Even though we fixed the pointer problem in DDI_DMA_SEGTOC, we are
4270 	 * not going to put this functionality into the 64-bit x86 kernel now.
4271 	 * It wasn't ported to the 64-bit kernel for s10, no reason to change
4272 	 * that in a future release.
4273 	 */
4274 	return (DDI_FAILURE);
4275 
4276 #else /* 32-bit x86 kernel */
4277 	ddi_dma_cookie_t lcookie;
4278 	ddi_dma_cookie_t *cookie;
4279 	rootnex_window_t *window;
4280 	ddi_dma_impl_t *hp;
4281 	rootnex_dma_t *dma;
4282 	uint_t nwin;
4283 	uint_t ccnt;
4284 	size_t len;
4285 	off_t off;
4286 	int e;
4287 
4288 
4289 	/*
4290 	 * DDI_DMA_SEGTOC, DDI_DMA_NEXTSEG, and DDI_DMA_NEXTWIN are a little
4291 	 * hacky since were optimizing for the current interfaces and so we can
4292 	 * cleanup the mess in genunix. Hopefully we will remove the this
4293 	 * obsoleted routines someday soon.
4294 	 */
4295 
4296 	switch (request) {
4297 
4298 	case DDI_DMA_SEGTOC: /* ddi_dma_segtocookie() */
4299 		hp = (ddi_dma_impl_t *)handle;
4300 		cookie = (ddi_dma_cookie_t *)objpp;
4301 
4302 		/*
4303 		 * convert segment to cookie. We don't distinguish between the
4304 		 * two :-)
4305 		 */
4306 		*cookie = *hp->dmai_cookie;
4307 		*lenp = cookie->dmac_size;
4308 		*offp = cookie->dmac_type & ~ROOTNEX_USES_COPYBUF;
4309 		return (DDI_SUCCESS);
4310 
4311 	case DDI_DMA_NEXTSEG: /* ddi_dma_nextseg() */
4312 		hp = (ddi_dma_impl_t *)handle;
4313 		dma = (rootnex_dma_t *)hp->dmai_private;
4314 
4315 		if ((*lenp != NULL) && ((uintptr_t)*lenp != (uintptr_t)hp)) {
4316 			return (DDI_DMA_STALE);
4317 		}
4318 
4319 		/* handle the case where we don't have any windows */
4320 		if (dma->dp_window == NULL) {
4321 			/*
4322 			 * if seg == NULL, and we don't have any windows,
4323 			 * return the first cookie in the sgl.
4324 			 */
4325 			if (*lenp == NULL) {
4326 				dma->dp_current_cookie = 0;
4327 				hp->dmai_cookie = dma->dp_cookies;
4328 				*objpp = (caddr_t)handle;
4329 				return (DDI_SUCCESS);
4330 
4331 			/* if we have more cookies, go to the next cookie */
4332 			} else {
4333 				if ((dma->dp_current_cookie + 1) >=
4334 				    dma->dp_sglinfo.si_sgl_size) {
4335 					return (DDI_DMA_DONE);
4336 				}
4337 				dma->dp_current_cookie++;
4338 				hp->dmai_cookie++;
4339 				return (DDI_SUCCESS);
4340 			}
4341 		}
4342 
4343 		/* We have one or more windows */
4344 		window = &dma->dp_window[dma->dp_current_win];
4345 
4346 		/*
4347 		 * if seg == NULL, return the first cookie in the current
4348 		 * window
4349 		 */
4350 		if (*lenp == NULL) {
4351 			dma->dp_current_cookie = 0;
4352 			hp->dmai_cookie = window->wd_first_cookie;
4353 
4354 		/*
4355 		 * go to the next cookie in the window then see if we done with
4356 		 * this window.
4357 		 */
4358 		} else {
4359 			if ((dma->dp_current_cookie + 1) >=
4360 			    window->wd_cookie_cnt) {
4361 				return (DDI_DMA_DONE);
4362 			}
4363 			dma->dp_current_cookie++;
4364 			hp->dmai_cookie++;
4365 		}
4366 		*objpp = (caddr_t)handle;
4367 		return (DDI_SUCCESS);
4368 
4369 	case DDI_DMA_NEXTWIN: /* ddi_dma_nextwin() */
4370 		hp = (ddi_dma_impl_t *)handle;
4371 		dma = (rootnex_dma_t *)hp->dmai_private;
4372 
4373 		if ((*offp != NULL) && ((uintptr_t)*offp != (uintptr_t)hp)) {
4374 			return (DDI_DMA_STALE);
4375 		}
4376 
4377 		/* if win == NULL, return the first window in the bind */
4378 		if (*offp == NULL) {
4379 			nwin = 0;
4380 
4381 		/*
4382 		 * else, go to the next window then see if we're done with all
4383 		 * the windows.
4384 		 */
4385 		} else {
4386 			nwin = dma->dp_current_win + 1;
4387 			if (nwin >= hp->dmai_nwin) {
4388 				return (DDI_DMA_DONE);
4389 			}
4390 		}
4391 
4392 		/* switch to the next window */
4393 		e = rootnex_dma_win(dip, rdip, handle, nwin, &off, &len,
4394 		    &lcookie, &ccnt);
4395 		ASSERT(e == DDI_SUCCESS);
4396 		if (e != DDI_SUCCESS) {
4397 			return (DDI_DMA_STALE);
4398 		}
4399 
4400 		/* reset the cookie back to the first cookie in the window */
4401 		if (dma->dp_window != NULL) {
4402 			window = &dma->dp_window[dma->dp_current_win];
4403 			hp->dmai_cookie = window->wd_first_cookie;
4404 		} else {
4405 			hp->dmai_cookie = dma->dp_cookies;
4406 		}
4407 
4408 		*objpp = (caddr_t)handle;
4409 		return (DDI_SUCCESS);
4410 
4411 	case DDI_DMA_FREE: /* ddi_dma_free() */
4412 		(void) rootnex_dma_unbindhdl(dip, rdip, handle);
4413 		(void) rootnex_dma_freehdl(dip, rdip, handle);
4414 		if (rootnex_state->r_dvma_call_list_id) {
4415 			ddi_run_callback(&rootnex_state->r_dvma_call_list_id);
4416 		}
4417 		return (DDI_SUCCESS);
4418 
4419 	case DDI_DMA_IOPB_ALLOC:	/* get contiguous DMA-able memory */
4420 	case DDI_DMA_SMEM_ALLOC:	/* get contiguous DMA-able memory */
4421 		/* should never get here, handled in genunix */
4422 		ASSERT(0);
4423 		return (DDI_FAILURE);
4424 
4425 	case DDI_DMA_KVADDR:
4426 	case DDI_DMA_GETERR:
4427 	case DDI_DMA_COFF:
4428 		return (DDI_FAILURE);
4429 	}
4430 
4431 	return (DDI_FAILURE);
4432 #endif /* defined(__amd64) */
4433 }
4434 
4435 
4436 /*
4437  * *********
4438  *  FMA Code
4439  * *********
4440  */
4441 
4442 /*
4443  * rootnex_fm_init()
4444  *    FMA init busop
4445  */
4446 /* ARGSUSED */
4447 static int
4448 rootnex_fm_init(dev_info_t *dip, dev_info_t *tdip, int tcap,
4449     ddi_iblock_cookie_t *ibc)
4450 {
4451 	*ibc = rootnex_state->r_err_ibc;
4452 
4453 	return (ddi_system_fmcap);
4454 }
4455 
4456 /*
4457  * rootnex_dma_check()
4458  *    Function called after a dma fault occurred to find out whether the
4459  *    fault address is associated with a driver that is able to handle faults
4460  *    and recover from faults.
4461  */
4462 /* ARGSUSED */
4463 static int
4464 rootnex_dma_check(dev_info_t *dip, const void *handle, const void *addr,
4465     const void *not_used)
4466 {
4467 	rootnex_window_t *window;
4468 	uint64_t start_addr;
4469 	uint64_t fault_addr;
4470 	ddi_dma_impl_t *hp;
4471 	rootnex_dma_t *dma;
4472 	uint64_t end_addr;
4473 	size_t csize;
4474 	int i;
4475 	int j;
4476 
4477 
4478 	/* The driver has to set DDI_DMA_FLAGERR to recover from dma faults */
4479 	hp = (ddi_dma_impl_t *)handle;
4480 	ASSERT(hp);
4481 
4482 	dma = (rootnex_dma_t *)hp->dmai_private;
4483 
4484 	/* Get the address that we need to search for */
4485 	fault_addr = *(uint64_t *)addr;
4486 
4487 	/*
4488 	 * if we don't have any windows, we can just walk through all the
4489 	 * cookies.
4490 	 */
4491 	if (dma->dp_window == NULL) {
4492 		/* for each cookie */
4493 		for (i = 0; i < dma->dp_sglinfo.si_sgl_size; i++) {
4494 			/*
4495 			 * if the faulted address is within the physical address
4496 			 * range of the cookie, return DDI_FM_NONFATAL.
4497 			 */
4498 			if ((fault_addr >= dma->dp_cookies[i].dmac_laddress) &&
4499 			    (fault_addr <= (dma->dp_cookies[i].dmac_laddress +
4500 			    dma->dp_cookies[i].dmac_size))) {
4501 				return (DDI_FM_NONFATAL);
4502 			}
4503 		}
4504 
4505 		/* fault_addr not within this DMA handle */
4506 		return (DDI_FM_UNKNOWN);
4507 	}
4508 
4509 	/* we have mutiple windows, walk through each window */
4510 	for (i = 0; i < hp->dmai_nwin; i++) {
4511 		window = &dma->dp_window[i];
4512 
4513 		/* Go through all the cookies in the window */
4514 		for (j = 0; j < window->wd_cookie_cnt; j++) {
4515 
4516 			start_addr = window->wd_first_cookie[j].dmac_laddress;
4517 			csize = window->wd_first_cookie[j].dmac_size;
4518 
4519 			/*
4520 			 * if we are trimming the first cookie in the window,
4521 			 * and this is the first cookie, adjust the start
4522 			 * address and size of the cookie to account for the
4523 			 * trim.
4524 			 */
4525 			if (window->wd_trim.tr_trim_first && (j == 0)) {
4526 				start_addr = window->wd_trim.tr_first_paddr;
4527 				csize = window->wd_trim.tr_first_size;
4528 			}
4529 
4530 			/*
4531 			 * if we are trimming the last cookie in the window,
4532 			 * and this is the last cookie, adjust the start
4533 			 * address and size of the cookie to account for the
4534 			 * trim.
4535 			 */
4536 			if (window->wd_trim.tr_trim_last &&
4537 			    (j == (window->wd_cookie_cnt - 1))) {
4538 				start_addr = window->wd_trim.tr_last_paddr;
4539 				csize = window->wd_trim.tr_last_size;
4540 			}
4541 
4542 			end_addr = start_addr + csize;
4543 
4544 			/*
4545 			 * if the faulted address is within the physical address
4546 			 * range of the cookie, return DDI_FM_NONFATAL.
4547 			 */
4548 			if ((fault_addr >= start_addr) &&
4549 			    (fault_addr <= end_addr)) {
4550 				return (DDI_FM_NONFATAL);
4551 			}
4552 		}
4553 	}
4554 
4555 	/* fault_addr not within this DMA handle */
4556 	return (DDI_FM_UNKNOWN);
4557 }
4558